diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 41099f2..087dcb0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,7 +15,7 @@ repos: files: \.go$ - id: check-schema - name: schema in sync with DSL types + name: schemas in sync with DSL and config types entry: make check-schema language: system pass_filenames: false diff --git a/Makefile b/Makefile index 1adedea..084ac40 100644 --- a/Makefile +++ b/Makefile @@ -44,12 +44,12 @@ pre-commit: ## Run all pre-commit hooks against every file changelog: ## Generate a changelog from git history git cliff --output CHANGELOG.md -gen-schema: ## Regenerate pkg/api/schema/v1/testfile.json from internal/dsl types +gen-schema: ## Regenerate the JSON Schemas from internal/dsl and internal/config types go generate ./pkg/api/schema/v1/... -check-schema: ## Fail if testfile.json is out of sync with `go generate` +check-schema: ## Fail if the generated JSON Schemas are out of sync with `go generate` go generate ./pkg/api/schema/v1/... - git diff --exit-code pkg/api/schema/v1/testfile.json + git diff --exit-code pkg/api/schema/v1/testfile.json pkg/api/schema/v1/config.json release-dry-run: ## Preview a release locally (requires goreleaser in PATH) goreleaser release --snapshot --clean diff --git a/README.md b/README.md index 104a4c1..b48586a 100644 --- a/README.md +++ b/README.md @@ -44,13 +44,14 @@ with **no pre-existing cluster**. Both drive the upstream CLI (vigie does not em need a container runtime — **docker or podman** — on the host. vigie resolves the `kind`/`k3d` binary in order: an explicit `--kind-binary` / `--k3d-binary` -path → `$PATH` → the vigie cache → download. Downloads are opt-in: on an interactive terminal +path → `test.cluster..binary` in `.vigie.yaml` → `$PATH` → the vigie cache → download. +Downloads are opt-in: on an interactive terminal vigie prompts for confirmation; in CI or with piped stdin it never downloads and errors with install guidance instead, unless you pass `--download-tools` (or set `VIGIE_AUTO_DOWNLOAD=1`). Minimum supported versions: **kind ≥ v0.20.0**, **k3d ≥ v5.4.0**. -Backend-specific provisioning flags go through `testApply.cluster.extraArgs` in `.vigie.yaml` -(e.g. a kind `--config` for a multi-node topology). Downloaded binaries are statically-linked +Backend-specific provisioning flags go through `test.cluster..extraArgs` in +`.vigie.yaml` (e.g. a kind `--config` for a multi-node topology). Downloaded binaries are statically-linked Go executables that run on NixOS as-is; a `nix profile install kind k3d` is picked up from `$PATH` before any download. @@ -77,7 +78,8 @@ go build -o vigie ./cmd/vigie ## Quick start -Drop test files under `tests/unit/` in your chart: +Drop test files under `tests/` in your chart — one root for every test file, scanned +recursively, with sub-directories purely for organisation: ```yaml # mychart/tests/unit/deployment_test.yaml @@ -124,8 +126,9 @@ Tests: 2 total, 2 passed (2ms total test time) ``` A complete, realistic example chart lives in -[`testdata/charts/basic`](./testdata/charts/basic) — its `tests/unit/` suite exercises the -full matcher library, `matrix`/`cases`, helper (`call:`) tests, and snapshots. +[`testdata/charts/basic`](./testdata/charts/basic) — its `tests/` root exercises the full +matcher library, `matrix`/`cases`, helper (`call:`) tests, and snapshots, plus apply-tier +suites with dependencies and live matchers. --- @@ -293,12 +296,17 @@ tests: ### Editor autocomplete -`vigie schema` prints the test-file JSON Schema. Reference it from a test file with a +`vigie schema` prints the test-file JSON Schema, `vigie schema config` the one for +`.vigie.yaml`. Reference either from the matching file with a [yaml-language-server](https://github.com/redhat-developer/yaml-language-server) modeline for completion and validation as you type — either the hosted schema: ```yaml +# in tests/**/*_test.yaml # yaml-language-server: $schema=https://raw.githubusercontent.com/fregateops/vigie/refs/heads/main/pkg/api/schema/v1/testfile.json + +# in .vigie.yaml +# yaml-language-server: $schema=https://raw.githubusercontent.com/fregateops/vigie/refs/heads/main/pkg/api/schema/v1/config.json ``` or a local copy for offline/pinned use: @@ -306,8 +314,15 @@ or a local copy for offline/pinned use: ```sh vigie schema > .vigie.schema.json # then: # yaml-language-server: $schema=./.vigie.schema.json + +vigie schema config > .vigie.config.schema.json +# then: # yaml-language-server: $schema=./.vigie.config.schema.json ``` +Both schemas are generated from the Go types they describe (`internal/dsl` and +`internal/config`), and `.vigie.yaml` is validated against its schema at load time — so a +mistyped key names itself instead of being silently ignored. + --- ## CLI reference @@ -344,7 +359,7 @@ vigie validate [chart] chart tier: render values.yaml + overlays, validat --set / --set-json / --set-literal value overrides (helm semantics) -p, --parallelism parallel scenarios (default: CPU count) -vigie schema print the test-file JSON Schema +vigie schema [target] print a JSON Schema: testfile (default) or config ``` Chart commands default `[chart]` to the current directory, so `vigie test` works from inside a @@ -381,17 +396,32 @@ validate: messageRegex: "networking.k8s.io/v1" test: - testsDir: tests/unit + testsDir: tests # single root holding every test file, scanned recursively skipSchema: false # kubeconform runs per test by default; true opts out kubeVersions: [1.36.1] # kubeconform runs once per version (matrix) -testApply: # the apply tier of `vigie test --cluster ` + # Per-backend settings for the cluster tiers. These do not select a tier — + # `--cluster ` does, and only that backend's block is read. cluster: - type: envtest # envtest|kubeconfig|kind|k3d - kubeVersion: 1.36.1 - extraArgs: [] # kind/k3d only, e.g. ["--config", "kind-3node.yaml"] + envtest: + kubeVersion: 1.36.1 # envtest binary assets (apiserver, etcd) + kind: + kubeVersion: 1.36.1 # node image + binary: "" # kind CLI; empty = PATH, then cache, then download + extraArgs: [] # e.g. ["--config", "kind-3node.yaml"] + k3d: + kubeVersion: 1.36.1 + binary: "" + extraArgs: [] # e.g. ["-v", "/host:/node"] + kubeconfig: + path: /home/me/.kube/config # no `~` expansion; required for --cluster kubeconfig ``` +CLI flags win over `.vigie.yaml`: `--kube-version`, `--kubeconfig`, `--kind-binary`, and +`--k3d-binary` each override the selected backend's block. `--download-tools` has no config +counterpart on purpose — whether a missing CLI may be fetched is an environment concern (TTY vs +CI), not a per-chart one. + ### Lint rule sets | Rule set | Checks | diff --git a/cmd/vigie/outcomes_test.go b/cmd/vigie/outcomes_test.go new file mode 100644 index 0000000..8b22115 --- /dev/null +++ b/cmd/vigie/outcomes_test.go @@ -0,0 +1,62 @@ +package main + +import ( + "testing" + + "github.com/fregateops/vigie/internal/runner" +) + +func suiteWith(results ...runner.TestResult) runner.SuiteResult { + return runner.SuiteResult{Results: results} +} + +func TestCountTestOutcomes_SplitsSkippedFromExecuted(t *testing.T) { + // Skipped tests carry Pass: true, so counting "cases" alone cannot tell a + // green run from one that verified nothing. + cases := []struct { + name string + results []runner.SuiteResult + wantExecuted int + wantSkipped int + }{ + { + name: "no suites", + results: nil, + wantExecuted: 0, + wantSkipped: 0, + }, + { + name: "files parsed but no tests", + results: []runner.SuiteResult{suiteWith()}, + wantExecuted: 0, + wantSkipped: 0, + }, + { + name: "every test skipped", + results: []runner.SuiteResult{suiteWith( + runner.TestResult{Pass: true, Skipped: true}, + runner.TestResult{Pass: true, Skipped: true}, + )}, + wantExecuted: 0, + wantSkipped: 2, + }, + { + name: "mixed across suites", + results: []runner.SuiteResult{ + suiteWith(runner.TestResult{Pass: true}, runner.TestResult{Pass: true, Skipped: true}), + suiteWith(runner.TestResult{Pass: false}), + }, + wantExecuted: 2, + wantSkipped: 1, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + executed, skipped := countTestOutcomes(tc.results) + if executed != tc.wantExecuted || skipped != tc.wantSkipped { + t.Errorf("countTestOutcomes = (%d executed, %d skipped), want (%d, %d)", + executed, skipped, tc.wantExecuted, tc.wantSkipped) + } + }) + } +} diff --git a/cmd/vigie/schema.go b/cmd/vigie/schema.go index a2fcc33..d60e0e5 100644 --- a/cmd/vigie/schema.go +++ b/cmd/vigie/schema.go @@ -4,22 +4,67 @@ import ( "fmt" "os" + "github.com/fregateops/vigie/internal/config" "github.com/fregateops/vigie/internal/dsl" "github.com/spf13/cobra" ) +// Schema targets accepted by `vigie schema`. testfile stays the default so the +// documented `vigie schema > .vigie.schema.json` idiom keeps working. +const ( + schemaTargetTestFile = "testfile" + schemaTargetConfig = "config" +) + var schemaCmd = &cobra.Command{ - Use: "schema", - Short: "Print the test file JSON Schema", - Example: ` # Save the schema for editor autocomplete, then reference it from a - # test file with: # yaml-language-server: $schema=./.vigie.schema.json - vigie schema > .vigie.schema.json`, - RunE: func(cmd *cobra.Command, args []string) error { - fmt.Fprintf(os.Stdout, "%s\n", dsl.SchemaJSON()) - return nil - }, + Use: "schema [testfile|config]", + Short: "Print a JSON Schema: the test file format (default) or .vigie.yaml", + Long: "Print one of vigie's JSON Schemas, for editor autocomplete and validation:\n\n" + + " testfile the test file format (tests/**/*_test.yaml) — the default\n" + + " config the per-chart configuration file (.vigie.yaml)", + Example: ` # Save the test file schema, then reference it from a test file with: + # # yaml-language-server: $schema=./.vigie.schema.json + vigie schema > .vigie.schema.json + + # Save the config schema, then reference it from .vigie.yaml with: + # # yaml-language-server: $schema=./.vigie.config.schema.json + vigie schema config > .vigie.config.schema.json`, + Args: cobra.MaximumNArgs(1), + ValidArgs: []string{schemaTargetTestFile, schemaTargetConfig}, + RunE: runSchemaCmd, } func init() { rootCmd.AddCommand(schemaCmd) } + +func runSchemaCmd(cmd *cobra.Command, args []string) error { + schema, err := schemaFor(schemaTarget(args)) + if err != nil { + exitErr(3, "%v", err) + } + fmt.Fprintf(os.Stdout, "%s\n", schema) + return nil +} + +// schemaTarget returns the requested target, defaulting to the test file so a +// bare `vigie schema` keeps printing it. +func schemaTarget(args []string) string { + if len(args) == 1 { + return args[0] + } + return schemaTargetTestFile +} + +// schemaFor returns the embedded schema a target name selects. +func schemaFor(target string) ([]byte, error) { + switch target { + case schemaTargetTestFile: + return dsl.SchemaJSON(), nil + case schemaTargetConfig: + return config.SchemaJSON(), nil + default: + return nil, fmt.Errorf("unknown schema %q: valid values are %s, %s", + target, schemaTargetTestFile, schemaTargetConfig) + } +} diff --git a/cmd/vigie/schema_test.go b/cmd/vigie/schema_test.go new file mode 100644 index 0000000..98cd9af --- /dev/null +++ b/cmd/vigie/schema_test.go @@ -0,0 +1,51 @@ +package main + +import ( + "bytes" + "strings" + "testing" +) + +func TestSchemaFor_SelectsTheRightDocument(t *testing.T) { + cases := []struct { + target string + idSuffix string + }{ + {schemaTargetTestFile, "/testfile.json"}, + {schemaTargetConfig, "/config.json"}, + } + for _, tc := range cases { + t.Run(tc.target, func(t *testing.T) { + schema, err := schemaFor(tc.target) + if err != nil { + t.Fatalf("schemaFor(%q): %v", tc.target, err) + } + if !bytes.Contains(schema, []byte(tc.idSuffix+`"`)) { + t.Errorf("schema for %q does not carry an $id ending in %q", tc.target, tc.idSuffix) + } + }) + } +} + +func TestSchemaFor_UnknownTargetListsValidOnes(t *testing.T) { + _, err := schemaFor("values") + if err == nil { + t.Fatal("schemaFor must reject an unknown target, got nil error") + } + for _, want := range []string{"values", schemaTargetTestFile, schemaTargetConfig} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error %q does not mention %q", err, want) + } + } +} + +// A bare `vigie schema` must keep printing the test-file schema: the README +// documents `vigie schema > .vigie.schema.json` as the way to get it. +func TestSchemaTarget_DefaultsToTheTestFile(t *testing.T) { + if got, want := schemaTarget(nil), schemaTargetTestFile; got != want { + t.Errorf("schemaTarget(nil): want %q, got %q", want, got) + } + if got, want := schemaTarget([]string{schemaTargetConfig}), schemaTargetConfig; got != want { + t.Errorf("schemaTarget([config]): want %q, got %q", want, got) + } +} diff --git a/cmd/vigie/test.go b/cmd/vigie/test.go index 1c03daf..85efeb9 100644 --- a/cmd/vigie/test.go +++ b/cmd/vigie/test.go @@ -14,6 +14,7 @@ import ( "github.com/fregateops/vigie/internal/cluster" "github.com/fregateops/vigie/internal/config" "github.com/fregateops/vigie/internal/doctor" + "github.com/fregateops/vigie/internal/dsl" "github.com/fregateops/vigie/internal/runner" "github.com/spf13/cobra" "golang.org/x/term" @@ -84,7 +85,7 @@ func init() { testCmd.Flags().BoolVar(&flagTestSchema, "schema", true, "Run the per-test kubeconform pass (template tier)") testCmd.Flags().StringSliceVar(&flagTestKubeVersions, "kube-version", nil, "Kubernetes version(s), repeatable: template tier matrixes kubeconform over all; cluster tier uses the first (default: 1.36.1)") testCmd.Flags().StringVar(&flagTestCluster, "cluster", clusterNone, "Cluster backend for the apply tier: none|envtest|simulated|kind|k3d|kubeconfig (default none = template tier)") - testCmd.Flags().StringVar(&flagTestKubeconfig, "kubeconfig", "", "Path to kubeconfig for --cluster kubeconfig (overrides testApply.cluster.kubeconfig)") + testCmd.Flags().StringVar(&flagTestKubeconfig, "kubeconfig", "", "Path to kubeconfig for --cluster kubeconfig (overrides test.cluster.kubeconfig.path)") testCmd.Flags().BoolVar(&flagTestFailFast, "fail-fast", false, "Cancel queued tests after the first failure") testCmd.Flags().BoolVar(&flagTestKeepCluster, "keep-cluster", false, "Keep the cluster running after the suite for debugging (node-backed backends only)") testCmd.Flags().StringVar(&flagTestMatch, "match", "", "Run only tests whose display name matches this regex") @@ -120,9 +121,12 @@ func runTestCmd(cmd *cobra.Command, args []string) error { } slog.Debug("discovered test files", "count", len(files), "testsDir", testsDir) + refuseUnrunnableFile() + // Warnings are non-fatal conditions that still shouldn't read as a green - // "0 tests" pass in CI (a typo'd path, an empty tests dir, or files with no - // tests). They fail the run (exit 5) by default; --pass-on-warning opts out. + // pass in CI: a typo'd path, an empty tests dir, files with no tests, or a + // run whose every test skipped. They fail the run (exit 5) by default; + // --pass-on-warning opts out. var warnings []string if len(files) == 0 { @@ -141,8 +145,15 @@ func runTestCmd(cmd *cobra.Command, args []string) error { if runner.AnyFailed(results) { os.Exit(1) } - if countTestCases(results) == 0 { + executed, skipped := countTestOutcomes(results) + switch { + case executed+skipped == 0: warnings = append(warnings, "test files were discovered but contained no tests") + case executed == 0: + // Every test skipped. Skips pass, so without this the run exits 0 + // having verified nothing - the failure mode that reads as success. + warnings = append(warnings, fmt.Sprintf( + "all %d test(s) were skipped: this run verified nothing (see the skip reasons above)", skipped)) } } @@ -150,6 +161,29 @@ func runTestCmd(cmd *cobra.Command, args []string) error { return nil } +// refuseUnrunnableFile fails the run when --file names a suite that cannot run +// a single test at the active tier. +// +// Silence is honest for a directory sweep - skipping the files that don't fit +// is the point - but not for a file the user named explicitly: running it to a +// green zero-assertion finish answers a question nobody asked. Refusing here +// also happens before any cluster is provisioned. +func refuseUnrunnableFile() { + if flagTestFile == "" { + return + } + suite, err := dsl.ParseFile(flagTestFile) + if err != nil { + // Leave malformed files to the runner, which reports parse errors with + // the full schema detail. + return + } + tier := runner.TierForBackend(flagTestCluster) + if unrunnable, reason := runner.UnrunnableAt(suite, tier); unrunnable { + exitErr(3, "%s cannot run at tier %s: %s", flagTestFile, tier, reason) + } +} + // discoverTests resolves the test-file list for the active tier. A single // --file short-circuits discovery; otherwise the apply tier accepts both unit // and integration suite shapes (DiscoverApplyTestFiles) while the template tier @@ -214,19 +248,29 @@ func runTests(ctx context.Context, chartPath string, cfg *config.Config, files [ }) } -// resolveClusterConfig builds the cluster.Config from the --cluster flag, -// layering --kube-version / --kubeconfig over the chart's testApply.cluster -// settings. The backend type comes from the flag (already known to be a real -// backend, not "none"). A cluster pins a single Kubernetes version, so when -// --kube-version lists several the first wins and the rest are warned about. +// resolveClusterConfig builds the cluster.Config for the backend named by +// --cluster (already known to be a real backend, not "none"): it reads that +// backend's block from `test.cluster.`, then layers the CLI overrides +// on top. A cluster pins a single Kubernetes version, so when --kube-version +// lists several the first wins and the rest are warned about. func resolveClusterConfig(cfg *config.Config) cluster.Config { - configured := cfg.TestApply.Cluster - resolved := cluster.Config{ - Type: flagTestCluster, - KubeVersion: configured.KubeVersion, - Kubeconfig: configured.Kubeconfig, - ExtraArgs: configured.ExtraArgs, + resolved := cluster.Config{Type: flagTestCluster} + configured := cfg.Test.Cluster + switch flagTestCluster { + case "envtest": + resolved.KubeVersion = configured.Envtest.KubeVersion + case "kind": + resolved.KubeVersion = configured.Kind.KubeVersion + resolved.ExtraArgs = configured.Kind.ExtraArgs + resolved.KindBinary = configured.Kind.Binary + case "k3d": + resolved.KubeVersion = configured.K3d.KubeVersion + resolved.ExtraArgs = configured.K3d.ExtraArgs + resolved.K3dBinary = configured.K3d.Binary + case "kubeconfig": + resolved.Kubeconfig = configured.Kubeconfig.Path } + if len(flagTestKubeVersions) > 0 { resolved.KubeVersion = flagTestKubeVersions[0] if len(flagTestKubeVersions) > 1 { @@ -237,10 +281,14 @@ func resolveClusterConfig(cfg *config.Config) cluster.Config { if flagTestKubeconfig != "" { resolved.Kubeconfig = flagTestKubeconfig } - // Node-backed backends (kind, k3d) resolve their CLI; carry the binary - // overrides and the download policy. Other backends ignore these fields. - resolved.KindBinary = flagTestKindBinary - resolved.K3dBinary = flagTestK3dBinary + if flagTestKindBinary != "" { + resolved.KindBinary = flagTestKindBinary + } + if flagTestK3dBinary != "" { + resolved.K3dBinary = flagTestK3dBinary + } + // The download policy is an environment concern (TTY vs CI), never a + // per-chart setting, so it has no `.vigie.yaml` counterpart. resolved.ToolDownload, resolved.ConfirmDownload = toolDownloadPolicy() resolved.Progress = os.Stderr return resolved @@ -282,13 +330,20 @@ func emitWarnings(warnings []string) { } } -// countTestCases totals the executed test cases across all suites. -func countTestCases(results []runner.SuiteResult) int { - total := 0 +// countTestOutcomes splits the test cases across all suites into those that +// actually ran and those that were skipped. Skipped tests carry Pass: true, so +// the two must be counted apart to tell "everything passed" from "nothing ran". +func countTestOutcomes(results []runner.SuiteResult) (executed, skipped int) { for _, sr := range results { - total += len(sr.Results) + for _, tr := range sr.Results { + if tr.Skipped { + skipped++ + continue + } + executed++ + } } - return total + return executed, skipped } // resolveTestsDir picks the CLI --tests flag when set, else the config value. diff --git a/cmd/vigie/test_test.go b/cmd/vigie/test_test.go index 470623b..b001d35 100644 --- a/cmd/vigie/test_test.go +++ b/cmd/vigie/test_test.go @@ -88,3 +88,34 @@ func TestTest_FailingChart_ReportsFailure(t *testing.T) { t.Errorf("pretty output should name the failing test; got:\n%s", out) } } + +// TestTest_TierGate_SkipsClusterMatchersAtTemplateTier pins the template-tier +// gate: a cluster-only matcher must be reported as skipped, with the flag that +// would run it, instead of hard-failing as though the test were broken. +func TestTest_TierGate_SkipsClusterMatchersAtTemplateTier(t *testing.T) { + results := runTestSuite(t, "../../testdata/charts/tier-gate", t.TempDir()) + if runner.AnyFailed(results) { + t.Fatalf("a cluster matcher at the template tier must skip, not fail; results: %+v", results) + } + + var passed, skipped int + var skipReason string + for _, sr := range results { + for _, tr := range sr.Results { + if tr.Skipped { + skipped++ + skipReason = tr.SkipReason + continue + } + passed++ + } + } + if passed != 1 || skipped != 1 { + t.Fatalf("want 1 passed and 1 skipped, got %d passed and %d skipped", passed, skipped) + } + for _, want := range []string{`"applies"`, "--cluster"} { + if !strings.Contains(skipReason, want) { + t.Errorf("skip reason %q does not mention %q", skipReason, want) + } + } +} diff --git a/examples/.vigie.yaml b/examples/.vigie.yaml index 8820a9f..72930e2 100644 --- a/examples/.vigie.yaml +++ b/examples/.vigie.yaml @@ -1,13 +1,17 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/fregateops/vigie/refs/heads/main/pkg/api/schema/v1/config.json # .vigie.yaml — vigie per-chart configuration # +# The modeline above gives editors autocomplete and validation for this file. +# For an offline or pinned copy: `vigie schema config > .vigie.config.schema.json` +# and point the modeline at it instead. +# # Drop this file at the root of your Helm chart (next to Chart.yaml) and # uncomment the keys you want to override. Every key is optional; if the file # is absent or empty, vigie uses the built-in defaults shown below. # # This reference documents the config surface that ships today (`vigie lint`, -# `vigie test`). Apply-tier (`test --cluster`) config is being reworked and is -# intentionally omitted here for now; `vigie validate` / `vigie run` config keys -# are not documented yet either — see the README. +# `vigie test`); `vigie validate` / `vigie run` config keys are not documented +# yet — see the README. # # Reference: https://github.com/fregateops/vigie @@ -71,12 +75,74 @@ lint: - templates/jobs/*.yaml # --------------------------------------------------------------------------- -# test — controls `vigie test` (template-tier: per-test render + assertions). +# test — controls `vigie test`, in every tier: the template tier (per-test +# render + assertions, the default) and the cluster tiers reached with +# `--cluster `. # -# CLI: `vigie test ./mychart [--file f] [--tests dir] [--snapshot-dir dir]`. +# CLI: `vigie test ./mychart [--file f] [--tests dir] [--snapshot-dir dir] +# [--cluster none|envtest|kind|k3d|kubeconfig]`. # --------------------------------------------------------------------------- test: - # Directory `vigie test` scans recursively for `*_test.yaml`. Relative - # paths resolve against the chart directory. Empty/omitted defaults to - # `/tests`. Overridden by the `--tests` CLI flag. - testsDir: tests/unit + # The single root `vigie test` scans recursively for `*_test.yaml` — it holds + # *every* test file, whatever tier each one targets. Sub-directories are for + # organisation only (e.g. unit/, integration/); a file's tier comes from its + # content and the active `--cluster`, never from where it sits. + # Relative paths resolve against the chart directory. Empty/omitted defaults + # to `/tests`. Overridden by the `--tests` CLI flag. + testsDir: tests + + # Disable the per-test kubeconform pass. Default: false (schema runs). + # Overridden by the `--schema=false` CLI flag. + skipSchema: false + + # Kubernetes versions for the per-test kubeconform pass; each version runs + # as a separate pass (matrix). Full `MAJOR.MINOR.PATCH` required. + # Empty/omitted uses the built-in default. Overridden by `--kube-version`. + kubeVersions: + - "1.36.1" + + # ------------------------------------------------------------------------- + # test.cluster — per-backend settings for the cluster tiers. + # + # These blocks do NOT select a tier: `vigie test` renders in-process unless + # you pass `--cluster `, and only the selected backend's block is + # read. Configure them once here and switch tiers from the CLI. + # ------------------------------------------------------------------------- + cluster: + # envtest — a real kube-apiserver + etcd in-process, no controllers. + envtest: + # Kubernetes version of the envtest binary assets (kube-apiserver, etcd). + # Empty/omitted uses the built-in default. Overridden by `--kube-version`. + kubeVersion: "1.36.1" + + # kind — a throwaway cluster provisioned by driving the external kind CLI. + kind: + # Node image version. Empty/omitted uses kind's own default. + # Overridden by `--kube-version`. + kubeVersion: "1.36.1" + # Path to the kind CLI. Empty/omitted resolves it from PATH, then the + # vigie binary cache, then an optional download. Overridden by + # `--kind-binary`. + binary: "" + # Extra flags passed verbatim to `kind create cluster`. + extraArgs: + - --config + - kind-3node.yaml + + # k3d — a throwaway k3s cluster provisioned by driving the external k3d CLI. + k3d: + kubeVersion: "1.36.1" + # Path to the k3d CLI; same resolution as kind. Overridden by `--k3d-binary`. + binary: "" + # Extra flags passed verbatim to `k3d cluster create`. + extraArgs: + - -v + - /host:/node + + # kubeconfig — a cluster you already run, reached via an external kubeconfig. + kubeconfig: + # Path to the kubeconfig file. Mirrors `helm --kubeconfig`. Required for + # `--cluster kubeconfig`. Overridden by the `--kubeconfig` CLI flag. + # No shell expansion happens here, so write it out in full rather than + # using `~`; relative paths resolve against the working directory. + path: /home/me/.kube/config diff --git a/internal/config/config.go b/internal/config/config.go index 5b4309b..2d04a68 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -11,25 +11,34 @@ import ( const filename = ".vigie.yaml" +// Config is the root of `.vigie.yaml`, the per-chart configuration file. Every +// key is optional; an absent or empty file uses the built-in defaults. +// +// The `json` tags mirror the `yaml` ones because tools/gen-schema reflects this +// type into the published JSON Schema, and invopop/jsonschema reads `json`. +// Every field is `omitempty` so the schema marks none of them required. type Config struct { - Defaults Defaults `yaml:"defaults"` - Lint LintConfig `yaml:"lint"` - Validate ValidateConfig `yaml:"validate"` - Test TestConfig `yaml:"test"` - TestApply TestApplyConfig `yaml:"testApply"` - Run RunConfig `yaml:"run"` + Defaults Defaults `yaml:"defaults" json:"defaults,omitempty"` + Lint LintConfig `yaml:"lint" json:"lint,omitempty"` + Validate ValidateConfig `yaml:"validate" json:"validate,omitempty"` + Test TestConfig `yaml:"test" json:"test,omitempty"` + Run RunConfig `yaml:"run" json:"run,omitempty"` } // LintConfig controls which rule sets run and what to ignore. type LintConfig struct { // RuleSets is an allowlist of rule sets to run. Empty means "all defaults". - RuleSets []string `yaml:"ruleSets"` + RuleSets []string `yaml:"ruleSets" json:"ruleSets,omitempty"` // DisableRules is a denylist of individual rule IDs that must not run, even // if their rule set is enabled. Distinct from Ignore, which filters // findings post-execution by path. - DisableRules []string `yaml:"disableRules"` - KubeVersions []string `yaml:"kubeVersions"` - Ignore []IgnoreRule `yaml:"ignore"` + DisableRules []string `yaml:"disableRules" json:"disableRules,omitempty"` + // KubeVersions lists the Kubernetes versions to render against for + // deprecation and version-aware rules. Each is a `MAJOR.MINOR` string. + // Empty means "all supported versions". + KubeVersions []string `yaml:"kubeVersions" json:"kubeVersions,omitempty"` + // Ignore suppresses findings for a rule, optionally scoped to file paths. + Ignore []IgnoreRule `yaml:"ignore" json:"ignore,omitempty"` } // IsRuleDisabled reports whether the given rule ID is in DisableRules. @@ -44,8 +53,12 @@ func (c *LintConfig) IsRuleDisabled(id string) bool { // IgnoreRule suppresses a specific rule, optionally scoped to file paths. type IgnoreRule struct { - Rule string `yaml:"rule"` - Paths []string `yaml:"paths"` + // Rule is the namespaced rule ID to suppress, e.g. + // `template-best-practices_missing-resource-limits`. + Rule string `yaml:"rule" json:"rule,omitempty"` + // Paths are glob patterns matched against the finding's source file path. + // Empty suppresses the rule everywhere. + Paths []string `yaml:"paths" json:"paths,omitempty"` } // EnabledRuleSets returns the configured rule sets, or all defaults if none specified. @@ -69,53 +82,47 @@ type ValidateConfig struct { // `values.yaml` (helm `-f overlay.yaml` semantics). Each entry produces one // independent render+kubeconform pass. Empty means "just the baseline // render against values.yaml". - ValuesFiles []string `yaml:"valuesFiles"` + ValuesFiles []string `yaml:"valuesFiles" json:"valuesFiles,omitempty"` // KubeVersions lists Kubernetes versions to validate against. Each // (overlay × kubeVersion) pair runs as a separate scenario. - KubeVersions []string `yaml:"kubeVersions"` + KubeVersions []string `yaml:"kubeVersions" json:"kubeVersions,omitempty"` // Set holds --set style key=value overrides (helm strvals semantics). Applied // as the base layer; values files take higher priority. - Set []string `yaml:"set"` + Set []string `yaml:"set" json:"set,omitempty"` // SetJSON holds --set-json style key=jsonValue overrides. - SetJSON []string `yaml:"setJson"` + SetJSON []string `yaml:"setJson" json:"setJson,omitempty"` // SetLiteral holds --set-literal style key=literalString overrides (no type coercion). - SetLiteral []string `yaml:"setLiteral"` + SetLiteral []string `yaml:"setLiteral" json:"setLiteral,omitempty"` // Ignore suppresses specific schema violations. - Ignore []ValidateIgnoreRule `yaml:"ignore"` + Ignore []ValidateIgnoreRule `yaml:"ignore" json:"ignore,omitempty"` } // ValidateIgnoreRule suppresses a kubeconform finding by kind, name, and // optional regex over the violation message. type ValidateIgnoreRule struct { - Kind string `yaml:"kind"` - Name string `yaml:"name"` - MessageRegex string `yaml:"messageRegex"` + // Kind is the Kubernetes kind whose findings are suppressed, e.g. `Ingress`. + Kind string `yaml:"kind" json:"kind,omitempty"` + // Name is the object name whose findings are suppressed. Empty matches any. + Name string `yaml:"name" json:"name,omitempty"` + // MessageRegex further narrows the suppression to violation messages + // matching this regular expression. Empty matches any. + MessageRegex string `yaml:"messageRegex" json:"messageRegex,omitempty"` } -// TestConfig controls `vigie test` (template-tier). +// TestConfig controls `vigie test` — both the template tier (render + assert +// in-process) and the cluster tiers reached with `--cluster `. type TestConfig struct { // SkipSchema disables the per-test kubeconform pass when true. - SkipSchema bool `yaml:"skipSchema"` + SkipSchema bool `yaml:"skipSchema" json:"skipSchema,omitempty"` // KubeVersions used by the per-test kubeconform pass. Has no effect when // SkipSchema is true. - KubeVersions []string `yaml:"kubeVersions"` - // TestsDir overrides the discovery root for `vigie test`. Relative paths - // resolve against the chart directory. Empty falls back to `/tests`. - // The directory is scanned recursively for `*_test.yaml`. - TestsDir string `yaml:"testsDir"` -} - -// TestApplyConfig configures the cluster (apply) tier of `vigie test`. The -// backend is selected via Cluster.Type — envtest (default), simulated, kind, -// k3d, or kubeconfig — surfaced on the CLI as `vigie test --cluster `. -type TestApplyConfig struct { - // Cluster pins the backend the apply tier runs against. - Cluster ClusterConfig `yaml:"cluster"` - // TestsDir overrides the discovery root for the apply tier. - // Relative paths resolve against the chart directory. Empty falls back - // to `/tests`. The directory is scanned recursively for - // `*_test.yaml`. - TestsDir string `yaml:"testsDir"` + KubeVersions []string `yaml:"kubeVersions" json:"kubeVersions,omitempty"` + // TestsDir overrides the discovery root for `vigie test`, in every tier. + // Relative paths resolve against the chart directory. Empty falls back to + // `/tests`. The directory is scanned recursively for `*_test.yaml`. + TestsDir string `yaml:"testsDir" json:"testsDir,omitempty"` + // Cluster holds the per-backend settings for the cluster tiers. + Cluster ClusterConfig `yaml:"cluster" json:"cluster,omitempty"` } // RunConfig controls `vigie run` — the orchestrated command that chains @@ -128,40 +135,76 @@ type RunConfig struct { // apply tier. Empty/omitted means "no apply tiers — just lint+validate+test". // Valid values: any cluster backend type (envtest, simulated, kind, k3d, // kubeconfig). - ApplyTiers []string `yaml:"applyTiers"` + ApplyTiers []string `yaml:"applyTiers" json:"applyTiers,omitempty"` } -// ClusterConfig selects the cluster backend for the apply tier of `vigie test`. -// Mirrors -// internal/cluster.Config but lives in config so charts can pin a backend in -// `.vigie.yaml` without depending on the cluster package. +// ClusterConfig groups the per-backend settings for the cluster tiers of +// `vigie test`. It does not select a backend — `--cluster ` does, and +// only the matching sub-block is read for a given run. Charts can therefore +// pin every backend's settings once and switch tiers from the CLI. type ClusterConfig struct { - // Type selects the implementation: envtest | simulated | kind | k3d | kubeconfig. - // Empty defaults to "envtest" — fast and dependency-free. - Type string `yaml:"type"` - // KubeVersion is the target Kubernetes server version. envtest uses it to - // pin the binary asset version; node-backed backends (kind, k3d) use it to - // pin the node image. - KubeVersion string `yaml:"kubeVersion"` - // Kubeconfig is the path to a kubeconfig file used when Type is - // "kubeconfig". Mirrors `helm --kubeconfig`. - Kubeconfig string `yaml:"kubeconfig"` - // ExtraArgs are additional flags passed verbatim to the node-backed - // backends' provisioning CLI (kind/k3d). Ignored by envtest/kubeconfig. - // Example: ["--config", "kind-3node.yaml"] for kind, or ["-v", "/host:/node"] - // for k3d. - ExtraArgs []string `yaml:"extraArgs"` + // Envtest configures the in-process apiserver backend (`--cluster envtest`). + Envtest EnvtestConfig `yaml:"envtest" json:"envtest,omitempty"` + // Kind configures the kind backend (`--cluster kind`). + Kind NodeBackendConfig `yaml:"kind" json:"kind,omitempty"` + // K3d configures the k3d backend (`--cluster k3d`). + K3d NodeBackendConfig `yaml:"k3d" json:"k3d,omitempty"` + // Kubeconfig configures the external-cluster backend (`--cluster kubeconfig`). + Kubeconfig KubeconfigBackendConfig `yaml:"kubeconfig" json:"kubeconfig,omitempty"` } +// EnvtestConfig configures the envtest backend, which runs a real +// kube-apiserver and etcd in-process with no controllers. +type EnvtestConfig struct { + // KubeVersion pins the envtest binary asset version. Empty uses the + // built-in default. Overridden by `--kube-version`. + KubeVersion string `yaml:"kubeVersion" json:"kubeVersion,omitempty"` +} + +// NodeBackendConfig configures a node-backed backend (kind or k3d), each of +// which provisions a throwaway cluster by driving its external CLI. +type NodeBackendConfig struct { + // KubeVersion pins the node image version. Empty uses the CLI's default. + // Overridden by `--kube-version`. + KubeVersion string `yaml:"kubeVersion" json:"kubeVersion,omitempty"` + // Binary is the path to the backend's CLI. Empty resolves it from PATH, + // then the vigie cache, then an optional download. Overridden by + // `--kind-binary` / `--k3d-binary`. + Binary string `yaml:"binary" json:"binary,omitempty"` + // ExtraArgs are additional flags passed verbatim to the provisioning CLI. + // Example: ["--config", "kind-3node.yaml"] for kind, or + // ["-v", "/host:/node"] for k3d. + ExtraArgs []string `yaml:"extraArgs" json:"extraArgs,omitempty"` +} + +// KubeconfigBackendConfig configures the kubeconfig backend, which runs against +// a cluster the user already operates. +type KubeconfigBackendConfig struct { + // Path is the kubeconfig file to reach the cluster with. Mirrors + // `helm --kubeconfig`. No shell expansion happens, so `~` is not a home + // directory here. Overridden by `--kubeconfig`. + Path string `yaml:"path" json:"path,omitempty"` +} + +// Defaults holds the values every test inherits unless a suite-level +// `defaults:` or a per-test `inputs:` block overrides them. type Defaults struct { - Release ReleaseDefaults `yaml:"release"` + // Release is the release identity passed to `helm template`. + Release ReleaseDefaults `yaml:"release" json:"release,omitempty"` } +// ReleaseDefaults is the Helm release identity used to render every test. type ReleaseDefaults struct { - Name string `yaml:"name"` - Namespace string `yaml:"namespace"` + // Name is the release name. Mirrors Helm's `--release-name`. + // Defaults to "release-name". + Name string `yaml:"name" json:"name,omitempty"` + // Namespace is the release namespace. Mirrors Helm's `--namespace`. + // Defaults to "default". + Namespace string `yaml:"namespace" json:"namespace,omitempty"` } +// DefaultConfig returns the built-in configuration used when `.vigie.yaml` is +// absent, and as the base every loaded file is decoded on top of. func DefaultConfig() *Config { return &Config{ Defaults: Defaults{ @@ -170,9 +213,6 @@ func DefaultConfig() *Config { Namespace: "default", }, }, - TestApply: TestApplyConfig{ - Cluster: ClusterConfig{Type: "envtest"}, - }, } } @@ -192,6 +232,14 @@ func Load(chartDir string) (*Config, error) { if len(bytes.TrimSpace(data)) == 0 { return cfg, nil } + if err := checkRetiredKeys(data); err != nil { + return nil, fmt.Errorf("%s: %w", path, err) + } + // The schema runs before the decode: it reports the whole set of violations + // at once, keyed by JSON pointer, where the decoder stops at the first one. + if err := Validate(data); err != nil { + return nil, fmt.Errorf("%s: %w", path, err) + } // Strict decoding so a misspelled or misplaced key fails loudly instead of // being silently ignored (e.g. `ruleSet:` for `ruleSets:`). dec := yaml.NewDecoder(bytes.NewReader(data)) @@ -205,11 +253,43 @@ func Load(chartDir string) (*Config, error) { return cfg, nil } +// retiredKeys maps top-level keys removed by the v2 config model to the +// migration hint shown when a stale `.vigie.yaml` still carries them. Strict +// decoding would reject them anyway, but with a bare "field not found" that +// says nothing about where the settings moved. +var retiredKeys = map[string]string{ + "testApply": "move `testApply.cluster` settings under `test.cluster.` " + + "(envtest, kind, k3d, kubeconfig) and `testApply.testsDir` to `test.testsDir`", +} + +// checkRetiredKeys reports a v1 config key with its v2 home, pointing at the +// line the key sits on. A malformed or non-mapping document is left to the +// strict decode, which reports the parse error. +func checkRetiredKeys(data []byte) error { + var doc yaml.Node + if err := yaml.Unmarshal(data, &doc); err != nil || len(doc.Content) == 0 { + return nil + } + root := doc.Content[0] + if root.Kind != yaml.MappingNode { + return nil + } + // A mapping's Content alternates key, value — only the keys interest us. + for i := 0; i < len(root.Content); i += 2 { + key := root.Content[i] + if hint, ok := retiredKeys[key.Value]; ok { + return fmt.Errorf("line %d: `%s:` was removed in the v2 config model: %s", key.Line, key.Value, hint) + } + } + return nil +} + // validateConfigKubeVersions vets every Kubernetes version field that feeds a -// binary download (kubeconform schemas, envtest/kcm/scheduler) so a truncated -// "1.30" surfaces at config-load time instead of as a 404 from dl.k8s.io. -// lint.kubeVersions is intentionally skipped — it accepts MAJOR.MINOR and only -// drives helm template's `.Capabilities.KubeVersion`, no binary download. +// binary download (kubeconform schemas, envtest/kcm/scheduler) or a node image +// so a truncated "1.30" surfaces at config-load time instead of as a 404 from +// dl.k8s.io. lint.kubeVersions is intentionally skipped — it accepts +// MAJOR.MINOR and only drives helm template's `.Capabilities.KubeVersion`, no +// binary download. func validateConfigKubeVersions(cfg *Config) error { if err := validateKubeVersions("validate.kubeVersions", cfg.Validate.KubeVersions); err != nil { return err @@ -217,8 +297,18 @@ func validateConfigKubeVersions(cfg *Config) error { if err := validateKubeVersions("test.kubeVersions", cfg.Test.KubeVersions); err != nil { return err } - if err := ValidateKubeVersion("testApply.cluster.kubeVersion", cfg.TestApply.Cluster.KubeVersion); err != nil { - return err + clusterVersions := []struct { + field string + version string + }{ + {"test.cluster.envtest.kubeVersion", cfg.Test.Cluster.Envtest.KubeVersion}, + {"test.cluster.kind.kubeVersion", cfg.Test.Cluster.Kind.KubeVersion}, + {"test.cluster.k3d.kubeVersion", cfg.Test.Cluster.K3d.KubeVersion}, + } + for _, cv := range clusterVersions { + if err := ValidateKubeVersion(cv.field, cv.version); err != nil { + return err + } } return nil } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 7bada3d..395a6d3 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -4,22 +4,19 @@ import ( "os" "path/filepath" "reflect" + "strings" "testing" "gopkg.in/yaml.v3" ) -func TestDefaultConfig_TestApplyDefaults(t *testing.T) { +func TestDefaultConfig_ClusterDefaultsAreEmpty(t *testing.T) { cfg := DefaultConfig() - if got, want := cfg.TestApply.Cluster.Type, "envtest"; got != want { - t.Errorf("TestApply.Cluster.Type: want %q, got %q", want, got) - } - if cfg.TestApply.Cluster.KubeVersion != "" { - t.Errorf("TestApply.Cluster.KubeVersion: want %q, got %q", "", cfg.TestApply.Cluster.KubeVersion) - } - if cfg.TestApply.Cluster.Kubeconfig != "" { - t.Errorf("TestApply.Cluster.Kubeconfig: want %q, got %q", "", cfg.TestApply.Cluster.Kubeconfig) + // No backend is pre-configured: `--cluster ` selects the tier and + // every backend falls back to its own built-in defaults. + if got, want := cfg.Test.Cluster, (ClusterConfig{}); !reflect.DeepEqual(got, want) { + t.Errorf("Test.Cluster: want zero value, got %+v", got) } if len(cfg.Run.ApplyTiers) != 0 { t.Errorf("Run.ApplyTiers: want empty, got %v", cfg.Run.ApplyTiers) @@ -33,13 +30,22 @@ func TestRunConfig_DefaultEmpty(t *testing.T) { } } -func TestTestApplyConfig_RoundTrip(t *testing.T) { +func TestClusterConfig_RoundTrip(t *testing.T) { const doc = ` -testApply: +test: cluster: - type: kubeconfig - kubeVersion: "1.30.0" - kubeconfig: /tmp/kubeconfig.yaml + envtest: + kubeVersion: "1.30.0" + kind: + kubeVersion: "1.31.0" + binary: /usr/local/bin/kind + extraArgs: + - --config + - kind-3node.yaml + k3d: + binary: /usr/local/bin/k3d + kubeconfig: + path: /tmp/kubeconfig.yaml run: applyTiers: - envtest @@ -51,14 +57,18 @@ run: t.Fatalf("yaml.Unmarshal: %v", err) } - if got, want := cfg.TestApply.Cluster.Type, "kubeconfig"; got != want { - t.Errorf("TestApply.Cluster.Type: want %q, got %q", want, got) - } - if got, want := cfg.TestApply.Cluster.KubeVersion, "1.30.0"; got != want { - t.Errorf("TestApply.Cluster.KubeVersion: want %q, got %q", want, got) - } - if got, want := cfg.TestApply.Cluster.Kubeconfig, "/tmp/kubeconfig.yaml"; got != want { - t.Errorf("TestApply.Cluster.Kubeconfig: want %q, got %q", want, got) + want := ClusterConfig{ + Envtest: EnvtestConfig{KubeVersion: "1.30.0"}, + Kind: NodeBackendConfig{ + KubeVersion: "1.31.0", + Binary: "/usr/local/bin/kind", + ExtraArgs: []string{"--config", "kind-3node.yaml"}, + }, + K3d: NodeBackendConfig{Binary: "/usr/local/bin/k3d"}, + Kubeconfig: KubeconfigBackendConfig{Path: "/tmp/kubeconfig.yaml"}, + } + if got := cfg.Test.Cluster; !reflect.DeepEqual(got, want) { + t.Errorf("Test.Cluster:\n got %+v\nwant %+v", got, want) } if got, want := cfg.Run.ApplyTiers, []string{"envtest", "kind"}; !reflect.DeepEqual(got, want) { t.Errorf("Run.ApplyTiers: want %v, got %v", want, got) @@ -69,8 +79,6 @@ func TestTestsDirConfig_RoundTrip(t *testing.T) { const doc = ` test: testsDir: ../tests/charts/my-chart-tests -testApply: - testsDir: /abs/path/tests ` var cfg Config @@ -80,8 +88,22 @@ testApply: if got, want := cfg.Test.TestsDir, "../tests/charts/my-chart-tests"; got != want { t.Errorf("Test.TestsDir: want %q, got %q", want, got) } - if got, want := cfg.TestApply.TestsDir, "/abs/path/tests"; got != want { - t.Errorf("TestApply.TestsDir: want %q, got %q", want, got) +} + +func TestLoad_RetiredTestApplyKeyPointsAtNewHome(t *testing.T) { + dir := t.TempDir() + body := "test:\n testsDir: tests/unit\ntestApply:\n cluster:\n type: kind\n" + if err := os.WriteFile(filepath.Join(dir, filename), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + _, err := Load(dir) + if err == nil { + t.Fatal("Load must reject the retired `testApply:` key, got nil error") + } + for _, want := range []string{"testApply", "test.cluster.", "line 3"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error %q does not mention %q", err, want) + } } } @@ -109,6 +131,33 @@ func TestLoad_EmptyFileKeepsDefaults(t *testing.T) { } } +// TestLoad_ExampleConfigIsValid keeps examples/.vigie.yaml honest: the strict +// loader rejects any key the structs no longer have, so the documented +// reference cannot drift away from the config model. +func TestLoad_ExampleConfigIsValid(t *testing.T) { + cfg, err := Load(filepath.Join("..", "..", "examples")) + if err != nil { + t.Fatalf("loading examples/.vigie.yaml: %v", err) + } + // Spot-check one key per top-level block so an example gutted by accident + // fails here rather than passing as a valid empty file. + if cfg.Defaults.Release.Name == "" { + t.Error("example documents no defaults.release.name") + } + if len(cfg.Lint.RuleSets) == 0 { + t.Error("example documents no lint.ruleSets") + } + if cfg.Test.TestsDir == "" { + t.Error("example documents no test.testsDir") + } + if cfg.Test.Cluster.Envtest.KubeVersion == "" { + t.Error("example documents no test.cluster.envtest.kubeVersion") + } + if cfg.Test.Cluster.Kubeconfig.Path == "" { + t.Error("example documents no test.cluster.kubeconfig.path") + } +} + func TestLoad_UnknownKeyIsRejected(t *testing.T) { dir := t.TempDir() // `ruleSet` is a typo for `ruleSets` — strict decoding must surface it diff --git a/internal/config/kubeversion_test.go b/internal/config/kubeversion_test.go index 55a783b..843728d 100644 --- a/internal/config/kubeversion_test.go +++ b/internal/config/kubeversion_test.go @@ -53,19 +53,24 @@ func TestValidateKubeVersion_ErrorMessageMentionsSource(t *testing.T) { } } -func TestLoad_RejectsTruncatedKubeVersionInTestApply(t *testing.T) { - dir := t.TempDir() - writeConfig(t, dir, `testApply: +func TestLoad_RejectsTruncatedKubeVersionPerClusterBackend(t *testing.T) { + for _, backend := range []string{"envtest", "kind", "k3d"} { + t.Run(backend, func(t *testing.T) { + dir := t.TempDir() + writeConfig(t, dir, `test: cluster: - type: envtest - kubeVersion: "1.30" + `+backend+`: + kubeVersion: "1.30" `) - _, err := Load(dir) - if err == nil { - t.Fatal("Load should have rejected truncated kubeVersion, got nil error") - } - if !strings.Contains(err.Error(), "testApply.cluster.kubeVersion") { - t.Errorf("error %q does not point at the offending field", err) + _, err := Load(dir) + if err == nil { + t.Fatal("Load should have rejected truncated kubeVersion, got nil error") + } + want := "test.cluster." + backend + ".kubeVersion" + if !strings.Contains(err.Error(), want) { + t.Errorf("error %q does not point at %s", err, want) + } + }) } } @@ -93,10 +98,13 @@ func TestLoad_AcceptsFullSemverEverywhere(t *testing.T) { test: kubeVersions: - "1.36.1" -testApply: cluster: - type: envtest - kubeVersion: "1.36.1" + envtest: + kubeVersion: "1.36.1" + kind: + kubeVersion: "1.36.1" + k3d: + kubeVersion: "1.36.1" `) if _, err := Load(dir); err != nil { t.Fatalf("Load: %v", err) diff --git a/internal/config/validator.go b/internal/config/validator.go new file mode 100644 index 0000000..0061ee1 --- /dev/null +++ b/internal/config/validator.go @@ -0,0 +1,22 @@ +package config + +import ( + "github.com/fregateops/vigie/internal/yamlschema" + schemav1 "github.com/fregateops/vigie/pkg/api/schema/v1" +) + +// SchemaJSON returns the embedded `.vigie.yaml` JSON Schema as raw bytes. +func SchemaJSON() []byte { + return schemav1.ConfigSchema +} + +// A config file is hand-maintained and full of commented-out keys, so a block +// left empty by commenting out its contents counts as unset rather than as a +// null that fails the schema. +var validator = yamlschema.New(schemav1.ConfigSchema, yamlschema.EmptyKeysAsAbsent()) + +// Validate checks raw `.vigie.yaml` bytes against the config JSON Schema +// (draft 2020-12). +func Validate(rawYAML []byte) error { + return validator.Validate(rawYAML) +} diff --git a/internal/config/validator_test.go b/internal/config/validator_test.go new file mode 100644 index 0000000..224adfd --- /dev/null +++ b/internal/config/validator_test.go @@ -0,0 +1,115 @@ +package config + +import ( + "encoding/json" + "strings" + "testing" +) + +func TestSchemaJSON_IsAValidSchemaDocument(t *testing.T) { + var doc struct { + ID string `json:"$id"` + Title string `json:"title"` + Type string `json:"type"` + } + if err := json.Unmarshal(SchemaJSON(), &doc); err != nil { + t.Fatalf("embedded config schema is not valid JSON: %v", err) + } + // A resolvable $id is what lets an editor fetch the schema from a modeline. + if !strings.HasSuffix(doc.ID, "/config.json") { + t.Errorf("$id %q does not point at config.json", doc.ID) + } + if doc.Title == "" { + t.Error("schema has no title") + } + if doc.Type != "object" { + t.Errorf("schema root type: want object, got %q", doc.Type) + } +} + +func TestValidate_RejectsUnknownKeyByName(t *testing.T) { + // `ruleSet` is a typo for `ruleSets`. The schema names the offending key and + // the block it sits in, which the strict decoder's message does not. + err := Validate([]byte("lint:\n ruleSet:\n - chart-yaml\n")) + if err == nil { + t.Fatal("Validate must reject an unknown key, got nil error") + } + for _, want := range []string{"/lint", "ruleSet"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error %q does not mention %q", err, want) + } + } +} + +func TestValidate_RejectsWrongType(t *testing.T) { + // kubeVersions is a list; a bare scalar is a common mistake. + err := Validate([]byte("test:\n kubeVersions: \"1.36.1\"\n")) + if err == nil { + t.Fatal("Validate must reject a scalar where a list is expected, got nil error") + } + if !strings.Contains(err.Error(), "/test/kubeVersions") { + t.Errorf("error %q does not point at the offending field", err) + } +} + +func TestValidate_EmptyBlockCountsAsUnset(t *testing.T) { + // Every sub-key commented out leaves the parent key with a null value. That + // is how a hand-edited config looks mid-experiment, so it must still load. + doc := ` +test: + cluster: + kind: +lint: +` + if err := Validate([]byte(doc)); err != nil { + t.Fatalf("Validate on a config with emptied blocks: %v", err) + } +} + +func TestValidate_AcceptsFullySpecifiedConfig(t *testing.T) { + doc := ` +defaults: + release: + name: release-name + namespace: default +lint: + ruleSets: [chart-yaml] + disableRules: [chart-yaml_name] + kubeVersions: ["1.30"] + ignore: + - rule: chart-yaml_name + paths: [templates/*.yaml] +validate: + valuesFiles: [values-prod.yaml] + kubeVersions: ["1.36.1"] + set: [a=b] + setJson: ['a={"b":1}'] + setLiteral: [a=b] + ignore: + - kind: Ingress + name: my-ingress + messageRegex: networking +test: + skipSchema: true + kubeVersions: ["1.36.1"] + testsDir: tests/unit + cluster: + envtest: + kubeVersion: "1.36.1" + kind: + kubeVersion: "1.36.1" + binary: /usr/local/bin/kind + extraArgs: [--config, kind.yaml] + k3d: + kubeVersion: "1.36.1" + binary: /usr/local/bin/k3d + extraArgs: [-v, /host:/node] + kubeconfig: + path: /tmp/kubeconfig.yaml +run: + applyTiers: [envtest, kind] +` + if err := Validate([]byte(doc)); err != nil { + t.Fatalf("Validate on a config exercising every key: %v", err) + } +} diff --git a/internal/deps/engine.go b/internal/deps/engine.go index 99ecf81..881292c 100644 --- a/internal/deps/engine.go +++ b/internal/deps/engine.go @@ -113,7 +113,7 @@ func Teardown(ctx context.Context, state *InstallState) error { reversed := reverseDeps(state.Installed) var teardownErrs []error for _, dep := range reversed { - if err := teardownOne(ctx, dep, state.restCfg); err != nil { + if err := teardownOne(ctx, dep, state.restCfg, state.baseDir); err != nil { slog.Warn("dep teardown failed", "dep", dep.Name, "err", err) teardownErrs = append(teardownErrs, err) } @@ -185,11 +185,11 @@ func installSource(ctx context.Context, dep dsl.Dependency, restCfg *rest.Config src := dep.Source switch { case src.Helm != nil: - return installHelm(ctx, dep, restCfg) + return installHelm(ctx, dep, restCfg, baseDir) case src.Manifest != "": - return applyManifest(ctx, dep, restCfg) + return applyManifest(ctx, dep, restCfg, baseDir) case src.Kustomize != "": - return applyKustomize(ctx, dep, restCfg) + return applyKustomize(ctx, dep, restCfg, baseDir) case src.Ref != "": // Refs are resolved before batching; reaching here is a programmer error. return fmt.Errorf("unresolved ref source %q: call ResolveRefs before Install", src.Ref) @@ -200,16 +200,19 @@ func installSource(ctx context.Context, dep dsl.Dependency, restCfg *rest.Config } } -// teardownOne dispatches to the appropriate source teardown function. -func teardownOne(ctx context.Context, dep dsl.Dependency, restCfg *rest.Config) error { +// teardownOne dispatches to the appropriate source teardown function. baseDir +// must match the one used at install time: manifest and kustomize teardowns +// re-read the source to know what to delete, so a different base resolves to a +// different file and leaves resources behind. +func teardownOne(ctx context.Context, dep dsl.Dependency, restCfg *rest.Config, baseDir string) error { src := dep.Source switch { case src.Helm != nil: return teardownHelm(ctx, dep, restCfg) case src.Manifest != "": - return teardownManifest(ctx, dep, restCfg) + return teardownManifest(ctx, dep, restCfg, baseDir) case src.Kustomize != "": - return teardownKustomize(ctx, dep, restCfg) + return teardownKustomize(ctx, dep, restCfg, baseDir) case src.Secret != nil: return teardownSecret(ctx, dep, restCfg) default: diff --git a/internal/deps/paths.go b/internal/deps/paths.go new file mode 100644 index 0000000..75183dc --- /dev/null +++ b/internal/deps/paths.go @@ -0,0 +1,18 @@ +package deps + +import "path/filepath" + +// resolveDepPath makes a relative path declared by a dependency absolute +// against baseDir - the directory of the test file that declared it - so a dep +// means the same thing however vigie was invoked. +// +// Without this, `manifest: ./fixtures/x.yaml` resolves against the process +// working directory: the suite passes when run from its own directory and fails +// from the repo root. Empty paths, absolute paths, and an empty baseDir are +// returned unchanged. +func resolveDepPath(path, baseDir string) string { + if path == "" || baseDir == "" || filepath.IsAbs(path) { + return path + } + return filepath.Join(baseDir, path) +} diff --git a/internal/deps/paths_test.go b/internal/deps/paths_test.go new file mode 100644 index 0000000..4008f4e --- /dev/null +++ b/internal/deps/paths_test.go @@ -0,0 +1,56 @@ +package deps + +import ( + "path/filepath" + "testing" +) + +func TestResolveDepPath(t *testing.T) { + cases := []struct { + name string + path string + baseDir string + want string + }{ + { + // The bug this exists to prevent: `manifest: ./fixtures/x.yaml` + // resolving against the working directory, so a suite passed from + // its own directory and failed from the repo root. + name: "relative path joins the base", + path: "./fixtures/landing-page.yaml", + baseDir: "testdata/charts/basic/tests/integration", + want: filepath.Join("testdata/charts/basic/tests/integration", "fixtures/landing-page.yaml"), + }, + { + name: "absolute path is left alone", + path: "/etc/manifests/x.yaml", + baseDir: "testdata/charts/basic/tests", + want: "/etc/manifests/x.yaml", + }, + { + name: "no base leaves the path as written", + path: "./fixtures/x.yaml", + baseDir: "", + want: "./fixtures/x.yaml", + }, + { + name: "empty path stays empty", + path: "", + baseDir: "testdata", + want: "", + }, + { + name: "parent-relative path resolves above the base", + path: "../shared/x.yaml", + baseDir: "testdata/charts/basic/tests/integration", + want: filepath.Join("testdata/charts/basic/tests", "shared/x.yaml"), + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := resolveDepPath(tc.path, tc.baseDir); got != tc.want { + t.Errorf("resolveDepPath(%q, %q) = %q, want %q", tc.path, tc.baseDir, got, tc.want) + } + }) + } +} diff --git a/internal/deps/source_helm.go b/internal/deps/source_helm.go index e9dde83..d1deef5 100644 --- a/internal/deps/source_helm.go +++ b/internal/deps/source_helm.go @@ -21,7 +21,7 @@ import ( // helm.sh/helm/v3 action API. It respects the dep's namespace and values overrides. // On scope=cluster, the caller is expected to have already checked the cache before // calling installHelm. -func installHelm(ctx context.Context, dep dsl.Dependency, restCfg *rest.Config) error { +func installHelm(ctx context.Context, dep dsl.Dependency, restCfg *rest.Config, baseDir string) error { slog.Debug("installing helm dep", "name", dep.Name, "chart", dep.Source.Helm.Chart, "repo", dep.Source.Helm.Repo, "version", dep.Source.Helm.Version, "namespace", dep.Namespace) @@ -31,7 +31,7 @@ func installHelm(ctx context.Context, dep dsl.Dependency, restCfg *rest.Config) namespace = dep.Name } - chrt, err := locateAndLoadChart(dep.Source.Helm) + chrt, err := locateAndLoadChart(dep.Source.Helm, baseDir) if err != nil { return fmt.Errorf("dep %q: locating helm chart: %w", dep.Name, err) } @@ -95,10 +95,11 @@ func teardownHelm(ctx context.Context, dep dsl.Dependency, restCfg *rest.Config) // locateAndLoadChart resolves the chart from the given HelmSource, downloading // from a remote repository or loading from a local path. -func locateAndLoadChart(src *dsl.HelmSource) (*helmchart.Chart, error) { - // If no repo URL is specified, treat Chart as a local path. +func locateAndLoadChart(src *dsl.HelmSource, baseDir string) (*helmchart.Chart, error) { + // If no repo URL is specified, treat Chart as a local path, relative to the + // test file that declared it rather than to the working directory. if src.Repo == "" { - return loader.Load(src.Chart) + return loader.Load(resolveDepPath(src.Chart, baseDir)) } cacheDir, err := os.MkdirTemp("", "vigie-chart-*") diff --git a/internal/deps/source_kustomize.go b/internal/deps/source_kustomize.go index fd20b3a..ee842b4 100644 --- a/internal/deps/source_kustomize.go +++ b/internal/deps/source_kustomize.go @@ -14,9 +14,10 @@ import ( // applyKustomize builds a kustomization in-process using the krusty API, // then applies the resulting documents via the dynamic client. -func applyKustomize(ctx context.Context, dep dsl.Dependency, restCfg *rest.Config) error { - slog.Debug("applying kustomize dep", "name", dep.Name, "path", dep.Source.Kustomize) - raw, err := buildKustomization(dep.Source.Kustomize) +func applyKustomize(ctx context.Context, dep dsl.Dependency, restCfg *rest.Config, baseDir string) error { + path := resolveDepPath(dep.Source.Kustomize, baseDir) + slog.Debug("applying kustomize dep", "name", dep.Name, "path", path) + raw, err := buildKustomization(path) if err != nil { return fmt.Errorf("dep %q: building kustomization: %w", dep.Name, err) } @@ -25,9 +26,9 @@ func applyKustomize(ctx context.Context, dep dsl.Dependency, restCfg *rest.Confi // teardownKustomize removes resources that were applied by applyKustomize by // rebuilding the manifest from kustomize and deleting each resulting resource. -func teardownKustomize(ctx context.Context, dep dsl.Dependency, restCfg *rest.Config) error { +func teardownKustomize(ctx context.Context, dep dsl.Dependency, restCfg *rest.Config, baseDir string) error { slog.Debug("tearing down kustomize dep", "name", dep.Name) - raw, err := buildKustomization(dep.Source.Kustomize) + raw, err := buildKustomization(resolveDepPath(dep.Source.Kustomize, baseDir)) if err != nil { slog.Debug("kustomize dep: rebuild failed during teardown, skipping", "dep", dep.Name, "err", err) return nil diff --git a/internal/deps/source_manifest.go b/internal/deps/source_manifest.go index d200c1b..cd166e7 100644 --- a/internal/deps/source_manifest.go +++ b/internal/deps/source_manifest.go @@ -27,19 +27,20 @@ const ( // applyManifest reads a multi-document YAML file and applies each document // via the dynamic client. -func applyManifest(ctx context.Context, dep dsl.Dependency, restCfg *rest.Config) error { - slog.Debug("applying manifest dep", "name", dep.Name, "path", dep.Source.Manifest) - raw, err := os.ReadFile(dep.Source.Manifest) +func applyManifest(ctx context.Context, dep dsl.Dependency, restCfg *rest.Config, baseDir string) error { + path := resolveDepPath(dep.Source.Manifest, baseDir) + slog.Debug("applying manifest dep", "name", dep.Name, "path", path) + raw, err := os.ReadFile(path) if err != nil { - return fmt.Errorf("dep %q: reading manifest %q: %w", dep.Name, dep.Source.Manifest, err) + return fmt.Errorf("dep %q: reading manifest %q: %w", dep.Name, path, err) } return applyRawDocs(ctx, dep.Name, raw, restCfg) } // teardownManifest deletes all resources previously applied by applyManifest. -func teardownManifest(ctx context.Context, dep dsl.Dependency, restCfg *rest.Config) error { +func teardownManifest(ctx context.Context, dep dsl.Dependency, restCfg *rest.Config, baseDir string) error { slog.Debug("tearing down manifest dep", "name", dep.Name) - raw, err := os.ReadFile(dep.Source.Manifest) + raw, err := os.ReadFile(resolveDepPath(dep.Source.Manifest, baseDir)) if err != nil { slog.Debug("manifest dep: source file missing during teardown, skipping", "dep", dep.Name) return nil diff --git a/internal/deps/source_secret.go b/internal/deps/source_secret.go index 44c4c67..8c72ee4 100644 --- a/internal/deps/source_secret.go +++ b/internal/deps/source_secret.go @@ -8,7 +8,6 @@ import ( "log/slog" "os" "os/exec" - "path/filepath" "strings" "time" @@ -148,10 +147,7 @@ func resolveOneSource(ctx context.Context, key dsl.SecretKeySpec, baseDir string return []byte(val), "env:" + key.Env, nil case key.File != "": - path := key.File - if !filepath.IsAbs(path) && baseDir != "" { - path = filepath.Join(baseDir, path) - } + path := resolveDepPath(key.File, baseDir) raw, err := os.ReadFile(path) if errors.Is(err, os.ErrNotExist) { return nil, "", nil diff --git a/internal/dsl/types.go b/internal/dsl/types.go index af84820..b7dfdd9 100644 --- a/internal/dsl/types.go +++ b/internal/dsl/types.go @@ -101,12 +101,6 @@ type Test struct { // Human-readable description of the scenario. It string `yaml:"it" json:"it"` - // Tiers this test applies to. Default: [template, validate]. - Tier []string `yaml:"tier" json:"tier,omitempty"` - - // Arbitrary labels for filtering tests. - Tags []string `yaml:"tags" json:"tags,omitempty"` - // Skip condition — boolean or CEL expression string. Skip any `yaml:"skip" json:"skip,omitempty"` diff --git a/internal/dsl/validator.go b/internal/dsl/validator.go index 220690f..b799343 100644 --- a/internal/dsl/validator.go +++ b/internal/dsl/validator.go @@ -1,14 +1,8 @@ package dsl import ( - "encoding/json" - "fmt" - "sort" - "strings" - + "github.com/fregateops/vigie/internal/yamlschema" schemav1 "github.com/fregateops/vigie/pkg/api/schema/v1" - "github.com/kaptinlin/jsonschema" - "gopkg.in/yaml.v3" ) // SchemaJSON returns the embedded test file JSON Schema as raw bytes. @@ -16,109 +10,9 @@ func SchemaJSON() []byte { return schemav1.TestFileSchema } -var compiledSchema *jsonschema.Schema - -func getSchema() (*jsonschema.Schema, error) { - if compiledSchema != nil { - return compiledSchema, nil - } - var err error - compiledSchema, err = jsonschema.NewCompiler().Compile(SchemaJSON()) - if err != nil { - return nil, fmt.Errorf("compiling schema: %w", err) - } - return compiledSchema, nil -} +var validator = yamlschema.New(schemav1.TestFileSchema) // Validate checks raw YAML bytes against the test file JSON Schema (draft 2020-12). func Validate(rawYAML []byte) error { - schema, err := getSchema() - if err != nil { - return err - } - - // YAML → JSON for the validator. - var doc any - if err := yaml.Unmarshal(rawYAML, &doc); err != nil { - return fmt.Errorf("invalid YAML: %w", err) - } - jsonBytes, err := json.Marshal(normalizeForJSON(doc)) - if err != nil { - return fmt.Errorf("converting to JSON: %w", err) - } - - result := schema.ValidateJSON(jsonBytes) - if result.IsValid() { - return nil - } - - // DetailedErrors drills into the failing leaf and keys each message by its - // instance location (a JSON pointer like /tests/0/asserts/0/eqaul), so a - // mistyped matcher points at the exact offending key instead of a vague - // top-level "tests does not match" rollup. - detailed := result.DetailedErrors() - if len(detailed) > 0 { - locations := make([]string, 0, len(detailed)) - for loc := range detailed { - locations = append(locations, loc) - } - sort.Strings(locations) - var msgs []string - for _, loc := range locations { - // Skip the structural rollup nodes ($ref/items/properties) that just - // say "does not match" — keep the concrete leaf violation (e.g. the - // additionalProperties error that names the offending key). - if isRollupLocation(loc) { - continue - } - where := loc - if where == "" { - where = "(root)" - } - msgs = append(msgs, fmt.Sprintf("%s: %s", where, detailed[loc])) - } - if len(msgs) > 0 { - return fmt.Errorf("schema validation errors:\n %s", strings.Join(msgs, "\n ")) - } - } - - // Fallback: the shallow, top-level errors if no leaf detail is available. - var msgs []string - for _, e := range result.Errors { - msgs = append(msgs, e.Error()) - } - return fmt.Errorf("schema validation errors:\n %s", strings.Join(msgs, "\n ")) -} - -// isRollupLocation reports whether a JSON-pointer instance location ends in a -// structural navigator keyword ($ref/items/properties). Those nodes carry only -// generic "does not match" rollups; the actionable message lives at the leaf. -func isRollupLocation(loc string) bool { - slash := strings.LastIndexByte(loc, '/') - last := loc[slash+1:] - switch last { - case "$ref", "items", "properties": - return true - } - return false -} - -// normalizeForJSON ensures all map keys are strings, as required by JSON marshaling. -func normalizeForJSON(v any) any { - switch val := v.(type) { - case map[string]any: - out := make(map[string]any, len(val)) - for k, v2 := range val { - out[k] = normalizeForJSON(v2) - } - return out - case []any: - out := make([]any, len(val)) - for i, v2 := range val { - out[i] = normalizeForJSON(v2) - } - return out - default: - return v - } + return validator.Validate(rawYAML) } diff --git a/internal/runner/apply.go b/internal/runner/apply.go index 4da4db1..7d13d81 100644 --- a/internal/runner/apply.go +++ b/internal/runner/apply.go @@ -30,7 +30,6 @@ import ( "github.com/fregateops/vigie/internal/deps" "github.com/fregateops/vigie/internal/dsl" "github.com/fregateops/vigie/internal/kubeclient" - "github.com/fregateops/vigie/internal/matchers" "github.com/fregateops/vigie/internal/render" "github.com/fregateops/vigie/internal/snapshot" ) @@ -70,9 +69,9 @@ type ApplyOptions struct { // RunApply runs the apply tier of `vigie test --cluster`: it starts the // configured cluster backend, walks each test file through the apply-tier state machine // (LOAD -> EXPAND -> EXECUTE -> REPORT), then stops the backend (unless -// KeepCluster). Integration-only features (dependencies, lifecycle hooks) are -// honoured only when the backend supports them; on envtest they are warned and -// skipped. +// KeepCluster). A suite declaring integration-only features (dependencies, +// lifecycle hooks) the active tier cannot provide is skipped whole rather than +// run with those features dropped. func RunApply(ctx context.Context, opts ApplyOptions) ([]SuiteResult, error) { slog.Debug("starting apply runner", "files", len(opts.TestFiles), "parallelism", opts.Parallelism, @@ -114,13 +113,12 @@ func RunApply(ctx context.Context, opts ApplyOptions) ([]SuiteResult, error) { } runner := &applyRunner{ - opts: opts, - kubeCfg: restCfg, - kubeconfig: opts.Backend.Kubeconfig(), - clientset: clientset, - matchRE: matchRE, - activeTier: backendTier(opts.BackendType), - integration: backendSupportsDeps(opts.BackendType), + opts: opts, + kubeCfg: restCfg, + kubeconfig: opts.Backend.Kubeconfig(), + clientset: clientset, + matchRE: matchRE, + activeTier: backendTier(opts.BackendType), } return runner.run(ctx) @@ -133,12 +131,10 @@ type applyRunner struct { kubeconfig string clientset *kubernetes.Clientset matchRE *regexp.Regexp - // activeTier is the value compared against each test's `tier:` field - // ("apiserver" for envtest, "e2e" for real-cluster backends). + // activeTier is the tier the backend runs at ("apiserver" for envtest, + // "e2e" for real-cluster backends), compared against what each matcher and + // each suite's integration features require. activeTier string - // integration is true when the backend supports integration-tier features - // (dependencies, lifecycle hooks). envtest sets this to false. - integration bool } // stopBackend honours --keep-cluster and uses a detached context so teardown @@ -224,23 +220,30 @@ func (r *applyRunner) runFile(ctx context.Context, filePath string) (SuiteResult baseDir := filepath.Dir(filePath) - // Warn-and-skip integration features when the backend doesn't support them. - // Tests with `dependencies:` running on envtest get a heads-up but the - // runner doesn't fail outright - the test's assertions may still pass if - // they don't depend on the deps. - clusterDeps, suiteDeps, testDeps := splitDepsByScope(suite.Dependencies) - if !r.integration && (len(clusterDeps)+len(suiteDeps)+len(testDeps) > 0) { - slog.Warn("dependencies declared but backend does not support them — skipping", - "file", filePath, "backend", r.opts.BackendType, - "clusterDeps", len(clusterDeps), "suiteDeps", len(suiteDeps), "testDeps", len(testDeps)) - clusterDeps, suiteDeps, testDeps = nil, nil, nil - } - if !r.integration && (len(suite.BeforeAll)+len(suite.AfterAll) > 0) { - slog.Warn("lifecycle hooks declared but backend does not support them — skipping", - "file", filePath, "backend", r.opts.BackendType) - suite.BeforeAll, suite.AfterAll = nil, nil + // A suite whose dependencies or hooks this tier cannot provide is skipped + // whole: its tests' premises cannot be established, so running them anyway + // reports on something other than what the author wrote. + if skip, reason := integrationFeatureSkip(suite, r.activeTier); skip { + slog.Debug("skipping suite (integration features unsupported)", + "file", filePath, "backend", r.opts.BackendType, "reason", reason) + for _, et := range expanded { + if r.matchRE != nil && !r.matchRE.MatchString(et.DisplayName) { + continue + } + sr.Results = append(sr.Results, TestResult{ + SuiteName: suite.SuiteName, + TestName: et.DisplayName, + Pass: true, + Skipped: true, + SkipReason: reason, + }) + } + sr.Duration = time.Since(start) + return sr, nil } + clusterDeps, suiteDeps, testDeps := splitDepsByScope(suite.Dependencies) + clusterState, _, err := deps.Install(ctx, clusterDeps, r.kubeCfg, deps.InstallOptions{ Parallelism: r.opts.Parallelism, BaseDir: baseDir, }) @@ -295,17 +298,6 @@ func (r *applyRunner) runFile(ctx context.Context, filePath string) (SuiteResult slog.Debug("skipping test (no match)", "test", et.DisplayName) continue } - if !tierApplies(et.Test.Tier, r.activeTier) { - slog.Debug("skipping test (tier filter)", "test", et.DisplayName, "tier", et.Test.Tier) - sr.Results = append(sr.Results, TestResult{ - SuiteName: suite.SuiteName, - TestName: et.DisplayName, - Pass: true, - Skipped: true, - SkipReason: fmt.Sprintf("tier %v excludes %s", et.Test.Tier, r.activeTier), - }) - continue - } if skip, reason := matcherTierSkip(et.Test.Asserts, r.activeTier); skip { slog.Debug("skipping test (matcher tier requirement)", "test", et.DisplayName, "reason", reason) @@ -393,20 +385,20 @@ func (r *applyRunner) runTest(ctx context.Context, et expandedTest, suite *dsl.S Test: et.DisplayName, Namespace: namespace, } - if r.integration { - if err := RunHooks(ctx, "setup", test.Setup, hookEnv); err != nil { - tr.Failures = append(tr.Failures, fmt.Sprintf(" → setup hook: %v", err)) - tr.Pass = false - return tr - } - defer func() { - teardownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - if err := RunHooks(teardownCtx, "teardown", test.Teardown, hookEnv); err != nil { - slog.Warn("teardown hook failed", "test", et.DisplayName, "err", err) - } - }() + // No tier guard here: runFile skips any suite declaring hooks the active + // tier cannot run, so reaching a test means they are available. + if err := RunHooks(ctx, "setup", test.Setup, hookEnv); err != nil { + tr.Failures = append(tr.Failures, fmt.Sprintf(" → setup hook: %v", err)) + tr.Pass = false + return tr } + defer func() { + teardownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + if err := RunHooks(teardownCtx, "teardown", test.Teardown, hookEnv); err != nil { + slog.Warn("teardown hook failed", "test", et.DisplayName, "err", err) + } + }() renderOpts := Options{ChartPath: r.opts.ChartPath, Cfg: r.opts.Cfg} req := buildRenderRequest(test, suite, renderOpts) @@ -592,40 +584,6 @@ func formatTestProgress(tr TestResult, displayName string, dur time.Duration) st return fmt.Sprintf("%s %s (%s)", status, displayName, durStr) } -// matcherTierSkip decides whether a test must be skipped because one of its -// matchers does not support the active backend's tier. The returned reason -// names the offending matcher so users can see *which* assertion caused the -// skip without re-reading the spec. -func matcherTierSkip(asserts []dsl.Assertion, activeTier string) (skip bool, reason string) { - name, needTier, ok := matchers.FindUnsupportedMatcher(asserts, activeTier) - if !ok { - return false, "" - } - return true, fmt.Sprintf("%q matcher requires tier %s; active tier is %s", name, needTier, activeTier) -} - -// tierApplies returns true when a test with the given tier list should run -// under the active tier. An empty/nil tier list means "all tiers". A "*" -// entry also means "all tiers". -func tierApplies(testTier []string, active string) bool { - if len(testTier) == 0 { - return true - } - for _, t := range testTier { - if t == "*" || t == active { - return true - } - } - return false -} - -// backendSupportsDeps reports whether a backend supports integration-tier -// features (`dependencies:` installation and lifecycle hooks). envtest does -// not - it has no controllers to reconcile Helm releases or hook jobs. -func backendSupportsDeps(backendType string) bool { - return backendType != "envtest" -} - // splitDepsByScope partitions deps by scope. The default scope (empty string) // is treated as "suite" to match DESIGN.md §12. func splitDepsByScope(allDeps []dsl.Dependency) (cluster, suite, test []dsl.Dependency) { diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 4082305..f43d506 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -171,6 +171,20 @@ func runFile(filePath string, opts Options, validators []versionedValidator, mat slog.Debug("skipping test (no match)", "test", et.DisplayName) continue } + // A test needing a cluster is skipped with the flag that would run it, + // mirroring the apply tier. Without this gate its matchers evaluate and + // hard-fail, reporting a broken test where the truth is a tier mismatch. + if skip, reason := matcherTierSkip(et.Test.Asserts, matchers.TierTemplate); skip { + slog.Debug("skipping test (matcher tier requirement)", "test", et.DisplayName, "reason", reason) + sr.Results = append(sr.Results, TestResult{ + SuiteName: suite.SuiteName, + TestName: et.DisplayName, + Pass: true, + Skipped: true, + SkipReason: reason, + }) + continue + } sr.Results = append(sr.Results, runTest(et, suite, opts, store, validators)) } sr.Duration = time.Since(start) diff --git a/internal/runner/tiergate.go b/internal/runner/tiergate.go new file mode 100644 index 0000000..d0fc523 --- /dev/null +++ b/internal/runner/tiergate.go @@ -0,0 +1,122 @@ +package runner + +import ( + "fmt" + + "github.com/fregateops/vigie/internal/dsl" + "github.com/fregateops/vigie/internal/matchers" +) + +// matcherTierSkip decides whether a test must be skipped because one of its +// matchers does not support the active backend's tier. The reason names the +// offending matcher, so users see *which* assertion caused the skip, and the +// `--cluster` value that would run it. +func matcherTierSkip(asserts []dsl.Assertion, activeTier string) (skip bool, reason string) { + name, needTier, ok := matchers.FindUnsupportedMatcher(asserts, activeTier) + if !ok { + return false, "" + } + return true, fmt.Sprintf("%q is not available at tier %s; run with %s", + name, activeTier, clusterHintForTier(needTier)) +} + +// TierForBackend returns the tier a `--cluster` value runs at. An empty value +// or "none" is the in-process template tier; every other backend maps through +// backendTier. +func TierForBackend(backendType string) string { + if backendType == "" || backendType == "none" { + return matchers.TierTemplate + } + return backendTier(backendType) +} + +// integrationFeatureSkip reports whether a suite declares integration-tier +// features - dependencies, or lifecycle hooks at suite or test scope - that the +// active tier cannot provide, and why. +// +// These used to be dropped with a log warning while the tests ran on, on the +// theory that assertions might still pass without them. They can, and that is +// exactly the problem: a suite whose staged dependencies were never installed +// reports green having verified a premise that never held. The suite's tests +// are skipped instead, so the gap is visible and counted. +func integrationFeatureSkip(suite *dsl.Suite, tier string) (skip bool, reason string) { + if tierAtLeast(tier, matchers.TierSimulated) { + return false, "" + } + hint := clusterHintForTier(matchers.TierSimulated) + switch { + case len(suite.Dependencies) > 0: + return true, fmt.Sprintf("suite declares dependencies, which tier %s cannot install; run with %s", tier, hint) + case len(suite.BeforeAll)+len(suite.AfterAll) > 0: + return true, fmt.Sprintf("suite declares beforeAll/afterAll hooks, which tier %s cannot run; run with %s", tier, hint) + case suiteHasTestHooks(suite): + return true, fmt.Sprintf("suite declares setup/teardown hooks, which tier %s cannot run; run with %s", tier, hint) + } + return false, "" +} + +func suiteHasTestHooks(suite *dsl.Suite) bool { + for _, test := range suite.Tests { + if len(test.Setup)+len(test.Teardown) > 0 { + return true + } + } + return false +} + +// tierAtLeast reports whether active sits at or above want on the ladder. An +// unrecognised tier ranks below everything, so it is treated as providing +// nothing rather than as satisfying a requirement by accident. +func tierAtLeast(active, want string) bool { + return tierRank(active) >= tierRank(want) +} + +func tierRank(tier string) int { + for i, t := range matchers.AllTiers { + if t == tier { + return i + } + } + return -1 +} + +// UnrunnableAt reports whether every test in a parsed suite would be skipped at +// the given tier, and why. It answers "this file cannot run here at all", +// which callers use to refuse an explicitly requested file instead of running +// it to a green zero-assertion finish. A suite with no tests is not unrunnable +// — that is a different warning. +func UnrunnableAt(suite *dsl.Suite, tier string) (unrunnable bool, reason string) { + if skip, reason := integrationFeatureSkip(suite, tier); skip { + return true, reason + } + if len(suite.Tests) == 0 { + return false, "" + } + for _, test := range suite.Tests { + skip, why := matcherTierSkip(test.Asserts, tier) + if !skip { + return false, "" + } + if reason == "" { + reason = why + } + } + return true, reason +} + +// clusterHintForTier names the `--cluster` values that satisfy needTier, +// listing only backends the factory can actually build. +// +// It deliberately does not echo needTier back at the user. Every matcher's +// tier list is a suffix of template < apiserver < simulated < e2e, so needTier +// is the *lowest* tier that would work - and for the waitFor/lookup family +// that is "simulated", a backend which arrives in a later release and which +// `--cluster simulated` currently rejects. Naming a flag value that errors out +// is worse than saying nothing, so the hint points at the backends that both +// exist and provide what the matcher needs. +func clusterHintForTier(needTier string) string { + if needTier == matchers.TierAPIServer { + return "--cluster envtest (or kind|k3d|kubeconfig)" + } + return "--cluster kind|k3d|kubeconfig" +} diff --git a/internal/runner/tiergate_test.go b/internal/runner/tiergate_test.go new file mode 100644 index 0000000..1f73cee --- /dev/null +++ b/internal/runner/tiergate_test.go @@ -0,0 +1,240 @@ +package runner + +import ( + "strings" + "testing" + + "github.com/fregateops/vigie/internal/dsl" + "github.com/fregateops/vigie/internal/matchers" +) + +func TestMatcherTierSkip_NamesTheMatcherAndAReachableFlag(t *testing.T) { + cases := []struct { + name string + asserts []dsl.Assertion + activeTier string + wantSkip bool + wantParts []string + }{ + { + name: "apiserver matcher at template tier", + asserts: []dsl.Assertion{{Applies: &dsl.AppliesSpec{}}}, + activeTier: matchers.TierTemplate, + wantSkip: true, + wantParts: []string{`"applies"`, "tier template", "--cluster envtest"}, + }, + { + name: "e2e matcher at apiserver tier", + asserts: []dsl.Assertion{{HTTP: &dsl.HTTPAssert{}}}, + activeTier: matchers.TierAPIServer, + wantSkip: true, + wantParts: []string{`"http"`, "tier apiserver", "--cluster kind|k3d|kubeconfig"}, + }, + { + name: "template matcher runs everywhere", + asserts: []dsl.Assertion{{Equal: &dsl.PathValue{Path: "kind"}}}, + activeTier: matchers.TierTemplate, + wantSkip: false, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + skip, reason := matcherTierSkip(tc.asserts, tc.activeTier) + if skip != tc.wantSkip { + t.Fatalf("matcherTierSkip skip = %v, want %v (reason %q)", skip, tc.wantSkip, reason) + } + for _, want := range tc.wantParts { + if !strings.Contains(reason, want) { + t.Errorf("reason %q does not contain %q", reason, want) + } + } + }) + } +} + +// The simulated backend is not implemented yet, so `--cluster simulated` errors +// out. A skip reason must never send the user there: for the waitFor/lookup +// family the lowest usable tier IS simulated, so the hint has to name the e2e +// backends that exist and also satisfy those matchers. +func TestClusterHintForTier_NeverNamesAnUnreachableBackend(t *testing.T) { + for _, tier := range matchers.AllTiers { + hint := clusterHintForTier(tier) + if strings.Contains(hint, matchers.TierSimulated) { + t.Errorf("hint for tier %q offers the unimplemented simulated backend: %q", tier, hint) + } + if !strings.Contains(hint, "--cluster") { + t.Errorf("hint for tier %q names no flag to pass: %q", tier, hint) + } + } +} + +func TestTierForBackend(t *testing.T) { + cases := map[string]string{ + "": matchers.TierTemplate, + "none": matchers.TierTemplate, + "envtest": matchers.TierAPIServer, + "kind": matchers.TierE2E, + "k3d": matchers.TierE2E, + "kubeconfig": matchers.TierE2E, + } + for backend, want := range cases { + if got := TierForBackend(backend); got != want { + t.Errorf("TierForBackend(%q) = %q, want %q", backend, got, want) + } + } +} + +func TestUnrunnableAt(t *testing.T) { + clusterTest := dsl.Test{It: "applies", Asserts: []dsl.Assertion{{Applies: &dsl.AppliesSpec{}}}} + renderTest := dsl.Test{It: "renders", Asserts: []dsl.Assertion{{Equal: &dsl.PathValue{Path: "kind"}}}} + + cases := []struct { + name string + suite dsl.Suite + tier string + wantUnrunnable bool + }{ + { + name: "every test needs a cluster", + suite: dsl.Suite{Tests: []dsl.Test{clusterTest, clusterTest}}, + tier: matchers.TierTemplate, + wantUnrunnable: true, + }, + { + // One runnable test is enough: the file has something to say here, + // so refusing it would be wrong. + name: "one test can still run", + suite: dsl.Suite{Tests: []dsl.Test{clusterTest, renderTest}}, + tier: matchers.TierTemplate, + wantUnrunnable: false, + }, + { + name: "the tier satisfies every test", + suite: dsl.Suite{Tests: []dsl.Test{clusterTest, renderTest}}, + tier: matchers.TierE2E, + wantUnrunnable: false, + }, + { + // An empty suite is a different problem, reported as its own warning. + name: "no tests at all", + suite: dsl.Suite{}, + tier: matchers.TierTemplate, + wantUnrunnable: false, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + unrunnable, reason := UnrunnableAt(&tc.suite, tc.tier) + if unrunnable != tc.wantUnrunnable { + t.Fatalf("UnrunnableAt = %v, want %v (reason %q)", unrunnable, tc.wantUnrunnable, reason) + } + if unrunnable && reason == "" { + t.Error("an unrunnable suite must explain why") + } + }) + } +} + +func TestIntegrationFeatureSkip(t *testing.T) { + renderTest := dsl.Test{It: "renders", Asserts: []dsl.Assertion{{Equal: &dsl.PathValue{Path: "kind"}}}} + hook := []dsl.LifecycleHook{{}} + + cases := []struct { + name string + suite dsl.Suite + tier string + wantSkip bool + wantPart string + }{ + { + name: "dependencies below simulated", + suite: dsl.Suite{Dependencies: []dsl.Dependency{{Name: "db"}}, Tests: []dsl.Test{renderTest}}, + tier: matchers.TierAPIServer, + wantSkip: true, + wantPart: "declares dependencies", + }, + { + // The premise the author wrote cannot be established, so the tests + // must not report on it - this is the hollow pass being closed. + name: "dependencies at e2e are fine", + suite: dsl.Suite{Dependencies: []dsl.Dependency{{Name: "db"}}, Tests: []dsl.Test{renderTest}}, + tier: matchers.TierE2E, + wantSkip: false, + }, + { + name: "suite hooks below simulated", + suite: dsl.Suite{BeforeAll: hook, Tests: []dsl.Test{renderTest}}, + tier: matchers.TierTemplate, + wantSkip: true, + wantPart: "beforeAll/afterAll", + }, + { + name: "per-test hooks below simulated", + suite: dsl.Suite{Tests: []dsl.Test{ + {It: "hooked", Setup: hook, Asserts: renderTest.Asserts}, + }}, + tier: matchers.TierAPIServer, + wantSkip: true, + wantPart: "setup/teardown", + }, + { + name: "plain suite is unaffected", + suite: dsl.Suite{Tests: []dsl.Test{renderTest}}, + tier: matchers.TierTemplate, + wantSkip: false, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + skip, reason := integrationFeatureSkip(&tc.suite, tc.tier) + if skip != tc.wantSkip { + t.Fatalf("integrationFeatureSkip = %v, want %v (reason %q)", skip, tc.wantSkip, reason) + } + if !skip { + return + } + if !strings.Contains(reason, tc.wantPart) { + t.Errorf("reason %q does not mention %q", reason, tc.wantPart) + } + if !strings.Contains(reason, "--cluster") { + t.Errorf("reason %q names no flag to pass", reason) + } + }) + } +} + +func TestTierAtLeast(t *testing.T) { + if !tierAtLeast(matchers.TierE2E, matchers.TierSimulated) { + t.Error("e2e must satisfy a simulated requirement") + } + if tierAtLeast(matchers.TierAPIServer, matchers.TierSimulated) { + t.Error("apiserver must not satisfy a simulated requirement") + } + if !tierAtLeast(matchers.TierTemplate, matchers.TierTemplate) { + t.Error("a tier must satisfy itself") + } + // An unknown tier provides nothing rather than accidentally satisfying a + // requirement. + if tierAtLeast("bogus", matchers.TierTemplate) { + t.Error("an unrecognised tier must not satisfy any requirement") + } +} + +func TestMatcherTierSkip_ReportsTheOffendingMatcherInsideAComposite(t *testing.T) { + // A composite's tier is the intersection of its children, so a cluster-only + // matcher nested in allOf must still be named - blaming "allOf" would leave + // the user hunting for the real cause. + asserts := []dsl.Assertion{{ + AllOf: []dsl.Assertion{ + {Equal: &dsl.PathValue{Path: "kind"}}, + {LogsContain: &dsl.LogsAssert{}}, + }, + }} + skip, reason := matcherTierSkip(asserts, matchers.TierTemplate) + if !skip { + t.Fatal("a logsContain nested in allOf must skip at the template tier") + } + if !strings.Contains(reason, `"logsContain"`) { + t.Errorf("reason %q does not name the nested matcher", reason) + } +} diff --git a/internal/yamlschema/validator.go b/internal/yamlschema/validator.go new file mode 100644 index 0000000..c726858 --- /dev/null +++ b/internal/yamlschema/validator.go @@ -0,0 +1,182 @@ +// Package yamlschema validates YAML documents against a JSON Schema +// (draft 2020-12), reporting violations at the key that caused them. +// +// It backs both schema-checked file formats: test files (internal/dsl) and +// `.vigie.yaml` (internal/config). +package yamlschema + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + "sync" + + "github.com/kaptinlin/jsonschema" + "gopkg.in/yaml.v3" +) + +// Validator compiles a JSON Schema once, on first use, and validates YAML +// documents against it. Safe for concurrent use — the runner validates test +// files in parallel. +type Validator struct { + raw []byte + + once sync.Once + compiled *jsonschema.Schema + compErr error + + emptyKeysAsAbsent bool +} + +// Option configures a Validator. +type Option func(*Validator) + +// EmptyKeysAsAbsent treats a key written with no value (`cluster:` on its own +// line) as unset instead of as an explicit null, so a block whose sub-keys are +// all commented out still validates. Without it such a key fails against an +// object-typed property. +func EmptyKeysAsAbsent() Option { + return func(v *Validator) { v.emptyKeysAsAbsent = true } +} + +// New returns a Validator for the given JSON Schema. Compilation is deferred to +// the first Validate call, so constructing one at package scope is free. +func New(schemaJSON []byte, opts ...Option) *Validator { + v := &Validator{raw: schemaJSON} + for _, opt := range opts { + opt(v) + } + return v +} + +func (v *Validator) schema() (*jsonschema.Schema, error) { + v.once.Do(func() { + v.compiled, v.compErr = jsonschema.NewCompiler().Compile(v.raw) + if v.compErr != nil { + v.compErr = fmt.Errorf("compiling schema: %w", v.compErr) + } + }) + return v.compiled, v.compErr +} + +// Validate checks raw YAML bytes against the schema. +func (v *Validator) Validate(rawYAML []byte) error { + schema, err := v.schema() + if err != nil { + return err + } + + // YAML → JSON for the validator. + var doc any + if err := yaml.Unmarshal(rawYAML, &doc); err != nil { + return fmt.Errorf("invalid YAML: %w", err) + } + normalized := normalizeForJSON(doc) + if v.emptyKeysAsAbsent { + normalized = pruneNulls(normalized) + } + jsonBytes, err := json.Marshal(normalized) + if err != nil { + return fmt.Errorf("converting to JSON: %w", err) + } + + result := schema.ValidateJSON(jsonBytes) + if result.IsValid() { + return nil + } + + // DetailedErrors drills into the failing leaf and keys each message by its + // instance location (a JSON pointer like /tests/0/asserts/0/eqaul), so a + // mistyped key points at the exact offending field instead of a vague + // top-level "does not match" rollup. + detailed := result.DetailedErrors() + if len(detailed) > 0 { + locations := make([]string, 0, len(detailed)) + for loc := range detailed { + locations = append(locations, loc) + } + sort.Strings(locations) + var msgs []string + for _, loc := range locations { + // Skip the structural rollup nodes ($ref/items/properties) that just + // say "does not match" — keep the concrete leaf violation (e.g. the + // additionalProperties error that names the offending key). + if isRollupLocation(loc) { + continue + } + where := loc + if where == "" { + where = "(root)" + } + msgs = append(msgs, fmt.Sprintf("%s: %s", where, detailed[loc])) + } + if len(msgs) > 0 { + return fmt.Errorf("schema validation errors:\n %s", strings.Join(msgs, "\n ")) + } + } + + // Fallback: the shallow, top-level errors if no leaf detail is available. + var msgs []string + for _, e := range result.Errors { + msgs = append(msgs, e.Error()) + } + return fmt.Errorf("schema validation errors:\n %s", strings.Join(msgs, "\n ")) +} + +// isRollupLocation reports whether a JSON-pointer instance location ends in a +// structural navigator keyword ($ref/items/properties). Those nodes carry only +// generic "does not match" rollups; the actionable message lives at the leaf. +func isRollupLocation(loc string) bool { + slash := strings.LastIndexByte(loc, '/') + last := loc[slash+1:] + switch last { + case "$ref", "items", "properties": + return true + } + return false +} + +// normalizeForJSON ensures all map keys are strings, as required by JSON marshaling. +func normalizeForJSON(v any) any { + switch val := v.(type) { + case map[string]any: + out := make(map[string]any, len(val)) + for k, v2 := range val { + out[k] = normalizeForJSON(v2) + } + return out + case []any: + out := make([]any, len(val)) + for i, v2 := range val { + out[i] = normalizeForJSON(v2) + } + return out + default: + return v + } +} + +// pruneNulls drops mapping entries whose value is nil, recursively. List +// elements keep their nils: only a key with nothing after it means "unset". +func pruneNulls(v any) any { + switch val := v.(type) { + case map[string]any: + out := make(map[string]any, len(val)) + for k, v2 := range val { + if v2 == nil { + continue + } + out[k] = pruneNulls(v2) + } + return out + case []any: + out := make([]any, len(val)) + for i, v2 := range val { + out[i] = pruneNulls(v2) + } + return out + default: + return v + } +} diff --git a/internal/yamlschema/validator_test.go b/internal/yamlschema/validator_test.go new file mode 100644 index 0000000..2e9b70a --- /dev/null +++ b/internal/yamlschema/validator_test.go @@ -0,0 +1,74 @@ +package yamlschema + +import ( + "strings" + "sync" + "testing" +) + +// objectSchema requires `block` to be an object when present, which is what a +// null value collides with. +const objectSchema = `{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "additionalProperties": false, + "properties": { + "block": { + "type": "object", + "additionalProperties": false, + "properties": {"key": {"type": "string"}} + } + } +}` + +func TestValidate_ReportsTheOffendingKey(t *testing.T) { + err := New([]byte(objectSchema)).Validate([]byte("block:\n ky: value\n")) + if err == nil { + t.Fatal("Validate must reject an unknown property, got nil error") + } + if !strings.Contains(err.Error(), "/block") { + t.Errorf("error %q does not point at the offending object", err) + } +} + +func TestValidate_EmptyKeyIsNullByDefault(t *testing.T) { + err := New([]byte(objectSchema)).Validate([]byte("block:\n")) + if err == nil { + t.Fatal("without EmptyKeysAsAbsent, a null block must fail an object-typed property") + } +} + +func TestValidate_EmptyKeysAsAbsentSkipsNullBlocks(t *testing.T) { + v := New([]byte(objectSchema), EmptyKeysAsAbsent()) + if err := v.Validate([]byte("block:\n")); err != nil { + t.Fatalf("EmptyKeysAsAbsent must treat `block:` as unset, got: %v", err) + } + // Pruning must not lose real violations nested under a populated block. + if err := v.Validate([]byte("block:\n key: 1\n")); err == nil { + t.Fatal("pruning nulls must not mask a wrong-typed value") + } +} + +func TestValidate_InvalidYAMLIsReportedAsSuch(t *testing.T) { + err := New([]byte(objectSchema)).Validate([]byte("block: [unclosed\n")) + if err == nil || !strings.Contains(err.Error(), "invalid YAML") { + t.Fatalf("want an invalid-YAML error, got: %v", err) + } +} + +// The runner validates test files in parallel, so a Validator shared at package +// scope must compile its schema exactly once without racing (run under -race). +func TestValidate_IsSafeForConcurrentUse(t *testing.T) { + v := New([]byte(objectSchema)) + var wg sync.WaitGroup + for range 8 { + wg.Add(1) + go func() { + defer wg.Done() + if err := v.Validate([]byte("block:\n key: value\n")); err != nil { + t.Errorf("Validate: %v", err) + } + }() + } + wg.Wait() +} diff --git a/pkg/api/schema/v1/config.json b/pkg/api/schema/v1/config.json new file mode 100644 index 0000000..86e56fd --- /dev/null +++ b/pkg/api/schema/v1/config.json @@ -0,0 +1,281 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://raw.githubusercontent.com/fregateops/vigie/refs/heads/main/pkg/api/schema/v1/config.json", + "$defs": { + "ClusterConfig": { + "properties": { + "envtest": { + "$ref": "#/$defs/EnvtestConfig", + "description": "Envtest configures the in-process apiserver backend (`--cluster envtest`)." + }, + "kind": { + "$ref": "#/$defs/NodeBackendConfig", + "description": "Kind configures the kind backend (`--cluster kind`)." + }, + "k3d": { + "$ref": "#/$defs/NodeBackendConfig", + "description": "K3d configures the k3d backend (`--cluster k3d`)." + }, + "kubeconfig": { + "$ref": "#/$defs/KubeconfigBackendConfig", + "description": "Kubeconfig configures the external-cluster backend (`--cluster kubeconfig`)." + } + }, + "additionalProperties": false, + "type": "object", + "description": "ClusterConfig groups the per-backend settings for the cluster tiers of `vigie test`." + }, + "Defaults": { + "properties": { + "release": { + "$ref": "#/$defs/ReleaseDefaults", + "description": "Release is the release identity passed to `helm template`." + } + }, + "additionalProperties": false, + "type": "object", + "description": "Defaults holds the values every test inherits unless a suite-level `defaults:` or a per-test `inputs:` block overrides them." + }, + "EnvtestConfig": { + "properties": { + "kubeVersion": { + "type": "string", + "description": "KubeVersion pins the envtest binary asset version. Empty uses the\nbuilt-in default. Overridden by `--kube-version`." + } + }, + "additionalProperties": false, + "type": "object", + "description": "EnvtestConfig configures the envtest backend, which runs a real kube-apiserver and etcd in-process with no controllers." + }, + "IgnoreRule": { + "properties": { + "rule": { + "type": "string", + "description": "Rule is the namespaced rule ID to suppress, e.g.\n`template-best-practices_missing-resource-limits`." + }, + "paths": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Paths are glob patterns matched against the finding's source file path.\nEmpty suppresses the rule everywhere." + } + }, + "additionalProperties": false, + "type": "object", + "description": "IgnoreRule suppresses a specific rule, optionally scoped to file paths." + }, + "KubeconfigBackendConfig": { + "properties": { + "path": { + "type": "string", + "description": "Path is the kubeconfig file to reach the cluster with. Mirrors\n`helm --kubeconfig`. No shell expansion happens, so `~` is not a home\ndirectory here. Overridden by `--kubeconfig`." + } + }, + "additionalProperties": false, + "type": "object", + "description": "KubeconfigBackendConfig configures the kubeconfig backend, which runs against a cluster the user already operates." + }, + "LintConfig": { + "properties": { + "ruleSets": { + "items": { + "type": "string" + }, + "type": "array", + "description": "RuleSets is an allowlist of rule sets to run. Empty means \"all defaults\"." + }, + "disableRules": { + "items": { + "type": "string" + }, + "type": "array", + "description": "DisableRules is a denylist of individual rule IDs that must not run, even\nif their rule set is enabled. Distinct from Ignore, which filters\nfindings post-execution by path." + }, + "kubeVersions": { + "items": { + "type": "string" + }, + "type": "array", + "description": "KubeVersions lists the Kubernetes versions to render against for\ndeprecation and version-aware rules. Each is a `MAJOR.MINOR` string.\nEmpty means \"all supported versions\"." + }, + "ignore": { + "items": { + "$ref": "#/$defs/IgnoreRule" + }, + "type": "array", + "description": "Ignore suppresses findings for a rule, optionally scoped to file paths." + } + }, + "additionalProperties": false, + "type": "object", + "description": "LintConfig controls which rule sets run and what to ignore." + }, + "NodeBackendConfig": { + "properties": { + "kubeVersion": { + "type": "string", + "description": "KubeVersion pins the node image version. Empty uses the CLI's default.\nOverridden by `--kube-version`." + }, + "binary": { + "type": "string", + "description": "Binary is the path to the backend's CLI. Empty resolves it from PATH,\nthen the vigie cache, then an optional download. Overridden by\n`--kind-binary` / `--k3d-binary`." + }, + "extraArgs": { + "items": { + "type": "string" + }, + "type": "array", + "description": "ExtraArgs are additional flags passed verbatim to the provisioning CLI.\nExample: [\"--config\", \"kind-3node.yaml\"] for kind, or\n[\"-v\", \"/host:/node\"] for k3d." + } + }, + "additionalProperties": false, + "type": "object", + "description": "NodeBackendConfig configures a node-backed backend (kind or k3d), each of which provisions a throwaway cluster by driving its external CLI." + }, + "ReleaseDefaults": { + "properties": { + "name": { + "type": "string", + "description": "Name is the release name. Mirrors Helm's `--release-name`.\nDefaults to \"release-name\"." + }, + "namespace": { + "type": "string", + "description": "Namespace is the release namespace. Mirrors Helm's `--namespace`.\nDefaults to \"default\"." + } + }, + "additionalProperties": false, + "type": "object", + "description": "ReleaseDefaults is the Helm release identity used to render every test." + }, + "RunConfig": { + "properties": { + "applyTiers": { + "items": { + "type": "string" + }, + "type": "array", + "description": "ApplyTiers selects which cluster backends `vigie run` exercises via the\napply tier. Empty/omitted means \"no apply tiers — just lint+validate+test\".\nValid values: any cluster backend type (envtest, simulated, kind, k3d,\nkubeconfig)." + } + }, + "additionalProperties": false, + "type": "object", + "description": "RunConfig controls `vigie run` — the orchestrated command that chains lint → validate → test." + }, + "TestConfig": { + "properties": { + "skipSchema": { + "type": "boolean", + "description": "SkipSchema disables the per-test kubeconform pass when true." + }, + "kubeVersions": { + "items": { + "type": "string" + }, + "type": "array", + "description": "KubeVersions used by the per-test kubeconform pass. Has no effect when\nSkipSchema is true." + }, + "testsDir": { + "type": "string", + "description": "TestsDir overrides the discovery root for `vigie test`, in every tier.\nRelative paths resolve against the chart directory. Empty falls back to\n`\u003cchart\u003e/tests`. The directory is scanned recursively for `*_test.yaml`." + }, + "cluster": { + "$ref": "#/$defs/ClusterConfig", + "description": "Cluster holds the per-backend settings for the cluster tiers." + } + }, + "additionalProperties": false, + "type": "object", + "description": "TestConfig controls `vigie test` — both the template tier (render + assert in-process) and the cluster tiers reached with `--cluster \u003cbackend\u003e`." + }, + "ValidateConfig": { + "properties": { + "valuesFiles": { + "items": { + "type": "string" + }, + "type": "array", + "description": "ValuesFiles lists value overlays to validate, layered on the chart's\n`values.yaml` (helm `-f overlay.yaml` semantics). Each entry produces one\nindependent render+kubeconform pass. Empty means \"just the baseline\nrender against values.yaml\"." + }, + "kubeVersions": { + "items": { + "type": "string" + }, + "type": "array", + "description": "KubeVersions lists Kubernetes versions to validate against. Each\n(overlay × kubeVersion) pair runs as a separate scenario." + }, + "set": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Set holds --set style key=value overrides (helm strvals semantics). Applied\nas the base layer; values files take higher priority." + }, + "setJson": { + "items": { + "type": "string" + }, + "type": "array", + "description": "SetJSON holds --set-json style key=jsonValue overrides." + }, + "setLiteral": { + "items": { + "type": "string" + }, + "type": "array", + "description": "SetLiteral holds --set-literal style key=literalString overrides (no type coercion)." + }, + "ignore": { + "items": { + "$ref": "#/$defs/ValidateIgnoreRule" + }, + "type": "array", + "description": "Ignore suppresses specific schema violations." + } + }, + "additionalProperties": false, + "type": "object", + "description": "ValidateConfig controls `vigie validate` (chart-level: render chart with values.yaml + each overlay, then run kubeconform against the rendered docs)." + }, + "ValidateIgnoreRule": { + "properties": { + "kind": { + "type": "string", + "description": "Kind is the Kubernetes kind whose findings are suppressed, e.g. `Ingress`." + }, + "name": { + "type": "string", + "description": "Name is the object name whose findings are suppressed. Empty matches any." + }, + "messageRegex": { + "type": "string", + "description": "MessageRegex further narrows the suppression to violation messages\nmatching this regular expression. Empty matches any." + } + }, + "additionalProperties": false, + "type": "object", + "description": "ValidateIgnoreRule suppresses a kubeconform finding by kind, name, and optional regex over the violation message." + } + }, + "properties": { + "defaults": { + "$ref": "#/$defs/Defaults" + }, + "lint": { + "$ref": "#/$defs/LintConfig" + }, + "validate": { + "$ref": "#/$defs/ValidateConfig" + }, + "test": { + "$ref": "#/$defs/TestConfig" + }, + "run": { + "$ref": "#/$defs/RunConfig" + } + }, + "additionalProperties": false, + "type": "object", + "title": "Vigie configuration file (.vigie.yaml)", + "description": "Config is the root of `.vigie.yaml`, the per-chart configuration file." +} diff --git a/pkg/api/schema/v1/schema.go b/pkg/api/schema/v1/schema.go index 94fe105..1d30d8c 100644 --- a/pkg/api/schema/v1/schema.go +++ b/pkg/api/schema/v1/schema.go @@ -1,9 +1,10 @@ -// Package v1 hosts the embedded JSON Schema for vigie test files. +// Package v1 hosts the embedded JSON Schemas for vigie: test files and the +// per-chart `.vigie.yaml` configuration file. // -// The schema is generated from the Go structs under internal/dsl by the -// tooling at tools/gen-schema/. Run `go generate ./pkg/api/schema/v1/...` -// (or `make check-schema`) to regenerate testfile.json after editing -// internal/dsl/types.go. +// Both are generated by the tooling at tools/gen-schema/ — testfile.json from +// the Go structs under internal/dsl, config.json from those under +// internal/config. Run `go generate ./pkg/api/schema/v1/...` (or +// `make check-schema`) to regenerate them after editing either package's types. package v1 import _ "embed" @@ -12,3 +13,6 @@ import _ "embed" //go:embed testfile.json var TestFileSchema []byte + +//go:embed config.json +var ConfigSchema []byte diff --git a/pkg/api/schema/v1/testfile.json b/pkg/api/schema/v1/testfile.json index 23f547e..5ce1dc9 100644 --- a/pkg/api/schema/v1/testfile.json +++ b/pkg/api/schema/v1/testfile.json @@ -1017,20 +1017,6 @@ "type": "string", "description": "Human-readable description of the scenario." }, - "tier": { - "items": { - "type": "string" - }, - "type": "array", - "description": "Tiers this test applies to. Default: [template, validate]." - }, - "tags": { - "items": { - "type": "string" - }, - "type": "array", - "description": "Arbitrary labels for filtering tests." - }, "skip": { "description": "Skip condition — boolean or CEL expression string." }, diff --git a/testdata/charts/basic/.vigie.yaml b/testdata/charts/basic/.vigie.yaml index dbeb3ab..8ea538a 100644 --- a/testdata/charts/basic/.vigie.yaml +++ b/testdata/charts/basic/.vigie.yaml @@ -12,6 +12,7 @@ lint: disableRules: - template-best-practices_missing-resource-limits -test: - # Unit tests live under tests/unit (also the default discovery root). - testsDir: tests/unit +# `test:` is omitted on purpose: every test file lives under the default +# discovery root `/tests`, scanned recursively. The unit/ and +# integration/ sub-directories are organisation only — the tier a file runs in +# comes from its content and the active `--cluster`, never from its path. diff --git a/testdata/charts/basic/tests/integration/deps_test.yaml b/testdata/charts/basic/tests/integration/deps_test.yaml index edeb733..44df09d 100644 --- a/testdata/charts/basic/tests/integration/deps_test.yaml +++ b/testdata/charts/basic/tests/integration/deps_test.yaml @@ -16,7 +16,7 @@ suite: dependency ordering and lifecycle hooks # teardown every test shares. cluster: - backend: kubeconfig + backend: kind defaults: release: diff --git a/testdata/charts/tier-gate/Chart.yaml b/testdata/charts/tier-gate/Chart.yaml new file mode 100644 index 0000000..f8acdfc --- /dev/null +++ b/testdata/charts/tier-gate/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v2 +name: tier-gate +description: Fixture for tier gating - one render-only test and one cluster-only test. +version: 0.1.0 +appVersion: "1.0.0" diff --git a/testdata/charts/tier-gate/templates/deployment.yaml b/testdata/charts/tier-gate/templates/deployment.yaml new file mode 100644 index 0000000..36ee766 --- /dev/null +++ b/testdata/charts/tier-gate/templates/deployment.yaml @@ -0,0 +1,21 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-app +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app: {{ .Release.Name }}-app + template: + metadata: + labels: + app: {{ .Release.Name }}-app + spec: + containers: + - name: app + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + resources: + limits: + cpu: 100m + memory: 128Mi diff --git a/testdata/charts/tier-gate/tests/mixed_test.yaml b/testdata/charts/tier-gate/tests/mixed_test.yaml new file mode 100644 index 0000000..080728c --- /dev/null +++ b/testdata/charts/tier-gate/tests/mixed_test.yaml @@ -0,0 +1,22 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/fregateops/vigie/refs/heads/main/pkg/api/schema/v1/testfile.json +# +# One suite spanning two tiers. At the template tier the render assertion runs +# and the `applies` assertion is skipped with the flag that would run it; at a +# cluster tier both run. Guards the tier gate in both runners. +suite: mixed tiers +templates: + - templates/deployment.yaml +tests: + - it: renders the workload name + target: + kind: Deployment + asserts: + - equal: + path: metadata.name + value: release-name-app + + - it: the Deployment is admitted by the API server + target: + kind: Deployment + asserts: + - applies: {} diff --git a/testdata/charts/tier-gate/values.yaml b/testdata/charts/tier-gate/values.yaml new file mode 100644 index 0000000..0ca97cf --- /dev/null +++ b/testdata/charts/tier-gate/values.yaml @@ -0,0 +1,4 @@ +replicaCount: 1 +image: + repository: myrepo/myapp + tag: "1.0.0" diff --git a/tools/gen-schema/main.go b/tools/gen-schema/main.go index 57d492c..3d5f70c 100644 --- a/tools/gen-schema/main.go +++ b/tools/gen-schema/main.go @@ -1,6 +1,9 @@ -// Command gen-schema generates pkg/api/schema/v1/testfile.json from the -// Go DSL types in internal/dsl using github.com/invopop/jsonschema with -// AddGoComments — field doc comments become JSON Schema descriptions. +// Command gen-schema generates the JSON Schemas under pkg/api/schema/v1 from +// their Go source types using github.com/invopop/jsonschema with AddGoComments +// — field doc comments become JSON Schema descriptions. +// +// Two schemas ship: testfile.json from the DSL types in internal/dsl, and +// config.json from the `.vigie.yaml` types in internal/config. package main import ( @@ -12,9 +15,23 @@ import ( "github.com/invopop/jsonschema" + "github.com/fregateops/vigie/internal/config" "github.com/fregateops/vigie/internal/dsl" ) +// schemaBaseURL is where the published schemas are fetched from by editors, so +// a `$schema=` modeline resolves. It points at the raw files on main. +const schemaBaseURL = "https://raw.githubusercontent.com/fregateops/vigie/refs/heads/main/pkg/api/schema/v1" + +// target describes one generated schema: the root Go type, the package whose +// doc comments annotate it, and the file it is written to. +type target struct { + file string + title string + pkgDir string + subject any +} + func main() { if err := run(); err != nil { fmt.Fprintln(os.Stderr, "gen-schema:", err) @@ -27,24 +44,46 @@ func run() error { return fmt.Errorf("chdir to repo root: %w", err) } + targets := []target{ + { + file: "testfile.json", + title: "Vigie test file", + pkgDir: "./internal/dsl", + subject: &dsl.Suite{}, + }, + { + file: "config.json", + title: "Vigie configuration file (.vigie.yaml)", + pkgDir: "./internal/config", + subject: &config.Config{}, + }, + } + for _, t := range targets { + if err := generate(t); err != nil { + return fmt.Errorf("%s: %w", t.file, err) + } + } + return nil +} + +func generate(t target) error { reflector := &jsonschema.Reflector{ ExpandedStruct: true, } - if err := reflector.AddGoComments("github.com/fregateops/vigie", "./internal/dsl"); err != nil { + if err := reflector.AddGoComments("github.com/fregateops/vigie", t.pkgDir); err != nil { return fmt.Errorf("loading comments: %w", err) } - schema := reflector.Reflect(&dsl.Suite{}) - // $id must resolve so editors can fetch it via a `$schema=` modeline. Point - // it at the raw file on main, matching the file's actual path/name. - schema.ID = "https://raw.githubusercontent.com/fregateops/vigie/refs/heads/main/pkg/api/schema/v1/testfile.json" - schema.Title = "Vigie test file" + schema := reflector.Reflect(t.subject) + // $id must resolve so editors can fetch it via a `$schema=` modeline. + schema.ID = jsonschema.ID(fmt.Sprintf("%s/%s", schemaBaseURL, t.file)) + schema.Title = t.title out, err := json.MarshalIndent(schema, "", " ") if err != nil { return fmt.Errorf("marshal: %w", err) } - return os.WriteFile(outputPath(), append(out, '\n'), 0o644) + return os.WriteFile(outputPath(t.file), append(out, '\n'), 0o644) } // main.go sits at tools/gen-schema/main.go — exactly 3 levels below the repo root. @@ -60,6 +99,6 @@ func repoRoot() string { return dir } -func outputPath() string { - return filepath.Join("pkg", "api", "schema", "v1", "testfile.json") +func outputPath(file string) string { + return filepath.Join("pkg", "api", "schema", "v1", file) }