From e4e419caec5c7dcacd4988866a563a61dd1172b1 Mon Sep 17 00:00:00 2001 From: Kevin Vu Date: Thu, 30 Oct 2025 02:17:08 -0700 Subject: [PATCH 1/2] testing: include file-level errors in JUnit skipped elements (#37806) * testing: include file-level errors in JUnit skipped elements Fixes #37801 * testing: use suite-level system-err for file errors --- .changes/v1.15/BUG FIXES-20251024-042900.yaml | 5 +++ internal/command/junit/junit.go | 26 ++++++++++- internal/command/junit/junit_test.go | 43 +++++++++++++++++++ 3 files changed, 72 insertions(+), 2 deletions(-) create mode 100644 .changes/v1.15/BUG FIXES-20251024-042900.yaml diff --git a/.changes/v1.15/BUG FIXES-20251024-042900.yaml b/.changes/v1.15/BUG FIXES-20251024-042900.yaml new file mode 100644 index 000000000000..989ac31a94e7 --- /dev/null +++ b/.changes/v1.15/BUG FIXES-20251024-042900.yaml @@ -0,0 +1,5 @@ +kind: BUG FIXES +body: 'testing: File-level error diagnostics are now included in JUnit XML skipped test elements, ensuring CI/CD pipelines can detect validation failures' +time: 2025-10-24T04:29:00.000000Z +custom: + Issue: "37801" diff --git a/internal/command/junit/junit.go b/internal/command/junit/junit.go index c62c51d5816d..0d7f4ffabd68 100644 --- a/internal/command/junit/junit.go +++ b/internal/command/junit/junit.go @@ -188,6 +188,9 @@ func junitXMLTestReport(suite *moduletest.Suite, suiteRunnerStopped bool, source }, }) + // Check if there are file-level errors that will be reported at suite level + hasFileLevelErrors := file.Status == moduletest.Error && file.Diagnostics.HasErrors() + for i, run := range file.Runs { // Each run is a "test case". @@ -209,7 +212,7 @@ func junitXMLTestReport(suite *moduletest.Suite, suiteRunnerStopped bool, source // Depending on run status, add either of: "skipped", "failure", or "error" elements switch run.Status { case moduletest.Skip: - testCase.Skipped = skipDetails(i, file, suiteRunnerStopped) + testCase.Skipped = skipDetails(i, file, suiteRunnerStopped, hasFileLevelErrors) case moduletest.Fail: // When the test fails we only use error diags that originate from failing assertions @@ -275,6 +278,16 @@ func junitXMLTestReport(suite *moduletest.Suite, suiteRunnerStopped bool, source }) } + // Add suite-level system-err if there are file-level errors + if hasFileLevelErrors { + systemErr := &withMessage{ + Body: getDiagString(file.Diagnostics, sources), + } + enc.EncodeElement(systemErr, xml.StartElement{ + Name: xml.Name{Local: "system-err"}, + }) + } + enc.EncodeToken(xml.EndElement{Name: suiteName}) } enc.EncodeToken(xml.EndElement{Name: suitesName}) @@ -300,8 +313,9 @@ func failureMessage(failedAssertions tfdiags.Diagnostics, checkCount int) string // Test can be skipped due to: // 1. terraform test recieving an interrupt from users; all unstarted tests will be skipped // 2. A previous run in a file has failed, causing subsequent run blocks to be skipped +// 3. File-level errors (e.g., invalid variable references) causing all tests to be skipped // The returned value is used to set content in the "skipped" element -func skipDetails(runIndex int, file *moduletest.File, suiteStopped bool) *withMessage { +func skipDetails(runIndex int, file *moduletest.File, suiteStopped bool, hasFileLevelErrors bool) *withMessage { if suiteStopped { // Test suite experienced an interrupt // This block only handles graceful Stop interrupts, as Cancel interrupts will prevent a JUnit file being produced at all @@ -323,6 +337,14 @@ func skipDetails(runIndex int, file *moduletest.File, suiteStopped bool) *withMe } } } + + // Check for file-level error diagnostics that caused tests to be skipped + // Note: Full diagnostic details are included in suite-level element + if hasFileLevelErrors { + return &withMessage{ + Message: "Testcase skipped due to file-level errors", + } + } } // Unhandled case: This results in with no attributes or body diff --git a/internal/command/junit/junit_test.go b/internal/command/junit/junit_test.go index 956571e56184..aace508351b8 100644 --- a/internal/command/junit/junit_test.go +++ b/internal/command/junit/junit_test.go @@ -12,6 +12,7 @@ import ( "github.com/hashicorp/terraform/internal/command/junit" "github.com/hashicorp/terraform/internal/configs/configload" "github.com/hashicorp/terraform/internal/moduletest" + "github.com/hashicorp/terraform/internal/tfdiags" ) // This test cannot access sources when contructing output for XML files. Due to this, the majority of testing @@ -114,6 +115,48 @@ func Test_TestJUnitXMLFile_Save(t *testing.T) { +`), + }, + "suite-level includes file-level error diagnostics when tests are skipped": { + filename: "output.xml", + runner: &local.TestSuiteRunner{}, + suite: func() moduletest.Suite { + file := &moduletest.File{ + Name: "file1.tftest.hcl", + Status: moduletest.Error, + Runs: []*moduletest.Run{ + { + Name: "my_test", + Status: moduletest.Skip, + }, + }, + } + // Simulate file-level error diagnostic (e.g., invalid variable reference) + var diags tfdiags.Diagnostics + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "Invalid reference", + "You can only reference global variables within the test file variables block.", + )) + file.AppendDiagnostics(diags) + return moduletest.Suite{ + Status: moduletest.Error, + Files: map[string]*moduletest.File{ + "file1.tftest.hcl": file, + }, + } + }(), + expectedOuput: []byte(` + + + + + + `), }, } From d7563c385bb7dd2af2a700b4a4f9027a6dff15d7 Mon Sep 17 00:00:00 2001 From: Sarah French Date: Thu, 30 Oct 2025 09:28:25 +0000 Subject: [PATCH 2/2] Move original change file to v1.14 folder --- .changes/{v1.15 => v1.14}/BUG FIXES-20251024-042900.yaml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .changes/{v1.15 => v1.14}/BUG FIXES-20251024-042900.yaml (100%) diff --git a/.changes/v1.15/BUG FIXES-20251024-042900.yaml b/.changes/v1.14/BUG FIXES-20251024-042900.yaml similarity index 100% rename from .changes/v1.15/BUG FIXES-20251024-042900.yaml rename to .changes/v1.14/BUG FIXES-20251024-042900.yaml