Use specific exit status for pytest test failures be able to retry infra failures

conftest.py

@@ -36,6 +36,18 @@
 _deselected_items: list[pytest.Item] = []
 setup_properties = SetupProperties()
 
+# Exit code returned by pytest when at least one test failed.
+# It is distinct from pytest's default exit codes (0-5) so that CI can tell
+# a test failure apart from other errors.
+TEST_FAIL_EXIT_CODE = 42
+
+# Track which test phases (setup / call / teardown) had failures, so that
+# we only use TEST_FAIL_EXIT_CODE when failures occurred exclusively during
+# the test ``call`` phase.  Setup/teardown failures are typically
+# infrastructure issues (Docker, network, ...) and should remain retryable
+# by CI (exit code 1).
+_failed_phases: set[str] = set()
+
 PytestOutcome = Literal["passed", "xpassed", "failed", "xfailed", "skipped", "error"]
 
 
@@ -528,6 +540,11 @@ def pytest_runtest_makereport(item: pytest.Item, call: pytest.CallInfo) -> Gener
         _set_outcome_properties(value, item.user_properties)
 
 
+def pytest_runtest_logreport(report: pytest.TestReport) -> None:
+    if report.failed:
+        _failed_phases.add(report.when)
+
+
 def _set_outcome_properties(outcome: PytestOutcome, user_properties: list[tuple]) -> None:
     if outcome in ("passed", "xpassed"):
         final_status = "pass"
@@ -564,6 +581,14 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None:
         exitstatus = pytest.ExitCode.OK
         session.exitstatus = pytest.ExitCode.OK
 
+    if exitstatus == pytest.ExitCode.TESTS_FAILED:
+        # Only use the dedicated test-failure exit code when every failure
+        # happened during the test ``call`` phase.  If any setup or teardown
+        # phase also failed, keep the default exit code (1) so that CI can
+        # retry the job — those failures are usually infrastructure issues.
+        if _failed_phases and _failed_phases <= {"call"}:
+            session.exitstatus = TEST_FAIL_EXIT_CODE
+
     if session.config.option.collectonly:
         return
 

tests/test_the_test/test_junit.py

@@ -55,7 +55,10 @@ def test_force_skip(self): ...
 @scenarios.test_the_test
 @pytest.mark.parametrize("use_xdist", [True, False])
 def test_main(use_xdist: bool):  # noqa: FBT001
-    run_system_tests(test_path="tests/test_the_test/test_junit.py", use_xdist=use_xdist, expected_return_code=1)
+    test_failure_exit_code = 42
+    run_system_tests(
+        test_path="tests/test_the_test/test_junit.py", use_xdist=use_xdist, expected_return_code=test_failure_exit_code
+    )
 
     observed_file = "logs_mock_the_test/reportJunit.xml"
     expected_file = "tests/test_the_test/reportJunit_expected.xml"

tests/test_the_test/test_strict.py

@@ -8,7 +8,8 @@
 @scenarios.test_the_test
 class Test_StrictMode:
     def test_strict_missing_features(self):
-        tests = run_system_tests(test_path=FILENAME, xfail_strict=True, expected_return_code=1)
+        test_failure_exit_code = 42
+        tests = run_system_tests(test_path=FILENAME, xfail_strict=True, expected_return_code=test_failure_exit_code)
 
         assert tests[f"{FILENAME}::test_strict_bug"]["outcome"] == "failed"
         assert tests[f"{FILENAME}::test_strict_missing_feature"]["outcome"] == "failed"

utils/ci/gitlab/system-tests.yml.j2

@@ -55,7 +55,9 @@ default:
       - unknown_failure
       - data_integrity_failure
     exit_codes:
-      - 255
+{% for code in range(1, 42) %}      - {{ code }}
+{% endfor %}{% for code in range(43, 256) %}      - {{ code }}
+{% endfor %}
 
 stages:
   - {{stage}}