feat: Improve linear solvers testing (#4108)

* Add hypre/hypredrive regression tests covering most MGR strategies
* Compare hypredrive and legacy hypre paths against common baselines
* Add iteration-count parity checks to detect solver quality regressions
* Add CI support for hypredrive and forced-legacy test passes
* Add warnings and unit tests for hypredrive fallback and numerical equivalence
* Fix hypredrive solver integration and configuration inconsistencies
* Make ILU configuration consistent between legacy hypre and hypredrive paths
* Fix MGR configurations for hybrid-FVM and reservoir solver strategies
* Add ILU F-relaxation support for MGR
* Fix issues uncovered by the expanded solver test coverage and update documentation/baselines

Author: victorapm

.github/workflows/build_and_test.yml

@@ -110,7 +110,11 @@ jobs:
       env:
         GITHUB_TOKEN: ${{ github.token }}
       run: |
+        # The device jobs query the same PR endpoint concurrently. Retry
+        # transient GitHub API throttling so an optional build does not turn a
+        # healthy matrix into a failure before compilation starts.
         pr_json=$(curl --fail --silent --show-error \
+          --retry 5 --retry-all-errors --retry-delay 5 --retry-max-time 90 \
           -H "Accept: application/vnd.github+json" \
           -H "Authorization: Bearer ${GITHUB_TOKEN}" \
           https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.number }})
@@ -387,6 +391,7 @@ jobs:
       run: |
         EXIT_STATUS=${GEOS_BUILD_EXIT_STATUS:-1}
         DATA_BASENAME=${GEOS_DATA_BASENAME}
+        DATA_BASENAME_WE=${DATA_BASENAME%%.*}
         DATA_EXCHANGE_DIR=${GEOS_DATA_EXCHANGE_DIR}
         UPLOAD_HEADER_PRINTED=false
         GCP_CREDENTIALS_STATUS="not set"
@@ -399,8 +404,8 @@ jobs:
             echo "Generated artifact at ${DATA_EXCHANGE_DIR}/${DATA_BASENAME}"
           fi
 
-          if [ -f "${DATA_EXCHANGE_DIR}/test_logs_${DATA_BASENAME}" ]; then
-            echo "Generated integrated test logs at ${DATA_EXCHANGE_DIR}/test_logs_${DATA_BASENAME}"
+          if [ -f "${DATA_EXCHANGE_DIR}/test_logs_${DATA_BASENAME_WE}.tar.gz" ]; then
+            echo "Generated integrated test logs at ${DATA_EXCHANGE_DIR}/test_logs_${DATA_BASENAME_WE}.tar.gz"
           fi
 
           if [[ -n "${{ inputs.LOCAL_BASELINE_DIR }}" ]]; then
@@ -409,8 +414,8 @@ jobs:
             mkdir -p "${TARGET_DIR}"
 
             for file in \
-              "${DATA_EXCHANGE_DIR}/baseline_${DATA_BASENAME}" \
-              "${DATA_EXCHANGE_DIR}/test_logs_${DATA_BASENAME}"
+              "${DATA_EXCHANGE_DIR}/baseline_${DATA_BASENAME_WE}.tar.gz" \
+              "${DATA_EXCHANGE_DIR}/test_logs_${DATA_BASENAME_WE}.tar.gz"
             do
               if [ -f "${file}" ]; then
                 cp "${file}" "${TARGET_DIR}"
@@ -419,11 +424,10 @@ jobs:
             done
           fi
 
-          if [ ! -f "${DATA_EXCHANGE_DIR}/baseline_${DATA_BASENAME}" ]; then
+          if [ ! -f "${DATA_EXCHANGE_DIR}/baseline_${DATA_BASENAME_WE}.tar.gz" ]; then
             echo "No new baseline archive was generated."
           fi
         fi
-
         if [[ "${{ inputs.UPLOAD_TO_GCP }}" == "true" && -n "${{ inputs.GCP_BUCKET }}" ]]; then
           if [[ "${{ inputs.BUILD_TYPE }}" = "integrated_tests" || ${EXIT_STATUS} -eq 0 ]]; then
             if ! CLOUDSDK_PYTHON=python3 gcloud config set storage/parallel_composite_upload_enabled True; then
@@ -434,8 +438,8 @@ jobs:
 
             for file in \
               "${DATA_EXCHANGE_DIR}/${DATA_BASENAME}" \
-              "${DATA_EXCHANGE_DIR}/test_logs_${DATA_BASENAME}" \
-              "${DATA_EXCHANGE_DIR}/baseline_${DATA_BASENAME}"
+              "${DATA_EXCHANGE_DIR}/test_logs_${DATA_BASENAME_WE}.tar.gz" \
+              "${DATA_EXCHANGE_DIR}/baseline_${DATA_BASENAME_WE}.tar.gz"
             do
               if [ -f "${file}" ]; then
                 case "$(basename "${file}")" in

.github/workflows/ci_tests.yml

@@ -275,6 +275,7 @@ jobs:
       DOCKER_IMAGE_TAG: ${{ needs.is_not_draft_pull_request.outputs.DOCKER_IMAGE_TAG }}
       DOCKER_REPOSITORY: geosx/ubuntu24.04-gcc12
       ENABLE_HYPRE: ON
+      ENABLE_HYPREDRV: ON
       ENABLE_TRILINOS: OFF
       GEOS_ENABLE_BOUNDS_CHECK: ON
       GCP_BUCKET: geosx/integratedTests
@@ -455,7 +456,7 @@ jobs:
             DOCKER_REPOSITORY: geosx/ubuntu24.04-amdclang19.0.0-rocm6.4.3
             ENABLE_HYPRE_DEVICE: HIP
             ENABLE_HYPRE: ON
-            ENABLE_HYPREDRV: OFF
+            ENABLE_HYPREDRV: ON
             ENABLE_TRILINOS: OFF
             GEOS_ENABLE_BOUNDS_CHECK: OFF
             RUNS_ON: streak2

.integrated_tests.yaml

@@ -1,6 +1,6 @@
 baselines:
   bucket: geosx
-  baseline: integratedTests/baseline_integratedTests-pr3994-17525-4ae3593
+  baseline: integratedTests/baseline_integratedTests-pr4108-17561-eb62184
 
 allow_fail:
   all: ''

BASELINE_NOTES.md

@@ -17,6 +17,12 @@ PR #4083 (2026-09-01) <https://storage.googleapis.com/geosx/integratedTests/base
 =====================
 Dispatch scaling wrt to fracture element type for ALM contact solver
 
+PR #4127 (2026-08-28) <https://storage.googleapis.com/geosx/integratedTests/baseline_integratedTests-pr4127-17440-e6bc364.tar.gz>
+=====================
+Refresh the integrated-test baseline after the Hypre/TPL update and the
+iterative test additions. The prior archive did not contain restart outputs
+for the iterative cases and caused ATS baseline-missing failures.
+
 PR #4096 (2026-08-27) <https://storage.googleapis.com/geosx/integratedTests/baseline_integratedTests-pr4096-17439-18734f3.tar.gz>
 =====================
 Refactor the phase-field module and add new benchmarks for baseline
@@ -37,6 +43,10 @@ PR #4088 (2026-07-27) <https://storage.googleapis.com/geosx/integratedTests/base
 =====================
 Fluid reset after convergence failure
 
+PR #3972 (2026-08-DD) <https://storage.googleapis.com/geosx/integratedTests/baseline_integratedTests-pr4108-17194-fec5349.tar.gz>
+=====================
+Add MGR tests to ATS
+
 PR #3972 (2026-07-28) <https://storage.googleapis.com/geosx/integratedTests/baseline_integratedTests-pr3972-17154-316e6d8.tar.gz>
 =====================
 Well model refactor .  Integrated test update due to schema changes

inputFiles/compositionalMultiphaseFlow/compositionalMultiphaseFlow.ats

@@ -77,6 +77,14 @@ decks = [
         restart_step=28,
         check_step=38,
         restartcheck_params=RestartcheckParameters(**restartcheck_params)),
+    TestDeck(
+        name="deadoil_3ph_staircase_3d_iterative",
+        description=
+        "hypre MGR variant (exercises the hypredrive path when available)",
+        partitions=((1, 1, 1), (2, 2, 2)),
+        restart_step=28,
+        check_step=38,
+        restartcheck_params=RestartcheckParameters(atol=1e-6, rtol=1e-5)),
     TestDeck(
         name="deadoil_3ph_staircase_hybrid_3d",
         description=
@@ -85,6 +93,14 @@ decks = [
         restart_step=28,
         check_step=38,
         restartcheck_params=RestartcheckParameters(**restartcheck_params)),
+    TestDeck(
+        name="deadoil_3ph_staircase_hybrid_3d_iterative",
+        description=
+        "hypre MGR variant (exercises the hypredrive path when available)",
+        partitions=((1, 1, 1), (2, 2, 2)),
+        restart_step=28,
+        check_step=38,
+        restartcheck_params=RestartcheckParameters(atol=1e-6, rtol=1e-5)),
     TestDeck(
         name="deadoil_3ph_staircase_obl_3d",
         description=
@@ -93,6 +109,14 @@ decks = [
         restart_step=28,
         check_step=38,
         restartcheck_params=RestartcheckParameters(**restartcheck_params)),
+    TestDeck(
+        name="deadoil_3ph_staircase_obl_3d_iterative",
+        description=
+        "hypre MGR variant (exercises the hypredrive path when available)",
+        partitions=((1, 1, 1), (2, 2, 2)),
+        restart_step=28,
+        check_step=38,
+        restartcheck_params=RestartcheckParameters(atol=1e-6, rtol=1e-5)),
     TestDeck(
         name="deadoil_2ph_staircase_gravity_segregation_3d",
         description=