PhilipWoulfe · PhilipWoulfe · Mar 22, 2026 · Mar 22, 2026 · Mar 22, 2026 · Mar 22, 2026
diff --git a/.env.example b/.env.example
@@ -54,6 +54,8 @@ DATA_SYNC_CONTINUE_ON_ERROR=false
 # Cloudflare Access audience (aud claim) expected by JWT validation.
 # Required.
 CLOUDFLARE_AUDIENCE=
+# Cloudflare Access issuer used by API JWT validation.
+CLOUDFLARE_ISSUER=https://f1-team.cloudflareaccess.com
 
 # Cloudflare claim type that carries group membership and the admin groups that grant Admin role.
 ADMIN_GROUP_CLAIM_TYPE=groups
@@ -72,6 +74,8 @@ CLOUDFLARE_TEST_SERVICE_TOKEN_EMAIL_DOMAIN=test.local
 # Host path inside the LXC where API log files will be stored.
 # Recommended: bind-mount a RAID-backed Proxmox dataset to /mnt/f1-logs and point HOST_LOG_PATH there.
 HOST_LOG_PATH=/mnt/f1-logs
+# Minimum free disk space required by scripts/deploy-preflight.sh before deploy.
+DEPLOY_MIN_FREE_DISK_MB=1024
 
 # Development-only mock identity used when the API simulates Cloudflare locally.
 DEV_MOCK_EMAIL=dev-user@example.com
@@ -93,4 +97,12 @@ E2E_STEP_TRACE_PATH=
 
 # --- Cloudflare Tunnel (Optional) ---
 # Used for the 'cloud' profile to expose the service publicly.
+# Profile examples:
+# - Test: COMPOSE_PROFILES=cloud
+# - Production: COMPOSE_PROFILES=cloud
+# COMPOSE_PROFILES=cloud
 # TUNNEL_TOKEN=
+
+# --- Tailscale (Out-of-Band Deploy/Admin Access) ---
+# Tailscale is expected to provide private VM access for deploy/admin traffic.
+# No compose variable is required yet; deployment workflow wiring will use the VM tailnet address/hostname.
diff --git a/.github/env/production.secrets.env.example b/.github/env/production.secrets.env.example
@@ -0,0 +1,9 @@
+# GitHub Environment secrets for production
+# Copy to production.secrets.env and fill real values before running scripts/github-sync-production-env.sh
+# Keep this file out of source control when populated with real secrets.
+
+TAILSCALE_OAUTH_CLIENT_ID=replace-me
+TAILSCALE_OAUTH_SECRET=replace-me
+VM_SSH_PRIVATE_KEY="-----BEGIN OPENSSH PRIVATE KEY-----\nreplace-me\n-----END OPENSSH PRIVATE KEY-----"
+VM_ENV_FILE="TAG=stable\nCONTAINER_NAME=f1-prod\nHOST_PORT=5000\nHOST_PORT_WEB=5001\nALLOWED_ORIGINS=https://f1.philipwoulfe.com\nBLAZOR_ENVIRONMENT=Production\nPOSTGRES_DB=f1competition\nPOSTGRES_USER=f1\nPOSTGRES_PASSWORD=replace-me\nPOSTGRES_PORT=5432\nDB_AUTO_MIGRATE=true\nDATA_SYNC_INTERVAL_MINUTES=60\nDATA_SYNC_AUTO_MIGRATE=false\nDATA_SYNC_HTTP_RETRY_COUNT=3\nDATA_SYNC_HTTP_RETRY_DELAY_MS=750\nDATA_SYNC_DEADLINE_MINUTES_BEFORE_START=30\nDATA_SYNC_JOLPICA_BASE_URL=https://api.jolpi.ca/ergast/f1/\nDATA_SYNC_CONTINUE_ON_ERROR=false\nCLOUDFLARE_AUDIENCE=replace-me\nCLOUDFLARE_ISSUER=https://f1-team.cloudflareaccess.com\nADMIN_GROUP_CLAIM_TYPE=groups\nADMIN_GROUPS=F1 Admins\nADMIN_EMAILS=\nCLOUDFLARE_ENABLE_TEST_SERVICE_TOKEN_FALLBACK=false\nCLOUDFLARE_TEST_SERVICE_TOKEN_SUBJECT_ALLOWLIST=\nCLOUDFLARE_TEST_SERVICE_TOKEN_ADMIN_SUBJECT_ALLOWLIST=\nCLOUDFLARE_TEST_SERVICE_TOKEN_EMAIL_DOMAIN=test.local\nHOST_LOG_PATH=/mnt/f1-logs\nDEPLOY_MIN_FREE_DISK_MB=1024\nDEV_MOCK_EMAIL=dev-user@example.com\nDEV_MOCK_GROUPS=F1 Admins,F1 Users\nDEV_ENABLE_DEBUG_ENDPOINTS=false"
+GHCR_READ_TOKEN=replace-me
diff --git a/.github/env/production.vars.env.example b/.github/env/production.vars.env.example
@@ -0,0 +1,7 @@
+# GitHub Environment variables for production
+# Copy to production.vars.env and fill real values before running scripts/github-sync-production-env.sh
+
+VM_TAILSCALE_HOST=replace-with-vm-tailnet-hostname-or-ip
+VM_DEPLOY_USER=deploy
+VM_DEPLOY_PATH=/opt/f1competition
+GHCR_USERNAME=replace-with-ghcr-username
diff --git a/.github/env/test.secrets.env.example b/.github/env/test.secrets.env.example
@@ -0,0 +1,10 @@
+# GitHub Environment secrets for test
+# Copy to test.secrets.env and fill real values before running scripts/github-sync-production-env.sh
+
+TAILSCALE_OAUTH_CLIENT_ID=replace-me
+TAILSCALE_OAUTH_SECRET=replace-me
+VM_SSH_PRIVATE_KEY="-----BEGIN OPENSSH PRIVATE KEY-----\nreplace-me\n-----END OPENSSH PRIVATE KEY-----"
+VM_ENV_FILE="TAG=test\nCONTAINER_NAME=f1-test\nHOST_PORT=5000\nHOST_PORT_WEB=5001\nALLOWED_ORIGINS=https://f1-test.philipwoulfe.com\nBLAZOR_ENVIRONMENT=Test\nPOSTGRES_DB=f1competition\nPOSTGRES_USER=f1\nPOSTGRES_PASSWORD=replace-me\nPOSTGRES_PORT=5432\nDB_AUTO_MIGRATE=true\nDATA_SYNC_INTERVAL_MINUTES=60\nDATA_SYNC_AUTO_MIGRATE=false\nDATA_SYNC_HTTP_RETRY_COUNT=3\nDATA_SYNC_HTTP_RETRY_DELAY_MS=750\nDATA_SYNC_DEADLINE_MINUTES_BEFORE_START=30\nDATA_SYNC_JOLPICA_BASE_URL=https://api.jolpi.ca/ergast/f1/\nDATA_SYNC_CONTINUE_ON_ERROR=false\nCLOUDFLARE_AUDIENCE=replace-me\nCLOUDFLARE_ISSUER=https://f1-team.cloudflareaccess.com\nADMIN_GROUP_CLAIM_TYPE=groups\nADMIN_GROUPS=F1 Admins\nADMIN_EMAILS=\nCLOUDFLARE_ENABLE_TEST_SERVICE_TOKEN_FALLBACK=false\nCLOUDFLARE_TEST_SERVICE_TOKEN_SUBJECT_ALLOWLIST=\nCLOUDFLARE_TEST_SERVICE_TOKEN_ADMIN_SUBJECT_ALLOWLIST=\nCLOUDFLARE_TEST_SERVICE_TOKEN_EMAIL_DOMAIN=test.local\nHOST_LOG_PATH=/mnt/f1-logs\nDEPLOY_MIN_FREE_DISK_MB=1024\nDEV_MOCK_EMAIL=dev-user@example.com\nDEV_MOCK_GROUPS=F1 Admins,F1 Users\nDEV_ENABLE_DEBUG_ENDPOINTS=false"
+GHCR_READ_TOKEN=replace-me
+E2E_CF_CLIENT_ID=replace-me
+E2E_CF_CLIENT_SECRET=replace-me
diff --git a/.github/env/test.vars.env.example b/.github/env/test.vars.env.example
@@ -0,0 +1,10 @@
+# GitHub Environment variables for test
+# Copy to test.vars.env and fill real values before running scripts/github-sync-production-env.sh
+
+VM_TAILSCALE_HOST=replace-with-test-vm-tailnet-hostname-or-ip
+VM_DEPLOY_USER=deploy
+VM_DEPLOY_PATH=/opt/f1competition
+GHCR_USERNAME=replace-with-ghcr-username
+E2E_BASE_URL=https://f1-test.philipwoulfe.com
+E2E_API_BASE_URL=https://f1-api-test.philipwoulfe.com
+E2E_RACE_ID=2025-24-yas_marina
diff --git a/.github/workflows/docker-build.yaml b/.github/workflows/docker-build.yaml
@@ -43,6 +43,18 @@ jobs:
         with:
           dotnet-version: 8.0.x
 
+      - name: Validate test environment configuration
+        env:
+          E2E_BASE_URL: ${{ vars.E2E_BASE_URL }}
+          E2E_API_BASE_URL: ${{ vars.E2E_API_BASE_URL }}
+          E2E_CF_CLIENT_ID: ${{ secrets.E2E_CF_CLIENT_ID }}
+          E2E_CF_CLIENT_SECRET: ${{ secrets.E2E_CF_CLIENT_SECRET }}
+        run: |
+          set -euo pipefail
+          [[ -n "${E2E_BASE_URL:-}" ]] || { echo "Missing test env var: E2E_BASE_URL"; exit 1; }
+          [[ -n "${E2E_CF_CLIENT_ID:-}" ]] || { echo "Missing test env secret: E2E_CF_CLIENT_ID"; exit 1; }
+          [[ -n "${E2E_CF_CLIENT_SECRET:-}" ]] || { echo "Missing test env secret: E2E_CF_CLIENT_SECRET"; exit 1; }
+
       - name: Run API/Backend Unit Tests with Coverage
         run: |
           dotnet test tests/F1.Api.Tests/F1.Api.Tests.csproj \
@@ -344,4 +356,121 @@ jobs:
         run: |
           docker buildx imagetools create \
             --tag ${{ env.REGISTRY }}/${{ needs.build-and-push.outputs.image_name_lower }}-data-sync:stable \
-            ${{ needs.build-and-push.outputs.data_sync_sha_tag }}
+            ${{ needs.build-and-push.outputs.data_sync_sha_tag }}
+
+  deploy-prod-vm:
+    needs:
+      - deploy-prod
+    if: github.ref == 'refs/heads/main' && github.event_name != 'pull_request'
+    runs-on: ubuntu-latest
+    environment: production
+    permissions:
+      contents: read
+    env:
+      VM_TAILSCALE_HOST: ${{ vars.VM_TAILSCALE_HOST }}
+      VM_DEPLOY_USER: ${{ vars.VM_DEPLOY_USER }}
+      VM_DEPLOY_PATH: ${{ vars.VM_DEPLOY_PATH != '' && vars.VM_DEPLOY_PATH || '/opt/f1competition' }}
+      GHCR_USERNAME: ${{ vars.GHCR_USERNAME }}
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Connect runner to tailnet
+        uses: tailscale/github-action@v3
+        with:
+          oauth-client-id: ${{ secrets.TAILSCALE_OAUTH_CLIENT_ID }}
+          oauth-secret: ${{ secrets.TAILSCALE_OAUTH_SECRET }}
+          tags: tag:ci
+
+      - name: Validate production environment configuration
+        env:
+          TAILSCALE_OAUTH_CLIENT_ID: ${{ secrets.TAILSCALE_OAUTH_CLIENT_ID }}
+          TAILSCALE_OAUTH_SECRET: ${{ secrets.TAILSCALE_OAUTH_SECRET }}
+          VM_SSH_PRIVATE_KEY: ${{ secrets.VM_SSH_PRIVATE_KEY }}
+          VM_ENV_FILE: ${{ secrets.VM_ENV_FILE }}
+          GHCR_READ_TOKEN: ${{ secrets.GHCR_READ_TOKEN }}
+          VM_TAILSCALE_HOST: ${{ vars.VM_TAILSCALE_HOST }}
+          VM_DEPLOY_USER: ${{ vars.VM_DEPLOY_USER }}
+          GHCR_USERNAME: ${{ vars.GHCR_USERNAME }}
+        run: |
+          set -euo pipefail
+          [[ -n "${TAILSCALE_OAUTH_CLIENT_ID:-}" ]] || { echo "Missing production secret: TAILSCALE_OAUTH_CLIENT_ID"; exit 1; }
+          [[ -n "${TAILSCALE_OAUTH_SECRET:-}" ]] || { echo "Missing production secret: TAILSCALE_OAUTH_SECRET"; exit 1; }
+          [[ -n "${VM_SSH_PRIVATE_KEY:-}" ]] || { echo "Missing production secret: VM_SSH_PRIVATE_KEY"; exit 1; }
+          [[ -n "${VM_ENV_FILE:-}" ]] || { echo "Missing production secret: VM_ENV_FILE"; exit 1; }
+          [[ -n "${GHCR_READ_TOKEN:-}" ]] || { echo "Missing production secret: GHCR_READ_TOKEN"; exit 1; }
+          [[ -n "${VM_TAILSCALE_HOST:-}" ]] || { echo "Missing production var: VM_TAILSCALE_HOST"; exit 1; }
+          [[ -n "${VM_DEPLOY_USER:-}" ]] || { echo "Missing production var: VM_DEPLOY_USER"; exit 1; }
+          [[ -n "${GHCR_USERNAME:-}" ]] || { echo "Missing production var: GHCR_USERNAME"; exit 1; }
+
+      - name: Configure SSH access
+        run: |
+          set -euo pipefail
+          mkdir -p ~/.ssh
+          printf '%s' '${{ secrets.VM_SSH_PRIVATE_KEY }}' > ~/.ssh/id_ed25519
+          chmod 600 ~/.ssh/id_ed25519
+          ssh-keyscan -H "$VM_TAILSCALE_HOST" >> ~/.ssh/known_hosts
+
+      - name: Prepare deployment files
+        run: |
+          set -euo pipefail
+          printf '%s' '${{ secrets.VM_ENV_FILE }}' > deploy.env
+          if grep -q '^TAG=' deploy.env; then
+            sed -i 's/^TAG=.*/TAG=stable/' deploy.env
+          else
+            echo 'TAG=stable' >> deploy.env
+          fi
+
+      - name: Ensure remote deploy directory exists
+        run: |
+          set -euo pipefail
+          ssh "$VM_DEPLOY_USER@$VM_TAILSCALE_HOST" "mkdir -p '$VM_DEPLOY_PATH/scripts'"
+
+      - name: Upload compose and deploy scripts
+        run: |
+          set -euo pipefail
+          scp docker-compose.yml "$VM_DEPLOY_USER@$VM_TAILSCALE_HOST:$VM_DEPLOY_PATH/docker-compose.yml"
+          scp scripts/deploy-preflight.sh "$VM_DEPLOY_USER@$VM_TAILSCALE_HOST:$VM_DEPLOY_PATH/scripts/deploy-preflight.sh"
+          scp scripts/deploy-smoke-check.sh "$VM_DEPLOY_USER@$VM_TAILSCALE_HOST:$VM_DEPLOY_PATH/scripts/deploy-smoke-check.sh"
+          scp deploy.env "$VM_DEPLOY_USER@$VM_TAILSCALE_HOST:$VM_DEPLOY_PATH/.env"
+
+      - name: Log remote host into GHCR
+        run: |
+          set -euo pipefail
+          printf '%s' '${{ secrets.GHCR_READ_TOKEN }}' | ssh "$VM_DEPLOY_USER@$VM_TAILSCALE_HOST" "docker login ghcr.io -u '$GHCR_USERNAME' --password-stdin"
+
+      - name: Deploy stable tag to VM
+        run: |
+          set -euo pipefail
+          ssh "$VM_DEPLOY_USER@$VM_TAILSCALE_HOST" "cd '$VM_DEPLOY_PATH' && chmod +x scripts/*.sh && ./scripts/deploy-preflight.sh .env && docker compose --env-file .env pull && docker compose --env-file .env up -d && ./scripts/deploy-smoke-check.sh .env"
+
+      - name: Capture remote deployment state
+        run: |
+          set -euo pipefail
+          ssh "$VM_DEPLOY_USER@$VM_TAILSCALE_HOST" "cd '$VM_DEPLOY_PATH' && docker compose --env-file .env ps" > deploy-compose-ps.txt
+          ssh "$VM_DEPLOY_USER@$VM_TAILSCALE_HOST" "cd '$VM_DEPLOY_PATH' && docker compose --env-file .env ps --format json" > deploy-compose-ps.json
+
+      - name: Write deploy summary
+        run: |
+          {
+            echo '## VM Deployment';
+            echo '';
+            echo '- Environment: production';
+            echo '- Target host: '"$VM_TAILSCALE_HOST";
+            echo '- Deploy path: '"$VM_DEPLOY_PATH";
+            echo '- Image tag: stable';
+            echo '- Commit SHA: '"$GITHUB_SHA";
+            echo '';
+            echo '### Remote docker compose ps';
+            echo '```text';
+            cat deploy-compose-ps.txt;
+            echo '```';
+          } >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload deploy state artifact
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: prod-vm-deploy-state
+          path: |
+            deploy-compose-ps.txt
+            deploy-compose-ps.json
diff --git a/.github/workflows/rollback-vm.yaml b/.github/workflows/rollback-vm.yaml
@@ -0,0 +1,124 @@
+name: Rollback VM Deployment
+
+on:
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: Image tag to deploy to the VM (for example sha-abcdef1 or stable)
+        required: true
+        type: string
+
+jobs:
+  rollback-prod-vm:
+    runs-on: ubuntu-latest
+    environment: production
+    permissions:
+      contents: read
+    env:
+      VM_TAILSCALE_HOST: ${{ vars.VM_TAILSCALE_HOST }}
+      VM_DEPLOY_USER: ${{ vars.VM_DEPLOY_USER }}
+      VM_DEPLOY_PATH: ${{ vars.VM_DEPLOY_PATH != '' && vars.VM_DEPLOY_PATH || '/opt/f1competition' }}
+      GHCR_USERNAME: ${{ vars.GHCR_USERNAME }}
+      ROLLBACK_TAG: ${{ inputs.tag }}
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Connect runner to tailnet
+        uses: tailscale/github-action@v3
+        with:
+          oauth-client-id: ${{ secrets.TAILSCALE_OAUTH_CLIENT_ID }}
+          oauth-secret: ${{ secrets.TAILSCALE_OAUTH_SECRET }}
+          tags: tag:ci
+
+      - name: Validate production environment configuration
+        env:
+          TAILSCALE_OAUTH_CLIENT_ID: ${{ secrets.TAILSCALE_OAUTH_CLIENT_ID }}
+          TAILSCALE_OAUTH_SECRET: ${{ secrets.TAILSCALE_OAUTH_SECRET }}
+          VM_SSH_PRIVATE_KEY: ${{ secrets.VM_SSH_PRIVATE_KEY }}
+          VM_ENV_FILE: ${{ secrets.VM_ENV_FILE }}
+          GHCR_READ_TOKEN: ${{ secrets.GHCR_READ_TOKEN }}
+          VM_TAILSCALE_HOST: ${{ vars.VM_TAILSCALE_HOST }}
+          VM_DEPLOY_USER: ${{ vars.VM_DEPLOY_USER }}
+          GHCR_USERNAME: ${{ vars.GHCR_USERNAME }}
+        run: |
+          set -euo pipefail
+          [[ -n "${TAILSCALE_OAUTH_CLIENT_ID:-}" ]] || { echo "Missing production secret: TAILSCALE_OAUTH_CLIENT_ID"; exit 1; }
+          [[ -n "${TAILSCALE_OAUTH_SECRET:-}" ]] || { echo "Missing production secret: TAILSCALE_OAUTH_SECRET"; exit 1; }
+          [[ -n "${VM_SSH_PRIVATE_KEY:-}" ]] || { echo "Missing production secret: VM_SSH_PRIVATE_KEY"; exit 1; }
+          [[ -n "${VM_ENV_FILE:-}" ]] || { echo "Missing production secret: VM_ENV_FILE"; exit 1; }
+          [[ -n "${GHCR_READ_TOKEN:-}" ]] || { echo "Missing production secret: GHCR_READ_TOKEN"; exit 1; }
+          [[ -n "${VM_TAILSCALE_HOST:-}" ]] || { echo "Missing production var: VM_TAILSCALE_HOST"; exit 1; }
+          [[ -n "${VM_DEPLOY_USER:-}" ]] || { echo "Missing production var: VM_DEPLOY_USER"; exit 1; }
+          [[ -n "${GHCR_USERNAME:-}" ]] || { echo "Missing production var: GHCR_USERNAME"; exit 1; }
+
+      - name: Configure SSH access
+        run: |
+          set -euo pipefail
+          mkdir -p ~/.ssh
+          printf '%s' '${{ secrets.VM_SSH_PRIVATE_KEY }}' > ~/.ssh/id_ed25519
+          chmod 600 ~/.ssh/id_ed25519
+          ssh-keyscan -H "$VM_TAILSCALE_HOST" >> ~/.ssh/known_hosts
+
+      - name: Prepare rollback env file
+        run: |
+          set -euo pipefail
+          printf '%s' '${{ secrets.VM_ENV_FILE }}' > deploy.env
+          if grep -q '^TAG=' deploy.env; then
+            sed -i "s/^TAG=.*/TAG=$ROLLBACK_TAG/" deploy.env
+          else
+            echo "TAG=$ROLLBACK_TAG" >> deploy.env
+          fi
+
+      - name: Ensure remote deploy directory exists
+        run: |
+          set -euo pipefail
+          ssh "$VM_DEPLOY_USER@$VM_TAILSCALE_HOST" "mkdir -p '$VM_DEPLOY_PATH/scripts'"
+
+      - name: Upload compose and deploy scripts
+        run: |
+          set -euo pipefail
+          scp docker-compose.yml "$VM_DEPLOY_USER@$VM_TAILSCALE_HOST:$VM_DEPLOY_PATH/docker-compose.yml"
+          scp scripts/deploy-preflight.sh "$VM_DEPLOY_USER@$VM_TAILSCALE_HOST:$VM_DEPLOY_PATH/scripts/deploy-preflight.sh"
+          scp scripts/deploy-smoke-check.sh "$VM_DEPLOY_USER@$VM_TAILSCALE_HOST:$VM_DEPLOY_PATH/scripts/deploy-smoke-check.sh"
+          scp deploy.env "$VM_DEPLOY_USER@$VM_TAILSCALE_HOST:$VM_DEPLOY_PATH/.env"
+
+      - name: Log remote host into GHCR
+        run: |
+          set -euo pipefail
+          printf '%s' '${{ secrets.GHCR_READ_TOKEN }}' | ssh "$VM_DEPLOY_USER@$VM_TAILSCALE_HOST" "docker login ghcr.io -u '$GHCR_USERNAME' --password-stdin"
+
+      - name: Roll back VM deployment
+        run: |
+          set -euo pipefail
+          ssh "$VM_DEPLOY_USER@$VM_TAILSCALE_HOST" "cd '$VM_DEPLOY_PATH' && chmod +x scripts/*.sh && ./scripts/deploy-preflight.sh .env && docker compose --env-file .env pull && docker compose --env-file .env up -d && ./scripts/deploy-smoke-check.sh .env"
+
+      - name: Capture remote rollback state
+        run: |
+          set -euo pipefail
+          ssh "$VM_DEPLOY_USER@$VM_TAILSCALE_HOST" "cd '$VM_DEPLOY_PATH' && docker compose --env-file .env ps" > rollback-compose-ps.txt
+          ssh "$VM_DEPLOY_USER@$VM_TAILSCALE_HOST" "cd '$VM_DEPLOY_PATH' && docker compose --env-file .env ps --format json" > rollback-compose-ps.json
+
+      - name: Write rollback summary
+        run: |
+          {
+            echo '## VM Rollback';
+            echo '';
+            echo '- Environment: production';
+            echo '- Target host: '"$VM_TAILSCALE_HOST";
+            echo '- Deploy path: '"$VM_DEPLOY_PATH";
+            echo '- Image tag: '"$ROLLBACK_TAG";
+            echo '';
+            echo '### Remote docker compose ps';
+            echo '```text';
+            cat rollback-compose-ps.txt;
+            echo '```';
+          } >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload rollback state artifact
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: prod-vm-rollback-state
+          path: |
+            rollback-compose-ps.txt
+            rollback-compose-ps.json