feat: COS solution tests on self-hosted runners

.github/workflows/_integration.yml

@@ -22,10 +22,114 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-      - name: Install dependencies
+      - name: Concierge prepare
+        if: ${{ runner.environment == 'github-hosted' }}
         run: |
           sudo snap install concierge --classic
           sudo concierge prepare --juju-channel 3.6/stable -p microk8s --extra-snaps just,astral-uv,terraform
+
+      # Setup for Self-hosted (PS7) runners
+      - name: Install snaps
+        if: ${{ runner.environment == 'self-hosted' }}
+        run: |
+          sudo snap install juju --classic --channel=3.6/stable
+          sudo snap install just --classic
+          sudo snap install astral-uv --classic
+          sudo snap install terraform --classic
+      - name: (IS hosted) Configure microk8s Docker Hub mirror
+        timeout-minutes: 10
+        if: ${{ runner.environment == 'self-hosted' }}
+        run: |
+          sudo snap install microk8s --channel "1.34-strict/stable" --classic
+          sudo adduser "$USER" snap_microk8s
+
+          # Wait for microk8s to populate iptables
+          # https://chat.canonical.com/canonical/pl/jo5cg6wqjjrudqd5ybj6hhttee
+          until sudo iptables --list | grep -q -i "microk8s"
+          do
+            echo "MicroK8s has not yet configured iptables."
+            sleep 10
+          done
+
+          sudo tee /var/snap/microk8s/current/args/certs.d/docker.io/hosts.toml << EOF
+          server = "$DOCKERHUB_MIRROR"
+          [host."${DOCKERHUB_MIRROR#'https://'}"]
+          capabilities = ["pull", "resolve"]
+          EOF
+          sudo microk8s stop
+          sudo microk8s start
+      - name: Set up microk8s
+        id: microk8s-setup
+        timeout-minutes: 15
+        if: ${{ runner.environment == 'self-hosted' }}
+        run: |
+          sudo apt-get update
+          sudo apt-get install retry -y
+
+          # `newgrp` does not work in GitHub Actions; use `sudo --user` instead
+          sudo --user "$USER" --preserve-env --preserve-env=PATH -- env -- microk8s status --wait-ready
+          sudo --user "$USER" --preserve-env --preserve-env=PATH -- env -- retry --times 3 --delay 5 -- sudo microk8s enable dns
+          sudo --user "$USER" --preserve-env --preserve-env=PATH -- env -- microk8s status --wait-ready
+          sudo --user "$USER" --preserve-env --preserve-env=PATH -- env -- microk8s.kubectl rollout status --namespace kube-system --watch --timeout=5m deployments/coredns
+          sudo --user "$USER" --preserve-env --preserve-env=PATH -- env -- retry --times 3 --delay 5 -- sudo microk8s enable hostpath-storage
+          sudo --user "$USER" --preserve-env --preserve-env=PATH -- env -- microk8s.kubectl rollout status --namespace kube-system --watch --timeout=5m deployments/hostpath-provisioner
+
+          IPADDR=$(ip -4 -j route get 2.2.2.2 | sed -n -e 's/^.*prefsrc\":"\([^ "]*\).*/\1/p')
+          sudo --user "$USER" --preserve-env --preserve-env=PATH -- env -- retry --times 3 --delay 5 -- sudo microk8s enable "metallb:$IPADDR-$IPADDR"
+          sudo --user "$USER" --preserve-env --preserve-env=PATH -- env -- microk8s status --wait-ready
+          sudo --user "$USER" --preserve-env --preserve-env=PATH -- env -- retry --times 3 --delay 5 -- sudo microk8s enable ingress
+          sudo --user "$USER" --preserve-env --preserve-env=PATH -- env -- microk8s status --wait-ready
+          sudo --user "$USER" --preserve-env --preserve-env=PATH -- env -- retry --times 3 --delay 5 -- sudo microk8s enable rbac
+          sudo --user "$USER" --preserve-env --preserve-env=PATH -- env -- microk8s status --wait-ready
+          sudo --user "$USER" --preserve-env --preserve-env=PATH -- env -- retry --times 3 --delay 5 -- sudo microk8s enable registry
+          sudo --user "$USER" --preserve-env --preserve-env=PATH -- env -- microk8s status --wait-ready
+
+          mkdir ~/.kube/
+          sudo --user "$USER" --preserve-env --preserve-env=PATH -- env -- microk8s config | sudo tee ~/.kube/config > /dev/null
+      - name: Set up environment
+        timeout-minutes: 15
+        if: ${{ runner.environment == 'self-hosted' }}
+        run: |
+          mkdir -p ~/.local/share/juju  # Workaround for juju 3 strict snap
+          sudo --user "$USER" --preserve-env --preserve-env=PATH -- env -- juju bootstrap microk8s --config model-logs-size=10G microk8s
+          juju model-defaults logging-config='<root>=INFO; unit=DEBUG'
+
+      # Runner-independent setup
+      - name: Install and configure microceph
+        if: ${{ inputs.product == 'cos' }}
+        run: |
+          # https://github.com/canonical/microceph-action/blob/main/microceph.sh
+          function check_ceph_ok_or_exit () {
+              i=0
+              for i in {1..5}; do
+                  if sudo microceph.ceph status | grep HEALTH_OK; then
+                      break
+                  else
+                      sudo microceph.ceph status
+                      sleep 30
+                      sudo microceph.ceph health detail
+                  fi
+              done
+              if [ "$i" -eq 5 ]; then
+                  exit 1
+              fi
+          }
+
+          sudo snap install microceph
+          sudo microceph cluster bootstrap
+          sleep 30s
+          sudo microceph.ceph config set "mon.$(hostname)" mon_data_avail_warn 6
+          sudo microceph disk add loop,2G,3
+          check_ceph_ok_or_exit
+
+          sudo microceph enable rgw --port 8080 --ssl-port 8443
+          sudo microceph.radosgw-admin user create --uid=user --display-name=User
+          sudo microceph.radosgw-admin key create --uid=user --key-type=s3 --access-key=access-key --secret-key=secret-key
+
       - name: Test deployment
         run: |
+          S3_ENDPOINT=http://$(ip -4 -j route get 2.2.2.2 | jq -r '.[] | .prefsrc'):8080
+          export S3_ENDPOINT
+          export S3_ACCESS_KEY=access-key
+          export S3_SECRET_KEY=secret-key
           just integration ${{ inputs.product }}/${{ matrix.scenario }}

.github/workflows/terraform.yml

@@ -48,4 +48,10 @@ jobs:
     uses: canonical/observability-stack/.github/workflows/_integration.yml@main
     with:
       product: cos_lite
-      runner: ubuntu-latest
+      runner: self-hosted-linux-amd64-noble-large
+  test-integration-cos:
+    name: COS Terraform integration
+    uses: canonical/observability-stack/.github/workflows/_integration.yml@main
+    with:
+      product: cos
+      runner: self-hosted-linux-amd64-noble-xlarge

tests/integration/cos_lite/tls_external/track-2.tf

@@ -39,5 +39,6 @@ module "cos-lite" {
   external_certificates_offer_url = "admin/${var.ca_model}.certificates"
   external_ca_cert_offer_url      = "admin/${var.ca_model}.send-ca-cert"
 
-  traefik           = { channel = "latest/edge" }  # TODO: Switch to latest/stable when rev257 hits stable
+  traefik    = { channel = "latest/edge" } # TODO: Switch to latest/stable when rev257 hits stable
+  prometheus = { revision = 279 }          # TODO: Remove when rev279 hits stable
 }

tests/integration/cos_lite/tls_full/track-2.tf

@@ -39,6 +39,7 @@ module "cos-lite" {
   external_certificates_offer_url = "admin/${var.ca_model}.certificates"
   external_ca_cert_offer_url      = "admin/${var.ca_model}.send-ca-cert"
 
-  traefik           = { channel = "latest/edge" }  # TODO: Switch to latest/stable when rev257 hits stable
+  traefik    = { channel = "latest/edge" } # TODO: Switch to latest/stable when rev257 hits stable
+  prometheus = { revision = 279 }          # TODO: Remove when rev279 hits stable
 }