repo: Add gh-actions/fs/overlay

.github/workflows/test-gcs-fuse.yml

@@ -0,0 +1,29 @@
+name: Test GCS FUSE Mount Action
+
+on:
+  push:
+    branches:
+    - main
+    - 'actions-v*'
+    paths:
+    - 'gh-actions/gcs/fuse/**'
+    - '.github/workflows/test-gcs-fuse.yml'
+  pull_request:
+    paths:
+    - 'gh-actions/gcs/fuse/**'
+    - '.github/workflows/test-gcs-fuse.yml'
+
+jobs:
+  test-external:
+    name: Test anonymous access (fork PRs)
+    if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: ./gh-actions/gcs/fuse
+      with:
+        bucket: gcp-public-data-sentinel-2
+        mount-point: /tmp/gcs-public-test
+    - run: |
+        ls -la /tmp/gcs-public-test
+        ls -la /tmp/gcs-public-test | grep -q index.csv

.github/workflows/test-overlay-fuse.yml

@@ -0,0 +1,77 @@
+name: Test Overlay with GCS FUSE Mount
+
+on:
+  push:
+    branches:
+    - main
+    - 'actions-v*'
+    paths:
+    - 'gh-actions/fs/overlay/**'
+    - 'gh-actions/gcs/fuse/**'
+    - '.github/workflows/test-overlay-fuse.yml'
+  pull_request:
+    paths:
+    - 'gh-actions/fs/overlay/**'
+    - 'gh-actions/gcs/fuse/**'
+    - '.github/workflows/test-overlay-fuse.yml'
+
+jobs:
+  test-overlay-public:
+    name: Test overlay with public GCS bucket
+    runs-on: ubuntu-24.04
+    steps:
+    - uses: actions/checkout@v4
+    - name: Mount public GCS bucket
+      uses: ./gh-actions/gcs/fuse
+      with:
+        bucket: gcp-public-data-sentinel-2
+        cache-dir: ${{ runner.temp }}/cache-fuse
+        mount-as-root: true
+        mount-point: ${{ runner.temp }}/gcs-public
+    - name: Create overlay directories
+      run: |
+        mkdir -p ${{ runner.temp }}/overlay-upper
+        mkdir -p ${{ runner.temp }}/overlay-mount
+        echo "Initial content in upper dir" > ${{ runner.temp }}/overlay-upper/upper-test-file.txt
+    - name: Mount overlay filesystem
+      uses: ./gh-actions/fs/overlay
+      with:
+        lower-dir: "${{ runner.temp }}/gcs-public"
+        upper-dir: "${{ runner.temp }}/overlay-upper"
+        mount-point: "${{ runner.temp }}/overlay-mount"
+    - name: Verify GCS data visibility
+      run: |
+        sudo du -ch "${{ runner.temp }}/cache-fuse" > /dev/null
+        ls -lah "${{ runner.temp }}/overlay-mount" | grep -q csv
+        ls -lah "${{ runner.temp }}/overlay-mount" | grep -q upper-test-file.txt
+        du -ch "${{ runner.temp }}/overlay-mount/tiles/01/C/CV/S2A_MSIL1C_20151221T205519_N0201_R028_T01CCV_20160329T181515.SAFE" > /dev/null
+        cat /home/runner/work/_temp/overlay-mount/tiles/01/C/CV/S2A_MSIL1C_20151221T205519_N0201_R028_T01CCV_20160329T181515.SAFE/GRANULE/S2A_OPER_MSI_L1C_TL_EPA__20160325T184811_A002599_T01CCV_N02.01/QI_DATA/S2A_OPER_PVI_L1C_TL_EPA__20160325T184811_A002599_T01CCV.jp2 > /dev/null
+    - name: Test write operations
+      run: |
+        echo "Overwrite existing"
+        echo BOOM > "${{ runner.temp }}/overlay-mount/index.csv.gz-backup"
+        ls -alh "${{ runner.temp }}/overlay-mount/index.csv.gz-backup"
+
+        echo "This is a new file created in the overlay" > "${{ runner.temp }}/overlay-mount/new-file.txt"
+        if [ -f "${{ runner.temp }}/overlay-mount/hello.txt" ]; then
+          echo "APPENDED CONTENT" >> "${{ runner.temp }}/overlay-mount/hello.txt
+          echo "✅ Successfully modified a GCS file through the overlay"
+          cat "${{ runner.temp }}/overlay-mount/hello.txt
+        fi
+        if [ -f "${{ runner.temp }}/overlay-mount/new-file.txt" ] && [ ! -f "${{ runner.temp }}/gcs-public/new-file.txt" ]; then
+          echo "✅ Write operations are working correctly"
+          echo "✅ Original GCS mount remains unchanged"
+        else
+          echo "❌ Overlay write test failed"
+          exit 1
+        fi
+        if [ -f "${{ runner.temp }}/overlay-upper/new-file.txt" ]; then
+          echo "✅ Changes correctly stored in upper layer"
+          cat ${{ runner.temp }}/overlay-upper/new-file.txt
+        else
+          echo "❌ Changes not properly stored in upper layer"
+          exit 1
+        fi
+    - name: Test cache dir
+      run: |
+        sudo du -ch "${{ runner.temp }}/cache-fuse"

gh-actions/cache/prime/action.yml

@@ -12,6 +12,9 @@ inputs:
   gcs-bucket:
     type: string
     default:
+  gcs-sync:
+    type: boolean
+    default: false
   lock-id:
     type: string
   lock-repository:
@@ -101,6 +104,40 @@ runs:
       bucket: ${{ inputs.gcs-bucket }}
       key: ${{ inputs.key }}
 
+  # RE/MOVE THIS!
+  - name: Populate bucket
+    if: >-
+      ${{ inputs.gcs-bucket
+          && steps.cache-restore-initial.outputs.cache-hit != 'true'
+          && steps.cache-restore.outputs.cache-hit != 'true'
+          && steps.gcs-object-recheck.outputs.exists != 'true' }}
+    shell: bash
+    env:
+      BUCKET: ${{ inputs.gcs-bucket }}
+    run: |
+      echo "Syncing ${BUCKET} -> ${BUCKET}-new"
+      # gsutil mv gs://${BUCKET}/91f032419402bae13580ed2563c0183c05ec6328793125fbece1f40f3a0565d8-arm64 gs://${BUCKET}/genesis-arm64
+      gcloud storage mv gs://${BUCKET}/91f032419402bae13580ed2563c0183c05ec6328793125fbece1f40f3a0565d8-x64 gs://${BUCKET}/genesis-x64 --quiet
+
+      echo "MOVED BUCKETS"
+      exit 1
+  - name: Sync remote bucket
+    if: >-
+      ${{ inputs.gcs-bucket
+          && steps.cache-restore-initial.outputs.cache-hit != 'true'
+          && steps.cache-restore.outputs.cache-hit != 'true'
+          && steps.gcs-object-recheck.outputs.exists != 'true' }}
+    shell: bash
+    env:
+      BUCKET: ${{ inputs.gcs-bucket }}
+      CACHE_KEY: ${{ inputs.key }}
+    run: |
+      ARCH="${CACHE_KEY#*-}"
+      CURRENT_CACHE="$(gsutil "gs://${BUCKET}/current.txt" -)"
+      CURRENT_CACHE="${CURRENT_CACHE}-${ARCH}"
+      echo "Syncing ${CURRENT_CACHE} -> ${CACHE_KEY}"
+      gsutil -m cp -r gs://${BUCKET}/${CURRENT_CACHE}/* gs://${BUCKET}/${CACHE_KEY}/
+
   - if: >-
       ${{ steps.cache-restore-initial.outputs.cache-hit != 'true'
           && steps.cache-restore.outputs.cache-hit != 'true'
@@ -156,11 +193,12 @@ runs:
       key: ${{ inputs.key }}
 
   - if: ${{ steps.restore.outputs.save == 'true' && inputs.gcs-bucket }}
-    uses: envoyproxy/toolshed/gh-actions/gcs/cache/save@actions-v0.3.16
+    uses: envoyproxy/toolshed/gh-actions/gcs/cache/save@80d124b5f6317c6222ebe21ff49d4c5aa169c13e
     with:
       bucket: ${{ inputs.gcs-bucket }}
       key: ${{ inputs.key }}
       path: ${{ inputs.path || inputs.path-tmp }}
+      sync: ${{ inputs.gcs-sync }}
 
   - if: ${{ steps.restore.outputs.save == 'true' && ! inputs.path }}
     run: |

gh-actions/fs/overlay/action.yml

@@ -0,0 +1,154 @@
+name: Overlay Filesystem Mount
+description: Create and mount an overlay filesystem to combine directories and allow writes to a read-only base
+author: Ryan Northey <[email protected]>
+
+inputs:
+  upper-dir:
+    type: string
+    required: true
+    description: |
+      Path to the directory that will contain the upper layer (write layer)
+  lower-dir:
+    type: string
+    required: true
+    description: |
+      Path to the base directory (read-only layer) - can be a colon-separated list for multiple lower layers
+  mount-point:
+    type: string
+    required: true
+    description: |
+      Directory path where the overlay filesystem will be mounted
+  work-dir:
+    type: string
+    description: |
+      Path to the directory used for overlay work files (created automatically if not specified)
+  create-dirs:
+    type: boolean
+    default: true
+    description: |
+      Automatically create the upper, work, and mount directories if they don't exist
+  options:
+    type: string
+    description: |
+      Additional mount options for the overlay filesystem
+  read-only:
+    type: boolean
+    default: false
+    description: |
+      Mount the overlay as read-only
+  verify-mount:
+    type: boolean
+    default: true
+    description: |
+      Verify that the mount was successful
+  debug:
+    type: boolean
+    default: false
+    description: |
+      Enable debug logging
+
+outputs:
+  mount-path:
+    description: 'The full path where the overlay filesystem was mounted'
+    value: ${{ steps.mount.outputs.mount-path }}
+
+  unmount-cmd:
+    description: 'Command to unmount the overlay filesystem (for use in post cleanup)'
+    value: ${{ steps.mount.outputs.unmount-cmd }}
+
+runs:
+  using: "composite"
+  steps:
+  - name: Check for overlay module
+    shell: bash
+    run: |
+      if ! grep -q overlay /proc/filesystems; then
+        echo "Overlay filesystem is not supported on this system!" >&2
+        exit 1
+      fi
+      if [[ "${{ inputs.debug }}" == "true" ]]; then
+        echo "Overlay filesystem is supported"
+      fi
+
+  - name: Create required directories
+    if: ${{ inputs.create-dirs == 'true' }}
+    shell: bash
+    run: |
+      mkdir -p "${{ inputs.upper-dir }}"
+      mkdir -p "${{ inputs.mount-point }}"
+
+      if [[ -n "${{ inputs.work-dir }}" ]]; then
+        WORK_DIR="${{ inputs.work-dir }}"
+      else
+        WORK_DIR="${{ inputs.upper-dir }}-work"
+      fi
+      mkdir -p "$WORK_DIR"
+      echo "OVERLAY_WORK_DIR=$WORK_DIR" >> $GITHUB_ENV
+      if [[ "${{ inputs.debug }}" == "true" ]]; then
+        echo "Created directories:"
+        echo "  Upper dir: ${{ inputs.upper-dir }}"
+        echo "  Mount point: ${{ inputs.mount-point }}"
+        echo "  Work dir: $WORK_DIR"
+      fi
+
+  - uses: envoyproxy/toolshed/gh-actions/github/script/[email protected]
+    name: Mount overlay filesystem
+    id: mount
+    with:
+      run: |
+        MOUNT_POINT="${{ inputs.mount-point }}"
+        UPPER_DIR="${{ inputs.upper-dir }}"
+        LOWER_DIR="${{ inputs.lower-dir }}"
+        WORK_DIR="${OVERLAY_WORK_DIR}"
+        MOUNT_OPTS="lowerdir=${LOWER_DIR},upperdir=${UPPER_DIR},workdir=${WORK_DIR}"
+        if [[ -n "${{ inputs.options }}" ]]; then
+          MOUNT_OPTS="${MOUNT_OPTS},${{ inputs.options }}"
+        fi
+        if [[ "${{ inputs.read-only }}" == "true" ]]; then
+          MOUNT_OPTS="${MOUNT_OPTS},ro"
+        fi
+        if [[ "${{ inputs.debug }}" == "true" ]]; then
+          echo "Mount options: $MOUNT_OPTS"
+        fi
+        CMD="sudo mount -t overlay -o ${MOUNT_OPTS} overlay ${MOUNT_POINT}"
+        echo "Mounting with: $CMD"
+        sudo mount -t overlay -o ${MOUNT_OPTS} overlay ${MOUNT_POINT}
+        RESULT=$?
+        if [[ $RESULT -ne 0 ]]; then
+          echo "Failed to mount overlay filesystem" >&2
+          exit 1
+        fi
+        sudo mount
+        sudo chown runner:runner ${MOUNT_POINT}
+        sudo ls -alh ${MOUNT_POINT} || echo "root cant see"
+        ls -alh ${MOUNT_POINT} || echo "runner cant see"
+        echo "mount-path=${MOUNT_POINT}" >> $GITHUB_OUTPUT
+        echo "unmount-cmd=umount ${MOUNT_POINT}" >> $GITHUB_OUTPUT
+      post: |
+        sudo umount ${{ inputs.mount-point }} || true
+
+  - name: Verify mount
+    if: ${{ inputs.verify-mount == 'true' }}
+    shell: bash
+    run: |
+      if ! mount | grep -q "${{ inputs.mount-point }} type overlay"; then
+        echo "Overlay mount verification failed!" >&2
+        exit 1
+      fi
+
+      if [[ "${{ inputs.debug }}" == "true" ]]; then
+        echo "Mount details:"
+        mount | grep overlay
+        echo "Filesystem contents:"
+        ls -la "${{ inputs.mount-point }}"
+      fi
+
+  - name: Mount information
+    shell: bash
+    run: |
+      echo "Overlay filesystem mounted successfully"
+      echo "  Mount point: ${{ inputs.mount-point }}"
+      echo "  Upper dir: ${{ inputs.upper-dir }}"
+      echo "  Lower dir: ${{ inputs.lower-dir }}"
+      echo "  Work dir: ${{ env.OVERLAY_WORK_DIR }}"
+      echo "  Read-only: ${{ inputs.read-only }}"

gh-actions/gcp/setup/action.yml

@@ -79,11 +79,16 @@ runs:
                 exit 1
             fi
             cp -a "$GCP_KEY_PATH" "${{ inputs.key-copy }}"
+
             GCP_KEY_FILENAME=$(basename $GCP_KEY_PATH)
             GCP_KEY_COPY_PATH="${{ inputs.key-copy }}/${GCP_KEY_FILENAME}"
             echo "GCP_KEY_COPY_PATH=${GCP_KEY_COPY_PATH}" >> $GITHUB_ENV
             echo "copy-path=${GCP_KEY_COPY_PATH}" >> $GITHUB_OUTPUT
         fi
+        export CLOUDSDK_CONFIG="$GITHUB_WORKSPACE/.gcloud"
+        mkdir -p "$CLOUDSDK_CONFIG"
+        gcloud auth activate-service-account --key-file="${GCP_KEY_PATH}"
+        echo "CLOUDSDK_CONFIG=$CLOUDSDK_CONFIG" >> $GITHUB_ENV
         if [[ ${{ inputs.boto }} != "true" ]]; then
             exit 0
         fi

gh-actions/gcs/cache/save/action.yml

@@ -15,16 +15,29 @@ inputs:
   ext:
     type: string
     default: zstd
+  sync:
+    type: boolean
+    default: false
 
 
 runs:
   using: "composite"
   steps:
   - name: Save GCS cache
+    if: ${{ ! inputs.sync }}
     run: |
       tar cf - -I "${{ inputs.compressor }}" -C ${{ inputs.path }} .  \
           | gsutil cp - "gs://${BUCKET}/${BLOB}"
     shell: bash
     env:
       BUCKET: ${{ inputs.bucket }}
       BLOB: ${{ inputs.key }}.${{ inputs.ext }}
+
+  - name: Sync GCS cache
+    if: ${{ inputs.sync }}
+    run: |
+      gsutil -q -m rsync -r ${{ inputs.path }} "gs://${BUCKET}/${BLOB}"
+    shell: bash
+    env:
+      BUCKET: ${{ inputs.bucket }}
+      BLOB: ${{ inputs.key }}