ci: validate Docker base image compatibility

.dockerignore

@@ -2,9 +2,17 @@
 /_build
 /db
 /deps
+/target
+/priv/native
 /*.ez
 .DS_Store
 
+# Files that cannot affect the production image
+/.git
+/.github
+/bench
+/test
+
 # Generated on crash by the VM
 erl_crash.dump
 

.github/workflows/docker-base-runner-daily.yml

@@ -14,11 +14,13 @@ jobs:
     outputs:
       short_sha: ${{steps.env.outputs.short_sha}}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v6
       - name: Setup outputs
         id: env
         run: |
           echo "short_sha=$(git rev-parse --short=7 HEAD)" >> "$GITHUB_OUTPUT"
+      - name: Validate Docker base images
+        run: python3 bin/docker-base-images check --warn-age-days 45
 
   build_base:
     needs: settings
@@ -32,9 +34,9 @@ jobs:
     runs-on: ${{ matrix.runner }}
     timeout-minutes: 30
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v6
       - run: docker context create builders
-      - uses: docker/setup-buildx-action@v3
+      - uses: docker/setup-buildx-action@v4
         with:
           endpoint: builders
       - name: Login to DockerHub
@@ -43,16 +45,17 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
       - id: build
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@v7
         with:
           file: ./Dockerfile.base
           push: true
+          pull: true
           tags: |
             supabase/logflare:base-${{ needs.settings.outputs.short_sha }}-${{ matrix.arch }}
             supabase/logflare:base-${{ matrix.arch }}
           platforms: linux/${{ matrix.arch }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+          cache-from: type=gha,scope=base-${{ matrix.arch }}
+          cache-to: type=gha,scope=base-${{ matrix.arch }},mode=max
 
   build_runner:
     needs: settings
@@ -66,9 +69,9 @@ jobs:
     runs-on: ${{ matrix.runner }}
     timeout-minutes: 30
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v6
       - run: docker context create builders
-      - uses: docker/setup-buildx-action@v3
+      - uses: docker/setup-buildx-action@v4
         with:
           endpoint: builders
       - name: Login to DockerHub
@@ -77,16 +80,17 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
       - id: build
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@v7
         with:
           file: ./Dockerfile.runner
           push: true
+          pull: true
           tags: |
             supabase/logflare:runner-${{ needs.settings.outputs.short_sha }}-${{ matrix.arch }}
             supabase/logflare:runner-${{ matrix.arch }}
           platforms: linux/${{ matrix.arch }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+          cache-from: type=gha,scope=runner-${{ matrix.arch }}
+          cache-to: type=gha,scope=runner-${{ matrix.arch }},mode=max
 
   merge_publish:
     needs:

.github/workflows/docker-ci-amd-and-arm.yml

@@ -4,23 +4,33 @@ on:
   workflow_dispatch:
   push:
     branches: [main]
+    paths-ignore:
+      - "test/**"
+      - "bench/**"
+      - "*.md"
 
 permissions:
   contents: read
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'push' }}
+
 jobs:
   settings:
     runs-on: ubuntu-latest
     outputs:
       version: ${{steps.env.outputs.version}}
       short_sha: ${{steps.env.outputs.short_sha}}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v6
       - name: Setup outputs
         id: env
         run: |
           echo "version=$(cat ./VERSION)" >> "$GITHUB_OUTPUT"
           echo "short_sha=$(git rev-parse --short=7 HEAD)" >> "$GITHUB_OUTPUT"
+      - name: Validate Docker base images
+        run: python3 bin/docker-base-images check --warn-age-days 45
   build:
     needs: settings
     strategy:
@@ -34,40 +44,41 @@ jobs:
     timeout-minutes: 120
     steps:
       # checkout git commit
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v6
       # Setup docker
       # https://github.com/supabase/postgres/blob/develop/.github/workflows/dockerhub-release.yml#LL41C11-L42C44
       - run: docker context create builders
-      - uses: docker/setup-buildx-action@v2
+      - uses: docker/setup-buildx-action@v4
         with:
           endpoint: builders
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
       # Build and push
       - id: build
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v7
         with:
           push: true
+          pull: true
           tags: supabase/logflare:dev-${{ needs.settings.outputs.short_sha }}-${{ matrix.arch }}
           platforms: linux/${{ matrix.arch }}
           # Baked into the image as LOGFLARE_COMMIT_SHA and emitted as an OTel
           # resource attribute at runtime (see Dockerfile / Logflare.Telemetry).
           build-args: |
             COMMIT_SHA=${{ needs.settings.outputs.short_sha }}
-          # caching
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+          # Keep architecture-specific caches from overwriting each other.
+          cache-from: type=gha,scope=app-${{ matrix.arch }}
+          cache-to: type=gha,scope=app-${{ matrix.arch }},mode=max
 
   merge_publish:
     needs:
       - settings
       - build
     runs-on: ubuntu-latest
     steps:
-      - uses: docker/login-action@v2
+      - uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}

.github/workflows/docker-test.yml

@@ -6,6 +6,8 @@ on:
     branches: [main]
     paths:
       - 'native/**'
+      - 'Cargo.toml'
+      - 'Cargo.lock'
       - 'priv/**'
       - 'rel/**'
       - 'config/**'
@@ -14,9 +16,13 @@ on:
       - 'Dockerfile.multi-step'
       - 'Dockerfile.base'
       - 'Dockerfile.runner'
+      - '.dockerignore'
       - '!assets/**'
       - '*.sh'
       - '.tool-versions'
+      - 'bin/docker-base-images'
+      - 'bin/docker_base_images.py'
+      - 'test/bin/docker_base_images_test.py'
       # Run suite when this file is changed
       - ".github/workflows/docker-test.yml"
 
@@ -29,12 +35,16 @@ jobs:
     timeout-minutes: 30
     steps:
       - uses: actions/checkout@v6
+      - name: Validate Docker base images
+        run: |
+          python3 -m unittest discover -s test/bin -p '*_test.py' -v
+          python3 bin/docker-base-images check --warn-age-days 45
       - uses: docker/setup-buildx-action@v4
       - uses: docker/build-push-action@v7
         with:
-          file: ./Dockerfile.multi-step
+          file: ./Dockerfile
           push: false
+          pull: true
           platforms: linux/amd64
-          # caching
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+          cache-from: type=gha,scope=docker-build-check
+          cache-to: type=gha,scope=docker-build-check,mode=max

.gitignore

@@ -13,6 +13,10 @@ erl_crash.dump
 # Generated on crash by NPM
 npm-debug.log
 
+# Python bytecode
+__pycache__/
+*.py[cod]
+
 # Static artifacts
 /assets/node_modules
 /assets/yarn.lock

Dockerfile

@@ -32,15 +32,19 @@ RUN mix do local.rebar --force + local.hex --force + deps.get + deps.compile
 COPY assets/package.json assets/package-lock.json assets/
 RUN npm --prefix assets ci
 
-COPY . ./
-
-# rust bin path and release optimizations
+# Compile native extensions before copying application source so ordinary Elixir
+# changes can reuse the expensive architecture-specific Rust build.
 ENV PATH="/root/.cargo/bin:${PATH}"
 ENV CARGO_PROFILE_RELEASE_LTO="thin"
 ENV CARGO_PROFILE_RELEASE_CODEGEN_UNITS=1
+COPY Cargo.toml Cargo.lock ./
+COPY native native/
+RUN set -e; \
+    for crate in arrowipc_ex ch_compression_ex mapper_ex sqlparser_ex; do \
+      cargo rustc --package "$crate" --release --locked; \
+    done
 
-# check installed correctly
-RUN cargo version
+COPY . ./
 
 # release
 RUN mix release  && \

bin/docker-base-images

@@ -0,0 +1,5 @@
+#!/usr/bin/env python3
+
+from docker_base_images import main
+
+raise SystemExit(main())