feat: Add reproducible builds release workflows and push images to DockerHub

.github/workflows/docker-reproducible.yml

@@ -0,0 +1,176 @@
+name: docker-reproducible
+
+on:
+  push:
+    branches:
+      - unstable
+      - stable
+    tags:
+      - v*
+  workflow_dispatch: # allows manual triggering for testing purposes and skips publishing an image
+
+env:
+  DOCKER_REPRODUCIBLE_IMAGE_NAME: >-
+    ${{ github.repository_owner }}/lighthouse-reproducible
+  DOCKER_PASSWORD: ${{ secrets.DH_KEY }}
+  DOCKER_USERNAME: ${{ secrets.DH_ORG }}
+
+jobs:
+  extract-version:
+    name: extract version
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Extract version
+        run: |
+          if [[ "${{ github.ref }}" == refs/tags/* ]]; then
+            # It's a tag (e.g., v1.2.3)
+            VERSION="${GITHUB_REF#refs/tags/}"
+          elif [[ "${{ github.ref }}" == refs/heads/stable ]]; then
+            # stable branch -> latest
+            VERSION="latest"
+          elif [[ "${{ github.ref }}" == refs/heads/unstable ]]; then
+            # unstable branch -> latest-unstable
+            VERSION="latest-unstable"
+          else
+            # For manual triggers from other branches and will not publish any image
+            VERSION="test-build"
+          fi
+          echo "VERSION=$VERSION" >> $GITHUB_OUTPUT
+        id: extract_version
+    outputs:
+      VERSION: ${{ steps.extract_version.outputs.VERSION }}
+
+  verify-and-build:
+    name: verify reproducibility and build
+    needs: extract-version
+    strategy:
+      matrix:
+        arch: [amd64, arm64]
+        include:
+          - arch: amd64
+            rust_target: x86_64-unknown-linux-gnu
+            rust_image: >-
+              rust:1.88-bullseye@sha256:8e3c421122bf4cd3b2a866af41a4dd52d87ad9e315fd2cb5100e87a7187a9816
+            platform: linux/amd64
+            runner: ubuntu-22.04
+          - arch: arm64
+            rust_target: aarch64-unknown-linux-gnu
+            rust_image: >-
+              rust:1.88-bullseye@sha256:8b22455a7ce2adb1355067638284ee99d21cc516fab63a96c4514beaf370aa94
+            platform: linux/arm64
+            runner: ubuntu-22.04-arm
+    runs-on: ${{ matrix.runner }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver: docker
+
+      - name: Verify reproducible builds (${{ matrix.arch }})
+        run: |
+          # Build first image
+          docker build -f Dockerfile.reproducible \
+            --platform ${{ matrix.platform }} \
+            --build-arg RUST_TARGET="${{ matrix.rust_target }}" \
+            --build-arg RUST_IMAGE="${{ matrix.rust_image }}" \
+            -t lighthouse-verify-1-${{ matrix.arch }} .
+
+          # Extract binary from first build
+          docker create --name extract-1-${{ matrix.arch }} lighthouse-verify-1-${{ matrix.arch }}
+          docker cp extract-1-${{ matrix.arch }}:/lighthouse ./lighthouse-1-${{ matrix.arch }}
+          docker rm extract-1-${{ matrix.arch }}
+
+          # Clean state for second build
+          docker buildx prune -f
+          docker system prune -f
+
+          # Build second image
+          docker build -f Dockerfile.reproducible \
+            --platform ${{ matrix.platform }} \
+            --build-arg RUST_TARGET="${{ matrix.rust_target }}" \
+            --build-arg RUST_IMAGE="${{ matrix.rust_image }}" \
+            -t lighthouse-verify-2-${{ matrix.arch }} .
+
+          # Extract binary from second build
+          docker create --name extract-2-${{ matrix.arch }} lighthouse-verify-2-${{ matrix.arch }}
+          docker cp extract-2-${{ matrix.arch }}:/lighthouse ./lighthouse-2-${{ matrix.arch }}
+          docker rm extract-2-${{ matrix.arch }}
+
+          # Compare binaries
+          echo "=== Comparing binaries ==="
+          echo "Build 1 SHA256: $(sha256sum lighthouse-1-${{ matrix.arch }})"
+          echo "Build 2 SHA256: $(sha256sum lighthouse-2-${{ matrix.arch }})"
+
+          if cmp lighthouse-1-${{ matrix.arch }} lighthouse-2-${{ matrix.arch }}; then
+            echo "Reproducible build verified for ${{ matrix.arch }}"
+          else
+            echo "Reproducible build FAILED for ${{ matrix.arch }}"
+            echo "BLOCKING RELEASE: Builds are not reproducible!"
+            echo "First 10 differences:"
+            cmp -l lighthouse-1-${{ matrix.arch }} lighthouse-2-${{ matrix.arch }} | head -10
+            exit 1
+          fi
+
+          # Clean up verification artifacts but keep one image for publishing
+          rm -f lighthouse-*-${{ matrix.arch }}
+          docker rmi lighthouse-verify-1-${{ matrix.arch }} || true
+
+          # Re-tag the second image for publishing (we verified it's identical to first)
+          VERSION=${{ needs.extract-version.outputs.VERSION }}
+          FINAL_TAG="${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${VERSION}-${{ matrix.arch }}"
+          docker tag lighthouse-verify-2-${{ matrix.arch }} "$FINAL_TAG"
+
+      - name: Log in to Docker Hub
+        if: ${{ github.event_name != 'workflow_dispatch' }}
+        uses: docker/login-action@v3
+        with:
+          username: ${{ env.DOCKER_USERNAME }}
+          password: ${{ env.DOCKER_PASSWORD }}
+
+      - name: Push verified image (${{ matrix.arch }})
+        if: ${{ github.event_name != 'workflow_dispatch' }}
+        run: |
+          VERSION=${{ needs.extract-version.outputs.VERSION }}
+          IMAGE_TAG="${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${VERSION}-${{ matrix.arch }}"
+          docker push "$IMAGE_TAG"
+
+      - name: Clean up local images
+        run: |
+          docker rmi lighthouse-verify-2-${{ matrix.arch }} || true
+          VERSION=${{ needs.extract-version.outputs.VERSION }}
+          docker rmi "${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${VERSION}-${{ matrix.arch }}" || true
+
+      - name: Upload verification artifacts (on failure)
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: verification-failure-${{ matrix.arch }}
+          path: |
+            lighthouse-*-${{ matrix.arch }}
+
+  create-manifest:
+    name: create multi-arch manifest
+    runs-on: ubuntu-22.04
+    needs: [extract-version, verify-and-build]
+    if: ${{ github.event_name != 'workflow_dispatch' }}
+    steps:
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ env.DOCKER_USERNAME }}
+          password: ${{ env.DOCKER_PASSWORD }}
+
+      - name: Create and push multi-arch manifest
+        run: |
+          IMAGE_NAME=${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}
+          VERSION=${{ needs.extract-version.outputs.VERSION }}
+
+          # Create manifest for the version tag
+          docker manifest create \
+            ${IMAGE_NAME}:${VERSION} \
+            ${IMAGE_NAME}:${VERSION}-amd64 \
+            ${IMAGE_NAME}:${VERSION}-arm64
+
+          docker manifest push ${IMAGE_NAME}:${VERSION}

Cargo.toml

@@ -288,13 +288,6 @@ lto = "fat"
 codegen-units = 1
 incremental = false
 
-[profile.reproducible]
-inherits = "release"
-debug = false
-panic = "abort"
-codegen-units = 1
-overflow-checks = true
-
 [profile.release-debug]
 inherits = "release"
 debug = true

Dockerfile.reproducible

@@ -3,42 +3,22 @@ ARG RUST_IMAGE="rust:1.88-bullseye@sha256:8e3c421122bf4cd3b2a866af41a4dd52d87ad9
 FROM ${RUST_IMAGE} AS builder
 
 # Install specific version of the build dependencies
-RUN apt-get update && apt-get install -y libclang-dev=1:11.0-51+nmu5 cmake=3.18.4-2+deb11u1
+RUN apt-get update && apt-get install -y libclang-dev=1:11.0-51+nmu5 cmake=3.18.4-2+deb11u1 libjemalloc-dev=5.2.1-3
 
-# Add target architecture argument with default value
 ARG RUST_TARGET="x86_64-unknown-linux-gnu"
 
 # Copy the project to the container
-COPY . /app
+COPY ./ /app
 WORKDIR /app
 
-# Get the latest commit timestamp and set SOURCE_DATE_EPOCH (default it to 0 if not passed)
-ARG SOURCE_DATE=0
-
-# Set environment variables for reproducibility
-ARG RUSTFLAGS="-C link-arg=-Wl,--build-id=none -C metadata='' --remap-path-prefix $(pwd)=."
-ENV SOURCE_DATE_EPOCH=$SOURCE_DATE \
-    CARGO_INCREMENTAL=0 \
-    LC_ALL=C \
-    TZ=UTC \
-    RUSTFLAGS="${RUSTFLAGS}"
-
-# Set the default features if not provided
-ARG FEATURES="gnosis,slasher-lmdb,slasher-mdbx,slasher-redb,jemalloc"
-
-# Set the default profile if not provided
-ARG PROFILE="reproducible"
-
 # Build the project with the reproducible settings
-RUN cargo build --bin lighthouse \
-    --features "${FEATURES}" \
-    --profile "${PROFILE}" \
-    --locked \
-    --target "${RUST_TARGET}"
+RUN make build-reproducible
 
-RUN mv /app/target/${RUST_TARGET}/${PROFILE}/lighthouse /lighthouse
+# Move the binary to a standard location
+RUN mv /app/target/${RUST_TARGET}/release/lighthouse /lighthouse
 
 # Create a minimal final image with just the binary
 FROM gcr.io/distroless/cc-debian12:nonroot-6755e21ccd99ddead6edc8106ba03888cbeed41a
 COPY --from=builder /lighthouse /lighthouse
+
 ENTRYPOINT [ "/lighthouse" ]

Makefile

@@ -81,36 +81,70 @@ build-lcli-aarch64:
 build-lcli-riscv64:
 	cross build --bin lcli --target riscv64gc-unknown-linux-gnu --features "portable" --profile "$(CROSS_PROFILE)" --locked
 
-# extracts the current source date for reproducible builds
-SOURCE_DATE := $(shell git log -1 --pretty=%ct)
+# Environment variables for reproducible builds
+# Initialize RUSTFLAGS
+RUST_BUILD_FLAGS =
+# Remove build ID from the binary to ensure reproducibility across builds
+RUST_BUILD_FLAGS += -C link-arg=-Wl,--build-id=none
+# Remove metadata hash from symbol names to ensure reproducible builds
+RUST_BUILD_FLAGS += -C metadata=''
+
+# Set timestamp from last git commit for reproducible builds
+SOURCE_DATE ?= $(shell git log -1 --pretty=%ct)
+
+# Disable incremental compilation to avoid non-deterministic artifacts
+CARGO_INCREMENTAL_VAL = 0
+# Set C locale for consistent string handling and sorting
+LOCALE_VAL = C
+# Set UTC timezone for consistent time handling across builds
+TZ_VAL = UTC
+
+# Default profile 
+PROFILE ?= release
+
+# Features for reproducible builds
+FEATURES_REPRODUCIBLE = $(CROSS_FEATURES),jemalloc-unprefixed
+
+# Derive the architecture-specific library path from RUST_TARGET
+JEMALLOC_LIB_ARCH = $(word 1,$(subst -, ,$(RUST_TARGET)))
+JEMALLOC_OVERRIDE = /usr/lib/$(JEMALLOC_LIB_ARCH)-linux-gnu/libjemalloc.a
+
+# Default target architecture
+RUST_TARGET ?= x86_64-unknown-linux-gnu
 
-# Default image for x86_64
+# Default images for different architectures
 RUST_IMAGE_AMD64 ?= rust:1.88-bullseye@sha256:8e3c421122bf4cd3b2a866af41a4dd52d87ad9e315fd2cb5100e87a7187a9816
+RUST_IMAGE_ARM64 ?= rust:1.88-bullseye@sha256:8b22455a7ce2adb1355067638284ee99d21cc516fab63a96c4514beaf370aa94
 
-# Reproducible build for x86_64
-build-reproducible-x86_64:
+.PHONY: build-reproducible
+build-reproducible: ## Build the lighthouse binary into `target` directory with reproducible builds
+	SOURCE_DATE_EPOCH=$(SOURCE_DATE) \
+	RUSTFLAGS="${RUST_BUILD_FLAGS} --remap-path-prefix $$(pwd)=." \
+	CARGO_INCREMENTAL=${CARGO_INCREMENTAL_VAL} \
+	LC_ALL=${LOCALE_VAL} \
+	TZ=${TZ_VAL} \
+	JEMALLOC_OVERRIDE=${JEMALLOC_OVERRIDE} \
+	cargo build --bin lighthouse --features "$(FEATURES_REPRODUCIBLE)" --profile "$(PROFILE)" --locked --target $(RUST_TARGET)
+
+.PHONY: build-reproducible-x86_64
+build-reproducible-x86_64: ## Build reproducible x86_64 Docker image
 	DOCKER_BUILDKIT=1 docker build \
 		--build-arg RUST_TARGET="x86_64-unknown-linux-gnu" \
 		--build-arg RUST_IMAGE=$(RUST_IMAGE_AMD64) \
-		--build-arg SOURCE_DATE=$(SOURCE_DATE) \
 		-f Dockerfile.reproducible \
 		-t lighthouse:reproducible-amd64 .
 
-# Default image for arm64
-RUST_IMAGE_ARM64 ?= rust:1.88-bullseye@sha256:8b22455a7ce2adb1355067638284ee99d21cc516fab63a96c4514beaf370aa94
-
-# Reproducible build for aarch64
-build-reproducible-aarch64:
+.PHONY: build-reproducible-aarch64
+build-reproducible-aarch64: ## Build reproducible aarch64 Docker image
 	DOCKER_BUILDKIT=1 docker build \
 		--platform linux/arm64 \
 		--build-arg RUST_TARGET="aarch64-unknown-linux-gnu" \
 		--build-arg RUST_IMAGE=$(RUST_IMAGE_ARM64) \
-		--build-arg SOURCE_DATE=$(SOURCE_DATE) \
 		-f Dockerfile.reproducible \
 		-t lighthouse:reproducible-arm64 .
 
-# Build both architectures
-build-reproducible-all: build-reproducible-x86_64 build-reproducible-aarch64
+.PHONY: build-reproducible-all
+build-reproducible-all: build-reproducible-x86_64 build-reproducible-aarch64 ## Build both x86_64 and aarch64 reproducible Docker images
 
 # Create a `.tar.gz` containing a binary for a specific target.
 define tarball_release_binary

common/malloc_utils/Cargo.toml

@@ -21,6 +21,8 @@ jemalloc-profiling = ["tikv-jemallocator/profiling"]
 # Force the use of system malloc (or glibc) rather than jemalloc.
 # This is a no-op on Windows where jemalloc is always disabled.
 sysmalloc = []
+# Enable jemalloc with unprefixed malloc (recommended for reproducible builds)
+jemalloc-unprefixed = ["jemalloc", "tikv-jemallocator/unprefixed_malloc_on_supported_platforms"]
 
 [dependencies]
 libc = "0.2.79"

testing/state_transition_vectors/Makefile

@@ -5,4 +5,4 @@ test:
 	cargo test --release --features "$(TEST_FEATURES)"
 
 clean:
-	rm -r vectors/
+	rm -rf vectors/