From 9a2d6dab1c9ca6c22ab095a93e6345169d87b190 Mon Sep 17 00:00:00 2001
From: Moe Mahhouk <mohammed-mahhouk@hotmail.com>
Date: Mon, 16 Jun 2025 17:45:20 +0000
Subject: [PATCH 01/17] chore: refactor reproducible builds

---
 Cargo.toml              |  7 -----
 Dockerfile.reproducible | 35 ++++++-----------------
 Makefile                | 61 ++++++++++++++++++++++++++++++-----------
 3 files changed, 53 insertions(+), 50 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index 4850b2f56c4..d044b2d20a4 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -286,12 +286,5 @@ lto = "fat"
 codegen-units = 1
 incremental = false
 
-[profile.reproducible]
-inherits = "release"
-debug = false
-panic = "abort"
-codegen-units = 1
-overflow-checks = true
-
 [patch.crates-io]
 quick-protobuf = { git = "https://github.com/sigp/quick-protobuf.git", rev = "681f413312404ab6e51f0b46f39b0075c6f4ebfd" }
diff --git a/Dockerfile.reproducible b/Dockerfile.reproducible
index df57616874d..f8f26f1b32e 100644
--- a/Dockerfile.reproducible
+++ b/Dockerfile.reproducible
@@ -1,44 +1,25 @@
-# Define the Rust image as an argument with a default to x86_64 Rust 1.82 image based on Debian Bullseye
-ARG RUST_IMAGE="rust:1.82-bullseye@sha256:ac7fe7b0c9429313c0fe87d3a8993998d1fe2be9e3e91b5e2ec05d3a09d87128"
+ARG RUST_IMAGE="rust:1.86-bullseye@sha256:1110399f568f1dbe838e58f15b4162d899cb95f450f5f0ffa739614f3a4c32f1"
+
 FROM ${RUST_IMAGE} AS builder
 
 # Install specific version of the build dependencies
 RUN apt-get update && apt-get install -y libclang-dev=1:11.0-51+nmu5 cmake=3.18.4-2+deb11u1
 
-# Add target architecture argument with default value
 ARG RUST_TARGET="x86_64-unknown-linux-gnu"
 
 # Copy the project to the container
-COPY . /app
+COPY ./ /app
 WORKDIR /app
 
-# Get the latest commit timestamp and set SOURCE_DATE_EPOCH (default it to 0 if not passed)
-ARG SOURCE_DATE=0
-
-# Set environment variables for reproducibility
-ARG RUSTFLAGS="-C link-arg=-Wl,--build-id=none -C metadata='' --remap-path-prefix $(pwd)=."
-ENV SOURCE_DATE_EPOCH=$SOURCE_DATE \
-    CARGO_INCREMENTAL=0 \
-    LC_ALL=C \
-    TZ=UTC \
-    RUSTFLAGS="${RUSTFLAGS}"
-
-# Set the default features if not provided
-ARG FEATURES="gnosis,slasher-lmdb,slasher-mdbx,slasher-redb,jemalloc"
-
-# Set the default profile if not provided
-ARG PROFILE="reproducible"
-
 # Build the project with the reproducible settings
-RUN cargo build --bin lighthouse \
-    --features "${FEATURES}" \
-    --profile "${PROFILE}" \
-    --locked \
-    --target "${RUST_TARGET}"
+RUN make build-reproducible
 
-RUN mv /app/target/${RUST_TARGET}/${PROFILE}/lighthouse /lighthouse
+# Move the binary to a standard location
+RUN mv /app/target/${RUST_TARGET}/release/lighthouse /lighthouse
 
 # Create a minimal final image with just the binary
 FROM gcr.io/distroless/cc-debian12:nonroot-6755e21ccd99ddead6edc8106ba03888cbeed41a
 COPY --from=builder /lighthouse /lighthouse
+
+EXPOSE 30303 30303/udp 9001 8545 8546
 ENTRYPOINT [ "/lighthouse" ]
diff --git a/Makefile b/Makefile
index 75b6811b743..a58dcfc4f90 100644
--- a/Makefile
+++ b/Makefile
@@ -82,36 +82,65 @@ build-lcli-aarch64:
 build-lcli-riscv64:
 	cross build --bin lcli --target riscv64gc-unknown-linux-gnu --features "portable" --profile "$(CROSS_PROFILE)" --locked
 
-# extracts the current source date for reproducible builds
-SOURCE_DATE := $(shell git log -1 --pretty=%ct)
+# Environment variables for reproducible builds
+# Initialize RUSTFLAGS
+RUST_BUILD_FLAGS =
+# Remove build ID from the binary to ensure reproducibility across builds
+RUST_BUILD_FLAGS += -C link-arg=-Wl,--build-id=none
+# Remove metadata hash from symbol names to ensure reproducible builds
+RUST_BUILD_FLAGS += -C metadata=''
+
+# Set timestamp from last git commit for reproducible builds
+SOURCE_DATE ?= $(shell git log -1 --pretty=%ct)
+
+# Disable incremental compilation to avoid non-deterministic artifacts
+CARGO_INCREMENTAL_VAL = 0
+# Set C locale for consistent string handling and sorting
+LOCALE_VAL = C
+# Set UTC timezone for consistent time handling across builds
+TZ_VAL = UTC
+
+# Default features for lighthouse
+FEATURES ?= gnosis,slasher-lmdb,slasher-mdbx,slasher-redb,jemalloc
+
+# Default profile 
+PROFILE ?= release
+
+# Default target architecture
+RUST_TARGET ?= x86_64-unknown-linux-gnu
+
+# Default images for different architectures
+RUST_IMAGE_AMD64 ?= rust:1.86-bullseye@sha256:1110399f568f1dbe838e58f15b4162d899cb95f450f5f0ffa739614f3a4c32f1
+RUST_IMAGE_ARM64 ?= rust:1.86-bullseye@sha256:36053eabadeb701e3e0406610a2ce72ccfa10b7828963cd08cffdcf660518b27
 
-# Default image for x86_64
-RUST_IMAGE_AMD64 ?= rust:1.82-bullseye@sha256:ac7fe7b0c9429313c0fe87d3a8993998d1fe2be9e3e91b5e2ec05d3a09d87128
+.PHONY: build-reproducible
+build-reproducible: ## Build the lighthouse binary into `target` directory with reproducible builds
+	SOURCE_DATE_EPOCH=$(SOURCE_DATE) \
+	RUSTFLAGS="${RUST_BUILD_FLAGS} --remap-path-prefix $$(pwd)=." \
+	CARGO_INCREMENTAL=${CARGO_INCREMENTAL_VAL} \
+	LC_ALL=${LOCALE_VAL} \
+	TZ=${TZ_VAL} \
+	cargo build --bin lighthouse --features "$(FEATURES)" --profile "$(PROFILE)" --locked --target $(RUST_TARGET)
 
-# Reproducible build for x86_64
-build-reproducible-x86_64:
+.PHONY: build-reproducible-x86_64
+build-reproducible-x86_64: ## Build reproducible x86_64 Docker image
 	DOCKER_BUILDKIT=1 docker build \
 		--build-arg RUST_TARGET="x86_64-unknown-linux-gnu" \
 		--build-arg RUST_IMAGE=$(RUST_IMAGE_AMD64) \
-		--build-arg SOURCE_DATE=$(SOURCE_DATE) \
-		-f Dockerfile.reproducible \
+		-f Dockerfile.test \
 		-t lighthouse:reproducible-amd64 .
 
-# Default image for arm64
-RUST_IMAGE_ARM64 ?= rust:1.82-bullseye@sha256:3c1b8b6487513ad4e753d008b960260f5bcc81bf110883460f6ed3cd72bf439b
-
-# Reproducible build for aarch64
-build-reproducible-aarch64:
+.PHONY: build-reproducible-aarch64
+build-reproducible-aarch64: ## Build reproducible aarch64 Docker image
 	DOCKER_BUILDKIT=1 docker build \
 		--platform linux/arm64 \
 		--build-arg RUST_TARGET="aarch64-unknown-linux-gnu" \
 		--build-arg RUST_IMAGE=$(RUST_IMAGE_ARM64) \
-		--build-arg SOURCE_DATE=$(SOURCE_DATE) \
 		-f Dockerfile.reproducible \
 		-t lighthouse:reproducible-arm64 .
 
-# Build both architectures
-build-reproducible-all: build-reproducible-x86_64 build-reproducible-aarch64
+.PHONY: build-reproducible-all
+build-reproducible-all: build-reproducible-x86_64 build-reproducible-aarch64 ## Build both x86_64 and aarch64 reproducible Docker images
 
 # Create a `.tar.gz` containing a binary for a specific target.
 define tarball_release_binary

From 10a3b59790945de0416c7117e87f7c112c7fcc49 Mon Sep 17 00:00:00 2001
From: MoeMahhouk <mohammed-mahhouk@hotmail.com>
Date: Mon, 16 Jun 2025 22:00:31 +0000
Subject: [PATCH 02/17] Add release-reproducible CI workflow

---
 .github/workflows/release-reproducible.yml | 131 +++++++++++++++++++++
 Makefile                                   |   2 +-
 2 files changed, 132 insertions(+), 1 deletion(-)
 create mode 100644 .github/workflows/release-reproducible.yml

diff --git a/.github/workflows/release-reproducible.yml b/.github/workflows/release-reproducible.yml
new file mode 100644
index 00000000000..80f7c670306
--- /dev/null
+++ b/.github/workflows/release-reproducible.yml
@@ -0,0 +1,131 @@
+# This workflow is for building and pushing reproducible Docker images for releases.
+name: release-reproducible
+
+on:
+  push:
+    tags:
+      - v*
+  workflow_dispatch:
+    inputs:
+      dry_run:
+        description: "Enable dry run mode (builds images but skips push to registry)"
+        type: boolean
+        default: false
+
+env:
+  DOCKER_REPRODUCIBLE_IMAGE_NAME: ${{ github.repository_owner }}/lighthouse-reproducible
+  DOCKER_PASSWORD: ${{ secrets.DH_KEY }}
+  DOCKER_USERNAME: ${{ secrets.DH_ORG }}
+
+jobs:
+  extract-version:
+    name: extract version
+    runs-on: ubuntu-latest
+    steps:
+      - name: Extract version
+        run: echo "VERSION=$(echo ${GITHUB_REF#refs/tags/})" >> $GITHUB_OUTPUT
+        id: extract_version
+    outputs:
+      VERSION: ${{ steps.extract_version.outputs.VERSION }}
+
+  build-reproducible:
+    name: build and push reproducible images
+    runs-on: ubuntu-latest
+    needs: extract-version
+    strategy:
+      matrix:
+        arch: [amd64, arm64]
+        include:
+          - arch: amd64
+            rust_target: x86_64-unknown-linux-gnu
+            rust_image: rust:1.86-bullseye@sha256:1110399f568f1dbe838e58f15b4162d899cb95f450f5f0ffa739614f3a4c32f1
+            platform: linux/amd64
+          - arch: arm64
+            rust_target: aarch64-unknown-linux-gnu
+            rust_image: rust:1.86-bullseye@sha256:36053eabadeb701e3e0406610a2ce72ccfa10b7828963cd08cffdcf660518b27
+            platform: linux/arm64
+    steps:
+      - uses: actions/checkout@v4
+      
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        
+      - name: Log in to Docker Hub
+        if: ${{ github.event.inputs.dry_run != 'true' }}
+        uses: docker/login-action@v3
+        with:
+          username: ${{ env.DOCKER_USERNAME }}
+          password: ${{ env.DOCKER_PASSWORD }}
+          
+      - name: Build reproducible image (${{ matrix.arch }})
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ./Dockerfile.reproducible
+          platforms: ${{ matrix.platform }}
+          push: ${{ github.event.inputs.dry_run != 'true' }}
+          tags: |
+            ${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}
+          build-args: |
+            RUST_TARGET=${{ matrix.rust_target }}
+            RUST_IMAGE=${{ matrix.rust_image }}
+          cache-from: type=gha,scope=${{ matrix.arch }}
+          cache-to: type=gha,mode=max,scope=${{ matrix.arch }}
+          provenance: false
+        env:
+          DOCKER_BUILD_RECORD_UPLOAD: false
+
+  create-manifest:
+    name: create multi-arch manifest
+    runs-on: ubuntu-latest
+    needs: [extract-version, build-reproducible]
+    if: ${{ github.event.inputs.dry_run != 'true' }}
+    steps:
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ env.DOCKER_USERNAME }}
+          password: ${{ env.DOCKER_PASSWORD }}
+          
+      - name: Create and push multi-arch manifest
+        run: |
+          # Create manifest for version tag
+          docker manifest create \
+            ${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${{ needs.extract-version.outputs.VERSION }} \
+            ${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${{ needs.extract-version.outputs.VERSION }}-amd64 \
+            ${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${{ needs.extract-version.outputs.VERSION }}-arm64
+          
+          docker manifest push ${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${{ needs.extract-version.outputs.VERSION }}
+          
+          # Create manifest for latest tag
+          docker manifest create \
+            ${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:latest \
+            ${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${{ needs.extract-version.outputs.VERSION }}-amd64 \
+            ${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${{ needs.extract-version.outputs.VERSION }}-arm64
+          
+          docker manifest push ${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:latest
+
+  dry-run-summary:
+    name: dry run summary
+    runs-on: ubuntu-latest
+    needs: [build-reproducible, extract-version]
+    if: ${{ github.event.inputs.dry_run == 'true' }}
+    steps:
+      - name: Summarize dry run
+        run: |
+          echo "## 🧪 Reproducible Build Dry Run Summary"
+          echo ""
+          echo "✅ Successfully completed dry run for version ${{ needs.extract-version.outputs.VERSION }}"
+          echo ""
+          echo "### What would happen in a real release:"
+          echo "- Multi-arch reproducible Docker images would be built"
+          echo "- Images would be pushed to Docker Hub as:"
+          echo "  - \`${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${{ needs.extract-version.outputs.VERSION }}\`"
+          echo "  - \`${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:latest\`"
+          echo ""
+          echo "### Architectures built:"
+          echo "- linux/amd64 (x86_64-unknown-linux-gnu)"
+          echo "- linux/arm64 (aarch64-unknown-linux-gnu)"
+          echo ""
+          echo "### Next Steps"
+          echo "To perform a real release, push a git tag (e.g., \`git tag v4.6.0 && git push origin v4.6.0\`)"
diff --git a/Makefile b/Makefile
index a58dcfc4f90..caae9f8ac2a 100644
--- a/Makefile
+++ b/Makefile
@@ -127,7 +127,7 @@ build-reproducible-x86_64: ## Build reproducible x86_64 Docker image
 	DOCKER_BUILDKIT=1 docker build \
 		--build-arg RUST_TARGET="x86_64-unknown-linux-gnu" \
 		--build-arg RUST_IMAGE=$(RUST_IMAGE_AMD64) \
-		-f Dockerfile.test \
+		-f Dockerfile.reproducible \
 		-t lighthouse:reproducible-amd64 .
 
 .PHONY: build-reproducible-aarch64

From a4348b2ef3a833e09df2ace1d90d077481c9b3fd Mon Sep 17 00:00:00 2001
From: MoeMahhouk <mohammed-mahhouk@hotmail.com>
Date: Mon, 16 Jun 2025 22:10:06 +0000
Subject: [PATCH 03/17] Add reproducible-build workflow to catch regressions

---
 .github/workflows/reproducible-build.yml | 178 +++++++++++++++++++++++
 1 file changed, 178 insertions(+)
 create mode 100644 .github/workflows/reproducible-build.yml

diff --git a/.github/workflows/reproducible-build.yml b/.github/workflows/reproducible-build.yml
new file mode 100644
index 00000000000..f2115eba271
--- /dev/null
+++ b/.github/workflows/reproducible-build.yml
@@ -0,0 +1,178 @@
+name: reproducible-build
+
+on:
+  workflow_dispatch: {}
+  schedule:
+    # Run every 2 days at 1 AM UTC
+    - cron: "0 1 */2 * *"
+  pull_request:
+    # Also run on PRs that might affect reproducible builds
+    paths:
+      - "Makefile"
+      - "Dockerfile.reproducible"
+      - ".github/workflows/reproducible-build.yml"
+      - "Cargo.toml"
+      - "Cargo.lock"
+
+jobs:
+  build-x86_64:
+    name: test reproducible builds (x86_64)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          target: x86_64-unknown-linux-gnu
+          
+      - name: Install build dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libclang-dev cmake
+          
+      - name: Install cargo-cache
+        run: cargo install cargo-cache
+        
+      - uses: Swatinem/rust-cache@v2
+        with:
+          cache-on-failure: true
+          key: reproducible-build-x86_64
+
+      - name: Build Lighthouse (first build)
+        run: |
+          make build-reproducible RUST_TARGET=x86_64-unknown-linux-gnu
+          cp target/x86_64-unknown-linux-gnu/release/lighthouse lighthouse-build-1
+          sha256sum lighthouse-build-1 > lighthouse-build-1.sha256
+          
+      - name: Clean build artifacts and cache
+        run: |
+          make clean
+          cargo cache -a
+          rm -rf target/
+          
+      - name: Build Lighthouse (second build)
+        run: |
+          make build-reproducible RUST_TARGET=x86_64-unknown-linux-gnu
+          cp target/x86_64-unknown-linux-gnu/release/lighthouse lighthouse-build-2
+          sha256sum lighthouse-build-2 > lighthouse-build-2.sha256
+          
+      - name: Compare binaries
+        run: |
+          echo "=== Build 1 SHA256 ==="
+          cat lighthouse-build-1.sha256
+          echo "=== Build 2 SHA256 ==="
+          cat lighthouse-build-2.sha256
+          echo "=== Binary Comparison ==="
+          if cmp lighthouse-build-1 lighthouse-build-2; then
+            echo "✅ Binaries are identical - reproducible build PASSED"
+          else
+            echo "❌ Binaries differ - reproducible build FAILED"
+            exit 1
+          fi
+          
+      - name: Upload build artifacts (on failure)
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: failed-reproducible-builds-x86_64
+          path: |
+            lighthouse-build-1
+            lighthouse-build-2
+            lighthouse-build-1.sha256
+            lighthouse-build-2.sha256
+
+  build-aarch64:
+    name: test reproducible builds (aarch64)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          target: aarch64-unknown-linux-gnu
+          
+      - name: Install build dependencies and cross-compilation tools
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libclang-dev cmake gcc-aarch64-linux-gnu
+          
+      - name: Install cargo-cache
+        run: cargo install cargo-cache
+        
+      - uses: Swatinem/rust-cache@v2
+        with:
+          cache-on-failure: true
+          key: reproducible-build-aarch64
+
+      - name: Build Lighthouse (first build)
+        run: |
+          make build-reproducible RUST_TARGET=aarch64-unknown-linux-gnu
+          cp target/aarch64-unknown-linux-gnu/release/lighthouse lighthouse-build-1-arm64
+          sha256sum lighthouse-build-1-arm64 > lighthouse-build-1-arm64.sha256
+          
+      - name: Clean build artifacts and cache
+        run: |
+          make clean
+          cargo cache -a
+          rm -rf target/
+          
+      - name: Build Lighthouse (second build)
+        run: |
+          make build-reproducible RUST_TARGET=aarch64-unknown-linux-gnu
+          cp target/aarch64-unknown-linux-gnu/release/lighthouse lighthouse-build-2-arm64
+          sha256sum lighthouse-build-2-arm64 > lighthouse-build-2-arm64.sha256
+          
+      - name: Compare binaries
+        run: |
+          echo "=== Build 1 SHA256 (ARM64) ==="
+          cat lighthouse-build-1-arm64.sha256
+          echo "=== Build 2 SHA256 (ARM64) ==="
+          cat lighthouse-build-2-arm64.sha256
+          echo "=== Binary Comparison ==="
+          if cmp lighthouse-build-1-arm64 lighthouse-build-2-arm64; then
+            echo "✅ ARM64 binaries are identical - reproducible build PASSED"
+          else
+            echo "❌ ARM64 binaries differ - reproducible build FAILED"
+            exit 1
+          fi
+          
+      - name: Upload build artifacts (on failure)
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: failed-reproducible-builds-aarch64
+          path: |
+            lighthouse-build-1-arm64
+            lighthouse-build-2-arm64
+            lighthouse-build-1-arm64.sha256
+            lighthouse-build-2-arm64.sha256
+
+  summary:
+    name: reproducible build summary
+    runs-on: ubuntu-latest
+    needs: [build-x86_64, build-aarch64]
+    if: always()
+    steps:
+      - name: Report results
+        run: |
+          echo "## 🔄 Reproducible Build Test Results"
+          echo ""
+          if [[ "${{ needs.build-x86_64.result }}" == "success" ]]; then
+            echo "✅ **x86_64**: Reproducible builds PASSED"
+          else
+            echo "❌ **x86_64**: Reproducible builds FAILED"
+          fi
+          
+          if [[ "${{ needs.build-aarch64.result }}" == "success" ]]; then
+            echo "✅ **aarch64**: Reproducible builds PASSED"
+          else
+            echo "❌ **aarch64**: Reproducible builds FAILED"
+          fi
+          
+          echo ""
+          if [[ "${{ needs.build-x86_64.result }}" == "success" ]] && [[ "${{ needs.build-aarch64.result }}" == "success" ]]; then
+            echo "🎉 **Overall**: All reproducible builds are working correctly!"
+          else
+            echo "⚠️  **Overall**: Some reproducible builds failed - check the logs above"
+            exit 1
+          fi

From 608caaba9e6296134e8ad35d1227cb2c5e0e8ca8 Mon Sep 17 00:00:00 2001
From: MoeMahhouk <mohammed-mahhouk@hotmail.com>
Date: Mon, 16 Jun 2025 22:23:21 +0000
Subject: [PATCH 04/17] fixing linting issues

---
 .github/workflows/release-reproducible.yml | 69 +++++++++++---------
 .github/workflows/reproducible-build.yml   | 74 ++++++++++++----------
 2 files changed, 82 insertions(+), 61 deletions(-)

diff --git a/.github/workflows/release-reproducible.yml b/.github/workflows/release-reproducible.yml
index 80f7c670306..10f668869ad 100644
--- a/.github/workflows/release-reproducible.yml
+++ b/.github/workflows/release-reproducible.yml
@@ -1,4 +1,3 @@
-# This workflow is for building and pushing reproducible Docker images for releases.
 name: release-reproducible
 
 on:
@@ -8,12 +7,14 @@ on:
   workflow_dispatch:
     inputs:
       dry_run:
-        description: "Enable dry run mode (builds images but skips push to registry)"
+        description: >-
+          Enable dry run mode (builds images but skips push to registry)
         type: boolean
         default: false
 
 env:
-  DOCKER_REPRODUCIBLE_IMAGE_NAME: ${{ github.repository_owner }}/lighthouse-reproducible
+  DOCKER_REPRODUCIBLE_IMAGE_NAME: >-
+    ${{ github.repository_owner }}/lighthouse-reproducible
   DOCKER_PASSWORD: ${{ secrets.DH_KEY }}
   DOCKER_USERNAME: ${{ secrets.DH_ORG }}
 
@@ -23,7 +24,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Extract version
-        run: echo "VERSION=$(echo ${GITHUB_REF#refs/tags/})" >> $GITHUB_OUTPUT
+        run: >-
+          echo "VERSION=$(echo ${GITHUB_REF#refs/tags/})" >> $GITHUB_OUTPUT
         id: extract_version
     outputs:
       VERSION: ${{ steps.extract_version.outputs.VERSION }}
@@ -38,42 +40,46 @@ jobs:
         include:
           - arch: amd64
             rust_target: x86_64-unknown-linux-gnu
-            rust_image: rust:1.86-bullseye@sha256:1110399f568f1dbe838e58f15b4162d899cb95f450f5f0ffa739614f3a4c32f1
+            rust_image: >-
+              rust:1.86-bullseye@sha256:1110399f568f1dbe838e58f15b4162d899cb95f450f5f0ffa739614f3a4c32f1
             platform: linux/amd64
           - arch: arm64
             rust_target: aarch64-unknown-linux-gnu
-            rust_image: rust:1.86-bullseye@sha256:36053eabadeb701e3e0406610a2ce72ccfa10b7828963cd08cffdcf660518b27
+            rust_image: >-
+              rust:1.86-bullseye@sha256:36053eabadeb701e3e0406610a2ce72ccfa10b7828963cd08cffdcf660518b27
             platform: linux/arm64
     steps:
       - uses: actions/checkout@v4
-      
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
-        
+
       - name: Log in to Docker Hub
         if: ${{ github.event.inputs.dry_run != 'true' }}
         uses: docker/login-action@v3
         with:
           username: ${{ env.DOCKER_USERNAME }}
           password: ${{ env.DOCKER_PASSWORD }}
-          
+
       - name: Build reproducible image (${{ matrix.arch }})
         uses: docker/build-push-action@v6
+        env:
+          IMAGE_BASE: ${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}
+          VERSION: ${{ needs.extract-version.outputs.VERSION }}
+          ARCH: ${{ matrix.arch }}
+          DOCKER_BUILD_RECORD_UPLOAD: false
         with:
           context: .
           file: ./Dockerfile.reproducible
           platforms: ${{ matrix.platform }}
           push: ${{ github.event.inputs.dry_run != 'true' }}
-          tags: |
-            ${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}
+          tags: ${{ env.IMAGE_BASE }}:${{ env.VERSION }}-${{ env.ARCH }}
           build-args: |
             RUST_TARGET=${{ matrix.rust_target }}
             RUST_IMAGE=${{ matrix.rust_image }}
           cache-from: type=gha,scope=${{ matrix.arch }}
           cache-to: type=gha,mode=max,scope=${{ matrix.arch }}
           provenance: false
-        env:
-          DOCKER_BUILD_RECORD_UPLOAD: false
 
   create-manifest:
     name: create multi-arch manifest
@@ -86,24 +92,26 @@ jobs:
         with:
           username: ${{ env.DOCKER_USERNAME }}
           password: ${{ env.DOCKER_PASSWORD }}
-          
+
       - name: Create and push multi-arch manifest
         run: |
+          IMAGE_NAME=${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}
+          VERSION=${{ needs.extract-version.outputs.VERSION }}
           # Create manifest for version tag
           docker manifest create \
-            ${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${{ needs.extract-version.outputs.VERSION }} \
-            ${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${{ needs.extract-version.outputs.VERSION }}-amd64 \
-            ${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${{ needs.extract-version.outputs.VERSION }}-arm64
-          
-          docker manifest push ${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${{ needs.extract-version.outputs.VERSION }}
-          
+            ${IMAGE_NAME}:${VERSION} \
+            ${IMAGE_NAME}:${VERSION}-amd64 \
+            ${IMAGE_NAME}:${VERSION}-arm64
+
+          docker manifest push ${IMAGE_NAME}:${VERSION}
+
           # Create manifest for latest tag
           docker manifest create \
-            ${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:latest \
-            ${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${{ needs.extract-version.outputs.VERSION }}-amd64 \
-            ${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${{ needs.extract-version.outputs.VERSION }}-arm64
-          
-          docker manifest push ${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:latest
+            ${IMAGE_NAME}:latest \
+            ${IMAGE_NAME}:${VERSION}-amd64 \
+            ${IMAGE_NAME}:${VERSION}-arm64
+
+          docker manifest push ${IMAGE_NAME}:latest
 
   dry-run-summary:
     name: dry run summary
@@ -113,19 +121,22 @@ jobs:
     steps:
       - name: Summarize dry run
         run: |
+          IMAGE_NAME=${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}
+          VERSION=${{ needs.extract-version.outputs.VERSION }}
           echo "## 🧪 Reproducible Build Dry Run Summary"
           echo ""
-          echo "✅ Successfully completed dry run for version ${{ needs.extract-version.outputs.VERSION }}"
+          echo "✅ Successfully completed dry run for version ${VERSION}"
           echo ""
           echo "### What would happen in a real release:"
           echo "- Multi-arch reproducible Docker images would be built"
           echo "- Images would be pushed to Docker Hub as:"
-          echo "  - \`${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${{ needs.extract-version.outputs.VERSION }}\`"
-          echo "  - \`${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:latest\`"
+          echo "  - \`${IMAGE_NAME}:${VERSION}\`"
+          echo "  - \`${IMAGE_NAME}:latest\`"
           echo ""
           echo "### Architectures built:"
           echo "- linux/amd64 (x86_64-unknown-linux-gnu)"
           echo "- linux/arm64 (aarch64-unknown-linux-gnu)"
           echo ""
           echo "### Next Steps"
-          echo "To perform a real release, push a git tag (e.g., \`git tag v4.6.0 && git push origin v4.6.0\`)"
+          echo "To perform a real release, push a git tag"
+          echo "(e.g., \`git tag v4.6.0 && git push origin v4.6.0\`)"
diff --git a/.github/workflows/reproducible-build.yml b/.github/workflows/reproducible-build.yml
index f2115eba271..843e9246966 100644
--- a/.github/workflows/reproducible-build.yml
+++ b/.github/workflows/reproducible-build.yml
@@ -3,10 +3,8 @@ name: reproducible-build
 on:
   workflow_dispatch: {}
   schedule:
-    # Run every 2 days at 1 AM UTC
     - cron: "0 1 */2 * *"
   pull_request:
-    # Also run on PRs that might affect reproducible builds
     paths:
       - "Makefile"
       - "Dockerfile.reproducible"
@@ -20,19 +18,19 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      
+
       - uses: dtolnay/rust-toolchain@stable
         with:
           target: x86_64-unknown-linux-gnu
-          
+
       - name: Install build dependencies
         run: |
           sudo apt-get update
           sudo apt-get install -y libclang-dev cmake
-          
+
       - name: Install cargo-cache
         run: cargo install cargo-cache
-        
+
       - uses: Swatinem/rust-cache@v2
         with:
           cache-on-failure: true
@@ -40,22 +38,26 @@ jobs:
 
       - name: Build Lighthouse (first build)
         run: |
-          make build-reproducible RUST_TARGET=x86_64-unknown-linux-gnu
-          cp target/x86_64-unknown-linux-gnu/release/lighthouse lighthouse-build-1
+          make build-reproducible \
+            RUST_TARGET=x86_64-unknown-linux-gnu
+          cp target/x86_64-unknown-linux-gnu/release/lighthouse \
+            lighthouse-build-1
           sha256sum lighthouse-build-1 > lighthouse-build-1.sha256
-          
+
       - name: Clean build artifacts and cache
         run: |
           make clean
           cargo cache -a
           rm -rf target/
-          
+
       - name: Build Lighthouse (second build)
         run: |
-          make build-reproducible RUST_TARGET=x86_64-unknown-linux-gnu
-          cp target/x86_64-unknown-linux-gnu/release/lighthouse lighthouse-build-2
+          make build-reproducible \
+            RUST_TARGET=x86_64-unknown-linux-gnu
+          cp target/x86_64-unknown-linux-gnu/release/lighthouse \
+            lighthouse-build-2
           sha256sum lighthouse-build-2 > lighthouse-build-2.sha256
-          
+
       - name: Compare binaries
         run: |
           echo "=== Build 1 SHA256 ==="
@@ -69,7 +71,7 @@ jobs:
             echo "❌ Binaries differ - reproducible build FAILED"
             exit 1
           fi
-          
+
       - name: Upload build artifacts (on failure)
         if: failure()
         uses: actions/upload-artifact@v4
@@ -86,19 +88,19 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      
+
       - uses: dtolnay/rust-toolchain@stable
         with:
           target: aarch64-unknown-linux-gnu
-          
+
       - name: Install build dependencies and cross-compilation tools
         run: |
           sudo apt-get update
           sudo apt-get install -y libclang-dev cmake gcc-aarch64-linux-gnu
-          
+
       - name: Install cargo-cache
         run: cargo install cargo-cache
-        
+
       - uses: Swatinem/rust-cache@v2
         with:
           cache-on-failure: true
@@ -106,22 +108,28 @@ jobs:
 
       - name: Build Lighthouse (first build)
         run: |
-          make build-reproducible RUST_TARGET=aarch64-unknown-linux-gnu
-          cp target/aarch64-unknown-linux-gnu/release/lighthouse lighthouse-build-1-arm64
-          sha256sum lighthouse-build-1-arm64 > lighthouse-build-1-arm64.sha256
-          
+          make build-reproducible \
+            RUST_TARGET=aarch64-unknown-linux-gnu
+          cp target/aarch64-unknown-linux-gnu/release/lighthouse \
+            lighthouse-build-1-arm64
+          sha256sum lighthouse-build-1-arm64 > \
+            lighthouse-build-1-arm64.sha256
+
       - name: Clean build artifacts and cache
         run: |
           make clean
           cargo cache -a
           rm -rf target/
-          
+
       - name: Build Lighthouse (second build)
         run: |
-          make build-reproducible RUST_TARGET=aarch64-unknown-linux-gnu
-          cp target/aarch64-unknown-linux-gnu/release/lighthouse lighthouse-build-2-arm64
-          sha256sum lighthouse-build-2-arm64 > lighthouse-build-2-arm64.sha256
-          
+          make build-reproducible \
+            RUST_TARGET=aarch64-unknown-linux-gnu
+          cp target/aarch64-unknown-linux-gnu/release/lighthouse \
+            lighthouse-build-2-arm64
+          sha256sum lighthouse-build-2-arm64 > \
+            lighthouse-build-2-arm64.sha256
+
       - name: Compare binaries
         run: |
           echo "=== Build 1 SHA256 (ARM64) ==="
@@ -135,7 +143,7 @@ jobs:
             echo "❌ ARM64 binaries differ - reproducible build FAILED"
             exit 1
           fi
-          
+
       - name: Upload build artifacts (on failure)
         if: failure()
         uses: actions/upload-artifact@v4
@@ -162,17 +170,19 @@ jobs:
           else
             echo "❌ **x86_64**: Reproducible builds FAILED"
           fi
-          
+
           if [[ "${{ needs.build-aarch64.result }}" == "success" ]]; then
             echo "✅ **aarch64**: Reproducible builds PASSED"
           else
             echo "❌ **aarch64**: Reproducible builds FAILED"
           fi
-          
+
           echo ""
-          if [[ "${{ needs.build-x86_64.result }}" == "success" ]] && [[ "${{ needs.build-aarch64.result }}" == "success" ]]; then
+          if [[ "${{ needs.build-x86_64.result }}" == "success" ]] \
+            && [[ "${{ needs.build-aarch64.result }}" == "success" ]]; then
             echo "🎉 **Overall**: All reproducible builds are working correctly!"
           else
-            echo "⚠️  **Overall**: Some reproducible builds failed - check the logs above"
+            echo "⚠️  **Overall**: Some reproducible builds failed"
+            echo "Check the logs above"
             exit 1
           fi

From d93242f508f2d57c6b28abdb670666612251842d Mon Sep 17 00:00:00 2001
From: MoeMahhouk <mohammed-mahhouk@hotmail.com>
Date: Tue, 24 Jun 2025 20:42:01 +0000
Subject: [PATCH 05/17] refine the reproducible build and test arm github
 runner

---
 .github/workflows/reproducible-build.yml  | 236 ++++++++++++----------
 testing/state_transition_vectors/Makefile |   2 +-
 2 files changed, 133 insertions(+), 105 deletions(-)

diff --git a/.github/workflows/reproducible-build.yml b/.github/workflows/reproducible-build.yml
index 843e9246966..abe37510243 100644
--- a/.github/workflows/reproducible-build.yml
+++ b/.github/workflows/reproducible-build.yml
@@ -19,142 +19,169 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - uses: dtolnay/rust-toolchain@stable
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
         with:
-          target: x86_64-unknown-linux-gnu
+          driver: docker  # Use docker driver instead of docker-container for reproducibility
 
-      - name: Install build dependencies
+      - name: Build first reproducible Docker image
         run: |
-          sudo apt-get update
-          sudo apt-get install -y libclang-dev cmake
+          echo "=== Building first Docker image (x86_64) ==="
+          docker build -f Dockerfile.reproducible \
+            --build-arg RUST_TARGET="x86_64-unknown-linux-gnu" \
+            --build-arg RUST_IMAGE="rust:1.86-bullseye@sha256:1110399f568f1dbe838e58f15b4162d899cb95f450f5f0ffa739614f3a4c32f1" \
+            -t build-lighthouse-1 .
+
+          echo "=== Extracting binary from first build ==="
+          docker create --name extract-lighthouse-1 build-lighthouse-1
+          docker cp extract-lighthouse-1:/lighthouse ./lighthouse-build-1
+          docker rm extract-lighthouse-1
+
+          echo "=== First build info ==="
+          ls -la lighthouse-build-1
+          sha256sum lighthouse-build-1
+          file lighthouse-build-1
+
+      - name: Clean Docker state completely
+        run: |
+          echo "=== Cleaning Docker state ==="
+          # Remove the first image
+          docker rmi build-lighthouse-1 || true
 
-      - name: Install cargo-cache
-        run: cargo install cargo-cache
+          # Remove all build cache (important for reproducibility testing)
+          docker buildx prune -f || true
+          docker system prune -f || true
 
-      - uses: Swatinem/rust-cache@v2
-        with:
-          cache-on-failure: true
-          key: reproducible-build-x86_64
+          # Clear any remaining containers
+          docker container prune -f || true
 
-      - name: Build Lighthouse (first build)
-        run: |
-          make build-reproducible \
-            RUST_TARGET=x86_64-unknown-linux-gnu
-          cp target/x86_64-unknown-linux-gnu/release/lighthouse \
-            lighthouse-build-1
-          sha256sum lighthouse-build-1 > lighthouse-build-1.sha256
+          echo "=== Docker state cleaned ==="
+          docker images
+          docker ps -a
 
-      - name: Clean build artifacts and cache
+      - name: Build second reproducible Docker image
         run: |
-          make clean
-          cargo cache -a
-          rm -rf target/
-
-      - name: Build Lighthouse (second build)
+          echo "=== Building second Docker image (x86_64) ==="
+          docker build -f Dockerfile.reproducible \
+            --build-arg RUST_TARGET="x86_64-unknown-linux-gnu" \
+            --build-arg RUST_IMAGE="rust:1.86-bullseye@sha256:1110399f568f1dbe838e58f15b4162d899cb95f450f5f0ffa739614f3a4c32f1" \
+            -t build-lighthouse-2 .
+
+          echo "=== Extracting binary from second build ==="
+          docker create --name extract-lighthouse-2 build-lighthouse-2
+          docker cp extract-lighthouse-2:/lighthouse ./lighthouse-build-2
+          docker rm extract-lighthouse-2
+
+          echo "=== Second build info ==="
+          ls -la lighthouse-build-2
+          sha256sum lighthouse-build-2
+          file lighthouse-build-2
+
+      - name: Compare Docker-built binaries
         run: |
-          make build-reproducible \
-            RUST_TARGET=x86_64-unknown-linux-gnu
-          cp target/x86_64-unknown-linux-gnu/release/lighthouse \
-            lighthouse-build-2
-          sha256sum lighthouse-build-2 > lighthouse-build-2.sha256
+          echo "=== Comparing Docker-built binaries (x86_64) ==="
+          echo "Build 1 info:"
+          ls -la lighthouse-build-1
+          echo "Build 2 info:"
+          ls -la lighthouse-build-2
 
-      - name: Compare binaries
-        run: |
-          echo "=== Build 1 SHA256 ==="
-          cat lighthouse-build-1.sha256
-          echo "=== Build 2 SHA256 ==="
-          cat lighthouse-build-2.sha256
-          echo "=== Binary Comparison ==="
+          echo "=== SHA256 checksums ==="
+          sha256sum lighthouse-build-*
+
+          echo "=== Binary comparison ==="
           if cmp lighthouse-build-1 lighthouse-build-2; then
-            echo "✅ Binaries are identical - reproducible build PASSED"
+            echo "✅ SUCCESS: Docker-built binaries are identical!"
+            echo "✅ Reproducible Docker build PASSED for x86_64"
           else
-            echo "❌ Binaries differ - reproducible build FAILED"
+            echo "❌ FAILED: Docker-built binaries differ"
+            echo "First 10 differences:"
+            cmp -l lighthouse-build-1 lighthouse-build-2 | head -10
             exit 1
           fi
 
-      - name: Upload build artifacts (on failure)
-        if: failure()
-        uses: actions/upload-artifact@v4
-        with:
-          name: failed-reproducible-builds-x86_64
-          path: |
-            lighthouse-build-1
-            lighthouse-build-2
-            lighthouse-build-1.sha256
-            lighthouse-build-2.sha256
-
   build-aarch64:
     name: test reproducible builds (aarch64)
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04-arm
     steps:
       - uses: actions/checkout@v4
 
-      - uses: dtolnay/rust-toolchain@stable
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
         with:
-          target: aarch64-unknown-linux-gnu
+          driver: docker
 
-      - name: Install build dependencies and cross-compilation tools
+      - name: Build first reproducible Docker image
         run: |
-          sudo apt-get update
-          sudo apt-get install -y libclang-dev cmake gcc-aarch64-linux-gnu
-
-      - name: Install cargo-cache
-        run: cargo install cargo-cache
+          echo "=== Building first Docker image (aarch64) ==="
+          docker build -f Dockerfile.reproducible \
+            --platform linux/arm64 \
+            --build-arg RUST_TARGET="aarch64-unknown-linux-gnu" \
+            --build-arg RUST_IMAGE="rust:1.86-bullseye@sha256:36053eabadeb701e3e0406610a2ce72ccfa10b7828963cd08cffdcf660518b27" \
+            -t build-lighthouse-1-arm64 .
+
+          echo "=== Extracting binary from first build ==="
+          docker create --name extract-lighthouse-1-arm64 build-lighthouse-1-arm64
+          docker cp extract-lighthouse-1-arm64:/lighthouse ./lighthouse-build-1-arm64
+          docker rm extract-lighthouse-1-arm64
+
+          echo "=== First build info ==="
+          ls -la lighthouse-build-1-arm64
+          sha256sum lighthouse-build-1-arm64
+          file lighthouse-build-1-arm64
+
+      - name: Clean Docker state completely
+        run: |
+          echo "=== Cleaning Docker state ==="
+          docker rmi build-lighthouse-1-arm64 || true
+          docker buildx prune -f || true
+          docker system prune -f || true
+          docker container prune -f || true
 
-      - uses: Swatinem/rust-cache@v2
-        with:
-          cache-on-failure: true
-          key: reproducible-build-aarch64
+          echo "=== Docker state cleaned ==="
+          docker images
+          docker ps -a
 
-      - name: Build Lighthouse (first build)
+      - name: Build second reproducible Docker image
         run: |
-          make build-reproducible \
-            RUST_TARGET=aarch64-unknown-linux-gnu
-          cp target/aarch64-unknown-linux-gnu/release/lighthouse \
-            lighthouse-build-1-arm64
-          sha256sum lighthouse-build-1-arm64 > \
-            lighthouse-build-1-arm64.sha256
-
-      - name: Clean build artifacts and cache
+          echo "=== Building second Docker image (aarch64) ==="
+          docker build -f Dockerfile.reproducible \
+            --platform linux/arm64 \
+            --build-arg RUST_TARGET="aarch64-unknown-linux-gnu" \
+            --build-arg RUST_IMAGE="rust:1.86-bullseye@sha256:36053eabadeb701e3e0406610a2ce72ccfa10b7828963cd08cffdcf660518b27" \
+            -t build-lighthouse-2-arm64 .
+
+          echo "=== Extracting binary from second build ==="
+          docker create --name extract-lighthouse-2-arm64 build-lighthouse-2-arm64
+          docker cp extract-lighthouse-2-arm64:/lighthouse ./lighthouse-build-2-arm64
+          docker rm extract-lighthouse-2-arm64
+
+          echo "=== Second build info ==="
+          ls -la lighthouse-build-2-arm64
+          sha256sum lighthouse-build-2-arm64
+          file lighthouse-build-2-arm64
+
+      - name: Compare Docker-built binaries
         run: |
-          make clean
-          cargo cache -a
-          rm -rf target/
+          echo "=== Comparing Docker-built binaries (aarch64) ==="
+          echo "Build 1 info:"
+          ls -la lighthouse-build-1-arm64
+          echo "Build 2 info:"
+          ls -la lighthouse-build-2-arm64
 
-      - name: Build Lighthouse (second build)
-        run: |
-          make build-reproducible \
-            RUST_TARGET=aarch64-unknown-linux-gnu
-          cp target/aarch64-unknown-linux-gnu/release/lighthouse \
-            lighthouse-build-2-arm64
-          sha256sum lighthouse-build-2-arm64 > \
-            lighthouse-build-2-arm64.sha256
-
-      - name: Compare binaries
-        run: |
-          echo "=== Build 1 SHA256 (ARM64) ==="
-          cat lighthouse-build-1-arm64.sha256
-          echo "=== Build 2 SHA256 (ARM64) ==="
-          cat lighthouse-build-2-arm64.sha256
-          echo "=== Binary Comparison ==="
+          echo "=== SHA256 checksums ==="
+          sha256sum lighthouse-build-*-arm64
+
+          echo "=== Binary comparison ==="
           if cmp lighthouse-build-1-arm64 lighthouse-build-2-arm64; then
-            echo "✅ ARM64 binaries are identical - reproducible build PASSED"
+            echo "✅ SUCCESS: Docker-built binaries are identical!"
+            echo "✅ Reproducible Docker build PASSED for aarch64"
           else
-            echo "❌ ARM64 binaries differ - reproducible build FAILED"
+            echo "❌ FAILED: Docker-built binaries differ"
+            echo "First 10 differences:"
+            cmp -l lighthouse-build-1-arm64 lighthouse-build-2-arm64 | head -10
             exit 1
           fi
 
-      - name: Upload build artifacts (on failure)
-        if: failure()
-        uses: actions/upload-artifact@v4
-        with:
-          name: failed-reproducible-builds-aarch64
-          path: |
-            lighthouse-build-1-arm64
-            lighthouse-build-2-arm64
-            lighthouse-build-1-arm64.sha256
-            lighthouse-build-2-arm64.sha256
-
   summary:
     name: reproducible build summary
     runs-on: ubuntu-latest
@@ -181,8 +208,9 @@ jobs:
           if [[ "${{ needs.build-x86_64.result }}" == "success" ]] \
             && [[ "${{ needs.build-aarch64.result }}" == "success" ]]; then
             echo "🎉 **Overall**: All reproducible builds are working correctly!"
+            echo "Docker containers are reproducible."
           else
             echo "⚠️  **Overall**: Some reproducible builds failed"
-            echo "Check the logs above"
+            echo "Check the logs above for details"
             exit 1
           fi
diff --git a/testing/state_transition_vectors/Makefile b/testing/state_transition_vectors/Makefile
index 437aa50b00a..c90810ad398 100644
--- a/testing/state_transition_vectors/Makefile
+++ b/testing/state_transition_vectors/Makefile
@@ -5,4 +5,4 @@ test:
 	cargo test --release --features "$(TEST_FEATURES)"
 
 clean:
-	rm -r vectors/
+	rm -rf vectors/

From febc1387bbaab7eb7b1d4e395dd8d239ccdfb121 Mon Sep 17 00:00:00 2001
From: MoeMahhouk <mohammed-mahhouk@hotmail.com>
Date: Mon, 30 Jun 2025 12:50:50 +0000
Subject: [PATCH 06/17] make release-reproducible verify reproducibility

---
 .github/workflows/release-reproducible.yml | 195 ++++++++++++++++-----
 1 file changed, 153 insertions(+), 42 deletions(-)

diff --git a/.github/workflows/release-reproducible.yml b/.github/workflows/release-reproducible.yml
index 10f668869ad..438dca5f4b5 100644
--- a/.github/workflows/release-reproducible.yml
+++ b/.github/workflows/release-reproducible.yml
@@ -30,9 +30,8 @@ jobs:
     outputs:
       VERSION: ${{ steps.extract_version.outputs.VERSION }}
 
-  build-reproducible:
-    name: build and push reproducible images
-    runs-on: ubuntu-latest
+  verify-and-build:
+    name: verify reproducibility and build
     needs: extract-version
     strategy:
       matrix:
@@ -43,16 +42,81 @@ jobs:
             rust_image: >-
               rust:1.86-bullseye@sha256:1110399f568f1dbe838e58f15b4162d899cb95f450f5f0ffa739614f3a4c32f1
             platform: linux/amd64
+            runner: ubuntu-latest
           - arch: arm64
             rust_target: aarch64-unknown-linux-gnu
             rust_image: >-
               rust:1.86-bullseye@sha256:36053eabadeb701e3e0406610a2ce72ccfa10b7828963cd08cffdcf660518b27
             platform: linux/arm64
+            runner: ubuntu-24.04-arm
+    runs-on: ${{ matrix.runner }}
     steps:
       - uses: actions/checkout@v4
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
+        with:
+          driver: docker
+
+      - name: Verify reproducible builds (${{ matrix.arch }})
+        run: |
+          echo "🔄 Verifying reproducible builds for ${{ matrix.arch }}..."
+
+          # Build first image
+          echo "=== Building first verification image ==="
+          docker build -f Dockerfile.reproducible \
+            --platform ${{ matrix.platform }} \
+            --build-arg RUST_TARGET="${{ matrix.rust_target }}" \
+            --build-arg RUST_IMAGE="${{ matrix.rust_image }}" \
+            -t lighthouse-verify-1-${{ matrix.arch }} .
+
+          # Extract binary from first build
+          docker create --name extract-1-${{ matrix.arch }} lighthouse-verify-1-${{ matrix.arch }}
+          docker cp extract-1-${{ matrix.arch }}:/lighthouse ./lighthouse-1-${{ matrix.arch }}
+          docker rm extract-1-${{ matrix.arch }}
+
+          # Clean state for second build
+          docker buildx prune -f
+          docker system prune -f
+
+          # Build second image
+          echo "=== Building second verification image ==="
+          docker build -f Dockerfile.reproducible \
+            --platform ${{ matrix.platform }} \
+            --build-arg RUST_TARGET="${{ matrix.rust_target }}" \
+            --build-arg RUST_IMAGE="${{ matrix.rust_image }}" \
+            -t lighthouse-verify-2-${{ matrix.arch }} .
+
+          # Extract binary from second build
+          docker create --name extract-2-${{ matrix.arch }} lighthouse-verify-2-${{ matrix.arch }}
+          docker cp extract-2-${{ matrix.arch }}:/lighthouse ./lighthouse-2-${{ matrix.arch }}
+          docker rm extract-2-${{ matrix.arch }}
+
+          # Compare binaries
+          echo "=== Comparing binaries ==="
+          echo "Build 1 SHA256: $(sha256sum lighthouse-1-${{ matrix.arch }})"
+          echo "Build 2 SHA256: $(sha256sum lighthouse-2-${{ matrix.arch }})"
+
+          if cmp lighthouse-1-${{ matrix.arch }} lighthouse-2-${{ matrix.arch }}; then
+            echo "✅ Reproducible build verified for ${{ matrix.arch }}"
+          else
+            echo "❌ Reproducible build FAILED for ${{ matrix.arch }}"
+            echo "🚨 BLOCKING RELEASE: Builds are not reproducible!"
+            echo "First 10 differences:"
+            cmp -l lighthouse-1-${{ matrix.arch }} lighthouse-2-${{ matrix.arch }} | head -10
+            exit 1
+          fi
+
+          # Clean up verification artifacts but keep one image for publishing
+          rm -f lighthouse-*-${{ matrix.arch }}
+          docker rmi lighthouse-verify-1-${{ matrix.arch }} || true
+
+          # Re-tag the second image for publishing (we verified it's identical to first)
+          VERSION=${{ needs.extract-version.outputs.VERSION }}
+          FINAL_TAG="${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${VERSION}-${{ matrix.arch }}"
+          docker tag lighthouse-verify-2-${{ matrix.arch }} "$FINAL_TAG"
+
+          echo "✅ Image ready for publishing: $FINAL_TAG"
 
       - name: Log in to Docker Hub
         if: ${{ github.event.inputs.dry_run != 'true' }}
@@ -61,30 +125,34 @@ jobs:
           username: ${{ env.DOCKER_USERNAME }}
           password: ${{ env.DOCKER_PASSWORD }}
 
-      - name: Build reproducible image (${{ matrix.arch }})
-        uses: docker/build-push-action@v6
-        env:
-          IMAGE_BASE: ${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}
-          VERSION: ${{ needs.extract-version.outputs.VERSION }}
-          ARCH: ${{ matrix.arch }}
-          DOCKER_BUILD_RECORD_UPLOAD: false
+      - name: Push verified image (${{ matrix.arch }})
+        if: ${{ github.event.inputs.dry_run != 'true' }}
+        run: |
+          VERSION=${{ needs.extract-version.outputs.VERSION }}
+          IMAGE_TAG="${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${VERSION}-${{ matrix.arch }}"
+
+          echo "📤 Pushing verified reproducible image: $IMAGE_TAG"
+          docker push "$IMAGE_TAG"
+          echo "✅ Successfully pushed $IMAGE_TAG"
+
+      - name: Clean up local images
+        run: |
+          docker rmi lighthouse-verify-2-${{ matrix.arch }} || true
+          VERSION=${{ needs.extract-version.outputs.VERSION }}
+          docker rmi "${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${VERSION}-${{ matrix.arch }}" || true
+
+      - name: Upload verification artifacts (on failure)
+        if: failure()
+        uses: actions/upload-artifact@v4
         with:
-          context: .
-          file: ./Dockerfile.reproducible
-          platforms: ${{ matrix.platform }}
-          push: ${{ github.event.inputs.dry_run != 'true' }}
-          tags: ${{ env.IMAGE_BASE }}:${{ env.VERSION }}-${{ env.ARCH }}
-          build-args: |
-            RUST_TARGET=${{ matrix.rust_target }}
-            RUST_IMAGE=${{ matrix.rust_image }}
-          cache-from: type=gha,scope=${{ matrix.arch }}
-          cache-to: type=gha,mode=max,scope=${{ matrix.arch }}
-          provenance: false
+          name: verification-failure-${{ matrix.arch }}
+          path: |
+            lighthouse-*-${{ matrix.arch }}
 
   create-manifest:
     name: create multi-arch manifest
     runs-on: ubuntu-latest
-    needs: [extract-version, build-reproducible]
+    needs: [extract-version, verify-and-build]
     if: ${{ github.event.inputs.dry_run != 'true' }}
     steps:
       - name: Log in to Docker Hub
@@ -97,6 +165,9 @@ jobs:
         run: |
           IMAGE_NAME=${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}
           VERSION=${{ needs.extract-version.outputs.VERSION }}
+
+          echo "🔗 Creating multi-arch manifest for $IMAGE_NAME:$VERSION"
+
           # Create manifest for version tag
           docker manifest create \
             ${IMAGE_NAME}:${VERSION} \
@@ -113,30 +184,70 @@ jobs:
 
           docker manifest push ${IMAGE_NAME}:latest
 
-  dry-run-summary:
-    name: dry run summary
+          echo "✅ Multi-arch manifests published:"
+          echo "  - ${IMAGE_NAME}:${VERSION}"
+          echo "  - ${IMAGE_NAME}:latest"
+
+  release-summary:
+    name: release summary
     runs-on: ubuntu-latest
-    needs: [build-reproducible, extract-version]
-    if: ${{ github.event.inputs.dry_run == 'true' }}
+    needs: [extract-version, verify-and-build, create-manifest]
+    if: always()
     steps:
-      - name: Summarize dry run
+      - name: Report release results
         run: |
-          IMAGE_NAME=${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}
           VERSION=${{ needs.extract-version.outputs.VERSION }}
-          echo "## 🧪 Reproducible Build Dry Run Summary"
-          echo ""
-          echo "✅ Successfully completed dry run for version ${VERSION}"
+          IMAGE_NAME=${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}
+
+          echo "## 🚀 Reproducible Release Summary for ${VERSION}"
           echo ""
-          echo "### What would happen in a real release:"
-          echo "- Multi-arch reproducible Docker images would be built"
-          echo "- Images would be pushed to Docker Hub as:"
-          echo "  - \`${IMAGE_NAME}:${VERSION}\`"
-          echo "  - \`${IMAGE_NAME}:latest\`"
+
+          if [[ "${{ needs.verify-and-build.result }}" == "success" ]]; then
+            echo "✅ **Reproducibility Verification & Build**: SUCCESS"
+            echo "- All architectures produce identical binaries"
+            echo "- Images built and ready for publishing"
+          else
+            echo "❌ **Reproducibility Verification & Build**: FAILED"
+            echo "- Builds are not reproducible OR build failed"
+            echo "- Release was blocked"
+          fi
+
           echo ""
-          echo "### Architectures built:"
-          echo "- linux/amd64 (x86_64-unknown-linux-gnu)"
-          echo "- linux/arm64 (aarch64-unknown-linux-gnu)"
+          if [[ "${{ github.event.inputs.dry_run }}" == "true" ]]; then
+            echo "🧪 **Mode**: DRY RUN"
+            echo "- Images were built and verified but NOT pushed"
+            echo "- Ready for real release"
+          elif [[ "${{ needs.create-manifest.result }}" == "success" ]]; then
+            echo "✅ **Publication**: SUCCESS"
+            echo "- Images published to Docker Hub"
+            echo "- Multi-arch manifests created"
+            echo ""
+            echo "### 📦 Published Images"
+            echo "- \`${IMAGE_NAME}:${VERSION}\`"
+            echo "- \`${IMAGE_NAME}:latest\`"
+            echo ""
+            echo "### 🏗️ Architectures"
+            echo "- linux/amd64 (\`${IMAGE_NAME}:${VERSION}-amd64\`)"
+            echo "- linux/arm64 (\`${IMAGE_NAME}:${VERSION}-arm64\`)"
+          else
+            echo "❌ **Publication**: FAILED"
+            echo "- Images were verified but failed to publish"
+          fi
+
           echo ""
-          echo "### Next Steps"
-          echo "To perform a real release, push a git tag"
-          echo "(e.g., \`git tag v4.6.0 && git push origin v4.6.0\`)"
+          if [[ "${{ needs.verify-and-build.result }}" == "success" ]] && [[ "${{ needs.create-manifest.result }}" == "success" ]] && [[ "${{ github.event.inputs.dry_run }}" != "true" ]]; then
+            echo "🎉 **Overall**: Secure release completed successfully!"
+            echo ""
+            echo "### 🔒 Security Guarantees"
+            echo "- ✅ Reproducible builds verified"
+            echo "- ✅ Identical binaries across architectures"
+            echo "- ✅ No build artifacts tampering"
+            echo "- ✅ Deterministic build process"
+          elif [[ "${{ github.event.inputs.dry_run }}" == "true" ]]; then
+            echo "🧪 **Overall**: Dry run completed successfully!"
+            echo "- Reproducibility verified ✅"
+            echo "- Ready for real release ✅"
+          else
+            echo "🚨 **Overall**: Release failed or incomplete"
+            echo "- Check logs above for details"
+          fi

From 19f2a0f823a280b8385b62f7bf05af1d381e34f3 Mon Sep 17 00:00:00 2001
From: MoeMahhouk <mohammed-mahhouk@hotmail.com>
Date: Mon, 30 Jun 2025 13:27:35 +0000
Subject: [PATCH 07/17] add label trigger for reproducible builds

---
 .github/workflows/reproducible-build.yml | 33 ++++++++++++++++++++++--
 1 file changed, 31 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/reproducible-build.yml b/.github/workflows/reproducible-build.yml
index abe37510243..6c7e0ca19a7 100644
--- a/.github/workflows/reproducible-build.yml
+++ b/.github/workflows/reproducible-build.yml
@@ -3,8 +3,9 @@ name: reproducible-build
 on:
   workflow_dispatch: {}
   schedule:
-    - cron: "0 1 */2 * *"
+    - cron: "0 1 */2 * *"  # Every 2 days to catch issues early
   pull_request:
+    types: [opened, synchronize, labeled]
     paths:
       - "Makefile"
       - "Dockerfile.reproducible"
@@ -16,6 +17,11 @@ jobs:
   build-x86_64:
     name: test reproducible builds (x86_64)
     runs-on: ubuntu-latest
+    # Run on schedule, workflow_dispatch, or PRs with the 'test-reproducible' label
+    if: >
+      github.event_name == 'schedule' ||
+      github.event_name == 'workflow_dispatch' ||
+      contains(github.event.pull_request.labels.*.name, 'test-reproducible')
     steps:
       - uses: actions/checkout@v4
 
@@ -102,6 +108,11 @@ jobs:
   build-aarch64:
     name: test reproducible builds (aarch64)
     runs-on: ubuntu-24.04-arm
+    # Run on schedule, workflow_dispatch, or PRs with the 'test-reproducible' label
+    if: >
+      github.event_name == 'schedule' ||
+      github.event_name == 'workflow_dispatch' ||
+      contains(github.event.pull_request.labels.*.name, 'test-reproducible')
     steps:
       - uses: actions/checkout@v4
 
@@ -192,14 +203,29 @@ jobs:
         run: |
           echo "## 🔄 Reproducible Build Test Results"
           echo ""
+
+          # Show trigger reason
+          if [[ "${{ github.event_name }}" == "schedule" ]]; then
+            echo "**Trigger**: Scheduled check (every 2 days)"
+          elif [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
+            echo "**Trigger**: Manual run"
+          elif [[ "${{ github.event_name }}" == "pull_request" ]]; then
+            echo "**Trigger**: PR with 'test-reproducible' label"
+          fi
+          echo ""
+
           if [[ "${{ needs.build-x86_64.result }}" == "success" ]]; then
             echo "✅ **x86_64**: Reproducible builds PASSED"
+          elif [[ "${{ needs.build-x86_64.result }}" == "skipped" ]]; then
+            echo "⏭️ **x86_64**: Skipped (no label or wrong trigger)"
           else
             echo "❌ **x86_64**: Reproducible builds FAILED"
           fi
 
           if [[ "${{ needs.build-aarch64.result }}" == "success" ]]; then
             echo "✅ **aarch64**: Reproducible builds PASSED"
+          elif [[ "${{ needs.build-aarch64.result }}" == "skipped" ]]; then
+            echo "⏭️ **aarch64**: Skipped (no label or wrong trigger)"
           else
             echo "❌ **aarch64**: Reproducible builds FAILED"
           fi
@@ -209,8 +235,11 @@ jobs:
             && [[ "${{ needs.build-aarch64.result }}" == "success" ]]; then
             echo "🎉 **Overall**: All reproducible builds are working correctly!"
             echo "Docker containers are reproducible."
+          elif [[ "${{ needs.build-x86_64.result }}" == "skipped" ]] \
+            && [[ "${{ needs.build-aarch64.result }}" == "skipped" ]]; then
+            echo "⏭️ **Overall**: Tests were skipped (add 'test-reproducible' label to run on PRs)"
           else
-            echo "⚠️  **Overall**: Some reproducible builds failed"
+            echo "⚠️ **Overall**: Some reproducible builds failed"
             echo "Check the logs above for details"
             exit 1
           fi

From bddc4f7a68c0413f1a12812cee8a2e2d42433a5b Mon Sep 17 00:00:00 2001
From: MoeMahhouk <mohammed-mahhouk@hotmail.com>
Date: Mon, 3 Nov 2025 15:46:31 +0000
Subject: [PATCH 08/17] update rust version for github actions

---
 .github/workflows/release-reproducible.yml | 4 ++--
 .github/workflows/reproducible-build.yml   | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/release-reproducible.yml b/.github/workflows/release-reproducible.yml
index 438dca5f4b5..5021161588e 100644
--- a/.github/workflows/release-reproducible.yml
+++ b/.github/workflows/release-reproducible.yml
@@ -40,13 +40,13 @@ jobs:
           - arch: amd64
             rust_target: x86_64-unknown-linux-gnu
             rust_image: >-
-              rust:1.86-bullseye@sha256:1110399f568f1dbe838e58f15b4162d899cb95f450f5f0ffa739614f3a4c32f1
+              rust:1.88-bullseye@sha256:8e3c421122bf4cd3b2a866af41a4dd52d87ad9e315fd2cb5100e87a7187a9816
             platform: linux/amd64
             runner: ubuntu-latest
           - arch: arm64
             rust_target: aarch64-unknown-linux-gnu
             rust_image: >-
-              rust:1.86-bullseye@sha256:36053eabadeb701e3e0406610a2ce72ccfa10b7828963cd08cffdcf660518b27
+              rust:1.88-bullseye@sha256:8b22455a7ce2adb1355067638284ee99d21cc516fab63a96c4514beaf370aa94
             platform: linux/arm64
             runner: ubuntu-24.04-arm
     runs-on: ${{ matrix.runner }}
diff --git a/.github/workflows/reproducible-build.yml b/.github/workflows/reproducible-build.yml
index 6c7e0ca19a7..dd999772660 100644
--- a/.github/workflows/reproducible-build.yml
+++ b/.github/workflows/reproducible-build.yml
@@ -35,7 +35,7 @@ jobs:
           echo "=== Building first Docker image (x86_64) ==="
           docker build -f Dockerfile.reproducible \
             --build-arg RUST_TARGET="x86_64-unknown-linux-gnu" \
-            --build-arg RUST_IMAGE="rust:1.86-bullseye@sha256:1110399f568f1dbe838e58f15b4162d899cb95f450f5f0ffa739614f3a4c32f1" \
+            --build-arg RUST_IMAGE="rust:1.88-bullseye@sha256:8e3c421122bf4cd3b2a866af41a4dd52d87ad9e315fd2cb5100e87a7187a9816" \
             -t build-lighthouse-1 .
 
           echo "=== Extracting binary from first build ==="
@@ -70,7 +70,7 @@ jobs:
           echo "=== Building second Docker image (x86_64) ==="
           docker build -f Dockerfile.reproducible \
             --build-arg RUST_TARGET="x86_64-unknown-linux-gnu" \
-            --build-arg RUST_IMAGE="rust:1.86-bullseye@sha256:1110399f568f1dbe838e58f15b4162d899cb95f450f5f0ffa739614f3a4c32f1" \
+            --build-arg RUST_IMAGE="rust:1.88-bullseye@sha256:8e3c421122bf4cd3b2a866af41a4dd52d87ad9e315fd2cb5100e87a7187a9816" \
             -t build-lighthouse-2 .
 
           echo "=== Extracting binary from second build ==="
@@ -127,7 +127,7 @@ jobs:
           docker build -f Dockerfile.reproducible \
             --platform linux/arm64 \
             --build-arg RUST_TARGET="aarch64-unknown-linux-gnu" \
-            --build-arg RUST_IMAGE="rust:1.86-bullseye@sha256:36053eabadeb701e3e0406610a2ce72ccfa10b7828963cd08cffdcf660518b27" \
+            --build-arg RUST_IMAGE="rust:1.88-bullseye@sha256:8b22455a7ce2adb1355067638284ee99d21cc516fab63a96c4514beaf370aa94" \
             -t build-lighthouse-1-arm64 .
 
           echo "=== Extracting binary from first build ==="
@@ -158,7 +158,7 @@ jobs:
           docker build -f Dockerfile.reproducible \
             --platform linux/arm64 \
             --build-arg RUST_TARGET="aarch64-unknown-linux-gnu" \
-            --build-arg RUST_IMAGE="rust:1.86-bullseye@sha256:36053eabadeb701e3e0406610a2ce72ccfa10b7828963cd08cffdcf660518b27" \
+            --build-arg RUST_IMAGE="rust:1.88-bullseye@sha256:8b22455a7ce2adb1355067638284ee99d21cc516fab63a96c4514beaf370aa94" \
             -t build-lighthouse-2-arm64 .
 
           echo "=== Extracting binary from second build ==="

From 5e21162dc244baf86a847b666bbb7263d606771c Mon Sep 17 00:00:00 2001
From: MoeMahhouk <mohammed-mahhouk@hotmail.com>
Date: Wed, 5 Nov 2025 16:04:23 +0000
Subject: [PATCH 09/17] addressing PR feedback

---
 ...producible.yml => docker-reproducible.yml} | 66 +++++++++---------
 .github/workflows/reproducible-build.yml      | 67 +++----------------
 Dockerfile.reproducible                       |  2 +-
 Makefile                                      |  5 +-
 4 files changed, 43 insertions(+), 97 deletions(-)
 rename .github/workflows/{release-reproducible.yml => docker-reproducible.yml} (81%)

diff --git a/.github/workflows/release-reproducible.yml b/.github/workflows/docker-reproducible.yml
similarity index 81%
rename from .github/workflows/release-reproducible.yml
rename to .github/workflows/docker-reproducible.yml
index 5021161588e..4d0c8138971 100644
--- a/.github/workflows/release-reproducible.yml
+++ b/.github/workflows/docker-reproducible.yml
@@ -1,4 +1,4 @@
-name: release-reproducible
+name: docker-reproducible
 
 on:
   push:
@@ -21,7 +21,7 @@ env:
 jobs:
   extract-version:
     name: extract version
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - name: Extract version
         run: >-
@@ -42,13 +42,13 @@ jobs:
             rust_image: >-
               rust:1.88-bullseye@sha256:8e3c421122bf4cd3b2a866af41a4dd52d87ad9e315fd2cb5100e87a7187a9816
             platform: linux/amd64
-            runner: ubuntu-latest
+            runner: ubuntu-22.04
           - arch: arm64
             rust_target: aarch64-unknown-linux-gnu
             rust_image: >-
               rust:1.88-bullseye@sha256:8b22455a7ce2adb1355067638284ee99d21cc516fab63a96c4514beaf370aa94
             platform: linux/arm64
-            runner: ubuntu-24.04-arm
+            runner: ubuntu-22.04-arm
     runs-on: ${{ matrix.runner }}
     steps:
       - uses: actions/checkout@v4
@@ -60,7 +60,7 @@ jobs:
 
       - name: Verify reproducible builds (${{ matrix.arch }})
         run: |
-          echo "🔄 Verifying reproducible builds for ${{ matrix.arch }}..."
+          echo "Verifying reproducible builds for ${{ matrix.arch }}..."
 
           # Build first image
           echo "=== Building first verification image ==="
@@ -98,10 +98,10 @@ jobs:
           echo "Build 2 SHA256: $(sha256sum lighthouse-2-${{ matrix.arch }})"
 
           if cmp lighthouse-1-${{ matrix.arch }} lighthouse-2-${{ matrix.arch }}; then
-            echo "✅ Reproducible build verified for ${{ matrix.arch }}"
+            echo "Reproducible build verified for ${{ matrix.arch }}"
           else
-            echo "❌ Reproducible build FAILED for ${{ matrix.arch }}"
-            echo "🚨 BLOCKING RELEASE: Builds are not reproducible!"
+            echo "Reproducible build FAILED for ${{ matrix.arch }}"
+            echo "BLOCKING RELEASE: Builds are not reproducible!"
             echo "First 10 differences:"
             cmp -l lighthouse-1-${{ matrix.arch }} lighthouse-2-${{ matrix.arch }} | head -10
             exit 1
@@ -116,7 +116,7 @@ jobs:
           FINAL_TAG="${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${VERSION}-${{ matrix.arch }}"
           docker tag lighthouse-verify-2-${{ matrix.arch }} "$FINAL_TAG"
 
-          echo "✅ Image ready for publishing: $FINAL_TAG"
+          echo "Image ready for publishing: $FINAL_TAG"
 
       - name: Log in to Docker Hub
         if: ${{ github.event.inputs.dry_run != 'true' }}
@@ -131,9 +131,9 @@ jobs:
           VERSION=${{ needs.extract-version.outputs.VERSION }}
           IMAGE_TAG="${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${VERSION}-${{ matrix.arch }}"
 
-          echo "📤 Pushing verified reproducible image: $IMAGE_TAG"
+          echo "Pushing verified reproducible image: $IMAGE_TAG"
           docker push "$IMAGE_TAG"
-          echo "✅ Successfully pushed $IMAGE_TAG"
+          echo "Successfully pushed $IMAGE_TAG"
 
       - name: Clean up local images
         run: |
@@ -151,7 +151,7 @@ jobs:
 
   create-manifest:
     name: create multi-arch manifest
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     needs: [extract-version, verify-and-build]
     if: ${{ github.event.inputs.dry_run != 'true' }}
     steps:
@@ -166,7 +166,7 @@ jobs:
           IMAGE_NAME=${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}
           VERSION=${{ needs.extract-version.outputs.VERSION }}
 
-          echo "🔗 Creating multi-arch manifest for $IMAGE_NAME:$VERSION"
+          echo "Creating multi-arch manifest for $IMAGE_NAME:$VERSION"
 
           # Create manifest for version tag
           docker manifest create \
@@ -184,13 +184,13 @@ jobs:
 
           docker manifest push ${IMAGE_NAME}:latest
 
-          echo "✅ Multi-arch manifests published:"
+          echo "Multi-arch manifests published:"
           echo "  - ${IMAGE_NAME}:${VERSION}"
           echo "  - ${IMAGE_NAME}:latest"
 
   release-summary:
     name: release summary
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     needs: [extract-version, verify-and-build, create-manifest]
     if: always()
     steps:
@@ -199,55 +199,55 @@ jobs:
           VERSION=${{ needs.extract-version.outputs.VERSION }}
           IMAGE_NAME=${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}
 
-          echo "## 🚀 Reproducible Release Summary for ${VERSION}"
+          echo "## Reproducible Release Summary for ${VERSION}"
           echo ""
 
           if [[ "${{ needs.verify-and-build.result }}" == "success" ]]; then
-            echo "✅ **Reproducibility Verification & Build**: SUCCESS"
+            echo "**Reproducibility Verification & Build**: SUCCESS"
             echo "- All architectures produce identical binaries"
             echo "- Images built and ready for publishing"
           else
-            echo "❌ **Reproducibility Verification & Build**: FAILED"
+            echo "**Reproducibility Verification & Build**: FAILED"
             echo "- Builds are not reproducible OR build failed"
             echo "- Release was blocked"
           fi
 
           echo ""
           if [[ "${{ github.event.inputs.dry_run }}" == "true" ]]; then
-            echo "🧪 **Mode**: DRY RUN"
+            echo "**Mode**: DRY RUN"
             echo "- Images were built and verified but NOT pushed"
             echo "- Ready for real release"
           elif [[ "${{ needs.create-manifest.result }}" == "success" ]]; then
-            echo "✅ **Publication**: SUCCESS"
+            echo "**Publication**: SUCCESS"
             echo "- Images published to Docker Hub"
             echo "- Multi-arch manifests created"
             echo ""
-            echo "### 📦 Published Images"
+            echo "### Published Images"
             echo "- \`${IMAGE_NAME}:${VERSION}\`"
             echo "- \`${IMAGE_NAME}:latest\`"
             echo ""
-            echo "### 🏗️ Architectures"
+            echo "### Architectures"
             echo "- linux/amd64 (\`${IMAGE_NAME}:${VERSION}-amd64\`)"
             echo "- linux/arm64 (\`${IMAGE_NAME}:${VERSION}-arm64\`)"
           else
-            echo "❌ **Publication**: FAILED"
+            echo "**Publication**: FAILED"
             echo "- Images were verified but failed to publish"
           fi
 
           echo ""
           if [[ "${{ needs.verify-and-build.result }}" == "success" ]] && [[ "${{ needs.create-manifest.result }}" == "success" ]] && [[ "${{ github.event.inputs.dry_run }}" != "true" ]]; then
-            echo "🎉 **Overall**: Secure release completed successfully!"
+            echo "**Overall**: Secure release completed successfully!"
             echo ""
-            echo "### 🔒 Security Guarantees"
-            echo "- ✅ Reproducible builds verified"
-            echo "- ✅ Identical binaries across architectures"
-            echo "- ✅ No build artifacts tampering"
-            echo "- ✅ Deterministic build process"
+            echo "### Security Guarantees"
+            echo "- Reproducible builds verified"
+            echo "- Identical binaries across architectures"
+            echo "- No build artifacts tampering"
+            echo "- Deterministic build process"
           elif [[ "${{ github.event.inputs.dry_run }}" == "true" ]]; then
-            echo "🧪 **Overall**: Dry run completed successfully!"
-            echo "- Reproducibility verified ✅"
-            echo "- Ready for real release ✅"
+            echo " **Overall**: Dry run completed successfully!"
+            echo "- Reproducibility verified"
+            echo "- Ready for real release"
           else
-            echo "🚨 **Overall**: Release failed or incomplete"
+            echo "**Overall**: Release failed or incomplete"
             echo "- Check logs above for details"
           fi
diff --git a/.github/workflows/reproducible-build.yml b/.github/workflows/reproducible-build.yml
index dd999772660..c17d0ef7ad0 100644
--- a/.github/workflows/reproducible-build.yml
+++ b/.github/workflows/reproducible-build.yml
@@ -16,7 +16,7 @@ on:
 jobs:
   build-x86_64:
     name: test reproducible builds (x86_64)
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     # Run on schedule, workflow_dispatch, or PRs with the 'test-reproducible' label
     if: >
       github.event_name == 'schedule' ||
@@ -96,10 +96,10 @@ jobs:
 
           echo "=== Binary comparison ==="
           if cmp lighthouse-build-1 lighthouse-build-2; then
-            echo "✅ SUCCESS: Docker-built binaries are identical!"
-            echo "✅ Reproducible Docker build PASSED for x86_64"
+            echo "SUCCESS: Docker-built binaries are identical!"
+            echo "Reproducible Docker build PASSED for x86_64"
           else
-            echo "❌ FAILED: Docker-built binaries differ"
+            echo "FAILED: Docker-built binaries differ"
             echo "First 10 differences:"
             cmp -l lighthouse-build-1 lighthouse-build-2 | head -10
             exit 1
@@ -107,7 +107,7 @@ jobs:
 
   build-aarch64:
     name: test reproducible builds (aarch64)
-    runs-on: ubuntu-24.04-arm
+    runs-on: ubuntu-22.04-arm
     # Run on schedule, workflow_dispatch, or PRs with the 'test-reproducible' label
     if: >
       github.event_name == 'schedule' ||
@@ -184,62 +184,11 @@ jobs:
 
           echo "=== Binary comparison ==="
           if cmp lighthouse-build-1-arm64 lighthouse-build-2-arm64; then
-            echo "✅ SUCCESS: Docker-built binaries are identical!"
-            echo "✅ Reproducible Docker build PASSED for aarch64"
+            echo "SUCCESS: Docker-built binaries are identical!"
+            echo "Reproducible Docker build PASSED for aarch64"
           else
-            echo "❌ FAILED: Docker-built binaries differ"
+            echo "FAILED: Docker-built binaries differ"
             echo "First 10 differences:"
             cmp -l lighthouse-build-1-arm64 lighthouse-build-2-arm64 | head -10
             exit 1
           fi
-
-  summary:
-    name: reproducible build summary
-    runs-on: ubuntu-latest
-    needs: [build-x86_64, build-aarch64]
-    if: always()
-    steps:
-      - name: Report results
-        run: |
-          echo "## 🔄 Reproducible Build Test Results"
-          echo ""
-
-          # Show trigger reason
-          if [[ "${{ github.event_name }}" == "schedule" ]]; then
-            echo "**Trigger**: Scheduled check (every 2 days)"
-          elif [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
-            echo "**Trigger**: Manual run"
-          elif [[ "${{ github.event_name }}" == "pull_request" ]]; then
-            echo "**Trigger**: PR with 'test-reproducible' label"
-          fi
-          echo ""
-
-          if [[ "${{ needs.build-x86_64.result }}" == "success" ]]; then
-            echo "✅ **x86_64**: Reproducible builds PASSED"
-          elif [[ "${{ needs.build-x86_64.result }}" == "skipped" ]]; then
-            echo "⏭️ **x86_64**: Skipped (no label or wrong trigger)"
-          else
-            echo "❌ **x86_64**: Reproducible builds FAILED"
-          fi
-
-          if [[ "${{ needs.build-aarch64.result }}" == "success" ]]; then
-            echo "✅ **aarch64**: Reproducible builds PASSED"
-          elif [[ "${{ needs.build-aarch64.result }}" == "skipped" ]]; then
-            echo "⏭️ **aarch64**: Skipped (no label or wrong trigger)"
-          else
-            echo "❌ **aarch64**: Reproducible builds FAILED"
-          fi
-
-          echo ""
-          if [[ "${{ needs.build-x86_64.result }}" == "success" ]] \
-            && [[ "${{ needs.build-aarch64.result }}" == "success" ]]; then
-            echo "🎉 **Overall**: All reproducible builds are working correctly!"
-            echo "Docker containers are reproducible."
-          elif [[ "${{ needs.build-x86_64.result }}" == "skipped" ]] \
-            && [[ "${{ needs.build-aarch64.result }}" == "skipped" ]]; then
-            echo "⏭️ **Overall**: Tests were skipped (add 'test-reproducible' label to run on PRs)"
-          else
-            echo "⚠️ **Overall**: Some reproducible builds failed"
-            echo "Check the logs above for details"
-            exit 1
-          fi
diff --git a/Dockerfile.reproducible b/Dockerfile.reproducible
index 4268669091e..6ca8bd1dc6a 100644
--- a/Dockerfile.reproducible
+++ b/Dockerfile.reproducible
@@ -21,5 +21,5 @@ RUN mv /app/target/${RUST_TARGET}/release/lighthouse /lighthouse
 FROM gcr.io/distroless/cc-debian12:nonroot-6755e21ccd99ddead6edc8106ba03888cbeed41a
 COPY --from=builder /lighthouse /lighthouse
 
-EXPOSE 30303 30303/udp 9001 8545 8546
+EXPOSE 8551 9000
 ENTRYPOINT [ "/lighthouse" ]
diff --git a/Makefile b/Makefile
index c798352462f..b478367dd04 100644
--- a/Makefile
+++ b/Makefile
@@ -99,9 +99,6 @@ LOCALE_VAL = C
 # Set UTC timezone for consistent time handling across builds
 TZ_VAL = UTC
 
-# Default features for lighthouse
-FEATURES ?= gnosis,slasher-lmdb,slasher-mdbx,slasher-redb,jemalloc
-
 # Default profile 
 PROFILE ?= release
 
@@ -119,7 +116,7 @@ build-reproducible: ## Build the lighthouse binary into `target` directory with
 	CARGO_INCREMENTAL=${CARGO_INCREMENTAL_VAL} \
 	LC_ALL=${LOCALE_VAL} \
 	TZ=${TZ_VAL} \
-	cargo build --bin lighthouse --features "$(FEATURES)" --profile "$(PROFILE)" --locked --target $(RUST_TARGET)
+	cargo build --bin lighthouse --features "$(CROSS_FEATURES)" --profile "$(PROFILE)" --locked --target $(RUST_TARGET)
 
 .PHONY: build-reproducible-x86_64
 build-reproducible-x86_64: ## Build reproducible x86_64 Docker image

From dbedf063a3dafe5a23496e42bebea002157d2dd1 Mon Sep 17 00:00:00 2001
From: MoeMahhouk <mohammed-mahhouk@hotmail.com>
Date: Thu, 6 Nov 2025 11:48:29 +0000
Subject: [PATCH 10/17] remove the release summary from docker-reproducible.yml

---
 .github/workflows/docker-reproducible.yml | 64 -----------------------
 1 file changed, 64 deletions(-)

diff --git a/.github/workflows/docker-reproducible.yml b/.github/workflows/docker-reproducible.yml
index 4d0c8138971..8b09848ae9a 100644
--- a/.github/workflows/docker-reproducible.yml
+++ b/.github/workflows/docker-reproducible.yml
@@ -187,67 +187,3 @@ jobs:
           echo "Multi-arch manifests published:"
           echo "  - ${IMAGE_NAME}:${VERSION}"
           echo "  - ${IMAGE_NAME}:latest"
-
-  release-summary:
-    name: release summary
-    runs-on: ubuntu-22.04
-    needs: [extract-version, verify-and-build, create-manifest]
-    if: always()
-    steps:
-      - name: Report release results
-        run: |
-          VERSION=${{ needs.extract-version.outputs.VERSION }}
-          IMAGE_NAME=${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}
-
-          echo "## Reproducible Release Summary for ${VERSION}"
-          echo ""
-
-          if [[ "${{ needs.verify-and-build.result }}" == "success" ]]; then
-            echo "**Reproducibility Verification & Build**: SUCCESS"
-            echo "- All architectures produce identical binaries"
-            echo "- Images built and ready for publishing"
-          else
-            echo "**Reproducibility Verification & Build**: FAILED"
-            echo "- Builds are not reproducible OR build failed"
-            echo "- Release was blocked"
-          fi
-
-          echo ""
-          if [[ "${{ github.event.inputs.dry_run }}" == "true" ]]; then
-            echo "**Mode**: DRY RUN"
-            echo "- Images were built and verified but NOT pushed"
-            echo "- Ready for real release"
-          elif [[ "${{ needs.create-manifest.result }}" == "success" ]]; then
-            echo "**Publication**: SUCCESS"
-            echo "- Images published to Docker Hub"
-            echo "- Multi-arch manifests created"
-            echo ""
-            echo "### Published Images"
-            echo "- \`${IMAGE_NAME}:${VERSION}\`"
-            echo "- \`${IMAGE_NAME}:latest\`"
-            echo ""
-            echo "### Architectures"
-            echo "- linux/amd64 (\`${IMAGE_NAME}:${VERSION}-amd64\`)"
-            echo "- linux/arm64 (\`${IMAGE_NAME}:${VERSION}-arm64\`)"
-          else
-            echo "**Publication**: FAILED"
-            echo "- Images were verified but failed to publish"
-          fi
-
-          echo ""
-          if [[ "${{ needs.verify-and-build.result }}" == "success" ]] && [[ "${{ needs.create-manifest.result }}" == "success" ]] && [[ "${{ github.event.inputs.dry_run }}" != "true" ]]; then
-            echo "**Overall**: Secure release completed successfully!"
-            echo ""
-            echo "### Security Guarantees"
-            echo "- Reproducible builds verified"
-            echo "- Identical binaries across architectures"
-            echo "- No build artifacts tampering"
-            echo "- Deterministic build process"
-          elif [[ "${{ github.event.inputs.dry_run }}" == "true" ]]; then
-            echo " **Overall**: Dry run completed successfully!"
-            echo "- Reproducibility verified"
-            echo "- Ready for real release"
-          else
-            echo "**Overall**: Release failed or incomplete"
-            echo "- Check logs above for details"
-          fi

From f4c1e46f93f480b8030f83c210efe1607b437b12 Mon Sep 17 00:00:00 2001
From: MoeMahhouk <mohammed-mahhouk@hotmail.com>
Date: Thu, 6 Nov 2025 11:48:57 +0000
Subject: [PATCH 11/17] removed exposed ports from the Dockerfile.reproducible

---
 Dockerfile.reproducible | 1 -
 1 file changed, 1 deletion(-)

diff --git a/Dockerfile.reproducible b/Dockerfile.reproducible
index 6ca8bd1dc6a..092b87b534a 100644
--- a/Dockerfile.reproducible
+++ b/Dockerfile.reproducible
@@ -21,5 +21,4 @@ RUN mv /app/target/${RUST_TARGET}/release/lighthouse /lighthouse
 FROM gcr.io/distroless/cc-debian12:nonroot-6755e21ccd99ddead6edc8106ba03888cbeed41a
 COPY --from=builder /lighthouse /lighthouse
 
-EXPOSE 8551 9000
 ENTRYPOINT [ "/lighthouse" ]

From 2bcc78494f4ecc56b111765dce163489d9d94c61 Mon Sep 17 00:00:00 2001
From: MoeMahhouk <mohammed-mahhouk@hotmail.com>
Date: Thu, 6 Nov 2025 16:18:07 +0000
Subject: [PATCH 12/17] add stable/unstable pushes to the docker-reproducible
 workflow

---
 .github/workflows/docker-reproducible.yml | 44 ++++++++++++++++-------
 1 file changed, 32 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/docker-reproducible.yml b/.github/workflows/docker-reproducible.yml
index 8b09848ae9a..f9891e626bf 100644
--- a/.github/workflows/docker-reproducible.yml
+++ b/.github/workflows/docker-reproducible.yml
@@ -2,6 +2,9 @@ name: docker-reproducible
 
 on:
   push:
+    branches:
+      - unstable
+      - stable
     tags:
       - v*
   workflow_dispatch:
@@ -24,11 +27,25 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
       - name: Extract version
-        run: >-
-          echo "VERSION=$(echo ${GITHUB_REF#refs/tags/})" >> $GITHUB_OUTPUT
+        run: |
+          if [[ "${{ github.ref }}" == refs/tags/* ]]; then
+            # It's a tag (e.g., v1.2.3)
+            VERSION="${GITHUB_REF#refs/tags/}"
+            echo "VERSION=$VERSION" >> $GITHUB_OUTPUT
+            echo "IS_TAG=true" >> $GITHUB_OUTPUT
+          elif [[ "${{ github.ref }}" == refs/heads/stable ]]; then
+            # stable branch -> latest
+            echo "VERSION=latest" >> $GITHUB_OUTPUT
+            echo "IS_TAG=false" >> $GITHUB_OUTPUT
+          elif [[ "${{ github.ref }}" == refs/heads/unstable ]]; then
+            # unstable branch -> latest-unstable
+            echo "VERSION=latest-unstable" >> $GITHUB_OUTPUT
+            echo "IS_TAG=false" >> $GITHUB_OUTPUT
+          fi
         id: extract_version
     outputs:
       VERSION: ${{ steps.extract_version.outputs.VERSION }}
+      IS_TAG: ${{ steps.extract_version.outputs.IS_TAG }}
 
   verify-and-build:
     name: verify reproducibility and build
@@ -165,10 +182,11 @@ jobs:
         run: |
           IMAGE_NAME=${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}
           VERSION=${{ needs.extract-version.outputs.VERSION }}
+          IS_TAG=${{ needs.extract-version.outputs.IS_TAG }}
 
           echo "Creating multi-arch manifest for $IMAGE_NAME:$VERSION"
 
-          # Create manifest for version tag
+          # Create manifest for the version tag
           docker manifest create \
             ${IMAGE_NAME}:${VERSION} \
             ${IMAGE_NAME}:${VERSION}-amd64 \
@@ -176,14 +194,16 @@ jobs:
 
           docker manifest push ${IMAGE_NAME}:${VERSION}
 
-          # Create manifest for latest tag
-          docker manifest create \
-            ${IMAGE_NAME}:latest \
-            ${IMAGE_NAME}:${VERSION}-amd64 \
-            ${IMAGE_NAME}:${VERSION}-arm64
+          echo "Published: ${IMAGE_NAME}:${VERSION}"
 
-          docker manifest push ${IMAGE_NAME}:latest
+          # For tagged releases, also update 'latest'
+          if [[ "$IS_TAG" == "true" ]]; then
+            echo "Creating 'latest' manifest for tagged release"
+            docker manifest create \
+              ${IMAGE_NAME}:latest \
+              ${IMAGE_NAME}:${VERSION}-amd64 \
+              ${IMAGE_NAME}:${VERSION}-arm64
 
-          echo "Multi-arch manifests published:"
-          echo "  - ${IMAGE_NAME}:${VERSION}"
-          echo "  - ${IMAGE_NAME}:latest"
+            docker manifest push ${IMAGE_NAME}:latest
+            echo "Published: ${IMAGE_NAME}:latest"
+          fi

From e838bfd6e62741f7c3368ff9f1cff2735ce68822 Mon Sep 17 00:00:00 2001
From: MoeMahhouk <mohammed-mahhouk@hotmail.com>
Date: Thu, 6 Nov 2025 16:18:47 +0000
Subject: [PATCH 13/17] remove reproducible-builds workflow

---
 .github/workflows/reproducible-build.yml | 194 -----------------------
 1 file changed, 194 deletions(-)
 delete mode 100644 .github/workflows/reproducible-build.yml

diff --git a/.github/workflows/reproducible-build.yml b/.github/workflows/reproducible-build.yml
deleted file mode 100644
index c17d0ef7ad0..00000000000
--- a/.github/workflows/reproducible-build.yml
+++ /dev/null
@@ -1,194 +0,0 @@
-name: reproducible-build
-
-on:
-  workflow_dispatch: {}
-  schedule:
-    - cron: "0 1 */2 * *"  # Every 2 days to catch issues early
-  pull_request:
-    types: [opened, synchronize, labeled]
-    paths:
-      - "Makefile"
-      - "Dockerfile.reproducible"
-      - ".github/workflows/reproducible-build.yml"
-      - "Cargo.toml"
-      - "Cargo.lock"
-
-jobs:
-  build-x86_64:
-    name: test reproducible builds (x86_64)
-    runs-on: ubuntu-22.04
-    # Run on schedule, workflow_dispatch, or PRs with the 'test-reproducible' label
-    if: >
-      github.event_name == 'schedule' ||
-      github.event_name == 'workflow_dispatch' ||
-      contains(github.event.pull_request.labels.*.name, 'test-reproducible')
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-        with:
-          driver: docker  # Use docker driver instead of docker-container for reproducibility
-
-      - name: Build first reproducible Docker image
-        run: |
-          echo "=== Building first Docker image (x86_64) ==="
-          docker build -f Dockerfile.reproducible \
-            --build-arg RUST_TARGET="x86_64-unknown-linux-gnu" \
-            --build-arg RUST_IMAGE="rust:1.88-bullseye@sha256:8e3c421122bf4cd3b2a866af41a4dd52d87ad9e315fd2cb5100e87a7187a9816" \
-            -t build-lighthouse-1 .
-
-          echo "=== Extracting binary from first build ==="
-          docker create --name extract-lighthouse-1 build-lighthouse-1
-          docker cp extract-lighthouse-1:/lighthouse ./lighthouse-build-1
-          docker rm extract-lighthouse-1
-
-          echo "=== First build info ==="
-          ls -la lighthouse-build-1
-          sha256sum lighthouse-build-1
-          file lighthouse-build-1
-
-      - name: Clean Docker state completely
-        run: |
-          echo "=== Cleaning Docker state ==="
-          # Remove the first image
-          docker rmi build-lighthouse-1 || true
-
-          # Remove all build cache (important for reproducibility testing)
-          docker buildx prune -f || true
-          docker system prune -f || true
-
-          # Clear any remaining containers
-          docker container prune -f || true
-
-          echo "=== Docker state cleaned ==="
-          docker images
-          docker ps -a
-
-      - name: Build second reproducible Docker image
-        run: |
-          echo "=== Building second Docker image (x86_64) ==="
-          docker build -f Dockerfile.reproducible \
-            --build-arg RUST_TARGET="x86_64-unknown-linux-gnu" \
-            --build-arg RUST_IMAGE="rust:1.88-bullseye@sha256:8e3c421122bf4cd3b2a866af41a4dd52d87ad9e315fd2cb5100e87a7187a9816" \
-            -t build-lighthouse-2 .
-
-          echo "=== Extracting binary from second build ==="
-          docker create --name extract-lighthouse-2 build-lighthouse-2
-          docker cp extract-lighthouse-2:/lighthouse ./lighthouse-build-2
-          docker rm extract-lighthouse-2
-
-          echo "=== Second build info ==="
-          ls -la lighthouse-build-2
-          sha256sum lighthouse-build-2
-          file lighthouse-build-2
-
-      - name: Compare Docker-built binaries
-        run: |
-          echo "=== Comparing Docker-built binaries (x86_64) ==="
-          echo "Build 1 info:"
-          ls -la lighthouse-build-1
-          echo "Build 2 info:"
-          ls -la lighthouse-build-2
-
-          echo "=== SHA256 checksums ==="
-          sha256sum lighthouse-build-*
-
-          echo "=== Binary comparison ==="
-          if cmp lighthouse-build-1 lighthouse-build-2; then
-            echo "SUCCESS: Docker-built binaries are identical!"
-            echo "Reproducible Docker build PASSED for x86_64"
-          else
-            echo "FAILED: Docker-built binaries differ"
-            echo "First 10 differences:"
-            cmp -l lighthouse-build-1 lighthouse-build-2 | head -10
-            exit 1
-          fi
-
-  build-aarch64:
-    name: test reproducible builds (aarch64)
-    runs-on: ubuntu-22.04-arm
-    # Run on schedule, workflow_dispatch, or PRs with the 'test-reproducible' label
-    if: >
-      github.event_name == 'schedule' ||
-      github.event_name == 'workflow_dispatch' ||
-      contains(github.event.pull_request.labels.*.name, 'test-reproducible')
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-        with:
-          driver: docker
-
-      - name: Build first reproducible Docker image
-        run: |
-          echo "=== Building first Docker image (aarch64) ==="
-          docker build -f Dockerfile.reproducible \
-            --platform linux/arm64 \
-            --build-arg RUST_TARGET="aarch64-unknown-linux-gnu" \
-            --build-arg RUST_IMAGE="rust:1.88-bullseye@sha256:8b22455a7ce2adb1355067638284ee99d21cc516fab63a96c4514beaf370aa94" \
-            -t build-lighthouse-1-arm64 .
-
-          echo "=== Extracting binary from first build ==="
-          docker create --name extract-lighthouse-1-arm64 build-lighthouse-1-arm64
-          docker cp extract-lighthouse-1-arm64:/lighthouse ./lighthouse-build-1-arm64
-          docker rm extract-lighthouse-1-arm64
-
-          echo "=== First build info ==="
-          ls -la lighthouse-build-1-arm64
-          sha256sum lighthouse-build-1-arm64
-          file lighthouse-build-1-arm64
-
-      - name: Clean Docker state completely
-        run: |
-          echo "=== Cleaning Docker state ==="
-          docker rmi build-lighthouse-1-arm64 || true
-          docker buildx prune -f || true
-          docker system prune -f || true
-          docker container prune -f || true
-
-          echo "=== Docker state cleaned ==="
-          docker images
-          docker ps -a
-
-      - name: Build second reproducible Docker image
-        run: |
-          echo "=== Building second Docker image (aarch64) ==="
-          docker build -f Dockerfile.reproducible \
-            --platform linux/arm64 \
-            --build-arg RUST_TARGET="aarch64-unknown-linux-gnu" \
-            --build-arg RUST_IMAGE="rust:1.88-bullseye@sha256:8b22455a7ce2adb1355067638284ee99d21cc516fab63a96c4514beaf370aa94" \
-            -t build-lighthouse-2-arm64 .
-
-          echo "=== Extracting binary from second build ==="
-          docker create --name extract-lighthouse-2-arm64 build-lighthouse-2-arm64
-          docker cp extract-lighthouse-2-arm64:/lighthouse ./lighthouse-build-2-arm64
-          docker rm extract-lighthouse-2-arm64
-
-          echo "=== Second build info ==="
-          ls -la lighthouse-build-2-arm64
-          sha256sum lighthouse-build-2-arm64
-          file lighthouse-build-2-arm64
-
-      - name: Compare Docker-built binaries
-        run: |
-          echo "=== Comparing Docker-built binaries (aarch64) ==="
-          echo "Build 1 info:"
-          ls -la lighthouse-build-1-arm64
-          echo "Build 2 info:"
-          ls -la lighthouse-build-2-arm64
-
-          echo "=== SHA256 checksums ==="
-          sha256sum lighthouse-build-*-arm64
-
-          echo "=== Binary comparison ==="
-          if cmp lighthouse-build-1-arm64 lighthouse-build-2-arm64; then
-            echo "SUCCESS: Docker-built binaries are identical!"
-            echo "Reproducible Docker build PASSED for aarch64"
-          else
-            echo "FAILED: Docker-built binaries differ"
-            echo "First 10 differences:"
-            cmp -l lighthouse-build-1-arm64 lighthouse-build-2-arm64 | head -10
-            exit 1
-          fi

From 19768d6e7d3430075640dca499ca23adee285c59 Mon Sep 17 00:00:00 2001
From: MoeMahhouk <mohammed-mahhouk@hotmail.com>
Date: Thu, 6 Nov 2025 16:53:46 +0000
Subject: [PATCH 14/17] remove unnecessary is_tag checks

---
 .github/workflows/docker-reproducible.yml | 23 +++--------------------
 1 file changed, 3 insertions(+), 20 deletions(-)

diff --git a/.github/workflows/docker-reproducible.yml b/.github/workflows/docker-reproducible.yml
index f9891e626bf..db724766cb0 100644
--- a/.github/workflows/docker-reproducible.yml
+++ b/.github/workflows/docker-reproducible.yml
@@ -31,21 +31,17 @@ jobs:
           if [[ "${{ github.ref }}" == refs/tags/* ]]; then
             # It's a tag (e.g., v1.2.3)
             VERSION="${GITHUB_REF#refs/tags/}"
-            echo "VERSION=$VERSION" >> $GITHUB_OUTPUT
-            echo "IS_TAG=true" >> $GITHUB_OUTPUT
           elif [[ "${{ github.ref }}" == refs/heads/stable ]]; then
             # stable branch -> latest
-            echo "VERSION=latest" >> $GITHUB_OUTPUT
-            echo "IS_TAG=false" >> $GITHUB_OUTPUT
+            VERSION="latest"
           elif [[ "${{ github.ref }}" == refs/heads/unstable ]]; then
             # unstable branch -> latest-unstable
-            echo "VERSION=latest-unstable" >> $GITHUB_OUTPUT
-            echo "IS_TAG=false" >> $GITHUB_OUTPUT
+            VERSION="latest-unstable"
           fi
+          echo "VERSION=$VERSION" >> $GITHUB_OUTPUT
         id: extract_version
     outputs:
       VERSION: ${{ steps.extract_version.outputs.VERSION }}
-      IS_TAG: ${{ steps.extract_version.outputs.IS_TAG }}
 
   verify-and-build:
     name: verify reproducibility and build
@@ -182,7 +178,6 @@ jobs:
         run: |
           IMAGE_NAME=${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}
           VERSION=${{ needs.extract-version.outputs.VERSION }}
-          IS_TAG=${{ needs.extract-version.outputs.IS_TAG }}
 
           echo "Creating multi-arch manifest for $IMAGE_NAME:$VERSION"
 
@@ -195,15 +190,3 @@ jobs:
           docker manifest push ${IMAGE_NAME}:${VERSION}
 
           echo "Published: ${IMAGE_NAME}:${VERSION}"
-
-          # For tagged releases, also update 'latest'
-          if [[ "$IS_TAG" == "true" ]]; then
-            echo "Creating 'latest' manifest for tagged release"
-            docker manifest create \
-              ${IMAGE_NAME}:latest \
-              ${IMAGE_NAME}:${VERSION}-amd64 \
-              ${IMAGE_NAME}:${VERSION}-arm64
-
-            docker manifest push ${IMAGE_NAME}:latest
-            echo "Published: ${IMAGE_NAME}:latest"
-          fi

From 3431d0349096e04f963c9ef65641d083193c3739 Mon Sep 17 00:00:00 2001
From: MoeMahhouk <mohammed-mahhouk@hotmail.com>
Date: Thu, 6 Nov 2025 17:17:28 +0000
Subject: [PATCH 15/17] manual workflow trigger is just for testing purposes
 only

---
 .github/workflows/docker-reproducible.yml | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/docker-reproducible.yml b/.github/workflows/docker-reproducible.yml
index db724766cb0..ff046918636 100644
--- a/.github/workflows/docker-reproducible.yml
+++ b/.github/workflows/docker-reproducible.yml
@@ -7,13 +7,7 @@ on:
       - stable
     tags:
       - v*
-  workflow_dispatch:
-    inputs:
-      dry_run:
-        description: >-
-          Enable dry run mode (builds images but skips push to registry)
-        type: boolean
-        default: false
+  workflow_dispatch: # allows manual triggering for testing purposes and skips publishing an image
 
 env:
   DOCKER_REPRODUCIBLE_IMAGE_NAME: >-
@@ -37,6 +31,9 @@ jobs:
           elif [[ "${{ github.ref }}" == refs/heads/unstable ]]; then
             # unstable branch -> latest-unstable
             VERSION="latest-unstable"
+          else
+            # For manual triggers from other branches and will not publish any image
+            VERSION="test-build"
           fi
           echo "VERSION=$VERSION" >> $GITHUB_OUTPUT
         id: extract_version
@@ -132,14 +129,14 @@ jobs:
           echo "Image ready for publishing: $FINAL_TAG"
 
       - name: Log in to Docker Hub
-        if: ${{ github.event.inputs.dry_run != 'true' }}
+        if: ${{ github.event_name != 'workflow_dispatch' }}
         uses: docker/login-action@v3
         with:
           username: ${{ env.DOCKER_USERNAME }}
           password: ${{ env.DOCKER_PASSWORD }}
 
       - name: Push verified image (${{ matrix.arch }})
-        if: ${{ github.event.inputs.dry_run != 'true' }}
+        if: ${{ github.event_name != 'workflow_dispatch' }}
         run: |
           VERSION=${{ needs.extract-version.outputs.VERSION }}
           IMAGE_TAG="${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${VERSION}-${{ matrix.arch }}"
@@ -166,7 +163,7 @@ jobs:
     name: create multi-arch manifest
     runs-on: ubuntu-22.04
     needs: [extract-version, verify-and-build]
-    if: ${{ github.event.inputs.dry_run != 'true' }}
+    if: ${{ github.event_name != 'workflow_dispatch' }}
     steps:
       - name: Log in to Docker Hub
         uses: docker/login-action@v3

From 12c057416da116a7809c29db2f1b7c57443e1a47 Mon Sep 17 00:00:00 2001
From: MoeMahhouk <mohammed-mahhouk@hotmail.com>
Date: Thu, 6 Nov 2025 19:12:39 +0000
Subject: [PATCH 16/17] Add reproducibility build for jemalloc-sys

---
 Dockerfile.reproducible        |  2 +-
 Makefile                       | 10 +++++++++-
 common/malloc_utils/Cargo.toml |  2 ++
 3 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/Dockerfile.reproducible b/Dockerfile.reproducible
index 092b87b534a..903515373f8 100644
--- a/Dockerfile.reproducible
+++ b/Dockerfile.reproducible
@@ -3,7 +3,7 @@ ARG RUST_IMAGE="rust:1.88-bullseye@sha256:8e3c421122bf4cd3b2a866af41a4dd52d87ad9
 FROM ${RUST_IMAGE} AS builder
 
 # Install specific version of the build dependencies
-RUN apt-get update && apt-get install -y libclang-dev=1:11.0-51+nmu5 cmake=3.18.4-2+deb11u1
+RUN apt-get update && apt-get install -y libclang-dev=1:11.0-51+nmu5 cmake=3.18.4-2+deb11u1 libjemalloc-dev=5.2.1-3
 
 ARG RUST_TARGET="x86_64-unknown-linux-gnu"
 
diff --git a/Makefile b/Makefile
index b478367dd04..697225d2282 100644
--- a/Makefile
+++ b/Makefile
@@ -102,6 +102,13 @@ TZ_VAL = UTC
 # Default profile 
 PROFILE ?= release
 
+# Features for reproducible builds
+FEATURES_REPRODUCIBLE = $(CROSS_FEATURES),jemalloc-unprefixed
+
+# Derive the architecture-specific library path from RUST_TARGET
+JEMALLOC_LIB_ARCH = $(word 1,$(subst -, ,$(RUST_TARGET)))
+JEMALLOC_OVERRIDE = /usr/lib/$(JEMALLOC_LIB_ARCH)-linux-gnu/libjemalloc.a
+
 # Default target architecture
 RUST_TARGET ?= x86_64-unknown-linux-gnu
 
@@ -116,7 +123,8 @@ build-reproducible: ## Build the lighthouse binary into `target` directory with
 	CARGO_INCREMENTAL=${CARGO_INCREMENTAL_VAL} \
 	LC_ALL=${LOCALE_VAL} \
 	TZ=${TZ_VAL} \
-	cargo build --bin lighthouse --features "$(CROSS_FEATURES)" --profile "$(PROFILE)" --locked --target $(RUST_TARGET)
+	JEMALLOC_OVERRIDE=${JEMALLOC_OVERRIDE} \
+	cargo build --bin lighthouse --features "$(FEATURES_REPRODUCIBLE)" --profile "$(PROFILE)" --locked --target $(RUST_TARGET)
 
 .PHONY: build-reproducible-x86_64
 build-reproducible-x86_64: ## Build reproducible x86_64 Docker image
diff --git a/common/malloc_utils/Cargo.toml b/common/malloc_utils/Cargo.toml
index 39c7137d4cb..1052128852a 100644
--- a/common/malloc_utils/Cargo.toml
+++ b/common/malloc_utils/Cargo.toml
@@ -21,6 +21,8 @@ jemalloc-profiling = ["tikv-jemallocator/profiling"]
 # Force the use of system malloc (or glibc) rather than jemalloc.
 # This is a no-op on Windows where jemalloc is always disabled.
 sysmalloc = []
+# Enable jemalloc with unprefixed malloc (recommended for reproducible builds)
+jemalloc-unprefixed = ["jemalloc", "tikv-jemallocator/unprefixed_malloc_on_supported_platforms"]
 
 [dependencies]
 libc = "0.2.79"

From 5613f2ad72c6bbd8df1397937afc933b43dc0c31 Mon Sep 17 00:00:00 2001
From: MoeMahhouk <mohammed-mahhouk@hotmail.com>
Date: Mon, 10 Nov 2025 08:38:21 +0000
Subject: [PATCH 17/17] chore: remove unnecessary echo

---
 .github/workflows/docker-reproducible.yml | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/.github/workflows/docker-reproducible.yml b/.github/workflows/docker-reproducible.yml
index ff046918636..f3479e9468d 100644
--- a/.github/workflows/docker-reproducible.yml
+++ b/.github/workflows/docker-reproducible.yml
@@ -70,10 +70,7 @@ jobs:
 
       - name: Verify reproducible builds (${{ matrix.arch }})
         run: |
-          echo "Verifying reproducible builds for ${{ matrix.arch }}..."
-
           # Build first image
-          echo "=== Building first verification image ==="
           docker build -f Dockerfile.reproducible \
             --platform ${{ matrix.platform }} \
             --build-arg RUST_TARGET="${{ matrix.rust_target }}" \
@@ -90,7 +87,6 @@ jobs:
           docker system prune -f
 
           # Build second image
-          echo "=== Building second verification image ==="
           docker build -f Dockerfile.reproducible \
             --platform ${{ matrix.platform }} \
             --build-arg RUST_TARGET="${{ matrix.rust_target }}" \
@@ -126,8 +122,6 @@ jobs:
           FINAL_TAG="${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${VERSION}-${{ matrix.arch }}"
           docker tag lighthouse-verify-2-${{ matrix.arch }} "$FINAL_TAG"
 
-          echo "Image ready for publishing: $FINAL_TAG"
-
       - name: Log in to Docker Hub
         if: ${{ github.event_name != 'workflow_dispatch' }}
         uses: docker/login-action@v3
@@ -140,10 +134,7 @@ jobs:
         run: |
           VERSION=${{ needs.extract-version.outputs.VERSION }}
           IMAGE_TAG="${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${VERSION}-${{ matrix.arch }}"
-
-          echo "Pushing verified reproducible image: $IMAGE_TAG"
           docker push "$IMAGE_TAG"
-          echo "Successfully pushed $IMAGE_TAG"
 
       - name: Clean up local images
         run: |
@@ -176,8 +167,6 @@ jobs:
           IMAGE_NAME=${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}
           VERSION=${{ needs.extract-version.outputs.VERSION }}
 
-          echo "Creating multi-arch manifest for $IMAGE_NAME:$VERSION"
-
           # Create manifest for the version tag
           docker manifest create \
             ${IMAGE_NAME}:${VERSION} \
@@ -185,5 +174,3 @@ jobs:
             ${IMAGE_NAME}:${VERSION}-arm64
 
           docker manifest push ${IMAGE_NAME}:${VERSION}
-
-          echo "Published: ${IMAGE_NAME}:${VERSION}"