Skip to content

Commit 014f115

Browse files
bakhtinbakhtin
authored
feat: Reproducible builds and *.deb packages (#19678)
Signed-off-by: bakhtin <[email protected]>
1 parent 07c5956 commit 014f115

File tree

9 files changed

+235
-67
lines changed

9 files changed

+235
-67
lines changed

‎.dockerignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,15 @@
44
# include source files
55
!/bin
66
!/crates
7+
!/pkg
78
!/testing
89
!book.toml
910
!Cargo.lock
1011
!Cargo.toml
1112
!Cross.toml
1213
!deny.toml
1314
!Makefile
15+
!README.md
1416

1517
# include for vergen constants
1618
!/.git

‎.github/workflows/release-reproducible.yml‎

Lines changed: 80 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,35 +1,53 @@
1-
# This workflow is for building and pushing reproducible Docker images for releases.
1+
# This workflow is for building and pushing reproducible artifacts for releases
22

33
name: release-reproducible
44

55
on:
6-
push:
7-
tags:
8-
- v*
6+
workflow_run:
7+
workflows: [release]
8+
types: [completed]
99

1010
env:
1111
DOCKER_REPRODUCIBLE_IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/reth-reproducible
1212

1313
jobs:
1414
extract-version:
1515
name: extract version
16+
if: ${{ github.event.workflow_run.conclusion == 'success' }}
1617
runs-on: ubuntu-latest
1718
steps:
18-
- name: Extract version
19-
run: echo "VERSION=$(echo ${GITHUB_REF#refs/tags/})" >> $GITHUB_OUTPUT
19+
- name: Extract version from triggering tag
2020
id: extract_version
21+
env:
22+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
23+
run: |
24+
# Get the tag that points to the head SHA of the triggering workflow
25+
TAG=$(gh api /repos/${{ github.repository }}/git/refs/tags \
26+
--jq '.[] | select(.object.sha == "${{ github.event.workflow_run.head_sha }}") | .ref' \
27+
| head -1 \
28+
| sed 's|refs/tags/||')
29+
30+
if [ -z "$TAG" ]; then
31+
echo "No tag found for SHA ${{ github.event.workflow_run.head_sha }}"
32+
exit 1
33+
fi
34+
35+
echo "VERSION=$TAG" >> $GITHUB_OUTPUT
2136
outputs:
2237
VERSION: ${{ steps.extract_version.outputs.VERSION }}
2338

2439
build-reproducible:
25-
name: build and push reproducible image
40+
name: build and push reproducible image and binaries
2641
runs-on: ubuntu-latest
27-
needs: extract-version
42+
needs: [extract-version]
2843
permissions:
2944
packages: write
30-
contents: read
45+
contents: write
3146
steps:
3247
- uses: actions/checkout@v6
48+
with:
49+
ref: ${{ needs.extract-version.outputs.VERSION }}
50+
3351
- name: Set up Docker Buildx
3452
uses: docker/setup-buildx-action@v3
3553

@@ -40,12 +58,37 @@ jobs:
4058
username: ${{ github.actor }}
4159
password: ${{ secrets.GITHUB_TOKEN }}
4260

43-
- name: Build and push reproducible image
61+
- name: Extract Rust version
62+
id: rust_version
63+
run: |
64+
RUST_TOOLCHAIN=$(rustc --version | cut -d' ' -f2)
65+
echo "RUST_TOOLCHAIN=$RUST_TOOLCHAIN" >> $GITHUB_OUTPUT
66+
67+
- name: Build reproducible artifacts
68+
uses: docker/build-push-action@v6
69+
id: docker_build
70+
with:
71+
context: .
72+
file: ./Dockerfile.reproducible
73+
build-args: |
74+
RUST_TOOLCHAIN=${{ steps.rust_version.outputs.RUST_TOOLCHAIN }}
75+
VERSION=${{ needs.extract-version.outputs.VERSION }}
76+
target: artifacts
77+
outputs: type=local,dest=./docker-artifacts
78+
cache-from: type=gha
79+
cache-to: type=gha,mode=max
80+
env:
81+
DOCKER_BUILD_RECORD_UPLOAD: false
82+
83+
- name: Build and push final image
4484
uses: docker/build-push-action@v6
4585
with:
4686
context: .
4787
file: ./Dockerfile.reproducible
4888
push: true
89+
build-args: |
90+
RUST_TOOLCHAIN=${{ steps.rust_version.outputs.RUST_TOOLCHAIN }}
91+
VERSION=${{ needs.extract-version.outputs.VERSION }}
4992
tags: |
5093
${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${{ needs.extract-version.outputs.VERSION }}
5194
${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:latest
@@ -54,3 +97,30 @@ jobs:
5497
provenance: false
5598
env:
5699
DOCKER_BUILD_RECORD_UPLOAD: false
100+
101+
- name: Prepare artifacts from Docker build
102+
run: |
103+
mkdir reproducible-artifacts
104+
cp docker-artifacts/reth reproducible-artifacts/reth-reproducible-${{ needs.extract-version.outputs.VERSION }}-x86_64-unknown-linux-gnu
105+
cp docker-artifacts/*.deb reproducible-artifacts/reth-${{ needs.extract-version.outputs.VERSION }}-x86_64-unknown-linux-gnu-reproducible.deb
106+
107+
- name: Configure GPG and create artifacts
108+
env:
109+
GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }}
110+
GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}
111+
run: |
112+
export GPG_TTY=$(tty)
113+
echo -n "$GPG_SIGNING_KEY" | base64 --decode | gpg --batch --import
114+
115+
cd reproducible-artifacts
116+
tar -czf reth-reproducible-${{ needs.extract-version.outputs.VERSION }}-x86_64-unknown-linux-gnu.tar.gz reth-reproducible-${{ needs.extract-version.outputs.VERSION }}-x86_64-unknown-linux-gnu --remove-files
117+
echo "$GPG_PASSPHRASE" | gpg --passphrase-fd 0 --pinentry-mode loopback --batch -ab reth-reproducible-${{ needs.extract-version.outputs.VERSION }}-x86_64-unknown-linux-gnu.tar.gz
118+
echo "$GPG_PASSPHRASE" | gpg --passphrase-fd 0 --pinentry-mode loopback --batch -ab reth-${{ needs.extract-version.outputs.VERSION }}-x86_64-unknown-linux-gnu-reproducible.deb
119+
120+
- name: Upload reproducible artifacts to release
121+
env:
122+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
123+
run: |
124+
gh release upload ${{ needs.extract-version.outputs.VERSION }} \
125+
reproducible-artifacts/*
126+

‎.github/workflows/reproducible-build.yml‎

Lines changed: 61 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -8,31 +8,73 @@ on:
88
jobs:
99
build:
1010
name: build reproducible binaries
11-
runs-on: ubuntu-latest
11+
runs-on: ${{ matrix.runner }}
12+
strategy:
13+
matrix:
14+
include:
15+
- runner: ubuntu-latest
16+
machine: machine-1
17+
- runner: ubuntu-22.04
18+
machine: machine-2
1219
steps:
1320
- uses: actions/checkout@v6
14-
- uses: rui314/setup-mold@v1
21+
1522
- uses: dtolnay/rust-toolchain@stable
1623
with:
1724
target: x86_64-unknown-linux-gnu
18-
- name: Install cross main
25+
26+
- name: Set up Docker Buildx
27+
uses: docker/setup-buildx-action@v3
28+
29+
- name: Build reproducible binary with Docker
1930
run: |
20-
cargo install cross --git https://github.com/cross-rs/cross
21-
- name: Install cargo-cache
31+
RUST_TOOLCHAIN=$(rustc --version | cut -d' ' -f2)
32+
docker build \
33+
--build-arg "RUST_TOOLCHAIN=${RUST_TOOLCHAIN}" \
34+
-f Dockerfile.reproducible -t reth:release \
35+
--target artifacts \
36+
--output type=local,dest=./target .
37+
38+
- name: Calculate SHA256
39+
id: sha256
2240
run: |
23-
cargo install cargo-cache
24-
- uses: Swatinem/rust-cache@v2
41+
sha256sum target/reth > checksum.sha256
42+
echo "Binaries SHA256 on ${{ matrix.machine }}: $(cat checksum.sha256)"
43+
44+
- name: Upload the hash
45+
uses: actions/upload-artifact@v4
2546
with:
26-
cache-on-failure: true
27-
- name: Build Reth
28-
run: |
29-
make build-reproducible
30-
mv target/x86_64-unknown-linux-gnu/release/reth reth-build-1
31-
- name: Clean cache
32-
run: make clean && cargo cache -a
33-
- name: Build Reth again
47+
name: checksum-${{ matrix.machine }}
48+
path: |
49+
checksum.sha256
50+
retention-days: 1
51+
52+
compare:
53+
name: compare reproducible binaries
54+
needs: build
55+
runs-on: ubuntu-latest
56+
steps:
57+
- name: Download artifacts from machine-1
58+
uses: actions/download-artifact@v4
59+
with:
60+
name: checksum-machine-1
61+
path: machine-1/
62+
- name: Download artifacts from machine-2
63+
uses: actions/download-artifact@v4
64+
with:
65+
name: checksum-machine-2
66+
path: machine-2/
67+
- name: Compare SHA256 hashes
3468
run: |
35-
make build-reproducible
36-
mv target/x86_64-unknown-linux-gnu/release/reth reth-build-2
37-
- name: Compare binaries
38-
run: cmp reth-build-1 reth-build-2
69+
echo "=== SHA256 Comparison ==="
70+
echo "Machine 1 hash:"
71+
cat machine-1/checksum.sha256
72+
echo "Machine 2 hash:"
73+
cat machine-2/checksum.sha256
74+
75+
if cmp -s machine-1/checksum.sha256 machine-2/checksum.sha256; then
76+
echo "✅ SUCCESS: Binaries are identical (reproducible build verified)"
77+
else
78+
echo "❌ FAILURE: Binaries differ (reproducible build failed)"
79+
exit 1
80+
fi

‎Cargo.toml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -328,6 +328,12 @@ inherits = "release"
328328
lto = "fat"
329329
codegen-units = 1
330330

331+
[profile.reproducible]
332+
inherits = "release"
333+
panic = "abort"
334+
codegen-units = 1
335+
incremental = false
336+
331337
[workspace.dependencies]
332338
# reth
333339
op-reth = { path = "crates/optimism/bin" }

‎Dockerfile.reproducible‎

Lines changed: 19 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,25 @@
1-
# Use the Rust 1.88 image based on Debian Bookworm
2-
FROM rust:1.88-bookworm AS builder
1+
ARG RUST_TOOLCHAIN=1.89.0
2+
FROM docker.io/rust:$RUST_TOOLCHAIN-trixie AS builder
33

4-
# Install specific version of libclang-dev
5-
RUN apt-get update && apt-get install -y libclang-dev=1:14.0-55.7~deb12u1
6-
7-
# Copy the project to the container
8-
COPY ./ /app
4+
ARG PROFILE
5+
ARG VERSION
6+
# Switch to snapshot repository to pin dependencies
7+
RUN sed -i '/^# http/{N;s|^# \(http[^ ]*\)\nURIs: .*|# \1\nURIs: \1|}' /etc/apt/sources.list.d/debian.sources
8+
RUN apt-get -o Acquire::Check-Valid-Until=false update && \
9+
apt-get install -y \
10+
libjemalloc-dev \
11+
libclang-dev \
12+
mold
913
WORKDIR /app
14+
COPY . .
15+
RUN RUSTFLAGS_REPRODUCIBLE_EXTRA="-Clink-arg=-fuse-ld=mold" make build-reth-reproducible && \
16+
PROFILE=${PROFILE:-reproducible} VERSION=$VERSION make build-deb-x86_64-unknown-linux-gnu
1017

11-
# Build the project with the reproducible settings
12-
RUN make build-reproducible
13-
14-
RUN mv /app/target/x86_64-unknown-linux-gnu/release/reth /reth
18+
FROM scratch AS artifacts
19+
COPY --from=builder /app/target/x86_64-unknown-linux-gnu/reproducible/reth /reth
20+
COPY --from=builder /app/target/x86_64-unknown-linux-gnu/reproducible/*.deb /
1521

16-
# Create a minimal final image with just the binary
17-
FROM gcr.io/distroless/cc-debian12:nonroot-6755e21ccd99ddead6edc8106ba03888cbeed41a
18-
COPY --from=builder /reth /reth
22+
FROM gcr.io/distroless/cc-debian13:nonroot-239cdd2c8a6b275b6a6f6ed1428c57de2fff3e50
23+
COPY --from=artifacts /reth /reth
1924
EXPOSE 30303 30303/udp 9001 8545 8546
2025
ENTRYPOINT [ "/reth" ]

‎Makefile‎

Lines changed: 31 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -65,33 +65,24 @@ build: ## Build the reth binary into `target` directory.
6565
cargo build --bin reth --features "$(FEATURES)" --profile "$(PROFILE)"
6666

6767
# Environment variables for reproducible builds
68-
# Initialize RUSTFLAGS
69-
RUST_BUILD_FLAGS =
70-
# Enable static linking to ensure reproducibility across builds
71-
RUST_BUILD_FLAGS += --C target-feature=+crt-static
72-
# Set the linker to use static libgcc to ensure reproducibility across builds
73-
RUST_BUILD_FLAGS += -C link-arg=-static-libgcc
74-
# Remove build ID from the binary to ensure reproducibility across builds
75-
RUST_BUILD_FLAGS += -C link-arg=-Wl,--build-id=none
76-
# Remove metadata hash from symbol names to ensure reproducible builds
77-
RUST_BUILD_FLAGS += -C metadata=''
7868
# Set timestamp from last git commit for reproducible builds
7969
SOURCE_DATE ?= $(shell git log -1 --pretty=%ct)
80-
# Disable incremental compilation to avoid non-deterministic artifacts
81-
CARGO_INCREMENTAL_VAL = 0
82-
# Set C locale for consistent string handling and sorting
83-
LOCALE_VAL = C
84-
# Set UTC timezone for consistent time handling across builds
85-
TZ_VAL = UTC
86-
87-
.PHONY: build-reproducible
88-
build-reproducible: ## Build the reth binary into `target` directory with reproducible builds. Only works for x86_64-unknown-linux-gnu currently
70+
71+
# Extra RUSTFLAGS for reproducible builds. Can be overridden via the environment.
72+
RUSTFLAGS_REPRODUCIBLE_EXTRA ?=
73+
74+
# `reproducible` only supports reth on x86_64-unknown-linux-gnu
75+
build-%-reproducible:
76+
@if [ "$*" != "reth" ]; then \
77+
echo "Error: Reproducible builds are only supported for reth, not $*"; \
78+
exit 1; \
79+
fi
8980
SOURCE_DATE_EPOCH=$(SOURCE_DATE) \
90-
RUSTFLAGS="${RUST_BUILD_FLAGS} --remap-path-prefix $$(pwd)=." \
91-
CARGO_INCREMENTAL=${CARGO_INCREMENTAL_VAL} \
92-
LC_ALL=${LOCALE_VAL} \
93-
TZ=${TZ_VAL} \
94-
cargo build --bin reth --features "$(FEATURES)" --profile "release" --locked --target x86_64-unknown-linux-gnu
81+
RUSTFLAGS="-C symbol-mangling-version=v0 -C strip=none -C link-arg=-Wl,--build-id=none -C metadata='' --remap-path-prefix $$(pwd)=. $(RUSTFLAGS_REPRODUCIBLE_EXTRA)" \
82+
LC_ALL=C \
83+
TZ=UTC \
84+
JEMALLOC_OVERRIDE=/usr/lib/x86_64-linux-gnu/libjemalloc.a \
85+
cargo build --bin reth --features "$(FEATURES) jemalloc-unprefixed" --profile "reproducible" --locked --target x86_64-unknown-linux-gnu
9586

9687
.PHONY: build-debug
9788
build-debug: ## Build the reth binary into `target/debug` directory.
@@ -155,6 +146,22 @@ op-build-x86_64-apple-darwin:
155146
op-build-aarch64-apple-darwin:
156147
$(MAKE) op-build-native-aarch64-apple-darwin
157148

149+
build-deb-%:
150+
@case "$*" in \
151+
x86_64-unknown-linux-gnu|aarch64-unknown-linux-gnu|riscv64gc-unknown-linux-gnu) \
152+
echo "Building debian package for $*"; \
153+
;; \
154+
*) \
155+
echo "Error: Debian packages are only supported for x86_64-unknown-linux-gnu, aarch64-unknown-linux-gnu, and riscv64gc-unknown-linux-gnu, not $*"; \
156+
exit 1; \
157+
;; \
158+
esac
159+
cargo install [email protected] --locked
160+
cargo deb --profile $(PROFILE) --no-build --no-dbgsym --no-strip \
161+
--target $* \
162+
$(if $(VERSION),--deb-version "1~$(VERSION)") \
163+
$(if $(VERSION),--output "target/$*/$(PROFILE)/reth-$(VERSION)-$*-$(PROFILE).deb")
164+
158165
# Create a `.tar.gz` containing a binary for a specific target.
159166
define tarball_release_binary
160167
cp $(CARGO_TARGET_DIR)/$(1)/$(PROFILE)/$(2) $(BIN_DIR)/$(2)

0 commit comments

Comments
 (0)