From b4d0cb46ba5580f972e921d5e0cfb3b2c490e7e6 Mon Sep 17 00:00:00 2001 From: Maxime Levillain Date: Mon, 19 Feb 2024 18:02:49 +0100 Subject: [PATCH] EVM/scripts: build reproducible wasm and preimages --- .dockerignore | 15 +++- etherlink/scripts/build-wasm.sh | 82 +++++++++++++++++++ etherlink/scripts/docker-compose/.env | 4 +- etherlink/scripts/docker-compose/.gitignore | 2 +- etherlink/scripts/docker-compose/README.md | 4 +- .../evm_kernel_builder.Dockerfile | 21 +++++ .../evm_kernel_builder/Dockerfile | 17 ---- etherlink/scripts/docker-compose/init.sh | 17 ++-- 8 files changed, 131 insertions(+), 31 deletions(-) create mode 100755 etherlink/scripts/build-wasm.sh create mode 100644 etherlink/scripts/docker-compose/evm_kernel_builder.Dockerfile delete mode 100644 etherlink/scripts/docker-compose/evm_kernel_builder/Dockerfile diff --git a/.dockerignore b/.dockerignore index de2b5bea0a20..fd0abaec75e5 100644 --- a/.dockerignore +++ b/.dockerignore @@ -77,10 +77,23 @@ _coverage_report .gitlab-ci.yml # Rust -target +**/target evm_kernel.wasm +evm_installer.wasm +evm_kernel_unstripped.wasm +evm-evaluation-assessor +smart-rollup-installer +_evm_installer_preimages/ +tx_kernel.wasm +tx_kernel_dal.wasm tx-demo-collector +dal_echo_kernel.wasm +risc-v-dummy.elf +sequencer.wasm +evm_benchmark_installer.wasm +evm_unstripped_installer.wasm +_evm_unstripped_installer_preimages/ # Terraform diff --git a/etherlink/scripts/build-wasm.sh b/etherlink/scripts/build-wasm.sh new file mode 100755 index 000000000000..9f7dbc59feb4 --- /dev/null +++ b/etherlink/scripts/build-wasm.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash + +set -e + +script_dir=$(dirname "$0") +etherlink_dir="$script_dir/.." +tezos_dir="$etherlink_dir/.." +commit=$(git rev-parse HEAD) + +rust_image=${rust_image:-"registry.gitlab.com/tezos/tezos/rust-toolchain"} +rust_image_tag=${rust_image_tag:-"master"} +platform=${platform:-"linux/amd64"} + +# register information about the rust-toolchain image +rust_toolchain_info() { + docker pull -q "${rust_image}:${rust_image_tag}" + docker inspect --format='{{index .RepoDigests 0}}' "${rust_image}:${rust_image_tag}" +} + +# build docker image with wasm inside it +build() { + evm_config=$1 + cp "$evm_config" "$etherlink_dir/config/.evm_config.yaml" + docker build -t etherlink_kernel:"$commit" --build-arg EVM_CONFIG="etherlink/config/.evm_config.yaml" --build-arg RUST_IMAGE="$rust_image" --build-arg RUST_TAG="$rust_image_tag" --build-arg CI_COMMIT_SHA="$commit" -f "$script_dir"/docker-compose/evm_kernel_builder.Dockerfile --platform "$platform" "$tezos_dir" + res_code=$? + rm -f "$etherlink_dir"/config/.evm_config.yaml + if [[ "${res_code}" -ne 0 ]]; then + echo "docker build of evm kernel failed" + exit 1 + fi +} + +# copy images in output directory +copy() { + output_dir=$1 + rust_image_version=$2 + container=$(docker create etherlink_kernel:"$commit") + docker cp "$container":/kernel "$output_dir" + { + echo "rust-toolchain: $rust_image_version" + echo "tezos: $commit" + } > "$output_dir/.versions" + echo "$container" +} + +# clean up +cleanup() { + container=$1 + _=$(docker container rm "$container") +} + +arg1=$1 + +case $arg1 in +--help) + cat << EOF +Reproducible EVM kernel builder +usage: [env_options] ./build-wasm.sh [[evm_config_file_path [output_directory]] +options: +- evm_config_file_path: input the config for the evm kernel installer (default=etherlink/config/dev.yaml) +- output_dir: directory where the wasm files (kernels, preimages) will be copied (default=etherlink/kernels-${commit}) +env_options: +- rust_image (default="registry.gitlab.com/tezos/tezos/rust-toolchain") +- rust_image_tag (default="master") +- platform (default="linux/amd64") +This script builds evm kernels and preimages using the configuration given in . +For it to be reproducible, they are build in a docker container then copied on the local filesystem. +EOF + ;; +*) + evm_config=${1:-"$etherlink_dir/config/dev.yaml"} + output_dir=${2:-"$etherlink_dir/kernels-$commit"} + echo "fetching rust-toolchain image info: $rust_image:$rust_image_tag" + rust_image_version=$(rust_toolchain_info) + echo "building the docker image with $evm_config" + build "$evm_config" + echo "copying kernels and preimages in $output_dir" + container=$(copy "$output_dir" "$rust_image_version") + echo "cleaning up" + cleanup "$container" + ;; +esac diff --git a/etherlink/scripts/docker-compose/.env b/etherlink/scripts/docker-compose/.env index d48644ef8bfd..455e2b8a8f9b 100644 --- a/etherlink/scripts/docker-compose/.env +++ b/etherlink/scripts/docker-compose/.env @@ -60,5 +60,5 @@ EVM_ACCOUNTS=("6ce4d79d4e77402e1ef3417fdda433aa744c6e1c" "b53dc01974176e5dff2298 SEQUENCER_ALIAS=${SEQUENCER_ALIAS:-"sequencer"} # sequencer secret key SEQUENCER_SECRET_KEY=${SEQUENCER_SECRET_KEY:-"edsk3gUfUPyBSfrS9CCgmCiQsTCHGkviBDusMxDJstFtojtc1zcpsh"} -# sequencer kernel config base file -SEQUENCER_CONFIG=${SEQUENCER_CONFIG:-$PWD/evm_config.yaml} +# evm kernel config base file +EVM_KERNEL_CONFIG=${EVM_KERNEL_CONFIG:-$PWD/evm_config.yaml} diff --git a/etherlink/scripts/docker-compose/.gitignore b/etherlink/scripts/docker-compose/.gitignore index 9702740a5200..1b2a27a642fc 100644 --- a/etherlink/scripts/docker-compose/.gitignore +++ b/etherlink/scripts/docker-compose/.gitignore @@ -1,2 +1,2 @@ /.etherlink-*-data -/evm_kernel_builder/evm_config.yaml \ No newline at end of file +/evm_config_tmp.yaml \ No newline at end of file diff --git a/etherlink/scripts/docker-compose/README.md b/etherlink/scripts/docker-compose/README.md index 0caa0b0a51e4..869a7bd6c344 100644 --- a/etherlink/scripts/docker-compose/README.md +++ b/etherlink/scripts/docker-compose/README.md @@ -70,8 +70,8 @@ EVM_ACCOUNTS=("6ce4d79d4e77402e1ef3417fdda433aa744c6e1c" "b53dc01974176e5dff2298 SEQUENCER_ALIAS=${SEQUENCER_ALIAS:-"sequencer"} # sequencer secret key SEQUENCER_SECRET_KEY=${SEQUENCER_SECRET_KEY:-"edsk3gUfUPyBSfrS9CCgmCiQsTCHGkviBDusMxDJstFtojtc1zcpsh"} -# sequencer kernel config base file -SEQUENCER_CONFIG=${SEQUENCER_CONFIG:-"$PWD/evm_config.yaml"} +# evm kernel kernel config base file +EVM_KERNEL_CONFIG=${EVM_KERNEL_CONFIG:-"$PWD/evm_config.yaml"} ``` You can you the dailynet by only setting `TZNETWORK` and removing `SNAPSHOT_URL`: diff --git a/etherlink/scripts/docker-compose/evm_kernel_builder.Dockerfile b/etherlink/scripts/docker-compose/evm_kernel_builder.Dockerfile new file mode 100644 index 000000000000..f45582caef66 --- /dev/null +++ b/etherlink/scripts/docker-compose/evm_kernel_builder.Dockerfile @@ -0,0 +1,21 @@ +ARG RUST_IMAGE=registry.gitlab.com/tezos/tezos/rust-toolchain +ARG RUST_TAG=master +ARG BASE_IMAGE=debian:sid-slim + +FROM ${RUST_IMAGE}:${RUST_TAG} AS kernel_build +RUN curl http://http.us.debian.org/debian/pool/main/p/prelink/execstack_0.0.20131005-1+b10_amd64.deb -o execstack.deb +RUN apt update && apt install -y ./execstack.deb +# Needed to clear execstack for rust 1.66 on mac os +RUN execstack -c $(find ~/.rustup/ -name libLLVM-*-rust-*-stable.so) +ARG EVM_CONFIG=etherlink/config/dev.yaml +ARG CI_COMMIT_SHA +WORKDIR /build +COPY kernels.mk etherlink.mk /build/ +COPY src/kernel_sdk /build/src/kernel_sdk +COPY etherlink /build/etherlink +RUN make -f etherlink.mk build-deps +RUN make -f etherlink.mk EVM_CONFIG=${EVM_CONFIG} evm_installer.wasm + +FROM ${BASE_IMAGE} +COPY --from=kernel_build /build/*.wasm /kernel/ +COPY --from=kernel_build /build/_evm_installer_preimages /kernel/_evm_installer_preimages diff --git a/etherlink/scripts/docker-compose/evm_kernel_builder/Dockerfile b/etherlink/scripts/docker-compose/evm_kernel_builder/Dockerfile deleted file mode 100644 index 482d28fcf9d7..000000000000 --- a/etherlink/scripts/docker-compose/evm_kernel_builder/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -FROM rust:1.66.0 AS kernel_build -RUN apt-get update -RUN apt-get install -y \ - make \ - libc-dev \ - git \ - wabt \ - clang -RUN rustup target add wasm32-unknown-unknown -RUN git clone -b master --single-branch https://gitlab.com/tezos/tezos.git /tezos -COPY evm_config.yaml /tezos/etherlink/config/sequencer.yaml -WORKDIR /tezos -RUN make -C /tezos/src/kernel_sdk/ build-deps && make -f etherlink.mk build-deps sequencer.wasm - -FROM alpine:3.14 -COPY --from=kernel_build /tezos/sequencer.wasm /tezos/evm_kernel.wasm /kernel/ -COPY --from=kernel_build /tezos/_evm_installer_preimages/ /kernel/_evm_installer_preimages diff --git a/etherlink/scripts/docker-compose/init.sh b/etherlink/scripts/docker-compose/init.sh index cfa833e6c981..a1b1d1ec89fb 100755 --- a/etherlink/scripts/docker-compose/init.sh +++ b/etherlink/scripts/docker-compose/init.sh @@ -54,7 +54,7 @@ add_kernel_config_contract() { alias="$3" label="$4" - if address=$(run_in_docker octez-client --endpoint "$ENDPOINT" show known contract "$alias"); then + if address=$(run_in_docker octez-client --endpoint "$ENDPOINT" show known contract "$alias" 2> /dev/null); then hex=$(printf '%s' "${address}" | xxd -p -c 36) add_kernel_config_set "$file" "$key" "$hex" "${label}: ${address}" fi @@ -95,11 +95,12 @@ create_kernel_config() { build_kernel() { mkdir -p "${HOST_TEZOS_DATA_DIR}/.tezos-client" - cp "${SEQUENCER_CONFIG}" evm_kernel_builder/evm_config.yaml - create_kernel_config evm_kernel_builder/evm_config.yaml + cp "${EVM_KERNEL_CONFIG}" "$script_dir"/evm_config_tmp.yaml + create_kernel_config "$script_dir"/evm_config_tmp.yaml # build kernel in an image (e.g. tezos/tezos-bare:master) with new chain id - docker build --no-cache -t etherlink_kernel:"${OCTEZ_TAG}" "${script_dir}"/evm_kernel_builder/ - container_name=$(docker create etherlink_kernel:"${OCTEZ_TAG}") + commit="$(git rev-parse HEAD)" + docker build --no-cache -t etherlink_kernel:"$commit" --build-arg EVM_CONFIG="etherlink/scripts/docker-compose/evm_config_tmp.yaml" --build-arg CI_COMMIT_SHA="$commit" -f "$script_dir/evm_kernel_builder.Dockerfile" "${script_dir}/../../.." + container_name=$(docker create etherlink_kernel:"$commit") docker cp "${container_name}":/kernel/ "${HOST_TEZOS_DATA_DIR}/" } @@ -191,7 +192,7 @@ originate_contracts() { init_rollup() { docker_update_images build_kernel - kernel="${HOST_TEZOS_DATA_DIR}"/kernel/sequencer.wasm + kernel="${HOST_TEZOS_DATA_DIR}"/kernel/evm_installer.wasm originate_evm_rollup "${kernel}" init_rollup_node_config } @@ -263,7 +264,7 @@ init_octez_node) build_kernel) docker_update_images build_kernel - kernel="${HOST_TEZOS_DATA_DIR}"/kernel/sequencer.wasm + kernel="${HOST_TEZOS_DATA_DIR}"/kernel/evm_installer.wasm ;; init_rollup) if [[ -n ${OPERATOR_ALIAS} ]]; then @@ -308,7 +309,7 @@ Available commands: - originate_contracts: originate contracts - build_kernel: - build lastest evm kernel + build latest evm kernel - init_rollup: build lastest evm kernel, originate the rollup, create operator, wait until operator balance is topped then create rollup node config. -- GitLab