diff --git a/.circleci/config.yml b/.circleci/config.yml index d17f13baadd4..288749974fb3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -33,26 +33,11 @@ defaults: - run_build: &run_build name: Build - command: | - set -ex - if [ "$CIRCLE_BRANCH" = release -o -n "$CIRCLE_TAG" -o -n "$FORCE_RELEASE" ]; then echo -n > prerelease.txt; else date -u +"nightly.%Y.%-m.%-d" > prerelease.txt; fi - echo -n "$CIRCLE_SHA1" > commit_hash.txt - mkdir -p build - cd build - [ -n "$COVERAGE" -a "$CIRCLE_BRANCH" != release -a -z "$CIRCLE_TAG" ] && CMAKE_OPTIONS="$CMAKE_OPTIONS -DCOVERAGE=ON" - cmake .. -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:-Release} $CMAKE_OPTIONS -G "Unix Makefiles" - make -j4 + command: scripts/ci/build.sh - run_build_ossfuzz: &run_build_ossfuzz name: Build_ossfuzz - command: | - mkdir -p build - cd build - protoc --proto_path=../test/tools/ossfuzz yulProto.proto --cpp_out=../test/tools/ossfuzz - protoc --proto_path=../test/tools/ossfuzz abiV2Proto.proto --cpp_out=../test/tools/ossfuzz - protoc --proto_path=../test/tools/ossfuzz solProto.proto --cpp_out=../test/tools/ossfuzz - cmake .. -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:-Release} $CMAKE_OPTIONS - make ossfuzz ossfuzz_proto ossfuzz_abiv2 -j4 + command: scripts/ci/build_ossfuzz.sh - run_proofs: &run_proofs name: Correctness proofs for optimization rules diff --git a/.github/workflows/buildpack-deps.yml b/.github/workflows/buildpack-deps.yml new file mode 100644 index 000000000000..48d83b77cd04 --- /dev/null +++ b/.github/workflows/buildpack-deps.yml @@ -0,0 +1,42 @@ +name: buildpack-deps + +on: + pull_request: + branches: [ develop ] + paths: + - 'scripts/docker/buildpack-deps/Dockerfile.emscripten' + - 'scripts/docker/buildpack-deps/Dockerfile.ubuntu1604.clang.ossfuzz' + - 'scripts/docker/buildpack-deps/Dockerfile.ubuntu1804' + - 'scripts/docker/buildpack-deps/Dockerfile.ubuntu2004.clang' + - 'scripts/docker/buildpack-deps/Dockerfile.ubuntu2004' + +jobs: + buildpack-deps: + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DOCKER_REPOSITORY: docker.pkg.github.com/${{ github.repository }} + IMAGE_NAME: buildpack-deps + + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + image_variant: [emscripten, ubuntu1604.clang.ossfuzz, ubuntu1804, ubuntu2004.clang, ubuntu2004] + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Upgrade ${{ env.IMAGE_NAME }}-${{ matrix.image_variant }} + run: | + echo "${GITHUB_TOKEN}" | docker login docker.pkg.github.com -u "${GITHUB_ACTOR}" --password-stdin + scripts/ci/docker_upgrade.sh ${{ env.IMAGE_NAME }} ${{ matrix.image_variant }} ${{ env.DOCKER_REPOSITORY }} + docker logout docker.pkg.github.com + + - name: comment PR + if: "env.DOCKER_IMAGE" + uses: aarlt/comment-on-pr@v1.2.0 + with: + msg: "`${{ env.DOCKER_IMAGE }} ${{ env.DOCKER_REPO_DIGEST }}`." \ No newline at end of file diff --git a/.gitignore b/.gitignore index 67e4507591ab..749860301975 100644 --- a/.gitignore +++ b/.gitignore @@ -32,7 +32,7 @@ prerelease.txt # Build directory build/ -build*/ +/build*/ emscripten_build/ docs/_build __pycache__ diff --git a/scripts/ci/build.sh b/scripts/ci/build.sh new file mode 100755 index 000000000000..1c8824abf46f --- /dev/null +++ b/scripts/ci/build.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -ex + +ROOTDIR="$(dirname "$0")/../.." +cd "${ROOTDIR}" + +# shellcheck disable=SC2166 +if [ "$CIRCLE_BRANCH" = release -o -n "$CIRCLE_TAG" -o -n "$FORCE_RELEASE" ]; then echo -n >prerelease.txt; else date -u +"nightly.%Y.%-m.%-d" >prerelease.txt; fi +if [ -n "$CIRCLE_SHA1" ] +then + echo -n "$CIRCLE_SHA1" >commit_hash.txt +fi + +mkdir -p build +cd build + +# shellcheck disable=SC2166 +[ -n "$COVERAGE" -a "$CIRCLE_BRANCH" != release -a -z "$CIRCLE_TAG" ] && CMAKE_OPTIONS="$CMAKE_OPTIONS -DCOVERAGE=ON" + +# shellcheck disable=SC2086 +cmake .. -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE:-Release}" $CMAKE_OPTIONS -G "Unix Makefiles" + +make -j 4 diff --git a/scripts/ci/build_ossfuzz.sh b/scripts/ci/build_ossfuzz.sh new file mode 100755 index 000000000000..42f7b3a3959c --- /dev/null +++ b/scripts/ci/build_ossfuzz.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -e + +ROOTDIR="$(dirname "$0")/../.." +BUILDDIR="${ROOTDIR}/build" + +mkdir -p "${BUILDDIR}" +cd "${BUILDDIR}" + +protoc --proto_path=../test/tools/ossfuzz yulProto.proto --cpp_out=../test/tools/ossfuzz +protoc --proto_path=../test/tools/ossfuzz abiV2Proto.proto --cpp_out=../test/tools/ossfuzz +protoc --proto_path=../test/tools/ossfuzz solProto.proto --cpp_out=../test/tools/ossfuzz +cmake .. -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE:-Release}" -DCMAKE_TOOLCHAIN_FILE=../cmake/toolchains/libfuzzer.cmake + +make ossfuzz ossfuzz_proto ossfuzz_abiv2 -j 4 diff --git a/scripts/ci/buildpack-deps_test_emscripten.sh b/scripts/ci/buildpack-deps_test_emscripten.sh new file mode 120000 index 000000000000..6e838fba7fb3 --- /dev/null +++ b/scripts/ci/buildpack-deps_test_emscripten.sh @@ -0,0 +1 @@ +../../scripts/travis-emscripten/build_emscripten.sh \ No newline at end of file diff --git a/scripts/ci/buildpack-deps_test_ubuntu1604.clang.ossfuzz.sh b/scripts/ci/buildpack-deps_test_ubuntu1604.clang.ossfuzz.sh new file mode 120000 index 000000000000..86beda7dd2c3 --- /dev/null +++ b/scripts/ci/buildpack-deps_test_ubuntu1604.clang.ossfuzz.sh @@ -0,0 +1 @@ +build_ossfuzz.sh \ No newline at end of file diff --git a/scripts/ci/buildpack-deps_test_ubuntu1804.sh b/scripts/ci/buildpack-deps_test_ubuntu1804.sh new file mode 120000 index 000000000000..c07a74de4fb4 --- /dev/null +++ b/scripts/ci/buildpack-deps_test_ubuntu1804.sh @@ -0,0 +1 @@ +build.sh \ No newline at end of file diff --git a/scripts/ci/buildpack-deps_test_ubuntu2004.clang.sh b/scripts/ci/buildpack-deps_test_ubuntu2004.clang.sh new file mode 120000 index 000000000000..c07a74de4fb4 --- /dev/null +++ b/scripts/ci/buildpack-deps_test_ubuntu2004.clang.sh @@ -0,0 +1 @@ +build.sh \ No newline at end of file diff --git a/scripts/ci/buildpack-deps_test_ubuntu2004.sh b/scripts/ci/buildpack-deps_test_ubuntu2004.sh new file mode 120000 index 000000000000..c07a74de4fb4 --- /dev/null +++ b/scripts/ci/buildpack-deps_test_ubuntu2004.sh @@ -0,0 +1 @@ +build.sh \ No newline at end of file diff --git a/scripts/ci/docker_upgrade.sh b/scripts/ci/docker_upgrade.sh new file mode 100755 index 000000000000..a8212c2c24d4 --- /dev/null +++ b/scripts/ci/docker_upgrade.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +set -e + +function error() { + echo >&2 "ERROR: ${1} Aborting." && false +} + +function warning() { + echo >&2 "WARNING: ${1}" +} + +[[ $# == 3 ]] || error "Expected exactly 3 parameters: '${0} '." + +IMAGE_NAME="${1}" +IMAGE_VARIANT="${2}" +DOCKER_REPOSITORY="${3}" +DOCKERFILE="scripts/docker/${IMAGE_NAME}/Dockerfile.${IMAGE_VARIANT}" + +echo "-- check_dockerfile_was_changed" + +# exit, if the dockerfile was not changed. +if git diff --quiet origin/develop HEAD -- "${DOCKERFILE}"; then + echo "${DOCKERFILE} was not changed. Nothing to do." + exit 0 +fi + +echo "-- check_version" + +PREV_VERSION=$(git diff origin/develop HEAD -- "${DOCKERFILE}" | grep -e '^\s*-LABEL\s\+version=".*"\s*$' | awk -F'"' '{ print $2 }') +NEXT_VERSION=$(git diff origin/develop HEAD -- "${DOCKERFILE}" | grep -e '^\s*+LABEL\s\+version=".*"\s*$' | awk -F'"' '{ print $2 }') + +[[ $NEXT_VERSION != "" ]] || error "No version label defined in Dockerfile. You may need to add 'LABEL version' in '${DOCKERFILE}'." + +[[ $PREV_VERSION != "" ]] || { + warning "no previous version found. Will set \$PREV_VERSION = 0." + PREV_VERSION=0 +} + +if [[ $((PREV_VERSION + 1)) != $((NEXT_VERSION)) ]]; then + error "Version label in Dockerfile was not incremented. You may need to change 'LABEL version' in '${DOCKERFILE}'." +fi + +echo "-- build_docker" + +# This is a workaround: we run `docker build` twice to prevent the `layer does not exist` problem. +# See https://github.com/moby/moby/issues/37965. +docker build "scripts/docker/${IMAGE_NAME}" --file "scripts/docker/${IMAGE_NAME}/Dockerfile.${IMAGE_VARIANT}" --tag "${IMAGE_NAME}" || + docker build "scripts/docker/${IMAGE_NAME}" --file "scripts/docker/${IMAGE_NAME}/Dockerfile.${IMAGE_VARIANT}" --tag "${IMAGE_NAME}" + +echo "-- test_docker @ '${PWD}'" + +docker run --rm --volume "${PWD}:/root/project" "${IMAGE_NAME}" "/root/project/scripts/ci/${IMAGE_NAME}_test_${IMAGE_VARIANT}.sh" + +echo "-- push_docker" + +VERSION=$(docker inspect --format='{{.Config.Labels.version}}' "${IMAGE_NAME}") +DOCKER_IMAGE_ID="${DOCKER_REPOSITORY}/${IMAGE_NAME}-${IMAGE_VARIANT}" + +docker tag "${IMAGE_NAME}" "${DOCKER_IMAGE_ID}:${VERSION}" +docker push "${DOCKER_IMAGE_ID}:${VERSION}" + +REPO_DIGEST=$(docker inspect --format='{{.RepoDigests}}' "${DOCKER_IMAGE_ID}:${VERSION}") + +docker tag "${IMAGE_NAME}" "${DOCKER_IMAGE_ID}:latest" +docker push "${DOCKER_IMAGE_ID}:latest" + +echo "::set-env name=DOCKER_IMAGE::${DOCKER_IMAGE_ID}:${VERSION}" +echo "::set-env name=DOCKER_REPO_DIGEST::${REPO_DIGEST}" diff --git a/.circleci/docker/Dockerfile.emscripten b/scripts/docker/buildpack-deps/Dockerfile.emscripten similarity index 99% rename from .circleci/docker/Dockerfile.emscripten rename to scripts/docker/buildpack-deps/Dockerfile.emscripten index d2456bb2ae0b..8885e1302508 100644 --- a/.circleci/docker/Dockerfile.emscripten +++ b/scripts/docker/buildpack-deps/Dockerfile.emscripten @@ -29,6 +29,7 @@ # make version=1.39.15 build # FROM emscripten/emsdk:1.39.15 AS base +LABEL version="0" ADD emscripten.jam /usr/src RUN set -ex; \ @@ -63,3 +64,4 @@ RUN set -ex; \ cxxflags="-s DISABLE_EXCEPTION_CATCHING=0 -Wno-unused-local-typedef -Wno-variadic-macros -Wno-c99-extensions -Wno-all" \ --prefix=/emsdk/emscripten/sdk/system install; \ rm -r /usr/src/boost_1_73_0 + diff --git a/.circleci/docker/Dockerfile.ubuntu1604.clang.ossfuzz b/scripts/docker/buildpack-deps/Dockerfile.ubuntu1604.clang.ossfuzz similarity index 99% rename from .circleci/docker/Dockerfile.ubuntu1604.clang.ossfuzz rename to scripts/docker/buildpack-deps/Dockerfile.ubuntu1604.clang.ossfuzz index f7dfc7001bdf..add0bfec433e 100644 --- a/.circleci/docker/Dockerfile.ubuntu1604.clang.ossfuzz +++ b/scripts/docker/buildpack-deps/Dockerfile.ubuntu1604.clang.ossfuzz @@ -22,6 +22,7 @@ # (c) 2016-2019 solidity contributors. #------------------------------------------------------------------------------ FROM gcr.io/oss-fuzz-base/base-clang as base +LABEL version="0" ARG DEBIAN_FRONTEND=noninteractive @@ -99,3 +100,4 @@ FROM base COPY --from=libraries /usr/lib /usr/lib COPY --from=libraries /usr/bin /usr/bin COPY --from=libraries /usr/include /usr/include + diff --git a/.circleci/docker/Dockerfile.ubuntu1804 b/scripts/docker/buildpack-deps/Dockerfile.ubuntu1804 similarity index 99% rename from .circleci/docker/Dockerfile.ubuntu1804 rename to scripts/docker/buildpack-deps/Dockerfile.ubuntu1804 index fa651900b873..34f1e674b866 100644 --- a/.circleci/docker/Dockerfile.ubuntu1804 +++ b/scripts/docker/buildpack-deps/Dockerfile.ubuntu1804 @@ -22,6 +22,7 @@ # (c) 2016-2019 solidity contributors. #------------------------------------------------------------------------------ FROM buildpack-deps:bionic AS base +LABEL version="0" ARG DEBIAN_FRONTEND=noninteractive @@ -91,3 +92,4 @@ FROM base COPY --from=libraries /usr/lib /usr/lib COPY --from=libraries /usr/bin /usr/bin COPY --from=libraries /usr/include /usr/include + diff --git a/.circleci/docker/Dockerfile.ubuntu2004 b/scripts/docker/buildpack-deps/Dockerfile.ubuntu2004 similarity index 99% rename from .circleci/docker/Dockerfile.ubuntu2004 rename to scripts/docker/buildpack-deps/Dockerfile.ubuntu2004 index a021b6533486..6c5486bd2776 100644 --- a/.circleci/docker/Dockerfile.ubuntu2004 +++ b/scripts/docker/buildpack-deps/Dockerfile.ubuntu2004 @@ -22,6 +22,7 @@ # (c) 2016-2019 solidity contributors. #------------------------------------------------------------------------------ FROM buildpack-deps:focal AS base +LABEL version="0" ARG DEBIAN_FRONTEND=noninteractive @@ -60,3 +61,4 @@ FROM base COPY --from=libraries /usr/lib /usr/lib COPY --from=libraries /usr/bin /usr/bin COPY --from=libraries /usr/include /usr/include + diff --git a/.circleci/docker/Dockerfile.ubuntu2004.clang b/scripts/docker/buildpack-deps/Dockerfile.ubuntu2004.clang similarity index 99% rename from .circleci/docker/Dockerfile.ubuntu2004.clang rename to scripts/docker/buildpack-deps/Dockerfile.ubuntu2004.clang index c8d057eb11a0..b38fe4f1a261 100644 --- a/.circleci/docker/Dockerfile.ubuntu2004.clang +++ b/scripts/docker/buildpack-deps/Dockerfile.ubuntu2004.clang @@ -22,6 +22,7 @@ # (c) 2016-2019 solidity contributors. #------------------------------------------------------------------------------ FROM buildpack-deps:focal AS base +LABEL version="0" ARG DEBIAN_FRONTEND=noninteractive @@ -62,3 +63,4 @@ FROM base COPY --from=libraries /usr/lib /usr/lib COPY --from=libraries /usr/bin /usr/bin COPY --from=libraries /usr/include /usr/include + diff --git a/scripts/docker/buildpack-deps/README.md b/scripts/docker/buildpack-deps/README.md new file mode 100644 index 000000000000..6b8855187d88 --- /dev/null +++ b/scripts/docker/buildpack-deps/README.md @@ -0,0 +1,35 @@ +# buildpack-deps docker images + +The `buildpack-deps` docker images are used to compile and test solidity within our CI. + +## GitHub Workflow + +The creation of the images are triggered by a single workflow, defined in `.github/workflows/buildpack-deps.yml`. +For each resulting `buildpack-deps` docker image a strategy is defined in the workflow file - the image variant. +The workflow gets triggered, if any Dockerfile defined in `scripts/docker/buildpack-deps/Dockerfile.*` were changed +within the PR. + +### Versioning + +The version of the docker images can be defined within the Dockerfile with `LABEL version`. A new docker image +will only be created and pushed, if the new version is incremented by `1` compared with the version of the Dockerfile +located in `develop`. + +### Build, Test & Push + +Note that the whole workflow - including all defined strategies (image variants) - will be triggered, +even if only a single Dockerfile was change. The full workflow will only gets executed, if the corresponding +Dockerfile was changed. The execution of workflows of unchanged Dockerfiles will not continue and just return success. +See `scripts/ci/docker_upgrade.sh`. + +If the version check was successful, the docker image will be built using the Dockerfile located in +`scripts/docker/buildpack-deps/Dockerfile.*`. + +The resulting docker image will be tested by executing +the corresponding `scripts/ci/buildpack-deps_test_*` scripts. These scripts are normally symlinked to `scripts/ci/build.sh`, +except for the `buildpack-deps-ubuntu1604.clang.ossfuzz` docker image, that is symlinked to `scripts/ci/build_ossfuzz.sh`. +These scripts `scripts/ci/build.sh` and `scripts/ci/build_ossfuzz.sh` are also used by CircleCI, see `.circleci/config.yml`. + +If the tests passed successfully, the docker image will get tagged by the version defined within the corresponding `Dockerfile`. +Finally, a comment will be added to the PR that contains the full repository, version and repository digest +of the freshly created docker image. \ No newline at end of file diff --git a/.circleci/docker/emscripten.jam b/scripts/docker/buildpack-deps/emscripten.jam similarity index 100% rename from .circleci/docker/emscripten.jam rename to scripts/docker/buildpack-deps/emscripten.jam