eosio contract build pipeline for release 2.2.x

.cicd/build.sh

@@ -2,9 +2,36 @@
 set -eo pipefail
 [[ "$ENABLE_INSTALL" == 'true' ]] || echo '--- :evergreen_tree: Configuring Environment'
 . ./.cicd/helpers/general.sh
+if [[ "$BUILDKITE_PIPELINE_SLUG" == 'eosio-contract-build' ]]; then
+    [[ "$RAW_PIPELINE_CONFIG" == '' ]] && export RAW_PIPELINE_CONFIG="$1"
+    [[ "$RAW_PIPELINE_CONFIG" == '' ]] && export RAW_PIPELINE_CONFIG='pipeline.jsonc'
+    [[ "$PIPELINE_CONFIG" == '' ]] && export PIPELINE_CONFIG='pipeline.json'
+    # read dependency file
+    if [[ -f "$RAW_PIPELINE_CONFIG" ]]; then
+        echo 'Reading pipeline configuration file...'
+        cat "$RAW_PIPELINE_CONFIG" | grep -Po '^[^"/]*("((?<=\\).|[^"])*"[^"/]*)*' | jq -c '.[env.BUILDKITE_PIPELINE_SLUG]' > "$PIPELINE_CONFIG"
+        CDT_VERSION=$(cat "$PIPELINE_CONFIG" | jq -r '.dependencies."eosio.cdt"')
+    else
+        echo 'ERROR: No pipeline configuration file or dependencies file found!'
+        exit 1
+    fi
+
+
+    CDT_COMMIT=$((curl -s https://api.github.com/repos/EOSIO/eosio.cdt/git/refs/tags/$CDT_VERSION && curl -s https://api.github.com/repos/EOSIO/eosio.cdt/git/refs/heads/$CDT_VERSION) | jq '.object.sha' | sed "s/null//g" | sed "/^$/d" | tr -d '"' | sed -n '1p')
+    test -z "$CDT_COMMIT" && CDT_COMMIT=$(echo $CDT_VERSION | tr -d '"' | tr -d "''" | cut -d ' ' -f 1) # if both searches returned nothing, the version is probably specified by commit hash already
+
+    echo "Using cdt ${CDT_COMMIT} from \"$CDT_VERSION\"..."
+    export CDT_URL="https://eos-public-oss-binaries.s3-us-west-2.amazonaws.com/${CDT_COMMIT:0:7}-eosio.cdt-ubuntu-20.04_amd64.deb"
+    export DEOSIO_COMPILE_TEST_CONTRACTS=true
+    CDT_COMMAND="curl -sSf $CDT_URL --output eosio.cdt.deb && apt install ./eosio.cdt.deb"
+fi
+
 mkdir -p "$BUILD_DIR"
+
 [[ -z "$DCMAKE_BUILD_TYPE" ]] && export DCMAKE_BUILD_TYPE='Release'
-CMAKE_EXTRAS="-DCMAKE_BUILD_TYPE=\"$DCMAKE_BUILD_TYPE\" -DENABLE_MULTIVERSION_PROTOCOL_TEST=\"true\" -DAMQP_CONN_STR=\"amqp://guest:guest@localhost:5672\""
+[[ -z "$DEOSIO_COMPILE_TEST_CONTRACTS" ]] && export DEOSIO_COMPILE_TEST_CONTRACTS=false
+CMAKE_EXTRAS="-DCMAKE_BUILD_TYPE=\"$DCMAKE_BUILD_TYPE\" -DEOSIO_COMPILE_TEST_CONTRACTS=\"$DEOSIO_COMPILE_TEST_CONTRACTS\" -DENABLE_MULTIVERSION_PROTOCOL_TEST=\"true\" -DAMQP_CONN_STR=\"amqp://guest:guest@localhost:5672\""
+
 if [[ "$(uname)" == 'Darwin' && "$FORCE_LINUX" != 'true' ]]; then
     # You can't use chained commands in execute
     if [[ "$GITHUB_ACTIONS" == 'true' ]]; then
@@ -40,8 +67,11 @@ else # Linux
     fi
     CMAKE_COMMAND="cmake \$CMAKE_EXTRAS .."
     MAKE_COMMAND="make -j $JOBS"
-    BUILD_COMMANDS="echo \"+++ :hammer_and_wrench: Building EOSIO\" && echo \"$ $CMAKE_COMMAND\" && eval $CMAKE_COMMAND && echo \"$ $MAKE_COMMAND\" && eval $MAKE_COMMAND"
-    # Docker Commands
+    if [[ "$BUILDKITE_PIPELINE_SLUG" == 'eosio-contract-build' ]]; then
+        BUILD_COMMANDS="echo \"+++ :hammer_and_wrench: Building EOSIO\" && echo \"$CDT_COMMAND\" && eval $CDT_COMMAND && echo \"$ $CMAKE_COMMAND\" && eval $CMAKE_COMMAND && echo \"$ $MAKE_COMMAND\" && eval $MAKE_COMMAND"
+    else
+        BUILD_COMMANDS="echo \"+++ :hammer_and_wrench: Building EOSIO\" && echo \"$ $CMAKE_COMMAND\" && eval $CMAKE_COMMAND && echo \"$ $MAKE_COMMAND\" && eval $MAKE_COMMAND"
+    fi    # Docker Commands
     if [[ "$BUILDKITE" == 'true' ]]; then
         # Generate Base Images
         BASE_IMAGE_COMMAND="\"$CICD_DIR/generate-base-images.sh\""
@@ -70,4 +100,4 @@ if [[ "$BUILDKITE" == 'true' && "$ENABLE_INSTALL" != 'true' ]]; then
     buildkite-agent artifact upload 'build.tar.gz'
     echo 'Done uploading artifacts.'
 fi
-[[ "$ENABLE_INSTALL" == 'true' ]] || echo '--- :white_check_mark: Done!'
+[[ "$ENABLE_INSTALL" == 'true' ]] || echo '--- :white_check_mark: Done!'

.cicd/create-docker-from-binary.sh

@@ -2,7 +2,11 @@
 echo '--- :evergreen_tree: Configuring Environment'
 set -euo pipefail
 . ./.cicd/helpers/general.sh
-buildkite-agent artifact download '*.deb' --step ':ubuntu: Ubuntu 18.04 - Package Builder' .
+if [[ "$BUILDKITE_PIPELINE_SLUG" == 'eosio-contract-build' ]]; then
+    buildkite-agent artifact download '*.deb' --step ':ubuntu: Ubuntu 20.04 - Package Builder' .
+else
+    buildkite-agent artifact download '*.deb' --step ':ubuntu: Ubuntu 18.04 - Package Builder' .
+fi
 SANITIZED_BRANCH="$(sanitize "$BUILDKITE_BRANCH")"
 echo "Branch '$BUILDKITE_BRANCH' sanitized as '$SANITIZED_BRANCH'."
 SANITIZED_TAG="$(sanitize "$BUILDKITE_TAG")"

.cicd/docker-tag.sh

@@ -2,7 +2,11 @@
 set -eo pipefail
 echo '--- :evergreen_tree: Configuring Environment'
 . ./.cicd/helpers/general.sh
-PREFIX='base-ubuntu-18.04'
+if [[ "$IMAGE_TAG" == 'ubuntu-20.04-unpinned' ]]; then
+    PREFIX='base-ubuntu-20.04'
+else
+    PREFIX='base-ubuntu-18.04'
+fi
 SANITIZED_BRANCH="$(sanitize "$BUILDKITE_BRANCH")"
 echo "Branch '$BUILDKITE_BRANCH' sanitized as '$SANITIZED_BRANCH'."
 SANITIZED_TAG="$(sanitize "$BUILDKITE_TAG")"

.cicd/generate-pipeline.sh

@@ -194,15 +194,24 @@ EOF
     fi
 done
 [[ -z "$TEST" ]] && cat <<EOF
-  - label: ":docker: Docker - Build and Install"
+  - label: ":docker: Docker - Build and Install on Ubuntu 18.04"
     command: "./.cicd/installation-build.sh"
     env:
       IMAGE_TAG: "ubuntu-18.04-$PLATFORM_TYPE"
       PLATFORM_TYPE: $PLATFORM_TYPE
     agents:
       queue: "$BUILDKITE_BUILD_AGENT_QUEUE"
     timeout: ${TIMEOUT:-180}
-    skip: ${SKIP_INSTALL}${SKIP_LINUX}${SKIP_DOCKER}${SKIP_CONTRACT_BUILDER}
+    skip: ${SKIP_INSTALL}${SKIP_UBUNTU_18_04}${SKIP_LINUX}${SKIP_DOCKER}${SKIP_CONTRACT_BUILDER}
+  - label: ":docker: Docker - Build and Install on Ubuntu 20.04"
+    command: "./.cicd/installation-build.sh"
+    env:
+      IMAGE_TAG: "ubuntu-20.04-unpinned"
+      PLATFORM_TYPE: "unpinned"
+    agents:
+      queue: "$BUILDKITE_BUILD_AGENT_QUEUE"
+    timeout: ${TIMEOUT:-180}
+    skip: ${SKIP_INSTALL}${SKIP_UBUNTU_20_04}${SKIP_LINUX}${SKIP_DOCKER}${SKIP_CONTRACT_BUILDER}
 
 EOF
 cat <<EOF
@@ -777,15 +786,25 @@ EOF
     timeout: ${TIMEOUT:-10}
     skip: ${SKIP_MACOS_11}${SKIP_PACKAGE_BUILDER}${SKIP_MAC}
 
-  - label: ":docker: Docker - Label Container with Git Branch and Git Tag"
+  - label: ":docker: Docker - Label Container with Git Branch and Git Tag on Ubuntu 18.04"
     command: .cicd/docker-tag.sh
     env:
       IMAGE_TAG: "ubuntu-18.04-$PLATFORM_TYPE"
       PLATFORM_TYPE: $PLATFORM_TYPE
     agents:
       queue: "$BUILDKITE_BUILD_AGENT_QUEUE"
     timeout: ${TIMEOUT:-10}
-    skip: ${SKIP_INSTALL}${SKIP_LINUX}${SKIP_DOCKER}${SKIP_CONTRACT_BUILDER}
+    skip: ${SKIP_INSTALL}${SKIP_UBUNTU_18_04}${SKIP_LINUX}${SKIP_DOCKER}${SKIP_CONTRACT_BUILDER}
+
+  - label: ":docker: Docker - Label Container with Git Branch and Git Tag on Ubuntu 20.04"
+    command: .cicd/docker-tag.sh
+    env:
+      IMAGE_TAG: "ubuntu-20.04-unpinned"
+      PLATFORM_TYPE: "unpinned"
+    agents:
+      queue: "$BUILDKITE_BUILD_AGENT_QUEUE"
+    timeout: ${TIMEOUT:-10}
+    skip: ${SKIP_INSTALL}${SKIP_UBUNTU_20_04}${SKIP_LINUX}${SKIP_DOCKER}${SKIP_CONTRACT_BUILDER}
 
   - wait
 
@@ -808,7 +827,14 @@ EOF
     command:  "./.cicd/create-docker-from-binary.sh"
     agents:
       queue: "$BUILDKITE_BUILD_AGENT_QUEUE"
-    skip: ${SKIP_INSTALL}${SKIP_LINUX}${SKIP_DOCKER}${SKIP_PACKAGE_BUILDER}${SKIP_PUBLIC_DOCKER}
+    skip: ${SKIP_INSTALL}${SKIP_UBUNTU_18_04}${SKIP_LINUX}${SKIP_DOCKER}${SKIP_PACKAGE_BUILDER}${SKIP_PUBLIC_DOCKER}
+    timeout: ${TIMEOUT:-10}
+
+  - label: ":docker: :ubuntu: Docker - Build 20.04 Docker Image"
+    command:  "./.cicd/create-docker-from-binary.sh"
+    agents:
+      queue: "$BUILDKITE_BUILD_AGENT_QUEUE"
+    skip: ${SKIP_INSTALL}${SKIP_UBUNTU_20_04}${SKIP_LINUX}${SKIP_DOCKER}${SKIP_PACKAGE_BUILDER}${SKIP_PUBLIC_DOCKER}
     timeout: ${TIMEOUT:-10}
 EOF
 IFS=$oIFS

.cicd/installation-build.sh

@@ -4,22 +4,32 @@ echo '--- :evergreen_tree: Configuring Environment'
 . ./.cicd/helpers/general.sh
 export ENABLE_INSTALL='true'
 export SANITIZED_BRANCH=$(sanitize "$BUILDKITE_BRANCH")
+if [[ "$IMAGE_TAG" == 'ubuntu-20.04-unpinned' ]]; then
+    export UBUNTU_VERSION=ubuntu-20.04
+else
+    export UBUNTU_VERSION=ubuntu-18.04
+fi
+
 echo "Branch '$BUILDKITE_BRANCH' sanitized as '$SANITIZED_BRANCH'."
-export CONTRACTS_BUILDER_TAG="eosio/ci-contracts-builder:base-ubuntu-18.04"
-export ARGS="--name ci-contracts-builder-$BUILDKITE_PIPELINE_SLUG-$BUILDKITE_BUILD_NUMBER --init -v \"\$(pwd):$MOUNTED_DIR\""
+export CONTRACTS_BUILDER_TAG="eosio/ci-contracts-builder:base-$UBUNTU_VERSION"
+export ARGS="--name ci-contracts-builder-$BUILDKITE_PIPELINE_SLUG-$BUILDKITE_BUILD_NUMBER-$UBUNTU_VERSION --init -v \"\$(pwd):$MOUNTED_DIR\""
 BUILD_COMMAND="'$CICD_DIR/build.sh'"
 echo "$ $BUILD_COMMAND"
 eval $BUILD_COMMAND
 echo '+++ :arrow_up: Pushing Container'
 for REGISTRY in "${CONTRACT_REGISTRIES[@]}"; do
     if [[ ! -z "$REGISTRY" ]]; then
-        COMMITS=("$REGISTRY:base-ubuntu-18.04-$BUILDKITE_COMMIT-$PLATFORM_TYPE")
-        # Platform agnostic elements should be unpinned
-        if [[ "$PLATFORM_TYPE" == 'unpinned' ]] ; then
-            COMMITS=(${COMMITS[@]} "$REGISTRY:base-ubuntu-18.04-$BUILDKITE_COMMIT" "$REGISTRY:base-ubuntu-18.04-$SANITIZED_BRANCH-$BUILDKITE_COMMIT")
+        if [[ "$BUILDKITE_PIPELINE_SLUG" == 'eosio-contract-build' ]]; then
+            COMMITS=("$REGISTRY:base-$UBUNTU_VERSION-$BUILDKITE_COMMIT" "$REGISTRY:base-$UBUNTU_VERSION-$BUILDKITE_COMMIT-$PLATFORM_TYPE" "$REGISTRY:base-$UBUNTU_VERSION-$SANITIZED_BRANCH-$BUILDKITE_COMMIT")
+        else
+            COMMITS=("$REGISTRY:base-ubuntu-18.04-$BUILDKITE_COMMIT-$PLATFORM_TYPE")
+            # Platform agnostic elements should be unpinned
+            if [[ "$PLATFORM_TYPE" == 'unpinned' ]] ; then
+                COMMITS=(${COMMITS[@]} "$REGISTRY:base-ubuntu-18.04-$BUILDKITE_COMMIT" "$REGISTRY:base-ubuntu-18.04-$SANITIZED_BRANCH-$BUILDKITE_COMMIT")
+            fi
         fi
         for COMMIT in "${COMMITS[@]}"; do
-            COMMIT_COMMAND="docker commit 'ci-contracts-builder-$BUILDKITE_PIPELINE_SLUG-$BUILDKITE_BUILD_NUMBER' '$COMMIT'"
+            COMMIT_COMMAND="docker commit 'ci-contracts-builder-$BUILDKITE_PIPELINE_SLUG-$BUILDKITE_BUILD_NUMBER-$UBUNTU_VERSION' '$COMMIT'"
             echo "$ $COMMIT_COMMAND"
             eval $COMMIT_COMMAND
             PUSH_COMMAND="docker push '$COMMIT'"
@@ -29,10 +39,10 @@ for REGISTRY in "${CONTRACT_REGISTRIES[@]}"; do
     fi
 done
 echo '--- :put_litter_in_its_place: Cleaning Up'
-DOCKER_STOP_COMMAND="docker stop 'ci-contracts-builder-$BUILDKITE_PIPELINE_SLUG-$BUILDKITE_BUILD_NUMBER'"
+DOCKER_STOP_COMMAND="docker stop 'ci-contracts-builder-$BUILDKITE_PIPELINE_SLUG-$BUILDKITE_BUILD_NUMBER-$UBUNTU_VERSION'"
 echo "$ $DOCKER_STOP_COMMAND"
 eval $DOCKER_STOP_COMMAND
-DOCKER_RM_COMMAND="docker rm 'ci-contracts-builder-$BUILDKITE_PIPELINE_SLUG-$BUILDKITE_BUILD_NUMBER'"
+DOCKER_RM_COMMAND="docker rm 'ci-contracts-builder-$BUILDKITE_PIPELINE_SLUG-$BUILDKITE_BUILD_NUMBER-$UBUNTU_VERSION' || :"
 echo "$ $DOCKER_RM_COMMAND"
 eval $DOCKER_RM_COMMAND
 echo '--- :white_check_mark: Done!'

.cicd/platforms/pinned/ubuntu-20.04-pinned.dockerfile

@@ -3,12 +3,44 @@ ENV VERSION 1
 # install dependencies.
 RUN apt-get update && \
     apt-get upgrade -y && \
-    DEBIAN_FRONTEND=noninteractive apt-get install -y git make \
-    bzip2 automake libbz2-dev libssl-dev doxygen graphviz libgmp3-dev \
-    autotools-dev python2.7 python2.7-dev python3 python3-pip python3-requests \
-    python3-dev python-configparser \
-    autoconf libtool g++ gcc curl zlib1g-dev sudo ruby libusb-1.0-0-dev \
-    libcurl4-gnutls-dev pkg-config patch vim-common jq gnupg libpq-dev postgresql postgresql-server-dev-all rabbitmq-server && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y \
+    autoconf \
+    automake \
+    autotools-dev \
+    bzip2 \
+    curl \
+    doxygen \
+    g++ \
+    gcc \
+    git \
+    gnupg \
+    graphviz \
+    jq \
+    libbz2-dev \
+    libcurl4-gnutls-dev \
+    libgmp3-dev \
+    libpq-dev \
+    libssl-dev \
+    libusb-1.0-0-dev \
+    libtinfo5 \
+    libtool \
+    make \
+    patch \
+    pkg-config \
+    postgresql \
+    postgresql-server-dev-all \
+    python2.7 \
+    python2.7-dev \
+    python3 \
+    python-configparser \
+    python3-dev \
+    python3-pip \
+    python3-requests \
+    rabbitmq-server \
+    ruby \
+    sudo \
+    vim-common \
+    zlib1g-dev && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 

.cicd/platforms/unpinned/ubuntu-20.04-unpinned.dockerfile

@@ -3,11 +3,44 @@ ENV VERSION 1
 # install dependencies.
 RUN apt-get update && \
     apt-get upgrade -y && \
-    DEBIAN_FRONTEND=noninteractive apt-get install -y git make \
-    bzip2 automake libbz2-dev libssl-dev doxygen graphviz libgmp3-dev \
-    autotools-dev python2.7 python2.7-dev python3 python3-dev python3-pip python3-requests \
-    autoconf libtool curl zlib1g-dev sudo ruby libusb-1.0-0-dev libpq-dev postgresql postgresql-server-dev-all \
-    libcurl4-gnutls-dev pkg-config patch llvm-7-dev clang-7 vim-common jq g++ gnupg rabbitmq-server && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y \
+    autoconf \
+    automake \
+    autotools-dev \
+    bzip2 \
+    curl \
+    clang-7 \
+    doxygen \
+    g++ \
+    git \
+    gnupg \
+    graphviz \
+    jq \
+    libbz2-dev \
+    libcurl4-gnutls-dev \
+    libgmp3-dev \
+    libpq-dev \
+    libssl-dev \
+    libusb-1.0-0-dev \
+    libtinfo5 \
+    libtool \
+    llvm-7-dev \
+    make \
+    patch \
+    pkg-config \
+    postgresql \
+    postgresql-server-dev-all \
+    python2.7 \
+    python2.7-dev \
+    python3 \
+    python3-dev \
+    python3-pip \
+    python3-requests \
+    rabbitmq-server \
+    ruby \
+    sudo \ 
+    vim-common \
+    zlib1g-dev && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 

pipeline.jsonc

@@ -43,5 +43,13 @@
                 "commit": "8fe13fffe4363dbc9028f3bdd549afc18802d511" // test backwards-compatibility forkdb v2 change.
             }
         ]
+    },
+    "eosio-contract-build":
+    {
+        "pipeline-branch": "master",
+        "dependencies": // dependencies to pull for a build of contracts, by branch, tag, or commit hash
+        {
+            "eosio.cdt": "release/1.8.x"
+        }
     }
 }