From 559955a820b6cfece086e0b58b97044492945fef Mon Sep 17 00:00:00 2001 From: Gavin Jefferies Date: Mon, 28 Feb 2022 10:24:50 -0800 Subject: [PATCH] RS-438: Migrate the build image from stackrox to rox-ci-image (#112) --- .circleci/changes_affect.sh | 23 ++-- .circleci/config.yml | 120 +++++++++++++----- .circleci/get_tag.sh | 22 +++- .editorconfig | 2 + Makefile | 17 +++ README.md | 4 +- images/centos8-rocksdb.Dockerfile | 39 ++++++ images/rocksdb.Dockerfile | 6 +- images/stackrox-build.Dockerfile | 49 +++++++ .../etc/yum.repos.d/google-cloud-sdk.repo | 8 ++ test/bats/get_tag.bats | 40 +++--- 11 files changed, 254 insertions(+), 76 deletions(-) create mode 100644 .editorconfig create mode 100644 Makefile create mode 100644 images/centos8-rocksdb.Dockerfile create mode 100644 images/stackrox-build.Dockerfile create mode 100644 images/static-contents/etc/yum.repos.d/google-cloud-sdk.repo diff --git a/.circleci/changes_affect.sh b/.circleci/changes_affect.sh index 925b4d47..365ec41f 100755 --- a/.circleci/changes_affect.sh +++ b/.circleci/changes_affect.sh @@ -24,33 +24,36 @@ all_changed_files() { echo "${all_changed_files[*]}" } - affects_all() { - [[ " $* " =~ images/static-contents ]] + [[ " $* " =~ [[:space:]]images/static-contents/bin/bash-wrapper[[:space:]] ]] } affects_collector() { - [[ " $* " =~ images/collector.Dockerfile ]] + [[ " $* " =~ [[:space:]]images/collector.Dockerfile[[:space:]] ]] } affects_scanner() { - [[ " $* " =~ images/rox.Dockerfile ]] \ - || [[ " $* " =~ images/base.Dockerfile ]] \ - || [[ " $* " =~ images/rocksdb.Dockerfile ]] + [[ " $* " =~ [[:space:]]images/rox.Dockerfile[[:space:]] ]] \ + || [[ " $* " =~ [[:space:]]images/base.Dockerfile[[:space:]] ]] \ + || [[ " $* " =~ [[:space:]]images/rocksdb.Dockerfile[[:space:]] ]] } affects_stackrox() { - [[ " $* " =~ images/rox.Dockerfile ]] \ - || [[ " $* " =~ images/base.Dockerfile ]] \ - || [[ " $* " =~ images/rocksdb.Dockerfile ]] + [[ " $* " =~ [[:space:]]images/rox.Dockerfile[[:space:]] ]] \ + || [[ " $* " =~ [[:space:]]images/base.Dockerfile[[:space:]] ]] \ + || [[ " $* " =~ [[:space:]]images/rocksdb.Dockerfile[[:space:]] ]] \ + || [[ " $* " =~ [[:space:]]images/centos8-rocksdb.Dockerfile[[:space:]] ]] \ + || [[ " $* " =~ [[:space:]]images/stackrox-build.Dockerfile[[:space:]] ]] \ + || [[ " $* " =~ [[:space:]]images/static-contents/etc/yum.repos.d/google-cloud-sdk.repo[[:space:]] ]] } affects_jenkins-plugin() { - [[ " $* " =~ images/jenkins-plugin.Dockerfile ]] + [[ " $* " =~ [[:space:]]images/jenkins-plugin.Dockerfile[[:space:]] ]] } main() { files="$(all_changed_files)" + echo "Changed files: >>>${files}<<<" if affects_all "${files}"; then return 0 fi diff --git a/.circleci/config.yml b/.circleci/config.yml index 050eba1b..10f02908 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -18,11 +18,9 @@ commands: repo: description: Name of the repo where the PR should be opened. type: string - default: "" - image-flavor: - description: The flavor of the apollo-ci image that the target repo uses. + image-flavors: + description: The flavors of the apollo-ci image that the target repo uses. A comma separated list. type: string - default: "" steps: - checkout - add_ssh_keys: @@ -33,15 +31,13 @@ commands: command: | git clone git@github.com:stackrox/<< parameters.repo >>.git /tmp/<< parameters.repo >> - run: - name: Create commit in << parameters.repo >> that updates the base image if necessary + name: Create a commit in << parameters.repo >> that updates dependent images if necessary command: | if ! .circleci/changes_affect.sh << parameters.repo >>; then echo "No need to open/update a PR against << parameters.repo >> - current changes are not affecting this repo." exit 0; fi - TAG="$(.circleci/get_tag.sh "<< parameters.image-flavor >>")" - pushd "/tmp/<< parameters.repo >>" git config user.email "roxbot@stackrox.com" @@ -57,6 +53,36 @@ commands: git push --set-upstream origin "${branch_name}" fi + todo="# TODO(do not merge): After upstream PR is merged, cut a tag and update this" + + IFS=',' read -r -a flavors \<<<"<< parameters.image-flavors >>" + for flavor in "${flavors[@]}"; do + echo "Doing image substitutions for $flavor" + prefix="$flavor-" + if [[ "$flavor" == "rox" ]]; then + prefix="" + fi + popd + tag="$(.circleci/get_tag.sh "$flavor")" + pushd "/tmp/<< parameters.repo >>" + + sed -r -i "s@(.*)/apollo-ci:${prefix}[0-9].*@\1/apollo-ci:${tag} ${todo}@g" .circleci/config.yml + + # If the image parameter was originally quoted, we need to close the quote + sed -r -i "s@\"(.*)/apollo-ci:${tag} # TODO@\"\1/apollo-ci:${tag}\" # TODO@g" .circleci/config.yml + + if [[ "$flavor" == "stackrox-build" ]] && [[ "<< parameters.repo >>" == "stackrox" ]]; then + echo "${tag} ${todo}" > BUILD_IMAGE_VERSION + fi + + if git diff-index --quiet HEAD; then + echo "There are no changes to commit in the dependent repo" + else + git commit -am "Bump apollo-ci:$flavor image tag to ${tag##:}" + fi + done + + git push origin "${branch_name}" popd # Open or update a PR and configure labels, assignees @@ -67,26 +93,11 @@ commands: "Bump version of apollo-ci image used in CircleCI" \ "ci-upgrade-tests" - pushd "/tmp/<< parameters.repo >>" - - sed -i "s@\(.*\)/apollo-ci:.*@\1/apollo-ci:${TAG} # TODO(do not merge): After upstream PR is merged, cut a tag and update this@g" .circleci/config.yml - - # If the image parameter was originally quoted, we need to close the quote - sed -i "s@\"\(.*\)/apollo-ci:${TAG} # TODO@\"\1/apollo-ci:${TAG}\" # TODO@g" .circleci/config.yml - - if git diff-index --quiet HEAD; then - echo "There is nothing new to commit - apparently the CI job has been restarted without any new changes in the code" - else - git commit --allow-empty -am "Bump base image tag to ${TAG##:}" - git push origin "${branch_name}" - fi - build-and-push-image: parameters: image-flavor: description: A flavor used to tag the apollo-ci image. type: string - default: "" dockerfile-path: description: Path to the Dockerfile type: string @@ -108,11 +119,23 @@ commands: rocksdb_sha="$(git hash-object images/rocksdb.Dockerfile)" BUILD_ARGS+=(--build-arg "ROCKSDB_TAG=rocksdb-<< pipeline.parameters.base_ubuntu_tag >>-${rocksdb_sha}") + centos8_rocksdb_sha="$(git hash-object images/centos8-rocksdb.Dockerfile)" + BUILD_ARGS+=(--build-arg "CENTOS8_ROCKSDB_TAG=centos8-rocksdb-${centos8_rocksdb_sha}") + TAG="$(.circleci/get_tag.sh "<< parameters.image-flavor >>")" + IMAGE="quay.io/rhacs-eng/apollo-ci:${TAG}" + + if [[ "<< parameters.image-flavor >>" == "centos8-rocksdb" ]] && \ + DOCKER_CLI_EXPERIMENTAL=enabled docker manifest inspect "$IMAGE" >/dev/null; then + echo "Image '$IMAGE' already exists - no need to build it" + circleci step halt + exit 0 + fi + docker build \ "${BUILD_ARGS[@]}" \ -f << parameters.dockerfile-path >> \ - -t "quay.io/rhacs-eng/apollo-ci:${TAG}" \ + -t "${IMAGE}" \ images/ for i in {1..5}; do @@ -124,7 +147,6 @@ commands: image-flavor: description: The flavor of apollo-ci image to check. type: string - default: "" steps: - run: name: Get roxctl @@ -151,7 +173,8 @@ jobs: <<: *defaults steps: - checkout - - setup_remote_docker + - setup_remote_docker: + version: 20.10.7 - run: name: "Ensure that rocksdb container is up-to-date" command: | @@ -176,7 +199,25 @@ jobs: steps: - build-and-push-image: dockerfile-path: images/rox.Dockerfile - - check-image + image-flavor: "rox" + - check-image: + image-flavor: "rox" + + build-and-push-centos8-rocksdb: + <<: *defaults + steps: + - build-and-push-image: + dockerfile-path: images/centos8-rocksdb.Dockerfile + image-flavor: "centos8-rocksdb" + + build-and-push-stackrox-build: + <<: *defaults + steps: + - build-and-push-image: + dockerfile-path: images/stackrox-build.Dockerfile + image-flavor: "stackrox-build" + - check-image: + image-flavor: "stackrox-build" test-cci-export: <<: *defaults @@ -288,26 +329,25 @@ jobs: steps: - open-test-pr: repo: stackrox - + image-flavors: "stackrox-build,rox" create-or-update-scanner-repo-pr: <<: *defaults steps: - open-test-pr: repo: scanner - + image-flavors: "rox" create-or-update-collector-repo-pr: <<: *defaults steps: - open-test-pr: repo: collector - image-flavor: "collector" - + image-flavors: "collector" create-or-update-jenkins-plugin-repo-pr: <<: *defaults steps: - open-test-pr: repo: jenkins-plugin - image-flavor: "jenkins-plugin" + image-flavors: "jenkins-plugin" workflows: version: 2 @@ -325,6 +365,13 @@ workflows: filters: tags: only: /.*/ + - build-and-push-centos8-rocksdb: + context: + - quay-rhacs-eng-readwrite + - docker-io-pull + filters: + tags: + only: /.*/ - build-and-push-base: context: - quay-rhacs-eng-readwrite @@ -352,6 +399,16 @@ workflows: - build-and-push-base - build-and-push-rocksdb - test-cci-export + - build-and-push-stackrox-build: + context: + - quay-rhacs-eng-readwrite + - stackrox-ci-instance + - docker-io-pull + filters: + tags: + only: /.*/ + requires: + - build-and-push-centos8-rocksdb - build-and-push-env-check: context: - quay-rhacs-eng-readwrite @@ -405,6 +462,7 @@ workflows: ignore: master requires: - build-and-push-rox + - build-and-push-stackrox-build - create-or-update-scanner-repo-pr: filters: branches: diff --git a/.circleci/get_tag.sh b/.circleci/get_tag.sh index c72c49c3..4951b263 100755 --- a/.circleci/get_tag.sh +++ b/.circleci/get_tag.sh @@ -2,19 +2,27 @@ set -euo pipefail -if [[ $# -ne 1 ]]; then +if [[ -z "${1:-}" ]]; then echo "Usage: $0 " exit 1 fi image_flavor="$1" -if [[ "$image_flavor" != "" ]]; then - image_flavor="${image_flavor}-" + +if [[ "$image_flavor" == "rocksdb" ]]; then + echo "rocksdb-$(git hash-object images/rocksdb.Dockerfile)" + exit 0 +fi + +if [[ "$image_flavor" == "centos8-rocksdb" ]]; then + echo "centos8-rocksdb-$(git hash-object images/centos8-rocksdb.Dockerfile)" + exit 0 fi -snapshot="" -if [[ "${CIRCLE_BRANCH:-}" != "master" && -z "${CIRCLE_TAG:-}" ]]; then - snapshot="snapshot-" +if [[ "$image_flavor" == "rox" ]]; then + image_prefix="" +else + image_prefix="${image_flavor}-" fi -echo "${snapshot}${image_flavor}$(git describe --tags --abbrev=10)" +echo "${image_prefix}$(git describe --tags --abbrev=10)" diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..02054bd4 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,2 @@ +[{Makefile,**.mk}] +indent_style = tab diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..2098b681 --- /dev/null +++ b/Makefile @@ -0,0 +1,17 @@ +ifeq ($(TAG),) +TAG=$(shell .circleci/get_tag.sh "stackrox-build") +endif +ifeq ($(ROCKSDB_TAG),) +ROCKSDB_TAG=$(shell .circleci/get_tag.sh "rocksdb") +endif + +.PHONY: stackrox-build-image +stackrox-build-image: + docker build images/ -f images/stackrox-build.Dockerfile \ + -t stackrox/apollo-ci:$(TAG) \ + --build-arg ROCKSDB_TAG=$(ROCKSDB_TAG) + +.PHONY: rocksdb-image +rocksdb-image: + docker build images/ -f images/centos8-rocksdb.Dockerfile \ + -t stackrox/apollo-ci:$(ROCKSDB_TAG) diff --git a/README.md b/README.md index 8cee8854..190b7381 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ [![CircleCI][circleci-badge]][circleci-link] [![Docker Hub][docker-badge]][docker-link] -# StackRox CI Base Images +# StackRox CI & Build Images -This repository holds the Dockerfiles for images used in StackRox CI builds. +This repository holds the Dockerfiles for images used in StackRox CI & builds. [circleci-badge]: https://circleci.com/gh/stackrox/rox-ci-image.svg?&style=shield&circle-token=f9c93b8793b8d77af175d0f34a200fe7261212d2 [circleci-link]: https://circleci.com/gh/stackrox/workflows/rox-ci-image/tree/master diff --git a/images/centos8-rocksdb.Dockerfile b/images/centos8-rocksdb.Dockerfile new file mode 100644 index 00000000..20d610fa --- /dev/null +++ b/images/centos8-rocksdb.Dockerfile @@ -0,0 +1,39 @@ +FROM quay.io/centos/centos:stream8 + +RUN yum update -y && \ + yum install -y epel-release dnf-plugins-core && \ + yum config-manager --set-enabled powertools && \ + yum -y groupinstall "Development Tools" && \ + yum install -y \ + bzip2-devel \ + libzstd-devel \ + lz4-devel \ + snappy-devel \ + wget \ + zlib-devel \ + && \ + yum clean all && \ + rm -rf /var/cache/yum + +# This compiles RocksDB without BMI and AVX2 instructions +ENV PORTABLE=1 TRY_SSE_ETC=0 TRY_SSE42="-msse4.2" TRY_PCLMUL="-mpclmul" CXXFLAGS="-fPIC" + +ARG ROCKSDB_VERSION="v6.7.3" +RUN mkdir -p /build && \ + cd /tmp && \ + git clone -b "${ROCKSDB_VERSION}" --depth 1 https://github.com/facebook/rocksdb.git && \ + cd rocksdb && \ + git ls-files -s | git hash-object --stdin >/build/ROCKSDB_HASH && \ + make static_lib + +RUN cd /tmp/rocksdb && \ + DEBUG_LEVEL=0 make ldb + +ARG UPX_VERSION=3.96 +ARG UPX_SHA256=ac75f5172c1c530d1b5ce7215ca9e94586c07b675a26af3b97f8421b8b8d413d +RUN url="https://github.com/upx/upx/releases/download/v${UPX_VERSION}/upx-${UPX_VERSION}-amd64_linux.tar.xz" && \ + wget --no-verbose -O upx.txz "$url" && \ + echo "${UPX_SHA256} *upx.txz" | sha256sum -c - && \ + tar -xJf upx.txz && \ + "upx-${UPX_VERSION}-amd64_linux/upx" -9 /tmp/rocksdb/ldb && \ + rm -rf upx.txz "upx-${UPX_VERSION}-amd64_linux" diff --git a/images/rocksdb.Dockerfile b/images/rocksdb.Dockerfile index 477786a0..b60cc89e 100644 --- a/images/rocksdb.Dockerfile +++ b/images/rocksdb.Dockerfile @@ -5,7 +5,6 @@ ARG BASE_UBUNTU_TAG FROM ubuntu:${BASE_UBUNTU_TAG} -ARG ROCKSDB_VERSION=v6.7.3 ENV PORTABLE=1 \ TRY_SSE_ETC=0 \ TRY_SSE42="-msse4.2" \ @@ -28,7 +27,10 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* \ && update-ca-certificates +ARG ROCKSDB_VERSION=v6.7.3 WORKDIR /tmp RUN git clone -b "${ROCKSDB_VERSION}" --depth 1 https://github.com/facebook/rocksdb.git WORKDIR /tmp/rocksdb -RUN make static_lib +RUN mkdir -p /build && \ + git ls-files -s | git hash-object --stdin >/build/ROCKSDB_HASH && \ + make static_lib diff --git a/images/stackrox-build.Dockerfile b/images/stackrox-build.Dockerfile new file mode 100644 index 00000000..40380547 --- /dev/null +++ b/images/stackrox-build.Dockerfile @@ -0,0 +1,49 @@ +ARG CENTOS8_ROCKSDB_TAG +FROM quay.io/rhacs-eng/apollo-ci:${CENTOS8_ROCKSDB_TAG} as builder + +FROM quay.io/centos/centos:stream8 + +RUN yum update -y && \ + yum install -y epel-release dnf-plugins-core && \ + yum config-manager --set-enabled powertools && \ + yum -y groupinstall "Development Tools" && \ + yum install -y \ + bzip2-devel \ + libzstd-devel \ + lz4-devel \ + snappy-devel \ + wget \ + zlib-devel \ + && \ + yum clean all && \ + rm -rf /var/cache/yum + +ARG GOLANG_VERSION=1.17.2 +ARG GOLANG_SHA256=f242a9db6a0ad1846de7b6d94d507915d14062660616a61ef7c808a76e4f1676 +ENV GOPATH /go +ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH +RUN url="https://dl.google.com/go/go${GOLANG_VERSION}.linux-amd64.tar.gz" && \ + wget --no-verbose -O go.tgz "$url" && \ + echo "${GOLANG_SHA256} *go.tgz" | sha256sum -c - && \ + tar -C /usr/local -xzf go.tgz && \ + rm go.tgz && \ + mkdir -p "$GOPATH/src" "$GOPATH/bin" && \ + chmod -R 777 "$GOPATH" + +RUN wget --quiet -O - https://rpm.nodesource.com/setup_lts.x | bash - && \ + wget --quiet -O - https://dl.yarnpkg.com/rpm/yarn.repo | tee /etc/yum.repos.d/yarn.repo && \ + yum update -y && \ + yum install -y nodejs yarn && \ + yum clean all && \ + rm -rf /var/cache/yum + +COPY --from=builder /tmp/rocksdb/librocksdb.a /lib/rocksdb/librocksdb.a +COPY --from=builder /tmp/rocksdb/include /lib/rocksdb/include +COPY --from=builder /tmp/rocksdb/ldb /usr/local/bin/ldb + +ENV CGO_CFLAGS="-I/lib/rocksdb/include" +ENV CGO_LDFLAGS="-L/lib/rocksdb -lrocksdb -lstdc++ -lm -lz -lbz2 -lsnappy -llz4 -lzstd" +ENV CGO_ENABLED=1 +ENV GOCACHE="/linux-gocache" + +WORKDIR /go/src/github.com/stackrox/rox diff --git a/images/static-contents/etc/yum.repos.d/google-cloud-sdk.repo b/images/static-contents/etc/yum.repos.d/google-cloud-sdk.repo new file mode 100644 index 00000000..c53061de --- /dev/null +++ b/images/static-contents/etc/yum.repos.d/google-cloud-sdk.repo @@ -0,0 +1,8 @@ +[google-cloud-sdk] +name=Google Cloud SDK +baseurl=https://packages.cloud.google.com/yum/repos/cloud-sdk-el8-x86_64 +enabled=1 +gpgcheck=1 +repo_gpgcheck=0 +gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg + https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg diff --git a/test/bats/get_tag.bats b/test/bats/get_tag.bats index 4dafd8ba..517cadb3 100644 --- a/test/bats/get_tag.bats +++ b/test/bats/get_tag.bats @@ -13,41 +13,33 @@ setup() { [ "$status" -eq 1 ] } -@test 'appends git describe' { - run .circleci/get_tag.sh something - [ "$status" -eq 0 ] - [[ "$output" =~ $describe$ ]] +@test "expects an image flavor value" { + run .circleci/get_tag.sh "" + [ "$status" -eq 1 ] } -@test 'adds image flavor' { +@test 'appends git describe to flavor' { run .circleci/get_tag.sh something [ "$status" -eq 0 ] - [[ "$output" == "snapshot-something-$describe" ]] -} - -@test 'only adds image flavor when not ""' { - run .circleci/get_tag.sh "" - [ "$status" -eq 0 ] - [[ "$output" == "snapshot-$describe" ]] + [[ "$output" == "something-$describe" ]] } -@test 'prepends snapshot on PRs' { - export CIRCLE_BRANCH=a-pr - run .circleci/get_tag.sh something +@test 'omits flavor for rox' { + run .circleci/get_tag.sh rox [ "$status" -eq 0 ] - [[ "$output" == "snapshot-something-$describe" ]] + [[ "$output" == "$describe" ]] } -@test 'does not prepend snapshot on master' { - export CIRCLE_BRANCH=master - run .circleci/get_tag.sh something +@test 'uses HASH for rocksdb' { + local hash="rocksdb-$(git hash-object images/rocksdb.Dockerfile)" + run .circleci/get_tag.sh rocksdb [ "$status" -eq 0 ] - [[ "$output" == "something-$describe" ]] + [[ "$output" == "$hash" ]] } -@test 'does not prepend snapshot on tags' { - export CIRCLE_TAG=1.2.3 - run .circleci/get_tag.sh something +@test 'uses HASH for centos8 rocksdb' { + local hash="centos8-rocksdb-$(git hash-object images/centos8-rocksdb.Dockerfile)" + run .circleci/get_tag.sh centos8-rocksdb [ "$status" -eq 0 ] - [[ "$output" == "something-$describe" ]] + [[ "$output" == "$hash" ]] }