From c546ab93021d5e5ba375b36812f6464e37c98a8f Mon Sep 17 00:00:00 2001 From: matt durham Date: Wed, 14 Feb 2024 16:31:54 -0500 Subject: [PATCH 1/9] Add build image for windows boringcrypto executable. --- .drone/drone.yml | 36 ++++++++- .drone/pipelines/build_images.jsonnet | 23 ++++++ .drone/util/build_image.jsonnet | 1 + build-image/boringcrypto/Dockerfile | 104 ++++++++++++++++++++++++++ 4 files changed, 163 insertions(+), 1 deletion(-) create mode 100644 build-image/boringcrypto/Dockerfile diff --git a/.drone/drone.yml b/.drone/drone.yml index 4a55da51c8e5..1397ef01777c 100644 --- a/.drone/drone.yml +++ b/.drone/drone.yml @@ -34,6 +34,40 @@ volumes: name: docker --- kind: pipeline +name: Create Linux build image for boringcrypto +platform: + arch: amd64 + os: linux +steps: +- commands: + - export IMAGE_TAG=${DRONE_TAG##build-image/v}-boringcrypto + - docker login -u $DOCKER_LOGIN -p $DOCKER_PASSWORD + - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + - docker buildx create --name multiarch --driver docker-container --use + - docker buildx build --push --platform linux/amd64,linux/arm64 -t grafana/agent-build-image:$IMAGE_TAG + ./build-image/boringcrypto + environment: + DOCKER_LOGIN: + from_secret: docker_login + DOCKER_PASSWORD: + from_secret: docker_password + image: docker + name: Build + volumes: + - name: docker + path: /var/run/docker.sock +trigger: + event: + - tag + ref: + - refs/tags/build-image/v* +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +--- +kind: pipeline name: Create Windows build image platform: arch: amd64 @@ -1317,6 +1351,6 @@ kind: secret name: updater_private_key --- kind: signature -hmac: a77b4f7b2708cc4417edd42fae39c307abcaec2c4f0f6176212665133bd20d5d +hmac: d66a28c3489b7a19b64e82590630ceed4fc71789a76b3da8564c2aae604253ad ... diff --git a/.drone/pipelines/build_images.jsonnet b/.drone/pipelines/build_images.jsonnet index 6df4f8f6cd2c..93fe4ff69130 100644 --- a/.drone/pipelines/build_images.jsonnet +++ b/.drone/pipelines/build_images.jsonnet @@ -40,6 +40,29 @@ local locals = { host: { path: '/var/run/docker.sock' }, }], }, + pipelines.linux('Create Linux build image for boringcrypto') { + trigger: locals.on_build_image_tag, + steps: [{ + name: 'Build', + image: 'docker', + volumes: [{ + name: 'docker', + path: '/var/run/docker.sock', + }], + environment: locals.docker_environment, + commands: [ + 'export IMAGE_TAG=${DRONE_TAG##build-image/v}-boringcrypto', + 'docker login -u $DOCKER_LOGIN -p $DOCKER_PASSWORD', + 'docker run --rm --privileged multiarch/qemu-user-static --reset -p yes', + 'docker buildx create --name multiarch --driver docker-container --use', + 'docker buildx build --push --platform linux/amd64,linux/arm64 -t grafana/agent-build-image:$IMAGE_TAG ./build-image/boringcrypto', + ], + }], + volumes: [{ + name: 'docker', + host: { path: '/var/run/docker.sock' }, + }], + }, pipelines.windows('Create Windows build image') { trigger: locals.on_build_image_tag, diff --git a/.drone/util/build_image.jsonnet b/.drone/util/build_image.jsonnet index 3d9f5d8a7dc3..a06704f2ef01 100644 --- a/.drone/util/build_image.jsonnet +++ b/.drone/util/build_image.jsonnet @@ -3,4 +3,5 @@ linux: 'grafana/agent-build-image:%s' % version, windows: 'grafana/agent-build-image:%s-windows' % version, + boringcrypto: 'grafana/agent-build-image:%s-boringcrytpo' % version, } diff --git a/build-image/boringcrypto/Dockerfile b/build-image/boringcrypto/Dockerfile new file mode 100644 index 000000000000..b6bcbabe5915 --- /dev/null +++ b/build-image/boringcrypto/Dockerfile @@ -0,0 +1,104 @@ +# syntax=docker/dockerfile:1.4 + +# NOTE: This Dockerfile can only be built using BuildKit. BuildKit is used by +# default when running `docker buildx build` or when DOCKER_BUILDKIT=1 is set +# in environment variables. + +# +# Dependencies +# +# We retrieve many of our dependencies by using various smaller containers. +# + +# Dependency: golangci-lint (for linting) +FROM alpine as golangci +RUN wget -O- -nv https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s v1.54.2 + +# Dependency: docker (for building images) +FROM alpine:3.17 as docker +RUN apk add --no-cache docker-cli docker-cli-buildx + +# Dependency: helm +FROM alpine:3.17 as helm +RUN apk add --no-cache helm + +# Dependency: Go and Go dependencies +FROM mcr.microsoft.com/oss/go/microsoft/golang:1.22-bullseye as golang + +# Keep in sync with cmd/grafana-agent-operator/DEVELOPERS.md +ENV CONTROLLER_GEN_VERSION v0.9.2 + +RUN go install sigs.k8s.io/controller-tools/cmd/controller-gen@$CONTROLLER_GEN_VERSION \ + && go install github.com/mitchellh/gox@v1.0.1 \ + && go install github.com/tcnksm/ghr@v0.15.0 \ + && go install github.com/grafana/tanka/cmd/tk@v0.22.1 \ + && go install github.com/jsonnet-bundler/jsonnet-bundler/cmd/jb@v0.5.1 \ + && go install github.com/google/go-jsonnet/cmd/jsonnet@v0.18.0 \ + && go install github.com/golang/protobuf/protoc-gen-go@v1.3.1 \ + && go install github.com/gogo/protobuf/protoc-gen-gogoslick@v1.3.0 \ + && go install github.com/gogo/protobuf/gogoproto/...@v1.3.0 \ + && go install github.com/ahmetb/gen-crd-api-reference-docs@v0.3.1-0.20220618162802-424739b250f5 \ + && go install github.com/norwoodj/helm-docs/cmd/helm-docs@v1.11.0 + +# +# Final image +# + +# rfratto/viceroy contains C cross compilers can be used for our Cgo +# dependencies. +FROM rfratto/viceroy:v0.4.0 + +# Install NodeJS LTS. This is needed because the most recent version of NodeJS +# from official Debian packages is v12, and we need LTS version v16. +# +# This must be done before installing other dependencies, otherwise nodesource +# will fail on installing NodeJS for all platforms instead of just our host +# platform. +# Source: https://github.com/nodesource/distributions#installation-instructions +RUN < Date: Wed, 14 Feb 2024 16:42:36 -0500 Subject: [PATCH 2/9] Update Dockerfile --- build-image/boringcrypto/Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build-image/boringcrypto/Dockerfile b/build-image/boringcrypto/Dockerfile index b6bcbabe5915..6352bcb71f00 100644 --- a/build-image/boringcrypto/Dockerfile +++ b/build-image/boringcrypto/Dockerfile @@ -1,5 +1,8 @@ # syntax=docker/dockerfile:1.4 +# NOTE: The main difference in this file vs the parent Dockerfile is using mcr.microsoft.com/oss/go/microsoft/golang:1.22-bullseye which is a microsoft +# fork of go that allows using windows crypto instead of boring crypto. Details at https://github.com/microsoft/go/tree/microsoft/main/eng/doc/fips + # NOTE: This Dockerfile can only be built using BuildKit. BuildKit is used by # default when running `docker buildx build` or when DOCKER_BUILDKIT=1 is set # in environment variables. From a699cbe1f8fd94cf2fa7b57d00e112596431d5a3 Mon Sep 17 00:00:00 2001 From: mattdurham Date: Wed, 14 Feb 2024 16:42:55 -0500 Subject: [PATCH 3/9] Update build_image.jsonnet --- .drone/util/build_image.jsonnet | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone/util/build_image.jsonnet b/.drone/util/build_image.jsonnet index a06704f2ef01..4d4f2ec1d4a7 100644 --- a/.drone/util/build_image.jsonnet +++ b/.drone/util/build_image.jsonnet @@ -3,5 +3,5 @@ linux: 'grafana/agent-build-image:%s' % version, windows: 'grafana/agent-build-image:%s-windows' % version, - boringcrypto: 'grafana/agent-build-image:%s-boringcrytpo' % version, + boringcrypto: 'grafana/agent-build-image:%s-boringcrypto' % version, } From 8ce168a395a406af6bcf8a878e9f056459991c3e Mon Sep 17 00:00:00 2001 From: matt durham Date: Fri, 16 Feb 2024 09:47:43 -0500 Subject: [PATCH 4/9] Use ARG instead of duplicate image --- .drone/drone.yml | 10 +-- .drone/pipelines/build_images.jsonnet | 2 +- build-image/Dockerfile | 6 +- build-image/boringcrypto/Dockerfile | 107 -------------------------- 4 files changed, 9 insertions(+), 116 deletions(-) delete mode 100644 build-image/boringcrypto/Dockerfile diff --git a/.drone/drone.yml b/.drone/drone.yml index 1397ef01777c..4238adadff30 100644 --- a/.drone/drone.yml +++ b/.drone/drone.yml @@ -44,8 +44,9 @@ steps: - docker login -u $DOCKER_LOGIN -p $DOCKER_PASSWORD - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - docker buildx create --name multiarch --driver docker-container --use - - docker buildx build --push --platform linux/amd64,linux/arm64 -t grafana/agent-build-image:$IMAGE_TAG - ./build-image/boringcrypto + - docker buildx build --build-arg="GO_RUNTIME=mcr.microsoft.com/oss/go/microsoft/golang:1.22-bullseye" + --push --platform linux/amd64,linux/arm64 -t grafana/agent-build-image:$IMAGE_TAG + ./build-image environment: DOCKER_LOGIN: from_secret: docker_login @@ -1349,8 +1350,3 @@ get: path: infra/data/ci/github/updater-app kind: secret name: updater_private_key ---- -kind: signature -hmac: d66a28c3489b7a19b64e82590630ceed4fc71789a76b3da8564c2aae604253ad - -... diff --git a/.drone/pipelines/build_images.jsonnet b/.drone/pipelines/build_images.jsonnet index 93fe4ff69130..868a0c25855b 100644 --- a/.drone/pipelines/build_images.jsonnet +++ b/.drone/pipelines/build_images.jsonnet @@ -55,7 +55,7 @@ local locals = { 'docker login -u $DOCKER_LOGIN -p $DOCKER_PASSWORD', 'docker run --rm --privileged multiarch/qemu-user-static --reset -p yes', 'docker buildx create --name multiarch --driver docker-container --use', - 'docker buildx build --push --platform linux/amd64,linux/arm64 -t grafana/agent-build-image:$IMAGE_TAG ./build-image/boringcrypto', + 'docker buildx build --build-arg="GO_RUNTIME=mcr.microsoft.com/oss/go/microsoft/golang:1.22-bullseye" --push --platform linux/amd64,linux/arm64 -t grafana/agent-build-image:$IMAGE_TAG ./build-image', ], }], volumes: [{ diff --git a/build-image/Dockerfile b/build-image/Dockerfile index 7bce2ddfde9e..708973b42512 100644 --- a/build-image/Dockerfile +++ b/build-image/Dockerfile @@ -4,6 +4,10 @@ # default when running `docker buildx build` or when DOCKER_BUILDKIT=1 is set # in environment variables. +# NOTE: The GO_RUNTIME is used to switch between the default google go runtime and mcr.microsoft.com/oss/go/microsoft/golang:1.22-bullseye which is a microsoft +# fork of go that allows using windows crypto instead of boring crypto. Details at https://github.com/microsoft/go/tree/microsoft/main/eng/doc/fips +ARG GO_RUNTIME=golang:1.22.0-bullseye + # # Dependencies # @@ -23,7 +27,7 @@ FROM alpine:3.17 as helm RUN apk add --no-cache helm # Dependency: Go and Go dependencies -FROM golang:1.22.0-bullseye as golang +FROM ${GO_RUNTIME} as golang # Keep in sync with cmd/grafana-agent-operator/DEVELOPERS.md ENV CONTROLLER_GEN_VERSION v0.9.2 diff --git a/build-image/boringcrypto/Dockerfile b/build-image/boringcrypto/Dockerfile deleted file mode 100644 index 6352bcb71f00..000000000000 --- a/build-image/boringcrypto/Dockerfile +++ /dev/null @@ -1,107 +0,0 @@ -# syntax=docker/dockerfile:1.4 - -# NOTE: The main difference in this file vs the parent Dockerfile is using mcr.microsoft.com/oss/go/microsoft/golang:1.22-bullseye which is a microsoft -# fork of go that allows using windows crypto instead of boring crypto. Details at https://github.com/microsoft/go/tree/microsoft/main/eng/doc/fips - -# NOTE: This Dockerfile can only be built using BuildKit. BuildKit is used by -# default when running `docker buildx build` or when DOCKER_BUILDKIT=1 is set -# in environment variables. - -# -# Dependencies -# -# We retrieve many of our dependencies by using various smaller containers. -# - -# Dependency: golangci-lint (for linting) -FROM alpine as golangci -RUN wget -O- -nv https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s v1.54.2 - -# Dependency: docker (for building images) -FROM alpine:3.17 as docker -RUN apk add --no-cache docker-cli docker-cli-buildx - -# Dependency: helm -FROM alpine:3.17 as helm -RUN apk add --no-cache helm - -# Dependency: Go and Go dependencies -FROM mcr.microsoft.com/oss/go/microsoft/golang:1.22-bullseye as golang - -# Keep in sync with cmd/grafana-agent-operator/DEVELOPERS.md -ENV CONTROLLER_GEN_VERSION v0.9.2 - -RUN go install sigs.k8s.io/controller-tools/cmd/controller-gen@$CONTROLLER_GEN_VERSION \ - && go install github.com/mitchellh/gox@v1.0.1 \ - && go install github.com/tcnksm/ghr@v0.15.0 \ - && go install github.com/grafana/tanka/cmd/tk@v0.22.1 \ - && go install github.com/jsonnet-bundler/jsonnet-bundler/cmd/jb@v0.5.1 \ - && go install github.com/google/go-jsonnet/cmd/jsonnet@v0.18.0 \ - && go install github.com/golang/protobuf/protoc-gen-go@v1.3.1 \ - && go install github.com/gogo/protobuf/protoc-gen-gogoslick@v1.3.0 \ - && go install github.com/gogo/protobuf/gogoproto/...@v1.3.0 \ - && go install github.com/ahmetb/gen-crd-api-reference-docs@v0.3.1-0.20220618162802-424739b250f5 \ - && go install github.com/norwoodj/helm-docs/cmd/helm-docs@v1.11.0 - -# -# Final image -# - -# rfratto/viceroy contains C cross compilers can be used for our Cgo -# dependencies. -FROM rfratto/viceroy:v0.4.0 - -# Install NodeJS LTS. This is needed because the most recent version of NodeJS -# from official Debian packages is v12, and we need LTS version v16. -# -# This must be done before installing other dependencies, otherwise nodesource -# will fail on installing NodeJS for all platforms instead of just our host -# platform. -# Source: https://github.com/nodesource/distributions#installation-instructions -RUN < Date: Fri, 16 Feb 2024 11:19:05 -0500 Subject: [PATCH 5/9] force the usage of arg --- .drone/drone.yml | 9 +++++++-- .drone/pipelines/build_images.jsonnet | 2 +- build-image/Dockerfile | 2 +- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.drone/drone.yml b/.drone/drone.yml index 4238adadff30..f521c097fe5f 100644 --- a/.drone/drone.yml +++ b/.drone/drone.yml @@ -10,8 +10,8 @@ steps: - docker login -u $DOCKER_LOGIN -p $DOCKER_PASSWORD - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - docker buildx create --name multiarch --driver docker-container --use - - docker buildx build --push --platform linux/amd64,linux/arm64 -t grafana/agent-build-image:$IMAGE_TAG - ./build-image + - docker buildx build --build-arg="GO_RUNTIME=golang:1.22.0-bullseye" --push --platform + linux/amd64,linux/arm64 -t grafana/agent-build-image:$IMAGE_TAG ./build-image environment: DOCKER_LOGIN: from_secret: docker_login @@ -1350,3 +1350,8 @@ get: path: infra/data/ci/github/updater-app kind: secret name: updater_private_key +--- +kind: signature +hmac: 2e439110a89f33a78d745a71635d47f9b1a99de6028bb84c258a0be9c09840f2 + +... diff --git a/.drone/pipelines/build_images.jsonnet b/.drone/pipelines/build_images.jsonnet index 868a0c25855b..328c7fc34443 100644 --- a/.drone/pipelines/build_images.jsonnet +++ b/.drone/pipelines/build_images.jsonnet @@ -32,7 +32,7 @@ local locals = { 'docker login -u $DOCKER_LOGIN -p $DOCKER_PASSWORD', 'docker run --rm --privileged multiarch/qemu-user-static --reset -p yes', 'docker buildx create --name multiarch --driver docker-container --use', - 'docker buildx build --push --platform linux/amd64,linux/arm64 -t grafana/agent-build-image:$IMAGE_TAG ./build-image', + 'docker buildx build --build-arg="GO_RUNTIME=golang:1.22.0-bullseye" --push --platform linux/amd64,linux/arm64 -t grafana/agent-build-image:$IMAGE_TAG ./build-image', ], }], volumes: [{ diff --git a/build-image/Dockerfile b/build-image/Dockerfile index 708973b42512..602e7cbd1aeb 100644 --- a/build-image/Dockerfile +++ b/build-image/Dockerfile @@ -6,7 +6,7 @@ # NOTE: The GO_RUNTIME is used to switch between the default google go runtime and mcr.microsoft.com/oss/go/microsoft/golang:1.22-bullseye which is a microsoft # fork of go that allows using windows crypto instead of boring crypto. Details at https://github.com/microsoft/go/tree/microsoft/main/eng/doc/fips -ARG GO_RUNTIME=golang:1.22.0-bullseye +ARG GO_RUNTIME=mustoverride # # Dependencies From 724d7a012e86f2864f73e29106d050d670a22961 Mon Sep 17 00:00:00 2001 From: matt durham Date: Fri, 16 Feb 2024 11:23:40 -0500 Subject: [PATCH 6/9] fix test step --- .github/workflows/check-linux-build-image.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/check-linux-build-image.yml b/.github/workflows/check-linux-build-image.yml index ca9201ede824..d5e78806c8f1 100644 --- a/.github/workflows/check-linux-build-image.yml +++ b/.github/workflows/check-linux-build-image.yml @@ -23,4 +23,6 @@ jobs: with: context: ./build-image push: false - tags: grafana/agent-build-image:latest \ No newline at end of file + tags: grafana/agent-build-image:latest + build-args: | + GO_RUNTIME=mcr.microsoft.com/oss/go/microsoft/golang:1.22-bullseye \ No newline at end of file From 58c81757c0898acade463a6e078135d77414835a Mon Sep 17 00:00:00 2001 From: matt durham Date: Fri, 16 Feb 2024 11:24:19 -0500 Subject: [PATCH 7/9] fix test step --- .github/workflows/check-linux-build-image.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/check-linux-build-image.yml b/.github/workflows/check-linux-build-image.yml index d5e78806c8f1..5ba307adadd8 100644 --- a/.github/workflows/check-linux-build-image.yml +++ b/.github/workflows/check-linux-build-image.yml @@ -19,6 +19,14 @@ jobs: uses: docker/setup-buildx-action@v3 - name: Create test Linux build image + uses: docker/build-push-action@v5 + with: + context: ./build-image + push: false + tags: grafana/agent-build-image:latest + build-args: | + GO_RUNTIME=golang:1.22.0-bullseyes + - name: Create test Linux build image for boring crypto uses: docker/build-push-action@v5 with: context: ./build-image From bca0507f584643cbaf7f80d66365af447d7df773 Mon Sep 17 00:00:00 2001 From: matt durham Date: Fri, 16 Feb 2024 11:24:31 -0500 Subject: [PATCH 8/9] fix test step --- .github/workflows/check-linux-build-image.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/check-linux-build-image.yml b/.github/workflows/check-linux-build-image.yml index 5ba307adadd8..ee3718f961e8 100644 --- a/.github/workflows/check-linux-build-image.yml +++ b/.github/workflows/check-linux-build-image.yml @@ -26,6 +26,7 @@ jobs: tags: grafana/agent-build-image:latest build-args: | GO_RUNTIME=golang:1.22.0-bullseyes + - name: Create test Linux build image for boring crypto uses: docker/build-push-action@v5 with: From 6f9dc03052b1c79e30592977c19fa25ee927fd78 Mon Sep 17 00:00:00 2001 From: matt durham Date: Fri, 16 Feb 2024 11:26:23 -0500 Subject: [PATCH 9/9] fix typo --- .github/workflows/check-linux-build-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-linux-build-image.yml b/.github/workflows/check-linux-build-image.yml index ee3718f961e8..32737b0b6afd 100644 --- a/.github/workflows/check-linux-build-image.yml +++ b/.github/workflows/check-linux-build-image.yml @@ -25,7 +25,7 @@ jobs: push: false tags: grafana/agent-build-image:latest build-args: | - GO_RUNTIME=golang:1.22.0-bullseyes + GO_RUNTIME=golang:1.22.0-bullseye - name: Create test Linux build image for boring crypto uses: docker/build-push-action@v5