From b187df36b22aaf9c85813cd9d7a6fa878dca323f Mon Sep 17 00:00:00 2001 From: erikzaadi Date: Tue, 15 Oct 2024 15:45:08 +0300 Subject: [PATCH 1/2] No more github action cache for docker builds --- .github/workflows/ci.yml | 22 ++++++++++++++++------ .github/workflows/release-integrations.yml | 4 +--- integrations/_infra/Dockerfile.Deb | 3 ++- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ea47a71ffa..f3ede6d70b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,7 +39,8 @@ jobs: build-integration: - runs-on: ubuntu-latest + # runs-on: ${{ matrix.platform == 'linux/arm64' && 'macos-13' || 'ubuntu-latest' }} + runs-on: 'ubuntu-latest' if: needs.prepare-matrix.outputs.matrix != '[]' outputs: is_dev_version: ${{ steps.prepare_tags.outputs.is_dev_version }} @@ -57,6 +58,10 @@ jobs: - name: Check out code uses: actions/checkout@v4 + # - name: Setup docker (missing on MacOS) + # if: matrix.platform == 'linux/arm64' + # uses: douglascamata/setup-docker-macos-action@v1-alpha + - name: Set up QEMU uses: docker/setup-qemu-action@v3 with: @@ -106,10 +111,17 @@ jobs: run: | echo "base_image=$(cat ${{ steps.prepare_tags.outputs.dockerfile_path }} | head -n 1 | awk -F '=' '{print $2}' )" >> $GITHUB_OUTPUT - - name: Cache Docker images - uses: ScribeMD/docker-cache@0.5.0 + - name: Login to Docker Hub + uses: docker/login-action@v3 with: - key: docker-${{ matrix.integration }}-${{ steps.get-docker-image.outputs.base_image }}-${{ matrix.platform }} + registry: ghcr.io + username: ${{ secrets.DOCKER_MACHINE_USER }} + password: ${{ secrets.DOCKER_MACHINE_TOKEN }} + + # - name: Cache Docker images + # uses: ScribeMD/docker-cache@0.5.0 + # with: + # key: docker-${{ matrix.integration }}-${{ steps.get-docker-image.outputs.base_image }}-${{ matrix.platform }} - name: Build uses: docker/build-push-action@v6 @@ -119,8 +131,6 @@ jobs: platforms: ${{ matrix.platform }} push: false load: true - cache-from: type=gha - cache-to: type=gha,mode=max tags: ${{ steps.prepare_tags.outputs.tags }} build-args: | BUILD_CONTEXT=${{ steps.prepare_tags.outputs.context_dir }} diff --git a/.github/workflows/release-integrations.yml b/.github/workflows/release-integrations.yml index f79a312754..4f01cd4bd5 100644 --- a/.github/workflows/release-integrations.yml +++ b/.github/workflows/release-integrations.yml @@ -57,7 +57,7 @@ jobs: needs: [prepare-matrix] strategy: # limit the number of parallel jobs to avoid hitting the ghcr.io rate limit - max-parallel: 5 + max-parallel: 10 matrix: integration: ${{fromJson(needs.prepare-matrix.outputs.matrix)}} steps: @@ -115,8 +115,6 @@ jobs: file: ${{ steps.prepare_tags.outputs.dockerfile_path }} platforms: linux/amd64,linux/arm64 push: true - cache-from: type=gha - cache-to: type=gha,mode=max tags: ${{ steps.prepare_tags.outputs.tags }} build-args: | BUILD_CONTEXT=${{ steps.prepare_tags.outputs.context_dir }} diff --git a/integrations/_infra/Dockerfile.Deb b/integrations/_infra/Dockerfile.Deb index 9c881105a5..eebce4cda5 100644 --- a/integrations/_infra/Dockerfile.Deb +++ b/integrations/_infra/Dockerfile.Deb @@ -12,6 +12,7 @@ ENV LIBRDKAFKA_VERSION=1.9.2 \ RUN apt-get update \ && apt-get install -y \ + --no-install-recommends \ wget \ g++ \ libssl-dev \ @@ -45,7 +46,7 @@ LABEL org.opencontainers.image.source=https://github.com/port-labs/ocean ENV PIP_ROOT_USER_ACTION=ignore RUN apt-get update \ - && apt-get install -y librdkafka-dev python3 \ + && apt-get install -y --no-install-recommends librdkafka-dev python3 \ && apt-get clean WORKDIR /app From 95003b9ecb48bd81b4d4fd593c43d96a9de12ec7 Mon Sep 17 00:00:00 2001 From: erikzaadi Date: Sun, 20 Oct 2024 12:42:13 +0300 Subject: [PATCH 2/2] Separate builder and runner image --- .../actions/build-docker-image/action.yml | 81 +++++++++++++++++++ .github/workflows/build-infra-images.yml | 34 ++++++++ .github/workflows/ci.yml | 52 ++---------- .github/workflows/detect-changes-matrix.yml | 9 +++ .../workflows/docker-images-security-scan.yml | 50 +++++------- .github/workflows/release-integrations.yml | 32 ++------ integrations/_infra/Dockerfile.Deb | 27 +------ integrations/_infra/Dockerfile.base.builder | 26 ++++++ integrations/_infra/Dockerfile.base.runner | 13 +++ 9 files changed, 203 insertions(+), 121 deletions(-) create mode 100644 .github/workflows/actions/build-docker-image/action.yml create mode 100644 .github/workflows/build-infra-images.yml create mode 100644 integrations/_infra/Dockerfile.base.builder create mode 100644 integrations/_infra/Dockerfile.base.runner diff --git a/.github/workflows/actions/build-docker-image/action.yml b/.github/workflows/actions/build-docker-image/action.yml new file mode 100644 index 0000000000..c8e622e24f --- /dev/null +++ b/.github/workflows/actions/build-docker-image/action.yml @@ -0,0 +1,81 @@ +name: Build docker images +description: Build Docker Images +# NOTE: In composite actions, all parameters are strings, +# thus flags are simply checked by being non empty strings, +# where there the default is an empty string +inputs: + dockerfile: + description: Dockerfile to build + required: true + tags: + description: Docker tags to publish + required: true + platforms: + description: Platforms to build (csv) + required: false + default: 'linux/arm64,linux/amd64' + test: + description: Test command to run on the created image (Optional) + required: false + default: '' + build-args: + description: Explicit docker build-args + required: false + default: '' + skip-init: + description: Skip docker init (if ran after another invocation of this action) + required: false + default: '' + docker-user: + required: true + description: Docker Hub User + docker-password: + required: true + description: Docker Hub User + skip-push: + required: false + description: Optionally skip push + default: '' + load-created-image: + required: false + description: Optionally load created docker image + default: '' + +runs: + using: 'composite' + steps: + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + if: ${{ inputs.skip-init == '' }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + if: ${{ inputs.skip-init == '' }} + + - name: Login to Docker Hub + uses: docker/login-action@v3 + if: ${{ inputs.skip-init == '' }} + with: + registry: ghcr.io + username: ${{ inputs.docker-user }} + password: ${{ inputs.docker-password }} + + - name: Build Runner Image + uses: docker/build-push-action@v6 + with: + context: . + file: ${{ inputs.dockerfile }} + platforms: ${{ inputs.platforms }} + push: ${{ inputs.skip-push == '' }} + load: ${{ inputs.test != '' || inputs.load-created-image != '' }} + tags: ${{ inputs.tags }} + build-args: | + ${{ inputs.build-args }} + + - name: Verify Built Image + shell: bash + if: ${{ inputs.test != '' }} + run: | + SINGLE_TAG=$(echo "${{ inputs.tags }}" | awk -F ',' '{print $1};' ) + SINGLE_PLATFORM=$(echo "${{ inputs.platforms }}" | awk -F ',' '{print $1};' ) + docker run --platform "${SINGLE_PLATFORM}" --rm --entrypoint bash "${SINGLE_TAG}" -c '${{ inputs.test }}' diff --git a/.github/workflows/build-infra-images.yml b/.github/workflows/build-infra-images.yml new file mode 100644 index 0000000000..58603c7ea7 --- /dev/null +++ b/.github/workflows/build-infra-images.yml @@ -0,0 +1,34 @@ +name: Build infra images +on: + pull_request: + workflow_dispatch: + +jobs: + detect-changes: + uses: ./.github/workflows/detect-changes-matrix.yml + build-infra: + runs-on: 'ubuntu-latest' + needs: detect-changes + if: ${{ needs.detect-changes.outputs.infra == 'true' }} + steps: + - name: Check out code + uses: actions/checkout@v4 + + - name: Build Docker Image + uses: ./.github/workflows/actions/build-docker-image + with: + dockerfile: ./integrations/_infra/Dockerfile.base.builder + platforms: linux/amd64,linux/arm64 + tags: ghcr.io/port-labs/port-ocean-base-builder:latest + docker-user: ${{ secrets.DOCKER_MACHINE_USER }} + docker-password: ${{ secrets.DOCKER_MACHINE_TOKEN }} + + - name: Build Docker Image + uses: ./.github/workflows/actions/build-docker-image + with: + dockerfile: ./integrations/_infra/Dockerfile.base.runner + platforms: linux/amd64,linux/arm64 + tags: ghcr.io/port-labs/port-ocean-base-runner:latest + docker-user: ${{ secrets.DOCKER_MACHINE_USER }} + docker-password: ${{ secrets.DOCKER_MACHINE_TOKEN }} + skip-init: 'true' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f3ede6d70b..7afc442208 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,10 +37,8 @@ jobs: echo $(echo ${integrations_to_build[@]} | jq -R -c 'split(" ")') echo "INTEGRATIONS_MATRIX=$(echo ${integrations_to_build[@]} | jq -R -c 'split(" ")')" >> $GITHUB_OUTPUT - build-integration: - # runs-on: ${{ matrix.platform == 'linux/arm64' && 'macos-13' || 'ubuntu-latest' }} - runs-on: 'ubuntu-latest' + runs-on: ubuntu-latest if: needs.prepare-matrix.outputs.matrix != '[]' outputs: is_dev_version: ${{ steps.prepare_tags.outputs.is_dev_version }} @@ -58,20 +56,6 @@ jobs: - name: Check out code uses: actions/checkout@v4 - # - name: Setup docker (missing on MacOS) - # if: matrix.platform == 'linux/arm64' - # uses: douglascamata/setup-docker-macos-action@v1-alpha - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - with: - platforms: ${{ matrix.platform }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - with: - platforms: ${{ matrix.platform }} - - name: Prepare Docker images tags id: prepare_tags run: | @@ -106,37 +90,15 @@ jobs: echo "is_dev_version=false" >> $GITHUB_OUTPUT fi - - name: Get used docker base image - id: get-docker-image - run: | - echo "base_image=$(cat ${{ steps.prepare_tags.outputs.dockerfile_path }} | head -n 1 | awk -F '=' '{print $2}' )" >> $GITHUB_OUTPUT - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ secrets.DOCKER_MACHINE_USER }} - password: ${{ secrets.DOCKER_MACHINE_TOKEN }} - - # - name: Cache Docker images - # uses: ScribeMD/docker-cache@0.5.0 - # with: - # key: docker-${{ matrix.integration }}-${{ steps.get-docker-image.outputs.base_image }}-${{ matrix.platform }} - - - name: Build - uses: docker/build-push-action@v6 + - name: Build Docker Image + uses: ./.github/workflows/actions/build-docker-image with: - context: . - file: ${{ steps.prepare_tags.outputs.dockerfile_path }} + dockerfile: ${{ steps.prepare_tags.outputs.dockerfile_path }} platforms: ${{ matrix.platform }} - push: false - load: true tags: ${{ steps.prepare_tags.outputs.tags }} build-args: | BUILD_CONTEXT=${{ steps.prepare_tags.outputs.context_dir }} INTEGRATION_VERSION=${{ steps.prepare_tags.outputs.version }} - - - name: Verify Built Image - run: | - SINGLE_TAG=$(echo "${{ steps.prepare_tags.outputs.tags }}" | awk -F ',' '{print $1};' ) - docker run --platform ${{ matrix.platform }} --rm --entrypoint bash "${SINGLE_TAG}" -c 'ocean version' + docker-user: ${{ secrets.DOCKER_MACHINE_USER }} + docker-password: ${{ secrets.DOCKER_MACHINE_TOKEN }} + skip-push: 'true' diff --git a/.github/workflows/detect-changes-matrix.yml b/.github/workflows/detect-changes-matrix.yml index 91dd9c31fc..4973fe16c9 100644 --- a/.github/workflows/detect-changes-matrix.yml +++ b/.github/workflows/detect-changes-matrix.yml @@ -11,6 +11,9 @@ on: core: value: ${{ jobs.detect-changes.outputs.core }} description: "Determine if any core changes per git commit changes" + infra: + value: ${{ jobs.detect-changes.outputs.infra }} + description: "Determine if any changes to docker infra" jobs: detect-changes: @@ -20,6 +23,7 @@ jobs: matrix: ${{ steps.set-all-matrix.outputs.matrix }} integrations: ${{ steps.set-all-matrix.outputs.integrations }} core: ${{ steps.set-all-matrix.outputs.core }} + infra: ${{ steps.set-all-matrix.outputs.infra}} steps: - name: Checkout Repo uses: actions/checkout@v4 @@ -42,6 +46,8 @@ jobs: - 'integrations/**' - '!integrations/**/*.md' - '!integrations/_infra/*' + infra: + - 'integrations/_infra/*' - name: Set integrations and all matrix id: set-all-matrix @@ -50,8 +56,11 @@ jobs: HAS_CORE=${{ steps.changed-files.outputs.core_all_changed_files != '[]' }} echo "Core changes : ${HAS_CORE}" MATRIX=$(node -e "integrations=${INTEGRATIONS}; hasCore=${HAS_CORE}; console.log(JSON.stringify(hasCore ? integrations.concat(['.']) : integrations))") + HAS_INFRA=${{ steps.changed-files.outputs.infra_all_changed_files != '[]' }} + echo "Infra changes : ${HAS_INFRA}" echo "Integration changes : ${INTEGRATIONS}" echo "All changes : ${MATRIX}" echo "core=${HAS_CORE}" >> $GITHUB_OUTPUT echo "integrations=${INTEGRATIONS}" >> $GITHUB_OUTPUT echo "matrix=${MATRIX}" >> $GITHUB_OUTPUT + echo "infra=${HAS_INFRA}" >> $GITHUB_OUTPUT diff --git a/.github/workflows/docker-images-security-scan.yml b/.github/workflows/docker-images-security-scan.yml index 13d78e5443..61bdef898e 100644 --- a/.github/workflows/docker-images-security-scan.yml +++ b/.github/workflows/docker-images-security-scan.yml @@ -8,33 +8,34 @@ on: # This is a bit annoying, there's no real way to display the integrations dynamically in a dropdown for the action dispatcher options: - all + - argocd - aws + - azure - azure-devops + - backstage + - datadog - dynatrace - fake-integration - - gcp - - jenkins - - kafka - - launchdarkly - - newrelic - - opencost - - pagerduty - - servicenow - - sonarqube - - terraform-cloud - - argocd - - azure - - datadog - firehydrant + - gcp - gitlab + - jenkins - jira + - kafka - kubecost + - launchdarkly - linear + - newrelic - octopus + - opencost - opsgenie + - pagerduty - sentry + - servicenow - snyk + - sonarqube - statuspage + - terraform-cloud - wiz jobs: @@ -77,14 +78,6 @@ jobs: - name: Checkout Repo uses: actions/checkout@v4 - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - with: - platforms: linux/amd64,linux/arm64 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Extract version and image tag id: enrich-version run: | @@ -97,17 +90,16 @@ jobs: echo "identifier=${IDENTIFIER}" >> ${GITHUB_OUTPUT} echo "image_tag=${IMAGE_FULL_TAG}" >> ${GITHUB_OUTPUT} - - name: Build - uses: docker/build-push-action@v6 + - name: Build Docker Image + uses: ./.github/workflows/actions/build-docker-image with: - context: . - file: ./integrations/_infra/Dockerfile + dockerfile: ./integrations/_infra/Dockerfile platforms: linux/amd64 - push: false + skip-push: 'true' tags: ${{ steps.enrich-version.outputs.image_tag }} - load: true - cache-from: type=gha - cache-to: type=gha,mode=max + load-created-image: 'true' + docker-user: ${{ secrets.DOCKER_MACHINE_USER }} + docker-password: ${{ secrets.DOCKER_MACHINE_TOKEN }} build-args: | BUILD_CONTEXT=./integrations/${{ steps.enrich-version.outputs.integration }} INTEGRATION_VERSION=${{ steps.enrich-version.outputs.version }} diff --git a/.github/workflows/release-integrations.yml b/.github/workflows/release-integrations.yml index 4f01cd4bd5..8047414391 100644 --- a/.github/workflows/release-integrations.yml +++ b/.github/workflows/release-integrations.yml @@ -13,12 +13,7 @@ jobs: steps: - name: Check out code uses: actions/checkout@v4 - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ secrets.DOCKER_MACHINE_USER }} - password: ${{ secrets.DOCKER_MACHINE_TOKEN }} + - name: Prepare matrix id: prepare-matrix run: | @@ -57,24 +52,13 @@ jobs: needs: [prepare-matrix] strategy: # limit the number of parallel jobs to avoid hitting the ghcr.io rate limit - max-parallel: 10 + max-parallel: 5 matrix: integration: ${{fromJson(needs.prepare-matrix.outputs.matrix)}} steps: - name: Check out code uses: actions/checkout@v4 - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - with: - platforms: linux/amd64,linux/arm64 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ secrets.DOCKER_MACHINE_USER }} - password: ${{ secrets.DOCKER_MACHINE_TOKEN }} + - name: Prepare Docker images tags id: prepare_tags run: | @@ -108,17 +92,17 @@ jobs: echo "is_dev_version=false" >> $GITHUB_OUTPUT fi - - name: Build and push - uses: docker/build-push-action@v6 + - name: Build Docker Image + uses: ./.github/workflows/actions/build-docker-image with: - context: . - file: ${{ steps.prepare_tags.outputs.dockerfile_path }} + dockerfile: ${{ steps.prepare_tags.outputs.dockerfile_path }} platforms: linux/amd64,linux/arm64 - push: true tags: ${{ steps.prepare_tags.outputs.tags }} build-args: | BUILD_CONTEXT=${{ steps.prepare_tags.outputs.context_dir }} INTEGRATION_VERSION=${{ steps.prepare_tags.outputs.version }} + docker-user: ${{ secrets.DOCKER_MACHINE_USER }} + docker-password: ${{ secrets.DOCKER_MACHINE_TOKEN }} upload-specs: runs-on: ubuntu-latest diff --git a/integrations/_infra/Dockerfile.Deb b/integrations/_infra/Dockerfile.Deb index eebce4cda5..5cd9cfba0e 100644 --- a/integrations/_infra/Dockerfile.Deb +++ b/integrations/_infra/Dockerfile.Deb @@ -1,6 +1,7 @@ -ARG BASE_PYTHON_IMAGE=debian:trixie-slim +ARG BASE_BUILDER_PYTHON_IMAGE=ghcr.io/port-labs/port-ocean-base-builder:latest +ARG BASE_RUNNER_PYTHON_IMAGE=ghcr.io/port-labs/port-ocean-base-runner:latest -FROM ${BASE_PYTHON_IMAGE} AS base +FROM ${BASE_BUILDER_PYTHON_IMAGE} AS base ARG BUILD_CONTEXT ARG BUILDPLATFORM @@ -10,29 +11,13 @@ ENV LIBRDKAFKA_VERSION=1.9.2 \ POETRY_VIRTUALENVS_IN_PROJECT=1 \ PIP_ROOT_USER_ACTION=ignore -RUN apt-get update \ - && apt-get install -y \ - --no-install-recommends \ - wget \ - g++ \ - libssl-dev \ - autoconf \ - automake \ - libtool \ - curl \ - librdkafka-dev \ - python3 \ - python3-pip \ - python3-poetry \ - && apt-get clean - WORKDIR /app COPY ./${BUILD_CONTEXT}/pyproject.toml ./${BUILD_CONTEXT}/poetry.lock /app/ RUN poetry install --without dev --no-root --no-interaction --no-ansi --no-cache -FROM ${BASE_PYTHON_IMAGE} AS prod +FROM ${BASE_RUNNER_PYTHON_IMAGE} AS prod ARG INTEGRATION_VERSION ARG BUILD_CONTEXT @@ -45,10 +30,6 @@ LABEL org.opencontainers.image.source=https://github.com/port-labs/ocean ENV PIP_ROOT_USER_ACTION=ignore -RUN apt-get update \ - && apt-get install -y --no-install-recommends librdkafka-dev python3 \ - && apt-get clean - WORKDIR /app # Copy the application code diff --git a/integrations/_infra/Dockerfile.base.builder b/integrations/_infra/Dockerfile.base.builder new file mode 100644 index 0000000000..a2ade3fe93 --- /dev/null +++ b/integrations/_infra/Dockerfile.base.builder @@ -0,0 +1,26 @@ +ARG BASE_PYTHON_IMAGE=debian:trixie-slim +# debian:trixie-slim - Python 3.12 +FROM ${BASE_PYTHON_IMAGE} + +LABEL org.opencontainers.image.source=https://github.com/port-labs/ocean + +ENV LIBRDKAFKA_VERSION=1.9.2 \ + PYTHONUNBUFFERED=1 \ + POETRY_VIRTUALENVS_IN_PROJECT=1 \ + PIP_ROOT_USER_ACTION=ignore + +RUN apt-get update \ + && apt-get install -y \ + --no-install-recommends \ + wget \ + g++ \ + libssl-dev \ + autoconf \ + automake \ + libtool \ + curl \ + librdkafka-dev \ + python3 \ + python3-pip \ + python3-poetry \ + && apt-get clean diff --git a/integrations/_infra/Dockerfile.base.runner b/integrations/_infra/Dockerfile.base.runner new file mode 100644 index 0000000000..61116df843 --- /dev/null +++ b/integrations/_infra/Dockerfile.base.runner @@ -0,0 +1,13 @@ +ARG BASE_PYTHON_IMAGE=debian:trixie-slim +# debian:trixie-slim - Python 3.12 +FROM ${BASE_PYTHON_IMAGE} + +LABEL org.opencontainers.image.source=https://github.com/port-labs/ocean + +ENV LIBRDKAFKA_VERSION=1.9.2 + +ENV PIP_ROOT_USER_ACTION=ignore + +RUN apt-get update \ + && apt-get install -y --no-install-recommends librdkafka-dev python3 \ + && apt-get clean