From fa8c7955c193be16f8c3eeca092536d53e364524 Mon Sep 17 00:00:00 2001 From: Charles Coggins Date: Thu, 28 Sep 2023 15:29:43 -0500 Subject: [PATCH 1/5] feat!: add lockfile generation support The changes in this PR are born out of the research spike to find the best way to add lockfile generation support to the `phylum-ci` image. Each [required tool](https://docs.phylum.io/docs/lockfile_generation) was examined, to find out how much overhead it would add to the image in each of the various ways it could be installed. The results of which are documented in the comment trails of this private story: https://github.com/phylum-dev/roadmap/issues/380 The actions taken here include: * Update the existing `Dockerfile` to install lockfile generation tools * Retain the previous functionality with a new `Dockerfile.slim` file * Switch base image from `python:3.11-slim-bullseye` to `-bookworm` * Format and refactor throughout * Be DRY about the version of `poetry` * Ensure `RUN` command contents adhere to `shellcheck` QA findings BREAKING CHANGE: The `phylum-ci` docker image created from the default `Dockerfile` is much larger, containing *all* the required tools for lockfile generation across all supported ecosystems. To retain the previous functionality, a new `slim` tag is offered for those instances where *no* manifest files are present and/or *only* lockfiles are used. --- Dockerfile | 114 ++++++++++++++++++++++++++++---- Dockerfile.slim | 169 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 271 insertions(+), 12 deletions(-) create mode 100644 Dockerfile.slim diff --git a/Dockerfile b/Dockerfile index d8254485..ca680d38 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,6 +14,7 @@ # This Dockerfile can also be used in CI, as part of the release pipeline. # The goal there is to ensure that the exact Python package that was built and # released is the one that is installed while creating this image. Prerequisites are: +# # * The package has already been built (e.g., `poetry build -vvv`) # * There is exactly one wheel available to reference by glob expression # @@ -63,7 +64,7 @@ # $ docker run --rm --entrypoint entrypoint.sh phylumio/phylum-ci:latest "ls -alh /" ########################################################################################## -FROM python:3.11-slim-bullseye AS builder +FROM python:3.11-slim-bookworm AS builder # PKG_SRC is the path to a built distribution/wheel and PKG_NAME is the name of the built # distribution/wheel. Both can optionally be specified in glob form. When not defined, @@ -78,6 +79,7 @@ ENV PHYLUM_VENV="/opt/venv" ENV PHYLUM_VENV_PIP="${PHYLUM_VENV}/bin/pip" ENV PIP_NO_COMPILE=1 ENV PIP_DISABLE_PIP_VERSION_CHECK=1 +ENV POETRY_VERSION="1.6.1" WORKDIR ${APP_PATH} @@ -87,7 +89,7 @@ RUN set -eux; \ RUN set -eux; \ python -m venv ${POETRY_VENV}; \ ${POETRY_VENV}/bin/pip install --no-cache-dir --upgrade pip setuptools; \ - ${POETRY_VENV}/bin/pip install --no-cache-dir poetry==1.6.1 poetry-plugin-export + ${POETRY_VENV}/bin/pip install --no-cache-dir poetry==${POETRY_VERSION} poetry-plugin-export # Copy the bare minimum needed for specifying dependencies. # This will enable better layer caching and faster builds when iterating locally. @@ -105,15 +107,16 @@ RUN --mount=type=cache,id=pip,target=/root/.cache/pip \ set -eux; \ ${PHYLUM_VENV_PIP} cache info; \ ${PHYLUM_VENV_PIP} cache list; \ - ${PHYLUM_VENV_PIP} install -r requirements.txt + ${PHYLUM_VENV_PIP} install -r requirements.txt pipenv poetry==${POETRY_VERSION} COPY "${PKG_SRC:-.}" . RUN ${PHYLUM_VENV_PIP} install --no-cache-dir ${PKG_NAME:-.} RUN find ${PHYLUM_VENV} -type f -name '*.pyc' -delete -# Place in a directory included in the final layer and also known to be part of the $PATH +# Place the ENTRYPOINT alternative script in a directory included +# in the final layer and also known to be part of the $PATH COPY entrypoint.sh ${PHYLUM_VENV}/bin/ -FROM python:3.11-slim-bullseye +FROM python:3.11-slim-bookworm # CLI_VER specifies the Phylum CLI version to install in the image. # Values should be provided in a format acceptable to the `phylum-init` script. @@ -129,24 +132,111 @@ ARG PHYLUM_API_URI # and therefore increase the API rate limit. ARG GITHUB_TOKEN +# Ref: https://docs.docker.com/engine/reference/builder/#automatic-platform-args-in-the-global-scope +ARG TARGETOS +ARG TARGETARCH + LABEL maintainer="Phylum, Inc. " LABEL org.opencontainers.image.source="https://github.com/phylum-dev/phylum-ci" ENV PHYLUM_VENV="/opt/venv" -ENV PATH=${PHYLUM_VENV}/bin:$PATH + +ENV GRADLE_PATH="/opt/gradle/bin" +ENV GO_PATH="/usr/local/go/bin" +ENV CARGO_PATH="/root/.cargo/bin" +ENV PATH=${PHYLUM_VENV}/bin:$PATH:${GRADLE_PATH}:${GO_PATH}:${CARGO_PATH} + ENV PYTHONDONTWRITEBYTECODE=1 -# Copy only Python packages to limit the image size +# Ref: https://learn.microsoft.com/en-us/dotnet/core/tools/dotnet-environment-variables +ENV DOTNET_CLI_TELEMETRY_OPTOUT=1 +ENV DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 +ENV DOTNET_NOLOGO=1 + +# Copy only Python packages to limit the image size. This includes the `phylum` package with it's +# `phylum-ci` and `phylum-init` entry points, plus the `pip`, `pipenv`, and `poetry` required tools. COPY --from=builder ${PHYLUM_VENV} ${PHYLUM_VENV} -RUN set -eux; \ +# Specify the shell options here, based on the use of pipelines. +# Ref: https://github.com/hadolint/hadolint/wiki/DL4006 +SHELL ["/bin/bash", "-euxo", "pipefail", "-c"] +RUN \ + # Install pre-requisites and package manager versions for `npm`, `bundle`, and `mvn` tools apt-get update; \ apt-get upgrade --yes; \ - apt-get install --yes --no-install-recommends git; \ - chmod +x ${PHYLUM_VENV}/bin/entrypoint.sh; \ - phylum-init -vvv --phylum-release ${CLI_VER:-latest} --global-install; \ + apt-get install --yes --no-install-recommends \ + git \ + npm \ + bundler \ + maven \ + curl \ + unzip \ + jq \ + ; \ + # Make ENTRYPOINT alternative script available + chmod +x "${PHYLUM_VENV}/bin/entrypoint.sh"; \ + # + # Install Phylum CLI + phylum-init -vvv --phylum-release "${CLI_VER:-latest}" --global-install; \ + # + # Install and enable `corepack` with cached instances of the latest major versions of `yarn` and `pnpm` tools + npm install -g corepack@latest; \ + corepack pack yarn@stable pnpm@latest; \ + corepack enable yarn pnpm; \ + # + # Create an "alias" for `curl` with secure and common options + printf '#!/bin/bash\ncurl --proto "=https" --tlsv1.2 -sSfL "$@"\n' > /usr/bin/curls; \ + chmod +x /usr/bin/curls; \ + # + # Manual install of `gradle` + GRADLE_VERSION=$(curls https://services.gradle.org/versions/current | jq -r '.version'); \ + GRADLE_DL="https://services.gradle.org/distributions/gradle-${GRADLE_VERSION}-bin.zip"; \ + GRADLE_DL_SHA256=$(curls "${GRADLE_DL}.sha256"); \ + curls -o gradle.zip "${GRADLE_DL}"; \ + printf "%s *gradle.zip" "${GRADLE_DL_SHA256}" | sha256sum -c -; \ + mkdir /opt/gradle; \ + unzip -d /opt/gradle gradle.zip; \ + mv /opt/gradle/gradle-"${GRADLE_VERSION}"/* /opt/gradle/; \ + rm gradle.zip && rmdir "/opt/gradle/gradle-${GRADLE_VERSION}/"; \ + # + # Manual install of `go` + # Ref: https://pkg.go.dev/golang.org/x/website/internal/dl + GO_DL_URL="https://go.dev/dl/?mode=json"; \ + GO_DL_REL=$(curls "${GO_DL_URL}" | jq '.[0].files[] | select(.os==env.TARGETOS and .arch==env.TARGETARCH)'); \ + GO_DL_FILENAME=$(echo "${GO_DL_REL}" | jq -r '.filename'); \ + GO_DL_SHA256=$(echo "${GO_DL_REL}" | jq -r '.sha256'); \ + curls -o go.tgz "https://go.dev/dl/${GO_DL_FILENAME}"; \ + printf "%s *go.tgz" "${GO_DL_SHA256}" | sha256sum -c -; \ + rm -rf /usr/local/go; \ + tar -C /usr/local -xzf go.tgz; \ + rm go.tgz; \ + # + # Manual install of Rust to get `cargo` tool + curls https://sh.rustup.rs | sh -s -- -y --profile minimal; \ + # + # Install .NET SDK to get `dotnet` tool + # Ref: https://github.com/dotnet/core/tree/main/release-notes + DOTNET_SDK_LATEST_CHANNEL_VER=$( \ + curls https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json | \ + jq -r '[."releases-index"[] | select(."support-phase"=="active")] | first."channel-version"' \ + ); \ + DEB_MAJ_VER=$(cut -d "." -f1 /etc/debian_version); \ + curls -o ms-prod.deb "https://packages.microsoft.com/config/debian/${DEB_MAJ_VER}/packages-microsoft-prod.deb"; \ + dpkg --install ms-prod.deb; \ + rm ms-prod.deb; \ + apt-get update; \ + apt-get install --yes --no-install-recommends "dotnet-sdk-${DOTNET_SDK_LATEST_CHANNEL_VER}"; \ + # + # Final cleanup + rm /usr/bin/curls; \ + apt-get remove --yes --auto-remove \ + curl \ + unzip \ + jq \ + ; \ apt-get purge --yes --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \ rm -rf /var/lib/apt/lists/*; \ - find / -type f -name '*.pyc' -delete + rm -rf /tmp/*; \ + find / -type f -name '*.pyc' -delete; CMD ["phylum-ci"] diff --git a/Dockerfile.slim b/Dockerfile.slim new file mode 100644 index 00000000..63689423 --- /dev/null +++ b/Dockerfile.slim @@ -0,0 +1,169 @@ +# syntax=docker/dockerfile:1 +# The syntax statement above is to make use of buildkit's build mounts for caching + +# NOTE: +# This Dockerfile is used to build the `slim` tags for the `phylum-ci` images. It differs +# from the primary Dockerfile in that it does not install the required tools needed for +# lockfile generation: https://docs.phylum.io/docs/lockfile_generation (with the exception +# of the `pip` tool). The image created from this Dockerfile is significantly smaller and +# will allow the integrations relying on it to complete faster. It is useful for those +# instances where *no* manifest files are present and/or *only* lockfiles are used. + +########################################################################################## +# This Dockerfile makes use of BuildKit mode, which needs to be enabled client side first: +# +# $ export DOCKER_BUILDKIT=1 +# +# This Dockerfile can be used to build the project's package within the image it creates. +# To do so, simply build the image WITHOUT any build args specified (e.g., `--build-arg`): +# +# $ docker build --tag phylum-ci . +# +# This Dockerfile can also be used in CI, as part of the release pipeline. +# The goal there is to ensure that the exact Python package that was built and +# released is the one that is installed while creating this image. Prerequisites are: +# +# * The package has already been built (e.g., `poetry build -vvv`) +# * There is exactly one wheel available to reference by glob expression +# +# To make use of this feature, build the image WITH build args specified: +# +# $ export PKG_SRC=dist/phylum-*.whl +# $ export PKG_NAME=phylum-*.whl +# $ docker build --tag phylum-ci --build-arg PKG_SRC --build-arg PKG_NAME . +# +# Another build arg is exposed to optionally specify the Phylum CLI version to install: +# +# $ docker build --tag phylum-ci --build-arg CLI_VER=v4.8.0 . +# +# The PHYLUM_API_URI build arg is exposed to optionally specify the URI of a Phylum API +# instance to use: +# +# $ export PHYLUM_API_URI=https://api.staging.phylum.io +# $ docker build --tag phylum-ci --build-arg PHYLUM_API_URI . +# +# Another build arg is exposed to optionally specify a GitHub Personal Access Token (PAT): +# +# $ docker build --tag phylum-ci --build-arg GITHUB_TOKEN . +# +# Providing a build argument like this (without a value) works when there is already an +# environment variable defined with the same name. Providing a GitHub PAT is useful to +# make authenticated requests and therefore increase the API rate limit. +# +# To make use of BuildKit's inline layer caching feature, add the `BUILDKIT_INLINE_CACHE` +# build argument to any instance of building an image. Then, that image can be used +# locally or remotely (if it was pushed to a repository) to warm the build cache by using +# the `--cache-from` argument: +# +# $ docker build --tag phylumio/phylum-ci:cache --build-arg BUILDKIT_INLINE_CACHE=1 . +# $ docker push phylumio/phylum-ci:cache && docker image rm phylumio/phylum-ci:cache +# $ docker build --tag phylumio/phylum-ci:faster --cache-from phylumio/phylum-ci:cache . +# +# There is no ENTRYPOINT in this Dockerfile by design. That way, it is possible to provide +# unquoted extra parameters to run arbitrary commands in the context of the container: +# +# $ docker run --rm phylumio/phylum-ci:latest ls -alh / +# +# However, there may be cases where an entrypoint is needed. One is provided and placed in +# a directory that will be included in the final layer and also known to be part of the +# $PATH. To make use of it, add the `--entrypoint` option to a docker run command, +# specifying the `entrypoint.sh` script, providing extra parameters as a *quoted* string: +# +# $ docker run --rm --entrypoint entrypoint.sh phylumio/phylum-ci:latest "ls -alh /" +########################################################################################## + +FROM python:3.11-slim-bookworm AS builder + +# PKG_SRC is the path to a built distribution/wheel and PKG_NAME is the name of the built +# distribution/wheel. Both can optionally be specified in glob form. When not defined, +# the values will default to the root of the package (i.e., `pyproject.toml` path). +ARG PKG_SRC +ARG PKG_NAME + +ENV APP_PATH="/app" +ENV POETRY_VENV="${APP_PATH}/.venv" +ENV POETRY_PATH="${POETRY_VENV}/bin/poetry" +ENV PHYLUM_VENV="/opt/venv" +ENV PHYLUM_VENV_PIP="${PHYLUM_VENV}/bin/pip" +ENV PIP_NO_COMPILE=1 +ENV PIP_DISABLE_PIP_VERSION_CHECK=1 +ENV POETRY_VERSION="1.6.1" + +WORKDIR ${APP_PATH} + +RUN set -eux; \ + python -m venv ${PHYLUM_VENV}; \ + ${PHYLUM_VENV_PIP} install --no-cache-dir --upgrade pip setuptools +RUN set -eux; \ + python -m venv ${POETRY_VENV}; \ + ${POETRY_VENV}/bin/pip install --no-cache-dir --upgrade pip setuptools; \ + ${POETRY_VENV}/bin/pip install --no-cache-dir poetry==${POETRY_VERSION} poetry-plugin-export + +# Copy the bare minimum needed for specifying dependencies. +# This will enable better layer caching and faster builds when iterating locally. +# `--without-hashes` is used to ensure the `pip` cache mount is used for packages that +# would otherwise only match the sdist and therefore have to be built for every run. +# References: +# * https://pythonspeed.com/articles/pipenv-docker/ +# * https://hub.docker.com/r/docker/dockerfile +COPY pyproject.toml poetry.lock ./ +RUN ${POETRY_PATH} export --without-hashes --format requirements.txt --output requirements.txt + +# Cache the pip installed dependencies for faster builds when iterating locally. +# NOTE: This `--mount` feature requires BUILDKIT to be used +RUN --mount=type=cache,id=pip,target=/root/.cache/pip \ + set -eux; \ + ${PHYLUM_VENV_PIP} cache info; \ + ${PHYLUM_VENV_PIP} cache list; \ + ${PHYLUM_VENV_PIP} install -r requirements.txt +COPY "${PKG_SRC:-.}" . +RUN ${PHYLUM_VENV_PIP} install --no-cache-dir ${PKG_NAME:-.} +RUN find ${PHYLUM_VENV} -type f -name '*.pyc' -delete + +# Place the ENTRYPOINT alternative script in a directory included +# in the final layer and also known to be part of the $PATH +COPY entrypoint.sh ${PHYLUM_VENV}/bin/ + +FROM python:3.11-slim-bookworm + +# CLI_VER specifies the Phylum CLI version to install in the image. +# Values should be provided in a format acceptable to the `phylum-init` script. +# When not defined, the value will default to `latest`. +ARG CLI_VER + +# PHYLUM_API_URI is an optional build argument that can be used to specify +# the URI of a Phylum API instance to use. +ARG PHYLUM_API_URI + +# GITHUB_TOKEN is an optional build argument that can be used to provide a +# GitHub Personal Access Token (PAT) in order to make authenticated requests +# and therefore increase the API rate limit. +ARG GITHUB_TOKEN + +LABEL maintainer="Phylum, Inc. " +LABEL org.opencontainers.image.source="https://github.com/phylum-dev/phylum-ci" + +ENV PHYLUM_VENV="/opt/venv" +ENV PATH=${PHYLUM_VENV}/bin:$PATH +ENV PYTHONDONTWRITEBYTECODE=1 + +# Copy only Python packages to limit the image size. This includes the +# `phylum` package with it's `phylum-ci` and `phylum-init` entry points. +COPY --from=builder ${PHYLUM_VENV} ${PHYLUM_VENV} + +RUN set -eux; \ + # Install pre-requisites + apt-get update; \ + apt-get upgrade --yes; \ + apt-get install --yes --no-install-recommends git; \ + # Make ENTRYPOINT alternative script available + chmod +x "${PHYLUM_VENV}/bin/entrypoint.sh"; \ + # Install Phylum CLI + phylum-init -vvv --phylum-release "${CLI_VER:-latest}" --global-install; \ + # Cleanup + apt-get purge --yes --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \ + rm -rf /var/lib/apt/lists/*; \ + rm -rf /tmp/*; \ + find / -type f -name '*.pyc' -delete; + +CMD ["phylum-ci"] From fe503e041f7eb9ea9ae70473933fe0e65f26cc34 Mon Sep 17 00:00:00 2001 From: Charles Coggins Date: Thu, 28 Sep 2023 15:47:34 -0500 Subject: [PATCH 2/5] test: add basic docker tests * Create `tests/docker_tests.sh` script to provide a basic set of tests * Confirm each of the expected tools is present within the image * The path and version of the tool is shown * Sometimes additional data is provided, like help or info output * Update the `Test` workflow * Turn the `docker` job into a true matrix job * Add different dockerfile inputs * Add build inputs to allow for building from a wheel or source * Test each built image with the new `docker_tests.sh` script * Simplify the logic in the `test-rollup` job --- .github/workflows/test.yml | 77 +++++++++++++-------------- tests/docker_tests.sh | 104 +++++++++++++++++++++++++++++++++++++ 2 files changed, 139 insertions(+), 42 deletions(-) create mode 100755 tests/docker_tests.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 22e8cd7d..1fd95265 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -91,15 +91,19 @@ jobs: - name: Run tox via poetry run: poetry run tox - # This job is meant to be a sanity check on the Docker image...that it can be created and + # This job is meant to be a sanity check on the Docker image...that it can be + # created with various Dockerfiles, from source or a built distribution, and # have the script entry points called without error. - docker: - name: Docker smoke test + docker-matrix: + name: ${{ matrix.dockerfile }} ${{ matrix.build }} smoke test runs-on: ubuntu-latest strategy: + fail-fast: false matrix: # It's only one Python version specified in a "matrix", but on purpose to stay DRY python-version: ["3.11"] + dockerfile: ["Dockerfile", "Dockerfile.slim"] + build: ["wheel", "source"] env: DOCKER_BUILDKIT: 1 steps: @@ -107,60 +111,61 @@ jobs: uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 - name: Install poetry + if: ${{ matrix.build == 'wheel' }} run: pipx install poetry==1.6.1 - name: Configure poetry + if: ${{ matrix.build == 'wheel' }} run: poetry config virtualenvs.in-project true - name: Set up Python + if: ${{ matrix.build == 'wheel' }} uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 with: python-version: ${{ matrix.python-version }} cache: 'poetry' - name: Install the project with poetry + if: ${{ matrix.build == 'wheel' }} run: | poetry env use python${{ matrix.python-version }} poetry check --lock poetry lock --no-update --no-cache poetry install --verbose --no-root --sync - - name: Build docker image from source - run: | - docker build \ - --tag phylum-ci:from-src \ - --cache-from phylumio/phylum-ci:latest \ - --build-arg GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} \ - . - - - name: Test docker image built from source - run: | - docker run --rm phylum-ci:from-src git --version - docker run --rm phylum-ci:from-src phylum-ci --version - docker run --rm phylum-ci:from-src phylum-ci --help - docker run --rm phylum-ci:from-src phylum-init --help - docker run --rm phylum-ci:from-src phylum --help - - name: Build wheel and source distribution + if: ${{ matrix.build == 'wheel' }} run: poetry build -vvv - name: Build docker image with pre-built distributions + if: ${{ matrix.build == 'wheel' }} run: | docker build \ - --tag phylum-ci:from-dist \ + --tag phylum-ci \ --cache-from phylumio/phylum-ci:latest \ --build-arg PKG_SRC=dist/phylum-*.whl \ --build-arg PKG_NAME=phylum-*.whl \ --build-arg GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} \ + --file ${{ matrix.dockerfile }} \ . - - name: Test docker image built from dist + - name: Build docker image from source + if: ${{ matrix.build == 'source' }} run: | - docker run --rm phylum-ci:from-dist git --version - docker run --rm phylum-ci:from-dist phylum-ci --version - docker run --rm phylum-ci:from-dist phylum-ci --help - docker run --rm phylum-ci:from-dist phylum-init --help - docker run --rm phylum-ci:from-dist phylum --help + docker build \ + --tag phylum-ci \ + --cache-from phylumio/phylum-ci:latest \ + --build-arg GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} \ + --file ${{ matrix.dockerfile }} \ + . + + - name: Test slim docker image built from ${{ matrix.build }} + if: ${{ matrix.dockerfile == 'Dockerfile.slim' }} + run: tests/docker_tests.sh --image phylum-ci --slim + + - name: Test full docker image built from ${{ matrix.build }} + if: ${{ matrix.dockerfile == 'Dockerfile' }} + run: tests/docker_tests.sh --image phylum-ci # This job reports the results of the test jobs above and is used to enforce status checks in # the repo settings without needing to update those settings everytime the test jobs are updated. @@ -168,20 +173,8 @@ jobs: name: Test rollup runs-on: ubuntu-latest if: always() - needs: [QA, test-matrix, docker] + needs: [QA, test-matrix, docker-matrix] steps: - - name: Check for test jobs failure - if: > - (needs.QA.result != 'success') - || (needs.test-matrix.result != 'success') - || (needs.docker.result != 'success') - run: | - echo "At least one test job was not successful" - exit 1 - - - name: Confirm test jobs success - if: > - (needs.QA.result == 'success') - && (needs.test-matrix.result == 'success') - && (needs.docker.result == 'success') - run: echo "All test jobs were successful" + - name: Check for test jobs failure or cancellation + if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') + run: exit 1 diff --git a/tests/docker_tests.sh b/tests/docker_tests.sh new file mode 100755 index 00000000..bc03fff7 --- /dev/null +++ b/tests/docker_tests.sh @@ -0,0 +1,104 @@ +#!/bin/sh + +# This script is meant to provide a basic set of tests. It simply confirms that +# each of the tools expected for operation is present within the image. The +# path and version of the tool is shown and in some cases additional information +# is provided, like help or info output. + +set -eu + +# Check for a required command +require_command() { + cmd="$1" + help_msg="${2:-}" + + if ! type "${cmd}" > /dev/null 2>&1; then + echo "ERROR: This script requires \`${cmd}\`. Please install it and re-run this script to continue." >&2 + if [ -n "${help_msg}" ]; then + printf "\n" >&2 + echo "${help_msg}" >&2 + fi + exit 1 + fi +} + +usage() { + cat 1>&2 <&2 + usage + exit 1 + ;; + esac +done + +if [ -z "${IMAGE:-}" ]; then + IMAGE="maxrake/phylum-ci:manifest_support" +fi + +# These are the commands to ensure the base pre-requisites are available +SLIM_COMMANDS=$(cat < Date: Thu, 28 Sep 2023 18:04:10 -0500 Subject: [PATCH 3/5] ci: update workflow files * Update `Docker` and `Release` workflows * Build with both the default and slim dockerfiles * Test with the new `docker_tests.sh` script * Create specific docker tags that include the `-slim` suffix * Create `slim` docker tags mirroring the `latest` slim image * Adhere to `actionlint` and `shellcheck` QA tools --- .github/workflows/auto_updates.yml | 2 +- .github/workflows/docker.yml | 54 ++++++++++++++++++---------- .github/workflows/preview.yml | 4 +-- .github/workflows/release.yml | 57 +++++++++++++++++++----------- 4 files changed, 76 insertions(+), 41 deletions(-) diff --git a/.github/workflows/auto_updates.yml b/.github/workflows/auto_updates.yml index aa22fa6d..9df36e5c 100644 --- a/.github/workflows/auto_updates.yml +++ b/.github/workflows/auto_updates.yml @@ -79,7 +79,7 @@ jobs: # NOTE: The git user name and email used for commits is already configured, # by the crazy-max/ghaction-import-gpg action. run: | - git commit -a -m "build: bump `poetry.lock` dependencies and `pre-commit` hooks" + git commit -a -m "build: bump \`poetry.lock\` dependencies and \`pre-commit\` hooks" git push --force origin HEAD:workflow-auto-updates - name: Create Pull Request diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index ecc82905..da348aaa 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -60,10 +60,10 @@ jobs: --exit-status \ .tag_name \ ) - REL_VER_WITHOUT_v=$(echo $REL_VER_WITH_v | sed 's/v//') - echo $REL_VER_WITH_v $REL_VER_WITHOUT_v - echo "REL_VER_WITH_v=$REL_VER_WITH_v" >> $GITHUB_ENV - echo "REL_VER_WITHOUT_v=$REL_VER_WITHOUT_v" >> $GITHUB_ENV + REL_VER_WITHOUT_v="${REL_VER_WITH_v//v/}" + echo "${REL_VER_WITH_v}" "${REL_VER_WITHOUT_v}" + echo "REL_VER_WITH_v=${REL_VER_WITH_v}" >> "${GITHUB_ENV}" + echo "REL_VER_WITHOUT_v=${REL_VER_WITHOUT_v}" >> "${GITHUB_ENV}" - name: Checkout the repo uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 @@ -76,7 +76,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: gh release download ${{ env.REL_VER_WITH_v }} --pattern '*.whl' - - name: Build docker image with latest phylum wheel + - name: Build default docker image with latest phylum wheel run: | docker build \ --tag phylum-ci \ @@ -87,13 +87,23 @@ jobs: --build-arg GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} \ . - - name: Test docker image with latest phylum wheel + - name: Test default docker image with latest phylum wheel + run: tests/docker_tests.sh --image phylum-ci + + - name: Build slim docker image with latest phylum wheel run: | - docker run --rm phylum-ci git --version - docker run --rm phylum-ci phylum-ci --version - docker run --rm phylum-ci phylum-ci --help - docker run --rm phylum-ci phylum-init --help - docker run --rm phylum-ci phylum --help + docker build \ + --tag phylum-ci-slim \ + --build-arg PKG_SRC=phylum-*.whl \ + --build-arg PKG_NAME=phylum-*.whl \ + --build-arg CLI_VER=${{ github.event.client_payload.CLI_version }} \ + --build-arg BUILDKIT_INLINE_CACHE=1 \ + --build-arg GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} \ + --file Dockerfile.slim \ + . + + - name: Test slim docker image with latest phylum wheel + run: tests/docker_tests.sh --image phylum-ci-slim --slim - name: Login to Docker Hub run: docker login --username ${{ secrets.DOCKER_HUB_USERNAME }} --password ${{ secrets.DOCKER_HUB_TOKEN }} @@ -101,24 +111,32 @@ jobs: - name: Login to GitHub Container Registry run: docker login --username ${{ github.actor }} --password ${{ secrets.GITHUB_TOKEN }} ghcr.io - - name: Tag and push unique docker image + - name: Create specific docker tags and push them run: | - export CLI_REL_VER=$(docker run --rm phylum-ci phylum --version | sed 's/phylum //') - docker tag phylum-ci phylumio/phylum-ci:${{ env.REL_VER_WITHOUT_v }}-CLI$CLI_REL_VER - docker tag phylum-ci ghcr.io/phylum-dev/phylum-ci:${{ env.REL_VER_WITHOUT_v }}-CLI$CLI_REL_VER - docker push phylumio/phylum-ci:${{ env.REL_VER_WITHOUT_v }}-CLI$CLI_REL_VER - docker push ghcr.io/phylum-dev/phylum-ci:${{ env.REL_VER_WITHOUT_v }}-CLI$CLI_REL_VER + CLI_REL_VER=$(docker run --rm phylum-ci phylum --version | sed 's/phylum //') + docker tag phylum-ci "phylumio/phylum-ci:${{ env.REL_VER_WITHOUT_v }}-CLI${CLI_REL_VER}" + docker tag phylum-ci-slim "phylumio/phylum-ci:${{ env.REL_VER_WITHOUT_v }}-CLI${CLI_REL_VER}-slim" + docker tag phylum-ci "ghcr.io/phylum-dev/phylum-ci:${{ env.REL_VER_WITHOUT_v }}-CLI${CLI_REL_VER}" + docker tag phylum-ci-slim "ghcr.io/phylum-dev/phylum-ci:${{ env.REL_VER_WITHOUT_v }}-CLI${CLI_REL_VER}-slim" + docker push "phylumio/phylum-ci:${{ env.REL_VER_WITHOUT_v }}-CLI${CLI_REL_VER}" + docker push "phylumio/phylum-ci:${{ env.REL_VER_WITHOUT_v }}-CLI${CLI_REL_VER}-slim" + docker push "ghcr.io/phylum-dev/phylum-ci:${{ env.REL_VER_WITHOUT_v }}-CLI${CLI_REL_VER}" + docker push "ghcr.io/phylum-dev/phylum-ci:${{ env.REL_VER_WITHOUT_v }}-CLI${CLI_REL_VER}-slim" - - name: Tag and push latest docker image + - name: Tag and push latest docker images # Only tag and push `latest` when it's not a CLI pre-release # NOTE: This is an instance where the expression syntax (`${{ }}`) is required for the `if` conditional, # contrary to the GitHub workflow syntax documentation. Do not remove the expression syntax. if: ${{ !contains(github.event.client_payload.CLI_version, 'rc') }} run: | docker tag phylum-ci phylumio/phylum-ci:latest + docker tag phylum-ci-slim phylumio/phylum-ci:slim docker tag phylum-ci ghcr.io/phylum-dev/phylum-ci:latest + docker tag phylum-ci-slim ghcr.io/phylum-dev/phylum-ci:slim docker push phylumio/phylum-ci:latest + docker push phylumio/phylum-ci:slim docker push ghcr.io/phylum-dev/phylum-ci:latest + docker push ghcr.io/phylum-dev/phylum-ci:slim - name: Logout of Docker Hub if: always() diff --git a/.github/workflows/preview.yml b/.github/workflows/preview.yml index e7cc0ebd..a71e78d4 100644 --- a/.github/workflows/preview.yml +++ b/.github/workflows/preview.yml @@ -61,8 +61,8 @@ jobs: # poetry version rules do not provide for developmental releases as specified in PEP440. # It can be pieced together with these commands. run: | - poetry version $(poetry run semantic-release print-version --next) - poetry version $(poetry version --short).dev$GITHUB_RUN_NUMBER + poetry version "$(poetry run semantic-release print-version --next)" + poetry version "$(poetry version --short).dev${GITHUB_RUN_NUMBER}" - name: Run tox via poetry run: poetry run tox diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e6fc41d3..b1244970 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -86,18 +86,18 @@ jobs: poetry install --verbose --sync --with test,ci - name: Set PHYLUM_ORIGINAL_VER value - run: echo "PHYLUM_ORIGINAL_VER=$(poetry version --short)" >> $GITHUB_ENV + run: echo "PHYLUM_ORIGINAL_VER=$(poetry version --short)" >> "${GITHUB_ENV}" - name: Set to next version for build run: | if [ "${{ inputs.prerelease }}" = "true" ]; then - poetry version $(poetry run semantic-release print-version --next --prerelease) + poetry version "$(poetry run semantic-release print-version --next --prerelease)" else - poetry version $(poetry run semantic-release print-version --next) + poetry version "$(poetry run semantic-release print-version --next)" fi - name: Set PHYLUM_REL_VER value - run: echo "PHYLUM_REL_VER=$(poetry version --short)" >> $GITHUB_ENV + run: echo "PHYLUM_REL_VER=$(poetry version --short)" >> "${GITHUB_ENV}" # NOTE: Run the tests for the current active Python version, as a sanity check. - name: Run tox via poetry @@ -150,9 +150,9 @@ jobs: else poetry run semantic-release publish -v DEBUG fi - echo "url=https://pypi.org/project/phylum/${{ env.PHYLUM_REL_VER }}/" >> $GITHUB_OUTPUT + echo "url=https://pypi.org/project/phylum/${{ env.PHYLUM_REL_VER }}/" >> "${GITHUB_OUTPUT}" - - name: Build docker image + - name: Build default docker image run: | docker build \ --tag phylum-ci \ @@ -162,13 +162,22 @@ jobs: --build-arg GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} \ . - - name: Test docker image with pre-built distributions + - name: Test default docker image with pre-built distributions + run: tests/docker_tests.sh --image phylum-ci + + - name: Build slim docker image run: | - docker run --rm phylum-ci git --version - docker run --rm phylum-ci phylum-ci --version - docker run --rm phylum-ci phylum-ci --help - docker run --rm phylum-ci phylum-init --help - docker run --rm phylum-ci phylum --help + docker build \ + --tag phylum-ci-slim \ + --build-arg PKG_SRC=dist/phylum-*.whl \ + --build-arg PKG_NAME=phylum-*.whl \ + --build-arg BUILDKIT_INLINE_CACHE=1 \ + --build-arg GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} \ + --file Dockerfile.slim \ + . + + - name: Test slim docker image with pre-built distributions + run: tests/docker_tests.sh --image phylum-ci-slim --slim - name: Login to Docker Hub run: docker login --username ${{ secrets.DOCKER_HUB_USERNAME }} --password ${{ secrets.DOCKER_HUB_TOKEN }} @@ -176,24 +185,32 @@ jobs: - name: Login to GitHub Container Registry run: docker login --username ${{ github.actor }} --password ${{ secrets.GITHUB_TOKEN }} ghcr.io - - name: Tag and push unique docker image + - name: Create specific docker tags and push them run: | - export CLI_REL_VER=$(docker run --rm phylum-ci phylum --version | sed 's/phylum //') - docker tag phylum-ci phylumio/phylum-ci:${{ env.PHYLUM_REL_VER }}-CLI$CLI_REL_VER - docker tag phylum-ci ghcr.io/phylum-dev/phylum-ci:${{ env.PHYLUM_REL_VER }}-CLI$CLI_REL_VER - docker push phylumio/phylum-ci:${{ env.PHYLUM_REL_VER }}-CLI$CLI_REL_VER - docker push ghcr.io/phylum-dev/phylum-ci:${{ env.PHYLUM_REL_VER }}-CLI$CLI_REL_VER - - - name: Tag and push latest docker image + CLI_REL_VER=$(docker run --rm phylum-ci phylum --version | sed 's/phylum //') + docker tag phylum-ci "phylumio/phylum-ci:${{ env.PHYLUM_REL_VER }}-CLI${CLI_REL_VER}" + docker tag phylum-ci-slim "phylumio/phylum-ci:${{ env.PHYLUM_REL_VER }}-CLI${CLI_REL_VER}-slim" + docker tag phylum-ci "ghcr.io/phylum-dev/phylum-ci:${{ env.PHYLUM_REL_VER }}-CLI${CLI_REL_VER}" + docker tag phylum-ci-slim "ghcr.io/phylum-dev/phylum-ci:${{ env.PHYLUM_REL_VER }}-CLI${CLI_REL_VER}-slim" + docker push "phylumio/phylum-ci:${{ env.PHYLUM_REL_VER }}-CLI${CLI_REL_VER}" + docker push "phylumio/phylum-ci:${{ env.PHYLUM_REL_VER }}-CLI${CLI_REL_VER}-slim" + docker push "ghcr.io/phylum-dev/phylum-ci:${{ env.PHYLUM_REL_VER }}-CLI${CLI_REL_VER}" + docker push "ghcr.io/phylum-dev/phylum-ci:${{ env.PHYLUM_REL_VER }}-CLI${CLI_REL_VER}-slim" + + - name: Tag and push latest docker images # Only tag and push `latest` when it's not a phylum-ci pre-release # NOTE: This is an instance where the expression syntax (`${{ }}`) is required for the `if` conditional, # contrary to the GitHub workflow syntax documentation. Do not remove the expression syntax. if: ${{ !inputs.prerelease }} run: | docker tag phylum-ci phylumio/phylum-ci:latest + docker tag phylum-ci-slim phylumio/phylum-ci:slim docker tag phylum-ci ghcr.io/phylum-dev/phylum-ci:latest + docker tag phylum-ci-slim ghcr.io/phylum-dev/phylum-ci:slim docker push phylumio/phylum-ci:latest + docker push phylumio/phylum-ci:slim docker push ghcr.io/phylum-dev/phylum-ci:latest + docker push ghcr.io/phylum-dev/phylum-ci:slim - name: Logout of Docker Hub if: always() From 34ba4b28687beb78b042d570f95ac8c8bcce7f48 Mon Sep 17 00:00:00 2001 From: Charles Coggins Date: Thu, 28 Sep 2023 20:52:56 -0500 Subject: [PATCH 4/5] test: move 'docker_tests.sh' script to be outside the python package --- .github/workflows/docker.yml | 4 ++-- .github/workflows/release.yml | 4 ++-- .github/workflows/test.yml | 4 ++-- {tests => scripts}/docker_tests.sh | 0 4 files changed, 6 insertions(+), 6 deletions(-) rename {tests => scripts}/docker_tests.sh (100%) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index da348aaa..0418dad1 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -88,7 +88,7 @@ jobs: . - name: Test default docker image with latest phylum wheel - run: tests/docker_tests.sh --image phylum-ci + run: scripts/docker_tests.sh --image phylum-ci - name: Build slim docker image with latest phylum wheel run: | @@ -103,7 +103,7 @@ jobs: . - name: Test slim docker image with latest phylum wheel - run: tests/docker_tests.sh --image phylum-ci-slim --slim + run: scripts/docker_tests.sh --image phylum-ci-slim --slim - name: Login to Docker Hub run: docker login --username ${{ secrets.DOCKER_HUB_USERNAME }} --password ${{ secrets.DOCKER_HUB_TOKEN }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b1244970..b677bdb6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -163,7 +163,7 @@ jobs: . - name: Test default docker image with pre-built distributions - run: tests/docker_tests.sh --image phylum-ci + run: scripts/docker_tests.sh --image phylum-ci - name: Build slim docker image run: | @@ -177,7 +177,7 @@ jobs: . - name: Test slim docker image with pre-built distributions - run: tests/docker_tests.sh --image phylum-ci-slim --slim + run: scripts/docker_tests.sh --image phylum-ci-slim --slim - name: Login to Docker Hub run: docker login --username ${{ secrets.DOCKER_HUB_USERNAME }} --password ${{ secrets.DOCKER_HUB_TOKEN }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1fd95265..8dae31ed 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -161,11 +161,11 @@ jobs: - name: Test slim docker image built from ${{ matrix.build }} if: ${{ matrix.dockerfile == 'Dockerfile.slim' }} - run: tests/docker_tests.sh --image phylum-ci --slim + run: scripts/docker_tests.sh --image phylum-ci --slim - name: Test full docker image built from ${{ matrix.build }} if: ${{ matrix.dockerfile == 'Dockerfile' }} - run: tests/docker_tests.sh --image phylum-ci + run: scripts/docker_tests.sh --image phylum-ci # This job reports the results of the test jobs above and is used to enforce status checks in # the repo settings without needing to update those settings everytime the test jobs are updated. diff --git a/tests/docker_tests.sh b/scripts/docker_tests.sh similarity index 100% rename from tests/docker_tests.sh rename to scripts/docker_tests.sh From a6a563beacd18c9f9fd3294837772df9e6eb7a33 Mon Sep 17 00:00:00 2001 From: Charles Coggins Date: Thu, 28 Sep 2023 20:58:51 -0500 Subject: [PATCH 5/5] docs: update documentation * Update the `README.md` with information about the `slim` tag options * Update the dockerfiles with information about how to perform testing --- Dockerfile | 5 +++++ Dockerfile.slim | 5 +++++ README.md | 34 ++++++++++++++++++++++++++-------- 3 files changed, 36 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index ca680d38..f7121253 100644 --- a/Dockerfile +++ b/Dockerfile @@ -62,6 +62,11 @@ # specifying the `entrypoint.sh` script, providing extra parameters as a *quoted* string: # # $ docker run --rm --entrypoint entrypoint.sh phylumio/phylum-ci:latest "ls -alh /" +# +# Images built from this Dockerfile can be tested for basic functionality: +# +# $ docker build --tag phylum-ci . +# $ scripts/docker_tests.sh --image phylum-ci ########################################################################################## FROM python:3.11-slim-bookworm AS builder diff --git a/Dockerfile.slim b/Dockerfile.slim index 63689423..cf9867d0 100644 --- a/Dockerfile.slim +++ b/Dockerfile.slim @@ -70,6 +70,11 @@ # specifying the `entrypoint.sh` script, providing extra parameters as a *quoted* string: # # $ docker run --rm --entrypoint entrypoint.sh phylumio/phylum-ci:latest "ls -alh /" +# +# Images built from this Dockerfile can be tested for basic functionality: +# +# $ docker build --tag phylum-ci . +# $ scripts/docker_tests.sh --image phylum-ci --slim ########################################################################################## FROM python:3.11-slim-bookworm AS builder diff --git a/README.md b/README.md index 35c3a6c4..4a4d55cc 100644 --- a/README.md +++ b/README.md @@ -79,27 +79,45 @@ docker run --rm phylumio/phylum-ci phylum-ci --help # Export a Phylum token (e.g., from `phylum auth token`) export PHYLUM_API_KEY=$(phylum auth token) -# Run it from a git repo directory containing at least one lockfile +# Run it from a git repo directory containing at least one supported lockfile or manifest docker run -it --rm -e PHYLUM_API_KEY --mount type=bind,src=$(pwd),dst=/phylum -w /phylum phylumio/phylum-ci ``` -The Docker image contains `git` and the installed `phylum` Python package. -It also contains an installed version of the Phylum CLI. -An advantage of using the Docker image is that the complete environment is packaged and made available with components -that are known to work together. +The default Docker image contains `git` and the installed `phylum` Python package. +It also contains an installed version of the Phylum CLI and all required tools needed for [lockfile generation]. +An advantage of using the default Docker image is that the complete environment is packaged and made available with +components that are known to work together. + +One disadvantage to the default image is it's size. It can take a while to download and may provide more tools than +required for your specific use case. Special `slim` tags of the `phylum-ci` image are provided as an alternative. +These tags differ from the default image in that they do not contain the required tools needed for [lockfile generation] +(with the exception of the `pip` tool). The `slim` tags are significantly smaller and will allow integrations relying +on them to complete faster. They are useful for those instances where *no* manifest files are present and/or *only* +lockfiles are used. + +```sh +# Get the "latest" `slim` tagged image +docker pull phylumio/phylum-ci:slim +``` When using the `latest` tagged image, the version of the Phylum CLI is the `latest` available. There are additional image tag options available to specify a specific release of the `phylum-ci` project and a specific -version of the Phylum CLI, in the form of `-CLIv`. Here are image tag examples: +version of the Phylum CLI, in the form of `-CLIv`. +Each of these also has a `-slim` variant that does not support [lockfile generation]. Here are image tag examples: ```sh # Get the most current release of *both* `phylum-ci` and the Phylum CLI docker pull phylumio/phylum-ci:latest -# Get the image with `phylum-ci` version 0.24.1 and Phylum CLI version 4.7.0 -docker pull phylumio/phylum-ci:0.24.1-CLIv4.7.0 +# Get the image with `phylum-ci` version 0.35.2 and Phylum CLI version 5.7.1 +docker pull phylumio/phylum-ci:0.35.2-CLIv5.7.1 + +# Get the `slim` image with `phylum-ci` version 0.36.0 and Phylum CLI version 5.7.1 +docker pull phylumio/phylum-ci:0.36.0-CLIv5.7.1-slim ``` +[lockfile generation]: https://docs.phylum.io/docs/lockfile_generation + #### `phylum-init` Script Entry Point The `phylum-init` script can be used to fetch and install the Phylum CLI.