diff --git a/.github/workflows/auto_updates.yml b/.github/workflows/auto_updates.yml index aa22fa6d..9df36e5c 100644 --- a/.github/workflows/auto_updates.yml +++ b/.github/workflows/auto_updates.yml @@ -79,7 +79,7 @@ jobs: # NOTE: The git user name and email used for commits is already configured, # by the crazy-max/ghaction-import-gpg action. run: | - git commit -a -m "build: bump `poetry.lock` dependencies and `pre-commit` hooks" + git commit -a -m "build: bump \`poetry.lock\` dependencies and \`pre-commit\` hooks" git push --force origin HEAD:workflow-auto-updates - name: Create Pull Request diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index ecc82905..0418dad1 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -60,10 +60,10 @@ jobs: --exit-status \ .tag_name \ ) - REL_VER_WITHOUT_v=$(echo $REL_VER_WITH_v | sed 's/v//') - echo $REL_VER_WITH_v $REL_VER_WITHOUT_v - echo "REL_VER_WITH_v=$REL_VER_WITH_v" >> $GITHUB_ENV - echo "REL_VER_WITHOUT_v=$REL_VER_WITHOUT_v" >> $GITHUB_ENV + REL_VER_WITHOUT_v="${REL_VER_WITH_v//v/}" + echo "${REL_VER_WITH_v}" "${REL_VER_WITHOUT_v}" + echo "REL_VER_WITH_v=${REL_VER_WITH_v}" >> "${GITHUB_ENV}" + echo "REL_VER_WITHOUT_v=${REL_VER_WITHOUT_v}" >> "${GITHUB_ENV}" - name: Checkout the repo uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 @@ -76,7 +76,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: gh release download ${{ env.REL_VER_WITH_v }} --pattern '*.whl' - - name: Build docker image with latest phylum wheel + - name: Build default docker image with latest phylum wheel run: | docker build \ --tag phylum-ci \ @@ -87,13 +87,23 @@ jobs: --build-arg GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} \ . - - name: Test docker image with latest phylum wheel + - name: Test default docker image with latest phylum wheel + run: scripts/docker_tests.sh --image phylum-ci + + - name: Build slim docker image with latest phylum wheel run: | - docker run --rm phylum-ci git --version - docker run --rm phylum-ci phylum-ci --version - docker run --rm phylum-ci phylum-ci --help - docker run --rm phylum-ci phylum-init --help - docker run --rm phylum-ci phylum --help + docker build \ + --tag phylum-ci-slim \ + --build-arg PKG_SRC=phylum-*.whl \ + --build-arg PKG_NAME=phylum-*.whl \ + --build-arg CLI_VER=${{ github.event.client_payload.CLI_version }} \ + --build-arg BUILDKIT_INLINE_CACHE=1 \ + --build-arg GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} \ + --file Dockerfile.slim \ + . + + - name: Test slim docker image with latest phylum wheel + run: scripts/docker_tests.sh --image phylum-ci-slim --slim - name: Login to Docker Hub run: docker login --username ${{ secrets.DOCKER_HUB_USERNAME }} --password ${{ secrets.DOCKER_HUB_TOKEN }} @@ -101,24 +111,32 @@ jobs: - name: Login to GitHub Container Registry run: docker login --username ${{ github.actor }} --password ${{ secrets.GITHUB_TOKEN }} ghcr.io - - name: Tag and push unique docker image + - name: Create specific docker tags and push them run: | - export CLI_REL_VER=$(docker run --rm phylum-ci phylum --version | sed 's/phylum //') - docker tag phylum-ci phylumio/phylum-ci:${{ env.REL_VER_WITHOUT_v }}-CLI$CLI_REL_VER - docker tag phylum-ci ghcr.io/phylum-dev/phylum-ci:${{ env.REL_VER_WITHOUT_v }}-CLI$CLI_REL_VER - docker push phylumio/phylum-ci:${{ env.REL_VER_WITHOUT_v }}-CLI$CLI_REL_VER - docker push ghcr.io/phylum-dev/phylum-ci:${{ env.REL_VER_WITHOUT_v }}-CLI$CLI_REL_VER + CLI_REL_VER=$(docker run --rm phylum-ci phylum --version | sed 's/phylum //') + docker tag phylum-ci "phylumio/phylum-ci:${{ env.REL_VER_WITHOUT_v }}-CLI${CLI_REL_VER}" + docker tag phylum-ci-slim "phylumio/phylum-ci:${{ env.REL_VER_WITHOUT_v }}-CLI${CLI_REL_VER}-slim" + docker tag phylum-ci "ghcr.io/phylum-dev/phylum-ci:${{ env.REL_VER_WITHOUT_v }}-CLI${CLI_REL_VER}" + docker tag phylum-ci-slim "ghcr.io/phylum-dev/phylum-ci:${{ env.REL_VER_WITHOUT_v }}-CLI${CLI_REL_VER}-slim" + docker push "phylumio/phylum-ci:${{ env.REL_VER_WITHOUT_v }}-CLI${CLI_REL_VER}" + docker push "phylumio/phylum-ci:${{ env.REL_VER_WITHOUT_v }}-CLI${CLI_REL_VER}-slim" + docker push "ghcr.io/phylum-dev/phylum-ci:${{ env.REL_VER_WITHOUT_v }}-CLI${CLI_REL_VER}" + docker push "ghcr.io/phylum-dev/phylum-ci:${{ env.REL_VER_WITHOUT_v }}-CLI${CLI_REL_VER}-slim" - - name: Tag and push latest docker image + - name: Tag and push latest docker images # Only tag and push `latest` when it's not a CLI pre-release # NOTE: This is an instance where the expression syntax (`${{ }}`) is required for the `if` conditional, # contrary to the GitHub workflow syntax documentation. Do not remove the expression syntax. if: ${{ !contains(github.event.client_payload.CLI_version, 'rc') }} run: | docker tag phylum-ci phylumio/phylum-ci:latest + docker tag phylum-ci-slim phylumio/phylum-ci:slim docker tag phylum-ci ghcr.io/phylum-dev/phylum-ci:latest + docker tag phylum-ci-slim ghcr.io/phylum-dev/phylum-ci:slim docker push phylumio/phylum-ci:latest + docker push phylumio/phylum-ci:slim docker push ghcr.io/phylum-dev/phylum-ci:latest + docker push ghcr.io/phylum-dev/phylum-ci:slim - name: Logout of Docker Hub if: always() diff --git a/.github/workflows/preview.yml b/.github/workflows/preview.yml index e7cc0ebd..a71e78d4 100644 --- a/.github/workflows/preview.yml +++ b/.github/workflows/preview.yml @@ -61,8 +61,8 @@ jobs: # poetry version rules do not provide for developmental releases as specified in PEP440. # It can be pieced together with these commands. run: | - poetry version $(poetry run semantic-release print-version --next) - poetry version $(poetry version --short).dev$GITHUB_RUN_NUMBER + poetry version "$(poetry run semantic-release print-version --next)" + poetry version "$(poetry version --short).dev${GITHUB_RUN_NUMBER}" - name: Run tox via poetry run: poetry run tox diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e6fc41d3..b677bdb6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -86,18 +86,18 @@ jobs: poetry install --verbose --sync --with test,ci - name: Set PHYLUM_ORIGINAL_VER value - run: echo "PHYLUM_ORIGINAL_VER=$(poetry version --short)" >> $GITHUB_ENV + run: echo "PHYLUM_ORIGINAL_VER=$(poetry version --short)" >> "${GITHUB_ENV}" - name: Set to next version for build run: | if [ "${{ inputs.prerelease }}" = "true" ]; then - poetry version $(poetry run semantic-release print-version --next --prerelease) + poetry version "$(poetry run semantic-release print-version --next --prerelease)" else - poetry version $(poetry run semantic-release print-version --next) + poetry version "$(poetry run semantic-release print-version --next)" fi - name: Set PHYLUM_REL_VER value - run: echo "PHYLUM_REL_VER=$(poetry version --short)" >> $GITHUB_ENV + run: echo "PHYLUM_REL_VER=$(poetry version --short)" >> "${GITHUB_ENV}" # NOTE: Run the tests for the current active Python version, as a sanity check. - name: Run tox via poetry @@ -150,9 +150,9 @@ jobs: else poetry run semantic-release publish -v DEBUG fi - echo "url=https://pypi.org/project/phylum/${{ env.PHYLUM_REL_VER }}/" >> $GITHUB_OUTPUT + echo "url=https://pypi.org/project/phylum/${{ env.PHYLUM_REL_VER }}/" >> "${GITHUB_OUTPUT}" - - name: Build docker image + - name: Build default docker image run: | docker build \ --tag phylum-ci \ @@ -162,13 +162,22 @@ jobs: --build-arg GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} \ . - - name: Test docker image with pre-built distributions + - name: Test default docker image with pre-built distributions + run: scripts/docker_tests.sh --image phylum-ci + + - name: Build slim docker image run: | - docker run --rm phylum-ci git --version - docker run --rm phylum-ci phylum-ci --version - docker run --rm phylum-ci phylum-ci --help - docker run --rm phylum-ci phylum-init --help - docker run --rm phylum-ci phylum --help + docker build \ + --tag phylum-ci-slim \ + --build-arg PKG_SRC=dist/phylum-*.whl \ + --build-arg PKG_NAME=phylum-*.whl \ + --build-arg BUILDKIT_INLINE_CACHE=1 \ + --build-arg GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} \ + --file Dockerfile.slim \ + . + + - name: Test slim docker image with pre-built distributions + run: scripts/docker_tests.sh --image phylum-ci-slim --slim - name: Login to Docker Hub run: docker login --username ${{ secrets.DOCKER_HUB_USERNAME }} --password ${{ secrets.DOCKER_HUB_TOKEN }} @@ -176,24 +185,32 @@ jobs: - name: Login to GitHub Container Registry run: docker login --username ${{ github.actor }} --password ${{ secrets.GITHUB_TOKEN }} ghcr.io - - name: Tag and push unique docker image + - name: Create specific docker tags and push them run: | - export CLI_REL_VER=$(docker run --rm phylum-ci phylum --version | sed 's/phylum //') - docker tag phylum-ci phylumio/phylum-ci:${{ env.PHYLUM_REL_VER }}-CLI$CLI_REL_VER - docker tag phylum-ci ghcr.io/phylum-dev/phylum-ci:${{ env.PHYLUM_REL_VER }}-CLI$CLI_REL_VER - docker push phylumio/phylum-ci:${{ env.PHYLUM_REL_VER }}-CLI$CLI_REL_VER - docker push ghcr.io/phylum-dev/phylum-ci:${{ env.PHYLUM_REL_VER }}-CLI$CLI_REL_VER - - - name: Tag and push latest docker image + CLI_REL_VER=$(docker run --rm phylum-ci phylum --version | sed 's/phylum //') + docker tag phylum-ci "phylumio/phylum-ci:${{ env.PHYLUM_REL_VER }}-CLI${CLI_REL_VER}" + docker tag phylum-ci-slim "phylumio/phylum-ci:${{ env.PHYLUM_REL_VER }}-CLI${CLI_REL_VER}-slim" + docker tag phylum-ci "ghcr.io/phylum-dev/phylum-ci:${{ env.PHYLUM_REL_VER }}-CLI${CLI_REL_VER}" + docker tag phylum-ci-slim "ghcr.io/phylum-dev/phylum-ci:${{ env.PHYLUM_REL_VER }}-CLI${CLI_REL_VER}-slim" + docker push "phylumio/phylum-ci:${{ env.PHYLUM_REL_VER }}-CLI${CLI_REL_VER}" + docker push "phylumio/phylum-ci:${{ env.PHYLUM_REL_VER }}-CLI${CLI_REL_VER}-slim" + docker push "ghcr.io/phylum-dev/phylum-ci:${{ env.PHYLUM_REL_VER }}-CLI${CLI_REL_VER}" + docker push "ghcr.io/phylum-dev/phylum-ci:${{ env.PHYLUM_REL_VER }}-CLI${CLI_REL_VER}-slim" + + - name: Tag and push latest docker images # Only tag and push `latest` when it's not a phylum-ci pre-release # NOTE: This is an instance where the expression syntax (`${{ }}`) is required for the `if` conditional, # contrary to the GitHub workflow syntax documentation. Do not remove the expression syntax. if: ${{ !inputs.prerelease }} run: | docker tag phylum-ci phylumio/phylum-ci:latest + docker tag phylum-ci-slim phylumio/phylum-ci:slim docker tag phylum-ci ghcr.io/phylum-dev/phylum-ci:latest + docker tag phylum-ci-slim ghcr.io/phylum-dev/phylum-ci:slim docker push phylumio/phylum-ci:latest + docker push phylumio/phylum-ci:slim docker push ghcr.io/phylum-dev/phylum-ci:latest + docker push ghcr.io/phylum-dev/phylum-ci:slim - name: Logout of Docker Hub if: always() diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 22e8cd7d..8dae31ed 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -91,15 +91,19 @@ jobs: - name: Run tox via poetry run: poetry run tox - # This job is meant to be a sanity check on the Docker image...that it can be created and + # This job is meant to be a sanity check on the Docker image...that it can be + # created with various Dockerfiles, from source or a built distribution, and # have the script entry points called without error. - docker: - name: Docker smoke test + docker-matrix: + name: ${{ matrix.dockerfile }} ${{ matrix.build }} smoke test runs-on: ubuntu-latest strategy: + fail-fast: false matrix: # It's only one Python version specified in a "matrix", but on purpose to stay DRY python-version: ["3.11"] + dockerfile: ["Dockerfile", "Dockerfile.slim"] + build: ["wheel", "source"] env: DOCKER_BUILDKIT: 1 steps: @@ -107,60 +111,61 @@ jobs: uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 - name: Install poetry + if: ${{ matrix.build == 'wheel' }} run: pipx install poetry==1.6.1 - name: Configure poetry + if: ${{ matrix.build == 'wheel' }} run: poetry config virtualenvs.in-project true - name: Set up Python + if: ${{ matrix.build == 'wheel' }} uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 with: python-version: ${{ matrix.python-version }} cache: 'poetry' - name: Install the project with poetry + if: ${{ matrix.build == 'wheel' }} run: | poetry env use python${{ matrix.python-version }} poetry check --lock poetry lock --no-update --no-cache poetry install --verbose --no-root --sync - - name: Build docker image from source - run: | - docker build \ - --tag phylum-ci:from-src \ - --cache-from phylumio/phylum-ci:latest \ - --build-arg GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} \ - . - - - name: Test docker image built from source - run: | - docker run --rm phylum-ci:from-src git --version - docker run --rm phylum-ci:from-src phylum-ci --version - docker run --rm phylum-ci:from-src phylum-ci --help - docker run --rm phylum-ci:from-src phylum-init --help - docker run --rm phylum-ci:from-src phylum --help - - name: Build wheel and source distribution + if: ${{ matrix.build == 'wheel' }} run: poetry build -vvv - name: Build docker image with pre-built distributions + if: ${{ matrix.build == 'wheel' }} run: | docker build \ - --tag phylum-ci:from-dist \ + --tag phylum-ci \ --cache-from phylumio/phylum-ci:latest \ --build-arg PKG_SRC=dist/phylum-*.whl \ --build-arg PKG_NAME=phylum-*.whl \ --build-arg GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} \ + --file ${{ matrix.dockerfile }} \ . - - name: Test docker image built from dist + - name: Build docker image from source + if: ${{ matrix.build == 'source' }} run: | - docker run --rm phylum-ci:from-dist git --version - docker run --rm phylum-ci:from-dist phylum-ci --version - docker run --rm phylum-ci:from-dist phylum-ci --help - docker run --rm phylum-ci:from-dist phylum-init --help - docker run --rm phylum-ci:from-dist phylum --help + docker build \ + --tag phylum-ci \ + --cache-from phylumio/phylum-ci:latest \ + --build-arg GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} \ + --file ${{ matrix.dockerfile }} \ + . + + - name: Test slim docker image built from ${{ matrix.build }} + if: ${{ matrix.dockerfile == 'Dockerfile.slim' }} + run: scripts/docker_tests.sh --image phylum-ci --slim + + - name: Test full docker image built from ${{ matrix.build }} + if: ${{ matrix.dockerfile == 'Dockerfile' }} + run: scripts/docker_tests.sh --image phylum-ci # This job reports the results of the test jobs above and is used to enforce status checks in # the repo settings without needing to update those settings everytime the test jobs are updated. @@ -168,20 +173,8 @@ jobs: name: Test rollup runs-on: ubuntu-latest if: always() - needs: [QA, test-matrix, docker] + needs: [QA, test-matrix, docker-matrix] steps: - - name: Check for test jobs failure - if: > - (needs.QA.result != 'success') - || (needs.test-matrix.result != 'success') - || (needs.docker.result != 'success') - run: | - echo "At least one test job was not successful" - exit 1 - - - name: Confirm test jobs success - if: > - (needs.QA.result == 'success') - && (needs.test-matrix.result == 'success') - && (needs.docker.result == 'success') - run: echo "All test jobs were successful" + - name: Check for test jobs failure or cancellation + if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') + run: exit 1 diff --git a/Dockerfile b/Dockerfile index d8254485..f7121253 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,6 +14,7 @@ # This Dockerfile can also be used in CI, as part of the release pipeline. # The goal there is to ensure that the exact Python package that was built and # released is the one that is installed while creating this image. Prerequisites are: +# # * The package has already been built (e.g., `poetry build -vvv`) # * There is exactly one wheel available to reference by glob expression # @@ -61,9 +62,14 @@ # specifying the `entrypoint.sh` script, providing extra parameters as a *quoted* string: # # $ docker run --rm --entrypoint entrypoint.sh phylumio/phylum-ci:latest "ls -alh /" +# +# Images built from this Dockerfile can be tested for basic functionality: +# +# $ docker build --tag phylum-ci . +# $ scripts/docker_tests.sh --image phylum-ci ########################################################################################## -FROM python:3.11-slim-bullseye AS builder +FROM python:3.11-slim-bookworm AS builder # PKG_SRC is the path to a built distribution/wheel and PKG_NAME is the name of the built # distribution/wheel. Both can optionally be specified in glob form. When not defined, @@ -78,6 +84,7 @@ ENV PHYLUM_VENV="/opt/venv" ENV PHYLUM_VENV_PIP="${PHYLUM_VENV}/bin/pip" ENV PIP_NO_COMPILE=1 ENV PIP_DISABLE_PIP_VERSION_CHECK=1 +ENV POETRY_VERSION="1.6.1" WORKDIR ${APP_PATH} @@ -87,7 +94,7 @@ RUN set -eux; \ RUN set -eux; \ python -m venv ${POETRY_VENV}; \ ${POETRY_VENV}/bin/pip install --no-cache-dir --upgrade pip setuptools; \ - ${POETRY_VENV}/bin/pip install --no-cache-dir poetry==1.6.1 poetry-plugin-export + ${POETRY_VENV}/bin/pip install --no-cache-dir poetry==${POETRY_VERSION} poetry-plugin-export # Copy the bare minimum needed for specifying dependencies. # This will enable better layer caching and faster builds when iterating locally. @@ -105,15 +112,16 @@ RUN --mount=type=cache,id=pip,target=/root/.cache/pip \ set -eux; \ ${PHYLUM_VENV_PIP} cache info; \ ${PHYLUM_VENV_PIP} cache list; \ - ${PHYLUM_VENV_PIP} install -r requirements.txt + ${PHYLUM_VENV_PIP} install -r requirements.txt pipenv poetry==${POETRY_VERSION} COPY "${PKG_SRC:-.}" . RUN ${PHYLUM_VENV_PIP} install --no-cache-dir ${PKG_NAME:-.} RUN find ${PHYLUM_VENV} -type f -name '*.pyc' -delete -# Place in a directory included in the final layer and also known to be part of the $PATH +# Place the ENTRYPOINT alternative script in a directory included +# in the final layer and also known to be part of the $PATH COPY entrypoint.sh ${PHYLUM_VENV}/bin/ -FROM python:3.11-slim-bullseye +FROM python:3.11-slim-bookworm # CLI_VER specifies the Phylum CLI version to install in the image. # Values should be provided in a format acceptable to the `phylum-init` script. @@ -129,24 +137,111 @@ ARG PHYLUM_API_URI # and therefore increase the API rate limit. ARG GITHUB_TOKEN +# Ref: https://docs.docker.com/engine/reference/builder/#automatic-platform-args-in-the-global-scope +ARG TARGETOS +ARG TARGETARCH + LABEL maintainer="Phylum, Inc. " LABEL org.opencontainers.image.source="https://github.com/phylum-dev/phylum-ci" ENV PHYLUM_VENV="/opt/venv" -ENV PATH=${PHYLUM_VENV}/bin:$PATH + +ENV GRADLE_PATH="/opt/gradle/bin" +ENV GO_PATH="/usr/local/go/bin" +ENV CARGO_PATH="/root/.cargo/bin" +ENV PATH=${PHYLUM_VENV}/bin:$PATH:${GRADLE_PATH}:${GO_PATH}:${CARGO_PATH} + ENV PYTHONDONTWRITEBYTECODE=1 -# Copy only Python packages to limit the image size +# Ref: https://learn.microsoft.com/en-us/dotnet/core/tools/dotnet-environment-variables +ENV DOTNET_CLI_TELEMETRY_OPTOUT=1 +ENV DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 +ENV DOTNET_NOLOGO=1 + +# Copy only Python packages to limit the image size. This includes the `phylum` package with it's +# `phylum-ci` and `phylum-init` entry points, plus the `pip`, `pipenv`, and `poetry` required tools. COPY --from=builder ${PHYLUM_VENV} ${PHYLUM_VENV} -RUN set -eux; \ +# Specify the shell options here, based on the use of pipelines. +# Ref: https://github.com/hadolint/hadolint/wiki/DL4006 +SHELL ["/bin/bash", "-euxo", "pipefail", "-c"] +RUN \ + # Install pre-requisites and package manager versions for `npm`, `bundle`, and `mvn` tools apt-get update; \ apt-get upgrade --yes; \ - apt-get install --yes --no-install-recommends git; \ - chmod +x ${PHYLUM_VENV}/bin/entrypoint.sh; \ - phylum-init -vvv --phylum-release ${CLI_VER:-latest} --global-install; \ + apt-get install --yes --no-install-recommends \ + git \ + npm \ + bundler \ + maven \ + curl \ + unzip \ + jq \ + ; \ + # Make ENTRYPOINT alternative script available + chmod +x "${PHYLUM_VENV}/bin/entrypoint.sh"; \ + # + # Install Phylum CLI + phylum-init -vvv --phylum-release "${CLI_VER:-latest}" --global-install; \ + # + # Install and enable `corepack` with cached instances of the latest major versions of `yarn` and `pnpm` tools + npm install -g corepack@latest; \ + corepack pack yarn@stable pnpm@latest; \ + corepack enable yarn pnpm; \ + # + # Create an "alias" for `curl` with secure and common options + printf '#!/bin/bash\ncurl --proto "=https" --tlsv1.2 -sSfL "$@"\n' > /usr/bin/curls; \ + chmod +x /usr/bin/curls; \ + # + # Manual install of `gradle` + GRADLE_VERSION=$(curls https://services.gradle.org/versions/current | jq -r '.version'); \ + GRADLE_DL="https://services.gradle.org/distributions/gradle-${GRADLE_VERSION}-bin.zip"; \ + GRADLE_DL_SHA256=$(curls "${GRADLE_DL}.sha256"); \ + curls -o gradle.zip "${GRADLE_DL}"; \ + printf "%s *gradle.zip" "${GRADLE_DL_SHA256}" | sha256sum -c -; \ + mkdir /opt/gradle; \ + unzip -d /opt/gradle gradle.zip; \ + mv /opt/gradle/gradle-"${GRADLE_VERSION}"/* /opt/gradle/; \ + rm gradle.zip && rmdir "/opt/gradle/gradle-${GRADLE_VERSION}/"; \ + # + # Manual install of `go` + # Ref: https://pkg.go.dev/golang.org/x/website/internal/dl + GO_DL_URL="https://go.dev/dl/?mode=json"; \ + GO_DL_REL=$(curls "${GO_DL_URL}" | jq '.[0].files[] | select(.os==env.TARGETOS and .arch==env.TARGETARCH)'); \ + GO_DL_FILENAME=$(echo "${GO_DL_REL}" | jq -r '.filename'); \ + GO_DL_SHA256=$(echo "${GO_DL_REL}" | jq -r '.sha256'); \ + curls -o go.tgz "https://go.dev/dl/${GO_DL_FILENAME}"; \ + printf "%s *go.tgz" "${GO_DL_SHA256}" | sha256sum -c -; \ + rm -rf /usr/local/go; \ + tar -C /usr/local -xzf go.tgz; \ + rm go.tgz; \ + # + # Manual install of Rust to get `cargo` tool + curls https://sh.rustup.rs | sh -s -- -y --profile minimal; \ + # + # Install .NET SDK to get `dotnet` tool + # Ref: https://github.com/dotnet/core/tree/main/release-notes + DOTNET_SDK_LATEST_CHANNEL_VER=$( \ + curls https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json | \ + jq -r '[."releases-index"[] | select(."support-phase"=="active")] | first."channel-version"' \ + ); \ + DEB_MAJ_VER=$(cut -d "." -f1 /etc/debian_version); \ + curls -o ms-prod.deb "https://packages.microsoft.com/config/debian/${DEB_MAJ_VER}/packages-microsoft-prod.deb"; \ + dpkg --install ms-prod.deb; \ + rm ms-prod.deb; \ + apt-get update; \ + apt-get install --yes --no-install-recommends "dotnet-sdk-${DOTNET_SDK_LATEST_CHANNEL_VER}"; \ + # + # Final cleanup + rm /usr/bin/curls; \ + apt-get remove --yes --auto-remove \ + curl \ + unzip \ + jq \ + ; \ apt-get purge --yes --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \ rm -rf /var/lib/apt/lists/*; \ - find / -type f -name '*.pyc' -delete + rm -rf /tmp/*; \ + find / -type f -name '*.pyc' -delete; CMD ["phylum-ci"] diff --git a/Dockerfile.slim b/Dockerfile.slim new file mode 100644 index 00000000..cf9867d0 --- /dev/null +++ b/Dockerfile.slim @@ -0,0 +1,174 @@ +# syntax=docker/dockerfile:1 +# The syntax statement above is to make use of buildkit's build mounts for caching + +# NOTE: +# This Dockerfile is used to build the `slim` tags for the `phylum-ci` images. It differs +# from the primary Dockerfile in that it does not install the required tools needed for +# lockfile generation: https://docs.phylum.io/docs/lockfile_generation (with the exception +# of the `pip` tool). The image created from this Dockerfile is significantly smaller and +# will allow the integrations relying on it to complete faster. It is useful for those +# instances where *no* manifest files are present and/or *only* lockfiles are used. + +########################################################################################## +# This Dockerfile makes use of BuildKit mode, which needs to be enabled client side first: +# +# $ export DOCKER_BUILDKIT=1 +# +# This Dockerfile can be used to build the project's package within the image it creates. +# To do so, simply build the image WITHOUT any build args specified (e.g., `--build-arg`): +# +# $ docker build --tag phylum-ci . +# +# This Dockerfile can also be used in CI, as part of the release pipeline. +# The goal there is to ensure that the exact Python package that was built and +# released is the one that is installed while creating this image. Prerequisites are: +# +# * The package has already been built (e.g., `poetry build -vvv`) +# * There is exactly one wheel available to reference by glob expression +# +# To make use of this feature, build the image WITH build args specified: +# +# $ export PKG_SRC=dist/phylum-*.whl +# $ export PKG_NAME=phylum-*.whl +# $ docker build --tag phylum-ci --build-arg PKG_SRC --build-arg PKG_NAME . +# +# Another build arg is exposed to optionally specify the Phylum CLI version to install: +# +# $ docker build --tag phylum-ci --build-arg CLI_VER=v4.8.0 . +# +# The PHYLUM_API_URI build arg is exposed to optionally specify the URI of a Phylum API +# instance to use: +# +# $ export PHYLUM_API_URI=https://api.staging.phylum.io +# $ docker build --tag phylum-ci --build-arg PHYLUM_API_URI . +# +# Another build arg is exposed to optionally specify a GitHub Personal Access Token (PAT): +# +# $ docker build --tag phylum-ci --build-arg GITHUB_TOKEN . +# +# Providing a build argument like this (without a value) works when there is already an +# environment variable defined with the same name. Providing a GitHub PAT is useful to +# make authenticated requests and therefore increase the API rate limit. +# +# To make use of BuildKit's inline layer caching feature, add the `BUILDKIT_INLINE_CACHE` +# build argument to any instance of building an image. Then, that image can be used +# locally or remotely (if it was pushed to a repository) to warm the build cache by using +# the `--cache-from` argument: +# +# $ docker build --tag phylumio/phylum-ci:cache --build-arg BUILDKIT_INLINE_CACHE=1 . +# $ docker push phylumio/phylum-ci:cache && docker image rm phylumio/phylum-ci:cache +# $ docker build --tag phylumio/phylum-ci:faster --cache-from phylumio/phylum-ci:cache . +# +# There is no ENTRYPOINT in this Dockerfile by design. That way, it is possible to provide +# unquoted extra parameters to run arbitrary commands in the context of the container: +# +# $ docker run --rm phylumio/phylum-ci:latest ls -alh / +# +# However, there may be cases where an entrypoint is needed. One is provided and placed in +# a directory that will be included in the final layer and also known to be part of the +# $PATH. To make use of it, add the `--entrypoint` option to a docker run command, +# specifying the `entrypoint.sh` script, providing extra parameters as a *quoted* string: +# +# $ docker run --rm --entrypoint entrypoint.sh phylumio/phylum-ci:latest "ls -alh /" +# +# Images built from this Dockerfile can be tested for basic functionality: +# +# $ docker build --tag phylum-ci . +# $ scripts/docker_tests.sh --image phylum-ci --slim +########################################################################################## + +FROM python:3.11-slim-bookworm AS builder + +# PKG_SRC is the path to a built distribution/wheel and PKG_NAME is the name of the built +# distribution/wheel. Both can optionally be specified in glob form. When not defined, +# the values will default to the root of the package (i.e., `pyproject.toml` path). +ARG PKG_SRC +ARG PKG_NAME + +ENV APP_PATH="/app" +ENV POETRY_VENV="${APP_PATH}/.venv" +ENV POETRY_PATH="${POETRY_VENV}/bin/poetry" +ENV PHYLUM_VENV="/opt/venv" +ENV PHYLUM_VENV_PIP="${PHYLUM_VENV}/bin/pip" +ENV PIP_NO_COMPILE=1 +ENV PIP_DISABLE_PIP_VERSION_CHECK=1 +ENV POETRY_VERSION="1.6.1" + +WORKDIR ${APP_PATH} + +RUN set -eux; \ + python -m venv ${PHYLUM_VENV}; \ + ${PHYLUM_VENV_PIP} install --no-cache-dir --upgrade pip setuptools +RUN set -eux; \ + python -m venv ${POETRY_VENV}; \ + ${POETRY_VENV}/bin/pip install --no-cache-dir --upgrade pip setuptools; \ + ${POETRY_VENV}/bin/pip install --no-cache-dir poetry==${POETRY_VERSION} poetry-plugin-export + +# Copy the bare minimum needed for specifying dependencies. +# This will enable better layer caching and faster builds when iterating locally. +# `--without-hashes` is used to ensure the `pip` cache mount is used for packages that +# would otherwise only match the sdist and therefore have to be built for every run. +# References: +# * https://pythonspeed.com/articles/pipenv-docker/ +# * https://hub.docker.com/r/docker/dockerfile +COPY pyproject.toml poetry.lock ./ +RUN ${POETRY_PATH} export --without-hashes --format requirements.txt --output requirements.txt + +# Cache the pip installed dependencies for faster builds when iterating locally. +# NOTE: This `--mount` feature requires BUILDKIT to be used +RUN --mount=type=cache,id=pip,target=/root/.cache/pip \ + set -eux; \ + ${PHYLUM_VENV_PIP} cache info; \ + ${PHYLUM_VENV_PIP} cache list; \ + ${PHYLUM_VENV_PIP} install -r requirements.txt +COPY "${PKG_SRC:-.}" . +RUN ${PHYLUM_VENV_PIP} install --no-cache-dir ${PKG_NAME:-.} +RUN find ${PHYLUM_VENV} -type f -name '*.pyc' -delete + +# Place the ENTRYPOINT alternative script in a directory included +# in the final layer and also known to be part of the $PATH +COPY entrypoint.sh ${PHYLUM_VENV}/bin/ + +FROM python:3.11-slim-bookworm + +# CLI_VER specifies the Phylum CLI version to install in the image. +# Values should be provided in a format acceptable to the `phylum-init` script. +# When not defined, the value will default to `latest`. +ARG CLI_VER + +# PHYLUM_API_URI is an optional build argument that can be used to specify +# the URI of a Phylum API instance to use. +ARG PHYLUM_API_URI + +# GITHUB_TOKEN is an optional build argument that can be used to provide a +# GitHub Personal Access Token (PAT) in order to make authenticated requests +# and therefore increase the API rate limit. +ARG GITHUB_TOKEN + +LABEL maintainer="Phylum, Inc. " +LABEL org.opencontainers.image.source="https://github.com/phylum-dev/phylum-ci" + +ENV PHYLUM_VENV="/opt/venv" +ENV PATH=${PHYLUM_VENV}/bin:$PATH +ENV PYTHONDONTWRITEBYTECODE=1 + +# Copy only Python packages to limit the image size. This includes the +# `phylum` package with it's `phylum-ci` and `phylum-init` entry points. +COPY --from=builder ${PHYLUM_VENV} ${PHYLUM_VENV} + +RUN set -eux; \ + # Install pre-requisites + apt-get update; \ + apt-get upgrade --yes; \ + apt-get install --yes --no-install-recommends git; \ + # Make ENTRYPOINT alternative script available + chmod +x "${PHYLUM_VENV}/bin/entrypoint.sh"; \ + # Install Phylum CLI + phylum-init -vvv --phylum-release "${CLI_VER:-latest}" --global-install; \ + # Cleanup + apt-get purge --yes --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \ + rm -rf /var/lib/apt/lists/*; \ + rm -rf /tmp/*; \ + find / -type f -name '*.pyc' -delete; + +CMD ["phylum-ci"] diff --git a/README.md b/README.md index 35c3a6c4..4a4d55cc 100644 --- a/README.md +++ b/README.md @@ -79,27 +79,45 @@ docker run --rm phylumio/phylum-ci phylum-ci --help # Export a Phylum token (e.g., from `phylum auth token`) export PHYLUM_API_KEY=$(phylum auth token) -# Run it from a git repo directory containing at least one lockfile +# Run it from a git repo directory containing at least one supported lockfile or manifest docker run -it --rm -e PHYLUM_API_KEY --mount type=bind,src=$(pwd),dst=/phylum -w /phylum phylumio/phylum-ci ``` -The Docker image contains `git` and the installed `phylum` Python package. -It also contains an installed version of the Phylum CLI. -An advantage of using the Docker image is that the complete environment is packaged and made available with components -that are known to work together. +The default Docker image contains `git` and the installed `phylum` Python package. +It also contains an installed version of the Phylum CLI and all required tools needed for [lockfile generation]. +An advantage of using the default Docker image is that the complete environment is packaged and made available with +components that are known to work together. + +One disadvantage to the default image is it's size. It can take a while to download and may provide more tools than +required for your specific use case. Special `slim` tags of the `phylum-ci` image are provided as an alternative. +These tags differ from the default image in that they do not contain the required tools needed for [lockfile generation] +(with the exception of the `pip` tool). The `slim` tags are significantly smaller and will allow integrations relying +on them to complete faster. They are useful for those instances where *no* manifest files are present and/or *only* +lockfiles are used. + +```sh +# Get the "latest" `slim` tagged image +docker pull phylumio/phylum-ci:slim +``` When using the `latest` tagged image, the version of the Phylum CLI is the `latest` available. There are additional image tag options available to specify a specific release of the `phylum-ci` project and a specific -version of the Phylum CLI, in the form of `-CLIv`. Here are image tag examples: +version of the Phylum CLI, in the form of `-CLIv`. +Each of these also has a `-slim` variant that does not support [lockfile generation]. Here are image tag examples: ```sh # Get the most current release of *both* `phylum-ci` and the Phylum CLI docker pull phylumio/phylum-ci:latest -# Get the image with `phylum-ci` version 0.24.1 and Phylum CLI version 4.7.0 -docker pull phylumio/phylum-ci:0.24.1-CLIv4.7.0 +# Get the image with `phylum-ci` version 0.35.2 and Phylum CLI version 5.7.1 +docker pull phylumio/phylum-ci:0.35.2-CLIv5.7.1 + +# Get the `slim` image with `phylum-ci` version 0.36.0 and Phylum CLI version 5.7.1 +docker pull phylumio/phylum-ci:0.36.0-CLIv5.7.1-slim ``` +[lockfile generation]: https://docs.phylum.io/docs/lockfile_generation + #### `phylum-init` Script Entry Point The `phylum-init` script can be used to fetch and install the Phylum CLI. diff --git a/scripts/docker_tests.sh b/scripts/docker_tests.sh new file mode 100755 index 00000000..bc03fff7 --- /dev/null +++ b/scripts/docker_tests.sh @@ -0,0 +1,104 @@ +#!/bin/sh + +# This script is meant to provide a basic set of tests. It simply confirms that +# each of the tools expected for operation is present within the image. The +# path and version of the tool is shown and in some cases additional information +# is provided, like help or info output. + +set -eu + +# Check for a required command +require_command() { + cmd="$1" + help_msg="${2:-}" + + if ! type "${cmd}" > /dev/null 2>&1; then + echo "ERROR: This script requires \`${cmd}\`. Please install it and re-run this script to continue." >&2 + if [ -n "${help_msg}" ]; then + printf "\n" >&2 + echo "${help_msg}" >&2 + fi + exit 1 + fi +} + +usage() { + cat 1>&2 <&2 + usage + exit 1 + ;; + esac +done + +if [ -z "${IMAGE:-}" ]; then + IMAGE="maxrake/phylum-ci:manifest_support" +fi + +# These are the commands to ensure the base pre-requisites are available +SLIM_COMMANDS=$(cat <