diff --git a/.github/workflows/create-env.yaml b/.github/workflows/create-env.yaml index 2ac0458f..21162b41 100644 --- a/.github/workflows/create-env.yaml +++ b/.github/workflows/create-env.yaml @@ -14,13 +14,19 @@ on: jobs: build: name: Build & Push - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 env: - IMAGE_VERSION: '2.2.1' + MAJOR_VERSION: 3 + MINOR_VERSION: 0 IMAGE_NAME: create-env steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + with: + platforms: arm64 - name: Prepare id: prepare @@ -28,81 +34,87 @@ jobs: curl -sSL \ 'https://raw.githubusercontent.com/bioconda/bioconda-common/master/common.sh' \ | sed -n \ - 's/BIOCONDA_UTILS_TAG=v/::set-output name=bioconda_utils_version::/p' + 's/BIOCONDA_UTILS_TAG=v/bioconda_utils_version=/p' \ + >> $GITHUB_OUTPUT - name: Build - id: buildah-build + id: build uses: redhat-actions/buildah-build@v2 with: image: ${{ env.IMAGE_NAME }} tags: >- latest - ${{ env.IMAGE_VERSION }} + ${{ env.MAJOR_VERSION }} + ${{ env.MAJOR_VERSION }}.${{ env.MINOR_VERSION }} + archs: amd64,arm64 context: ./images/${{ env.IMAGE_NAME }} - dockerfiles: | + containerfiles: | ./images/${{ env.IMAGE_NAME }}/Dockerfile build-args: | bioconda_utils_version=${{ steps.prepare.outputs.bioconda_utils_version }} - name: Test run: | - image='${{ steps.buildah-build.outputs.image }}' - ids="$( - for tag in ${{ steps.buildah-build.outputs.tags }} ; do - buildah images --quiet --no-trunc "${image}:${tag}" - done - )" - ids="$( printf %s "${ids}" | sort -u )" - for id in ${ids} ; do - podman history "${id}" - buildah bud \ - --build-arg=base="${id}" \ - --file=Dockerfile.test \ - "images/${image}" - done + set -x + image='${{ steps.build.outputs.image }}' + for tag in ${{ steps.build.outputs.tags }} ; do + buildah manifest inspect \ + "${image}:${tag}" \ + | jq '.manifests|{([.[].digest]|sort|join("+")): [.[]|["'"${tag}"'", .platform.architecture, .digest]|join(" ")]}' + done \ + | jq -rs 'add|add[]' \ + | while read tag arch digest ; do + podman images --format='{{.ID}}|{{.Digest}}|{{.RepoDigests}}' \ + | sed -n "/${digest}/{s/|.*//p;q}" \ + | xargs -n1 -- \ + sh -xc \ + 'podman history "${1}" ; podman inspect "${1}"' -- + buildah bud \ + --arch="${arch}" \ + --build-arg=base="${image}:${tag}" \ + --file=Dockerfile.test \ + "images/${image}" + done buildah rmi --prune || true - name: Check Tags run: | # FIX upstream: Quay.io does not support immutable images currently. # => Try to use the REST API to check for duplicate tags. - page=0 - has_additional='true' - while [ "${has_additional}" = 'true' ] ; do - page="$(( page + 1 ))" - respone="$( - curl -sL \ - 'https://quay.io/api/v1/repository/bioconda/${{ steps.buildah-build.outputs.image }}/tag?limit=100&page='"${page}" - )" - has_additional="$( printf %s "${respone}" | jq -r '.has_additional' )" - existing_tags="$( - printf %s "${respone}" \ - | jq -r '.tags[]|.name' - )" \ - || { - >&2 printf %s\\n \ - 'Could not get list of image tags.' \ - 'Does the repository exist on Quay.io?' \ - 'Quay.io REST API response was:' \ - "${respone}" - exit 1 - } - for tag in ${{ steps.buildah-build.outputs.tags }} ; do - if [ \! "${tag}" = latest ] ; then + response="$( + curl -sL \ + 'https://quay.io/api/v1/repository/bioconda/${{ steps.build.outputs.image }}/tag/' + )" + + existing_tags="$( + printf %s "${response}" \ + | jq -r '.tags[]|select(.end_ts == null or .end_ts >= now)|.name' + )" \ + || { + printf %s\\n \ + 'Could not get list of image tags.' \ + 'Does the repository exist on Quay.io?' \ + 'Quay.io REST API response was:' \ + "${response}" + exit 1 + } + for tag in ${{ steps.build.outputs.tags }} ; do + case "${tag}" in + latest | '${{ env.MAJOR_VERSION }}' ) ;; + * ) if printf %s "${existing_tags}" | grep -qxF "${tag}" ; then - >&2 printf 'Tag %s already exists!\n' "${tag}" + printf 'Tag %s already exists!\n' "${tag}" exit 1 fi - fi - done + esac done - if: ${{ github.ref == 'refs/heads/main' }} name: Push uses: redhat-actions/push-to-registry@v2 with: - image: ${{ steps.buildah-build.outputs.image }} - tags: ${{ steps.buildah-build.outputs.tags }} + image: ${{ steps.build.outputs.image }} + tags: ${{ steps.build.outputs.tags }} registry: ${{ secrets.QUAY_BIOCONDA_REPO }} username: ${{ secrets.QUAY_BIOCONDA_USERNAME }} password: ${{ secrets.QUAY_BIOCONDA_TOKEN }} @@ -110,18 +122,24 @@ jobs: - if: ${{ github.ref == 'refs/heads/main' }} name: Test Pushed run: | - image='${{ steps.buildah-build.outputs.image }}' - ids="$( - for tag in ${{ steps.buildah-build.outputs.tags }} ; do - buildah images --quiet --no-trunc "${image}:${tag}" - done - )" - ids="$( printf %s "${ids}" | sort -u )" - for id in ${ids} ; do - podman history "${id}" - buildah bud \ - --build-arg=base="${id}" \ - --file=Dockerfile.test \ - "images/${image}" - done + set -x + image='${{ steps.build.outputs.image }}' + for tag in ${{ steps.build.outputs.tags }} ; do + buildah manifest inspect \ + "${image}:${tag}" \ + | jq '.manifests|{([.[].digest]|sort|join("+")): [.[]|["'"${tag}"'", .platform.architecture, .digest]|join(" ")]}' + done \ + | jq -rs 'add|add[]' \ + | while read tag arch digest ; do + podman images --format='{{.ID}}|{{.Digest}}|{{.RepoDigests}}' \ + | sed -n "/${digest}/{s/|.*//p;q}" \ + | xargs -n1 -- \ + sh -xc \ + 'podman history "${1}" ; podman inspect "${1}"' -- + buildah bud \ + --arch="${arch}" \ + --build-arg=base="${image}:${tag}" \ + --file=Dockerfile.test \ + "images/${image}" + done buildah rmi --prune || true diff --git a/images/create-env/CHANGELOG.md b/images/create-env/CHANGELOG.md index 33195a62..cd5a8d32 100644 --- a/images/create-env/CHANGELOG.md +++ b/images/create-env/CHANGELOG.md @@ -1,6 +1,23 @@ # Changelog +## bioconda/create-env 3.0 (2023-10-17) + +### Changed + +- Add linux-aarch64 image; bioconda/create-env is now a multiplatform manifest. + +- Change to a simple "major.minor" version scheme and offer mutable "major" tag. + +- Drop defaults channel from included config. + +- Use Miniforge installer to build this image. + +- Rebuilt on the latest base image with Debian 12.2 / BusyBox 1.36.1. + +- Do not install findutils, sed if provided by the base image (as is currently). + + ## bioconda/create-env 2.2.1 (2022-10-14) ### Changed diff --git a/images/create-env/Dockerfile b/images/create-env/Dockerfile index aeefa564..93b83948 100644 --- a/images/create-env/Dockerfile +++ b/images/create-env/Dockerfile @@ -1,5 +1,5 @@ # Use the exact conda, mamba versions as used in bioconda-recipes' builds. -ARG bioconda_utils_version='1.1.3' +ARG bioconda_utils_version FROM quay.io/bioconda/bioconda-utils-build-env-cos7:${bioconda_utils_version} as bioconda-build-env RUN /opt/conda/bin/conda list \ --export '^(conda|mamba)$' \ @@ -7,13 +7,15 @@ RUN /opt/conda/bin/conda list \ > /tmp/requirements.txt -# Use Debian instead of BusyBox base since Miniconda currently needs external zlib. -FROM quay.io/bioconda/base-glibc-debian-bash as build +FROM quay.io/bioconda/base-glibc-busybox-bash as build WORKDIR /tmp/work COPY --from=bioconda-build-env /tmp/requirements.txt ./ COPY install-conda print-env-activate create-env ./ -ADD https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh ./miniconda.sh +RUN arch="$( uname -m )" \ + && \ + wget --quiet -O ./miniconda.sh \ + "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-${arch}.sh" RUN ./install-conda ./requirements.txt /opt/create-env diff --git a/images/create-env/Dockerfile.test b/images/create-env/Dockerfile.test index 1eb7bf75..9c2566ae 100644 --- a/images/create-env/Dockerfile.test +++ b/images/create-env/Dockerfile.test @@ -1,43 +1,48 @@ ARG base FROM "${base}" -RUN CONDA_PKGS_DIRS="/tmp/pkgs" \ +RUN set -x && \ + CONDA_PKGS_DIRS="/tmp/pkgs" \ /opt/create-env/env-execute \ create-env \ --conda=mamba \ /usr/local \ - file findutils -RUN . /usr/local/env-activate.sh && \ + file findutils grep +RUN set -x && \ + . /usr/local/env-activate.sh && \ if find /opt/create-env \ + -xdev \ -type f \ -exec file {} \+ \ - 2> /dev/null \ | grep 'not stripped' \ ; then \ >&2 printf 'found unstripped binaries\n' ; exit 1 \ ; fi -RUN . /usr/local/env-activate.sh && \ +RUN set -x && \ + . /usr/local/env-activate.sh && \ if find /opt/create-env \ + -xdev \ -type f \ -name \*.a \ - 2> /dev/null \ | grep . \ ; then \ >&2 printf 'found static libraries\n' ; exit 1 \ ; fi -FROM "${base}" as build_bioconda_utils -RUN /opt/create-env/env-execute \ +FROM "${base}" as build_bioconda_package +RUN set -x && \ + /opt/create-env/env-execute \ create-env \ --conda=mamba \ --strip-files=\* \ /usr/local \ - bioconda-utils -FROM bioconda/base-glibc-busybox-bash -COPY --from=build_bioconda_utils /usr/local /usr/local -RUN /usr/local/env-execute \ - bioconda-utils --version \ + catfasta2phyml +FROM quay.io/bioconda/base-glibc-busybox-bash +COPY --from=build_bioconda_package /usr/local /usr/local +RUN set -x && \ + /usr/local/env-execute \ + catfasta2phyml --version \ && \ [ ! "${CONDA_PREFIX}" = /usr/local ] \ && \ @@ -45,11 +50,12 @@ RUN /usr/local/env-execute \ && \ [ "${CONDA_PREFIX}" = /usr/local ] \ && \ - bioconda-utils --version + catfasta2phyml --version FROM "${base}" as build_conda -RUN /opt/create-env/env-execute \ +RUN set -x && \ + /opt/create-env/env-execute \ create-env \ --conda=mamba \ --env-activate-args='--prefix-is-base' \ @@ -58,10 +64,11 @@ RUN /opt/create-env/env-execute \ --remove-paths=\*.pyc \ /opt/conda \ conda -FROM bioconda/base-glibc-busybox-bash +FROM quay.io/bioconda/base-glibc-busybox-bash COPY --from=build_conda /opt/conda /opt/conda COPY --from=build_conda /opt/conda/env-activate.sh /usr/local/ -RUN /usr/local/env-execute \ +RUN set -x && \ + /usr/local/env-execute \ conda info --all \ && \ { set -x && . /usr/local/env-activate.sh && set +x ; } \ diff --git a/images/create-env/create-env b/images/create-env/create-env index 324f0581..fde5bffc 100755 --- a/images/create-env/create-env +++ b/images/create-env/create-env @@ -158,16 +158,33 @@ if [ -n "${strip_files_globs}" ] ; then # Strip binaries. (Run strip on all files; ignore errors for non-ELF files.) # Limit open fds (ulimit -n) for strip (small number chosen arbitrarily). # (To avoid "could not create temporary file to hold stripped copy: Too many open files") + + # Filter out the binaries currently in use by the pipeline via sed below. + skip_inode_expressions="$( + command -v -- find xargs sed strip \ + | xargs -- stat -L -c '-e /^%d,%i:/d' -- + )" find "${prefix}" \ -type f \ \( "${@}" \) \ -print0 \ | xargs \ - --null \ + -0 \ + -n 64 \ + -- \ + stat -L -c '%d,%i:%n' -- \ + | sed \ + ${skip_inode_expressions} \ + -e 's/^[^:]*://' \ + | tr \\n \\0 \ + | + xargs \ + -0 \ -n 64 \ + -- \ strip -- \ 2>&1 \ - | grep -v ': file format not recognized' \ + | sed '/: file format not recognized/d' \ || true ) fi diff --git a/images/create-env/install-conda b/images/create-env/install-conda index e86d5057..a3b9b332 100755 --- a/images/create-env/install-conda +++ b/images/create-env/install-conda @@ -1,4 +1,4 @@ -#! /bin/bash -eu +#! /bin/bash -eux requirements_file="${1}" conda_install_prefix="${2}" @@ -7,6 +7,14 @@ conda_install_prefix="${2}" miniconda_boostrap_prefix="$( pwd )/miniconda" # Run the following in a subshell to avoid environment changes from bootstrap. ( + + # Use the base image-provided tools if they work for us: + tools='' + find -print0 -maxdepth 0 && xargs -0 true < /dev/null \ + || tools="${tools} findutils" + sed -e '' < /dev/null \ + || tools="${tools} sed" + sh ./miniconda.sh \ -b \ -p "${miniconda_boostrap_prefix}" @@ -21,17 +29,16 @@ miniconda_boostrap_prefix="$( pwd )/miniconda" # Only need `strip` executable from binutils. Other binaries from the package # and especially the "sysroot" dependency is only bloat for this container # image. (NOTE: The binary needs libgcc-ng which is explicitly added later.) - conda create --yes \ + mamba create --yes \ --prefix="${conda_install_prefix}" \ --channel=conda-forge \ binutils - tmp_dir="${miniconda_boostrap_prefix}/tmp-strip" - mkdir "${tmp_dir}" - mv "${conda_install_prefix}/bin/"*strip "${tmp_dir}/" - conda remove --yes --all \ + cp -aL "${conda_install_prefix}/bin/strip" ./strip + conda run --prefix="${conda_install_prefix}" strip -- ./strip + mamba remove --yes --all \ --prefix="${conda_install_prefix}" - conda create --yes \ + mamba create --yes \ --prefix="${conda_install_prefix}" \ --channel=conda-forge \ \ @@ -40,15 +47,14 @@ miniconda_boostrap_prefix="$( pwd )/miniconda" tini \ \ libgcc-ng \ - findutils \ + ${tools} \ ; mv \ ./print-env-activate \ ./create-env \ - "${tmp_dir}/"* \ + ./strip \ "${conda_install_prefix}/bin/" - rmdir "${tmp_dir}" ) # Activate the new base environment. @@ -60,6 +66,13 @@ eval "${activate_script}" set -u unset activate_script +# Strip find/xargs/sed beforehand as they are excluded in the strip pipeline. +for prog in find xargs sed ; do + case "$( command -v "${prog}" )" in + "${conda_install_prefix%%/}"/* ) + strip -- "$( command -v "${prog}" )" + esac +done # Use --conda=: to turn the `conda create` into a no-op, but do continue to # run strip, remove files and output the activate/execute scripts. @@ -79,8 +92,11 @@ rm -rf "${miniconda_boostrap_prefix}" conda config \ --append channels conda-forge \ --append channels bioconda \ - --append channels defaults \ ; +conda config \ + --remove channels defaults \ + 2> /dev/null \ + || true conda config \ --remove repodata_fns current_repodata.json \ 2> /dev/null \