Merge pull request #59 from mbargull/create-env-3

Add aarch64 create-env image
bioconda · Oct 18, 2023 · e75e4dd · e75e4dd
2 parents 8cca845 + ac2fffc
commit e75e4dd
Show file tree

Hide file tree

Showing 6 changed files with 174 additions and 97 deletions.
diff --git a/.github/workflows/create-env.yaml b/.github/workflows/create-env.yaml
@@ -14,114 +14,132 @@ on:
 jobs:
   build:
     name: Build & Push
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     env:
-      IMAGE_VERSION: '2.2.1'
+      MAJOR_VERSION: 3
+      MINOR_VERSION: 0
       IMAGE_NAME: create-env
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
+
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v2
+      with:
+        platforms: arm64
 
     - name: Prepare
       id: prepare
       run: |
         curl -sSL \
           'https://raw.githubusercontent.com/bioconda/bioconda-common/master/common.sh' \
           | sed -n \
-            's/BIOCONDA_UTILS_TAG=v/::set-output name=bioconda_utils_version::/p'
+            's/BIOCONDA_UTILS_TAG=v/bioconda_utils_version=/p' \
+            >> $GITHUB_OUTPUT
 
     - name: Build
-      id: buildah-build
+      id: build
       uses: redhat-actions/buildah-build@v2
       with:
         image: ${{ env.IMAGE_NAME }}
         tags: >-
           latest
-          ${{ env.IMAGE_VERSION }}
+          ${{ env.MAJOR_VERSION }}
+          ${{ env.MAJOR_VERSION }}.${{ env.MINOR_VERSION }}
+        archs: amd64,arm64
         context: ./images/${{ env.IMAGE_NAME }}
-        dockerfiles: |
+        containerfiles: |
           ./images/${{ env.IMAGE_NAME }}/Dockerfile
         build-args: |
           bioconda_utils_version=${{ steps.prepare.outputs.bioconda_utils_version }}
 
     - name: Test
       run: |
-        image='${{ steps.buildah-build.outputs.image }}'
-        ids="$(
-          for tag in ${{ steps.buildah-build.outputs.tags }} ; do
-            buildah images --quiet --no-trunc "${image}:${tag}"
-          done
-          )"
-        ids="$( printf %s "${ids}" | sort -u )"
-        for id in ${ids} ; do
-          podman history "${id}"
-          buildah bud \
-            --build-arg=base="${id}" \
-            --file=Dockerfile.test \
-            "images/${image}"
-        done
+        set -x
+        image='${{ steps.build.outputs.image }}'
+        for tag in ${{ steps.build.outputs.tags }} ; do
+          buildah manifest inspect \
+            "${image}:${tag}" \
+            | jq '.manifests|{([.[].digest]|sort|join("+")): [.[]|["'"${tag}"'", .platform.architecture, .digest]|join(" ")]}'
+          done \
+          | jq -rs 'add|add[]' \
+          | while read tag arch digest ; do
+              podman images --format='{{.ID}}|{{.Digest}}|{{.RepoDigests}}' \
+                | sed -n "/${digest}/{s/|.*//p;q}" \
+                | xargs -n1 -- \
+                  sh -xc \
+                  'podman history "${1}" ; podman inspect "${1}"' --
+              buildah bud \
+                --arch="${arch}" \
+                --build-arg=base="${image}:${tag}" \
+                --file=Dockerfile.test \
+                "images/${image}"
+            done
         buildah rmi --prune || true
 
     - name: Check Tags
       run: |
         # FIX upstream: Quay.io does not support immutable images currently.
         #               => Try to use the REST API to check for duplicate tags.
-        page=0
-        has_additional='true'
-        while [ "${has_additional}" = 'true' ] ; do
-          page="$(( page + 1 ))"
-          respone="$(
-            curl -sL \
-              'https://quay.io/api/v1/repository/bioconda/${{ steps.buildah-build.outputs.image }}/tag?limit=100&page='"${page}"
-            )"
-          has_additional="$( printf %s "${respone}" | jq -r '.has_additional' )"
-          existing_tags="$(
-            printf %s "${respone}" \
-              | jq -r '.tags[]|.name'
-            )" \
-            || {
-              >&2 printf %s\\n \
-                'Could not get list of image tags.' \
-                'Does the repository exist on Quay.io?' \
-                'Quay.io REST API response was:' \
-                "${respone}"
-              exit 1
-            }
-          for tag in ${{ steps.buildah-build.outputs.tags }} ; do
-            if [ \! "${tag}" = latest ] ; then
+        response="$(
+          curl -sL \
+            'https://quay.io/api/v1/repository/bioconda/${{ steps.build.outputs.image }}/tag/'
+          )"
+
+        existing_tags="$(
+          printf %s "${response}" \
+            | jq -r '.tags[]|select(.end_ts == null or .end_ts >= now)|.name'
+          )" \
+          || {
+            printf %s\\n \
+              'Could not get list of image tags.' \
+              'Does the repository exist on Quay.io?' \
+              'Quay.io REST API response was:' \
+              "${response}"
+            exit 1
+          }
+        for tag in ${{ steps.build.outputs.tags }} ; do
+          case "${tag}" in
+            latest | '${{ env.MAJOR_VERSION }}' ) ;;
+            * )
               if printf %s "${existing_tags}" | grep -qxF "${tag}" ; then
-                >&2 printf 'Tag %s already exists!\n' "${tag}"
+                printf 'Tag %s already exists!\n' "${tag}"
                 exit 1
               fi
-            fi
-          done
+          esac
         done
 
     - if: ${{ github.ref == 'refs/heads/main' }}
       name: Push
       uses: redhat-actions/push-to-registry@v2
       with:
-        image: ${{ steps.buildah-build.outputs.image }}
-        tags: ${{ steps.buildah-build.outputs.tags }}
+        image: ${{ steps.build.outputs.image }}
+        tags: ${{ steps.build.outputs.tags }}
         registry: ${{ secrets.QUAY_BIOCONDA_REPO }}
         username: ${{ secrets.QUAY_BIOCONDA_USERNAME }}
         password: ${{ secrets.QUAY_BIOCONDA_TOKEN }}
 
     - if: ${{ github.ref == 'refs/heads/main' }}
       name: Test Pushed
       run: |
-        image='${{ steps.buildah-build.outputs.image }}'
-        ids="$(
-          for tag in ${{ steps.buildah-build.outputs.tags }} ; do
-            buildah images --quiet --no-trunc "${image}:${tag}"
-          done
-          )"
-        ids="$( printf %s "${ids}" | sort -u )"
-        for id in ${ids} ; do
-          podman history "${id}"
-          buildah bud \
-            --build-arg=base="${id}" \
-            --file=Dockerfile.test \
-            "images/${image}"
-        done
+        set -x
+        image='${{ steps.build.outputs.image }}'
+        for tag in ${{ steps.build.outputs.tags }} ; do
+          buildah manifest inspect \
+            "${image}:${tag}" \
+            | jq '.manifests|{([.[].digest]|sort|join("+")): [.[]|["'"${tag}"'", .platform.architecture, .digest]|join(" ")]}'
+          done \
+          | jq -rs 'add|add[]' \
+          | while read tag arch digest ; do
+              podman images --format='{{.ID}}|{{.Digest}}|{{.RepoDigests}}' \
+                | sed -n "/${digest}/{s/|.*//p;q}" \
+                | xargs -n1 -- \
+                  sh -xc \
+                  'podman history "${1}" ; podman inspect "${1}"' --
+              buildah bud \
+                --arch="${arch}" \
+                --build-arg=base="${image}:${tag}" \
+                --file=Dockerfile.test \
+                "images/${image}"
+            done
         buildah rmi --prune || true
diff --git a/images/create-env/CHANGELOG.md b/images/create-env/CHANGELOG.md
@@ -1,6 +1,23 @@
 # Changelog
 
 
+## bioconda/create-env 3.0 (2023-10-17)
+
+### Changed
+
+- Add linux-aarch64 image; bioconda/create-env is now a multiplatform manifest.
+
+- Change to a simple "major.minor" version scheme and offer mutable "major" tag.
+
+- Drop defaults channel from included config.
+
+- Use Miniforge installer to build this image.
+
+- Rebuilt on the latest base image with Debian 12.2 / BusyBox 1.36.1.
+
+- Do not install findutils, sed if provided by the base image (as is currently).
+
+
 ## bioconda/create-env 2.2.1 (2022-10-14)
 
 ### Changed

diff --git a/images/create-env/Dockerfile b/images/create-env/Dockerfile
@@ -1,19 +1,21 @@
 # Use the exact conda, mamba versions as used in bioconda-recipes' builds.
-ARG bioconda_utils_version='1.1.3'
+ARG bioconda_utils_version
 FROM quay.io/bioconda/bioconda-utils-build-env-cos7:${bioconda_utils_version} as bioconda-build-env
 RUN /opt/conda/bin/conda list \
       --export '^(conda|mamba)$' \
       | sed -n 's/=[^=]*$//p' \
       > /tmp/requirements.txt
 
 
-# Use Debian instead of BusyBox base since Miniconda currently needs external zlib.
-FROM quay.io/bioconda/base-glibc-debian-bash as build
+FROM quay.io/bioconda/base-glibc-busybox-bash as build
 
 WORKDIR /tmp/work
 COPY --from=bioconda-build-env /tmp/requirements.txt ./
 COPY install-conda print-env-activate create-env ./
-ADD https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh ./miniconda.sh
+RUN arch="$( uname -m )" \
+    && \
+    wget --quiet -O ./miniconda.sh \
+      "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-${arch}.sh"
 
 RUN ./install-conda ./requirements.txt /opt/create-env
 

diff --git a/images/create-env/Dockerfile.test b/images/create-env/Dockerfile.test
@@ -1,55 +1,61 @@
 ARG base
 
 FROM "${base}"
-RUN CONDA_PKGS_DIRS="/tmp/pkgs" \
+RUN set -x && \
+    CONDA_PKGS_DIRS="/tmp/pkgs" \
       /opt/create-env/env-execute \
         create-env \
           --conda=mamba \
           /usr/local \
-          file findutils
-RUN . /usr/local/env-activate.sh && \
+          file findutils grep
+RUN set -x && \
+    . /usr/local/env-activate.sh && \
     if find /opt/create-env \
+      -xdev \
       -type f \
       -exec file {} \+ \
-      2> /dev/null \
       | grep 'not stripped' \
     ; then \
       >&2 printf 'found unstripped binaries\n' ; exit 1 \
     ; fi
-RUN . /usr/local/env-activate.sh && \
+RUN set -x && \
+    . /usr/local/env-activate.sh && \
     if find /opt/create-env \
+      -xdev \
       -type f \
       -name \*.a \
-      2> /dev/null \
       | grep . \
     ; then \
       >&2 printf 'found static libraries\n' ; exit 1 \
     ; fi
 
 
-FROM "${base}" as build_bioconda_utils
-RUN /opt/create-env/env-execute \
+FROM "${base}" as build_bioconda_package
+RUN set -x && \
+    /opt/create-env/env-execute \
       create-env \
         --conda=mamba \
         --strip-files=\* \
         /usr/local \
-        bioconda-utils
-FROM bioconda/base-glibc-busybox-bash
-COPY --from=build_bioconda_utils /usr/local /usr/local
-RUN /usr/local/env-execute \
-      bioconda-utils --version \
+        catfasta2phyml
+FROM quay.io/bioconda/base-glibc-busybox-bash
+COPY --from=build_bioconda_package /usr/local /usr/local
+RUN set -x && \
+    /usr/local/env-execute \
+      catfasta2phyml --version \
     && \
     [ ! "${CONDA_PREFIX}" = /usr/local ] \
     && \
     { set -x && . /usr/local/env-activate.sh && set +x ; } \
     && \
     [ "${CONDA_PREFIX}" = /usr/local ] \
     && \
-    bioconda-utils --version
+    catfasta2phyml --version
 
 
 FROM "${base}" as build_conda
-RUN /opt/create-env/env-execute \
+RUN set -x && \
+    /opt/create-env/env-execute \
       create-env \
         --conda=mamba \
         --env-activate-args='--prefix-is-base' \
@@ -58,10 +64,11 @@ RUN /opt/create-env/env-execute \
         --remove-paths=\*.pyc \
         /opt/conda \
         conda
-FROM bioconda/base-glibc-busybox-bash
+FROM quay.io/bioconda/base-glibc-busybox-bash
 COPY --from=build_conda /opt/conda /opt/conda
 COPY --from=build_conda /opt/conda/env-activate.sh /usr/local/
-RUN /usr/local/env-execute \
+RUN set -x && \
+    /usr/local/env-execute \
       conda info --all \
     && \
     { set -x && . /usr/local/env-activate.sh && set +x ; } \

diff --git a/images/create-env/create-env b/images/create-env/create-env
@@ -158,16 +158,33 @@ if [ -n "${strip_files_globs}" ] ; then
     # Strip binaries. (Run strip on all files; ignore errors for non-ELF files.)
     # Limit open fds (ulimit -n) for strip (small number chosen arbitrarily).
     # (To avoid "could not create temporary file to hold stripped copy: Too many open files")
+
+    # Filter out the binaries currently in use by the pipeline via sed below.
+    skip_inode_expressions="$(
+      command -v -- find xargs sed strip \
+      | xargs -- stat -L -c '-e /^%d,%i:/d' --
+    )"
     find "${prefix}" \
       -type f \
       \( "${@}" \) \
       -print0 \
       | xargs \
-          --null \
+          -0 \
+          -n 64 \
+          -- \
+          stat -L -c '%d,%i:%n' -- \
+      | sed \
+        ${skip_inode_expressions} \
+        -e 's/^[^:]*://' \
+      | tr \\n \\0 \
+      |
+        xargs \
+          -0 \
           -n 64 \
+          -- \
           strip -- \
         2>&1 \
-      | grep -v ': file format not recognized' \
+      | sed '/: file format not recognized/d' \
       || true
   )
 fi