diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index 6dd68c1273..7de96cffde 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -14,7 +14,15 @@ jobs: build: name: Build image runs-on: ubuntu-20.04 - + strategy: + matrix: + include: + - dockerfile: Dockerfile.aarch64 + arch: arm64 + image: bioconda-utils-build-env-cos7-aarch64 + - dockerfile: Dockerfile + arch: amd64 + image: bioconda-utils-build-env-cos7 steps: - uses: actions/checkout@v3 with: @@ -28,16 +36,22 @@ jobs: # printf %s "::set-output name=tag::${tag#v}" printf %s "tag=${tag#v}" >> $GITHUB_OUTPUT + - name: Install qemu dependency + run: | + sudo apt-get update + sudo apt-get install -y qemu-user-static + - name: Build image id: buildah-build uses: redhat-actions/buildah-build@v2 with: - image: bioconda-utils-build-env-cos7 + image: ${{ matrix.image }} + arch: ${{ matrix.arch }} tags: >- latest ${{ steps.get-tag.outputs.tag }} dockerfiles: | - ./Dockerfile + ./${{ matrix.dockerfile }} - name: Test built image run: | diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index 86cb768a19..3838274b2a 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -8,6 +8,15 @@ name: release-please jobs: release-please: runs-on: ubuntu-latest + strategy: + matrix: + include: + - dockerfile: Dockerfile.aarch64 + arch: arm64 + image: bioconda-utils-build-env-cos7-aarch64 + - dockerfile: Dockerfile + arch: amd64 + image: bioconda-utils-build-env-cos7 steps: - uses: GoogleCloudPlatform/release-please-action@v2 @@ -28,17 +37,23 @@ jobs: tag=${{ steps.release.outputs.tag_name }} printf %s "::set-output name=tag::${tag#v}" + - name: Install qemu dependency + run: | + sudo apt-get update + sudo apt-get install -y qemu-user-static + - name: Build Image if: ${{ steps.release.outputs.release_created }} id: buildah-build uses: redhat-actions/buildah-build@v2 with: - image: bioconda-utils-build-env-cos7 + image: ${{ matrix.image }} + arch: ${{ matrix.arch }} tags: >- latest ${{ steps.get-tag.outputs.tag }} dockerfiles: | - ./Dockerfile + ./${{ matrix.dockerfile }} - name: Test Built Image if: ${{ steps.release.outputs.release_created }} diff --git a/Dockerfile.aarch64 b/Dockerfile.aarch64 new file mode 100644 index 0000000000..58661be992 --- /dev/null +++ b/Dockerfile.aarch64 @@ -0,0 +1,78 @@ +# Specify the base image to support multi-arch images, such as +# - 'quay.io/condaforge/linux-anvil-aarch64' for Linux aarch64 +# - 'quay.io/condaforge/linux-anvil-cos7-x86_64' for Linux x86_64 +ARG BASE_IMAGE=quay.io/condaforge/linux-anvil-aarch64 + +FROM quay.io/condaforge/linux-anvil-aarch64 as base + +# Copy over C.UTF-8 locale from our base image to make it consistently available during build. +COPY --from=quay.io/bioconda/base-glibc-busybox-bash /usr/lib/locale/C.UTF-8 /usr/lib/locale/C.UTF-8 + +# Provide system deps unconditionally until we are able to offer per-recipe installs. +# (Addresses, e.g., "ImportError: libGL.so.1" in tests directly invoked by conda-build.) +# Also install packages that have been installed historically (openssh-client). +RUN yum install -y mesa-libGL-devel \ + && \ + yum install -y openssh-clients \ + && \ + yum clean all && \ + rm -rf /var/cache/yum/* + +# This changes root's .condarc which ENTRYPOINT copies to /home/conda/.condarc later. +RUN . /opt/conda/etc/profile.d/conda.sh && \ + conda config \ + --prepend channels defaults \ + --prepend channels bioconda \ + --prepend channels conda-forge \ + && \ + { conda config --remove repodata_fns current_repodata.json 2> /dev/null || true ; } && \ + conda config --prepend repodata_fns repodata.json && \ + conda config --set channel_priority flexible && \ + conda config --set auto_update_conda False + +FROM base as build +WORKDIR /tmp/repo +COPY . ./ +RUN . /opt/conda/etc/profile.d/conda.sh && conda list +RUN . /opt/conda/etc/profile.d/conda.sh && conda activate base && \ + pip wheel . && \ + mkdir - /opt/bioconda-utils && \ + cp ./bioconda_utils-*.whl \ + ./bioconda_utils/bioconda_utils-requirements.txt \ + /opt/bioconda-utils/ \ + && \ + chgrp -R lucky /opt/bioconda-utils && \ + chmod -R g=u /opt/bioconda-utils + +# It's for multi-arch support. We do runtime involucro build here to solve +# the chicken-and-egg conundrum. See also: +# https://github.com/bioconda/bioconda-recipes/pull/40144 +FROM golang as involucro_build +RUN wget https://github.com/involucro/involucro/archive/v1.1.2.tar.gz && \ + tar -zxvf v1.1.2.tar.gz && \ + mkdir -p src/github.com/involucro/ && \ + mv involucro-1.1.2/ src/github.com/involucro/involucro && \ + module_path="$( go env GOPATH )"/src/github.com/involucro/involucro/cmd/involucro && \ + GO111MODULE=auto GOBIN="$( go env GOBIN )" go build -o ./bin/involucro -ldflags="-s -X github.com/involucro/involucro/app.version=1.1.2" "${module_path}" + +FROM base +COPY --from=build /opt/bioconda-utils /opt/bioconda-utils +COPY --from=involucro_build /go/bin/involucro /opt/conda/bin/ +RUN . /opt/conda/etc/profile.d/conda.sh && conda activate base && \ + # Make sure we get the (working) conda we want before installing the rest. + sed -nE \ + '/^conda([>=4.26 # Progress monitor ruamel_yaml=0.15.* # Recipe YAML parsing pyaml=17.12.* # Faster YAML parser (deprecate?) networkx=2.* -pandas=0.23.* +pandas=1.4.* numpy=1.19.* # Avoid breaking pandas on OSX libblas=*=*openblas # Avoid large mkl package (pulled in by pandas) boltons=18.* -jsonschema=2.6.* # JSON schema verification +jsonschema=3.2.* # JSON schema verification simplejson # Used by bioconda bot worker (NEEDED?) pyopenssl>=22.1 # Stay compatible with cryptography @@ -28,11 +28,11 @@ conda-forge-pinning=2022.08.25.15.20.42 # tools anaconda-client=1.6.* # anaconda_upload involucro=1.1.* # mulled test and container build -skopeo=0.1.35 # docker upload +skopeo=1.11.* # docker upload git=2.* # well - git # hosters - special regex not supported by RE -regex=2018.08.29 +regex=2022.7.9 # asyncio aiohttp=3.8.* # HTTP lib @@ -51,7 +51,7 @@ gidgethub=3.0.* # githubhandler pyjwt>=2.4.0 # githubhandler (JWT signing), needs >=2.4.0, CVE-2022-29217 # unknown -beautifulsoup4=4.6.* +beautifulsoup4=4.8.* galaxy-lib>=18.9.1 jinja2>=2.10.1,<3 markupsafe<2.1 # markupsafe 2.1 breaks jinja2 @@ -69,4 +69,4 @@ graphviz requests=2.22.* # merge handling -pygithub \ No newline at end of file +pygithub diff --git a/bioconda_utils/cli.py b/bioconda_utils/cli.py index cb17c24c78..d2b60135b5 100644 --- a/bioconda_utils/cli.py +++ b/bioconda_utils/cli.py @@ -423,12 +423,15 @@ def do_lint(recipe_folder, config, packages="*", cache=None, list_checks=False, than one worker, then make sure to give each a different offset!''') @arg('--keep-old-work', action='store_true', help='''Do not remove anything from environment, even after successful build and test.''') +@arg('--docker-base-image', help='''Name of base image that can be used in\ + **dockerfile_template**.''') @enable_logging() def build(recipe_folder, config, packages="*", git_range=None, testonly=False, force=False, docker=None, mulled_test=False, build_script_template=None, pkg_dir=None, anaconda_upload=False, mulled_upload_target=None, build_image=False, keep_image=False, lint=False, lint_exclude=None, - check_channels=None, n_workers=1, worker_offset=0, keep_old_work=False): + check_channels=None, n_workers=1, worker_offset=0, keep_old_work=False, + docker_base_image='quay.io/bioconda/bioconda-utils-build-env-cos7:{}'.format(VERSION.replace('+', '_'))): cfg = utils.load_config(config) setup = cfg.get('setup', None) if setup: @@ -454,6 +457,7 @@ def build(recipe_folder, config, packages="*", git_range=None, testonly=False, use_host_conda_bld=use_host_conda_bld, keep_image=keep_image, build_image=build_image, + docker_base_image=docker_base_image ) else: docker_builder = None diff --git a/bioconda_utils/docker_utils.py b/bioconda_utils/docker_utils.py index a43d864646..b6d21e8bed 100644 --- a/bioconda_utils/docker_utils.py +++ b/bioconda_utils/docker_utils.py @@ -60,6 +60,8 @@ import conda import conda_build +from conda import exports as conda_exports + from . import utils from . import __version__ @@ -90,9 +92,10 @@ # # Note that if the directory didn't exist on the host, then the staging area # will exist in the container but will be empty. Channels expect at least -# a linux-64 and noarch directory within that directory, so we make sure it -# exists before adding the channel. +# a linux-64/linux-aarch64 and noarch directory within that directory, so we +# make sure it exists before adding the channel. mkdir -p {self.container_staging}/linux-64 +mkdir -p {self.container_staging}/linux-aarch64 mkdir -p {self.container_staging}/noarch touch {self.container_staging}/noarch/repodata.json conda config --add channels file://{self.container_staging} 2> >( @@ -442,8 +445,9 @@ def build_recipe(self, recipe_dir, build_args, env, noarch=False): # Write build script to tempfile build_dir = os.path.realpath(tempfile.mkdtemp()) + # conda_exports.subdir is {platform}-{arch} like: 'linux-64' 'linux-aarch64' script = self.build_script_template.format( - self=self, arch='noarch' if noarch else 'linux-64') + self=self, arch='noarch' if noarch else conda_exports.subdir) with open(os.path.join(build_dir, 'build_script.bash'), 'w') as fout: fout.write(script) build_script = fout.name