diff --git a/.github/workflows/toolshed.yml b/.github/workflows/toolshed.yml index 398e60f..febe259 100644 --- a/.github/workflows/toolshed.yml +++ b/.github/workflows/toolshed.yml @@ -8,37 +8,31 @@ on: branches: [ "main" ] env: - ORG_NAMESPACE: opencyphal - IMAGE_NAME: toolshed + REGISTRY_IMAGE: ghcr.io/opencyphal/toolshed IMAGE_VERSION_PREFIX: ts jobs: - - dry_run: + release-ts: if: ${{ github.event_name == 'pull_request' || ( github.event_name == 'release' && startsWith( github.ref, 'refs/tags/ts') && github.event.release.prerelease ) }} runs-on: ubuntu-latest - + strategy: + matrix: + platform: + - linux/amd64 + - linux/arm64 steps: - - uses: actions/checkout@v3 - - - # Add QEMU to allow building non-native containers as - # part of multi-platform container builds. - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - # Setup buildx so we can do multi-platform container builds. - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + - name: Prepare + run: | + platform=${{ matrix.platform }} + echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV - - name: build - run: cd ${{ env.IMAGE_NAME}} && docker buildx build --platform linux/arm64,linux/amd64 . + - uses: actions/checkout@v4 - release: - if: ${{ github.event_name == 'release' && startsWith( github.ref, 'refs/tags/ts' ) && !github.event.release.prerelease }} - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY_IMAGE }} - # Add QEMU to allow building non-native containers as # part of multi-platform container builds. @@ -49,16 +43,70 @@ jobs: name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Login to GHCR - uses: docker/login-action@v2 + - name: Build and push by digest + id: build + uses: docker/build-push-action@v5 with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: form image tag - id: image_tag - uses: olegtarasov/get-tag@v2.1.2 + context: toolshed + platforms: ${{ matrix.platform }} + labels: ${{ steps.meta.outputs.labels }} + outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=${{ github.event_name == 'pull_request' && 'false' || 'true' }} + - + name: Export digest + if: ${{ github.event_name != 'pull_request'}} + run: | + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + - + name: Upload digest + if: ${{ github.event_name != 'pull_request'}} + uses: actions/upload-artifact@v4 + with: + name: digests-${{ env.PLATFORM_PAIR }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 - - name: build-and-push - run: cd ${{ env.IMAGE_NAME }} && docker buildx build --platform linux/arm64,linux/amd64 --push -t ghcr.io/${{ env.ORG_NAMESPACE }}/${{ env.IMAGE_NAME }}:${{ steps.image_tag.outputs.tag }} . + merge-ts: + if: ${{ github.event_name == 'release' && startsWith( github.ref, 'refs/tags/ts' ) && !github.event.release.prerelease }} + runs-on: ubuntu-latest + needs: + - release-ts + steps: + - + name: Download digests + uses: actions/download-artifact@v4 + with: + path: /tmp/digests + pattern: digests-* + merge-multiple: true + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - + name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + tags: type=ref,event=tag + images: ${{ env.REGISTRY_IMAGE }} + + - name: Login to GHCR + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - + name: Create manifest list and push + working-directory: /tmp/digests + run: | + docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ + $(printf '${{ env.REGISTRY_IMAGE }}@sha256:%s ' *) + - + name: Inspect image + run: | + docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:${{ steps.meta.outputs.version }} diff --git a/.github/workflows/toxic.yml b/.github/workflows/toxic.yml index c00c9e0..048b369 100644 --- a/.github/workflows/toxic.yml +++ b/.github/workflows/toxic.yml @@ -8,37 +8,31 @@ on: branches: [ "main" ] env: - ORG_NAMESPACE: opencyphal - IMAGE_NAME: toxic + REGISTRY_IMAGE: ghcr.io/opencyphal/toxic IMAGE_VERSION_PREFIX: tx jobs: - - dry_run: + release-tx: if: ${{ github.event_name == 'pull_request' || ( github.event_name == 'release' && startsWith( github.ref, 'refs/tags/tx') && github.event.release.prerelease ) }} runs-on: ubuntu-latest - + strategy: + matrix: + platform: + - linux/amd64 + - linux/arm64 steps: - - uses: actions/checkout@v3 - - - # Add QEMU to allow building non-native containers as - # part of multi-platform container builds. - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - # Setup buildx so we can do multi-platform container builds. - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + - name: Prepare + run: | + platform=${{ matrix.platform }} + echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV - - name: build - run: cd ${{ env.IMAGE_NAME }} && docker buildx build --platform linux/arm64,linux/amd64 . + - uses: actions/checkout@v4 - release: - if: ${{ github.event_name == 'release' && startsWith( github.ref, 'refs/tags/tx' ) && !github.event.release.prerelease }} - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY_IMAGE }} - # Add QEMU to allow building non-native containers as # part of multi-platform container builds. @@ -49,16 +43,70 @@ jobs: name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Login to GHCR - uses: docker/login-action@v2 + - name: Build and push by digest + id: build + uses: docker/build-push-action@v5 with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: form image tag - id: image_tag - uses: olegtarasov/get-tag@v2.1.2 + context: toxic + platforms: ${{ matrix.platform }} + labels: ${{ steps.meta.outputs.labels }} + outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=${{ github.event_name == 'pull_request' && 'false' || 'true' }} + - + name: Export digest + if: ${{ github.event_name != 'pull_request'}} + run: | + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + - + name: Upload digest + if: ${{ github.event_name != 'pull_request'}} + uses: actions/upload-artifact@v4 + with: + name: digests-${{ env.PLATFORM_PAIR }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 - - name: build-and-push - run: cd ${{ env.IMAGE_NAME }} && docker buildx build --platform linux/arm64,linux/amd64 --push -t ghcr.io/${{ env.ORG_NAMESPACE }}/${{ env.IMAGE_NAME }}:${{ steps.image_tag.outputs.tag }} . + merge-tx: + if: ${{ github.event_name == 'release' && startsWith( github.ref, 'refs/tags/tx' ) && !github.event.release.prerelease }} + runs-on: ubuntu-latest + needs: + - release-tx + steps: + - + name: Download digests + uses: actions/download-artifact@v4 + with: + path: /tmp/digests + pattern: digests-* + merge-multiple: true + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - + name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + tags: type=ref,event=tag + images: ${{ env.REGISTRY_IMAGE }} + + - name: Login to GHCR + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - + name: Create manifest list and push + working-directory: /tmp/digests + run: | + docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ + $(printf '${{ env.REGISTRY_IMAGE }}@sha256:%s ' *) + - + name: Inspect image + run: | + docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:${{ steps.meta.outputs.version }} diff --git a/README.md b/README.md index a796c0c..a7e1278 100644 --- a/README.md +++ b/README.md @@ -25,12 +25,13 @@ like [libcanard](https://github.com/OpenCyphal/libcanard) and | tag | Python | Gcc | Clang | Cmake | Platforms | Other Utilities | |----------|--------|-----|-------|-------|-----------|-----------------| -| [ts20.4.1](https://github.com/OpenCyphal/docker_toolchains/pkgs/container/toolshed) | 3.10 | 9.3.0 | 10.0.0 | 3.16.3 | | | -| [ts22.4.1](https://github.com/OpenCyphal/docker_toolchains/pkgs/container/toolshed) | 3.10 | 11.3.0 | 15.0.7 | 3.22.1 | | -| [ts22.4.2](https://github.com/OpenCyphal/docker_toolchains/pkgs/container/toolshed) | 3.10 | 11.3.0 | 15.0.7 | 3.22.1 | | -| [ts22.4.3](https://github.com/OpenCyphal/docker_toolchains/pkgs/container/toolshed) | 3.10 | 11.3.0 | 15.0.7 | 3.22.1 | | -| ts22.4.4 - An experimental release that shouldn't be used. | +| [ts22.4.7](https://github.com/OpenCyphal/docker_toolchains/pkgs/container/toolshed) | 3.10 | 12.3.0 | 18.1.3 | 3.22.1 | | +| [ts22.4.6](https://github.com/OpenCyphal/docker_toolchains/pkgs/container/toolshed) | 3.10 | 11.4.0 | 18.0.0 | 3.22.1 | | | [ts22.4.5](https://github.com/OpenCyphal/docker_toolchains/pkgs/container/toolshed) | 3.10 | 11.4.0 | 18.0.0 | 3.22.1 | | +| [ts22.4.3](https://github.com/OpenCyphal/docker_toolchains/pkgs/container/toolshed) | 3.10 | 11.3.0 | 15.0.7 | 3.22.1 | | +| [ts22.4.2](https://github.com/OpenCyphal/docker_toolchains/pkgs/container/toolshed) | 3.10 | 11.3.0 | 15.0.7 | 3.22.1 | | +| [ts22.4.1](https://github.com/OpenCyphal/docker_toolchains/pkgs/container/toolshed) | 3.10 | 11.3.0 | 15.0.7 | 3.22.1 | | + You can use this in your workflow yaml like this: @@ -39,7 +40,7 @@ You can use this in your workflow yaml like this: jobs: my-job: runs-on: ubuntu-latest - container: ghcr.io/opencyphal/toolshed:ts22.4.3 + container: ghcr.io/opencyphal/toolshed:ts22.4.7 ``` @@ -53,8 +54,8 @@ is based on Ubuntu and provides the necessary compilers and utilities to author | tag | Python | Tex Live | git | Platforms | Other Utilities | |----------|--------|----------|-----|-----------|-----------------| -| [te20.4.1](https://github.com/OpenCyphal/docker_toolchains/pkgs/container/texer) | 3.11 | 2019 | 2.25.1 | | | | [te22.4.1](https://github.com/OpenCyphal/docker_toolchains/pkgs/container/texer) | 3.11 | 2022 | 2.34.1 | | | +| [te20.4.1](https://github.com/OpenCyphal/docker_toolchains/pkgs/container/texer) | 3.11 | 2019 | 2.25.1 | | | You can use this in your workflow yaml like this: @@ -78,9 +79,10 @@ is based on Ubuntu and provides all modern Python3 distributions, pip, tox, and | tag | Base Python | Python Versions | Tox | Nox | pip | Platforms | Other Utilities | |----------|-------------|-----------------|-----|-----|-----|-----------|-----------------| -| [tx20.4.1](https://github.com/OpenCyphal/docker_toolchains/pkgs/container/toxic) | 3.8 | 3.6, 3.7, 3.8, 3.9, 3.10, 3.11, 3.12 | 4.4.5 | (not available) | 20.0.2 | | | -| [tx22.4.1](https://github.com/OpenCyphal/docker_toolchains/pkgs/container/toxic) | 3.8 | 3.7, 3.8, 3.9, 3.10, 3.11, 3.12 | 4.4.5 | 2023.4.22 | 20.0.2 | | | | [tx22.4.2](https://github.com/OpenCyphal/docker_toolchains/pkgs/container/toxic) | 3.10 | 3.7, 3.8, 3.9, 3.10, 3.11, 3.12 | 4.13.0 | 2023.4.22 | 20.0.2 | | | +| [tx22.4.1](https://github.com/OpenCyphal/docker_toolchains/pkgs/container/toxic) | 3.8 | 3.7, 3.8, 3.9, 3.10, 3.11, 3.12 | 4.4.5 | 2023.4.22 | 20.0.2 | | | +| [tx20.4.1](https://github.com/OpenCyphal/docker_toolchains/pkgs/container/toxic) | 3.8 | 3.6, 3.7, 3.8, 3.9, 3.10, 3.11, 3.12 | 4.4.5 | (not available) | 20.0.2 | | | + You can use this in your workflow yaml like this: @@ -88,7 +90,7 @@ You can use this in your workflow yaml like this: jobs: my-job: runs-on: ubuntu-latest - container: ghcr.io/opencyphal/toxic:tx20.4.1 + container: ghcr.io/opencyphal/toxic:tx20.4.2 ``` --------------------------- diff --git a/toolshed/Dockerfile b/toolshed/Dockerfile index cdfae7d..ce62446 100644 --- a/toolshed/Dockerfile +++ b/toolshed/Dockerfile @@ -3,7 +3,9 @@ # FROM ubuntu:22.04 AS base -LABEL org.opencontainers.image.description "Provides a consistent build and test environment for development, continuous-integration, and test automation of OpenCyphal C and C++ based projects." +LABEL org.opencontainers.image.source=https://github.com/OpenCyphal/docker_toolchains +LABEL org.opencontainers.image.description="Provides a consistent build and test environment for development, continuous-integration, and test automation of OpenCyphal C and C++ based projects." +LABEL org.opencontainers.image.licenses=MIT VOLUME /repo @@ -13,24 +15,39 @@ ENV LANG=en_US.UTF-8 ENV LANGUAGE=en_US:en ENV LC_ALL=en_US.UTF-8 + # +---------------------------------------------------------------------------+ -# | BASE INSTALL +# | UBUNTU PROVISIONING # +---------------------------------------------------------------------------+ -COPY provision.sh / -RUN /provision.sh +FROM base AS provisioning +WORKDIR /tmp +ADD pre-provision.sh ./pre-provision.sh +ADD provision.sh ./provision.sh +ADD post-provision.sh ./post-provision.sh + +RUN ./pre-provision.sh +RUN ./provision.sh +RUN ./post-provision.sh + +# INSTALL THE PATH FOR INTERATIVE SESSIONS +RUN echo "export PATH=$PATH" >> ~/.bashrc +# LEAVE THE WORKDIR AS /repo +WORKDIR /repo # +---------------------------------------------------------------------------+ # | NVM/nodejs # +---------------------------------------------------------------------------+ -FROM base AS node +FROM provisioning AS node WORKDIR /tmp ADD nvm-install.sh ./nvm-install.sh -RUN . ./nvm-install.sh +RUN cat ./nvm-install.sh | bash ENV NVM_DIR="~/.nvm" -RUN . ~/.nvm/nvm.sh && nvm install v16.20.0 +RUN . ~/.nvm/nvm.sh && nvm install v20 +# LEAVE THE WORKDIR AS /repo +WORKDIR /repo # +---------------------------------------------------------------------------+ # | INSTALL PYTHON STUFF @@ -44,6 +61,8 @@ RUN pip3 install nox RUN pip3 install gcovr RUN pip3 list +# LEAVE THE WORKDIR AS /repo +WORKDIR /repo # +---------------------------------------------------------------------------+ # | LLVM @@ -56,99 +75,53 @@ ADD llvm-select.sh ./llvm-select.sh RUN ./llvm-install.sh 18 all RUN ./llvm-select.sh 18 +# LEAVE THE WORKDIR AS /repo +WORKDIR /repo # +---------------------------------------------------------------------------+ -# | GCC -# +---------------------------------------------------------------------------+ -FROM llvm AS gcc-arm64 - -ARG GCC_HOST_PLATFORM=aarch64 - -FROM llvm AS gcc-amd64 - -ARG GCC_HOST_PLATFORM=x86_64 - -FROM gcc-$TARGETARCH AS gcc -WORKDIR /tmp - -# ARM: M-Cores, Bare Metal -ARG GCC_ARM_NONE_EABI_PLATFORM=${GCC_HOST_PLATFORM} -ARG GCC_ARM_NONE_EABI_VERSION=11.2-2022.02 -ARG GCC_ARM_NONE_EABI_TRIPLE=arm-none-eabi -ARG GCC_ARM_NONE_EABI_BASENAME=gcc-arm-${GCC_ARM_NONE_EABI_VERSION}-${GCC_ARM_NONE_EABI_PLATFORM}-${GCC_ARM_NONE_EABI_TRIPLE} -ARG GCC_ARM_NONE_EABI_REV=99a2bce6f4464be08eca01eda13e4e96 -ARG GCC_ARM_NONE_EABI_HASH=C371F8D384D7F8DC08BFE154352AA3AE - -WORKDIR /tmp -COPY ${GCC_ARM_NONE_EABI_BASENAME}.sha512.txt . -ADD https://developer.arm.com/-/media/Files/downloads/gnu/${GCC_ARM_NONE_EABI_VERSION}/binrel/${GCC_ARM_NONE_EABI_BASENAME}.tar.xz?rev=${GCC_ARM_NONE_EABI_REV}&hash=${GCC_ARM_NONE_EABI_HASH} /tmp/${GCC_ARM_NONE_EABI_BASENAME}.tar.xz -RUN sha512sum -c ${GCC_ARM_NONE_EABI_BASENAME}.sha512.txt -RUN tar xJf ${GCC_ARM_NONE_EABI_BASENAME}.tar.xz -C /usr/share/ -RUN rm ${GCC_ARM_NONE_EABI_BASENAME}.tar.xz - -ENV PATH="/usr/share/${GCC_ARM_NONE_EABI_BASENAME}/bin:${PATH}" - -RUN ${GCC_ARM_NONE_EABI_TRIPLE}-gcc --version - -# ARM: A-Cores, Linux -ARG GCC_ARM_NONE_LINUX_PLATFORM=${GCC_HOST_PLATFORM} -ARG GCC_ARM_NONE_LINUX_VERSION=11.2-2022.02 -ARG GCC_ARM_NONE_LINUX_TRIPLE=arm-none-linux-gnueabihf -ARG GCC_ARM_NONE_LINUX_BASENAME=gcc-arm-${GCC_ARM_NONE_LINUX_VERSION}-${GCC_ARM_NONE_LINUX_PLATFORM}-${GCC_ARM_NONE_LINUX_TRIPLE} -ARG GCC_ARM_NONE_LINUX_REV=ffc49e4af4cb4c298c2110a4d887716c -ARG GCC_ARM_NONE_LINUX_HASH=13DC7A96AEA6FB1D479EB23B782E79EB - -COPY ${GCC_ARM_NONE_LINUX_BASENAME}.sha512.txt . -ADD https://developer.arm.com/-/media/Files/downloads/gnu/${GCC_ARM_NONE_LINUX_VERSION}/binrel/${GCC_ARM_NONE_LINUX_BASENAME}.tar.xz?rev=${GCC_ARM_NONE_LINUX_REV}&hash=${GCC_ARM_NONE_LINUX_HASH} /tmp/${GCC_ARM_NONE_LINUX_BASENAME}.tar.xz -RUN sha512sum -c ${GCC_ARM_NONE_LINUX_BASENAME}.sha512.txt -RUN tar xJf ${GCC_ARM_NONE_LINUX_BASENAME}.tar.xz -C /usr/share/ -RUN rm ${GCC_ARM_NONE_LINUX_BASENAME}.tar.xz - -ENV PATH="/usr/share/${GCC_ARM_NONE_LINUX_BASENAME}/bin:${PATH}" - -RUN ${GCC_ARM_NONE_LINUX_TRIPLE}-gcc --version - - -# +---------------------------------------------------------------------------+ -# | DOXYGEN BUILD +# | DOXYGEN BUILD (PARALLEL) # +---------------------------------------------------------------------------+ -FROM gcc AS doxygen-arm64 - -ARG USE_LIB_CLANG=OFF - -FROM gcc AS doxygen-amd64 - -ARG USE_LIB_CLANG=OFF - -FROM doxygen-$TARGETARCH AS doxygen +FROM llvm AS doxygen WORKDIR /tmp +RUN mkdir doxygen-Release +RUN mkdir doxygen-Release/build +COPY cached-download.sh . -ARG DOXYGEN_VERSION=1_9_6 +ARG DOXYGEN_VERSION=1_10_0 COPY doxygen-Release_${DOXYGEN_VERSION}.sha512.txt . -ADD https://github.com/doxygen/doxygen/archive/refs/tags/Release_${DOXYGEN_VERSION}.tar.gz doxygen-Release_${DOXYGEN_VERSION}.tar.gz -RUN sha512sum -c doxygen-Release_${DOXYGEN_VERSION}.sha512.txt -RUN mkdir doxygen-Release -RUN tar -xvf doxygen-Release_${DOXYGEN_VERSION}.tar.gz -C doxygen-Release --strip-components 1 -RUN rm doxygen-Release_${DOXYGEN_VERSION}.tar.gz -RUN mkdir doxygen-Release/build +RUN --mount=type=cache,target=/doxygen-download \ + ./cached-download.sh \ + --cache-target "/doxygen-download" \ + --download-url "https://github.com/doxygen/doxygen/archive/refs/tags/Release_${DOXYGEN_VERSION}.tar.gz" \ + --download-to-file "/doxygen-download/doxygen-Release_${DOXYGEN_VERSION}.tar.gz" \ + --checksums "/tmp/doxygen-Release_${DOXYGEN_VERSION}.sha512.txt" \ + --tar-args "--strip-components=1" \ + --unpack-dir "/tmp/doxygen-Release" + WORKDIR /tmp/doxygen-Release/build -RUN cmake -G "Unix Makefiles" -Dbuild_parse:BOOL=ON -Duse_libclang:BOOL=${USE_LIB_CLANG} .. -RUN make -RUN make install +RUN cmake -G "Ninja" -Dbuild_parse:BOOL=ON .. +RUN cmake --build . +RUN cmake --install . WORKDIR /tmp RUN rm -rf doxygen-Release +# LEAVE THE WORKDIR AS /repo +WORKDIR /repo # +---------------------------------------------------------------------------+ -# | FINALIZE +# | GCC::native # +---------------------------------------------------------------------------+ -FROM doxygen AS all +FROM doxygen AS gcc-select -# INSTALL THE PATH FOR INTERATIVE SESSIONS -RUN echo "export PATH=$PATH" >> ~/.bashrc +WORKDIR /tmp + +ADD gcc-select.sh ./gcc-select.sh +RUN ./gcc-select.sh 12 +RUN update-alternatives --display gcc # LEAVE THE WORKDIR AS /repo WORKDIR /repo + diff --git a/toolshed/README.md b/toolshed/README.md index 700ff9d..e2f5338 100644 --- a/toolshed/README.md +++ b/toolshed/README.md @@ -18,6 +18,13 @@ These instructions are for maintainers with permissions to push to the [OpenCyphal organization on Github](https://github.com/OpenCyphal/). Normally the container should be published by a github action but these instructions provide a way to manually update the container from any developer environment. +> **IMPORTANT NOTE** +> +> You must enable [containerd](https://containerd.io/) if you are using Docker Desktop to build locally (this is available as a general setting in Docker desktop). Docker desktop does not support multi-platform images and, if you try to use Docker Desktop without containerd, these instructions will fail with the message: +> +> *WARNING: No output specified with docker-container driver. Build result will only remain in the build cache.* +> + First create a temporary (7-day expiration please) personal access token (classic) with write:packages and read:packages scope. See [this github help page](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) for instructions. @@ -44,25 +51,16 @@ docker buildx use cyphalbuild ... then build the container: ```bash -docker buildx build --platform linux/amd64,linux/arm64 -t ghcr.io/opencyphal/toolshed:ts22.4.x . +docker buildx build --platform linux/amd64,linux/arm64 --load -t ghcr.io/opencyphal/toolshed:ts22.4.x . ``` -(where x is the next version number for the container) - -When the build completes you'll see the following warning: - -> WARNING: No output specified with docker-container driver. Build result will only remain in the build cache. To push result image into registry use --push or to load image into docker use --load - -It's important you don't restart your docker build container before you load and/or push since it may blow away the cache you need to push or load from. +After this completes you'll see your image using the classic `docker images` command or the newer `buildx imagetools` command to inspect the multi-architecture manifest: -### Load -Currently you can't use --load with docker desktop so the only way to test the container is to rebuild for one platform only: - -```bash -docker buildx build --platform linux/arm64 --load -t ghcr.io/opencyphal/toolshed:ts22.4.x . +``` +docker buildx imagetools inspect ghcr.io/opencyphal/toolshed:ts22.4.x ``` -This will be operating off of the cache so it shouldn't take very long to complete. After it does you'll see your image using the classic `docker images` command. Now you can login to the container to test it out: + Now you can login to the container to test it out: ```bash docker run --rm -it -v ${PWD}:/repo ghcr.io/opencyphal/toolshed:ts22.4.x diff --git a/toolshed/cached-download.sh b/toolshed/cached-download.sh new file mode 100755 index 0000000..898b06b --- /dev/null +++ b/toolshed/cached-download.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +# +----------------------------------------------------------+ +# | BASH : Modifying Shell Behaviour +# | (https://www.gnu.org/software/bash/manual) +# +----------------------------------------------------------+ +# Exit immediately if a pipeline returns a non-zero status. +set -o errexit + +# If set, the return value of a pipeline is the value of the +# last (rightmost) command to exit with a non-zero status, or +# zero if all commands in the pipeline exit successfully. +set -o pipefail + +# +----------------------------------------------------------+ +# | Command Line Arguments +# +----------------------------------------------------------+ +# Usage: cached-download.sh [OPTIONS] +# Generate usage message +echo_usage() { + echo "Usage: $0 [OPTIONS]" + echo " -c, --cache-target DIR Docker cache mount point" + echo " -d, --download-url URL URL to download" + echo " -f, --download-to-file NAME The filename to output the download to" + echo " -s, --checksums FILE Checksums file" + echo " -u, --unpack-dir DIR Unpack directory" + echo " -t, --tar-args ARGS (optional) Additional arguments for (un)tar command" + echo " -h, --help Show help message" +} + +# Check if any required option is unset (except help) +check_options() { + if [[ -z $cache_target || -z $download_url || -z $download_to_file || -z $checksums_file || -z $unpack_dir ]]; then + echo "Error: Missing required options" + echo_usage + exit 1 + fi +} + +tar_args="" + +# Parse command line arguments +while [[ $# -gt 0 ]]; do + key="$1" + case $key in + -c|--cache-target) + cache_target="$2" + shift + shift + ;; + -d|--download-url) + download_url="$2" + shift + shift + ;; + -f|--download-to-file) + download_to_file="$2" + shift + shift + ;; + -s|--checksums) + checksums_file="$2" + shift + shift + ;; + -u|--unpack-dir) + unpack_dir="$2" + shift + shift + ;; + -t|--tar-args) + tar_args="$2" + shift + shift + ;; + -h|--help) + echo_usage + exit 0 + ;; + *) + echo "Unknown option: $1" + echo_usage + exit 1 + ;; + esac +done + +# Check if any required option is unset (except help) +check_options + +cd "$cache_target" +wget --no-config --show-progress --progress=bar:noscroll:force -O "$download_to_file" "$download_url" +sha512sum -c "$checksums_file" +tar -xvf "$download_to_file" $tar_args -C "$unpack_dir" diff --git a/toolshed/doxygen-Release_1_10_0.sha512.txt b/toolshed/doxygen-Release_1_10_0.sha512.txt new file mode 100644 index 0000000..4dbd14c --- /dev/null +++ b/toolshed/doxygen-Release_1_10_0.sha512.txt @@ -0,0 +1 @@ +d8731796d5513ff5939c234f32f7f31aabc45f5557947741a330b3d33b368fb5ad770ef1d3d7384a402d98229eafeb1f955ef76e9932310b83c1d35a2eef500f doxygen-Release_1_10_0.tar.gz diff --git a/toolshed/gcc-select.sh b/toolshed/gcc-select.sh new file mode 100755 index 0000000..9d3b683 --- /dev/null +++ b/toolshed/gcc-select.sh @@ -0,0 +1,32 @@ +#!/bin/bash + + +GCC_VERSION=${1} + +setup_gcc_alternatives() +{ + local version=${1} + local priority=${2} + local group=${3} + local members=${4} + local path=${5} + local cmdln + + cmdln="--verbose --install ${path}${group} ${group} ${path}${group}-${version} ${priority}" + for member in ${members}; do + cmdln="${cmdln} --slave ${path}${member} ${member} ${path}${member}-${version}" + done + update-alternatives ${cmdln} +} + +GCC_ALTERNATIVES_PRI=99 +GCC_ALTERNATIVES_PATH="/usr/bin/" + +# setup llvm-config group +GCC_ALTERNATIVES_GROUP_GCC_CONFIG="" +GCC_ALTERNATIVES_GROUP_GCC_CONFIG="${GCC_ALTERNATIVES_GROUP_GCC_CONFIG} g++" +GCC_ALTERNATIVES_GROUP_GCC_CONFIG="${GCC_ALTERNATIVES_GROUP_GCC_CONFIG} gcc-ar" +GCC_ALTERNATIVES_GROUP_GCC_CONFIG="${GCC_ALTERNATIVES_GROUP_GCC_CONFIG} gcc-nm" +GCC_ALTERNATIVES_GROUP_GCC_CONFIG="${GCC_ALTERNATIVES_GROUP_GCC_CONFIG} gcc-ranlib" + +setup_gcc_alternatives "${GCC_VERSION}" "${GCC_ALTERNATIVES_PRI}" "gcc" "${GCC_ALTERNATIVES_GROUP_GCC_CONFIG}" "${GCC_ALTERNATIVES_PATH}" diff --git a/toolshed/nvm-install.sh b/toolshed/nvm-install.sh old mode 100644 new mode 100755 index 258f6e6..0f6af49 --- a/toolshed/nvm-install.sh +++ b/toolshed/nvm-install.sh @@ -1,7 +1,5 @@ #!/usr/bin/env bash -# Downloaded from https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.1/install.sh on 2023-03-31 - { # this ensures the entire script is downloaded # nvm_has() { @@ -12,6 +10,12 @@ nvm_echo() { command printf %s\\n "$*" 2>/dev/null } +if [ -z "${BASH_VERSION}" ] || [ -n "${ZSH_VERSION}" ]; then + # shellcheck disable=SC2016 + nvm_echo >&2 'Error: the install instructions explicitly say to pipe the install script to `bash`; please follow them' + exit 1 +fi + nvm_grep() { GREP_OPTIONS='' command grep "$@" } @@ -29,14 +33,14 @@ nvm_install_dir() { } nvm_latest_version() { - nvm_echo "v0.39.1" + nvm_echo "v0.39.7" } nvm_profile_is_bash_or_zsh() { local TEST_PROFILE TEST_PROFILE="${1-}" case "${TEST_PROFILE-}" in - *"/.bashrc" | *"/.bash_profile" | *"/.zshrc") + *"/.bashrc" | *"/.bash_profile" | *"/.zshrc" | *"/.zprofile") return ;; *) @@ -48,12 +52,28 @@ nvm_profile_is_bash_or_zsh() { # # Outputs the location to NVM depending on: # * The availability of $NVM_SOURCE +# * The presence of $NVM_INSTALL_GITHUB_REPO # * The method used ("script" or "git" in the script, defaults to "git") # NVM_SOURCE always takes precedence unless the method is "script-nvm-exec" # nvm_source() { local NVM_GITHUB_REPO NVM_GITHUB_REPO="${NVM_INSTALL_GITHUB_REPO:-nvm-sh/nvm}" + if [ "${NVM_GITHUB_REPO}" != 'nvm-sh/nvm' ]; then + { nvm_echo >&2 "$(cat)" ; } << EOF +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@ WARNING: REMOTE REPO IDENTIFICATION HAS CHANGED! @ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! + +The default repository for this install is \`nvm-sh/nvm\`, +but the environment variables \`\$NVM_INSTALL_GITHUB_REPO\` is +currently set to \`${NVM_GITHUB_REPO}\`. + +If this is not intentional, interrupt this installation and +verify your environment variables. +EOF + fi local NVM_VERSION NVM_VERSION="${NVM_INSTALL_VERSION:-$(nvm_latest_version)}" local NVM_METHOD @@ -278,11 +298,13 @@ nvm_detect_profile() { elif [ "${SHELL#*zsh}" != "$SHELL" ]; then if [ -f "$HOME/.zshrc" ]; then DETECTED_PROFILE="$HOME/.zshrc" + elif [ -f "$HOME/.zprofile" ]; then + DETECTED_PROFILE="$HOME/.zprofile" fi fi if [ -z "$DETECTED_PROFILE" ]; then - for EACH_PROFILE in ".profile" ".bashrc" ".bash_profile" ".zshrc" + for EACH_PROFILE in ".profile" ".bashrc" ".bash_profile" ".zprofile" ".zshrc" do if DETECTED_PROFILE="$(nvm_try_profile "${HOME}/${EACH_PROFILE}")"; then break @@ -358,6 +380,14 @@ nvm_do_install() { exit 1 fi fi + # Disable the optional which check, https://www.shellcheck.net/wiki/SC2230 + # shellcheck disable=SC2230 + if nvm_has xcode-select && [ "$(xcode-select -p >/dev/null 2>/dev/null ; echo $?)" = '2' ] && [ "$(which git)" = '/usr/bin/git' ] && [ "$(which curl)" = '/usr/bin/curl' ]; then + nvm_echo >&2 'You may be on a Mac, and need to install the Xcode Command Line Developer Tools.' + # shellcheck disable=SC2016 + nvm_echo >&2 'If so, run `xcode-select --install` and try again. If not, please report this!' + exit 1 + fi if [ -z "${METHOD}" ]; then # Autodetect install method if nvm_has git; then @@ -403,7 +433,7 @@ nvm_do_install() { if [ -n "${PROFILE}" ]; then TRIED_PROFILE="${NVM_PROFILE} (as defined in \$PROFILE), " fi - nvm_echo "=> Profile not found. Tried ${TRIED_PROFILE-}~/.bashrc, ~/.bash_profile, ~/.zshrc, and ~/.profile." + nvm_echo "=> Profile not found. Tried ${TRIED_PROFILE-}~/.bashrc, ~/.bash_profile, ~/.zprofile, ~/.zshrc, and ~/.profile." nvm_echo "=> Create one of them and run this script again" nvm_echo " OR" nvm_echo "=> Append the following lines to the correct file yourself:" @@ -462,4 +492,4 @@ nvm_reset() { [ "_$NVM_ENV" = "_testing" ] || nvm_do_install -} # this ensures the entire script is downloaded # +} # this ensures the entire script is downloaded # \ No newline at end of file diff --git a/toolshed/post-provision.sh b/toolshed/post-provision.sh new file mode 100755 index 0000000..2ceef1b --- /dev/null +++ b/toolshed/post-provision.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +# +----------------------------------------------------------+ +# | BASH : Modifying Shell Behaviour +# | (https://www.gnu.org/software/bash/manual) +# +----------------------------------------------------------+ +# Treat unset variables and parameters other than the special +# parameters ‘@’ or ‘*’ as an error when performing parameter +# expansion. An error message will be written to the standard +# error, and a non-interactive shell will exit. +set -o nounset + +# Exit immediately if a pipeline returns a non-zero status. +set -o errexit + +# If set, the return value of a pipeline is the value of the +# last (rightmost) command to exit with a non-zero status, or +# zero if all commands in the pipeline exit successfully. +set -o pipefail + +# +----------------------------------------------------------+ + +echo "export LANG=en_US.UTF-8" >> ~/.bashrc +echo "export LANGUAGE=en_US:en" >> ~/.bashrc +echo "export LC_ALL=en_US.UTF-8" >> ~/.bashrc +echo "alias la=\"ls -lah\"" >> ~/.bashrc diff --git a/toolshed/pre-provision.sh b/toolshed/pre-provision.sh new file mode 100755 index 0000000..994aead --- /dev/null +++ b/toolshed/pre-provision.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +# +----------------------------------------------------------+ +# | BASH : Modifying Shell Behaviour +# | (https://www.gnu.org/software/bash/manual) +# +----------------------------------------------------------+ +# Treat unset variables and parameters other than the special +# parameters ‘@’ or ‘*’ as an error when performing parameter +# expansion. An error message will be written to the standard +# error, and a non-interactive shell will exit. +set -o nounset + +# Exit immediately if a pipeline returns a non-zero status. +set -o errexit + +# If set, the return value of a pipeline is the value of the +# last (rightmost) command to exit with a non-zero status, or +# zero if all commands in the pipeline exit successfully. +set -o pipefail + +# +----------------------------------------------------------+ +export DEBIAN_FRONTEND=noninteractive + +apt-get update +apt-get -y install unzip +apt-get -y install zip +apt-get -y install debconf + + +# setup locales in the container so Python can default to utf-8. +apt-get -y install locales +# from http://jaredmarkell.com/docker-and-locales/ +locale-gen en_US.UTF-8 +# See Dockerfile for exports + +apt-get -y install software-properties-common +add-apt-repository -y ppa:deadsnakes/ppa +apt-get update diff --git a/toolshed/provision.sh b/toolshed/provision.sh index 74ac797..81ce501 100755 --- a/toolshed/provision.sh +++ b/toolshed/provision.sh @@ -21,21 +21,6 @@ set -o pipefail # +----------------------------------------------------------+ export DEBIAN_FRONTEND=noninteractive -apt-get update -apt-get -y install unzip -apt-get -y install zip - -# setup locales in the container so Python can default to utf-8. -apt-get -y install locales -# from http://jaredmarkell.com/docker-and-locales/ -locale-gen en_US.UTF-8 -export ENV LANG=en_US.UTF-8 -export ENV LANGUAGE=en_US:en -export ENV LC_ALL=en_US.UTF-8 - -apt-get -y install software-properties-common -add-apt-repository -y ppa:deadsnakes/ppa -apt-get update apt-get -y install apt-utils apt-get -y install python3.10 apt-get -y install python3-pip @@ -54,8 +39,4 @@ apt-get -y install lsb-release apt-get -y install wget apt-get -y install gnupg apt-get -y install vim - -echo "export LANG=en_US.UTF-8" >> ~/.bashrc -echo "export LANGUAGE=en_US:en" >> ~/.bashrc -echo "export LC_ALL=en_US.UTF-8" >> ~/.bashrc -echo "alias la=\"ls -lah\"" >> ~/.bashrc +apt-get -y install g++-12 diff --git a/toxic/Dockerfile b/toxic/Dockerfile index 86d8000..10af34a 100644 --- a/toxic/Dockerfile +++ b/toxic/Dockerfile @@ -1,23 +1,42 @@ # -# Builds a docker image to use for development and CI. +# Builds the toxic/ubuntu-22.04 toolchain container. # -FROM ubuntu:22.04 +FROM ubuntu:22.04 AS base -LABEL org.opencontainers.image.description "Provides a consistent build and test environment for development, continuous-integration, and test automation of OpenCyphal Python based projects." +LABEL org.opencontainers.image.source=https://github.com/OpenCyphal/docker_toolchains +LABEL org.opencontainers.image.description="Provides a consistent build and test environment for development, continuous-integration, and test automation of OpenCyphal Python based projects." +LABEL org.opencontainers.image.licenses=MIT VOLUME /repo WORKDIR /repo -ARG DEBIAN_FRONTEND=noninteractive - -RUN apt-get update && apt-get -y install apt-utils && apt-get -y install locales -RUN locale-gen en_US.UTF-8 - ENV LANG=en_US.UTF-8 ENV LANGUAGE=en_US:en ENV LC_ALL=en_US.UTF-8 -COPY provision.sh / +# +---------------------------------------------------------------------------+ +# | UBUNTU PROVISIONING +# +---------------------------------------------------------------------------+ +FROM base AS provisioning +WORKDIR /tmp +ADD pre-provision.sh ./pre-provision.sh +ADD provision.sh ./provision.sh +ADD post-provision.sh ./post-provision.sh + +RUN ./pre-provision.sh +RUN ./provision.sh +RUN ./pre-provision.sh + -RUN /provision.sh + +# +---------------------------------------------------------------------------+ +# | FINALIZE +# +---------------------------------------------------------------------------+ +FROM provisioning AS all + +# INSTALL THE PATH FOR INTERATIVE SESSIONS +RUN echo "export PATH=$PATH" >> ~/.bashrc + +# LEAVE THE WORKDIR AS /repo +WORKDIR /repo diff --git a/toxic/README.md b/toxic/README.md index 46878c5..d087669 100644 --- a/toxic/README.md +++ b/toxic/README.md @@ -18,6 +18,13 @@ These instructions are for maintainers with permissions to push to the [OpenCyphal organization on Github](https://github.com/OpenCyphal/). Normally the container should be published by a github action but these instructions provide a way to manually update the container from any developer environment. +> **IMPORTANT NOTE** +> +> You must enable [containerd](https://containerd.io/) if you are using Docker Desktop to build locally (this is available as a general setting in Docker desktop). Docker desktop does not support multi-platform images and, if you try to use Docker Desktop without containerd, these instructions will fail with the message: +> +> *WARNING: No output specified with docker-container driver. Build result will only remain in the build cache.* +> + First create a temporary (7-day expiration please) personal access token (classic) with write:packages and read:packages scope. See [this github help page](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) for instructions. @@ -44,26 +51,18 @@ docker buildx use cyphalbuild ... then build the container: ```bash -docker buildx build --platform linux/amd64,linux/arm64 -t ghcr.io/opencyphal/toxic:tx22.4.x . +docker buildx build --platform linux/amd64,linux/arm64 --load -t ghcr.io/opencyphal/toxic:tx22.4.x . ``` (where x is the next version number for the container) +After this completes you'll see your image using the classic `docker images` command or the newer `buildx imagetools` command to inspect the multi-architecture manifest: -When the build completes you'll see the following warning: - -> WARNING: No output specified with docker-container driver. Build result will only remain in the build cache. To push result image into registry use --push or to load image into docker use --load - -It's important you don't restart your docker build container before you load and/or push since it may blow away the cache you need to push or load from. - -### Load -Currently you can't use --load with docker desktop so the only way to test the container is to rebuild for one platform only: - -```bash -docker buildx build --platform linux/arm64 --load -t ghcr.io/opencyphal/toxic:tx22.4.x . +``` +docker buildx imagetools inspect ghcr.io/opencyphal/toxic:tx22.4.x ``` -This will be operating off of the cache so it shouldn't take very long to complete. After it does you'll see your image using the classic `docker images` command. Now you can login to the container to test it out: + Now you can login to the container to test it out: ```bash docker run --rm -it -v ${PWD}:/repo ghcr.io/opencyphal/toxic:tx22.4.x diff --git a/toxic/post-provision.sh b/toxic/post-provision.sh new file mode 100755 index 0000000..2ceef1b --- /dev/null +++ b/toxic/post-provision.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +# +----------------------------------------------------------+ +# | BASH : Modifying Shell Behaviour +# | (https://www.gnu.org/software/bash/manual) +# +----------------------------------------------------------+ +# Treat unset variables and parameters other than the special +# parameters ‘@’ or ‘*’ as an error when performing parameter +# expansion. An error message will be written to the standard +# error, and a non-interactive shell will exit. +set -o nounset + +# Exit immediately if a pipeline returns a non-zero status. +set -o errexit + +# If set, the return value of a pipeline is the value of the +# last (rightmost) command to exit with a non-zero status, or +# zero if all commands in the pipeline exit successfully. +set -o pipefail + +# +----------------------------------------------------------+ + +echo "export LANG=en_US.UTF-8" >> ~/.bashrc +echo "export LANGUAGE=en_US:en" >> ~/.bashrc +echo "export LC_ALL=en_US.UTF-8" >> ~/.bashrc +echo "alias la=\"ls -lah\"" >> ~/.bashrc diff --git a/toxic/pre-provision.sh b/toxic/pre-provision.sh new file mode 100755 index 0000000..2465286 --- /dev/null +++ b/toxic/pre-provision.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +# +----------------------------------------------------------+ +# | BASH : Modifying Shell Behaviour +# | (https://www.gnu.org/software/bash/manual) +# +----------------------------------------------------------+ +# Treat unset variables and parameters other than the special +# parameters ‘@’ or ‘*’ as an error when performing parameter +# expansion. An error message will be written to the standard +# error, and a non-interactive shell will exit. +set -o nounset + +# Exit immediately if a pipeline returns a non-zero status. +set -o errexit + +# If set, the return value of a pipeline is the value of the +# last (rightmost) command to exit with a non-zero status, or +# zero if all commands in the pipeline exit successfully. +set -o pipefail + +# +----------------------------------------------------------+ +export DEBIAN_FRONTEND=noninteractive + +apt-get update + +# setup locales in the container so Python can default to utf-8. +apt-get -y install locales +# from http://jaredmarkell.com/docker-and-locales/ +locale-gen en_US.UTF-8 +# See Dockerfile for exports + +apt-get -y install software-properties-common +add-apt-repository -y ppa:deadsnakes/ppa +apt-get update + diff --git a/toxic/provision.sh b/toxic/provision.sh index bf1c228..fbe386d 100755 --- a/toxic/provision.sh +++ b/toxic/provision.sh @@ -19,10 +19,12 @@ set -o errexit set -o pipefail # +----------------------------------------------------------+ +export DEBIAN_FRONTEND=noninteractive apt-get -y install software-properties-common apt-get -y install git apt-get -y install curl +apt-get -y install wget apt-get -y install unzip # deadsnakes maintains a bunch of python versions for Ubuntu.