Matrix UBI images (#260)

Build per version UBI images for Nodejs, Python and Dotnet. We don't build Nodejs 22 or Python 3.10 as these are no prebuilt packages for these.
pulumi · Sep 23, 2024 · 797a4a4 · 797a4a4
1 parent caad030
commit 797a4a4
Show file tree

Hide file tree

Showing 11 changed files with 129 additions and 92 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -384,22 +384,46 @@ jobs:
             ${{ env.IMAGE_NAME }} \
             -test.parallel=1 -test.timeout=1h -test.v -test.run "TestPulumiTemplateTests|TestEnvironment"
 
+  define-ubi-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.define-matrix-sdk-manifests.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@master
+      - name: Define Matrix for UBI SDK Manifests
+        id: define-matrix-sdk-manifests
+        # Filter out the nodejs 22 and version from the matrix, as it is not supported on UBI.
+        # https://access.redhat.com/articles/3376841
+        # Filter out the python 3.10 version from the matrix, as it is not supported on UBI.
+        # https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/installing_and_using_dynamic_programming_languages/assembly_introduction-to-python_installing-and-using-dynamic-programming-languages#con_python-versions_assembly_introduction-to-python
+        run: |
+          echo matrix=$(python ./.github/scripts/matrix/gen-matrix.py --no-arch |
+            jq '.include |= map(
+              select(
+                (.sdk != "nodejs" or .language_version != "22") and
+                (.sdk != "python" or .language_version != "3.10")
+              )
+            )') >> "$GITHUB_OUTPUT"
+
   ubi-sdk:
     name: UBI SDK images
     runs-on: ubuntu-latest
     permissions:
       id-token: write
+    needs: define-ubi-matrix
     strategy:
       fail-fast: false
-      matrix:
-        sdk: ["nodejs", "python", "dotnet", "go", "java"]
+      matrix: ${{ fromJSON(needs.define-ubi-matrix.outputs.matrix) }}
     steps:
       # If no version of Pulumi is supplied by the incoming event (e.g. in the
       # case of a PR or merge to main), we use the latest production version:
       - name: Set version to latest
         if: ${{ !env.PULUMI_VERSION }}
         run: |
           echo "PULUMI_VERSION=$(curl https://www.pulumi.com/latest-version)" >> $GITHUB_ENV
+      - name: Set image name
+        run: |
+          echo "IMAGE_NAME=${{ env.DOCKER_ORG }}/pulumi-${{ matrix.sdk }}${{ matrix.suffix }}:${{ env.PULUMI_VERSION }}-ubi" >> $GITHUB_ENV
       - uses: actions/checkout@master
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
@@ -411,15 +435,13 @@ jobs:
         # We only build UBI for amd64 due to arm64 builds hanging on `npm
         # install -g yarn` with no additional output, plus the apparent
         # requirement of a paid subscription in order to file a bug with RedHat.
-        #
-        # We supply a working directory to the command below due to the
-        # dnf/nodej2.module file that has to mounted into the container.
         run: |
           docker build \
             -f docker/${{ matrix.sdk }}/Dockerfile.ubi \
             --platform linux/amd64 \
-            -t ${{ env.DOCKER_ORG }}/pulumi-${{ matrix.sdk }}:${{ env.PULUMI_VERSION }}-ubi \
+            -t ${{ env.IMAGE_NAME }} \
             --build-arg PULUMI_VERSION=${{ env.PULUMI_VERSION }} \
+            --build-arg LANGUAGE_VERSION=${{ matrix.language_version }} \
             docker/${{ matrix.sdk }} \
             --load
       - name: Install go
@@ -480,5 +502,5 @@ jobs:
             --mount type=bind,source=$GOOGLE_APPLICATION_CREDENTIALS,target=/src/creds.json \
             --volume /tmp:/src \
             --entrypoint /src/pulumi-test-containers \
-            ${{ env.DOCKER_ORG }}/pulumi-${{ matrix.sdk }}:${{ env.PULUMI_VERSION }}-ubi \
+            ${{ env.IMAGE_NAME }} \
             -test.parallel=1 -test.timeout=1h -test.v -test.run "TestPulumiTemplateTests|TestEnvironment"
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -319,7 +319,7 @@ jobs:
             ${{ env.DOCKER_ORG }}/pulumi-base:${{ env.PULUMI_VERSION }}-debian-amd64
           docker manifest push ${{ env.DOCKER_ORG }}/pulumi-base:latest
 
-  define-matrix:
+  define-debian-matrix:
     runs-on: ubuntu-latest
     outputs:
       matrix: ${{ steps.define-matrix.outputs.matrix }}
@@ -332,13 +332,13 @@ jobs:
 
   debian-sdk:
     name: Debian SDK images
-    needs: define-matrix
+    needs: define-debian-matrix
     runs-on: ubuntu-latest
     permissions:
       id-token: write
     strategy:
       fail-fast: false
-      matrix: ${{ fromJSON(needs.define-matrix.outputs.matrix) }}
+      matrix: ${{ fromJSON(needs.define-debian-matrix.outputs.matrix) }}
     steps:
       - uses: actions/checkout@master
       - name: Set image name
@@ -519,17 +519,40 @@ jobs:
             ${{ env.DOCKER_ORG }}/pulumi-${{ matrix.sdk }}${{ matrix.suffix }}:${{ env.PULUMI_VERSION }}-debian-amd64
           docker manifest push ${{ env.DOCKER_ORG }}/pulumi-${{ matrix.sdk }}:latest
 
+  define-ubi-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.define-matrix.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@master
+      - name: Define Matrix for UBI SDK Manifests
+        id: define-matrix
+        # Filter out the nodejs 22 and version from the matrix, as it is not supported on UBI.
+        # https://access.redhat.com/articles/3376841
+        # Filter out the python 3.10 version from the matrix, as it is not supported on UBI.
+        # https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/installing_and_using_dynamic_programming_languages/assembly_introduction-to-python_installing-and-using-dynamic-programming-languages#con_python-versions_assembly_introduction-to-python
+        run: |
+          echo matrix=$(python ./.github/scripts/matrix/gen-matrix.py --no-arch |
+            jq '.include |= map(
+              select(
+                (.sdk != "nodejs" or .language_version != "22") and
+                (.sdk != "python" or .language_version != "3.10")
+              )
+            )') >> "$GITHUB_OUTPUT"
 
   ubi-sdk:
     name: UBI SDK images
     runs-on: ubuntu-latest
     permissions:
       id-token: write
+    needs: define-ubi-matrix
     strategy:
       fail-fast: false
-      matrix:
-        sdk: ["nodejs", "python", "dotnet", "go", "java"]
+      matrix: ${{ fromJSON(needs.define-ubi-matrix.outputs.matrix) }}
     steps:
+      - name: Set image name
+        run: |
+          echo "IMAGE_NAME=${{ env.DOCKER_ORG }}/pulumi-${{ matrix.sdk }}${{ matrix.suffix }}:${{ env.PULUMI_VERSION }}-ubi" >> $GITHUB_ENV
       - uses: actions/checkout@master
       - name: Login to Docker Hub
         uses: docker/login-action@v1
@@ -545,15 +568,13 @@ jobs:
       - name: Build
         # We only build UBI for amd64 due to arm64 builds hanging on `npm
         # install -g yarn` with no additional output.
-        #
-        # We supply a working directory to the command below due to the
-        # dnf/nodej2.module file that has to mounted into the container.
         run: |
           docker build \
             -f docker/${{ matrix.sdk }}/Dockerfile.ubi \
             --platform linux/amd64 \
-            -t ${{ env.DOCKER_ORG }}/pulumi-${{ matrix.sdk }}:${{ env.PULUMI_VERSION }}-ubi \
+            -t ${{ env.IMAGE_NAME }} \
             --build-arg PULUMI_VERSION=${{ env.PULUMI_VERSION }} \
+            --build-arg LANGUAGE_VERSION=${{ matrix.language_version }} \
             docker/${{ matrix.sdk }} \
             --load
       - name: Install go
@@ -565,7 +586,6 @@ jobs:
         working-directory: tests
         run: |
           GOOS=linux GOARCH=amd64 go test -c -o /tmp/pulumi-test-containers ./...
-
       - name: Set SDKS_TO_TEST (dotnet)
         if: ${{ matrix.sdk == 'dotnet' }}
         run: echo "SDKS_TO_TEST=csharp" >> $GITHUB_ENV
@@ -615,11 +635,11 @@ jobs:
             --mount type=bind,source=$GOOGLE_APPLICATION_CREDENTIALS,target=/src/creds.json \
             --volume /tmp:/src \
             --entrypoint /src/pulumi-test-containers \
-            ${{ env.DOCKER_ORG }}/pulumi-${{ matrix.sdk }}:${{ env.PULUMI_VERSION }}-ubi \
+            ${{ env.IMAGE_NAME }} \
             -test.parallel=1 -test.timeout=1h -test.v -test.run "TestPulumiTemplateTests|TestEnvironment"
       - name: Push image
         run: |
-          docker push ${{ env.DOCKER_ORG }}/pulumi-${{ matrix.sdk }}:${{ env.PULUMI_VERSION }}-ubi
+          docker push ${{ env.IMAGE_NAME }}
 
   start-syncs:
     name: Start syncs to other container registries

diff --git a/.github/workflows/sync-ecr.yml b/.github/workflows/sync-ecr.yml
@@ -135,21 +135,25 @@ jobs:
             public.ecr.aws/${{ env.DOCKER_USERNAME }}/${{ matrix.image }}:${{ env.PULUMI_VERSION }}-debian-arm64
           docker manifest push public.ecr.aws/${{ env.DOCKER_USERNAME }}/${{ matrix.image }}:latest
 
+  define-ubi-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.define-matrix.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@master
+      - name: Define Matrix
+        id: define-matrix
+        run: |
+          echo matrix=$(python ./.github/scripts/matrix/gen-sync-matrix.py) >> "$GITHUB_OUTPUT"
+
   # NOTE: If UBI images become multi platform, this job can be replaced by adding a similar step to "-debian" for "-ubi" the previous job.
   ubi-images:
     name: UBI SDK and base images
     runs-on: ubuntu-latest
+    needs: define-ubi-matrix
     strategy:
       fail-fast: false
-      matrix:
-        image:
-          [
-            "pulumi-base",
-            "pulumi-go",
-            "pulumi-nodejs",
-            "pulumi-python",
-            "pulumi-dotnet",
-          ]
+      matrix: ${{ fromJSON(needs.define-ubi-matrix.outputs.matrix) }}
     steps:
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@v1

diff --git a/.github/workflows/sync-ghcr.yml b/.github/workflows/sync-ghcr.yml
@@ -114,21 +114,25 @@ jobs:
             ghcr.io/${{ env.DOCKER_USERNAME }}/${{ matrix.image }}:${{ env.PULUMI_VERSION }}-debian-arm64
           docker manifest push ghcr.io/${{ env.DOCKER_USERNAME }}/${{ matrix.image }}:latest
 
+  define-ubi-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.define-matrix.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@master
+      - name: Define Matrix
+        id: define-matrix
+        run: |
+          echo matrix=$(python ./.github/scripts/matrix/gen-sync-matrix.py) >> "$GITHUB_OUTPUT"
+
   # NOTE: If UBI images become multi platform, this job can be replaced by adding a similar step to "-debian" for "-ubi" the previous job.
   ubi-images:
     name: UBI SDK and base images
     runs-on: ubuntu-latest
+    needs: define-ubi-matrix
     strategy:
       fail-fast: false
-      matrix:
-        image:
-          [
-            "pulumi-base",
-            "pulumi-go",
-            "pulumi-nodejs",
-            "pulumi-python",
-            "pulumi-dotnet",
-          ]
+      matrix: ${{ fromJSON(needs.define-ubi-matrix.outputs.matrix) }}
     steps:
       - name: Login to GitHub Container Registry
         uses: docker/login-action@v1

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,9 @@
 
 ## Unreleased
 
+- Add per language versions for ubi images
+  ([#260](https://github.com/pulumi/pulumi-docker-containers/pull/260))
+
 ## 3.132.0
 
 - Add dotnet 8.0 to the kitchen sink image

diff --git a/docker/dotnet/Dockerfile.ubi b/docker/dotnet/Dockerfile.ubi
@@ -1,37 +1,31 @@
 # syntax = docker/dockerfile:experimental
 # Interim container so we can copy pulumi binaries
-# Must be defined first
-FROM registry.access.redhat.com/ubi8/ubi-minimal:latest as builder
+FROM redhat/ubi8-minimal:latest as builder
 ARG PULUMI_VERSION
 RUN microdnf install -y \
     curl \
-    make \
-    gcc \
-    git \
-    tar \
-    gcc-c++
-
+    gzip \
+    tar
 # Install the Pulumi SDK, including the CLI and language runtimes.
 RUN curl -fsSL https://get.pulumi.com/ | bash -s -- --version $PULUMI_VERSION
 
-
 # The runtime container
 FROM redhat/ubi8-minimal:latest
+ARG LANGUAGE_VERSION
 LABEL org.opencontainers.image.description="Pulumi CLI container for dotnet"
-ARG DOTNET_VERSION=6.0
 WORKDIR /pulumi/projects
 
 RUN microdnf install -y \
     ca-certificates \
-    tar \
-    git \
     curl \
     # Required by the dotnet-install script, which calls `find`:
-    findutils
+    findutils \
+    git \
+    tar
 
-# Install dotnet 6.0 using instructions from:
+# Install dotnet using instructions from:
 # https://docs.microsoft.com/en-us/dotnet/core/tools/dotnet-install-script
-RUN curl -fsSL https://dot.net/v1/dotnet-install.sh | bash -s -- -channel ${DOTNET_VERSION} 
+RUN curl -fsSL https://dot.net/v1/dotnet-install.sh | bash -s -- -channel ${LANGUAGE_VERSION}
 
 # Uses the workdir, copies from pulumi interim container
 COPY --from=builder /root/.pulumi/bin/pulumi /pulumi/bin/pulumi

diff --git a/docker/go/Dockerfile.ubi b/docker/go/Dockerfile.ubi
@@ -1,16 +1,11 @@
 # syntax = docker/dockerfile:experimental
-
-# Build container
+# Interim container so we can copy pulumi binaries
 FROM redhat/ubi8-minimal:latest as builder
 ARG PULUMI_VERSION
 RUN microdnf install -y \
     curl \
-    make \
-    gcc \
-    git \
-    tar \
-    gcc-c++
-
+    gzip \
+    tar
 # Install the Pulumi SDK, including the CLI and language runtimes.
 RUN curl -fsSL https://get.pulumi.com/ | bash -s -- --version $PULUMI_VERSION
 
@@ -19,13 +14,6 @@ ARG RUNTIME_VERSION=1.21.1
 ENV RUNTIME_SHA256 b3075ae1ce5dab85f89bc7905d1632de23ca196bd8336afd93fa97434cfa55ae
 
 WORKDIR /golang
-RUN microdnf install -y \
-    curl \
-    make \
-    gcc \
-    git \
-    tar \
-    gcc-c++
 RUN curl -fsSLo /tmp/go.tgz https://golang.org/dl/go${RUNTIME_VERSION}.linux-amd64.tar.gz && \
     echo "${RUNTIME_SHA256} /tmp/go.tgz" | sha256sum -c -; \
     mkdir -p bin; \
@@ -41,6 +29,7 @@ WORKDIR /pulumi/projects
 
 # Install needed tools, like git
 RUN microdnf install -y \
+    curl \
     git \
     tar \
     ca-certificates; \

diff --git a/docker/nodejs/Dockerfile.ubi b/docker/nodejs/Dockerfile.ubi
@@ -1,32 +1,33 @@
 # syntax = docker/dockerfile:experimental
 # Interim container so we can copy pulumi binaries
-# Must be defined first
 FROM redhat/ubi8-minimal:latest as builder
 ARG PULUMI_VERSION
 RUN microdnf install -y \
     curl \
-    make \
-    gcc \
-    git \
-    tar \
-    gcc-c++
+    gzip \
+    tar
 # Install the Pulumi SDK, including the CLI and language runtimes.
 RUN curl -fsSL https://get.pulumi.com/ | bash -s -- --version $PULUMI_VERSION
 
 # The runtime container
 FROM redhat/ubi8-minimal:latest
+ARG LANGUAGE_VERSION
 LABEL org.opencontainers.image.description="Pulumi CLI container for nodejs"
 WORKDIR /pulumi/projects
 
 COPY dnf/nodejs.module /etc/dnf/modules.d/nodejs.module
+RUN sed -i s"/__LANGUAGE_VERSION_PLACEHOLDER__/${LANGUAGE_VERSION}/g" /etc/dnf/modules.d/nodejs.module
+RUN cat /etc/dnf/modules.d/nodejs.module
 
 RUN microdnf install -y \
+    ca-certificates \
+    curl \
     git \
-    tar \
     nodejs \
-    ca-certificates && \
+    tar && \
     npm install -g [email protected] && \
-    npm install -g yarn
+    npm install -g corepack && \
+    corepack install -g yarn@1
 
 # Uses the workdir, copies from pulumi interim container
 COPY --from=builder /root/.pulumi/bin/pulumi /pulumi/bin/pulumi