RHOAIENG-15333: chore(ci): create a Python script that reads Workbench image manifests and outputs snippet suited for inclusion in Docs

.github/pull.yml

@@ -0,0 +1,19 @@
+---
+version: "1"
+conflictLabel: "needs-rebase"
+rules:
+  - base: main
+    upstream: opendatahub-io:main
+    mergeMethod: none
+  - base: release-2023a
+    upstream: opendatahub-io:2023a
+    mergeMethod: none
+  - base: release-2023b
+    upstream: opendatahub-io:2023b
+    mergeMethod: none
+  - base: release-2024a
+    upstream: opendatahub-io:2024a
+    mergeMethod: none
+  - base: release-2024b
+    upstream: opendatahub-io:2024b
+    mergeMethod: none

.github/workflows/build-notebooks.yaml

@@ -515,48 +515,6 @@
             },
             "secrets": "inherit"
         },
-        "base-c9s-python-3_11": {
-            "needs": [],
-            "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml",
-            "with": {
-                "target": "base-c9s-python-3.11",
-                "github": "${{ toJSON(github) }}"
-            },
-            "secrets": "inherit"
-        },
-        "cuda-c9s-python-3_11": {
-            "needs": [
-                "base-c9s-python-3_11"
-            ],
-            "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml",
-            "with": {
-                "target": "cuda-c9s-python-3.11",
-                "github": "${{ toJSON(github) }}"
-            },
-            "secrets": "inherit"
-        },
-        "rstudio-c9s-python-3_11": {
-            "needs": [
-                "base-c9s-python-3_11"
-            ],
-            "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml",
-            "with": {
-                "target": "rstudio-c9s-python-3.11",
-                "github": "${{ toJSON(github) }}"
-            },
-            "secrets": "inherit"
-        },
-        "cuda-rstudio-c9s-python-3_11": {
-            "needs": [
-                "cuda-c9s-python-3_11"
-            ],
-            "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml",
-            "with": {
-                "target": "cuda-rstudio-c9s-python-3.11",
-                "github": "${{ toJSON(github) }}"
-            },
-            "secrets": "inherit"
-        },
         "rocm-ubi9-python-3_9": {
             "needs": [
                 "base-ubi9-python-3_9"

.github/workflows/docs.yaml

@@ -0,0 +1,61 @@
+---
+"name": "Docs (release notes)"
+"on":
+  "push":
+  "pull_request":
+  "workflow_dispatch":
+
+permissions:
+  contents: read
+
+env:
+  poetry_version: '1.8.3'
+
+jobs:
+  generate-releasenotes:
+    name: Generate list of images for release notes
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Cache poetry in ~/.local
+        uses: actions/cache/restore@v4
+        id: cache-poetry-restore
+        with:
+          path: ~/.local
+          key: "${{ runner.os }}-local-${{ env.poetry_version }}"
+
+      - name: Install poetry
+        if: steps.cache-poetry-restore.outputs.cache-hit != 'true'
+        run: pipx install poetry==${{ env.poetry_version }}
+        env:
+          PIPX_HOME: /home/runner/.local/pipx
+          PIPX_BIN_DIR: /home/runner/.local/bin
+
+      - name: Check poetry is installed correctly
+        run: poetry env info
+
+      - name: Save cache
+        if: steps.cache-poetry-restore.outputs.cache-hit != 'true'
+        uses: actions/cache/save@v4
+        with:
+          path: ~/.local
+          key: ${{ steps.cache-poetry-restore.outputs.cache-primary-key }}
+
+      - name: Set up Python
+        id: setup-python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+          cache: 'poetry'
+
+      - name: Configure poetry
+        run: poetry env use "${{ steps.setup-python.outputs.python-path }}"
+
+      - name: Install deps
+        run: poetry install --sync
+
+      - name: Run the release notes script
+        run: |
+          set -Eeuxo pipefail
+          poetry run ci/package_versions.py | tee ${GITHUB_STEP_SUMMARY}

.github/workflows/notebooks-digest-updater-upstream.yaml → .github/workflows/notebook-digest-updater.yaml

@@ -1,6 +1,6 @@
 ---
-# The aim of this GitHub workflow is to update the params.env file with the latest digest.
-name: Update notebook image build commit hashes
+# The aim of this GitHub workflow is to update the params.env and commit.env files with the latest builds.
+name: Update notebook image build references (downstream)
 on:  # yamllint disable-line rule:truthy
   workflow_dispatch:
     inputs:
@@ -51,12 +51,12 @@ jobs:
           git config --global user.email "github-actions[bot]@users.noreply.github.com"
           git config --global user.name "GitHub Actions"
 
-      # Get latest build commit from the https://github.com/opendatahub-io/notebooks/${release_branch} using this as identifier for the latest tag name
+        # Get latest build commit from the https://github.com/red-hat-data-services/notebooks/${release_branch} using this as identifier for the latest tag name
       - name: Retrive latest commit hash from the release branch
         id: hash-n
         shell: bash
         run: |
-          PAYLOAD=$(curl --silent -H 'Accept: application/vnd.github.v4.raw' https://api.github.com/repos/opendatahub-io/notebooks/commits?sha=$RELEASE_VERSION_N&per_page=1)
+          PAYLOAD=$(curl --silent -H 'Accept: application/vnd.github.v4.raw' https://api.github.com/repos/red-hat-data-services/notebooks/commits?sha=release-$RELEASE_VERSION_N&per_page=1)
           echo "HASH_N=$(echo $PAYLOAD | jq -r '.[0].sha[0:7]')" >> ${GITHUB_OUTPUT}
 
       # Checkout the release branch to apply the updates
@@ -66,31 +66,41 @@ jobs:
           ref: ${{ env.DIGEST_UPDATER_BRANCH }}
 
       - name: Update the params.env file
+        shell: bash
         run: |
-          PARAMS_ENV_PATH="manifests/base/params.env"
-
           echo Latest commit is: ${{ steps.hash-n.outputs.HASH_N }} on ${{ env.RELEASE_VERSION_N }}
 
           # Get the complete list of images N-version to update
+          PARAMS_ENV_PATH="manifests/base/params.env"
           IMAGES=$(grep "\-n=" "${PARAMS_ENV_PATH}" | cut -d "=" -f 1)
 
+          # The order of the regexes array should match with the params.env file
+          REGEXES=("v3-${{ env.RELEASE_VERSION_N }}-\d{8}-+${{ steps.hash-n.outputs.HASH_N }}" \
+                  "cuda-[a-z]+-minimal-[a-z0-9]+-[a-z]+-3.11-${{ env.RELEASE_VERSION_N }}-\d{8}-${{ steps.hash-n.outputs.HASH_N }}" \
+                  "v3-${{ env.RELEASE_VERSION_N }}-\d{8}-+${{ steps.hash-n.outputs.HASH_N }}" \
+                  "v3-${{ env.RELEASE_VERSION_N }}-\d{8}-+${{ steps.hash-n.outputs.HASH_N }}" \
+                  "cuda-[a-z]+-tensorflow-[a-z0-9]+-[a-z]+-3.11-${{ env.RELEASE_VERSION_N }}-\d{8}-${{ steps.hash-n.outputs.HASH_N }}" \
+                  "v3-${{ env.RELEASE_VERSION_N }}-\d{8}-+${{ steps.hash-n.outputs.HASH_N }}" \
+                  "codeserver-[a-z0-9]+-[a-z]+-3.11-${{ env.RELEASE_VERSION_N }}-\d{8}-${{ steps.hash-n.outputs.HASH_N }}" \
+                  "rocm-[a-z]+-minimal-[a-z0-9]+-[a-z]+-3.11-${{ env.RELEASE_VERSION_N }}-\d{8}-${{ steps.hash-n.outputs.HASH_N }}" \
+                  "rocm-[a-z]+-pytorch-[a-z0-9]+-[a-z]+-3.11-${{ env.RELEASE_VERSION_N }}-\d{8}-${{ steps.hash-n.outputs.HASH_N }}" \
+                  "rocm-[a-z]+-tensorflow-[a-z0-9]+-[a-z]+-3.11-${{ env.RELEASE_VERSION_N }}-\d{8}-${{ steps.hash-n.outputs.HASH_N }}")
+
+          i=0
           for image in ${IMAGES}; do
             echo "CHECKING: '${image}'"
             img=$(grep -E "${image}=" "${PARAMS_ENV_PATH}" | cut -d '=' -f2)
             registry=$(echo "${img}" | cut -d '@' -f1)
 
+            regex=${REGEXES[$i]}
             skopeo_metadata=$(skopeo inspect --retry-times 3 "docker://${img}")
-
-            src_tag=$(echo "${skopeo_metadata}" | jq '.Env[] | select(startswith("OPENSHIFT_BUILD_NAME=")) | split("=")[1]' | tr -d '"' | sed 's/-amd64$//')
-            regex="^$src_tag-${{ env.RELEASE_VERSION_N}}-\d+-${{ steps.hash-n.outputs.HASH_N }}\$"
             latest_tag=$(echo "${skopeo_metadata}" | jq -r --arg regex "$regex" '.RepoTags | map(select(. | test($regex))) | .[0]')
-            # use `--no-tags` for skopeo once available in newer version
             digest=$(skopeo inspect --retry-times 3 "docker://${registry}:${latest_tag}" | jq .Digest | tr -d '"')
             output="${registry}@${digest}"
             echo "NEW: ${output}"
             sed -i "s|${image}=.*|${image}=${output}|" "${PARAMS_ENV_PATH}"
+            i=$((i+1))
           done
-
           if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then
             git fetch origin ${{ env.DIGEST_UPDATER_BRANCH }} && \
             git pull origin ${{ env.DIGEST_UPDATER_BRANCH }} && \
@@ -103,9 +113,10 @@ jobs:
 
       - name: Update the commit.env file
         run: |
+          echo Latest commit is: ${{ steps.hash-n.outputs.HASH_N }} on ${{ env.RELEASE_VERSION_N }}
+
           COMMIT_ENV_PATH="manifests/base/commit.env"
 
-          echo Latest commit is: ${{ steps.hash-n.outputs.HASH_N }} on ${{ env.RELEASE_VERSION_N }}
           # Get the complete list of commits N-version to update
           COMMIT=$(grep "\-n=" "${COMMIT_ENV_PATH}" | cut -d "=" -f 1)
 
@@ -135,12 +146,12 @@ jobs:
           git config --global user.email "github-actions[bot]@users.noreply.github.com"
           git config --global user.name "GitHub Actions"
 
-      # Get latest build commit from the https://github.com/opendatahub-io/notebooks/${release_branch} using this as identifier for the latest tag name
+      # Get latest build commit from the https://github.com/red-hat-data-services/notebooks/${release_branch} using this as identifier for the latest tag name
       - name: Retrive latest commit hash from the release branch
         id: hash-n-1
         shell: bash
         run: |
-          PAYLOAD=$(curl --silent -H 'Accept: application/vnd.github.v4.raw' https://api.github.com/repos/opendatahub-io/notebooks/commits?sha=$RELEASE_VERSION_N_1&per_page=1)
+          PAYLOAD=$(curl --silent -H 'Accept: application/vnd.github.v4.raw' https://api.github.com/repos/red-hat-data-services/notebooks/commits?sha=release-$RELEASE_VERSION_N_1&per_page=1)
           echo "HASH_N_1=$(echo $PAYLOAD | jq -r '.[0].sha[0:7]')" >> ${GITHUB_OUTPUT}
 
       # Checkout the release branch to apply the updates
@@ -149,30 +160,37 @@ jobs:
         with:
           ref: ${{ env.DIGEST_UPDATER_BRANCH }}
 
-      - name: Update the param.env file
+      - name: Update the params.env file
+        shell: bash
         run: |
-          PARAMS_ENV_PATH="manifests/base/params.env"
-
-          echo Latest commit is: ${{ steps.hash-n-1.outputs.HASH_N_1 }} on ${{ env.RELEASE_VERSION_N_1 }}
+          echo Latest commit is: ${{ steps.hash-n-1.outputs.HASH_N_1 }} on ${{ env.RELEASE_VERSION_N_1}}
 
           # Get the complete list of images N-1-version to update
+          PARAMS_ENV_PATH="manifests/base/params.env"
           IMAGES=$(grep "\-n-1=" "${PARAMS_ENV_PATH}" | cut -d "=" -f 1)
 
+          # The order of the regexes array should match with the params.env file
+          REGEXES=("v2-${{ env.RELEASE_VERSION_N_1 }}-\d{8}+-${{ steps.hash-n-1.outputs.HASH_N_1 }}" \
+                  "cuda-[a-z]+-minimal-[a-z0-9]+-[a-z]+-3.9-${{ env.RELEASE_VERSION_N_1 }}-\d{8}-${{ steps.hash-n-1.outputs.HASH_N_1 }}" \
+                  "v2-${{ env.RELEASE_VERSION_N_1 }}-\d{8}+-${{ steps.hash-n-1.outputs.HASH_N_1 }}" \
+                  "v2-${{ env.RELEASE_VERSION_N_1 }}-\d{8}+-${{ steps.hash-n-1.outputs.HASH_N_1 }}" \
+                  "cuda-[a-z]+-tensorflow-[a-z0-9]+-[a-z]+-3.9-${{ env.RELEASE_VERSION_N_1 }}-\d{8}-${{ steps.hash-n-1.outputs.HASH_N_1 }}" \
+                  "v2-${{ env.RELEASE_VERSION_N_1 }}-\d{8}+-${{ steps.hash-n-1.outputs.HASH_N_1 }}" \
+                  "codeserver-[a-z0-9]+-[a-z]+-3.9-${{ env.RELEASE_VERSION_N_1 }}-\d{8}-${{ steps.hash-n-1.outputs.HASH_N_1 }}" )
+
+          i=0
           for image in ${IMAGES}; do
             echo "CHECKING: '${image}'"
             img=$(grep -E "${image}=" "${PARAMS_ENV_PATH}" | cut -d '=' -f2)
             registry=$(echo "${img}" | cut -d '@' -f1)
-
+            regex=${REGEXES[$i]}
             skopeo_metadata=$(skopeo inspect --retry-times 3 "docker://${img}")
-
-            src_tag=$(echo "${skopeo_metadata}" | jq '.Env[] | select(startswith("OPENSHIFT_BUILD_NAME=")) | split("=")[1]' | tr -d '"' | sed 's/-amd64$//')
-            regex="^$src_tag-${{ env.RELEASE_VERSION_N_1}}-\d+-${{ steps.hash-n-1.outputs.HASH_N_1 }}\$"
             latest_tag=$(echo "${skopeo_metadata}" | jq -r --arg regex "$regex" '.RepoTags | map(select(. | test($regex))) | .[0]')
-            # use `--no-tags` for skopeo once available in newer version
             digest=$(skopeo inspect --retry-times 3 "docker://${registry}:${latest_tag}" | jq .Digest | tr -d '"')
             output="${registry}@${digest}"
             echo "NEW: ${output}"
             sed -i "s|${image}=.*|${image}=${output}|" "${PARAMS_ENV_PATH}"
+            i=$((i+1))
           done
 
           if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then