Merge sbom manipulation steps

The steps merge-syft-sboms, merge-cachi2-sbom, create-purl-sbom, create-base-images-sbom were merged into single "prepare-sboms" step. This is done because the tekton results size for a task depends on the number of steps. By reducing the number of the steps we increase the limit. The scripts from the individual steps were moved to the build-tasks-dockerfiles repo that also contains the Dockerfile for the sbom-utility-scripts image. STONEBLD-2608 Signed-off-by: mkosiarc <[email protected]>
konflux-ci · Aug 14, 2024 · 1e73297 · 1e73297
1 parent 7a8a7aa
commit 1e73297
Show file tree

Hide file tree

Showing 6 changed files with 42 additions and 342 deletions.
diff --git a/renovate.json b/renovate.json
@@ -27,7 +27,8 @@
         "quay.io/redhat-appstudio/hacbs-jvm-build-request-processor",
         "quay.io/redhat-appstudio/build-definitions-source-image-build-utils",
         "quay.io/redhat-appstudio/cosign",
-        "quay.io/redhat-appstudio/cachi2"
+        "quay.io/redhat-appstudio/cachi2",
+        "quay.io/redhat-appstudio/sbom-utility-scripts-image"
       ],
       "groupName": "build",
       "reviewers": ["mmorhun", "tkdchen", "rcerven", "mkosiarc", "brunoapimentel", "chmeliik"]

diff --git a/task/buildah-oci-ta/0.2/buildah-oci-ta.yaml b/task/buildah-oci-ta/0.2/buildah-oci-ta.yaml
@@ -475,96 +475,24 @@ spec:
           memory: 256Mi
       securityContext:
         runAsUser: 0
-    - name: merge-syft-sboms
-      image: registry.access.redhat.com/ubi9/python-39:1-192.1722518946@sha256:0176b477075984d5a502253f951d2502f0763c551275f9585ac515b9f241d73d
+    - name: prepare-sboms
+      image: quay.io/redhat-appstudio/sbom-utility-scripts-image@sha256:064eeec1cfb75abd21b533bf0d9870a7a3fb138f30192596dce28061798cbf06
       workingDir: /var/workdir
       script: |
-        #!/bin/python3
-        import json
+        echo "Merging contents of sbom-source.json and sbom-image.json into sbom-cyclonedx.json"
+        python3 /scripts/merge_syft_sboms.py
 
-        # load SBOMs
-        with open("./sbom-image.json") as f:
-          image_sbom = json.load(f)
-
-        with open("./sbom-source.json") as f:
-          source_sbom = json.load(f)
-
-        # fetch unique components from available SBOMs
-        def get_identifier(component):
-          return component["name"] + '@' + component.get("version", "")
-
-        image_sbom_components = image_sbom.setdefault("components", [])
-        existing_components = [get_identifier(component) for component in image_sbom_components]
-
-        source_sbom_components = source_sbom.get("components", [])
-        for component in source_sbom_components:
-          if get_identifier(component) not in existing_components:
-            image_sbom_components.append(component)
-            existing_components.append(get_identifier(component))
-
-        image_sbom_components.sort(key=lambda c: get_identifier(c))
-
-        # write the CycloneDX unified SBOM
-        with open("./sbom-cyclonedx.json", "w") as f:
-          json.dump(image_sbom, f, indent=4)
-      computeResources:
-        limits:
-          cpu: 200m
-          memory: 512Mi
-        requests:
-          cpu: 100m
-          memory: 256Mi
-      securityContext:
-        runAsUser: 0
-    - name: merge-cachi2-sbom
-      image: quay.io/redhat-appstudio/cachi2:0.9.1@sha256:df67f9e063b544a8c49a271359377fed560562615e0278f6d0b9a3485f3f8fad
-      workingDir: /var/workdir
-      script: |
         if [ -f "sbom-cachi2.json" ]; then
           echo "Merging contents of sbom-cachi2.json into sbom-cyclonedx.json"
-          merge_syft_sbom sbom-cachi2.json sbom-cyclonedx.json >sbom-temp.json
+          python3 /scripts/merge_cachi2_sboms.py sbom-cachi2.json sbom-cyclonedx.json >sbom-temp.json
           mv sbom-temp.json sbom-cyclonedx.json
-        else
-          echo "Skipping step since no Cachi2 SBOM was produced"
         fi
-      computeResources:
-        limits:
-          cpu: 200m
-          memory: 512Mi
-        requests:
-          cpu: 100m
-          memory: 256Mi
-      securityContext:
-        runAsUser: 0
-    - name: create-purl-sbom
-      image: registry.access.redhat.com/ubi9/python-39:1-192.1722518946@sha256:0176b477075984d5a502253f951d2502f0763c551275f9585ac515b9f241d73d
-      workingDir: /var/workdir
-      script: |
-        #!/bin/python3
-        import json
 
-        with open("./sbom-cyclonedx.json") as f:
-          cyclonedx_sbom = json.load(f)
+        echo "Creating sbom-purl.json"
+        python3 /scripts/create_purl_sbom.py
 
-        purls = [{"purl": component["purl"]} for component in cyclonedx_sbom.get("components", []) if "purl" in component]
-        purl_content = {"image_contents": {"dependencies": purls}}
-
-        with open("sbom-purl.json", "w") as output_file:
-          json.dump(purl_content, output_file, indent=4)
-      computeResources:
-        limits:
-          cpu: 200m
-          memory: 512Mi
-        requests:
-          cpu: 100m
-          memory: 256Mi
-      securityContext:
-        runAsUser: 0
-    - name: create-base-images-sbom
-      image: quay.io/redhat-appstudio/base-images-sbom-script@sha256:667669e3def018f9dbb8eaf8868887a40bc07842221e9a98f6787edcff021840
-      workingDir: /var/workdir
-      script: |
-        python3 /app/base_images_sbom_script.py \
+        echo "Adding base images data to sbom-cyclonedx.json"
+        python3 /scripts/base_images_sbom_script.py \
           --sbom=sbom-cyclonedx.json \
           --base-images-from-dockerfile=/shared/base_images_from_dockerfile \
           --base-images-digests=/shared/base_images_digests

diff --git a/task/buildah-remote-oci-ta/0.2/buildah-remote-oci-ta.yaml b/task/buildah-remote-oci-ta/0.2/buildah-remote-oci-ta.yaml
@@ -564,105 +564,28 @@ spec:
       requests:
         cpu: 100m
         memory: 256Mi
-    image: registry.access.redhat.com/ubi9/python-39:1-192.1722518946@sha256:0176b477075984d5a502253f951d2502f0763c551275f9585ac515b9f241d73d
-    name: merge-syft-sboms
-    script: |
-      #!/bin/python3
-      import json
-
-      # load SBOMs
-      with open("./sbom-image.json") as f:
-        image_sbom = json.load(f)
-
-      with open("./sbom-source.json") as f:
-        source_sbom = json.load(f)
-
-      # fetch unique components from available SBOMs
-      def get_identifier(component):
-        return component["name"] + '@' + component.get("version", "")
-
-      image_sbom_components = image_sbom.setdefault("components", [])
-      existing_components = [get_identifier(component) for component in image_sbom_components]
-
-      source_sbom_components = source_sbom.get("components", [])
-      for component in source_sbom_components:
-        if get_identifier(component) not in existing_components:
-          image_sbom_components.append(component)
-          existing_components.append(get_identifier(component))
-
-      image_sbom_components.sort(key=lambda c: get_identifier(c))
-
-      # write the CycloneDX unified SBOM
-      with open("./sbom-cyclonedx.json", "w") as f:
-        json.dump(image_sbom, f, indent=4)
-    securityContext:
-      runAsUser: 0
-    workingDir: /var/workdir
-  - computeResources:
-      limits:
-        cpu: 200m
-        memory: 512Mi
-      requests:
-        cpu: 100m
-        memory: 256Mi
-    image: quay.io/redhat-appstudio/cachi2:0.9.1@sha256:df67f9e063b544a8c49a271359377fed560562615e0278f6d0b9a3485f3f8fad
-    name: merge-cachi2-sbom
+    image: quay.io/redhat-appstudio/sbom-utility-scripts-image@sha256:064eeec1cfb75abd21b533bf0d9870a7a3fb138f30192596dce28061798cbf06
+    name: prepare-sboms
     script: |
       #!/bin/bash
       set -e
       if [[ "${IMAGE##*-}" != "${PLATFORM##*/}" ]]; then
         export IMAGE="${IMAGE}-${PLATFORM##*/}"
       fi
+      echo "Merging contents of sbom-source.json and sbom-image.json into sbom-cyclonedx.json"
+      python3 /scripts/merge_syft_sboms.py
+
       if [ -f "sbom-cachi2.json" ]; then
         echo "Merging contents of sbom-cachi2.json into sbom-cyclonedx.json"
-        merge_syft_sbom sbom-cachi2.json sbom-cyclonedx.json >sbom-temp.json
+        python3 /scripts/merge_cachi2_sboms.py sbom-cachi2.json sbom-cyclonedx.json >sbom-temp.json
         mv sbom-temp.json sbom-cyclonedx.json
-      else
-        echo "Skipping step since no Cachi2 SBOM was produced"
       fi
-    securityContext:
-      runAsUser: 0
-    workingDir: /var/workdir
-  - computeResources:
-      limits:
-        cpu: 200m
-        memory: 512Mi
-      requests:
-        cpu: 100m
-        memory: 256Mi
-    image: registry.access.redhat.com/ubi9/python-39:1-192.1722518946@sha256:0176b477075984d5a502253f951d2502f0763c551275f9585ac515b9f241d73d
-    name: create-purl-sbom
-    script: |
-      #!/bin/python3
-      import json
-
-      with open("./sbom-cyclonedx.json") as f:
-        cyclonedx_sbom = json.load(f)
 
-      purls = [{"purl": component["purl"]} for component in cyclonedx_sbom.get("components", []) if "purl" in component]
-      purl_content = {"image_contents": {"dependencies": purls}}
+      echo "Creating sbom-purl.json"
+      python3 /scripts/create_purl_sbom.py
 
-      with open("sbom-purl.json", "w") as output_file:
-        json.dump(purl_content, output_file, indent=4)
-    securityContext:
-      runAsUser: 0
-    workingDir: /var/workdir
-  - computeResources:
-      limits:
-        cpu: 200m
-        memory: 512Mi
-      requests:
-        cpu: 100m
-        memory: 256Mi
-    image: quay.io/redhat-appstudio/base-images-sbom-script@sha256:667669e3def018f9dbb8eaf8868887a40bc07842221e9a98f6787edcff021840
-    name: create-base-images-sbom
-    script: |
-      #!/bin/bash
-      set -e
-      if [[ "${IMAGE##*-}" != "${PLATFORM##*/}" ]]; then
-        export IMAGE="${IMAGE}-${PLATFORM##*/}"
-      fi
-      python3 /app/base_images_sbom_script.py \
+      echo "Adding base images data to sbom-cyclonedx.json"
+      python3 /scripts/base_images_sbom_script.py \
         --sbom=sbom-cyclonedx.json \
         --base-images-from-dockerfile=/shared/base_images_from_dockerfile \
         --base-images-digests=/shared/base_images_digests

diff --git a/task/buildah-remote/0.2/buildah-remote.yaml b/task/buildah-remote/0.2/buildah-remote.yaml
@@ -546,105 +546,28 @@ spec:
       requests:
         cpu: 100m
         memory: 256Mi
-    image: registry.access.redhat.com/ubi9/python-39:1-192.1722518946@sha256:0176b477075984d5a502253f951d2502f0763c551275f9585ac515b9f241d73d
-    name: merge-syft-sboms
-    script: |
-      #!/bin/python3
-      import json
-
-      # load SBOMs
-      with open("./sbom-image.json") as f:
-        image_sbom = json.load(f)
-
-      with open("./sbom-source.json") as f:
-        source_sbom = json.load(f)
-
-      # fetch unique components from available SBOMs
-      def get_identifier(component):
-        return component["name"] + '@' + component.get("version", "")
-
-      image_sbom_components = image_sbom.setdefault("components", [])
-      existing_components = [get_identifier(component) for component in image_sbom_components]
-
-      source_sbom_components = source_sbom.get("components", [])
-      for component in source_sbom_components:
-        if get_identifier(component) not in existing_components:
-          image_sbom_components.append(component)
-          existing_components.append(get_identifier(component))
-
-      image_sbom_components.sort(key=lambda c: get_identifier(c))
-
-      # write the CycloneDX unified SBOM
-      with open("./sbom-cyclonedx.json", "w") as f:
-        json.dump(image_sbom, f, indent=4)
-    securityContext:
-      runAsUser: 0
-    workingDir: $(workspaces.source.path)
-  - computeResources:
-      limits:
-        cpu: 200m
-        memory: 512Mi
-      requests:
-        cpu: 100m
-        memory: 256Mi
-    image: quay.io/redhat-appstudio/cachi2:0.9.1@sha256:df67f9e063b544a8c49a271359377fed560562615e0278f6d0b9a3485f3f8fad
-    name: merge-cachi2-sbom
+    image: quay.io/redhat-appstudio/sbom-utility-scripts-image@sha256:064eeec1cfb75abd21b533bf0d9870a7a3fb138f30192596dce28061798cbf06
+    name: prepare-sboms
     script: |
       #!/bin/bash
       set -e
       if [[ "${IMAGE##*-}" != "${PLATFORM##*/}" ]]; then
         export IMAGE="${IMAGE}-${PLATFORM##*/}"
       fi
+      echo "Merging contents of sbom-source.json and sbom-image.json into sbom-cyclonedx.json"
+      python3 /scripts/merge_syft_sboms.py
+
       if [ -f "sbom-cachi2.json" ]; then
         echo "Merging contents of sbom-cachi2.json into sbom-cyclonedx.json"
-        merge_syft_sbom sbom-cachi2.json sbom-cyclonedx.json > sbom-temp.json
+        python3 /scripts/merge_cachi2_sboms.py sbom-cachi2.json sbom-cyclonedx.json > sbom-temp.json
         mv sbom-temp.json sbom-cyclonedx.json
-      else
-        echo "Skipping step since no Cachi2 SBOM was produced"
       fi
-    securityContext:
-      runAsUser: 0
-    workingDir: $(workspaces.source.path)
-  - computeResources:
-      limits:
-        cpu: 200m
-        memory: 512Mi
-      requests:
-        cpu: 100m
-        memory: 256Mi
-    image: registry.access.redhat.com/ubi9/python-39:1-192.1722518946@sha256:0176b477075984d5a502253f951d2502f0763c551275f9585ac515b9f241d73d
-    name: create-purl-sbom
-    script: |
-      #!/bin/python3
-      import json
-
-      with open("./sbom-cyclonedx.json") as f:
-        cyclonedx_sbom = json.load(f)
 
-      purls = [{"purl": component["purl"]} for component in cyclonedx_sbom.get("components", []) if "purl" in component]
-      purl_content = {"image_contents": {"dependencies": purls}}
+      echo "Creating sbom-purl.json"
+      python3 /scripts/create_purl_sbom.py
 
-      with open("sbom-purl.json", "w") as output_file:
-        json.dump(purl_content, output_file, indent=4)
-    securityContext:
-      runAsUser: 0
-    workingDir: $(workspaces.source.path)
-  - computeResources:
-      limits:
-        cpu: 200m
-        memory: 512Mi
-      requests:
-        cpu: 100m
-        memory: 256Mi
-    image: quay.io/redhat-appstudio/base-images-sbom-script@sha256:667669e3def018f9dbb8eaf8868887a40bc07842221e9a98f6787edcff021840
-    name: create-base-images-sbom
-    script: |
-      #!/bin/bash
-      set -e
-      if [[ "${IMAGE##*-}" != "${PLATFORM##*/}" ]]; then
-        export IMAGE="${IMAGE}-${PLATFORM##*/}"
-      fi
-      python3 /app/base_images_sbom_script.py \
+      echo "Adding base images data to sbom-cyclonedx.json"
+      python3 /scripts/base_images_sbom_script.py \
         --sbom=sbom-cyclonedx.json \
         --base-images-from-dockerfile=/shared/base_images_from_dockerfile \
         --base-images-digests=/shared/base_images_digests