Add sast-find-unicode-control task for testing

Signed-off-by: Chuntao Han <[email protected]>

.tekton/pull-request.yaml

@@ -5,7 +5,7 @@ metadata:
   name: build-definitions-pull-request
   annotations:
     pipelinesascode.tekton.dev/on-cel-expression: (event == "pull_request" && target_branch == "main" && ( !has(body.pull_request) || !body.pull_request.draft) ) || (event == "push" && target_branch.startsWith("gh-readonly-queue/main/"))
-    pipelinesascode.tekton.dev/task: "[task/git-clone/0.1/git-clone.yaml, .tekton/tasks/buildah.yaml, .tekton/tasks/yaml-lint.yaml, .tekton/tasks/e2e-test.yaml, task/sast-snyk-check/0.2/sast-snyk-check.yaml]"
+    pipelinesascode.tekton.dev/task: "[task/git-clone/0.1/git-clone.yaml, .tekton/tasks/buildah.yaml, .tekton/tasks/yaml-lint.yaml, .tekton/tasks/e2e-test.yaml, task/sast-snyk-check/0.2/sast-snyk-check.yaml task/sast-find-unicode-control/0.1/sast-find-unicode-control.yaml]"
     pipelinesascode.tekton.dev/task-2: "yaml-lint"
     pipelinesascode.tekton.dev/max-keep-runs: "5"
 spec:
@@ -59,6 +59,14 @@ spec:
         workspaces:
           - name: workspace
             workspace: workspace
+      - name: sast-find-unicode-control
+        runAfter:
+          - fetch-repository
+        taskRef:
+          name: sast-find-unicode-control
+        workspaces:
+          - name: workspace
+            workspace: workspace
       - name: build-container
         runAfter:
           - yaml-lint-check

task/sast-find-unicode-control/0.1/README.md

@@ -0,0 +1,30 @@
+# sast-snyk-check task
+
+## Description:
+
+The sast-find-unicode-control task uses [find-unicode-control](https://github.com/siddhesh/find-unicode-control.git)  tool to perform Static Application Security Testing (SAST) to look for non-printable unicode characters in all text files in a source tree.
+
+## Parameters:
+
+| name                         | description                                                                                      |
+|------------------------------|--------------------------------------------------------------------------------------------------|
+| FIND_UNICODE_CONTROL_GIT_URL | URL from repository to find unicode control                                                      |
+| KFP_GIT_URL                  | Link to the known-false-positives repository. If left blank, results won't be filtered           |
+| PROJECT_NVR                  | Name-Version-Release (NVR) of the scanned project, used to find path exclusions (it is optional) |
+| RECORD_EXCLUDED              | File to store all excluded findings to (it is optional)                                          |
+
+
+## Results:
+
+| name        | description              |
+|-------------|--------------------------|
+| TEST_OUTPUT | Tekton task test output. |
+
+## Source repository for image:
+
+https://github.com/konflux-ci/konflux-test
+
+## Additional links:
+
+* https://github.com/siddhesh/find-unicode-control.git
+* https://gitlab.cee.redhat.com/osh/known-false-positives.git

task/sast-find-unicode-control/0.1/sast-find-unicode-control.yaml

@@ -0,0 +1,232 @@
+apiVersion: tekton.dev/v1
+kind: Task
+metadata:
+  labels:
+    app.kubernetes.io/version: "0.1"
+  annotations:
+    tekton.dev/pipelines.minVersion: "0.12.1"
+    tekton.dev/tags: "konflux"
+  name: sast-find-unicode-control
+spec:
+  description: >-
+    Scans source code for non-printable unicode characters in all text files.
+  results:
+    - description: Tekton task test output.
+      name: TEST_OUTPUT
+  params:
+    - description: Image URL.
+      name: image-url
+      type: string
+      # In a future 0.4 version of the task, drop the default to make this required
+      default: ""
+    - description: Image digest to report findings for.
+      name: image-digest
+      type: string
+      # In a future 0.4 version of the task, drop the default to make this required
+      default: ""
+    - name: FIND_UNICODE_CONTROL_GIT_URL
+      type: string
+      description: URL from repository to find unicode control
+      default: "https://github.com/siddhesh/find-unicode-control.git"
+    - name: KFP_GIT_URL
+      type: string
+      description: URL from repository to download known false positives files
+      default: "https://gitlab.cee.redhat.com/osh/known-false-positives.git"
+    - default: ""
+      description: Name-Version-Release (NVR) of the scanned project, used to find path exclusions (it is optional)
+      name: PROJECT_NVR
+      type: string
+    - default: ""
+      description: File to store all excluded findings to (it is optional)
+      name: RECORD_EXCLUDED
+      type: string
+  volumes:
+  steps:
+    - name: sast-find-unicode-control
+      image: quay.io/redhat-appstudio/konflux-test:v1.4.5@sha256:801a105ba0f9c7f58f5ba5cde1a3b4404009fbebb1028779ca2c5de211e94940
+      # per https://kubernetes.io/docs/concepts/containers/images/#imagepullpolicy-defaulting
+      # the cluster will set imagePullPolicy to IfNotPresent
+      workingDir: $(workspaces.workspace.path)/hacbs/$(context.task.name)
+      volumeMounts:
+      env:
+        - name: KFP_GIT_URL
+          value: $(params.KFP_GIT_URL)
+        - name: FIND_UNICODE_CONTROL_GIT_URL
+          value: $(params.FIND_UNICODE_CONTROL_GIT_URL)
+        - name: PROJECT_NVR
+          value: $(params.PROJECT_NVR)
+        - name: RECORD_EXCLUDED
+          value: $(params.RECORD_EXCLUDED)
+        - name: SOURCE_CODE_DIR
+          value: $(workspaces.workspace.path)
+      script: |
+        #!/usr/bin/env bash
+        set -euo pipefail
+        . /utils.sh
+        trap 'handle_error $(results.TEST_OUTPUT.path)' EXIT
+
+        SCAN_RESULT=""
+
+        # Installation of Red Hat certificates for cloning Red Hat internal repositories
+        curl -sS https://certs.corp.redhat.com/certs/2015-IT-Root-CA.pem > /etc/pki/ca-trust/source/anchors/2015-RH-IT-Root-CA.crt
+        curl -sS https://certs.corp.redhat.com/certs/2022-IT-Root-CA.pem > /etc/pki/ca-trust/source/anchors/2022-IT-Root-CA.pem
+        update-ca-trust
+        
+
+        # Clone the source code from upstream repo
+        clone_unicontrol()
+        {
+            # Clone find-unicode-control repository
+            git clone --depth=1 "${FIND_UNICODE_CONTROL_GIT_URL}" cloned
+            status=$?
+            if [ ${status} -ne 0 ]; then
+                echo "Error: Failed to clone the repository: ${FIND_UNICODE_CONTROL_GIT_URL}" >&2
+                show_error
+                return 1
+            else
+                echo "Message: Successfully cloned the repository: ${FIND_UNICODE_CONTROL_GIT_URL}" >&2
+            fi
+        
+            # Check usage of find-unicode-control to confirm the tool can work
+            ./cloned/find_unicode_control.py --help >&2
+            status=$?
+            if [ ${status} -ne 0 ]; then
+                echo "Error: Failed to check usage of find-unicode-control" >&2
+                show_error
+                return 1
+            else
+                echo "Message: Succeed to check usage of find-unicode-control" >&2
+            fi
+        }
+        
+        # Find unicode control
+        unicontrol_scan()
+        {
+            LANG=en_US.utf8 ./cloned/find_unicode_control.py -d -v -p bidi --notests "$SOURCE_CODE_DIR" >raw_sast_find_unicode_control_check_out.txt 2>raw_sast_find_unicode_control_check_out.log
+            status=$?
+            if [ "$status" -ne 0 ] && [ "$status" -ne 1 ]; then
+                echo "Error: sast-find-unicode-control test failed because of the following issues:"  >&2
+                cat raw_sast_find_unicode_control_check_out.log
+                show_error
+                return 1
+            else
+                echo "Message: Succeed to run find-unicode-control" >&2
+            fi
+        
+            # translate the output format
+            sed -i raw_sast_find_unicode_control_check_out.txt -E -e 's|(.*:[0-9]+)(.*)|\1: warning:\2|' -e 's|^|Error: UNICONTROL_WARNING:\n|'
+            status=$?
+            if [ "$status" -ne 0 ] && [ "$status" -ne 1 ]; then
+                echo "Error: failed to translate the unicontrol output format" >&2
+                show_error
+                return 1
+            else
+                echo "Message: Succeed to translate the unicontrol output format\n" >&2
+                SCAN_RESULT="raw_sast_find_unicode_control_check_out.txt"
+            fi
+        }
+        
+        # Filter known false positive
+        filter_kfp()
+        {
+            # We check if the KFP_GIT_URL variable is set to apply the filters or not
+            if [[ -z "${KFP_GIT_URL}" ]]; then
+                echo "KFP_GIT_URL variable not defined. False positives won't be filtered" >&2
+                mv sast_snyk_check_out_all_findings.json filtered_sast_snyk_check_out.json
+                return 0
+            else
+                echo "Filtering false positives in results files using csfilter-kfp..." >&2
+                cmd=（
+                    csfilter-kfp
+                    --verbose 
+                    --kfp-git-url=$KFP_GIT_URL"
+                ）
+
+                if [[ -z "${PROJECT_NVR}" ]]; then
+                    cmd+=("--project-nvr=$PROJECT_NVR")
+                if
+
+                if [[ -z "${RECORD_EXCLUDED}" ]]; then
+                    cmd+=("--record-excluded=RECORD_EXCLUDED")
+                if
+
+                "${cmd[@]}" raw_sast_find_unicode_control_check_out.txt > filtered_sast_find_unicode_control_check_out.json
+                status=$?
+                if [ "$status" -ne 0 ]; then
+                    echo "Error: failed to filter known false positives" >&2
+                    show_error
+                    return 1
+                else
+                    echo "Message: Succeed to filter known false positives" >&2
+                    SCAN_RESULT="filtered_sast_find_unicode_control_check_out.json"
+                fi
+            fi
+        }
+
+        # Process all results as configured with CSGERP_OPTS
+        process_all_results()
+        {
+            CSGERP_OPTS=(
+                --mode=json
+                --event='error|warning'
+                --remove-duplicates
+                --embed-context=3
+            )
+            # In order to generate csdiff/v1, we need to add the whole path of the source code as Snyk only provides an URI to embed the context
+            set -x
+            csgrep --mode=json --prepend-path-prefix="$SOURCE_CODE_DIR"/ "$SCAN_RESULT" | csgrep "${CSGERP_OPTS[@]}" | csgrep --mode=json --strip-path-prefix="$SOURCE_CODE_DIR"/source/  > processed_sast_find_unicode_control_check_out.json
+            set +x
+
+            csgrep --mode=evtstat processed_sast_find_unicode_control_check_out.json
+            csgrep --mode=sarif processed_sast_find_unicode_control_check_out.json > sast_find_unicode_control_check_out.sarif
+        } 
+
+        show_error ()
+        {
+            note="Task $(context.task.name) failed: For details, check Tekton task log."
+            ERROR_OUTPUT=$(make_result_json -r ERROR -t "$note")
+            echo "${ERROR_OUTPUT}" | tee $(results.TEST_OUTPUT.path)
+        }
+
+        main() 
+        {
+            cd ${SOURCE_CODE_DIR}
+            clone_unicontrol
+            unicontrol_scan
+            filter_kfp
+            process_all_results
+        }
+        
+        main
+    - name: upload
+      image: quay.io/konflux-ci/oras:latest@sha256:f4b891ee3038a5f13cd92ff4f473faad5601c2434d1c6b9bccdfc134d9d5f820
+      workingDir: $(workspaces.workspace.path)/hacbs/$(context.task.name)
+      env:
+        - name: IMAGE_URL
+          value: $(params.image-url)
+        - name: IMAGE_DIGEST
+          value: $(params.image-digest)
+      script: |
+        #!/usr/bin/env bash
+
+        UPLOAD_FILE=sast_find_unicode_control_check_out.sarif
+        MEDIA_TYPE=application/sarif+json
+
+        if [ -z "${IMAGE_URL}" ] || [ -z "${IMAGE_DIGEST}"  ]; then
+          echo 'No image-url or image-digest param provided. Skipping upload.'
+          exit 0;
+        fi
+
+        if [ ! -f "${UPLOAD_FILE}" ]; then
+          echo "No ${UPLOAD_FILE} exists. Skipping upload."
+          exit 0;
+        fi
+
+        echo "Selecting auth"
+        select-oci-auth $IMAGE_URL > $HOME/auth.json
+        echo "Attaching to ${IMAGE_URL} via the OCI 1.1 Referrers API"
+        oras attach --no-tty --registry-config "$HOME/auth.json" --distribution-spec v1.1-referrers-api --artifact-type "${MEDIA_TYPE}" "${IMAGE_URL}" "${UPLOAD_FILE}:${MEDIA_TYPE}"
+        echo "Attaching to ${IMAGE_URL} via the OCI 1.1 Referrers Tag"
+        oras attach --no-tty --registry-config "$HOME/auth.json" --distribution-spec v1.1-referrers-tag --artifact-type "${MEDIA_TYPE}" "${IMAGE_URL}" "${UPLOAD_FILE}:${MEDIA_TYPE}"
+  workspaces:
+  - name: workspace