-
Notifications
You must be signed in to change notification settings - Fork 114
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
sast-unicode-check: add version 0.1 task
Resolves: https://issues.redhat.com/browse/OSH-739 Signed-off-by: Chuntao Han <[email protected]>
- Loading branch information
1 parent
0ea2f67
commit 3aa3d8e
Showing
4 changed files
with
325 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,30 @@ | ||
# sast-unicode-check task | ||
|
||
## Description: | ||
|
||
The sast-unicode-check task uses [find-unicode-control](https://github.com/siddhesh/find-unicode-control.git) tool to perform Static Application Security Testing (SAST) to look for non-printable unicode characters in all text files in a source tree. | ||
|
||
## Parameters: | ||
|
||
| name | description | Default Value | Required | | ||
|------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------|----------| | ||
| FIND_UNICODE_CONTROL_GIT_URL | URL from repository to find unicode control. | "https://github.com/siddhesh/find-unicode-control.git#c2accbfbba7553a8bc1ebd97089ae08ad8347e58" | Yes | | ||
| FIND_UNICODE_CONTROL_ARGS | arguments for find-unicode-control command. | "-p bidi -v -d -t" | Yes | | ||
| KFP_GIT_URL | Known False Positives git URL, optionally taking a revision delimited by #; If empty, filtering of known false positives is disabled. | "" | No | | ||
| PROJECT_NVR | Name-Version-Release (NVR) of the scanned project. It is used to find path exclusions (it is optional). | "" | No | | ||
| RECORD_EXCLUDED | Whether to record the excluded findings (defaults to false). If `true`, the the excluded findings will be stored in `excluded-findings.json`. | "false" | No | | ||
|
||
## Results: | ||
|
||
| name | description | | ||
|---------------|------------------------------------------| | ||
| TEST_OUTPUT | Tekton task test output. | | ||
|
||
## Source repository for image: | ||
|
||
https://github.com/konflux-ci/konflux-test | ||
|
||
## Additional links: | ||
|
||
* https://github.com/siddhesh/find-unicode-control.git | ||
* https://gitlab.cee.redhat.com/osh/known-false-positives.git |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,281 @@ | ||
apiVersion: tekton.dev/v1 | ||
kind: Task | ||
metadata: | ||
labels: | ||
app.kubernetes.io/version: "0.1" | ||
annotations: | ||
tekton.dev/pipelines.minVersion: "0.12.1" | ||
tekton.dev/tags: "konflux" | ||
name: sast-unicode-check | ||
spec: | ||
description: >- | ||
Scans source code for non-printable unicode characters in all text files. | ||
results: | ||
- description: Tekton task test output. | ||
name: TEST_OUTPUT | ||
params: | ||
- name: image-url | ||
type: string | ||
description: Image URL. | ||
default: "" | ||
- name: image-digest | ||
type: string | ||
description: Image digest to report findings for. | ||
default: "" | ||
- name: FIND_UNICODE_CONTROL_GIT_URL | ||
type: string | ||
description: URL from repository to find unicode control. | ||
default: "https://github.com/siddhesh/find-unicode-control.git#c2accbfbba7553a8bc1ebd97089ae08ad8347e58" | ||
- name: FIND_UNICODE_CONTROL_ARGS | ||
type: string | ||
description: arguments for find-unicode-control command. | ||
default: "-p bidi -v -d -t" | ||
- name: KFP_GIT_URL | ||
type: string | ||
description: URL from repository to download known false positives files. | ||
# FIXME: Red Hat internal projects will default to https://gitlab.cee.redhat.com/osh/known-false-positives.git when KONFLUX-4530 is resolved | ||
default: "" | ||
- name: PROJECT_NVR | ||
type: string | ||
description: | | ||
Name-Version-Release (NVR) of the scanned project. | ||
It is used to find path exclusions (it is optional). | ||
default: "" | ||
- name: RECORD_EXCLUDED | ||
type: string | ||
description: | | ||
Whether to record the excluded findings (defaults to false). | ||
If `true`, the the excluded findings will be stored in `excluded-findings.json`. | ||
default: "false" | ||
- name: caTrustConfigMapName | ||
type: string | ||
description: The name of the ConfigMap to read CA bundle data from. | ||
default: trusted-ca | ||
- name: caTrustConfigMapKey | ||
type: string | ||
description: The name of the key in the ConfigMap that contains the CA bundle data. | ||
default: ca-bundle.crt | ||
volumes: | ||
- name: trusted-ca | ||
configMap: | ||
name: $(params.caTrustConfigMapName) | ||
items: | ||
- key: $(params.caTrustConfigMapKey) | ||
path: ca-bundle.crt | ||
optional: true | ||
steps: | ||
- name: sast-unicode-check | ||
image: quay.io/redhat-appstudio/konflux-test:v1.4.6@sha256:5f298d8d990dfa82023e50029b71b08e19c3c9cedb181dfc4bc86c9ecad8700c | ||
# per https://kubernetes.io/docs/concepts/containers/images/#imagepullpolicy-defaulting | ||
# the cluster will set imagePullPolicy to IfNotPresent | ||
workingDir: $(workspaces.workspace.path)/hacbs/$(context.task.name) | ||
volumeMounts: | ||
env: | ||
- name: KFP_GIT_URL | ||
value: $(params.KFP_GIT_URL) | ||
- name: FIND_UNICODE_CONTROL_GIT_URL | ||
value: $(params.FIND_UNICODE_CONTROL_GIT_URL) | ||
- name: FIND_UNICODE_CONTROL_ARGS | ||
value: $(params.FIND_UNICODE_CONTROL_ARGS) | ||
- name: PROJECT_NVR | ||
value: $(params.PROJECT_NVR) | ||
- name: RECORD_EXCLUDED | ||
value: $(params.RECORD_EXCLUDED) | ||
- name: SOURCE_CODE_DIR | ||
value: $(workspaces.workspace.path) | ||
script: | | ||
#!/usr/bin/env bash | ||
set -exuo pipefail | ||
. /utils.sh | ||
trap 'handle_error $(results.TEST_OUTPUT.path)' EXIT | ||
SCAN_PROP="" | ||
ca_bundle=/mnt/trusted-ca/ca-bundle.crt | ||
if [ -f "$ca_bundle" ]; then | ||
echo "INFO: Using mounted CA bundle: $ca_bundle" | ||
cp -vf $ca_bundle /etc/pki/ca-trust/source/anchors | ||
update-ca-trust | ||
fi | ||
# Clone the source code from upstream repo | ||
clone_unicontrol() | ||
{ | ||
GIT_URL=$(echo "${FIND_UNICODE_CONTROL_GIT_URL}" | awk -F'#' '{print $1}') | ||
REV=$(echo "${FIND_UNICODE_CONTROL_GIT_URL}" | awk -F'#' '{print $2}') | ||
# Clone find-unicode-control repository | ||
CLONE_EXIT_CODE=0 | ||
git clone "${GIT_URL}" find-unicode-control >&2 || CLONE_EXIT_CODE=$? | ||
if [[ "${CLONE_EXIT_CODE}" -eq 0 ]]; then | ||
echo "Message: Succeed to clone the repository: ${GIT_URL}" >&2 | ||
else | ||
echo "Error: Failed to clone the repository: ${GIT_URL}" >&2 | ||
return 1 | ||
fi | ||
if [[ -n "${REV}" ]]; then | ||
# Get git url suffix | ||
git_url_suffix=$(git -C ./find-unicode-control/ rev-parse HEAD) | ||
if [[ "${git_url_suffix}" != "${REV}" ]]; then | ||
git -C ./find-unicode-control/ checkout --quiet "${REV}" | ||
fi | ||
SCAN_PROP="find-unicode-control-git-url:${FIND_UNICODE_CONTROL_GIT_URL}" | ||
else | ||
SCAN_PROP="find-unicode-control-git-url:${FIND_UNICODE_CONTROL_GIT_URL}#${git_url_suffix}" | ||
fi | ||
# Check usage of find-unicode-control to confirm the tool can work | ||
USAGE_EXIT_CODE=0 | ||
./find-unicode-control/find_unicode_control.py --help >&2 || USAGE_EXIT_CODE=$? | ||
if [[ "${USAGE_EXIT_CODE}" -eq 0 ]]; then | ||
echo "Message: Succeed to check usage of find-unicode-control" >&2 | ||
else | ||
echo "Error: Failed to check usage of find-unicode-control" >&2 | ||
return 1 | ||
fi | ||
} | ||
# Find unicode control | ||
unicontrol_scan() | ||
{ | ||
FUC_EXIT_CODE=0 | ||
LANG=en_US.utf8 ./find-unicode-control/find_unicode_control.py ${FIND_UNICODE_CONTROL_ARGS} "${SOURCE_CODE_DIR}/source" \ | ||
>raw_sast_unicode_check_out.txt \ | ||
2>raw_sast_unicode_check_out.log \ | ||
|| FUC_EXIT_CODE=$? | ||
if [[ "${FUC_EXIT_CODE}" -eq 0 || "${FUC_EXIT_CODE}" -eq 1 ]]; then | ||
# No non-printable unicode characters found, or successfully found | ||
echo "Message: Succeed to run find-unicode-control command" >&2 | ||
else | ||
# Failed to run the command | ||
echo "Error: failed to run find unicode control command" >&2 | ||
return 1 | ||
fi | ||
# Translate the output format | ||
TRANSLATE_EXIT_CODE=0 | ||
sed -i raw_sast_unicode_check_out.txt -E -e 's|(.*:[0-9]+)(.*)|\1: warning:\2|' -e 's|^|Error: UNICONTROL_WARNING:\n|' || TRANSLATE_EXIT_CODE=$? | ||
if [[ "${TRANSLATE_EXIT_CODE}" -eq 0 ]]; then | ||
echo "Message: Succeed to translate the unicontrol output format" >&2 | ||
else | ||
echo "Error: failed to translate the unicontrol output format" >&2 | ||
return 1 | ||
fi | ||
} | ||
# Process all results as configured with CSGERP_OPTS | ||
process_all_results() | ||
{ | ||
CSGERP_OPTS=( | ||
--mode=json | ||
--remove-duplicates | ||
--embed-context=3 | ||
--set-scan-prop="${SCAN_PROP}" | ||
--strip-path-prefix="${SOURCE_CODE_DIR}"/source/ | ||
) | ||
# In order to generate csdiff/v1, we need to add the whole path of the source code as sast-unicode-check only provides an URI to embed the context | ||
csgrep "${CSGERP_OPTS[@]}" raw_sast_unicode_check_out.txt > processed_sast_unicode_check_out.json | ||
csgrep --mode=evtstat processed_sast_unicode_check_out.json | ||
} | ||
# Filter known false positive | ||
csfilter_kfp() | ||
{ | ||
echo "Message: Filtering false positives in results files using ${KFP_GIT_URL}..." >&2 | ||
# Build initial csfilter-kfp command | ||
csfilter_kfp_cmd="csfilter-kfp --verbose --kfp-git-url=${KFP_GIT_URL}" | ||
# Append --project-nvr option if PROJECT_NVR is set | ||
if [[ -n "${PROJECT_NVR}" ]]; then | ||
csfilter_kfp_cmd+=" --project-nvr="${PROJECT_NVR}"" | ||
fi | ||
# Append --record-excluded option if RECORD_EXCLUDED is true | ||
if [[ "${RECORD_EXCLUDED}" == "true" ]]; then | ||
csfilter_kfp_cmd+=" --record-excluded=raw-excluded-findings.json" | ||
fi | ||
KFP_EXIT_CODE=0 | ||
${csfilter_kfp_cmd[@]} processed_sast_unicode_check_out.json > sast_unicode_check_out.json || KFP_EXIT_CODE=$? | ||
if [ "${KFP_EXIT_CODE}" -eq 0 ]; then | ||
echo "Message: Succeed to filter known false positives" >&2 | ||
else | ||
echo "Error: failed to filter known false positives" >&2 | ||
return 1 | ||
fi | ||
if [ -f "raw-excluded-findings.json" ]; then | ||
csgrep --mode=json --strip-path-prefix="${SOURCE_CODE_DIR}"/source/ raw-excluded-findings.json > excluded-findings.json | ||
fi | ||
} | ||
generate_sarif_report() | ||
{ | ||
csgrep --mode=sarif sast_unicode_check_out.json > sast_unicode_check_out.sarif | ||
if [[ "${FUC_EXIT_CODE}" -eq 0 ]]; then | ||
note="Task $(context.task.name) success: No finding was detected" | ||
ERROR_OUTPUT=$(make_result_json -r SUCCESS -t "$note") | ||
elif [[ "${FUC_EXIT_CODE}" -eq 1 ]] && [[ ! -s sast_unicode_check_out.sarif ]]; then | ||
note="Task $(context.task.name) success: Some findings were detected, but filtered by known false positive" | ||
ERROR_OUTPUT=$(make_result_json -r SUCCESS -t "$note") | ||
else | ||
echo "sast-unicode-check test failed because of the following issues:" | ||
cat processed_sast_unicode_check_out.json | ||
TEST_OUTPUT= | ||
parse_test_output $(context.task.name) sarif sast_unicode_check_out.sarif || true | ||
note="Task $(context.task.name) failed: For details, check Tekton task log." | ||
ERROR_OUTPUT=$(make_result_json -r ERROR -t "$note") | ||
fi | ||
echo "${TEST_OUTPUT:-${ERROR_OUTPUT}}" | tee $(results.TEST_OUTPUT.path) | ||
} | ||
main() | ||
{ | ||
clone_unicontrol | ||
unicontrol_scan | ||
process_all_results | ||
csfilter_kfp | ||
generate_sarif_report | ||
} | ||
main | ||
- name: upload | ||
image: quay.io/konflux-ci/oras:latest@sha256:f4b891ee3038a5f13cd92ff4f473faad5601c2434d1c6b9bccdfc134d9d5f820 | ||
workingDir: $(workspaces.workspace.path)/hacbs/$(context.task.name) | ||
env: | ||
- name: IMAGE_URL | ||
value: $(params.image-url) | ||
- name: IMAGE_DIGEST | ||
value: $(params.image-digest) | ||
script: | | ||
#!/usr/bin/env bash | ||
if [ -z "${IMAGE_URL}" ] || [ -z "${IMAGE_DIGEST}" ]; then | ||
echo 'No image-url or image-digest param provided. Skipping upload.' | ||
exit 0; | ||
fi | ||
UPLOAD_FILES="sast_unicode_check_out.sarif excluded-findings.json" | ||
for UPLOAD_FILE in ${UPLOAD_FILES}; do | ||
if [ ! -f "${UPLOAD_FILE}" ]; then | ||
echo "No ${UPLOAD_FILE} exists. Skipping upload." | ||
continue; | ||
fi | ||
if [ "${UPLOAD_FILES}" == "excluded-findings.json" ]; then | ||
MEDIA_TYPE=application/json | ||
else | ||
MEDIA_TYPE=application/sarif+json | ||
fi | ||
echo "Selecting auth" | ||
select-oci-auth "${IMAGE_URL}" > "${HOME}/auth.json" | ||
echo "Attaching to ${IMAGE_URL}" | ||
oras attach --no-tty --registry-config "$HOME/auth.json" --artifact-type "${MEDIA_TYPE}" "${IMAGE_URL}" "${UPLOAD_FILE}:${MEDIA_TYPE}" | ||
done | ||
workspaces: | ||
- name: workspace |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,5 @@ | ||
# See the OWNERS docs: https://go.k8s.io/owners | ||
approvers: | ||
- integration-team | ||
reviewers: | ||
- integration-team |