Ssml

Jenkinsfile

@@ -0,0 +1,169 @@
+@Library('flexy') _
+
+// rename build
+def userId = currentBuild.rawBuild.getCause(hudson.model.Cause$UserIdCause)?.userId
+if (userId) {
+  currentBuild.displayName = userId
+}
+
+def RETURNSTATUS = "default"
+def output = ""
+pipeline {
+  agent none
+  parameters {
+        string(name: 'BUILD_NUMBER', defaultValue: '', description: 'Build number of job that has installed the cluster.')
+        string(name: "DAST_IMAGE", defaultValue: "quay.io/redhatproductsecurity/rapidast", description: 'Image to use as the base for running zap.')
+        string(name: "DAST_IMAGE_TAG", defaultValue: "latest", description: 'Image tag to use as the base for running zap.')
+        string(name: 'DAST_TOOL_URL', defaultValue: 'https://github.com/RedHatProductSecurity/rapidast.git', description: 'Rapidast tool github url .')
+        string(name: 'DAST_TOOL_BRANCH', defaultValue: 'development', description: 'Rapdiast tool github barnch to checkout.')
+        string(name: 'SE_TOOL_URL', defaultValue: 'https://github.com/openshift-qe/ocpqe-security-tools.git', description: 'OCPQE security tool github url.')
+        string(name: 'SEC_TOOL_BRANCH', defaultValue: 'main', description: 'OCPQE security tool github barnch to checkout.')
+        string(name: 'API_URL_LIST', defaultValue: 'admissionregistration.k8s.io/v1', description: 
+        '''List of api files to scan against.
+        Api docs you can find using <b>kubectl api-versions</b>''')
+        string(name: 'POLICY_FILE', defaultValue: 'API-scan-minimal', description: 'List of policies to check apis against.')
+        string(name:'JENKINS_AGENT_LABEL',defaultValue:'oc415',description:
+        '''
+        scale-ci-static: for static agent that is specific to scale-ci, useful when the jenkins dynamic agent isn't stable<br>
+        4.y: oc4y || mac-installer || rhel8-installer-4y <br/>
+            e.g, for 4.8, use oc48 || mac-installer || rhel8-installer-48 <br/>
+        3.11: ansible-2.6 <br/>
+        3.9~3.10: ansible-2.4 <br/>
+        3.4~3.7: ansible-2.4-extra || ansible-2.3 <br/>
+        '''
+        )
+       text(name: 'ENV_VARS', defaultValue: '', description:'''<p>
+               Enter list of additional (optional) Env Vars you'd want to pass to the script, one pair on each line. <br>
+               See https://github.com/cloud-bulldozer/kraken-hub/blob/main/docs/cerberus.md for list of variables to pass <br>
+               e.g.<br>
+               SOMEVAR1='env-test'<br>
+               SOMEVAR2='env2-test'<br>
+               ...<br>
+               SOMEVARn='envn-test'<br>
+               </p>'''
+            )
+     }
+  stages {
+    stage('SSMl Run'){
+        agent {
+          kubernetes {
+            cloud 'PSI OCP-C1 agents'
+            yaml """\
+              apiVersion: v1
+              kind: Pod
+              metadata:
+                labels:
+                  label: ${JENKINS_AGENT_LABEL}
+              spec:
+                containers:
+                - name: "jnlp"
+                  image: "image-registry.openshift-image-registry.svc:5000/aosqe/cucushift:${JENKINS_AGENT_LABEL}-rhel8"
+                  resources:
+                    requests:
+                      memory: "8Gi"
+                      cpu: "2"
+                    limits:
+                      memory: "8Gi"
+                      cpu: "2"
+                  imagePullPolicy: Always
+                  workingDir: "/home/jenkins/ws"
+                  tty: true
+              """.stripIndent()
+          }
+        }
+      steps{
+        deleteDir()
+        checkout([
+          $class: 'GitSCM',
+          branches: [[name: params.SEC_TOOL_BRANCH ]],
+          doGenerateSubmoduleConfigurations: false,
+          userRemoteConfigs: [[url: params.SE_TOOL_URL ]
+          ]])
+        checkout([
+            $class: 'GitSCM',
+            branches: [[name: params.DAST_TOOL_BRANCH ]],
+            doGenerateSubmoduleConfigurations: false,
+            extensions: [
+                [$class: 'CloneOption', noTags: true, reference: '', shallow: true],
+                [$class: 'PruneStaleBranch'],
+                [$class: 'CleanCheckout'],
+                [$class: 'IgnoreNotifyCommit'],
+                [$class: 'RelativeTargetDirectory', relativeTargetDir: 'dast_tool']
+            ],
+            userRemoteConfigs: [[url: params.DAST_TOOL_URL ]]
+        ])
+        copyArtifacts(
+            filter: '',
+            fingerprintArtifacts: true,
+            projectName: 'ocp-common/Flexy-install',
+            selector: specific(params.BUILD_NUMBER),
+            target: 'flexy-artifacts'
+        )
+        script {
+          buildinfo = readYaml file: "flexy-artifacts/BUILDINFO.yml"
+          currentBuild.displayName = "${currentBuild.displayName}-${params.BUILD_NUMBER}"
+          currentBuild.description = "Copying Artifact from Flexy-install build <a href=\"${buildinfo.buildUrl}\">Flexy-install#${params.BUILD_NUMBER}</a>"
+          buildinfo.params.each { env.setProperty(it.key, it.value) }
+        }
+        script {
+          RETURNSTATUS = sh(returnStatus: true, script: '''
+           # Get ENV VARS Supplied by the user to this job and store in .env_override
+          echo "$ENV_VARS" > .env_override
+          # Export those env vars so they could be used by CI Job
+          set -a && source .env_override && set +a
+          mkdir -p ~/.kube
+          cp $WORKSPACE/flexy-artifacts/workdir/install-dir/auth/kubeconfig ~/.kube/config
+          ls
+          oc login -u kubeadmin -p $(cat $WORKSPACE/flexy-artifacts/workdir/install-dir/auth/kubeadmin-password)
+          HELM_DIR=$(mktemp -d)
+          curl -sS -L https://get.helm.sh/helm-v3.11.2-linux-amd64.tar.gz | tar -xzC ${HELM_DIR}/ linux-amd64/helm
+
+          ${HELM_DIR}/linux-amd64/helm version  
+
+          mv ${HELM_DIR}/linux-amd64/helm $WORKSPACE/helm
+          PATH=$PATH:$WORKSPACE
+          helm version
+
+          cd dast
+          ls
+          export DAST_PATH=../dast_tool
+          set +e
+          ./deploy_ssml_api.sh
+          api_run_status=$?
+
+          echo "api_run_status $api_run_status"
+          exit $api_run_status
+
+          ''')
+          sh "echo $RETURNSTATUS"
+          archiveArtifacts(
+              artifacts: 'dast/results/**',
+              allowEmptyArchive: true,
+              fingerprint: true
+          )
+        }
+      script{
+            def status = "FAIL"
+            sh "echo $RETURNSTATUS"
+            if( RETURNSTATUS.toString() == "0") {
+                status = "PASS"
+            }else {
+                currentBuild.result = "FAILURE"
+           }
+      }
+     }
+   }
+  }
+  post {
+      always {
+          script {
+                      build job: 'scale-ci/e2e-benchmarking-multibranch-pipeline/post-to-slack',
+                      parameters: [
+                          string(name: 'BUILD_NUMBER', value: BUILD_NUMBER), string(name: 'WORKLOAD', value: "ssml"),
+                          text(name: "BUILD_URL", value: env.BUILD_URL), string(name: 'BUILD_ID', value: currentBuild.number.toString()),
+                          string(name: 'RESULT', value:currentBuild.currentResult)
+                      ], propagate: false
+          }
+      }
+  }
+}

_helpers.tpl

@@ -0,0 +1,66 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "rapidast-chart.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "rapidast-chart.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "rapidast-chart.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create job spec
+*/}}
+
+{{- define "rapidast-chart.job" -}}
+template:
+  metadata:
+    name: {{ .Release.Name }}-job
+  spec:
+    containers:
+    - name: "{{ .Chart.Name }}"
+      securityContext: {{ .Values.secContext }}
+      image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
+      # Since Helm configmap cannot handle the dash character but the policy name undner  scanPolicyXML' in 'values.yaml' is 'helm-custom-scan', the dest file name of the copy command is 'helm-custom-scan.policy'.
+      # This file will be used if the rapidast config specifies 'helm-custom-scan' for the activeScan policy.
+      # Otherwise, '/home/rapidast/.ZAP/policies/API-scan-minimal.policy' will be used by default.
+      command: ["sh", "-c", "cp /helm/config/helmcustomscan.policy /opt/rapidast/scanners/zap/policies/helm-custom-scan.policy && rapidast.py --config /helm/config/rapidastconfig.yaml"]
+      imagePullPolicy: {{ .Values.image.pullPolicy }}
+      resources:
+        {{- toYaml .Values.resources | nindent 8 }}
+      volumeMounts:
+      - name: config-volume
+        mountPath: /helm/config
+      - name: results-volume
+        mountPath: /zap/results/
+    volumes:
+      - name: config-volume
+        configMap:
+          name: {{ .Release.Name }}-configmap
+      - name: results-volume
+        persistentVolumeClaim:
+          claimName: {{ .Values.pvc }}
+    restartPolicy: Never
+{{- end }}

deploy_ssml.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+oc label ns default security.openshift.io/scc.podSecurityLabelSync=false pod-security.kubernetes.io/enforce=privileged pod-security.kubernetes.io/audit=privileged pod-security.kubernetes.io/warn=privileged --overwrite
+
+export CONSOLE_URL=$(oc get routes console -n openshift-console -o jsonpath='{.spec.host}')
+
+export TOKEN=$(oc whoami -t)
+
+# path for local testing
+#dast_tool_path=../rapidast/
+dast_tool_path=./dast_tool
+echo "$CONSOLE_URL"
+#curl -k "https://${CONSOLE_URL}/api/kubernetes/openapi/v2" -H "Cookie: openshift-session-token=${TOKEN}"  -H "Accept: application/json"  >> openapi.json
+mkdir results 
+for api_doc in ${API_URL_LIST}; do 
+  echo "api doc $api_doc"
+  export API_URL="https://raw.githubusercontent.com/paigerube14/ocp-qe-perfscale-ci/ssml/apidocs/$api_doc"
+  echo "api url: $API_URL"
+  #edit rapidast config file
+  envsubst < values.yaml.template > $dast_tool_path/helm/chart/value_test.yaml
+
+  helm install rapidast $dast_tool_path/helm/chart -f $dast_tool_path/helm/chart/value_test.yaml
+
+  # wait for pod to be completed or error
+  rapidast_pod=$(oc get pods -n default -l job-name=rapidast-job -o name)
+  echo "rapidast current pod $rapidast_pod"
+  oc wait --for=condition=Ready $rapidast_pod --timeout=120s
+  oc get $rapidast_pod -o 'jsonpath={..status.conditions}'
+  while [[ $(oc get $rapidast_pod -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}') == "True" ]]; do
+    echo "sleeping 5"
+    sleep 5
+
+  done
+  mkdir results/$api_doc
+  cp $dast_tool_path/helm/chart/value_test.yaml results/$api_doc/value.yaml
+
+  oc logs $rapidast_pod -n default >> results/$api_doc/pod_logs.out
+
+  ./results.sh rapidast-pvc results/$api_doc
+
+  phase=$(oc get $rapidast_pod -o jsonpath='{.status.phase}')
+  helm uninstall rapidast 
+  oc delete pvc rapidast-pvc
+done
+
+if [ $phase != "Succeeded" ]; then
+    echo "Pod $rapidast_pod failed. Look at pod logs in archives (results/*/pod_logs.out)"
+    exit 1
+fi

deploy_ssml_api.sh

@@ -0,0 +1,73 @@
+#!/bin/bash
+
+export CONSOLE_URL=$(oc get routes console -n openshift-console -o jsonpath='{.spec.host}')
+
+export CLUSTER_NAME=$(oc get machineset -n openshift-machine-api -o=go-template='{{(index (index .items 0).metadata.labels "machine.openshift.io/cluster-api-cluster" )}}')
+
+export BASE_API_URL=$(oc get infrastructure -o jsonpath="{.items[*].status.apiServerURL}")
+export TOKEN=$(oc whoami -t)
+export NAMESPACE=${NAMESPACE:-default}
+
+oc label ns $NAMESPACE security.openshift.io/scc.podSecurityLabelSync=false pod-security.kubernetes.io/enforce=privileged pod-security.kubernetes.io/audit=privileged pod-security.kubernetes.io/warn=privileged --overwrite
+
+# path for local testing
+#dast_tool_path=../rapidast/
+dast_tool_path=${DAST_PATH:-./dast_tool}
+echo "$CONSOLE_URL"
+#curl -k "https://${CONSOLE_URL}/api/kubernetes/openapi/v2" -H "Cookie: openshift-session-token=${TOKEN}"  -H "Accept: application/json"  >> openapi.json
+mkdir results 
+
+counter=0
+#for api_doc in $(kubectl api-versions); do
+for api_doc in ${API_URL_LIST}; do
+  echo "api doc $api_doc"
+  # export API_URL="https://raw.githubusercontent.com/paigerube14/ocp-qe-perfscale-ci/ssml/apidocs/$api_doc"
+  if [[ "$api_doc" == *"/"* ]]; then
+    export API_URL="$BASE_API_URL/openapi/v3/apis/$api_doc"
+  else   # e.g. 'v1'
+    export API_URL="$BASE_API_URL/openapi/v3/api/$api_doc"
+  fi
+
+  echo "api url: $API_URL"
+  #edit rapidast config file
+  envsubst < values.yaml.template > $dast_tool_path/helm/chart/value_test.yaml
+  helm install rapidast $dast_tool_path/helm/chart -f $dast_tool_path/helm/chart/value_test.yaml
+
+  # wait for pod to be completed or error
+  rapidast_pod=$(oc get pods -n default -l job-name=rapidast-job -o name)
+  echo "rapidast current pod $rapidast_pod"
+  oc wait --for=condition=Ready $rapidast_pod --timeout=120s
+
+  folder_api_name=$(echo "$api_doc" | tr "/" .)
+  mkdir results/$folder_api_name
+
+  #oc get $rapidast_pod -n default -o yaml >> results/$folder_api_name/pod_yaml.yaml
+
+  oc get $rapidast_pod -o 'jsonpath={..status.conditions}'
+  while [[ $(oc get $rapidast_pod -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}') == "True" ]]; do
+    echo "sleeping 5"
+    sleep 5
+
+  done
+
+  #cp $dast_tool_path/helm/chart/value_test.yaml results/$folder_api_name/value.yaml
+
+  oc logs $rapidast_pod -n default >> results/$folder_api_name/pod_logs.out
+
+  ./results.sh rapidast-pvc results/$folder_api_name
+  ls results
+
+  ls results/$folder_api_name
+
+  phase=$(oc get $rapidast_pod -o jsonpath='{.status.phase}')
+  helm uninstall rapidast 
+  oc delete pvc rapidast-pvc
+  (( counter++ ))
+done
+
+python find_alert_types.py
+
+if [ $phase != "Succeeded" ]; then
+    echo "Pod $rapidast_pod failed. Look at pod logs in archives (results/*/pod_logs.out)"
+    exit 1
+fi