From 57eeb5a7e83c303bcfc3e4f7ea9e6d9d47cf8708 Mon Sep 17 00:00:00 2001 From: Ishwar Kanse Date: Thu, 24 Oct 2024 11:20:21 +0530 Subject: [PATCH] Test OpenTelemetry must-gather script --- .../must-gather/chainsaw-test.yaml | 105 ++++++++++++++++++ .../must-gather/check_must_gather.sh | 52 +++++++++ .../must-gather/install-app-assert.yaml | 77 +++++++++++++ .../must-gather/install-app.yaml | 31 ++++++ .../install-collector-sidecar.yaml | 22 ++++ .../must-gather/install-instrumentation.yaml | 33 ++++++ .../install-target-allocator-assert.yaml | 58 ++++++++++ .../must-gather/install-target-allocator.yaml | 70 ++++++++++++ .../00-install-jaeger.yaml | 7 -- .../otlp-metrics-traces/chainsaw-test.yaml | 22 ++++ .../otlp-metrics-traces/check_must_gather.sh | 41 +++++++ 11 files changed, 511 insertions(+), 7 deletions(-) create mode 100755 tests/e2e-openshift/must-gather/chainsaw-test.yaml create mode 100755 tests/e2e-openshift/must-gather/check_must_gather.sh create mode 100644 tests/e2e-openshift/must-gather/install-app-assert.yaml create mode 100644 tests/e2e-openshift/must-gather/install-app.yaml create mode 100644 tests/e2e-openshift/must-gather/install-collector-sidecar.yaml create mode 100644 tests/e2e-openshift/must-gather/install-instrumentation.yaml create mode 100644 tests/e2e-openshift/must-gather/install-target-allocator-assert.yaml create mode 100644 tests/e2e-openshift/must-gather/install-target-allocator.yaml create mode 100755 tests/e2e-openshift/otlp-metrics-traces/check_must_gather.sh diff --git a/tests/e2e-openshift/must-gather/chainsaw-test.yaml b/tests/e2e-openshift/must-gather/chainsaw-test.yaml new file mode 100755 index 0000000000..1f7f295db1 --- /dev/null +++ b/tests/e2e-openshift/must-gather/chainsaw-test.yaml @@ -0,0 +1,105 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: must-gather +spec: + namespace: chainsaw-must-gather + steps: + - name: Install Target Allocator + try: + - apply: + template: true + file: install-target-allocator.yaml + - assert: + file: install-target-allocator-assert.yaml + catch: + - podLogs: + selector: app.kubernetes.io/component=opentelemetry-targetallocator + - name: assert-data-propagates + try: + - assert: + resource: + apiVersion: v1 + kind: ConfigMap + metadata: + name: stateful-targetallocator + namespace: ($namespace) + data: + targetallocator.yaml: + (parse_yaml(@)): + allocation_strategy: consistent-hashing + collector_selector: + matchlabels: + app.kubernetes.io/component: opentelemetry-collector + app.kubernetes.io/instance: (join('.', [$namespace, 'stateful'])) + app.kubernetes.io/managed-by: opentelemetry-operator + app.kubernetes.io/part-of: opentelemetry + matchexpressions: [ ] + config: + global: + scrape_interval: 30s + scrape_protocols: + - PrometheusProto + - OpenMetricsText1.0.0 + - OpenMetricsText0.0.1 + - PrometheusText0.0.4 + scrape_configs: + - job_name: otel-collector + scrape_interval: 10s + static_configs: + - targets: + - 0.0.0.0:8888 + filter_strategy: relabel-config + - name: Create instrumentation CR and sidecar collector instance + try: + # In OpenShift, when a namespace is created, all necessary SCC annotations are automatically added. However, if a namespace is created using a resource file with only selected SCCs, the other auto-added SCCs are not included. Therefore, the UID-range and supplemental groups SCC annotations must be set after the namespace is created. + - command: + entrypoint: kubectl + args: + - annotate + - namespace + - ${NAMESPACE} + - openshift.io/sa.scc.uid-range=1000/1000 + - --overwrite + - command: + entrypoint: kubectl + args: + - annotate + - namespace + - ${NAMESPACE} + - openshift.io/sa.scc.supplemental-groups=3000/3000 + - --overwrite + - apply: + file: install-collector-sidecar.yaml + - apply: + file: install-instrumentation.yaml + - name: Install app + try: + - apply: + file: install-app.yaml + - assert: + file: install-app-assert.yaml + catch: + - podLogs: + selector: app=my-nodejs + - name: Run the must-gather and verify the contents + try: + - command: + entrypoint: oc + args: + - get + - pods + - -A + - -l control-plane=controller-manager + - -l app.kubernetes.io/name=opentelemetry-operator + - -o + - jsonpath={.items[0].metadata.namespace} + outputs: + - name: OTEL_NAMESPACE + value: ($stdout) + - script: + env: + - name: otelnamespace + value: ($OTEL_NAMESPACE) + timeout: 5m + content: ./check_must_gather.sh diff --git a/tests/e2e-openshift/must-gather/check_must_gather.sh b/tests/e2e-openshift/must-gather/check_must_gather.sh new file mode 100755 index 0000000000..078107ffae --- /dev/null +++ b/tests/e2e-openshift/must-gather/check_must_gather.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +# Create the directory to store must-gather +MUST_GATHER_DIR=/tmp/must-gather +mkdir -p $MUST_GATHER_DIR + +# Run the must-gather script +oc adm must-gather --dest-dir=$MUST_GATHER_DIR --image=ghcr.io/open-telemetry/opentelemetry-operator/must-gather:latest -- /usr/bin/must-gather --operator-namespace $otelnamespace + +# Define required files and directories +REQUIRED_ITEMS=( + event-filter.html + ghcr-io-open-telemetry-opentelemetry-operator-must-gather-sha256-*/olm/*opentelemetry-operato*.yaml + ghcr-io-open-telemetry-opentelemetry-operator-must-gather-sha256-*/olm/clusterserviceversion-opentelemetry-operator-v*.yaml + ghcr-io-open-telemetry-opentelemetry-operator-must-gather-sha256-*/olm/installplan-install-*.yaml + ghcr-io-open-telemetry-opentelemetry-operator-must-gather-sha256-*/olm/subscription-opentelemetry-operator-v*-sub.yaml + ghcr-io-open-telemetry-opentelemetry-operator-must-gather-sha256-*/namespaces/chainsaw-must-gather/stateful/service-stateful-collector-headless.yaml + ghcr-io-open-telemetry-opentelemetry-operator-must-gather-sha256-*/namespaces/chainsaw-must-gather/stateful/service-stateful-collector.yaml + ghcr-io-open-telemetry-opentelemetry-operator-must-gather-sha256-*/namespaces/chainsaw-must-gather/stateful/deployment-stateful-targetallocator.yaml + ghcr-io-open-telemetry-opentelemetry-operator-must-gather-sha256-*/namespaces/chainsaw-must-gather/stateful/service-stateful-collector-monitoring.yaml + ghcr-io-open-telemetry-opentelemetry-operator-must-gather-sha256-*/namespaces/chainsaw-must-gather/stateful/poddisruptionbudget-stateful-targetallocator.yaml + ghcr-io-open-telemetry-opentelemetry-operator-must-gather-sha256-*/namespaces/chainsaw-must-gather/stateful/poddisruptionbudget-stateful-collector.yaml + ghcr-io-open-telemetry-opentelemetry-operator-must-gather-sha256-*/namespaces/chainsaw-must-gather/stateful/service-stateful-targetallocator.yaml + ghcr-io-open-telemetry-opentelemetry-operator-must-gather-sha256-*/namespaces/chainsaw-must-gather/stateful/configmap-stateful-collector-7a42612e.yaml + ghcr-io-open-telemetry-opentelemetry-operator-must-gather-sha256-*/namespaces/chainsaw-must-gather/stateful/configmap-stateful-targetallocator.yaml + ghcr-io-open-telemetry-opentelemetry-operator-must-gather-sha256-*/namespaces/chainsaw-must-gather/stateful/statefulset-stateful-collector.yaml + ghcr-io-open-telemetry-opentelemetry-operator-must-gather-sha256-*/namespaces/chainsaw-must-gather/stateful/opentelemetrycollector-stateful.yaml + ghcr-io-open-telemetry-opentelemetry-operator-must-gather-sha256-*/namespaces/chainsaw-must-gather/stateful/serviceaccount-stateful-collector.yaml + ghcr-io-open-telemetry-opentelemetry-operator-must-gather-sha256-*/namespaces/chainsaw-must-gather/sidecar/service-sidecar-collector.yaml + ghcr-io-open-telemetry-opentelemetry-operator-must-gather-sha256-*/namespaces/chainsaw-must-gather/sidecar/opentelemetrycollector-sidecar.yaml + ghcr-io-open-telemetry-opentelemetry-operator-must-gather-sha256-*/namespaces/chainsaw-must-gather/sidecar/service-sidecar-collector-monitoring.yaml + ghcr-io-open-telemetry-opentelemetry-operator-must-gather-sha256-*/namespaces/chainsaw-must-gather/sidecar/configmap-sidecar-collector-aec5aa11.yaml + ghcr-io-open-telemetry-opentelemetry-operator-must-gather-sha256-*/namespaces/chainsaw-must-gather/sidecar/serviceaccount-sidecar-collector.yaml + ghcr-io-open-telemetry-opentelemetry-operator-must-gather-sha256-*/namespaces/chainsaw-must-gather/sidecar/service-sidecar-collector-headless.yaml + ghcr-io-open-telemetry-opentelemetry-operator-must-gather-sha256-*/chainsaw-must-gather/instrumentation-nodejs.yaml + ghcr-io-open-telemetry-opentelemetry-operator-must-gather-sha256-*/opentelemetry-operator-controller-manager-* + ghcr-io-open-telemetry-opentelemetry-operator-must-gather-sha256-*/deployment-opentelemetry-operator-controller-manager.yaml + timestamp +) + +# Verify each required item +for item in "${REQUIRED_ITEMS[@]}"; do + if ! find "$MUST_GATHER_DIR" -path "$MUST_GATHER_DIR/$item" -print -quit | grep -q .; then + echo "Missing: $item" + exit 1 + else + echo "Found: $item" + fi +done + +# Cleanup the must-gather directory +rm -rf $MUST_GATHER_DIR diff --git a/tests/e2e-openshift/must-gather/install-app-assert.yaml b/tests/e2e-openshift/must-gather/install-app-assert.yaml new file mode 100644 index 0000000000..719727ca68 --- /dev/null +++ b/tests/e2e-openshift/must-gather/install-app-assert.yaml @@ -0,0 +1,77 @@ +apiVersion: v1 +kind: Pod +metadata: + annotations: + instrumentation.opentelemetry.io/inject-nodejs: "true" + sidecar.opentelemetry.io/inject: "true" + labels: + app: my-nodejs +spec: + containers: + - env: + - name: OTEL_NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: OTEL_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_PATH + value: /usr/local/lib/node_modules + - name: OTEL_NODEJS_DEBUG + value: "true" + - name: NODE_OPTIONS + value: ' --require /otel-auto-instrumentation-nodejs/autoinstrumentation.js' + - name: OTEL_TRACES_EXPORTER + value: otlp + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: http://localhost:4317 + - name: OTEL_EXPORTER_OTLP_TIMEOUT + value: "20" + - name: OTEL_TRACES_SAMPLER + value: parentbased_traceidratio + - name: OTEL_TRACES_SAMPLER_ARG + value: "0.85" + - name: SPLUNK_TRACE_RESPONSE_HEADER_ENABLED + value: "true" + - name: OTEL_METRICS_EXPORTER + value: prometheus + - name: OTEL_SERVICE_NAME + value: my-nodejs + - name: OTEL_RESOURCE_ATTRIBUTES_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: OTEL_RESOURCE_ATTRIBUTES_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: OTEL_PROPAGATORS + value: jaeger,b3 + - name: OTEL_RESOURCE_ATTRIBUTES + name: myapp + volumeMounts: + - mountPath: /var/run/secrets/kubernetes.io/serviceaccount + readOnly: true + - mountPath: /otel-auto-instrumentation-nodejs + name: opentelemetry-auto-instrumentation-nodejs + - args: + - --config=env:OTEL_CONFIG + name: otc-container + initContainers: + - name: opentelemetry-auto-instrumentation-nodejs +status: + containerStatuses: + - name: myapp + ready: true + started: true + - name: otc-container + ready: true + started: true + initContainerStatuses: + - name: opentelemetry-auto-instrumentation-nodejs + ready: true + phase: Running diff --git a/tests/e2e-openshift/must-gather/install-app.yaml b/tests/e2e-openshift/must-gather/install-app.yaml new file mode 100644 index 0000000000..45de0d5f01 --- /dev/null +++ b/tests/e2e-openshift/must-gather/install-app.yaml @@ -0,0 +1,31 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: my-nodejs +spec: + selector: + matchLabels: + app: my-nodejs + replicas: 1 + template: + metadata: + labels: + app: my-nodejs + annotations: + sidecar.opentelemetry.io/inject: "true" + instrumentation.opentelemetry.io/inject-nodejs: "true" + spec: + securityContext: + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 3000 + containers: + - name: myapp + image: ghcr.io/open-telemetry/opentelemetry-operator/e2e-test-app-nodejs:main + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + env: + - name: NODE_PATH + value: /usr/local/lib/node_modules diff --git a/tests/e2e-openshift/must-gather/install-collector-sidecar.yaml b/tests/e2e-openshift/must-gather/install-collector-sidecar.yaml new file mode 100644 index 0000000000..34a26ebb2c --- /dev/null +++ b/tests/e2e-openshift/must-gather/install-collector-sidecar.yaml @@ -0,0 +1,22 @@ +apiVersion: opentelemetry.io/v1alpha1 +kind: OpenTelemetryCollector +metadata: + name: sidecar +spec: + config: | + receivers: + otlp: + protocols: + grpc: + http: + processors: + + exporters: + debug: + + service: + pipelines: + traces: + receivers: [otlp] + exporters: [debug] + mode: sidecar diff --git a/tests/e2e-openshift/must-gather/install-instrumentation.yaml b/tests/e2e-openshift/must-gather/install-instrumentation.yaml new file mode 100644 index 0000000000..a87939b6c2 --- /dev/null +++ b/tests/e2e-openshift/must-gather/install-instrumentation.yaml @@ -0,0 +1,33 @@ +apiVersion: opentelemetry.io/v1alpha1 +kind: Instrumentation +metadata: + name: nodejs +spec: + env: + - name: OTEL_TRACES_EXPORTER + value: otlp + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: http://localhost:4317 + - name: OTEL_EXPORTER_OTLP_TIMEOUT + value: "20" + - name: OTEL_TRACES_SAMPLER + value: parentbased_traceidratio + - name: OTEL_TRACES_SAMPLER_ARG + value: "0.85" + - name: SPLUNK_TRACE_RESPONSE_HEADER_ENABLED + value: "true" + - name: OTEL_METRICS_EXPORTER + value: prometheus + exporter: + endpoint: http://localhost:4317 + propagators: + - jaeger + - b3 + sampler: + type: parentbased_traceidratio + argument: "0.25" + nodejs: + env: + - name: OTEL_NODEJS_DEBUG + value: "true" + diff --git a/tests/e2e-openshift/must-gather/install-target-allocator-assert.yaml b/tests/e2e-openshift/must-gather/install-target-allocator-assert.yaml new file mode 100644 index 0000000000..1ba3d195e3 --- /dev/null +++ b/tests/e2e-openshift/must-gather/install-target-allocator-assert.yaml @@ -0,0 +1,58 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: stateful-collector +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: stateful-targetallocator +status: + observedGeneration: 1 + readyReplicas: 1 + replicas: 1 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: stateful-targetallocator +--- +apiVersion: v1 +data: + collector.yaml: | + exporters: + debug: null + receivers: + jaeger: + protocols: + grpc: + endpoint: 0.0.0.0:14250 + prometheus: + config: + global: + scrape_interval: 30s + scrape_protocols: + - PrometheusProto + - OpenMetricsText1.0.0 + - OpenMetricsText0.0.1 + - PrometheusText0.0.4 + target_allocator: + collector_id: ${POD_NAME} + endpoint: http://stateful-targetallocator:80 + interval: 30s + service: + pipelines: + traces: + exporters: + - debug + receivers: + - jaeger + telemetry: + metrics: + address: 0.0.0.0:8888 +kind: ConfigMap +metadata: + name: stateful-collector-7a42612e diff --git a/tests/e2e-openshift/must-gather/install-target-allocator.yaml b/tests/e2e-openshift/must-gather/install-target-allocator.yaml new file mode 100644 index 0000000000..806224f6ee --- /dev/null +++ b/tests/e2e-openshift/must-gather/install-target-allocator.yaml @@ -0,0 +1,70 @@ +apiVersion: v1 +automountServiceAccountToken: true +kind: ServiceAccount +metadata: + name: ta +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: smoke-targetallocator +rules: +- apiGroups: + - "" + resources: + - pods + - namespaces + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: (join('-', ['default-view', $namespace])) +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: smoke-targetallocator +subjects: +- kind: ServiceAccount + name: ta + namespace: ($namespace) +--- +apiVersion: opentelemetry.io/v1alpha1 +kind: OpenTelemetryCollector +metadata: + name: stateful +spec: + config: | + receivers: + jaeger: + protocols: + grpc: + + # Collect own metrics + prometheus: + config: + global: + scrape_interval: 30s + scrape_protocols: ['PrometheusProto','OpenMetricsText1.0.0','OpenMetricsText0.0.1','PrometheusText0.0.4'] + scrape_configs: + - job_name: 'otel-collector' + scrape_interval: 10s + static_configs: + - targets: [ '0.0.0.0:8888' ] + + processors: + + exporters: + debug: + service: + pipelines: + traces: + receivers: [jaeger] + exporters: [debug] + mode: statefulset + targetAllocator: + enabled: true + serviceAccount: ta diff --git a/tests/e2e-openshift/otlp-metrics-traces/00-install-jaeger.yaml b/tests/e2e-openshift/otlp-metrics-traces/00-install-jaeger.yaml index 7ad775f831..9a2cae1b6b 100644 --- a/tests/e2e-openshift/otlp-metrics-traces/00-install-jaeger.yaml +++ b/tests/e2e-openshift/otlp-metrics-traces/00-install-jaeger.yaml @@ -1,11 +1,4 @@ #For this test case you'll need to install the Jaeger operator (OpenShift Distributed Tracing Platform in OpenShift) - -apiVersion: v1 -kind: Namespace -metadata: - name: chainsaw-otlp-metrics - ---- apiVersion: jaegertracing.io/v1 kind: Jaeger metadata: diff --git a/tests/e2e-openshift/otlp-metrics-traces/chainsaw-test.yaml b/tests/e2e-openshift/otlp-metrics-traces/chainsaw-test.yaml index 50ed4185e7..de34a5a232 100755 --- a/tests/e2e-openshift/otlp-metrics-traces/chainsaw-test.yaml +++ b/tests/e2e-openshift/otlp-metrics-traces/chainsaw-test.yaml @@ -7,6 +7,7 @@ metadata: spec: # Avoid running this test case in parallel to prevent the deletion of shared resources used by multiple tests, specifically in the context of OpenShift user workload monitoring. concurrent: false + namespace: chainsaw-otlp-metrics steps: - name: step-00 try: @@ -42,3 +43,24 @@ spec: - script: timeout: 5m content: ./check_metrics.sh + - name: Run the must-gather and verify the contents + try: + - command: + entrypoint: oc + args: + - get + - pods + - -A + - -l control-plane=controller-manager + - -l app.kubernetes.io/name=opentelemetry-operator + - -o + - jsonpath={.items[0].metadata.namespace} + outputs: + - name: OTEL_NAMESPACE + value: ($stdout) + - script: + env: + - name: otelnamespace + value: ($OTEL_NAMESPACE) + timeout: 5m + content: ./check_must_gather.sh diff --git a/tests/e2e-openshift/otlp-metrics-traces/check_must_gather.sh b/tests/e2e-openshift/otlp-metrics-traces/check_must_gather.sh new file mode 100755 index 0000000000..fc67dbb6b3 --- /dev/null +++ b/tests/e2e-openshift/otlp-metrics-traces/check_must_gather.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# Create the directory to store must-gather +MUST_GATHER_DIR=/tmp/otlp-metrics-traces +mkdir -p $MUST_GATHER_DIR + +# Run the must-gather script +oc adm must-gather --dest-dir=$MUST_GATHER_DIR --image=ghcr.io/open-telemetry/opentelemetry-operator/must-gather:latest -- /usr/bin/must-gather --operator-namespace $otelnamespace + +# Define required files and directories +REQUIRED_ITEMS=( + event-filter.html + ghcr-io-open-telemetry-opentelemetry-operator-must-gather-sha256-*/olm/clusterserviceversion-opentelemetry-operator-*.yaml + ghcr-io-open-telemetry-opentelemetry-operator-must-gather-sha256-*/olm/*opentelemetry-operator*.yaml + ghcr-io-open-telemetry-opentelemetry-operator-must-gather-sha256-*/olm/installplan-install-*.yaml + ghcr-io-open-telemetry-opentelemetry-operator-must-gather-sha256-*/olm/subscription-opentelemetry-operator-v*-sub.yaml + ghcr-io-open-telemetry-opentelemetry-operator-must-gather-sha256-*/namespaces/chainsaw-otlp-metrics/cluster-collector/service-cluster-collector-collector-headless.yaml + ghcr-io-open-telemetry-opentelemetry-operator-must-gather-sha256-*/namespaces/chainsaw-otlp-metrics/cluster-collector/deployment-cluster-collector-collector.yaml + ghcr-io-open-telemetry-opentelemetry-operator-must-gather-sha256-*/namespaces/chainsaw-otlp-metrics/cluster-collector/service-cluster-collector-collector-monitoring.yaml + ghcr-io-open-telemetry-opentelemetry-operator-must-gather-sha256-*/namespaces/chainsaw-otlp-metrics/cluster-collector/opentelemetrycollector-cluster-collector.yaml + ghcr-io-open-telemetry-opentelemetry-operator-must-gather-sha256-*/namespaces/chainsaw-otlp-metrics/cluster-collector/configmap-cluster-collector-collector-3d90ce58.yaml + ghcr-io-open-telemetry-opentelemetry-operator-must-gather-sha256-*/namespaces/chainsaw-otlp-metrics/cluster-collector/serviceaccount-cluster-collector-collector.yaml + ghcr-io-open-telemetry-opentelemetry-operator-must-gather-sha256-*/namespaces/chainsaw-otlp-metrics/cluster-collector/service-cluster-collector-collector.yaml + ghcr-io-open-telemetry-opentelemetry-operator-must-gather-sha256-*/namespaces/chainsaw-otlp-metrics/cluster-collector/poddisruptionbudget-cluster-collector-collector.yaml + ghcr-io-open-telemetry-opentelemetry-operator-must-gather-sha256-*/opentelemetry-operator-controller-manager-* + ghcr-io-open-telemetry-opentelemetry-operator-must-gather-sha256-*/deployment-opentelemetry-operator-controller-manager.yaml + timestamp +) + +# Verify each required item +for item in "${REQUIRED_ITEMS[@]}"; do + if ! find "$MUST_GATHER_DIR" -path "$MUST_GATHER_DIR/$item" -print -quit | grep -q .; then + echo "Missing: $item" + exit 1 + else + echo "Found: $item" + fi +done + +# Cleanup the must-gather directory +rm -rf $MUST_GATHER_DIR