diff --git a/tests/e2e-otel/groupbyattrsprocessor/chainsaw-test.yaml b/tests/e2e-otel/groupbyattrsprocessor/chainsaw-test.yaml new file mode 100644 index 0000000..94f77cc --- /dev/null +++ b/tests/e2e-otel/groupbyattrsprocessor/chainsaw-test.yaml @@ -0,0 +1,33 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: groupbyattrsprocessor +spec: + namespace: chainsaw-gba + description: + steps: + - name: Enable user workload monitoring + try: + - apply: + file: workload-monitoring.yaml + - assert: + file: workload-monitoring-assert.yaml + - name: Create OTEl collector with kubletstats receiver + try: + - apply: + file: otel-collector.yaml + - assert: + file: otel-collector-assert.yaml + - name: Create OTEL collector with groupbyattrs processor + try: + - apply: + file: otel-groupbyattributes.yaml + - assert: + file: otel-groupbyattributes-assert.yaml + - name: Check the groupbyattrs metrics + try: + - apply: + file: monitoring-view-role.yaml + - script: + timeout: 5m + content: ./check_metrics.sh diff --git a/tests/e2e-otel/groupbyattrsprocessor/check_metrics.sh b/tests/e2e-otel/groupbyattrsprocessor/check_metrics.sh new file mode 100755 index 0000000..663cdfb --- /dev/null +++ b/tests/e2e-otel/groupbyattrsprocessor/check_metrics.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +TOKEN=$(oc create token prometheus-user-workload -n openshift-user-workload-monitoring) +THANOS_QUERIER_HOST=$(oc get route thanos-querier -n openshift-monitoring -o json | jq -r '.spec.host') + +#Check metrics for OpenTelemetry collector instance. +metrics="otelcol_processor_groupbyattrs_metric_groups_sum otelcol_processor_groupbyattrs_num_non_grouped_metrics otelcol_processor_groupbyattrs_metric_groups_bucket otelcol_processor_groupbyattrs_metric_groups_count" + +for metric in $metrics; do +query="$metric" +count=0 + +# Keep fetching and checking the metrics until metrics with value is present. +while [[ $count -eq 0 ]]; do + response=$(curl -k -H "Authorization: Bearer $TOKEN" -H "Content-type: application/json" "https://$THANOS_QUERIER_HOST/api/v1/query?query=$query") + count=$(echo "$response" | jq -r '.data.result | length') + + if [[ $count -eq 0 ]]; then + echo "No metric '$metric' with value present. Retrying..." + sleep 5 # Wait for 5 seconds before retrying + else + echo "Metric '$metric' with value is present." + fi + done +done + diff --git a/tests/e2e-otel/groupbyattrsprocessor/monitoring-view-role.yaml b/tests/e2e-otel/groupbyattrsprocessor/monitoring-view-role.yaml new file mode 100644 index 0000000..496aa62 --- /dev/null +++ b/tests/e2e-otel/groupbyattrsprocessor/monitoring-view-role.yaml @@ -0,0 +1,23 @@ +# Add the clusterrole and rolebinding required for fetching metrics from Thanos querier. Refer https://issues.redhat.com/browse/MON-3379 +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: chainsaw-gba +rules: +- apiGroups: ["monitoring.coreos.com"] + resources: ["prometheuses/api"] + verbs: ["get", "list", "watch"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: chainsaw-gba +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: chainsaw-gba +subjects: +- kind: ServiceAccount + name: prometheus-user-workload + namespace: openshift-user-workload-monitoring diff --git a/tests/e2e-otel/groupbyattrsprocessor/otel-collector-assert.yaml b/tests/e2e-otel/groupbyattrsprocessor/otel-collector-assert.yaml new file mode 100644 index 0000000..edb2214 --- /dev/null +++ b/tests/e2e-otel/groupbyattrsprocessor/otel-collector-assert.yaml @@ -0,0 +1,64 @@ +apiVersion: project.openshift.io/v1 +kind: Project +metadata: + name: chainsaw-gba +status: + phase: Active + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: chainsaw-gba + namespace: chainsaw-gba + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: chainsaw-gba-role +rules: +- apiGroups: + - "" + resources: + - nodes/stats + verbs: + - get + - watch + - list +- apiGroups: + - "" + resources: + - nodes/proxy + verbs: + - get + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: chainsaw-gba-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: chainsaw-gba-role +subjects: +- kind: ServiceAccount + name: chainsaw-gba + namespace: chainsaw-gba + +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: chainsaw-gba-collector + namespace: chainsaw-gba +spec: + updateStrategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + type: RollingUpdate +status: + numberMisscheduled: 0 + (desiredNumberScheduled == numberReady): true \ No newline at end of file diff --git a/tests/e2e-otel/groupbyattrsprocessor/otel-collector.yaml b/tests/e2e-otel/groupbyattrsprocessor/otel-collector.yaml new file mode 100644 index 0000000..9320331 --- /dev/null +++ b/tests/e2e-otel/groupbyattrsprocessor/otel-collector.yaml @@ -0,0 +1,70 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: chainsaw-gba + namespace: chainsaw-gba + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: chainsaw-gba-role +rules: + - apiGroups: [''] + resources: ['nodes/stats'] + verbs: ['get', 'watch', 'list'] + - apiGroups: [""] + resources: ["nodes/proxy"] + verbs: ["get"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: chainsaw-gba-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: chainsaw-gba-role +subjects: + - kind: ServiceAccount + name: chainsaw-gba + namespace: chainsaw-gba + +--- +apiVersion: opentelemetry.io/v1alpha1 +kind: OpenTelemetryCollector +metadata: + name: chainsaw-gba + namespace: chainsaw-gba +spec: + mode: daemonset + image: ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-contrib:0.98.0 + serviceAccount: chainsaw-gba + serviceAccountName: chainsaw-gba + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + config: | + receivers: + kubeletstats: + collection_interval: 20s + auth_type: "serviceAccount" + endpoint: "https://${env:K8S_NODE_NAME}:10250" + insecure_skip_verify: true + exporters: + otlp: + endpoint: gba-main-collector.chainsaw-gba.svc:4317 + tls: + insecure: true + service: + pipelines: + metrics: + receivers: [kubeletstats] + exporters: [otlp] + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule diff --git a/tests/e2e-otel/groupbyattrsprocessor/otel-groupbyattributes-assert.yaml b/tests/e2e-otel/groupbyattrsprocessor/otel-groupbyattributes-assert.yaml new file mode 100644 index 0000000..f1316e8 --- /dev/null +++ b/tests/e2e-otel/groupbyattrsprocessor/otel-groupbyattributes-assert.yaml @@ -0,0 +1,32 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: gba-main-collector + namespace: chainsaw-gba +status: + availableReplicas: 1 + readyReplicas: 1 + replicas: 1 + +--- +apiVersion: v1 +kind: Service +metadata: + name: gba-main-collector + namespace: chainsaw-gba +spec: + ports: + - appProtocol: grpc + name: otlp-grpc + port: 4317 + protocol: TCP + targetPort: 4317 + - name: prometheus + port: 8889 + protocol: TCP + targetPort: 8889 + selector: + app.kubernetes.io/component: opentelemetry-collector + app.kubernetes.io/instance: chainsaw-gba.gba-main + app.kubernetes.io/managed-by: opentelemetry-operator + app.kubernetes.io/part-of: opentelemetry diff --git a/tests/e2e-otel/groupbyattrsprocessor/otel-groupbyattributes.yaml b/tests/e2e-otel/groupbyattrsprocessor/otel-groupbyattributes.yaml new file mode 100644 index 0000000..4502a3b --- /dev/null +++ b/tests/e2e-otel/groupbyattrsprocessor/otel-groupbyattributes.yaml @@ -0,0 +1,39 @@ +apiVersion: opentelemetry.io/v1alpha1 +kind: OpenTelemetryCollector +metadata: + name: gba-main + namespace: chainsaw-gba +spec: + mode: deployment + image: ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-contrib:0.98.0 + observability: + metrics: + enableMetrics: true + config: | + receivers: + otlp: + protocols: + grpc: + processors: + groupbyattrs: + keys: + - k8s.namespace.name + - k8s.container.name + - k8s.pod.name + batch: + attributes: + actions: + - key: otelpipeline + value: gba + action: insert + exporters: + prometheus: + endpoint: 0.0.0.0:8889 + resource_to_telemetry_conversion: + enabled: true # by default resource attributes are dropped + service: + pipelines: + metrics: + receivers: [otlp] + processors: [attributes, batch, groupbyattrs] + exporters: [prometheus] diff --git a/tests/e2e-otel/groupbyattrsprocessor/workload-monitoring-assert.yaml b/tests/e2e-otel/groupbyattrsprocessor/workload-monitoring-assert.yaml new file mode 100644 index 0000000..a94fda1 --- /dev/null +++ b/tests/e2e-otel/groupbyattrsprocessor/workload-monitoring-assert.yaml @@ -0,0 +1,28 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prometheus-operator + namespace: openshift-user-workload-monitoring +(status.replicas == spec.replicas): true +spec: + (replicas >= `1`): true + +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: prometheus-user-workload + namespace: openshift-user-workload-monitoring +(status.replicas == spec.replicas): true +spec: + (replicas >= `1`): true + +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: thanos-ruler-user-workload + namespace: openshift-user-workload-monitoring +(status.replicas == spec.replicas): true +spec: + (replicas >= `1`): true diff --git a/tests/e2e-otel/groupbyattrsprocessor/workload-monitoring.yaml b/tests/e2e-otel/groupbyattrsprocessor/workload-monitoring.yaml new file mode 100644 index 0000000..1d5c5e4 --- /dev/null +++ b/tests/e2e-otel/groupbyattrsprocessor/workload-monitoring.yaml @@ -0,0 +1,11 @@ +# oc -n openshift-user-workload-monitoring get pod +# https://docs.openshift.com/container-platform/4.13/monitoring/enabling-monitoring-for-user-defined-projects.html#accessing-metrics-from-outside-cluster_enabling-monitoring-for-user-defined-projects + +apiVersion: v1 +kind: ConfigMap +metadata: + name: cluster-monitoring-config + namespace: openshift-monitoring +data: + config.yaml: | + enableUserWorkload: true diff --git a/tests/e2e-otel/hostmetricsreceiver/chainsaw-test.yaml b/tests/e2e-otel/hostmetricsreceiver/chainsaw-test.yaml index 22690d8..c103f65 100644 --- a/tests/e2e-otel/hostmetricsreceiver/chainsaw-test.yaml +++ b/tests/e2e-otel/hostmetricsreceiver/chainsaw-test.yaml @@ -3,6 +3,7 @@ kind: Test metadata: name: hostmetricsreceiver spec: + namespace: chainsaw-hostmetrics steps: - name: Create OTEL collector with hostmetricsreceiver receiver try: diff --git a/tests/e2e-otel/hostmetricsreceiver/check_logs.sh b/tests/e2e-otel/hostmetricsreceiver/check_logs.sh index e7be85d..1448004 100755 --- a/tests/e2e-otel/hostmetricsreceiver/check_logs.sh +++ b/tests/e2e-otel/hostmetricsreceiver/check_logs.sh @@ -3,7 +3,7 @@ # Define the label selector LABEL_SELECTOR="app.kubernetes.io/component=opentelemetry-collector" -NAMESPACE=chainsaw-hostmetricsreceiver +NAMESPACE=chainsaw-hostmetrics # Define the search strings SEARCH_STRING1='process.pid' @@ -54,4 +54,4 @@ while ! $FOUND1 || ! $FOUND2 || ! $FOUND3 || ! $FOUND4 || ! $FOUND5; do done done -echo "Found all the host metrics in OpenTelemetry collector." \ No newline at end of file +echo "Found all the host metrics in OpenTelemetry collector." diff --git a/tests/e2e-otel/hostmetricsreceiver/otel-hostmetricsreceiver-assert.yaml b/tests/e2e-otel/hostmetricsreceiver/otel-hostmetricsreceiver-assert.yaml index 6ce86b9..93e25c2 100644 --- a/tests/e2e-otel/hostmetricsreceiver/otel-hostmetricsreceiver-assert.yaml +++ b/tests/e2e-otel/hostmetricsreceiver/otel-hostmetricsreceiver-assert.yaml @@ -1,8 +1,8 @@ apiVersion: apps/v1 kind: DaemonSet metadata: - name: otel-hostmetricsreceiver-collector - namespace: chainsaw-hostmetricsreceiver + name: otel-hstmtrs-collector + namespace: chainsaw-hostmetrics spec: updateStrategy: rollingUpdate: @@ -17,8 +17,8 @@ status: apiVersion: v1 kind: Service metadata: - name: otel-hostmetricsreceiver-collector-monitoring - namespace: chainsaw-hostmetricsreceiver + name: otel-hstmtrs-collector-monitoring + namespace: chainsaw-hostmetrics spec: ports: - name: monitoring @@ -27,6 +27,6 @@ spec: targetPort: 8888 selector: app.kubernetes.io/component: opentelemetry-collector - app.kubernetes.io/instance: chainsaw-hostmetricsreceiver.otel-hostmetricsreceiver + app.kubernetes.io/instance: chainsaw-hostmetrics.otel-hstmtrs app.kubernetes.io/managed-by: opentelemetry-operator app.kubernetes.io/part-of: opentelemetry \ No newline at end of file diff --git a/tests/e2e-otel/hostmetricsreceiver/otel-hostmetricsreceiver.yaml b/tests/e2e-otel/hostmetricsreceiver/otel-hostmetricsreceiver.yaml index f0f44ac..8386562 100644 --- a/tests/e2e-otel/hostmetricsreceiver/otel-hostmetricsreceiver.yaml +++ b/tests/e2e-otel/hostmetricsreceiver/otel-hostmetricsreceiver.yaml @@ -1,65 +1,52 @@ apiVersion: v1 -kind: Namespace -metadata: - name: chainsaw-hostmetricsreceiver - labels: - security.openshift.io/scc.podSecurityLabelSync: "false" - pod-security.kubernetes.io/enforce: "privileged" - pod-security.kubernetes.io/audit: "privileged" - pod-security.kubernetes.io/warn: "privileged" - ---- -apiVersion: v1 kind: ServiceAccount metadata: - name: privileged-sa - namespace: chainsaw-hostmetricsreceiver + name: otel-hostfs-daemonset + namespace: chainsaw-hostmetrics --- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding +apiVersion: security.openshift.io/v1 +kind: SecurityContextConstraints +allowHostDirVolumePlugin: true +allowHostIPC: false +allowHostNetwork: false +allowHostPID: true +allowHostPorts: false +allowPrivilegeEscalation: true +allowPrivilegedContainer: true +allowedCapabilities: null +defaultAddCapabilities: +- SYS_ADMIN +fsGroup: + type: RunAsAny +groups: [] metadata: - name: chainsaw-hostmetricsreceiver-binding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:openshift:scc:privileged -subjects: -- kind: ServiceAccount - name: privileged-sa - namespace: chainsaw-hostmetricsreceiver + name: otel-hostmetrics +readOnlyRootFilesystem: true +runAsUser: + type: RunAsAny +seLinuxContext: + type: RunAsAny +supplementalGroups: + type: RunAsAny +users: +- system:serviceaccount:chainsaw-hostmetrics:otel-hostfs-daemonset +volumes: +- configMap +- emptyDir +- hostPath +- projected --- apiVersion: opentelemetry.io/v1alpha1 kind: OpenTelemetryCollector metadata: - name: otel-hostmetricsreceiver - namespace: chainsaw-hostmetricsreceiver + name: otel-hstmtrs spec: mode: daemonset image: ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-contrib:0.98.0 - serviceAccount: privileged-sa - serviceAccountName: privileged-sa - securityContext: - runAsUser: 0 - runAsGroup: 0 - allowPrivilegeEscalation: false - capabilities: - drop: - - CHOWN - - DAC_OVERRIDE - - FOWNER - - FSETID - - KILL - - NET_BIND_SERVICE - - SETGID - - SETPCAP - - SETUID - readOnlyRootFilesystem: true - seLinuxOptions: - type: spc_t - seccompProfile: - type: RuntimeDefault + serviceAccount: otel-hostfs-daemonset + serviceAccountName: otel-hostfs-daemonset config: | receivers: hostmetrics: