From 5f7bd358236836ad21a0c933e0bc4a59c2dc8127 Mon Sep 17 00:00:00 2001 From: ryanohnemus Date: Fri, 12 Jan 2024 10:10:01 -0600 Subject: [PATCH] ensure tests run helm uninstall in teardown Signed-off-by: ryanohnemus --- tests/chunk-rollover/basic.bats | 9 +++-- .../resources/helm/fluentbit-basic.yaml | 8 ++--- tests/elasticsearch/basic.bats | 2 ++ tests/elasticsearch/compress.bats | 2 ++ tests/kubernetes-plugins/basic.bats | 33 ++++++++++++++----- .../resources/fluentbit-basic.yaml | 1 + tests/opensearch/basic.bats | 2 ++ tests/opensearch/hosted.bats | 14 ++++++-- 8 files changed, 51 insertions(+), 20 deletions(-) diff --git a/tests/chunk-rollover/basic.bats b/tests/chunk-rollover/basic.bats index e7b6fd8..717ca11 100644 --- a/tests/chunk-rollover/basic.bats +++ b/tests/chunk-rollover/basic.bats @@ -27,6 +27,7 @@ setup_file() { teardown_file() { if [[ "${SKIP_TEARDOWN:-no}" != "yes" ]]; then + helm uninstall -n $TEST_NAMESPACE fluent-bit run kubectl delete namespace "$TEST_NAMESPACE" rm -f ${HELM_VALUES_EXTRA_FILE} fi @@ -58,10 +59,9 @@ DETIK_CLIENT_NAMESPACE="${TEST_NAMESPACE}" # replace the namespace for svc FQDN - sed -i "s/payload-receiver-service.test.svc.cluster.local/payload-receiver-service.$TEST_NAMESPACE.svc.cluster.local/g" "${BATS_TEST_DIRNAME}/resources/helm/fluentbit-basic.yaml" helm upgrade --install --debug --create-namespace --namespace "$TEST_NAMESPACE" fluent-bit fluent/fluent-bit \ --values ${BATS_TEST_DIRNAME}/resources/helm/fluentbit-basic.yaml \ - --set image.repository=${FLUENTBIT_IMAGE_REPOSITORY},image.tag=${FLUENTBIT_IMAGE_TAG} \ + --set image.repository=${FLUENTBIT_IMAGE_REPOSITORY},image.tag=${FLUENTBIT_IMAGE_TAG},env[0].name=TEST_NAMESPACE,env[0].value=${TEST_NAMESPACE} \ --values "$HELM_VALUES_EXTRA_FILE" \ --timeout "${HELM_FB_TIMEOUT:-5m0s}" \ --wait @@ -75,9 +75,8 @@ DETIK_CLIENT_NAMESPACE="${TEST_NAMESPACE}" fi # replace the namespace for crb - sed -i "s/namespace: test/namespace: $TEST_NAMESPACE/g" "${BATS_TEST_DIRNAME}/resources/manifests/rbac/cluster-role-binding.yaml" - kubectl create -f ${BATS_TEST_DIRNAME}/resources/manifests/rbac/cluster-role-binding.yaml --dry-run=client -o yaml | kubectl apply -f - - + sed "s/namespace: test/namespace: $TEST_NAMESPACE/g" "${BATS_TEST_DIRNAME}/resources/manifests/rbac/cluster-role-binding.yaml" | kubectl apply -f - + # Time interval in seconds to check the pods status INTERVAL=10 diff --git a/tests/chunk-rollover/resources/helm/fluentbit-basic.yaml b/tests/chunk-rollover/resources/helm/fluentbit-basic.yaml index 8c02951..9003426 100644 --- a/tests/chunk-rollover/resources/helm/fluentbit-basic.yaml +++ b/tests/chunk-rollover/resources/helm/fluentbit-basic.yaml @@ -48,7 +48,7 @@ config: [OUTPUT] name http match * - host payload-receiver-service.test.svc.cluster.local + host payload-receiver-service.${TEST_NAMESPACE}.svc.cluster.local port 8088 retry_limit 3 storage.total_limit_size 300M @@ -58,7 +58,7 @@ config: [OUTPUT] name http match * - host payload-receiver-service.test.svc.cluster.local + host payload-receiver-service.${TEST_NAMESPACE}.svc.cluster.local port 8088 retry_limit 3 storage.total_limit_size 300M @@ -68,7 +68,7 @@ config: [OUTPUT] name http match * - host payload-receiver-service.test.svc.cluster.local + host payload-receiver-service.${TEST_NAMESPACE}.svc.cluster.local port 8088 retry_limit 3 storage.total_limit_size 300M @@ -78,7 +78,7 @@ config: [OUTPUT] name http match * - host payload-receiver-service.test.svc.cluster.local + host payload-receiver-service.${TEST_NAMESPACE}.svc.cluster.local port 8088 retry_limit 3 storage.total_limit_size 300M diff --git a/tests/elasticsearch/basic.bats b/tests/elasticsearch/basic.bats index 59fe579..05a3bff 100644 --- a/tests/elasticsearch/basic.bats +++ b/tests/elasticsearch/basic.bats @@ -27,6 +27,8 @@ setup_file() { teardown_file() { if [[ "${SKIP_TEARDOWN:-no}" != "yes" ]]; then + helm uninstall -n $TEST_NAMESPACE elasticsearch + helm uninstall -n $TEST_NAMESPACE fluent-bit run kubectl delete namespace "$TEST_NAMESPACE" rm -f ${HELM_VALUES_EXTRA_FILE} fi diff --git a/tests/elasticsearch/compress.bats b/tests/elasticsearch/compress.bats index 3ccf9e0..ed8aaea 100644 --- a/tests/elasticsearch/compress.bats +++ b/tests/elasticsearch/compress.bats @@ -27,6 +27,8 @@ setup_file() { teardown_file() { if [[ "${SKIP_TEARDOWN:-no}" != "yes" ]]; then + helm uninstall -n $TEST_NAMESPACE elasticsearch + helm uninstall -n $TEST_NAMESPACE fluent-bit run kubectl delete namespace "$TEST_NAMESPACE" rm -f ${HELM_VALUES_EXTRA_FILE} fi diff --git a/tests/kubernetes-plugins/basic.bats b/tests/kubernetes-plugins/basic.bats index f60245c..bce434d 100644 --- a/tests/kubernetes-plugins/basic.bats +++ b/tests/kubernetes-plugins/basic.bats @@ -27,6 +27,7 @@ setup() { teardown() { if [[ "${SKIP_TEARDOWN:-no}" != "yes" ]]; then + helm uninstall fluent-bit -n $TEST_NAMESPACE run kubectl delete namespace "$TEST_NAMESPACE" rm -f ${HELM_VALUES_EXTRA_FILE} fi @@ -39,7 +40,7 @@ DETIK_CLIENT_NAME="kubectl -n $TEST_NAMESPACE" DETIK_CLIENT_NAMESPACE="${TEST_NAMESPACE}" -@test "test fluent-bit reads k8s labels" { +function deploy_fluent_bit() { helm repo add fluent https://fluent.github.io/helm-charts/ || helm repo add fluent https://fluent.github.io/helm-charts helm repo update --fail-on-repo-update-fail @@ -50,9 +51,27 @@ DETIK_CLIENT_NAMESPACE="${TEST_NAMESPACE}" --timeout "${HELM_FB_TIMEOUT:-5m0s}" \ --wait - try "at most 15 times every 2s " \ - "to find 1 pods named 'fluent-bit' " \ - "with 'status' being 'running'" + kubectl wait pods -n "$TEST_NAMESPACE" -l "app.kubernetes.io/name=fluent-bit" --for condition=Ready --timeout=60s + + # Now we wait until the nodeName is shown in the k8s spec, which surprisingly can take a second to update even after condition=Ready + attempts=0 + while true; do + run kubectl get pods -l "app.kubernetes.io/name=fluent-bit" -o jsonpath='{.items[0].spec}' + if [[ "$output" != *"nodeName"* ]]; then + if [ "$attempt" -lt 5 ]; then + attempt=$(( attempt + 1 )) + sleep 5 + else + fail "did not find fluent-bit nodeName after $attempt attempts" + fi + else + break + fi + done +} + +@test "test fluent-bit adds k8s labels to records" { + deploy_fluent_bit # The hello-world-1 container MUST be on the same node as the fluentbit worker, so we use a nodeSelector to specify the same node name run kubectl get pods -l "app.kubernetes.io/name=fluent-bit" -o jsonpath='{.items[0].spec.nodeName}' @@ -64,10 +83,8 @@ DETIK_CLIENT_NAMESPACE="${TEST_NAMESPACE}" --overrides="{\"apiVersion\":\"v1\",\"spec\":{\"nodeSelector\":{\"kubernetes.io/hostname\":\"$node_name\"}}}" \ --command -- sh -c 'while true; do echo "hello world"; sleep 1; done' - try "at most 15 times every 5s " \ - "to find 1 pods named 'hello-world-1' " \ - "with 'status' being 'Running'" - + kubectl wait pod/hello-world-1 -n "$TEST_NAMESPACE" --for condition=Ready --timeout=30s + # We are sleeping here specifically for the Fluent-Bit's tail input's # configured Refresh_Interval to have enough time to detect the new pod's log file # and to have processed part of it. diff --git a/tests/kubernetes-plugins/resources/fluentbit-basic.yaml b/tests/kubernetes-plugins/resources/fluentbit-basic.yaml index 2551318..55d5862 100644 --- a/tests/kubernetes-plugins/resources/fluentbit-basic.yaml +++ b/tests/kubernetes-plugins/resources/fluentbit-basic.yaml @@ -1,4 +1,5 @@ kind: Deployment +fullnameOverride: fluent-bit-k8s-test replicaCount: 1 rbac: create: true diff --git a/tests/opensearch/basic.bats b/tests/opensearch/basic.bats index b58b998..3d9e74c 100644 --- a/tests/opensearch/basic.bats +++ b/tests/opensearch/basic.bats @@ -27,6 +27,8 @@ function setup_file() { function teardown_file() { if [[ "${SKIP_TEARDOWN:-no}" != "yes" ]]; then + helm uninstall -n $TEST_NAMESPACE opensearch + helm uninstall -n $TEST_NAMESPACE fluent-bit run kubectl delete namespace "$TEST_NAMESPACE" rm -f ${HELM_VALUES_EXTRA_FILE} fi diff --git a/tests/opensearch/hosted.bats b/tests/opensearch/hosted.bats index 1322317..1ae4224 100644 --- a/tests/opensearch/hosted.bats +++ b/tests/opensearch/hosted.bats @@ -27,9 +27,12 @@ setup_file() { teardown_file() { if [[ "${SKIP_TEARDOWN:-no}" != "yes" ]]; then - run kubectl delete namespace "$TEST_NAMESPACE" - rm -f ${HELM_VALUES_EXTRA_FILE} - rm -f ${BATS_TEST_DIRNAME}/resources/helm/fluentbit-hosted.yaml + if [[ $HOSTED_OPENSEARCH_HOST != "localhost" ]]; then + helm uninstall -n $TEST_NAMESPACE fluent-bit + run kubectl delete namespace "$TEST_NAMESPACE" + rm -f ${HELM_VALUES_EXTRA_FILE} + rm -f ${BATS_TEST_DIRNAME}/resources/helm/fluentbit-hosted.yaml + fi fi } @@ -47,6 +50,11 @@ DETIK_CLIENT_NAMESPACE="${TEST_NAMESPACE}" @test "test fluent-bit forwards logs to AWS OpenSearch hosted service default index" { + if [[ $HOSTED_OPENSEARCH_HOST == "localhost" ]]; then + # then we aren't actually using a hosted opensearch + return + fi + envsubst < "${BATS_TEST_DIRNAME}/resources/helm/fluentbit-hosted.yaml.tpl" > "${BATS_TEST_DIRNAME}/resources/helm/fluentbit-hosted.yaml" helm upgrade --install --debug --create-namespace --namespace "$TEST_NAMESPACE" fluent-bit fluent/fluent-bit \