ensure tests run helm uninstall in teardown

Signed-off-by: ryanohnemus <[email protected]>
fluent · Jan 13, 2024 · 5f7bd35 · 5f7bd35
1 parent 4ae1006
commit 5f7bd35
Show file tree

Hide file tree

Showing 8 changed files with 51 additions and 20 deletions.
diff --git a/tests/chunk-rollover/basic.bats b/tests/chunk-rollover/basic.bats
@@ -27,6 +27,7 @@ setup_file() {
 
 teardown_file() {
     if [[ "${SKIP_TEARDOWN:-no}" != "yes" ]]; then
+        helm uninstall -n $TEST_NAMESPACE fluent-bit
         run kubectl delete namespace "$TEST_NAMESPACE"
         rm -f ${HELM_VALUES_EXTRA_FILE}
     fi
@@ -58,10 +59,9 @@ DETIK_CLIENT_NAMESPACE="${TEST_NAMESPACE}"
 
 
     # replace the namespace for svc FQDN
-    sed -i "s/payload-receiver-service.test.svc.cluster.local/payload-receiver-service.$TEST_NAMESPACE.svc.cluster.local/g" "${BATS_TEST_DIRNAME}/resources/helm/fluentbit-basic.yaml"
     helm upgrade --install --debug --create-namespace --namespace "$TEST_NAMESPACE" fluent-bit fluent/fluent-bit \
         --values ${BATS_TEST_DIRNAME}/resources/helm/fluentbit-basic.yaml \
-        --set image.repository=${FLUENTBIT_IMAGE_REPOSITORY},image.tag=${FLUENTBIT_IMAGE_TAG} \
+        --set image.repository=${FLUENTBIT_IMAGE_REPOSITORY},image.tag=${FLUENTBIT_IMAGE_TAG},env[0].name=TEST_NAMESPACE,env[0].value=${TEST_NAMESPACE} \
         --values "$HELM_VALUES_EXTRA_FILE" \
         --timeout "${HELM_FB_TIMEOUT:-5m0s}" \
         --wait
@@ -75,9 +75,8 @@ DETIK_CLIENT_NAMESPACE="${TEST_NAMESPACE}"
     fi
 
     # replace the namespace for crb
-    sed -i "s/namespace: test/namespace: $TEST_NAMESPACE/g" "${BATS_TEST_DIRNAME}/resources/manifests/rbac/cluster-role-binding.yaml"
-    kubectl create -f ${BATS_TEST_DIRNAME}/resources/manifests/rbac/cluster-role-binding.yaml --dry-run=client -o yaml | kubectl apply -f -
-
+    sed "s/namespace: test/namespace: $TEST_NAMESPACE/g" "${BATS_TEST_DIRNAME}/resources/manifests/rbac/cluster-role-binding.yaml" | kubectl apply -f -
+
     # Time interval in seconds to check the pods status
     INTERVAL=10
 

diff --git a/tests/chunk-rollover/resources/helm/fluentbit-basic.yaml b/tests/chunk-rollover/resources/helm/fluentbit-basic.yaml
@@ -48,7 +48,7 @@ config:
     [OUTPUT]
         name                      http
         match                     *
-        host                      payload-receiver-service.test.svc.cluster.local
+        host                      payload-receiver-service.${TEST_NAMESPACE}.svc.cluster.local
         port                      8088
         retry_limit               3
         storage.total_limit_size  300M
@@ -58,7 +58,7 @@ config:
     [OUTPUT]
         name                      http
         match                     *
-        host                      payload-receiver-service.test.svc.cluster.local
+        host                      payload-receiver-service.${TEST_NAMESPACE}.svc.cluster.local
         port                      8088
         retry_limit               3
         storage.total_limit_size  300M
@@ -68,7 +68,7 @@ config:
     [OUTPUT]
         name                      http
         match                     *
-        host                      payload-receiver-service.test.svc.cluster.local
+        host                      payload-receiver-service.${TEST_NAMESPACE}.svc.cluster.local
         port                      8088
         retry_limit               3
         storage.total_limit_size  300M
@@ -78,7 +78,7 @@ config:
     [OUTPUT]
         name                      http
         match                     *
-        host                      payload-receiver-service.test.svc.cluster.local
+        host                      payload-receiver-service.${TEST_NAMESPACE}.svc.cluster.local
         port                      8088
         retry_limit               3
         storage.total_limit_size  300M

diff --git a/tests/elasticsearch/basic.bats b/tests/elasticsearch/basic.bats
@@ -27,6 +27,8 @@ setup_file() {
 
 teardown_file() {
     if [[ "${SKIP_TEARDOWN:-no}" != "yes" ]]; then
+        helm uninstall -n $TEST_NAMESPACE elasticsearch
+        helm uninstall -n $TEST_NAMESPACE fluent-bit
         run kubectl delete namespace "$TEST_NAMESPACE"
         rm -f ${HELM_VALUES_EXTRA_FILE}
     fi

diff --git a/tests/elasticsearch/compress.bats b/tests/elasticsearch/compress.bats
@@ -27,6 +27,8 @@ setup_file() {
 
 teardown_file() {
     if [[ "${SKIP_TEARDOWN:-no}" != "yes" ]]; then
+        helm uninstall -n $TEST_NAMESPACE elasticsearch
+        helm uninstall -n $TEST_NAMESPACE fluent-bit
         run kubectl delete namespace "$TEST_NAMESPACE"
         rm -f ${HELM_VALUES_EXTRA_FILE}
     fi

diff --git a/tests/kubernetes-plugins/basic.bats b/tests/kubernetes-plugins/basic.bats
@@ -27,6 +27,7 @@ setup() {
 
 teardown() {
     if [[ "${SKIP_TEARDOWN:-no}" != "yes" ]]; then
+        helm uninstall fluent-bit -n $TEST_NAMESPACE
         run kubectl delete namespace "$TEST_NAMESPACE"
         rm -f ${HELM_VALUES_EXTRA_FILE}
     fi
@@ -39,7 +40,7 @@ DETIK_CLIENT_NAME="kubectl -n $TEST_NAMESPACE"
 DETIK_CLIENT_NAMESPACE="${TEST_NAMESPACE}"
 
 
-@test "test fluent-bit reads k8s labels" {
+function deploy_fluent_bit() {
     helm repo add fluent https://fluent.github.io/helm-charts/ || helm repo add fluent https://fluent.github.io/helm-charts
     helm repo update --fail-on-repo-update-fail
 
@@ -50,9 +51,27 @@ DETIK_CLIENT_NAMESPACE="${TEST_NAMESPACE}"
         --timeout "${HELM_FB_TIMEOUT:-5m0s}" \
         --wait
 
-    try "at most 15 times every 2s " \
-        "to find 1 pods named 'fluent-bit' " \
-        "with 'status' being 'running'"
+    kubectl wait pods -n "$TEST_NAMESPACE" -l "app.kubernetes.io/name=fluent-bit" --for condition=Ready --timeout=60s
+
+    # Now we wait until the nodeName is shown in the k8s spec, which surprisingly can take a second to update even after condition=Ready
+    attempts=0
+    while true; do
+    	run kubectl get pods -l "app.kubernetes.io/name=fluent-bit" -o jsonpath='{.items[0].spec}'
+        if [[ "$output" != *"nodeName"* ]]; then
+            if [ "$attempt" -lt 5 ]; then
+                attempt=$(( attempt + 1 ))
+                sleep 5
+            else
+                fail "did not find fluent-bit nodeName after $attempt attempts"
+            fi
+        else
+            break
+        fi
+    done
+}
+
+@test "test fluent-bit adds k8s labels to records" {
+    deploy_fluent_bit
 
     # The hello-world-1 container MUST be on the same node as the fluentbit worker, so we use a nodeSelector to specify the same node name
     run kubectl get pods -l "app.kubernetes.io/name=fluent-bit" -o jsonpath='{.items[0].spec.nodeName}'
@@ -64,10 +83,8 @@ DETIK_CLIENT_NAMESPACE="${TEST_NAMESPACE}"
         --overrides="{\"apiVersion\":\"v1\",\"spec\":{\"nodeSelector\":{\"kubernetes.io/hostname\":\"$node_name\"}}}" \
         --command -- sh -c 'while true; do echo "hello world"; sleep 1; done'
 
-    try "at most 15 times every 5s " \
-        "to find 1 pods named 'hello-world-1' " \
-        "with 'status' being 'Running'"
-
+    kubectl wait pod/hello-world-1 -n "$TEST_NAMESPACE" --for condition=Ready --timeout=30s
+
     # We are sleeping here specifically for the Fluent-Bit's tail input's
     # configured Refresh_Interval to have enough time to detect the new pod's log file
     # and to have processed part of it.

diff --git a/tests/kubernetes-plugins/resources/fluentbit-basic.yaml b/tests/kubernetes-plugins/resources/fluentbit-basic.yaml
@@ -1,4 +1,5 @@
 kind: Deployment
+fullnameOverride: fluent-bit-k8s-test
 replicaCount: 1
 rbac:
   create: true

diff --git a/tests/opensearch/basic.bats b/tests/opensearch/basic.bats
@@ -27,6 +27,8 @@ function setup_file() {
 
 function teardown_file() {
     if [[ "${SKIP_TEARDOWN:-no}" != "yes" ]]; then
+        helm uninstall -n $TEST_NAMESPACE opensearch
+        helm uninstall -n $TEST_NAMESPACE fluent-bit
         run kubectl delete namespace "$TEST_NAMESPACE"
         rm -f ${HELM_VALUES_EXTRA_FILE}
     fi

diff --git a/tests/opensearch/hosted.bats b/tests/opensearch/hosted.bats
@@ -27,9 +27,12 @@ setup_file() {
 
 teardown_file() {
     if [[ "${SKIP_TEARDOWN:-no}" != "yes" ]]; then
-        run kubectl delete namespace "$TEST_NAMESPACE"
-        rm -f ${HELM_VALUES_EXTRA_FILE}
-        rm -f ${BATS_TEST_DIRNAME}/resources/helm/fluentbit-hosted.yaml
+        if [[ $HOSTED_OPENSEARCH_HOST != "localhost" ]]; then
+            helm uninstall -n $TEST_NAMESPACE fluent-bit
+            run kubectl delete namespace "$TEST_NAMESPACE"
+            rm -f ${HELM_VALUES_EXTRA_FILE}
+            rm -f ${BATS_TEST_DIRNAME}/resources/helm/fluentbit-hosted.yaml
+        fi
     fi
 }
 
@@ -47,6 +50,11 @@ DETIK_CLIENT_NAMESPACE="${TEST_NAMESPACE}"
 
 
 @test "test fluent-bit forwards logs to AWS OpenSearch hosted service default index" {
+    if [[ $HOSTED_OPENSEARCH_HOST == "localhost" ]]; then
+        # then we aren't actually using a hosted opensearch
+        return
+    fi
+
     envsubst < "${BATS_TEST_DIRNAME}/resources/helm/fluentbit-hosted.yaml.tpl" > "${BATS_TEST_DIRNAME}/resources/helm/fluentbit-hosted.yaml"
 
     helm upgrade --install --debug --create-namespace --namespace "$TEST_NAMESPACE" fluent-bit fluent/fluent-bit \