backup

astronomer · Aug 12, 2024 · ffacc91 · ffacc91
1 parent 6e12308
commit ffacc91
Show file tree

Hide file tree

Showing 6 changed files with 118 additions and 95 deletions.
diff --git a/.github/workflows/test_kubernetes.yml b/.github/workflows/test_kubernetes.yml
@@ -11,23 +11,6 @@ on:
 jobs:
   run-kubernets-tests:
     runs-on: ubuntu-latest
-    services:
-      # Label used to access the service container
-      postgres:
-        # Docker Hub image
-        image: postgres
-        # Provide the password for postgres
-        env:
-          POSTGRES_PASSWORD: postgres
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          # Maps tcp port 5432 on service container to the host
-          - 5432:5432
     steps:
       - uses: actions/checkout@v3
       - name: Kubernetes KinD Cluster
@@ -38,5 +21,3 @@ jobs:
           pip install hatch
           hatch -e tests.py3.9-2.9 run pip freeze
           hatch run tests.py3.9-2.9:test-kubernetes
-          kubectl get pods -o wide
-          kubectl logs postgres-postgresql-0
diff --git a/dev/Dockerfile.postgres_profile_docker_k8s b/dev/Dockerfile.postgres_profile_docker_k8s
@@ -3,7 +3,7 @@ FROM python:3.9
 RUN pip install dbt-postgres==1.8.2 psycopg2==2.9.3 pytz
 
 ENV POSTGRES_DATABASE=postgres
-ENV POSTGRES_HOST=postgres-postgresql.default.svc.cluster.local
+ENV POSTGRES_HOST=127.0.0.1
 ENV POSTGRES_PASSWORD=<postgres_password>
 ENV POSTGRES_PORT=5432
 ENV POSTGRES_SCHEMA=public

diff --git a/dev/dags/dbt/jaffle_shop/profiles.yml b/dev/dags/dbt/jaffle_shop/profiles.yml
@@ -16,9 +16,9 @@ postgres_profile:
   outputs:
     dev:
       type: postgres
-      dbname: '{{ env_var(''POSTGRES_DATABASE'') }}'
-      host: '{{ env_var(''POSTGRES_HOST'') }}'
-      pass: '{{ env_var(''POSTGRES_PASSWORD'') }}'
-      port: '{{ env_var(''POSTGRES_PORT'') | as_number }}'
-      schema: '{{ env_var(''POSTGRES_SCHEMA'') }}'
-      user: '{{ env_var(''POSTGRES_USER'') }}'
+      dbname: postgres #'{{ env_var(''POSTGRES_DATABASE'') }}'
+      host: "0.0.0.0" #'{{ env_var(''POSTGRES_HOST'') }}'
+      pass: postgres #'{{ env_var(''POSTGRES_PASSWORD'') }}'
+      port: 5432 #'{{ env_var(''POSTGRES_PORT'') | as_number }}'
+      schema: postgres # '{{ env_var(''POSTGRES_SCHEMA'') }}'
+      user: "postgres" #'{{ env_var(''POSTGRES_USER'') }}'
diff --git a/scripts/test/integration-kubernetes.sh b/scripts/test/integration-kubernetes.sh
@@ -3,6 +3,7 @@
 set -x
 set -e
 
+# export POSTGRES_DB=postgres
 
 airflow db reset -y
 

diff --git a/scripts/test/kubernetes-setup.sh b/scripts/test/kubernetes-setup.sh
@@ -3,75 +3,85 @@
 set -x
 set -e
 
-check_nodes_ready() {
-    # Get the list of node statuses
-    node_statuses=$(kubectl get nodes --no-headers | awk '{print $2}')
-    # Check if all nodes are in the "Ready" state
-    for status in $node_statuses; do
-        if [ "$status" != "Ready" ]; then
-            return 1
-        fi
-    done
-    return 0
-}
-
-wait_for_nodes_ready() {
-    local max_attempts=60
-    local interval=5
-    local attempt=0
-
-    echo "Waiting for nodes in the kind cluster to be in 'Ready' state..."
+#check_nodes_ready() {
+#    # Get the list of node statuses
+#    node_statuses=$(kubectl get nodes --no-headers | awk '{print $2}')
+#    # Check if all nodes are in the "Ready" state
+#    for status in $node_statuses; do
+#        if [ "$status" != "Ready" ]; then
+#            return 1
+#        fi
+#    done
+#    return 0
+#}
+#
+#wait_for_nodes_ready() {
+#    local max_attempts=60
+#    local interval=5
+#    local attempt=0
+#
+#    echo "Waiting for nodes in the kind cluster to be in 'Ready' state..."
+#
+#    while [ $attempt -lt $max_attempts ]; do
+#        if check_nodes_ready; then
+#            echo "All nodes in the kind cluster are in 'Ready' state."
+#            return 0
+#        else
+#            echo "Nodes are not yet ready. Checking again in $interval seconds..."
+#            sleep $interval
+#            attempt=$((attempt + 1))
+#        fi
+#    done
+#
+#    echo "Timeout waiting for nodes in the kind cluster to be in 'Ready' state."
+#    return 1
+#}
+#
+#kubectl config set-context default
+#
+## Create a docker image containing the dbt project files and dbt profile
+#cd dev && docker build -t dbt-jaffle-shop:1.0.0 -f Dockerfile.postgres_profile_docker_k8s .
+## Make the build image available in the Kind K8s cluster
+#kind load docker-image dbt-jaffle-shop:1.0.0
+#
+## Deploy a Postgres pod to Kind
+##helm repo add bitnami https://charts.bitnami.com/bitnami
+##helm repo update
+##helm install postgres bitnami/postgresql --set postgresqlExtendedConf.huge_pages="off" # -f scripts/test/values.yaml
+#
+## Retrieve the Postgres password and set it as an environment variable
+##POSTGRES_PASSWORD=$(kubectl get secret --namespace default postgres-postgresql -o jsonpath="{.data.postgres-password}" | base64 -d)
+##export POSTGRES_PASSWORD
+#
+#kubectl create secret generic postgres-secrets --from-literal=host=postgres-postgresql.default.svc.cluster.local --from-literal=password=$POSTGRES_PASSWORD
+#
+#sleep 120
+## Expose the Postgres to the host running Docker/Kind
+##kubectl port-forward --namespace default postgres-postgresql-0  5432:5432 &
+##kubectl port-forward --namespace default svc/postgres-postgresql 5432:5432 &
+##wait_for_nodes_ready
+##
+### Wait for the kind cluster to be in 'Ready' state
+##wait_for_nodes_ready
+#
+## For Debugging
+#echo "nodes"
+#kubectl get nodes
+#echo "helm"
+#helm list
+#echo "pod service"
+#kubectl get pods --namespace default
+#kubectl get svc --namespace default
+#echo "pg log"
+#kubectl logs postgres-postgresql-0 -c postgresql
+#kubectl describe pod postgres-postgresql-0
 
-    while [ $attempt -lt $max_attempts ]; do
-        if check_nodes_ready; then
-            echo "All nodes in the kind cluster are in 'Ready' state."
-            return 0
-        else
-            echo "Nodes are not yet ready. Checking again in $interval seconds..."
-            sleep $interval
-            attempt=$((attempt + 1))
-        fi
-    done
 
-    echo "Timeout waiting for nodes in the kind cluster to be in 'Ready' state."
-    return 1
-}
+kubectl create secret generic postgres-secrets --from-literal=host=0.0.0.0 --from-literal=password=postgres
 
-kubectl config set-context default
+kubectl apply -f scripts/test/postgres-deployment.yaml
 
-# Create a docker image containing the dbt project files and dbt profile
 cd dev && docker build -t dbt-jaffle-shop:1.0.0 -f Dockerfile.postgres_profile_docker_k8s .
-# Make the build image available in the Kind K8s cluster
 kind load docker-image dbt-jaffle-shop:1.0.0
 
-# Deploy a Postgres pod to Kind
-helm repo add bitnami https://charts.bitnami.com/bitnami
-helm repo update
-helm install postgres bitnami/postgresql --set postgresqlExtendedConf.huge_pages="off" # -f scripts/test/values.yaml
-
-# Retrieve the Postgres password and set it as an environment variable
-POSTGRES_PASSWORD=$(kubectl get secret --namespace default postgres-postgresql -o jsonpath="{.data.postgres-password}" | base64 -d)
-export POSTGRES_PASSWORD
-
-kubectl create secret generic postgres-secrets --from-literal=host=postgres-postgresql.default.svc.cluster.local --from-literal=password=$POSTGRES_PASSWORD
-
-sleep 120
-# Expose the Postgres to the host running Docker/Kind
-#kubectl port-forward --namespace default postgres-postgresql-0  5432:5432 &
-kubectl port-forward --namespace default svc/postgres-postgresql 5432:5432 &
-#wait_for_nodes_ready
-#
-## Wait for the kind cluster to be in 'Ready' state
-#wait_for_nodes_ready
-
-# For Debugging
-echo "nodes"
-kubectl get nodes
-echo "helm"
-helm list
-echo "pod service"
-kubectl get pods --namespace default
-kubectl get svc --namespace default
-echo "pg log"
-kubectl logs postgres-postgresql-0 -c postgresql
-kubectl describe pod postgres-postgresql-0
+kubectl get pod
diff --git a/scripts/test/postgres-deployment.yaml b/scripts/test/postgres-deployment.yaml
@@ -2,6 +2,7 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: postgres
+  namespace: default
 spec:
   replicas: 1
   selector:
@@ -13,14 +14,44 @@ spec:
         app: postgres
     spec:
       containers:
-
       - name: postgres
-        image: bitnami/postgres:latest  # Replace with desired Bitnami Postgres image
+        image: bitnami/postgresql:latest
         env:
+        - name: POSTGRES_DB
+          value: postgres
+        - name: POSTGRES_HOST
+          valueFrom:
+            secretKeyRef:
+              name: postgres-secrets
+              key: host  # Adjust according to your secret's key
         - name: POSTGRES_PASSWORD
           valueFrom:
             secretKeyRef:
-              name: postgres-postgresql
-              key: postgres-password
+              name: postgres-secrets
+              key: password
         ports:
         - containerPort: 5432
+
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: postgres-secrets
+type: Opaque
+data:
+  host: MC4wLjAuMA==
+  password: cG9zdGdyZXM=
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: postgres
+  namespace: default  # Ensure this matches the namespace of the Deployment
+spec:
+  type: LoadBalancer
+  ports:
+  - port: 5432
+    targetPort: 5432
+  selector:
+    app: postgres