diff --git a/.github/workflows/test_kubernetes.yml b/.github/workflows/test_kubernetes.yml index e341a3232..3987752b5 100644 --- a/.github/workflows/test_kubernetes.yml +++ b/.github/workflows/test_kubernetes.yml @@ -11,23 +11,6 @@ on: jobs: run-kubernets-tests: runs-on: ubuntu-latest - services: - # Label used to access the service container - postgres: - # Docker Hub image - image: postgres - # Provide the password for postgres - env: - POSTGRES_PASSWORD: postgres - # Set health checks to wait until postgres has started - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - # Maps tcp port 5432 on service container to the host - - 5432:5432 steps: - uses: actions/checkout@v3 - name: Kubernetes KinD Cluster @@ -38,5 +21,3 @@ jobs: pip install hatch hatch -e tests.py3.9-2.9 run pip freeze hatch run tests.py3.9-2.9:test-kubernetes - kubectl get pods -o wide - kubectl logs postgres-postgresql-0 diff --git a/dev/Dockerfile.postgres_profile_docker_k8s b/dev/Dockerfile.postgres_profile_docker_k8s index 9b4768a7a..c369f144b 100644 --- a/dev/Dockerfile.postgres_profile_docker_k8s +++ b/dev/Dockerfile.postgres_profile_docker_k8s @@ -3,7 +3,7 @@ FROM python:3.9 RUN pip install dbt-postgres==1.8.2 psycopg2==2.9.3 pytz ENV POSTGRES_DATABASE=postgres -ENV POSTGRES_HOST=postgres-postgresql.default.svc.cluster.local +ENV POSTGRES_HOST=127.0.0.1 ENV POSTGRES_PASSWORD= ENV POSTGRES_PORT=5432 ENV POSTGRES_SCHEMA=public diff --git a/dev/dags/dbt/jaffle_shop/profiles.yml b/dev/dags/dbt/jaffle_shop/profiles.yml index 43f0aae7d..b5fa5caa2 100644 --- a/dev/dags/dbt/jaffle_shop/profiles.yml +++ b/dev/dags/dbt/jaffle_shop/profiles.yml @@ -16,9 +16,9 @@ postgres_profile: outputs: dev: type: postgres - dbname: '{{ env_var(''POSTGRES_DATABASE'') }}' - host: '{{ env_var(''POSTGRES_HOST'') }}' - pass: '{{ env_var(''POSTGRES_PASSWORD'') }}' - port: '{{ env_var(''POSTGRES_PORT'') | as_number }}' - schema: '{{ env_var(''POSTGRES_SCHEMA'') }}' - user: '{{ env_var(''POSTGRES_USER'') }}' + dbname: postgres #'{{ env_var(''POSTGRES_DATABASE'') }}' + host: "0.0.0.0" #'{{ env_var(''POSTGRES_HOST'') }}' + pass: postgres #'{{ env_var(''POSTGRES_PASSWORD'') }}' + port: 5432 #'{{ env_var(''POSTGRES_PORT'') | as_number }}' + schema: postgres # '{{ env_var(''POSTGRES_SCHEMA'') }}' + user: "postgres" #'{{ env_var(''POSTGRES_USER'') }}' diff --git a/scripts/test/integration-kubernetes.sh b/scripts/test/integration-kubernetes.sh index 2ef58db99..de7b46675 100644 --- a/scripts/test/integration-kubernetes.sh +++ b/scripts/test/integration-kubernetes.sh @@ -3,6 +3,7 @@ set -x set -e +# export POSTGRES_DB=postgres airflow db reset -y diff --git a/scripts/test/kubernetes-setup.sh b/scripts/test/kubernetes-setup.sh index 989e6e402..a8727cea1 100644 --- a/scripts/test/kubernetes-setup.sh +++ b/scripts/test/kubernetes-setup.sh @@ -3,75 +3,85 @@ set -x set -e -check_nodes_ready() { - # Get the list of node statuses - node_statuses=$(kubectl get nodes --no-headers | awk '{print $2}') - # Check if all nodes are in the "Ready" state - for status in $node_statuses; do - if [ "$status" != "Ready" ]; then - return 1 - fi - done - return 0 -} - -wait_for_nodes_ready() { - local max_attempts=60 - local interval=5 - local attempt=0 - - echo "Waiting for nodes in the kind cluster to be in 'Ready' state..." +#check_nodes_ready() { +# # Get the list of node statuses +# node_statuses=$(kubectl get nodes --no-headers | awk '{print $2}') +# # Check if all nodes are in the "Ready" state +# for status in $node_statuses; do +# if [ "$status" != "Ready" ]; then +# return 1 +# fi +# done +# return 0 +#} +# +#wait_for_nodes_ready() { +# local max_attempts=60 +# local interval=5 +# local attempt=0 +# +# echo "Waiting for nodes in the kind cluster to be in 'Ready' state..." +# +# while [ $attempt -lt $max_attempts ]; do +# if check_nodes_ready; then +# echo "All nodes in the kind cluster are in 'Ready' state." +# return 0 +# else +# echo "Nodes are not yet ready. Checking again in $interval seconds..." +# sleep $interval +# attempt=$((attempt + 1)) +# fi +# done +# +# echo "Timeout waiting for nodes in the kind cluster to be in 'Ready' state." +# return 1 +#} +# +#kubectl config set-context default +# +## Create a docker image containing the dbt project files and dbt profile +#cd dev && docker build -t dbt-jaffle-shop:1.0.0 -f Dockerfile.postgres_profile_docker_k8s . +## Make the build image available in the Kind K8s cluster +#kind load docker-image dbt-jaffle-shop:1.0.0 +# +## Deploy a Postgres pod to Kind +##helm repo add bitnami https://charts.bitnami.com/bitnami +##helm repo update +##helm install postgres bitnami/postgresql --set postgresqlExtendedConf.huge_pages="off" # -f scripts/test/values.yaml +# +## Retrieve the Postgres password and set it as an environment variable +##POSTGRES_PASSWORD=$(kubectl get secret --namespace default postgres-postgresql -o jsonpath="{.data.postgres-password}" | base64 -d) +##export POSTGRES_PASSWORD +# +#kubectl create secret generic postgres-secrets --from-literal=host=postgres-postgresql.default.svc.cluster.local --from-literal=password=$POSTGRES_PASSWORD +# +#sleep 120 +## Expose the Postgres to the host running Docker/Kind +##kubectl port-forward --namespace default postgres-postgresql-0 5432:5432 & +##kubectl port-forward --namespace default svc/postgres-postgresql 5432:5432 & +##wait_for_nodes_ready +## +### Wait for the kind cluster to be in 'Ready' state +##wait_for_nodes_ready +# +## For Debugging +#echo "nodes" +#kubectl get nodes +#echo "helm" +#helm list +#echo "pod service" +#kubectl get pods --namespace default +#kubectl get svc --namespace default +#echo "pg log" +#kubectl logs postgres-postgresql-0 -c postgresql +#kubectl describe pod postgres-postgresql-0 - while [ $attempt -lt $max_attempts ]; do - if check_nodes_ready; then - echo "All nodes in the kind cluster are in 'Ready' state." - return 0 - else - echo "Nodes are not yet ready. Checking again in $interval seconds..." - sleep $interval - attempt=$((attempt + 1)) - fi - done - echo "Timeout waiting for nodes in the kind cluster to be in 'Ready' state." - return 1 -} +kubectl create secret generic postgres-secrets --from-literal=host=0.0.0.0 --from-literal=password=postgres -kubectl config set-context default +kubectl apply -f scripts/test/postgres-deployment.yaml -# Create a docker image containing the dbt project files and dbt profile cd dev && docker build -t dbt-jaffle-shop:1.0.0 -f Dockerfile.postgres_profile_docker_k8s . -# Make the build image available in the Kind K8s cluster kind load docker-image dbt-jaffle-shop:1.0.0 -# Deploy a Postgres pod to Kind -helm repo add bitnami https://charts.bitnami.com/bitnami -helm repo update -helm install postgres bitnami/postgresql --set postgresqlExtendedConf.huge_pages="off" # -f scripts/test/values.yaml - -# Retrieve the Postgres password and set it as an environment variable -POSTGRES_PASSWORD=$(kubectl get secret --namespace default postgres-postgresql -o jsonpath="{.data.postgres-password}" | base64 -d) -export POSTGRES_PASSWORD - -kubectl create secret generic postgres-secrets --from-literal=host=postgres-postgresql.default.svc.cluster.local --from-literal=password=$POSTGRES_PASSWORD - -sleep 120 -# Expose the Postgres to the host running Docker/Kind -#kubectl port-forward --namespace default postgres-postgresql-0 5432:5432 & -kubectl port-forward --namespace default svc/postgres-postgresql 5432:5432 & -#wait_for_nodes_ready -# -## Wait for the kind cluster to be in 'Ready' state -#wait_for_nodes_ready - -# For Debugging -echo "nodes" -kubectl get nodes -echo "helm" -helm list -echo "pod service" -kubectl get pods --namespace default -kubectl get svc --namespace default -echo "pg log" -kubectl logs postgres-postgresql-0 -c postgresql -kubectl describe pod postgres-postgresql-0 +kubectl get pod \ No newline at end of file diff --git a/scripts/test/postgres-deployment.yaml b/scripts/test/postgres-deployment.yaml index 05aa006f4..473439cff 100644 --- a/scripts/test/postgres-deployment.yaml +++ b/scripts/test/postgres-deployment.yaml @@ -2,6 +2,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: postgres + namespace: default spec: replicas: 1 selector: @@ -13,14 +14,44 @@ spec: app: postgres spec: containers: - - name: postgres - image: bitnami/postgres:latest # Replace with desired Bitnami Postgres image + image: bitnami/postgresql:latest env: + - name: POSTGRES_DB + value: postgres + - name: POSTGRES_HOST + valueFrom: + secretKeyRef: + name: postgres-secrets + key: host # Adjust according to your secret's key - name: POSTGRES_PASSWORD valueFrom: secretKeyRef: - name: postgres-postgresql - key: postgres-password + name: postgres-secrets + key: password ports: - containerPort: 5432 + +--- +apiVersion: v1 +kind: Secret +metadata: + name: postgres-secrets +type: Opaque +data: + host: MC4wLjAuMA== + password: cG9zdGdyZXM= + +--- +apiVersion: v1 +kind: Service +metadata: + name: postgres + namespace: default # Ensure this matches the namespace of the Deployment +spec: + type: LoadBalancer + ports: + - port: 5432 + targetPort: 5432 + selector: + app: postgres