Add support for Google Cloud

.github/workflows/conductor.yaml

@@ -32,7 +32,7 @@ jobs:
       fail-fast: false
       matrix:
         kube_version:
-          - "1.25.8"
+          - "1.29.8"
     steps:
       - uses: actions/checkout@v4
       - name: Install system dependencies

.github/workflows/operator.yaml

@@ -16,14 +16,14 @@ on:
     branches:
       - main
     paths:
-    - '.github/workflows/operator.yaml'
-    - 'tembo-operator/**'
+      - ".github/workflows/operator.yaml"
+      - "tembo-operator/**"
   push:
     branches:
       - main
     paths:
-    - '.github/workflows/operator.yaml'
-    - 'tembo-operator/**'
+      - ".github/workflows/operator.yaml"
+      - "tembo-operator/**"
 
 jobs:
   functional_test:
@@ -39,7 +39,7 @@ jobs:
         # Go here for a list of versions:
         # https://github.com/kubernetes-sigs/kind/releases
         kube_version:
-          - '1.25.8'
+          - "1.29.8"
     steps:
       - uses: actions/checkout@v4
       - name: Install system dependencies

charts/tembo-operator/Chart.lock

@@ -1,6 +1,6 @@
 dependencies:
 - name: cloudnative-pg
   repository: https://cloudnative-pg.github.io/charts
-  version: 0.20.1
-digest: sha256:8b7ed89dc3d149784f369ed4035d79268e9348f232b5cbebd5096c2d29e9ded7
-generated: "2024-02-12T14:57:18.051558882-06:00"
+  version: 0.21.6
+digest: sha256:3922d990e9dec07c6dda1f7b8799e9cfd2ef28450357f5a3f260a3d4773e5db2
+generated: "2024-09-04T09:47:10.610286988-05:00"

charts/tembo-operator/Chart.yaml

@@ -3,7 +3,7 @@ name: tembo-operator
 description: "Helm chart to deploy the tembo-operator"
 type: application
 icon: https://cloud.tembo.io/images/TemboElephant.png
-version: 0.7.2
+version: 0.7.3
 home: https://tembo.io
 sources:
   - https://github.com/tembo-io/tembo
@@ -17,6 +17,6 @@ maintainers:
     url: https://tembocommunity.slack.com
 dependencies:
   - name: cloudnative-pg
-    version: 0.20.1
+    version: 0.21.6
     repository: https://cloudnative-pg.github.io/charts
     condition: cloudnative-pg.enabled

charts/tembo-operator/templates/crd.yaml

@@ -40,7 +40,7 @@ spec:
                 description: |-
                   A AffinityConfiguration provides a way to configure the CoreDB instance to run on specific nodes in the cluster based off of nodeSelector, nodeAffinity and tolerations
 
-                  For more informaton on AffinityConfiguration please see the [Cloudnative-PG documentation](https://cloudnative-pg.io/documentation/1.22/cloudnative-pg.v1/#postgresql-cnpg-io-v1-AffinityConfiguration)
+                  For more information on AffinityConfiguration please see the [Cloudnative-PG documentation](https://cloudnative-pg.io/documentation/1.22/cloudnative-pg.v1/#postgresql-cnpg-io-v1-AffinityConfiguration)
 
                   **Default**: ```yaml apiVersion: coredb.io/v1alpha1 kind: CoreDB metadata: name: test-db-restore spec: affinityConfiguration: podAntiAffinityType: preferred topologyKey: topology.kubernetes.io/zone ```
                 nullable: true
@@ -1924,20 +1924,11 @@ spec:
                 nullable: true
                 type: array
               backup:
-                default:
-                  destinationPath: s3://
-                  encryption: AES256
-                  retentionPolicy: '30'
-                  schedule: 0 0 * * *
-                  endpointURL: null
-                  s3Credentials:
-                    inheritFromIAMRole: true
-                  volumeSnapshot:
-                    enabled: false
                 description: |-
                   The backup configuration for the CoreDB instance to facilitate database backups and WAL archive uploads to an S3 compatible object store.
 
                   **Default**: disabled
+                nullable: true
                 properties:
                   destinationPath:
                     default: s3://
@@ -1950,17 +1941,38 @@ spec:
                     nullable: true
                     type: string
                   endpointURL:
-                    description: The S3 compatable endpoint URL
+                    description: The S3 compatible endpoint URL
                     nullable: true
                     type: string
+                  googleCredentials:
+                    default: {}
+                    description: The Google Cloud Storage credentials to use for backups
+                    nullable: true
+                    properties:
+                      applicationCredentials:
+                        description: The reference to the secret containing the Google Cloud Storage JSON file with the credentials
+                        nullable: true
+                        properties:
+                          key:
+                            type: string
+                          name:
+                            type: string
+                        required:
+                        - key
+                        - name
+                        type: object
+                      gkeEnvironment:
+                        description: Use the role based authentication without providing explicitly the keys.
+                        nullable: true
+                        type: boolean
+                    type: object
                   retentionPolicy:
                     default: '30'
                     description: The number of days to retain backups for
                     nullable: true
                     type: string
                   s3Credentials:
-                    default:
-                      inheritFromIAMRole: true
+                    default: {}
                     description: The S3 credentials to use for backups (if not using IAM Role)
                     nullable: true
                     properties:
@@ -2174,7 +2186,7 @@ spec:
               extensions:
                 default: []
                 description: |-
-                  A list of extensions to enable on the CoreDB instance. This list should be a lits of extension names that are already available on the Postgres instance you are running.  To install extensions at runtime please see the `trunk_installs` field.
+                  A list of extensions to enable on the CoreDB instance. This list should be a list of extension names that are already available on the Postgres instance you are running.  To install extensions at runtime please see the `trunk_installs` field.
 
                   **Default**: []
                 items:
@@ -2377,12 +2389,35 @@ spec:
                       The object storage path and bucket name of the instance you wish to restore from.  This maps to the `Backup` `destinationPath` field for the original instance.
 
                       **Example**: If you have an instance with `spec.backup.destinationPath` set to `s3://my-bucket/v2/test-db` then you would set `backupsPath` to `s3://my-bucket/v2/test-db`. And backups are saved in that bucket under `s3://my-bucket/v2/test-db/server_name`
+
+                      For GCS Buckets, the path should be in the format `gs://my-bucket/v2/test-db`
                     nullable: true
                     type: string
                   endpointURL:
-                    description: endpointURL is the S3 compatable endpoint URL
+                    description: endpointURL is the S3 compatible endpoint URL
                     nullable: true
                     type: string
+                  googleCredentials:
+                    description: googleCredentials is the Google Cloud Storage credentials to use for backups.
+                    nullable: true
+                    properties:
+                      applicationCredentials:
+                        description: The reference to the secret containing the Google Cloud Storage JSON file with the credentials
+                        nullable: true
+                        properties:
+                          key:
+                            type: string
+                          name:
+                            type: string
+                        required:
+                        - key
+                        - name
+                        type: object
+                      gkeEnvironment:
+                        description: Use the role based authentication without providing explicitly the keys.
+                        nullable: true
+                        type: boolean
+                    type: object
                   recoveryTargetTime:
                     description: recovery_target_time is the time base target for point-in-time recovery.
                     nullable: true
@@ -2690,7 +2725,7 @@ spec:
                 description: |-
                   The topologySpreadConstraints provides a way to spread matching pods among the given topology
 
-                  For more information see the Kubernetes documentation on [Topology Spread Constraints](https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/) Tembo is compatable with the `v1` version of the TopologySpreadConstraints up to [Kubernetes 1.25](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.25/#topologyspreadconstraint-v1-core)
+                  For more information see the Kubernetes documentation on [Topology Spread Constraints](https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/) Tembo is compatible with the `v1` version of the TopologySpreadConstraints up to [Kubernetes 1.25](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.25/#topologyspreadconstraint-v1-core)
 
                   **Default**: `None`
                 items:

conductor/justfile

@@ -2,17 +2,15 @@ NAME := "conductor"
 VERSION := `git rev-parse HEAD`
 SEMVER_VERSION := `grep version Cargo.toml | awk -F"\"" '{print $2}' | head -n 1`
 NAMESPACE := "default"
-KUBE_VERSION := env_var_or_default('KUBE_VERSION', '1.25.8')
+KUBE_VERSION := env_var_or_default('KUBE_VERSION', '1.29.8')
 RUST_LOG := "info"
 
 default:
   @just --list --unsorted --color=always | rg -v "    default"
 
 install-traefik:
 	kubectl create namespace traefik || true
-	helm upgrade --install --namespace=traefik --version=20.8.0 --values=./testdata/traefik-values.yaml traefik traefik/traefik
-	# https://github.com/traefik/traefik-helm-chart/issues/757#issuecomment-1753995542
-	kubectl apply -f https://raw.githubusercontent.com/traefik/traefik/v3.0.0-beta2/docs/content/reference/dynamic-configuration/kubernetes-crd-definition-v1.yml
+	helm upgrade --install --namespace=traefik --version=29.0.1 --values=./testdata/traefik-values.yaml traefik traefik/traefik
 
 install-operator:
   just install-cert-manager
@@ -70,7 +68,7 @@ fmt:
 	cargo clippy --fix
 	cargo +nightly fmt
 
-watch:
+watch: run-operator
 	POSTGRES_QUEUE_CONNECTION=postgresql://postgres:[email protected]:5431/postgres \
 	RUST_BACKTRACE=1 \
 	RUST_LOG={{RUST_LOG}} \

conductor/src/main.rs

@@ -771,7 +771,7 @@ async fn init_cloud_perms(
         ..Default::default()
     };
 
-    coredb_spec.backup = backup;
+    coredb_spec.backup = Some(backup);
     coredb_spec.serviceAccountTemplate = service_account_template;
 
     Ok(())

conductor/testdata/traefik-values.yaml

@@ -1,5 +1,7 @@
 image:
-  tag: v3.0.0-beta2
+  tag: v3.0.3-tembo.1
+  registry: quay.io/tembo
+  repository: traefik
 logs:
   general:
     level: DEBUG
@@ -15,22 +17,38 @@ additionalArguments:
   - "--api.debug=true"
 ports:
   postgresql:
-    expose: true
+    expose:
+      default: true
     port: 5432
     exposedPort: 5432
     nodePort: 32432
     protocol: TCP
-  web:
-    expose: true
-    port: 8080
-    exposedPort: 8080
-    nodePort: 32430
+  # web:
+  #   expose: true
+  #   port: 8080
+  #   exposedPort: 8080
+  #   nodePort: 32430
+  #   protocol: TCP
+  websecure:
+    expose:
+      default: true
+    port: 8443
+    exposedPort: 8443
+    nodePort: 32443
     protocol: TCP
   traefik:
-    expose: true
+    expose:
+      default: true
     port: 9000
     exposedPort: 9000
     nodePort: 32431
     protocol: TCP
 deployment:
   replicas: 1
+resources:
+  requests:
+    cpu: "200m"
+    memory: "100Mi"
+  limits:
+    cpu: "400m"
+    memory: "300Mi"

conductor/tests/integration_tests.rs

@@ -313,12 +313,19 @@ mod test {
         let new_backup_spec = update_coredb.spec.backup.clone();
 
         // assert that the backup.schedule for old_backup_spec are equal to new_backup_spec
-        assert_eq!(old_backup_spec.schedule, new_backup_spec.schedule);
+        assert_eq!(
+            old_backup_spec.as_ref().and_then(|b| b.schedule.clone()),
+            new_backup_spec.as_ref().and_then(|b| b.schedule.clone())
+        );
 
         // assert that the destination paths for old_backup_spec are equal to new_backup_spec
         assert_eq!(
-            old_backup_spec.destinationPath,
-            new_backup_spec.destinationPath
+            old_backup_spec
+                .as_ref()
+                .and_then(|b| b.destinationPath.clone()),
+            new_backup_spec
+                .as_ref()
+                .and_then(|b| b.destinationPath.clone())
         );
 
         // Lets now test sending an Event::Restart to the queue and see if the