fix: StatefulHA changes added (#1049)

* statefulha shanges added * Missing license header * remove etag from the dashboard * header license year update to 2024 * fix the dashboard * Readme, ci and deployment yamls edited * Cleanup README.md --------- Co-authored-by: Dmitrii Ganochenko <[email protected]>
GoogleCloudPlatform · Jan 12, 2024 · 2db26ab · 2db26ab
1 parent 6309f87
commit 2db26ab
Show file tree

Hide file tree

Showing 30 changed files with 142 additions and 37 deletions.
diff --git a/.github/workflows/databases-qdrant-ci.yml b/.github/workflows/databases-qdrant-ci.yml
@@ -0,0 +1,41 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: databases-qdrant.yml
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - '.github/workflows/databases-qdrant.yml'
+      - 'databases/qdrant/**'
+  pull_request:
+    paths:
+      - '.github/workflows/databases-qdrant.yml'
+      - 'databases/qdrant/**'
+jobs:
+  job:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+      - name: Validate GKE Standard TF for Qdrant
+        run: |
+          cd databases/qdrant/terraform/gke-standard 
+          terraform init
+          terraform validate
+      - name: Validate GKE Autopilot TF for CQdrant
+        run: |
+          cd databases/qdrant/terraform/gke-autopilot
+          terraform init
+          terraform validate
diff --git a/databases/qdrant/README.md b/databases/qdrant/README.md
@@ -0,0 +1,5 @@
+# Deploying a Qdrant vector database and StatefullHA operator on GKE
+
+[![Open in Cloud Shell](https://gstatic.com/cloudssh/images/open-btn.svg)](https://ssh.cloud.google.com/cloudshell/editor?cloudshell_git_repo=https://github.com/GoogleCloudPlatform/kubernetes-engine-samples&cloudshell_tutorial=cloudshell/tutorial.md&cloudshell_workspace=gke-qdrant)
+
+This example shows how to build and deploy a Qdrant vector database using [Kubernetes](https://kubernetes.io) and Day 1 & 2 operations such as Monitoring with [Cloud Monitoring](https://cloud.google.com/monitoring) and disaster recovery with [Backup for GKE](https://cloud.google.com/kubernetes-engine/docs/add-on/backup-for-gke/concepts/backup-for-gke).
diff --git a/databases/qdrant/manifests/01-regional-pd/ha-app.yaml b/databases/qdrant/manifests/01-regional-pd/ha-app.yaml
@@ -0,0 +1,36 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# [START gke_databases_qdrant_01_regional_pd_ha_app]
+kind: HighAvailabilityApplication
+apiVersion: ha.gke.io/v1alpha1
+metadata:
+  name: qdrant-ha-app
+  namespace: qdrant
+spec:
+  resourceSelection:
+    type: "Selector"
+    resourceKind: StatefulSet
+    selector:
+      matchLabels:
+        app.kubernetes.io/name: qdrant
+
+  policy:
+    storageSettings:
+      requireRegionalStorage: true
+    failoverSettings:
+      forceDeleteStrategy: AfterNodeUnreachable
+      afterNodeUnreachable:
+        afterNodeUnreachableSeconds: 20 # 60 seconds total
+# [END gke_databases_qdrant_01_regional_pd_ha_app]
diff --git a/databases/qdrant/manifests/01-regional-pd/regional-pd.yaml b/databases/qdrant/manifests/01-regional-pd/regional-pd.yaml
@@ -1,4 +1,4 @@
-# Copyright 2023 Google LLC
+# Copyright 2024 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,15 +12,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# [START gke_databases_qdrant_01_regional_pd_storage]
 apiVersion: storage.k8s.io/v1
 kind: StorageClass
 allowVolumeExpansion: true
 metadata:
-  name: regional-pd
+  name: ha-regional
 parameters:
-  type: pd-ssd
   replication-type: regional-pd
+  type: pd-ssd
+  availability-class: regional-hard-failover
 provisioner: pd.csi.storage.gke.io
 reclaimPolicy: Retain
 volumeBindingMode: WaitForFirstConsumer
-
+# [END gke_databases_qdrant_01_regional_pd_storage]
diff --git a/databases/qdrant/manifests/02-values-file/values.yaml b/databases/qdrant/manifests/02-values-file/values.yaml
@@ -1,4 +1,4 @@
-# Copyright 2023 Google LLC
+# Copyright 2024 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# [START gke_databases_qdrant_02_values_file_config]
 replicaCount: 3
 
 config:
@@ -63,7 +64,7 @@ podDisruptionBudget:
 persistence:
   accessModes: ["ReadWriteOnce"]
   size: 10Gi
-  storageClassName: regional-pd
+  storageClassName: ha-regional
 
 apiKey: true
-
+# [END gke_databases_qdrant_02_values_file_config]
diff --git a/databases/qdrant/manifests/03-prometheus-metrics/metrics-proxy.yaml b/databases/qdrant/manifests/03-prometheus-metrics/metrics-proxy.yaml
@@ -1,4 +1,4 @@
-# Copyright 2023 Google LLC
+# Copyright 2024 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
----
+# [START gke_databases_qdrant_03_prometheus_metrics_proxy]
 apiVersion: v1
 kind: ConfigMap
 metadata:
@@ -76,3 +76,4 @@ spec:
           items:
             - key: default.conf.template
               path: default.conf.template
+# [END gke_databases_qdrant_03_prometheus_metrics_proxy]
diff --git a/databases/qdrant/manifests/03-prometheus-metrics/pod-monitoring.yaml b/databases/qdrant/manifests/03-prometheus-metrics/pod-monitoring.yaml
@@ -1,4 +1,4 @@
-# Copyright 2023 Google LLC
+# Copyright 2024 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# [START gke_databases_qdrant_03_prometheus_metrics_monitor]
 apiVersion: monitoring.googleapis.com/v1
 kind: PodMonitoring
 metadata:
@@ -25,3 +26,4 @@ spec:
   - port: 80
     interval: 30s
     path: / 
+# [END gke_databases_qdrant_03_prometheus_metrics_monitor]
diff --git a/databases/qdrant/manifests/04-qdrant-client/client-pod.yaml b/databases/qdrant/manifests/04-qdrant-client/client-pod.yaml
@@ -1,4 +1,4 @@
-# Copyright 2023 Google LLC
+# Copyright 2024 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# [START gke_databases_qdrant_04_client_pod]
 apiVersion: v1
 kind: Pod
 metadata:
@@ -37,4 +38,4 @@ spec:
   - name: books-dataset
     configMap:
       name: books-dataset
-
+ # [END gke_databases_qdrant_04_client_pod]
diff --git a/databases/qdrant/monitoring/dashboard.json b/databases/qdrant/monitoring/dashboard.json
@@ -9,7 +9,6 @@
     }
   ],
   "displayName": "Qdrant Overview",
-  "etag": "ae937deb98835d06d54420de51422a98",
   "labels": {},
   "mosaicLayout": {
     "columns": 48,
@@ -131,6 +130,5 @@
         "yPos": 16
       }
     ]
-  },
-  "name": "projects/233959860574/dashboards/afd5daa6-94cd-49d0-9d73-aa08baab8c5b"
-}
+  }
+}
diff --git a/databases/qdrant/terraform/cloud-storage/main.tf b/databases/qdrant/terraform/cloud-storage/main.tf
@@ -1,4 +1,4 @@
-# Copyright 2023 Google LLC
+# Copyright 2024 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

diff --git a/databases/qdrant/terraform/cloud-storage/variables.tf b/databases/qdrant/terraform/cloud-storage/variables.tf
@@ -1,4 +1,4 @@
-# Copyright 2023 Google LLC
+# Copyright 2024 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

diff --git a/databases/qdrant/terraform/cloud-storage/versions.tf b/databases/qdrant/terraform/cloud-storage/versions.tf
@@ -1,4 +1,4 @@
-# Copyright 2023 Google LLC
+# Copyright 2024 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

diff --git a/databases/qdrant/terraform/gke-autopilot/main.tf b/databases/qdrant/terraform/gke-autopilot/main.tf
@@ -1,4 +1,4 @@
-# Copyright 2023 Google LLC
+# Copyright 2024 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

diff --git a/databases/qdrant/terraform/gke-autopilot/variables.tf b/databases/qdrant/terraform/gke-autopilot/variables.tf
@@ -1,4 +1,4 @@
-# Copyright 2023 Google LLC
+# Copyright 2024 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

diff --git a/databases/qdrant/terraform/gke-autopilot/versions.tf b/databases/qdrant/terraform/gke-autopilot/versions.tf
@@ -1,4 +1,4 @@
-# Copyright 2023 Google LLC
+# Copyright 2024 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

diff --git a/databases/qdrant/terraform/gke-standard/main.tf b/databases/qdrant/terraform/gke-standard/main.tf
@@ -1,4 +1,4 @@
-# Copyright 2023 Google LLC
+# Copyright 2024 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

diff --git a/databases/qdrant/terraform/gke-standard/variables.tf b/databases/qdrant/terraform/gke-standard/variables.tf
@@ -1,4 +1,4 @@
-# Copyright 2023 Google LLC
+# Copyright 2024 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

diff --git a/databases/qdrant/terraform/gke-standard/versions.tf b/databases/qdrant/terraform/gke-standard/versions.tf
@@ -1,4 +1,4 @@
-# Copyright 2023 Google LLC
+# Copyright 2024 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

diff --git a/databases/qdrant/terraform/modules/cloud-storage/main.tf b/databases/qdrant/terraform/modules/cloud-storage/main.tf
@@ -1,4 +1,4 @@
-# Copyright 2023 Google LLC
+# Copyright 2024 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

diff --git a/databases/qdrant/terraform/modules/cloud-storage/variables.tf b/databases/qdrant/terraform/modules/cloud-storage/variables.tf
@@ -1,4 +1,4 @@
-# Copyright 2023 Google LLC
+# Copyright 2024 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

diff --git a/databases/qdrant/terraform/modules/cloud-storage/versions.tf b/databases/qdrant/terraform/modules/cloud-storage/versions.tf
@@ -1,4 +1,4 @@
-# Copyright 2023 Google LLC
+# Copyright 2024 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

diff --git a/databases/qdrant/terraform/modules/cluster-autopilot/main.tf b/databases/qdrant/terraform/modules/cluster-autopilot/main.tf
@@ -1,4 +1,4 @@
-# Copyright 2023 Google LLC
+# Copyright 2024 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

diff --git a/databases/qdrant/terraform/modules/cluster-autopilot/variables.tf b/databases/qdrant/terraform/modules/cluster-autopilot/variables.tf
@@ -1,4 +1,4 @@
-# Copyright 2023 Google LLC
+# Copyright 2024 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

diff --git a/databases/qdrant/terraform/modules/cluster-autopilot/versions.tf b/databases/qdrant/terraform/modules/cluster-autopilot/versions.tf
@@ -1,4 +1,4 @@
-# Copyright 2023 Google LLC
+# Copyright 2024 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

diff --git a/databases/qdrant/terraform/modules/cluster/main.tf b/databases/qdrant/terraform/modules/cluster/main.tf
@@ -1,4 +1,4 @@
-# Copyright 2023 Google LLC
+# Copyright 2024 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

diff --git a/databases/qdrant/terraform/modules/cluster/variables.tf b/databases/qdrant/terraform/modules/cluster/variables.tf
@@ -1,4 +1,4 @@
-# Copyright 2023 Google LLC
+# Copyright 2024 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

diff --git a/databases/qdrant/terraform/modules/cluster/versions.tf b/databases/qdrant/terraform/modules/cluster/versions.tf
@@ -1,4 +1,4 @@
-# Copyright 2023 Google LLC
+# Copyright 2024 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

diff --git a/databases/qdrant/terraform/modules/network/main.tf b/databases/qdrant/terraform/modules/network/main.tf
@@ -1,4 +1,4 @@
-# Copyright 2023 Google LLC
+# Copyright 2024 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -44,6 +44,24 @@ module "gcp-network" {
   }
 }
 
+module "firewall_rules" {
+  source       = "terraform-google-modules/network/google//modules/firewall-rules"
+  version = "~> 8.0"
+  project_id   = var.project_id
+  network_name = module.gcp-network.network_name
+
+  ingress_rules = [{
+    name                    = "allow-webhook"
+    description             = "open webhook port for statefulha operator"
+    source_ranges           = ["172.16.0.0/28",]
+    allow = [{
+      protocol = "tcp"
+      ports    = ["443","9443"]
+    }]
+    deny = []
+  }]
+}
+
 output "network_name" {
   value = module.gcp-network.network_name
 }

diff --git a/databases/qdrant/terraform/modules/network/variables.tf b/databases/qdrant/terraform/modules/network/variables.tf
@@ -1,4 +1,4 @@
-# Copyright 2023 Google LLC
+# Copyright 2024 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

diff --git a/databases/qdrant/terraform/modules/network/versions.tf b/databases/qdrant/terraform/modules/network/versions.tf
@@ -1,4 +1,4 @@
-# Copyright 2023 Google LLC
+# Copyright 2024 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.