test

Change-Id: Ie80a37694fe35e98e801b0c5a425f7793837b092
GoogleCloudPlatform · Sep 18, 2024 · a7b7743 · a7b7743
1 parent 6e48fd5
commit a7b7743
Show file tree

Hide file tree

Showing 4 changed files with 183 additions and 429 deletions.
diff --git a/applications/jupyter/main.tf b/applications/jupyter/main.tf
@@ -122,13 +122,6 @@ provider "helm" {
   }
 }
 
-module "gcs" {
-  source      = "../../modules/gcs"
-  count       = var.create_gcs_bucket ? 1 : 0
-  project_id  = var.project_id
-  bucket_name = var.gcs_bucket
-}
-
 # create namespace
 module "namespace" {
   source           = "../../modules/kubernetes-namespace"
@@ -164,5 +157,5 @@ module "jupyterhub" {
   k8s_backend_service_port = var.k8s_backend_service_port
   domain                   = var.domain
   members_allowlist        = var.members_allowlist != "" ? split(",", var.members_allowlist) : []
-  depends_on               = [module.gcs, module.namespace]
+  depends_on               = [module.namespace]
 }
diff --git a/applications/jupyter/workloads.tfvars b/applications/jupyter/workloads.tfvars
@@ -14,27 +14,28 @@
 
 ##common variables
 ## Need to pull this variables from tf output from previous infrastructure stage
-project_id = "<your project ID>"
+project_id = "gke-ai-eco-dev"
 
 ## This is required for terraform to connect to GKE cluster and deploy workloads.
-cluster_name     = "ml-cluster"
-cluster_location = "us-central1"
+cluster_name     = "ml-a236619-827-2ec62154-cluster"
+cluster_location = "us-east4"
 
 ## If terraform should create a new GKE cluster, fill in this section as well.
 ##    By default, a public autopilot GKE cluster will be created in the default network.
 ##    Set the autopilot_cluster variable to false to create a standard cluster instead.
 create_cluster        = false
-autopilot_cluster     = true
+autopilot_cluster     = false
 cluster_membership_id = "" # required for private cluster, defaults to `cluster_name`
 
 #######################################################
 ####    APPLICATIONS
 #######################################################
 
 ## JupyterHub variables
-kubernetes_namespace              = "ai-on-gke"
-create_gcs_bucket                 = true
-gcs_bucket                        = "gcs-bucket-<unique-suffix>" # Choose a globally unique bucket name.
+kubernetes_namespace              = "jupyter-test"
+create_gcs_bucket                 = false
+gcs_bucket                        = "gke-aieco-rag-a236619-2ec62154" 
+# Choose a globally unique bucket name.
 workload_identity_service_account = "jupyter-sa"
 
 # IAP Configs

diff --git a/applications/rag/example_notebooks/rag-kaggle-ray-sql-interactive.ipynb b/applications/rag/example_notebooks/rag-kaggle-ray-sql-interactive.ipynb
@@ -37,9 +37,9 @@
    "id": "c7ff518d-f4d2-481b-b408-2c2507565611",
    "metadata": {},
    "source": [
-    "## Creating the Database Connection\n",
+    "## Download the Data\n",
     "\n",
-    "Let's now set up a connection to your CloudSQL database:"
+    "Let's now import required modules:"
    ]
   },
   {
@@ -60,42 +60,7 @@
     "from datasets import load_dataset_builder, load_dataset, Dataset\n",
     "from huggingface_hub import snapshot_download\n",
     "from google.cloud.sql.connector import Connector, IPTypes\n",
-    "import sqlalchemy\n",
-    "\n",
-    "# initialize parameters\n",
-    "\n",
-    "INSTANCE_CONNECTION_NAME = os.environ[\"CLOUDSQL_INSTANCE_CONNECTION_NAME\"]\n",
-    "print(f\"Your instance connection name is: {INSTANCE_CONNECTION_NAME}\")\n",
-    "DB_NAME = \"pgvector-database\"\n",
-    "\n",
-    "db_username_file = open(\"/etc/secret-volume/username\", \"r\")\n",
-    "DB_USER = db_username_file.read()\n",
-    "db_username_file.close()\n",
-    "\n",
-    "db_password_file = open(\"/etc/secret-volume/password\", \"r\")\n",
-    "DB_PASS = db_password_file.read()\n",
-    "db_password_file.close()\n",
-    "\n",
-    "# initialize Connector object\n",
-    "connector = Connector()\n",
-    "\n",
-    "# function to return the database connection object\n",
-    "def getconn():\n",
-    "    conn = connector.connect(\n",
-    "        INSTANCE_CONNECTION_NAME,\n",
-    "        \"pg8000\",\n",
-    "        user=DB_USER,\n",
-    "        password=DB_PASS,\n",
-    "        db=DB_NAME,\n",
-    "        ip_type=IPTypes.PRIVATE\n",
-    "    )\n",
-    "    return conn\n",
-    "\n",
-    "# create connection pool with 'creator' argument to our connection object function\n",
-    "pool = sqlalchemy.create_engine(\n",
-    "    \"postgresql+pg8000://\",\n",
-    "    creator=getconn,\n",
-    ")"
+    "import sqlalchemy"
    ]
   },
   {
@@ -150,7 +115,9 @@
     "os.makedirs(SHARED_DATA_BASEPATH, exist_ok=True)\n",
     "\n",
     "# One time download of the sentence transformer model to a shared persistent storage available to the ray workers\n",
-    "snapshot_download(repo_id=SENTENCE_TRANSFORMER_MODEL, revision=SENTENCE_TRANSFORMER_MODEL_SNAPSHOT, cache_dir=SHARED_DATA_BASEPATH)"
+    "snapshot_download(repo_id=SENTENCE_TRANSFORMER_MODEL, revision=SENTENCE_TRANSFORMER_MODEL_SNAPSHOT, cache_dir=\"~/data/rag\")\n",
+    "\n",
+    "!cp -r \"~/data/rag\" \"/data/rag/st\""
    ]
   },
   {
@@ -322,6 +289,40 @@
     "from sqlalchemy.orm import scoped_session, sessionmaker, mapped_column\n",
     "from pgvector.sqlalchemy import Vector\n",
     "\n",
+    "# initialize parameters\n",
+    "\n",
+    "INSTANCE_CONNECTION_NAME = os.environ[\"CLOUDSQL_INSTANCE_CONNECTION_NAME\"]\n",
+    "print(f\"Your instance connection name is: {INSTANCE_CONNECTION_NAME}\")\n",
+    "DB_NAME = \"pgvector-database\"\n",
+    "\n",
+    "db_username_file = open(\"/etc/secret-volume/username\", \"r\")\n",
+    "DB_USER = db_username_file.read()\n",
+    "db_username_file.close()\n",
+    "\n",
+    "db_password_file = open(\"/etc/secret-volume/password\", \"r\")\n",
+    "DB_PASS = db_password_file.read()\n",
+    "db_password_file.close()\n",
+    "\n",
+    "# initialize Connector object\n",
+    "connector = Connector()\n",
+    "\n",
+    "# function to return the database connection object\n",
+    "def getconn():\n",
+    "    conn = connector.connect(\n",
+    "        INSTANCE_CONNECTION_NAME,\n",
+    "        \"pg8000\",\n",
+    "        user=DB_USER,\n",
+    "        password=DB_PASS,\n",
+    "        db=DB_NAME,\n",
+    "        ip_type=IPTypes.PRIVATE\n",
+    "    )\n",
+    "    return conn\n",
+    "\n",
+    "# create connection pool with 'creator' argument to our connection object function\n",
+    "pool = sqlalchemy.create_engine(\n",
+    "    \"postgresql+pg8000://\",\n",
+    "    creator=getconn,\n",
+    ")\n",
     "\n",
     "Base = declarative_base()\n",
     "DBSession = scoped_session(sessionmaker())\n",