Deprecate nip.io

1. ask user to reverse the ip before they use the iap
2. ask user to create the DNS record
3. TF will not auto reserve ip
4. TF will load the ip based on name

Tested-by: zlq

applications/jupyter/README.md

@@ -103,11 +103,14 @@ gcloud auth application-default login
 
 7. Run `terraform apply --var-file=./workloads.tfvars`. It can take upto 5 minutes on standard clusters & upto 10 minutes on AutoPilot clusters. Due to some IAP limitations, this is expected to fail with an error `Error retrieving IAM policy for iap webbackendservice` which will be resolved by the next step.
 
-8. If using authentication with IAP (i.e. `add_auth = true`), rerun terraform apply again. This is needed to configure Jupyter with IAP correctly.
-
-    * Verify the backend service for IAP has been created (takes 5-10 mins) with `gcloud compute backend-services list`
-        - Should have `jupyter-proxy-public` in the name eg.: `k8s1-63da503a-jupyter-proxy-public-80-74043627`.
-    * Run `terraform apply --var-file=./workloads.tfvars`
+8. If using authentication with IAP (i.e. `add_auth = true`)      
+   * Requirement: Follow [how to reserve a static IP](https://cloud.google.com/compute/docs/ip-addresses/reserve-static-external-ip-address#reserve_new_static) to get a **global** IP
+   * set up your DNS service to point to this public IP.
+      *  If the DNS service of your domain is managed by [Google Cloud DNS managed zone](https://cloud.google.com/dns/docs/zones), we have two ways to add the A record:
+      1. Go to https://console.cloud.google.com/net-services/dns/zones, select the zone and click ADD STANDARD, fill in your domain name and public IP address.
+      2. Run command: `gcloud dns record-sets create <domain address>. --zone=<zone name> --type="A" --ttl=<ttl in seconds> --rrdatas="<public ip address>"`
+   * Set `jupyter_domain` to specify a domain, you need to be able to configure DNS records of that domain, you can [register a Domain on Google Cloud Domains](https://cloud.google.com/domains/docs/register-domain#registering-a-domain), or use the domain registrar of your choice.
+   * Set `jupyter_static_ip_name` to the IP you reversed in static IP and register with their domain.
 
 ## Using JupyterHub
 
@@ -188,7 +191,7 @@ Ensure that the following variables within `workloads.tfvars` are set:
 * if you do bring your own OAuth client, you must add to the `Authorized redirect URIs` Field:  `https://iap.googleapis.com/v1/oauth/clientIds/<client ID>:handleRedirect`
 
 **Note:**
-This module uses `<ip>.nip.io` as the domain name with a global static ipv4 address that is configured automatically. Optionaly you can also use a custom doamin & existing ingress ip address in the `workloads.tfvars` file.
+You must use a custom doamin & existing static **global** ip address name in the `workloads.tfvars` file.
 
 ## Additional Information
 

applications/jupyter/main.tf

@@ -163,6 +163,7 @@ module "jupyterhub" {
   k8s_backend_service_name = var.k8s_backend_service_name
   k8s_backend_service_port = var.k8s_backend_service_port
   domain                   = var.domain
+  static_ip_name           = var.static_ip_name
   members_allowlist        = var.members_allowlist != "" ? split(",", var.members_allowlist) : []
   depends_on               = [module.gcs, module.namespace]
 }

applications/jupyter/variables.tf

@@ -108,10 +108,16 @@ variable "create_brand" {
 
 variable "domain" {
   type        = string
-  description = "Provide domain for ingress resource and ssl certificate. If it's empty, it will use nip.io wildcard dns"
+  description = "Provide domain for ingress resource and ssl certificate."
   default     = ""
 }
 
+variable "static_ip_name" {
+  type        = string
+  description = "Static ip name reversed in gcp"
+  default     = "rag-jupyter-hub"
+}
+
 variable "support_email" {
   type        = string
   description = "Email for users to contact with questions about their consent"

applications/jupyter/workloads-auto-create-brand.example.tfvars

@@ -42,8 +42,8 @@ k8s_backend_config_name  = "jupyter-iap-config"
 k8s_backend_service_name = "proxy-public"
 k8s_backend_service_port = 80
 
-url_domain_addr   = ""
-url_domain_name   = ""
+domain            = "" ## Provide domain for ingress resource and ssl certificate.
+static_ip_name    = ""
 client_id         = ""
 client_secret     = ""
 members_allowlist = "user:<email>,group:<email>,serviceAccount:<email>,domain:google.com"

applications/jupyter/workloads-existing-brand-auto-create-client.example.tfvars

@@ -41,8 +41,8 @@ k8s_backend_config_name  = "jupyter-iap-config"
 k8s_backend_service_name = "proxy-public"
 k8s_backend_service_port = 80
 
-url_domain_addr   = ""
-url_domain_name   = ""
+domain            = "" ## Provide domain for ingress resource and ssl certificate.
+static_ip_name    = ""
 client_id         = ""
 client_secret     = ""
 members_allowlist = "user:<email>,group:<email>,serviceAccount:<email>,domain:google.com"

applications/jupyter/workloads-existing-brand-existing-client.example.tfvars

@@ -42,8 +42,8 @@ k8s_backend_config_name  = "jupyter-iap-config"
 k8s_backend_service_name = "proxy-public"
 k8s_backend_service_port = 80
 
-url_domain_addr   = ""
-url_domain_name   = ""
+domain            = "" ## Provide domain for ingress resource and ssl certificate.
+static_ip_name    = ""
 client_id         = "<client_id>"
 client_secret     = "<client_secret>"
 members_allowlist = "user:<email>,group:<email>,serviceAccount:<email>,domain:google.com"

applications/jupyter/workloads.tfvars

@@ -45,7 +45,8 @@ k8s_backend_config_name  = "jupyter-iap-config"
 k8s_backend_service_name = "proxy-public"
 k8s_backend_service_port = 80
 
-domain            = "" ## Provide domain for ingress resource and ssl certificate. If it's empty, it will use nip.io wildcard dns
+domain            = "" ## Provide domain for ingress resource and ssl certificate.
+static_ip_name    = ""
 client_id         = "" ## Ensure brand is Internal, to autogenerate client credentials
 client_secret     = ""
 members_allowlist = "user:<email>,group:<email>,serviceAccount:<email>,domain:google.com"

applications/rag/README.md

@@ -99,7 +99,6 @@ gcloud container clusters get-credentials ${CLUSTER_NAME} --location=${CLUSTER_L
             * password: use `terraform output jupyterhub_password` to fetch the password value
    * If IAP is enabled (`jupyter_add_auth = true`):
         - Fetch the domain: `terraform output jupyterhub_uri`
-        - If you used a custom domain, ensure you configured your DNS as described above. This can be skipped if using `nip.io`.
         - Verify the domain status is `Active`:
             - `kubectl get managedcertificates jupyter-managed-cert -n ${NAMESPACE} --output jsonpath='{.status.domainStatus[0].status}'`
             - Note: This can take up to 20 minutes to propagate.
@@ -126,7 +125,6 @@ gcloud container clusters get-credentials ${CLUSTER_NAME} --location=${CLUSTER_L
             - Go to `localhost:8265` in a browser
         - If IAP is enabled (`ray_dashboard_add_auth = true`):
             - Fetch the domain: `terraform output ray-dashboard-managed-cert`
-            - If you used a custom domain, ensure you configured your DNS as described above. This can be skipped if using `nip.io`.
             - Verify the domain status is `Active`:
                 - `kubectl get managedcertificates ray-dashboard-managed-cert -n rag --output jsonpath='{.status.domainStatus[0].status}'`
                 - Note: This can take up to 20 minutes to propagate.
@@ -141,7 +139,6 @@ gcloud container clusters get-credentials ${CLUSTER_NAME} --location=${CLUSTER_L
         - Go to `localhost:8080` in a browser
     * If IAP is enabled (`frontend_add_auth = true`):
         - Fetch the domain: `terraform output frontend_uri`
-        - If you used a custom domain, ensure you configured your DNS as described above. This can be skipped if using `nip.io`.
         - Verify the domain status is `Active`:
             - `kubectl get managedcertificates frontend-managed-cert -n rag --output jsonpath='{.status.domainStatus[0].status}'`
             - Note: This can take up to 20 minutes to propagate.
@@ -160,16 +157,15 @@ We recommend you configure authenticated access via IAP for your services.
     * `frontend_add_auth = true`
     * `ray_dashboard_add_auth = true`
 3) Allowlist principals for your services via `jupyter_members_allowlist`, `frontend_members_allowlist` and `ray_dashboard_members_allowlist`.
-4) Configure custom domains names via `jupyter_domain`, `frontend_domain` and `ray_dashboard_domain` for your services. If left blank, we'll provision test `nip.io` domains for you.
-5) Configure DNS records for your custom domains:
-    - [Register a Domain on Google Cloud Domains](https://cloud.google.com/domains/docs/register-domain#registering-a-domain) or use a domain registrar of your choice. You can skip this if using `nip.io`.
-    - Set up your DNS service to point to the public IP
-        * Run `terraform output frontend_ip_address` to get the public ip address of frontend, and add an A record in your DNS configuration to point to the public IP address.
-        * Run `terraform output jupyterhub_ip_address` to get the public ip address of jupyterhub, and add an A record in your DNS configuration to point to the public IP address.
-        * Run `terraform output ray_dashboard_ip_address` to get the public ip address of ray dashboard, and add an A record in your DNS configuration to point to the public IP address.
-    - Add an A record: If the DNS service of your domain is managed by [Google Cloud DNS managed zone](https://cloud.google.com/dns/docs/zones), there are two options to add the A record:
+4) Configure custom domains names via `jupyter_domain`, `frontend_domain` and `ray_dashboard_domain` for your services. 
+    * Requirement: Follow [how to reserve a static IP](https://cloud.google.com/compute/docs/ip-addresses/reserve-static-external-ip-address#reserve_new_static) to get a **global** IP
+    * set up your DNS service to point to this public IP.
+        *  If the DNS service of your domain is managed by [Google Cloud DNS managed zone](https://cloud.google.com/dns/docs/zones), we have two ways to add the A record:
         1. Go to https://console.cloud.google.com/net-services/dns/zones, select the zone and click ADD STANDARD, fill in your domain name and public IP address.
-        2. Run `gcloud dns record-sets create <domain address>. --zone=<zone name> --type="A" --ttl=<ttl in seconds> --rrdatas="<public ip address>"`
+        2. Run command: `gcloud dns record-sets create <domain address>. --zone=<zone name> --type="A" --ttl=<ttl in seconds> --rrdatas="<public ip address>"`
+    * For each iap you enabled, prepare the domain and static IP **name**
+    * Set `jupyter_domain`, `frontend_domain`, `ray_dashboard_domain` to specify a domain, you need to be able to configure DNS records of that domain, you can [register a Domain on Google Cloud Domains](https://cloud.google.com/domains/docs/register-domain#registering-a-domain), or use the domain registrar of your choice.
+    * Set `jupyter_static_ip_name`, `frontend_static_ip_name`, `ray_dashboard_static_ip_name` to the IP you reversed in static IP and register with their domain.
 
 # Cleanup
 

applications/rag/frontend/main.tf

@@ -42,6 +42,7 @@ module "iap_auth" {
   client_id                = var.client_id
   client_secret            = var.client_secret
   domain                   = var.domain
+  static_ip_name           = var.static_ip_name
   members_allowlist        = var.members_allowlist
   depends_on = [
     kubernetes_service.rag_frontend_service, kubernetes_deployment.rag_frontend_deployment

applications/rag/frontend/variables.tf

@@ -129,10 +129,16 @@ variable "support_email" {
 
 variable "domain" {
   type        = string
-  description = "Provide domain for ingress resource and ssl certificate. If it's empty, it will use nip.io wildcard dns"
+  description = "Provide domain for ingress resource and ssl certificate."
   default     = ""
 }
 
+variable "static_ip_name" {
+  type        = string
+  description = "Static ip name reversed in gcp"
+  default     = "rag-frontend"
+}
+
 variable "client_id" {
   type        = string
   description = "Client ID used for enabling IAP"

applications/rag/main.tf

@@ -204,6 +204,7 @@ module "jupyterhub" {
   k8s_backend_service_name = var.jupyter_k8s_backend_service_name
   k8s_backend_service_port = var.jupyter_k8s_backend_service_port
   domain                   = var.jupyter_domain
+  static_ip_name           = var.jupyter_static_ip_name
   members_allowlist        = var.jupyter_members_allowlist != "" ? split(",", var.jupyter_members_allowlist) : []
 
   depends_on = [module.namespace, module.gcs]
@@ -245,6 +246,7 @@ module "kuberay-cluster" {
   k8s_backend_config_name  = var.ray_dashboard_k8s_backend_config_name
   k8s_backend_service_port = var.ray_dashboard_k8s_backend_service_port
   domain                   = var.ray_dashboard_domain
+  static_ip_name           = var.ray_dashboard_static_ip_name
   members_allowlist        = var.ray_dashboard_members_allowlist
 }
 
@@ -297,6 +299,7 @@ module "frontend" {
   k8s_backend_service_name = var.frontend_k8s_backend_service_name
   k8s_backend_service_port = var.frontend_k8s_backend_service_port
   domain                   = var.frontend_domain
+  static_ip_name           = var.frontend_static_ip_name
   members_allowlist        = var.frontend_members_allowlist != "" ? split(",", var.frontend_members_allowlist) : []
   depends_on               = [module.namespace]
 }

applications/rag/variables.tf

@@ -152,7 +152,13 @@ variable "frontend_k8s_backend_service_port" {
 
 variable "frontend_domain" {
   type        = string
-  description = "Domain used for SSL certificate. If it's empty, *.nip.io DNS is used."
+  description = "Domain used for SSL certificate."
+  default     = ""
+}
+
+variable "frontend_static_ip_name" {
+  type        = string
+  description = "Static ip name reversed in gcp"
   default     = ""
 }
 
@@ -211,7 +217,13 @@ variable "ray_dashboard_k8s_backend_service_port" {
 
 variable "ray_dashboard_domain" {
   type        = string
-  description = "Domain used for SSL certificate. If it's empty, *.nip.io DNS is used."
+  description = "Domain used for SSL certificate."
+  default     = ""
+}
+
+variable "ray_dashboard_static_ip_name" {
+  type        = string
+  description = "Static ip name reversed in gcp"
   default     = ""
 }
 
@@ -278,7 +290,13 @@ variable "jupyter_k8s_backend_service_port" {
 
 variable "jupyter_domain" {
   type        = string
-  description = "Domain used for SSL certificate. If it's empty, *.nip.io DNS is used."
+  description = "Domain used for SSL certificate."
+  default     = ""
+}
+
+variable "jupyter_static_ip_name" {
+  type        = string
+  description = "Static ip name reversed in gcp"
   default     = ""
 }
 

applications/rag/workloads.tfvars

@@ -58,15 +58,18 @@ dataset_embeddings_table_name = "netflix_reviews_db"
 
 ## Jupyter IAP Settings
 jupyter_add_auth          = false                                                                 # Set to true to enable authenticated access via IAP.
-jupyter_domain            = ""                                                                    # Custom domain for ingress resource and ssl certificate. If empty, it will use nip.io wildcard DNS.
+jupyter_domain            = ""
+jupyter_static_ip_name    = ""                                                               # Custom domain for ingress resource and ssl certificate.
 jupyter_members_allowlist = "user:<email>,group:<email>,serviceAccount:<email>,domain:google.com" # Allowlist principals for access.
 
 ## Frontend IAP Settings
 frontend_add_auth          = false                                                                 # Set to true to enable authenticated access via IAP.
-frontend_domain            = ""                                                                    # Custom domain for ingress resource and ssl certificate. If empty, it will use nip.io wildcard DNS.
+frontend_domain            = ""
+frontend_static_ip_name    = ""                                                                    # Custom domain for ingress resource and ssl certificate.
 frontend_members_allowlist = "user:<email>,group:<email>,serviceAccount:<email>,domain:google.com" # Allowlist principals for access.
 
 ## Ray Dashboard IAP Settings
 ray_dashboard_add_auth          = false                                                                 # Set to true to enable authenticated access via IAP.
-ray_dashboard_domain            = ""                                                                    # Custom domain for ingress resource and ssl certificate. If empty, it will use nip.io wildcard DNS.
+ray_dashboard_domain            = ""
+ray_dashboard_static_ip_name    = ""                                                                    # Custom domain for ingress resource and ssl certificate.
 ray_dashboard_members_allowlist = "user:<email>,group:<email>,serviceAccount:<email>,domain:google.com" # Allowlist principals for access.

applications/ray/main.tf

@@ -201,6 +201,7 @@ module "kuberay-cluster" {
   k8s_backend_config_name  = var.ray_dashboard_k8s_backend_config_name
   k8s_backend_service_port = var.ray_dashboard_k8s_backend_service_port
   domain                   = var.ray_dashboard_domain
+  static_ip_name           = var.ray_dashboard_static_ip_name
   members_allowlist        = var.ray_dashboard_members_allowlist
 
   depends_on = [module.gcs, module.kuberay-operator]

applications/ray/variables.tf

@@ -229,7 +229,13 @@ variable "ray_dashboard_k8s_backend_service_port" {
 
 variable "ray_dashboard_domain" {
   type        = string
-  description = "Domain used for SSL certificate. If it's empty, *.nip.io DNS is used."
+  description = "Domain used for SSL certificate."
+  default     = ""
+}
+
+variable "ray_dashboard_static_ip_name" {
+  type        = string
+  description = "Static ip name reversed in gcp"
   default     = ""
 }
 

applications/ray/workloads.tfvars

@@ -54,6 +54,7 @@ ray_dashboard_k8s_backend_config_name  = "ray-dashboard-iap-config"
 ray_dashboard_k8s_backend_service_port = 8265
 
 ray_dashboard_domain            = ""
+ray_dashboard_static_ip_name    = ""
 ray_dashboard_client_id         = ""
 ray_dashboard_client_secret     = ""
 ray_dashboard_members_allowlist = "user:<email>,group:<email>,serviceAccount:<email>,domain:google.com"

modules/iap/iap.tf

@@ -40,13 +40,8 @@ resource "random_string" "random" {
   upper   = false
 }
 
-# IAP
-resource "google_compute_global_address" "ip_address" {
-  provider     = google-beta
-  project      = var.project_id
-  name         = "${var.app_name}-address-${random_string.random.result}"
-  address_type = "EXTERNAL"
-  ip_version   = "IPV4"
+data "dns_a_record_set" "ip_address" {
+  host          = var.domain
 }
 
 # Helm Chart IAP
@@ -84,12 +79,12 @@ resource "helm_release" "iap" {
 
   set {
     name  = "iap.managedCertificate.domain"
-    value = var.domain != "" ? var.domain : "${google_compute_global_address.ip_address.address}.nip.io"
+    value = var.domain
   }
 
   set {
     name  = "iap.ingress.staticIpName"
-    value = google_compute_global_address.ip_address.name
+    value = var.static_ip_name
   }
 
   set {

modules/iap/outputs.tf

@@ -13,9 +13,9 @@
 # limitations under the License.
 
 output "domain" {
-  value = var.domain == "" ? "${google_compute_global_address.ip_address.address}.nip.io" : var.domain
+  value = var.domain
 }
 
 output "ip_address" {
-  value = google_compute_global_address.ip_address.address
+  value = data.dns_a_record_set.ip_address.addrs[0]
 }

modules/iap/variables.tf

@@ -65,7 +65,7 @@ variable "k8s_backend_service_port" {
 
 variable "domain" {
   type        = string
-  description = "Provide domain for ingress resource and ssl certificate. If it's empty, it will use nip.io wildcard dns"
+  description = "Provide domain for ingress resource and ssl certificate."
   default     = ""
 }
 
@@ -87,6 +87,11 @@ variable "client_secret" {
   default     = ""
 }
 
+variable "static_ip_name" {
+  type        = string
+  description = "Static ip name reversed in gcp"
+}
+
 variable "members_allowlist" {
   type    = list(string)
   default = []

modules/jupyter/main.tf

@@ -42,6 +42,7 @@ module "iap_auth" {
   client_id                = var.client_id
   client_secret            = var.client_secret
   domain                   = var.domain
+  static_ip_name           = var.static_ip_name
   members_allowlist        = var.members_allowlist
   depends_on = [
     helm_release.jupyterhub