From ce75b6947f6fa34b490ed439ba98715226178190 Mon Sep 17 00:00:00 2001
From: Leo Yongsul Kim <ystory84@gmail.com>
Date: Sun, 4 Jun 2023 21:55:38 +0900
Subject: [PATCH] feat: Refactor post-provisioning operations (#12)

- Move post-provisioning operations from terraform-oci-always-free-oke module to examples/private-cluster/modules

#patch
---
 examples/private-cluster/main.tf              | 31 ++++++++++++++++---
 .../modules}/kubernetes/bastion-tennel.tf     | 27 +++-------------
 .../modules}/kubernetes/ingress-nginx.tf      | 12 +++++--
 .../modules}/kubernetes/kubeconfig.tf         | 10 +++---
 .../modules}/kubernetes/locals.tf             |  3 --
 .../resources/ingress-nginx-deployment.yaml   |  0
 .../oke-admin-service-account-token.yaml      |  0
 .../resources/oke-admin-service-account.yaml  |  0
 .../scripts/create_bastion_tunnel_template.sh | 20 ++++++------
 .../scripts/create_kubeconfig_template.sh     |  0
 .../modules}/kubernetes/service-account.tf    | 10 +++++-
 .../modules}/kubernetes/variables.tf          |  0
 .../private-cluster/modules}/tls/outputs.tf   |  0
 .../modules}/tls/tls-private-key.tf           |  0
 .../private-cluster/modules}/tls/versions.tf  |  0
 .../private-cluster/terraform.tfvars.example  |  1 +
 examples/private-cluster/variables.tf         |  6 ++++
 main.tf                                       | 28 ++---------------
 outputs.tf                                    |  5 +++
 variables.tf                                  |  6 ----
 20 files changed, 80 insertions(+), 79 deletions(-)
 rename {modules => examples/private-cluster/modules}/kubernetes/bastion-tennel.tf (51%)
 rename {modules => examples/private-cluster/modules}/kubernetes/ingress-nginx.tf (66%)
 rename {modules => examples/private-cluster/modules}/kubernetes/kubeconfig.tf (72%)
 rename {modules => examples/private-cluster/modules}/kubernetes/locals.tf (91%)
 rename {modules => examples/private-cluster/modules}/kubernetes/resources/ingress-nginx-deployment.yaml (100%)
 rename {modules => examples/private-cluster/modules}/kubernetes/resources/oke-admin-service-account-token.yaml (100%)
 rename {modules => examples/private-cluster/modules}/kubernetes/resources/oke-admin-service-account.yaml (100%)
 rename {modules => examples/private-cluster/modules}/kubernetes/scripts/create_bastion_tunnel_template.sh (78%)
 rename {modules => examples/private-cluster/modules}/kubernetes/scripts/create_kubeconfig_template.sh (100%)
 rename {modules => examples/private-cluster/modules}/kubernetes/service-account.tf (53%)
 rename {modules => examples/private-cluster/modules}/kubernetes/variables.tf (100%)
 rename {modules => examples/private-cluster/modules}/tls/outputs.tf (100%)
 rename {modules => examples/private-cluster/modules}/tls/tls-private-key.tf (100%)
 rename {modules => examples/private-cluster/modules}/tls/versions.tf (100%)

diff --git a/examples/private-cluster/main.tf b/examples/private-cluster/main.tf
index b76b3c5..766ae48 100644
--- a/examples/private-cluster/main.tf
+++ b/examples/private-cluster/main.tf
@@ -1,6 +1,12 @@
+module "tls" {
+  source = "./modules/tls"
+
+  count = var.create_ssh_key_pair == true ? 1 : 0
+}
+
 module "free_k8s" {
-  source  = "../../"
-#  version = "0.0.5"
+  source = "../../"
+  #  version = "0.0.5"
 
   tenancy_id  = var.tenancy_id
   home_region = var.home_region
@@ -12,9 +18,26 @@ module "free_k8s" {
   control_plane_type          = "private"
   control_plane_allowed_cidrs = ["0.0.0.0/0"]
 
-  create_ssh_key_pair = true
-
   providers = {
     oci.home = oci.home
   }
 }
+
+module "kubernetes" {
+  source = "./modules/kubernetes"
+
+  control_plane_bastion_service_id = module.free_k8s.bastion_ids["cp"]
+  workers_bastion_service_id       = module.free_k8s.bastion_ids["workers"]
+
+  # ssh keys
+  ssh_private_key      = var.create_ssh_key_pair ? chomp(module.tls[0].ssh_private_key) : var.ssh_private_key
+  ssh_private_key_path = var.ssh_private_key_path
+  ssh_public_key       = var.create_ssh_key_pair ? chomp(module.tls[0].ssh_public_key) : var.ssh_public_key
+  ssh_public_key_path  = var.ssh_public_key_path
+
+
+  cluster_id        = module.free_k8s.cluster_id
+  cluster_endpoints = module.free_k8s.cluster_endpoints
+
+  region = var.region
+}
diff --git a/modules/kubernetes/bastion-tennel.tf b/examples/private-cluster/modules/kubernetes/bastion-tennel.tf
similarity index 51%
rename from modules/kubernetes/bastion-tennel.tf
rename to examples/private-cluster/modules/kubernetes/bastion-tennel.tf
index 9dcddb0..9d32490 100644
--- a/modules/kubernetes/bastion-tennel.tf
+++ b/examples/private-cluster/modules/kubernetes/bastion-tennel.tf
@@ -1,24 +1,13 @@
-variable "always_run_bastion_tunnel" {
-  description = "A boolean variable to decide whether to always trigger the bastion_tunnel null_resource on each 'terraform apply'. Set to 'true' to always trigger, and 'false' to maintain the trigger state."
-  default     = true
-}
-
-locals {
-  trigger_value_bastion_tunnel = var.always_run_bastion_tunnel ? uuid() : ""
-}
-
 resource "null_resource" "bastion_tunnel" {
   depends_on = [null_resource.kubeconfig, local_file.bastion_tunnel]
 
   triggers = {
-    always_run = local.trigger_value_bastion_tunnel
+    always_run = uuid()
   }
 
   provisioner "local-exec" {
-    command = "${path.root}/cluster_access.sh"
+    command = local_file.bastion_tunnel.filename
   }
-
-  count = local.post_provisioning_ops_enabled ? 1 : 0
 }
 
 resource "local_file" "bastion_tunnel" {
@@ -26,30 +15,24 @@ resource "local_file" "bastion_tunnel" {
   content    = templatefile("${path.module}/scripts/create_bastion_tunnel_template.sh",
     {
       bastion_id       = var.control_plane_bastion_service_id
-      public_key_file  = local_file.id_rsa_pub[0].filename
-      private_key_file = local_file.id_rsa[0].filename
+      public_key_file  = local_file.id_rsa_pub.filename
+      private_key_file = local_file.id_rsa.filename
       cluster_ip       = local.private_endpoint_ip
       cluster_port     = local.private_endpoint_port
       region           = var.region
     }
   )
-  filename = "${path.root}/cluster_access.sh"
-
-  count = local.post_provisioning_ops_enabled ? 1 : 0
+  filename = "${path.root}/connect_to_cluster.sh"
 }
 
 resource "local_file" "id_rsa" {
   content         = local.ssh_private_key
   filename        = "${path.root}/id_rsa"
   file_permission = "0600"
-
-  count = local.post_provisioning_ops_enabled ? 1 : 0
 }
 
 resource "local_file" "id_rsa_pub" {
   content         = local.ssh_authorized_keys
   filename        = "${path.root}/id_rsa.pub"
   file_permission = "0600"
-
-  count = local.post_provisioning_ops_enabled ? 1 : 0
 }
diff --git a/modules/kubernetes/ingress-nginx.tf b/examples/private-cluster/modules/kubernetes/ingress-nginx.tf
similarity index 66%
rename from modules/kubernetes/ingress-nginx.tf
rename to examples/private-cluster/modules/kubernetes/ingress-nginx.tf
index d4541c7..b2fb36e 100644
--- a/modules/kubernetes/ingress-nginx.tf
+++ b/examples/private-cluster/modules/kubernetes/ingress-nginx.tf
@@ -8,8 +8,16 @@ resource "null_resource" "ingress_nginx" {
 
   provisioner "local-exec" {
     when    = destroy
-    command = "kubectl --kubeconfig ~/.kube/ociconfig delete -f ${path.module}/resources/ingress-nginx-deployment.yaml"
+    command = "${path.root}/create_kubeconfig.sh"
+  }
+
+  provisioner "local-exec" {
+    when    = destroy
+    command = "${path.root}/connect_to_cluster.sh"
   }
 
-  count = local.post_provisioning_ops_enabled ? 1 : 0
+  provisioner "local-exec" {
+    when    = destroy
+    command = "kubectl --kubeconfig ~/.kube/ociconfig delete -f ${path.module}/resources/ingress-nginx-deployment.yaml"
+  }
 }
diff --git a/modules/kubernetes/kubeconfig.tf b/examples/private-cluster/modules/kubernetes/kubeconfig.tf
similarity index 72%
rename from modules/kubernetes/kubeconfig.tf
rename to examples/private-cluster/modules/kubernetes/kubeconfig.tf
index a2c06a1..6f58983 100644
--- a/modules/kubernetes/kubeconfig.tf
+++ b/examples/private-cluster/modules/kubernetes/kubeconfig.tf
@@ -1,11 +1,13 @@
 resource "null_resource" "kubeconfig" {
   depends_on = [local_file.create_kubeconfig]
 
-  provisioner "local-exec" {
-    command = "${path.root}/create_kubeconfig.sh"
+  triggers = {
+    always_run = uuid()
   }
 
-  count = local.post_provisioning_ops_enabled ? 1 : 0
+  provisioner "local-exec" {
+    command = local_file.create_kubeconfig.filename
+  }
 }
 
 resource "local_file" "create_kubeconfig" {
@@ -17,6 +19,4 @@ resource "local_file" "create_kubeconfig" {
     }
   )
   filename = "${path.root}/create_kubeconfig.sh"
-
-  count = local.post_provisioning_ops_enabled ? 1 : 0
 }
diff --git a/modules/kubernetes/locals.tf b/examples/private-cluster/modules/kubernetes/locals.tf
similarity index 91%
rename from modules/kubernetes/locals.tf
rename to examples/private-cluster/modules/kubernetes/locals.tf
index c477905..86e09e1 100644
--- a/modules/kubernetes/locals.tf
+++ b/examples/private-cluster/modules/kubernetes/locals.tf
@@ -1,7 +1,4 @@
 locals {
-  # TODO
-  post_provisioning_ops_enabled = true
-
   private_endpoint_ip   = split(":", element(var.cluster_endpoints, 0)["private_endpoint"])[0]
   private_endpoint_port = split(":", element(var.cluster_endpoints, 0)["private_endpoint"])[1]
 
diff --git a/modules/kubernetes/resources/ingress-nginx-deployment.yaml b/examples/private-cluster/modules/kubernetes/resources/ingress-nginx-deployment.yaml
similarity index 100%
rename from modules/kubernetes/resources/ingress-nginx-deployment.yaml
rename to examples/private-cluster/modules/kubernetes/resources/ingress-nginx-deployment.yaml
diff --git a/modules/kubernetes/resources/oke-admin-service-account-token.yaml b/examples/private-cluster/modules/kubernetes/resources/oke-admin-service-account-token.yaml
similarity index 100%
rename from modules/kubernetes/resources/oke-admin-service-account-token.yaml
rename to examples/private-cluster/modules/kubernetes/resources/oke-admin-service-account-token.yaml
diff --git a/modules/kubernetes/resources/oke-admin-service-account.yaml b/examples/private-cluster/modules/kubernetes/resources/oke-admin-service-account.yaml
similarity index 100%
rename from modules/kubernetes/resources/oke-admin-service-account.yaml
rename to examples/private-cluster/modules/kubernetes/resources/oke-admin-service-account.yaml
diff --git a/modules/kubernetes/scripts/create_bastion_tunnel_template.sh b/examples/private-cluster/modules/kubernetes/scripts/create_bastion_tunnel_template.sh
similarity index 78%
rename from modules/kubernetes/scripts/create_bastion_tunnel_template.sh
rename to examples/private-cluster/modules/kubernetes/scripts/create_bastion_tunnel_template.sh
index f604341..2e8c880 100755
--- a/modules/kubernetes/scripts/create_bastion_tunnel_template.sh
+++ b/examples/private-cluster/modules/kubernetes/scripts/create_bastion_tunnel_template.sh
@@ -3,18 +3,18 @@
 # Create a port-forwarding session on the bastion
 oci_bastion_session_create() {
   oci bastion session create-port-forwarding \
-    --bastion-id "${bastion_id}" \
+    --bastion-id ${bastion_id} \
     --display-name oke-tunnel \
-    --ssh-public-key-file "${public_key_file}" \
+    --ssh-public-key-file ${public_key_file} \
     --key-type PUB \
-    --target-private-ip "${cluster_ip}" \
-    --target-port "${cluster_port}" \
+    --target-private-ip ${cluster_ip} \
+    --target-port ${cluster_port} \
     --session-ttl 10800
 }
 
 oci_bastion_session_list() {
   oci bastion session list \
-    --bastion-id "${bastion_id}" \
+    --bastion-id ${bastion_id} \
     --display-name oke-tunnel \
     --limit 1 \
     --session-lifecycle-state ACTIVE \
@@ -25,7 +25,7 @@ oci_bastion_session_list() {
 
 oci_bastion_session_state() {
   session_id="$1"
-  oci bastion session get --session-id "$session_id" |
+  oci bastion session get --session-id $session_id |
     jq -r '.data."lifecycle-state"'
 }
 
@@ -56,7 +56,7 @@ oci_bastion_session_init() {
     sleep 10
   fi
 
-  echo >&2 "$state"
+  echo >&2 "$state $session_id"
 }
 
 oci_bastion_session_init
@@ -65,10 +65,10 @@ oci_bastion_session_init
 if lsof -t -i:6443; then
   kill "$(lsof -t -i:6443)"
 fi
-nohup ssh -i "${private_key_file}" \
+nohup ssh -i ${private_key_file} \
   -o HostKeyAlgorithms=+ssh-rsa \
   -o PubkeyAcceptedAlgorithms=+ssh-rsa \
-  -N -L 6443:"${cluster_ip}:${cluster_port}" \
+  -N -L 6443:${cluster_ip}:${cluster_port} \
   -p 22 \
   -o StrictHostKeyChecking=no \
-  "$session_id"@host.bastion."${region}".oci.oraclecloud.com >/dev/null 2>&1 &
+  "$session_id"@host.bastion.${region}.oci.oraclecloud.com >/dev/null 2>&1 &
diff --git a/modules/kubernetes/scripts/create_kubeconfig_template.sh b/examples/private-cluster/modules/kubernetes/scripts/create_kubeconfig_template.sh
similarity index 100%
rename from modules/kubernetes/scripts/create_kubeconfig_template.sh
rename to examples/private-cluster/modules/kubernetes/scripts/create_kubeconfig_template.sh
diff --git a/modules/kubernetes/service-account.tf b/examples/private-cluster/modules/kubernetes/service-account.tf
similarity index 53%
rename from modules/kubernetes/service-account.tf
rename to examples/private-cluster/modules/kubernetes/service-account.tf
index cc75bcb..5ae1ae2 100644
--- a/modules/kubernetes/service-account.tf
+++ b/examples/private-cluster/modules/kubernetes/service-account.tf
@@ -9,5 +9,13 @@ resource "null_resource" "create_service_account" {
     command = "kubectl --kubeconfig ~/.kube/ociconfig apply -f ${path.module}/resources/oke-admin-service-account-token.yaml"
   }
 
-  count = local.post_provisioning_ops_enabled ? 1 : 0
+  provisioner "local-exec" {
+    when    = destroy
+    command = "kubectl --kubeconfig ~/.kube/ociconfig delete -f ${path.module}/resources/oke-admin-service-account-token.yaml"
+  }
+
+  provisioner "local-exec" {
+    when    = destroy
+    command = "kubectl --kubeconfig ~/.kube/ociconfig delete -f ${path.module}/resources/oke-admin-service-account.yaml"
+  }
 }
diff --git a/modules/kubernetes/variables.tf b/examples/private-cluster/modules/kubernetes/variables.tf
similarity index 100%
rename from modules/kubernetes/variables.tf
rename to examples/private-cluster/modules/kubernetes/variables.tf
diff --git a/modules/tls/outputs.tf b/examples/private-cluster/modules/tls/outputs.tf
similarity index 100%
rename from modules/tls/outputs.tf
rename to examples/private-cluster/modules/tls/outputs.tf
diff --git a/modules/tls/tls-private-key.tf b/examples/private-cluster/modules/tls/tls-private-key.tf
similarity index 100%
rename from modules/tls/tls-private-key.tf
rename to examples/private-cluster/modules/tls/tls-private-key.tf
diff --git a/modules/tls/versions.tf b/examples/private-cluster/modules/tls/versions.tf
similarity index 100%
rename from modules/tls/versions.tf
rename to examples/private-cluster/modules/tls/versions.tf
diff --git a/examples/private-cluster/terraform.tfvars.example b/examples/private-cluster/terraform.tfvars.example
index c85bd88..43bd0f8 100644
--- a/examples/private-cluster/terraform.tfvars.example
+++ b/examples/private-cluster/terraform.tfvars.example
@@ -12,6 +12,7 @@ region = ""
 home_region = ""
 
 # SSH keys
+create_ssh_key_pair = ""
 # ssh_private_key    = <<EOT
 #-----BEGIN RSA PRIVATE KEY-----
 #content+of+api+key
diff --git a/examples/private-cluster/variables.tf b/examples/private-cluster/variables.tf
index 90246c6..3a07a30 100644
--- a/examples/private-cluster/variables.tf
+++ b/examples/private-cluster/variables.tf
@@ -52,6 +52,12 @@ variable "user_id" {
 }
 
 # ssh keys
+variable "create_ssh_key_pair" {
+  default     = false
+  description = "Whether to create an SSH key pair. If set to true, a new SSH key pair is created."
+  type        = bool
+}
+
 variable "ssh_private_key" {
   default     = ""
   description = "The contents of the private ssh key file, optionally base64-encoded."
diff --git a/main.tf b/main.tf
index 435bef8..b789912 100644
--- a/main.tf
+++ b/main.tf
@@ -4,12 +4,6 @@ module "compartment" {
   name = var.name
 }
 
-module "tls" {
-  source = "./modules/tls"
-
-  count = var.create_ssh_key_pair == true ? 1 : 0
-}
-
 module "oke" {
   source  = "oracle-terraform-modules/oke/oci"
   version = "4.5.9"
@@ -20,9 +14,9 @@ module "oke" {
   home_region = var.home_region
 
   # ssh keys
-  ssh_private_key      = var.create_ssh_key_pair ? chomp(module.tls[0].ssh_private_key) : var.ssh_private_key
+  ssh_private_key      = var.ssh_private_key
   ssh_private_key_path = var.ssh_private_key_path
-  ssh_public_key       = var.create_ssh_key_pair ? chomp(module.tls[0].ssh_public_key) : var.ssh_public_key
+  ssh_public_key       = var.ssh_public_key
   ssh_public_key_path  = var.ssh_public_key_path
 
   # general oci parameters
@@ -99,21 +93,3 @@ module "bastion_service_workers" {
   bastion_service_target_subnet = module.oke.subnet_ids["workers"]
   vcn_id                        = module.oke.vcn_id
 }
-
-module "kubernetes" {
-  source = "./modules/kubernetes"
-
-  control_plane_bastion_service_id = module.bastion_service_control_plane.bastion_id
-  workers_bastion_service_id       = module.bastion_service_workers.bastion_id
-
-  # ssh keys
-  ssh_private_key      = var.create_ssh_key_pair ? chomp(module.tls[0].ssh_private_key) : var.ssh_private_key
-  ssh_private_key_path = var.ssh_private_key_path
-  ssh_public_key       = var.create_ssh_key_pair ? chomp(module.tls[0].ssh_public_key) : var.ssh_public_key
-  ssh_public_key_path  = var.ssh_public_key_path
-
-  cluster_id        = module.oke.cluster_id
-  cluster_endpoints = module.oke.cluster_endpoints
-
-  region = var.region
-}
\ No newline at end of file
diff --git a/outputs.tf b/outputs.tf
index fe02059..4fded25 100644
--- a/outputs.tf
+++ b/outputs.tf
@@ -1,3 +1,8 @@
+output "cluster_id" {
+  description = "ID of the Kubernetes cluster"
+  value       = module.oke.cluster_id
+}
+
 output "cluster_endpoints" {
   description = "Endpoints for the Kubernetes cluster"
   value       = module.oke.cluster_endpoints
diff --git a/variables.tf b/variables.tf
index 1bac07a..071835a 100644
--- a/variables.tf
+++ b/variables.tf
@@ -29,12 +29,6 @@ variable "label_prefix" {
 }
 
 # SSH keys
-variable "create_ssh_key_pair" {
-  default     = false
-  description = "Whether to create an SSH key pair. If set to true, a new SSH key pair is created."
-  type        = bool
-}
-
 variable "ssh_private_key" {
   default     = ""
   description = "The contents of the private ssh key file, optionally base64-encoded."