From 67ff8a14ebd506f9db2a5db3abd0822e41b0f9a9 Mon Sep 17 00:00:00 2001 From: Marques Johansson Date: Mon, 15 Nov 2021 22:03:23 -0500 Subject: [PATCH 1/7] include kubeconfig filename in outputs, make content sensitive Signed-off-by: Marques Johansson --- .gitignore | 3 +++ modules/controller_pool/main.tf | 2 +- modules/controller_pool/outputs.tf | 12 +++++++++--- outputs.tf | 8 +++++++- 4 files changed, 20 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 7db02e6..667dd2a 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,6 @@ *.tfvars examples/inventory.yaml examples/main.retry +kubeconfig +metal-key +metal-key.pub diff --git a/modules/controller_pool/main.tf b/modules/controller_pool/main.tf index 49c0f0e..b4f31a2 100644 --- a/modules/controller_pool/main.tf +++ b/modules/controller_pool/main.tf @@ -83,7 +83,7 @@ resource "null_resource" "kubeconfig" { } data "local_file" "kubeconfig" { - filename = "${path.root}/kubeconfig" + filename = abspath("${path.root}/kubeconfig") depends_on = [ null_resource.kubeconfig diff --git a/modules/controller_pool/outputs.tf b/modules/controller_pool/outputs.tf index cc4b764..1d362ea 100644 --- a/modules/controller_pool/outputs.tf +++ b/modules/controller_pool/outputs.tf @@ -9,6 +9,12 @@ output "controller_addresses" { # } output "kubeconfig" { - description = "Kubeconfig for the newly created cluster" - value = data.local_file.kubeconfig -} \ No newline at end of file + description = "Kubeconfig content for the newly created cluster" + value = data.local_file.kubeconfig.content + sensitive = true +} + +output "kubeconfig_filename" { + description = "Kubeconfig file for the newly created cluster" + value = data.local_file.kubeconfig.filename +} diff --git a/outputs.tf b/outputs.tf index 14c1840..770baf6 100644 --- a/outputs.tf +++ b/outputs.tf @@ -10,4 +10,10 @@ output "kubernetes_api_address" { output "kubernetes_kubeconfig" { description = "Kubeconfig for the newly created cluster" value = module.controllers.kubeconfig -} \ No newline at end of file + sensitive = true +} + +output "kubernetes_kubeconfig_file" { + description = "Kubecobnfig file for the newly created cluster" + value = module.controllers.kubeconfig_filename +} From c580497dbebdba918c2cbd57731209dace732ebd Mon Sep 17 00:00:00 2001 From: Marques Johansson Date: Mon, 15 Nov 2021 22:26:35 -0500 Subject: [PATCH 2/7] enable kube-vip and ccm in integration tests Signed-off-by: Marques Johansson --- .github/workflows/integration.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 4228996..c504873 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -14,6 +14,10 @@ jobs: name: Integration Tests runs-on: ${{ matrix.os }} env: + TF_VAR_secrets_encryption: false + TF_VAR_metro: "sv" + TF_VAR_ccm_enabled: true + TF_VAR_loadbalancer_type: "kube-vip" TF_IN_AUTOMATION: 1 TF_VERSION: ${{ matrix.tf }} TF_VAR_control_plane_node_count: 0 From d78def2fa229e1d568ee2b8b78bef554cfe70d46 Mon Sep 17 00:00:00 2001 From: Keith Mattix II Date: Thu, 18 Nov 2021 10:45:43 -0600 Subject: [PATCH 3/7] Add kubeconfig file for kube-vip apply commands --- modules/controller_pool/controller-primary.tpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/controller_pool/controller-primary.tpl b/modules/controller_pool/controller-primary.tpl index 12a7826..dad42a0 100644 --- a/modules/controller_pool/controller-primary.tpl +++ b/modules/controller_pool/controller-primary.tpl @@ -108,7 +108,7 @@ EOF } function kube_vip { - kubectl apply -f https://kube-vip.io/manifests/rbac.yaml + kubectl --kubeconfig=/etc/kubernetes/admin.conf apply -f https://kube-vip.io/manifests/rbac.yaml GATEWAY_IP=$(curl https://metadata.platformequinix.com/metadata | jq -r ".network.addresses[] | select(.public == false) | .gateway"); ip route add 169.254.255.1 via $GATEWAY_IP ip route add 169.254.255.2 via $GATEWAY_IP @@ -118,7 +118,7 @@ function kube_vip { --services \ --bgp \ --annotations metal.equinix.com \ - --inCluster | kubectl apply -f - + --inCluster | kubectl --kubeconfig=/etc/kubernetes/admin.conf apply -f - } function ceph_pre_check { From d87ce6d85971b4ac35732e8a1c800839b97d56fe Mon Sep 17 00:00:00 2001 From: Keith Mattix II Date: Sat, 20 Nov 2021 17:33:23 -0600 Subject: [PATCH 4/7] Add even more time to wait for kube-vip cases --- modules/controller_pool/assets/kubeconfig_copy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/controller_pool/assets/kubeconfig_copy.sh b/modules/controller_pool/assets/kubeconfig_copy.sh index d6aa93e..5391fa0 100755 --- a/modules/controller_pool/assets/kubeconfig_copy.sh +++ b/modules/controller_pool/assets/kubeconfig_copy.sh @@ -1,5 +1,5 @@ #!/bin/bash /usr/bin/ssh -i $ssh_private_key_path -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@$controller "while true; do if ! type kubeadm > /dev/null; then sleep 20; else break; fi; done" -sleep 360 +sleep 520 /usr/bin/scp -i $ssh_private_key_path -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -q root@$controller:/etc/kubernetes/admin.conf $local_path/kubeconfig; From 33080c4d0eb15876a52d1ca7e5820f1932ef4e2a Mon Sep 17 00:00:00 2001 From: Keith Mattix II Date: Sat, 20 Nov 2021 18:05:01 -0600 Subject: [PATCH 5/7] Remove kube-vip alias --- modules/controller_pool/controller-primary.tpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/controller_pool/controller-primary.tpl b/modules/controller_pool/controller-primary.tpl index dad42a0..d7663c7 100644 --- a/modules/controller_pool/controller-primary.tpl +++ b/modules/controller_pool/controller-primary.tpl @@ -112,8 +112,8 @@ function kube_vip { GATEWAY_IP=$(curl https://metadata.platformequinix.com/metadata | jq -r ".network.addresses[] | select(.public == false) | .gateway"); ip route add 169.254.255.1 via $GATEWAY_IP ip route add 169.254.255.2 via $GATEWAY_IP - alias kube-vip="docker run --network host --rm ghcr.io/kube-vip/kube-vip:v0.3.8" - kube-vip manifest daemonset \ + alias kube-vip="" + docker run --network host --rm ghcr.io/kube-vip/kube-vip:v0.3.8 manifest daemonset \ --interface lo \ --services \ --bgp \ From 7bd61ec4c4c92400c705a7b084e54830cdd5ba22 Mon Sep 17 00:00:00 2001 From: Keith Mattix II Date: Mon, 29 Nov 2021 15:02:12 -0600 Subject: [PATCH 6/7] Apply suggestions from code review Co-authored-by: Marques Johansson --- modules/controller_pool/controller-primary.tpl | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/controller_pool/controller-primary.tpl b/modules/controller_pool/controller-primary.tpl index d7663c7..095308d 100644 --- a/modules/controller_pool/controller-primary.tpl +++ b/modules/controller_pool/controller-primary.tpl @@ -112,7 +112,6 @@ function kube_vip { GATEWAY_IP=$(curl https://metadata.platformequinix.com/metadata | jq -r ".network.addresses[] | select(.public == false) | .gateway"); ip route add 169.254.255.1 via $GATEWAY_IP ip route add 169.254.255.2 via $GATEWAY_IP - alias kube-vip="" docker run --network host --rm ghcr.io/kube-vip/kube-vip:v0.3.8 manifest daemonset \ --interface lo \ --services \ From 2855ee7390f33910636a8d279347be990f00f791 Mon Sep 17 00:00:00 2001 From: Marques Johansson Date: Thu, 9 Dec 2021 10:07:09 -0500 Subject: [PATCH 7/7] update kube-vip to 0.4.0 to remove need of static ToR routes Signed-off-by: Marques Johansson --- modules/controller_pool/controller-primary.tpl | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/modules/controller_pool/controller-primary.tpl b/modules/controller_pool/controller-primary.tpl index 095308d..e614d43 100644 --- a/modules/controller_pool/controller-primary.tpl +++ b/modules/controller_pool/controller-primary.tpl @@ -109,10 +109,7 @@ EOF function kube_vip { kubectl --kubeconfig=/etc/kubernetes/admin.conf apply -f https://kube-vip.io/manifests/rbac.yaml - GATEWAY_IP=$(curl https://metadata.platformequinix.com/metadata | jq -r ".network.addresses[] | select(.public == false) | .gateway"); - ip route add 169.254.255.1 via $GATEWAY_IP - ip route add 169.254.255.2 via $GATEWAY_IP - docker run --network host --rm ghcr.io/kube-vip/kube-vip:v0.3.8 manifest daemonset \ + docker run --network host --rm ghcr.io/kube-vip/kube-vip:v0.4.0 manifest daemonset \ --interface lo \ --services \ --bgp \