From 57854ac077206e93f154efbfcb6d8376c068a1ea Mon Sep 17 00:00:00 2001 From: Julian Katz Date: Mon, 8 May 2023 08:19:43 -0700 Subject: [PATCH 1/7] feat(create_snapshot.py): Add validating webhook configs (#596) * Add validating webhook configs to create_snapshot.py Many Anthos products (Policy Controller, for example) use ValidatingWebhookConfigurations. As webhooks are called in requests to the API server, they are also a common source of problems on the cluster. Adding them to this script will better enable engineers to understand the state of the cluster. * Update troubleshooting/create_snapshot.py Co-authored-by: Andrew Peabody * Update troubleshooting/create_snapshot.py change backtick to single quote Co-authored-by: Andrew Peabody * Update troubleshooting/create_snapshot.py Co-authored-by: Andrew Peabody * Apply suggestions from code review add noqa comment Co-authored-by: Andrew Peabody * Apply suggestions from code review Add additional space before comment Co-authored-by: Andrew Peabody * Apply suggestions from code review Add another space before a comment Co-authored-by: Andrew Peabody --------- Co-authored-by: Andrew Peabody Co-authored-by: Shabir Mohamed Abdul Samadh <7249208+Shabirmean@users.noreply.github.com> --- troubleshooting/create_snapshot.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/troubleshooting/create_snapshot.py b/troubleshooting/create_snapshot.py index 439eebf3..058c600e 100644 --- a/troubleshooting/create_snapshot.py +++ b/troubleshooting/create_snapshot.py @@ -61,6 +61,8 @@ 'kubectl describe clusterrolebindings {kubeconfig_arg} --request-timeout {timeout}', # noqa: E501 'kubectl describe crd {kubeconfig_arg} --request-timeout {timeout}', 'kubectl describe nodes {kubeconfig_arg} --request-timeout {timeout}', + 'kubectl get validatingwebhookconfigurations -o wide {kubeconfig_arg} --request-timeout {timeout}', # noqa: E501 + 'kubectl get validatingwebhookconfigurations -o yaml {kubeconfig_arg} --request-timeout {timeout}', # noqa: E501 ] KUBECTL_PER_NS_CMDS = [ From ceabb9206bc0e67ac20f9e6ebc04d06bfdfd8271 Mon Sep 17 00:00:00 2001 From: Curtis Gavin Date: Mon, 8 May 2023 13:30:15 -0700 Subject: [PATCH 2/7] Adding the Anthos On-prem API (gkeonprem.googleapis.com) to the list of APIs that are enabled. (#603) Co-authored-by: Shabir Mohamed Abdul Samadh <7249208+Shabirmean@users.noreply.github.com> --- anthos-bm-gcp-bash/install_admin_cluster.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/anthos-bm-gcp-bash/install_admin_cluster.sh b/anthos-bm-gcp-bash/install_admin_cluster.sh index 3298a52b..9e811f96 100644 --- a/anthos-bm-gcp-bash/install_admin_cluster.sh +++ b/anthos-bm-gcp-bash/install_admin_cluster.sh @@ -66,6 +66,7 @@ gcloud services enable \ container.googleapis.com \ gkeconnect.googleapis.com \ gkehub.googleapis.com \ + gkeonprem.googleapis.com \ serviceusage.googleapis.com \ stackdriver.googleapis.com \ monitoring.googleapis.com \ From 3e958847d903dd9645da3632e917995dab550f7a Mon Sep 17 00:00:00 2001 From: Curtis Gavin Date: Mon, 8 May 2023 13:33:18 -0700 Subject: [PATCH 3/7] Adding the Anthos On-prem API (gkeonprem.googleapis.com) to the list of APIs that are enabled. (#603) Co-authored-by: Shabir Mohamed Abdul Samadh <7249208+Shabirmean@users.noreply.github.com> From eecb1da249ac6a2deab32402137c4421714ab6d7 Mon Sep 17 00:00:00 2001 From: Shabir Mohamed Abdul Samadh <7249208+Shabirmean@users.noreply.github.com> Date: Mon, 15 May 2023 19:34:38 -0400 Subject: [PATCH 4/7] feature: add user cluster create with terraform sample (metallb) (#593) * chore: add samples for metalln usercluster terraform * chore: add cluster name variable * doc: add intro to user cluster with manuallb * doc: update readme for the user cluster create * linkt: fix errors * lint: fix linting errors * lint: fix linting errors * chore: update tfvars files * chore: add dynamic admin user block * chore: add region tags * chore: update tfvars sample * doc: update readme * chore: fix sample terraform vars file * lint: add whitespace at end * feature: add sample for Anthos bare metal on VMware (metal-lb) (#597) * Add sample for Anthos VMware on metal lb * Update sample file with placeholders * Remove ignored annotations as the field is defaulted * reformat sample file * fix lint error * fix lint error by running terraform fmt * change back the update on abm directory * fix: update TF sample files of AVMW for beta release (#604) * Update provider to google-beta * Add enable_load_balancer in node pool * update lb_address_pools sample value to keep consistent with ABM sample * add authorization field to support admin_users var * Add instructions of preps for cluster upgrade * remove google-private in versions and add admin_users in variable * update the var name for admin_cluster and readme * Correct gkectl command in readme file * Update provider in ABM samples to google-beta * Add upgrade via gcloud command example in readme * update description for upgrade in readme file * chore: add readme link to AVMW * chore: fix docs --------- Co-authored-by: Naitian Liu <83430653+naitianliu-google@users.noreply.github.com> --- anthos-bm-gcp-terraform/README.md | 48 +++++- anthos-multi-cloud/Azure/provider.tf | 9 +- anthos-onprem-terraform/README.md | 37 ++++ .../abm_user_cluster_metallb/README.md | 99 +++++++++++ .../abm_user_cluster_metallb/main.tf | 163 ++++++++++++++++++ .../terraform.tfvars.sample | 17 ++ .../abm_user_cluster_metallb/variables.tf | 135 +++++++++++++++ .../abm_user_cluster_metallb/versions.tf | 43 +++++ .../avmw_user_cluster_metallb/README.md | 91 ++++++++++ .../avmw_user_cluster_metallb/main.tf | 112 ++++++++++++ .../terraform.tfvars.sample | 14 ++ .../avmw_user_cluster_metallb/variables.tf | 129 ++++++++++++++ .../avmw_user_cluster_metallb/versions.tf | 43 +++++ 13 files changed, 930 insertions(+), 10 deletions(-) create mode 100644 anthos-onprem-terraform/README.md create mode 100644 anthos-onprem-terraform/abm_user_cluster_metallb/README.md create mode 100644 anthos-onprem-terraform/abm_user_cluster_metallb/main.tf create mode 100644 anthos-onprem-terraform/abm_user_cluster_metallb/terraform.tfvars.sample create mode 100644 anthos-onprem-terraform/abm_user_cluster_metallb/variables.tf create mode 100644 anthos-onprem-terraform/abm_user_cluster_metallb/versions.tf create mode 100644 anthos-onprem-terraform/avmw_user_cluster_metallb/README.md create mode 100644 anthos-onprem-terraform/avmw_user_cluster_metallb/main.tf create mode 100644 anthos-onprem-terraform/avmw_user_cluster_metallb/terraform.tfvars.sample create mode 100644 anthos-onprem-terraform/avmw_user_cluster_metallb/variables.tf create mode 100644 anthos-onprem-terraform/avmw_user_cluster_metallb/versions.tf diff --git a/anthos-bm-gcp-terraform/README.md b/anthos-bm-gcp-terraform/README.md index 066de45c..d2206138 100644 --- a/anthos-bm-gcp-terraform/README.md +++ b/anthos-bm-gcp-terraform/README.md @@ -1,7 +1,14 @@ +> #### This is a terraform script to provision the GCE infrastructure in which Anthos clusters on bare metal (ABM) will be installed. The cluster installation is done using `bash` scripts. This does NOT use the `google_gkeonprem_*` terraform resources of the `google` provider to create the ABM cluster. If you are looking for a sample that is fully based off of the terraform provider, see the [anthos-onprem-terraform](/anthos-onprem-terraform/) directory. +--- ## Anthos Baremetal on Google Compute Engine VMs with Terraform -This repository shows you how to use Terraform to try Anthos clusters on bare metal in High Availability (HA) mode using Virtual Machines (VMs) running on Compute Engine. For information about how to use the `gcloud` command-line tool to try this, see [Try Anthos clusters on bare metal on Compute Engine VMs](https://cloud.google.com/anthos/clusters/docs/bare-metal/latest/try/gce-vms). +> Read the dosclaimer on top of this README before you continue. + +This repository shows you how to use Terraform to try Anthos clusters on bare +metal in High Availability (HA) mode using Virtual Machines (VMs) running on +Compute Engine. For information about how to use the `gcloud` command-line tool +to try this, see [Try Anthos clusters on bare metal on Compute Engine VMs](https://cloud.google.com/anthos/clusters/docs/bare-metal/latest/try/gce-vms). ### Pre-requisites @@ -12,14 +19,19 @@ This repository shows you how to use Terraform to try Anthos clusters on bare me - A [Google Cloud Project](https://console.cloud.google.com/cloud-resource-manager?_ga=2.187862184.1029435410.1614837439-1338907320.1614299892) _(in which the resources for the setup will be provisioned)_ -- A [Service Account](https://cloud.google.com/iam/docs/creating-managing-service-accounts) in the project that satisfies **one** of the following requirements and its **[key file downloaded](docs/create_sa_key.md)** to the workstation: +- A [Service Account](https://cloud.google.com/iam/docs/creating-managing-service-accounts) + in the project that satisfies **one** of the following requirements and its + **[key file downloaded](docs/create_sa_key.md)** to the workstation: - The Service Account has `Owner` permissions - The Service Account has both `Editor` and `Project IAM Admin` permissions --- ### Bare metal infrastructure on Google Cloud using Compute Engine VMs -The [Quick starter](docs/quickstart.md) guide sets up the following infrastructure in Google Cloud using Compute Engine VMs. The diagram assumes that the none of the default values for the [variables](variables.tf) were changed other than the ones mentioned in the quick starter. +The [Quick starter](docs/quickstart.md) guide sets up the following +infrastructure in Google Cloud using Compute Engine VMs. The diagram assumes +that the none of the default values for the [variables](variables.tf) were +changed other than the ones mentioned in the quick starter. ![Bare metal infrastructure on Google Cloud using Compute Engine VMs](docs/images/abm_gcp_infra.svg) @@ -29,22 +41,42 @@ The [Quick starter](docs/quickstart.md) guide sets up the following infrastructu - [Terraform Module Information _(includes variables definitions)_](docs/variables.md) - [Quick start guide](docs/quickstart.md): - - The terraform script sets up the GCE VM environment. The output of the script prints out the commands to follow to install **Anthos on bare metal** in the provisioned GCE VMs. + - The terraform script sets up the GCE VM environment. The output of the + script prints out the commands to follow to install + **Anthos on bare metal** in the provisioned GCE VMs. - [All in one install](docs/one_click_install.md): - - The terraform script sets up the GCE VM environment and also triggers the **Anthos on bare metal** installation on the provisioned GCE VMs. The output of the script prints out the commands to SSH into the *admin workstation VM* and monitor the Anthos on bare metal installation process. + - The terraform script sets up the GCE VM environment and also triggers the + **Anthos on bare metal** installation on the provisioned GCE VMs. The + output of the script prints out the commands to SSH into the + *admin workstation VM* and monitor the Anthos on bare metal installation + process. - [Manual LB install](docs/manuallb_install.md): - - The terraform script sets up the GCE environment and triggers the **Anthos on bare metal** installation similar to the [all-in-one install](docs/one_click_install.md). However, in this mode **Anthos on bare metal** is installed with a [`Manual Loadbalancer`](https://cloud.google.com/anthos/clusters/docs/bare-metal/latest/installing/manual-lb) instead of the default [`Bundled LB`](https://cloud.google.com/anthos/clusters/docs/bare-metal/latest/installing/bundled-lb). We use [Google Cloud Loadbalancer](https://cloud.google.com/load-balancing/docs/load-balancing-overview) as the manual loadbalancer for the Anthos on bare metal cluster. The output of the script prints out the same instructions as the all-in-one install; additionally it also prints out the **Public IP** addresses of the loadbalancers. + - The terraform script sets up the GCE environment and triggers the + **Anthos on bare metal** installation similar to the + [all-in-one install](docs/one_click_install.md). However, in this mode + **Anthos on bare metal** is installed with a + [`Manual Loadbalancer`](https://cloud.google.com/anthos/clusters/docs/bare-metal/latest/installing/manual-lb) instead of the default + [`Bundled LB`](https://cloud.google.com/anthos/clusters/docs/bare-metal/latest/installing/bundled-lb). + We use + [Google Cloud Loadbalancer](https://cloud.google.com/load-balancing/docs/load-balancing-overview) + as the manual loadbalancer for the Anthos on bare metal cluster. The + output of the script prints out the same instructions as the all-in-one + install; additionally it also prints out the **Public IP** addresses of + the loadbalancers. - [NFS Shared Storage](docs/nfs.md): - - An optional NFS server is provisioned in conjunction with any of the install methods above to provide shared storage to the **Anthos on bare metal** cluster. + - An optional NFS server is provisioned in conjunction with any of the + install methods above to provide shared storage to the + **Anthos on bare metal** cluster. --- ## Contributing #### Pre-requisites -- The same [pre-requisites](#pre-requisites) to run this sample is required for testing as well +- The same [pre-requisites](#pre-requisites) to run this sample is required for + testing as well #### Pull requests - For improvements to this sample submit your pull requests to the `main` branch diff --git a/anthos-multi-cloud/Azure/provider.tf b/anthos-multi-cloud/Azure/provider.tf index 3e47cf96..3278f8ca 100644 --- a/anthos-multi-cloud/Azure/provider.tf +++ b/anthos-multi-cloud/Azure/provider.tf @@ -17,7 +17,8 @@ terraform { required_providers { azuread = { - source = "hashicorp/azuread" + source = "hashicorp/azuread" + version = ">= 2.14.0" } azurerm = { source = "hashicorp/azurerm" @@ -35,8 +36,12 @@ provider "azurerm" { } provider "azuread" { - version = ">= 2.14.0" + /** + * update this block with your default Azure Active Directory information + * like tenant_id or client_id. + */ } + provider "google" { project = var.gcp_project_id } diff --git a/anthos-onprem-terraform/README.md b/anthos-onprem-terraform/README.md new file mode 100644 index 00000000..1d1d0c5b --- /dev/null +++ b/anthos-onprem-terraform/README.md @@ -0,0 +1,37 @@ +## Install Anthos onPrem clusters using Terraform + +This directory hosts samples and how-to's for installing Anthos onPrem clusters +_(i.e. Anthos on bare metal and Anthos on VMware)_ using the +`google_gkeonprem-*` resources of the canonical +[Google Cloud Terraform provider](https://registry.terraform.io/providers/hashicorp/google/latest/docs). + + +All the samples here, first **emulates** a bare metal infrastucture using +Compute Engine (GCE) VMs. It is on this imaginary bare metal environment the +clusters are installed. Thus, as a prestep to all the guides here, you will see +a step for provisioning the bare metal insfrastructure. If you have your own +bare metal infrastructure, you can skip that section and adjust the sample to +suit your environment. + + +--- + +### Anthos clusters on bare metal (ABM) +- Create **admin** clusters with Terraform `(coming soon)` + - See [guide](https://cloud.google.com/anthos/clusters/docs/bare-metal/latest/installing/creating-clusters/create-admin-cluster-api) for creating an admin cluster using other clients +- Create **user** clusters with Terraform + - [MetalLB](./abm_user_cluster_metallb/) + - ManualLB `(coming soon)` +- Create **standalone** clusters with Terraform `(coming soon)` +--- + +### Anthos clusters on VMware (AVMware) +- Create **admin** clusters with Terraform `(coming soon)` +- Create **user** clusters with Terraform + - [MetalLB](./avmw_user_cluster_metallb/) + - ManualLB `(coming soon)` +- Create **standalone** clusters with Terraform `(coming soon)` + +--- diff --git a/anthos-onprem-terraform/abm_user_cluster_metallb/README.md b/anthos-onprem-terraform/abm_user_cluster_metallb/README.md new file mode 100644 index 00000000..bd6df1dc --- /dev/null +++ b/anthos-onprem-terraform/abm_user_cluster_metallb/README.md @@ -0,0 +1,99 @@ +## Create Anthos on bare metal **user** clusters (MetalLB) with Terraform + +The steps here acheive the same result as what is explained in the +[Create an Anthos on bare metal user cluster on Compute Engine VMs using Anthos On-Prem API clients](https://cloud.google.com/anthos/clusters/docs/bare-metal/latest/try/admin-user-gce-vms) +public documentation. We show an example of how to create an Anthos on bare +metal **user cluster** with **MetalLB** using the Google provider for Terraform. + +The sample here has a prerequisite step of creating an **admin cluster** using +the [script available in this repository](/anthos-bm-gcp-bash/install_admin_cluster.sh). +Thus, the default variables _(especially IP addresses)_ used in this sample are +based on the assumption that the admin cluster and the GCE VM based bare metal +infrstructure for the user cluster was created using that script. If you +already have an **admin cluster and bare metal nodes for the user cluster**, you +may skip running this script. However, you will have to update the sample to use +values appropriate to your environment. + +--- +### Prerequisite + +#### Create admin cluster and VMs for user cluster + +> **Note:** If you already have an admin cluster and the bare metal nodes for +> a new user cluster, then you can skip this step. But don't forget to update +> the sample with the values appropriate to your environment. + +- First you will have to create an admin cluster that will manage your user + cluster. At the time of writing this guide, admin cluster creation + **using Terraform** is not supported. + +- Follow the [instructions here](/anthos-bm-gcp-bash/docs/admin.md) to create an + admin cluster and to provision the GCE VMs for the user cluster using the + [install_admin_cluster.sh](/anthos-bm-gcp-bash/install_admin_cluster.sh) + script. + +- Upon completion, you will see the node information for the GCE VMs printed on + screen. + + ```sh + |---------------------------------------------------------------------------------------------------------| + | VM Name | L2 Network IP (VxLAN) | INFO | + |---------------------------------------------------------------------------------------------------------| + | abm-admin-cluster-cp1 | 10.200.0.3 | Has control plane of admin cluster running inside | + | abm-user-cluster-cp1 | 10.200.0.4 | 🌟 Ready for use as control plane for the user cluster | + | abm-user-cluster-w1 | 10.200.0.5 | 🌟 Ready for use as worker for the user cluster | + | abm-user-cluster-w2 | 10.200.0.6 | 🌟 Ready for use as worker for the user cluster | + |---------------------------------------------------------------------------------------------------------| + ``` + +#### Create the user cluster with terraform + +The steps that follow assumes that you already have this repo cloned locally and +have changed directory to where this samples is: +`/anthos-onprem-terraform/abm_user_cluster_metallb`. + +- Make a copy of the `terraform.tfvars.sample` file: + + ```sh + cp terraform.tfvars.sample terraform.tfvars + ``` + The sample terraform vaiables file has most of the default values filled in + based on the output of the [install_admin_cluster.sh](/anthos-bm-gcp-bash/install_admin_cluster.sh) + script from the previous section. + +- Update missing variables in the `terraform.tfvars` file: + - **`project_id`**: The GCP project of the admin cluster and where the user + cluster will be created. + - **`region`**: The Google Cloud region in which the Anthos On-Prem API + runs. + - **`admin_cluster_name`**: The name of the admin cluster that will manage the + new user cluster. If you used the [install_admin_cluster.sh](/anthos-bm-gcp-bash/install_admin_cluster.sh) + script and used the default name, then this must be `abm-admin-cluster`. + - **`bare_metal_version`**: The Anthos clusters on bare metal version for + your user cluster. This must be same as the admin cluster version or one + minor version less, at most. It cannot be higher in any case - minor or + patch. + + - **`admin_user_emails`**: List of GCP accounts that must be given + administrator rights on the user cluster. + +- Execute terraform: + + ```sh + terraform init + ``` + ```sh + terraform plan + ``` + ```sh + terraform apply + ``` + + Once completed you will see an output as follows: + ```sh + ... + + ``` + + You can view your user cluster in the + [Anthos clusters page](https://console.cloud.google.com/anthos/clusters). diff --git a/anthos-onprem-terraform/abm_user_cluster_metallb/main.tf b/anthos-onprem-terraform/abm_user_cluster_metallb/main.tf new file mode 100644 index 00000000..34cab41d --- /dev/null +++ b/anthos-onprem-terraform/abm_user_cluster_metallb/main.tf @@ -0,0 +1,163 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#[START anthos_onprem_terraform_bare_metal_user_cluster_metallb_main] +module "enable_google_apis_primary" { + source = "terraform-google-modules/project-factory/google//modules/project_services" + version = "~> 14.0" + project_id = var.project_id + activate_apis = [ + "cloudresourcemanager.googleapis.com", + "anthos.googleapis.com", + "anthosgke.googleapis.com", + "container.googleapis.com", + "gkeconnect.googleapis.com", + "gkehub.googleapis.com", + "serviceusage.googleapis.com", + "stackdriver.googleapis.com", + "monitoring.googleapis.com", + "logging.googleapis.com", + "iam.googleapis.com", + "compute.googleapis.com", + "anthosaudit.googleapis.com", + "opsconfigmonitoring.googleapis.com", + "file.googleapis.com", + "connectgateway.googleapis.com" + ] + disable_services_on_destroy = false +} + +# Enable GKE OnPrem API +resource "google_project_service" "default" { + project = var.project_id + service = "gkeonprem.googleapis.com" + disable_on_destroy = false +} + +# Create an anthos baremetal user cluster and enroll it with the gkeonprem API +resource "google_gkeonprem_bare_metal_cluster" "default" { + name = var.cluster_name + description = "Anthos bare metal user cluster with MetalLB" + provider = google-beta + depends_on = [google_project_service.default] + location = var.region + bare_metal_version = var.bare_metal_version + admin_cluster_membership = "projects/${var.project_id}/locations/global/memberships/${var.admin_cluster_name}" + network_config { + island_mode_cidr { + service_address_cidr_blocks = ["172.26.0.0/16"] + pod_address_cidr_blocks = ["10.240.0.0/13"] + } + } + control_plane { + control_plane_node_pool_config { + node_pool_config { + operating_system = "LINUX" + dynamic "node_configs" { + for_each = var.control_plane_ips + content { + node_ip = node_configs.value + } + } + } + } + } + load_balancer { + port_config { + control_plane_load_balancer_port = 443 + } + vip_config { + control_plane_vip = var.control_plane_vip + ingress_vip = var.ingress_vip + } + metal_lb_config { + dynamic "address_pools" { + for_each = var.lb_address_pools + content { + pool = address_pools.value.name + addresses = address_pools.value.addresses + } + } + } + } + storage { + lvp_share_config { + lvp_config { + path = "/mnt/localpv-share" + storage_class = "local-shared" + } + shared_path_pv_count = 5 + } + lvp_node_mounts_config { + path = "/mnt/localpv-disk" + storage_class = "local-disks" + } + } + + dynamic "security_config" { + for_each = length(var.admin_user_emails) == 0 ? [] : [1] + content { + authorization { + dynamic "admin_users" { + for_each = var.admin_user_emails + content { + username = admin_users.value + } + } + } + } + } + + lifecycle { + ignore_changes = [ + annotations["onprem.cluster.gke.io/user-cluster-resource-link"], + annotations["alpha.baremetal.cluster.gke.io/cluster-metrics-webhook"], + annotations["baremetal.cluster.gke.io/operation"], + annotations["baremetal.cluster.gke.io/operation-id"], + annotations["baremetal.cluster.gke.io/start-time"], + annotations["baremetal.cluster.gke.io/upgrade-from-version"] + ] + } +} + +# Create a node pool of worker nodes for the anthos baremetal user cluster +resource "google_gkeonprem_bare_metal_node_pool" "default" { + name = "${var.cluster_name}-nodepool" + display_name = "Nodepool for ${var.cluster_name}" + provider = google-beta + bare_metal_cluster = google_gkeonprem_bare_metal_cluster.default.name + location = var.region + node_pool_config { + operating_system = "LINUX" + labels = {} + + dynamic "node_configs" { + for_each = var.worker_node_ips + content { + labels = {} + node_ip = node_configs.value + } + } + } + + lifecycle { + ignore_changes = [ + annotations["baremetal.cluster.gke.io/gke-version"], + annotations["baremetal.cluster.gke.io/version"], + ] + } +} +#[END anthos_onprem_terraform_bare_metal_user_cluster_metallb_main] diff --git a/anthos-onprem-terraform/abm_user_cluster_metallb/terraform.tfvars.sample b/anthos-onprem-terraform/abm_user_cluster_metallb/terraform.tfvars.sample new file mode 100644 index 00000000..0436ef16 --- /dev/null +++ b/anthos-onprem-terraform/abm_user_cluster_metallb/terraform.tfvars.sample @@ -0,0 +1,17 @@ +# [START anthos_onprem_terraform_bare_metal_user_cluster_metallb_tfvars] + +project_id = "PROJECT_ID" +region = "ON_PREM_API_REGION" +admin_cluster_name = "ADMIN_CLUSTER_NAME" +bare_metal_version = "VERSION" +admin_user_emails = ["YOUR_EMAIL_ADDRESS", "ADMIN_2_EMAIL_ADDRESS"] +cluster_name = "abm-user-cluster-metallb" +control_plane_ips = ["10.200.0.4"] +worker_node_ips = ["10.200.0.5", "10.200.0.6"] +control_plane_vip = "10.200.0.50" +ingress_vip = "10.200.0.51" +lb_address_pools = [ + { name = "lbpool_1", addresses = ["10.200.0.51-10.200.0.70"] } +] + +# [END anthos_onprem_terraform_bare_metal_user_cluster_metallb_tfvars] diff --git a/anthos-onprem-terraform/abm_user_cluster_metallb/variables.tf b/anthos-onprem-terraform/abm_user_cluster_metallb/variables.tf new file mode 100644 index 00000000..cc566974 --- /dev/null +++ b/anthos-onprem-terraform/abm_user_cluster_metallb/variables.tf @@ -0,0 +1,135 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Unique identifer of the Google Cloud Project that is to be used" + type = string +} + +variable "region" { + description = </anthos-onprem-terraform/avmw_user_cluster_metallb`. + +- Make a copy of the `terraform.tfvars.sample` file: + + ```sh + cp terraform.tfvars.sample terraform.tfvars + ``` + The sample terraform variables file has most of the default values filled in. + +- Update missing variables in the `terraform.tfvars` file: + - **`project_id`**: The GCP project of the admin cluster and where the user + cluster will be created. + - **`region`**: The Google Cloud region in which the Anthos On-Prem API + runs. + - **`admin_cluster_name`**: The name of the admin cluster that will manage the + new user cluster. + - **`on_prem_version`**: The Anthos clusters on VMware version for + your user cluster. + +- Execute terraform: + + ```sh + terraform init + ``` + ```sh + terraform plan + ``` + ```sh + terraform apply + ``` + + Once completed you will see an output as follows: + ```sh + ... + + ``` + + You can view your user cluster in the + [Anthos clusters page](https://console.cloud.google.com/anthos/clusters). +--- + +### Upgrade the user cluster with terraform + +Before upgrading the user cluster, please make sure the admin cluster platform +controller has been upgraded to the target version. The steps to upgrade the +admin cluster platform controller is listed in the +[public documentation](https://cloud.google.com/anthos/clusters/docs/on-prem/latest/how-to/upgrading#:~:text=In%20the%20Cluster%20basics%20section,Click%20Upgrade.). + +An example is shown below: + +```bash +gcloud beta container vmware admin-clusters update \ + --required-platform-version= \ + --location \ + --project +``` + +Then, following the steps below to upgrade the user cluster via terraform. + +- Update the version variable in the `terraform.tfvars` file: + - **`on_prem_version`**: The Anthos clusters on VMware version for your user + cluster. + +- Execute terraform: + + ```sh + terraform init + ``` + ```sh + terraform plan + ``` + ```sh + terraform apply + ``` diff --git a/anthos-onprem-terraform/avmw_user_cluster_metallb/main.tf b/anthos-onprem-terraform/avmw_user_cluster_metallb/main.tf new file mode 100644 index 00000000..cee19258 --- /dev/null +++ b/anthos-onprem-terraform/avmw_user_cluster_metallb/main.tf @@ -0,0 +1,112 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#[START anthos_onprem_terraform_vmware_user_cluster_metallb_main] +module "enable_google_apis_primary" { + source = "terraform-google-modules/project-factory/google//modules/project_services" + version = "~> 14.0" + project_id = var.project_id + activate_apis = [ + "cloudresourcemanager.googleapis.com", + "anthos.googleapis.com", + "anthosgke.googleapis.com", + "container.googleapis.com", + "gkeconnect.googleapis.com", + "gkehub.googleapis.com", + "serviceusage.googleapis.com", + "stackdriver.googleapis.com", + "monitoring.googleapis.com", + "logging.googleapis.com", + "iam.googleapis.com", + "compute.googleapis.com", + "anthosaudit.googleapis.com", + "opsconfigmonitoring.googleapis.com", + "file.googleapis.com", + "connectgateway.googleapis.com" + ] + disable_services_on_destroy = false +} + +# Enable GKE OnPrem API +resource "google_project_service" "default" { + project = var.project_id + service = "gkeonprem.googleapis.com" + disable_on_destroy = false +} + +# Create an anthos vmware user cluster and enroll it with the gkeonprem API +resource "google_gkeonprem_vmware_cluster" "default" { + name = var.cluster_name + description = "Anthos VMware user cluster with MetalLB" + provider = google-beta + depends_on = [google_project_service.default] + location = var.region + on_prem_version = var.on_prem_version + admin_cluster_membership = "projects/${var.project_id}/locations/global/memberships/${var.admin_cluster_name}" + network_config { + service_address_cidr_blocks = ["10.96.0.0/12"] + pod_address_cidr_blocks = ["192.168.0.0/16"] + dhcp_ip_config { + enabled = true + } + } + control_plane_node { + cpus = var.control_plane_node_cpus + memory = var.control_plane_node_memory + replicas = var.control_plane_node_replicas + } + load_balancer { + vip_config { + control_plane_vip = var.control_plane_vip + ingress_vip = var.ingress_vip + } + metal_lb_config { + dynamic "address_pools" { + for_each = var.lb_address_pools + content { + pool = address_pools.value.name + addresses = address_pools.value.addresses + } + } + } + } + authorization { + dynamic "admin_users" { + for_each = var.admin_user_emails + content { + username = admin_users.value + } + } + } +} + +# Create a node pool for the anthos vmware user cluster +resource "google_gkeonprem_vmware_node_pool" "default" { + name = "${var.cluster_name}-nodepool" + display_name = "Nodepool for ${var.cluster_name}" + provider = google-beta + vmware_cluster = google_gkeonprem_vmware_cluster.default.name + location = var.region + config { + replicas = 3 + image_type = "ubuntu_containerd" + enable_load_balancer = true + } + depends_on = [ + google_gkeonprem_vmware_cluster.default + ] +} +#[END anthos_onprem_terraform_vmware_user_cluster_metallb_main] diff --git a/anthos-onprem-terraform/avmw_user_cluster_metallb/terraform.tfvars.sample b/anthos-onprem-terraform/avmw_user_cluster_metallb/terraform.tfvars.sample new file mode 100644 index 00000000..10899862 --- /dev/null +++ b/anthos-onprem-terraform/avmw_user_cluster_metallb/terraform.tfvars.sample @@ -0,0 +1,14 @@ +project_id = "FLEET_HOST_PROJECT_ID" +region = "REGION" +admin_cluster_name = "ADMIN_CLUSTER_NAME" +on_prem_version = "VERSION" +admin_user_emails = ["YOUR_EMAIL_ADDRESS", "ADMIN_2_EMAIL_ADDRESS"] +cluster_name = "avmw-user-cluster-metallb" +control_plane_node_cpus = 4 +control_plane_node_memory = 8192 +control_plane_node_replicas = 3 +control_plane_vip = "CONTROL_PLANE_VIP" +ingress_vip = "INGRESS_VIP" +lb_address_pools = [ + { name = "lbpool_1", addresses = ["10.200.0.51-10.200.0.70"] } +] diff --git a/anthos-onprem-terraform/avmw_user_cluster_metallb/variables.tf b/anthos-onprem-terraform/avmw_user_cluster_metallb/variables.tf new file mode 100644 index 00000000..59b386b3 --- /dev/null +++ b/anthos-onprem-terraform/avmw_user_cluster_metallb/variables.tf @@ -0,0 +1,129 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Unique identifer of the Google Cloud Project that is to be used" + type = string +} + +variable "region" { + description = < Date: Tue, 16 May 2023 13:31:05 -0700 Subject: [PATCH 5/7] doc: update anthos-onprem-terraform README.md (#610) * Update anthos-onprem-terraform README.md Removed "coming soon", added links to each resource in the Terraform registry, and some minor edits. * docs: fix reference to samples --------- Co-authored-by: Shabir Mohamed Abdul Samadh <7249208+Shabirmean@users.noreply.github.com> --- anthos-onprem-terraform/README.md | 41 +++++++++++-------------------- 1 file changed, 15 insertions(+), 26 deletions(-) diff --git a/anthos-onprem-terraform/README.md b/anthos-onprem-terraform/README.md index 1d1d0c5b..fa080c14 100644 --- a/anthos-onprem-terraform/README.md +++ b/anthos-onprem-terraform/README.md @@ -1,37 +1,26 @@ -## Install Anthos onPrem clusters using Terraform +## Install Anthos On-Prem clusters using Terraform -This directory hosts samples and how-to's for installing Anthos onPrem clusters +This directory hosts samples and how-to's for installing Anthos On-Prem clusters _(i.e. Anthos on bare metal and Anthos on VMware)_ using the -`google_gkeonprem-*` resources of the canonical +`google_gkeonprem-*` resources in the [Google Cloud Terraform provider](https://registry.terraform.io/providers/hashicorp/google/latest/docs). +For more information, see the reference documentation for each resource. -All the samples here, first **emulates** a bare metal infrastucture using -Compute Engine (GCE) VMs. It is on this imaginary bare metal environment the -clusters are installed. Thus, as a prestep to all the guides here, you will see -a step for provisioning the bare metal insfrastructure. If you have your own -bare metal infrastructure, you can skip that section and adjust the sample to -suit your environment. - - --- -### Anthos clusters on bare metal (ABM) -- Create **admin** clusters with Terraform `(coming soon)` - - See [guide](https://cloud.google.com/anthos/clusters/docs/bare-metal/latest/installing/creating-clusters/create-admin-cluster-api) for creating an admin cluster using other clients -- Create **user** clusters with Terraform - - [MetalLB](./abm_user_cluster_metallb/) - - ManualLB `(coming soon)` -- Create **standalone** clusters with Terraform `(coming soon)` +#### Anthos clusters on bare metal (ABM) + +| Type | Sample _(by loadbalancer type)_ | Terraform resources | +| ---------------- | ---------------------------------------------- | ------------------- | +| **user** cluster | Bundled [MetalLB](./abm_user_cluster_metallb/) | [google_gkeonprem_bare_metal_cluster](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/gkeonprem_bare_metal_cluster)
[google_gkeonprem_vmware_node_pool](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/gkeonprem_vmware_node_pool) | + --- -### Anthos clusters on VMware (AVMware) -- Create **admin** clusters with Terraform `(coming soon)` -- Create **user** clusters with Terraform - - [MetalLB](./avmw_user_cluster_metallb/) - - ManualLB `(coming soon)` -- Create **standalone** clusters with Terraform `(coming soon)` +#### Anthos clusters on VMware (AVMware) + +| Type | Sample _(by loadbalancer type)_ | Terraform resources | +| ---------------- | ----------------------------------------------- | ------------------- | +| **user** cluster | Bundled [MetalLB](./avmw_user_cluster_metallb/) | [google_gkeonprem_vmware_cluster](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/gkeonprem_vmware_cluster)
[google_gkeonprem_vmware_node_pool](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/gkeonprem_vmware_node_pool) | --- From d5d12f5484072486c9da11d7ce7e49b25b16ed24 Mon Sep 17 00:00:00 2001 From: EB Date: Tue, 16 May 2023 16:31:21 -0400 Subject: [PATCH 6/7] feat(create_snapshot.py): Adding debug information for mutating webhook configs (#611) --- troubleshooting/create_snapshot.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/troubleshooting/create_snapshot.py b/troubleshooting/create_snapshot.py index 058c600e..45eb8373 100644 --- a/troubleshooting/create_snapshot.py +++ b/troubleshooting/create_snapshot.py @@ -63,6 +63,8 @@ 'kubectl describe nodes {kubeconfig_arg} --request-timeout {timeout}', 'kubectl get validatingwebhookconfigurations -o wide {kubeconfig_arg} --request-timeout {timeout}', # noqa: E501 'kubectl get validatingwebhookconfigurations -o yaml {kubeconfig_arg} --request-timeout {timeout}', # noqa: E501 + 'kubectl get mutatingwebhookconfigurations -o wide {kubeconfig_arg} --request-timeout {timeout}', # noqa: E501 + 'kubectl get mutatingwebhookconfigurations -o yaml {kubeconfig_arg} --request-timeout {timeout}', # noqa: E501 ] KUBECTL_PER_NS_CMDS = [ From 1757b325226b2549a5c9f9676c5284033e8c76cd Mon Sep 17 00:00:00 2001 From: Naitian Liu <83430653+naitianliu-google@users.noreply.github.com> Date: Fri, 19 May 2023 11:33:47 -0700 Subject: [PATCH 7/7] Update doc link for updating platform controller version (#616) --- anthos-onprem-terraform/avmw_user_cluster_metallb/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/anthos-onprem-terraform/avmw_user_cluster_metallb/README.md b/anthos-onprem-terraform/avmw_user_cluster_metallb/README.md index 1949032e..827452a3 100644 --- a/anthos-onprem-terraform/avmw_user_cluster_metallb/README.md +++ b/anthos-onprem-terraform/avmw_user_cluster_metallb/README.md @@ -61,7 +61,7 @@ have changed directory to where this samples is: Before upgrading the user cluster, please make sure the admin cluster platform controller has been upgraded to the target version. The steps to upgrade the admin cluster platform controller is listed in the -[public documentation](https://cloud.google.com/anthos/clusters/docs/on-prem/latest/how-to/upgrading#:~:text=In%20the%20Cluster%20basics%20section,Click%20Upgrade.). +[public documentation](https://cloud.google.com/anthos/clusters/docs/on-prem/latest/how-to/upgrade-on-prem-api#available_versions_for_upgrades). An example is shown below: