From 2191cc7bd6efbea80a75a4cd41046540a76dd08a Mon Sep 17 00:00:00 2001 From: Benjamin Schimke Date: Mon, 8 Jul 2024 09:50:39 +0200 Subject: [PATCH 01/18] add cluster-templates --- templates/aws/cluster-template.yaml | 107 ++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 templates/aws/cluster-template.yaml diff --git a/templates/aws/cluster-template.yaml b/templates/aws/cluster-template.yaml new file mode 100644 index 00000000..efe200e5 --- /dev/null +++ b/templates/aws/cluster-template.yaml @@ -0,0 +1,107 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: ${CLUSTER_NAME} +spec: + clusterNetwork: + pods: + cidrBlocks: + - 10.1.0.0/16 + services: + cidrBlocks: + - 10.152.0.0/16 + serviceDomain: cluster.local + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + kind: CK8sControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: AWSCluster + name: ${CLUSTER_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: AWSCluster +metadata: + name: ${CLUSTER_NAME} +spec: + region: ${AWS_REGION} + sshKeyName: ${AWS_SSH_KEY_NAME} + bastion: + enabled: ${AWS_CREATE_BASTION:=false} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta2 +kind: CK8sControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + machineTemplate: + infrastructureTemplate: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: AWSMachineTemplate + name: ${CLUSTER_NAME}-control-plane + spec: + airGapped: true + controlPlane: + extraKubeAPIServerArgs: + --anonymous-auth: "true" + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: AWSMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane +spec: + template: + spec: + iamInstanceProfile: control-plane.cluster-api-provider-aws.sigs.k8s.io + instanceType: ${AWS_CONTROL_PLANE_INSTANCE_TYPE:=t3.large} + publicIP: ${AWS_PUBLIC_IP:=false} + sshKeyName: ${AWS_SSH_KEY_NAME} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-worker-md-0 +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + template: + spec: + version: ${KUBERNETES_VERSION} + clusterName: ${CLUSTER_NAME} + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 + kind: CK8sConfigTemplate + name: ${CLUSTER_NAME}-md-0 + infrastructureRef: + name: "${CLUSTER_NAME}-md-0" + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: AWSMachineTemplate +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: AWSMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + iamInstanceProfile: nodes.cluster-api-provider-aws.sigs.k8s.io + instanceType: ${AWS_NODE_INSTANCE_TYPE:=t3.large} + publicIP: ${AWS_PUBLIC_IP:=false} + sshKeyName: ${AWS_SSH_KEY_NAME} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 +kind: CK8sConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + airGapped: true From 484f9e26a734f8b6a07d46b43b8f686e164be749 Mon Sep 17 00:00:00 2001 From: Benjamin Schimke Date: Mon, 8 Jul 2024 13:04:35 +0200 Subject: [PATCH 02/18] Add clusterctl config --- docs/clusterctl.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 docs/clusterctl.yaml diff --git a/docs/clusterctl.yaml b/docs/clusterctl.yaml new file mode 100644 index 00000000..0c2e052c --- /dev/null +++ b/docs/clusterctl.yaml @@ -0,0 +1,8 @@ +--- +providers: + - name: ck8s + type: BootstrapProvider + url: "https://github.com/canonical/cluster-api-k8s/releases/latest/bootstrap-components.yaml" + - name: ck8s + type: ControlPlaneProvider + url: "https://github.com/canonical/cluster-api-k8s/releases/latest/control-plane-components.yaml" From 63026cdbdfb82240c1c42ab0a1ff9d8cdf0c2ba8 Mon Sep 17 00:00:00 2001 From: Benjamin Schimke Date: Wed, 10 Jul 2024 10:20:40 +0200 Subject: [PATCH 03/18] docs --- docs/aws.md | 3 + docs/overview.md | 149 ++++++++++++++++++++++++++++ templates/aws/cluster-template.yaml | 2 - 3 files changed, 152 insertions(+), 2 deletions(-) create mode 100644 docs/aws.md create mode 100644 docs/overview.md diff --git a/docs/aws.md b/docs/aws.md new file mode 100644 index 00000000..1d59f820 --- /dev/null +++ b/docs/aws.md @@ -0,0 +1,3 @@ +# Setting up the AWS Infrastructure provider + +This How-To walks you to the steps to set up CAPI with the AWS infrastructure provider diff --git a/docs/overview.md b/docs/overview.md new file mode 100644 index 00000000..368b099e --- /dev/null +++ b/docs/overview.md @@ -0,0 +1,149 @@ +# Cluster Provisioning with CAPI + +This guide covers how to deploy a Canonical Kubernetes multi-node cluster using Cluster API (CAPI). + +## Install `clusterctl` + +The `clusterctl` CLI tool manages the lifecycle of a Cluster API management cluster. To install it, follow the [upstream instructions]. Typically, this involves fetching the executable that matches your hardware architecture and placing it in your PATH. For example, at the time this guide was written, for `amd64` you would: + +```sh +curl -L https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.7.3/clusterctl-linux-amd64 -o clusterctl +sudo install -o root -g root -m 0755 clusterctl /usr/local/bin/clusterctl +``` + +### Set up a management Cluster + +The management cluster hosts the CAPI providers. You can use a Canonical Kubernetes cluster as a management cluster: + +```sh +sudo snap install k8s --classic +sudo k8s bootstrap +sudo k8s status --wait-ready +mkdir -p ~/.kube/ +sudo k8s config > ~/.kube/config +``` + +When setting up the management cluster, place its kubeconfig under `~/.kube/config` so other tools such as `clusterctl` can discover and interact with it. + +### Prepare the Infrastructure Provider + +Before generating a cluster, you need to configure the infrastructure provider. Each provider has its own prerequisites. Please follow the Cluster API instructions for the additional infrastructure-specific configuration. + +#### Example Using AWS + +The AWS infrastructure provider requires the `clusterawsadm` tool to be installed: + +```sh +curl -L https://github.com/kubernetes-sigs/cluster-api-provider-aws/releases/download/v2.0.2/clusterawsadm-linux-amd64 -o clusterawsadm +chmod +x clusterawsadm +sudo mv clusterawsadm /usr/local/bin +``` + +With `clusterawsadm`, you can bootstrap the AWS environment that CAPI will use. + +Start by setting up environment variables defining the AWS account to use, if these are not already defined: + +```sh +export AWS_REGION= +export AWS_ACCESS_KEY_ID= +export AWS_SECRET_ACCESS_KEY= +``` + +If you are using multi-factor authentication, you will also need: + +```sh +export AWS_SESSION_TOKEN= # If you are using Multi-Factor Auth. +``` + +The `clusterawsadm` uses these details to create a CloudFormation stack in your AWS account with the correct IAM resources: + +```sh +clusterawsadm bootstrap iam create-cloudformation-stack +``` + +The credentials should also be encoded and stored as a Kubernetes secret: + +```sh +export AWS_B64ENCODED_CREDENTIALS=$(clusterawsadm bootstrap credentials encode-as-profile) +``` + +### Initialize the Management Cluster + +To initialize the management cluster with the latest released version of the providers and the infrastructure of your choice: + +```sh +clusterctl init --bootstrap ck8s --control-plane ck8s -i +``` + +### Generate a Cluster Spec Manifest + +Once the bootstrap and control-plane controllers are up and running, you can apply the cluster manifests with the specifications of the cluster you want to provision. + +For Canonical Kubernetes, there are example manifests in the bootstrap provider examples directory on GitHub. + +Alternatively, you can generate a cluster manifest for a selected set of commonly used infrastructures via templates provided by the Canonical Kubernetes team. Visit the usage instructions for a list of different providers and their deployment. + +Ensure you have initialized the desired infrastructure provider and fetch the Canonical Kubernetes bootstrap provider repository: + +```sh +git clone https://github.com/canonical/cluster-api-bootstrap-provider-microk8s +``` + +Review the list of variables needed for the cluster template: + +```sh +cd cluster-api-bootstrap-provider-microk8s +clusterctl generate cluster k8s- --from ./templates/cluster-template-.yaml --list-variables +``` + +Set the respective environment variables by editing the rc file as needed before sourcing it. Then generate the cluster manifest: + +```sh +source ./templates/cluster-template-.rc +clusterctl generate cluster k8s- --from ./templates/cluster-template-.yaml > cluster.yaml +``` + +Each provisioned node is associated with a `K8sConfig`, through which you can set the cluster’s properties. Review the available options in the respective definitions file and edit the cluster manifest (`cluster.yaml` above) to match your needs. Note that the configuration structure is similar to that of `kubeadm` - in the `K8sConfig`, you will find a `ClusterConfiguration` and an `InitConfiguration` section. + +### Deploy the Cluster + +To deploy the cluster, run: + +```sh +sudo microk8s kubectl apply -f cluster.yaml +``` + +To see the deployed machines: + +```sh +sudo microk8s kubectl get machine +``` + +After the first control plane node is provisioned, you can get the kubeconfig of the workload cluster: + +```sh +clusterctl get kubeconfig > kubeconfig +``` + +You can then see the workload nodes using: + +```sh +KUBECONFIG=./kubeconfig kubectl get node +``` + +### Delete the Cluster + +To get the list of provisioned clusters: + +```sh +sudo microk8s kubectl get clusters +``` + +To delete a cluster: + +```sh +sudo microk8s kubectl delete cluster +``` + + +[upstream instructions]: https://cluster-api.sigs.k8s.io/user/quick-start#install-clusterctl diff --git a/templates/aws/cluster-template.yaml b/templates/aws/cluster-template.yaml index efe200e5..307351ee 100644 --- a/templates/aws/cluster-template.yaml +++ b/templates/aws/cluster-template.yaml @@ -42,7 +42,6 @@ spec: kind: AWSMachineTemplate name: ${CLUSTER_NAME}-control-plane spec: - airGapped: true controlPlane: extraKubeAPIServerArgs: --anonymous-auth: "true" @@ -104,4 +103,3 @@ metadata: spec: template: spec: - airGapped: true From 2e51475c88fc0b4f237042a79a096905738640cc Mon Sep 17 00:00:00 2001 From: Benjamin Schimke Date: Thu, 11 Jul 2024 14:00:12 +0200 Subject: [PATCH 04/18] move clusterctl and update template-variables --- docs/clusterctl.yaml => clusterctl.yaml | 0 templates/aws/cluster-template.yaml | 4 ++++ templates/aws/template-variables.rc | 14 ++++++++++++++ 3 files changed, 18 insertions(+) rename docs/clusterctl.yaml => clusterctl.yaml (100%) create mode 100644 templates/aws/template-variables.rc diff --git a/docs/clusterctl.yaml b/clusterctl.yaml similarity index 100% rename from docs/clusterctl.yaml rename to clusterctl.yaml diff --git a/templates/aws/cluster-template.yaml b/templates/aws/cluster-template.yaml index 307351ee..57165e52 100644 --- a/templates/aws/cluster-template.yaml +++ b/templates/aws/cluster-template.yaml @@ -55,6 +55,8 @@ metadata: spec: template: spec: + ami: + id: ${AWS_AMI_ID:=ami-0ad50e72a79228704} iamInstanceProfile: control-plane.cluster-api-provider-aws.sigs.k8s.io instanceType: ${AWS_CONTROL_PLANE_INSTANCE_TYPE:=t3.large} publicIP: ${AWS_PUBLIC_IP:=false} @@ -91,6 +93,8 @@ metadata: spec: template: spec: + ami: + id: ${AWS_AMI_ID:=ami-0ad50e72a79228704} iamInstanceProfile: nodes.cluster-api-provider-aws.sigs.k8s.io instanceType: ${AWS_NODE_INSTANCE_TYPE:=t3.large} publicIP: ${AWS_PUBLIC_IP:=false} diff --git a/templates/aws/template-variables.rc b/templates/aws/template-variables.rc new file mode 100644 index 00000000..0288c057 --- /dev/null +++ b/templates/aws/template-variables.rc @@ -0,0 +1,14 @@ +# Kubernetes cluster configuration +export KUBERNETES_VERSION=v1.30.0 +export CONTROL_PLANE_MACHINE_COUNT=1 +export WORKER_MACHINE_COUNT=1 + +# AWS region +export AWS_REGION="eu-central-1" + +# AWS machine configuration +export AWS_CREATE_BASTION=true +export AWS_PUBLIC_IP=false +export AWS_CONTROL_PLANE_MACHINE_FLAVOR=t3.large +export AWS_NODE_MACHINE_FLAVOR=t3.large +export AWS_SSH_KEY_NAME=default From 3cded26f61d59e72efa89e24ae6de21ce797033c Mon Sep 17 00:00:00 2001 From: Benjamin Schimke Date: Fri, 12 Jul 2024 06:18:22 +0200 Subject: [PATCH 05/18] delete obsolete docs --- docs/aws.md | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 docs/aws.md diff --git a/docs/aws.md b/docs/aws.md deleted file mode 100644 index 1d59f820..00000000 --- a/docs/aws.md +++ /dev/null @@ -1,3 +0,0 @@ -# Setting up the AWS Infrastructure provider - -This How-To walks you to the steps to set up CAPI with the AWS infrastructure provider From f24f3f664e457c5449c645e6d79f586d7a731f3b Mon Sep 17 00:00:00 2001 From: Benjamin Schimke Date: Tue, 16 Jul 2024 08:24:22 +0200 Subject: [PATCH 06/18] overwrite install scriptes --- templates/aws/cluster-template.yaml | 238 ++++++++++++++++++++++++++++ 1 file changed, 238 insertions(+) diff --git a/templates/aws/cluster-template.yaml b/templates/aws/cluster-template.yaml index 57165e52..60ce6d48 100644 --- a/templates/aws/cluster-template.yaml +++ b/templates/aws/cluster-template.yaml @@ -29,6 +29,14 @@ spec: sshKeyName: ${AWS_SSH_KEY_NAME} bastion: enabled: ${AWS_CREATE_BASTION:=false} + controlPlaneLoadBalancer: + healthCheckProtocol: TCP + network: + cni: + cniIngressRules: + - description: microcluster + protocol: tcp + toPort: 2380 --- apiVersion: controlplane.cluster.x-k8s.io/v1beta2 kind: CK8sControlPlane @@ -42,7 +50,22 @@ spec: kind: AWSMachineTemplate name: ${CLUSTER_NAME}-control-plane spec: + files: + # note(ben): This is only required as long as k8s does not have a stable release. + - path: /capi/scripts/install.sh + permissions: "0500" + owner: "root:root" + content: | + #!/bin/bash -xe + snap install k8s --classic --edge + preRunCommands: + - systemctl stop kubelet || true + - systemctl disable kubelet || true + - systemctl stop containerd || true + - systemctl disable containerd || true + controlPlane: + cloudProvider: external extraKubeAPIServerArgs: --anonymous-auth: "true" replicas: ${CONTROL_PLANE_MACHINE_COUNT} @@ -107,3 +130,218 @@ metadata: spec: template: spec: + files: + # note(ben): This is only required as long as k8s does not have a stable release. + - path: /capi/scripts/install.sh + permissions: "0500" + owner: "root:root" + content: | + #!/bin/bash -xe + snap install k8s --classic --edge + - path: /capi/scripts/bootstrap.sh + permissions: "0500" + owner: "root:root" + content: | + #!/bin/bash -xe + if [ ! -f /etc/kubernetes/pki/ca.crt ]; then + k8s bootstrap --name '{{ ds.meta_data.local_hostname }}' --address '$(cat /capi/etc/microcluster-address)' --file '/capi/etc/config.yaml' + fi + - path: /capi/scripts/join-cluster.sh + permissions: "0500" + owner: "root:root" + content: | + #!/bin/bash -xe + k8s join-cluster --name '{{ ds.meta_data.local_hostname }}' "$(cat /capi/etc/join-token)" --address "$(cat /capi/etc/microcluster-address)" --file "/capi/etc/config.yaml" + preRunCommands: + - systemctl stop kubelet || true + - systemctl disable kubelet || true + - systemctl stop containerd || true + - systemctl disable containerd || true +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm +spec: + clusterSelector: + matchLabels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon +--- +apiVersion: v1 +data: + aws-ccm.yaml: | + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: aws-cloud-controller-manager + namespace: kube-system + labels: + k8s-app: aws-cloud-controller-manager + spec: + selector: + matchLabels: + k8s-app: aws-cloud-controller-manager + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + k8s-app: aws-cloud-controller-manager + spec: + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/master + effect: NoSchedule + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + serviceAccountName: cloud-controller-manager + containers: + - name: aws-cloud-controller-manager + image: gcr.io/k8s-staging-provider-aws/cloud-controller-manager:v1.20.0-alpha.0 + args: + - --v=2 + resources: + requests: + cpu: 200m + hostNetwork: true + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: cloud-controller-manager:apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - '*' + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - configmaps + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - list + - watch + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system +kind: ConfigMap +metadata: + name: cloud-controller-manager-addon + namespace: default From 10151ab0c3b87167e408a4622f2b29d6e083ff26 Mon Sep 17 00:00:00 2001 From: eaudetcobello Date: Wed, 14 Aug 2024 20:11:24 -0400 Subject: [PATCH 07/18] update template configuration - Change service CIDR block to match our standard range - Create a bastion by default - Copy cluster bootstrap/join overrides to control-plane - Remove unneeded prerunCommands - Remove tolerations that don't apply to our setup (kubernetes.io/master) - Add args from default provider template to aws-cloud-controller-manager args - Restore some RBAC rules from default provider template --- templates/aws/cluster-template.yaml | 212 ++++++++++++++-------------- 1 file changed, 108 insertions(+), 104 deletions(-) diff --git a/templates/aws/cluster-template.yaml b/templates/aws/cluster-template.yaml index 60ce6d48..a310818d 100644 --- a/templates/aws/cluster-template.yaml +++ b/templates/aws/cluster-template.yaml @@ -9,8 +9,7 @@ spec: - 10.1.0.0/16 services: cidrBlocks: - - 10.152.0.0/16 - serviceDomain: cluster.local + - 10.152.183.0/24 controlPlaneRef: apiVersion: controlplane.cluster.x-k8s.io/v1beta2 kind: CK8sControlPlane @@ -28,7 +27,7 @@ spec: region: ${AWS_REGION} sshKeyName: ${AWS_SSH_KEY_NAME} bastion: - enabled: ${AWS_CREATE_BASTION:=false} + enabled: ${AWS_CREATE_BASTION:=true} controlPlaneLoadBalancer: healthCheckProtocol: TCP network: @@ -58,16 +57,22 @@ spec: content: | #!/bin/bash -xe snap install k8s --classic --edge - preRunCommands: - - systemctl stop kubelet || true - - systemctl disable kubelet || true - - systemctl stop containerd || true - - systemctl disable containerd || true - + - path: /capi/scripts/bootstrap.sh + permissions: "0500" + owner: "root:root" + content: | + #!/bin/bash -xe + if [ ! -f /etc/kubernetes/pki/ca.crt ]; then + k8s bootstrap --name '{{ ds.meta_data.local_hostname }}' --address "$(cat /capi/etc/microcluster-address)" --file '/capi/etc/config.yaml' + fi + - path: /capi/scripts/join-cluster.sh + permissions: "0500" + owner: "root:root" + content: | + #!/bin/bash -xe + k8s join-cluster --name '{{ ds.meta_data.local_hostname }}' "$(cat /capi/etc/join-token)" --address "$(cat /capi/etc/microcluster-address)" --file "/capi/etc/config.yaml" controlPlane: cloudProvider: external - extraKubeAPIServerArgs: - --anonymous-auth: "true" replicas: ${CONTROL_PLANE_MACHINE_COUNT} version: ${KUBERNETES_VERSION} --- @@ -144,7 +149,7 @@ spec: content: | #!/bin/bash -xe if [ ! -f /etc/kubernetes/pki/ca.crt ]; then - k8s bootstrap --name '{{ ds.meta_data.local_hostname }}' --address '$(cat /capi/etc/microcluster-address)' --file '/capi/etc/config.yaml' + k8s bootstrap --name '{{ ds.meta_data.local_hostname }}' --address "$(cat /capi/etc/microcluster-address)" --file '/capi/etc/config.yaml' fi - path: /capi/scripts/join-cluster.sh permissions: "0500" @@ -152,11 +157,7 @@ spec: content: | #!/bin/bash -xe k8s join-cluster --name '{{ ds.meta_data.local_hostname }}' "$(cat /capi/etc/join-token)" --address "$(cat /capi/etc/microcluster-address)" --file "/capi/etc/config.yaml" - preRunCommands: - - systemctl stop kubelet || true - - systemctl disable kubelet || true - - systemctl stop containerd || true - - systemctl disable containerd || true + --- apiVersion: addons.cluster.x-k8s.io/v1beta1 kind: ClusterResourceSet @@ -169,10 +170,11 @@ spec: resources: - kind: ConfigMap name: cloud-controller-manager-addon + strategy: ApplyOnce --- apiVersion: v1 data: - aws-ccm.yaml: | + aws-ccm-external.yaml: | --- apiVersion: apps/v1 kind: DaemonSet @@ -192,12 +194,12 @@ data: labels: k8s-app: aws-cloud-controller-manager spec: + nodeSelector: + node-role.kubernetes.io/control-plane: "" tolerations: - key: node.cloudprovider.kubernetes.io/uninitialized value: "true" effect: NoSchedule - - key: node-role.kubernetes.io/master - effect: NoSchedule - effect: NoSchedule key: node-role.kubernetes.io/control-plane affinity: @@ -207,15 +209,15 @@ data: - matchExpressions: - key: node-role.kubernetes.io/control-plane operator: Exists - - matchExpressions: - - key: node-role.kubernetes.io/master - operator: Exists serviceAccountName: cloud-controller-manager containers: - name: aws-cloud-controller-manager - image: gcr.io/k8s-staging-provider-aws/cloud-controller-manager:v1.20.0-alpha.0 + image: registry.k8s.io/provider-aws/cloud-controller-manager:v1.28.3 args: - --v=2 + - --cloud-provider=aws + - --use-service-account-credentials=true + - --configure-cloud-routes=false resources: requests: cpu: 200m @@ -247,86 +249,88 @@ data: metadata: name: system:cloud-controller-manager rules: - - apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - update - - apiGroups: - - "" - resources: - - nodes - verbs: - - '*' - - apiGroups: - - "" - resources: - - nodes/status - verbs: - - patch - - apiGroups: - - "" - resources: - - services - verbs: - - list - - patch - - update - - watch - - apiGroups: - - "" - resources: - - services/status - verbs: - - list - - patch - - update - - watch - - apiGroups: - - "" - resources: - - serviceaccounts - verbs: - - create - - apiGroups: - - "" - resources: - - persistentvolumes - verbs: - - get - - list - - update - - watch - - apiGroups: - - "" - resources: - - configmaps - verbs: - - list - - watch - - apiGroups: - - "" - resources: - - endpoints - verbs: - - create - - get - - list - - watch - - update - - apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - create - - get - - list - - watch - - update + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - '*' + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - serviceaccounts/token + verbs: + - create --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 From b6561c2593c60f84f908cdfe339800cd897fff09 Mon Sep 17 00:00:00 2001 From: eaudetcobello Date: Wed, 14 Aug 2024 20:18:58 -0400 Subject: [PATCH 08/18] configurable ccm image --- templates/aws/cluster-template.yaml | 2 +- templates/aws/template-variables.rc | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/templates/aws/cluster-template.yaml b/templates/aws/cluster-template.yaml index a310818d..a5ff78f6 100644 --- a/templates/aws/cluster-template.yaml +++ b/templates/aws/cluster-template.yaml @@ -212,7 +212,7 @@ data: serviceAccountName: cloud-controller-manager containers: - name: aws-cloud-controller-manager - image: registry.k8s.io/provider-aws/cloud-controller-manager:v1.28.3 + image: ${AWS_CCM_IMAGE} args: - --v=2 - --cloud-provider=aws diff --git a/templates/aws/template-variables.rc b/templates/aws/template-variables.rc index 0288c057..5559ea69 100644 --- a/templates/aws/template-variables.rc +++ b/templates/aws/template-variables.rc @@ -12,3 +12,5 @@ export AWS_PUBLIC_IP=false export AWS_CONTROL_PLANE_MACHINE_FLAVOR=t3.large export AWS_NODE_MACHINE_FLAVOR=t3.large export AWS_SSH_KEY_NAME=default + +export AWS_CCM_IMAGE=registry.k8s.io/provider-aws/cloud-controller-manager:v1.28.3 From b07ac72d65e47e3e11b90fc7d13547934072a593 Mon Sep 17 00:00:00 2001 From: eaudetcobello Date: Tue, 20 Aug 2024 21:42:41 -0400 Subject: [PATCH 09/18] remove clusterctl --- clusterctl.yaml | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 clusterctl.yaml diff --git a/clusterctl.yaml b/clusterctl.yaml deleted file mode 100644 index 0c2e052c..00000000 --- a/clusterctl.yaml +++ /dev/null @@ -1,8 +0,0 @@ ---- -providers: - - name: ck8s - type: BootstrapProvider - url: "https://github.com/canonical/cluster-api-k8s/releases/latest/bootstrap-components.yaml" - - name: ck8s - type: ControlPlaneProvider - url: "https://github.com/canonical/cluster-api-k8s/releases/latest/control-plane-components.yaml" From c15f2842bb60206e4ab194fd15544e783b928456 Mon Sep 17 00:00:00 2001 From: eaudetcobello Date: Tue, 20 Aug 2024 21:54:06 -0400 Subject: [PATCH 10/18] review comments no hardcoded defaults, empty variables by default, use new nodeName field to specify node name --- templates/aws/cluster-template.yaml | 45 ++++++----------------------- templates/aws/template-variables.rc | 16 +++++----- 2 files changed, 17 insertions(+), 44 deletions(-) diff --git a/templates/aws/cluster-template.yaml b/templates/aws/cluster-template.yaml index a5ff78f6..976551e4 100644 --- a/templates/aws/cluster-template.yaml +++ b/templates/aws/cluster-template.yaml @@ -27,7 +27,7 @@ spec: region: ${AWS_REGION} sshKeyName: ${AWS_SSH_KEY_NAME} bastion: - enabled: ${AWS_CREATE_BASTION:=true} + enabled: ${AWS_CREATE_BASTION} controlPlaneLoadBalancer: healthCheckProtocol: TCP network: @@ -49,6 +49,7 @@ spec: kind: AWSMachineTemplate name: ${CLUSTER_NAME}-control-plane spec: + nodeName: {{ ds.meta_data.local_hostname }} files: # note(ben): This is only required as long as k8s does not have a stable release. - path: /capi/scripts/install.sh @@ -57,20 +58,6 @@ spec: content: | #!/bin/bash -xe snap install k8s --classic --edge - - path: /capi/scripts/bootstrap.sh - permissions: "0500" - owner: "root:root" - content: | - #!/bin/bash -xe - if [ ! -f /etc/kubernetes/pki/ca.crt ]; then - k8s bootstrap --name '{{ ds.meta_data.local_hostname }}' --address "$(cat /capi/etc/microcluster-address)" --file '/capi/etc/config.yaml' - fi - - path: /capi/scripts/join-cluster.sh - permissions: "0500" - owner: "root:root" - content: | - #!/bin/bash -xe - k8s join-cluster --name '{{ ds.meta_data.local_hostname }}' "$(cat /capi/etc/join-token)" --address "$(cat /capi/etc/microcluster-address)" --file "/capi/etc/config.yaml" controlPlane: cloudProvider: external replicas: ${CONTROL_PLANE_MACHINE_COUNT} @@ -84,10 +71,10 @@ spec: template: spec: ami: - id: ${AWS_AMI_ID:=ami-0ad50e72a79228704} + id: ${AWS_AMI_ID} iamInstanceProfile: control-plane.cluster-api-provider-aws.sigs.k8s.io - instanceType: ${AWS_CONTROL_PLANE_INSTANCE_TYPE:=t3.large} - publicIP: ${AWS_PUBLIC_IP:=false} + instanceType: ${AWS_CONTROL_PLANE_INSTANCE_TYPE} + publicIP: ${AWS_PUBLIC_IP} sshKeyName: ${AWS_SSH_KEY_NAME} --- apiVersion: cluster.x-k8s.io/v1beta1 @@ -122,10 +109,10 @@ spec: template: spec: ami: - id: ${AWS_AMI_ID:=ami-0ad50e72a79228704} + id: ${AWS_AMI_ID} iamInstanceProfile: nodes.cluster-api-provider-aws.sigs.k8s.io - instanceType: ${AWS_NODE_INSTANCE_TYPE:=t3.large} - publicIP: ${AWS_PUBLIC_IP:=false} + instanceType: ${AWS_NODE_INSTANCE_TYPE} + publicIP: ${AWS_PUBLIC_IP} sshKeyName: ${AWS_SSH_KEY_NAME} --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 @@ -135,6 +122,7 @@ metadata: spec: template: spec: + nodeName: {{ ds.meta_data.local_hostname }} files: # note(ben): This is only required as long as k8s does not have a stable release. - path: /capi/scripts/install.sh @@ -143,21 +131,6 @@ spec: content: | #!/bin/bash -xe snap install k8s --classic --edge - - path: /capi/scripts/bootstrap.sh - permissions: "0500" - owner: "root:root" - content: | - #!/bin/bash -xe - if [ ! -f /etc/kubernetes/pki/ca.crt ]; then - k8s bootstrap --name '{{ ds.meta_data.local_hostname }}' --address "$(cat /capi/etc/microcluster-address)" --file '/capi/etc/config.yaml' - fi - - path: /capi/scripts/join-cluster.sh - permissions: "0500" - owner: "root:root" - content: | - #!/bin/bash -xe - k8s join-cluster --name '{{ ds.meta_data.local_hostname }}' "$(cat /capi/etc/join-token)" --address "$(cat /capi/etc/microcluster-address)" --file "/capi/etc/config.yaml" - --- apiVersion: addons.cluster.x-k8s.io/v1beta1 kind: ClusterResourceSet diff --git a/templates/aws/template-variables.rc b/templates/aws/template-variables.rc index 5559ea69..e2defa49 100644 --- a/templates/aws/template-variables.rc +++ b/templates/aws/template-variables.rc @@ -1,16 +1,16 @@ # Kubernetes cluster configuration export KUBERNETES_VERSION=v1.30.0 -export CONTROL_PLANE_MACHINE_COUNT=1 -export WORKER_MACHINE_COUNT=1 +export CONTROL_PLANE_MACHINE_COUNT= # e.g. 1 +export WORKER_MACHINE_COUNT= # AWS region -export AWS_REGION="eu-central-1" +export AWS_REGION="" # e.g. us-east-2 # AWS machine configuration -export AWS_CREATE_BASTION=true -export AWS_PUBLIC_IP=false -export AWS_CONTROL_PLANE_MACHINE_FLAVOR=t3.large -export AWS_NODE_MACHINE_FLAVOR=t3.large -export AWS_SSH_KEY_NAME=default +export AWS_CREATE_BASTION= # e.g. true +export AWS_PUBLIC_IP= # e.g. true +export AWS_CONTROL_PLANE_MACHINE_FLAVOR= # e.g. t3.large +export AWS_NODE_MACHINE_FLAVOR= # e.g. t3.large +export AWS_SSH_KEY_NAME= # e.g. default export AWS_CCM_IMAGE=registry.k8s.io/provider-aws/cloud-controller-manager:v1.28.3 From ae9ea7dc8e2d64bac52e95dc139eb3c46d99e2d7 Mon Sep 17 00:00:00 2001 From: eaudetcobello Date: Tue, 20 Aug 2024 21:55:25 -0400 Subject: [PATCH 11/18] reformat --- templates/aws/template-variables.rc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/templates/aws/template-variables.rc b/templates/aws/template-variables.rc index e2defa49..e8cef3b5 100644 --- a/templates/aws/template-variables.rc +++ b/templates/aws/template-variables.rc @@ -4,13 +4,13 @@ export CONTROL_PLANE_MACHINE_COUNT= # e.g. 1 export WORKER_MACHINE_COUNT= # AWS region -export AWS_REGION="" # e.g. us-east-2 +export AWS_REGION="" # e.g. "us-east-2" # AWS machine configuration -export AWS_CREATE_BASTION= # e.g. true -export AWS_PUBLIC_IP= # e.g. true -export AWS_CONTROL_PLANE_MACHINE_FLAVOR= # e.g. t3.large -export AWS_NODE_MACHINE_FLAVOR= # e.g. t3.large -export AWS_SSH_KEY_NAME= # e.g. default +export AWS_CREATE_BASTION= # e.g. "true" +export AWS_PUBLIC_IP= # e.g. "true" +export AWS_CONTROL_PLANE_MACHINE_FLAVOR= # e.g. "t3.large" +export AWS_NODE_MACHINE_FLAVOR= # e.g. "t3.large" +export AWS_SSH_KEY_NAME= # e.g. "default" export AWS_CCM_IMAGE=registry.k8s.io/provider-aws/cloud-controller-manager:v1.28.3 From 34873ec60bf121ac5c86b3b22fd4b7b178cf509f Mon Sep 17 00:00:00 2001 From: eaudetcobello Date: Tue, 20 Aug 2024 22:02:09 -0400 Subject: [PATCH 12/18] remove microk8s references --- docs/overview.md | 28 +++++++++++----------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/docs/overview.md b/docs/overview.md index 368b099e..cdcace65 100644 --- a/docs/overview.md +++ b/docs/overview.md @@ -16,11 +16,11 @@ sudo install -o root -g root -m 0755 clusterctl /usr/local/bin/clusterctl The management cluster hosts the CAPI providers. You can use a Canonical Kubernetes cluster as a management cluster: ```sh -sudo snap install k8s --classic +sudo snap install k8s --classic --edge sudo k8s bootstrap sudo k8s status --wait-ready mkdir -p ~/.kube/ -sudo k8s config > ~/.kube/config +sudo k8s kubectl config view --raw > ~/.kube/config ``` When setting up the management cluster, place its kubeconfig under `~/.kube/config` so other tools such as `clusterctl` can discover and interact with it. @@ -44,7 +44,7 @@ With `clusterawsadm`, you can bootstrap the AWS environment that CAPI will use. Start by setting up environment variables defining the AWS account to use, if these are not already defined: ```sh -export AWS_REGION= +export AWS_REGION= export AWS_ACCESS_KEY_ID= export AWS_SECRET_ACCESS_KEY= ``` @@ -79,20 +79,14 @@ clusterctl init --bootstrap ck8s --control-plane ck8s -i clusterctl generate cluster k8s- --from ./templates/cluster-template-.yaml --list-variables ``` @@ -103,20 +97,20 @@ source ./templates/cluster-template-.rc clusterctl generate cluster k8s- --from ./templates/cluster-template-.yaml > cluster.yaml ``` -Each provisioned node is associated with a `K8sConfig`, through which you can set the cluster’s properties. Review the available options in the respective definitions file and edit the cluster manifest (`cluster.yaml` above) to match your needs. Note that the configuration structure is similar to that of `kubeadm` - in the `K8sConfig`, you will find a `ClusterConfiguration` and an `InitConfiguration` section. +Each provisioned node is associated with a `K8sConfig`, through which you can set the cluster’s properties. Review the available options in the respective definitions file and edit the cluster manifest (`cluster.yaml` above) to match your needs. Note that the configuration structure is similar to that of `kubeadm` - in the `CK8sConfig`, you will find a `ClusterConfiguration` and an `InitConfiguration` section. ### Deploy the Cluster To deploy the cluster, run: ```sh -sudo microk8s kubectl apply -f cluster.yaml +sudo k8s kubectl apply -f cluster.yaml ``` To see the deployed machines: ```sh -sudo microk8s kubectl get machine +sudo k8s kubectl get machine ``` After the first control plane node is provisioned, you can get the kubeconfig of the workload cluster: @@ -136,13 +130,13 @@ KUBECONFIG=./kubeconfig kubectl get node To get the list of provisioned clusters: ```sh -sudo microk8s kubectl get clusters +sudo k8s kubectl get clusters ``` To delete a cluster: ```sh -sudo microk8s kubectl delete cluster +sudo k8s kubectl delete cluster ``` From 394ff177bfa1834e7d8527b8c91d502b1ed90b21 Mon Sep 17 00:00:00 2001 From: eaudetcobello Date: Tue, 20 Aug 2024 22:11:13 -0400 Subject: [PATCH 13/18] add ami id to .rc file --- templates/aws/template-variables.rc | 1 + 1 file changed, 1 insertion(+) diff --git a/templates/aws/template-variables.rc b/templates/aws/template-variables.rc index e8cef3b5..34bc801b 100644 --- a/templates/aws/template-variables.rc +++ b/templates/aws/template-variables.rc @@ -12,5 +12,6 @@ export AWS_PUBLIC_IP= # e.g. "true" export AWS_CONTROL_PLANE_MACHINE_FLAVOR= # e.g. "t3.large" export AWS_NODE_MACHINE_FLAVOR= # e.g. "t3.large" export AWS_SSH_KEY_NAME= # e.g. "default" +export AWS_AMI_ID= # e.g. "ami-0ad50e72a79228704" export AWS_CCM_IMAGE=registry.k8s.io/provider-aws/cloud-controller-manager:v1.28.3 From 0f07038938ea05e677e48bd9fcb373fa7d2c3dbb Mon Sep 17 00:00:00 2001 From: eaudetcobello Date: Wed, 21 Aug 2024 11:49:11 -0400 Subject: [PATCH 14/18] use nodename field, ccm label --- templates/aws/cluster-template.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/templates/aws/cluster-template.yaml b/templates/aws/cluster-template.yaml index 976551e4..09602f2c 100644 --- a/templates/aws/cluster-template.yaml +++ b/templates/aws/cluster-template.yaml @@ -2,6 +2,8 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: name: ${CLUSTER_NAME} + labels: + ccm: external spec: clusterNetwork: pods: @@ -49,7 +51,7 @@ spec: kind: AWSMachineTemplate name: ${CLUSTER_NAME}-control-plane spec: - nodeName: {{ ds.meta_data.local_hostname }} + nodeName: "{{ ds.meta_data.local_hostname }}" files: # note(ben): This is only required as long as k8s does not have a stable release. - path: /capi/scripts/install.sh @@ -122,7 +124,7 @@ metadata: spec: template: spec: - nodeName: {{ ds.meta_data.local_hostname }} + nodeName: "{{ ds.meta_data.local_hostname }}" files: # note(ben): This is only required as long as k8s does not have a stable release. - path: /capi/scripts/install.sh @@ -139,7 +141,7 @@ metadata: spec: clusterSelector: matchLabels: - cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + ccm: external resources: - kind: ConfigMap name: cloud-controller-manager-addon From 77559421d03c00a6bd6d3e8f6f1ad1a922fd16ae Mon Sep 17 00:00:00 2001 From: eaudetcobello Date: Wed, 21 Aug 2024 14:02:47 -0400 Subject: [PATCH 15/18] configurable root volume size. fix var. name --- templates/aws/cluster-template.yaml | 4 ++++ templates/aws/template-variables.rc | 6 ++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/templates/aws/cluster-template.yaml b/templates/aws/cluster-template.yaml index 09602f2c..9a960294 100644 --- a/templates/aws/cluster-template.yaml +++ b/templates/aws/cluster-template.yaml @@ -78,6 +78,8 @@ spec: instanceType: ${AWS_CONTROL_PLANE_INSTANCE_TYPE} publicIP: ${AWS_PUBLIC_IP} sshKeyName: ${AWS_SSH_KEY_NAME} + rootVolume: + size: ${AWS_CONTROL_PLANE_ROOT_VOLUME_SIZE} --- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineDeployment @@ -116,6 +118,8 @@ spec: instanceType: ${AWS_NODE_INSTANCE_TYPE} publicIP: ${AWS_PUBLIC_IP} sshKeyName: ${AWS_SSH_KEY_NAME} + rootVolume: + size: ${AWS_NODE_ROOT_VOLUME_SIZE} --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 kind: CK8sConfigTemplate diff --git a/templates/aws/template-variables.rc b/templates/aws/template-variables.rc index 34bc801b..48313d8a 100644 --- a/templates/aws/template-variables.rc +++ b/templates/aws/template-variables.rc @@ -9,8 +9,10 @@ export AWS_REGION="" # e.g. "us-east-2" # AWS machine configuration export AWS_CREATE_BASTION= # e.g. "true" export AWS_PUBLIC_IP= # e.g. "true" -export AWS_CONTROL_PLANE_MACHINE_FLAVOR= # e.g. "t3.large" -export AWS_NODE_MACHINE_FLAVOR= # e.g. "t3.large" +export AWS_CONTROL_PLANE_INSTANCE_TYPE= # e.g. "t3.large" +export AWS_NODE_INSTANCE_TYPE= # e.g. "t3.large" +export AWS_CONTROL_PLANE_ROOT_VOLUME_SIZE= # in Gi, e.g. 16 +export AWS_NODE_ROOT_VOLUME_SIZE= # in Gi, e.g. 16 export AWS_SSH_KEY_NAME= # e.g. "default" export AWS_AMI_ID= # e.g. "ami-0ad50e72a79228704" From 078f92d123f0e667c2c7fe8b216a0053e3733817 Mon Sep 17 00:00:00 2001 From: eaudetcobello Date: Wed, 21 Aug 2024 14:12:47 -0400 Subject: [PATCH 16/18] remove hardcoded namespace --- templates/aws/cluster-template.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/templates/aws/cluster-template.yaml b/templates/aws/cluster-template.yaml index 9a960294..883270ec 100644 --- a/templates/aws/cluster-template.yaml +++ b/templates/aws/cluster-template.yaml @@ -43,7 +43,6 @@ apiVersion: controlplane.cluster.x-k8s.io/v1beta2 kind: CK8sControlPlane metadata: name: ${CLUSTER_NAME}-control-plane - namespace: default spec: machineTemplate: infrastructureTemplate: @@ -327,4 +326,3 @@ data: kind: ConfigMap metadata: name: cloud-controller-manager-addon - namespace: default From 03ac130df2f65bb50668023d3c05f19afc93b7dd Mon Sep 17 00:00:00 2001 From: eaudetcobello Date: Mon, 26 Aug 2024 12:44:03 -0400 Subject: [PATCH 17/18] remove docs (now in k8s-snap dir) --- docs/overview.md | 143 ----------------------------------------------- 1 file changed, 143 deletions(-) delete mode 100644 docs/overview.md diff --git a/docs/overview.md b/docs/overview.md deleted file mode 100644 index cdcace65..00000000 --- a/docs/overview.md +++ /dev/null @@ -1,143 +0,0 @@ -# Cluster Provisioning with CAPI - -This guide covers how to deploy a Canonical Kubernetes multi-node cluster using Cluster API (CAPI). - -## Install `clusterctl` - -The `clusterctl` CLI tool manages the lifecycle of a Cluster API management cluster. To install it, follow the [upstream instructions]. Typically, this involves fetching the executable that matches your hardware architecture and placing it in your PATH. For example, at the time this guide was written, for `amd64` you would: - -```sh -curl -L https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.7.3/clusterctl-linux-amd64 -o clusterctl -sudo install -o root -g root -m 0755 clusterctl /usr/local/bin/clusterctl -``` - -### Set up a management Cluster - -The management cluster hosts the CAPI providers. You can use a Canonical Kubernetes cluster as a management cluster: - -```sh -sudo snap install k8s --classic --edge -sudo k8s bootstrap -sudo k8s status --wait-ready -mkdir -p ~/.kube/ -sudo k8s kubectl config view --raw > ~/.kube/config -``` - -When setting up the management cluster, place its kubeconfig under `~/.kube/config` so other tools such as `clusterctl` can discover and interact with it. - -### Prepare the Infrastructure Provider - -Before generating a cluster, you need to configure the infrastructure provider. Each provider has its own prerequisites. Please follow the Cluster API instructions for the additional infrastructure-specific configuration. - -#### Example Using AWS - -The AWS infrastructure provider requires the `clusterawsadm` tool to be installed: - -```sh -curl -L https://github.com/kubernetes-sigs/cluster-api-provider-aws/releases/download/v2.0.2/clusterawsadm-linux-amd64 -o clusterawsadm -chmod +x clusterawsadm -sudo mv clusterawsadm /usr/local/bin -``` - -With `clusterawsadm`, you can bootstrap the AWS environment that CAPI will use. - -Start by setting up environment variables defining the AWS account to use, if these are not already defined: - -```sh -export AWS_REGION= -export AWS_ACCESS_KEY_ID= -export AWS_SECRET_ACCESS_KEY= -``` - -If you are using multi-factor authentication, you will also need: - -```sh -export AWS_SESSION_TOKEN= # If you are using Multi-Factor Auth. -``` - -The `clusterawsadm` uses these details to create a CloudFormation stack in your AWS account with the correct IAM resources: - -```sh -clusterawsadm bootstrap iam create-cloudformation-stack -``` - -The credentials should also be encoded and stored as a Kubernetes secret: - -```sh -export AWS_B64ENCODED_CREDENTIALS=$(clusterawsadm bootstrap credentials encode-as-profile) -``` - -### Initialize the Management Cluster - -To initialize the management cluster with the latest released version of the providers and the infrastructure of your choice: - -```sh -clusterctl init --bootstrap ck8s --control-plane ck8s -i -``` - -### Generate a Cluster Spec Manifest - -Once the bootstrap and control-plane controllers are up and running, you can apply the cluster manifests with the specifications of the cluster you want to provision. - -You can generate a cluster manifest for an infrastructure using templates provided by the Canonical Kubernetes team. The templates/ folder contains templates for common clouds. - -Ensure you have initialized the desired infrastructure provider and fetch the Canonical Kubernetes bootstrap provider repository. - -Review the list of variables needed for the cluster template: - -```sh -cd templates/ -clusterctl generate cluster k8s- --from ./templates/cluster-template-.yaml --list-variables -``` - -Set the respective environment variables by editing the rc file as needed before sourcing it. Then generate the cluster manifest: - -```sh -source ./templates/cluster-template-.rc -clusterctl generate cluster k8s- --from ./templates/cluster-template-.yaml > cluster.yaml -``` - -Each provisioned node is associated with a `K8sConfig`, through which you can set the cluster’s properties. Review the available options in the respective definitions file and edit the cluster manifest (`cluster.yaml` above) to match your needs. Note that the configuration structure is similar to that of `kubeadm` - in the `CK8sConfig`, you will find a `ClusterConfiguration` and an `InitConfiguration` section. - -### Deploy the Cluster - -To deploy the cluster, run: - -```sh -sudo k8s kubectl apply -f cluster.yaml -``` - -To see the deployed machines: - -```sh -sudo k8s kubectl get machine -``` - -After the first control plane node is provisioned, you can get the kubeconfig of the workload cluster: - -```sh -clusterctl get kubeconfig > kubeconfig -``` - -You can then see the workload nodes using: - -```sh -KUBECONFIG=./kubeconfig kubectl get node -``` - -### Delete the Cluster - -To get the list of provisioned clusters: - -```sh -sudo k8s kubectl get clusters -``` - -To delete a cluster: - -```sh -sudo k8s kubectl delete cluster -``` - - -[upstream instructions]: https://cluster-api.sigs.k8s.io/user/quick-start#install-clusterctl From 9484e0658345d458f2cf076443d2f1619e2d8cf4 Mon Sep 17 00:00:00 2001 From: eaudetcobello Date: Mon, 26 Aug 2024 12:50:02 -0400 Subject: [PATCH 18/18] gi -> gigabyte --- templates/aws/template-variables.rc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/aws/template-variables.rc b/templates/aws/template-variables.rc index 48313d8a..4251a012 100644 --- a/templates/aws/template-variables.rc +++ b/templates/aws/template-variables.rc @@ -11,8 +11,8 @@ export AWS_CREATE_BASTION= # e.g. "true" export AWS_PUBLIC_IP= # e.g. "true" export AWS_CONTROL_PLANE_INSTANCE_TYPE= # e.g. "t3.large" export AWS_NODE_INSTANCE_TYPE= # e.g. "t3.large" -export AWS_CONTROL_PLANE_ROOT_VOLUME_SIZE= # in Gi, e.g. 16 -export AWS_NODE_ROOT_VOLUME_SIZE= # in Gi, e.g. 16 +export AWS_CONTROL_PLANE_ROOT_VOLUME_SIZE= # in Gigabyte, e.g. 16 +export AWS_NODE_ROOT_VOLUME_SIZE= # in Gigabyte, e.g. 16 export AWS_SSH_KEY_NAME= # e.g. "default" export AWS_AMI_ID= # e.g. "ami-0ad50e72a79228704"