Support for building windows images

Signed-off-by: Waleed Malik <[email protected]>
kubermatic · Mar 7, 2023 · e3bff57 · e3bff57
1 parent d163811
commit e3bff57
Show file tree

Hide file tree

Showing 30 changed files with 1,254 additions and 3 deletions.
diff --git a/README.md b/README.md
@@ -30,7 +30,7 @@
 
 ### What works
 
-- Creation of worker nodes on AWS, Digitalocean, Openstack, Azure, Google Cloud Platform, Nutanix, VMWare Cloud Director, VMWare Vsphere, Linode, Hetzner cloud and Kubevirt (experimental)
+- Creation of worker nodes on AWS, Digitalocean, Openstack, Azure, Google Cloud Platform, Nutanix, VMWare Cloud Director, VMWare Vsphere, Linode, Hetzner cloud and Kubevirt.
 - Using Ubuntu, Flatcar or CentOS 7 distributions ([not all distributions work on all providers](/docs/operating-system.md))
 
 ### Supported Kubernetes versions

diff --git a/docs/operating-system.md b/docs/operating-system.md
@@ -4,7 +4,7 @@
 
 ### Cloud provider
 
-|   | Ubuntu | CentOS | Flatcar | RHEL | Amazon Linux 2 | Rocky Linux | Windows |
+|   | Ubuntu | CentOS | Flatcar | RHEL | Amazon Linux 2 | Rocky Linux | Windows[^1] |
 |---|---|---|---|---|---|---|---|
 | AWS | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | x |
 | Azure | ✓ | ✓ | ✓ | ✓ | x | ✓ | ✓ |
@@ -18,6 +18,8 @@
 | VMware Cloud Director | ✓ | x | x | x | x | x | x |
 | VSphere | ✓ | ✓ | ✓ | ✓ | x | ✓ | x |
 
+[^1]: Windows support is considered experimental.
+
 ## Configuring a operating system
 
 The operating system to use can be set via `machine.spec.providerConfig.operatingSystem`.
@@ -45,3 +47,4 @@ Machine controller may work with other OS versions that are not listed in the ta
 | RHEL | 8.x |
 | Rocky Linux | 8.5 |
 | Ubuntu | 20.04 LTS, 22.04 LTS |
+| Windows | windows-2019, windows-2022 |
diff --git a/hack/verify-boilerplate.sh b/hack/verify-boilerplate.sh
@@ -22,4 +22,5 @@ boilerplate \
   -boilerplates hack/boilerplate \
   -exclude pkg/machines/v1alpha1 \
   -exclude pkg/signals \
-  -exclude pkg/userdata/scripts
+  -exclude pkg/userdata/scripts \
+  -exclude image-builder/windows
diff --git a/image-builder/windows/Makefile b/image-builder/windows/Makefile
@@ -0,0 +1,91 @@
+# Copyright 2023 The Machine Controller Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+SHELL = /bin/bash -eu -o pipefail
+
+## --------------------------------------
+## Packer flags
+## --------------------------------------
+
+# Set Packer color to true if not already set in env variables
+# Only valid for builds
+ifneq (,$(findstring build-, $(MAKECMDGOALS)))
+	# A build target
+	PACKER_COLOR ?= true
+	PACKER_FLAGS += -color=$(PACKER_COLOR)
+endif
+
+# If FOREGROUND=1 then Packer will set headless to false, causing local builds
+# to build in the foreground, with a UI. This is very useful when debugging new
+# platforms or issues with existing ones.
+ifeq (1,$(strip $(FOREGROUND)))
+PACKER_FLAGS += -var="headless=false"
+endif
+
+# If ON_ERROR_ASK=1 then Packer will set -on-error to ask, causing the Packer
+# build to pause when any error happens, instead of simply exiting. This is
+# useful when debugging unknown issues logging into the remote machine via ssh.
+ifeq (1,$(strip $(ON_ERROR_ASK)))
+PACKER_FLAGS += -on-error=ask
+endif
+
+# ssh_private_key_file and ssh_public_key are needed to pass ssh keypair
+# from its host to the packer guest machine, so boot managers like ignition
+# could make use of the key in its config.
+# SSH_PRIVATE_KEY_FILE is name of the file that contains the private key.
+# SSH_PUBLIC_KEY_FILE is name of the file that contains the public key.
+ifneq (,$(strip $(SSH_PRIVATE_KEY_FILE)))
+PACKER_FLAGS += -var ssh_private_key_file="$(SSH_PRIVATE_KEY_FILE)"
+endif
+
+ifneq (,$(strip $(SSH_PUBLIC_KEY_FILE)))
+PACKER_FLAGS += -var ssh_public_key="$(shell cat ${SSH_PUBLIC_KEY_FILE})"
+endif
+
+# Since OpenSSH 9.0+ 'scp' uses SFTP protocol instead of legacy SCP protocol, which causes building errors like:
+#
+# bash: line 1: /usr/lib/sftp-server: No such file or directory\nscp: Connection closed\r\n""
+#
+# However, -O option is not available in older OpenSSH version, so we cannot always set it as an option to use.
+# To provide better out-of-the-box experience for users with newer versions of OpenSSH, we conditionally ensure
+# -O is used when used OpenSSH version requires it.
+#
+# See https://github.com/kubernetes-sigs/image-builder/issues/859 and
+# https://github.com/hashicorp/packer-plugin-ansible/issues/100 for more details.
+ifeq ($(shell test $$(ssh -V 2>&1 | cut -d _ -f2 | cut -d . -f1) -ge 9; echo $$?),0)
+	# Use ?= to retain possible existing value of environment variable. If it is already declared, we assume user to be
+	# aware of OpenSSH version they use and it is up to the user to specify "-O" option as well if needed.
+  export ANSIBLE_SCP_EXTRA_ARGS ?= "-O"
+endif
+
+# If DEBUG=1 then Packer will set -debug, enabling debug mode for builds, providing
+# more verbose logging
+ifeq (1,$(strip $(DEBUG)))
+PACKER_FLAGS += -debug
+endif
+
+COMMON_WINDOWS_VAR_FILES :=	packer/common/ansible-args-windows.json \
+					packer/common/common.json \
+					packer/common/cloudbase-init.json
+
+PACKER_WINDOWS_NODE_FLAGS := $(foreach f,$(abspath $(COMMON_WINDOWS_VAR_FILES)),-var-file="$(f)" )
+
+azure-deps:
+	hack/ensure-ansible.sh
+	hack/ensure-ansible-windows.sh
+	hack/ensure-packer.sh
+
+build-azure-image: azure-deps
+	packer validate $(PACKER_WINDOWS_NODE_FLAGS) -var-file="$(abspath packer/azure/azure-config.json)" -var-file="$(abspath packer/azure/$(distribution))"  $(abspath packer/azure/packer-windows.json)
+	packer build $(PACKER_WINDOWS_NODE_FLAGS) -var-file="$(abspath packer/azure/azure-config.json)" -var-file="$(abspath packer/azure/$(distribution))"  $(abspath packer/azure/packer-windows.json)
diff --git a/image-builder/windows/README.md b/image-builder/windows/README.md
@@ -0,0 +1,28 @@
+# Build image for Windows Nodes
+
+These scripts are used to build a custom image for windows. The stock/base images don't have support for a provisioning utility that can process the userdata and perform the required configurations on the machine. To overcome this limitation, we resort to custom images with some pre-configurations including [cloudbase-init](https://cloudbase.it/cloudbase-init/); the provisioning utility for windows.
+
+This is an indirect clone of [image-builder-capi](https://github.com/kubernetes-sigs/image-builder/tree/master/images/capi) and extends it to make the images compatible with machine-controller.
+
+## Prerequisites
+
+- [Packer](https://www.packer.io/intro/getting-started/install.html) version >= 1.6.0
+- [Ansible](http://docs.ansible.com/ansible/latest/intro_installation.html) version >= 2.10.0
+
+## Azure
+
+Following distributions are supported:
+
+- windows-2019
+- windows-2022
+
+### Requirements
+
+- An Azure account
+- The Azure CLI installed and configured
+- Set environment variables for `AZURE_SUBSCRIPTION_ID`, `AZURE_CLIENT_ID`, `AZURE_CLIENT_SECRET`
+- Set optional environment variables `RESOURCE_GROUP_NAME`, `STORAGE_ACCOUNT_NAME`, `AZURE_LOCATION` & `GALLERY_NAME` to override the default values
+
+### Build command
+
+`make build-azure-image distribution=windows-2022.json`
diff --git a/image-builder/windows/ansible/ansible_winrm.ps1 b/image-builder/windows/ansible/ansible_winrm.ps1
@@ -0,0 +1,51 @@
+# Copyright 2020 The Kubernetes Authors.
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file is from packer documentation:
+# https://www.packer.io/docs/provisioners/ansible.html#winrm-communicator
+# https://www.packer.io/docs/builders/amazon/ebs#connecting-to-windows-instances-using-winrm
+
+Write-Host "Changing PS execution policy to Unrestricted" -ForegroundColor Cyan
+Set-ExecutionPolicy Unrestricted -Scope LocalMachine -Force -ErrorAction Ignore
+
+# Don't set this before Set-ExecutionPolicy as it throws an error
+$ErrorActionPreference = "stop"
+
+Write-Host "Execution Policy configured" -ForegroundColor Cyan
+
+# Remove HTTP listener
+Remove-Item -Path WSMan:\Localhost\listener\listener* -Recurse
+
+# Create a self-signed certificate to let ssl work
+$Cert = New-SelfSignedCertificate -CertstoreLocation Cert:\LocalMachine\My -DnsName "packer"
+New-Item -Path WSMan:\LocalHost\Listener -Transport HTTPS -Address * -CertificateThumbPrint $Cert.Thumbprint -Force
+
+# WinRM
+write-output "Setting up WinRM"
+write-host "(host) setting up WinRM"
+
+cmd.exe /c winrm quickconfig -q
+cmd.exe /c winrm set "winrm/config" '@{MaxTimeoutms="1800000"}'
+cmd.exe /c winrm set "winrm/config/winrs" '@{MaxMemoryPerShellMB="1024"}'
+cmd.exe /c winrm set "winrm/config/service" '@{AllowUnencrypted="true"}'
+cmd.exe /c winrm set "winrm/config/client" '@{AllowUnencrypted="true"}'
+cmd.exe /c winrm set "winrm/config/service/auth" '@{Basic="true"}'
+cmd.exe /c winrm set "winrm/config/client/auth" '@{Basic="true"}'
+cmd.exe /c winrm set "winrm/config/service/auth" '@{CredSSP="true"}'
+cmd.exe /c winrm set "winrm/config/listener?Address=*+Transport=HTTPS" "@{Port=`"5986`";Hostname=`"packer`";CertificateThumbprint=`"$($Cert.Thumbprint)`"}"
+cmd.exe /c netsh advfirewall firewall set rule group="remote administration" new enable=yes
+cmd.exe /c netsh firewall add portopening TCP 5986 "Port 5986"
+cmd.exe /c net stop winrm
+cmd.exe /c sc config winrm start= auto
+cmd.exe /c net start winrm
diff --git a/image-builder/windows/ansible/example.vars.yml b/image-builder/windows/ansible/example.vars.yml
@@ -0,0 +1,29 @@
+# Copyright 2020 The Kubernetes Authors.
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+cloudbase_init_url: https://github.com/cloudbase/cloudbase-init/releases/download/1.1.4/CloudbaseInitSetup_1_1_4_x64.msi
+
+windows_service_manager: 'nssm'
+pause_image: "registry.k8s.io/pause:3.9"
+load_additional_components: true
+prepull: false
+distribution_version: 2019
+
+cloudbase_metadata_services: "cloudbaseinit.metadata.services.azureservice.AzureService, cloudbaseinit.metadata.services.ovfservice.OvfService"
+cloudbase_plugins: "cloudbaseinit.plugins.common.ephemeraldisk.EphemeralDiskPlugin, cloudbaseinit.plugins.common.userdata.UserDataPlugin, cloudbaseinit.plugins.common.localscripts.LocalScriptsPlugin"
+cloudbase_metadata_services_unattend: "cloudbaseinit.metadata.services.azureservice.AzureService, cloudbaseinit.metadata.services.ovfservice.OvfService"
+cloudbase_plugins_unattend: "cloudbaseinit.plugins.common.mtu.MTUPlugin"
+
+debug_tools: true
+additional_debug_files: "https://raw.githubusercontent.com/kubernetes-sigs/sig-windows-tools/master/hack/DebugWindowsNode.ps1"
diff --git a/image-builder/windows/ansible/node_windows.yml b/image-builder/windows/ansible/node_windows.yml
@@ -0,0 +1,90 @@
+# Copyright 2020 The Kubernetes Authors.
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+- hosts: all
+  vars:
+    node_custom_roles_pre: ""
+    node_custom_roles_post: ""
+    custom_role_names: ""
+
+  tasks:
+    - name: Check if cloudbase-init url is set
+      set_fact:
+        install_cloudbase_init: '{{ true if (cloudbase_init_url is defined) and (cloudbase_init_url|length > 0) else false }}'
+
+    # https://docs.ansible.com/ansible/latest/user_guide/windows_performance.html
+    - name: Optimise powershell
+      win_shell: |
+        function Optimize-PowershellAssemblies {
+          # NGEN powershell assembly, improves startup time of powershell by 10x
+          $old_path = $env:path
+          try {
+            $env:path = [Runtime.InteropServices.RuntimeEnvironment]::GetRuntimeDirectory()
+            [AppDomain]::CurrentDomain.GetAssemblies() | % {
+              if (! $_.location) {continue}
+              $Name = Split-Path $_.location -leaf
+              if ($Name.startswith("Microsoft.PowerShell.")) {
+                Write-Progress -Activity "Native Image Installation" -Status "$name"
+                ngen install $_.location | % {"`t$_"}
+              }
+            }
+          } finally {
+            $env:path = $old_path
+          }
+        }
+        Optimize-PowershellAssemblies
+      become: yes
+      become_method: runas
+      become_user: SYSTEM
+
+    - name: Get Install Drive
+      win_shell: $env:SYSTEMDRIVE
+      register: systemdrive
+
+    - name: Get Program Files Directory
+      win_shell: $env:ProgramFiles
+      register: programfiles
+
+    - name: Get All Users profile path
+      win_shell: $env:ALLUSERSPROFILE.Replace("\", "\\")
+      register: alluserprofile
+
+    - name: Get TEMP Directory
+      win_shell: $env:TEMP
+      register: tempdir
+
+    - include_role:
+        name: "{{ role }}"
+      loop: "{{ node_custom_roles_pre.split() }}"
+      loop_control:
+        loop_var: role
+      when: node_custom_roles_pre != ""
+    - include_role:
+        name: systemprep
+    - include_role:
+        name: cloudbase-init
+      when: install_cloudbase_init
+    - include_role:
+        name: providers
+    - include_role:
+        name: "{{ role }}"
+      loop: "{{ custom_role_names.split() + node_custom_roles_post.split() }}"
+      loop_control:
+        loop_var: role
+      when: custom_role_names != "" or node_custom_roles_post != ""
+
+  environment:
+    HTTP_PROXY: "{{ http_proxy | default('') }}"
+    HTTPS_PROXY: "{{ https_proxy | default('') }}"
+    NO_PROXY: "{{ no_proxy | default('') }}"
diff --git a/image-builder/windows/ansible/roles/cloudbase-init/tasks/main.yml b/image-builder/windows/ansible/roles/cloudbase-init/tasks/main.yml
@@ -0,0 +1,53 @@
+# Copyright 2020 The Kubernetes Authors.
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+- name: Download Cloudbase-init
+  win_get_url:
+    url: "{{ cloudbase_init_url }}"
+    dest: '{{ tempdir.stdout | trim }}\CloudbaseInitSetup.msi'
+  register: installer
+  retries: 5
+  delay: 3
+  until: installer is not failed
+
+- name: Ensure log directory
+  win_file:
+    path: '{{ systemdrive.stdout | trim }}\logs'
+    state: directory
+
+- name: Install Cloudbase-init
+  win_package:
+    path: '{{ installer.dest }}'
+    log_path: '{{ systemdrive.stdout | trim }}\logs\cloudbase-install-log-{{lookup("pipe", "date +%Y%m%dT%H%M%S")}}.log'
+
+# configuration modified from https://github.com/cloudbase/windows-openstack-imaging-tools/tree/master/Examples/config/azure
+- name: Set up cloudbase-init unattend configuration
+  win_template:
+    src: templates/cloudbase-init-unattend.conf
+    dest: '{{ programfiles.stdout | trim }}\Cloudbase Solutions\Cloudbase-Init\conf\cloudbase-init-unattend.conf'
+
+# configuration modified from https://github.com/cloudbase/windows-openstack-imaging-tools/tree/master/Examples/config/azure
+- name: Set up cloudbase-init configuration
+  win_template:
+    src: templates/cloudbase-init.conf
+    dest: '{{ programfiles.stdout | trim }}\Cloudbase Solutions\Cloudbase-Init\conf\cloudbase-init.conf'
+
+- name: Configure set up complete
+  win_shell: |
+    # If this file already exists then the following command fails
+    Remove-Item -Force {{ systemdrive.stdout | trim }}\Windows\Setup\Scripts\SetupComplete.cmd
+    & "{{ programfiles.stdout | trim }}\Cloudbase Solutions\Cloudbase-Init\bin\SetSetupComplete.cmd"
+  become: yes
+  become_method: runas
+  become_user: System