diff --git a/test/images/Makefile b/test/images/Makefile
index 7b9981cb19e08..c025a0e1b0fcb 100644
--- a/test/images/Makefile
+++ b/test/images/Makefile
@@ -33,10 +33,11 @@ endif
 all: all-container
 
 all-container:
-	./image-util.sh build $(WHAT)
+	./image-util.sh build $(WHAT) "docker"
 
-all-push: all-container
-	./image-util.sh push $(WHAT)
+all-push:
+	bash -x ./image-util.sh build $(WHAT) "registry"
+	bash -x ./image-util.sh push $(WHAT)
 
 all-build-and-push:
 	./image-util.sh build_and_push ${WHAT}
diff --git a/test/images/README.md b/test/images/README.md
index e4ed1ddf5856c..93a9352ed60d4 100644
--- a/test/images/README.md
+++ b/test/images/README.md
@@ -11,80 +11,18 @@ new images, test the changes made, promote the newly built staging images.
 
 ## Prerequisites
 
-In order to build the docker test images, a Linux node is required. The node will require `make`
-and `docker (version 18.06.0 or newer)`. Manifest lists were introduced in 18.03.0, but 18.06.0
-is recommended in order to avoid certain issues.
+In order to build the docker test images, a Linux node is required. The node will require `make`,
+`docker (version 19.03.0 or newer)`, and ``docker buildx``, which will be used to build multiarch
+images, as well as Windows images. In order to properly build multiarch and Windows images, some
+initialization is required:
 
-The node must be able to push the images to the desired container registry, make sure you are
-authenticated with the registry you're pushing to.
-
-Windows Container images are not built by default, since they cannot be built on Linux. For
-that, a Windows node with Docker installed and configured for remote management is required.
-
-
-### Windows node(s) setup
-
-In order to build the Windows container images, a node with Windows 10 or Windows Server 2019
-with the latest updates installed is required. The node will have to have Docker installed,
-preferably version 18.06.0 or newer.
-
-Keep in mind that the Windows node might not be able to build container images for newer OS versions
-than itself (even with `--isolation=hyperv`), so keeping the node up to date and / or upgrading it
-to the latest Windows Server edition is ideal.
-
-Windows test images must be built for Windows Server 2019 (1809) and Windows Server 1903, thus,
-if the node does not have Hyper-V enabled, or it is not supported, multiple Windows nodes are required,
-one per OS version.
-
-Additionally, remote management must be configured for the node's Docker daemon. Exposing the
-Docker daemon without requiring any authentication is not recommended, and thus, it must be
-configured with TLS to ensure that only authorised people can interact with it. For this, the
-following `powershell` script can be executed:
-
-```powershell
-mkdir .docker
-docker run --isolation=hyperv --user=ContainerAdministrator --rm `
-  -e SERVER_NAME=$(hostname) `
-  -e IP_ADDRESSES=127.0.0.1,YOUR_WINDOWS_BUILD_NODE_IP `
-  -v "c:\programdata\docker:c:\programdata\docker" `
-  -v "$env:USERPROFILE\.docker:c:\users\containeradministrator\.docker" stefanscherer/dockertls-windows:2.5.5
-# restart the Docker daemon.
-Restart-Service docker
-```
-
-For more information about the above commands, you can check [here](https://hub.docker.com/r/stefanscherer/dockertls-windows/).
-
-A firewall rule to allow connections to the Docker daemon is necessary:
-
-```powershell
-New-NetFirewallRule -DisplayName 'Docker SSL Inbound' -Profile @('Domain', 'Public', 'Private') -Direction Inbound -Action Allow -Protocol TCP -LocalPort 2376
-```
-
-If your Windows build node is hosted by a cloud provider, make sure the port `2376` is open for the node.
-For example, in Azure, this is done by running the following command:
-
-```console
-az vm open-port -g GROUP-NAME -n NODE-NAME --port 2376
-```
-
-The `ca.pem`, `cert.pem`, and `key.pem` files that can be found in `$env:USERPROFILE\.docker`
-will have to copied to the `~/.docker-${os_version)/` on the Linux build node, where `${os_version}`
-is `1809` or `1903`.
-
-```powershell
-scp.exe -r $env:USERPROFILE\.docker ubuntu@YOUR_LINUX_BUILD_NODE:/home/ubuntu/.docker-$os_version
-```
-
-After all this, the Linux build node should be able to connect to the Windows build node:
-
-```bash
-docker --tlsverify --tlscacert ~/.docker-${os_version}/ca.pem --tlscert ~/.docker-${os_version}/cert.pem --tlskey ~/.docker-${os_version}/key.pem -H "$REMOTE_DOCKER_URL" version
+```shell
+docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+docker buildx create --name img-builder --use
+docker buildx inspect --bootstrap
 ```
 
-For more information and troubleshooting about enabling Docker remote management, see
-[here](https://docs.microsoft.com/en-us/virtualization/windowscontainers/management/manage_remotehost)
-
-Finally, the node must be able to push the images to the desired container registry, make sure you are
+The node must be able to push the images to the desired container registry, make sure you are
 authenticated with the registry you're pushing to.
 
 
@@ -136,6 +74,38 @@ The images are built through `make`. Since some images (e.g.: `busybox`) are use
 other images, it is recommended to build them first, if needed.
 
 
+### Windows test images considerations
+
+Ideally, the same `Dockerfile` can be used to build both Windows and Linux images. However, that isn't
+always possible. If a different `Dockerfile` is needed for an image, it should be named `Dockerfile_windows`.
+When building, `image-util.sh` will first check for this file name when building Windows images.
+
+The building process uses `docker buildx` to build both Windows and Linux images, but there are a few
+limitations when it comes to the Windows images:
+
+- The Dockerfile can have multiple stages, including Windows and Linux stages for the same image, but
+  the Windows stage cannot have any `RUN` commands (see the agnhost's `Dockerfile_windows` as an example).
+- The Windows stage cannot have any `WORKDIR` commands due to a bug (https://github.com/docker/buildx/issues/378)
+- When copying Windows symlink files to a Windows image, `docker buildx` changes the symlink target,
+  prepending `Files\` to them (https://github.com/docker/buildx/issues/373) (for example, the symlink
+  target `C:\bin\busybox.exe` becomes `Files\C:\bin\busybox.exe`). This can be avoided by having symlink
+  targets with relative paths and having the target duplicated (for example, the symlink target
+  `busybox.exe` becomes `Files\busybox.exe` when copied, so the binary `C:\bin\Files\busybox.exe`
+  should exist in order for the symlink to be used correctly). See the busybox's `Dockerfile_windows` as
+  an example.
+- `docker buildx` overwrites the image's PATH environment variable to a Linux PATH environment variable,
+  which won't work properly on Windows. See https://github.com/moby/buildkit/issues/1560
+- The base image for all the Windows images is nanoserver, which is ~10 times smaller than Windows Servercore.
+  Most binaries added to the image will work out of the box, but some will not due to missing dependencies
+  (**atention**: the image will still build successfully, even if the added binaries will not work).
+  For example, `coredns.exe` requires `netapi32.dll`, which cannot be found on a nanoserver image, but
+  we can copy it from a servercore image (see the agnhost image's `Dockerfile_windows` file as an example).
+  A good rule of thumb is to use 64-bit applications instead of 32-bit as they have fewer dependencies.
+  You can determine what dependencies are missing by running `procmon.exe` on the container's host
+  (make sure that process isolation is used, not Hyper-V isolation).
+  [This](https://stefanscherer.github.io/find-dependencies-in-windows-containers/) is a useful guide on how to use `procmon.exe`.
+
+
 ## Building images
 
 The images are built through `make`. Since some images (`agnhost`) are used as a base for other images,
@@ -160,14 +130,6 @@ registry. That can changed by running this command instead:
 REGISTRY=foo_registry make all-push WHAT=agnhost
 ```
 
-In order to also include Windows Container images into the final manifest lists, the `REMOTE_DOCKER_URL` argument
-in the form `tcp://[host]:[port][path]` (for more details, see [here]([https://docs.docker.com/engine/reference/commandline/dockerd/#daemon-socket-option]/))
-will also have to be specified:
-
-```bash
-REMOTE_DOCKER_URL_1909=remote_docker_url_1909 REMOTE_DOCKER_URL_1903=remote_docker_url_1903 REMOTE_DOCKER_URL_1809=remote_docker_url_1809 REGISTRY=foo_registry make all-push WHAT=test-webserver
-```
-
 *NOTE* (for test `gcr.io` image publishers): Some tests (e.g.: `should serve a basic image on each replica with a private image`)
 require the `agnhost` image to be published in an authenticated repo as well:
 
@@ -215,15 +177,3 @@ After all the above has been done, run the desired tests.
 ```bash
 sudo chmod o+x /etc/docker
 ```
-
-`nc` is being used by some E2E tests, which is why we are including a Linux-like `nc.exe` into the Windows `busybox` image. The image could fail to build during that step with an error that looks like this:
-
-```console
-re-exec error: exit status 1: output: time="..." level=error msg="hcsshim::ImportLayer failed in Win32: The system cannot find the path specified. (0x3) path=\\\\?\\C:\\ProgramData\\...
-```
-
-The issue is caused by the Windows Defender which is removing the `nc.exe` binary from the filesystem. For more details on this issue, see [here](https://github.com/diegocr/netcat/issues/6). To fix this, you can simply run the following powershell command to temporarily disable Windows Defender:
-
-```powershell
-Set-MpPreference -DisableRealtimeMonitoring $true
-```
diff --git a/test/images/agnhost/Dockerfile_windows b/test/images/agnhost/Dockerfile_windows
index 4572e5c17bbb4..94b8ae45cb89f 100644
--- a/test/images/agnhost/Dockerfile_windows
+++ b/test/images/agnhost/Dockerfile_windows
@@ -14,7 +14,23 @@
 
 ARG BASEIMAGE
 ARG REGISTRY
-FROM $REGISTRY/windows-image-builder-helper:1.0 as helper
+ARG OS_VERSION
+
+# We're using a Linux image to unpack the archives, then we're copying them over to Windows.
+FROM --platform=linux/amd64 alpine:3.6 as prep
+
+ADD https://github.com/coredns/coredns/releases/download/v1.5.0/coredns_1.5.0_windows_amd64.tgz /coredns.tgz
+ADD https://iperf.fr/download/windows/iperf-2.0.9-win64.zip /iperf.zip
+
+# we're also creating an empty /uploads folder, which we're copying over to Windows  because
+# we can't RUN commands on a Windows image.
+RUN tar -xzvf /coredns.tgz &&\
+    unzip iperf.zip &&\
+    mv iperf-2.0.9-win64 iperf &&\
+    mkdir /uploads
+
+FROM e2eteam/busybox-helper:1.29.0 as busybox-helper
+FROM --platform=linux/amd64 $REGISTRY/windows-servercore-cache:1.0-linux-amd64-$OS_VERSION as servercore-helper
 FROM $BASEIMAGE
 
 # from dnsutils image
@@ -26,20 +42,13 @@ FROM $BASEIMAGE
 # - curl, nc: used by a lot of e2e tests (inherited from BASEIMAGE)
 # from iperf image
 # install necessary packages: iperf
-COPY --from=helper /dig /dig
-COPY --from=helper /Windows/System32/netapi32.dll /Windows/System32/netapi32.dll
-
-RUN setx /M PATH "C:\dig\;%PATH%
-RUN powershell -Command "\
-    curl.exe -L 'https://github.com/coredns/coredns/releases/download/v1.5.0/coredns_1.5.0_windows_amd64.tgz' -o C:\coredns.tgz;\
-    tar.exe -xzvf C:\coredns.tgz;\
-    Remove-Item C:\coredns.tgz"
+COPY --from=busybox-helper /dig /dig
+COPY --from=servercore-helper /Windows/System32/netapi32.dll /Windows/System32/netapi32.dll
+COPY --from=prep /coredns.exe /coredns.exe
+COPY --from=prep /iperf /iperf
 
-RUN powershell -Command "\
-    curl.exe -L 'https://iperf.fr/download/windows/iperf-2.0.9-win64.zip' -o C:\iperf.zip;\
-    Expand-Archive -Path C:\iperf.zip -DestinationPath C:\ -Force;\
-    Rename-Item C:\iperf-2.0.9-win64 C:\iperf;\
-    Remove-Item C:\iperf.zip"
+# NOTE(claudiub): docker buildx sets the PATH env variable to a Linux-like PATH, which is not desirable.
+ENV PATH="C:\dig\;C:\bin\;C:\curl\;C:\Windows\system32;C:\Windows;C:\Program Files\PowerShell;"
 
 # PORT 80 needed by: test-webserver
 # PORT 8080 needed by: netexec, nettest
@@ -48,7 +57,7 @@ RUN powershell -Command "\
 EXPOSE 80 8080 8081 9376
 
 # from netexec
-RUN mkdir C:\uploads
+COPY --from=prep /uploads /uploads
 
 # from porter
 ADD porter/localhost.crt localhost.crt
@@ -57,12 +66,12 @@ ADD porter/localhost.key localhost.key
 # from mounttest
 ADD mounttest/filePermissions.ps1 filePermissions.ps1
 
-ADD agnhost agnhost
+ADD agnhost /agnhost
 
 # needed for the entrypoint-tester related tests. Some of the entrypoint-tester related tests
 # overrides this image's entrypoint with agnhost-2 binary, and will verify that the correct
 # entrypoint is used by the containers.
-RUN mklink agnhost-2 agnhost
+ADD agnhost /agnhost-2
 
 ENTRYPOINT ["/agnhost"]
 CMD ["pause"]
diff --git a/test/images/busybox/Dockerfile_windows b/test/images/busybox/Dockerfile_windows
index be60037e0c2b8..5b25e06c0e9f3 100644
--- a/test/images/busybox/Dockerfile_windows
+++ b/test/images/busybox/Dockerfile_windows
@@ -13,59 +13,45 @@
 # limitations under the License.
 
 ARG BASEIMAGE
-from $BASEIMAGE as prep
+ARG REGISTRY
 
-ENV CURL_VERSION=7.57.0 \
-    PS_VERSION=6.2.0
-WORKDIR /curl
-ADD https://skanthak.homepage.t-online.de/download/curl-$CURL_VERSION.cab curl.cab
-ADD https://github.com/PowerShell/PowerShell/releases/download/v$PS_VERSION/PowerShell-$PS_VERSION-win-x64.zip /PowerShell/powershell.zip
-ADD https://eternallybored.org/misc/netcat/netcat-win32-1.12.zip /netcat//netcat.zip
+# We're using a Linux image to unpack the archive, then we're copying it over to Windows.
+FROM --platform=linux/amd64 alpine:3.6 as prep
 
-USER ContainerAdministrator
-RUN expand /R curl.cab /F:* . &\
-    cd C:\PowerShell &\
-    tar.exe -xf powershell.zip &\
-    del powershell.zip &\
-    mklink powershell.exe pwsh.exe &\
-    cd C:\netcat &\
-    tar.exe -xf netcat.zip &\
-    del netcat.zip
+RUN mkdir /tmp-dir
 
+FROM e2eteam/busybox-helper:1.29.0 as busybox-helper
+FROM e2eteam/powershell-helper:6.2.7 as ps-helper
 FROM $BASEIMAGE
 
-COPY --from=prep /curl/AMD64 /curl/CURL.LIC /curl/
-COPY --from=prep ["/PowerShell", "/Program Files/PowerShell"]
-COPY --from=prep /netcat/nc64.exe /bin/nc.exe
-ADD https://github.com/kubernetes-sigs/windows-testing/raw/master/images/busybox/busybox.exe /bin/busybox.exe
+COPY --from=prep /tmp-dir /tmp
+COPY --from=busybox-helper /bin /bin
+
+# NOTE(claudiub): Unfortunately, docker buildx has some issues copying over Windows symlinks.
+# "Files/" is always prepended to the symlink target. The symlinks themselves are relative paths,
+# so, in order to make use of them, we can simply add a busybox binary to C:\bin\Files\busybox.exe.
+COPY --from=busybox-helper /bin/busybox.exe /bin/Files/
+COPY --from=busybox-helper /curl /curl
+COPY --from=busybox-helper /netcat/nc64.exe /bin/nc.exe
+
+# include powershell and its Module analysis cache.
+COPY --from=ps-helper ["/PowerShell", "/Program Files/PowerShell"]
+
+# NOTE(claudiub): For the same reason mentioned above, we have to copy pwsh.exe to Files/pwsh.exe
+COPY --from=ps-helper ["/PowerShell/pwsh.exe", "/Program Files/PowerShell/Files/"]
+COPY --from=ps-helper /Users/ContainerAdministrator/AppData/Local/Microsoft/Windows/PowerShell/docker/ModuleAnalysisCache /Users/ContainerAdministrator/AppData/Local/Microsoft/Windows/PowerShell/docker/ModuleAnalysisCache
+
 ADD hostname /bin/hostname.exe
 
 USER ContainerAdministrator
-RUN FOR /f "tokens=*" %i IN ('C:\bin\busybox --list') DO mklink C:\bin\%i.exe C:\bin\busybox.exe
-# Set the path
-RUN setx /M PATH "C:\bin;C:\curl\;%PATH%;C:\Program Files\PowerShell;" &\
-    mkdir C:\tmp
 
-# Copy PowerShell Core from the installer container
-ENV ProgramFiles="C:\Program Files" \
+# NOTE(claudiub): docker buildx sets the PATH env variable to a Linux-like PATH, which is not desirable.
+ENV PATH="C:\bin;C:\curl;C:\Windows\System32;C:\Windows;C:\Program Files\PowerShell;" \
+    ProgramFiles="C:\Program Files" \
     # set a fixed location for the Module analysis cache
     LOCALAPPDATA="C:\Users\ContainerAdministrator\AppData\Local" \
-    PSModuleAnalysisCachePath="$LOCALAPPDATA\Microsoft\Windows\PowerShell\docker\ModuleAnalysisCache" \
+    PSModuleAnalysisCachePath="C:\Users\ContainerAdministrator\AppData\Local\Microsoft\Windows\PowerShell\docker\ModuleAnalysisCache" \
     # Persist %PSCORE% ENV variable for user convenience
-    PSCORE="$ProgramFiles\PowerShell\pwsh.exe"
-
-# intialize powershell module cache
-RUN powershell \
-        -NoLogo \
-        -NoProfile \
-        -Command " \
-          $stopTime = (get-date).AddMinutes(15); \
-          $ErrorActionPreference = 'Stop' ; \
-          $ProgressPreference = 'SilentlyContinue' ; \
-          while(!(Test-Path -Path $env:PSModuleAnalysisCachePath)) {  \
-            Write-Host "'Waiting for $env:PSModuleAnalysisCachePath'" ; \
-            if((get-date) -gt $stopTime) { throw 'timout expired'} \
-            Start-Sleep -Seconds 6 ; \
-          }"
+    PSCORE="C:\Program Files\PowerShell\pwsh.exe"
 
 ENTRYPOINT ["cmd.exe", "/s", "/c"]
diff --git a/test/images/cloudbuild.yaml b/test/images/cloudbuild.yaml
index 9384694338ffb..a4e900a778044 100644
--- a/test/images/cloudbuild.yaml
+++ b/test/images/cloudbuild.yaml
@@ -9,25 +9,8 @@ options:
   substitution_option: ALLOW_LOOSE
   machineType: 'N1_HIGHCPU_8'
 steps:
-  - name: gcr.io/cloud-builders/gcloud
+  - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20200422-b25d964'
     entrypoint: 'bash'
-    # NOTE(claudiub): We need to get the ca.pem, cert.pem, key.pem files and put create the
-    # /certs/.docker-1809/, /certs/.docker-1903/, /certs/.docker-1909/ folders, which will contain the files.
-    args:
-    - -c
-    - |
-      mkdir -p .docker-windows
-      gcloud secrets versions access latest --project=k8s-infra-prow-build-trusted --secret=windows-remote-docker_ca-pem > .docker-windows/ca.pem
-      gcloud secrets versions access latest --project=k8s-infra-prow-build-trusted --secret=windows-remote-docker_cert-pem > .docker-windows/cert.pem
-      gcloud secrets versions access latest --project=k8s-infra-prow-build-trusted --secret=windows-remote-docker_key-pem > .docker-windows/key.pem
-      cp -r .docker-windows /certs/.docker-1809
-      cp -r .docker-windows /certs/.docker-1903
-      cp -r .docker-windows /certs/.docker-1909
-    volumes:
-    - name: 'certs'
-      path: '/certs'
-  - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20190906-745fed4'
-    entrypoint: make
     dir: ./test/images/
     env:
     - DOCKER_CLI_EXPERIMENTAL=enabled
@@ -35,15 +18,13 @@ steps:
     - BASE_REF=$_PULL_BASE_REF
     - WHAT=$_WHAT
     - REGISTRY=gcr.io/k8s-staging-e2e-test-images
-    - DOCKER_CERT_BASE_PATH=/certs
-    - REMOTE_DOCKER_URL_1809=tcp://img-promoter-1809.eastus.cloudapp.azure.com:2376
-    - REMOTE_DOCKER_URL_1903=tcp://img-promoter-1903.eastus.cloudapp.azure.com:2376
-    - REMOTE_DOCKER_URL_1909=tcp://img-promoter-1909.eastus.cloudapp.azure.com:2376
     args:
-    - all-build-and-push
-    volumes:
-    - name: 'certs'
-      path: '/certs'
+    - '-c'
+    - |
+      docker run --rm --privileged multiarch/qemu-user-static --reset -p yes \
+      && docker buildx create --name img-builder --use \
+      && docker buildx inspect --bootstrap \
+      && make all-build-and-push
 substitutions:
   # _GIT_TAG will be filled with a git-based tag for the image, of the form vYYYYMMDD-hash, and
   # can be used as a substitution
diff --git a/test/images/image-util.sh b/test/images/image-util.sh
index beeb72085e952..25a18488f8421 100755
--- a/test/images/image-util.sh
+++ b/test/images/image-util.sh
@@ -33,6 +33,19 @@ source "${KUBE_ROOT}/hack/lib/util.sh"
 # Mapping of go ARCH to actual architectures shipped part of multiarch/qemu-user-static project
 declare -A QEMUARCHS=( ["amd64"]="x86_64" ["arm"]="arm" ["arm64"]="aarch64" ["ppc64le"]="ppc64le" ["s390x"]="s390x" )
 
+windows_os_versions=(1809 1903 1909 2004)
+declare -A WINDOWS_OS_VERSIONS_MAP
+
+initWindowsOsVersions() {
+  for os_version in "${windows_os_versions[@]}"; do
+    img_base="mcr.microsoft.com/windows/nanoserver:${os_version}"
+    full_version=$(docker manifest inspect "${img_base}" | grep "os.version" | head -n 1 | awk '{print $2}') || true
+    WINDOWS_OS_VERSIONS_MAP["${os_version}"]="${full_version}"
+  done
+}
+
+initWindowsOsVersions
+
 # Returns list of all supported architectures from BASEIMAGE file
 listOsArchs() {
   image=$1
@@ -50,15 +63,10 @@ splitOsArch() {
       arch=$(echo "$os_arch" | cut -d "/" -f 2)
       os_version=$(echo "$os_arch" | cut -d "/" -f 3)
       suffix="$os_name-$arch-$os_version"
-
-      # currently, GCE does not have Hyper-V support, which means that the same node cannot be used to build
-      # multiple versions of Windows images. Which is why we have $REMOTE_DOCKER_URL_$os_version URLs configured.
-      # TODO(claudiub): once Hyper-V support has been added to GCE, revert this to just $REMOTE_DOCKER_URL.
-      remote_docker_url_name="REMOTE_DOCKER_URL_$os_version"
-      REMOTE_DOCKER_URL=$(eval echo "\${${remote_docker_url_name}:-}")
     elif [[ $os_arch =~ .*/.* ]]; then
       os_name=$(echo "$os_arch" | cut -d "/" -f 1)
       arch=$(echo "$os_arch" | cut -d "/" -f 2)
+      os_version=""
       suffix="$os_name-$arch"
     else
       echo "The BASEIMAGE file for the ${image} image is not properly formatted. Expected entries to start with 'os/arch', found '${os_arch}' instead."
@@ -78,6 +86,9 @@ getBaseImage() {
 # arm64, ppc64le, s390x
 build() {
   image=$1
+  output_type=$2
+  docker_version_check
+
   if [[ -f ${image}/BASEIMAGE ]]; then
     os_archs=$(listOsArchs "$image")
   else
@@ -89,10 +100,8 @@ build() {
 
   for os_arch in ${os_archs}; do
     splitOsArch "${image}" "${os_arch}"
-    if [[ "${os_name}" == "windows" && -z "${REMOTE_DOCKER_URL}" ]]; then
-      # If we have a Windows os_arch entry but no Remote Docker Daemon for it,
-      # we should skip it, so we don't have to build any binaries for it.
-      echo "Cannot build the image '${image}' for ${os_arch}. REMOTE_DOCKER_URL_$os_version should be set, containing the URL to a Windows docker daemon."
+    if [[ "${os_name}" == "windows" && "${output_type}" == "docker" ]]; then
+      echo "Cannot build the image '${image}' for ${os_arch}. Built Windows container images need to be pushed to a registry."
       continue
     fi
 
@@ -148,28 +157,19 @@ build() {
       fi
     fi
 
-    if [[ "$os_name" = "linux" ]]; then
-      docker build --pull -t "${REGISTRY}/${image}:${TAG}-${os_name}-${arch}" --build-arg BASEIMAGE="${BASEIMAGE}" .
-    elif [[ -n "${REMOTE_DOCKER_URL:-}" ]]; then
-      # NOTE(claudiub): We're using a remote Windows node to build the Windows Docker images.
-      # The node requires TLS authentication, and thus it is expected that the
-      # ca.pem, cert.pem, key.pem files can be found in the ${HOME}/.docker-${os_version} folder.
-      # TODO(claudiub): add "build --isolation=hyperv" once GCE introduces Hyper-V support.
-      docker --tlsverify --tlscacert "${DOCKER_CERT_BASE_PATH}/.docker-${os_version}/ca.pem" \
-        --tlscert "${DOCKER_CERT_BASE_PATH}/.docker-${os_version}/cert.pem" --tlskey "${DOCKER_CERT_BASE_PATH}/.docker-${os_version}/key.pem" \
-        -H "${REMOTE_DOCKER_URL}" build --pull -t "${REGISTRY}/${image}:${TAG}-${os_name}-${arch}-${os_version}" \
-        --build-arg BASEIMAGE="${BASEIMAGE}" --build-arg REGISTRY="${REGISTRY}" -f $dockerfile_name .
-    fi
+    docker buildx build --no-cache --pull --output=type="${output_type}" --platform "${os_name}/${arch}" \
+        --build-arg BASEIMAGE="${BASEIMAGE}" --build-arg REGISTRY="${REGISTRY}" --build-arg OS_VERSION="${os_version}" \
+        -t "${REGISTRY}/${image}:${TAG}-${suffix}" -f "${dockerfile_name}" .
+
     popd
   done
 }
 
 docker_version_check() {
-  # The reason for this version check is even though "docker manifest" command is available in 18.03, it does
-  # not work properly in that version. So we insist on 18.06.0 or higher.
+  # docker buildx has been introduced in 19.03, so we need to make sure we have it.
   docker_version=$(docker version --format '{{.Client.Version}}' | cut -d"-" -f1)
-  if [[ ${docker_version} != 18.06.0 && ${docker_version} < 18.06.0 ]]; then
-    echo "Minimum docker version 18.06.0 is required for creating and pushing manifest images[found: ${docker_version}]"
+  if [[ ${docker_version} != 19.03.0 && ${docker_version} < 19.03.0 ]]; then
+    echo "Minimum docker version 19.03.0 is required for using docker buildx: ${docker_version}]"
     exit 1
   fi
 }
@@ -178,6 +178,7 @@ docker_version_check() {
 push() {
   image=$1
   docker_version_check
+
   TAG=$(<"${image}"/VERSION)
   if [[ -f ${image}/BASEIMAGE ]]; then
     os_archs=$(listOsArchs "$image")
@@ -185,28 +186,6 @@ push() {
     # prepend linux/ to the QEMUARCHS items.
     os_archs=$(printf 'linux/%s\n' "${!QEMUARCHS[*]}")
   fi
-  for os_arch in ${os_archs}; do
-    splitOsArch "${image}" "${os_arch}"
-
-    if [[ "$os_name" = "linux" ]]; then
-      docker push "${REGISTRY}/${image}:${TAG}-${os_name}-${arch}"
-    elif [[ -n "${REMOTE_DOCKER_URL:-}" ]]; then
-      # NOTE(claudiub): We're pushing the image we built on the remote Windows node.
-      docker --tlsverify --tlscacert "${DOCKER_CERT_BASE_PATH}/.docker-${os_version}/ca.pem" \
-        --tlscert "${DOCKER_CERT_BASE_PATH}/.docker-${os_version}/cert.pem" --tlskey "${DOCKER_CERT_BASE_PATH}/.docker-${os_version}/key.pem" \
-        -H "${REMOTE_DOCKER_URL}" push "${REGISTRY}/${image}:${TAG}-${os_name}-${arch}-${os_version}"
-    else
-      echo "Cannot push the image '${image}' for ${os_arch}. REMOTE_DOCKER_URL_${os_version} should be set, containing the URL to a Windows docker daemon."
-      # we should exclude this image from the manifest list as well, we couldn't build / push it.
-      os_archs=$(printf "%s\n" "$os_archs" | grep -v "$os_arch" || true)
-    fi
-  done
-
-  if test -z "${os_archs}"; then
-    # this can happen for Windows-only images if they have been skipped entirely.
-    echo "No image for the manifest list. Skipping ${image}."
-    return
-  fi
 
   kube::util::ensure-gnu-sed
 
@@ -217,9 +196,31 @@ push() {
   # Make os_archs list into image manifest. Eg: 'linux/amd64 linux/ppc64le' to '${REGISTRY}/${image}:${TAG}-linux-amd64 ${REGISTRY}/${image}:${TAG}-linux-ppc64le'
   while IFS='' read -r line; do manifest+=("$line"); done < <(echo "$os_archs" | ${SED} "s~\/~-~g" | ${SED} -e "s~[^ ]*~$REGISTRY\/$image:$TAG\-&~g")
   docker manifest create --amend "${REGISTRY}/${image}:${TAG}" "${manifest[@]}"
+
+  # We will need the full registry name in order to set the "os.version" for Windows images.
+  # If the ${REGISTRY} dcesn't have any slashes, it means that it's on dockerhub.
+  registry_prefix=""
+  if [[ ! $REGISTRY =~ .*/.* ]]; then
+    registry_prefix="docker.io/"
+  fi
+  # The images in the manifest list are stored locally. The folder / file name is almost the same,
+  # with a few changes.
+  manifest_image_folder=$(echo "${registry_prefix}${REGISTRY}/${image}:${TAG}" | sed "s|/|_|g" | sed "s/:/-/")
+
   for os_arch in ${os_archs}; do
     splitOsArch "${image}" "${os_arch}"
     docker manifest annotate --os "${os_name}" --arch "${arch}" "${REGISTRY}/${image}:${TAG}" "${REGISTRY}/${image}:${TAG}-${suffix}"
+
+    # For Windows images, we also need to include the "os.version" in the manifest list, so the Windows node
+    # can pull the proper image it needs.
+    if [[ "$os_name" = "windows" ]]; then
+      full_version="${WINDOWS_OS_VERSIONS_MAP[$os_version]}"
+
+      # At the moment, docker manifest annotate doesn't allow us to set the os.version, so we'll have to
+      # it ourselves. The manifest list can be found locally as JSONs.
+      sed -i -r "s/(\"os\"\:\"windows\")/\0,\"os.version\":$full_version/" \
+        "${HOME}/.docker/manifests/${manifest_image_folder}/${manifest_image_folder}-${suffix}"
+    fi
   done
   docker manifest push --purge "${REGISTRY}/${image}:${TAG}"
 }
@@ -228,7 +229,7 @@ push() {
 # This will allow images to be pushed immediately after they've been built.
 build_and_push() {
   image=$1
-  build "${image}"
+  build "${image}" "registry"
   push "${image}"
 }
 
@@ -256,9 +257,10 @@ if [[ "${WHAT}" == "all-conformance" ]]; then
   # no point in rebuilding all of them every time. This will only build the Conformance-related images.
   # Discussed during Conformance Office Hours Meeting (2019.12.17):
   # https://docs.google.com/document/d/1W31nXh9RYAb_VaYkwuPLd1hFxuRX3iU0DmaQ4lkCsX8/edit#heading=h.l87lu17xm9bh
-  conformance_images=("windows-image-builder-helper" "busybox" "agnhost" "echoserver" "jessie-dnsutils" "kitten" "nautilus" "nonewprivs" "resource-consumer" "sample-apiserver")
+  shift
+  conformance_images=("busybox" "agnhost" "echoserver" "jessie-dnsutils" "kitten" "nautilus" "nonewprivs" "resource-consumer" "sample-apiserver")
   for image in "${conformance_images[@]}"; do
-    eval "${TASK}" "${image}"
+    eval "${TASK}" "${image}" "$@"
   done
 else
   eval "${TASK}" "$@"
diff --git a/test/images/sample-apiserver/Dockerfile b/test/images/sample-apiserver/Dockerfile
index dbec210d46de9..2ab6420e79e8e 100644
--- a/test/images/sample-apiserver/Dockerfile
+++ b/test/images/sample-apiserver/Dockerfile
@@ -13,7 +13,7 @@
 # limitations under the License.
 
 ARG BASEIMAGE
-FROM k8s.gcr.io/build-image/kube-cross:v1.15.2-1 as build_k8s_1_17_sample_apiserver
+FROM --platform=linux/amd64 k8s.gcr.io/build-image/kube-cross:v1.15.2-1 as build_k8s_1_17_sample_apiserver
 
 ENV GOPATH /go
 RUN mkdir -p ${GOPATH}/src ${GOPATH}/bin
diff --git a/test/images/windows-image-builder-helper/BASEIMAGE b/test/images/windows-image-builder-helper/BASEIMAGE
deleted file mode 100644
index ce71d103b81bd..0000000000000
--- a/test/images/windows-image-builder-helper/BASEIMAGE
+++ /dev/null
@@ -1,3 +0,0 @@
-windows/amd64/1809=mcr.microsoft.com/windows/servercore:ltsc2019
-windows/amd64/1903=mcr.microsoft.com/windows/servercore:1903
-windows/amd64/1909=mcr.microsoft.com/windows/servercore:1909
diff --git a/test/images/windows-servercore-cache/BASEIMAGE b/test/images/windows-servercore-cache/BASEIMAGE
new file mode 100644
index 0000000000000..26f9f306f7e6f
--- /dev/null
+++ b/test/images/windows-servercore-cache/BASEIMAGE
@@ -0,0 +1,4 @@
+linux/amd64/1809=mcr.microsoft.com/windows/servercore:ltsc2019
+linux/amd64/1903=mcr.microsoft.com/windows/servercore:1903
+linux/amd64/1909=mcr.microsoft.com/windows/servercore:1909
+linux/amd64/2004=mcr.microsoft.com/windows/servercore:2004
diff --git a/test/images/windows-servercore-cache/Dockerfile b/test/images/windows-servercore-cache/Dockerfile
new file mode 100644
index 0000000000000..b92bffab20c0a
--- /dev/null
+++ b/test/images/windows-servercore-cache/Dockerfile
@@ -0,0 +1,22 @@
+# Copyright 2020 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG OS_VERSION
+FROM --platform=windows/amd64 mcr.microsoft.com/windows/servercore:$OS_VERSION as prep
+FROM scratch
+
+COPY --from=prep /Windows/System32/en-US/nltest.exe.mui /Windows/System32/en-US/nltest.exe.mui
+COPY --from=prep /Windows/System32/nltest.exe /Windows/System32/nltest.exe
+COPY --from=prep /Windows/System32/netapi32.dll /Windows/System32/netapi32.dll
+COPY --from=prep /Windows/System32/ntdsapi.dll /Windows/System32/ntdsapi.dll
diff --git a/test/images/windows-image-builder-helper/VERSION b/test/images/windows-servercore-cache/VERSION
similarity index 100%
rename from test/images/windows-image-builder-helper/VERSION
rename to test/images/windows-servercore-cache/VERSION
diff --git a/test/images/windows/Makefile b/test/images/windows/Makefile
new file mode 100644
index 0000000000000..368db16319762
--- /dev/null
+++ b/test/images/windows/Makefile
@@ -0,0 +1,38 @@
+# Copyright 2020 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+REGISTRY ?= gcr.io/kubernetes-e2e-test-images
+REMOTE_DOCKER_URL ?=
+DOCKER_CERT_PATH ?= "$(HOME)/.docker"
+export
+
+ALL_IMAGES = busybox-helper powershell-helper
+
+sub-build-%:
+	img_version=$(shell cat $*/VERSION); \
+	docker --tlsverify --tlscacert "$(DOCKER_CERT_PATH)/ca.pem" \
+		--tlscert "$(DOCKER_CERT_PATH)/cert.pem" --tlskey "$(DOCKER_CERT_PATH)/key.pem" \
+		-H "$(REMOTE_DOCKER_URL)" build --no-cache --pull -t "$(REGISTRY)/$*:$${img_version}" $*/
+
+sub-push-%:
+	img_version=`cat $*/VERSION`; \
+	docker --tlsverify --tlscacert "$(DOCKER_CERT_PATH)/ca.pem" \
+		--tlscert "$(DOCKER_CERT_PATH)/cert.pem" --tlskey "$(DOCKER_CERT_PATH)/key.pem" \
+		-H "$(REMOTE_DOCKER_URL)" push "$(REGISTRY)/$*:$${img_version}"
+
+all-build: $(foreach image, ${ALL_IMAGES}, sub-build-${image})
+
+all-push: all-build $(foreach image, ${ALL_IMAGES}, sub-push-${image})
+
+.PHONY: all-build all-push
diff --git a/test/images/windows-image-builder-helper/Dockerfile_windows b/test/images/windows/busybox-helper/Dockerfile
similarity index 63%
rename from test/images/windows-image-builder-helper/Dockerfile_windows
rename to test/images/windows/busybox-helper/Dockerfile
index c9cc2c1c0ceb2..00dfe3756ca89 100644
--- a/test/images/windows-image-builder-helper/Dockerfile_windows
+++ b/test/images/windows/busybox-helper/Dockerfile
@@ -12,23 +12,32 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-ARG BASEIMAGE
-from $BASEIMAGE as prep
+FROM mcr.microsoft.com/windows/servercore:ltsc2019 as prep
 
 ENV CURL_VERSION=7.57.0
-WORKDIR /curl
-ADD https://skanthak.homepage.t-online.de/download/curl-$CURL_VERSION.cab curl.cab
+
 ADD https://github.com/kubernetes-sigs/windows-testing/raw/master/images/busybox/busybox.exe /bin/busybox.exe
+ADD https://skanthak.homepage.t-online.de/download/curl-$CURL_VERSION.cab /curl/curl.cab
+ADD https://eternallybored.org/misc/netcat/netcat-win32-1.12.zip /netcat/netcat.zip
+ADD https://downloads.isc.org/isc/bind9/9.14.10/BIND9.14.10.x64.zip /bind.zip
 
 USER ContainerAdministrator
-RUN FOR /f "tokens=*" %i IN ('C:\bin\busybox --list') DO mklink C:\bin\%i.exe C:\bin\busybox.exe
-RUN expand /R curl.cab /F:* . &\
+
+# NOTE(claudiub): We have to create relative path symlinks because docker buildx has an issue when copying
+# over symlinks, it prepends "Files\" to the symlink target. "Files\C:\bin\busybox.exe" would be an
+# invalid path.
+RUN cd C:\bin && FOR /f "tokens=*" %i IN ('.\busybox --list') DO mklink .\%i.exe busybox.exe
+
+RUN cd C:\curl &\
+    expand /R curl.cab /F:* . &\
     del C:\curl\curl.cab &\
+    cd C:\netcat &\
+    tar.exe -xf netcat.zip &\
+    del netcat.zip &\
     setx /M PATH "C:\bin;C:\curl\;%PATH%"
 
-# download bind and prepare a folder just for dig.
+# Download bind and prepare a folder just for dig.
 RUN powershell -Command "\
-      curl.exe 'https://downloads.isc.org/isc/bind9/9.14.10/BIND9.14.10.x64.zip' -o /bind.zip; \
       Expand-Archive -Path C:\bind.zip -DestinationPath C:\bind; \
       $s = [System.Diagnostics.Process]::Start('C:\bind\vcredist_x64.exe', '/quiet'); \
       $s.WaitForExit(); \
@@ -39,13 +48,9 @@ RUN powershell -Command "\
       cp C:\Windows\System32\vcruntime140.dll C:\dig\; \
       rm C:\bind.zip;"
 
-FROM $BASEIMAGE
+FROM mcr.microsoft.com/windows/nanoserver:1809
 
-COPY --from=prep /curl/AMD64 /curl/CURL.LIC /curl/
 COPY --from=prep /bin /bin
+COPY --from=prep /curl/AMD64 /curl/CURL.LIC /curl/
 COPY --from=prep /dig /dig
-
-ENV chocolateyUseWindowsCompression false
-RUN powershell -Command "\
-    iex ((new-object net.webclient).DownloadString('https://chocolatey.org/install.ps1')); \
-    choco feature disable --name showDownloadProgress;
+COPY --from=prep /netcat /netcat
diff --git a/test/images/windows/busybox-helper/VERSION b/test/images/windows/busybox-helper/VERSION
new file mode 100644
index 0000000000000..5e57fb89558c5
--- /dev/null
+++ b/test/images/windows/busybox-helper/VERSION
@@ -0,0 +1 @@
+1.29.0
diff --git a/test/images/windows/powershell-helper/Dockerfile b/test/images/windows/powershell-helper/Dockerfile
new file mode 100644
index 0000000000000..f813e9a792164
--- /dev/null
+++ b/test/images/windows/powershell-helper/Dockerfile
@@ -0,0 +1,51 @@
+# Copyright 2020 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM mcr.microsoft.com/windows/servercore:ltsc2019 as prep
+
+ENV PS_VERSION=6.2.7
+ADD https://github.com/PowerShell/PowerShell/releases/download/v$PS_VERSION/PowerShell-$PS_VERSION-win-x64.zip /PowerShell/powershell.zip
+
+RUN cd C:\PowerShell &\
+    tar.exe -xf powershell.zip &\
+    del powershell.zip &\
+    mklink powershell.exe pwsh.exe
+
+FROM mcr.microsoft.com/windows/nanoserver:1809
+
+COPY --from=prep /PowerShell /PowerShell
+
+# set a fixed location for the Module analysis cache
+ENV LOCALAPPDATA="C:\Users\ContainerAdministrator\AppData\Local" \
+    PSModuleAnalysisCachePath="C:\Users\ContainerAdministrator\AppData\Local\Microsoft\Windows\PowerShell\docker\ModuleAnalysisCache" \
+    # Persist %PSCORE% ENV variable for user convenience
+    PSCORE="C:\PowerShell\pwsh.exe"
+
+# use downloaded powershell
+USER ContainerAdministrator
+RUN setx /M PATH "C:\Powershell\;%PATH%"
+
+# intialize powershell module cache
+RUN powershell \
+        -NoLogo \
+        -NoProfile \
+        -Command " \
+          $stopTime = (get-date).AddMinutes(15); \
+          $ErrorActionPreference = 'Stop' ; \
+          $ProgressPreference = 'SilentlyContinue' ; \
+          while(!(Test-Path -Path $env:PSModuleAnalysisCachePath)) {  \
+            Write-Host "'Waiting for $env:PSModuleAnalysisCachePath'" ; \
+            if((get-date) -gt $stopTime) { throw 'timout expired'} \
+            Start-Sleep -Seconds 6 ; \
+          }"
diff --git a/test/images/windows/powershell-helper/VERSION b/test/images/windows/powershell-helper/VERSION
new file mode 100644
index 0000000000000..d938f420bfb06
--- /dev/null
+++ b/test/images/windows/powershell-helper/VERSION
@@ -0,0 +1 @@
+6.2.7