Add make cluster/image command; Build image and cluster in parallel f…

…or CI

Signed-off-by: Connor Catlett <[email protected]>

Makefile

@@ -112,6 +112,10 @@ cluster/create: bin/kops bin/eksctl bin/aws
 cluster/kubeconfig:
 	@./hack/e2e/kubeconfig.sh
 
+.PHONY: cluster/image
+cluster/image: bin/aws
+	@./hack/e2e/build-image.sh
+
 .PHONY: cluster/delete
 cluster/delete: bin/kops bin/eksctl
 	./hack/e2e/delete-cluster.sh

docs/makefile.md

@@ -153,6 +153,10 @@ export CLUSTER_TYPE="eksctl"
 make cluster/create
 ```
 
+### `make cluster/image`
+
+Builds an image for use in the E2E tests. This will automatically build the most appropriate image (for example, skipping Windows builds unless `WINDOWS` is set to `true`).
+
 ### `make cluster/kubeconfig`
 
 Prints the `KUBECONFIG` environment variable for a cluster. You must pass the same `CLUSTER_TYPE` and `CLUSTER_NAME` as used when creating the cluster. This command must be `eval`ed to import the environment variables into your shell.
@@ -176,7 +180,7 @@ Deletes a cluster created by `make cluster/create`. You must pass the same `CLUS
 
 ## E2E Tests
 
-Run E2E tests against a cluster created by `make cluster/create`. You must pass the same `CLUSTER_TYPE` and `CLUSTER_NAME` as used when creating the cluster.
+Run E2E tests against a cluster created by `make cluster/create`. You must pass the same `CLUSTER_TYPE` and `CLUSTER_NAME` as used when creating the cluster. You must have already run `make cluster/image` to build the image for the cluster, or provide an image of your own.
 
 Alternatively, you may run on an externally created cluster by passing `CLUSTER_TYPE` (required to determine which `values.yaml` to deploy) and `KUBECONFIG`. For `kops` clusters, the node IAM role should include the appropriate IAM policies to use the driver (see [the installation docs](./install.md#set-up-driver-permissions)). For `eksctl` clusters, the `ebs-csi-controller-sa` service account should be pre-created and setup to supply an IRSA role with the appropriate policies.
 

hack/e2e/ecr.sh → hack/e2e/build-image.sh

@@ -14,9 +14,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# This script builds the EBS CSI Driver image for the e2e tests
+# Environment variables have default values (see config.sh) but
+# many can be overridden on demand if needed
+
 set -euo pipefail
 
-function ecr_build_and_push() {
+BASE_DIR="$(dirname "$(realpath "${BASH_SOURCE[0]}")")"
+BIN="${BASE_DIR}/../../bin"
+
+source "${BASE_DIR}/config.sh"
+source "${BASE_DIR}/util.sh"
+
+function build_and_push() {
   REGION=${1}
   AWS_ACCOUNT_ID=${2}
   IMAGE_NAME=${3}
@@ -25,7 +35,7 @@ function ecr_build_and_push() {
 
   # https://docs.aws.amazon.com/AmazonECR/latest/userguide/service-quotas.html
   MAX_IMAGES=10000
-  IMAGE_COUNT=$(aws ecr list-images --repository-name ${IMAGE_NAME} --region ${REGION} --query 'length(imageIds[])')
+  IMAGE_COUNT=$(aws ecr list-images --repository-name "${IMAGE_NAME##*/}" --region "${REGION}" --query 'length(imageIds[])')
 
   if [ $IMAGE_COUNT -ge $MAX_IMAGES ]; then
     loudecho "Repository image limit reached. Unable to push new images."
@@ -38,7 +48,9 @@ function ecr_build_and_push() {
   if [ -n "${PROW_JOB_ID:-}" ]; then
     trap "docker buildx rm ebs-csi-multiarch-builder" EXIT
     docker buildx create --driver-opt=image=moby/buildkit:v0.12.5 --bootstrap --use --name ebs-csi-multiarch-builder
-    docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+    # Ignore failures: Sometimes, this fails if run in parallel across multiple jobs
+    # If it fails "for real" the build later will fail, so it is safe to proceed
+    docker run --rm --privileged multiarch/qemu-user-static --reset -p yes || true
   fi
 
   export IMAGE="${IMAGE_NAME}"
@@ -54,3 +66,16 @@ function ecr_build_and_push() {
   fi
   make -j $(nproc) all-push
 }
+
+if [[ "${CREATE_MISSING_ECR_REPO}" == true ]]; then
+  REPO_CHECK=$(aws ecr describe-repositories --region "${AWS_REGION}")
+  if [ $(jq ".repositories | map(.repositoryName) | index(\"${IMAGE_NAME##*/}\")" <<<"${REPO_CHECK}") == "null" ]; then
+    aws ecr create-repository --region "${AWS_REGION}" --repository-name aws-ebs-csi-driver >/dev/null
+  fi
+fi
+
+build_and_push "${AWS_REGION}" \
+  "${AWS_ACCOUNT_ID}" \
+  "${IMAGE_NAME}" \
+  "${IMAGE_TAG}" \
+  "${IMAGE_ARCH}"

hack/e2e/run.sh

@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# This script builds and deploys the EBS CSI Driver and runs e2e tests
+# This script deploys the EBS CSI Driver and runs e2e tests
 # CLUSTER_NAME and CLUSTER_TYPE are expected to be specified by the caller
 # All other environment variables have default values (see config.sh) but
 # many can be overridden on demand if needed
@@ -26,7 +26,6 @@ BIN="${BASE_DIR}/../../bin"
 
 source "${BASE_DIR}/config.sh"
 source "${BASE_DIR}/util.sh"
-source "${BASE_DIR}/ecr.sh"
 source "${BASE_DIR}/metrics/metrics.sh"
 
 ## Setup
@@ -48,21 +47,6 @@ else
   NODE_OS_DISTRO="linux"
 fi
 
-## Build image
-
-if [[ "${CREATE_MISSING_ECR_REPO}" == true ]]; then
-  REPO_CHECK=$(aws ecr describe-repositories --region "${AWS_REGION}")
-  if [ $(jq '.repositories | map(.repositoryName) | index("aws-ebs-csi-driver")' <<<"${REPO_CHECK}") == "null" ]; then
-    aws ecr create-repository --region "${AWS_REGION}" --repository-name aws-ebs-csi-driver >/dev/null
-  fi
-fi
-
-ecr_build_and_push "${AWS_REGION}" \
-  "${AWS_ACCOUNT_ID}" \
-  "${IMAGE_NAME}" \
-  "${IMAGE_TAG}" \
-  "${IMAGE_ARCH}"
-
 ## Deploy
 
 if [[ "${EBS_INSTALL_SNAPSHOT}" == true ]]; then

hack/prow-e2e.sh

@@ -17,6 +17,10 @@
 # This script runs tests in CI by creating a cluster, running the tests,
 # cleaning up (regardless of test success/failure), and passing out the result
 
+# Prevent race conditions by frontloading tool download
+# TODO: Find a way to lock pip installs to prevent pip concurrency bugs from hurting us
+make bin/aws
+
 case ${1} in
 test-e2e-single-az)
   TEST="single-az"
@@ -67,11 +71,18 @@ export KOPS_BUCKET=${KOPS_BUCKET:-"k8s-kops-csi-shared-e2e"}
 # Always use us-west-2 in CI, no matter where the local client is
 export AWS_REGION=us-west-2
 
-if make cluster/create; then
+make cluster/create &
+PIDS[1]=$!
+make cluster/image &
+PIDS[2]=$!
+
+for PID in "${PIDS[@]}"; do
+  wait $PID || E2E_PASSED=1
+done
+
+if [[ $E2E_PASSED -eq 0 ]]; then
   make e2e/${TEST}
   E2E_PASSED=$?
-else
-  E2E_PASSED=1
 fi
 make cluster/delete