Commonize logic for creating a kind cluster

.gitignore

@@ -11,5 +11,10 @@ hack/release/release
 node/windows-packaging/CalicoWindows/confd/config-bgp.ps1
 node/windows-packaging/CalicoWindows/confd/config-bgp.psm1
 _output
-hack/release/ghr
 builder.coverprofile
+hack/release/ghr
+
+/* Created by local kind cluster */
+hack/test/kind/kind
+hack/test/kind/kubectl
+hack/test/kind/kubeconfig.yaml

libcalico-go/test/kind-config.yaml → hack/test/kind/control-plane-only.yaml

@@ -26,3 +26,5 @@ kubeadmConfigPatches:
   metadata:
     name: config
   mode: ipvs
+  conntrack:
+    maxPerCore: 0

node/tests/kind-config.yaml → hack/test/kind/multinode-kind.yaml

@@ -17,6 +17,9 @@ kubeadmConfigPatches:
     name: config
   featureGates:
     IPv6DualStack: true
+  controllerManager:
+    extraArgs:
+      cluster-cidr: "192.168.0.0/16"
 - |
   apiVersion: kubeproxy.config.k8s.io/v1alpha1
   kind: KubeProxyConfiguration

lib.Makefile

@@ -251,8 +251,8 @@ DOCKER_RUN := mkdir -p ../.go-pkg-cache bin $(GOMOD_CACHE) && \
 		-e OS=$(BUILDOS) \
 		-e GOOS=$(BUILDOS) \
 		-e GOFLAGS=$(GOFLAGS) \
-		-v $(CURDIR)/..:/go/src/github.com/projectcalico/calico:rw \
-		-v $(CURDIR)/../.go-pkg-cache:/go-cache:rw \
+		-v $(REPO_ROOT):/go/src/github.com/projectcalico/calico:rw \
+		-v $(REPO_ROOT)/.go-pkg-cache:/go-cache:rw \
 		-w /go/src/$(PACKAGE_NAME)
 
 DOCKER_RUN_RO := mkdir -p .go-pkg-cache bin $(GOMOD_CACHE) && \
@@ -267,8 +267,8 @@ DOCKER_RUN_RO := mkdir -p .go-pkg-cache bin $(GOMOD_CACHE) && \
 		-e OS=$(BUILDOS) \
 		-e GOOS=$(BUILDOS) \
 		-e GOFLAGS=$(GOFLAGS) \
-		-v $(CURDIR)/..:/go/src/github.com/projectcalico/calico:ro \
-		-v $(CURDIR)/../.go-pkg-cache:/go-cache:rw \
+		-v $(REPO_ROOT):/go/src/github.com/projectcalico/calico:ro \
+		-v $(REPO_ROOT)/.go-pkg-cache:/go-cache:rw \
 		-w /go/src/$(PACKAGE_NAME)
 
 DOCKER_GO_BUILD := $(DOCKER_RUN) $(CALICO_BUILD)
@@ -1117,7 +1117,6 @@ run-k8s-apiserver: stop-k8s-apiserver run-etcd
 		--name $(APISERVER_NAME) \
 		-v $(REPO_ROOT):/go/src/github.com/projectcalico/calico \
 		-v $(CERTS_PATH):/home/user/certs \
-		-v $(CURDIR)/config:/config \
 		-e KUBECONFIG=/home/user/certs/kubeconfig \
 		$(CALICO_BUILD) kube-apiserver \
 		--etcd-servers=http://$(LOCAL_IP_ENV):2379 \
@@ -1177,6 +1176,50 @@ run-k8s-controller-manager: stop-k8s-controller-manager run-k8s-apiserver
 stop-k8s-controller-manager:
 	@-docker rm -f $(CONTROLLER_MANAGER_NAME)
 
+###############################################################################
+# Common functions for create a local kind cluster.
+###############################################################################
+KIND_DIR := $(REPO_ROOT)/hack/test/kind
+KIND_KUBECONFIG?=$(KIND_DIR)/kubeconfig.yaml
+KIND ?= $(KIND_DIR)/kind
+KUBECTL ?= $(KIND_DIR)/kubectl
+
+# Different tests may require different kind configurations.
+KIND_CONFIG ?= $(KIND_DIR)/multinode-kind.yaml
+KIND_NAME = $(basename $(notdir $(KIND_CONFIG)))
+
+kind-cluster-create: $(REPO_ROOT)/.$(KIND_NAME).created
+$(REPO_ROOT)/.$(KIND_NAME).created: $(KUBECTL) $(KIND)
+	# First make sure any previous cluster is deleted
+	$(MAKE) kind-cluster-destroy
+
+	# Create a kind cluster.
+	$(KIND) create cluster \
+		--config $(KIND_CONFIG) \
+		--kubeconfig $(KIND_KUBECONFIG) \
+		--name $(KIND_NAME) \
+		--image kindest/node:$(K8S_VERSION)
+
+	# Wait for controller manager to be running and healthy, then create Calico CRDs.
+	while ! KUBECONFIG=$(KIND_KUBECONFIG) $(KUBECTL) get serviceaccount default; do echo "Waiting for default serviceaccount to be created..."; sleep 2; done
+	while ! KUBECONFIG=$(KIND_KUBECONFIG) $(KUBECTL) create -f $(REPO_ROOT)/libcalico-go/config/crd; do echo "Waiting for CRDs to be created"; sleep 2; done
+	touch $@
+
+kind-cluster-destroy: $(KIND) $(KUBECTL)
+	-$(KUBECTL) --kubeconfig=$(KIND_KUBECONFIG) drain kind-control-plane kind-worker kind-worker2 kind-worker3 --ignore-daemonsets --force
+	-$(KIND) delete cluster --name $(KIND_NAME)
+	rm -f $(KIND_KUBECONFIG)
+	rm -f $(REPO_ROOT)/.$(KIND_NAME).created
+
+kind $(KIND):
+	mkdir -p $(KIND_DIR)
+	$(DOCKER_GO_BUILD) sh -c "GOBIN=/go/src/github.com/projectcalico/calico/hack/test/kind go install sigs.k8s.io/[email protected]"
+
+kubectl $(KUBECTL):
+	mkdir -p $(KIND_DIR)
+	curl -L https://storage.googleapis.com/kubernetes-release/release/$(K8S_VERSION)/bin/linux/amd64/kubectl -o $@
+	chmod +x $@
+
 ###############################################################################
 # Common functions for launching a local etcd instance.
 ###############################################################################

libcalico-go/Makefile

@@ -4,6 +4,8 @@ PACKAGE_NAME = github.com/projectcalico/calico/libcalico-go
 
 LOCAL_CHECKS = goimports check-gen-files
 
+KIND_CONFIG = $(KIND_DIR)/control-plane-only.yaml
+
 ###############################################################################
 # Download and include ../lib.Makefile
 #   Additions to EXTRA_DOCKER_ARGS need to happen before the include since
@@ -137,7 +139,7 @@ fv: fv-setup
 fv-setup: run-etcd run-etcd-tls cluster-create run-coredns
 
 ## Teardown the FVs resources.
-fv-teardown: stop-etcd stop-etcd-tls cluster-destroy stop-coredns
+fv-teardown: stop-etcd stop-etcd-tls kind-cluster-destroy stop-coredns
 
 ## Run the FVs without any setup or teardown. Useful when writing FVs.
 .PHONY:fv-fast
@@ -148,6 +150,13 @@ fv-fast:
 		--dns $(shell docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' coredns) \
 		$(CALICO_BUILD) sh -c 'cd /go/src/$(PACKAGE_NAME) && ginkgo -r -focus "$(GINKGO_FOCUS).*\[Datastore\]|\[Datastore\].*$(GINKGO_FOCUS)" $(WHAT)'
 
+# Create a kind cluster, and deploy some resources required for the libcalico-go tests.
+# TODO: Remove the need for this extra target, should all be in lib.Makefile.
+cluster-create: kind-cluster-create
+	@export KUBECONFIG=$(KIND_KUBECONFIG) && \
+		$(KUBECTL) create -f test/mock-node.yaml && \
+		$(KUBECTL) create -f test/namespaces.yaml
+
 ## Run an etcd with TLS enabled, needed for some libcalico-go UTs.
 run-etcd-tls: stop-etcd-tls
 	# TODO: We shouldn't need to enable the v2 API, but some of our test code 
@@ -173,45 +182,6 @@ run-etcd-tls: stop-etcd-tls
 stop-etcd-tls:
 	-docker rm -f calico-etcd-tls
 
-## Create a local kind dual stack cluster.
-KUBECONFIG?=kubeconfig.yaml
-cluster-create: $(BINDIR)/kubectl $(BINDIR)/kind
-	# First make sure any previous cluster is deleted
-	make cluster-destroy
-
-	# Create a kind cluster.
-	$(BINDIR)/kind create cluster \
-		--config ./test/kind-config.yaml \
-		--kubeconfig $(KUBECONFIG) \
-		--image kindest/node:$(K8S_VERSION)
-
-	# Deploy resources needed in test env.
-	$(MAKE) deploy-test-resources
-
-	# Wait for controller manager to be running and healthy.
-	while ! KUBECONFIG=$(KUBECONFIG) $(BINDIR)/kubectl get serviceaccount default; do echo "Waiting for default serviceaccount to be created..."; sleep 2; done
-
-
-## Deploy resources needed for UTs.
-deploy-test-resources: $(BINDIR)/kubectl
-	@export KUBECONFIG=$(KUBECONFIG) && \
-		./$(BINDIR)/kubectl create -f config/crd && \
-		./$(BINDIR)/kubectl create -f test/mock-node.yaml && \
-		./$(BINDIR)/kubectl create -f test/namespaces.yaml
-
-## Destroy local kind cluster
-cluster-destroy: $(BINDIR)/kind
-	-$(BINDIR)/kind delete cluster
-	rm -f $(KUBECONFIG)
-
-$(BINDIR)/kind:
-	$(DOCKER_GO_BUILD) sh -c "GOBIN=/go/src/$(PACKAGE_NAME)/$(BINDIR) go install sigs.k8s.io/kind"
-
-$(BINDIR)/kubectl:
-	mkdir -p $(BINDIR)
-	curl -L https://storage.googleapis.com/kubernetes-release/release/$(KUBECTL_VERSION)/bin/linux/amd64/kubectl -o $@
-	chmod +x $(BINDIR)/kubectl
-
 run-coredns: stop-coredns
 	docker run \
 		--detach \

node/Makefile

@@ -292,43 +292,6 @@ fv: run-k8s-apiserver
 	-e ETCD_ENDPOINTS=http://$(LOCAL_IP_ENV):2379 \
 	$(CALICO_BUILD) ginkgo -cover -r -skipPackage vendor pkg/lifecycle/startup pkg/allocateip $(GINKGO_ARGS)
 
-## Create a local kind dual stack cluster.
-KUBECONFIG?=kubeconfig.yaml
-cluster-create: $(BINDIR)/kubectl $(BINDIR)/kind
-	# First make sure any previous cluster is deleted
-	make cluster-destroy
-
-	# Create a kind cluster.
-	$(BINDIR)/kind create cluster \
-	        --config ./tests/kind-config.yaml \
-	        --kubeconfig $(KUBECONFIG) \
-	        --image kindest/node:$(K8S_VERSION)
-
-	# Deploy resources needed in test env.
-	$(MAKE) deploy-test-resources
-
-	# Wait for controller manager to be running and healthy.
-	while ! KUBECONFIG=$(KUBECONFIG) $(BINDIR)/kubectl get serviceaccount default; do echo "Waiting for default serviceaccount to be created..."; sleep 2; done
-
-## Deploy resources on the kind cluster that are needed for tests
-deploy-test-resources: $(BINDIR)/kubectl $(K8ST_IMAGE_TARS)
-	KUBECONFIG=$(KUBECONFIG) ./tests/k8st/deploy_resources_on_kind_cluster.sh
-
-## Destroy local kind cluster
-cluster-destroy: $(BINDIR)/kubectl $(BINDIR)/kind
-	-$(BINDIR)/kubectl --kubeconfig=$(KUBECONFIG) drain kind-control-plane kind-worker kind-worker2 kind-worker3 --ignore-daemonsets --force
-	-$(BINDIR)/kind delete cluster
-	rm -f ./tests/k8st/infra/calico.yaml.tmp
-	rm -f $(KUBECONFIG)
-
-$(BINDIR)/kind:
-	$(DOCKER_GO_BUILD) sh -c "GOBIN=/go/src/$(PACKAGE_NAME)/$(BINDIR) go install sigs.k8s.io/kind"
-
-$(BINDIR)/kubectl:
-	mkdir -p $(BINDIR)
-	curl -L https://storage.googleapis.com/kubernetes-release/release/$(KUBECTL_VERSION)/bin/linux/$(ARCH)/kubectl -o $@
-	chmod +x $(BINDIR)/kubectl
-
 # Skip packages containing FV tests.
 UT_PACKAGES_TO_SKIP?=pkg/lifecycle/startup,pkg/allocateip,pkg/status
 .PHONY: ut
@@ -389,14 +352,14 @@ kube-controllers.tar: ../go.mod $(shell find ../kube-controllers -name '*.go')
 	make -C ../kube-controllers image
 	docker save --output $@ calico/kube-controllers:latest-$(ARCH)
 
-load-container-images: $(K8ST_IMAGE_TARS) $(BINDIR)/kubectl
+load-container-images: $(K8ST_IMAGE_TARS) $(KUBECTL)
 	# Load the latest tar files onto the currently running kind cluster.
-	KUBECONFIG=$(KUBECONFIG) ./tests/k8st/load_images_on_kind_cluster.sh
+	KUBECONFIG=$(KIND_KUBECONFIG) ./tests/k8st/load_images_on_kind_cluster.sh
 	# Restart the Calico containers so they launch with the newly loaded code.
 	# TODO: We should be able to do this without restarting everything in kube-system.
-	KUBECONFIG=$(KUBECONFIG) $(BINDIR)/kubectl delete pods -n kube-system --all 
+	KUBECONFIG=$(KIND_KUBECONFIG) $(KUBECTL) delete pods -n kube-system --all 
 	# calicoctl is deployed as a pod on the cluster and needs to be recreated.
-	KUBECONFIG=$(KUBECONFIG) $(BINDIR)/kubectl apply -f tests/k8st/infra/calicoctl.yaml
+	KUBECONFIG=$(KIND_KUBECONFIG) $(KUBECTL) apply -f tests/k8st/infra/calicoctl.yaml
 
 .PHONY: st-checks
 st-checks:
@@ -415,15 +378,16 @@ k8s-test:
 	$(MAKE) kind-k8st-cleanup
 
 .PHONY: kind-k8st-setup
-kind-k8st-setup: $(K8ST_IMAGE_TARS) cluster-create
+kind-k8st-setup: $(K8ST_IMAGE_TARS) kind-cluster-create
+	KUBECONFIG=$(KIND_KUBECONFIG) ./tests/k8st/deploy_resources_on_kind_cluster.sh
 
 .PHONY: kind-k8st-run-test
-kind-k8st-run-test: calico_test.created $(KUBECONFIG)
+kind-k8st-run-test: calico_test.created $(KIND_KUBECONFIG)
 	docker run -t --rm \
 	    -v $(CURDIR):/code \
 	    -v /var/run/docker.sock:/var/run/docker.sock \
-	    -v $(CURDIR)/$(KUBECONFIG):/root/.kube/config \
-	    -v $(CURDIR)/$(BINDIR)/kubectl:/bin/kubectl \
+	    -v $(KIND_KUBECONFIG):/root/.kube/config \
+	    -v $(KUBECTL):/bin/kubectl \
 	    -e ROUTER_IMAGE=$(BIRD_IMAGE) \
 	    --privileged \
 	    --net host \
@@ -432,7 +396,7 @@ kind-k8st-run-test: calico_test.created $(KUBECONFIG)
 	     cd /code/tests/k8st && nosetests $(K8ST_TO_RUN) -v --with-xunit --xunit-file="/code/report/k8s-tests.xml" --with-timer'
 
 .PHONY: kind-k8st-cleanup
-kind-k8st-cleanup: cluster-destroy
+kind-k8st-cleanup: kind-cluster-destroy
 
 # Needed for Semaphore CI (where disk space is a real issue during k8s-test)
 .PHONY: remove-go-build-image