Merge pull request #1915 from wfernandes/e2e-delete-resources

🏃E2e delete resources

go.mod

@@ -12,8 +12,8 @@ require (
 	github.com/google/go-cmp v0.3.1 // indirect
 	github.com/hashicorp/golang-lru v0.5.3 // indirect
 	github.com/imdario/mergo v0.3.8 // indirect
-	github.com/onsi/ginkgo v1.10.1
-	github.com/onsi/gomega v1.7.0
+	github.com/onsi/ginkgo v1.10.3
+	github.com/onsi/gomega v1.7.1
 	github.com/pkg/errors v0.8.1
 	github.com/prometheus/client_golang v1.0.0
 	github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90

go.sum

@@ -201,11 +201,15 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W
 github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo=
 github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.10.3 h1:OoxbjfXVZyod1fmWYhI7SEyaD8B00ynP3T+D5GiyHOY=
+github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.3.0/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.7.1 h1:K0jcRCwNQM3vFGh1ppMtDh/+7ApJrjldlX8fA0jDTLQ=
+github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
 github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=

test/e2e/go.mod

@@ -1,14 +1,14 @@
 module sigs.k8s.io/cluster-api/test/e2e
 
-go 1.12
+go 1.13
 
 require (
 	github.com/onsi/ginkgo v1.10.3
 	github.com/onsi/gomega v1.7.1
 	k8s.io/api v0.0.0-20191121015604-11707872ac1c
 	k8s.io/apimachinery v0.0.0-20191121015412-41065c7a8c2a
 	k8s.io/client-go v11.0.0+incompatible
-	sigs.k8s.io/cluster-api v0.2.6-0.20191216153338-db9baf9ade0c
+	sigs.k8s.io/cluster-api v0.2.6-0.20191223162332-fd807a3d843b
 	sigs.k8s.io/cluster-api/test/framework v0.0.0-00010101000000-000000000000
 	sigs.k8s.io/controller-runtime v0.4.0
 )

test/framework/multi_node_control_plane.go → test/framework/control_plane.go

@@ -23,42 +23,51 @@ import (
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
 	v1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/wait"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3"
+	cabpkv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1alpha3"
+	controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1alpha3"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
-// MultiNodeControlplaneClusterInput defines the necessary dependencies to run a multi-node control plane cluster.
-type MultiNodeControlplaneClusterInput struct {
-	Management        ManagementCluster
-	Cluster           *clusterv1.Cluster
-	InfraCluster      runtime.Object
-	ControlplaneNodes []Node
-	CreateTimeout     time.Duration
+// ControlplaneClusterInput defines the necessary dependencies to run a multi-node control plane cluster.
+type ControlplaneClusterInput struct {
+	Management    ManagementCluster
+	Cluster       *clusterv1.Cluster
+	InfraCluster  runtime.Object
+	Nodes         []Node
+	CreateTimeout time.Duration
+	DeleteTimeout time.Duration
 }
 
 // SetDefaults defaults the struct fields if necessary.
-func (m *MultiNodeControlplaneClusterInput) SetDefaults() {
+func (m *ControlplaneClusterInput) SetDefaults() {
 	if m.CreateTimeout == 0 {
 		m.CreateTimeout = 10 * time.Minute
 	}
+
+	if m.DeleteTimeout == 0 {
+		m.DeleteTimeout = 5 * time.Minute
+	}
 }
 
-// MulitNodeControlPlaneCluster creates an n node control plane cluster.
+// ControlPlaneCluster creates an n node control plane cluster.
 // Assertions:
 //  * The number of nodes in the created cluster will equal the number of nodes in the input data.
-func MultiNodeControlPlaneCluster(input *MultiNodeControlplaneClusterInput) {
+func (input *ControlplaneClusterInput) ControlPlaneCluster() {
 	ctx := context.Background()
 	Expect(input.Management).ToNot(BeNil())
 
 	mgmtClient, err := input.Management.GetClient()
 	Expect(err).NotTo(HaveOccurred(), "stack: %+v", err)
 
 	By("creating an InfrastructureCluster resource")
-	Expect(mgmtClient.Create(ctx, input.InfraCluster)).NotTo(HaveOccurred())
+	Expect(mgmtClient.Create(ctx, input.InfraCluster)).To(Succeed())
 
 	By("creating a Cluster resource linked to the InfrastructureCluster resource")
 	Eventually(func() error {
@@ -70,15 +79,15 @@ func MultiNodeControlPlaneCluster(input *MultiNodeControlplaneClusterInput) {
 	}, input.CreateTimeout, 10*time.Second).Should(BeNil())
 
 	// create all the machines at once
-	for _, node := range input.ControlplaneNodes {
+	for _, node := range input.Nodes {
 		By("creating an InfrastructureMachine resource")
-		Expect(mgmtClient.Create(ctx, node.InfraMachine)).NotTo(HaveOccurred())
+		Expect(mgmtClient.Create(ctx, node.InfraMachine)).To(Succeed())
 
 		By("creating a bootstrap config")
-		Expect(mgmtClient.Create(ctx, node.BootstrapConfig)).NotTo(HaveOccurred())
+		Expect(mgmtClient.Create(ctx, node.BootstrapConfig)).To(Succeed())
 
 		By("creating a core Machine resource with a linked InfrastructureMachine and BootstrapConfig")
-		Expect(mgmtClient.Create(ctx, node.Machine)).NotTo(HaveOccurred())
+		Expect(mgmtClient.Create(ctx, node.Machine)).To(Succeed())
 	}
 
 	// Wait for the cluster infrastructure
@@ -96,7 +105,7 @@ func MultiNodeControlPlaneCluster(input *MultiNodeControlplaneClusterInput) {
 
 	// wait for all the machines to be running
 	By("waiting for all machines to be running")
-	for _, node := range input.ControlplaneNodes {
+	for _, node := range input.Nodes {
 		Eventually(func() string {
 			machine := &clusterv1.Machine{}
 			key := client.ObjectKey{
@@ -129,7 +138,64 @@ func MultiNodeControlPlaneCluster(input *MultiNodeControlplaneClusterInput) {
 		By("getting the workload client and listing the nodes")
 		workloadClient, err := input.Management.GetWorkloadClient(ctx, input.Cluster.Namespace, input.Cluster.Name)
 		Expect(err).NotTo(HaveOccurred(), "Stack:\n%+v\n", err)
-		Expect(workloadClient.List(ctx, &nodes)).NotTo(HaveOccurred())
+		Expect(workloadClient.List(ctx, &nodes)).To(Succeed())
 		return nodes.Items
-	}, input.CreateTimeout, 10*time.Second).Should(HaveLen(len(input.ControlplaneNodes)))
+	}, input.CreateTimeout, 10*time.Second).Should(HaveLen(len(input.Nodes)))
+}
+
+// CleanUp deletes the cluster and waits for everything to be gone.
+// Assertions:
+//   * Deletes Machines
+//   * Deletes MachineSets
+//   * Deletes MachineDeployments
+//   * Deletes KubeadmConfigs
+//   * Deletes Secrets
+func (input *ControlplaneClusterInput) CleanUpCoreArtifacts() {
+	input.SetDefaults()
+	ctx := context.Background()
+	mgmtClient, err := input.Management.GetClient()
+	Expect(err).NotTo(HaveOccurred(), "stack: %+v", err)
+
+	By(fmt.Sprintf("deleting cluster %s", input.Cluster.GetName()))
+	Expect(mgmtClient.Delete(ctx, input.Cluster)).To(Succeed())
+
+	Eventually(func() []clusterv1.Cluster {
+		clusters := clusterv1.ClusterList{}
+		Expect(mgmtClient.List(ctx, &clusters)).To(Succeed())
+		return clusters.Items
+	}, input.DeleteTimeout, 10*time.Second).Should(HaveLen(0))
+
+	lbl, err := labels.Parse(fmt.Sprintf("%s=%s", clusterv1.ClusterLabelName, input.Cluster.GetClusterName()))
+	Expect(err).ToNot(HaveOccurred())
+	listOpts := &client.ListOptions{LabelSelector: lbl}
+
+	By("ensuring all CAPI artifacts have been deleted")
+	ensureArtifactsDeleted(ctx, mgmtClient, listOpts)
+}
+
+func ensureArtifactsDeleted(ctx context.Context, mgmtClient client.Client, opt *client.ListOptions) {
+	// assertions
+	ml := &clusterv1.MachineList{}
+	Expect(mgmtClient.List(ctx, ml, opt)).To(Succeed())
+	Expect(ml.Items).To(HaveLen(0))
+
+	msl := &clusterv1.MachineSetList{}
+	Expect(mgmtClient.List(ctx, msl, opt)).To(Succeed())
+	Expect(msl.Items).To(HaveLen(0))
+
+	mdl := &clusterv1.MachineDeploymentList{}
+	Expect(mgmtClient.List(ctx, mdl, opt)).To(Succeed())
+	Expect(mdl.Items).To(HaveLen(0))
+
+	kcpl := &controlplanev1.KubeadmControlPlaneList{}
+	Expect(mgmtClient.List(ctx, kcpl, opt)).To(Succeed())
+	Expect(kcpl.Items).To(HaveLen(0))
+
+	kcl := &cabpkv1.KubeadmConfigList{}
+	Expect(mgmtClient.List(ctx, kcl, opt)).To(Succeed())
+	Expect(kcl.Items).To(HaveLen(0))
+
+	sl := &corev1.SecretList{}
+	Expect(mgmtClient.List(ctx, sl, opt)).To(Succeed())
+	Expect(sl.Items).To(HaveLen(0))
 }

test/framework/convenience.go

@@ -35,7 +35,7 @@ func InstallComponents(ctx context.Context, mgmt Applier, components ...Componen
 			By(fmt.Sprintf("installing %s", component.GetName()))
 			c, err := component.Manifests(ctx)
 			Expect(err).NotTo(HaveOccurred())
-			Expect(mgmt.Apply(ctx, c)).NotTo(HaveOccurred())
+			Expect(mgmt.Apply(ctx, c)).To(Succeed())
 		}
 	})
 }

test/framework/go.mod

@@ -7,8 +7,11 @@ require (
 	github.com/onsi/gomega v1.7.1
 	github.com/pkg/errors v0.8.1
 	k8s.io/api v0.0.0-20191121015604-11707872ac1c
+	k8s.io/apiextensions-apiserver v0.0.0-20190918201827-3de75813f604 // indirect
 	k8s.io/apimachinery v0.0.0-20191121015412-41065c7a8c2a
-	k8s.io/client-go v0.0.0-20190918160344-1fbdaa4c8d90
-	sigs.k8s.io/cluster-api v0.2.6-0.20191216153338-db9baf9ade0c
+	k8s.io/client-go v11.0.0+incompatible
+	sigs.k8s.io/cluster-api v0.2.6-0.20191223162332-fd807a3d843b
 	sigs.k8s.io/controller-runtime v0.4.0
 )
+
+replace k8s.io/client-go => k8s.io/client-go v0.0.0-20190918160344-1fbdaa4c8d90