Merge pull request #538 from yussufsh/ittest

feature: ability to run remote IT on PowerVS

Makefile

@@ -126,3 +126,6 @@ init-buildx:
 	# Register gcloud as a Docker credential helper.
 	# Required for "docker buildx build --push".
 	gcloud auth configure-docker --quiet
+
+test-integration:
+	go test -v -timeout 100m sigs.k8s.io/ibm-powervs-block-csi-driver/tests/it -run ^TestIntegration$

tests/it/README.md

@@ -0,0 +1,34 @@
+# Integration Testing
+
+## Introduction
+
+The integration test will create a PowerVS instance and run the CSI driver tests on it. The driver will be built locally for ppc64le and copied over to the pvm instance. The driver will run on the instance and a SSH tunnel will be created. A grpc client will connect to the driver via a tcp port and run the Node and Controller tests.
+
+You can also run the test on an existing PowerVS instance itself.
+
+## Environments
+The following environment variables will be required for running the integration test.
+```
+export IBMCLOUD_API_KEY=<api key>               # IBM Cloud API key
+export POWERVS_ZONE="mon01",                  # IBM Cloud zone
+
+```
+Below environment variable is needed when running the tests within a PowerVS machine.
+```
+export POWERVS_INSTANCE_ID="<instance_id>"    # The pvm instance id where the test is running
+```
+Below environment variables are used if you want to use run tests from a remote machine.
+```
+export TEST_REMOTE_NODE="1"                   # Set to 1 if you want to run the remote node tests
+export IBMCLOUD_ACCOUNT_ID="<account_id>"     # IBM Cloud account to use for creating the remote node
+export POWERVS_CLOUD_INSTANCE_ID="<pid>"      # Workspace guid to use for creating the remote node
+export POWERVS_NETWORK="pub-network"          # (Optional) The network to use for creating the remote node
+export POWERVS_IMAGE="CentOS-Stream-8"        # (Optional) The image to use for creating the remote node
+```
+
+
+## Run the test
+To run the test use the following command.
+```
+make test-integration
+```

tests/it/integration_test.go

@@ -0,0 +1,181 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package it
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/container-storage-interface/spec/lib/go/csi"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"k8s.io/klog/v2"
+)
+
+var (
+	stdVolCap = []*csi.VolumeCapability{
+		{
+			AccessType: &csi.VolumeCapability_Mount{
+				Mount: &csi.VolumeCapability_MountVolume{},
+			},
+			AccessMode: &csi.VolumeCapability_AccessMode{
+				Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER,
+			},
+		},
+	}
+	stdCapRange    = &csi.CapacityRange{RequiredBytes: int64(1 * 1024 * 1024 * 1024)}
+	testVolumeName = fmt.Sprintf("ibm-powervs-csi-driver-it-%d", rand.New(rand.NewSource(time.Now().UnixNano())).Uint64())
+)
+
+func TestIntegration(t *testing.T) {
+	flag.Parse()
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "IBM PowerVS CSI Driver Integration Tests")
+}
+
+var _ = Describe("IBM PowerVS CSI Driver", func() {
+
+	It("Should create, attach, stage and mount volume, check if it's writable, unmount, unstage, detach, delete, and check if it's deleted", func() {
+
+		testCreateAttachWriteReadDetachDelete()
+
+	})
+})
+
+func testCreateAttachWriteReadDetachDelete() {
+	klog.Infof("Creating volume with name %s", testVolumeName)
+	start := time.Now()
+	resp, err := csiClient.ctrl.CreateVolume(context.Background(), &csi.CreateVolumeRequest{
+		Name:               testVolumeName,
+		CapacityRange:      stdCapRange,
+		VolumeCapabilities: stdVolCap,
+		Parameters:         nil,
+	})
+	Expect(err).To(BeNil(), "error during create volume")
+	volume := resp.GetVolume()
+	Expect(volume).NotTo(BeNil(), "volume is nil")
+	klog.Infof("Created volume %s in %v", volume, time.Since(start))
+
+	defer func() {
+		klog.Infof("Deleting volume %s", volume.VolumeId)
+		start := time.Now()
+		_, err = csiClient.ctrl.DeleteVolume(context.Background(), &csi.DeleteVolumeRequest{VolumeId: volume.VolumeId})
+		Expect(err).To(BeNil(), "error during delete volume")
+		klog.Infof("Deleted volume %s in %v", volume.VolumeId, time.Since(start))
+	}()
+
+	klog.Info("Running ValidateVolumeCapabilities")
+	vcResp, err := csiClient.ctrl.ValidateVolumeCapabilities(context.Background(), &csi.ValidateVolumeCapabilitiesRequest{
+		VolumeId:           volume.VolumeId,
+		VolumeCapabilities: stdVolCap,
+	})
+	Expect(err).To(BeNil())
+	klog.Infof("Ran ValidateVolumeCapabilities with response %v", vcResp)
+
+	klog.Info("Running NodeGetInfo")
+	niResp, err := csiClient.node.NodeGetInfo(context.Background(), &csi.NodeGetInfoRequest{})
+	Expect(err).To(BeNil())
+	klog.Infof("Ran NodeGetInfo with response %v", niResp)
+
+	testAttachStagePublishDetach(volume.VolumeId)
+}
+
+func testAttachStagePublishDetach(volumeID string) {
+	klog.Infof("Attaching volume %s to node %s", volumeID, nodeID)
+	start := time.Now()
+	respAttach, err := csiClient.ctrl.ControllerPublishVolume(context.Background(), &csi.ControllerPublishVolumeRequest{
+		VolumeId:         volumeID,
+		NodeId:           nodeID,
+		VolumeCapability: stdVolCap[0],
+	})
+	Expect(err).To(BeNil(), "failed attaching volume %s to node %s", volumeID, nodeID)
+	// assertAttachmentState(volumeID, "attached")
+	klog.Infof("Attached volume with response %v in %v", respAttach.PublishContext, time.Since(start))
+
+	defer func() {
+		klog.Infof("Detaching volume %s from node %s", volumeID, nodeID)
+		start := time.Now()
+		_, err = csiClient.ctrl.ControllerUnpublishVolume(context.Background(), &csi.ControllerUnpublishVolumeRequest{
+			VolumeId: volumeID,
+			NodeId:   nodeID,
+		})
+		Expect(err).To(BeNil(), "failed detaching volume %s from node %s", volumeID, nodeID)
+		// assertAttachmentState(volumeID, "detached")
+		klog.Infof("Detached volume %s from node %s in %v", volumeID, nodeID, time.Since(start))
+	}()
+
+	if os.Getenv("TEST_REMOTE_NODE") == "1" {
+		testStagePublish(volumeID, respAttach.PublishContext["wwn"])
+	}
+}
+
+func testStagePublish(volumeID, wwn string) {
+	// Node Stage
+	volDir := filepath.Join("/tmp/", testVolumeName)
+	stageDir := filepath.Join(volDir, "stage")
+	klog.Infof("Staging volume %s to path %s", volumeID, stageDir)
+	start := time.Now()
+	_, err := csiClient.node.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{
+		VolumeId:          volumeID,
+		StagingTargetPath: stageDir,
+		VolumeCapability:  stdVolCap[0],
+		PublishContext:    map[string]string{"wwn": wwn},
+	})
+	Expect(err).To(BeNil(), "failed staging volume %s to path %s", volumeID, stageDir)
+	klog.Infof("Staged volume %s to path %s in %v", volumeID, stageDir, time.Since(start))
+
+	defer func() {
+		klog.Infof("Unstaging volume %s from path %s", volumeID, stageDir)
+		start := time.Now()
+		_, err = csiClient.node.NodeUnstageVolume(context.Background(), &csi.NodeUnstageVolumeRequest{VolumeId: volumeID, StagingTargetPath: stageDir})
+		Expect(err).To(BeNil(), "failed unstaging volume %s from path %s", volumeID, stageDir)
+		err = os.RemoveAll(volDir)
+		Expect(err).To(BeNil(), "failed to remove volume directory")
+		klog.Infof("Unstaged volume %s from path %s in %v", volumeID, stageDir, time.Since(start))
+	}()
+
+	// Node Publish
+	publishDir := filepath.Join("/tmp/", testVolumeName, "mount")
+	klog.Infof("Publishing volume %s to path %s", volumeID, publishDir)
+	start = time.Now()
+	_, err = csiClient.node.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{
+		VolumeId:          volumeID,
+		StagingTargetPath: stageDir,
+		TargetPath:        publishDir,
+		VolumeCapability:  stdVolCap[0],
+		PublishContext:    map[string]string{"wwn": wwn},
+	})
+	Expect(err).To(BeNil(), "failed publishing volume %s to path %s", volumeID, publishDir)
+	klog.Infof("Published volume %s to path %s in %v", volumeID, publishDir, time.Since(start))
+
+	defer func() {
+		klog.Infof("Unpublishing volume %s from path %s", volumeID, publishDir)
+		start := time.Now()
+		_, err = csiClient.node.NodeUnpublishVolume(context.Background(), &csi.NodeUnpublishVolumeRequest{
+			VolumeId:   volumeID,
+			TargetPath: publishDir,
+		})
+		Expect(err).To(BeNil(), "failed unpublishing volume %s from path %s", volumeID, publishDir)
+		klog.Infof("Unpublished volume %s from path %s in %v", volumeID, publishDir, time.Since(start))
+	}()
+}

tests/it/utils.go

@@ -0,0 +1,143 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package it
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"os"
+	"time"
+
+	"github.com/container-storage-interface/spec/lib/go/csi"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/ibm-powervs-block-csi-driver/pkg/driver"
+	"sigs.k8s.io/ibm-powervs-block-csi-driver/pkg/util"
+	"sigs.k8s.io/ibm-powervs-block-csi-driver/tests/remote"
+)
+
+const (
+	port = 10000
+)
+
+var (
+	drv       *driver.Driver
+	csiClient *CSIClient
+	nodeID    string
+	r         *remote.Remote
+	endpoint  = fmt.Sprintf("tcp://localhost:%d", port)
+)
+
+// Before runs the driver and creates a CSI client
+var _ = BeforeSuite(func() {
+	var err error
+
+	runRemotely := os.Getenv("TEST_REMOTE_NODE") == "1"
+
+	verifyRequiredEnvVars(runRemotely)
+
+	if runRemotely {
+		r = remote.NewRemote()
+		err = r.SetupNewDriver(endpoint)
+		Expect(err).To(BeNil(), "error while driver setup")
+		nodeID = os.Getenv("POWERVS_INSTANCE_ID")
+		Expect(nodeID).NotTo(BeEmpty(), "Missing env var POWERVS_INSTANCE_ID")
+	} else {
+		drv, err = driver.NewDriver(driver.WithEndpoint(endpoint))
+		Expect(err).To(BeNil())
+		// Run the driver in goroutine
+		go func() {
+			err = drv.Run()
+			Expect(err).To(BeNil())
+		}()
+	}
+
+	csiClient, err = newCSIClient()
+	Expect(err).To(BeNil(), "failed to create CSI Client")
+	Expect(csiClient).NotTo(BeNil())
+})
+
+// After stops the driver
+var _ = AfterSuite(func() {
+	if os.Getenv("TEST_REMOTE_NODE") == "1" {
+		r.TeardownDriver()
+	} else if drv != nil {
+		drv.Stop()
+	}
+})
+
+// CSIClient controller and node clients
+type CSIClient struct {
+	ctrl csi.ControllerClient
+	node csi.NodeClient
+}
+
+// verifyRequiredEnvVars Verify that PowerVS details are set in env
+func verifyRequiredEnvVars(runRemotely bool) {
+	Expect(os.Getenv("IBMCLOUD_API_KEY")).NotTo(BeEmpty(), "Missing env var IBMCLOUD_API_KEY")
+	Expect(os.Getenv("POWERVS_CLOUD_INSTANCE_ID")).NotTo(BeEmpty(), "Missing env var POWERVS_CLOUD_INSTANCE_ID")
+	Expect(os.Getenv("POWERVS_ZONE")).NotTo(BeEmpty(), "Missing env var POWERVS_ZONE")
+
+	// POWERVS_INSTANCE_ID is required when not running remotely
+	if !runRemotely {
+		nodeID = os.Getenv("POWERVS_INSTANCE_ID")
+		Expect(nodeID).NotTo(BeEmpty(), "Missing env var POWERVS_INSTANCE_ID")
+	}
+}
+
+// newCSIClient creates as CSI client
+func newCSIClient() (*CSIClient, error) {
+	opts := []grpc.DialOption{
+		grpc.WithTransportCredentials(insecure.NewCredentials()),
+		// grpc.WithBlock(),
+		grpc.WithContextDialer(
+			func(context.Context, string) (net.Conn, error) {
+				scheme, addr, err := util.ParseEndpoint(endpoint)
+				if err != nil {
+					return nil, err
+				}
+				var conn net.Conn
+				err = wait.PollUntilContextTimeout(context.Background(), 10*time.Second, 3*time.Minute, true, func(context.Context) (bool, error) {
+					conn, err = net.Dial(scheme, addr)
+					if err != nil {
+						klog.Warningf("Client failed to dial endpoint %v", endpoint)
+						return false, nil
+					}
+					klog.Infof("Client succeeded to dial endpoint %v", endpoint)
+					return true, nil
+				})
+				if err != nil || conn == nil {
+					return nil, fmt.Errorf("failed to get client connection: %v", err)
+				}
+				return conn, err
+			},
+		),
+	}
+	grpcClient, err := grpc.Dial(endpoint, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return &CSIClient{
+		ctrl: csi.NewControllerClient(grpcClient),
+		node: csi.NewNodeClient(grpcClient),
+	}, nil
+}