From 478509e32c19e976c9ede2395d6db37bec9a36b9 Mon Sep 17 00:00:00 2001 From: Yussuf Shaikh Date: Sat, 11 Nov 2023 21:53:08 +0530 Subject: [PATCH] feature: ability to run remote IT on PowerVS Signed-off-by: Yussuf Shaikh --- Makefile | 3 + tests/it/README.md | 34 ++++ tests/it/integration_test.go | 181 +++++++++++++++++++++ tests/it/utils.go | 143 +++++++++++++++++ tests/remote/pi_resources.go | 302 +++++++++++++++++++++++++++++++++++ tests/remote/setup.go | 243 ++++++++++++++++++++++++++++ tests/remote/util.go | 92 +++++++++++ 7 files changed, 998 insertions(+) create mode 100644 tests/it/README.md create mode 100644 tests/it/integration_test.go create mode 100644 tests/it/utils.go create mode 100644 tests/remote/pi_resources.go create mode 100644 tests/remote/setup.go create mode 100644 tests/remote/util.go diff --git a/Makefile b/Makefile index 5312487ba..cc1ee3b5c 100644 --- a/Makefile +++ b/Makefile @@ -126,3 +126,6 @@ init-buildx: # Register gcloud as a Docker credential helper. # Required for "docker buildx build --push". gcloud auth configure-docker --quiet + +test-integration: + go test -v -timeout 100m sigs.k8s.io/ibm-powervs-block-csi-driver/tests/it -run ^TestIntegration$ diff --git a/tests/it/README.md b/tests/it/README.md new file mode 100644 index 000000000..aca61ef39 --- /dev/null +++ b/tests/it/README.md @@ -0,0 +1,34 @@ +# Integration Testing + +## Introduction + +The integration test will create a PowerVS instance and run the CSI driver tests on it. The driver will be built locally for ppc64le and copied over to the pvm instance. The driver will run on the instance and a SSH tunnel will be created. A grpc client will connect to the driver via a tcp port and run the Node and Controller tests. + +You can also run the test on an existing PowerVS instance itself. + +## Environments +The following environment variables will be required for running the integration test. +``` +export IBMCLOUD_API_KEY= # IBM Cloud API key +export POWERVS_ZONE="mon01", # IBM Cloud zone + +``` +Below environment variable is needed when running the tests within a PowerVS machine. +``` +export POWERVS_INSTANCE_ID="" # The pvm instance id where the test is running +``` +Below environment variables are used if you want to use run tests from a remote machine. +``` +export TEST_REMOTE_NODE="1" # Set to 1 if you want to run the remote node tests +export IBMCLOUD_ACCOUNT_ID="" # IBM Cloud account to use for creating the remote node +export POWERVS_CLOUD_INSTANCE_ID="" # Workspace guid to use for creating the remote node +export POWERVS_NETWORK="pub-network" # (Optional) The network to use for creating the remote node +export POWERVS_IMAGE="CentOS-Stream-8" # (Optional) The image to use for creating the remote node +``` + + +## Run the test +To run the test use the following command. +``` +make test-integration +``` diff --git a/tests/it/integration_test.go b/tests/it/integration_test.go new file mode 100644 index 000000000..75989fde2 --- /dev/null +++ b/tests/it/integration_test.go @@ -0,0 +1,181 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package it + +import ( + "context" + "flag" + "fmt" + "math/rand" + "os" + "path/filepath" + "testing" + "time" + + "github.com/container-storage-interface/spec/lib/go/csi" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/klog/v2" +) + +var ( + stdVolCap = []*csi.VolumeCapability{ + { + AccessType: &csi.VolumeCapability_Mount{ + Mount: &csi.VolumeCapability_MountVolume{}, + }, + AccessMode: &csi.VolumeCapability_AccessMode{ + Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, + }, + }, + } + stdCapRange = &csi.CapacityRange{RequiredBytes: int64(1 * 1024 * 1024 * 1024)} + testVolumeName = fmt.Sprintf("ibm-powervs-csi-driver-it-%d", rand.New(rand.NewSource(time.Now().UnixNano())).Uint64()) +) + +func TestIntegration(t *testing.T) { + flag.Parse() + RegisterFailHandler(Fail) + RunSpecs(t, "IBM PowerVS CSI Driver Integration Tests") +} + +var _ = Describe("IBM PowerVS CSI Driver", func() { + + It("Should create, attach, stage and mount volume, check if it's writable, unmount, unstage, detach, delete, and check if it's deleted", func() { + + testCreateAttachWriteReadDetachDelete() + + }) +}) + +func testCreateAttachWriteReadDetachDelete() { + klog.Infof("Creating volume with name %s", testVolumeName) + start := time.Now() + resp, err := csiClient.ctrl.CreateVolume(context.Background(), &csi.CreateVolumeRequest{ + Name: testVolumeName, + CapacityRange: stdCapRange, + VolumeCapabilities: stdVolCap, + Parameters: nil, + }) + Expect(err).To(BeNil(), "error during create volume") + volume := resp.GetVolume() + Expect(volume).NotTo(BeNil(), "volume is nil") + klog.Infof("Created volume %s in %v", volume, time.Since(start)) + + defer func() { + klog.Infof("Deleting volume %s", volume.VolumeId) + start := time.Now() + _, err = csiClient.ctrl.DeleteVolume(context.Background(), &csi.DeleteVolumeRequest{VolumeId: volume.VolumeId}) + Expect(err).To(BeNil(), "error during delete volume") + klog.Infof("Deleted volume %s in %v", volume.VolumeId, time.Since(start)) + }() + + klog.Info("Running ValidateVolumeCapabilities") + vcResp, err := csiClient.ctrl.ValidateVolumeCapabilities(context.Background(), &csi.ValidateVolumeCapabilitiesRequest{ + VolumeId: volume.VolumeId, + VolumeCapabilities: stdVolCap, + }) + Expect(err).To(BeNil()) + klog.Infof("Ran ValidateVolumeCapabilities with response %v", vcResp) + + klog.Info("Running NodeGetInfo") + niResp, err := csiClient.node.NodeGetInfo(context.Background(), &csi.NodeGetInfoRequest{}) + Expect(err).To(BeNil()) + klog.Infof("Ran NodeGetInfo with response %v", niResp) + + testAttachStagePublishDetach(volume.VolumeId) +} + +func testAttachStagePublishDetach(volumeID string) { + klog.Infof("Attaching volume %s to node %s", volumeID, nodeID) + start := time.Now() + respAttach, err := csiClient.ctrl.ControllerPublishVolume(context.Background(), &csi.ControllerPublishVolumeRequest{ + VolumeId: volumeID, + NodeId: nodeID, + VolumeCapability: stdVolCap[0], + }) + Expect(err).To(BeNil(), "failed attaching volume %s to node %s", volumeID, nodeID) + // assertAttachmentState(volumeID, "attached") + klog.Infof("Attached volume with response %v in %v", respAttach.PublishContext, time.Since(start)) + + defer func() { + klog.Infof("Detaching volume %s from node %s", volumeID, nodeID) + start := time.Now() + _, err = csiClient.ctrl.ControllerUnpublishVolume(context.Background(), &csi.ControllerUnpublishVolumeRequest{ + VolumeId: volumeID, + NodeId: nodeID, + }) + Expect(err).To(BeNil(), "failed detaching volume %s from node %s", volumeID, nodeID) + // assertAttachmentState(volumeID, "detached") + klog.Infof("Detached volume %s from node %s in %v", volumeID, nodeID, time.Since(start)) + }() + + if os.Getenv("TEST_REMOTE_NODE") == "1" { + testStagePublish(volumeID, respAttach.PublishContext["wwn"]) + } +} + +func testStagePublish(volumeID, wwn string) { + // Node Stage + volDir := filepath.Join("/tmp/", testVolumeName) + stageDir := filepath.Join(volDir, "stage") + klog.Infof("Staging volume %s to path %s", volumeID, stageDir) + start := time.Now() + _, err := csiClient.node.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ + VolumeId: volumeID, + StagingTargetPath: stageDir, + VolumeCapability: stdVolCap[0], + PublishContext: map[string]string{"wwn": wwn}, + }) + Expect(err).To(BeNil(), "failed staging volume %s to path %s", volumeID, stageDir) + klog.Infof("Staged volume %s to path %s in %v", volumeID, stageDir, time.Since(start)) + + defer func() { + klog.Infof("Unstaging volume %s from path %s", volumeID, stageDir) + start := time.Now() + _, err = csiClient.node.NodeUnstageVolume(context.Background(), &csi.NodeUnstageVolumeRequest{VolumeId: volumeID, StagingTargetPath: stageDir}) + Expect(err).To(BeNil(), "failed unstaging volume %s from path %s", volumeID, stageDir) + err = os.RemoveAll(volDir) + Expect(err).To(BeNil(), "failed to remove volume directory") + klog.Infof("Unstaged volume %s from path %s in %v", volumeID, stageDir, time.Since(start)) + }() + + // Node Publish + publishDir := filepath.Join("/tmp/", testVolumeName, "mount") + klog.Infof("Publishing volume %s to path %s", volumeID, publishDir) + start = time.Now() + _, err = csiClient.node.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ + VolumeId: volumeID, + StagingTargetPath: stageDir, + TargetPath: publishDir, + VolumeCapability: stdVolCap[0], + PublishContext: map[string]string{"wwn": wwn}, + }) + Expect(err).To(BeNil(), "failed publishing volume %s to path %s", volumeID, publishDir) + klog.Infof("Published volume %s to path %s in %v", volumeID, publishDir, time.Since(start)) + + defer func() { + klog.Infof("Unpublishing volume %s from path %s", volumeID, publishDir) + start := time.Now() + _, err = csiClient.node.NodeUnpublishVolume(context.Background(), &csi.NodeUnpublishVolumeRequest{ + VolumeId: volumeID, + TargetPath: publishDir, + }) + Expect(err).To(BeNil(), "failed unpublishing volume %s from path %s", volumeID, publishDir) + klog.Infof("Unpublished volume %s from path %s in %v", volumeID, publishDir, time.Since(start)) + }() +} diff --git a/tests/it/utils.go b/tests/it/utils.go new file mode 100644 index 000000000..4cecdebfd --- /dev/null +++ b/tests/it/utils.go @@ -0,0 +1,143 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package it + +import ( + "context" + "fmt" + "net" + "os" + "time" + + "github.com/container-storage-interface/spec/lib/go/csi" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/klog/v2" + "sigs.k8s.io/ibm-powervs-block-csi-driver/pkg/driver" + "sigs.k8s.io/ibm-powervs-block-csi-driver/pkg/util" + "sigs.k8s.io/ibm-powervs-block-csi-driver/tests/remote" +) + +const ( + port = 10000 +) + +var ( + drv *driver.Driver + csiClient *CSIClient + nodeID string + r *remote.Remote + endpoint = fmt.Sprintf("tcp://localhost:%d", port) +) + +// Before runs the driver and creates a CSI client +var _ = BeforeSuite(func() { + var err error + + runRemotely := os.Getenv("TEST_REMOTE_NODE") == "1" + + verifyRequiredEnvVars(runRemotely) + + if runRemotely { + r = remote.NewRemote() + err = r.SetupNewDriver(endpoint) + Expect(err).To(BeNil(), "error while driver setup") + nodeID = os.Getenv("POWERVS_INSTANCE_ID") + Expect(nodeID).NotTo(BeEmpty(), "Missing env var POWERVS_INSTANCE_ID") + } else { + drv, err = driver.NewDriver(driver.WithEndpoint(endpoint)) + Expect(err).To(BeNil()) + // Run the driver in goroutine + go func() { + err = drv.Run() + Expect(err).To(BeNil()) + }() + } + + csiClient, err = newCSIClient() + Expect(err).To(BeNil(), "failed to create CSI Client") + Expect(csiClient).NotTo(BeNil()) +}) + +// After stops the driver +var _ = AfterSuite(func() { + if os.Getenv("TEST_REMOTE_NODE") == "1" { + r.TeardownDriver() + } else if drv != nil { + drv.Stop() + } +}) + +// CSIClient controller and node clients +type CSIClient struct { + ctrl csi.ControllerClient + node csi.NodeClient +} + +// verifyRequiredEnvVars Verify that PowerVS details are set in env +func verifyRequiredEnvVars(runRemotely bool) { + Expect(os.Getenv("IBMCLOUD_API_KEY")).NotTo(BeEmpty(), "Missing env var IBMCLOUD_API_KEY") + Expect(os.Getenv("POWERVS_CLOUD_INSTANCE_ID")).NotTo(BeEmpty(), "Missing env var POWERVS_CLOUD_INSTANCE_ID") + Expect(os.Getenv("POWERVS_ZONE")).NotTo(BeEmpty(), "Missing env var POWERVS_ZONE") + + // POWERVS_INSTANCE_ID is required when not running remotely + if !runRemotely { + nodeID = os.Getenv("POWERVS_INSTANCE_ID") + Expect(nodeID).NotTo(BeEmpty(), "Missing env var POWERVS_INSTANCE_ID") + } +} + +// newCSIClient creates as CSI client +func newCSIClient() (*CSIClient, error) { + opts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + // grpc.WithBlock(), + grpc.WithContextDialer( + func(context.Context, string) (net.Conn, error) { + scheme, addr, err := util.ParseEndpoint(endpoint) + if err != nil { + return nil, err + } + var conn net.Conn + err = wait.PollUntilContextTimeout(context.Background(), 10*time.Second, 3*time.Minute, true, func(context.Context) (bool, error) { + conn, err = net.Dial(scheme, addr) + if err != nil { + klog.Warningf("Client failed to dial endpoint %v", endpoint) + return false, nil + } + klog.Infof("Client succeeded to dial endpoint %v", endpoint) + return true, nil + }) + if err != nil || conn == nil { + return nil, fmt.Errorf("failed to get client connection: %v", err) + } + return conn, err + }, + ), + } + grpcClient, err := grpc.Dial(endpoint, opts...) + if err != nil { + return nil, err + } + return &CSIClient{ + ctrl: csi.NewControllerClient(grpcClient), + node: csi.NewNodeClient(grpcClient), + }, nil +} diff --git a/tests/remote/pi_resources.go b/tests/remote/pi_resources.go new file mode 100644 index 000000000..409b48be4 --- /dev/null +++ b/tests/remote/pi_resources.go @@ -0,0 +1,302 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package remote + +import ( + "context" + "fmt" + "os" + "time" + + "github.com/IBM-Cloud/power-go-client/clients/instance" + "github.com/IBM-Cloud/power-go-client/power/models" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/klog/v2" +) + +func (r *Remote) createPVSResources() (err error) { + var image, network string + + if err = r.initPowerVSClients(); err != nil { + return err + } + + if image, err = getEnvVar("POWERVS_IMAGE"); err != nil { + image = "CentOS-Stream-8" + if err = r.createImage(image); err != nil { + return fmt.Errorf("error while creating ibm pi image: %v", err) + } + } + + if network, err = getEnvVar("POWERVS_NETWORK"); err != nil { + network = "pub-network" + netID, err := r.createPublicNetwork(network) + if err != nil { + return fmt.Errorf("error while creating ibm pi network: %v", err) + } + // Use network ID instead of name (there are cache issue at PowerVS) + network = netID + } + + if err = r.createSSHKey(); err != nil { + return fmt.Errorf("error while creating ibm pi ssh key: %v", err) + } + + insID, pubIP, err := r.createInstance(image, network) + if err != nil { + return err + } + os.Setenv("POWERVS_INSTANCE_ID", insID) + r.publicIP = pubIP + + return nil +} + +func (r *Remote) initPowerVSClients() error { + piSession, err := getIBMPISession() + if err != nil { + return fmt.Errorf("error while creating ibm pi session: %v", err) + } + piID, err := getPiID() + if err != nil { + return fmt.Errorf("error while getting ibm pi id: %v", err) + } + r.powervsClients.ic = instance.NewIBMPIInstanceClient(context.Background(), piSession, piID) + r.powervsClients.imgc = instance.NewIBMPIImageClient(context.Background(), piSession, piID) + r.powervsClients.nc = instance.NewIBMPINetworkClient(context.Background(), piSession, piID) + r.powervsClients.kc = instance.NewIBMPIKeyClient(context.Background(), piSession, piID) + return nil +} + +func (r *Remote) createImage(image string) error { + imgc := r.powervsClients.imgc + if _, err := imgc.Get(image); err != nil { + // If no image then copy one + stockImages, err := imgc.GetAllStockImages(false, false) + if err != nil { + return err + } + var stockImageID string + for _, sI := range stockImages.Images { + if *sI.Name == image { + stockImageID = *sI.ImageID + } + } + if stockImageID == "" { + return fmt.Errorf("cannot find image: %s", image) + } + if _, err := imgc.Create(&models.CreateImage{ImageID: stockImageID}); err != nil { + return err + } + } + return nil +} + +// If given network name is not found in the network list +// create a new public network with the same name and use it +func (r *Remote) createPublicNetwork(network string) (string, error) { + nc := r.powervsClients.nc + netFound := "" + nets, err := nc.GetAll() + if err != nil { + return "", err + } + for _, n := range nets.Networks { + if *n.Name == network { + netFound = *n.NetworkID + } + } + + // If no public network then create one + if netFound == "" { + netType := "pub-vlan" + if _, err := nc.Create(&models.NetworkCreate{Name: network, Type: &netType}); err != nil { + return "", err + } + net, err := waitForNetworkVLAN(network, nc) + if err != nil { + return "", err + } + netFound = *net.NetworkID + } + return netFound, nil +} + +// Generate a new SSH key pair and create a SSHKey using the pub key +func (r *Remote) createSSHKey() error { + kc := r.powervsClients.kc + sshKey := r.resName + // Create SSH key pair + out, err := runCommand("ssh-keygen", "-t", "rsa", "-f", sshDefaultKey, "-N", "") + klog.Infof("ssh-keygen command output: %s err: %v", out, err) + + publicKey, err := os.ReadFile(sshDefaultKey + ".pub") + if err != nil { + return fmt.Errorf("error while creating and reading SSH key files: %v", err) + } + + sshPubKey := string(publicKey) + _, err = kc.Create(&models.SSHKey{Name: &sshKey, SSHKey: &sshPubKey}) + + return err +} + +// Create a pvm instance +func (r *Remote) createInstance(image, network string) (string, string, error) { + ic := r.powervsClients.ic + name := r.resName + memory := 4.0 + processors := 0.5 + procType := "shared" + sysType := "e980" + storageType := "tier1" + + nets := []*models.PVMInstanceAddNetwork{{NetworkID: &network}} + req := &models.PVMInstanceCreate{ + ImageID: &image, + KeyPairName: name, + Networks: nets, + ServerName: &name, + Memory: &memory, + Processors: &processors, + ProcType: &procType, + SysType: sysType, + StorageType: storageType, + } + resp, err := ic.Create(req) + if err != nil { + return "", "", fmt.Errorf("error while creating pvm instance: %v", err) + } + + insID := "" + for _, in := range *resp { + insID = *in.PvmInstanceID + } + + if insID == "" { + return "", "", fmt.Errorf("error while fetching pvm instance: %v", err) + } + + in, err := waitForInstanceHealth(insID, ic) + if err != nil { + return "", "", fmt.Errorf("error while waiting for pvm instance status: %v", err) + } + + publicIP := "" + insNets := in.Networks + for _, net := range insNets { + publicIP = net.ExternalIP + } + + if publicIP == "" { + return "", "", fmt.Errorf("error while getting pvm instance public IP") + } + + err = waitForInstanceSSH(publicIP) + if err != nil { + return "", "", fmt.Errorf("error while waiting for pvm instance ssh connection: %v", err) + } + + return insID, publicIP, err +} + +// Wait till VLAN is attached to the network +func waitForNetworkVLAN(netID string, nc *instance.IBMPINetworkClient) (*models.Network, error) { + var network *models.Network + err := wait.PollUntilContextTimeout(context.Background(), 15*time.Second, 5*time.Minute, true, func(context.Context) (bool, error) { + var err error + network, err = nc.Get(netID) + if err != nil || network == nil { + return false, err + } + if network.VlanID != nil { + return true, nil + } + return false, nil + }) + + if err != nil || network == nil { + return nil, fmt.Errorf("failed to get target instance status: %v", err) + } + return network, err +} + +// Wait for VM status is ACTIVE and VM Health status to be OK/Warning +func waitForInstanceHealth(insID string, ic *instance.IBMPIInstanceClient) (*models.PVMInstance, error) { + var pvm *models.PVMInstance + err := wait.PollUntilContextTimeout(context.Background(), 30*time.Second, 45*time.Minute, true, func(context.Context) (bool, error) { + var err error + pvm, err = ic.Get(insID) + if err != nil { + return false, err + } + if *pvm.Status == "ERROR" { + if pvm.Fault != nil { + return false, fmt.Errorf("failed to create the lpar: %s", pvm.Fault.Message) + } + return false, fmt.Errorf("failed to create the lpar") + } + // Check for `instanceReadyStatus` health status and also the final health status "OK" + if *pvm.Status == "ACTIVE" && (pvm.Health.Status == "WARNING" || pvm.Health.Status == "OK") { + return true, nil + } + + return false, nil + }) + + if err != nil || pvm == nil { + return nil, fmt.Errorf("failed to get target instance status: %v", err) + } + return pvm, err +} + +// Wait till SSH test is complete +func waitForInstanceSSH(publicIP string) error { + err := wait.PollUntilContextTimeout(context.Background(), 20*time.Second, 30*time.Minute, true, func(context.Context) (bool, error) { + var err error + outp, err := runRemoteCommand(publicIP, "hostname") + klog.Infof("out: %s err: %v", outp, err) + if err != nil { + return false, nil + } + return true, nil + }) + + if err != nil { + return fmt.Errorf("failed to get ssh connection: %v", err) + } + + return err +} + +// Delete the VM and SSH key +func (r *Remote) destroyPVSResources() { + if r.powervsClients.ic == nil || r.powervsClients.kc == nil { + klog.Warning("failed to retrieve PowerVS instance and key clients, ignoring") + return + } + + err := r.powervsClients.kc.Delete(r.resName) + if err != nil { + klog.Warningf("failed to destroy pvs resources, might leave behind some stale instances: %v", err) + } + + err = r.powervsClients.ic.Delete(r.resName) + if err != nil { + klog.Warningf("failed to destroy pvs resources, might leave behind some stale ssh keys: %v", err) + } +} diff --git a/tests/remote/setup.go b/tests/remote/setup.go new file mode 100644 index 000000000..698ff253c --- /dev/null +++ b/tests/remote/setup.go @@ -0,0 +1,243 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package remote + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "github.com/IBM-Cloud/power-go-client/clients/instance" + "k8s.io/klog/v2" +) + +const ( + osLinux = "linux" + arch = "ppc64le" + binName = "ibm-powervs-block-csi-driver" + binPath = "bin/" + binName + tarName = "powervs-csi-driver-binary.tar.gz" + timestampFormat = "20060102T150405" + sshDefaultKey = "/tmp/id_rsa" + sshUser = "root" + outputFile = "prog.out" +) + +type Remote struct { + resName string + publicIP string + sshPID int + tarPath string + remoteDir string + powervsClients PowerVSClients +} + +type PowerVSClients struct { + ic *instance.IBMPIInstanceClient + imgc *instance.IBMPIImageClient + nc *instance.IBMPINetworkClient + kc *instance.IBMPIKeyClient +} + +func NewRemote() *Remote { + return &Remote{ + resName: "csi-test-" + time.Now().Format(timestampFormat), + } +} + +func (r *Remote) SetupNewDriver(endpoint string) (err error) { + if err = r.createDriverArchive(); err != nil { + return err + } + defer func() { + if err := os.Remove(r.tarPath); err != nil { + klog.Warningf("failed to remove archive file %s: %v", r.tarPath, err) + } + }() + + if err = r.createPVSResources(); err != nil { + return err + } + + if err = r.uploadAndRun(endpoint); err != nil { + return err + } + + if err = r.createSSHTunnel(endpoint); err != nil { + return fmt.Errorf("SSH Tunnel pid %v encountered error: %v", r.sshPID, err.Error()) + } + + return nil +} + +// Create binary and archive it +func (r *Remote) createDriverArchive() (err error) { + tarDir, err := os.MkdirTemp("", "powervscsidriver") + if err != nil { + return fmt.Errorf("failed to create temporary directory %v", err) + } + defer os.RemoveAll(tarDir) + + err = setupBinariesLocally(tarDir) + if err != nil { + return fmt.Errorf("failed to setup test package %s: %v", tarDir, err) + } + + // Fetch tar path at current dir + dir, err := os.Getwd() + if err != nil { + return fmt.Errorf("failed to get working directory %v", err) + } + r.tarPath = filepath.Join(dir, tarName) + + return nil +} + +func setupBinariesLocally(tarDir string) error { + cmd := exec.Command("make", "GOOS="+osLinux, "GOARCH="+arch, "driver") + cmd.Dir = "../.." + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to run make driver: %s: %v", string(out), err) + } + + // Copy binaries + bPath := "../../" + binPath + if _, err := os.Stat(bPath); err != nil { + return fmt.Errorf("failed to locate test binary %s: %v", binPath, err) + } + out, err = exec.Command("cp", bPath, tarDir).CombinedOutput() + if err != nil { + return fmt.Errorf("failed to copy %s: %s: %v", bPath, string(out), err) + } + + // Build the tar + out, err = exec.Command("tar", "-zcvf", tarName, "-C", tarDir, ".").CombinedOutput() + if err != nil { + return fmt.Errorf("failed to build tar: %s: %v", string(out), err) + } + + return nil +} + +// Upload archive to instance and run binaries +// TODO: grep pid of driver and kill it later +// Also, use random port +// May be track the job id if given +func (r *Remote) uploadAndRun(endpoint string) error { + r.remoteDir = filepath.Join("/tmp", "powervs-csi-"+time.Now().Format(timestampFormat)) + + klog.Infof("Staging test on %s", r.remoteDir) + if output, err := r.runSSHCommand("mkdir", r.remoteDir); err != nil { + return fmt.Errorf("failed to create remote directory %s on the instance: %s: %v", r.remoteDir, output, err) + } + + klog.Infof("Copying test archive %s to %s:%s/", r.tarPath, r.publicIP, r.remoteDir) + if output, err := runCommand("scp", "-i", sshDefaultKey, r.tarPath, fmt.Sprintf("%s@%s:%s/", sshUser, r.publicIP, r.remoteDir)); err != nil { + return fmt.Errorf("failed to copy test archive: %s: %v", output, err) + } + + klog.Infof("Extract test archive at %s:%s/", r.publicIP, r.remoteDir) + extractCmd := fmt.Sprintf("'cd %s && tar -xzvf ./%s'", r.remoteDir, tarName) + if output, err := r.runSSHCommand("sh", "-c", extractCmd); err != nil { + return fmt.Errorf("failed to extract test archive: %s: %v", output, err) + } + + klog.Infof("Run the driver on %s", r.publicIP) + // TODO: env file + exportStr := fmt.Sprintf("export IBMCLOUD_API_KEY=%s;", os.Getenv("IBMCLOUD_API_KEY")) + exportStr += fmt.Sprintf("export POWERVS_CLOUD_INSTANCE_ID=%s;", os.Getenv("POWERVS_CLOUD_INSTANCE_ID")) + exportStr += fmt.Sprintf("export POWERVS_ZONE=%s;", os.Getenv("POWERVS_ZONE")) + exportStr += fmt.Sprintf("export POWERVS_INSTANCE_ID=%s;", os.Getenv("POWERVS_INSTANCE_ID")) + + // Below vars needed for staging env + if iamEndpoint, err := getEnvVar("IBMCLOUD_IAM_API_ENDPOINT"); err == nil { + exportStr += fmt.Sprintf("export IBMCLOUD_IAM_API_ENDPOINT=%s;", iamEndpoint) + } + if piEndpoint, err := getEnvVar("IBMCLOUD_POWER_API_ENDPOINT"); err == nil { + exportStr += fmt.Sprintf("export IBMCLOUD_POWER_API_ENDPOINT=%s;", piEndpoint) + } + if rcEndpoint, err := getEnvVar("IBMCLOUD_RESOURCE_CONTROLLER_ENDPOINT"); err == nil { + exportStr += fmt.Sprintf("export IBMCLOUD_RESOURCE_CONTROLLER_ENDPOINT=%s;", rcEndpoint) + } + + runCmd := fmt.Sprintf("'%s /usr/bin/nohup %s/%s -v=6 --endpoint=%s 2> %s/%s < /dev/null > /dev/null &'", + exportStr, r.remoteDir, binName, endpoint, r.remoteDir, outputFile) + if output, err := r.runSSHCommand("sh", "-c", runCmd); err != nil { + return fmt.Errorf("failed to run the driver: %s: %v", output, err) + } + + return nil +} + +func (r *Remote) runSSHCommand(arg ...string) (string, error) { + return runRemoteCommand(r.publicIP, arg...) +} + +func (r *Remote) createSSHTunnel(endpoint string) error { + port := endpoint[strings.LastIndex(endpoint, ":")+1:] + + args := []string{"-i", sshDefaultKey, "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", + "-nNT", "-L", fmt.Sprintf("%s:localhost:%s", port, port), fmt.Sprintf("%s@%s", sshUser, r.publicIP)} + + klog.Infof("Executing SSH command: ssh %v", args) + sshCmd := exec.Command("ssh", args...) + err := sshCmd.Start() + if err != nil { + return err + } + r.sshPID = sshCmd.Process.Pid + return nil +} + +func (r *Remote) TeardownDriver() { + // sshPID=0 means something went wrong, no ssh tunnel created + if r.sshPID != 0 { + // Close the SSH tunnel + err := findAndKillProcess(r.sshPID) + if err != nil { + klog.Warningf("failed to find ssh PID, might leave behind some stale tunnel process: %v", err) + } + } + + // Kill the driver process? + // killCmd := "$'kill $(ps -ef | grep ibm-powervs-block-csi-driver | grep -v grep | awk \\'{print $2}\\')'" + // if output, err := r.runSSHCommand("sh", "-c", killCmd); err != nil { + // klog.Warningf("failed to kill remote driver, might leave behind some stale driver process: %s: %v", output, err) + // } + + r.printRemoteLog() + + // Delete any created ssh key files and tarball + os.Remove(sshDefaultKey) + os.Remove(sshDefaultKey + ".pub") + os.Remove(r.tarPath) + + r.destroyPVSResources() +} + +func (r *Remote) printRemoteLog() { + if r.publicIP == "" || r.remoteDir == "" { + return + } + klog.Infof("printing remote log from %s/%s:", r.remoteDir, outputFile) + out, _ := r.runSSHCommand("cat", fmt.Sprintf("%s/%s", r.remoteDir, outputFile)) + klog.Info(out) +} diff --git a/tests/remote/util.go b/tests/remote/util.go new file mode 100644 index 000000000..1eb960c07 --- /dev/null +++ b/tests/remote/util.go @@ -0,0 +1,92 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package remote + +import ( + "fmt" + "os" + "os/exec" + "strings" + + "github.com/IBM-Cloud/power-go-client/ibmpisession" + "github.com/IBM/go-sdk-core/v5/core" + "k8s.io/klog/v2" +) + +func runRemoteCommand(publicIP string, arg ...string) (string, error) { + args := []string{"-i", sshDefaultKey, "-o", "StrictHostKeyChecking no", fmt.Sprintf("%s@%s", sshUser, publicIP), "--"} + args = append(args, arg...) + + // Should we print?; May contain sensitive information + klog.Infof("Executing SSH command: ssh %v", args) + return runCommand("ssh", args...) +} + +func runCommand(cmd string, args ...string) (string, error) { + output, err := exec.Command(cmd, args...).CombinedOutput() + if err != nil { + return string(output), fmt.Errorf("command [%s %s] failed with error: %v", cmd, strings.Join(args, " "), err.Error()) + } + return string(output), err +} + +func getEnvVar(envName string) (string, error) { + ret := os.Getenv(envName) + if ret == "" { + return "", fmt.Errorf("missing %s env variable", envName) + } + return ret, nil +} + +func getIBMPISession() (*ibmpisession.IBMPISession, error) { + var err error + var apiKey, accountID, zone string + + if apiKey, err = getEnvVar("IBMCLOUD_API_KEY"); err != nil { + return nil, err + } + if accountID, err = getEnvVar("IBMCLOUD_ACCOUNT_ID"); err != nil { + return nil, err + } + if zone, err = getEnvVar("POWERVS_ZONE"); err != nil { + return nil, err + } + + o := &ibmpisession.IBMPIOptions{ + Authenticator: &core.IamAuthenticator{ApiKey: apiKey, URL: os.Getenv("IBMCLOUD_IAM_API_ENDPOINT")}, + UserAccount: accountID, + Zone: zone, + Debug: true, + } + + return ibmpisession.NewIBMPISession(o) +} + +func getPiID() (string, error) { + return getEnvVar("POWERVS_CLOUD_INSTANCE_ID") +} + +func findAndKillProcess(sshPID int) error { + proc, err := os.FindProcess(sshPID) + if err != nil { + return fmt.Errorf("unable to find ssh tunnel process %v: %v", sshPID, err) + } + if err = proc.Kill(); err != nil { + return fmt.Errorf("failed to kill ssh tunnel process %v: %v", sshPID, err) + } + return nil +}