Adding ssie events for longevity (#1929)

.travis.yml

@@ -2,15 +2,20 @@ sudo: required
 dist: focal
 services:
   - docker
-language: go
+  - mongodb
+language:
+  - go
+  - minimal
 go:
-  - 1.19.x
+  - 1.21.x
 cache:
   directories:
     - $GOPATH/pkg/mod
 before_install:
+  - wget -qO - https://www.mongodb.org/static/pgp/server-4.4.asc | sudo apt-key add -
+  - echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu focal/mongodb-org/4.4 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-4.4.list
   - sudo apt-get update -yq
-  - sudo apt-get install apt-transport-https ca-certificates -y
+  - sudo apt-get install apt-transport-https ca-certificates mongodb-org -y 
   - sudo update-ca-certificates
 script:
   - make all
@@ -29,6 +34,10 @@ after_success:
       sudo docker tag "${DOCKER_HUB_REPO}/torpedo-backup:${DOCKER_HUB_TAG}" "${DOCKER_HUB_REPO}/torpedo-backup:${TRAVIS_BRANCH}";
       sudo docker push "${DOCKER_HUB_REPO}/torpedo-backup:${DOCKER_HUB_TAG}";
       sudo docker push "${DOCKER_HUB_REPO}/torpedo-backup:${TRAVIS_BRANCH}";
+      make container-longevity;
+      sudo docker tag "${DOCKER_HUB_REPO}/torpedo-longevity:${DOCKER_HUB_TAG}" "${DOCKER_HUB_REPO}/torpedo-longevity:${TRAVIS_BRANCH}";
+      sudo docker push "${DOCKER_HUB_REPO}/torpedo-longevity:${DOCKER_HUB_TAG}";
+      sudo docker push "${DOCKER_HUB_REPO}/torpedo-longevity:${TRAVIS_BRANCH}";
     fi
 notifications:
   email:

Dockerfile

@@ -1,13 +1,13 @@
-FROM golang:1.19.5-alpine AS build
+FROM golang:1.21.6-alpine AS build
 LABEL maintainer="[email protected]"
 ARG MAKE_TARGET
 
 WORKDIR /go/src/github.com/portworx/torpedo
 
 # Install setup dependencies
-RUN apk update && apk add --no-cache bash git gcc musl-dev make curl openssh-client
+RUN apk update && apk add --no-cache bash git gcc musl-dev make curl openssh-client coreutils python3
 
-RUN GOFLAGS= GO111MODULE=on go install github.com/onsi/ginkgo/ginkgo@v1.16.5
+RUN GOFLAGS= GO111MODULE=on go install -mod=mod github.com/onsi/ginkgo/v2/ginkgo@v2.15.0
 
 # Install aws-iam-authenticator
 # This is needed by test running inside EKS cluster and creating aws entities like bucket etc.
@@ -18,21 +18,22 @@ RUN mkdir bin && \
 
 # Install IBM Cloud SDK
 RUN curl -fsSL https://clis.cloud.ibm.com/install/linux | sh && \
-    ibmcloud plugin install -f vpc-infrastructure && \
+    ibmcloud plugin install -v 11.3.0 -f vpc-infrastructure && \
     ibmcloud plugin install -f container-service
 
 # Install vCluster binary
 RUN curl -L -o vcluster "https://github.com/loft-sh/vcluster/releases/latest/download/vcluster-linux-amd64"  \
     && install -c -m 0755 vcluster /usr/local/bin  \
     && rm -f vcluster
 
+
+
 # No need to copy *everything*. This keeps the cache useful
 COPY vendor vendor
 COPY Makefile Makefile
 COPY go.mod go.mod
 COPY go.sum go.sum
 COPY pkg pkg
-COPY porx porx
 COPY scripts scripts
 COPY drivers drivers
 COPY deployments deployments
@@ -47,14 +48,21 @@ COPY tests tests
 RUN --mount=type=cache,target=/root/.cache/go-build make $MAKE_TARGET
 
 # Build a fresh container with just the binaries
-FROM alpine
+FROM alpine:3.18.5
 
-RUN apk add --no-cache ca-certificates bash curl jq libc6-compat
+RUN apk add --no-cache ca-certificates bash curl jq libc6-compat coreutils
 
  # Install Azure Cli
 RUN apk add --no-cache --update python3 py3-pip
 RUN apk add --no-cache --update --virtual=build gcc musl-dev python3-dev libffi-dev openssl-dev cargo make && pip3 install "pyyaml<=5.3.1" && pip3 install --no-cache-dir --prefer-binary azure-cli && apk del build
 
+
+# Install Oracle CLI
+RUN bash -c "$(curl -L https://raw.githubusercontent.com/oracle/oci-cli/master/scripts/install/install.sh)" -- --python-install-location /usr/bin/python3 --accept-all-defaults && \
+    chmod a+x /root/bin/oci && \
+    mv -f /root/bin/oci /usr/bin/oci
+
+
 # Install kubectl from Docker Hub.
 COPY --from=lachlanevenson/k8s-kubectl:latest /usr/local/bin/kubectl /usr/local/bin/kubectl
 
@@ -66,11 +74,31 @@ WORKDIR /torpedo
 COPY deployments deployments
 COPY scripts scripts
 
+# Install Postman-Newman Dependencies
+RUN apk update && apk upgrade \
+    && apk add --no-cache \
+        nodejs \
+        npm \
+    && rm -rf /var/cache/apk/*
+
+# Install Newman globally using npm
+RUN npm install -g newman
+
 WORKDIR /go/src/github.com/portworx/torpedo
 
 # Install docker
 RUN apk add --update --no-cache docker
 
+# Install openssh and sshpass
+RUN apk add --no-cache openssh sshpass
+
+# Install dependancy for OCP 4.14 CLI
+RUN apk --update add gcompat
+
+# Install yq
+RUN wget https://github.com/mikefarah/yq/releases/download/v4.25.1/yq_linux_amd64 -O /usr/bin/yq && \
+    chmod +x /usr/bin/yq
+
 # Copy ginkgo & binaries over from previous container
 COPY --from=build /go/bin/ginkgo /bin/ginkgo
 COPY --from=build /go/src/github.com/portworx/torpedo/bin bin
@@ -80,5 +108,5 @@ COPY --from=build /usr/local/bin/vcluster /bin/vcluster
 COPY --from=build /root/.bluemix/plugins /root/.bluemix/plugins
 COPY drivers drivers
 
-ENTRYPOINT ["ginkgo", "--failFast", "--slowSpecThreshold", "180", "-v", "-trace"]
+ENTRYPOINT ["ginkgo", "--fail-fast", "--poll-progress-after", "3m", "-v", "-trace", "--junit-report=/testresults/junit_basic.xml"]
 CMD []

Dockerfile-taas

@@ -1,4 +1,4 @@
-FROM golang:1.19.5-alpine AS build
+FROM golang:1.21.6-alpine AS build
 LABEL maintainer="[email protected]"
 ARG MAKE_TARGET
 
@@ -13,7 +13,6 @@ COPY Makefile Makefile
 COPY go.mod go.mod
 COPY go.sum go.sum
 COPY pkg pkg
-COPY porx porx
 COPY scripts scripts
 COPY drivers drivers
 COPY deployments deployments
@@ -44,6 +43,7 @@ RUN apk add --update --no-cache docker
 
 # Copy binaries over from previous container
 COPY --from=build /go/src/github.com/portworx/torpedo/bin bin
+COPY drivers drivers
 
 ENTRYPOINT []
 CMD []

Makefile

@@ -10,6 +10,9 @@
 # make {build-backup|container-backup}:
 #	 create backup.test binary/container (<repo>/torpedo-backup:<tag>)
 #
+# make {build-longevity|container-longevity}:
+#	 create longevity.test binary/container (<repo>/torpedo-longevity:<tag>)
+#
 # make all:
 #	 verify that all test binaries build successfully
 #
@@ -64,7 +67,7 @@ PGBENCH_IMG=$(DOCKER_HUB_REPO)/torpedo-pgbench:latest
 ESLOAD_IMG=$(DOCKER_HUB_REPO)/torpedo-esload:latest
 
 
-all: vet build build-pds build-backup build-taas fmt
+all: vet build build-pds build-backup build-taas build-longevity fmt
 
 deps:
 	go get -d -v $(PKGS)
@@ -84,7 +87,7 @@ fmt:
 
 # Tools
 $(GOPATH)/bin/ginkgo:
-	GOFLAGS= GO111MODULE=on go install github.com/onsi/ginkgo/ginkgo@v1.16.5
+	GOFLAGS= GO111MODULE=on go install -mod=mod github.com/onsi/ginkgo/v2/ginkgo@v2.15.0
 
 # this target builds the basic.test binary only. Use build-pds for pds.test binary.
 build: GINKGO_BUILD_DIR=./tests/basic
@@ -116,6 +119,16 @@ build-backup: $(GOPATH)/bin/ginkgo
 	find $(GINKGO_BUILD_DIR) -name '*.test' | awk '{cmd="cp  "$$1"  $(BIN)"; system(cmd)}'
 	chmod -R 755 bin/*
 
+build-longevity: GINKGO_BUILD_DIR=./tests/longevity
+build-longevity: $(GOPATH)/bin/ginkgo
+	mkdir -p $(BIN)
+	go build -tags "$(TAGS)" $(BUILDFLAGS) $(PKGS)
+
+	ginkgo build -r $(GINKGO_BUILD_DIR)
+	find $(GINKGO_BUILD_DIR) -name '*.test' | awk '{cmd="cp  "$$1"  $(BIN)"; system(cmd)}'
+	chmod -R 755 bin/*
+
+
 # this target builds the taas binary only.
 build-taas: TAAS_BUILD_DIR=./apiServer/taas
 build-taas:
@@ -184,6 +197,13 @@ container-taas:
 	@echo "Building taas container "$(TORPEDO_IMG)
 	sudo DOCKER_BUILDKIT=1 docker build --tag $(TORPEDO_IMG) --build-arg MAKE_TARGET=build-taas -f Dockerfile-taas .
 
+# this target builds a container with torpedo longevity binary only. Repo is hardcoded to ".../longevity".
+container-longevity: TORPEDO_IMG=$(DOCKER_HUB_REPO)/torpedo-longevity:$(DOCKER_HUB_TAG)
+container-longevity:
+	@echo "Building longevity container "$(TORPEDO_IMG)
+	sudo DOCKER_BUILDKIT=1 docker build --tag $(TORPEDO_IMG) --build-arg MAKE_TARGET=build-longevity -f Dockerfile .
+
+
 deploy: TORPEDO_IMG=$(DOCKER_HUB_REPO)/torpedo:$(DOCKER_HUB_TAG)
 deploy: container
 	sudo docker push $(TORPEDO_IMG)
@@ -200,6 +220,10 @@ deploy-taas: TORPEDO_IMG=$(DOCKER_HUB_REPO)/taas:$(DOCKER_HUB_TAG)
 deploy-taas: container-taas
 	sudo docker push $(TORPEDO_IMG)
 
+deploy-longevity: TORPEDO_IMG=$(DOCKER_HUB_REPO)/torpedo-longevity:$(DOCKER_HUB_TAG)
+deploy-longevity: container-longevity
+	sudo docker push $(TORPEDO_IMG)
+
 clean:
 	-sudo rm -rf bin
 	-sudo find . -type f -name "*.test" -delete
@@ -211,6 +235,8 @@ clean:
 	-docker rmi -f $(DOCKER_HUB_REPO)/torpedo-backup:$(DOCKER_HUB_TAG)
 	@echo "Deleting taas image"
 	-docker rmi -f $(DOCKER_HUB_REPO)/torpedo-taas:$(DOCKER_HUB_TAG)
+	@echo "Deleting longevity image"
+		-docker rmi -f $(DOCKER_HUB_REPO)/torpedo-longevity:$(DOCKER_HUB_TAG)
 	go clean -i $(PKGS)
 
 sidecar: sidecar-wp-cli

apiServer/taas/apiserver.go

@@ -19,13 +19,20 @@ func main() {
 	router.GET("taas/storagenodes", utils.GetStorageNodes)
 	router.GET("taas/storagelessnodes", utils.GetStorageLessNodes)
 	router.POST("taas/collectsupport", utils.CollectSupport)
-	router.POST("taas/scheduleapps/:appName", utils.ScheduleAppsAndValidate)
+	router.POST("taas/scheduleapps", utils.ScheduleAppsAndValidate)
 	router.POST("taas/deploypxagent", utils.ExecuteHelmCmd)
 	router.GET("taas/getclusterid/:namespace", utils.GetNamespaceID)
 	router.GET("taas/getclusternodestatus", utils.GetNodeStatus)
 	router.POST("taas/runhelmcmd", utils.ExecuteHelmCmd)
 	router.GET("taas/pxversion", utils.GetPxVersion)
 	router.GET("taas/ispxinstalled", utils.IsPxInstalled)
 	router.GET("taas/getpxctloutput", utils.GetPxctlStatusOutput)
+	router.GET("taas/getkubevirtvmsbyns", utils.GetVMsInNamespaces)
+	router.GET("taas/getkubevirtvmsbynslabels", utils.GetVMsWithNamespaceLabels)
+	router.POST("taas/namespaces/addLabel", utils.AddNSLabel)
+	router.POST("taas/stork/upgrade", utils.UpgradeStork)
+	router.DELETE("taas/deletepod", utils.DeletePod)
+	router.GET("taas/getpxbackupnamespace", utils.GetPxBackupNamespace)
+	router.POST("taas/createvolumesnapshotclass", utils.CreateVolumeSnapshotClass)
 	log.Fatal(router.Run(":8080"))
 }

apiServer/taas/utils/constants.go

@@ -1,8 +1,9 @@
 package utils
 
 import (
-	"github.com/portworx/sched-ops/k8s/core"
 	"time"
+
+	"github.com/portworx/sched-ops/k8s/core"
 )
 
 type HelmPayload struct {
@@ -24,8 +25,6 @@ const (
 )
 
 var (
-	errors  []error
-	errChan = make(chan error, 100)
 	k8sCore = core.Instance()
 )
 

apiServer/taas/utils/k8sutils.go

@@ -4,6 +4,8 @@ import (
 	"fmt"
 	"github.com/gin-gonic/gin"
 	"github.com/portworx/torpedo/drivers/pds/lib"
+	"github.com/portworx/torpedo/pkg/log"
+	"github.com/portworx/torpedo/tests"
 	v1 "k8s.io/api/core/v1"
 	"net/http"
 	"os/exec"
@@ -117,3 +119,34 @@ func GetNodeStatus(c *gin.Context) {
 		"DegradedNodes":  degradedNodes,
 	})
 }
+
+// CreateVolumeSnapshotClass creates volume snapshot class
+func CreateVolumeSnapshotClass(c *gin.Context) {
+	log.Infof("Creating volume snapshot class")
+	var createVolumeSnapshotClassRequest struct {
+		VolumeSnapshotClassName      string `json:"volumeSnapshotClassName"`
+		Provisioner                  string `json:"provisioner"`
+		IsDefaultVolumeSnapshotClass bool   `json:"isDefaultVolumeSnapshotClass"`
+		DeletePolicy                 string `json:"deletePolicy"`
+	}
+	if !checkTorpedoInit(c) {
+		c.JSON(http.StatusInternalServerError, gin.H{
+			"error in init": fmt.Errorf("error in InitInstance()"),
+		})
+		return
+	}
+	if err := c.BindJSON(&createVolumeSnapshotClassRequest); err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error for request": err.Error()})
+		return
+	}
+	if len(createVolumeSnapshotClassRequest.VolumeSnapshotClassName) == 0 || len(createVolumeSnapshotClassRequest.Provisioner) == 0 {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "volumesnapshotclass name  or provisioner cannot be empty", "vsc name": createVolumeSnapshotClassRequest.VolumeSnapshotClassName, "provisioner": createVolumeSnapshotClassRequest.Provisioner})
+		return
+	}
+	_, err := tests.Inst().S.CreateVolumeSnapshotClasses(createVolumeSnapshotClassRequest.VolumeSnapshotClassName, createVolumeSnapshotClassRequest.Provisioner, createVolumeSnapshotClassRequest.IsDefaultVolumeSnapshotClass, createVolumeSnapshotClassRequest.DeletePolicy)
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+		return
+	}
+	c.JSON(http.StatusOK, gin.H{"message": fmt.Sprintf("VolumeSnapshotClass %v with provisioner %s and setting it as default: %v created successfully", createVolumeSnapshotClassRequest.VolumeSnapshotClassName, createVolumeSnapshotClassRequest.Provisioner, createVolumeSnapshotClassRequest.IsDefaultVolumeSnapshotClass)})
+}