[YUNIKORN-2305] E2E test: Upload stdout logs to Github Action artifact

.github/workflows/pre-commit.yml

@@ -68,3 +68,9 @@ jobs:
         env:
           KIND_NODE_IMAGE: ${{ matrix.k8s }}
           KIND_EXTRA_ARGS: ${{ matrix.plugin }}
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        if: ${{ failure() }}
+        with:
+          name: ${{ github.job }} stdout (${{ matrix.k8s }}${{ matrix.plugin == '--plugin' && format(', {0}', matrix.plugin) || matrix.plugin }}) 
+          path: build/e2e

test/e2e/admission_controller/admission_controller_suite_test.go

@@ -21,6 +21,7 @@ package admission_controller_test
 import (
 	"fmt"
 	"path/filepath"
+	"runtime"
 	"testing"
 
 	. "github.com/onsi/ginkgo/v2"
@@ -45,6 +46,7 @@ func init() {
 
 const appName = "sleep"
 
+var suiteName string
 var kubeClient k8s.KubeCtl
 var ns string
 var bypassNs = "kube-system"
@@ -188,6 +190,8 @@ func TestAdmissionController(t *testing.T) {
 }
 
 var _ = BeforeSuite(func() {
+	_, filename, _, _ := runtime.Caller(0)
+	suiteName = common.GetSuiteName(filename)
 	restClient = yunikorn.RClient{}
 
 	kubeClient = k8s.KubeCtl{}

test/e2e/admission_controller/admission_controller_test.go

@@ -30,6 +30,7 @@ import (
 
 	amConf "github.com/apache/yunikorn-k8shim/pkg/admission/conf"
 	"github.com/apache/yunikorn-k8shim/pkg/common/constants"
+	tests "github.com/apache/yunikorn-k8shim/test/e2e"
 	"github.com/apache/yunikorn-k8shim/test/e2e/framework/configmanager"
 	"github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/common"
 	"github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/k8s"
@@ -448,6 +449,8 @@ var _ = ginkgo.Describe("AdmissionController", func() {
 	})
 
 	ginkgo.AfterEach(func() {
+		tests.DumpClusterInfoIfSpecFailed(suiteName, []string{ns})
+
 		ginkgo.By("Tear down namespace: " + ns)
 		err := kubeClient.TearDownNamespace(ns)
 		gomega.Ω(err).NotTo(gomega.HaveOccurred())

test/e2e/basic_scheduling/basic_scheduling_test.go

@@ -19,6 +19,7 @@
 package basicscheduling_test
 
 import (
+	"runtime"
 	"time"
 
 	"github.com/onsi/ginkgo/v2"
@@ -32,6 +33,7 @@ import (
 	"github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/yunikorn"
 )
 
+var suiteName string
 var kClient k8s.KubeCtl
 var restClient yunikorn.RClient
 var sleepRespPod *v1.Pod
@@ -44,6 +46,8 @@ var oldConfigMap = new(v1.ConfigMap)
 var sleepPodConfigs = k8s.SleepPodConfig{Name: "sleepjob", NS: dev}
 
 var _ = ginkgo.BeforeSuite(func() {
+	_, filename, _, _ := runtime.Caller(0)
+	suiteName = common.GetSuiteName(filename)
 	// Initializing kubectl client
 	kClient = k8s.KubeCtl{}
 	gomega.Ω(kClient.SetClient()).To(gomega.BeNil())
@@ -119,11 +123,7 @@ var _ = ginkgo.Describe("", func() {
 	})
 
 	ginkgo.AfterEach(func() {
-		testDescription := ginkgo.CurrentSpecReport()
-		if testDescription.Failed() {
-			tests.LogTestClusterInfoWrapper(testDescription.FailureMessage(), []string{dev})
-			tests.LogYunikornContainer(testDescription.FailureMessage())
-		}
+		tests.DumpClusterInfoIfSpecFailed(suiteName, []string{dev})
 		// call the healthCheck api to check scheduler health
 		ginkgo.By("Check Yunikorn's health")
 		checks, err := yunikorn.GetFailedHealthChecks()

test/e2e/bin_packing/bin_packing_suite_test.go

@@ -20,6 +20,7 @@ package bin_packing
 
 import (
 	"path/filepath"
+	"runtime"
 	"testing"
 
 	"github.com/onsi/ginkgo/v2"
@@ -53,11 +54,14 @@ func TestBinPacking(t *testing.T) {
 	ginkgo.RunSpecs(t, "TestBinPacking", ginkgo.Label("TestBinPacking"))
 }
 
+var suiteName string
 var oldConfigMap = new(v1.ConfigMap)
 var annotation = "ann-" + common.RandSeq(10)
 var kClient = k8s.KubeCtl{} //nolint
 
 var _ = BeforeSuite(func() {
+	_, filename, _, _ := runtime.Caller(0)
+	suiteName = common.GetSuiteName(filename)
 
 	Ω(kClient.SetClient()).To(BeNil())
 	/* Sample configMap. Post-update, Yunikorn will use binpacking node sort and fair app sort

test/e2e/bin_packing/bin_packing_test.go

@@ -23,7 +23,6 @@ import (
 	"sort"
 	"time"
 
-	"github.com/onsi/ginkgo/v2"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -193,11 +192,7 @@ var _ = Describe("", func() {
 	})
 
 	AfterEach(func() {
-		testDescription := ginkgo.CurrentSpecReport()
-		if testDescription.Failed() {
-			tests.LogTestClusterInfoWrapper(testDescription.FailureMessage(), []string{ns})
-			tests.LogYunikornContainer(testDescription.FailureMessage())
-		}
+		tests.DumpClusterInfoIfSpecFailed(suiteName, []string{ns})
 		By("Tear down namespace: " + ns)
 		err := kClient.TearDownNamespace(ns)
 		Ω(err).NotTo(HaveOccurred())

test/e2e/framework/configmanager/constants.go

@@ -18,16 +18,11 @@
 
 package configmanager
 
-import (
-	"os"
-)
-
 const (
 	TestResultsPath = "test_results/"
 
-	// LogPerm is the permission for files that are created by this framework
-	// that contain logs, outputs etc
-	LogPerm = os.FileMode(0666)
+	// LogPath is the path to store the dumped log files and should be equal to the artifact path set in pre-commit.yaml.
+	LogPath = "build/e2e/"
 
 	YKScheduler          = "yunikorn-scheduler"
 	YKSchedulerContainer = "yunikorn-scheduler-k8s"
@@ -50,6 +45,7 @@ const (
 	GroupUsagePath    = "ws/v1/partition/%s/usage/group/%s"
 	HealthCheckPath   = "ws/v1/scheduler/healthcheck"
 	ValidateConfPath  = "ws/v1/validate-conf"
+	FullStateDumpPath = "ws/v1/fullstatedump"
 
 	// YuniKorn Service Details
 	DefaultYuniKornHost   = "localhost"

test/e2e/framework/helpers/common/utils.go

@@ -89,20 +89,6 @@ func CreateJUnitReportDir() error {
 	return err
 }
 
-// CreateLogFile creates the ReportDirectory if it is not present, writes the
-// given testdata to the given filename.
-func CreateLogFile(filename string, data []byte) error {
-	path, err := CreateReportDirectory()
-	if err != nil {
-		fmt.Fprintf(ginkgo.GinkgoWriter, "ReportDirectory cannot be created: %v\n", err)
-		return err
-	}
-
-	finalPath := filepath.Join(path, filename)
-	err = os.WriteFile(finalPath, data, configmanager.LogPerm)
-	return err
-}
-
 func GetFileContents(filename string) ([]byte, error) {
 	data, err := os.ReadFile(filename)
 	return data, err
@@ -254,3 +240,53 @@ func RunShellCmdForeground(cmdStr string) (string, error) {
 
 	return stdOutStream.String(), nil
 }
+
+func CreateLogFile(suiteName string, specName string, logType string, extension string) (*os.File, error) {
+	filePath, err := getLogFilePath(suiteName, specName, logType, extension)
+	if err != nil {
+		return nil, err
+	}
+
+	dir := filepath.Dir(filePath)
+	if _, err = os.Stat(dir); os.IsNotExist(err) {
+		err = os.MkdirAll(dir, 0755)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	file, err := os.Create(filePath)
+	if err != nil {
+		return nil, err
+	}
+	fmt.Fprintf(ginkgo.GinkgoWriter, "Created log file: %s\n", filePath)
+	return file, nil
+}
+
+func getLogFilePath(suiteName string, specName string, logType string, extension string) (string, error) {
+	gitRoot, runErr := RunShellCmdForeground("git rev-parse --show-toplevel")
+	if runErr != nil {
+		return "", runErr
+	}
+	gitRoot = strings.TrimSpace(gitRoot)
+	suiteName = replaceInvalidFileChars(strings.TrimSpace(suiteName))
+	specName = replaceInvalidFileChars(strings.TrimSpace(specName))
+
+	dumpLogFilePath := filepath.Join(gitRoot, configmanager.LogPath, suiteName, fmt.Sprintf("%s_%s.%s", specName, logType, extension))
+	return dumpLogFilePath, nil
+}
+
+func replaceInvalidFileChars(str string) string {
+	// some charaters are not allowed in upload-artifact : https://github.com/actions/upload-artifact/issues/333
+	invalidChars := []string{"\"", ":", "<", ">", "|", "*", "?", "\r", "\n"}
+	for _, char := range invalidChars {
+		str = strings.ReplaceAll(str, char, "_")
+	}
+	return str
+}
+
+func GetSuiteName(testFilePath string) string {
+	dir := filepath.Dir(testFilePath)
+	suiteName := filepath.Base(dir)
+	return suiteName
+}

test/e2e/framework/helpers/k8s/k8s_utils.go

@@ -631,14 +631,47 @@ func GetConfigMapObj(yamlPath string) (*v1.ConfigMap, error) {
 	return c.(*v1.ConfigMap), err
 }
 
-func LogNamespaceInfo(ns string) error {
-	fmt.Fprintf(ginkgo.GinkgoWriter, "Log namespace info from %s\n", ns)
+func (k *KubeCtl) LogNamespaceInfo(file *os.File, ns string) error {
+	fmt.Fprintf(file, "Log namespace info, ns: %s\n", ns)
 	cmd := fmt.Sprintf("kubectl cluster-info dump --namespaces=%s", ns)
 	out, runErr := common.RunShellCmdForeground(cmd)
 	if runErr != nil {
 		return runErr
 	}
-	ginkgo.By("Cluster dump output:\n" + out)
+	_, err := fmt.Fprintln(file, out)
+	return err
+}
+
+func (k *KubeCtl) LogPodsInfo(file *os.File) error {
+	fmt.Fprintln(file, "Log pods info:")
+	pods, err := k.GetPodsByOptions(metav1.ListOptions{})
+	if err != nil {
+		return err
+	} else {
+		fmt.Fprintf(file, "Pod count is %d\n", len(pods.Items))
+		for _, pod := range pods.Items {
+			fmt.Fprintf(file, "Pod name is %s\n", pod.Name)
+			fmt.Fprintf(file, "Pod details: %s\n", pod.String())
+		}
+	}
+	return nil
+}
+
+func (k *KubeCtl) LogNodesInfo(file *os.File) error {
+	fmt.Fprintln(file, "Log nodes info:")
+	nodes, err := k.GetNodes()
+	if err != nil {
+		return err
+	}
+	fmt.Fprintf(file, "Node count is %d\n", len(nodes.Items))
+	for _, node := range nodes.Items {
+		fmt.Fprintf(file, "Node: %s\n", node.Name)
+		nodeInfo, err := k.DescribeNode(node)
+		if err != nil {
+			fmt.Fprintf(file, "Failed to describe node: %s, err: %v\n", node.Name, err)
+		}
+		fmt.Fprintln(file, nodeInfo)
+	}
 	return nil
 }
 
@@ -1443,15 +1476,13 @@ func (k *KubeCtl) GetNodesAvailRes(nodes v1.NodeList) map[string]v1.ResourceList
 	return nodeAvailRes
 }
 
-// DescribeNode Describe Node
-func (k *KubeCtl) DescribeNode(node v1.Node) error {
+func (k *KubeCtl) DescribeNode(node v1.Node) (string, error) {
 	cmd := "kubectl describe node " + node.Name
 	out, runErr := common.RunShellCmdForeground(cmd)
 	if runErr != nil {
-		return runErr
+		return "", runErr
 	}
-	ginkgo.By("describe output for node is:\n" + out)
-	return nil
+	return out, nil
 }
 
 func (k *KubeCtl) SetNodeLabel(name, key, value string) error {

test/e2e/framework/helpers/yunikorn/rest_api_utils.go

@@ -87,6 +87,19 @@ func (c *RClient) do(req *http.Request, v interface{}) (*http.Response, error) {
 	return resp, err
 }
 
+func (c *RClient) getBody(req *http.Request) (string, error) {
+	resp, err := c.httpClient.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return "", err
+	}
+	return string(body), nil
+}
+
 func (c *RClient) GetQueues(partition string) (*dao.PartitionQueueDAOInfo, error) {
 	req, err := c.newRequest("GET", fmt.Sprintf(configmanager.QueuesPath, partition), nil)
 	if err != nil {
@@ -214,6 +227,16 @@ func (c *RClient) GetAllocationLog(partition string, queueName string, appID str
 	return nil, errors.New("allocation is empty")
 }
 
+func (c *RClient) GetFullStateDump() (string, error) {
+	req, err := c.newRequest("GET", configmanager.FullStateDumpPath, nil)
+	if err != nil {
+		return "", err
+	}
+
+	fullStateDump, err := c.getBody(req)
+	return fullStateDump, err
+}
+
 func (c *RClient) isAllocLogPresent(partition string, queueName string, appID string, podName string) wait.ConditionFunc {
 	return func() (bool, error) {
 		log, err := c.GetAllocationLog(partition, queueName, appID, podName)

test/e2e/gang_scheduling/gang_scheduling_suite_test.go

@@ -20,6 +20,7 @@ package gangscheduling_test
 
 import (
 	"path/filepath"
+	"runtime"
 	"testing"
 
 	"github.com/onsi/ginkgo/v2"
@@ -52,11 +53,14 @@ func TestGangScheduling(t *testing.T) {
 	ginkgo.RunSpecs(t, "TestGangScheduling", ginkgo.Label("TestGangScheduling"))
 }
 
+var suiteName string
 var oldConfigMap = new(v1.ConfigMap)
 var annotation = "ann-" + common.RandSeq(10)
 var kClient = k8s.KubeCtl{} //nolint
 
 var _ = BeforeSuite(func() {
+	_, filename, _, _ := runtime.Caller(0)
+	suiteName = common.GetSuiteName(filename)
 	annotation = "ann-" + common.RandSeq(10)
 	yunikorn.EnsureYuniKornConfigsPresent()
 	yunikorn.UpdateConfigMapWrapper(oldConfigMap, "fifo", annotation)

test/e2e/gang_scheduling/gang_scheduling_test.go

@@ -571,11 +571,7 @@ var _ = Describe("", func() {
 	})
 
 	AfterEach(func() {
-		testDescription := ginkgo.CurrentSpecReport()
-		if testDescription.Failed() {
-			tests.LogTestClusterInfoWrapper(testDescription.FailureMessage(), []string{ns})
-			tests.LogYunikornContainer(testDescription.FailureMessage())
-		}
+		tests.DumpClusterInfoIfSpecFailed(suiteName, []string{ns})
 
 		By(fmt.Sprintf("Cleanup jobs: %v", jobNames))
 		for _, jobName := range jobNames {