test: Write logs also for jobs that have timed out

project-codeflare · Jul 5, 2023 · 8c1831a · 8c1831a
1 parent 4048a26
commit 8c1831a
Show file tree

Hide file tree

Showing 4 changed files with 14 additions and 9 deletions.
diff --git a/test/e2e/mnist_pytorch_mcad_job_test.go b/test/e2e/mnist_pytorch_mcad_job_test.go
@@ -145,16 +145,16 @@ func TestMNISTPyTorchMCAD(t *testing.T) {
 	test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutMedium).
 		Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive)))
 
+	// Retrieving the job logs once it has completed or timed out
+	defer WriteJobLogs(test, job.Namespace, job.Name)
+
 	test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name)
 	test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should(
 		Or(
 			WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue)),
 			WithTransform(ConditionStatus(batchv1.JobFailed), Equal(corev1.ConditionTrue)),
 		))
 
-	// Retrieve the job logs
-	WriteJobLogs(test, job.Namespace, job.Name)
-
 	// Assert the job has completed successfully
 	test.Expect(GetJob(test, job.Namespace, job.Name)).
 		To(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue)))

diff --git a/test/e2e/mnist_raycluster_sdk_test.go b/test/e2e/mnist_raycluster_sdk_test.go
@@ -194,16 +194,16 @@ func TestMNISTRayClusterSDK(t *testing.T) {
 	test.Expect(err).NotTo(HaveOccurred())
 	test.T().Logf("Created Job %s/%s successfully", job.Namespace, job.Name)
 
+	// Retrieving the job logs once it has completed or timed out
+	defer WriteJobLogs(test, job.Namespace, job.Name)
+
 	test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name)
 	test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should(
 		Or(
 			WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue)),
 			WithTransform(ConditionStatus(batchv1.JobFailed), Equal(corev1.ConditionTrue)),
 		))
 
-	// Retrieve the job logs
-	WriteJobLogs(test, job.Namespace, job.Name)
-
 	// Assert the job has completed successfully
 	test.Expect(GetJob(test, job.Namespace, job.Name)).
 		To(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue)))

diff --git a/test/e2e/mnist_rayjob_mcad_raycluster_test.go b/test/e2e/mnist_rayjob_mcad_raycluster_test.go
@@ -252,13 +252,13 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) {
 	test.Expect(err).NotTo(HaveOccurred())
 	test.T().Logf("Created RayJob %s/%s successfully", rayJob.Namespace, rayJob.Name)
 
+	// Retrieving the job logs once it has completed or timed out
+	defer WriteRayJobLogs(test, rayJob.Namespace, rayJob.Name)
+
 	test.T().Logf("Waiting for RayJob %s/%s to complete", rayJob.Namespace, rayJob.Name)
 	test.Eventually(RayJob(test, rayJob.Namespace, rayJob.Name), TestTimeoutLong).
 		Should(WithTransform(RayJobStatus, Satisfy(rayv1alpha1.IsJobTerminal)))
 
-	test.T().Logf("Retrieving RayJob %s/%s logs", rayJob.Namespace, rayJob.Name)
-	WriteToOutputDir(test, rayJob.Name, Log, GetRayJobLogs(test, rayJob.Namespace, rayJob.Name))
-
 	// Assert the Ray job has completed successfully
 	test.Expect(GetRayJob(test, rayJob.Namespace, rayJob.Name)).
 		To(WithTransform(RayJobStatus, Equal(rayv1alpha1.JobStatusSucceeded)))

diff --git a/test/support/ray.go b/test/support/ray.go
@@ -63,3 +63,8 @@ func GetRayJobLogs(t Test, namespace, name string) []byte {
 
 	return []byte(body["logs"])
 }
+
+func WriteRayJobLogs(t Test, namespace, name string) {
+	t.T().Logf("Retrieving RayJob %s/%s logs", namespace, name)
+	WriteToOutputDir(t, name, Log, GetRayJobLogs(t, namespace, name))
+}