From 8c1831ae50d2d56ad19e3d0b1c36b2b10c67b385 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Wed, 5 Jul 2023 14:30:52 +0200 Subject: [PATCH] test: Write logs also for jobs that have timed out --- test/e2e/mnist_pytorch_mcad_job_test.go | 6 +++--- test/e2e/mnist_raycluster_sdk_test.go | 6 +++--- test/e2e/mnist_rayjob_mcad_raycluster_test.go | 6 +++--- test/support/ray.go | 5 +++++ 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/test/e2e/mnist_pytorch_mcad_job_test.go b/test/e2e/mnist_pytorch_mcad_job_test.go index 41376a0b..168c7b13 100644 --- a/test/e2e/mnist_pytorch_mcad_job_test.go +++ b/test/e2e/mnist_pytorch_mcad_job_test.go @@ -145,6 +145,9 @@ func TestMNISTPyTorchMCAD(t *testing.T) { test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutMedium). Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) + // Retrieving the job logs once it has completed or timed out + defer WriteJobLogs(test, job.Namespace, job.Name) + test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name) test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should( Or( @@ -152,9 +155,6 @@ func TestMNISTPyTorchMCAD(t *testing.T) { WithTransform(ConditionStatus(batchv1.JobFailed), Equal(corev1.ConditionTrue)), )) - // Retrieve the job logs - WriteJobLogs(test, job.Namespace, job.Name) - // Assert the job has completed successfully test.Expect(GetJob(test, job.Namespace, job.Name)). To(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue))) diff --git a/test/e2e/mnist_raycluster_sdk_test.go b/test/e2e/mnist_raycluster_sdk_test.go index 202f3bcc..e8bb1f5c 100644 --- a/test/e2e/mnist_raycluster_sdk_test.go +++ b/test/e2e/mnist_raycluster_sdk_test.go @@ -194,6 +194,9 @@ func TestMNISTRayClusterSDK(t *testing.T) { test.Expect(err).NotTo(HaveOccurred()) test.T().Logf("Created Job %s/%s successfully", job.Namespace, job.Name) + // Retrieving the job logs once it has completed or timed out + defer WriteJobLogs(test, job.Namespace, job.Name) + test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name) test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should( Or( @@ -201,9 +204,6 @@ func TestMNISTRayClusterSDK(t *testing.T) { WithTransform(ConditionStatus(batchv1.JobFailed), Equal(corev1.ConditionTrue)), )) - // Retrieve the job logs - WriteJobLogs(test, job.Namespace, job.Name) - // Assert the job has completed successfully test.Expect(GetJob(test, job.Namespace, job.Name)). To(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue))) diff --git a/test/e2e/mnist_rayjob_mcad_raycluster_test.go b/test/e2e/mnist_rayjob_mcad_raycluster_test.go index b6b01653..958b3047 100644 --- a/test/e2e/mnist_rayjob_mcad_raycluster_test.go +++ b/test/e2e/mnist_rayjob_mcad_raycluster_test.go @@ -252,13 +252,13 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { test.Expect(err).NotTo(HaveOccurred()) test.T().Logf("Created RayJob %s/%s successfully", rayJob.Namespace, rayJob.Name) + // Retrieving the job logs once it has completed or timed out + defer WriteRayJobLogs(test, rayJob.Namespace, rayJob.Name) + test.T().Logf("Waiting for RayJob %s/%s to complete", rayJob.Namespace, rayJob.Name) test.Eventually(RayJob(test, rayJob.Namespace, rayJob.Name), TestTimeoutLong). Should(WithTransform(RayJobStatus, Satisfy(rayv1alpha1.IsJobTerminal))) - test.T().Logf("Retrieving RayJob %s/%s logs", rayJob.Namespace, rayJob.Name) - WriteToOutputDir(test, rayJob.Name, Log, GetRayJobLogs(test, rayJob.Namespace, rayJob.Name)) - // Assert the Ray job has completed successfully test.Expect(GetRayJob(test, rayJob.Namespace, rayJob.Name)). To(WithTransform(RayJobStatus, Equal(rayv1alpha1.JobStatusSucceeded))) diff --git a/test/support/ray.go b/test/support/ray.go index 3833d43b..cd5a9d87 100644 --- a/test/support/ray.go +++ b/test/support/ray.go @@ -63,3 +63,8 @@ func GetRayJobLogs(t Test, namespace, name string) []byte { return []byte(body["logs"]) } + +func WriteRayJobLogs(t Test, namespace, name string) { + t.T().Logf("Retrieving RayJob %s/%s logs", namespace, name) + WriteToOutputDir(t, name, Log, GetRayJobLogs(t, namespace, name)) +}