test: Print Job logs after successfull or failed completion

Author: astefanutti

test/e2e/mnist_pytorch_mcad_job_test.go

@@ -25,7 +25,6 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/labels"
 
 	. "github.com/project-codeflare/codeflare-operator/test/support"
 	mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1"
@@ -179,22 +178,17 @@ torchvision==0.12.0
 	test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutMedium).
 		Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive)))
 
-	defer JobTroubleshooting(test, job)
-
-	test.T().Logf("Waiting for Job %s/%s to complete successfully", job.Namespace, job.Name)
-	test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).
-		Should(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue)))
-
-	// Refresh the job to get the generated pod selector
-	job = GetJob(test, job.Namespace, job.Name)
-
-	// Get the job Pod
-	pods := GetPods(test, job.Namespace, metav1.ListOptions{
-		LabelSelector: labels.FormatLabels(job.Spec.Selector.MatchLabels)},
-	)
-	test.Expect(pods).To(HaveLen(1))
+	test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name)
+	test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should(
+		Or(
+			WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue)),
+			WithTransform(ConditionStatus(batchv1.JobFailed), Equal(corev1.ConditionTrue)),
+		))
 
 	// Print the job logs
-	test.T().Logf("Printing Job %s/%s logs", job.Namespace, job.Name)
-	test.T().Log(GetPodLogs(test, &pods[0], corev1.PodLogOptions{}))
+	PrintJobLogs(test, job.Namespace, job.Name)
+
+	// Assert the job has completed successfully
+	test.Expect(GetJob(test, job.Namespace, job.Name)).
+		To(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue)))
 }

test/e2e/mnist_raycluster_sdk_test.go

@@ -25,7 +25,6 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/labels"
 
 	rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1"
 
@@ -187,29 +186,17 @@ func TestMNISTRayClusterSDK(t *testing.T) {
 	test.Expect(err).NotTo(HaveOccurred())
 	test.T().Logf("Created Job %s/%s successfully", job.Namespace, job.Name)
 
-	defer JobTroubleshooting(test, job)
-
 	test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name)
 	test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should(
 		Or(
 			WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue)),
 			WithTransform(ConditionStatus(batchv1.JobFailed), Equal(corev1.ConditionTrue)),
 		))
 
-	// Refresh the job to get the generated pod selector
-	job = GetJob(test, job.Namespace, job.Name)
-
-	// Get the job Pod
-	pods := GetPods(test, job.Namespace, metav1.ListOptions{
-		LabelSelector: labels.FormatLabels(job.Spec.Selector.MatchLabels)},
-	)
-	test.Expect(pods).To(HaveLen(1))
-
 	// Print the job logs
-	test.T().Logf("Printing Job %s/%s logs", job.Namespace, job.Name)
-	test.T().Log(GetPodLogs(test, &pods[0], corev1.PodLogOptions{}))
+	PrintJobLogs(test, job.Namespace, job.Name)
 
 	// Assert the job has completed successfully
-	test.T().Logf("Checking the Job %s/%s has completed successfully", job.Namespace, job.Name)
-	test.Expect(job).To(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue)))
+	test.Expect(GetJob(test, job.Namespace, job.Name)).
+		To(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue)))
 }

test/support/batch.go

@@ -38,24 +38,21 @@ func GetJob(t Test, namespace, name string) *batchv1.Job {
 	return Job(t, namespace, name)(t)
 }
 
-func JobTroubleshooting(test Test, job *batchv1.Job) {
-	if !test.T().Failed() {
-		return
-	}
-	job = GetJob(test, job.Namespace, job.Name)
+func PrintJobLogs(t Test, namespace, name string) {
+	t.T().Helper()
 
-	test.T().Errorf("Job %s/%s hasn't completed in time: %s", job.Namespace, job.Name, job)
+	job := GetJob(t, namespace, name)
 
-	pods := GetPods(test, job.Namespace, metav1.ListOptions{
+	pods := GetPods(t, job.Namespace, metav1.ListOptions{
 		LabelSelector: labels.FormatLabels(job.Spec.Selector.MatchLabels)},
 	)
 
 	if len(pods) == 0 {
-		test.T().Errorf("Job %s/%s has no pods scheduled", job.Namespace, job.Name)
+		t.T().Errorf("Job %s/%s has no pods scheduled", job.Namespace, job.Name)
 	} else {
 		for i, pod := range pods {
-			test.T().Logf("Printing Pod %s/%s logs", pod.Namespace, pod.Name)
-			test.T().Log(GetPodLogs(test, &pods[i], corev1.PodLogOptions{}))
+			t.T().Logf("Printing Pod %s/%s logs", pod.Namespace, pod.Name)
+			t.T().Log(GetPodLogs(t, &pods[i], corev1.PodLogOptions{}))
 		}
 	}
 }