Implement upgrade test for Training operator

Test verifies that Training operator is able to monitor PyTorchJob created by previous version.

Author: sutaakar

.github/workflows/odh-release.yml

@@ -38,7 +38,8 @@ jobs:
 
     - name: Compile tests
       run: |
-        go test -c -o compiled-tests/kfto ./tests/kfto/
+        go test -c -o compiled-tests/kfto ./tests/kfto/core
+        go test -c -o compiled-tests/kfto-upgrade ./tests/kfto/upgrade
 
     - name: Creates a release in GitHub
       run: |

tests/kfto/config.json tests/kfto/core/config.jsontests/kfto/config.json renamed to tests/kfto/core/config.json

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package kfto
+package core
 
 import (
 	"os"
@@ -28,7 +28,7 @@ const (
 )
 
 func GetFmsHfTuningImage() string {
-	return lookupEnvOrDefault(fmsHfTuningImageEnvVar, "quay.io/modh/fms-hf-tuning:d0bd35b0297c28b87ee6caa32d5966d77587591f")
+	return lookupEnvOrDefault(fmsHfTuningImageEnvVar, "quay.io/modh/fms-hf-tuning:bd8bf628cd739c7a201a976bc3c1096785353f1a")
 }
 
 func GetBloomModelImage() string {

tests/kfto/environment.go tests/kfto/core/environment.gotests/kfto/environment.go renamed to tests/kfto/core/environment.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package kfto
+package core
 
 import (
 	"testing"
@@ -30,39 +30,6 @@ import (
 	kftov1 "github.com/kubeflow/training-operator/pkg/apis/kubeflow.org/v1"
 )
 
-func PytorchJob(t Test, namespace, name string) func(g Gomega) *kftov1.PyTorchJob {
-	return func(g Gomega) *kftov1.PyTorchJob {
-		job, err := t.Client().Kubeflow().KubeflowV1().PyTorchJobs(namespace).Get(t.Ctx(), name, metav1.GetOptions{})
-		g.Expect(err).NotTo(HaveOccurred())
-		return job
-	}
-}
-
-func PytorchJobConditionRunning(job *kftov1.PyTorchJob) corev1.ConditionStatus {
-	return PytorchJobCondition(job, kftov1.JobRunning)
-}
-
-func PytorchJobConditionSucceeded(job *kftov1.PyTorchJob) corev1.ConditionStatus {
-	return PytorchJobCondition(job, kftov1.JobSucceeded)
-}
-
-func PytorchJobConditionSuspended(job *kftov1.PyTorchJob) corev1.ConditionStatus {
-	return PytorchJobCondition(job, kftov1.JobSuspended)
-}
-
-func PytorchJobCondition(job *kftov1.PyTorchJob, conditionType kftov1.JobConditionType) corev1.ConditionStatus {
-	for _, condition := range job.Status.Conditions {
-		if condition.Type == conditionType {
-			return condition.Status
-		}
-	}
-	return corev1.ConditionUnknown
-}
-
-func OwnerReferenceName(meta metav1.Object) string {
-	return meta.GetOwnerReferences()[0].Name
-}
-
 func TestPytorchjobWithSFTtrainer(t *testing.T) {
 	test := With(t)
 
@@ -119,7 +86,7 @@ func TestPytorchjobWithSFTtrainer(t *testing.T) {
 		)
 
 	// Make sure the PyTorch job is running
-	test.Eventually(PytorchJob(test, namespace.Name, tuningJob.Name), TestTimeoutShort).
+	test.Eventually(PytorchJob(test, namespace.Name, tuningJob.Name), TestTimeoutLong).
 		Should(WithTransform(PytorchJobConditionRunning, Equal(corev1.ConditionTrue)))
 
 	// Make sure the PyTorch job succeed
@@ -174,7 +141,7 @@ func TestPytorchjobUsingKueueQuota(t *testing.T) {
 	tuningJob := createPyTorchJob(test, namespace.Name, localQueue.Name, *config)
 
 	// Make sure the PyTorch job is running
-	test.Eventually(PytorchJob(test, namespace.Name, tuningJob.Name), TestTimeoutShort).
+	test.Eventually(PytorchJob(test, namespace.Name, tuningJob.Name), TestTimeoutLong).
 		Should(WithTransform(PytorchJobConditionRunning, Equal(corev1.ConditionTrue)))
 
 	// Create second training PyTorch job

tests/kfto/kfto_kueue_sft_test.go tests/kfto/core/kfto_kueue_sft_test.gotests/kfto/kfto_kueue_sft_test.go renamed to tests/kfto/core/kfto_kueue_sft_test.go

@@ -0,0 +1,68 @@
+/*
+Copyright 2023.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package core
+
+import (
+	"embed"
+
+	. "github.com/onsi/gomega"
+	. "github.com/project-codeflare/codeflare-common/support"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	kftov1 "github.com/kubeflow/training-operator/pkg/apis/kubeflow.org/v1"
+)
+
+//go:embed *.json
+var files embed.FS
+
+func ReadFile(t Test, fileName string) []byte {
+	t.T().Helper()
+	file, err := files.ReadFile(fileName)
+	t.Expect(err).NotTo(HaveOccurred())
+	return file
+}
+
+func PytorchJob(t Test, namespace, name string) func(g Gomega) *kftov1.PyTorchJob {
+	return func(g Gomega) *kftov1.PyTorchJob {
+		job, err := t.Client().Kubeflow().KubeflowV1().PyTorchJobs(namespace).Get(t.Ctx(), name, metav1.GetOptions{})
+		g.Expect(err).NotTo(HaveOccurred())
+		return job
+	}
+}
+
+func PytorchJobConditionRunning(job *kftov1.PyTorchJob) corev1.ConditionStatus {
+	return PytorchJobCondition(job, kftov1.JobRunning)
+}
+
+func PytorchJobConditionSucceeded(job *kftov1.PyTorchJob) corev1.ConditionStatus {
+	return PytorchJobCondition(job, kftov1.JobSucceeded)
+}
+
+func PytorchJobConditionSuspended(job *kftov1.PyTorchJob) corev1.ConditionStatus {
+	return PytorchJobCondition(job, kftov1.JobSuspended)
+}
+
+func PytorchJobCondition(job *kftov1.PyTorchJob, conditionType kftov1.JobConditionType) corev1.ConditionStatus {
+	for _, condition := range job.Status.Conditions {
+		if condition.Type == conditionType {
+			return condition.Status
+		}
+	}
+	return corev1.ConditionUnknown
+}