Skip to content

Commit 7f2dbf7

Browse files
abhijeet-dhumalopenshift-merge-bot[bot]
abhijeet-dhumal
authored and
openshift-merge-bot[bot]
committed
Added modelName parameter to be considered for testing finetune demo
1 parent 2a0386c commit 7f2dbf7

File tree

5 files changed

+41
-90
lines changed

5 files changed

+41
-90
lines changed

‎tests/odh/mnist_ray_test.go‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -131,7 +131,6 @@ func mnistRay(t *testing.T, numGpus int) {
131131

132132
// Fetch created raycluster
133133
rayClusterName := "mnisttest"
134-
// Wait until raycluster is up and running
135134
rayCluster, err := test.Client().Ray().RayV1().RayClusters(namespace.Name).Get(test.Ctx(), rayClusterName, metav1.GetOptions{})
136135
test.Expect(err).ToNot(HaveOccurred())
137136

‎tests/odh/mnist_raytune_hpo_test.go‎

Lines changed: 0 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -129,53 +129,12 @@ func mnistRayTuneHpo(t *testing.T, numGpus int) {
129129
ContainElement(WithTransform(KueueWorkloadAdmitted, BeTrueBecause("Workload failed to be admitted"))),
130130
),
131131
)
132-
time.Sleep(30 * time.Second)
133132

134133
// Fetch created raycluster
135134
rayClusterName := "mnisthpotest"
136135
rayCluster, err := test.Client().Ray().RayV1().RayClusters(namespace.Name).Get(test.Ctx(), rayClusterName, metav1.GetOptions{})
137136
test.Expect(err).ToNot(HaveOccurred())
138137

139-
// Initialise raycluster client to interact with raycluster to get rayjob details using REST-API
140-
dashboardUrl := GetDashboardUrl(test, namespace, rayCluster)
141-
rayClusterClientConfig := RayClusterClientConfig{Address: dashboardUrl.String(), Client: nil, InsecureSkipVerify: true}
142-
rayClient, err := NewRayClusterClient(rayClusterClientConfig, test.Config().BearerToken)
143-
if err != nil {
144-
test.T().Errorf("%s", err)
145-
}
146-
147-
jobID := GetTestJobId(test, rayClient, dashboardUrl.Host)
148-
test.Expect(jobID).ToNot(Equal(nil))
149-
150-
// Wait for the job to be succeeded or failed
151-
var rayJobStatus string
152-
fmt.Printf("Waiting for job to be Succeeded...\n")
153-
test.Eventually(func() string {
154-
resp, err := rayClient.GetJobDetails(jobID)
155-
test.Expect(err).ToNot(HaveOccurred())
156-
rayJobStatusVal := resp.Status
157-
if rayJobStatusVal == "SUCCEEDED" || rayJobStatusVal == "FAILED" {
158-
fmt.Printf("JobStatus : %s\n", rayJobStatusVal)
159-
rayJobStatus = rayJobStatusVal
160-
return rayJobStatus
161-
}
162-
if rayJobStatus != rayJobStatusVal && rayJobStatusVal != "SUCCEEDED" {
163-
fmt.Printf("JobStatus : %s...\n", rayJobStatusVal)
164-
rayJobStatus = rayJobStatusVal
165-
}
166-
return rayJobStatus
167-
}, TestTimeoutDouble, 3*time.Second).Should(Or(Equal("SUCCEEDED"), Equal("FAILED")), "Job did not complete within the expected time")
168-
test.Expect(rayJobStatus).To(Equal("SUCCEEDED"), "RayJob failed !")
169-
170-
// Store job logs in output directory
171-
WriteRayJobAPILogs(test, rayClient, jobID)
172-
173-
// Fetch created raycluster
174-
rayClusterName := "mnisthpotest"
175-
// Wait until raycluster is up and running
176-
rayCluster, err := test.Client().Ray().RayV1().RayClusters(namespace.Name).Get(test.Ctx(), rayClusterName, metav1.GetOptions{})
177-
test.Expect(err).ToNot(HaveOccurred())
178-
179138
// Initialise raycluster client to interact with raycluster to get rayjob details using REST-API
180139
dashboardUrl := GetDashboardUrl(test, namespace, rayCluster)
181140
rayClusterClientConfig := RayClusterClientConfig{Address: dashboardUrl.String(), Client: nil, InsecureSkipVerify: true}

‎tests/odh/notebook.go‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,6 @@ import (
2929
"k8s.io/apimachinery/pkg/util/yaml"
3030
)
3131

32-
const recommendedTagAnnotation = "opendatahub.io/workbench-image-recommended"
33-
3432
var notebookResource = schema.GroupVersionResource{Group: "kubeflow.org", Version: "v1", Resource: "notebooks"}
3533

3634
type NotebookProps struct {

‎tests/odh/ray_finetune_llm_deepspeed_test.go‎

Lines changed: 38 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -30,13 +30,13 @@ import (
3030
)
3131

3232
func TestRayFinetuneLlmDeepspeedDemoLlama_2_7b(t *testing.T) {
33-
rayFinetuneLlmDeepspeed(t, 1, "zero_3_llama_2_7b.json")
33+
rayFinetuneLlmDeepspeed(t, 1, "meta-llama/Llama-2-7b-chat-hf", "zero_3_llama_2_7b.json")
3434
}
3535
func TestRayFinetuneLlmDeepspeedDemoLlama_31_8b(t *testing.T) {
36-
rayFinetuneLlmDeepspeed(t, 1, "zero_3_offload_optim_param.json")
36+
rayFinetuneLlmDeepspeed(t, 1, "meta-llama/Meta-Llama-3.1-8B", "zero_3_offload_optim_param.json")
3737
}
3838

39-
func rayFinetuneLlmDeepspeed(t *testing.T, numGpus int, modelConfigFile string) {
39+
func rayFinetuneLlmDeepspeed(t *testing.T, numGpus int, modelName string, modelConfigFile string) {
4040
test := With(t)
4141

4242
// Create a namespace
@@ -56,21 +56,22 @@ func rayFinetuneLlmDeepspeed(t *testing.T, numGpus int, modelConfigFile string)
5656
"import os": "import os,time,sys",
5757
"import sys": "!cp /opt/app-root/notebooks/* ./\\n\",\n\t\"!ls",
5858
"from codeflare_sdk.cluster.auth import TokenAuthentication": "from codeflare_sdk.cluster.auth import TokenAuthentication\\n\",\n\t\"from codeflare_sdk.job import RayJobClient",
59-
"token = ''": fmt.Sprintf("token = '%s'", userToken),
60-
"server = ''": fmt.Sprintf("server = '%s'", GetOpenShiftApiUrl(test)),
61-
"namespace='ray-finetune-llm-deepspeed'": fmt.Sprintf("namespace='%s'", namespace.Name),
62-
"head_cpus=16": "head_cpus=2",
63-
"head_extended_resource_requests=1": "head_extended_resource_requests=0",
64-
"num_workers=7": "num_workers=1",
65-
"worker_cpu_requests=16": "worker_cpu_requests=4",
66-
"worker_cpu_limits=16": "worker_cpu_limits=4",
67-
"worker_memory_requests=128": "worker_memory_requests=64",
68-
"worker_memory_limits=256": "worker_memory_limits=128",
69-
"head_memory=128": "head_memory=48",
70-
"client = cluster.job_client": "ray_dashboard = cluster.cluster_dashboard_uri()\\n\",\n\t\"header = {\\\"Authorization\\\": \\\"Bearer " + userToken + "\\\"}\\n\",\n\t\"client = RayJobClient(address=ray_dashboard, headers=header, verify=False)\\n",
71-
"--num-devices=8": fmt.Sprintf("--num-devices=%d", numGpus),
72-
"--num-epochs=3": fmt.Sprintf("--num-epochs=%d", 1),
73-
"--ds-config=./deepspeed_configs/zero_3_offload_optim+param.json": fmt.Sprintf("--ds-config=./%s \\\"\\n\",\n\t\" \\\"--lora-config=./lora.json \\\"\\n\",\n\t\" \\\"--as-test", modelConfigFile),
59+
"token = ''": fmt.Sprintf("token = '%s'", userToken),
60+
"server = ''": fmt.Sprintf("server = '%s'", GetOpenShiftApiUrl(test)),
61+
"namespace='ray-finetune-llm-deepspeed'": fmt.Sprintf("namespace='%s'", namespace.Name),
62+
"head_cpus=16": "head_cpus=2",
63+
"head_extended_resource_requests=1": "head_extended_resource_requests=0",
64+
"num_workers=7": "num_workers=1",
65+
"worker_cpu_requests=16": "worker_cpu_requests=4",
66+
"worker_cpu_limits=16": "worker_cpu_limits=4",
67+
"worker_memory_requests=128": "worker_memory_requests=64",
68+
"worker_memory_limits=256": "worker_memory_limits=128",
69+
"head_memory=128": "head_memory=48",
70+
"client = cluster.job_client": "ray_dashboard = cluster.cluster_dashboard_uri()\\n\",\n\t\"header = {\\\"Authorization\\\": \\\"Bearer " + userToken + "\\\"}\\n\",\n\t\"client = RayJobClient(address=ray_dashboard, headers=header, verify=False)\\n",
71+
"--num-devices=8": fmt.Sprintf("--num-devices=%d", numGpus),
72+
"--num-epochs=3": fmt.Sprintf("--num-epochs=%d", 1),
73+
"--model-name=meta-llama/Meta-Llama-3.1-8B": fmt.Sprintf("--model-name=%s", modelName),
74+
"--ds-config=./deepspeed_configs/zero_3_offload_optim_param.json": fmt.Sprintf("--ds-config=./%s \\\"\\n\",\n\t\" \\\"--lora-config=./lora.json \\\"\\n\",\n\t\" \\\"--as-test", modelConfigFile),
7475
"--batch-size-per-device=32": "--batch-size-per-device=6",
7576
"--eval-batch-size-per-device=32": "--eval-batch-size-per-device=6",
7677
"'pip': 'requirements.txt'": "'pip': '/opt/app-root/src/requirements.txt'",
@@ -83,7 +84,6 @@ func rayFinetuneLlmDeepspeed(t *testing.T, numGpus int, modelConfigFile string)
8384
updatedNotebookContent = strings.Replace(updatedNotebookContent, oldValue, newValue, -1)
8485
}
8586
updatedNotebook := []byte(updatedNotebookContent)
86-
os.WriteFile("demo.ipynb", updatedNotebook, 0644)
8787

8888
// Test configuration
8989
jupyterNotebookConfigMapFileName := "ray_finetune_llm_deepspeed.ipynb"
@@ -117,8 +117,6 @@ func rayFinetuneLlmDeepspeed(t *testing.T, numGpus int, modelConfigFile string)
117117
),
118118
)
119119

120-
time.Sleep(30 * time.Second)
121-
122120
// Fetch created raycluster
123121
rayClusterName := "ray"
124122
rayCluster, err := test.Client().Ray().RayV1().RayClusters(namespace.Name).Get(test.Ctx(), rayClusterName, metav1.GetOptions{})
@@ -128,37 +126,44 @@ func rayFinetuneLlmDeepspeed(t *testing.T, numGpus int, modelConfigFile string)
128126
dashboardUrl := GetDashboardUrl(test, namespace, rayCluster)
129127
rayClusterClientConfig := RayClusterClientConfig{Address: dashboardUrl.String(), Client: nil, InsecureSkipVerify: true}
130128
rayClient, err := NewRayClusterClient(rayClusterClientConfig, test.Config().BearerToken)
131-
if err != nil {
132-
test.T().Errorf("%s", err)
133-
}
129+
test.Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Failed to create new raycluster client: %s", err))
134130

131+
// wait until rayjob exists
132+
test.Eventually(func() []RayJobDetailsResponse {
133+
rayJobs, err := rayClient.GetJobs()
134+
test.Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Failed to fetch ray-jobs : %s", err))
135+
return *rayJobs
136+
}, TestTimeoutMedium, 1*time.Second).Should(HaveLen(1), "Ray job not found")
137+
138+
// Get test job-id
135139
jobID := GetTestJobId(test, rayClient, dashboardUrl.Host)
136-
test.Expect(jobID).ToNot(Equal(nil))
140+
test.Expect(jobID).ToNot(BeEmpty())
137141

138142
// Wait for the job to be succeeded or failed
139143
var rayJobStatus string
140-
fmt.Printf("Waiting for job to be Succeeded...\n")
144+
test.T().Logf("Waiting for job to be Succeeded...\n")
141145
test.Eventually(func() string {
142146
resp, err := rayClient.GetJobDetails(jobID)
143-
test.Expect(err).ToNot(HaveOccurred())
147+
test.Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Failed to get job details :%s", err))
144148
rayJobStatusVal := resp.Status
145149
if rayJobStatusVal == "SUCCEEDED" || rayJobStatusVal == "FAILED" {
146-
fmt.Printf("JobStatus : %s\n", rayJobStatusVal)
150+
test.T().Logf("JobStatus - %s\n", rayJobStatusVal)
147151
rayJobStatus = rayJobStatusVal
148-
WriteRayJobAPILogs(test, rayClient, jobID)
149152
return rayJobStatus
150153
}
151154
if rayJobStatus != rayJobStatusVal && rayJobStatusVal != "SUCCEEDED" {
152-
fmt.Printf("JobStatus : %s...\n", rayJobStatusVal)
155+
test.T().Logf("JobStatus - %s...\n", rayJobStatusVal)
153156
rayJobStatus = rayJobStatusVal
154157
}
155158
return rayJobStatus
156-
}, TestTimeoutDouble, 3*time.Second).Should(Or(Equal("SUCCEEDED"), Equal("FAILED")), "Job did not complete within the expected time")
159+
}, TestTimeoutDouble, 1*time.Second).Should(Or(Equal("SUCCEEDED"), Equal("FAILED")), "Job did not complete within the expected time")
157160
// Store job logs in output directory
158161
WriteRayJobAPILogs(test, rayClient, jobID)
162+
163+
// Assert ray-job status after job execution
159164
test.Expect(rayJobStatus).To(Equal("SUCCEEDED"), "RayJob failed !")
160165

161166
// Make sure the RayCluster finishes and is deleted
162-
test.Eventually(RayClusters(test, namespace.Name), TestTimeoutMedium).
163-
Should(HaveLen(0))
167+
test.Eventually(RayClusters(test, namespace.Name), TestTimeoutLong).
168+
Should(BeEmpty())
164169
}

‎tests/odh/support.go‎

Lines changed: 3 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -18,14 +18,11 @@ package odh
1818

1919
import (
2020
"embed"
21-
"net/http"
2221
"net/url"
2322
"os"
2423

25-
. "github.com/onsi/gomega"
2624
gomega "github.com/onsi/gomega"
2725
"github.com/project-codeflare/codeflare-common/support"
28-
. "github.com/project-codeflare/codeflare-common/support"
2926
rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1"
3027
v1 "k8s.io/api/core/v1"
3128
)
@@ -49,24 +46,17 @@ func ReadFileExt(t support.Test, fileName string) []byte {
4946

5047
func GetDashboardUrl(test support.Test, namespace *v1.Namespace, rayCluster *rayv1.RayCluster) *url.URL {
5148
dashboardName := "ray-dashboard-" + rayCluster.Name
52-
test.T().Logf("Raycluster created : %s\n", rayCluster.Name)
53-
route := GetRoute(test, namespace.Name, dashboardName)
49+
route := support.GetRoute(test, namespace.Name, dashboardName)
5450
hostname := route.Status.Ingress[0].Host
5551
dashboardUrl, _ := url.Parse("https://" + hostname)
5652
test.T().Logf("Ray-dashboard route : %s\n", dashboardUrl.String())
5753

5854
return dashboardUrl
5955
}
6056

61-
func GetTestJobId(test Test, rayClient RayClusterClient, hostName string) string {
62-
listJobsReq, err := http.NewRequest("GET", "https://"+hostName+"/api/jobs/", nil)
63-
if err != nil {
64-
test.T().Errorf("failed to do get request: %s\n", err)
65-
}
66-
listJobsReq.Header.Add("Authorization", "Bearer "+test.Config().BearerToken)
67-
57+
func GetTestJobId(test support.Test, rayClient support.RayClusterClient, hostName string) string {
6858
allJobsData, err := rayClient.GetJobs()
69-
test.Expect(err).ToNot(HaveOccurred())
59+
test.Expect(err).ToNot(gomega.HaveOccurred())
7060

7161
jobID := (*allJobsData)[0].SubmissionID
7262
if len(*allJobsData) > 0 {

0 commit comments

Comments
 (0)