Skip to content

Commit

Permalink
Convert MNIST PyTorch and MNIST Ray test cases to go
Browse files Browse the repository at this point in the history
  • Loading branch information
sutaakar committed Sep 11, 2023
1 parent dcc99ce commit ac95a82
Show file tree
Hide file tree
Showing 15 changed files with 922 additions and 17 deletions.
6 changes: 3 additions & 3 deletions tests/go.mod
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
module github.com/opendatahub-io/distributed-workloads/tests/new-tests
module github.com/opendatahub-io/distributed-workloads/tests

require (
github.com/onsi/gomega v1.27.10
github.com/openshift/api v0.0.0-20230718161610-2a3e8b481cec
github.com/project-codeflare/codeflare-operator v0.2.2
github.com/project-codeflare/codeflare-operator v0.2.3-0.20230906080827-4ae9e8f9e640
github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.1-0.20230831150053-05b1af83014c
github.com/ray-project/kuberay/ray-operator v0.0.0-20230830082034-e7fbf7d73576
k8s.io/api v0.27.2
k8s.io/apimachinery v0.27.2
Expand All @@ -30,7 +31,6 @@ require (
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/openshift/client-go v0.0.0-20230718165156-6014fb98e86a // indirect
github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/stretchr/testify v1.8.4 // indirect
golang.org/x/net v0.12.0 // indirect
Expand Down
8 changes: 4 additions & 4 deletions tests/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -107,10 +107,10 @@ github.com/openshift/client-go v0.0.0-20230718165156-6014fb98e86a h1:ZKewwwEIURD
github.com/openshift/client-go v0.0.0-20230718165156-6014fb98e86a/go.mod h1:EjhPQjEm8HM3GThz5ywNGLEec1P1IjTn08kwzdvupvA=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/project-codeflare/codeflare-operator v0.2.2 h1:7Vf1Hv+q0muzEGbTzPBdKjXOd903NbMu1bsdL02PCr0=
github.com/project-codeflare/codeflare-operator v0.2.2/go.mod h1:2oYGL1iF92OzEhw+FRd7S4kFAEF9dZZnDCJLsuYZ4r0=
github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.0 h1:4J3vW+vuCBHyocd3huZy3FJhOw5acaBHFLYwutkTE/E=
github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.0/go.mod h1:gtTl8Tsl+X+bGhqVudLoveINR6IkN+sVvH0J+VZIP40=
github.com/project-codeflare/codeflare-operator v0.2.3-0.20230906080827-4ae9e8f9e640 h1:D9oeAMirZcWTNzO7G8IDXy2IEKLnH8EBXjgiaufzGx8=
github.com/project-codeflare/codeflare-operator v0.2.3-0.20230906080827-4ae9e8f9e640/go.mod h1:q+Y/3BS/N3MDzljiD7XRaUae3yZvzzXvhegMJjAaasE=
github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.1-0.20230831150053-05b1af83014c h1:LjCgRcjtkd9byePczvxGBLjdM22M2BCTB76FoEk2fkU=
github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.1-0.20230831150053-05b1af83014c/go.mod h1:gtTl8Tsl+X+bGhqVudLoveINR6IkN+sVvH0J+VZIP40=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/ray-project/kuberay/ray-operator v0.0.0-20230830082034-e7fbf7d73576 h1:NYt+ipGo+jMLYCFI5S04DzVFSXn8WCR0m6m3bFXPk+4=
github.com/ray-project/kuberay/ray-operator v0.0.0-20230830082034-e7fbf7d73576/go.mod h1:7PBzsR4L0b/f5tTxq1XGnzvudbrMyPadiLiNiP9pOl0=
Expand Down
99 changes: 99 additions & 0 deletions tests/integration/mcad_ray_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
/*
Copyright 2023.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
"testing"

. "github.com/onsi/gomega"
cfosupport "github.com/project-codeflare/codeflare-operator/test/support"
mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1"
rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1"

corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

support "github.com/opendatahub-io/distributed-workloads/tests/integration/support"
)

func TestMCADRay(t *testing.T) {
test := cfosupport.With(t)

// Create a namespace
namespace := test.NewTestNamespace()

// Test configuration
jupyterNotebookConfigMapFileName := "mnist_ray_mini.ipynb"
config := &corev1.ConfigMap{
TypeMeta: metav1.TypeMeta{
APIVersion: corev1.SchemeGroupVersion.String(),
Kind: "ConfigMap",
},
ObjectMeta: metav1.ObjectMeta{
Name: "notebooks-ray",
},
BinaryData: map[string][]byte{
// MNIST MCAD Notebook
jupyterNotebookConfigMapFileName: ReadFile(test, "resources/mnist_ray_mini.ipynb"),
"mnist.py": ReadFile(test, "resources/mnist.py"),
"requirements.txt": ReadFile(test, "resources/requirements.txt"),
},
Immutable: cfosupport.Ptr(true),
}
config, err := test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), config, metav1.CreateOptions{})
test.Expect(err).NotTo(HaveOccurred())
test.T().Logf("Created ConfigMap %s/%s successfully", config.Namespace, config.Name)

// Create RBAC, retrieve token for user with limited rights
policyRules := []rbacv1.PolicyRule{
{
Verbs: []string{"get", "create", "delete", "list", "patch", "update"},
// APIGroups: []string{mcadv1beta1.GroupName},
APIGroups: []string{"mcad.ibm.com"},
Resources: []string{"appwrappers"},
},
{
Verbs: []string{"get", "list"},
APIGroups: []string{rayv1alpha1.GroupVersion.Group},
Resources: []string{"rayclusters", "rayclusters/status"},
},
{
Verbs: []string{"get", "list"},
APIGroups: []string{"route.openshift.io"},
Resources: []string{"routes"},
},
}
token := support.CreateTestRBAC(test, namespace, policyRules)

// Create Notebook CR
support.CreateNotebook(test, namespace, token, config.Name, jupyterNotebookConfigMapFileName)

// Make sure the AppWrapper is created and running
test.Eventually(cfosupport.AppWrappers(test, namespace), cfosupport.TestTimeoutLong).
Should(
And(
HaveLen(1),
ContainElement(WithTransform(cfosupport.AppWrapperName, HavePrefix("mnistjob"))),
ContainElement(WithTransform(cfosupport.AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))),
),
)

// Make sure the AppWrapper finishes and is deleted
test.Eventually(cfosupport.AppWrappers(test, namespace), cfosupport.TestTimeoutLong).
Should(HaveLen(0))
}
92 changes: 92 additions & 0 deletions tests/integration/pytorch_mcad_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
/*
Copyright 2023.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
"testing"

. "github.com/onsi/gomega"
cfosupport "github.com/project-codeflare/codeflare-operator/test/support"
mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1"

corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

support "github.com/opendatahub-io/distributed-workloads/tests/integration/support"
)

func TestMnistPyTorchMCAD(t *testing.T) {
test := cfosupport.With(t)

// Create a namespace
namespace := test.NewTestNamespace()

// Test configuration
jupyterNotebookConfigMapFileName := "mnist_mcad_mini.ipynb"
config := &corev1.ConfigMap{
TypeMeta: metav1.TypeMeta{
APIVersion: corev1.SchemeGroupVersion.String(),
Kind: "ConfigMap",
},
ObjectMeta: metav1.ObjectMeta{
Name: "notebooks-mcad",
},
BinaryData: map[string][]byte{
// MNIST MCAD Notebook
jupyterNotebookConfigMapFileName: ReadFile(test, "resources/mnist_mcad_mini.ipynb"),
},
Immutable: cfosupport.Ptr(true),
}
config, err := test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), config, metav1.CreateOptions{})
test.Expect(err).NotTo(HaveOccurred())
test.T().Logf("Created ConfigMap %s/%s successfully", config.Namespace, config.Name)

// Create RBAC, retrieve token for user with limited rights
policyRules := []rbacv1.PolicyRule{
{
Verbs: []string{"get", "create", "delete", "list", "patch", "update"},
// APIGroups: []string{mcadv1beta1.GroupName},
APIGroups: []string{"mcad.ibm.com"},
Resources: []string{"appwrappers"},
},
// Needed for job.logs()
{
Verbs: []string{"get"},
APIGroups: []string{""},
Resources: []string{"pods/log"},
},
}
token := support.CreateTestRBAC(test, namespace, policyRules)

// Create Notebook CR
support.CreateNotebook(test, namespace, token, config.Name, jupyterNotebookConfigMapFileName)

// Make sure the AppWrapper is created and running
test.Eventually(cfosupport.AppWrappers(test, namespace), cfosupport.TestTimeoutLong).
Should(
And(
HaveLen(1),
ContainElement(WithTransform(cfosupport.AppWrapperName, HavePrefix("mnistjob"))),
ContainElement(WithTransform(cfosupport.AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))),
),
)

// Make sure the AppWrapper finishes and is deleted
test.Eventually(cfosupport.AppWrappers(test, namespace), cfosupport.TestTimeoutLong).
Should(HaveLen(0))
}
2 changes: 1 addition & 1 deletion tests/integration/resources/mnist.py
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,7 @@ def test_dataloader(self):
trainer = Trainer(
accelerator="auto",
# devices=1 if torch.cuda.is_available() else None, # limiting got iPython runs
max_epochs=5,
max_epochs=2,
callbacks=[TQDMProgressBar(refresh_rate=20)],
num_nodes=int(os.environ.get("GROUP_WORLD_SIZE", 1)),
devices=int(os.environ.get("LOCAL_WORLD_SIZE", 1)),
Expand Down
93 changes: 93 additions & 0 deletions tests/integration/resources/mnist_mcad_mini.ipynb
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"id": "b55bc3ea-4ce3-49bf-bb1f-e209de8ca47a",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# Import pieces from codeflare-sdk\n",
"from codeflare_sdk.job.jobs import DDPJobDefinition\n",
"from time import sleep"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "47ca5c15",
"metadata": {
"tags": ["parameters"]
},
"outputs": [],
"source": [
"#parameters\n",
"namespace = \"default\""
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "26b21373",
"metadata": {},
"outputs": [],
"source": [
"job = DDPJobDefinition(name=\"mnistjob\", script=\"mnist.py\", scheduler_args={\"namespace\": namespace}, j=\"1x1\", gpu=0, cpu=1, memMB=2000, image=\"quay.io/project-codeflare/mnist-job-test:v0.0.1\").submit()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d24e9f95",
"metadata": {},
"outputs": [],
"source": [
"finished = False\n",
"while not finished:\n",
" sleep(1)\n",
" try:\n",
" finished = (\"Epoch 4: 100%\" in job.logs())\n",
" except:\n",
" finished = False"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "f078b7cd",
"metadata": {},
"outputs": [],
"source": [
"job.cancel()"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.13"
},
"vscode": {
"interpreter": {
"hash": "f9f85f796d01129d0dd105a088854619f454435301f6ffec2fea96ecbd9be4ac"
}
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Loading

0 comments on commit ac95a82

Please sign in to comment.