Convert MNIST PyTorch test case to go

tests/go.mod

@@ -4,6 +4,7 @@ require (
 	github.com/onsi/gomega v1.27.10
 	github.com/openshift/api v0.0.0-20230718161610-2a3e8b481cec
 	github.com/project-codeflare/codeflare-operator v0.2.2
+	github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.0
 	github.com/ray-project/kuberay/ray-operator v0.0.0-20230830082034-e7fbf7d73576
 	k8s.io/api v0.27.2
 	k8s.io/apimachinery v0.27.2
@@ -30,7 +31,6 @@ require (
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/openshift/client-go v0.0.0-20230718165156-6014fb98e86a // indirect
-	github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/stretchr/testify v1.8.4 // indirect
 	golang.org/x/net v0.12.0 // indirect
@@ -54,4 +54,6 @@ require (
 	sigs.k8s.io/yaml v1.3.0 // indirect
 )
 
+replace github.com/project-codeflare/codeflare-operator => /home/ksuta/src/github.com/project-codeflare/codeflare-operator
+
 go 1.19

tests/go.sum

@@ -107,8 +107,6 @@ github.com/openshift/client-go v0.0.0-20230718165156-6014fb98e86a h1:ZKewwwEIURD
 github.com/openshift/client-go v0.0.0-20230718165156-6014fb98e86a/go.mod h1:EjhPQjEm8HM3GThz5ywNGLEec1P1IjTn08kwzdvupvA=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/project-codeflare/codeflare-operator v0.2.2 h1:7Vf1Hv+q0muzEGbTzPBdKjXOd903NbMu1bsdL02PCr0=
-github.com/project-codeflare/codeflare-operator v0.2.2/go.mod h1:2oYGL1iF92OzEhw+FRd7S4kFAEF9dZZnDCJLsuYZ4r0=
 github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.0 h1:4J3vW+vuCBHyocd3huZy3FJhOw5acaBHFLYwutkTE/E=
 github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.0/go.mod h1:gtTl8Tsl+X+bGhqVudLoveINR6IkN+sVvH0J+VZIP40=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=

tests/integration/mcad_pytorch_test.go

@@ -0,0 +1,171 @@
+/*
+Copyright 2023.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package integration
+
+import (
+	"bytes"
+	"html/template"
+	"os"
+	"os/exec"
+	"strings"
+	"testing"
+
+	. "github.com/onsi/gomega"
+	support "github.com/project-codeflare/codeflare-operator/test/support"
+	mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/yaml"
+)
+
+var notebookResource = schema.GroupVersionResource{Group: "kubeflow.org", Version: "v1", Resource: "notebooks"}
+
+type NotebookProps struct {
+	IngressDomain           string
+	OpenShiftApiUrl         string
+	KubernetesBearerToken   string
+	Namespace               string
+	OpenDataHubNamespace    string
+	CodeFlareImageStreamTag string
+	JobType                 string
+	NotebookPVC             string
+}
+
+func TestMnistPyTorchMCAD(t *testing.T) {
+	test := support.With(t)
+	test.T().Parallel()
+
+	// Create a namespace
+	namespace := test.NewTestNamespace()
+
+	// Test configuration
+	config := &corev1.ConfigMap{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: corev1.SchemeGroupVersion.String(),
+			Kind:       "ConfigMap",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "notebooks-mcad",
+		},
+		BinaryData: map[string][]byte{
+			// MNIST MCAD Notebook
+			"mnist_mcad_mini.ipynb": ReadFile(test, "resources/mnist_mcad_mini.ipynb"),
+		},
+		Immutable: support.Ptr(true),
+	}
+	config, err := test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), config, metav1.CreateOptions{})
+	test.Expect(err).NotTo(HaveOccurred())
+	test.T().Logf("Created ConfigMap %s/%s successfully", config.Namespace, config.Name)
+
+	// Create PVC for Notebook
+	notebookPvc := &corev1.PersistentVolumeClaim{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: corev1.SchemeGroupVersion.String(),
+			Kind:       "PersistentVolumeClaim",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "jupyterhub-nb-kube-3aadmin-pvc",
+		},
+		Spec: corev1.PersistentVolumeClaimSpec{
+			AccessModes: []corev1.PersistentVolumeAccessMode{
+				corev1.ReadWriteOnce,
+			},
+			Resources: corev1.ResourceRequirements{
+				Requests: corev1.ResourceList{
+					corev1.ResourceStorage: resource.MustParse("10Gi"),
+				},
+			},
+		},
+	}
+	notebookPvc, err = test.Client().Core().CoreV1().PersistentVolumeClaims(namespace.Name).Create(test.Ctx(), notebookPvc, metav1.CreateOptions{})
+	test.Expect(err).NotTo(HaveOccurred())
+	test.T().Logf("Created PersistentVolumeClaim %s/%s successfully", notebookPvc.Namespace, notebookPvc.Name)
+
+	// Read the Notebook CR from resources and perform replacements for custom values using go template
+	notebookProps := NotebookProps{
+		IngressDomain:           getIngressDomain(test),
+		OpenShiftApiUrl:         getOpenShiftApiUrl(test),
+		KubernetesBearerToken:   getKubernetesBearerToken(test),
+		Namespace:               namespace.Name,
+		OpenDataHubNamespace:    GetOpenDataHubNamespace(),
+		CodeFlareImageStreamTag: getCodeFlareImageStreamTag(test),
+		JobType:                 "mcad",
+		NotebookPVC:             "jupyterhub-nb-kube-3aadmin-pvc",
+	}
+	notebookTemplate := string(ReadFile(test, "resources/custom-nb-small.yaml"))
+	parsedNotebookTemplate, err := template.New("notebook").Parse(notebookTemplate)
+	test.Expect(err).NotTo(HaveOccurred())
+
+	// Filter template and store results to the buffer
+	notebookBuffer := new(bytes.Buffer)
+	err = parsedNotebookTemplate.Execute(notebookBuffer, notebookProps)
+	test.Expect(err).NotTo(HaveOccurred())
+
+	// Create Notebook CR using OC client, didn't find a Notebook go lang client for programmatic creation
+	notebookCR := &unstructured.Unstructured{}
+	err = yaml.NewYAMLOrJSONDecoder(notebookBuffer, 8192).Decode(notebookCR)
+	test.Expect(err).NotTo(HaveOccurred())
+	_, err = test.Client().Dynamic().Resource(notebookResource).Namespace(namespace.Name).Create(test.Ctx(), notebookCR, metav1.CreateOptions{})
+	test.Expect(err).NotTo(HaveOccurred())
+
+	// Make sure the AppWrapper is created and running
+	test.Eventually(support.AppWrappersWithPrefix(test, namespace, "mnistjob"), support.TestTimeoutLong).
+		Should(And(HaveLen(1), ContainElement(WithTransform(support.AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive)))))
+
+	// Make sure the AppWrapper finishes and is deleted
+	test.Eventually(support.AppWrappersWithPrefix(test, namespace, "mnistjob"), support.TestTimeoutLong).
+		Should(HaveLen(0))
+}
+
+func getIngressDomain(test support.Test) string {
+	domain, err := executeCommand("oc", "get", "ingresses.config/cluster", "-o", "jsonpath={.spec.domain}")
+	test.T().Logf("Domain %s", domain)
+	test.Expect(err).NotTo(HaveOccurred())
+	return domain
+}
+
+func getOpenShiftApiUrl(test support.Test) string {
+	openShiftApiUrl, err := executeCommand("oc", "whoami", "--show-server=true")
+	openShiftApiDomain := strings.TrimPrefix(openShiftApiUrl, "https://")
+	test.T().Logf("Domain %s", openShiftApiDomain)
+	test.Expect(err).NotTo(HaveOccurred())
+	return openShiftApiDomain
+}
+
+func getKubernetesBearerToken(test support.Test) string {
+	token, err := executeCommand("oc", "whoami", "--show-token=true")
+	test.T().Logf("Token %s", token)
+	test.Expect(err).NotTo(HaveOccurred())
+	return token
+}
+
+func getCodeFlareImageStreamTag(test support.Test) string {
+	v, ok := os.LookupEnv("CODEFLARE_IMAGESTREAM_TAG")
+	if !ok {
+		test.T().Fatalf("CODEFLARE_IMAGESTREAM_TAG not defined, cannot continue")
+	}
+	return v
+}
+
+func executeCommand(name string, arg ...string) (string, error) {
+	outputBytes, err := exec.Command(name, arg...).CombinedOutput()
+	return string(outputBytes), err
+}

tests/integration/resources/custom-nb-small.yaml

@@ -0,0 +1,165 @@
+# This template maybe used to spin up a custom notebook image
+# i.e.: sed s/{{ .IngressDomain}}/$(oc get ingresses.config/cluster -o jsonpath={.spec.domain})/g tests/resources/custom-nb.template | oc apply -f -
+# resources generated:
+# pod/jupyter-nb-kube-3aadmin-0
+# service/jupyter-nb-kube-3aadmin
+# route.route.openshift.io/jupyter-nb-kube-3aadmin (jupyter-nb-kube-3aadmin-opendatahub.apps.tedbig412.cp.fyre.ibm.com)
+# service/jupyter-nb-kube-3aadmin-tls
+apiVersion: kubeflow.org/v1
+kind: Notebook
+metadata:
+  annotations:
+    notebooks.opendatahub.io/inject-oauth: "true"
+    notebooks.opendatahub.io/last-image-selection: codeflare-notebook:latest
+    notebooks.opendatahub.io/last-size-selection: Small
+    notebooks.opendatahub.io/oauth-logout-url: https://odh-dashboard-{{ .OpenDataHubNamespace}}.{{ .IngressDomain}}/notebookController/kube-3aadmin/home
+    opendatahub.io/link: https://jupyter-nb-kube-3aadmin-{{ .Namespace}}.{{ .IngressDomain}}/notebook/{{ .Namespace}}/jupyter-nb-kube-3aadmin
+    opendatahub.io/username: kube:admin
+  generation: 1
+  labels:
+    app: jupyter-nb-kube-3aadmin
+    opendatahub.io/dashboard: "true"
+    opendatahub.io/odh-managed: "true"
+    opendatahub.io/user: kube-3aadmin
+  name: jupyter-nb-kube-3aadmin
+  namespace: {{ .Namespace}}
+spec:
+  template:
+    spec:
+      affinity:
+        nodeAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - preference:
+              matchExpressions:
+              - key: nvidia.com/gpu.present
+                operator: NotIn
+                values:
+                - "true"
+            weight: 1
+      containers:
+      - env:
+        - name: NOTEBOOK_ARGS
+          value: |-
+            --ServerApp.port=8888
+                              --ServerApp.token=''
+                              --ServerApp.password=''
+                              --ServerApp.base_url=/notebook/{{ .Namespace}}/jupyter-nb-kube-3aadmin
+                              --ServerApp.quit_button=False
+                              --ServerApp.tornado_settings={"user":"kube-3aadmin","hub_host":"https://odh-dashboard-{{ .OpenDataHubNamespace}}.{{ .IngressDomain}}","hub_prefix":"/notebookController/kube-3aadmin"}
+        - name: JUPYTER_IMAGE
+          value: image-registry.openshift-image-registry.svc:5000/{{ .OpenDataHubNamespace}}/codeflare-notebook:{{ .CodeFlareImageStreamTag}}
+        - name: JUPYTER_NOTEBOOK_PORT
+          value: "8888"
+        - name: OCP_SERVER
+          value: https://{{ .OpenShiftApiUrl}}
+        - name: OCP_TOKEN
+          value: {{ .KubernetesBearerToken}}
+        image: image-registry.openshift-image-registry.svc:5000/{{ .OpenDataHubNamespace}}/codeflare-notebook:{{ .CodeFlareImageStreamTag}}
+        command: ["/bin/sh", "-c", "pip install papermill && oc login --token=${OCP_TOKEN} --server=${OCP_SERVER} --insecure-skip-tls-verify=true && papermill /opt/app-root/notebooks-{{ .JobType}}/mnist_{{ .JobType}}_mini.ipynb /opt/app-root/src/mcad-out.ipynb && sleep infinity"]
+        # args: ["pip install papermill && oc login --token=${OCP_TOKEN} --server=${OCP_SERVER} --insecure-skip-tls-verify=true && papermill /opt/app-root/notebooks/mcad.ipynb /opt/app-root/src/mcad-out.ipynb" ]
+        imagePullPolicy: Always
+        # livenessProbe:
+        #   failureThreshold: 3
+        #   httpGet:
+        #     path: /notebook/{{ .Namespace}}/jupyter-nb-kube-3aadmin/api
+        #     port: notebook-port
+        #     scheme: HTTP
+        #   initialDelaySeconds: 10
+        #   periodSeconds: 5
+        #   successThreshold: 1
+        #   timeoutSeconds: 1
+        name: jupyter-nb-kube-3aadmin
+        ports:
+        - containerPort: 8888
+          name: notebook-port
+          protocol: TCP
+        resources:
+          limits:
+            cpu: "2"
+            memory: 3Gi
+          requests:
+            cpu: "1"
+            memory: 3Gi
+        volumeMounts:
+        - mountPath: /opt/app-root/src
+          name: jupyterhub-nb-kube-3aadmin-pvc
+        - mountPath: /opt/app-root/notebooks-{{ .JobType}}
+          name: notebooks-{{ .JobType}}
+        workingDir: /opt/app-root/src
+      - args:
+        - --provider=openshift
+        - --https-address=:8443
+        - --http-address=
+        - --openshift-service-account=jupyter-nb-kube-3aadmin
+        - --cookie-secret-file=/etc/oauth/config/cookie_secret
+        - --cookie-expire=24h0m0s
+        - --tls-cert=/etc/tls/private/tls.crt
+        - --tls-key=/etc/tls/private/tls.key
+        - --upstream=http://localhost:8888
+        - --upstream-ca=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+        - --skip-auth-regex=^(?:/notebook/$(NAMESPACE)/jupyter-nb-kube-3aadmin)?/api$
+        - --email-domain=*
+        - --skip-provider-button
+        - --openshift-sar={"verb":"get","resource":"notebooks","resourceAPIGroup":"kubeflow.org","resourceName":"jupyter-nb-kube-3aadmin","namespace":"$(NAMESPACE)"}
+        - --logout-url=https://odh-dashboard-{{ .OpenDataHubNamespace}}.{{ .IngressDomain}}/notebookController/kube-3aadmin/home
+        env:
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        image: registry.redhat.io/openshift4/ose-oauth-proxy:v4.10
+        imagePullPolicy: Always
+        livenessProbe:
+          failureThreshold: 3
+          httpGet:
+            path: /oauth/healthz
+            port: oauth-proxy
+            scheme: HTTPS
+          initialDelaySeconds: 30
+          periodSeconds: 5
+          successThreshold: 1
+          timeoutSeconds: 1
+        name: oauth-proxy
+        ports:
+        - containerPort: 8443
+          name: oauth-proxy
+          protocol: TCP
+        readinessProbe:
+          failureThreshold: 3
+          httpGet:
+            path: /oauth/healthz
+            port: oauth-proxy
+            scheme: HTTPS
+          initialDelaySeconds: 5
+          periodSeconds: 5
+          successThreshold: 1
+          timeoutSeconds: 1
+        resources:
+          limits:
+            cpu: 100m
+            memory: 64Mi
+          requests:
+            cpu: 100m
+            memory: 64Mi
+        volumeMounts:
+        - mountPath: /etc/oauth/config
+          name: oauth-config
+        - mountPath: /etc/tls/private
+          name: tls-certificates
+      enableServiceLinks: false
+      serviceAccountName: jupyter-nb-kube-3aadmin
+      volumes:
+      - name: jupyterhub-nb-kube-3aadmin-pvc
+        persistentVolumeClaim:
+          claimName: {{ .NotebookPVC}}
+      - name: oauth-config
+        secret:
+          defaultMode: 420
+          secretName: jupyter-nb-kube-3aadmin-oauth-config
+      - name: tls-certificates
+        secret:
+          defaultMode: 420
+          secretName: jupyter-nb-kube-3aadmin-tls
+      - name: notebooks-{{ .JobType}}
+        configMap:
+          name: notebooks-{{ .JobType}}