feat(conformance): create v1.7 conformance test directory (kubeflow#7123

)

* feat(conformance): fork v1.7 conformance files

Updated KFP test image to 1.7

* chore(conformance): restore v1.5 conformance files.

* Address code review comments.

conformance/1.5/README.md

@@ -2,8 +2,6 @@
 
 Before running the conformance tests, you need to configure kubectl default context to point to the k8s cluster that is hosting Kubeflow.
 
-TODO: Make the kubeflow namespace configurable.
-
 To run version <x.y> of the conformance test.
 
 `cd kubeflow/conformance/<x.y>`

conformance/1.5/katib-conformance.yaml

@@ -1,3 +1,17 @@
+# Copyright 2023 The Kubeflow Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 apiVersion: apps/v1
 kind: Deployment
 metadata:

conformance/1.5/report-pod.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 
-# Copyright 2019 The Kubeflow Authors.
+# Copyright 2022 The Kubeflow Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

conformance/1.5/training-operator-conformance.yaml

@@ -1,3 +1,17 @@
+# Copyright 2023 The Kubeflow Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 apiVersion: apps/v1
 kind: Deployment
 metadata:

conformance/1.7/Makefile

@@ -0,0 +1,69 @@
+# Copyright 2022 The Kubeflow Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+TEST_PROFILE ?= kf-conformance-test
+KUBEFLOW_NAMESPACE ?= kubeflow
+
+.PHONY: setup
+setup:
+	# Create the conformance report directory
+	mkdir -p /tmp/kf-conformance
+
+	kubectl apply -f ./setup.yaml
+
+.PHONY: run
+run: setup run-kfp run-katib run-training-operator
+
+.PHONY: report
+report: report-kfp report-katib report-training-operator
+
+.PHONY: clean
+clean:
+	kubectl delete -f ./setup.yaml
+	rm -rf /tmp/kf-conformance
+
+# Component specific tasks
+
+.PHONY: run-kfp
+run-kfp:
+	kubectl apply -f ./kfp-conformance.yaml
+
+.PHONY: report-kfp
+report-kfp:
+	./report-pod.sh \
+		$(shell kubectl get pod -n kf-conformance --selector=app=kfp-conformance -o jsonpath='{.items[*].metadata.name}') \
+		/tmp/kfp-conformance.done \
+		/tmp/kfp-conformance.log
+
+.PHONY: run-training-operator
+run-training-operator:
+	kubectl apply -f ./training-operator-conformance.yaml
+
+.PHONY: report-training-operator
+report-training-operator:
+	./report-pod.sh \
+		$(shell kubectl get pod -n kf-conformance --selector=app=training-operator-conformance -o jsonpath='{.items[*].metadata.name}') \
+		/tmp/training-operator-conformance.done \
+		/tmp/training-operator-conformance.log
+
+.PHONY: run-katib
+run-katib:
+	kubectl apply -f ./katib-conformance.yaml
+
+.PHONY: report-katib
+report-katib:
+	./report-pod.sh \
+		$(shell kubectl get pod -n kf-conformance --selector=app=katib-conformance -o jsonpath='{.items[*].metadata.name}') \
+		/tmp/katib-conformance.done \
+		/tmp/katib-conformance.log

conformance/1.7/README.md

@@ -0,0 +1,40 @@
+# Kubeflow conformance program (WIP)
+
+Before running the conformance tests, you need to configure kubectl default context to point to the k8s cluster that is hosting Kubeflow.
+
+To run version <x.y> of the conformance test.
+
+`cd kubeflow/conformance/<x.y>`
+
+**Run conformance test**
+
+Before running conformance tests, run setup:
+`make setup`
+
+To run all conformance tests for all components:
+`make run`
+
+You can run conformance test for a specific component:
+`make run-kfp`
+`make run-katib`
+`make run-training-operator`
+
+**Download test report**
+
+To download test report for all components:
+`make report`
+
+You can download test report for a specific component:
+`make report-kfp`
+`make report-katib`
+`make report-training-operator`
+
+The reports are downloaded to /tmp/kf-conformance
+
+**Submit the test report to Kubeflow conformance body for review**
+
+TBD
+
+**Clean up test resources**
+
+`make clean`

conformance/1.7/katib-conformance.yaml

@@ -0,0 +1,42 @@
+# Copyright 2023 The Kubeflow Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: katib-conformance
+  namespace: kf-conformance
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: katib-conformance
+  template:
+    metadata:
+      labels:
+        app: katib-conformance
+      annotations:
+        sidecar.istio.io/inject: "false"
+    spec:
+      serviceAccountName: kf-conformance
+      containers:
+      - name: katib-container
+        image: kubeflowkatib/katib-conformance:v1beta1-195ce77
+        resources:
+          limits:
+            memory: "128Mi"
+            cpu: "1"
+          requests:
+            memory: "64Mi"
+            cpu: "0.5"

conformance/1.7/kfp-conformance.yaml

@@ -0,0 +1,64 @@
+# Copyright 2023 The Kubeflow Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kfp-conformance
+  namespace: kf-conformance
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: kfp-conformance
+  template:
+    metadata:
+      labels:
+        app: kfp-conformance
+      annotations:
+        proxy.istio.io/config: '{ "holdApplicationUntilProxyStarts": true }'
+    spec:
+      terminationGracePeriodSeconds: 0
+      serviceAccountName: kf-conformance
+      containers:
+      - name: kfp-conformance
+        # To build the conformance test image, see
+        # https://github.com/kubeflow/pipelines/tree/release-2.0-alpha/backend
+        image: gcr.io/ml-pipeline/kfp-conformance:1.7
+        resources:
+          limits:
+            memory: "128Mi"
+            cpu: "1"
+          requests:
+            memory: "64Mi"
+            cpu: "0.5"
+        imagePullPolicy: Always
+        volumeMounts:
+        - mountPath: /var/run/secrets/kubeflow/pipelines
+          name: volume-kf-pipeline-token
+          readOnly: true
+        env:
+        - name: KF_PIPELINES_SA_TOKEN_PATH
+          value: /var/run/secrets/kubeflow/pipelines/token
+        # To enable debug mode, uncomment the env variable ADDITIONAL_FLAGS.
+        # - name: ADDITIONAL_FLAGS
+        #   value: "-isDebugMode"
+      volumes:
+      - name: volume-kf-pipeline-token
+        projected:
+          sources:
+            - serviceAccountToken:
+                path: token
+                expirationSeconds: 7200
+                audience: pipelines.kubeflow.org

conformance/1.7/report-pod.sh

@@ -0,0 +1,30 @@
+#!/bin/sh
+
+# Copyright 2022 The Kubeflow Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Runs a pod ($1). Waits for the done file ($2) to appear.
+# Copy out the test report file ($3)
+
+# Wait for the pod to generate a done file.
+until kubectl exec $1 -n kf-conformance -- ls $2
+do
+    sleep 30
+    echo "Waiting for $1 to finish ..."
+done
+
+REPORT_PATH=/tmp/kf-conformance/$(basename $3)
+kubectl cp kf-conformance/$1:$3 $REPORT_PATH
+
+echo "Test report copied to $REPORT_PATH"

conformance/1.7/setup.yaml

@@ -0,0 +1,73 @@
+# Copyright 2022 The Kubeflow Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Conformance test profile
+apiVersion: kubeflow.org/v1beta1
+kind: Profile
+metadata:
+  name: kf-conformance
+spec:
+  owner:
+    kind: User
+    name: [email protected]
+  resourceQuotaSpec:
+   hard:
+     cpu: "4"
+     memory: 4Gi
+     requests.storage: "5Gi"
+---
+# Service account used by conformance test
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kf-conformance
+  namespace: kf-conformance
+---
+# Bind service account to kubeflow-admin role
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: kf-conformance
+  namespace: kf-conformance
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kubeflow-admin
+subjects:
+- kind: ServiceAccount
+  name: kf-conformance
+  namespace: kf-conformance
+---
+# TODO: workaround until this issue can be fixed: https://github.com/kubeflow/pipelines/issues/7657
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kfp-conformance-patch
+rules:
+- apiGroups: [""]
+  resources: ["pipelines"]
+  verbs: ["create"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kfp-conformance-patch
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kfp-conformance-patch
+subjects:
+- kind: ServiceAccount
+  name: kf-conformance
+  namespace: kf-conformance