Skip to content

Commit

Permalink
test: fix test for rhoai to be used for e2e weekly
Browse files Browse the repository at this point in the history 
Signed-off-by: Wen Zhou <[email protected]>
  • Loading branch information
@zdtsw @openshift-merge-bot
zdtsw authored and openshift-merge-bot[bot] committed Jun 17, 2024
1 parent 338dcb6 commit 1269d52
Show file tree
Hide file tree
Showing 5 changed files with 96 additions and 47 deletions.
6 changes: 3 additions & 3 deletions tests/e2e/controller_setup_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,17 +77,17 @@ func NewTestContext() (*testContext, error) { //nolint:golint,revive // Only use
}

// setup DSCI CR since we do not create automatically by operator
testDSCI := setupDSCICR()
testDSCI := setupDSCICR("e2e-test-dsci")
// Setup DataScienceCluster CR
testDSC := setupDSCInstance()
testDSC := setupDSCInstance("e2e-test")

return &testContext{
cfg: config,
kubeClient: kc,
customClient: custClient,
operatorNamespace: opNamespace,
applicationsNamespace: testDSCI.Spec.ApplicationsNamespace,
resourceCreationTimeout: time.Minute * 2,
resourceCreationTimeout: time.Minute * 3, // 5 pods of dashboard will take longer time than ODH
resourceRetryInterval: time.Second * 10,
ctx: context.TODO(),
testDsc: testDSC,
Expand Down
86 changes: 66 additions & 20 deletions tests/e2e/dsc_creation_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,10 +40,18 @@ func creationTestSuite(t *testing.T) {
err = testCtx.testDSCICreation()
require.NoError(t, err, "error creating DSCI CR")
})
// do not enable test when we do not have webhook in place
// t.Run("Creation of more than one of DSCInitialization instance", func(t *testing.T) {
// testCtx.testDSCIDuplication(t)
// })
t.Run("Creation of DataScienceCluster instance", func(t *testing.T) {
err = testCtx.testDSCCreation()
require.NoError(t, err, "error creating DataScienceCluster instance")
})
// do not enable test when we do not have webhook in place
// t.Run("Creation of more than one of DataScienceCluster instance", func(t *testing.T) {
// testCtx.testDSCDuplication(t)
// })
t.Run("Validate all deployed components", func(t *testing.T) {
err = testCtx.testAllApplicationCreation(t)
require.NoError(t, err, "error testing deployments for DataScienceCluster: "+testCtx.testDsc.Name)
Expand Down Expand Up @@ -167,6 +175,49 @@ func (tc *testContext) testDSCCreation() error {
return waitDSCReady(tc)
}

// func (tc *testContext) requireInstalled(t *testing.T, gvk schema.GroupVersionKind) {
// t.Helper()
// list := &unstructured.UnstructuredList{}
// list.SetGroupVersionKind(gvk)
// err := tc.customClient.List(tc.ctx, list)
// require.NoErrorf(t, err, "Could not get %s list", gvk.Kind)
// require.Greaterf(t, len(list.Items), 0, "%s has not been installed", gvk.Kind)
// }

// func (tc *testContext) testDuplication(t *testing.T, gvk schema.GroupVersionKind, o any) {
// t.Helper()
// tc.requireInstalled(t, gvk)
// u, err := runtime.DefaultUnstructuredConverter.ToUnstructured(o)
// require.NoErrorf(t, err, "Could not unstructure %s", gvk.Kind)
// obj := &unstructured.Unstructured{
// Object: u,
// }
// obj.SetGroupVersionKind(gvk)
// err = tc.customClient.Create(tc.ctx, obj)
// require.Errorf(t, err, "Could create second %s", gvk.Kind)
// }

// func (tc *testContext) testDSCIDuplication(t *testing.T) { //nolint:thelper
// gvk := schema.GroupVersionKind{
// Group: "dscinitialization.opendatahub.io",
// Version: "v1",
// Kind: "DSCInitialization",
// }
// dup := setupDSCICR("e2e-test-dsci-dup")
// tc.testDuplication(t, gvk, dup)
// }

// func (tc *testContext) testDSCDuplication(t *testing.T) { //nolint:thelper
// gvk := schema.GroupVersionKind{
// Group: "datasciencecluster.opendatahub.io",
// Version: "v1",
// Kind: "DataScienceCluster",
// }

// dup := setupDSCInstance("e2e-test-dsc-dup")
// tc.testDuplication(t, gvk, dup)
// }

func (tc *testContext) testAllApplicationCreation(t *testing.T) error { //nolint:funlen,thelper
// Validate test instance is in Ready state

Expand Down Expand Up @@ -209,24 +260,19 @@ func (tc *testContext) testAllApplicationCreation(t *testing.T) error { //nolint
})
}

t.Run("Validate TrainingOperator", func(t *testing.T) {
// speed testing in parallel
t.Parallel()
err = tc.testApplicationCreation(&(tc.testDsc.Spec.Components.TrainingOperator))
if tc.testDsc.Spec.Components.TrainingOperator.ManagementState == operatorv1.Managed {
require.NoError(t, err, "error validating application %v when enabled", tc.testDsc.Spec.Components.TrainingOperator.GetComponentName())
} else {
require.Error(t, err, "error validating application %v when disabled", tc.testDsc.Spec.Components.TrainingOperator.GetComponentName())
}
})
return nil
}

func (tc *testContext) testApplicationCreation(component components.ComponentInterface) error {
err := wait.PollUntilContextTimeout(tc.ctx, tc.resourceRetryInterval, tc.resourceCreationTimeout, false, func(ctx context.Context) (bool, error) {
// TODO: see if checking deployment is a good test, CF does not create deployment
componentName := component.GetComponentName()
if componentName == "dashboard" { // special case for RHOAI dashboard name
componentName = "rhods-dashboard"
}

appList, err := tc.kubeClient.AppsV1().Deployments(tc.applicationsNamespace).List(context.TODO(), metav1.ListOptions{
LabelSelector: odhLabelPrefix + component.GetComponentName(),
LabelSelector: odhLabelPrefix + componentName,
})
if err != nil {
log.Printf("error listing application deployments :%v. Trying again...", err)
Expand All @@ -249,7 +295,7 @@ func (tc *testContext) testApplicationCreation(component components.ComponentInt
// when no deployment is found
// check Reconcile failed with missing dependent operator error
for _, Condition := range tc.testDsc.Status.Conditions {
if strings.Contains(Condition.Message, "Please install the operator before enabling "+component.GetComponentName()) {
if strings.Contains(Condition.Message, "Please install the operator before enabling "+componentName) {
return true, err
}
}
Expand Down Expand Up @@ -329,7 +375,7 @@ func (tc *testContext) testUpdateComponentReconcile() error {
// Test Updating Dashboard Replicas

appDeployments, err := tc.kubeClient.AppsV1().Deployments(tc.applicationsNamespace).List(context.TODO(), metav1.ListOptions{
LabelSelector: odhLabelPrefix + tc.testDsc.Spec.Components.Dashboard.GetComponentName(),
LabelSelector: odhLabelPrefix + "rhods-dashboard",
})
if err != nil {
return err
Expand All @@ -343,7 +389,7 @@ func (tc *testContext) testUpdateComponentReconcile() error {
Namespace: testDeployment.Namespace,
},
Spec: autoscalingv1.ScaleSpec{
Replicas: 3,
Replicas: 4, // rhoai has 5 pods
},
Status: autoscalingv1.ScaleStatus{},
}
Expand All @@ -355,8 +401,8 @@ func (tc *testContext) testUpdateComponentReconcile() error {
return fmt.Errorf("failed to patch replicas : expect to be %v but got %v", patchedReplica.Spec.Replicas, retrievedDep.Spec.Replicas)
}

// Sleep for 20 seconds to allow the operator to reconcile
time.Sleep(2 * tc.resourceRetryInterval)
// Sleep for 40 seconds to allow the operator to reconcile
time.Sleep(4 * tc.resourceRetryInterval)
revertedDep, err := tc.kubeClient.AppsV1().Deployments(tc.applicationsNamespace).Get(context.TODO(), testDeployment.Name, metav1.GetOptions{})
if err != nil {
return fmt.Errorf("error getting component resource after reconcile: %w", err)
Expand All @@ -375,10 +421,10 @@ func (tc *testContext) testUpdateDSCComponentEnabled() error {

if tc.testDsc.Spec.Components.Dashboard.ManagementState == operatorv1.Managed {
appDeployments, err := tc.kubeClient.AppsV1().Deployments(tc.applicationsNamespace).List(context.TODO(), metav1.ListOptions{
LabelSelector: odhLabelPrefix + tc.testDsc.Spec.Components.Dashboard.GetComponentName(),
LabelSelector: odhLabelPrefix + "rhods-dashboard", // here is not same label as comopnent name in DSC
})
if err != nil {
return fmt.Errorf("error getting enabled component %v", tc.testDsc.Spec.Components.Dashboard.GetComponentName())
return fmt.Errorf("error getting enabled component %v", "rhods-dashboard")
}
if len(appDeployments.Items) > 0 {
dashboardDeploymentName = appDeployments.Items[0].Name
Expand Down Expand Up @@ -413,8 +459,8 @@ func (tc *testContext) testUpdateDSCComponentEnabled() error {
return fmt.Errorf("error after retry %w", err)
}

// Sleep for 20 seconds to allow the operator to reconcile
time.Sleep(2 * tc.resourceRetryInterval)
// Sleep for 40 seconds to allow the operator to reconcile
time.Sleep(4 * tc.resourceRetryInterval)
_, err = tc.kubeClient.AppsV1().Deployments(tc.applicationsNamespace).Get(context.TODO(), dashboardDeploymentName, metav1.GetOptions{})
if err != nil {
if errors.IsNotFound(err) {
Expand Down
15 changes: 9 additions & 6 deletions tests/e2e/dsc_deletion_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ func deletionTestSuite(t *testing.T) {
require.NoError(t, err, "Error to delete DSC instance")
})
t.Run("Deletion: Application Resource", func(t *testing.T) {
err = testCtx.testAllApplicationDeletion()
err = testCtx.testAllApplicationDeletion(t)
require.NoError(t, err, "Error to delete component")
})
t.Run("Deletion: DSCI instance", func(t *testing.T) {
Expand Down Expand Up @@ -83,7 +83,7 @@ func (tc *testContext) testApplicationDeletion(component components.ComponentInt
return nil
}

func (tc *testContext) testAllApplicationDeletion() error {
func (tc *testContext) testAllApplicationDeletion(t *testing.T) error { //nolint:thelper
// Deletion all listed components' deployments

components, err := tc.testDsc.GetComponents()
Expand All @@ -92,9 +92,12 @@ func (tc *testContext) testAllApplicationDeletion() error {
}

for _, c := range components {
if err = tc.testApplicationDeletion(c); err != nil {
return err
}
c := c
t.Run("Delete "+c.GetComponentName(), func(t *testing.T) {
t.Parallel()
err = tc.testApplicationDeletion(c)
require.NoError(t, err)
})
}

return nil
Expand All @@ -119,5 +122,5 @@ func (tc *testContext) testDeletionExistDSCI() error {
return fmt.Errorf("error getting DSCI instance :%w", err)
}
}
return err
return nil
}
28 changes: 14 additions & 14 deletions tests/e2e/helper_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,9 @@ import (
)

const (
servicemeshNamespace = "openshift-operators"
servicemeshOpName = "servicemeshoperator"
serverlessOpName = "serverless-operator"
depedentOperatorNamespace = "openshift-operators"
servicemeshOpName = "servicemeshoperator"
serverlessOpName = "serverless-operator"
)

func (tc *testContext) waitForControllerDeployment(name string, replicas int32) error {
Expand Down Expand Up @@ -67,16 +67,16 @@ func (tc *testContext) waitForControllerDeployment(name string, replicas int32)
return err
}

func setupDSCICR() *dsci.DSCInitialization {
func setupDSCICR(name string) *dsci.DSCInitialization {
dsciTest := &dsci.DSCInitialization{
ObjectMeta: metav1.ObjectMeta{
Name: "e2e-test-dsci",
Name: name,
},
Spec: dsci.DSCInitializationSpec{
ApplicationsNamespace: "opendatahub",
ApplicationsNamespace: "redhat-ods-applications",
Monitoring: dsci.Monitoring{
ManagementState: "Managed",
Namespace: "opendatahub",
Namespace: "redhat-ods-monitoring",
},
TrustedCABundle: dsci.TrustedCABundleSpec{
ManagementState: "Managed",
Expand All @@ -90,10 +90,10 @@ func setupDSCICR() *dsci.DSCInitialization {
return dsciTest
}

func setupDSCInstance() *dsc.DataScienceCluster {
func setupDSCInstance(name string) *dsc.DataScienceCluster {
dscTest := &dsc.DataScienceCluster{
ObjectMeta: metav1.ObjectMeta{
Name: "e2e-test-dsc",
Name: name,
},
Spec: dsc.DataScienceClusterSpec{
Components: dsc.Components{
Expand All @@ -110,7 +110,7 @@ func setupDSCInstance() *dsc.DataScienceCluster {
},
ModelMeshServing: modelmeshserving.ModelMeshServing{
Component: components.Component{
ManagementState: operatorv1.Removed,
ManagementState: operatorv1.Managed,
},
},
DataSciencePipelines: datasciencepipelines.DataSciencePipelines{
Expand Down Expand Up @@ -148,7 +148,7 @@ func setupDSCInstance() *dsc.DataScienceCluster {
},
TrainingOperator: trainingoperator.TrainingOperator{
Component: components.Component{
ManagementState: operatorv1.Managed,
ManagementState: operatorv1.Removed,
},
},
},
Expand Down Expand Up @@ -373,7 +373,7 @@ func ensureOperator(tc *testContext, name string, ns string) error {
return waitCSV(tc, name, ns)
}

func ensureServicemeshOperators(t *testing.T, tc *testContext) error { //nolint: thelper
func ensureDepedentOperators(t *testing.T, tc *testContext) error { //nolint: thelper
ops := []string{
serverlessOpName,
servicemeshOpName,
Expand All @@ -385,7 +385,7 @@ func ensureServicemeshOperators(t *testing.T, tc *testContext) error { //nolint:
op := op // to avoid loop variable in the closures
t.Logf("Ensuring %s is installed", op)
go func(op string) {
err := ensureOperator(tc, op, servicemeshNamespace)
err := ensureOperator(tc, op, depedentOperatorNamespace)
c <- err
}(op)
}
Expand All @@ -399,5 +399,5 @@ func ensureServicemeshOperators(t *testing.T, tc *testContext) error { //nolint:
}

func (tc *testContext) setUp(t *testing.T) error { //nolint: thelper
return ensureServicemeshOperators(t, tc)
return ensureDepedentOperators(t, tc)
}
8 changes: 4 additions & 4 deletions tests/e2e/odh_manager_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,14 @@ func testODHOperatorValidation(t *testing.T) {
testCtx, err := NewTestContext()
require.NoError(t, err)

t.Run("validate ODH Operator pod", testCtx.testODHDeployment)
t.Run("validate RHOAI Operator pod", testCtx.testRHOAIDeployment)
t.Run("validate CRDs owned by the operator", testCtx.validateOwnedCRDs)
}

func (tc *testContext) testODHDeployment(t *testing.T) {
func (tc *testContext) testRHOAIDeployment(t *testing.T) {
// Verify if the operator deployment is created
require.NoErrorf(t, tc.waitForControllerDeployment("opendatahub-operator-controller-manager", 1),
"error in validating odh operator deployment")
require.NoErrorf(t, tc.waitForControllerDeployment("redhat-ods-operator-controller-manager", 1),
"error in validating rhoai operator deployment")
}

func (tc *testContext) validateOwnedCRDs(t *testing.T) {
Expand Down

0 comments on commit 1269d52

Please sign in to comment.