Merge pull request #618 from weherdh/adopkointest

Fix PKO integration tests

Makefile

@@ -49,8 +49,8 @@ DEPENDENCY_VERSIONS:=$(abspath $(DEPENDENCIES)/$(UNAME_OS)/$(UNAME_ARCH)/version
 export PATH:=$(DEPENDENCY_BIN):$(PATH)
 
 # Config
-KIND_KUBECONFIG_DIR:=.cache/integration
-KIND_KUBECONFIG:=$(KIND_KUBECONFIG_DIR)/kubeconfig
+KIND_KUBECONFIG_DIR:=.cache/dev-env
+KIND_KUBECONFIG:=$(KIND_KUBECONFIG_DIR)/kubeconfig.yaml
 export KUBECONFIG?=$(abspath $(KIND_KUBECONFIG))
 export GOLANGCI_LINT_CACHE=$(abspath .cache/golangci-lint)
 export SKIP_TEARDOWN?=
@@ -66,15 +66,15 @@ WEBHOOK_PORT?=8080
 IMAGE_ORG?=quay.io/app-sre
 ADDON_OPERATOR_MANAGER_IMAGE?=$(IMAGE_ORG)/addon-operator-manager:$(VERSION)
 ADDON_OPERATOR_WEBHOOK_IMAGE?=$(IMAGE_ORG)/addon-operator-webhook:$(VERSION)
-API_MOCK_IMAGE?=$(IMAGE_ORG)/api-mock:$(VERSION)
+API_MOCK_IM?=$(IMAGE_ORG)/api-mock:$(VERSION)
+export API_MOCK_IMAGE?=$(API_MOCK_IM)
 
 # COLORS
 GREEN  := $(shell tput -Txterm setaf 2)
 YELLOW := $(shell tput -Txterm setaf 3)
 RESET  := $(shell tput -Txterm sgr0)
 
 OUTPUT=$(shell grep -e 'webhookdefinitions' $(PROJECT_DIR)/bundle/manifests/addon-operator.clusterserviceversion.yaml)
-
 
 # ---------
 ##@ General

integration/integration_test.go

@@ -5,6 +5,9 @@ import (
 	"log"
 	"testing"
 
+	"k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/client-go/util/retry"
+
 	"github.com/stretchr/testify/suite"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
@@ -18,13 +21,67 @@ type integrationTestSuite struct {
 
 func (s *integrationTestSuite) SetupSuite() {
 	if !testing.Short() {
-		s.Setup()
+		ctx := context.Background()
+		addonOperator := addonsv1alpha1.AddonOperator{}
+
+		// Wait for API to be created
+		s.T().Log("Checking AddonOperator is created")
+		err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
+			err := integration.Client.Get(ctx, client.ObjectKey{
+				Name: addonsv1alpha1.DefaultAddonOperatorName,
+			}, &addonOperator)
+			return err
+		})
+		s.Require().NoError(err)
+		err = integration.WaitForObject(
+			ctx,
+			s.T(), defaultAddonAvailabilityTimeout, &addonOperator, "to be Available",
+			func(obj client.Object) (done bool, err error) {
+				a := obj.(*addonsv1alpha1.AddonOperator)
+				return meta.IsStatusConditionTrue(
+					a.Status.Conditions, addonsv1alpha1.Available), nil
+			})
+		s.Require().NoError(err)
+		s.T().Log("AddonOperator exists")
+
+		s.T().Log("Patch AddonOperator with OCM mock configuration")
+		addonOperatorPointer := &addonsv1alpha1.AddonOperator{}
+		if err := integration.Client.Get(ctx, client.ObjectKey{
+			Name: addonsv1alpha1.DefaultAddonOperatorName,
+		}, addonOperatorPointer); err != nil {
+			s.T().Fatalf("get AddonOperator object: %v", err)
+		}
+
+		addonOperatorPointer.Spec.OCM = &addonsv1alpha1.AddonOperatorOCM{
+			Endpoint: integration.OCMAPIEndpoint,
+			Secret: addonsv1alpha1.ClusterSecretReference{
+				Name:      "pull-secret",
+				Namespace: "api-mock",
+			},
+		}
+		if err := integration.Client.Update(ctx, addonOperatorPointer); err != nil {
+			s.T().Fatalf("patch AddonOperator object: %v", err)
+		}
+		s.T().Log("AddonOperator patched")
 	}
 }
 
 func (s *integrationTestSuite) TearDownSuite() {
 	if !testing.Short() {
-		s.Teardown()
+		ctx := context.Background()
+
+		// assert that all addons are gone before teardown
+		addonList := &addonsv1alpha1.AddonList{}
+		err := integration.Client.List(ctx, addonList)
+		s.Assert().NoError(err)
+		addonNames := []string{}
+		for _, a := range addonList.Items {
+			addonNames = append(addonNames, a.GetName())
+			addon := a
+			s.T().Logf("Cleaning up addons from the tear down: %s", addonNames)
+			s.addonCleanup(&addon, ctx)
+		}
+		s.Assert().Len(addonNames, 0, "expected all Addons to be gone before teardown, but some still exist. Cleaned up all of them")
 	}
 
 	if err := integration.PrintPodStatusAndLogs("addon-operator"); err != nil {
@@ -43,16 +100,19 @@ func (s *integrationTestSuite) addonCleanup(addon *addonsv1alpha1.Addon,
 	// wait until Addon is gone
 	err = integration.WaitToBeGone(ctx, s.T(), defaultAddonDeletionTimeout, addon)
 	s.Require().NoError(err, "wait for Addon to be deleted")
+	s.T().Logf("Addon %s is deleted", addon.Name)
 }
 
 func TestIntegration(t *testing.T) {
 	// Run kube-apiserver proxy during tests
+	t.Log("Integration testing starts, setting the proxy")
 	apiProxyCloseCh := make(chan struct{})
 	defer close(apiProxyCloseCh)
 	if err := integration.RunAPIServerProxy(apiProxyCloseCh); err != nil {
 		t.Fatal(err)
 	}
 
+	t.Log("Integration testing starts, setting the OCMclient")
 	// Initialize the OCM client after the proxy has started.
 	if err := integration.InitOCMClient(); err != nil {
 		t.Fatal(err)

integration/pko_test.go

@@ -64,7 +64,7 @@ func (s *integrationTestSuite) TestPackageOperatorReconcilerStatusPropagatedToAd
 				OLMOwnNamespace: &addonsv1alpha1.AddonInstallOLMOwnNamespace{
 					AddonInstallOLMCommon: addonsv1alpha1.AddonInstallOLMCommon{
 						Namespace:          namespace,
-						CatalogSourceImage: referenceAddonCatalogSourceImageWorking,
+						CatalogSourceImage: referenceAddonCatalogSourceImageWorkingv6,
 						Channel:            "alpha",
 						PackageName:        "reference-addon",
 						Config:             &addonsv1alpha1.SubscriptionConfig{EnvironmentVariables: referenceAddonConfigEnvObjects},
@@ -207,17 +207,23 @@ func (s *integrationTestSuite) TestPackageOperatorReconcilerSourceParameterInjec
 	var wg sync.WaitGroup
 	// set the WaitGroup counter to the total number of jobs
 	wg.Add(len(tests))
+	// Create the error channel for tests
+	errCh := make(chan error)
 	// launch workers
 	for i := 0; i < numWorkers; i++ {
-		go testWorker(s, i, numWorkers, &wg, tests)
+		go testWorker(s, i, numWorkers, &wg, tests, errCh)
 	}
+
 	// wait for all jobs to finish
-	timedOut := waitTimeout(&wg, 15*time.Minute)
+	timedOut := waitTimeout(&wg, 30*time.Minute, errCh)
+	for err := range errCh {
+		s.Assert().NoError(err, "ClusterPackage failed to check")
+	}
 	// check that the timeout didn't trigger
 	s.Require().False(timedOut)
 }
 
-func testWorker(s *integrationTestSuite, workerID int, numWorkers int, wg *sync.WaitGroup, testSources []TestPKOSourcesData) {
+func testWorker(s *integrationTestSuite, workerID int, numWorkers int, wg *sync.WaitGroup, testSources []TestPKOSourcesData, errCh chan error) {
 	testIdx := workerID
 
 	for counter := 1; testIdx < len(testSources); counter++ {
@@ -243,7 +249,7 @@ func testWorker(s *integrationTestSuite, workerID int, numWorkers int, wg *sync.
 			s.createSendGridSecret(ctx, testAddonName, testAddonNamespace)
 		}
 
-		s.waitForClusterPackage(
+		if err := s.waitForClusterPackage(
 			ctx,
 			testAddonName,
 			testAddonNamespace,
@@ -252,18 +258,21 @@ func testWorker(s *integrationTestSuite, workerID int, numWorkers int, wg *sync.
 			testSource.deployDeadMansSnitchSecret,
 			testSource.deployPagerDutySecret,
 			testSource.deploySendGridSecret,
-		)
+		); err != nil {
+			errCh <- err
+		}
 
 		s.addonCleanup(addon, ctx)
 		wg.Done()
 		testIdx = workerID + counter*numWorkers
 	}
 }
 
-func waitTimeout(wg *sync.WaitGroup, timeout time.Duration) bool {
+func waitTimeout(wg *sync.WaitGroup, timeout time.Duration, errCh chan error) bool {
 	c := make(chan struct{})
 	go func() {
 		defer close(c)
+		defer close(errCh)
 		wg.Wait()
 	}()
 	select {
@@ -288,7 +297,7 @@ func (s *integrationTestSuite) createAddon(ctx context.Context, addonName string
 				OLMOwnNamespace: &addonsv1alpha1.AddonInstallOLMOwnNamespace{
 					AddonInstallOLMCommon: addonsv1alpha1.AddonInstallOLMCommon{
 						Namespace:          addonNamespace,
-						CatalogSourceImage: referenceAddonCatalogSourceImageWorking,
+						CatalogSourceImage: referenceAddonCatalogSourceImageWorkingv6,
 						Channel:            "alpha",
 						PackageName:        "reference-addon",
 						Config:             &addonsv1alpha1.SubscriptionConfig{EnvironmentVariables: referenceAddonConfigEnvObjects},
@@ -348,13 +357,17 @@ func (s *integrationTestSuite) createSecret(ctx context.Context, name string, na
 // and check if their env variables corresponds to the secrets
 func (s *integrationTestSuite) waitForClusterPackage(ctx context.Context, addonName string, addonNamespace string, conditionType string,
 	addonParametersValuePresent bool, deadMansSnitchUrlValuePresent bool, pagerDutyValuePresent bool, sendGridValuePresent bool,
-) {
+) error {
 	logger := testutil.NewLogger(s.T())
 	cp := &v1alpha1.ClusterPackage{ObjectMeta: metav1.ObjectMeta{Name: addonName}}
-	err := integration.WaitForObjectWithInterval(ctx, s.T(), 20*time.Second,
+	if err := integration.WaitForObjectWithInterval(ctx, s.T(), 30*time.Second,
 		defaultAddonAvailabilityTimeout, cp, "to be "+conditionType,
-		clusterPackageChecker(&logger, addonNamespace, conditionType, addonParametersValuePresent, deadMansSnitchUrlValuePresent, pagerDutyValuePresent, sendGridValuePresent))
-	s.Require().NoError(err)
+		clusterPackageChecker(&logger, addonNamespace, conditionType, addonParametersValuePresent, deadMansSnitchUrlValuePresent,
+			pagerDutyValuePresent, sendGridValuePresent)); err != nil {
+		s.T().Log("ClusterPackage failed to check")
+		return err
+	}
+	return nil
 }
 
 func clusterPackageChecker(
@@ -401,6 +414,10 @@ func clusterPackageChecker(
 
 		targetNamespace, present := addonsv1[addon.TargetNamespaceConfigKey]
 		targetNamespaceValueOk := present && targetNamespace == addonNamespace
+		if !targetNamespaceValueOk {
+			logger.Info("result: false target namespace value wrong")
+			return false, nil
+		}
 
 		clusterID, present := addonsv1[addon.ClusterIDConfigKey]
 		clusterIDValueOk := false
@@ -412,8 +429,17 @@ func clusterPackageChecker(
 		ocmClusterID, present := addonsv1[addon.OcmClusterIDConfigKey]
 		ocmClusterIDValueOk := present && len(fmt.Sprintf("%v", ocmClusterID)) > 0
 
+		if !ocmClusterIDValueOk {
+			logger.Info("result: false ocm cluster id value wrong")
+			return false, nil
+		}
+
 		ocmClusterName, present := addonsv1[addon.OcmClusterNameConfigKey]
 		ocmClusterNameValueOk := present && len(fmt.Sprintf("%v", ocmClusterName)) > 0
+		if !ocmClusterNameValueOk {
+			logger.Info("result: false ocm cluster name value wrong")
+			return false, nil
+		}
 
 		addonParametersValueOk, deadMansSnitchUrlValueOk, pagerDutyValueOk, sendGridValueOk := false, false, false, false
 		if addonParametersValuePresent {
@@ -428,20 +454,36 @@ func clusterPackageChecker(
 			_, present := addonsv1[addon.ParametersConfigKey]
 			addonParametersValueOk = !present
 		}
+		if !addonParametersValueOk {
+			logger.Info("result: false addon parameters value wrong")
+			return false, nil
+		}
+
 		if deadMansSnitchUrlValuePresent {
 			value, present := addonsv1[addon.DeadMansSnitchUrlConfigKey]
 			deadMansSnitchUrlValueOk = present && fmt.Sprint(value) == deadMansSnitchUrlValue
 		} else {
 			_, present := addonsv1[addon.DeadMansSnitchUrlConfigKey]
 			deadMansSnitchUrlValueOk = !present
 		}
+
+		if !deadMansSnitchUrlValueOk {
+			logger.Info("result: false DM url value wrong")
+			return false, nil
+		}
+
 		if pagerDutyValuePresent {
 			value, present := addonsv1[addon.PagerDutyKeyConfigKey]
 			pagerDutyValueOk = present && fmt.Sprint(value) == pagerDutyKeyValue
 		} else {
 			_, present := addonsv1[addon.PagerDutyKeyConfigKey]
 			pagerDutyValueOk = !present
 		}
+		if !pagerDutyValueOk {
+			logger.Info("result: false PD value wrong")
+			return false, nil
+		}
+
 		if sendGridValuePresent {
 			value, present := addonsv1[addon.SendGridConfigKey]
 			if present {
@@ -454,6 +496,10 @@ func clusterPackageChecker(
 			_, present := addonsv1[addon.SendGridConfigKey]
 			sendGridValueOk = !present
 		}
+		if !sendGridValueOk {
+			logger.Info("result: false SG value wrong")
+			return false, nil
+		}
 
 		logger.Info(fmt.Sprintf("targetNamespace=%t, clusterID=%t, ocmClusterID=%t, ocmClusterName=%t, addonParameters=%t, deadMansSnitchUrl=%t, pagerDutyKey=%t, sendGrid=%t",
 			targetNamespaceValueOk,

internal/featuretoggle/addons_plug_and_play_feature_toggle_testable.go

@@ -17,7 +17,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-var pkoVersion = "1.6.5"
+var pkoVersion = "1.11.0"
 
 func (h *AddonsPlugAndPlayFeatureToggle) Enable(ctx context.Context) error {
 	adoInCluster := addonsv1alpha1.AddonOperator{}