Add integration tests and update workflow for testing (#17)

* feat: add integration tests and update workflow for testing

* refactor: disable lll linter and clean up integration test variable shadowing

* feat: update DeployControllerManager to include image parameter

* feat: implement BuildImage function to build Docker image for controller manager

* fix: update cache key annotations and test namespace for consistency

* fix: reorder BuildImage function call in test environment setup

* fix: update test namespace for consistency in integration tests

* feat: add integration tests workflow for end-to-end testing

* refactor: reorganize integration tests and remove unused cache resources

Author: ninjadq

.github/workflows/test-integration.yml

@@ -0,0 +1,23 @@
+name: Integration Tests
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  test-e2e:
+    name: Run on Ubuntu
+    runs-on: ubuntu-latest
+    steps:
+      - name: Clone the code
+        uses: actions/checkout@v4
+
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+
+      - name: Running integration tests
+        run: |
+          go mod tidy
+          make test-integration

.golangci.yml

@@ -30,7 +30,7 @@ linters:
     - gosimple
     - govet
     - ineffassign
-    - lll
+    # - lll
     - misspell
     - nakedret
     - prealloc

Makefile

@@ -59,7 +59,7 @@ vet: ## Run go vet against code.
 
 .PHONY: test
 test: manifests generate fmt vet setup-envtest ## Run tests.
-	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile ./cover.out -covermode=atomic -coverpkg=./...
+	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e | grep -v /integration) -coverprofile ./cover.out -covermode=atomic -coverpkg=./...
 
 # TODO(user): To use a different vendor for e2e tests, modify the setup under 'tests/e2e'.
 # The default setup assumes Kind is pre-installed and builds/loads the Manager Docker image locally.
@@ -77,6 +77,10 @@ test-e2e: manifests generate fmt vet ## Run the e2e tests. Expected an isolated
 	}
 	go test ./test/e2e/ -v -ginkgo.v
 
+.Phony: test-integration
+test-integration: manifests generate fmt vet ## Run the integration tests. Expected an isolated environment using Kind.
+	go test ./test/integration
+
 .PHONY: lint
 lint: golangci-lint ## Run golangci-lint linter
 	$(GOLANGCI_LINT) run

go.mod

@@ -16,6 +16,7 @@ require (
 	k8s.io/apimachinery v0.32.1
 	k8s.io/client-go v0.32.1
 	sigs.k8s.io/controller-runtime v0.20.4
+	sigs.k8s.io/e2e-framework v0.6.0
 )
 
 require (
@@ -46,14 +47,17 @@ require (
 	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db // indirect
+	github.com/gorilla/websocket v1.5.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/moby/spdystream v0.5.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/prometheus/client_golang v1.19.1 // indirect
@@ -63,6 +67,7 @@ require (
 	github.com/spf13/cobra v1.8.1 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/stoewer/go-strcase v1.3.0 // indirect
+	github.com/vladimirvivien/gexe v0.4.1 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect
 	go.opentelemetry.io/otel v1.28.0 // indirect

go.sum

@@ -2,6 +2,8 @@ cel.dev/expr v0.18.0 h1:CJ6drgk+Hf96lkLikr4rFf19WrU0BOWEihyZnI2TAzo=
 cel.dev/expr v0.18.0/go.mod h1:MrpN08Q+lEBs+bGYdLxxHkZoUSsCp0nSKTs0nTymJgw=
 github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI=
 github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g=
+github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
+github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -67,6 +69,8 @@ github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgY
 github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
+github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
@@ -86,13 +90,17 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/moby/spdystream v0.5.0 h1:7r0J1Si3QO/kjRitvSLVVFUjxMEb/YLj6S9FF62JBCU=
+github.com/moby/spdystream v0.5.0/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus=
+github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
 github.com/onsi/ginkgo/v2 v2.22.0 h1:Yed107/8DjTr0lKCNt7Dn8yQ6ybuDRQoMGrNFKzMfHg=
 github.com/onsi/ginkgo/v2 v2.22.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo=
 github.com/onsi/gomega v1.36.1 h1:bJDPBO7ibjxcbHMgSCoo4Yj18UWbKDlLwX1x9sybDcw=
@@ -130,6 +138,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
 github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/vladimirvivien/gexe v0.4.1 h1:W9gWkp8vSPjDoXDu04Yp4KljpVMaSt8IQuHswLDd5LY=
+github.com/vladimirvivien/gexe v0.4.1/go.mod h1:3gjgTqE2c0VyHnU5UOIwk7gyNzZDGulPb/DJPgcw64E=
 github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
 github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -243,6 +253,8 @@ sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.0 h1:CPT0ExVicCzcp
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.0/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw=
 sigs.k8s.io/controller-runtime v0.20.4 h1:X3c+Odnxz+iPTRobG4tp092+CvBU9UK0t/bRf+n0DGU=
 sigs.k8s.io/controller-runtime v0.20.4/go.mod h1:xg2XB0K5ShQzAgsoujxuKN4LNXR2LfwwHsPj7Iaw+XY=
+sigs.k8s.io/e2e-framework v0.6.0 h1:p7hFzHnLKO7eNsWGI2AbC1Mo2IYxidg49BiT4njxkrM=
+sigs.k8s.io/e2e-framework v0.6.0/go.mod h1:IREnCHnKgRCioLRmNi0hxSJ1kJ+aAdjEKK/gokcZu4k=
 sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8=
 sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo=
 sigs.k8s.io/structured-merge-diff/v4 v4.4.2 h1:MdmvkGuXi/8io6ixD5wud3vOLwc1rj0aNqRlpuvjmwA=

internal/utils/controller/const.go

@@ -1,12 +1,12 @@
 package controller
 
 const (
-	LabelNameCacheKey = "github.com/Azure/operation-cache-controller/cache-key"
+	LabelNameCacheKey = "operation-cache-controller.azure.github.com/cache-key"
 )
 
 const (
-	AnnotationNameCacheMode = "github.com/Azure/operation-cache-controller/cache-mode"
-	AnnotationNameCacheKey  = "github.com/Azure/operation-cache-controller/cache-key-annotation"
+	AnnotationNameCacheMode = "operation-cache-controller.azure.github.com/cache-mode"
+	AnnotationNameCacheKey  = "operation-cache-controller.azure.github.com/cache-key"
 	AnnotationValueTrue     = "true"
 	AnnotationValueFalse    = "false"
 )

test/integration/integration_test.go

@@ -0,0 +1,62 @@
+package integration
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"sigs.k8s.io/e2e-framework/pkg/envconf"
+	"sigs.k8s.io/e2e-framework/pkg/features"
+
+	v1 "github.com/Azure/operation-cache-controller/api/v1"
+	rqutils "github.com/Azure/operation-cache-controller/internal/utils/controller/requirement"
+	"github.com/Azure/operation-cache-controller/test/utils"
+)
+
+type requirementKey struct{}
+
+const (
+	testRequirementName = "test-requirement"
+)
+
+var CacheFeature = features.New("Simple Requirements").
+	Setup(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context {
+		// start a deployment
+		requiremnt := utils.NewRequirement(testRequirementName, utils.TestNamespcae)
+		requiremnt.Namespace = utils.TestNamespcae
+		if err := c.Client().Resources().Create(ctx, requiremnt); err != nil {
+			t.Fatal(err)
+		}
+		time.Sleep(2 * time.Second)
+
+		return ctx
+	}).
+	Assess("create requirement", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+		var requirement v1.Requirement
+		if err := cfg.Client().Resources().Get(ctx, testRequirementName, utils.TestNamespcae, &requirement); err != nil {
+			t.Fatal(err)
+		}
+		assert.Equal(t, testRequirementName, requirement.Name)
+		if err := wait.PollUntilContextTimeout(ctx, 10*time.Second, 60*time.Second, true, func(ctx context.Context) (bool, error) {
+			requirement := &v1.Requirement{}
+			if err := cfg.Client().Resources().Get(ctx, testRequirementName, utils.TestNamespcae, requirement); err != nil {
+				return false, err
+			}
+			if requirement.Status.Phase != rqutils.PhaseReady {
+				return false, nil
+			}
+			return true, nil
+		}); err != nil {
+			t.Fatal(err, "operations not ready")
+		}
+		return context.WithValue(ctx, requirementKey{}, &requirement)
+	}).
+	Teardown(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+		requirement := ctx.Value(requirementKey{}).(*v1.Requirement)
+		if err := cfg.Client().Resources().Delete(ctx, requirement); err != nil {
+			t.Fatal(err)
+		}
+		return ctx
+	}).Feature()

test/integration/main_test.go

@@ -0,0 +1,90 @@
+package integration
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"testing"
+
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/client-go/kubernetes/scheme"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+	"sigs.k8s.io/e2e-framework/pkg/env"
+	"sigs.k8s.io/e2e-framework/pkg/envconf"
+	"sigs.k8s.io/e2e-framework/pkg/envfuncs"
+	"sigs.k8s.io/e2e-framework/support/kind"
+
+	v1 "github.com/Azure/operation-cache-controller/api/v1"
+	"github.com/Azure/operation-cache-controller/test/utils"
+)
+
+var testenv env.Environment
+
+// projectImage is the name of the image which will be build and loaded
+// with the code source changes to be tested.
+var projectImage = "example.com/operation-cache-controller:v0.0.1"
+var kindClusterName = "integration-test-cluster"
+
+func init() {
+	log.SetLogger(zap.New(zap.WriteTo(os.Stdout), zap.UseDevMode(true)))
+	utilruntime.Must(v1.AddToScheme(scheme.Scheme))
+}
+
+func TestMain(m *testing.M) {
+	// Create a new test environment configuration
+	testenv = env.New()
+
+	// Setup the test environment with Kind cluster and necessary resources
+	testenv = testenv.Setup(
+		BuildImage,
+		envfuncs.CreateCluster(kind.NewProvider(), kindClusterName),
+		envfuncs.LoadDockerImageToCluster(kindClusterName, projectImage),
+		envfuncs.CreateNamespace(utils.TestNamespcae),
+		InstallCRD,
+		DeployControllerManager,
+	)
+
+	// Teardown the test environment
+	testenv = testenv.Finish(
+		envfuncs.DeleteNamespace(utils.TestNamespcae),
+		envfuncs.DestroyCluster(kindClusterName),
+	)
+
+	// Run the tests
+	os.Exit(testenv.Run(m))
+}
+
+func BuildImage(ctx context.Context, cfg *envconf.Config) (context.Context, error) {
+	// Build the Docker image for the controller manager
+	cmd := exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", projectImage))
+	_, err := utils.Run(cmd)
+	return ctx, err
+}
+
+func InstallCRD(ctx context.Context, cfg *envconf.Config) (context.Context, error) {
+	// Install the CRD in the test environment
+	cmd := exec.Command("make", "install")
+	_, err := utils.Run(cmd)
+	return ctx, err
+}
+
+func DeployControllerManager(ctx context.Context, cfg *envconf.Config) (context.Context, error) {
+	// Deploy the controller manager in the test environment
+	cmd := exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", projectImage))
+	_, err := utils.Run(cmd)
+	return ctx, err
+}
+
+func UninstallCRD(ctx context.Context, cfg *envconf.Config) (context.Context, error) {
+	// Uninstall the CRD in the test environment
+	cmd := exec.Command("make", "uninstall")
+	_, err := utils.Run(cmd)
+	return ctx, err
+}
+func TestRealCluster(t *testing.T) {
+	// Create a new test environment configuration
+	// Run the integration tests against the Kind cluster
+	testenv.Test(t, CacheFeature)
+}

test/utils/resources.go

@@ -0,0 +1,57 @@
+package utils
+
+import (
+	appsv1 "github.com/Azure/operation-cache-controller/api/v1"
+	batchv1 "k8s.io/api/batch/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+const (
+	TestNamespcae = "operation-cache-controller-system"
+)
+
+func NewTestJobSpec(name string) batchv1.JobSpec {
+	return batchv1.JobSpec{
+		Template: corev1.PodTemplateSpec{
+			ObjectMeta: metav1.ObjectMeta{},
+			Spec: corev1.PodSpec{
+				Containers: []corev1.Container{
+					{
+						Name:    name,
+						Image:   "mcr.microsoft.com/azurelinux/busybox:1.36",
+						Command: []string{"echo", name + " job"},
+					},
+				},
+			},
+		},
+	}
+}
+
+func NewTestApplicationSpec(name string) appsv1.ApplicationSpec {
+	return appsv1.ApplicationSpec{
+		Name:      name,
+		Provision: NewTestJobSpec("provision"),
+		Teardown:  NewTestJobSpec("teardown"),
+	}
+}
+
+func NewSimpleOperationSpec(name string) *appsv1.OperationSpec {
+	return &appsv1.OperationSpec{
+		Applications: []appsv1.ApplicationSpec{NewTestApplicationSpec("app1")},
+	}
+}
+
+func NewRequirement(name, namespace string) *appsv1.Requirement {
+	return &appsv1.Requirement{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: namespace,
+		},
+		Spec: appsv1.RequirementSpec{
+			Template: appsv1.OperationSpec{
+				Applications: []appsv1.ApplicationSpec{NewTestApplicationSpec("app1")},
+			},
+		},
+	}
+}

test/utils/utils.go

@@ -198,6 +198,7 @@ func GetProjectDir() (string, error) {
 		return wd, err
 	}
 	wd = strings.Replace(wd, "/test/e2e", "", -1)
+	wd = strings.Replace(wd, "/test/integration", "", -1)
 	return wd, nil
 }