diff --git a/Makefile b/Makefile index e2b87737..4af946f6 100644 --- a/Makefile +++ b/Makefile @@ -76,6 +76,7 @@ SKIP_RESOURCE_CLEANUP ?= false USE_EXISTING_CLUSTER ?= false USE_EKS ?= true ISOLATED_MODE ?= false +GITEA_CUSTOM_INGRESS ?= false GINKGO_NOCOLOR ?= false GINKGO_LABEL_FILTER ?= short GINKGO_TESTS ?= $(ROOT_DIR)/$(TEST_DIR)/e2e/suites/... @@ -535,7 +536,8 @@ test-e2e: $(GINKGO) $(HELM) $(CLUSTERCTL) kubectl e2e-image ## Run the end-to-en -e2e.skip-resource-cleanup=$(SKIP_RESOURCE_CLEANUP) \ -e2e.use-existing-cluster=$(USE_EXISTING_CLUSTER) \ -e2e.isolated-mode=$(ISOLATED_MODE) \ - -e2e.use-eks=$(USE_EKS) + -e2e.use-eks=$(USE_EKS) \ + -e2e.gitea-custom-ingress=$(GITEA_CUSTOM_INGRESS) .PHONY: e2e-image e2e-image: ## Build the image for e2e tests diff --git a/test/e2e/const.go b/test/e2e/const.go index c76613ee..fe2ade11 100644 --- a/test/e2e/const.go +++ b/test/e2e/const.go @@ -80,6 +80,9 @@ var ( //go:embed data/cluster-templates/vsphere-kubeadm.yaml CAPIvSphereKubeadm []byte + + //go:embed data/gitea/ingress.yaml + GiteaIngress []byte ) const ( diff --git a/test/e2e/data/cluster-templates/vsphere-kubeadm.yaml b/test/e2e/data/cluster-templates/vsphere-kubeadm.yaml index 9067f4e2..68c5825b 100644 --- a/test/e2e/data/cluster-templates/vsphere-kubeadm.yaml +++ b/test/e2e/data/cluster-templates/vsphere-kubeadm.yaml @@ -57,6 +57,7 @@ spec: numCPUs: 2 os: Linux powerOffMode: hard + resourcePool: '${VSPHERE_RESOURCE_POOL}' server: '${VSPHERE_SERVER}' template: '${VSPHERE_TEMPLATE}' thumbprint: '${VSPHERE_TLS_THUMBPRINT}' @@ -186,6 +187,7 @@ spec: server: '${VSPHERE_SERVER}' template: '${VSPHERE_TEMPLATE}' thumbprint: '${VSPHERE_TLS_THUMBPRINT}' + resourcePool: '${VSPHERE_RESOURCE_POOL}' --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 kind: KubeadmConfigTemplate diff --git a/test/e2e/data/gitea/ingress.yaml b/test/e2e/data/gitea/ingress.yaml new file mode 100644 index 00000000..8fdd03f1 --- /dev/null +++ b/test/e2e/data/gitea/ingress.yaml @@ -0,0 +1,18 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: gitea-http + namespace: default +spec: + ingressClassName: ngrok # This ingress in intended to be used only with ngrok + rules: + - host: gitea.${RANCHER_HOSTNAME} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: gitea-http + port: + number: 3000 diff --git a/test/e2e/flags.go b/test/e2e/flags.go index 0a3cdeca..249007b5 100644 --- a/test/e2e/flags.go +++ b/test/e2e/flags.go @@ -54,6 +54,9 @@ type FlagValues struct { // ClusterctlBinaryPath is the path to the clusterctl binary to use. ClusterctlBinaryPath string + + // GiteaCustomIngress is the flag to enable custom ingress for Gitea. + GiteaCustomIngress bool } // InitFlags is used to specify the standard flags for the e2e tests. @@ -68,4 +71,5 @@ func InitFlags(values *FlagValues) { flag.StringVar(&values.ClusterctlBinaryPath, "e2e.clusterctl-binary-path", "helm", "path to the clusterctl binary") flag.StringVar(&values.ChartPath, "e2e.chart-path", "", "path to the operator chart") flag.BoolVar(&values.IsolatedMode, "e2e.isolated-mode", false, "if true, the test will run without ngrok and exposing the cluster to the internet. This setup will only work with CAPD or other providers that run in the same network as the bootstrap cluster.") + flag.BoolVar(&values.GiteaCustomIngress, "e2e.gitea-custom-ingress", false, "if true, the test will use a custom ingress for Gitea") } diff --git a/test/e2e/helpers.go b/test/e2e/helpers.go index 49c2208f..625ef02a 100644 --- a/test/e2e/helpers.go +++ b/test/e2e/helpers.go @@ -41,6 +41,7 @@ import ( managementv3 "github.com/rancher/turtles/internal/rancher/management/v3" provisioningv1 "github.com/rancher/turtles/internal/rancher/provisioning/v1" turtlesframework "github.com/rancher/turtles/test/framework" + networkingv1 "k8s.io/api/networking/v1" ) func SetupSpecNamespace(ctx context.Context, specName string, clusterProxy framework.ClusterProxy, artifactFolder string) (*corev1.Namespace, context.CancelFunc) { @@ -86,6 +87,7 @@ func InitScheme() *runtime.Scheme { Expect(clusterv1.AddToScheme(scheme)).To(Succeed()) Expect(provisioningv1.AddToScheme(scheme)).To(Succeed()) Expect(managementv3.AddToScheme(scheme)).To(Succeed()) + Expect(networkingv1.AddToScheme(scheme)).To(Succeed()) return scheme } diff --git a/test/e2e/specs/import_gitops.go b/test/e2e/specs/import_gitops.go index 2b657ae0..952e9489 100644 --- a/test/e2e/specs/import_gitops.go +++ b/test/e2e/specs/import_gitops.go @@ -277,6 +277,7 @@ func CreateUsingGitOpsSpec(ctx context.Context, inputGetter func() CreateUsingGi turtlesframework.RancherGetOriginalKubeconfig(ctx, turtlesframework.RancherGetClusterKubeconfigInput{ Getter: input.BootstrapClusterProxy.GetClient(), SecretName: fmt.Sprintf("%s-kubeconfig", capiCluster.Name), + ClusterName: capiCluster.Name, Namespace: capiCluster.Namespace, WriteToTempFile: true, }, originalKubeconfig) diff --git a/test/e2e/specs/import_gitops_mgmtv3.go b/test/e2e/specs/import_gitops_mgmtv3.go index 90e56382..8cc6e6a0 100644 --- a/test/e2e/specs/import_gitops_mgmtv3.go +++ b/test/e2e/specs/import_gitops_mgmtv3.go @@ -297,6 +297,7 @@ func CreateMgmtV3UsingGitOpsSpec(ctx context.Context, inputGetter func() CreateM turtlesframework.RancherGetOriginalKubeconfig(ctx, turtlesframework.RancherGetClusterKubeconfigInput{ Getter: input.BootstrapClusterProxy.GetClient(), SecretName: fmt.Sprintf("%s-kubeconfig", capiCluster.Name), + ClusterName: capiCluster.Name, Namespace: capiCluster.Namespace, WriteToTempFile: true, }, originalKubeconfig) diff --git a/test/e2e/specs/migrate_gitops_provv1_mgmtv3.go b/test/e2e/specs/migrate_gitops_provv1_mgmtv3.go index 069ccb22..d45092d2 100644 --- a/test/e2e/specs/migrate_gitops_provv1_mgmtv3.go +++ b/test/e2e/specs/migrate_gitops_provv1_mgmtv3.go @@ -341,6 +341,7 @@ func MigrateToV3UsingGitOpsSpec(ctx context.Context, inputGetter func() MigrateT turtlesframework.RancherGetOriginalKubeconfig(ctx, turtlesframework.RancherGetClusterKubeconfigInput{ Getter: input.BootstrapClusterProxy.GetClient(), SecretName: fmt.Sprintf("%s-kubeconfig", capiCluster.Name), + ClusterName: capiCluster.Name, Namespace: capiCluster.Namespace, WriteToTempFile: true, }, originalKubeconfig) diff --git a/test/e2e/suites/embedded-capi-disabled/suite_test.go b/test/e2e/suites/embedded-capi-disabled/suite_test.go index 5a2c749a..07e5e018 100644 --- a/test/e2e/suites/embedded-capi-disabled/suite_test.go +++ b/test/e2e/suites/embedded-capi-disabled/suite_test.go @@ -29,6 +29,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" "k8s.io/klog/v2" "sigs.k8s.io/cluster-api/test/framework/clusterctl" ctrl "sigs.k8s.io/controller-runtime" @@ -243,10 +244,15 @@ var _ = BeforeSuite(func() { giteaValues := map[string]string{ "gitea.admin.username": e2eConfig.GetVariable(e2e.GiteaUserNameVar), "gitea.admin.password": e2eConfig.GetVariable(e2e.GiteaUserPasswordVar), - "service.http.type": "NodePort", } + + giteaServiceType := corev1.ServiceTypeNodePort if flagVals.UseEKS { - giteaValues["service.http.type"] = "LoadBalancer" + giteaServiceType = corev1.ServiceTypeLoadBalancer + } + + if flagVals.GiteaCustomIngress { + giteaServiceType = corev1.ServiceTypeClusterIP } giteaResult = testenv.DeployGitea(ctx, testenv.DeployGiteaInput{ @@ -263,6 +269,9 @@ var _ = BeforeSuite(func() { AuthSecretName: e2e.AuthSecretName, Username: e2eConfig.GetVariable(e2e.GiteaUserNameVar), Password: e2eConfig.GetVariable(e2e.GiteaUserPasswordVar), + ServiceType: giteaServiceType, + CustomIngressConfig: e2e.GiteaIngress, + Variables: e2eConfig.Variables, }) }) diff --git a/test/e2e/suites/import-gitops-v3/suite_test.go b/test/e2e/suites/import-gitops-v3/suite_test.go index 4fcc9475..86462b9e 100644 --- a/test/e2e/suites/import-gitops-v3/suite_test.go +++ b/test/e2e/suites/import-gitops-v3/suite_test.go @@ -29,14 +29,14 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "k8s.io/klog/v2" - "sigs.k8s.io/cluster-api/test/framework/clusterctl" - ctrl "sigs.k8s.io/controller-runtime" - "github.com/rancher/turtles/test/e2e" "github.com/rancher/turtles/test/framework" turtlesframework "github.com/rancher/turtles/test/framework" "github.com/rancher/turtles/test/testenv" + corev1 "k8s.io/api/core/v1" + "k8s.io/klog/v2" + "sigs.k8s.io/cluster-api/test/framework/clusterctl" + ctrl "sigs.k8s.io/controller-runtime" ) // Test suite flags. @@ -237,10 +237,15 @@ var _ = BeforeSuite(func() { giteaValues := map[string]string{ "gitea.admin.username": e2eConfig.GetVariable(e2e.GiteaUserNameVar), "gitea.admin.password": e2eConfig.GetVariable(e2e.GiteaUserPasswordVar), - "service.http.type": "NodePort", } + + giteaServiceType := corev1.ServiceTypeNodePort if flagVals.UseEKS { - giteaValues["service.http.type"] = "LoadBalancer" + giteaServiceType = corev1.ServiceTypeLoadBalancer + } + + if flagVals.GiteaCustomIngress { + giteaServiceType = corev1.ServiceTypeClusterIP } giteaResult = testenv.DeployGitea(ctx, testenv.DeployGiteaInput{ @@ -257,6 +262,9 @@ var _ = BeforeSuite(func() { AuthSecretName: e2e.AuthSecretName, Username: e2eConfig.GetVariable(e2e.GiteaUserNameVar), Password: e2eConfig.GetVariable(e2e.GiteaUserPasswordVar), + ServiceType: giteaServiceType, + CustomIngressConfig: e2e.GiteaIngress, + Variables: e2eConfig.Variables, }) }) diff --git a/test/e2e/suites/import-gitops/suite_test.go b/test/e2e/suites/import-gitops/suite_test.go index 252946bd..c0375e35 100644 --- a/test/e2e/suites/import-gitops/suite_test.go +++ b/test/e2e/suites/import-gitops/suite_test.go @@ -29,14 +29,14 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "k8s.io/klog/v2" - "sigs.k8s.io/cluster-api/test/framework/clusterctl" - ctrl "sigs.k8s.io/controller-runtime" - "github.com/rancher/turtles/test/e2e" "github.com/rancher/turtles/test/framework" turtlesframework "github.com/rancher/turtles/test/framework" "github.com/rancher/turtles/test/testenv" + corev1 "k8s.io/api/core/v1" + "k8s.io/klog/v2" + "sigs.k8s.io/cluster-api/test/framework/clusterctl" + ctrl "sigs.k8s.io/controller-runtime" ) // Test suite flags. @@ -301,10 +301,15 @@ var _ = BeforeSuite(func() { giteaValues := map[string]string{ "gitea.admin.username": e2eConfig.GetVariable(e2e.GiteaUserNameVar), "gitea.admin.password": e2eConfig.GetVariable(e2e.GiteaUserPasswordVar), - "service.http.type": "NodePort", } + + giteaServiceType := corev1.ServiceTypeNodePort if flagVals.UseEKS { - giteaValues["service.http.type"] = "LoadBalancer" + giteaServiceType = corev1.ServiceTypeLoadBalancer + } + + if flagVals.GiteaCustomIngress { + giteaServiceType = corev1.ServiceTypeClusterIP } giteaResult = testenv.DeployGitea(ctx, testenv.DeployGiteaInput{ @@ -321,6 +326,9 @@ var _ = BeforeSuite(func() { AuthSecretName: e2e.AuthSecretName, Username: e2eConfig.GetVariable(e2e.GiteaUserNameVar), Password: e2eConfig.GetVariable(e2e.GiteaUserPasswordVar), + ServiceType: giteaServiceType, + CustomIngressConfig: e2e.GiteaIngress, + Variables: e2eConfig.Variables, }) }) diff --git a/test/e2e/suites/migrate-gitops/suite_test.go b/test/e2e/suites/migrate-gitops/suite_test.go index 36f0f891..651e8f7c 100644 --- a/test/e2e/suites/migrate-gitops/suite_test.go +++ b/test/e2e/suites/migrate-gitops/suite_test.go @@ -29,13 +29,13 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "k8s.io/klog/v2" - "sigs.k8s.io/cluster-api/test/framework/clusterctl" - ctrl "sigs.k8s.io/controller-runtime" - "github.com/rancher/turtles/test/e2e" turtlesframework "github.com/rancher/turtles/test/framework" "github.com/rancher/turtles/test/testenv" + corev1 "k8s.io/api/core/v1" + "k8s.io/klog/v2" + "sigs.k8s.io/cluster-api/test/framework/clusterctl" + ctrl "sigs.k8s.io/controller-runtime" ) // Test suite flags. @@ -188,7 +188,15 @@ var _ = BeforeSuite(func() { giteaValues := map[string]string{ "gitea.admin.username": e2eConfig.GetVariable(e2e.GiteaUserNameVar), "gitea.admin.password": e2eConfig.GetVariable(e2e.GiteaUserPasswordVar), - "service.http.type": "NodePort", + } + + giteaServiceType := corev1.ServiceTypeNodePort + if flagVals.UseEKS { + giteaServiceType = corev1.ServiceTypeLoadBalancer + } + + if flagVals.GiteaCustomIngress { + giteaServiceType = corev1.ServiceTypeClusterIP } giteaResult = testenv.DeployGitea(ctx, testenv.DeployGiteaInput{ @@ -205,6 +213,9 @@ var _ = BeforeSuite(func() { AuthSecretName: e2e.AuthSecretName, Username: e2eConfig.GetVariable(e2e.GiteaUserNameVar), Password: e2eConfig.GetVariable(e2e.GiteaUserPasswordVar), + ServiceType: giteaServiceType, + CustomIngressConfig: e2e.GiteaIngress, + Variables: e2eConfig.Variables, }) }) diff --git a/test/framework/helper.go b/test/framework/helper.go index 6fd08dd0..0bf3db73 100644 --- a/test/framework/helper.go +++ b/test/framework/helper.go @@ -38,7 +38,7 @@ type VariableLookupFunc func(key string) string // GetVariable is used to get the value for a variable. The expectation is that the variable exists in one of // the sources. Assertion will fail if its not found. The order of precedence when checking for variables is: // 1. Environment variables -// 2. Base variablesß +// 2. Base variables // This is a re-implementation of the CAPI function to add additional logging. func GetVariable(vars VariableCollection) VariableLookupFunc { Expect(vars).ToNot(BeNil(), "Variable should not be nil") diff --git a/test/framework/kube_helper.go b/test/framework/kube_helper.go index f3c3a728..11712b38 100644 --- a/test/framework/kube_helper.go +++ b/test/framework/kube_helper.go @@ -23,6 +23,7 @@ import ( . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/klog/v2" @@ -32,7 +33,7 @@ import ( const ( retryableOperationInterval = 3 * time.Second - retryableOperationTimeout = 3 * time.Minute + retryableOperationTimeout = 99 * time.Minute ) // GetNodeAddressInput is th einput to GetNodeAddress. @@ -233,3 +234,28 @@ func CreateDockerRegistrySecret(ctx context.Context, input CreateDockerRegistryS Expect(cmdCreateSecret.Error).NotTo(HaveOccurred(), "Failed creating docker registry k8s secret") Expect(cmdCreateSecret.ExitCode).To(Equal(0), "Creating secret return non-zero exit code") } + +// GetIngressHostInput is the input to GetIngressHost. +type GetIngressHostInput struct { + GetLister framework.GetLister + IngressName string + IngressNamespace string + IngressRuleIndex int +} + +// GetIngressHost gets the host from an ingress object. +func GetIngressHost(ctx context.Context, input GetIngressHostInput) string { + Expect(ctx).NotTo(BeNil(), "ctx is required for GetNodeAddress") + Expect(input.GetLister).ToNot(BeNil(), "Invalid argument. input.GetLister can't be nil when calling GetIngressHost") + + ingress := &networkingv1.Ingress{} + Eventually(func() error { + return input.GetLister.Get(ctx, client.ObjectKey{Namespace: input.IngressNamespace, Name: input.IngressName}, ingress) + }).Should(Succeed(), "Failed to get ingress") + + Expect(ingress.Spec.Rules).NotTo(HaveLen(0), "Expected ingress to have at least 1 rule") + Expect(len(ingress.Spec.Rules) >= input.IngressRuleIndex).To(BeTrue(), "Ingress rule index is greater than number of rules") + + rule := ingress.Spec.Rules[input.IngressRuleIndex] + return rule.Host +} diff --git a/test/framework/rancher_helpers.go b/test/framework/rancher_helpers.go index 0ac92340..fcd602af 100644 --- a/test/framework/rancher_helpers.go +++ b/test/framework/rancher_helpers.go @@ -43,6 +43,7 @@ type RancherGetClusterKubeconfigInput struct { Getter framework.Getter SecretName string Namespace string + ClusterName string RancherServerURL string WriteToTempFile bool } @@ -160,7 +161,7 @@ func RancherGetOriginalKubeconfig(ctx context.Context, input RancherGetClusterKu func (i *RancherGetClusterKubeconfigInput) isDockerCluster(ctx context.Context) bool { cluster := &clusterv1.Cluster{} key := client.ObjectKey{ - Name: i.SecretName, + Name: i.ClusterName, Namespace: i.Namespace, } diff --git a/test/testenv/gitea.go b/test/testenv/gitea.go index 9c0194c1..263d7280 100644 --- a/test/testenv/gitea.go +++ b/test/testenv/gitea.go @@ -21,6 +21,7 @@ import ( "fmt" "net/http" + "github.com/drone/envsubst/v2" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -47,6 +48,9 @@ type DeployGiteaInput struct { Username string Password string AuthSecretName string + CustomIngressConfig []byte + ServiceType corev1.ServiceType + Variables turtlesframework.VariableCollection } type DeployGiteaResult struct { @@ -63,12 +67,21 @@ func DeployGitea(ctx context.Context, input DeployGiteaInput) *DeployGiteaResult Expect(input.ChartVersion).ToNot(BeEmpty(), "Chartversion is required for DeployGitea") Expect(input.RolloutWaitInterval).ToNot(BeNil(), "RolloutWaitInterval is required for DeployGitea") Expect(input.ServiceWaitInterval).ToNot(BeNil(), "ServiceWaitInterval is required for DeployGitea") + Expect(input.ServiceType).ToNot(BeEmpty(), "ServiceType is required for DeployGitea") if input.Username != "" { Expect(input.Password).ToNot(BeEmpty(), "Password is required for DeployGitea if a username is supplied") Expect(input.AuthSecretName).ToNot(BeEmpty(), "AuthSecretName is required for DeployGitea if a username is supplied") } + if input.ServiceType == corev1.ServiceTypeClusterIP { + Expect(input.CustomIngressConfig).ToNot(BeEmpty(), "CustomIngressConfig is required for DeployGitea if service type is ClusterIP") + } + + if input.Values["service.http.type"] == "" { + input.Values["service.http.type"] = string(input.ServiceType) + } + result := &DeployGiteaResult{} By("Installing gitea chart") @@ -126,7 +139,8 @@ func DeployGitea(ctx context.Context, input DeployGiteaInput) *DeployGiteaResult }, input.ServiceWaitInterval...) Expect(port.NodePort).ToNot(Equal(0), "Node port for Gitea service is not set") - if input.Values["service.http.type"] == "NodePort" { + switch input.Values["service.http.type"] { + case string(corev1.ServiceTypeNodePort): By("Get Git server node port") addr := turtlesframework.GetNodeAddress(ctx, turtlesframework.GetNodeAddressInput{ Lister: input.BootstrapClusterProxy.GetClient(), @@ -135,7 +149,7 @@ func DeployGitea(ctx context.Context, input DeployGiteaInput) *DeployGiteaResult }) result.GitAddress = fmt.Sprintf("http://%s:%d", addr, port.NodePort) - } else { + case string(corev1.ServiceTypeLoadBalancer): By("Getting git server ingress address") svcRes := &WaitForServiceIngressHostnameResult{} WaitForServiceIngressHostname(ctx, WaitForServiceIngressHostnameInput{ @@ -145,6 +159,22 @@ func DeployGitea(ctx context.Context, input DeployGiteaInput) *DeployGiteaResult IngressWaitInterval: input.ServiceWaitInterval, }, svcRes) result.GitAddress = fmt.Sprintf("http://%s:%d", svcRes.Hostname, port.Port) + case string(corev1.ServiceTypeClusterIP): + By("Creating custom ingress for gitea") + variableGetter := turtlesframework.GetVariable(input.Variables) + ingress, err := envsubst.Eval(string(input.CustomIngressConfig), variableGetter) + Expect(err).ToNot(HaveOccurred()) + Expect(input.BootstrapClusterProxy.Apply(ctx, []byte(ingress))).To(Succeed()) + + By("Getting git server ingress address") + host := turtlesframework.GetIngressHost(ctx, turtlesframework.GetIngressHostInput{ + GetLister: input.BootstrapClusterProxy.GetClient(), + IngressRuleIndex: 0, + IngressName: "gitea-http", + IngressNamespace: "default", + }) + + result.GitAddress = fmt.Sprintf("https://%s", host) } if input.Username == "" {