From 7864fb04c296604a719e4988eec31d8b7e470e2c Mon Sep 17 00:00:00 2001 From: Richard Case Date: Mon, 19 Feb 2024 09:56:28 +0000 Subject: [PATCH 1/6] chore: change the number of control plane nodes Signed-off-by: Richard Case --- test/e2e/e2e_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 12641697..8b17a1d6 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -221,7 +221,7 @@ var _ = Describe("Workload cluster creation", func() { Namespace: namespace.Name, ClusterName: clusterName, KubernetesVersion: e2eConfig.GetVariable(KubernetesVersion), - ControlPlaneMachineCount: pointer.Int64Ptr(1), + ControlPlaneMachineCount: pointer.Int64Ptr(3), //TODO: change this back to 1 when scaling is supported WorkerMachineCount: pointer.Int64Ptr(1), }, WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"), From e115e4cf3bfa82634831388c984ca1902fb91606 Mon Sep 17 00:00:00 2001 From: Furkat Gofurov Date: Wed, 24 Jan 2024 22:23:28 +0200 Subject: [PATCH 2/6] Fix e2e tests Signed-off-by: Furkat Gofurov --- Makefile | 2 +- metadata.yaml | 2 +- test/e2e/config/e2e_conf.yaml | 119 +++++++++++------- .../cluster-template-docker.yaml | 8 +- 4 files changed, 77 insertions(+), 54 deletions(-) diff --git a/Makefile b/Makefile index cbbb7b29..c6c5bab9 100644 --- a/Makefile +++ b/Makefile @@ -99,7 +99,7 @@ GOLANGCI_LINT_VER := v1.55.1 GOLANGCI_LINT_BIN := golangci-lint GOLANGCI_LINT := $(abspath $(TOOLS_BIN_DIR)/$(GOLANGCI_LINT_BIN)) -GINKGO_VER := v2.9.4 +GINKGO_VER := v2.14.0 GINKGO_BIN := ginkgo GINKGO := $(abspath $(TOOLS_BIN_DIR)/$(GINKGO_BIN)-$(GINKGO_VER)) GINKGO_PKG := github.com/onsi/ginkgo/v2/ginkgo diff --git a/metadata.yaml b/metadata.yaml index e54a1f5a..db2c55a9 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -11,4 +11,4 @@ releaseSeries: contract: v1beta1 - major: 0 minor: 2 - contract: v1beta1 \ No newline at end of file + contract: v1beta1 diff --git a/test/e2e/config/e2e_conf.yaml b/test/e2e/config/e2e_conf.yaml index 0928ffe7..2cfaaed2 100644 --- a/test/e2e/config/e2e_conf.yaml +++ b/test/e2e/config/e2e_conf.yaml @@ -10,63 +10,86 @@ providers: - name: cluster-api type: CoreProvider versions: - - name: v1.3.0 - value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.3.0/core-components.yaml - type: url - contract: v1beta1 - files: - - sourcePath: "../data/shared/v1beta1/metadata.yaml" - replacements: - - old: "imagePullPolicy: Always" - new: "imagePullPolicy: IfNotPresent" - - old: "--leader-elect" - new: "--leader-elect=false" + - name: "v1.6.0" + value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.6.0/core-components.yaml" + type: "url" + contract: v1beta1 + files: + - sourcePath: "../data/shared/v1beta1/metadata.yaml" + replacements: + - old: "imagePullPolicy: Always" + new: "imagePullPolicy: IfNotPresent" + - old: "--leader-elect" + new: "--leader-elect=false" - name: docker type: InfrastructureProvider versions: - - name: v1.3.3 - value: "https://github.com/belgaied2/cluster-api/releases/download/v1.3.3-cabpr-fix/infrastructure-components.yaml" # NOTE: this uses the docker form for the time being - type: url - files: - - sourcePath: "../data/infrastructure/cluster-template-docker.yaml" - - sourcePath: "../data/shared/v1beta1/metadata.yaml" - replacements: - - old: "imagePullPolicy: Always" - new: "imagePullPolicy: IfNotPresent" - - old: "--leader-elect" - new: "--leader-elect=false" - - old: --metrics-addr=127.0.0.1:8080 - new: --metrics-addr=:8080 - contract: v1beta1 - + - name: "v1.6.0" + value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.6.0/infrastructure-components-development.yaml" + type: "url" + contract: v1beta1 + files: + - sourcePath: "../data/infrastructure/cluster-template-docker.yaml" + - sourcePath: "../data/shared/v1beta1/metadata.yaml" + replacements: + - old: "imagePullPolicy: Always" + new: "imagePullPolicy: IfNotPresent" + - old: "--leader-elect" + new: "--leader-elect=false" + - old: --metrics-addr=127.0.0.1:8080 + new: --metrics-addr=:8080 - name: rke2-control-plane type: ControlPlaneProvider versions: - - name: v0.1.99 - value: "../../../controlplane/config/default" - contract: v1beta1 - files: - - sourcePath: "../../../metadata.yaml" - targetName: "metadata.yaml" - replacements: - - old: "imagePullPolicy: Always" - new: "imagePullPolicy: IfNotPresent" - - old: "--leader-elect" - new: "--leader-elect=false" + - name: "v0.2.3" + value: "https://github.com/rancher-sandbox/cluster-api-provider-rke2/releases/download/v0.2.3/control-plane-components.yaml" + type: "url" + contract: v1beta1 + files: + - sourcePath: "../../../metadata.yaml" + targetName: "metadata.yaml" + replacements: + - old: "imagePullPolicy: Always" + new: "imagePullPolicy: IfNotPresent" + - old: "--leader-elect" + new: "--leader-elect=false" + - name: v0.2.99 + value: "../../../controlplane/config/default" + contract: v1beta1 + files: + - sourcePath: "../../../metadata.yaml" + targetName: "metadata.yaml" + replacements: + - old: "imagePullPolicy: Always" + new: "imagePullPolicy: IfNotPresent" + - old: "--leader-elect" + new: "--leader-elect=false" - name: rke2-bootstrap type: BootstrapProvider versions: - - name: v0.1.99 - value: ../../../bootstrap/config/default - contract: v1beta1 - files: - - sourcePath: "../../../metadata.yaml" - targetName: "metadata.yaml" - replacements: - - old: "imagePullPolicy: Always" - new: "imagePullPolicy: IfNotPresent" - - old: "--leader-elect" - new: "--leader-elect=false" + - name: "v0.2.3" + value: "https://github.com/rancher-sandbox/cluster-api-provider-rke2/releases/download/v0.2.3/bootstrap-components.yaml" + type: "url" + contract: v1beta1 + files: + - sourcePath: "../../../metadata.yaml" + targetName: "metadata.yaml" + replacements: + - old: "imagePullPolicy: Always" + new: "imagePullPolicy: IfNotPresent" + - old: "--leader-elect" + new: "--leader-elect=false" + - name: v0.2.99 + value: ../../../bootstrap/config/default + contract: v1beta1 + files: + - sourcePath: "../../../metadata.yaml" + targetName: "metadata.yaml" + replacements: + - old: "imagePullPolicy: Always" + new: "imagePullPolicy: IfNotPresent" + - old: "--leader-elect" + new: "--leader-elect=false" variables: KUBERNETES_VERSION_MANAGEMENT: "v1.28.0" diff --git a/test/e2e/data/infrastructure/cluster-template-docker.yaml b/test/e2e/data/infrastructure/cluster-template-docker.yaml index 017e2189..43af5fe8 100644 --- a/test/e2e/data/infrastructure/cluster-template-docker.yaml +++ b/test/e2e/data/infrastructure/cluster-template-docker.yaml @@ -13,7 +13,7 @@ spec: - 10.46.0.0/16 serviceDomain: cluster.local controlPlaneRef: - apiVersion: controlplane.cluster.x-k8s.io/v1alpha1 + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 kind: RKE2ControlPlane name: ${CLUSTER_NAME}-control-plane infrastructureRef: @@ -26,7 +26,7 @@ kind: DockerCluster metadata: name: ${CLUSTER_NAME} --- -apiVersion: controlplane.cluster.x-k8s.io/v1alpha1 +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 kind: RKE2ControlPlane metadata: name: ${CLUSTER_NAME}-control-plane @@ -67,7 +67,7 @@ spec: clusterName: ${CLUSTER_NAME} bootstrap: configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1alpha1 + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 kind: RKE2ConfigTemplate name: ${CLUSTER_NAME}-md-0 infrastructureRef: @@ -84,7 +84,7 @@ spec: spec: customImage: kindest/node:${KIND_IMAGE_VERSION} --- -apiVersion: bootstrap.cluster.x-k8s.io/v1alpha1 +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 kind: RKE2ConfigTemplate metadata: name: ${CLUSTER_NAME}-md-0 From f637daff090a66630575d2df86d2162d7fd83686 Mon Sep 17 00:00:00 2001 From: Danil Grigorev Date: Mon, 19 Feb 2024 13:41:01 +0100 Subject: [PATCH 3/6] Revert api bump in docker tempalte for release-0.2 Signed-off-by: Danil Grigorev --- test/e2e/data/infrastructure/cluster-template-docker.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/e2e/data/infrastructure/cluster-template-docker.yaml b/test/e2e/data/infrastructure/cluster-template-docker.yaml index 43af5fe8..017e2189 100644 --- a/test/e2e/data/infrastructure/cluster-template-docker.yaml +++ b/test/e2e/data/infrastructure/cluster-template-docker.yaml @@ -13,7 +13,7 @@ spec: - 10.46.0.0/16 serviceDomain: cluster.local controlPlaneRef: - apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + apiVersion: controlplane.cluster.x-k8s.io/v1alpha1 kind: RKE2ControlPlane name: ${CLUSTER_NAME}-control-plane infrastructureRef: @@ -26,7 +26,7 @@ kind: DockerCluster metadata: name: ${CLUSTER_NAME} --- -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +apiVersion: controlplane.cluster.x-k8s.io/v1alpha1 kind: RKE2ControlPlane metadata: name: ${CLUSTER_NAME}-control-plane @@ -67,7 +67,7 @@ spec: clusterName: ${CLUSTER_NAME} bootstrap: configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + apiVersion: bootstrap.cluster.x-k8s.io/v1alpha1 kind: RKE2ConfigTemplate name: ${CLUSTER_NAME}-md-0 infrastructureRef: @@ -84,7 +84,7 @@ spec: spec: customImage: kindest/node:${KIND_IMAGE_VERSION} --- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +apiVersion: bootstrap.cluster.x-k8s.io/v1alpha1 kind: RKE2ConfigTemplate metadata: name: ${CLUSTER_NAME}-md-0 From a140d7bd8c83b4a6e8bd3b4eb623025085d28f07 Mon Sep 17 00:00:00 2001 From: Danil Grigorev Date: Tue, 20 Feb 2024 16:14:42 +0100 Subject: [PATCH 4/6] :seedling: Enable e2e as a pre-submit job Enabling e2e workflow as a pre-submit job, now that the e2e nightly tests are passing. Signed-off-by: Danil Grigorev --- .github/workflows/e2e.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index dffe39c3..482f1d7e 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -1,5 +1,6 @@ name: E2E tests on: + pull_request: schedule: - cron: '0 1 * * *' workflow_dispatch: @@ -38,4 +39,4 @@ jobs: with: name: e2e-artifacts path: _artifacts - if-no-files-found: ignore \ No newline at end of file + if-no-files-found: ignore From afa2c9682debad501324b7f7d660fc4205328c70 Mon Sep 17 00:00:00 2001 From: Richard Case Date: Fri, 17 Nov 2023 14:22:28 +0100 Subject: [PATCH 5/6] fix: bump versions and change template for e2e Signed-off-by: Richard Case --- test/e2e/config/e2e_conf.yaml | 6 ++ .../cluster-template-docker.yaml | 71 +++++++++++++++++++ test/e2e/data/shared/v1beta1/metadata.yaml | 2 +- 3 files changed, 78 insertions(+), 1 deletion(-) diff --git a/test/e2e/config/e2e_conf.yaml b/test/e2e/config/e2e_conf.yaml index 2cfaaed2..10597bc8 100644 --- a/test/e2e/config/e2e_conf.yaml +++ b/test/e2e/config/e2e_conf.yaml @@ -5,6 +5,12 @@ images: loadBehavior: mustLoad - name: ghcr.io/rancher-sandbox/cluster-api-provider-rke2-controlplane:dev loadBehavior: mustLoad + - name: quay.io/jetstack/cert-manager-cainjector:v1.13.2 + loadBehavior: tryLoad + - name: quay.io/jetstack/cert-manager-webhook:v1.13.2 + loadBehavior: tryLoad + - name: quay.io/jetstack/cert-manager-controller:v1.13.2 + loadBehavior: tryLoad providers: - name: cluster-api diff --git a/test/e2e/data/infrastructure/cluster-template-docker.yaml b/test/e2e/data/infrastructure/cluster-template-docker.yaml index 017e2189..2c0f01c1 100644 --- a/test/e2e/data/infrastructure/cluster-template-docker.yaml +++ b/test/e2e/data/infrastructure/cluster-template-docker.yaml @@ -1,3 +1,66 @@ +apiVersion: v1 +data: + value: |- + # generated by kind + global + log /dev/log local0 + log /dev/log local1 notice + daemon + # limit memory usage to approximately 18 MB + # (see https://github.com/kubernetes-sigs/kind/pull/3115) + maxconn 100000 + + resolvers docker + nameserver dns 127.0.0.11:53 + + defaults + log global + mode tcp + option dontlognull + # TODO: tune these + timeout connect 5000 + timeout client 50000 + timeout server 50000 + # allow to boot despite dns don't resolve backends + default-server init-addr none + + frontend stats + bind *:8404 + stats enable + stats uri / + stats refresh 10s + + frontend control-plane + bind *:{{ .FrontendControlPlanePort }} + {{ if .IPv6 -}} + bind :::{{ .FrontendControlPlanePort }}; + {{- end }} + default_backend kube-apiservers + + backend kube-apiservers + option httpchk GET /healthz + http-check expect status 401 + # TODO: we should be verifying (!) + {{range $server, $address := .BackendServers}} + server {{ $server }} {{ JoinHostPort $address $.BackendControlPlanePort }} check check-ssl verify none resolvers docker resolve-prefer {{ if $.IPv6 -}} ipv6 {{- else -}} ipv4 {{- end }} + {{- end}} + + frontend rke2-join + bind *:9345 + {{ if .IPv6 -}} + bind :::9345; + {{- end }} + default_backend rke2-servers + + backend rke2-servers + option httpchk GET /v1-rke2/readyz + http-check expect status 403 + {{range $server, $address := .BackendServers}} + server {{ $server }} {{ $address }}:9345 check check-ssl verify none + {{- end}} +kind: ConfigMap +metadata: + name: ${CLUSTER_NAME}-lb-config --- apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster @@ -25,6 +88,10 @@ apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: DockerCluster metadata: name: ${CLUSTER_NAME} +spec: + loadBalancer: + customHAProxyConfigTemplateRef: + name: ${CLUSTER_NAME}-lb-config --- apiVersion: controlplane.cluster.x-k8s.io/v1alpha1 kind: RKE2ControlPlane @@ -36,6 +103,10 @@ spec: version: ${KUBERNETES_VERSION}+rke2r1 nodeAnnotations: test: "true" + serverConfig: + disableComponents: + kubernetesComponents: + - cloudController infrastructureRef: apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: DockerMachineTemplate diff --git a/test/e2e/data/shared/v1beta1/metadata.yaml b/test/e2e/data/shared/v1beta1/metadata.yaml index 01fd595b..37d745d2 100644 --- a/test/e2e/data/shared/v1beta1/metadata.yaml +++ b/test/e2e/data/shared/v1beta1/metadata.yaml @@ -2,5 +2,5 @@ apiVersion: clusterctl.cluster.x-k8s.io/v1alpha3 kind: Metadata releaseSeries: - major: 1 - minor: 3 + minor: 6 contract: v1beta1 \ No newline at end of file From caad9b444589d798bb2b3013ec0e5bf959a36667 Mon Sep 17 00:00:00 2001 From: Richard Case Date: Sat, 18 Nov 2023 20:02:34 +0100 Subject: [PATCH 6/6] test: add disk space cleanup Signed-off-by: Richard Case --- .github/workflows/e2e.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 482f1d7e..65ba0c56 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -9,6 +9,11 @@ jobs: e2e-tests: runs-on: ubuntu-latest steps: + - name: Free Disk Space + uses: jlumbroso/free-disk-space@main + with: + tool-cache: false + docker-images: false - name: Checkout code uses: actions/checkout@v4 with: