Merge branch 'rancher-sandbox:main' into add-vsphere-e2e

rancher · Dec 7, 2023 · 60b85b5 · 60b85b5
2 parents 72bc2dd + b955ceb
commit 60b85b5
Show file tree

Hide file tree

Showing 15 changed files with 97 additions and 66 deletions.
diff --git a/.github/workflows/nightly-chart-and-image-publish.yaml b/.github/workflows/nightly-chart-and-image-publish.yaml
@@ -8,13 +8,10 @@ on:
 env:
   TAG: v0.0.0-${{ github.sha }}
   REGISTRY: ghcr.io
-  PROD_REGISTRY: ${{ secrets.REGISTRY_ENDPOINT }}
-  PROD_USERNAME: ${{ secrets.REGISTRY_USERNAME }}
-  PROD_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
   PROD_ORG: rancher-sandbox
 
 jobs:
-  build-ghcr:
+  build-and-publish-ghcr:
     runs-on: ubuntu-latest
     permissions:
       contents: read
@@ -35,14 +32,14 @@ jobs:
         username: ${{ github.actor }}
         password: ${{ secrets.GITHUB_TOKEN }}
     - name: Build docker image
-      run: make docker-build-all TAG=${{ env.TAG }}
+      run: make docker-build-all TAG=${{ env.TAG }} ORG=${{ env.PROD_ORG }}
     - name: Push docker image
-      run: make docker-push-all TAG=${{ env.TAG }}
+      run: make docker-push-all TAG=${{ env.TAG }} ORG=${{ env.PROD_ORG }}
 
   publish-helm-chart-ghcr:
     name: Publish Helm chart to GHCR
-    needs: 
-    - build-ghcr
+    needs:
+    - build-and-publish-ghcr
     permissions:
       contents: read
       packages: write
@@ -59,7 +56,7 @@ jobs:
         with:
           version: 3.8.0
       - name: Build Helm chart
-        run: make release-chart RELEASE_TAG=v0.0.0-${{ github.sha }}
+        run: make release-chart RELEASE_TAG=${{ env.TAG }} CONTROLLER_IMAGE_VERSION=${{ env.TAG }} 
       - name: Login to ghcr.io using Helm
         run: |
           echo ${{ secrets.GITHUB_TOKEN }} | helm registry login ghcr.io --username ${{ github.repository_owner }} --password-stdin
@@ -68,31 +65,10 @@ jobs:
           GHCR_REPOSITORY: ${{ github.repository_owner }}/rancher-turtles-chart
         run: |
           helm push out/package/rancher-turtles-0.0.0-${{ github.sha }}.tgz oci://ghcr.io/${{ github.repository_owner }}/rancher-turtles-chart
-
-  publish-ghcr-helm-chart-oci-image:
-    name: Publish GHCR Helm chart as OCI image
-    needs:
-      - publish-helm-chart-ghcr
-    permissions:
-      contents: read
-      packages: write
-    env:
-      GHCR_REPOSITORY: ${{ github.repository_owner }}/rancher-turtles-chart
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          ref: ${{ github.ref }}
-
-      - name: Install Helm
-        uses: Azure/setup-helm@v3
-        with:
-          version: 3.8.0
-
-      - name: Publish Helm chart as OCI image to ghcr
+      - name: Print helm install command
         run: |
-          helm chart export out/package/rancher-turtles-0.0.0-${{ github.sha }}.tgz oci://ghcr.io/${{ github.repository_owner }}/rancher-turtles-chart
+          echo "Nightly build can be installed using the following command:"
+          echo "helm install rancher-turtles oci://ghcr.io/${{ github.repository_owner }}/rancher-turtles-chart --version 0.0.0-${{ github.sha }}  -n rancher-turtles-system --create-namespace --wait"
 
   build-and-publish-prod:
     runs-on: ubuntu-latest
@@ -111,16 +87,16 @@ jobs:
     - name: Docker login to prod registry
       uses: docker/login-action@v3
       with:
-        registry: ${{ secrets.PROD_REGISTRY_ENDPOINT }}
-        username: ${{ secrets.PROD_REGISTRY_USERNAME }}
-        password: ${{ secrets.PROD_REGISTRY_PASSWORD }}
+        registry: ${{ secrets.REGISTRY_ENDPOINT }}
+        username: ${{ secrets.REGISTRY_USERNAME }}
+        password: ${{ secrets.REGISTRY_PASSWORD }}
     - name: Build Docker image for prod registry
-      run: make docker-build-all TAG=${{ env.TAG }} REGISTRY=${{ secrets.PROD_REGISTRY_ENDPOINT }} ORG=${{ secrets.PROD_ORG }}
+      run: make docker-build-all TAG=${{ env.TAG }} REGISTRY=${{ secrets.REGISTRY_ENDPOINT }} ORG=${{ env.PROD_ORG }}
     - name: Push Docker image to prod registry
-      run: make docker-push-all TAG=${{ env.TAG }} REGISTRY=${{ secrets.PROD_REGISTRY_ENDPOINT }} ORG=${{ secrets.PROD_ORG }}
+      run: make docker-push-all TAG=${{ env.TAG }} REGISTRY=${{ secrets.REGISTRY_ENDPOINT }} ORG=${{ env.PROD_ORG }}
 
-  build-and-publish-prod-helm-chart-oci-image:
-    name: Build Helm Chart and publish as OCI image to prod registry
+  publish-helm-chart-prod:
+    name: Publish OCI image of helm chart to prod registry
     needs:
       - build-and-publish-prod
     permissions:
@@ -139,10 +115,10 @@ jobs:
         with:
           version: 3.8.0
       - name: Build Helm chart
-        run: make release-chart RELEASE_TAG=${{ env.TAG }}
+        run: make release-chart RELEASE_TAG=${{ env.TAG }} CONTROLLER_IMAGE_VERSION=${{ env.TAG }} 
       - name: Login to prod registry using Helm
         run: |
-          echo ${{ secrets.PROD_REGISTRY_PASSWORD }} | helm registry login ${{ secrets.PROD_REGISTRY_ENDPOINT }} --username ${{ secrets.PROD_REGISTRY_USERNAME }} --password-stdin
+          echo ${{ secrets.REGISTRY_PASSWORD }} | helm registry login ${{ secrets.REGISTRY_ENDPOINT }} --username ${{ secrets.REGISTRY_USERNAME }} --password-stdin
       - name: Publish Helm chart as OCI image to prod registry
         run: |
-          helm push out/package/rancher-turtles-0.0.0-${{ github.sha }}.tgz oci://$${{ secrets.PROD_REGISTRY_ENDPOINT }}/$${{ secrets.PROD_ORG }}/rancher-turtles-chart
+          helm push out/package/rancher-turtles-0.0.0-${{ github.sha }}}.tgz oci://${{ secrets.REGISTRY_ENDPOINT }}/${{ env.PROD_ORG }}/rancher-turtles-chart
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -82,10 +82,11 @@ jobs:
 
   release:
     name: Create helm release
-    needs: [build-push-services]
+    needs: [multiarch]
     runs-on: ubuntu-latest
     env:
       TAG: ${{ github.ref_name }}
+      CONTROLLER_IMG: ${{ vars.REGISTRY_IMAGE }}
       PROD_REGISTRY: ${{ secrets.REGISTRY_ENDPOINT }}
       PROD_ORG: rancher-sandbox
       RELEASE_DIR: .cr-release-packages
@@ -95,12 +96,17 @@ jobs:
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
+      - name: Get prod multiarch image digest
+        run: | 
+          docker pull ${{ env.CONTROLLER_IMG }}:${{ env.TAG }}
+          multiarch_digest=$( docker inspect --format='{{index .RepoDigests 0}}' ${{ env.CONTROLLER_IMG }}:${{ env.TAG }} | sed 's/.*@//' )
+          echo "multiarch_digest=${multiarch_digest}" >> $GITHUB_ENV
       - name: Configure Git
         run: |
           git config user.name "$GITHUB_ACTOR"
           git config user.email "[email protected]"
       - name: Package operator chart
-        run: RELEASE_TAG=${GITHUB_REF##*/} CHART_PACKAGE_DIR=${RELEASE_DIR} REGISTRY=${{ env.PROD_REGISTRY }} ORG=${{ env.PROD_ORG }} make release
+        run: RELEASE_TAG=${GITHUB_REF##*/} CONTROLLER_IMAGE_VERSION=${{ env.multiarch_digest }} CHART_PACKAGE_DIR=${RELEASE_DIR} REGISTRY=${{ env.PROD_REGISTRY }} ORG=${{ env.PROD_ORG }} make release
 
       - name: Install chart-releaser
         uses: helm/[email protected]

diff --git a/.github/workflows/test_chart.yaml b/.github/workflows/test_chart.yaml
@@ -7,6 +7,7 @@ env:
   RELEASE_TAG: v0.0.1
   MANIFEST_IMG: controller
   CONTROLLER_IMG: controller
+  CONTROLLER_IMAGE_VERSION: v0.0.1
   PULL_POLICY: Never
   CERT_MANAGER_VERSION: v1.12.3
   RANCHER_VERSION: v2.7.5

diff --git a/Makefile b/Makefile
@@ -135,8 +135,9 @@ ORG ?= rancher-sandbox
 CONTROLLER_IMAGE_NAME ?= rancher-turtles
 CONTROLLER_IMG ?= $(REGISTRY)/$(ORG)/$(CONTROLLER_IMAGE_NAME)
 MANIFEST_IMG ?= $(CONTROLLER_IMG)-$(ARCH)
+CONTROLLER_IMAGE_VERSION ?= $(shell git describe --abbrev=0 2>/dev/null)
 
-# Relase
+# Release
 RELEASE_TAG ?= $(shell git describe --abbrev=0 2>/dev/null)
 PREVIOUS_TAG ?= $(shell git describe --abbrev=0 --exclude $(RELEASE_TAG) 2>/dev/null)
 HELM_CHART_TAG := $(shell echo $(RELEASE_TAG) | cut -c 2-)
@@ -476,8 +477,8 @@ release: clean-release $(RELEASE_DIR)  ## Builds and push container images using
 build-chart: $(HELM) $(KUSTOMIZE) $(RELEASE_DIR) $(CHART_RELEASE_DIR) $(CHART_PACKAGE_DIR) ## Builds the chart to publish with a release
 	$(KUSTOMIZE) build ./config/chart > $(CHART_DIR)/templates/rancher-turtles-components.yaml
 	cp -rf $(CHART_DIR)/* $(CHART_RELEASE_DIR)
-	sed -i'' -e 's@tag: .*@tag: '"$(RELEASE_TAG)"'@' $(CHART_RELEASE_DIR)/values.yaml
 	sed -i'' -e 's@image: .*@image: '"$(CONTROLLER_IMG)"'@' $(CHART_RELEASE_DIR)/values.yaml
+	sed -i'' -e 's@imageVersion: .*@imageVersion: '"$(CONTROLLER_IMAGE_VERSION)"'@' $(CHART_RELEASE_DIR)/values.yaml
 	sed -i'' -e 's@imagePullPolicy: .*@imagePullPolicy: '"$(PULL_POLICY)"'@' $(CHART_RELEASE_DIR)/values.yaml
 	cd $(CHART_RELEASE_DIR) && $(HELM) dependency update
 	$(HELM) package $(CHART_RELEASE_DIR) --app-version=$(HELM_CHART_TAG) --version=$(HELM_CHART_TAG) --destination=$(CHART_PACKAGE_DIR)
@@ -489,7 +490,6 @@ release-chart: $(HELM) $(NOTES) build-chart verify-gen
 
 .PHONY: test-e2e
 test-e2e: $(GINKGO) $(HELM) $(CLUSTERCTL) kubectl e2e-image ## Run the end-to-end tests
-	RANCHER_HOSTNAME=$(RANCHER_HOSTNAME) \
 	$(GINKGO) -v --trace -poll-progress-after=$(GINKGO_POLL_PROGRESS_AFTER) \
 		-poll-progress-interval=$(GINKGO_POLL_PROGRESS_INTERVAL) --tags=e2e --focus="$(GINKGO_FOCUS)" --label-filter="$(GINKGO_LABEL_FILTER)" \
 		$(_SKIP_ARGS) --nodes=$(GINKGO_NODES) --timeout=$(GINKGO_TIMEOUT) --no-color=$(GINKGO_NOCOLOR) \
@@ -506,7 +506,7 @@ test-e2e: $(GINKGO) $(HELM) $(CLUSTERCTL) kubectl e2e-image ## Run the end-to-en
 .PHONY: e2e-image
 e2e-image: ## Build the image for e2e tests
 	TAG=v0.0.1 $(MAKE) docker-build
-	RELEASE_TAG=v0.0.1 CONTROLLER_IMG=$(MANIFEST_IMG) $(MAKE) build-chart
+	RELEASE_TAG=v0.0.1 CONTROLLER_IMG=$(MANIFEST_IMG) CONTROLLER_IMAGE_VERSION=v0.0.1 $(MAKE) build-chart
 
 .PHONY: compile-e2e
 e2e-compile: ## Test e2e compilation

diff --git a/charts/rancher-turtles/templates/deployment.yaml b/charts/rancher-turtles/templates/deployment.yaml
@@ -45,7 +45,11 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.uid
-        image: '{{ .Values.rancherTurtles.image }}:{{ .Values.rancherTurtles.tag }}'
+        {{- if (contains "sha256:" .Values.rancherTurtles.imageVersion) }}
+        image: '{{ .Values.rancherTurtles.image }}@{{ .Values.rancherTurtles.imageVersion }}'
+        {{- else }}
+        image: '{{ .Values.rancherTurtles.image }}:{{ .Values.rancherTurtles.imageVersion }}'
+        {{- end}}
         imagePullPolicy: '{{ .Values.rancherTurtles.imagePullPolicy }}'
         livenessProbe:
           httpGet:

diff --git a/charts/rancher-turtles/values.yaml b/charts/rancher-turtles/values.yaml
@@ -1,6 +1,6 @@
 rancherTurtles:
   image: controller
-  tag: v0.0.0
+  imageVersion: v0.0.0
   imagePullPolicy: Never
   namespace: rancher-turtles-system
   managerArguments: []

diff --git a/config/chart/kustomization.yaml b/config/chart/kustomization.yaml
@@ -2,7 +2,7 @@ namespace: "{{ .Values.rancherTurtles.namespace }}"
 images:
 - name: controller
   newName: "{{ .Values.rancherTurtles.image }}"
-  newTag: "{{ .Values.rancherTurtles.tag }}"
+  newTag: "{{ .Values.rancherTurtles.imageVersion }}"
 namePrefix: rancher-turtles-
 bases:
 - ../rbac

diff --git a/test/e2e/flags.go b/test/e2e/flags.go
@@ -37,6 +37,9 @@ type FlagValues struct {
 	// HelmBinaryPath is the path to the helm binary.
 	HelmBinaryPath string
 
+	// HelmExtraValuesDir is the location where extra values files will be stored.
+	HelmExtraValuesDir string
+
 	// ChartPath is the path to the operator chart.
 	ChartPath string
 
@@ -55,6 +58,7 @@ func InitFlags(values *FlagValues) {
 	flag.BoolVar(&values.SkipCleanup, "e2e.skip-resource-cleanup", false, "if true, the resource cleanup after tests will be skipped")
 	flag.BoolVar(&values.UseExistingCluster, "e2e.use-existing-cluster", false, "if true, the test uses the current cluster instead of creating a new one (default discovery rules apply)")
 	flag.StringVar(&values.HelmBinaryPath, "e2e.helm-binary-path", "helm", "path to the helm binary")
+	flag.StringVar(&values.HelmExtraValuesDir, "e2e.helm-extra-values-path", "/tmp", "path to the extra values file")
 	flag.StringVar(&values.ClusterctlBinaryPath, "e2e.clusterctl-binary-path", "helm", "path to the clusterctl binary")
 	flag.StringVar(&values.ChartPath, "e2e.chart-path", "", "path to the operator chart")
 	flag.BoolVar(&values.IsolatedMode, "e2e.isolated-mode", false, "if true, the test will run without ngrok and exposing the cluster to the internet. This setup will only work with CAPD or other providers that run in the same network as the bootstrap cluster.")

diff --git a/test/e2e/suites/embedded-capi-disabled/suite_test.go b/test/e2e/suites/embedded-capi-disabled/suite_test.go
@@ -103,11 +103,11 @@ var _ = BeforeSuite(func() {
 	if flagVals.IsolatedMode {
 		hostName = setupClusterResult.IsolatedHostName
 	}
-	turtlesframework.Byf("Rancher hostname is %s", hostName)
 
 	testenv.RancherDeployIngress(ctx, testenv.RancherDeployIngressInput{
 		BootstrapClusterProxy:    setupClusterResult.BootstrapClusterProxy,
 		HelmBinaryPath:           flagVals.HelmBinaryPath,
+		HelmExtraValuesPath:      filepath.Join(flagVals.HelmExtraValuesDir, "deploy-rancher-ingress.yaml"),
 		IsolatedMode:             flagVals.IsolatedMode,
 		NginxIngress:             e2e.NginxIngress,
 		NginxIngressNamespace:    e2e.NginxIngressNamespace,
@@ -125,6 +125,7 @@ var _ = BeforeSuite(func() {
 	testenv.DeployRancher(ctx, testenv.DeployRancherInput{
 		BootstrapClusterProxy:  setupClusterResult.BootstrapClusterProxy,
 		HelmBinaryPath:         flagVals.HelmBinaryPath,
+		HelmExtraValuesPath:    filepath.Join(flagVals.HelmExtraValuesDir, "deploy-rancher.yaml"),
 		InstallCertManager:     true,
 		CertManagerChartPath:   e2eConfig.GetVariable(e2e.CertManagerPathVar),
 		CertManagerUrl:         e2eConfig.GetVariable(e2e.CertManagerUrlVar),

diff --git a/test/e2e/suites/import-gitops/suite_test.go b/test/e2e/suites/import-gitops/suite_test.go
@@ -103,11 +103,11 @@ var _ = BeforeSuite(func() {
 	if flagVals.IsolatedMode {
 		hostName = setupClusterResult.IsolatedHostName
 	}
-	turtlesframework.Byf("Rancher hostname is %s", hostName)
 
 	testenv.RancherDeployIngress(ctx, testenv.RancherDeployIngressInput{
 		BootstrapClusterProxy:    setupClusterResult.BootstrapClusterProxy,
 		HelmBinaryPath:           flagVals.HelmBinaryPath,
+		HelmExtraValuesPath:      filepath.Join(flagVals.HelmExtraValuesDir, "deploy-rancher-ingress.yaml"),
 		IsolatedMode:             flagVals.IsolatedMode,
 		NginxIngress:             e2e.NginxIngress,
 		NginxIngressNamespace:    e2e.NginxIngressNamespace,
@@ -123,6 +123,7 @@ var _ = BeforeSuite(func() {
 	testenv.DeployRancher(ctx, testenv.DeployRancherInput{
 		BootstrapClusterProxy:  setupClusterResult.BootstrapClusterProxy,
 		HelmBinaryPath:         flagVals.HelmBinaryPath,
+		HelmExtraValuesPath:    filepath.Join(flagVals.HelmExtraValuesDir, "deploy-rancher.yaml"),
 		InstallCertManager:     true,
 		CertManagerChartPath:   e2eConfig.GetVariable(e2e.CertManagerPathVar),
 		CertManagerUrl:         e2eConfig.GetVariable(e2e.CertManagerUrlVar),

diff --git a/test/e2e/suites/update-labels/suite_test.go b/test/e2e/suites/update-labels/suite_test.go
@@ -102,7 +102,6 @@ var _ = BeforeSuite(func() {
 	if flagVals.IsolatedMode {
 		hostName = setupClusterResult.IsolatedHostName
 	}
-	turtlesframework.Byf("Rancher hostname is %s", hostName)
 
 	testenv.DeployRancherTurtles(ctx, testenv.DeployRancherTurtlesInput{
 		BootstrapClusterProxy:        setupClusterResult.BootstrapClusterProxy,
@@ -126,6 +125,7 @@ var _ = BeforeSuite(func() {
 	testenv.RancherDeployIngress(ctx, testenv.RancherDeployIngressInput{
 		BootstrapClusterProxy:    setupClusterResult.BootstrapClusterProxy,
 		HelmBinaryPath:           flagVals.HelmBinaryPath,
+		HelmExtraValuesPath:      filepath.Join(flagVals.HelmExtraValuesDir, "deploy-rancher-ingress.yaml"),
 		IsolatedMode:             flagVals.IsolatedMode,
 		NginxIngress:             e2e.NginxIngress,
 		NginxIngressNamespace:    e2e.NginxIngressNamespace,
@@ -141,6 +141,7 @@ var _ = BeforeSuite(func() {
 	testenv.DeployRancher(ctx, testenv.DeployRancherInput{
 		BootstrapClusterProxy:  setupClusterResult.BootstrapClusterProxy,
 		HelmBinaryPath:         flagVals.HelmBinaryPath,
+		HelmExtraValuesPath:    filepath.Join(flagVals.HelmExtraValuesDir, "deploy-rancher.yaml"),
 		InstallCertManager:     false,
 		RancherChartRepoName:   "rancher-latest",
 		RancherChartURL:        "https://releases.rancher.com/server-charts/latest",

diff --git a/test/e2e/suites/v2prov/suite_test.go b/test/e2e/suites/v2prov/suite_test.go
@@ -102,11 +102,11 @@ var _ = BeforeSuite(func() {
 	if flagVals.IsolatedMode {
 		hostName = setupClusterResult.IsolatedHostName
 	}
-	turtlesframework.Byf("Rancher hostname is %s", hostName)
 
 	testenv.RancherDeployIngress(ctx, testenv.RancherDeployIngressInput{
 		BootstrapClusterProxy:    setupClusterResult.BootstrapClusterProxy,
 		HelmBinaryPath:           flagVals.HelmBinaryPath,
+		HelmExtraValuesPath:      filepath.Join(flagVals.HelmExtraValuesDir, "deploy-rancher-ingress.yaml"),
 		IsolatedMode:             flagVals.IsolatedMode,
 		NginxIngress:             e2e.NginxIngress,
 		NginxIngressNamespace:    e2e.NginxIngressNamespace,
@@ -122,6 +122,7 @@ var _ = BeforeSuite(func() {
 	testenv.DeployRancher(ctx, testenv.DeployRancherInput{
 		BootstrapClusterProxy:  setupClusterResult.BootstrapClusterProxy,
 		HelmBinaryPath:         flagVals.HelmBinaryPath,
+		HelmExtraValuesPath:    filepath.Join(flagVals.HelmExtraValuesDir, "deploy-rancher.yaml"),
 		InstallCertManager:     true,
 		CertManagerChartPath:   e2eConfig.GetVariable(e2e.CertManagerPathVar),
 		CertManagerUrl:         e2eConfig.GetVariable(e2e.CertManagerUrlVar),

diff --git a/test/framework/rancher_helpers.go b/test/framework/rancher_helpers.go
@@ -81,8 +81,6 @@ func RancherGetClusterKubeconfig(ctx context.Context, input RancherGetClusterKub
 	serverURL.Host = input.RancherServerURL
 	cluster.Server = serverURL.String()
 
-	Byf("Updated kubeconfig with new server-url of %s", cluster.Server)
-
 	content, err = clientcmd.Write(*cfg)
 	Expect(err).NotTo(HaveOccurred(), "Failed to save updated kubeconfig")