Merge branch 'master' into preflight_add_update

.github/workflows/golint.yml

@@ -25,7 +25,7 @@ jobs:
         go-version-file: go.mod
 
     - name: Run golangci-lint
-      uses: golangci/golangci-lint-action@v4
+      uses: golangci/golangci-lint-action@v6
       with:
         version: v1.56.2
         args: -v --timeout 15m

.sha256sum

@@ -6,4 +6,4 @@ b1f1de0fe40d05de90742b17928968923b936adc294000f58974f50a297581dd  swagger/redhat
 c023515341196746454c0ae7af077d40d3ec13f6b88b33cb558f0a7ab17a5a24  swagger/redhatopenshift/resource-manager/Microsoft.RedHatOpenShift/preview/2023-07-01-preview/redhatopenshift.json
 440748951dd1c3b34b5ccbdcb7cd966e3b89490887a1f1d64429561fad789515  swagger/redhatopenshift/resource-manager/Microsoft.RedHatOpenShift/stable/2023-09-04/redhatopenshift.json
 74a46fdde6ceb0121fe1515c7e11e902dd921b54cffe693307fb02b3dc88f26e  swagger/redhatopenshift/resource-manager/Microsoft.RedHatOpenShift/stable/2023-11-22/redhatopenshift.json
-a27184734436629e24b344c3b5c015437f144e18e7eddce7e252a1ed4cda7bca  swagger/redhatopenshift/resource-manager/Microsoft.RedHatOpenShift/preview/2024-08-12-preview/redhatopenshift.json
+f1e9d42d7c4c0081282e065e7845455db28ed6924687f1acecafb5fbc43ae0c3  swagger/redhatopenshift/resource-manager/Microsoft.RedHatOpenShift/preview/2024-08-12-preview/redhatopenshift.json

Dockerfile.ci-azext-aro

@@ -0,0 +1,12 @@
+FROM mcr.microsoft.com/azure-cli:2.61.0 AS builder
+
+RUN pip install pytest
+COPY /python /data/
+
+WORKDIR /data/az/aro
+RUN pytest --ignore=azext_aro/tests/latest/integration
+RUN python3 setup.py bdist_wheel
+
+FROM mcr.microsoft.com/azure-cli:2.61.0-cbl-mariner2.0 AS final
+COPY --from=builder /data/az/aro/dist /opt/az
+RUN az extension add --yes --source /opt/az/aro-*-py2.py3-none-any.whl

Dockerfile.ci-rp

@@ -1,11 +1,12 @@
 ARG REGISTRY
-ARG VERSION
+ARG ARO_VERSION
 
 ###############################################################################
 # Stage 1: Build the SRE Portal Assets
 # builder is responsible for all compilation and validation of the RP
 ###############################################################################
 FROM ${REGISTRY}/ubi8/nodejs-16  as portal-build
+LABEL aro-portal-build=true
 WORKDIR /build/portal/v2
 USER root
 
@@ -22,6 +23,8 @@ RUN npm run lint && npm run build
 # Stage 2: Compile the Golang RP code
 ###############################################################################
 FROM ${REGISTRY}/ubi8/go-toolset:1.20.12-5 AS builder
+ARG ARO_VERSION
+LABEL aro-builder=true
 USER root
 WORKDIR /app
 
@@ -49,17 +52,18 @@ COPY --from=portal-build /build/pkg/portal/assets/v2/build /app/pkg/portal/asset
 # Lint, generate, build, and test
 RUN golangci-lint run --verbose
 RUN go generate ./...
-RUN go build -ldflags "-X github.com/Azure/ARO-RP/pkg/util/version.GitCommit=${VERSION}" ./cmd/aro
-RUN go test ./test/e2e/... -tags e2e,codec.safe -c -ldflags "-X github.com/Azure/ARO-RP/pkg/util/version.GitCommit=${VERSION}" -o e2e.test
+RUN go build -ldflags "-X github.com/Azure/ARO-RP/pkg/util/version.GitCommit=${ARO_VERSION}" ./cmd/aro
+RUN go test ./test/e2e/... -tags e2e,codec.safe -c -ldflags "-X github.com/Azure/ARO-RP/pkg/util/version.GitCommit=${ARO_VERSION}" -o e2e.test
 
 # Additional tests
-RUN ARO_RUN_PKI_TESTS=nope go run gotest.tools/[email protected] --format pkgname --junitfile report.xml -- -coverprofile=cover.out ./...
+RUN ARO_SKIP_PKI_TESTS=true go run gotest.tools/[email protected] --format pkgname --junitfile report.xml -- -coverprofile=cover.out ./...
 RUN hack/fips/validate-fips.sh ./aro
 
 ###############################################################################
 # Stage 3: final is our slim image with minimal layers and tools
 ###############################################################################
 FROM ${REGISTRY}/ubi8/ubi-minimal AS final
+LABEL aro-final=true
 RUN microdnf update && microdnf clean all
 COPY --from=builder /app/aro /app/e2e.test /usr/local/bin/
 ENTRYPOINT ["aro"]

Dockerfile.gatekeeper

@@ -14,7 +14,7 @@ WORKDIR ${GOPATH}/src/github.com/open-policy-agent/gatekeeper
 USER root
 RUN curl -Lq $DOWNLOAD_URL | tar -xz --strip-components=1
 
-RUN go build -mod vendor -a -ldflags "-X github.com/open-policy-agent/gatekeeper/pkg/version.Version=latest" -o manager
+RUN go build -mod vendor -a -ldflags "-X github.com/open-policy-agent/gatekeeper/pkg/version.Version=$GATEKEEPER_VERSION" -o manager
 
 #### Runtime container
 FROM ${REGISTRY}/ubi8/ubi-minimal:latest

Makefile

@@ -14,8 +14,7 @@ FLUENTBIT_VERSION = 1.9.10
 FLUENTBIT_IMAGE ?= ${RP_IMAGE_ACR}.azurecr.io/fluentbit:$(FLUENTBIT_VERSION)-cm$(MARINER_VERSION)
 AUTOREST_VERSION = 3.6.3
 AUTOREST_IMAGE = quay.io/openshift-on-azure/autorest:${AUTOREST_VERSION}
-GATEKEEPER_VERSION = v3.10.0
-GATEKEEPER_IMAGE ?= ${RP_IMAGE_ACR}.azurecr.io/gatekeeper:$(GATEKEEPER_VERSION)
+GATEKEEPER_VERSION = v3.15.1
 GOTESTSUM = gotest.tools/[email protected]
 
 ifneq ($(shell uname -s),Darwin)
@@ -40,6 +39,7 @@ else
 endif
 
 ARO_IMAGE ?= $(ARO_IMAGE_BASE):$(VERSION)
+GATEKEEPER_IMAGE ?= ${REGISTRY}/gatekeeper:$(GATEKEEPER_VERSION)
 
 check-release:
 # Check that VERSION is a valid tag when building an official release (when RELEASE=true).
@@ -67,6 +67,10 @@ az: pyenv
 	python3 ./setup.py bdist_wheel || true && \
 	rm -f ~/.azure/commandIndex.json # https://github.com/Azure/azure-cli/issues/14997
 
+.PHONY: azext-aro
+azext-aro:
+	docker build --platform=linux/amd64 . -f Dockerfile.ci-azext-aro --no-cache=$(NO_CACHE) -t azext-aro:latest
+
 clean:
 	rm -rf python/az/aro/{aro.egg-info,build,dist} aro
 	find python -type f -name '*.pyc' -delete
@@ -77,7 +81,10 @@ client: generate
 	hack/build-client.sh "${AUTOREST_IMAGE}" 2020-04-30 2021-09-01-preview 2022-04-01 2022-09-04 2023-04-01 2023-07-01-preview 2023-09-04 2023-11-22 2024-08-12-preview
 
 ci-rp: fix-macos-vendor
-	docker build . -f Dockerfile.ci-rp --ulimit=nofile=4096:4096 --build-arg REGISTRY=$(REGISTRY) --build-arg VERSION=$(VERSION) --no-cache=$(NO_CACHE)
+	docker build . -f Dockerfile.ci-rp --ulimit=nofile=4096:4096 --build-arg REGISTRY=$(REGISTRY) --build-arg ARO_VERSION=$(VERSION) --no-cache=$(NO_CACHE)
+
+ci-clean:
+	docker image prune --all --filter="label=aro-*=true"
 
 # TODO: hard coding dev-config.yaml is clunky; it is also probably convenient to
 # override COMMIT.
@@ -278,4 +285,4 @@ vendor:
 install-go-tools:
 	go install ${GOTESTSUM}
 
-.PHONY: admin.kubeconfig aks.kubeconfig aro az ci-portal ci-rp clean client deploy dev-config.yaml discoverycache fix-macos-vendor generate image-aro-multistage image-fluentbit image-proxy init-contrib lint-go runlocal-rp proxy publish-image-aro-multistage publish-image-fluentbit publish-image-proxy secrets secrets-update e2e.test tunnel test-e2e test-go test-python vendor build-all validate-go unit-test-go coverage-go validate-fips install-go-tools
+.PHONY: admin.kubeconfig aks.kubeconfig aro az ci-rp ci-clean clean client deploy dev-config.yaml discoverycache fix-macos-vendor generate image-aro-multistage image-fluentbit image-proxy init-contrib lint-go runlocal-rp proxy publish-image-aro-multistage publish-image-fluentbit publish-image-proxy secrets secrets-update e2e.test tunnel test-e2e test-go test-python vendor build-all validate-go unit-test-go coverage-go validate-fips install-go-tools

cmd/aro/mirror.go

@@ -12,7 +12,6 @@ import (
 	"os"
 	"strings"
 
-	"github.com/Azure/go-autorest/autorest/azure"
 	"github.com/containers/image/v5/types"
 	"github.com/sirupsen/logrus"
 
@@ -74,33 +73,10 @@ func mirror(ctx context.Context, log *logrus.Entry) error {
 		return err
 	}
 
-	// Geneva allows anonymous pulls
-	var srcAuthGeneva *types.DockerAuthConfig
-
 	// We can lose visibility of early image mirroring errors because logs are trimmed in the output of Ev2 pipelines.
 	// If images fail to mirror, those errors need to be returned together and logged at the end of the execution.
 	var imageMirroringErrors []string
 
-	// Geneva mirroring from upstream only takes place in Public Cloud, in
-	// sovereign clouds a separate mirror process mirrors from the public cloud
-	if env.Environment().Environment == azure.PublicCloud {
-		srcAcrGeneva := "linuxgeneva-microsoft" + acrDomainSuffix
-		mirrorImages := []string{
-			// https://eng.ms/docs/products/geneva/collect/references/linuxcontainers
-			srcAcrGeneva + "/distroless/genevamdm:2.2024.328.1744-c5fb79-20240328t1935",
-			srcAcrGeneva + "/distroless/genevamdsd:mariner_20240327.2",
-		}
-		for _, ref := range mirrorImages {
-			log.Printf("mirroring %s -> %s", ref, pkgmirror.DestLastIndex(dstAcr+acrDomainSuffix, ref))
-			err = pkgmirror.Copy(ctx, pkgmirror.DestLastIndex(dstAcr+acrDomainSuffix, ref), ref, dstAuth, srcAuthGeneva)
-			if err != nil {
-				imageMirroringErrors = append(imageMirroringErrors, fmt.Sprintf("%s: %s\n", ref, err))
-			}
-		}
-	} else {
-		log.Printf("skipping Geneva mirroring due to not being in Public")
-	}
-
 	for _, ref := range []string{
 
 		// https://mcr.microsoft.com/en-us/product/azure-cli/about
@@ -134,7 +110,7 @@ func mirror(ctx context.Context, log *logrus.Entry) error {
 		"quay.io/app-sre/managed-upgrade-operator:v0.1.952-44b631a",
 
 		// https://quay.io/repository/app-sre/hive?tab=tags
-		"quay.io/app-sre/hive:83aedb9f6e",
+		"quay.io/app-sre/hive:d7ead609f4",
 	} {
 		log.Printf("mirroring %s -> %s", ref, pkgmirror.Dest(dstAcr+acrDomainSuffix, ref))
 

cmd/aro/rp.go

@@ -163,6 +163,11 @@ func rp(ctx context.Context, log, audit *logrus.Entry) error {
 		return err
 	}
 
+	dbPlatformWorkloadIdentityRoleSets, err := database.NewPlatformWorkloadIdentityRoleSets(ctx, dbc, dbName)
+	if err != nil {
+		return err
+	}
+
 	go database.EmitMetrics(ctx, log, dbOpenShiftClusters, metrics)
 
 	feAead, err := encryption.NewMulti(ctx, _env.ServiceKeyvault(), env.FrontendEncryptionSecretV2Name, env.FrontendEncryptionSecretName)
@@ -173,7 +178,7 @@ func rp(ctx context.Context, log, audit *logrus.Entry) error {
 	if err != nil {
 		return err
 	}
-	f, err := frontend.NewFrontend(ctx, audit, log.WithField("component", "frontend"), _env, dbAsyncOperations, dbClusterManagerConfiguration, dbOpenShiftClusters, dbSubscriptions, dbOpenShiftVersions, api.APIs, metrics, clusterm, feAead, hiveClusterManager, adminactions.NewKubeActions, adminactions.NewAzureActions, clusterdata.NewParallelEnricher(metrics, _env))
+	f, err := frontend.NewFrontend(ctx, audit, log.WithField("component", "frontend"), _env, dbAsyncOperations, dbClusterManagerConfiguration, dbOpenShiftClusters, dbSubscriptions, dbOpenShiftVersions, dbPlatformWorkloadIdentityRoleSets, api.APIs, metrics, clusterm, feAead, hiveClusterManager, adminactions.NewKubeActions, adminactions.NewAzureActions, clusterdata.NewParallelEnricher(metrics, _env))
 	if err != nil {
 		return err
 	}

docs/prepare-your-dev-environment.md

@@ -34,7 +34,7 @@ This document goes through the development dependencies one requires in order to
         If you're using podman-machine, you will need to export the socket, for example::
 
             ```bash
-            export ARO_PODMAN_SOCKET=unix:///$HOME/.local/share/containers/podman/machine/qemu/podman.sock
+            export ARO_PODMAN_SOCKET=unix://$HOME/.local/share/containers/podman/machine/qemu/podman.sock
             ```
         
         You will also need to ensure that podman machine has enough resources::