diff --git a/.github/workflows/golint.yml b/.github/workflows/golint.yml index ddcecae2a30..710879d1efb 100644 --- a/.github/workflows/golint.yml +++ b/.github/workflows/golint.yml @@ -25,7 +25,7 @@ jobs: go-version-file: go.mod - name: Run golangci-lint - uses: golangci/golangci-lint-action@v4 + uses: golangci/golangci-lint-action@v6 with: version: v1.56.2 args: -v --timeout 15m diff --git a/.sha256sum b/.sha256sum index 8fae07dc734..cca696c9d93 100644 --- a/.sha256sum +++ b/.sha256sum @@ -6,4 +6,4 @@ b1f1de0fe40d05de90742b17928968923b936adc294000f58974f50a297581dd swagger/redhat c023515341196746454c0ae7af077d40d3ec13f6b88b33cb558f0a7ab17a5a24 swagger/redhatopenshift/resource-manager/Microsoft.RedHatOpenShift/preview/2023-07-01-preview/redhatopenshift.json 440748951dd1c3b34b5ccbdcb7cd966e3b89490887a1f1d64429561fad789515 swagger/redhatopenshift/resource-manager/Microsoft.RedHatOpenShift/stable/2023-09-04/redhatopenshift.json 74a46fdde6ceb0121fe1515c7e11e902dd921b54cffe693307fb02b3dc88f26e swagger/redhatopenshift/resource-manager/Microsoft.RedHatOpenShift/stable/2023-11-22/redhatopenshift.json -a27184734436629e24b344c3b5c015437f144e18e7eddce7e252a1ed4cda7bca swagger/redhatopenshift/resource-manager/Microsoft.RedHatOpenShift/preview/2024-08-12-preview/redhatopenshift.json +f1e9d42d7c4c0081282e065e7845455db28ed6924687f1acecafb5fbc43ae0c3 swagger/redhatopenshift/resource-manager/Microsoft.RedHatOpenShift/preview/2024-08-12-preview/redhatopenshift.json diff --git a/Dockerfile.ci-azext-aro b/Dockerfile.ci-azext-aro new file mode 100644 index 00000000000..758686c3f8b --- /dev/null +++ b/Dockerfile.ci-azext-aro @@ -0,0 +1,12 @@ +FROM mcr.microsoft.com/azure-cli:2.61.0 AS builder + +RUN pip install pytest +COPY /python /data/ + +WORKDIR /data/az/aro +RUN pytest --ignore=azext_aro/tests/latest/integration +RUN python3 setup.py bdist_wheel + +FROM mcr.microsoft.com/azure-cli:2.61.0-cbl-mariner2.0 AS final +COPY --from=builder /data/az/aro/dist /opt/az +RUN az extension add --yes --source /opt/az/aro-*-py2.py3-none-any.whl diff --git a/Dockerfile.ci-rp b/Dockerfile.ci-rp index 23ee37330ef..69c06f318d7 100644 --- a/Dockerfile.ci-rp +++ b/Dockerfile.ci-rp @@ -1,11 +1,12 @@ ARG REGISTRY -ARG VERSION +ARG ARO_VERSION ############################################################################### # Stage 1: Build the SRE Portal Assets # builder is responsible for all compilation and validation of the RP ############################################################################### FROM ${REGISTRY}/ubi8/nodejs-16 as portal-build +LABEL aro-portal-build=true WORKDIR /build/portal/v2 USER root @@ -22,6 +23,8 @@ RUN npm run lint && npm run build # Stage 2: Compile the Golang RP code ############################################################################### FROM ${REGISTRY}/ubi8/go-toolset:1.20.12-5 AS builder +ARG ARO_VERSION +LABEL aro-builder=true USER root WORKDIR /app @@ -49,17 +52,18 @@ COPY --from=portal-build /build/pkg/portal/assets/v2/build /app/pkg/portal/asset # Lint, generate, build, and test RUN golangci-lint run --verbose RUN go generate ./... -RUN go build -ldflags "-X github.com/Azure/ARO-RP/pkg/util/version.GitCommit=${VERSION}" ./cmd/aro -RUN go test ./test/e2e/... -tags e2e,codec.safe -c -ldflags "-X github.com/Azure/ARO-RP/pkg/util/version.GitCommit=${VERSION}" -o e2e.test +RUN go build -ldflags "-X github.com/Azure/ARO-RP/pkg/util/version.GitCommit=${ARO_VERSION}" ./cmd/aro +RUN go test ./test/e2e/... -tags e2e,codec.safe -c -ldflags "-X github.com/Azure/ARO-RP/pkg/util/version.GitCommit=${ARO_VERSION}" -o e2e.test # Additional tests -RUN ARO_RUN_PKI_TESTS=nope go run gotest.tools/gotestsum@v1.11.0 --format pkgname --junitfile report.xml -- -coverprofile=cover.out ./... +RUN ARO_SKIP_PKI_TESTS=true go run gotest.tools/gotestsum@v1.11.0 --format pkgname --junitfile report.xml -- -coverprofile=cover.out ./... RUN hack/fips/validate-fips.sh ./aro ############################################################################### # Stage 3: final is our slim image with minimal layers and tools ############################################################################### FROM ${REGISTRY}/ubi8/ubi-minimal AS final +LABEL aro-final=true RUN microdnf update && microdnf clean all COPY --from=builder /app/aro /app/e2e.test /usr/local/bin/ ENTRYPOINT ["aro"] diff --git a/Dockerfile.gatekeeper b/Dockerfile.gatekeeper index ea7a8540140..8b781e8e3de 100644 --- a/Dockerfile.gatekeeper +++ b/Dockerfile.gatekeeper @@ -14,7 +14,7 @@ WORKDIR ${GOPATH}/src/github.com/open-policy-agent/gatekeeper USER root RUN curl -Lq $DOWNLOAD_URL | tar -xz --strip-components=1 -RUN go build -mod vendor -a -ldflags "-X github.com/open-policy-agent/gatekeeper/pkg/version.Version=latest" -o manager +RUN go build -mod vendor -a -ldflags "-X github.com/open-policy-agent/gatekeeper/pkg/version.Version=$GATEKEEPER_VERSION" -o manager #### Runtime container FROM ${REGISTRY}/ubi8/ubi-minimal:latest diff --git a/Makefile b/Makefile index e5b53a309c8..032896fca46 100644 --- a/Makefile +++ b/Makefile @@ -14,8 +14,7 @@ FLUENTBIT_VERSION = 1.9.10 FLUENTBIT_IMAGE ?= ${RP_IMAGE_ACR}.azurecr.io/fluentbit:$(FLUENTBIT_VERSION)-cm$(MARINER_VERSION) AUTOREST_VERSION = 3.6.3 AUTOREST_IMAGE = quay.io/openshift-on-azure/autorest:${AUTOREST_VERSION} -GATEKEEPER_VERSION = v3.10.0 -GATEKEEPER_IMAGE ?= ${RP_IMAGE_ACR}.azurecr.io/gatekeeper:$(GATEKEEPER_VERSION) +GATEKEEPER_VERSION = v3.15.1 GOTESTSUM = gotest.tools/gotestsum@v1.11.0 ifneq ($(shell uname -s),Darwin) @@ -40,6 +39,7 @@ else endif ARO_IMAGE ?= $(ARO_IMAGE_BASE):$(VERSION) +GATEKEEPER_IMAGE ?= ${REGISTRY}/gatekeeper:$(GATEKEEPER_VERSION) check-release: # Check that VERSION is a valid tag when building an official release (when RELEASE=true). @@ -67,6 +67,10 @@ az: pyenv python3 ./setup.py bdist_wheel || true && \ rm -f ~/.azure/commandIndex.json # https://github.com/Azure/azure-cli/issues/14997 +.PHONY: azext-aro +azext-aro: + docker build --platform=linux/amd64 . -f Dockerfile.ci-azext-aro --no-cache=$(NO_CACHE) -t azext-aro:latest + clean: rm -rf python/az/aro/{aro.egg-info,build,dist} aro find python -type f -name '*.pyc' -delete @@ -77,7 +81,10 @@ client: generate hack/build-client.sh "${AUTOREST_IMAGE}" 2020-04-30 2021-09-01-preview 2022-04-01 2022-09-04 2023-04-01 2023-07-01-preview 2023-09-04 2023-11-22 2024-08-12-preview ci-rp: fix-macos-vendor - docker build . -f Dockerfile.ci-rp --ulimit=nofile=4096:4096 --build-arg REGISTRY=$(REGISTRY) --build-arg VERSION=$(VERSION) --no-cache=$(NO_CACHE) + docker build . -f Dockerfile.ci-rp --ulimit=nofile=4096:4096 --build-arg REGISTRY=$(REGISTRY) --build-arg ARO_VERSION=$(VERSION) --no-cache=$(NO_CACHE) + +ci-clean: + docker image prune --all --filter="label=aro-*=true" # TODO: hard coding dev-config.yaml is clunky; it is also probably convenient to # override COMMIT. @@ -278,4 +285,4 @@ vendor: install-go-tools: go install ${GOTESTSUM} -.PHONY: admin.kubeconfig aks.kubeconfig aro az ci-portal ci-rp clean client deploy dev-config.yaml discoverycache fix-macos-vendor generate image-aro-multistage image-fluentbit image-proxy init-contrib lint-go runlocal-rp proxy publish-image-aro-multistage publish-image-fluentbit publish-image-proxy secrets secrets-update e2e.test tunnel test-e2e test-go test-python vendor build-all validate-go unit-test-go coverage-go validate-fips install-go-tools +.PHONY: admin.kubeconfig aks.kubeconfig aro az ci-rp ci-clean clean client deploy dev-config.yaml discoverycache fix-macos-vendor generate image-aro-multistage image-fluentbit image-proxy init-contrib lint-go runlocal-rp proxy publish-image-aro-multistage publish-image-fluentbit publish-image-proxy secrets secrets-update e2e.test tunnel test-e2e test-go test-python vendor build-all validate-go unit-test-go coverage-go validate-fips install-go-tools diff --git a/cmd/aro/mirror.go b/cmd/aro/mirror.go index db0c7ec926b..2d031707a02 100644 --- a/cmd/aro/mirror.go +++ b/cmd/aro/mirror.go @@ -12,7 +12,6 @@ import ( "os" "strings" - "github.com/Azure/go-autorest/autorest/azure" "github.com/containers/image/v5/types" "github.com/sirupsen/logrus" @@ -74,33 +73,10 @@ func mirror(ctx context.Context, log *logrus.Entry) error { return err } - // Geneva allows anonymous pulls - var srcAuthGeneva *types.DockerAuthConfig - // We can lose visibility of early image mirroring errors because logs are trimmed in the output of Ev2 pipelines. // If images fail to mirror, those errors need to be returned together and logged at the end of the execution. var imageMirroringErrors []string - // Geneva mirroring from upstream only takes place in Public Cloud, in - // sovereign clouds a separate mirror process mirrors from the public cloud - if env.Environment().Environment == azure.PublicCloud { - srcAcrGeneva := "linuxgeneva-microsoft" + acrDomainSuffix - mirrorImages := []string{ - // https://eng.ms/docs/products/geneva/collect/references/linuxcontainers - srcAcrGeneva + "/distroless/genevamdm:2.2024.328.1744-c5fb79-20240328t1935", - srcAcrGeneva + "/distroless/genevamdsd:mariner_20240327.2", - } - for _, ref := range mirrorImages { - log.Printf("mirroring %s -> %s", ref, pkgmirror.DestLastIndex(dstAcr+acrDomainSuffix, ref)) - err = pkgmirror.Copy(ctx, pkgmirror.DestLastIndex(dstAcr+acrDomainSuffix, ref), ref, dstAuth, srcAuthGeneva) - if err != nil { - imageMirroringErrors = append(imageMirroringErrors, fmt.Sprintf("%s: %s\n", ref, err)) - } - } - } else { - log.Printf("skipping Geneva mirroring due to not being in Public") - } - for _, ref := range []string{ // https://mcr.microsoft.com/en-us/product/azure-cli/about @@ -134,7 +110,7 @@ func mirror(ctx context.Context, log *logrus.Entry) error { "quay.io/app-sre/managed-upgrade-operator:v0.1.952-44b631a", // https://quay.io/repository/app-sre/hive?tab=tags - "quay.io/app-sre/hive:83aedb9f6e", + "quay.io/app-sre/hive:d7ead609f4", } { log.Printf("mirroring %s -> %s", ref, pkgmirror.Dest(dstAcr+acrDomainSuffix, ref)) diff --git a/cmd/aro/rp.go b/cmd/aro/rp.go index 5dfd3eb7f2d..99accccecc6 100644 --- a/cmd/aro/rp.go +++ b/cmd/aro/rp.go @@ -163,6 +163,11 @@ func rp(ctx context.Context, log, audit *logrus.Entry) error { return err } + dbPlatformWorkloadIdentityRoleSets, err := database.NewPlatformWorkloadIdentityRoleSets(ctx, dbc, dbName) + if err != nil { + return err + } + go database.EmitMetrics(ctx, log, dbOpenShiftClusters, metrics) feAead, err := encryption.NewMulti(ctx, _env.ServiceKeyvault(), env.FrontendEncryptionSecretV2Name, env.FrontendEncryptionSecretName) @@ -173,7 +178,7 @@ func rp(ctx context.Context, log, audit *logrus.Entry) error { if err != nil { return err } - f, err := frontend.NewFrontend(ctx, audit, log.WithField("component", "frontend"), _env, dbAsyncOperations, dbClusterManagerConfiguration, dbOpenShiftClusters, dbSubscriptions, dbOpenShiftVersions, api.APIs, metrics, clusterm, feAead, hiveClusterManager, adminactions.NewKubeActions, adminactions.NewAzureActions, clusterdata.NewParallelEnricher(metrics, _env)) + f, err := frontend.NewFrontend(ctx, audit, log.WithField("component", "frontend"), _env, dbAsyncOperations, dbClusterManagerConfiguration, dbOpenShiftClusters, dbSubscriptions, dbOpenShiftVersions, dbPlatformWorkloadIdentityRoleSets, api.APIs, metrics, clusterm, feAead, hiveClusterManager, adminactions.NewKubeActions, adminactions.NewAzureActions, clusterdata.NewParallelEnricher(metrics, _env)) if err != nil { return err } diff --git a/docs/prepare-your-dev-environment.md b/docs/prepare-your-dev-environment.md index 159acc7662a..f5b67a9e8cd 100644 --- a/docs/prepare-your-dev-environment.md +++ b/docs/prepare-your-dev-environment.md @@ -34,7 +34,7 @@ This document goes through the development dependencies one requires in order to If you're using podman-machine, you will need to export the socket, for example:: ```bash - export ARO_PODMAN_SOCKET=unix:///$HOME/.local/share/containers/podman/machine/qemu/podman.sock + export ARO_PODMAN_SOCKET=unix://$HOME/.local/share/containers/podman/machine/qemu/podman.sock ``` You will also need to ensure that podman machine has enough resources:: diff --git a/docs/upstream-differences.md b/docs/upstream-differences.md deleted file mode 100644 index f94f7ee7182..00000000000 --- a/docs/upstream-differences.md +++ /dev/null @@ -1,204 +0,0 @@ -# Upstream differences - -This file catalogues the differences of install approach between ARO and -upstream OCP. - -## Installer carry patches - -1. Clone ARO and upstream repos: - ```sh - # clone our forked installer - git clone https://github.com/openshift/installer-aro.git - cd installer-aro - - # add the upstream as a remote source - git remote add upstream https://github.com/openshift/installer.git - git fetch upstream -a - ``` -1. See carry patches from previous release: - ```sh - # list patches - git log upstream/release-X.Y-1..origin/release-X.Y-1-azure - - # see diff of patches - git show upstream/release-X.Y-1..origin/release-X.Y-1-azure - ``` - -## Installation differences - -* ARO does not use Terraform to create clusters, and instead uses ARM templates directly - -* ARO persists the install graph in the cluster storage account in a new "aro" - container / "graph" blob. - -* No managed identity (for now). - -* No IPv6 support (for now). - -* Upstream installer closely binds the installConfig (cluster) name, cluster - domain name, infra ID and Azure resource name prefix. ARO separates these out - a little. The installConfig (cluster) name and the domain name remain bound; - the infra ID and Azure resource name prefix are taken from the ARO resource - name. - -* API server public IP domain name label is not set. - -* ARO uses first party RHCOS OS images published by Microsoft. - -* ARO never creates xxxxx-bootstrap-pip-* for bootstrap VM, or the corresponding - NSG rule. - -* ARO does not create a outbound-provider Service on port 27627. - -* ARO deploys a private link service in order for the RP to be able to - communicate with the cluster. - -* ARO runs a dnsmasq service on the nodes through the use of a machineconfig to resolve api-int and *.apps domains on the node locally allowing for custom DNS configured on the VNET. - -# Introducing new OCP release into ARO RP - -To support a new version of OpenShift on ARO, you will need to reconcile [upstream changes](https://github.com/openshift/installer) with our [forked installer](https://github.com/openshift/installer-aro). This will not be a merge, but a cherry-pick of patches we've implemented. - -## Update installer fork - -To bring new OCP release branch into ARO installer fork: - -1. If not done already, fetch our fork and upstream repos: - ```sh - # clone our forked installer - # Alternatively, fork openshift/installer-aro and clone your fork - git clone https://github.com/openshift/installer-aro.git - cd installer-aro - - # add the upstream as a remote source - git remote add upstream https://github.com/openshift/installer.git - git fetch upstream -a - ``` -1. Assess and document differences in X.Y and X.Y-1 in upstream - ```sh - # diff the upstream X.Y with X.Y-1 and search for architecture changes - git diff upstream/release-X.Y-1 upstream/release-X.Y - - # pay particular attention to Terraform files, which may need to be moved into ARO's ARM templates - git diff upstream/release-X.Y-1 upstream/release-X.Y */azure/*.tf - ``` -2. Create a new X.Y release branch in our forked installer - ```sh - # create a new release branch in the fork based on the upstream - git checkout upstream/release-X.Y - git checkout -b release-X.Y-azure - ``` -3. If there is a golang version bump in this release, modify `./hack/build.sh` and `./hack/go-test.sh` with the new version, then verify these scripts still work and commit them -4. Determine the patches you need to cherry-pick, based on the last (Y-1) release - ```sh - # find commit shas to cherry-pick from last time - git checkout release-X.Y-1-azure - git log - ``` -5. For every commit you need to cherry-pick (in-order), do: - ```sh - # WARNING: when you reach the commit for `commit data/assets_vfsdata.go`, look ahead - git cherry-pick abc123 # may require manually fixing a merge - ./hack/build.sh # fix any failures - ./hack/go-test.sh # fix any failures - # if you had to manually merge, you can now `git cherry-pick --continue` - ``` - - When cherry-picking the specific patch `commit data/assets_vfsdata.go`, instead run: - ```sh - git cherry-pick abc123 # may require manually fixing a merge - ./hack/build.sh # fix any failures - ./hack/go-test.sh # fix any failures - # if you had to manually merge, you can now `git cherry-pick --continue` - pushd ./hack/assets && go run ./assets.go && popd - ./hack/build.sh # fix any failures - ./hack/go-test.sh # fix any failures - git add data/assets_vfsdata.go - git commit --amend - ``` - -**Note:** If any changes are required during the process, make sure to amend the relevant patch or create a new one. -Each commit should be atomic/complete - you should be able to cherry-pick it into the upstream installer and bring -the fix or feature it carries in full, without a need to cherry-pick additional commits. -This makes it easier to understand the nature of the patch as well as contribute our carry patches -back to the upstream installer. - -## Update Installer Fork Again - -In the far or near future after you have [initially patched the installer](#update-installer-fork), you may need to pull in additional upstream changes that have happened since you patched the installer (e.g. upstream added a bugfix since your cherry-picking). The easiest way to pull in these changes safely is: - -```sh -git fetch upstream -a -git checkout release-X.Y-azure -git pull upstream/release-X.Y --rebase=interactive -``` - -When you get to the editor mode to set up your rebase, you should do a few things: - -1. after each `pick` line, add the verification commands with: - ```text - exec ./hack/build.sh - exec ./hack/go-test.sh - ``` -1. change the line for commit `data/assets_vfsdata.go` from `pick` to `edit` so you can regenerate assets as outlined [above](#update-installer-fork), and verify it manually. - -By the end of this editing process, you should have a rebase file that looks something like: - -```text -pick abc123 patch A -exec ./hack/build.sh -exec ./hack/go-test.sh -pick def234 patch B -exec ./hack/build.sh -exec ./hack/go-test.sh -edit ghi345 patch data/assets_vfsdata.go -exec ./hack/build.sh -exec ./hack/go-test.sh -pick jkl456 patch C -exec ./hack/build.sh -exec ./hack/go-test.sh -{...} -``` - -When you are finished, you can write/close the file editor to perform the rebase on the upstream differences while verifying that every patch still works along the way. - -Get a repo admin to create `release-X.Y-azure` in [openshift/installer-aro](https://github.com/openshift/installer-aro). You can now push your local branch `release-X.Y-azure` to your own fork of [openshift/installer-aro](https://github.com/openshift/installer-aro) and send a PR. - -# Update ARO Installer Wrapper and ARO-RP - -Once installer fork is ready, perform the following changes in the [ARO Installer Wrapper](https://github.com/openshift/installer-aro-wrapper): - -1. Update `go mod edit -replace` calls in `hack/update-go-module-dependencies.sh` to use a new release-X.Y branch. - * Make sure to read comments in the script. -1. `make vendor`. - * You most likely will have to make changes to the codebase at this point to adjust it to new versions of dependencies. - * Also you likely will have to repeat this step several time until you resolve all conflicting dependencies. - Follow `go mod` failures, which will tell you what module requires what other module. - You will probably need to look at the `go.mod` files of these modules and see whether they set own replace directives, - as the script is likely to fail with something like this: - - ``` - go: github.com/openshift/installer@v0.16.1 requires - github.com/openshift/cluster-api-provider-kubevirt@v0.0.0-20201214114543-e5aed9c73f1f requires - kubevirt.io/client-go@v0.0.0-00010101000000-000000000000: invalid version: unknown revision 000000000000 - ``` - - In the example above you need to: - * Checkout `github.com/openshift/cluster-api-provider-kubevirt` at commit `e5aed9c73f1f`. - * In go.mod find a replace directive for `kubevirt.io/client-go`. - * Add/update relevant replace directive in `go.mod`. -1. `make generate`. -1. A local image can be built for testing purposes and pushed to a dev ACR repo. The process for [publishing the final image uses ADO](https://msazure.visualstudio.com/AzureRedHatOpenShift/_wiki/wikis/ARO.wiki/452838/ARO-Installer-Image-Deployment-Process) - -In the ARO-RP codebase: - -1. Update `pkg/util/version/const.go` to point to the new release. - * You should be able to find latest published release and image hash [on quay.io](https://quay.io/repository/openshift-release-dev/ocp-release?tab=tags). -1. (Optional) `make discoverycache`. - * This command requires a running cluster with the new version. -1. The list of the hard-coded namespaces in `pkg/util/namespace/namespace.go` needs to be updated regularly as every - minor version of upstream OCP introduces a new namespace or two. - - -Publish RHCOS image to the Azure Cloud Partner Portal: -1. Publish RHCOS image. See [this document](https://github.com/openshift/installer-aro-wrapper/blob/main/docs/publish-rhcos-image.md). -1. After this point, you should be able to create a dev cluster using the RP and it should use the new release. diff --git a/env.example b/env.example index a265550d1dc..2f420e76437 100644 --- a/env.example +++ b/env.example @@ -3,4 +3,6 @@ export ARO_IMAGE=arointsvc.azurecr.io/aro:latest export NO_CACHE=false export AZURE_EXTENSION_DEV_SOURCES="$(pwd)/python" +export ARO_SKIP_PKI_TESTS=true + . secrets/env diff --git a/go.mod b/go.mod index 3c3c3f0d45a..a5a34cb8899 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/alvaroloes/enumer v1.1.2 github.com/apparentlymart/go-cidr v1.1.0 github.com/codahale/etm v0.0.0-20141003032925-c00c9e6fb4c9 - github.com/containers/image/v5 v5.29.2 + github.com/containers/image/v5 v5.29.3 github.com/containers/podman/v4 v4.9.4 github.com/coreos/go-oidc v2.2.1+incompatible github.com/coreos/go-semver v0.3.0 @@ -29,7 +29,7 @@ require ( github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 github.com/go-bindata/go-bindata v3.1.2+incompatible github.com/go-chi/chi/v5 v5.0.8 - github.com/go-logr/logr v1.4.1 + github.com/go-logr/logr v1.4.2 github.com/go-test/deep v1.1.0 github.com/gofrs/uuid v4.2.0+incompatible github.com/golang-jwt/jwt/v4 v4.5.0 @@ -59,7 +59,7 @@ require ( github.com/openshift/api v3.9.1-0.20191111211345-a27ff30ebf09+incompatible github.com/openshift/client-go v0.0.0-20220525160904-9e1acff93e4a github.com/openshift/cloud-credential-operator v0.0.0-00010101000000-000000000000 - github.com/openshift/hive/apis v0.0.0-20240510150258-83aedb9f6e73 + github.com/openshift/hive/apis v0.0.0-20240529172037-d7ead609f495 github.com/openshift/library-go v0.0.0-20220525173854-9b950a41acdc github.com/openshift/machine-config-operator v0.0.1-0.20230519222939-1abc13efbb0d github.com/pires/go-proxyproto v0.6.2 @@ -80,9 +80,9 @@ require ( golang.org/x/sync v0.6.0 golang.org/x/text v0.15.0 golang.org/x/tools v0.19.0 - k8s.io/api v0.30.0 + k8s.io/api v0.30.1 k8s.io/apiextensions-apiserver v0.25.0 - k8s.io/apimachinery v0.30.0 + k8s.io/apimachinery v0.30.1 k8s.io/cli-runtime v0.25.16 k8s.io/client-go v0.26.2 k8s.io/code-generator v0.25.16 diff --git a/go.sum b/go.sum index cb7db633d62..a2971170158 100644 --- a/go.sum +++ b/go.sum @@ -122,8 +122,8 @@ github.com/containers/buildah v1.33.7 h1:Y2kNea+hNNyZ74ppYFWmD0cLc/DwZ5A4NEUPQWP github.com/containers/buildah v1.33.7/go.mod h1:pphfdjrwtTWkuIy1aDyZMEVyMfmm0DsbvxLGxxEU1cM= github.com/containers/common v0.57.4 h1:kmfBad92kUjP5X44BPpOwMe+eZQqaKETfS+ASeL0g+g= github.com/containers/common v0.57.4/go.mod h1:o3L3CyOI9yr+JC8l4dZgvqTxcjs3qdKmkek00uchgvw= -github.com/containers/image/v5 v5.29.2 h1:b8U0XYWhaQbKucK73IbmSm8WQyKAhKDbAHQc45XlsOw= -github.com/containers/image/v5 v5.29.2/go.mod h1:kQ7qcDsps424ZAz24thD+x7+dJw1vgur3A9tTDsj97E= +github.com/containers/image/v5 v5.29.3 h1:RJHdxP+ZiC+loIFG2DTmjlVNWTS7o5jrdrRScUrY1VE= +github.com/containers/image/v5 v5.29.3/go.mod h1:kQ7qcDsps424ZAz24thD+x7+dJw1vgur3A9tTDsj97E= github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01 h1:Qzk5C6cYglewc+UyGf6lc8Mj2UaPTHy/iF2De0/77CA= github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01/go.mod h1:9rfv8iPl1ZP7aqh9YA68wnZv2NUDbXdcdPHVz0pFbPY= github.com/containers/ocicrypt v1.1.9 h1:2Csfba4jse85Raxk5HIyEk8OwZNjRvfkhEGijOjIdEM= @@ -233,8 +233,8 @@ github.com/go-jose/go-jose/v3 v3.0.3/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQr github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.2.3 h1:a9vnzlIBPQBBkeaR9IuMUfmVOrQlkoC4YfPoFkX3T7A= diff --git a/pkg/api/admin/openshiftcluster.go b/pkg/api/admin/openshiftcluster.go index d1bab4bb6ea..ed328772e43 100644 --- a/pkg/api/admin/openshiftcluster.go +++ b/pkg/api/admin/openshiftcluster.go @@ -418,9 +418,13 @@ type IngressProfile struct { // PlatformWorkloadIdentityProfile encapsulates all information that is specific to workload identity clusters. type PlatformWorkloadIdentityProfile struct { + UpgradeableTo *UpgradeableTo `json:"upgradeableTo,omitempty"` PlatformWorkloadIdentities []PlatformWorkloadIdentity `json:"platformWorkloadIdentities,omitempty"` } +// UpgradeableTo stores a single OpenShift version a workload identity cluster can be upgraded to +type UpgradeableTo string + // PlatformWorkloadIdentity stores information representing a single workload identity. type PlatformWorkloadIdentity struct { OperatorName string `json:"operatorName,omitempty"` diff --git a/pkg/api/admin/platformworkloadidentityroleset.go b/pkg/api/admin/platformworkloadidentityroleset.go index 9ed4680bceb..d2b26e9a6fb 100644 --- a/pkg/api/admin/platformworkloadidentityroleset.go +++ b/pkg/api/admin/platformworkloadidentityroleset.go @@ -33,14 +33,14 @@ type PlatformWorkloadIdentityRoleSetProperties struct { // PlatformWorkloadIdentityRole represents a mapping from a particular OCP operator to the built-in role that should be assigned to that operator's corresponding managed identity. type PlatformWorkloadIdentityRole struct { // OperatorName represents the name of the operator that this role is for. - OperatorName string `json:"operatorName,omitempty" mutable:"true"` + OperatorName string `json:"operatorName,omitempty" mutable:"true" validate:"required"` // RoleDefinitionName represents the name of the role. - RoleDefinitionName string `json:"roleDefinitionName,omitempty" mutable:"true"` + RoleDefinitionName string `json:"roleDefinitionName,omitempty" mutable:"true" validate:"required"` // RoleDefinitionID represents the resource ID of the role definition. - RoleDefinitionID string `json:"roleDefinitionId,omitempty" mutable:"true"` + RoleDefinitionID string `json:"roleDefinitionId,omitempty" mutable:"true" validate:"required"` // ServiceAccounts represents the set of service accounts associated with the given operator, since each service account needs its own federated credential. - ServiceAccounts []string `json:"serviceAccounts,omitempty" mutable:"true"` + ServiceAccounts []string `json:"serviceAccounts,omitempty" mutable:"true" validate:"required"` } diff --git a/pkg/api/admin/platformworkloadidentityroleset_convert.go b/pkg/api/admin/platformworkloadidentityroleset_convert.go index b1caa21cb63..b7bf175b1e6 100644 --- a/pkg/api/admin/platformworkloadidentityroleset_convert.go +++ b/pkg/api/admin/platformworkloadidentityroleset_convert.go @@ -1,11 +1,10 @@ package admin +import "github.com/Azure/ARO-RP/pkg/api" + // Copyright (c) Microsoft Corporation. // Licensed under the Apache License 2.0. -/* -TODO: Uncomment once API endpoints have been implemented and this code is being used. - type platformWorkloadIdentityRoleSetConverter struct{} // platformWorkloadIdentityRoleSetConverter.ToExternal returns a new external representation @@ -21,12 +20,17 @@ func (c platformWorkloadIdentityRoleSetConverter) ToExternal(s *api.PlatformWork }, } - for i, r := range s.Properties.PlatformWorkloadIdentityRoles { - out.Properties.PlatformWorkloadIdentityRoles[i].OperatorName = r.OperatorName - out.Properties.PlatformWorkloadIdentityRoles[i].RoleDefinitionName = r.RoleDefinitionName - out.Properties.PlatformWorkloadIdentityRoles[i].RoleDefinitionID = r.RoleDefinitionID - out.Properties.PlatformWorkloadIdentityRoles[i].ServiceAccounts = make([]string, 0, len(r.ServiceAccounts)) - out.Properties.PlatformWorkloadIdentityRoles[i].ServiceAccounts = append(out.Properties.PlatformWorkloadIdentityRoles[i].ServiceAccounts, r.ServiceAccounts...) + for _, r := range s.Properties.PlatformWorkloadIdentityRoles { + role := PlatformWorkloadIdentityRole{ + OperatorName: r.OperatorName, + RoleDefinitionName: r.RoleDefinitionName, + RoleDefinitionID: r.RoleDefinitionID, + ServiceAccounts: make([]string, 0, len(r.ServiceAccounts)), + } + + role.ServiceAccounts = append(role.ServiceAccounts, r.ServiceAccounts...) + + out.Properties.PlatformWorkloadIdentityRoles = append(out.Properties.PlatformWorkloadIdentityRoles, role) } return out @@ -56,12 +60,16 @@ func (c platformWorkloadIdentityRoleSetConverter) ToInternal(_new interface{}, o out.Properties.OpenShiftVersion = new.Properties.OpenShiftVersion out.Properties.PlatformWorkloadIdentityRoles = make([]api.PlatformWorkloadIdentityRole, 0, len(new.Properties.PlatformWorkloadIdentityRoles)) - for i, r := range new.Properties.PlatformWorkloadIdentityRoles { - out.Properties.PlatformWorkloadIdentityRoles[i].OperatorName = r.OperatorName - out.Properties.PlatformWorkloadIdentityRoles[i].RoleDefinitionName = r.RoleDefinitionName - out.Properties.PlatformWorkloadIdentityRoles[i].RoleDefinitionID = r.RoleDefinitionID - out.Properties.PlatformWorkloadIdentityRoles[i].ServiceAccounts = make([]string, 0, len(r.ServiceAccounts)) - out.Properties.PlatformWorkloadIdentityRoles[i].ServiceAccounts = append(out.Properties.PlatformWorkloadIdentityRoles[i].ServiceAccounts, r.ServiceAccounts...) + for _, r := range new.Properties.PlatformWorkloadIdentityRoles { + role := api.PlatformWorkloadIdentityRole{ + OperatorName: r.OperatorName, + RoleDefinitionName: r.RoleDefinitionName, + RoleDefinitionID: r.RoleDefinitionID, + ServiceAccounts: make([]string, 0, len(r.ServiceAccounts)), + } + + role.ServiceAccounts = append(role.ServiceAccounts, r.ServiceAccounts...) + + out.Properties.PlatformWorkloadIdentityRoles = append(out.Properties.PlatformWorkloadIdentityRoles, role) } } -*/ diff --git a/pkg/api/admin/platformworkloadidentityroleset_validatestatic.go b/pkg/api/admin/platformworkloadidentityroleset_validatestatic.go index e70bffc0555..d6fcfb9c1b6 100644 --- a/pkg/api/admin/platformworkloadidentityroleset_validatestatic.go +++ b/pkg/api/admin/platformworkloadidentityroleset_validatestatic.go @@ -1,11 +1,17 @@ package admin +import ( + "fmt" + "net/http" + "strings" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/api/util/immutable" +) + // Copyright (c) Microsoft Corporation. // Licensed under the Apache License 2.0. -/* -TODO: Uncomment once API endpoints have been implemented and this code is being used. - type platformWorkloadIdentityRoleSetStaticValidator struct{} func (sv platformWorkloadIdentityRoleSetStaticValidator) Static(_new interface{}, _current *api.PlatformWorkloadIdentityRoleSet) error { @@ -37,27 +43,31 @@ func (sv platformWorkloadIdentityRoleSetStaticValidator) validate(new *PlatformW return api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidParameter, "properties.platformWorkloadIdentityRoles", "Must be provided and must be non-empty") } - errs := []error{} + missingProperties := []string{} for i, r := range new.Properties.PlatformWorkloadIdentityRoles { if r.OperatorName == "" { - errs = append(errs, api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidParameter, fmt.Sprintf("properties.platformWorkloadIdentityRoles[%d].operatorName", i), "Must be provided")) + missingProperties = append(missingProperties, fmt.Sprintf("properties.platformWorkloadIdentityRoles[%d].operatorName", i)) } if r.RoleDefinitionName == "" { - errs = append(errs, api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidParameter, fmt.Sprintf("properties.platformWorkloadIdentityRoles[%d].roleDefinitionName", i), "Must be provided")) + missingProperties = append(missingProperties, fmt.Sprintf("properties.platformWorkloadIdentityRoles[%d].roleDefinitionName", i)) } if r.RoleDefinitionID == "" { - errs = append(errs, api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidParameter, fmt.Sprintf("properties.platformWorkloadIdentityRoles[%d].roleDefinitionId", i), "Must be provided")) + missingProperties = append(missingProperties, fmt.Sprintf("properties.platformWorkloadIdentityRoles[%d].roleDefinitionId", i)) } if r.ServiceAccounts == nil || len(r.ServiceAccounts) == 0 { - errs = append(errs, api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidParameter, fmt.Sprintf("properties.platformWorkloadIdentityRoles[%d].serviceAccounts", i), "Must be provided and must be non-empty")) + missingProperties = append(missingProperties, fmt.Sprintf("properties.platformWorkloadIdentityRoles[%d].serviceAccounts", i)) } } - return errors.Join(errs...) + if len(missingProperties) > 0 { + return api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidParameter, strings.Join(missingProperties, ", "), "Must be provided") + } + + return nil } func (sv platformWorkloadIdentityRoleSetStaticValidator) validateDelta(new, current *PlatformWorkloadIdentityRoleSet) error { @@ -68,4 +78,3 @@ func (sv platformWorkloadIdentityRoleSetStaticValidator) validateDelta(new, curr } return nil } -*/ diff --git a/pkg/api/admin/register.go b/pkg/api/admin/register.go index ac29676945f..4a0613c276e 100644 --- a/pkg/api/admin/register.go +++ b/pkg/api/admin/register.go @@ -12,9 +12,11 @@ const APIVersion = "admin" func init() { api.APIs[APIVersion] = &api.Version{ - OpenShiftClusterConverter: openShiftClusterConverter{}, - OpenShiftClusterStaticValidator: openShiftClusterStaticValidator{}, - OpenShiftVersionConverter: openShiftVersionConverter{}, - OpenShiftVersionStaticValidator: openShiftVersionStaticValidator{}, + OpenShiftClusterConverter: openShiftClusterConverter{}, + OpenShiftClusterStaticValidator: openShiftClusterStaticValidator{}, + OpenShiftVersionConverter: openShiftVersionConverter{}, + OpenShiftVersionStaticValidator: openShiftVersionStaticValidator{}, + PlatformWorkloadIdentityRoleSetConverter: platformWorkloadIdentityRoleSetConverter{}, + PlatformWorkloadIdentityRoleSetStaticValidator: platformWorkloadIdentityRoleSetStaticValidator{}, } } diff --git a/pkg/api/openshiftcluster.go b/pkg/api/openshiftcluster.go index 0826520d0bb..caa157d71b8 100644 --- a/pkg/api/openshiftcluster.go +++ b/pkg/api/openshiftcluster.go @@ -773,9 +773,13 @@ type HiveProfile struct { type PlatformWorkloadIdentityProfile struct { MissingFields + UpgradeableTo *UpgradeableTo `json:"upgradeableTo,omitempty"` PlatformWorkloadIdentities []PlatformWorkloadIdentity `json:"platformWorkloadIdentities,omitempty"` } +// UpgradeableTo stores a single OpenShift version a workload identity cluster can be upgraded to +type UpgradeableTo string + // PlatformWorkloadIdentity stores information representing a single workload identity. type PlatformWorkloadIdentity struct { MissingFields @@ -803,4 +807,5 @@ type Identity struct { Type string `json:"type,omitempty"` UserAssignedIdentities UserAssignedIdentities `json:"userAssignedIdentities,omitempty"` + IdentityURL string `json:"identityURL,omitempty" mutable:"true"` } diff --git a/pkg/api/register.go b/pkg/api/register.go index 89b986ac55d..5e91693909b 100644 --- a/pkg/api/register.go +++ b/pkg/api/register.go @@ -37,6 +37,16 @@ type OpenShiftVersionStaticValidator interface { Static(interface{}, *OpenShiftVersion) error } +type PlatformWorkloadIdentityRoleSetConverter interface { + ToExternal(*PlatformWorkloadIdentityRoleSet) interface{} + ToExternalList([]*PlatformWorkloadIdentityRoleSet) interface{} + ToInternal(interface{}, *PlatformWorkloadIdentityRoleSet) +} + +type PlatformWorkloadIdentityRoleSetStaticValidator interface { + Static(interface{}, *PlatformWorkloadIdentityRoleSet) error +} + type SyncSetConverter interface { ToExternal(*SyncSet) interface{} ToExternalList([]*SyncSet) interface{} @@ -63,18 +73,20 @@ type SecretConverter interface { // Version is a set of endpoints implemented by each API version type Version struct { - OpenShiftClusterConverter OpenShiftClusterConverter - OpenShiftClusterStaticValidator OpenShiftClusterStaticValidator - OpenShiftClusterCredentialsConverter OpenShiftClusterCredentialsConverter - OpenShiftClusterAdminKubeconfigConverter OpenShiftClusterAdminKubeconfigConverter - OpenShiftVersionConverter OpenShiftVersionConverter - OpenShiftVersionStaticValidator OpenShiftVersionStaticValidator - OperationList OperationList - SyncSetConverter SyncSetConverter - MachinePoolConverter MachinePoolConverter - SyncIdentityProviderConverter SyncIdentityProviderConverter - SecretConverter SecretConverter - ClusterManagerStaticValidator ClusterManagerStaticValidator + OpenShiftClusterConverter OpenShiftClusterConverter + OpenShiftClusterStaticValidator OpenShiftClusterStaticValidator + OpenShiftClusterCredentialsConverter OpenShiftClusterCredentialsConverter + OpenShiftClusterAdminKubeconfigConverter OpenShiftClusterAdminKubeconfigConverter + OpenShiftVersionConverter OpenShiftVersionConverter + OpenShiftVersionStaticValidator OpenShiftVersionStaticValidator + PlatformWorkloadIdentityRoleSetConverter PlatformWorkloadIdentityRoleSetConverter + PlatformWorkloadIdentityRoleSetStaticValidator PlatformWorkloadIdentityRoleSetStaticValidator + OperationList OperationList + SyncSetConverter SyncSetConverter + MachinePoolConverter MachinePoolConverter + SyncIdentityProviderConverter SyncIdentityProviderConverter + SecretConverter SecretConverter + ClusterManagerStaticValidator ClusterManagerStaticValidator } // APIs is the map of registered API versions diff --git a/pkg/api/v20240812preview/openshiftcluster.go b/pkg/api/v20240812preview/openshiftcluster.go index 09a5a632dbf..28e7d4bd171 100644 --- a/pkg/api/v20240812preview/openshiftcluster.go +++ b/pkg/api/v20240812preview/openshiftcluster.go @@ -290,15 +290,19 @@ type IngressProfile struct { // PlatformWorkloadIdentityProfile encapsulates all information that is specific to workload identity clusters. type PlatformWorkloadIdentityProfile struct { - PlatformWorkloadIdentities []PlatformWorkloadIdentity `json:"platformWorkloadIdentities,omitempty"` + UpgradeableTo *UpgradeableTo `json:"upgradeableTo,omitempty" mutable:"true"` + PlatformWorkloadIdentities []PlatformWorkloadIdentity `json:"platformWorkloadIdentities,omitempty" mutable:"true"` } +// UpgradeableTo stores a single OpenShift version a workload identity cluster can be upgraded to +type UpgradeableTo string + // PlatformWorkloadIdentity stores information representing a single workload identity. type PlatformWorkloadIdentity struct { - OperatorName string `json:"operatorName,omitempty"` - ResourceID string `json:"resourceId,omitempty"` - ClientID string `json:"clientId,omitempty" swagger:"readOnly"` - ObjectID string `json:"objectId,omitempty" swagger:"readOnly"` + OperatorName string `json:"operatorName,omitempty" mutable:"true"` + ResourceID string `json:"resourceId,omitempty" mutable:"true"` + ClientID string `json:"clientId,omitempty" swagger:"readOnly" mutable:"true"` + ObjectID string `json:"objectId,omitempty" swagger:"readOnly" mutable:"true"` } // ClusterUserAssignedIdentity stores information about a user-assigned managed identity in a predefined format required by Microsoft's Managed Identity team. diff --git a/pkg/api/v20240812preview/openshiftcluster_convert.go b/pkg/api/v20240812preview/openshiftcluster_convert.go index 1d6ab240a1c..80d0793b617 100644 --- a/pkg/api/v20240812preview/openshiftcluster_convert.go +++ b/pkg/api/v20240812preview/openshiftcluster_convert.go @@ -141,6 +141,12 @@ func (c openShiftClusterConverter) ToExternal(oc *api.OpenShiftCluster) interfac if oc.Properties.PlatformWorkloadIdentityProfile != nil && oc.Properties.PlatformWorkloadIdentityProfile.PlatformWorkloadIdentities != nil { out.Properties.PlatformWorkloadIdentityProfile = &PlatformWorkloadIdentityProfile{} + + if oc.Properties.PlatformWorkloadIdentityProfile.UpgradeableTo != nil { + temp := UpgradeableTo(*oc.Properties.PlatformWorkloadIdentityProfile.UpgradeableTo) + out.Properties.PlatformWorkloadIdentityProfile.UpgradeableTo = &temp + } + out.Properties.PlatformWorkloadIdentityProfile.PlatformWorkloadIdentities = make([]PlatformWorkloadIdentity, len(oc.Properties.PlatformWorkloadIdentityProfile.PlatformWorkloadIdentities)) for i := range oc.Properties.PlatformWorkloadIdentityProfile.PlatformWorkloadIdentities { @@ -225,6 +231,12 @@ func (c openShiftClusterConverter) ToInternal(_oc interface{}, out *api.OpenShif } if oc.Properties.PlatformWorkloadIdentityProfile != nil && oc.Properties.PlatformWorkloadIdentityProfile.PlatformWorkloadIdentities != nil { out.Properties.PlatformWorkloadIdentityProfile = &api.PlatformWorkloadIdentityProfile{} + + if oc.Properties.PlatformWorkloadIdentityProfile.UpgradeableTo != nil { + temp := api.UpgradeableTo(*oc.Properties.PlatformWorkloadIdentityProfile.UpgradeableTo) + out.Properties.PlatformWorkloadIdentityProfile.UpgradeableTo = &temp + } + out.Properties.PlatformWorkloadIdentityProfile.PlatformWorkloadIdentities = make([]api.PlatformWorkloadIdentity, len(oc.Properties.PlatformWorkloadIdentityProfile.PlatformWorkloadIdentities)) for i := range oc.Properties.PlatformWorkloadIdentityProfile.PlatformWorkloadIdentities { diff --git a/pkg/api/v20240812preview/openshiftcluster_validatestatic.go b/pkg/api/v20240812preview/openshiftcluster_validatestatic.go index c027b1cd709..b5011fdd210 100644 --- a/pkg/api/v20240812preview/openshiftcluster_validatestatic.go +++ b/pkg/api/v20240812preview/openshiftcluster_validatestatic.go @@ -12,6 +12,7 @@ import ( azcorearm "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" "github.com/Azure/go-autorest/autorest/azure" + "github.com/coreos/go-semver/semver" "github.com/Azure/ARO-RP/pkg/api" "github.com/Azure/ARO-RP/pkg/api/util/immutable" @@ -442,6 +443,14 @@ func (sv openShiftClusterStaticValidator) validatePlatformWorkloadIdentityProfil return api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidParameter, fmt.Sprintf("%s.PlatformWorkloadIdentities[%d].resourceID", path, n), "Resource must be a user assigned identity.") } } + + if pwip.UpgradeableTo != nil { + _, err := semver.NewVersion(string(*pwip.UpgradeableTo)) + if err != nil { + return api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidParameter, fmt.Sprintf("%s.UpgradeableTo[%v]", path, *pwip.UpgradeableTo), "UpgradeableTo must be a valid OpenShift version in the format 'x.y.z'.") + } + } + return nil } diff --git a/pkg/api/v20240812preview/openshiftcluster_validatestatic_test.go b/pkg/api/v20240812preview/openshiftcluster_validatestatic_test.go index 97e0b802fb5..884df39ea8a 100644 --- a/pkg/api/v20240812preview/openshiftcluster_validatestatic_test.go +++ b/pkg/api/v20240812preview/openshiftcluster_validatestatic_test.go @@ -1163,6 +1163,9 @@ func TestOpenShiftClusterStaticValidateDelta(t *testing.T) { } func TestOpenShiftClusterStaticValidatePlatformWorkloadIdentityProfile(t *testing.T) { + validUpgradeableToValue := UpgradeableTo("4.14.29") + invalidUpgradeableToValue := UpgradeableTo("16.107.invalid") + createTests := []*validateTest{ { name: "valid empty workloadIdentityProfile", @@ -1323,6 +1326,27 @@ func TestOpenShiftClusterStaticValidatePlatformWorkloadIdentityProfile(t *testin }, wantErr: "400: InvalidParameter: properties.servicePrincipalProfile: Must provide either an identity or service principal credentials.", }, + { + name: "valid UpgradeableTo value", + modify: func(oc *OpenShiftCluster) { + oc.Identity = &Identity{} + oc.Properties.ServicePrincipalProfile = nil + oc.Properties.PlatformWorkloadIdentityProfile = &PlatformWorkloadIdentityProfile{ + UpgradeableTo: &validUpgradeableToValue, + } + }, + }, + { + name: "invalid UpgradeableTo value", + modify: func(oc *OpenShiftCluster) { + oc.Identity = &Identity{} + oc.Properties.ServicePrincipalProfile = nil + oc.Properties.PlatformWorkloadIdentityProfile = &PlatformWorkloadIdentityProfile{ + UpgradeableTo: &invalidUpgradeableToValue, + } + }, + wantErr: `400: InvalidParameter: properties.platformWorkloadIdentityProfile.UpgradeableTo[16.107.invalid]: UpgradeableTo must be a valid OpenShift version in the format 'x.y.z'.`, + }, } runTests(t, testModeCreate, createTests) diff --git a/pkg/api/v20240812preview/platformworkloadidentityroleset_convert.go b/pkg/api/v20240812preview/platformworkloadidentityroleset_convert.go index a651229c16f..ffd92a6ef05 100644 --- a/pkg/api/v20240812preview/platformworkloadidentityroleset_convert.go +++ b/pkg/api/v20240812preview/platformworkloadidentityroleset_convert.go @@ -24,10 +24,13 @@ func (c platformWorkloadIdentityRoleSetConverter) ToExternal(s *api.PlatformWork }, } - for i, r := range s.Properties.PlatformWorkloadIdentityRoles { - out.Properties.PlatformWorkloadIdentityRoles[i].OperatorName = r.OperatorName - out.Properties.PlatformWorkloadIdentityRoles[i].RoleDefinitionName = r.RoleDefinitionName - out.Properties.PlatformWorkloadIdentityRoles[i].RoleDefinitionID = r.RoleDefinitionID + for _, r := range s.Properties.PlatformWorkloadIdentityRoles { + role := PlatformWorkloadIdentityRole{ + OperatorName: r.OperatorName, + RoleDefinitionName: r.RoleDefinitionName, + RoleDefinitionID: r.RoleDefinitionID, + } + out.Properties.PlatformWorkloadIdentityRoles = append(out.Properties.PlatformWorkloadIdentityRoles, role) } return out @@ -57,9 +60,12 @@ func (c platformWorkloadIdentityRoleSetConverter) ToInternal(_new interface{}, o out.Properties.OpenShiftVersion = new.Properties.OpenShiftVersion out.Properties.PlatformWorkloadIdentityRoles = make([]api.PlatformWorkloadIdentityRole, 0, len(new.Properties.PlatformWorkloadIdentityRoles)) - for i, r := range new.Properties.PlatformWorkloadIdentityRoles { - out.Properties.PlatformWorkloadIdentityRoles[i].OperatorName = r.OperatorName - out.Properties.PlatformWorkloadIdentityRoles[i].RoleDefinitionName = r.RoleDefinitionName - out.Properties.PlatformWorkloadIdentityRoles[i].RoleDefinitionID = r.RoleDefinitionID + for _, r := range new.Properties.PlatformWorkloadIdentityRoles { + role := api.PlatformWorkloadIdentityRole{ + OperatorName: r.OperatorName, + RoleDefinitionName: r.RoleDefinitionName, + RoleDefinitionID: r.RoleDefinitionID, + } + out.Properties.PlatformWorkloadIdentityRoles = append(out.Properties.PlatformWorkloadIdentityRoles, role) } } diff --git a/pkg/api/v20240812preview/register.go b/pkg/api/v20240812preview/register.go index 2083e837d59..51d13fe8f60 100644 --- a/pkg/api/v20240812preview/register.go +++ b/pkg/api/v20240812preview/register.go @@ -22,6 +22,7 @@ func init() { OpenShiftClusterCredentialsConverter: openShiftClusterCredentialsConverter{}, OpenShiftClusterAdminKubeconfigConverter: openShiftClusterAdminKubeconfigConverter{}, OpenShiftVersionConverter: openShiftVersionConverter{}, + PlatformWorkloadIdentityRoleSetConverter: platformWorkloadIdentityRoleSetConverter{}, OperationList: api.OperationList{ Operations: []api.Operation{ api.OperationResultsRead, diff --git a/pkg/client/services/redhatopenshift/mgmt/2024-08-12-preview/redhatopenshift/models.go b/pkg/client/services/redhatopenshift/mgmt/2024-08-12-preview/redhatopenshift/models.go index e438d5fc15b..2535b27f728 100644 --- a/pkg/client/services/redhatopenshift/mgmt/2024-08-12-preview/redhatopenshift/models.go +++ b/pkg/client/services/redhatopenshift/mgmt/2024-08-12-preview/redhatopenshift/models.go @@ -1525,6 +1525,7 @@ func (pwi PlatformWorkloadIdentity) MarshalJSON() ([]byte, error) { // PlatformWorkloadIdentityProfile platformWorkloadIdentityProfile encapsulates all information that is // specific to workload identity clusters. type PlatformWorkloadIdentityProfile struct { + UpgradeableTo *string `json:"upgradeableTo,omitempty"` PlatformWorkloadIdentities *[]PlatformWorkloadIdentity `json:"platformWorkloadIdentities,omitempty"` } diff --git a/pkg/cluster/arooperator.go b/pkg/cluster/arooperator.go index 7afbb5b3c9d..ca187e23fbe 100644 --- a/pkg/cluster/arooperator.go +++ b/pkg/cluster/arooperator.go @@ -65,6 +65,10 @@ func (m *manager) ensureCredentialsRequest(ctx context.Context) error { return m.aroOperatorDeployer.CreateOrUpdateCredentialsRequest(ctx) } +func (m *manager) ensureUpgradeAnnotation(ctx context.Context) error { + return m.aroOperatorDeployer.EnsureUpgradeAnnotation(ctx) +} + func (m *manager) renewMDSDCertificate(ctx context.Context) error { return m.aroOperatorDeployer.RenewMDSDCertificate(ctx) } diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index b92520d266e..5c990d819f0 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -7,6 +7,8 @@ import ( "context" "time" + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" "github.com/Azure/go-autorest/autorest" "github.com/Azure/go-autorest/autorest/azure" configclient "github.com/openshift/client-go/config/clientset/versioned" @@ -31,6 +33,7 @@ import ( aroclient "github.com/Azure/ARO-RP/pkg/operator/clientset/versioned" "github.com/Azure/ARO-RP/pkg/operator/deploy" "github.com/Azure/ARO-RP/pkg/util/azureclient/azuresdk/armnetwork" + "github.com/Azure/ARO-RP/pkg/util/azureclient/azuresdk/common" "github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/authorization" "github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/compute" "github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/features" @@ -72,7 +75,8 @@ type manager struct { virtualMachines compute.VirtualMachinesClient interfaces network.InterfacesClient // TODO: use armInterfaces instead. armInterfaces armnetwork.InterfacesClient - publicIPAddresses network.PublicIPAddressesClient + publicIPAddresses network.PublicIPAddressesClient // TODO: use armPublicIPAddresses instead. + armPublicIPAddresses armnetwork.PublicIPAddressesClient loadBalancers network.LoadBalancersClient // TODO: use armLoadBalancers instead. armLoadBalancers armnetwork.LoadBalancersClient privateEndpoints network.PrivateEndpointsClient @@ -159,12 +163,24 @@ func New(ctx context.Context, log *logrus.Entry, _env env.Interface, db database return nil, err } - armLoadBalancersClient, err := armnetwork.NewLoadBalancersClient(_env.Environment(), r.SubscriptionID, fpCredential) + clientOptions := arm.ClientOptions{ + ClientOptions: azcore.ClientOptions{ + Cloud: _env.Environment().Cloud, + Retry: common.RetryOptions, + }, + } + + armLoadBalancersClient, err := armnetwork.NewLoadBalancersClient(r.SubscriptionID, fpCredential, &clientOptions) + if err != nil { + return nil, err + } + + armInterfacesClient, err := armnetwork.NewInterfacesClient(r.SubscriptionID, fpCredential, &clientOptions) if err != nil { return nil, err } - armInterfacesClient, err := armnetwork.NewInterfacesClient(_env.Environment(), r.SubscriptionID, fpCredential) + armPublicIPAddressesClient, err := armnetwork.NewPublicIPAddressesClient(r.SubscriptionID, fpCredential, &clientOptions) if err != nil { return nil, err } @@ -186,6 +202,7 @@ func New(ctx context.Context, log *logrus.Entry, _env env.Interface, db database interfaces: network.NewInterfacesClient(_env.Environment(), r.SubscriptionID, fpAuthorizer), armInterfaces: armInterfacesClient, publicIPAddresses: network.NewPublicIPAddressesClient(_env.Environment(), r.SubscriptionID, fpAuthorizer), + armPublicIPAddresses: armPublicIPAddressesClient, loadBalancers: network.NewLoadBalancersClient(_env.Environment(), r.SubscriptionID, fpAuthorizer), armLoadBalancers: armLoadBalancersClient, privateEndpoints: network.NewPrivateEndpointsClient(_env.Environment(), r.SubscriptionID, fpAuthorizer), diff --git a/pkg/cluster/install.go b/pkg/cluster/install.go index 5e14d57ef16..072b9cb29c2 100644 --- a/pkg/cluster/install.go +++ b/pkg/cluster/install.go @@ -213,6 +213,7 @@ func (m *manager) Update(ctx context.Context) error { steps.Action(m.updateAROSecret), steps.Action(m.restartAROOperatorMaster), // depends on m.updateOpenShiftSecret; the point of restarting is to pick up any changes made to the secret steps.Condition(m.aroDeploymentReady, 5*time.Minute, true), + steps.Action(m.ensureUpgradeAnnotation), steps.Action(m.reconcileLoadBalancerProfile), } @@ -524,7 +525,7 @@ func (m *manager) initializeKubernetesClients(ctx context.Context) error { // initializeKubernetesClients initializes clients which are used // once the cluster is up later on in the install process. func (m *manager) initializeOperatorDeployer(ctx context.Context) (err error) { - m.aroOperatorDeployer, err = deploy.New(m.log, m.env, m.doc.OpenShiftCluster, m.arocli, m.client, m.extensionscli, m.kubernetescli) + m.aroOperatorDeployer, err = deploy.New(m.log, m.env, m.doc.OpenShiftCluster, m.arocli, m.client, m.extensionscli, m.kubernetescli, m.operatorcli) return } diff --git a/pkg/cluster/ipaddresses.go b/pkg/cluster/ipaddresses.go index aad66abec2e..95f05559e33 100644 --- a/pkg/cluster/ipaddresses.go +++ b/pkg/cluster/ipaddresses.go @@ -164,25 +164,27 @@ func (m *manager) populateDatabaseIntIP(ctx context.Context) error { return err } -// this function can only be called on create - not on update - because it +// updateAPIIPEarly updates the `doc` with the public and private IP of the API server, +// and updates the DNS record of the API server according to the API server visibility. +// This function can only be called on create - not on update - because it // refers to -pip-v4, which doesn't exist on pre-DNS change clusters. func (m *manager) updateAPIIPEarly(ctx context.Context) error { infraID := m.doc.OpenShiftCluster.Properties.InfraID resourceGroup := stringutils.LastTokenByte(m.doc.OpenShiftCluster.Properties.ClusterProfile.ResourceGroupID, '/') - lb, err := m.loadBalancers.Get(ctx, resourceGroup, infraID+"-internal", "") + lb, err := m.armLoadBalancers.Get(ctx, resourceGroup, infraID+"-internal", nil) if err != nil { return err } - intIPAddress := *((*lb.FrontendIPConfigurations)[0].PrivateIPAddress) + intIPAddress := *lb.Properties.FrontendIPConfigurations[0].Properties.PrivateIPAddress ipAddress := intIPAddress if m.doc.OpenShiftCluster.Properties.APIServerProfile.Visibility == api.VisibilityPublic { - ip, err := m.publicIPAddresses.Get(ctx, resourceGroup, infraID+"-pip-v4", "") + ip, err := m.armPublicIPAddresses.Get(ctx, resourceGroup, infraID+"-pip-v4", nil) if err != nil { return err } - ipAddress = *ip.IPAddress + ipAddress = *ip.Properties.IPAddress } err = m.dns.Update(ctx, m.doc.OpenShiftCluster, ipAddress) diff --git a/pkg/cluster/ipaddresses_test.go b/pkg/cluster/ipaddresses_test.go index d2f95a40643..eac3d534e13 100644 --- a/pkg/cluster/ipaddresses_test.go +++ b/pkg/cluster/ipaddresses_test.go @@ -8,6 +8,7 @@ import ( "strings" "testing" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2" mgmtnetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2020-08-01/network" "github.com/Azure/go-autorest/autorest/to" "github.com/golang/mock/gomock" @@ -18,6 +19,7 @@ import ( "github.com/Azure/ARO-RP/pkg/api" "github.com/Azure/ARO-RP/pkg/database/cosmosdb" + mock_armnetwork "github.com/Azure/ARO-RP/pkg/util/mocks/azureclient/azuresdk/armnetwork" mock_network "github.com/Azure/ARO-RP/pkg/util/mocks/azureclient/mgmt/network" mock_dns "github.com/Azure/ARO-RP/pkg/util/mocks/dns" mock_env "github.com/Azure/ARO-RP/pkg/util/mocks/env" @@ -26,6 +28,11 @@ import ( utilerror "github.com/Azure/ARO-RP/test/util/error" ) +const ( + privateIP = "10.0.0.1" + publicIP = "1.2.3.4" +) + func TestCreateOrUpdateRouterIPFromCluster(t *testing.T) { ctx := context.Background() @@ -61,7 +68,7 @@ func TestCreateOrUpdateRouterIPFromCluster(t *testing.T) { fixture.AddOpenShiftClusterDocuments(doc) doc.Dequeues = 1 - doc.OpenShiftCluster.Properties.IngressProfiles[0].IP = "1.2.3.4" + doc.OpenShiftCluster.Properties.IngressProfiles[0].IP = publicIP checker.AddOpenShiftClusterDocuments(doc) }, mocks: func(dns *mock_dns.MockManager) { @@ -77,7 +84,7 @@ func TestCreateOrUpdateRouterIPFromCluster(t *testing.T) { Status: corev1.ServiceStatus{ LoadBalancer: corev1.LoadBalancerStatus{ Ingress: []corev1.LoadBalancerIngress{{ - IP: "1.2.3.4", + IP: publicIP, }}, }, }, @@ -217,7 +224,7 @@ func TestCreateOrUpdateRouterIPEarly(t *testing.T) { fixture.AddOpenShiftClusterDocuments(doc) doc.Dequeues = 1 - doc.OpenShiftCluster.Properties.IngressProfiles[0].IP = "1.2.3.4" + doc.OpenShiftCluster.Properties.IngressProfiles[0].IP = publicIP checker.AddOpenShiftClusterDocuments(doc) }, mocks: func(publicIPAddresses *mock_network.MockPublicIPAddressesClient, dns *mock_dns.MockManager, subnet *mock_subnet.MockManager) { @@ -225,7 +232,7 @@ func TestCreateOrUpdateRouterIPEarly(t *testing.T) { Get(gomock.Any(), "clusterResourceGroup", "infra-default-v4", ""). Return(mgmtnetwork.PublicIPAddress{ PublicIPAddressPropertiesFormat: &mgmtnetwork.PublicIPAddressPropertiesFormat{ - IPAddress: to.StringPtr("1.2.3.4"), + IPAddress: to.StringPtr(publicIP), }, }, nil) dns.EXPECT(). @@ -262,13 +269,13 @@ func TestCreateOrUpdateRouterIPEarly(t *testing.T) { fixture.AddOpenShiftClusterDocuments(doc) doc.Dequeues = 1 - doc.OpenShiftCluster.Properties.IngressProfiles[0].IP = "1.2.3.4" + doc.OpenShiftCluster.Properties.IngressProfiles[0].IP = publicIP checker.AddOpenShiftClusterDocuments(doc) }, mocks: func(publicIPAddresses *mock_network.MockPublicIPAddressesClient, dns *mock_dns.MockManager, subnet *mock_subnet.MockManager) { subnet.EXPECT(). GetHighestFreeIP(gomock.Any(), "subnetid"). - Return("1.2.3.4", nil) + Return(publicIP, nil) dns.EXPECT(). CreateOrUpdateRouter(gomock.Any(), gomock.Any(), gomock.Any()). Return(nil) @@ -308,13 +315,13 @@ func TestCreateOrUpdateRouterIPEarly(t *testing.T) { fixture.AddOpenShiftClusterDocuments(doc) doc.Dequeues = 1 - doc.OpenShiftCluster.Properties.IngressProfiles[0].IP = "1.2.3.4" + doc.OpenShiftCluster.Properties.IngressProfiles[0].IP = publicIP checker.AddOpenShiftClusterDocuments(doc) }, mocks: func(publicIPAddresses *mock_network.MockPublicIPAddressesClient, dns *mock_dns.MockManager, subnet *mock_subnet.MockManager) { subnet.EXPECT(). GetHighestFreeIP(gomock.Any(), "enricheWPsubnetid"). - Return("1.2.3.4", nil) + Return(publicIP, nil) dns.EXPECT(). CreateOrUpdateRouter(gomock.Any(), gomock.Any(), gomock.Any()). Return(nil) @@ -402,7 +409,7 @@ func TestPopulateDatabaseIntIP(t *testing.T) { fixture.AddOpenShiftClusterDocuments(doc) doc.Dequeues = 1 - doc.OpenShiftCluster.Properties.APIServerProfile.IntIP = "10.0.0.1" + doc.OpenShiftCluster.Properties.APIServerProfile.IntIP = privateIP checker.AddOpenShiftClusterDocuments(doc) }, mocks: func(loadBalancers *mock_network.MockLoadBalancersClient) { @@ -413,7 +420,7 @@ func TestPopulateDatabaseIntIP(t *testing.T) { FrontendIPConfigurations: &[]mgmtnetwork.FrontendIPConfiguration{ { FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{ - PrivateIPAddress: to.StringPtr("10.0.0.1"), + PrivateIPAddress: to.StringPtr(privateIP), }, }, }, @@ -441,7 +448,7 @@ func TestPopulateDatabaseIntIP(t *testing.T) { fixture.AddOpenShiftClusterDocuments(doc) doc.Dequeues = 1 - doc.OpenShiftCluster.Properties.APIServerProfile.IntIP = "10.0.0.1" + doc.OpenShiftCluster.Properties.APIServerProfile.IntIP = privateIP checker.AddOpenShiftClusterDocuments(doc) }, mocks: func(loadBalancers *mock_network.MockLoadBalancersClient) { @@ -452,7 +459,7 @@ func TestPopulateDatabaseIntIP(t *testing.T) { FrontendIPConfigurations: &[]mgmtnetwork.FrontendIPConfiguration{ { FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{ - PrivateIPAddress: to.StringPtr("10.0.0.1"), + PrivateIPAddress: to.StringPtr(privateIP), }, }, }, @@ -472,7 +479,7 @@ func TestPopulateDatabaseIntIP(t *testing.T) { ResourceGroupID: resourceGroupID, }, APIServerProfile: api.APIServerProfile{ - IntIP: "10.0.0.1", + IntIP: privateIP, }, ProvisioningState: api.ProvisioningStateCreating, InfraID: "infra", @@ -540,7 +547,7 @@ func TestUpdateAPIIPEarly(t *testing.T) { for _, tt := range []struct { name string fixtureChecker func(*testdatabase.Fixture, *testdatabase.Checker, *cosmosdb.FakeOpenShiftClusterDocumentClient) - mocks func(*mock_network.MockLoadBalancersClient, *mock_network.MockPublicIPAddressesClient, *mock_dns.MockManager) + mocks func(*mock_armnetwork.MockLoadBalancersClient, *mock_armnetwork.MockPublicIPAddressesClient, *mock_dns.MockManager) wantErr string }{ { @@ -565,33 +572,37 @@ func TestUpdateAPIIPEarly(t *testing.T) { fixture.AddOpenShiftClusterDocuments(doc) doc.Dequeues = 1 - doc.OpenShiftCluster.Properties.APIServerProfile.IP = "1.2.3.4" - doc.OpenShiftCluster.Properties.APIServerProfile.IntIP = "10.0.0.1" + doc.OpenShiftCluster.Properties.APIServerProfile.IP = publicIP + doc.OpenShiftCluster.Properties.APIServerProfile.IntIP = privateIP checker.AddOpenShiftClusterDocuments(doc) }, - mocks: func(loadBalancers *mock_network.MockLoadBalancersClient, publicIPAddresses *mock_network.MockPublicIPAddressesClient, dns *mock_dns.MockManager) { + mocks: func(loadBalancers *mock_armnetwork.MockLoadBalancersClient, publicIPAddresses *mock_armnetwork.MockPublicIPAddressesClient, dns *mock_dns.MockManager) { loadBalancers.EXPECT(). - Get(gomock.Any(), "clusterResourceGroup", "infra-internal", ""). - Return(mgmtnetwork.LoadBalancer{ - LoadBalancerPropertiesFormat: &mgmtnetwork.LoadBalancerPropertiesFormat{ - FrontendIPConfigurations: &[]mgmtnetwork.FrontendIPConfiguration{ - { - FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{ - PrivateIPAddress: to.StringPtr("10.0.0.1"), + Get(gomock.Any(), "clusterResourceGroup", "infra-internal", nil). + Return(armnetwork.LoadBalancersClientGetResponse{ + LoadBalancer: armnetwork.LoadBalancer{ + Properties: &armnetwork.LoadBalancerPropertiesFormat{ + FrontendIPConfigurations: []*armnetwork.FrontendIPConfiguration{ + { + Properties: &armnetwork.FrontendIPConfigurationPropertiesFormat{ + PrivateIPAddress: to.StringPtr(privateIP), + }, }, }, }, }, }, nil) publicIPAddresses.EXPECT(). - Get(gomock.Any(), "clusterResourceGroup", "infra-pip-v4", ""). - Return(mgmtnetwork.PublicIPAddress{ - PublicIPAddressPropertiesFormat: &mgmtnetwork.PublicIPAddressPropertiesFormat{ - IPAddress: to.StringPtr("1.2.3.4"), + Get(gomock.Any(), "clusterResourceGroup", "infra-pip-v4", nil). + Return(armnetwork.PublicIPAddressesClientGetResponse{ + PublicIPAddress: armnetwork.PublicIPAddress{ + Properties: &armnetwork.PublicIPAddressPropertiesFormat{ + IPAddress: to.StringPtr(publicIP), + }, }, }, nil) dns.EXPECT(). - Update(gomock.Any(), gomock.Any(), gomock.Any()). + Update(gomock.Any(), gomock.Any(), publicIP). Return(nil) }, }, @@ -617,26 +628,28 @@ func TestUpdateAPIIPEarly(t *testing.T) { fixture.AddOpenShiftClusterDocuments(doc) doc.Dequeues = 1 - doc.OpenShiftCluster.Properties.APIServerProfile.IP = "10.0.0.1" - doc.OpenShiftCluster.Properties.APIServerProfile.IntIP = "10.0.0.1" + doc.OpenShiftCluster.Properties.APIServerProfile.IP = privateIP + doc.OpenShiftCluster.Properties.APIServerProfile.IntIP = privateIP checker.AddOpenShiftClusterDocuments(doc) }, - mocks: func(loadBalancers *mock_network.MockLoadBalancersClient, publicIPAddresses *mock_network.MockPublicIPAddressesClient, dns *mock_dns.MockManager) { + mocks: func(loadBalancers *mock_armnetwork.MockLoadBalancersClient, publicIPAddresses *mock_armnetwork.MockPublicIPAddressesClient, dns *mock_dns.MockManager) { loadBalancers.EXPECT(). - Get(gomock.Any(), "clusterResourceGroup", "infra-internal", ""). - Return(mgmtnetwork.LoadBalancer{ - LoadBalancerPropertiesFormat: &mgmtnetwork.LoadBalancerPropertiesFormat{ - FrontendIPConfigurations: &[]mgmtnetwork.FrontendIPConfiguration{ - { - FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{ - PrivateIPAddress: to.StringPtr("10.0.0.1"), + Get(gomock.Any(), "clusterResourceGroup", "infra-internal", nil). + Return(armnetwork.LoadBalancersClientGetResponse{ + LoadBalancer: armnetwork.LoadBalancer{ + Properties: &armnetwork.LoadBalancerPropertiesFormat{ + FrontendIPConfigurations: []*armnetwork.FrontendIPConfiguration{ + { + Properties: &armnetwork.FrontendIPConfigurationPropertiesFormat{ + PrivateIPAddress: to.StringPtr(privateIP), + }, }, }, }, }, }, nil) dns.EXPECT(). - Update(gomock.Any(), gomock.Any(), gomock.Any()). + Update(gomock.Any(), gomock.Any(), privateIP). Return(nil) }, }, @@ -645,8 +658,8 @@ func TestUpdateAPIIPEarly(t *testing.T) { controller := gomock.NewController(t) defer controller.Finish() - loadBalancers := mock_network.NewMockLoadBalancersClient(controller) - publicIPAddresses := mock_network.NewMockPublicIPAddressesClient(controller) + loadBalancers := mock_armnetwork.NewMockLoadBalancersClient(controller) + publicIPAddresses := mock_armnetwork.NewMockPublicIPAddressesClient(controller) dns := mock_dns.NewMockManager(controller) if tt.mocks != nil { tt.mocks(loadBalancers, publicIPAddresses, dns) @@ -671,11 +684,11 @@ func TestUpdateAPIIPEarly(t *testing.T) { } m := &manager{ - doc: doc, - db: dbOpenShiftClusters, - publicIPAddresses: publicIPAddresses, - loadBalancers: loadBalancers, - dns: dns, + doc: doc, + db: dbOpenShiftClusters, + armPublicIPAddresses: publicIPAddresses, + armLoadBalancers: loadBalancers, + dns: dns, } err = m.updateAPIIPEarly(ctx) @@ -706,7 +719,7 @@ func TestEnsureGatewayCreate(t *testing.T) { }, { name: "noop: IP set", - gatewayPrivateEndpointIP: "1.2.3.4", + gatewayPrivateEndpointIP: privateIP, }, { name: "error: private endpoint connection not found", @@ -720,7 +733,7 @@ func TestEnsureGatewayCreate(t *testing.T) { IPConfigurations: &[]mgmtnetwork.InterfaceIPConfiguration{ { InterfaceIPConfigurationPropertiesFormat: &mgmtnetwork.InterfaceIPConfigurationPropertiesFormat{ - PrivateIPAddress: to.StringPtr("1.2.3.4"), + PrivateIPAddress: to.StringPtr(privateIP), }, }, }, @@ -751,7 +764,7 @@ func TestEnsureGatewayCreate(t *testing.T) { IPConfigurations: &[]mgmtnetwork.InterfaceIPConfiguration{ { InterfaceIPConfigurationPropertiesFormat: &mgmtnetwork.InterfaceIPConfigurationPropertiesFormat{ - PrivateIPAddress: to.StringPtr("1.2.3.4"), + PrivateIPAddress: to.StringPtr(privateIP), }, }, }, @@ -815,7 +828,7 @@ func TestEnsureGatewayCreate(t *testing.T) { ID: resourceID, Properties: api.OpenShiftClusterProperties{ NetworkProfile: api.NetworkProfile{ - GatewayPrivateEndpointIP: "1.2.3.4", + GatewayPrivateEndpointIP: privateIP, GatewayPrivateLinkID: "1234", }, }, diff --git a/pkg/cluster/loadbalancerprofile.go b/pkg/cluster/loadbalancerprofile.go index ba7ed73e19c..578fc4d7538 100644 --- a/pkg/cluster/loadbalancerprofile.go +++ b/pkg/cluster/loadbalancerprofile.go @@ -19,6 +19,16 @@ import ( const outboundRuleV4 = "outbound-rule-v4" +type deleteIPResult struct { + name string + err error +} + +type createIPResult struct { + ip mgmtnetwork.PublicIPAddress + err error +} + func (m *manager) reconcileLoadBalancerProfile(ctx context.Context) error { if m.doc.OpenShiftCluster.Properties.NetworkProfile.OutboundType != api.OutboundTypeLoadbalancer || m.doc.OpenShiftCluster.Properties.ArchitectureVersion == api.ArchitectureVersionV1 { return nil @@ -68,7 +78,7 @@ func (m *manager) reconcileOutboundRuleV4IPsInner(ctx context.Context, lb mgmtne } } - desiredOutboundIPs, err := m.getDesiredOutboundIPs(ctx) + desiredOutboundIPs, err := m.reconcileOutboundIPs(ctx) if err != nil { return err } @@ -95,9 +105,29 @@ func (m *manager) reconcileOutboundRuleV4IPsInner(ctx context.Context, lb mgmtne return nil } -// Remove all frontend ip config in use by outbound-rule-v4. Frontend IP config that is used by load balancer rules will be saved. +// Remove outbound-rule-v4 IPs and corresponding frontendIPConfig from load balancer func removeOutboundIPsFromLB(lb mgmtnetwork.LoadBalancer) { - // get all outbound rule fip config to remove + removeOutboundRuleV4FrontendIPConfig(lb) + setOutboundRuleV4(lb, []mgmtnetwork.SubResource{}) +} + +func removeOutboundRuleV4FrontendIPConfig(lb mgmtnetwork.LoadBalancer) { + var savedFIPConfig = make([]mgmtnetwork.FrontendIPConfiguration, 0, len(*lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations)) + var outboundRuleFrontendConfig = getOutboundRuleV4FIPConfigs(lb) + + for i := 0; i < len(*lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations); i++ { + fipConfigID := *(*lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations)[i].ID + fipConfig := (*lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations)[i] + hasLBRules := (*lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations)[i].LoadBalancingRules != nil + if _, ok := outboundRuleFrontendConfig[fipConfigID]; ok && !hasLBRules { + continue + } + savedFIPConfig = append(savedFIPConfig, fipConfig) + } + lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations = &savedFIPConfig +} + +func getOutboundRuleV4FIPConfigs(lb mgmtnetwork.LoadBalancer) map[string]mgmtnetwork.SubResource { var obRuleV4FIPConfigs = make(map[string]mgmtnetwork.SubResource) for _, obRule := range *lb.LoadBalancerPropertiesFormat.OutboundRules { if *obRule.Name == outboundRuleV4 { @@ -106,36 +136,22 @@ func removeOutboundIPsFromLB(lb mgmtnetwork.LoadBalancer) { fipConfig := (*obRule.OutboundRulePropertiesFormat.FrontendIPConfigurations)[i] obRuleV4FIPConfigs[fipConfigID] = fipConfig } - // clear outbound-rule-v4 frontend ip config - *obRule.FrontendIPConfigurations = []mgmtnetwork.SubResource{} break } } - - // rebuild frontend ip config without outbound-rule-v4 frontend ip config, preserving - // the public api server frontend ip config if the api server is public - var savedFIPConfig = make([]mgmtnetwork.FrontendIPConfiguration, 0, len(*lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations)) - for i := 0; i < len(*lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations); i++ { - fipConfigID := *(*lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations)[i].ID - fipConfig := (*lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations)[i] - fipLBRules := (*lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations)[i].LoadBalancingRules - if _, ok := obRuleV4FIPConfigs[fipConfigID]; ok && fipLBRules == nil { - continue - } - savedFIPConfig = append(savedFIPConfig, fipConfig) - } - lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations = &savedFIPConfig + return obRuleV4FIPConfigs } -// return a map of Frontend IP Configs where the key is the ID of the Frontend IP Config +// Returns a map of Frontend IP Configurations. Frontend IP Configurations can be looked up by Public IP Address ID or Frontend IP Configuration ID func getFrontendIPConfigs(lb mgmtnetwork.LoadBalancer) map[string]mgmtnetwork.FrontendIPConfiguration { - // map out frontendConfig to ID of public IP addresses for quick lookup var frontendIPConfigs = make(map[string]mgmtnetwork.FrontendIPConfiguration, len(*lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations)) for i := 0; i < len(*lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations); i++ { - fipConfigIPID := *(*lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations)[i].FrontendIPConfigurationPropertiesFormat.PublicIPAddress.ID + fipConfigID := *(*lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations)[i].ID + fipConfigIPAddressID := *(*lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations)[i].FrontendIPConfigurationPropertiesFormat.PublicIPAddress.ID fipConfig := (*lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations)[i] - frontendIPConfigs[fipConfigIPID] = fipConfig + frontendIPConfigs[fipConfigID] = fipConfig + frontendIPConfigs[fipConfigIPAddressID] = fipConfig } return frontendIPConfigs @@ -147,21 +163,24 @@ func addOutboundIPsToLB(resourceGroupID string, lb mgmtnetwork.LoadBalancer, obI outboundRuleV4FrontendIPConfig := []mgmtnetwork.SubResource{} // add IP Addresses to frontendConfig - for _, obIPOrPrefix := range obIPsOrIPPrefixes { + for _, obIPOrIPPrefix := range obIPsOrIPPrefixes { // check if the frontend config exists in the map to avoid duplicate entries - if _, ok := frontendIPConfigs[obIPOrPrefix.ID]; !ok { - frontendIPConfigName := stringutils.LastTokenByte(obIPOrPrefix.ID, '/') + if _, ok := frontendIPConfigs[obIPOrIPPrefix.ID]; !ok { + frontendIPConfigName := stringutils.LastTokenByte(obIPOrIPPrefix.ID, '/') frontendConfigID := fmt.Sprintf("%s/providers/Microsoft.Network/loadBalancers/%s/frontendIPConfigurations/%s", resourceGroupID, *lb.Name, frontendIPConfigName) - *lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations = append(*lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations, newFrontendIPConfig(frontendIPConfigName, frontendConfigID, obIPOrPrefix.ID)) + *lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations = append(*lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations, newFrontendIPConfig(frontendIPConfigName, frontendConfigID, obIPOrIPPrefix.ID)) outboundRuleV4FrontendIPConfig = append(outboundRuleV4FrontendIPConfig, newOutboundRuleFrontendIPConfig(frontendConfigID)) } else { // frontendIPConfig already exists and just needs to be added to the outbound rule - frontendConfig := frontendIPConfigs[obIPOrPrefix.ID] + frontendConfig := frontendIPConfigs[obIPOrIPPrefix.ID] outboundRuleV4FrontendIPConfig = append(outboundRuleV4FrontendIPConfig, newOutboundRuleFrontendIPConfig(*frontendConfig.ID)) } } - // update outbound-rule-v4 + setOutboundRuleV4(lb, outboundRuleV4FrontendIPConfig) +} + +func setOutboundRuleV4(lb mgmtnetwork.LoadBalancer, outboundRuleV4FrontendIPConfig []mgmtnetwork.SubResource) { for _, outboundRule := range *lb.LoadBalancerPropertiesFormat.OutboundRules { if *outboundRule.Name == outboundRuleV4 { outboundRule.OutboundRulePropertiesFormat.FrontendIPConfigurations = &outboundRuleV4FrontendIPConfig @@ -174,16 +193,56 @@ func addOutboundIPsToLB(resourceGroupID string, lb mgmtnetwork.LoadBalancer, obI // The default outbound ip is saved if the api server is public. func (m *manager) deleteUnusedManagedIPs(ctx context.Context) error { resourceGroupName := stringutils.LastTokenByte(m.doc.OpenShiftCluster.Properties.ClusterProfile.ResourceGroupID, '/') + + unusedManagedIPs, err := m.getUnusedManagedIPs(ctx) + if err != nil { + return err + } + + ch := make(chan deleteIPResult) + defer close(ch) + var cleanupErrors []string + + for _, id := range unusedManagedIPs { + ipName := stringutils.LastTokenByte(id, '/') + go m.deleteIPAddress(ctx, resourceGroupName, ipName, ch) + } + + for range unusedManagedIPs { + result := <-ch + if result.err != nil { + cleanupErrors = append(cleanupErrors, fmt.Sprintf("deletion of unused managed ip %s failed with error: %v", result.name, result.err)) + } + } + + if cleanupErrors != nil { + return fmt.Errorf("failed to cleanup unused managed ips\n%s", strings.Join(cleanupErrors, "\n")) + } + + return nil +} + +func (m *manager) deleteIPAddress(ctx context.Context, resourceGroupName string, ipName string, ch chan<- deleteIPResult) { + m.log.Infof("deleting managed public IP Address: %s", ipName) + err := m.publicIPAddresses.DeleteAndWait(ctx, resourceGroupName, ipName) + ch <- deleteIPResult{ + name: ipName, + err: err, + } +} + +func (m *manager) getUnusedManagedIPs(ctx context.Context) ([]string, error) { + resourceGroupName := stringutils.LastTokenByte(m.doc.OpenShiftCluster.Properties.ClusterProfile.ResourceGroupID, '/') infraID := m.doc.OpenShiftCluster.Properties.InfraID managedIPs, err := m.getClusterManagedIPs(ctx) if err != nil { - return err + return nil, err } lb, err := m.loadBalancers.Get(ctx, resourceGroupName, infraID, "") if err != nil { - return err + return nil, err } outboundIPs := getOutboundIPsFromLB(lb) @@ -191,31 +250,22 @@ func (m *manager) deleteUnusedManagedIPs(ctx context.Context) error { for i := 0; i < len(outboundIPs); i++ { outboundIPMap[strings.ToLower(outboundIPs[i].ID)] = outboundIPs[i] } - var cleanupErrors []string + var unusedManagedIPs []string for _, ip := range managedIPs { // don't delete api server ip if *ip.Name == infraID+"-pip-v4" && m.doc.OpenShiftCluster.Properties.APIServerProfile.Visibility == api.VisibilityPublic { continue } if _, ok := outboundIPMap[strings.ToLower(*ip.ID)]; !ok && strings.Contains(strings.ToLower(*ip.ID), strings.ToLower(m.doc.OpenShiftCluster.Properties.ClusterProfile.ResourceGroupID)) { - ipName := stringutils.LastTokenByte(*ip.ID, '/') - m.log.Infof("deleting managed public IP Address: %s", ipName) - err := m.publicIPAddresses.DeleteAndWait(ctx, resourceGroupName, ipName) - if err != nil { - cleanupErrors = append(cleanupErrors, fmt.Sprintf("deletion of unused managed ip %s failed with error: %v", ipName, err)) - } + unusedManagedIPs = append(unusedManagedIPs, *ip.ID) } } - if cleanupErrors != nil { - return fmt.Errorf("failed to cleanup unused managed ips\n%s", strings.Join(cleanupErrors, "\n")) - } - - return nil + return unusedManagedIPs, nil } // Returns the desired RP managed outbound publicIPAddresses. Additional Managed Outbound IPs // will be created as required to satisfy ManagedOutboundIP.Count. -func (m *manager) getDesiredOutboundIPs(ctx context.Context) ([]api.ResourceReference, error) { +func (m *manager) reconcileOutboundIPs(ctx context.Context) ([]api.ResourceReference, error) { // Determine source of outbound IPs // TODO: add customer provided ip and ip prefixes if m.doc.OpenShiftCluster.Properties.NetworkProfile.LoadBalancerProfile.ManagedOutboundIPs != nil { @@ -230,23 +280,27 @@ func (m *manager) getDesiredOutboundIPs(ctx context.Context) ([]api.ResourceRefe func (m *manager) reconcileDesiredManagedIPs(ctx context.Context) ([]api.ResourceReference, error) { infraID := m.doc.OpenShiftCluster.Properties.InfraID managedOBIPCount := m.doc.OpenShiftCluster.Properties.NetworkProfile.LoadBalancerProfile.ManagedOutboundIPs.Count - desiredIPAddresses := make([]api.ResourceReference, 0, managedOBIPCount) ipAddresses, err := m.getClusterManagedIPs(ctx) if err != nil { return nil, err } - // create additional IPs if needed numToCreate := managedOBIPCount - len(ipAddresses) - for i := 0; i < numToCreate; i++ { - ipAddress, err := m.createPublicIPAddress(ctx) + + if numToCreate > 0 { + err = m.createPublicIPAddresses(ctx, ipAddresses, numToCreate) if err != nil { return nil, err } - ipAddresses[*ipAddress.Name] = ipAddress } + desiredIPAddresses := getDesiredOutboundIPs(managedOBIPCount, ipAddresses, infraID) + return desiredIPAddresses, nil +} + +func getDesiredOutboundIPs(managedOBIPCount int, ipAddresses map[string]mgmtnetwork.PublicIPAddress, infraID string) []api.ResourceReference { + desiredIPAddresses := make([]api.ResourceReference, 0, managedOBIPCount) // ensure that when scaling managed ips down the default outbound IP is reused incase the api server visibility is public desiredCount := 0 if defaultIP, ok := ipAddresses[infraID+"-pip-v4"]; ok { @@ -263,8 +317,31 @@ func (m *manager) reconcileDesiredManagedIPs(ctx context.Context) ([]api.Resourc break } } + return desiredIPAddresses +} - return desiredIPAddresses, nil +func (m *manager) createPublicIPAddresses(ctx context.Context, ipAddresses map[string]mgmtnetwork.PublicIPAddress, numToCreate int) error { + ch := make(chan createIPResult) + defer close(ch) + var errResults []string + // create additional IPs if needed + for i := 0; i < numToCreate; i++ { + go m.createPublicIPAddress(ctx, ch) + } + + for i := 0; i < numToCreate; i++ { + result := <-ch + if result.err != nil { + errResults = append(errResults, fmt.Sprintf("creation of ip address %s failed with error: %s", *result.ip.Name, result.err.Error())) + } else { + ipAddresses[*result.ip.Name] = result.ip + } + } + + if len(errResults) > 0 { + return fmt.Errorf("failed to create required IPs\n%s", strings.Join(errResults, "\n")) + } + return nil } // Get all current managed IP Addresses in cluster resource group based on naming convention. @@ -279,7 +356,7 @@ func (m *manager) getClusterManagedIPs(ctx context.Context) (map[string]mgmtnetw } for i := 0; i < len(result); i++ { - // -pip-v4 is not necessarily managed but is the default installed outbound IP + // -pip-v4 is the default installed outbound IP if *result[i].Name == infraID+"-pip-v4" || strings.Contains(*result[i].Name, "-outbound-pip-v4") { ipAddresses[*result[i].Name] = result[i] } @@ -293,41 +370,23 @@ func genManagedOutboundIPName() string { } // Create a managed outbound IP Address. -func (m *manager) createPublicIPAddress(ctx context.Context) (mgmtnetwork.PublicIPAddress, error) { +func (m *manager) createPublicIPAddress(ctx context.Context, ch chan<- createIPResult) { name := genManagedOutboundIPName() resourceGroupName := stringutils.LastTokenByte(m.doc.OpenShiftCluster.Properties.ClusterProfile.ResourceGroupID, '/') resourceID := fmt.Sprintf("%s/providers/Microsoft.Network/publicIPAddresses/%s", m.doc.OpenShiftCluster.Properties.ClusterProfile.ResourceGroupID, name) m.log.Infof("creating public IP Address: %s", name) - publicIPAddress := mgmtnetwork.PublicIPAddress{ - Name: &name, - ID: &resourceID, - Location: &m.doc.OpenShiftCluster.Location, - PublicIPAddressPropertiesFormat: &mgmtnetwork.PublicIPAddressPropertiesFormat{ - PublicIPAllocationMethod: mgmtnetwork.Static, - PublicIPAddressVersion: mgmtnetwork.IPv4, - }, - Sku: &mgmtnetwork.PublicIPAddressSku{ - Name: mgmtnetwork.PublicIPAddressSkuNameStandard, - }, - } + publicIPAddress := newPublicIPAddress(name, resourceID, m.doc.OpenShiftCluster.Location) err := m.publicIPAddresses.CreateOrUpdateAndWait(ctx, resourceGroupName, name, publicIPAddress) - if err != nil { - return mgmtnetwork.PublicIPAddress{}, err + ch <- createIPResult{ + ip: publicIPAddress, + err: err, } - - return publicIPAddress, nil } func getOutboundIPsFromLB(lb mgmtnetwork.LoadBalancer) []api.ResourceReference { var outboundIPs []api.ResourceReference - fipConfigs := make(map[string]mgmtnetwork.FrontendIPConfiguration, len(*lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations)) - - for i := 0; i < len(*lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations); i++ { - fipConfigID := *(*lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations)[i].ID - fipConfig := (*lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations)[i] - fipConfigs[fipConfigID] = fipConfig - } + fipConfigs := getFrontendIPConfigs(lb) for _, obRule := range *lb.LoadBalancerPropertiesFormat.OutboundRules { if *obRule.Name == outboundRuleV4 { @@ -360,6 +419,21 @@ func (m *manager) patchEffectiveOutboundIPs(ctx context.Context, outboundIPs []a return nil } +func newPublicIPAddress(name, resourceID, location string) mgmtnetwork.PublicIPAddress { + return mgmtnetwork.PublicIPAddress{ + Name: &name, + ID: &resourceID, + Location: &location, + PublicIPAddressPropertiesFormat: &mgmtnetwork.PublicIPAddressPropertiesFormat{ + PublicIPAllocationMethod: mgmtnetwork.Static, + PublicIPAddressVersion: mgmtnetwork.IPv4, + }, + Sku: &mgmtnetwork.PublicIPAddressSku{ + Name: mgmtnetwork.PublicIPAddressSkuNameStandard, + }, + } +} + func newFrontendIPConfig(name string, id string, publicIPorIPPrefixID string) mgmtnetwork.FrontendIPConfiguration { // TODO: add check for publicIPorIPPrefixID return mgmtnetwork.FrontendIPConfiguration{ diff --git a/pkg/cluster/loadbalancerprofile_test.go b/pkg/cluster/loadbalancerprofile_test.go index ceea42229d2..15d5832c477 100644 --- a/pkg/cluster/loadbalancerprofile_test.go +++ b/pkg/cluster/loadbalancerprofile_test.go @@ -23,7 +23,7 @@ import ( testdatabase "github.com/Azure/ARO-RP/test/database" ) -func TestGetDesiredOutboundIPs(t *testing.T) { +func TestReconcileOutboundIPs(t *testing.T) { ctx := context.Background() infraID := "infraID" clusterRGID := "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG" @@ -121,8 +121,8 @@ func TestGetDesiredOutboundIPs(t *testing.T) { } tt.m.publicIPAddresses = publicIPAddressClient - // Run getDesiredOutboundIPs and assert the correct results - outboundIPs, err := tt.m.getDesiredOutboundIPs(ctx) + // Run reconcileOutboundIPs and assert the correct results + outboundIPs, err := tt.m.reconcileOutboundIPs(ctx) assert.Equal(t, tt.expectedErr, err, "Unexpected error exception") // results are not deterministic when scaling down so just check desired length assert.Len(t, outboundIPs, tt.m.doc.OpenShiftCluster.Properties.NetworkProfile.LoadBalancerProfile.ManagedOutboundIPs.Count) @@ -303,8 +303,60 @@ func TestAddOutboundIPsToLB(t *testing.T) { ID: "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/publicIPAddresses/infraID-pip-v4", }, }, - currentLB: getClearedLB(), - expectedLB: fakeUpdatedLoadBalancer(0), + currentLB: getClearedLB(), + expectedLB: mgmtnetwork.LoadBalancer{ + Name: to.StringPtr("infraID"), + LoadBalancerPropertiesFormat: &mgmtnetwork.LoadBalancerPropertiesFormat{ + FrontendIPConfigurations: &[]mgmtnetwork.FrontendIPConfiguration{ + { + Name: to.StringPtr("ae3506385907e44eba9ef9bf76eac973"), + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/loadBalancers/infraID/frontendIPConfigurations/ae3506385907e44eba9ef9bf76eac973"), + FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{ + LoadBalancingRules: &[]mgmtnetwork.SubResource{ + { + ID: to.StringPtr("ae3506385907e44eba9ef9bf76eac973-TCP-80"), + }, + { + ID: to.StringPtr("ae3506385907e44eba9ef9bf76eac973-TCP-443"), + }, + }, + PublicIPAddress: &mgmtnetwork.PublicIPAddress{ + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/publicIPAddresses/infraID-default-v4"), + }, + }, + }, + { + Name: to.StringPtr("public-lb-ip-v4"), + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/loadBalancers/infraID/frontendIPConfigurations/public-lb-ip-v4"), + FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{ + LoadBalancingRules: &[]mgmtnetwork.SubResource{ + { + ID: to.StringPtr("api-internal-v4"), + }, + }, + OutboundRules: &[]mgmtnetwork.SubResource{{ + ID: to.StringPtr(outboundRuleV4), + }}, + PublicIPAddress: &mgmtnetwork.PublicIPAddress{ + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/publicIPAddresses/infraID-pip-v4"), + }, + }, + }, + }, + OutboundRules: &[]mgmtnetwork.OutboundRule{ + { + Name: to.StringPtr(outboundRuleV4), + OutboundRulePropertiesFormat: &mgmtnetwork.OutboundRulePropertiesFormat{ + FrontendIPConfigurations: &[]mgmtnetwork.SubResource{ + { + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/loadBalancers/infraID/frontendIPConfigurations/public-lb-ip-v4"), + }, + }, + }, + }, + }, + }, + }, }, { name: "add multiple outbound IPs to LB", @@ -316,8 +368,72 @@ func TestAddOutboundIPsToLB(t *testing.T) { ID: "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/publicIPAddresses/uuid1-outbound-pip-v4", }, }, - currentLB: getClearedLB(), - expectedLB: fakeUpdatedLoadBalancer(1), + currentLB: getClearedLB(), + expectedLB: mgmtnetwork.LoadBalancer{ + Name: to.StringPtr("infraID"), + LoadBalancerPropertiesFormat: &mgmtnetwork.LoadBalancerPropertiesFormat{ + FrontendIPConfigurations: &[]mgmtnetwork.FrontendIPConfiguration{ + { + Name: to.StringPtr("ae3506385907e44eba9ef9bf76eac973"), + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/loadBalancers/infraID/frontendIPConfigurations/ae3506385907e44eba9ef9bf76eac973"), + FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{ + LoadBalancingRules: &[]mgmtnetwork.SubResource{ + { + ID: to.StringPtr("ae3506385907e44eba9ef9bf76eac973-TCP-80"), + }, + { + ID: to.StringPtr("ae3506385907e44eba9ef9bf76eac973-TCP-443"), + }, + }, + PublicIPAddress: &mgmtnetwork.PublicIPAddress{ + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/publicIPAddresses/infraID-default-v4"), + }, + }, + }, + { + Name: to.StringPtr("public-lb-ip-v4"), + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/loadBalancers/infraID/frontendIPConfigurations/public-lb-ip-v4"), + FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{ + LoadBalancingRules: &[]mgmtnetwork.SubResource{ + { + ID: to.StringPtr("api-internal-v4"), + }, + }, + OutboundRules: &[]mgmtnetwork.SubResource{{ + ID: to.StringPtr(outboundRuleV4), + }}, + PublicIPAddress: &mgmtnetwork.PublicIPAddress{ + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/publicIPAddresses/infraID-pip-v4"), + }, + }, + }, + { + Name: to.StringPtr("uuid1-outbound-pip-v4"), + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/loadBalancers/infraID/frontendIPConfigurations/uuid1-outbound-pip-v4"), + FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{ + PublicIPAddress: &mgmtnetwork.PublicIPAddress{ + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/publicIPAddresses/uuid1-outbound-pip-v4"), + }, + }, + }, + }, + OutboundRules: &[]mgmtnetwork.OutboundRule{ + { + Name: to.StringPtr(outboundRuleV4), + OutboundRulePropertiesFormat: &mgmtnetwork.OutboundRulePropertiesFormat{ + FrontendIPConfigurations: &[]mgmtnetwork.SubResource{ + { + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/loadBalancers/infraID/frontendIPConfigurations/public-lb-ip-v4"), + }, + { + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/loadBalancers/infraID/frontendIPConfigurations/uuid1-outbound-pip-v4"), + }, + }, + }, + }, + }, + }, + }, }, } { t.Run(tt.name, func(t *testing.T) { @@ -339,7 +455,7 @@ func TestRemoveOutboundIPsFromLB(t *testing.T) { name: "remove all outbound-rule-v4 fip config except api server", currentLB: fakeLoadBalancersGet(1, api.VisibilityPublic), expectedLB: mgmtnetwork.LoadBalancer{ - Name: &infraID, + Name: to.StringPtr("infraID"), LoadBalancerPropertiesFormat: &mgmtnetwork.LoadBalancerPropertiesFormat{ FrontendIPConfigurations: &[]mgmtnetwork.FrontendIPConfiguration{ { @@ -392,7 +508,7 @@ func TestRemoveOutboundIPsFromLB(t *testing.T) { name: "remove all outbound-rule-v4 fip config", currentLB: fakeLoadBalancersGet(1, api.VisibilityPrivate), expectedLB: mgmtnetwork.LoadBalancer{ - Name: &infraID, + Name: to.StringPtr("infraID"), LoadBalancerPropertiesFormat: &mgmtnetwork.LoadBalancerPropertiesFormat{ FrontendIPConfigurations: &[]mgmtnetwork.FrontendIPConfiguration{ { @@ -955,8 +1071,6 @@ func TestReconcileLoadBalancerProfile(t *testing.T) { loadBalancersClient.EXPECT(). Get(gomock.Any(), clusterRGName, infraID, ""). Return(fakeLoadBalancersGet(0, api.VisibilityPublic), nil) - // loadBalancersClient.EXPECT(). - // CreateOrUpdateAndWait(ctx, clusterRGName, infraID, fakeUpdatedLoadBalancer(0)).Return(nil) publicIPAddressClient.EXPECT(). List(gomock.Any(), clusterRGName). Return(getFakePublicIPList(1), nil) @@ -975,7 +1089,7 @@ func TestReconcileLoadBalancerProfile(t *testing.T) { }, }, }, - expectedErr: []error{fmt.Errorf("multiple errors occurred while updating outbound-rule-v4\nfailed to create ip\nfailed to cleanup unused managed ips\ndeletion of unused managed ip uuid1-outbound-pip-v4 failed with error: error")}, + expectedErr: []error{fmt.Errorf("multiple errors occurred while updating outbound-rule-v4\nfailed to create required IPs\ncreation of ip address uuid2-outbound-pip-v4 failed with error: failed to create ip\nfailed to cleanup unused managed ips\ndeletion of unused managed ip uuid1-outbound-pip-v4 failed with error: error")}, }, } { t.Run(tt.name, func(t *testing.T) { @@ -1070,7 +1184,7 @@ func fakeLoadBalancersGet(additionalIPCount int, apiServerVisibility api.Visibil } } lb := mgmtnetwork.LoadBalancer{ - Name: &infraID, + Name: to.StringPtr("infraID"), LoadBalancerPropertiesFormat: &mgmtnetwork.LoadBalancerPropertiesFormat{ FrontendIPConfigurations: &[]mgmtnetwork.FrontendIPConfiguration{ { diff --git a/pkg/deploy/assets/gateway-production-parameters.json b/pkg/deploy/assets/gateway-production-parameters.json index 5fc523531a7..61dbb88b64e 100644 --- a/pkg/deploy/assets/gateway-production-parameters.json +++ b/pkg/deploy/assets/gateway-production-parameters.json @@ -38,9 +38,6 @@ "gatewayServicePrincipalId": { "value": "" }, - "gatewayStorageAccountDomain": { - "value": "" - }, "gatewayVmSize": { "value": "Standard_D4s_v3" }, diff --git a/pkg/deploy/assets/gateway-production.json b/pkg/deploy/assets/gateway-production.json index 19f123da724..1fc31cd9dfa 100644 --- a/pkg/deploy/assets/gateway-production.json +++ b/pkg/deploy/assets/gateway-production.json @@ -40,9 +40,6 @@ "gatewayServicePrincipalId": { "type": "string" }, - "gatewayStorageAccountDomain": { - "type": "string" - }, "gatewayVmSize": { "type": "string", "defaultValue": "Standard_D4s_v3" @@ -106,15 +103,6 @@ } }, "resources": [ - { - "sku": { - "name": "Standard_LRS" - }, - "location": "[resourceGroup().location]", - "name": "[substring(parameters('gatewayStorageAccountDomain'), 0, indexOf(parameters('gatewayStorageAccountDomain'), '.'))]", - "type": "Microsoft.Storage/storageAccounts", - "apiVersion": "2019-06-01" - }, { "sku": { "name": "Standard" @@ -294,8 +282,7 @@ }, "diagnosticsProfile": { "bootDiagnostics": { - "enabled": true, - "storageUri": "[concat('https://', parameters('gatewayStorageAccountDomain'), '/')]" + "enabled": true } }, "extensionProfile": { @@ -309,7 +296,7 @@ "autoUpgradeMinorVersion": true, "settings": {}, "protectedSettings": { - "script": "[base64(concat(base64ToString('c2V0IC1leAoK'),'ACRRESOURCEID=$(base64 -d \u003c\u003c\u003c''',base64(parameters('acrResourceId')),''')\n','AZURECLOUDNAME=$(base64 -d \u003c\u003c\u003c''',base64(parameters('azureCloudName')),''')\n','AZURESECPACKQUALYSURL=$(base64 -d \u003c\u003c\u003c''',base64(parameters('azureSecPackQualysUrl')),''')\n','AZURESECPACKVSATENANTID=$(base64 -d \u003c\u003c\u003c''',base64(parameters('azureSecPackVSATenantId')),''')\n','DATABASEACCOUNTNAME=$(base64 -d \u003c\u003c\u003c''',base64(parameters('databaseAccountName')),''')\n','DBTOKENCLIENTID=$(base64 -d \u003c\u003c\u003c''',base64(parameters('dbtokenClientId')),''')\n','DBTOKENURL=$(base64 -d \u003c\u003c\u003c''',base64(parameters('dbtokenUrl')),''')\n','MDMFRONTENDURL=$(base64 -d \u003c\u003c\u003c''',base64(parameters('mdmFrontendUrl')),''')\n','MDSDENVIRONMENT=$(base64 -d \u003c\u003c\u003c''',base64(parameters('mdsdEnvironment')),''')\n','FLUENTBITIMAGE=$(base64 -d \u003c\u003c\u003c''',base64(parameters('fluentbitImage')),''')\n','GATEWAYMDSDCONFIGVERSION=$(base64 -d \u003c\u003c\u003c''',base64(parameters('gatewayMdsdConfigVersion')),''')\n','GATEWAYDOMAINS=$(base64 -d \u003c\u003c\u003c''',base64(parameters('gatewayDomains')),''')\n','GATEWAYFEATURES=$(base64 -d \u003c\u003c\u003c''',base64(parameters('gatewayFeatures')),''')\n','KEYVAULTDNSSUFFIX=$(base64 -d \u003c\u003c\u003c''',base64(parameters('keyvaultDNSSuffix')),''')\n','KEYVAULTPREFIX=$(base64 -d \u003c\u003c\u003c''',base64(parameters('keyvaultPrefix')),''')\n','RPIMAGE=$(base64 -d \u003c\u003c\u003c''',base64(parameters('rpImage')),''')\n','RPMDMACCOUNT=$(base64 -d \u003c\u003c\u003c''',base64(parameters('rpMdmAccount')),''')\n','RPMDSDACCOUNT=$(base64 -d \u003c\u003c\u003c''',base64(parameters('rpMdsdAccount')),''')\n','RPMDSDNAMESPACE=$(base64 -d \u003c\u003c\u003c''',base64(parameters('rpMdsdNamespace')),''')\n','MDMIMAGE=''/genevamdm:2.2024.328.1744-c5fb79-20240328t1935''\n','LOCATION=$(base64 -d \u003c\u003c\u003c''',base64(resourceGroup().location),''')\n','SUBSCRIPTIONID=$(base64 -d \u003c\u003c\u003c''',base64(subscription().subscriptionId),''')\n','RESOURCEGROUPNAME=$(base64 -d \u003c\u003c\u003c''',base64(resourceGroup().name),''')\n','\n',base64ToString('IyEvYmluL2Jhc2gKCmVjaG8gInNldHRpbmcgc3NoIHBhc3N3b3JkIGF1dGhlbnRpY2F0aW9uIgojIFdlIG5lZWQgdG8gbWFudWFsbHkgc2V0IFBhc3N3b3JkQXV0aGVudGljYXRpb24gdG8gdHJ1ZSBpbiBvcmRlciBmb3IgdGhlIFZNU1MgQWNjZXNzIEpJVCB0byB3b3JrCnNlZCAtaSAncy9QYXNzd29yZEF1dGhlbnRpY2F0aW9uIG5vL1Bhc3N3b3JkQXV0aGVudGljYXRpb24geWVzL2cnIC9ldGMvc3NoL3NzaGRfY29uZmlnCnN5c3RlbWN0bCByZWxvYWQgc3NoZC5zZXJ2aWNlCgojQWRkaW5nIHJldHJ5IGxvZ2ljIHRvIHl1bSBjb21tYW5kcyBpbiBvcmRlciB0byBhdm9pZCBzdGFsbGluZyBvdXQgb24gcmVzb3VyY2UgbG9ja3MKZWNobyAicnVubmluZyBSSFVJIGZpeCIKZm9yIGF0dGVtcHQgaW4gezEuLjYwfTsgZG8KICB5dW0gdXBkYXRlIC15IC0tZGlzYWJsZXJlcG89JyonIC0tZW5hYmxlcmVwbz0ncmh1aS1taWNyb3NvZnQtYXp1cmUqJyAmJiBicmVhawogIGlmIFtbICR7YXR0ZW1wdH0gLWx0IDYwIF1dOyB0aGVuIHNsZWVwIDMwOyBlbHNlIGV4aXQgMTsgZmkKZG9uZQoKZWNobyAicnVubmluZyB5dW0gdXBkYXRlIgpmb3IgYXR0ZW1wdCBpbiB7MS4uNjB9OyBkbwogIHl1bSAteSAteCBXQUxpbnV4QWdlbnQgLXggV0FMaW51eEFnZW50LXVkZXYgdXBkYXRlIC0tYWxsb3dlcmFzaW5nICYmIGJyZWFrCiAgaWYgW1sgJHthdHRlbXB0fSAtbHQgNjAgXV07IHRoZW4gc2xlZXAgMzA7IGVsc2UgZXhpdCAxOyBmaQpkb25lCgplY2hvICJleHRlbmRpbmcgcGFydGl0aW9uIHRhYmxlIgojIExpbnV4IGJsb2NrIGRldmljZXMgYXJlIGluY29uc2lzdGVudGx5IG5hbWVkCiMgaXQncyBkaWZmaWN1bHQgdG8gdGllIHRoZSBsdm0gcHYgdG8gdGhlIHBoeXNpY2FsIGRpc2sgdXNpbmcgL2Rldi9kaXNrIGZpbGVzLCB3aGljaCBpcyB3aHkgbHZzIGlzIHVzZWQgaGVyZQpwaHlzaWNhbF9kaXNrPSIkKGx2cyAtbyBkZXZpY2VzIC1hIHwgaGVhZCAtbjIgfCB0YWlsIC1uMSB8IGN1dCAtZCAnICcgLWYgMyB8IGN1dCAtZCBcKCAtZiAxIHwgdHIgLWQgJ1s6ZGlnaXQ6XScpIgpncm93cGFydCAiJHBoeXNpY2FsX2Rpc2siIDIKCmVjaG8gImV4dGVuZGluZyBmaWxlc3lzdGVtcyIKbHZleHRlbmQgLWwgKzIwJUZSRUUgL2Rldi9yb290dmcvcm9vdGx2Cnhmc19ncm93ZnMgLwoKbHZleHRlbmQgLWwgKzEwMCVGUkVFIC9kZXYvcm9vdHZnL3Zhcmx2Cnhmc19ncm93ZnMgL3ZhcgoKcnBtIC0taW1wb3J0IGh0dHBzOi8vZGwuZmVkb3JhcHJvamVjdC5vcmcvcHViL2VwZWwvUlBNLUdQRy1LRVktRVBFTC04CnJwbSAtLWltcG9ydCBodHRwczovL3BhY2thZ2VzLm1pY3Jvc29mdC5jb20va2V5cy9taWNyb3NvZnQuYXNjCgpmb3IgYXR0ZW1wdCBpbiB7MS4uNjB9OyBkbwogIHl1bSAteSBpbnN0YWxsIGh0dHBzOi8vZGwuZmVkb3JhcHJvamVjdC5vcmcvcHViL2VwZWwvZXBlbC1yZWxlYXNlLWxhdGVzdC04Lm5vYXJjaC5ycG0gJiYgYnJlYWsKICBpZiBbWyAke2F0dGVtcHR9IC1sdCA2MCBdXTsgdGhlbiBzbGVlcCAzMDsgZWxzZSBleGl0IDE7IGZpCmRvbmUKCmVjaG8gImNvbmZpZ3VyaW5nIGxvZ3JvdGF0ZSIKCiMgZ2F0ZXdheV9sb2dkaXIgaXMgYSByZWFkb25seSB2YXJpYWJsZSB0aGF0IHNwZWNpZmllcyB0aGUgaG9zdCBwYXRoIG1vdW50IHBvaW50IGZvciB0aGUgZ2F0ZXdheSBjb250YWluZXIgbG9nIGZpbGUKIyBmb3IgdGhlIHB1cnBvc2Ugb2Ygcm90YXRpbmcgdGhlIGdhdGV3YXkgbG9ncwpkZWNsYXJlIC1yIGdhdGV3YXlfbG9nZGlyPScvdmFyL2xvZy9hcm8tZ2F0ZXdheScKCmNhdCA+L2V0Yy9sb2dyb3RhdGUuY29uZiA8PEVPRgojIHNlZSAibWFuIGxvZ3JvdGF0ZSIgZm9yIGRldGFpbHMKIyByb3RhdGUgbG9nIGZpbGVzIHdlZWtseQp3ZWVrbHkKCiMga2VlcCAyIHdlZWtzIHdvcnRoIG9mIGJhY2tsb2dzCnJvdGF0ZSAyCgojIGNyZWF0ZSBuZXcgKGVtcHR5KSBsb2cgZmlsZXMgYWZ0ZXIgcm90YXRpbmcgb2xkIG9uZXMKY3JlYXRlCgojIHVzZSBkYXRlIGFzIGEgc3VmZml4IG9mIHRoZSByb3RhdGVkIGZpbGUKZGF0ZWV4dAoKIyB1bmNvbW1lbnQgdGhpcyBpZiB5b3Ugd2FudCB5b3VyIGxvZyBmaWxlcyBjb21wcmVzc2VkCmNvbXByZXNzCgojIFJQTSBwYWNrYWdlcyBkcm9wIGxvZyByb3RhdGlvbiBpbmZvcm1hdGlvbiBpbnRvIHRoaXMgZGlyZWN0b3J5CmluY2x1ZGUgL2V0Yy9sb2dyb3RhdGUuZAoKIyBubyBwYWNrYWdlcyBvd24gd3RtcCBhbmQgYnRtcCAtLSB3ZSdsbCByb3RhdGUgdGhlbSBoZXJlCi92YXIvbG9nL3d0bXAgewogICAgbW9udGhseQogICAgY3JlYXRlIDA2NjQgcm9vdCB1dG1wCiAgICAgICAgbWluc2l6ZSAxTQogICAgcm90YXRlIDEKfQoKL3Zhci9sb2cvYnRtcCB7CiAgICBtaXNzaW5nb2sKICAgIG1vbnRobHkKICAgIGNyZWF0ZSAwNjAwIHJvb3QgdXRtcAogICAgcm90YXRlIDEKfQoKIyBNYXhpbXVtIGxvZyBkaXJlY3Rvcnkgc2l6ZSBpcyAxMDBHIHdpdGggdGhpcyBjb25maWd1cmF0aW9uCiMgU2V0dGluZyBsaW1pdCB0byAxMDBHIHRvIGFsbG93IHNwYWNlIGZvciBvdGhlciBsb2dnaW5nIHNlcnZpY2VzCiMgY29weXRydW5jYXRlIGlzIGEgY3JpdGljYWwgb3B0aW9uIHVzZWQgdG8gcHJldmVudCBsb2dzIGZyb20gYmVpbmcgc2hpcHBlZCB0d2ljZQoke2dhdGV3YXlfbG9nZGlyfSB7CiAgICBzaXplIDIwRwogICAgcm90YXRlIDUKICAgIGNyZWF0ZSAwNjAwIHJvb3Qgcm9vdAogICAgY29weXRydW5jYXRlCiAgICBub29sZGRpcgogICAgY29tcHJlc3MKfQpFT0YKCmVjaG8gImNvbmZpZ3VyaW5nIHl1bSByZXBvc2l0b3J5IGFuZCBydW5uaW5nIHl1bSB1cGRhdGUiCmNhdCA+L2V0Yy95dW0ucmVwb3MuZC9henVyZS5yZXBvIDw8J0VPRicKW2F6dXJlLWNsaV0KbmFtZT1henVyZS1jbGkKYmFzZXVybD1odHRwczovL3BhY2thZ2VzLm1pY3Jvc29mdC5jb20veXVtcmVwb3MvYXp1cmUtY2xpCmVuYWJsZWQ9eWVzCmdwZ2NoZWNrPXllcwoKW2F6dXJlY29yZV0KbmFtZT1henVyZWNvcmUKYmFzZXVybD1odHRwczovL3BhY2thZ2VzLm1pY3Jvc29mdC5jb20veXVtcmVwb3MvYXp1cmVjb3JlCmVuYWJsZWQ9eWVzCmdwZ2NoZWNrPW5vCkVPRgoKc2VtYW5hZ2UgZmNvbnRleHQgLWEgLXQgdmFyX2xvZ190ICIvdmFyL2xvZy9qb3VybmFsKC8uKik/Igpta2RpciAtcCAvdmFyL2xvZy9qb3VybmFsCgpmb3IgYXR0ZW1wdCBpbiB7MS4uNjB9OyBkbwogIHl1bSAteSBpbnN0YWxsIGNsYW1hdiBhenNlYy1jbGFtYXYgYXpzZWMtbW9uaXRvciBhenVyZS1jbGkgYXp1cmUtbWRzZCBhenVyZS1zZWN1cml0eSBwb2RtYW4tZG9ja2VyIG9wZW5zc2wtcGVybCBweXRob24zICYmIGJyZWFrCiAgIyBoYWNrIC0gd2UgYXJlIGluc3RhbGxpbmcgcHl0aG9uMyBvbiBob3N0cyBkdWUgdG8gYW4gaXNzdWUgd2l0aCBBenVyZSBMaW51eCBFeHRlbnNpb25zIGh0dHBzOi8vZ2l0aHViLmNvbS9BenVyZS9henVyZS1saW51eC1leHRlbnNpb25zL3B1bGwvMTUwNQogIGlmIFtbICR7YXR0ZW1wdH0gLWx0IDYwIF1dOyB0aGVuIHNsZWVwIDMwOyBlbHNlIGV4aXQgMTsgZmkKZG9uZQoKZWNobyAiYXBwbHlpbmcgZmlyZXdhbGwgcnVsZXMiCiMgaHR0cHM6Ly9hY2Nlc3MucmVkaGF0LmNvbS9zZWN1cml0eS9jdmUvY3ZlLTIwMjAtMTM0MDEKY2F0ID4vZXRjL3N5c2N0bC5kLzAyLWRpc2FibGUtYWNjZXB0LXJhLmNvbmYgPDwnRU9GJwpuZXQuaXB2Ni5jb25mLmFsbC5hY2NlcHRfcmE9MApFT0YKCmNhdCA+L2V0Yy9zeXNjdGwuZC8wMS1kaXNhYmxlLWNvcmUuY29uZiA8PCdFT0YnCmtlcm5lbC5jb3JlX3BhdHRlcm4gPSB8L2Jpbi90cnVlCkVPRgpzeXNjdGwgLS1zeXN0ZW0KCmZpcmV3YWxsLWNtZCAtLWFkZC1wb3J0PTgwL3RjcCAtLXBlcm1hbmVudApmaXJld2FsbC1jbWQgLS1hZGQtcG9ydD04MDgxL3RjcCAtLXBlcm1hbmVudApmaXJld2FsbC1jbWQgLS1hZGQtcG9ydD00NDMvdGNwIC0tcGVybWFuZW50CgplY2hvICJsb2dnaW5nIGludG8gcHJvZCBhY3IiCmV4cG9ydCBBWlVSRV9DTE9VRF9OQU1FPSRBWlVSRUNMT1VETkFNRQpheiBsb2dpbiAtaSAtLWFsbG93LW5vLXN1YnNjcmlwdGlvbnMKCiMgVGhlIG1hbmFnZWQgaWRlbnRpdHkgdGhhdCB0aGUgVk0gcnVucyBhcyBvbmx5IGhhcyBhIHNpbmdsZSByb2xlYXNzaWdubWVudC4KIyBUaGlzIHJvbGUgYXNzaWdubWVudCBpcyBBQ1JQdWxsIHdoaWNoIGlzIG5vdCBuZWNlc3NhcmlseSBwcmVzZW50IGluIHRoZQojIHN1YnNjcmlwdGlvbiB3ZSdyZSBkZXBsb3lpbmcgaW50by4gIElmIHRoZSBpZGVudGl0eSBkb2VzIG5vdCBoYXZlIGFueQojIHJvbGUgYXNzaWdubWVudHMgc2NvcGVkIG9uIHRoZSBzdWJzY3JpcHRpb24gd2UncmUgZGVwbG95aW5nIGludG8sIGl0IHdpbGwKIyBub3Qgc2hvdyBvbiBheiBsb2dpbiAtaSwgd2hpY2ggaXMgd2h5IHRoZSBiZWxvdyBsaW5lIGlzIGNvbW1lbnRlZC4KIyBheiBhY2NvdW50IHNldCAtcyAiJFNVQlNDUklQVElPTklEIgoKIyBTdXBwcmVzcyBlbXVsYXRpb24gb3V0cHV0IGZvciBwb2RtYW4gaW5zdGVhZCBvZiBkb2NrZXIgZm9yIGF6IGFjciBjb21wYXRhYmlsaXR5Cm1rZGlyIC1wIC9ldGMvY29udGFpbmVycy8KdG91Y2ggL2V0Yy9jb250YWluZXJzL25vZG9ja2VyCgpta2RpciAtcCAvcm9vdC8uZG9ja2VyClJFR0lTVFJZX0FVVEhfRklMRT0vcm9vdC8uZG9ja2VyL2NvbmZpZy5qc29uIGF6IGFjciBsb2dpbiAtLW5hbWUgIiQoc2VkIC1lICdzfC4qL3x8JyA8PDwiJEFDUlJFU09VUkNFSUQiKSIKCk1ETUlNQUdFPSIke1JQSU1BR0UlJS8qfS8ke01ETUlNQUdFIyMqL30iCmRvY2tlciBwdWxsICIkTURNSU1BR0UiCmRvY2tlciBwdWxsICIkUlBJTUFHRSIKZG9ja2VyIHB1bGwgIiRGTFVFTlRCSVRJTUFHRSIKCmF6IGxvZ291dAoKZWNobyAiY29uZmlndXJpbmcgZmx1ZW50Yml0IHNlcnZpY2UiCm1rZGlyIC1wIC9ldGMvZmx1ZW50Yml0Lwpta2RpciAtcCAvdmFyL2xpYi9mbHVlbnQKCmNhdCA+L2V0Yy9mbHVlbnRiaXQvZmx1ZW50Yml0LmNvbmYgPDwnRU9GJwpbSU5QVVRdCglOYW1lIHN5c3RlbWQKCVRhZyBqb3VybmFsZAoJU3lzdGVtZF9GaWx0ZXIgX0NPTU09YXJvCglEQiAvdmFyL2xpYi9mbHVlbnQvam91cm5hbGRiCgpbRklMVEVSXQoJTmFtZSBtb2RpZnkKCU1hdGNoIGpvdXJuYWxkCglSZW1vdmVfd2lsZGNhcmQgXwoJUmVtb3ZlIFRJTUVTVEFNUAoKW09VVFBVVF0KCU5hbWUgZm9yd2FyZAoJTWF0Y2ggKgoJUG9ydCAyOTIzMApFT0YKCmVjaG8gIkZMVUVOVEJJVElNQUdFPSRGTFVFTlRCSVRJTUFHRSIgPi9ldGMvc3lzY29uZmlnL2ZsdWVudGJpdAoKY2F0ID4vZXRjL3N5c3RlbWQvc3lzdGVtL2ZsdWVudGJpdC5zZXJ2aWNlIDw8J0VPRicKW1VuaXRdCkFmdGVyPW5ldHdvcmstb25saW5lLnRhcmdldApXYW50cz1uZXR3b3JrLW9ubGluZS50YXJnZXQKU3RhcnRMaW1pdEludGVydmFsU2VjPTAKCltTZXJ2aWNlXQpSZXN0YXJ0U2VjPTFzCkVudmlyb25tZW50RmlsZT0vZXRjL3N5c2NvbmZpZy9mbHVlbnRiaXQKRXhlY1N0YXJ0UHJlPS0vdXNyL2Jpbi9kb2NrZXIgcm0gLWYgJU4KRXhlY1N0YXJ0PS91c3IvYmluL2RvY2tlciBydW4gXAogIC0tc2VjdXJpdHktb3B0IGxhYmVsPWRpc2FibGUgXAogIC0tZW50cnlwb2ludCAvb3B0L3RkLWFnZW50LWJpdC9iaW4vdGQtYWdlbnQtYml0IFwKICAtLW5ldD1ob3N0IFwKICAtLWhvc3RuYW1lICVIIFwKICAtLW5hbWUgJU4gXAogIC0tcm0gXAogIC0tY2FwLWRyb3AgbmV0X3JhdyBcCiAgLXYgL2V0Yy9mbHVlbnRiaXQvZmx1ZW50Yml0LmNvbmY6L2V0Yy9mbHVlbnRiaXQvZmx1ZW50Yml0LmNvbmYgXAogIC12IC92YXIvbGliL2ZsdWVudDovdmFyL2xpYi9mbHVlbnQ6eiBcCiAgLXYgL3Zhci9sb2cvam91cm5hbDovdmFyL2xvZy9qb3VybmFsOnJvIFwKICAtdiAvZXRjL21hY2hpbmUtaWQ6L2V0Yy9tYWNoaW5lLWlkOnJvIFwKICAkRkxVRU5UQklUSU1BR0UgXAogIC1jIC9ldGMvZmx1ZW50Yml0L2ZsdWVudGJpdC5jb25mCgpFeGVjU3RvcD0vdXNyL2Jpbi9kb2NrZXIgc3RvcCAlTgpSZXN0YXJ0PWFsd2F5cwpSZXN0YXJ0U2VjPTUKU3RhcnRMaW1pdEludGVydmFsPTAKCltJbnN0YWxsXQpXYW50ZWRCeT1tdWx0aS11c2VyLnRhcmdldApFT0YKCmVjaG8gImNvbmZpZ3VyaW5nIG1kbSBzZXJ2aWNlIgpjYXQgPi9ldGMvc3lzY29uZmlnL21kbSA8PEVPRgpNRE1GUk9OVEVORFVSTD0nJE1ETUZST05URU5EVVJMJwpNRE1JTUFHRT0nJE1ETUlNQUdFJwpNRE1TT1VSQ0VFTlZJUk9OTUVOVD0nJExPQ0FUSU9OJwpNRE1TT1VSQ0VST0xFPWdhdGV3YXkKTURNU09VUkNFUk9MRUlOU1RBTkNFPSckKGhvc3RuYW1lKScKRU9GCgpta2RpciAvdmFyL2V0dwpjYXQgPi9ldGMvc3lzdGVtZC9zeXN0ZW0vbWRtLnNlcnZpY2UgPDwnRU9GJwpbVW5pdF0KQWZ0ZXI9bmV0d29yay1vbmxpbmUudGFyZ2V0CldhbnRzPW5ldHdvcmstb25saW5lLnRhcmdldAoKW1NlcnZpY2VdCkVudmlyb25tZW50RmlsZT0vZXRjL3N5c2NvbmZpZy9tZG0KRXhlY1N0YXJ0UHJlPS0vdXNyL2Jpbi9kb2NrZXIgcm0gLWYgJU4KRXhlY1N0YXJ0PS91c3IvYmluL2RvY2tlciBydW4gXAogIC0tZW50cnlwb2ludCAvdXNyL3NiaW4vTWV0cmljc0V4dGVuc2lvbiBcCiAgLS1ob3N0bmFtZSAlSCBcCiAgLS1uYW1lICVOIFwKICAtLXJtIFwKICAtLWNhcC1kcm9wIG5ldF9yYXcgXAogIC1tIDJnIFwKICAtdiAvZXRjL21kbS5wZW06L2V0Yy9tZG0ucGVtIFwKICAtdiAvdmFyL2V0dzovdmFyL2V0dzp6IFwKICAkTURNSU1BR0UgXAogIC1DZXJ0RmlsZSAvZXRjL21kbS5wZW0gXAogIC1Gcm9udEVuZFVybCAkTURNRlJPTlRFTkRVUkwgXAogIC1Mb2dnZXIgQ29uc29sZSBcCiAgLUxvZ0xldmVsIFdhcm5pbmcgXAogIC1Qcml2YXRlS2V5RmlsZSAvZXRjL21kbS5wZW0gXAogIC1Tb3VyY2VFbnZpcm9ubWVudCAkTURNU09VUkNFRU5WSVJPTk1FTlQgXAogIC1Tb3VyY2VSb2xlICRNRE1TT1VSQ0VST0xFIFwKICAtU291cmNlUm9sZUluc3RhbmNlICRNRE1TT1VSQ0VST0xFSU5TVEFOQ0UKRXhlY1N0b3A9L3Vzci9iaW4vZG9ja2VyIHN0b3AgJU4KUmVzdGFydD1hbHdheXMKUmVzdGFydFNlYz0xClN0YXJ0TGltaXRJbnRlcnZhbD0wCgpbSW5zdGFsbF0KV2FudGVkQnk9bXVsdGktdXNlci50YXJnZXQKRU9GCgplY2hvICJjb25maWd1cmluZyBhcm8tZ2F0ZXdheSBzZXJ2aWNlIgpjYXQgPi9ldGMvc3lzY29uZmlnL2Fyby1nYXRld2F5IDw8RU9GCkFDUl9SRVNPVVJDRV9JRD0nJEFDUlJFU09VUkNFSUQnCkRBVEFCQVNFX0FDQ09VTlRfTkFNRT0nJERBVEFCQVNFQUNDT1VOVE5BTUUnCkFaVVJFX0RCVE9LRU5fQ0xJRU5UX0lEPSckREJUT0tFTkNMSUVOVElEJwpEQlRPS0VOX1VSTD0nJERCVE9LRU5VUkwnCk1ETV9BQ0NPVU5UPSIkUlBNRE1BQ0NPVU5UIgpNRE1fTkFNRVNQQUNFPUdhdGV3YXkKR0FURVdBWV9ET01BSU5TPSckR0FURVdBWURPTUFJTlMnCkdBVEVXQVlfRkVBVFVSRVM9JyRHQVRFV0FZRkVBVFVSRVMnClJQSU1BR0U9JyRSUElNQUdFJwpFT0YKCmNhdCA+L2V0Yy9zeXN0ZW1kL3N5c3RlbS9hcm8tZ2F0ZXdheS5zZXJ2aWNlIDw8RU9GCltVbml0XQpBZnRlcj1uZXR3b3JrLW9ubGluZS50YXJnZXQKV2FudHM9bmV0d29yay1vbmxpbmUudGFyZ2V0CgpbU2VydmljZV0KRW52aXJvbm1lbnRGaWxlPS9ldGMvc3lzY29uZmlnL2Fyby1nYXRld2F5CkV4ZWNTdGFydFByZT0tL3Vzci9iaW4vZG9ja2VyIHJtIC1mICVOCkV4ZWNTdGFydFByZT0vdXNyL2Jpbi9ta2RpciAtcCAke2dhdGV3YXlfbG9nZGlyfQpFeGVjU3RhcnQ9L3Vzci9iaW4vZG9ja2VyIHJ1biBcCiAgLS1ob3N0bmFtZSAlSCBcCiAgLS1uYW1lICVOIFwKICAtLXJtIFwKICAtLWNhcC1kcm9wIG5ldF9yYXcgXAogIC1lIEFDUl9SRVNPVVJDRV9JRCBcCiAgLWUgREFUQUJBU0VfQUNDT1VOVF9OQU1FIFwKICAtZSBBWlVSRV9EQlRPS0VOX0NMSUVOVF9JRCBcCiAgLWUgREJUT0tFTl9VUkwgXAogIC1lIEdBVEVXQVlfRE9NQUlOUyBcCiAgLWUgR0FURVdBWV9GRUFUVVJFUyBcCiAgLWUgTURNX0FDQ09VTlQgXAogIC1lIE1ETV9OQU1FU1BBQ0UgXAogIC1tIDJnIFwKICAtcCA4MDo4MDgwIFwKICAtcCA4MDgxOjgwODEgXAogIC1wIDQ0Mzo4NDQzIFwKICAtdiAvcnVuL3N5c3RlbWQvam91cm5hbDovcnVuL3N5c3RlbWQvam91cm5hbCBcCiAgLXYgL3Zhci9ldHc6L3Zhci9ldHc6eiBcCiAgLXYgJHtnYXRld2F5X2xvZ2Rpcn06L2N0ci5sb2c6eiBcCiAgXCRSUElNQUdFIFwKICBnYXRld2F5CkV4ZWNTdG9wPS91c3IvYmluL2RvY2tlciBzdG9wIC10IDM2MDAgJU4KVGltZW91dFN0b3BTZWM9MzYwMApSZXN0YXJ0PWFsd2F5cwpSZXN0YXJ0U2VjPTEKU3RhcnRMaW1pdEludGVydmFsPTAKCltJbnN0YWxsXQpXYW50ZWRCeT1tdWx0aS11c2VyLnRhcmdldApFT0YKCmNoY29uIC1SIHN5c3RlbV91Om9iamVjdF9yOnZhcl9sb2dfdDpzMCAvdmFyL29wdC9taWNyb3NvZnQvbGludXhtb25hZ2VudAoKbWtkaXIgLXAgL3Zhci9saWIvd2FhZ2VudC9NaWNyb3NvZnQuQXp1cmUuS2V5VmF1bHQuU3RvcmUKCmVjaG8gImNvbmZpZ3VyaW5nIG1kc2QgYW5kIG1kbSBzZXJ2aWNlcyIKZm9yIHZhciBpbiAibWRzZCIgIm1kbSI7IGRvCmNhdCA+L2V0Yy9zeXN0ZW1kL3N5c3RlbS9kb3dubG9hZC0kdmFyLWNyZWRlbnRpYWxzLnNlcnZpY2UgPDxFT0YKW1VuaXRdCkRlc2NyaXB0aW9uPVBlcmlvZGljICR2YXIgY3JlZGVudGlhbHMgcmVmcmVzaAoKW1NlcnZpY2VdClR5cGU9b25lc2hvdApFeGVjU3RhcnQ9L3Vzci9sb2NhbC9iaW4vZG93bmxvYWQtY3JlZGVudGlhbHMuc2ggJHZhcgpFT0YKCmNhdCA+L2V0Yy9zeXN0ZW1kL3N5c3RlbS9kb3dubG9hZC0kdmFyLWNyZWRlbnRpYWxzLnRpbWVyIDw8RU9GCltVbml0XQpEZXNjcmlwdGlvbj1QZXJpb2RpYyAkdmFyIGNyZWRlbnRpYWxzIHJlZnJlc2gKQWZ0ZXI9bmV0d29yay1vbmxpbmUudGFyZ2V0CldhbnRzPW5ldHdvcmstb25saW5lLnRhcmdldAoKW1RpbWVyXQpPbkJvb3RTZWM9MG1pbgpPbkNhbGVuZGFyPTAvMTI6MDA6MDAKQWNjdXJhY3lTZWM9NXMKCltJbnN0YWxsXQpXYW50ZWRCeT10aW1lcnMudGFyZ2V0CkVPRgpkb25lCgpjYXQgPi91c3IvbG9jYWwvYmluL2Rvd25sb2FkLWNyZWRlbnRpYWxzLnNoIDw8RU9GCiMhL2Jpbi9iYXNoCnNldCAtZXUKCkNPTVBPTkVOVD0iXCQxIgplY2hvICJEb3dubG9hZCBcJENPTVBPTkVOVCBjcmVkZW50aWFscyIKClRFTVBfRElSPVwkKG1rdGVtcCAtZCkKZXhwb3J0IEFaVVJFX0NPTkZJR19ESVI9XCQobWt0ZW1wIC1kKQoKZWNobyAiTG9nZ2luZyBpbnRvIEF6dXJlLi4uIgpSRVRSSUVTPTMKd2hpbGUgWyAiXCRSRVRSSUVTIiAtZ3QgMCBdOyBkbwogICAgaWYgYXogbG9naW4gLWkgLS1hbGxvdy1uby1zdWJzY3JpcHRpb25zCiAgICB0aGVuCiAgICAgICAgZWNobyAiYXogbG9naW4gc3VjY2Vzc2Z1bCIKICAgICAgICBicmVhawogICAgZWxzZQogICAgICAgIGVjaG8gImF6IGxvZ2luIGZhaWxlZC4gUmV0cnlpbmcuLi4iCiAgICAgICAgbGV0IFJFVFJJRVMtPTEKICAgICAgICBzbGVlcCA1CiAgICBmaQpkb25lCgp0cmFwICJjbGVhbnVwIiBFWElUCgpjbGVhbnVwKCkgewogIGF6IGxvZ291dAogIFtbICJcJFRFTVBfRElSIiA9fiAvdG1wLy4rIF1dICYmIHJtIC1yZiBcJFRFTVBfRElSCiAgW1sgIlwkQVpVUkVfQ09ORklHX0RJUiIgPX4gL3RtcC8uKyBdXSAmJiBybSAtcmYgXCRBWlVSRV9DT05GSUdfRElSCn0KCmlmIFsgIlwkQ09NUE9ORU5UIiA9ICJtZG0iIF07IHRoZW4KICBDVVJSRU5UX0NFUlRfRklMRT0iL2V0Yy9tZG0ucGVtIgplbGlmIFsgIlwkQ09NUE9ORU5UIiA9ICJtZHNkIiBdOyB0aGVuCiAgQ1VSUkVOVF9DRVJUX0ZJTEU9Ii92YXIvbGliL3dhYWdlbnQvTWljcm9zb2Z0LkF6dXJlLktleVZhdWx0LlN0b3JlL21kc2QucGVtIgplbHNlCiAgZWNobyBJbnZhbGlkIHVzYWdlICYmIGV4aXQgMQpmaQoKU0VDUkVUX05BTUU9Imd3eS1cJHtDT01QT05FTlR9IgpORVdfQ0VSVF9GSUxFPSJcJFRFTVBfRElSL1wkQ09NUE9ORU5ULnBlbSIKZm9yIGF0dGVtcHQgaW4gezEuLjV9OyBkbwogIGF6IGtleXZhdWx0IHNlY3JldCBkb3dubG9hZCAtLWZpbGUgXCRORVdfQ0VSVF9GSUxFIC0taWQgImh0dHBzOi8vJEtFWVZBVUxUUFJFRklYLWd3eS4kS0VZVkFVTFRETlNTVUZGSVgvc2VjcmV0cy9cJFNFQ1JFVF9OQU1FIiAmJiBicmVhawogIGlmIFtbIFwkYXR0ZW1wdCAtbHQgNSBdXTsgdGhlbiBzbGVlcCAxMDsgZWxzZSBleGl0IDE7IGZpCmRvbmUKCmlmIFsgLWYgXCRORVdfQ0VSVF9GSUxFIF07IHRoZW4KICBpZiBbICJcJENPTVBPTkVOVCIgPSAibWRzZCIgXTsgdGhlbgogICAgY2hvd24gc3lzbG9nOnN5c2xvZyBcJE5FV19DRVJUX0ZJTEUKICBlbHNlCiAgICBzZWQgLWkgLW5lICcxLC9FTkQgQ0VSVElGSUNBVEUvIHAnIFwkTkVXX0NFUlRfRklMRQogIGZpCgogIG5ld19jZXJ0X3NuPSJcJChvcGVuc3NsIHg1MDkgLWluICJcJE5FV19DRVJUX0ZJTEUiIC1ub291dCAtc2VyaWFsIHwgYXdrIC1GPSAne3ByaW50IFwkMn0nKSIKICBjdXJyZW50X2NlcnRfc249IlwkKG9wZW5zc2wgeDUwOSAtaW4gIlwkQ1VSUkVOVF9DRVJUX0ZJTEUiIC1ub291dCAtc2VyaWFsIHwgYXdrIC1GPSAne3ByaW50IFwkMn0nKSIKICBpZiBbWyAhIC16IFwkbmV3X2NlcnRfc24gXV0gJiYgW1sgXCRuZXdfY2VydF9zbiAhPSAiXCRjdXJyZW50X2NlcnRfc24iIF1dOyB0aGVuCiAgICBlY2hvIHVwZGF0aW5nIGNlcnRpZmljYXRlIGZvciBcJENPTVBPTkVOVAogICAgY2htb2QgMDYwMCBcJE5FV19DRVJUX0ZJTEUKICAgIG12IFwkTkVXX0NFUlRfRklMRSBcJENVUlJFTlRfQ0VSVF9GSUxFCiAgZmkKZWxzZQogIGVjaG8gRmFpbGVkIHRvIHJlZnJlc2ggY2VydGlmaWNhdGUgZm9yIFwkQ09NUE9ORU5UICYmIGV4aXQgMQpmaQpFT0YKCmNobW9kIHUreCAvdXNyL2xvY2FsL2Jpbi9kb3dubG9hZC1jcmVkZW50aWFscy5zaAoKc3lzdGVtY3RsIGVuYWJsZSBkb3dubG9hZC1tZHNkLWNyZWRlbnRpYWxzLnRpbWVyCnN5c3RlbWN0bCBlbmFibGUgZG93bmxvYWQtbWRtLWNyZWRlbnRpYWxzLnRpbWVyCgovdXNyL2xvY2FsL2Jpbi9kb3dubG9hZC1jcmVkZW50aWFscy5zaCBtZHNkCi91c3IvbG9jYWwvYmluL2Rvd25sb2FkLWNyZWRlbnRpYWxzLnNoIG1kbQpNRFNEQ0VSVElGSUNBVEVTQU49JChvcGVuc3NsIHg1MDkgLWluIC92YXIvbGliL3dhYWdlbnQvTWljcm9zb2Z0LkF6dXJlLktleVZhdWx0LlN0b3JlL21kc2QucGVtIC1ub291dCAtc3ViamVjdCB8IHNlZCAtZSAncy8uKkNOID0gLy8nKQoKY2F0ID4vZXRjL3N5c3RlbWQvc3lzdGVtL3dhdGNoLW1kbS1jcmVkZW50aWFscy5zZXJ2aWNlIDw8RU9GCltVbml0XQpEZXNjcmlwdGlvbj1XYXRjaCBmb3IgY2hhbmdlcyBpbiBtZG0ucGVtIGFuZCByZXN0YXJ0cyB0aGUgbWRtIHNlcnZpY2UKCltTZXJ2aWNlXQpUeXBlPW9uZXNob3QKRXhlY1N0YXJ0PS91c3IvYmluL3N5c3RlbWN0bCByZXN0YXJ0IG1kbS5zZXJ2aWNlCgpbSW5zdGFsbF0KV2FudGVkQnk9bXVsdGktdXNlci50YXJnZXQKRU9GCgpjYXQgPi9ldGMvc3lzdGVtZC9zeXN0ZW0vd2F0Y2gtbWRtLWNyZWRlbnRpYWxzLnBhdGggPDxFT0YKW1BhdGhdClBhdGhNb2RpZmllZD0vZXRjL21kbS5wZW0KCltJbnN0YWxsXQpXYW50ZWRCeT1tdWx0aS11c2VyLnRhcmdldApFT0YKCnN5c3RlbWN0bCBlbmFibGUgd2F0Y2gtbWRtLWNyZWRlbnRpYWxzLnBhdGgKc3lzdGVtY3RsIHN0YXJ0IHdhdGNoLW1kbS1jcmVkZW50aWFscy5wYXRoCgpta2RpciAvZXRjL3N5c3RlbWQvc3lzdGVtL21kc2Quc2VydmljZS5kCmNhdCA+L2V0Yy9zeXN0ZW1kL3N5c3RlbS9tZHNkLnNlcnZpY2UuZC9vdmVycmlkZS5jb25mIDw8J0VPRicKW1VuaXRdCkFmdGVyPW5ldHdvcmstb25saW5lLnRhcmdldApFT0YKCmNhdCA+L2V0Yy9kZWZhdWx0L21kc2QgPDxFT0YKTURTRF9ST0xFX1BSRUZJWD0vdmFyL3J1bi9tZHNkL2RlZmF1bHQKTURTRF9PUFRJT05TPSItQSAtZCAtciBcJE1EU0RfUk9MRV9QUkVGSVgiCgpleHBvcnQgTU9OSVRPUklOR19HQ1NfRU5WSVJPTk1FTlQ9JyRNRFNERU5WSVJPTk1FTlQnCmV4cG9ydCBNT05JVE9SSU5HX0dDU19BQ0NPVU5UPSckUlBNRFNEQUNDT1VOVCcKZXhwb3J0IE1PTklUT1JJTkdfR0NTX1JFR0lPTj0nJExPQ0FUSU9OJwpleHBvcnQgTU9OSVRPUklOR19HQ1NfQVVUSF9JRF9UWVBFPUF1dGhLZXlWYXVsdApleHBvcnQgTU9OSVRPUklOR19HQ1NfQVVUSF9JRD0nJE1EU0RDRVJUSUZJQ0FURVNBTicKZXhwb3J0IE1PTklUT1JJTkdfR0NTX05BTUVTUEFDRT0nJFJQTURTRE5BTUVTUEFDRScKZXhwb3J0IE1PTklUT1JJTkdfQ09ORklHX1ZFUlNJT049JyRHQVRFV0FZTURTRENPTkZJR1ZFUlNJT04nCmV4cG9ydCBNT05JVE9SSU5HX1VTRV9HRU5FVkFfQ09ORklHX1NFUlZJQ0U9dHJ1ZQoKZXhwb3J0IE1PTklUT1JJTkdfVEVOQU5UPSckTE9DQVRJT04nCmV4cG9ydCBNT05JVE9SSU5HX1JPTEU9Z2F0ZXdheQpleHBvcnQgTU9OSVRPUklOR19ST0xFX0lOU1RBTkNFPSckKGhvc3RuYW1lKScKCmV4cG9ydCBNRFNEX01TR1BBQ0tfU09SVF9DT0xVTU5TPTEKRU9GCgojIHNldHRpbmcgTU9OSVRPUklOR19HQ1NfQVVUSF9JRF9UWVBFPUF1dGhLZXlWYXVsdCBzZWVtcyB0byBoYXZlIGNhdXNlZCBtZHNkIG5vdAojIHRvIGhvbm91ciBTU0xfQ0VSVF9GSUxFIGFueSBtb3JlLCBoZWF2ZW4gb25seSBrbm93cyB3aHkuCm1rZGlyIC1wIC91c3IvbGliL3NzbC9jZXJ0cwpjc3BsaXQgLWYgL3Vzci9saWIvc3NsL2NlcnRzL2NlcnQtIC1iICUwM2QucGVtIC9ldGMvcGtpL3Rscy9jZXJ0cy9jYS1idW5kbGUuY3J0IC9eJC8xIHsqfSA+L2Rldi9udWxsCmNfcmVoYXNoIC91c3IvbGliL3NzbC9jZXJ0cwoKIyB3ZSBsZWF2ZSBjbGllbnRJZCBibGFuayBhcyBsb25nIGFzIG9ubHkgMSBtYW5hZ2VkIGlkZW50aXR5IGFzc2lnbmVkIHRvIHZtc3MKIyBpZiB3ZSBoYXZlIG1vcmUgdGhhbiAxLCB3ZSB3aWxsIG5lZWQgdG8gcG9wdWxhdGUgd2l0aCBjbGllbnRJZCB1c2VkIGZvciBvZmYtbm9kZSBzY2FubmluZwpjYXQgPi9ldGMvZGVmYXVsdC92c2Etbm9kZXNjYW4tYWdlbnQuY29uZmlnIDw8RU9GCnsKICAgICJOaWNlIjogMTksCiAgICAiVGltZW91dCI6IDEwODAwLAogICAgIkNsaWVudElkIjogIiIsCiAgICAiVGVuYW50SWQiOiAiJEFaVVJFU0VDUEFDS1ZTQVRFTkFOVElEIiwKICAgICJRdWFseXNTdG9yZUJhc2VVcmwiOiAiJEFaVVJFU0VDUEFDS1FVQUxZU1VSTCIsCiAgICAiUHJvY2Vzc1RpbWVvdXQiOiAzMDAsCiAgICAiQ29tbWFuZERlbGF5IjogMAogIH0KRU9GCgplY2hvICJlbmFibGluZyBhcm8gc2VydmljZXMiCmZvciBzZXJ2aWNlIGluIGFyby1nYXRld2F5IGF1b21zIGF6c2VjZCBhenNlY21vbmQgbWRzZCBtZG0gY2hyb255ZCBmbHVlbnRiaXQ7IGRvCiAgc3lzdGVtY3RsIGVuYWJsZSAkc2VydmljZS5zZXJ2aWNlCmRvbmUKCmZvciBzY2FuIGluIGJhc2VsaW5lIGNsYW1hdiBzb2Z0d2FyZTsgZG8KICAvdXNyL2xvY2FsL2Jpbi9henNlY2QgY29uZmlnIC1zICRzY2FuIC1kIFAxRApkb25lCgplY2hvICJyZWJvb3RpbmciCnJlc3RvcmVjb24gLVJGIC92YXIvbG9nLyoKKHNsZWVwIDMwOyByZWJvb3QpICYK')))]" + "script": "[base64(concat(base64ToString('c2V0IC1leAoK'),'ACRRESOURCEID=$(base64 -d \u003c\u003c\u003c''',base64(parameters('acrResourceId')),''')\n','AZURECLOUDNAME=$(base64 -d \u003c\u003c\u003c''',base64(parameters('azureCloudName')),''')\n','AZURESECPACKQUALYSURL=$(base64 -d \u003c\u003c\u003c''',base64(parameters('azureSecPackQualysUrl')),''')\n','AZURESECPACKVSATENANTID=$(base64 -d \u003c\u003c\u003c''',base64(parameters('azureSecPackVSATenantId')),''')\n','DATABASEACCOUNTNAME=$(base64 -d \u003c\u003c\u003c''',base64(parameters('databaseAccountName')),''')\n','DBTOKENCLIENTID=$(base64 -d \u003c\u003c\u003c''',base64(parameters('dbtokenClientId')),''')\n','DBTOKENURL=$(base64 -d \u003c\u003c\u003c''',base64(parameters('dbtokenUrl')),''')\n','MDMFRONTENDURL=$(base64 -d \u003c\u003c\u003c''',base64(parameters('mdmFrontendUrl')),''')\n','MDSDENVIRONMENT=$(base64 -d \u003c\u003c\u003c''',base64(parameters('mdsdEnvironment')),''')\n','FLUENTBITIMAGE=$(base64 -d \u003c\u003c\u003c''',base64(parameters('fluentbitImage')),''')\n','GATEWAYMDSDCONFIGVERSION=$(base64 -d \u003c\u003c\u003c''',base64(parameters('gatewayMdsdConfigVersion')),''')\n','GATEWAYDOMAINS=$(base64 -d \u003c\u003c\u003c''',base64(parameters('gatewayDomains')),''')\n','GATEWAYFEATURES=$(base64 -d \u003c\u003c\u003c''',base64(parameters('gatewayFeatures')),''')\n','KEYVAULTDNSSUFFIX=$(base64 -d \u003c\u003c\u003c''',base64(parameters('keyvaultDNSSuffix')),''')\n','KEYVAULTPREFIX=$(base64 -d \u003c\u003c\u003c''',base64(parameters('keyvaultPrefix')),''')\n','RPIMAGE=$(base64 -d \u003c\u003c\u003c''',base64(parameters('rpImage')),''')\n','RPMDMACCOUNT=$(base64 -d \u003c\u003c\u003c''',base64(parameters('rpMdmAccount')),''')\n','RPMDSDACCOUNT=$(base64 -d \u003c\u003c\u003c''',base64(parameters('rpMdsdAccount')),''')\n','RPMDSDNAMESPACE=$(base64 -d \u003c\u003c\u003c''',base64(parameters('rpMdsdNamespace')),''')\n','MDMIMAGE=''/distroless/genevamdm:2.2024.517.533-b73893-20240522t0954@sha256:939df9d7b6660874697f8ebed1fe56504f86d92f99801a9dc6fd98e9176d3f75''\n','LOCATION=$(base64 -d \u003c\u003c\u003c''',base64(resourceGroup().location),''')\n','SUBSCRIPTIONID=$(base64 -d \u003c\u003c\u003c''',base64(subscription().subscriptionId),''')\n','RESOURCEGROUPNAME=$(base64 -d \u003c\u003c\u003c''',base64(resourceGroup().name),''')\n','\n',base64ToString('IyEvYmluL2Jhc2gKCmVjaG8gInNldHRpbmcgc3NoIHBhc3N3b3JkIGF1dGhlbnRpY2F0aW9uIgojIFdlIG5lZWQgdG8gbWFudWFsbHkgc2V0IFBhc3N3b3JkQXV0aGVudGljYXRpb24gdG8gdHJ1ZSBpbiBvcmRlciBmb3IgdGhlIFZNU1MgQWNjZXNzIEpJVCB0byB3b3JrCnNlZCAtaSAncy9QYXNzd29yZEF1dGhlbnRpY2F0aW9uIG5vL1Bhc3N3b3JkQXV0aGVudGljYXRpb24geWVzL2cnIC9ldGMvc3NoL3NzaGRfY29uZmlnCnN5c3RlbWN0bCByZWxvYWQgc3NoZC5zZXJ2aWNlCgojQWRkaW5nIHJldHJ5IGxvZ2ljIHRvIHl1bSBjb21tYW5kcyBpbiBvcmRlciB0byBhdm9pZCBzdGFsbGluZyBvdXQgb24gcmVzb3VyY2UgbG9ja3MKZWNobyAicnVubmluZyBSSFVJIGZpeCIKZm9yIGF0dGVtcHQgaW4gezEuLjYwfTsgZG8KICB5dW0gdXBkYXRlIC15IC0tZGlzYWJsZXJlcG89JyonIC0tZW5hYmxlcmVwbz0ncmh1aS1taWNyb3NvZnQtYXp1cmUqJyAmJiBicmVhawogIGlmIFtbICR7YXR0ZW1wdH0gLWx0IDYwIF1dOyB0aGVuIHNsZWVwIDMwOyBlbHNlIGV4aXQgMTsgZmkKZG9uZQoKZWNobyAicnVubmluZyB5dW0gdXBkYXRlIgpmb3IgYXR0ZW1wdCBpbiB7MS4uNjB9OyBkbwogIHl1bSAteSAteCBXQUxpbnV4QWdlbnQgLXggV0FMaW51eEFnZW50LXVkZXYgdXBkYXRlIC0tYWxsb3dlcmFzaW5nICYmIGJyZWFrCiAgaWYgW1sgJHthdHRlbXB0fSAtbHQgNjAgXV07IHRoZW4gc2xlZXAgMzA7IGVsc2UgZXhpdCAxOyBmaQpkb25lCgplY2hvICJleHRlbmRpbmcgcGFydGl0aW9uIHRhYmxlIgojIExpbnV4IGJsb2NrIGRldmljZXMgYXJlIGluY29uc2lzdGVudGx5IG5hbWVkCiMgaXQncyBkaWZmaWN1bHQgdG8gdGllIHRoZSBsdm0gcHYgdG8gdGhlIHBoeXNpY2FsIGRpc2sgdXNpbmcgL2Rldi9kaXNrIGZpbGVzLCB3aGljaCBpcyB3aHkgbHZzIGlzIHVzZWQgaGVyZQpwaHlzaWNhbF9kaXNrPSIkKGx2cyAtbyBkZXZpY2VzIC1hIHwgaGVhZCAtbjIgfCB0YWlsIC1uMSB8IGN1dCAtZCAnICcgLWYgMyB8IGN1dCAtZCBcKCAtZiAxIHwgdHIgLWQgJ1s6ZGlnaXQ6XScpIgpncm93cGFydCAiJHBoeXNpY2FsX2Rpc2siIDIKCmVjaG8gImV4dGVuZGluZyBmaWxlc3lzdGVtcyIKbHZleHRlbmQgLWwgKzIwJUZSRUUgL2Rldi9yb290dmcvcm9vdGx2Cnhmc19ncm93ZnMgLwoKbHZleHRlbmQgLWwgKzEwMCVGUkVFIC9kZXYvcm9vdHZnL3Zhcmx2Cnhmc19ncm93ZnMgL3ZhcgoKcnBtIC0taW1wb3J0IGh0dHBzOi8vZGwuZmVkb3JhcHJvamVjdC5vcmcvcHViL2VwZWwvUlBNLUdQRy1LRVktRVBFTC04CnJwbSAtLWltcG9ydCBodHRwczovL3BhY2thZ2VzLm1pY3Jvc29mdC5jb20va2V5cy9taWNyb3NvZnQuYXNjCgpmb3IgYXR0ZW1wdCBpbiB7MS4uNjB9OyBkbwogIHl1bSAteSBpbnN0YWxsIGh0dHBzOi8vZGwuZmVkb3JhcHJvamVjdC5vcmcvcHViL2VwZWwvZXBlbC1yZWxlYXNlLWxhdGVzdC04Lm5vYXJjaC5ycG0gJiYgYnJlYWsKICBpZiBbWyAke2F0dGVtcHR9IC1sdCA2MCBdXTsgdGhlbiBzbGVlcCAzMDsgZWxzZSBleGl0IDE7IGZpCmRvbmUKCmVjaG8gImNvbmZpZ3VyaW5nIGxvZ3JvdGF0ZSIKCiMgZ2F0ZXdheV9sb2dkaXIgaXMgYSByZWFkb25seSB2YXJpYWJsZSB0aGF0IHNwZWNpZmllcyB0aGUgaG9zdCBwYXRoIG1vdW50IHBvaW50IGZvciB0aGUgZ2F0ZXdheSBjb250YWluZXIgbG9nIGZpbGUKIyBmb3IgdGhlIHB1cnBvc2Ugb2Ygcm90YXRpbmcgdGhlIGdhdGV3YXkgbG9ncwpkZWNsYXJlIC1yIGdhdGV3YXlfbG9nZGlyPScvdmFyL2xvZy9hcm8tZ2F0ZXdheScKCmNhdCA+L2V0Yy9sb2dyb3RhdGUuY29uZiA8PEVPRgojIHNlZSAibWFuIGxvZ3JvdGF0ZSIgZm9yIGRldGFpbHMKIyByb3RhdGUgbG9nIGZpbGVzIHdlZWtseQp3ZWVrbHkKCiMga2VlcCAyIHdlZWtzIHdvcnRoIG9mIGJhY2tsb2dzCnJvdGF0ZSAyCgojIGNyZWF0ZSBuZXcgKGVtcHR5KSBsb2cgZmlsZXMgYWZ0ZXIgcm90YXRpbmcgb2xkIG9uZXMKY3JlYXRlCgojIHVzZSBkYXRlIGFzIGEgc3VmZml4IG9mIHRoZSByb3RhdGVkIGZpbGUKZGF0ZWV4dAoKIyB1bmNvbW1lbnQgdGhpcyBpZiB5b3Ugd2FudCB5b3VyIGxvZyBmaWxlcyBjb21wcmVzc2VkCmNvbXByZXNzCgojIFJQTSBwYWNrYWdlcyBkcm9wIGxvZyByb3RhdGlvbiBpbmZvcm1hdGlvbiBpbnRvIHRoaXMgZGlyZWN0b3J5CmluY2x1ZGUgL2V0Yy9sb2dyb3RhdGUuZAoKIyBubyBwYWNrYWdlcyBvd24gd3RtcCBhbmQgYnRtcCAtLSB3ZSdsbCByb3RhdGUgdGhlbSBoZXJlCi92YXIvbG9nL3d0bXAgewogICAgbW9udGhseQogICAgY3JlYXRlIDA2NjQgcm9vdCB1dG1wCiAgICAgICAgbWluc2l6ZSAxTQogICAgcm90YXRlIDEKfQoKL3Zhci9sb2cvYnRtcCB7CiAgICBtaXNzaW5nb2sKICAgIG1vbnRobHkKICAgIGNyZWF0ZSAwNjAwIHJvb3QgdXRtcAogICAgcm90YXRlIDEKfQoKIyBNYXhpbXVtIGxvZyBkaXJlY3Rvcnkgc2l6ZSBpcyAxMDBHIHdpdGggdGhpcyBjb25maWd1cmF0aW9uCiMgU2V0dGluZyBsaW1pdCB0byAxMDBHIHRvIGFsbG93IHNwYWNlIGZvciBvdGhlciBsb2dnaW5nIHNlcnZpY2VzCiMgY29weXRydW5jYXRlIGlzIGEgY3JpdGljYWwgb3B0aW9uIHVzZWQgdG8gcHJldmVudCBsb2dzIGZyb20gYmVpbmcgc2hpcHBlZCB0d2ljZQoke2dhdGV3YXlfbG9nZGlyfSB7CiAgICBzaXplIDIwRwogICAgcm90YXRlIDUKICAgIGNyZWF0ZSAwNjAwIHJvb3Qgcm9vdAogICAgY29weXRydW5jYXRlCiAgICBub29sZGRpcgogICAgY29tcHJlc3MKfQpFT0YKCmVjaG8gImNvbmZpZ3VyaW5nIHl1bSByZXBvc2l0b3J5IGFuZCBydW5uaW5nIHl1bSB1cGRhdGUiCmNhdCA+L2V0Yy95dW0ucmVwb3MuZC9henVyZS5yZXBvIDw8J0VPRicKW2F6dXJlLWNsaV0KbmFtZT1henVyZS1jbGkKYmFzZXVybD1odHRwczovL3BhY2thZ2VzLm1pY3Jvc29mdC5jb20veXVtcmVwb3MvYXp1cmUtY2xpCmVuYWJsZWQ9eWVzCmdwZ2NoZWNrPXllcwoKW2F6dXJlY29yZV0KbmFtZT1henVyZWNvcmUKYmFzZXVybD1odHRwczovL3BhY2thZ2VzLm1pY3Jvc29mdC5jb20veXVtcmVwb3MvYXp1cmVjb3JlCmVuYWJsZWQ9eWVzCmdwZ2NoZWNrPW5vCkVPRgoKc2VtYW5hZ2UgZmNvbnRleHQgLWEgLXQgdmFyX2xvZ190ICIvdmFyL2xvZy9qb3VybmFsKC8uKik/Igpta2RpciAtcCAvdmFyL2xvZy9qb3VybmFsCgpmb3IgYXR0ZW1wdCBpbiB7MS4uNjB9OyBkbwogIHl1bSAteSBpbnN0YWxsIGNsYW1hdiBhenNlYy1jbGFtYXYgYXpzZWMtbW9uaXRvciBhenVyZS1jbGkgYXp1cmUtbWRzZCBhenVyZS1zZWN1cml0eSBwb2RtYW4tZG9ja2VyIG9wZW5zc2wtcGVybCBweXRob24zICYmIGJyZWFrCiAgIyBoYWNrIC0gd2UgYXJlIGluc3RhbGxpbmcgcHl0aG9uMyBvbiBob3N0cyBkdWUgdG8gYW4gaXNzdWUgd2l0aCBBenVyZSBMaW51eCBFeHRlbnNpb25zIGh0dHBzOi8vZ2l0aHViLmNvbS9BenVyZS9henVyZS1saW51eC1leHRlbnNpb25zL3B1bGwvMTUwNQogIGlmIFtbICR7YXR0ZW1wdH0gLWx0IDYwIF1dOyB0aGVuIHNsZWVwIDMwOyBlbHNlIGV4aXQgMTsgZmkKZG9uZQoKZWNobyAiYXBwbHlpbmcgZmlyZXdhbGwgcnVsZXMiCiMgaHR0cHM6Ly9hY2Nlc3MucmVkaGF0LmNvbS9zZWN1cml0eS9jdmUvY3ZlLTIwMjAtMTM0MDEKY2F0ID4vZXRjL3N5c2N0bC5kLzAyLWRpc2FibGUtYWNjZXB0LXJhLmNvbmYgPDwnRU9GJwpuZXQuaXB2Ni5jb25mLmFsbC5hY2NlcHRfcmE9MApFT0YKCmNhdCA+L2V0Yy9zeXNjdGwuZC8wMS1kaXNhYmxlLWNvcmUuY29uZiA8PCdFT0YnCmtlcm5lbC5jb3JlX3BhdHRlcm4gPSB8L2Jpbi90cnVlCkVPRgpzeXNjdGwgLS1zeXN0ZW0KCmZpcmV3YWxsLWNtZCAtLWFkZC1wb3J0PTgwL3RjcCAtLXBlcm1hbmVudApmaXJld2FsbC1jbWQgLS1hZGQtcG9ydD04MDgxL3RjcCAtLXBlcm1hbmVudApmaXJld2FsbC1jbWQgLS1hZGQtcG9ydD00NDMvdGNwIC0tcGVybWFuZW50CgplY2hvICJsb2dnaW5nIGludG8gcHJvZCBhY3IiCmV4cG9ydCBBWlVSRV9DTE9VRF9OQU1FPSRBWlVSRUNMT1VETkFNRQpheiBsb2dpbiAtaSAtLWFsbG93LW5vLXN1YnNjcmlwdGlvbnMKCiMgVGhlIG1hbmFnZWQgaWRlbnRpdHkgdGhhdCB0aGUgVk0gcnVucyBhcyBvbmx5IGhhcyBhIHNpbmdsZSByb2xlYXNzaWdubWVudC4KIyBUaGlzIHJvbGUgYXNzaWdubWVudCBpcyBBQ1JQdWxsIHdoaWNoIGlzIG5vdCBuZWNlc3NhcmlseSBwcmVzZW50IGluIHRoZQojIHN1YnNjcmlwdGlvbiB3ZSdyZSBkZXBsb3lpbmcgaW50by4gIElmIHRoZSBpZGVudGl0eSBkb2VzIG5vdCBoYXZlIGFueQojIHJvbGUgYXNzaWdubWVudHMgc2NvcGVkIG9uIHRoZSBzdWJzY3JpcHRpb24gd2UncmUgZGVwbG95aW5nIGludG8sIGl0IHdpbGwKIyBub3Qgc2hvdyBvbiBheiBsb2dpbiAtaSwgd2hpY2ggaXMgd2h5IHRoZSBiZWxvdyBsaW5lIGlzIGNvbW1lbnRlZC4KIyBheiBhY2NvdW50IHNldCAtcyAiJFNVQlNDUklQVElPTklEIgoKIyBTdXBwcmVzcyBlbXVsYXRpb24gb3V0cHV0IGZvciBwb2RtYW4gaW5zdGVhZCBvZiBkb2NrZXIgZm9yIGF6IGFjciBjb21wYXRhYmlsaXR5Cm1rZGlyIC1wIC9ldGMvY29udGFpbmVycy8KdG91Y2ggL2V0Yy9jb250YWluZXJzL25vZG9ja2VyCgpta2RpciAtcCAvcm9vdC8uZG9ja2VyClJFR0lTVFJZX0FVVEhfRklMRT0vcm9vdC8uZG9ja2VyL2NvbmZpZy5qc29uIGF6IGFjciBsb2dpbiAtLW5hbWUgIiQoc2VkIC1lICdzfC4qL3x8JyA8PDwiJEFDUlJFU09VUkNFSUQiKSIKCk1ETUlNQUdFPSIke1JQSU1BR0UlJS8qfS8ke01ETUlNQUdFIyovfSIKZG9ja2VyIHB1bGwgIiRNRE1JTUFHRSIKZG9ja2VyIHB1bGwgIiRSUElNQUdFIgpkb2NrZXIgcHVsbCAiJEZMVUVOVEJJVElNQUdFIgoKYXogbG9nb3V0CgplY2hvICJjb25maWd1cmluZyBmbHVlbnRiaXQgc2VydmljZSIKbWtkaXIgLXAgL2V0Yy9mbHVlbnRiaXQvCm1rZGlyIC1wIC92YXIvbGliL2ZsdWVudAoKY2F0ID4vZXRjL2ZsdWVudGJpdC9mbHVlbnRiaXQuY29uZiA8PCdFT0YnCltJTlBVVF0KCU5hbWUgc3lzdGVtZAoJVGFnIGpvdXJuYWxkCglTeXN0ZW1kX0ZpbHRlciBfQ09NTT1hcm8KCURCIC92YXIvbGliL2ZsdWVudC9qb3VybmFsZGIKCltGSUxURVJdCglOYW1lIG1vZGlmeQoJTWF0Y2ggam91cm5hbGQKCVJlbW92ZV93aWxkY2FyZCBfCglSZW1vdmUgVElNRVNUQU1QCgpbT1VUUFVUXQoJTmFtZSBmb3J3YXJkCglNYXRjaCAqCglQb3J0IDI5MjMwCkVPRgoKZWNobyAiRkxVRU5UQklUSU1BR0U9JEZMVUVOVEJJVElNQUdFIiA+L2V0Yy9zeXNjb25maWcvZmx1ZW50Yml0CgpjYXQgPi9ldGMvc3lzdGVtZC9zeXN0ZW0vZmx1ZW50Yml0LnNlcnZpY2UgPDwnRU9GJwpbVW5pdF0KQWZ0ZXI9bmV0d29yay1vbmxpbmUudGFyZ2V0CldhbnRzPW5ldHdvcmstb25saW5lLnRhcmdldApTdGFydExpbWl0SW50ZXJ2YWxTZWM9MAoKW1NlcnZpY2VdClJlc3RhcnRTZWM9MXMKRW52aXJvbm1lbnRGaWxlPS9ldGMvc3lzY29uZmlnL2ZsdWVudGJpdApFeGVjU3RhcnRQcmU9LS91c3IvYmluL2RvY2tlciBybSAtZiAlTgpFeGVjU3RhcnQ9L3Vzci9iaW4vZG9ja2VyIHJ1biBcCiAgLS1zZWN1cml0eS1vcHQgbGFiZWw9ZGlzYWJsZSBcCiAgLS1lbnRyeXBvaW50IC9vcHQvdGQtYWdlbnQtYml0L2Jpbi90ZC1hZ2VudC1iaXQgXAogIC0tbmV0PWhvc3QgXAogIC0taG9zdG5hbWUgJUggXAogIC0tbmFtZSAlTiBcCiAgLS1ybSBcCiAgLS1jYXAtZHJvcCBuZXRfcmF3IFwKICAtdiAvZXRjL2ZsdWVudGJpdC9mbHVlbnRiaXQuY29uZjovZXRjL2ZsdWVudGJpdC9mbHVlbnRiaXQuY29uZiBcCiAgLXYgL3Zhci9saWIvZmx1ZW50Oi92YXIvbGliL2ZsdWVudDp6IFwKICAtdiAvdmFyL2xvZy9qb3VybmFsOi92YXIvbG9nL2pvdXJuYWw6cm8gXAogIC12IC9ldGMvbWFjaGluZS1pZDovZXRjL21hY2hpbmUtaWQ6cm8gXAogICRGTFVFTlRCSVRJTUFHRSBcCiAgLWMgL2V0Yy9mbHVlbnRiaXQvZmx1ZW50Yml0LmNvbmYKCkV4ZWNTdG9wPS91c3IvYmluL2RvY2tlciBzdG9wICVOClJlc3RhcnQ9YWx3YXlzClJlc3RhcnRTZWM9NQpTdGFydExpbWl0SW50ZXJ2YWw9MAoKW0luc3RhbGxdCldhbnRlZEJ5PW11bHRpLXVzZXIudGFyZ2V0CkVPRgoKZWNobyAiY29uZmlndXJpbmcgbWRtIHNlcnZpY2UiCmNhdCA+L2V0Yy9zeXNjb25maWcvbWRtIDw8RU9GCk1ETUZST05URU5EVVJMPSckTURNRlJPTlRFTkRVUkwnCk1ETUlNQUdFPSckTURNSU1BR0UnCk1ETVNPVVJDRUVOVklST05NRU5UPSckTE9DQVRJT04nCk1ETVNPVVJDRVJPTEU9Z2F0ZXdheQpNRE1TT1VSQ0VST0xFSU5TVEFOQ0U9JyQoaG9zdG5hbWUpJwpFT0YKCm1rZGlyIC92YXIvZXR3CmNhdCA+L2V0Yy9zeXN0ZW1kL3N5c3RlbS9tZG0uc2VydmljZSA8PCdFT0YnCltVbml0XQpBZnRlcj1uZXR3b3JrLW9ubGluZS50YXJnZXQKV2FudHM9bmV0d29yay1vbmxpbmUudGFyZ2V0CgpbU2VydmljZV0KRW52aXJvbm1lbnRGaWxlPS9ldGMvc3lzY29uZmlnL21kbQpFeGVjU3RhcnRQcmU9LS91c3IvYmluL2RvY2tlciBybSAtZiAlTgpFeGVjU3RhcnQ9L3Vzci9iaW4vZG9ja2VyIHJ1biBcCiAgLS1lbnRyeXBvaW50IC91c3Ivc2Jpbi9NZXRyaWNzRXh0ZW5zaW9uIFwKICAtLWhvc3RuYW1lICVIIFwKICAtLW5hbWUgJU4gXAogIC0tcm0gXAogIC0tY2FwLWRyb3AgbmV0X3JhdyBcCiAgLW0gMmcgXAogIC12IC9ldGMvbWRtLnBlbTovZXRjL21kbS5wZW0gXAogIC12IC92YXIvZXR3Oi92YXIvZXR3OnogXAogICRNRE1JTUFHRSBcCiAgLUNlcnRGaWxlIC9ldGMvbWRtLnBlbSBcCiAgLUZyb250RW5kVXJsICRNRE1GUk9OVEVORFVSTCBcCiAgLUxvZ2dlciBDb25zb2xlIFwKICAtTG9nTGV2ZWwgV2FybmluZyBcCiAgLVByaXZhdGVLZXlGaWxlIC9ldGMvbWRtLnBlbSBcCiAgLVNvdXJjZUVudmlyb25tZW50ICRNRE1TT1VSQ0VFTlZJUk9OTUVOVCBcCiAgLVNvdXJjZVJvbGUgJE1ETVNPVVJDRVJPTEUgXAogIC1Tb3VyY2VSb2xlSW5zdGFuY2UgJE1ETVNPVVJDRVJPTEVJTlNUQU5DRQpFeGVjU3RvcD0vdXNyL2Jpbi9kb2NrZXIgc3RvcCAlTgpSZXN0YXJ0PWFsd2F5cwpSZXN0YXJ0U2VjPTEKU3RhcnRMaW1pdEludGVydmFsPTAKCltJbnN0YWxsXQpXYW50ZWRCeT1tdWx0aS11c2VyLnRhcmdldApFT0YKCmVjaG8gImNvbmZpZ3VyaW5nIGFyby1nYXRld2F5IHNlcnZpY2UiCmNhdCA+L2V0Yy9zeXNjb25maWcvYXJvLWdhdGV3YXkgPDxFT0YKQUNSX1JFU09VUkNFX0lEPSckQUNSUkVTT1VSQ0VJRCcKREFUQUJBU0VfQUNDT1VOVF9OQU1FPSckREFUQUJBU0VBQ0NPVU5UTkFNRScKQVpVUkVfREJUT0tFTl9DTElFTlRfSUQ9JyREQlRPS0VOQ0xJRU5USUQnCkRCVE9LRU5fVVJMPSckREJUT0tFTlVSTCcKTURNX0FDQ09VTlQ9IiRSUE1ETUFDQ09VTlQiCk1ETV9OQU1FU1BBQ0U9R2F0ZXdheQpHQVRFV0FZX0RPTUFJTlM9JyRHQVRFV0FZRE9NQUlOUycKR0FURVdBWV9GRUFUVVJFUz0nJEdBVEVXQVlGRUFUVVJFUycKUlBJTUFHRT0nJFJQSU1BR0UnCkVPRgoKY2F0ID4vZXRjL3N5c3RlbWQvc3lzdGVtL2Fyby1nYXRld2F5LnNlcnZpY2UgPDxFT0YKW1VuaXRdCkFmdGVyPW5ldHdvcmstb25saW5lLnRhcmdldApXYW50cz1uZXR3b3JrLW9ubGluZS50YXJnZXQKCltTZXJ2aWNlXQpFbnZpcm9ubWVudEZpbGU9L2V0Yy9zeXNjb25maWcvYXJvLWdhdGV3YXkKRXhlY1N0YXJ0UHJlPS0vdXNyL2Jpbi9kb2NrZXIgcm0gLWYgJU4KRXhlY1N0YXJ0UHJlPS91c3IvYmluL21rZGlyIC1wICR7Z2F0ZXdheV9sb2dkaXJ9CkV4ZWNTdGFydD0vdXNyL2Jpbi9kb2NrZXIgcnVuIFwKICAtLWhvc3RuYW1lICVIIFwKICAtLW5hbWUgJU4gXAogIC0tcm0gXAogIC0tY2FwLWRyb3AgbmV0X3JhdyBcCiAgLWUgQUNSX1JFU09VUkNFX0lEIFwKICAtZSBEQVRBQkFTRV9BQ0NPVU5UX05BTUUgXAogIC1lIEFaVVJFX0RCVE9LRU5fQ0xJRU5UX0lEIFwKICAtZSBEQlRPS0VOX1VSTCBcCiAgLWUgR0FURVdBWV9ET01BSU5TIFwKICAtZSBHQVRFV0FZX0ZFQVRVUkVTIFwKICAtZSBNRE1fQUNDT1VOVCBcCiAgLWUgTURNX05BTUVTUEFDRSBcCiAgLW0gMmcgXAogIC1wIDgwOjgwODAgXAogIC1wIDgwODE6ODA4MSBcCiAgLXAgNDQzOjg0NDMgXAogIC12IC9ydW4vc3lzdGVtZC9qb3VybmFsOi9ydW4vc3lzdGVtZC9qb3VybmFsIFwKICAtdiAvdmFyL2V0dzovdmFyL2V0dzp6IFwKICAtdiAke2dhdGV3YXlfbG9nZGlyfTovY3RyLmxvZzp6IFwKICBcJFJQSU1BR0UgXAogIGdhdGV3YXkKRXhlY1N0b3A9L3Vzci9iaW4vZG9ja2VyIHN0b3AgLXQgMzYwMCAlTgpUaW1lb3V0U3RvcFNlYz0zNjAwClJlc3RhcnQ9YWx3YXlzClJlc3RhcnRTZWM9MQpTdGFydExpbWl0SW50ZXJ2YWw9MAoKW0luc3RhbGxdCldhbnRlZEJ5PW11bHRpLXVzZXIudGFyZ2V0CkVPRgoKY2hjb24gLVIgc3lzdGVtX3U6b2JqZWN0X3I6dmFyX2xvZ190OnMwIC92YXIvb3B0L21pY3Jvc29mdC9saW51eG1vbmFnZW50Cgpta2RpciAtcCAvdmFyL2xpYi93YWFnZW50L01pY3Jvc29mdC5BenVyZS5LZXlWYXVsdC5TdG9yZQoKZWNobyAiY29uZmlndXJpbmcgbWRzZCBhbmQgbWRtIHNlcnZpY2VzIgpmb3IgdmFyIGluICJtZHNkIiAibWRtIjsgZG8KY2F0ID4vZXRjL3N5c3RlbWQvc3lzdGVtL2Rvd25sb2FkLSR2YXItY3JlZGVudGlhbHMuc2VydmljZSA8PEVPRgpbVW5pdF0KRGVzY3JpcHRpb249UGVyaW9kaWMgJHZhciBjcmVkZW50aWFscyByZWZyZXNoCgpbU2VydmljZV0KVHlwZT1vbmVzaG90CkV4ZWNTdGFydD0vdXNyL2xvY2FsL2Jpbi9kb3dubG9hZC1jcmVkZW50aWFscy5zaCAkdmFyCkVPRgoKY2F0ID4vZXRjL3N5c3RlbWQvc3lzdGVtL2Rvd25sb2FkLSR2YXItY3JlZGVudGlhbHMudGltZXIgPDxFT0YKW1VuaXRdCkRlc2NyaXB0aW9uPVBlcmlvZGljICR2YXIgY3JlZGVudGlhbHMgcmVmcmVzaApBZnRlcj1uZXR3b3JrLW9ubGluZS50YXJnZXQKV2FudHM9bmV0d29yay1vbmxpbmUudGFyZ2V0CgpbVGltZXJdCk9uQm9vdFNlYz0wbWluCk9uQ2FsZW5kYXI9MC8xMjowMDowMApBY2N1cmFjeVNlYz01cwoKW0luc3RhbGxdCldhbnRlZEJ5PXRpbWVycy50YXJnZXQKRU9GCmRvbmUKCmNhdCA+L3Vzci9sb2NhbC9iaW4vZG93bmxvYWQtY3JlZGVudGlhbHMuc2ggPDxFT0YKIyEvYmluL2Jhc2gKc2V0IC1ldQoKQ09NUE9ORU5UPSJcJDEiCmVjaG8gIkRvd25sb2FkIFwkQ09NUE9ORU5UIGNyZWRlbnRpYWxzIgoKVEVNUF9ESVI9XCQobWt0ZW1wIC1kKQpleHBvcnQgQVpVUkVfQ09ORklHX0RJUj1cJChta3RlbXAgLWQpCgplY2hvICJMb2dnaW5nIGludG8gQXp1cmUuLi4iClJFVFJJRVM9Mwp3aGlsZSBbICJcJFJFVFJJRVMiIC1ndCAwIF07IGRvCiAgICBpZiBheiBsb2dpbiAtaSAtLWFsbG93LW5vLXN1YnNjcmlwdGlvbnMKICAgIHRoZW4KICAgICAgICBlY2hvICJheiBsb2dpbiBzdWNjZXNzZnVsIgogICAgICAgIGJyZWFrCiAgICBlbHNlCiAgICAgICAgZWNobyAiYXogbG9naW4gZmFpbGVkLiBSZXRyeWluZy4uLiIKICAgICAgICBsZXQgUkVUUklFUy09MQogICAgICAgIHNsZWVwIDUKICAgIGZpCmRvbmUKCnRyYXAgImNsZWFudXAiIEVYSVQKCmNsZWFudXAoKSB7CiAgYXogbG9nb3V0CiAgW1sgIlwkVEVNUF9ESVIiID1+IC90bXAvLisgXV0gJiYgcm0gLXJmIFwkVEVNUF9ESVIKICBbWyAiXCRBWlVSRV9DT05GSUdfRElSIiA9fiAvdG1wLy4rIF1dICYmIHJtIC1yZiBcJEFaVVJFX0NPTkZJR19ESVIKfQoKaWYgWyAiXCRDT01QT05FTlQiID0gIm1kbSIgXTsgdGhlbgogIENVUlJFTlRfQ0VSVF9GSUxFPSIvZXRjL21kbS5wZW0iCmVsaWYgWyAiXCRDT01QT05FTlQiID0gIm1kc2QiIF07IHRoZW4KICBDVVJSRU5UX0NFUlRfRklMRT0iL3Zhci9saWIvd2FhZ2VudC9NaWNyb3NvZnQuQXp1cmUuS2V5VmF1bHQuU3RvcmUvbWRzZC5wZW0iCmVsc2UKICBlY2hvIEludmFsaWQgdXNhZ2UgJiYgZXhpdCAxCmZpCgpTRUNSRVRfTkFNRT0iZ3d5LVwke0NPTVBPTkVOVH0iCk5FV19DRVJUX0ZJTEU9IlwkVEVNUF9ESVIvXCRDT01QT05FTlQucGVtIgpmb3IgYXR0ZW1wdCBpbiB7MS4uNX07IGRvCiAgYXoga2V5dmF1bHQgc2VjcmV0IGRvd25sb2FkIC0tZmlsZSBcJE5FV19DRVJUX0ZJTEUgLS1pZCAiaHR0cHM6Ly8kS0VZVkFVTFRQUkVGSVgtZ3d5LiRLRVlWQVVMVEROU1NVRkZJWC9zZWNyZXRzL1wkU0VDUkVUX05BTUUiICYmIGJyZWFrCiAgaWYgW1sgXCRhdHRlbXB0IC1sdCA1IF1dOyB0aGVuIHNsZWVwIDEwOyBlbHNlIGV4aXQgMTsgZmkKZG9uZQoKaWYgWyAtZiBcJE5FV19DRVJUX0ZJTEUgXTsgdGhlbgogIGlmIFsgIlwkQ09NUE9ORU5UIiA9ICJtZHNkIiBdOyB0aGVuCiAgICBjaG93biBzeXNsb2c6c3lzbG9nIFwkTkVXX0NFUlRfRklMRQogIGVsc2UKICAgIHNlZCAtaSAtbmUgJzEsL0VORCBDRVJUSUZJQ0FURS8gcCcgXCRORVdfQ0VSVF9GSUxFCiAgZmkKCiAgbmV3X2NlcnRfc249IlwkKG9wZW5zc2wgeDUwOSAtaW4gIlwkTkVXX0NFUlRfRklMRSIgLW5vb3V0IC1zZXJpYWwgfCBhd2sgLUY9ICd7cHJpbnQgXCQyfScpIgogIGN1cnJlbnRfY2VydF9zbj0iXCQob3BlbnNzbCB4NTA5IC1pbiAiXCRDVVJSRU5UX0NFUlRfRklMRSIgLW5vb3V0IC1zZXJpYWwgfCBhd2sgLUY9ICd7cHJpbnQgXCQyfScpIgogIGlmIFtbICEgLXogXCRuZXdfY2VydF9zbiBdXSAmJiBbWyBcJG5ld19jZXJ0X3NuICE9ICJcJGN1cnJlbnRfY2VydF9zbiIgXV07IHRoZW4KICAgIGVjaG8gdXBkYXRpbmcgY2VydGlmaWNhdGUgZm9yIFwkQ09NUE9ORU5UCiAgICBjaG1vZCAwNjAwIFwkTkVXX0NFUlRfRklMRQogICAgbXYgXCRORVdfQ0VSVF9GSUxFIFwkQ1VSUkVOVF9DRVJUX0ZJTEUKICBmaQplbHNlCiAgZWNobyBGYWlsZWQgdG8gcmVmcmVzaCBjZXJ0aWZpY2F0ZSBmb3IgXCRDT01QT05FTlQgJiYgZXhpdCAxCmZpCkVPRgoKY2htb2QgdSt4IC91c3IvbG9jYWwvYmluL2Rvd25sb2FkLWNyZWRlbnRpYWxzLnNoCgpzeXN0ZW1jdGwgZW5hYmxlIGRvd25sb2FkLW1kc2QtY3JlZGVudGlhbHMudGltZXIKc3lzdGVtY3RsIGVuYWJsZSBkb3dubG9hZC1tZG0tY3JlZGVudGlhbHMudGltZXIKCi91c3IvbG9jYWwvYmluL2Rvd25sb2FkLWNyZWRlbnRpYWxzLnNoIG1kc2QKL3Vzci9sb2NhbC9iaW4vZG93bmxvYWQtY3JlZGVudGlhbHMuc2ggbWRtCk1EU0RDRVJUSUZJQ0FURVNBTj0kKG9wZW5zc2wgeDUwOSAtaW4gL3Zhci9saWIvd2FhZ2VudC9NaWNyb3NvZnQuQXp1cmUuS2V5VmF1bHQuU3RvcmUvbWRzZC5wZW0gLW5vb3V0IC1zdWJqZWN0IHwgc2VkIC1lICdzLy4qQ04gPSAvLycpCgpjYXQgPi9ldGMvc3lzdGVtZC9zeXN0ZW0vd2F0Y2gtbWRtLWNyZWRlbnRpYWxzLnNlcnZpY2UgPDxFT0YKW1VuaXRdCkRlc2NyaXB0aW9uPVdhdGNoIGZvciBjaGFuZ2VzIGluIG1kbS5wZW0gYW5kIHJlc3RhcnRzIHRoZSBtZG0gc2VydmljZQoKW1NlcnZpY2VdClR5cGU9b25lc2hvdApFeGVjU3RhcnQ9L3Vzci9iaW4vc3lzdGVtY3RsIHJlc3RhcnQgbWRtLnNlcnZpY2UKCltJbnN0YWxsXQpXYW50ZWRCeT1tdWx0aS11c2VyLnRhcmdldApFT0YKCmNhdCA+L2V0Yy9zeXN0ZW1kL3N5c3RlbS93YXRjaC1tZG0tY3JlZGVudGlhbHMucGF0aCA8PEVPRgpbUGF0aF0KUGF0aE1vZGlmaWVkPS9ldGMvbWRtLnBlbQoKW0luc3RhbGxdCldhbnRlZEJ5PW11bHRpLXVzZXIudGFyZ2V0CkVPRgoKc3lzdGVtY3RsIGVuYWJsZSB3YXRjaC1tZG0tY3JlZGVudGlhbHMucGF0aApzeXN0ZW1jdGwgc3RhcnQgd2F0Y2gtbWRtLWNyZWRlbnRpYWxzLnBhdGgKCm1rZGlyIC9ldGMvc3lzdGVtZC9zeXN0ZW0vbWRzZC5zZXJ2aWNlLmQKY2F0ID4vZXRjL3N5c3RlbWQvc3lzdGVtL21kc2Quc2VydmljZS5kL292ZXJyaWRlLmNvbmYgPDwnRU9GJwpbVW5pdF0KQWZ0ZXI9bmV0d29yay1vbmxpbmUudGFyZ2V0CkVPRgoKY2F0ID4vZXRjL2RlZmF1bHQvbWRzZCA8PEVPRgpNRFNEX1JPTEVfUFJFRklYPS92YXIvcnVuL21kc2QvZGVmYXVsdApNRFNEX09QVElPTlM9Ii1BIC1kIC1yIFwkTURTRF9ST0xFX1BSRUZJWCIKCmV4cG9ydCBNT05JVE9SSU5HX0dDU19FTlZJUk9OTUVOVD0nJE1EU0RFTlZJUk9OTUVOVCcKZXhwb3J0IE1PTklUT1JJTkdfR0NTX0FDQ09VTlQ9JyRSUE1EU0RBQ0NPVU5UJwpleHBvcnQgTU9OSVRPUklOR19HQ1NfUkVHSU9OPSckTE9DQVRJT04nCmV4cG9ydCBNT05JVE9SSU5HX0dDU19BVVRIX0lEX1RZUEU9QXV0aEtleVZhdWx0CmV4cG9ydCBNT05JVE9SSU5HX0dDU19BVVRIX0lEPSckTURTRENFUlRJRklDQVRFU0FOJwpleHBvcnQgTU9OSVRPUklOR19HQ1NfTkFNRVNQQUNFPSckUlBNRFNETkFNRVNQQUNFJwpleHBvcnQgTU9OSVRPUklOR19DT05GSUdfVkVSU0lPTj0nJEdBVEVXQVlNRFNEQ09ORklHVkVSU0lPTicKZXhwb3J0IE1PTklUT1JJTkdfVVNFX0dFTkVWQV9DT05GSUdfU0VSVklDRT10cnVlCgpleHBvcnQgTU9OSVRPUklOR19URU5BTlQ9JyRMT0NBVElPTicKZXhwb3J0IE1PTklUT1JJTkdfUk9MRT1nYXRld2F5CmV4cG9ydCBNT05JVE9SSU5HX1JPTEVfSU5TVEFOQ0U9JyQoaG9zdG5hbWUpJwoKZXhwb3J0IE1EU0RfTVNHUEFDS19TT1JUX0NPTFVNTlM9MQpFT0YKCiMgc2V0dGluZyBNT05JVE9SSU5HX0dDU19BVVRIX0lEX1RZUEU9QXV0aEtleVZhdWx0IHNlZW1zIHRvIGhhdmUgY2F1c2VkIG1kc2Qgbm90CiMgdG8gaG9ub3VyIFNTTF9DRVJUX0ZJTEUgYW55IG1vcmUsIGhlYXZlbiBvbmx5IGtub3dzIHdoeS4KbWtkaXIgLXAgL3Vzci9saWIvc3NsL2NlcnRzCmNzcGxpdCAtZiAvdXNyL2xpYi9zc2wvY2VydHMvY2VydC0gLWIgJTAzZC5wZW0gL2V0Yy9wa2kvdGxzL2NlcnRzL2NhLWJ1bmRsZS5jcnQgL14kLzEgeyp9ID4vZGV2L251bGwKY19yZWhhc2ggL3Vzci9saWIvc3NsL2NlcnRzCgojIHdlIGxlYXZlIGNsaWVudElkIGJsYW5rIGFzIGxvbmcgYXMgb25seSAxIG1hbmFnZWQgaWRlbnRpdHkgYXNzaWduZWQgdG8gdm1zcwojIGlmIHdlIGhhdmUgbW9yZSB0aGFuIDEsIHdlIHdpbGwgbmVlZCB0byBwb3B1bGF0ZSB3aXRoIGNsaWVudElkIHVzZWQgZm9yIG9mZi1ub2RlIHNjYW5uaW5nCmNhdCA+L2V0Yy9kZWZhdWx0L3ZzYS1ub2Rlc2Nhbi1hZ2VudC5jb25maWcgPDxFT0YKewogICAgIk5pY2UiOiAxOSwKICAgICJUaW1lb3V0IjogMTA4MDAsCiAgICAiQ2xpZW50SWQiOiAiIiwKICAgICJUZW5hbnRJZCI6ICIkQVpVUkVTRUNQQUNLVlNBVEVOQU5USUQiLAogICAgIlF1YWx5c1N0b3JlQmFzZVVybCI6ICIkQVpVUkVTRUNQQUNLUVVBTFlTVVJMIiwKICAgICJQcm9jZXNzVGltZW91dCI6IDMwMCwKICAgICJDb21tYW5kRGVsYXkiOiAwCiAgfQpFT0YKCmVjaG8gImVuYWJsaW5nIGFybyBzZXJ2aWNlcyIKZm9yIHNlcnZpY2UgaW4gYXJvLWdhdGV3YXkgYXVvbXMgYXpzZWNkIGF6c2VjbW9uZCBtZHNkIG1kbSBjaHJvbnlkIGZsdWVudGJpdDsgZG8KICBzeXN0ZW1jdGwgZW5hYmxlICRzZXJ2aWNlLnNlcnZpY2UKZG9uZQoKZm9yIHNjYW4gaW4gYmFzZWxpbmUgY2xhbWF2IHNvZnR3YXJlOyBkbwogIC91c3IvbG9jYWwvYmluL2F6c2VjZCBjb25maWcgLXMgJHNjYW4gLWQgUDFECmRvbmUKCmVjaG8gInJlYm9vdGluZyIKcmVzdG9yZWNvbiAtUkYgL3Zhci9sb2cvKgooc2xlZXAgMzA7IHJlYm9vdCkgJgo=')))]" } } } diff --git a/pkg/deploy/assets/rp-production-parameters.json b/pkg/deploy/assets/rp-production-parameters.json index 9e40f9398b6..8e60daafb5c 100644 --- a/pkg/deploy/assets/rp-production-parameters.json +++ b/pkg/deploy/assets/rp-production-parameters.json @@ -159,9 +159,6 @@ "sshPublicKey": { "value": "" }, - "storageAccountDomain": { - "value": "" - }, "subscriptionResourceGroupName": { "value": "" }, diff --git a/pkg/deploy/assets/rp-production.json b/pkg/deploy/assets/rp-production.json index 26282772f7b..2116b86e69d 100644 --- a/pkg/deploy/assets/rp-production.json +++ b/pkg/deploy/assets/rp-production.json @@ -184,9 +184,6 @@ "sshPublicKey": { "type": "string" }, - "storageAccountDomain": { - "type": "string" - }, "subscriptionResourceGroupName": { "type": "string" }, @@ -493,8 +490,7 @@ }, "diagnosticsProfile": { "bootDiagnostics": { - "enabled": true, - "storageUri": "[concat('https://', parameters('storageAccountDomain'), '/')]" + "enabled": true } }, "extensionProfile": { @@ -508,7 +504,7 @@ "autoUpgradeMinorVersion": true, "settings": {}, "protectedSettings": { - "script": "[base64(concat(base64ToString('c2V0IC1leAoK'),'ACRRESOURCEID=$(base64 -d \u003c\u003c\u003c''',base64(parameters('acrResourceId')),''')\n','ADMINAPICLIENTCERTCOMMONNAME=$(base64 -d \u003c\u003c\u003c''',base64(parameters('adminApiClientCertCommonName')),''')\n','ARMAPICLIENTCERTCOMMONNAME=$(base64 -d \u003c\u003c\u003c''',base64(parameters('armApiClientCertCommonName')),''')\n','ARMCLIENTID=$(base64 -d \u003c\u003c\u003c''',base64(parameters('armClientId')),''')\n','AZURECLOUDNAME=$(base64 -d \u003c\u003c\u003c''',base64(parameters('azureCloudName')),''')\n','AZURESECPACKQUALYSURL=$(base64 -d \u003c\u003c\u003c''',base64(parameters('azureSecPackQualysUrl')),''')\n','AZURESECPACKVSATENANTID=$(base64 -d \u003c\u003c\u003c''',base64(parameters('azureSecPackVSATenantId')),''')\n','CLUSTERMDMACCOUNT=$(base64 -d \u003c\u003c\u003c''',base64(parameters('clusterMdmAccount')),''')\n','CLUSTERMDSDACCOUNT=$(base64 -d \u003c\u003c\u003c''',base64(parameters('clusterMdsdAccount')),''')\n','CLUSTERMDSDCONFIGVERSION=$(base64 -d \u003c\u003c\u003c''',base64(parameters('clusterMdsdConfigVersion')),''')\n','CLUSTERMDSDNAMESPACE=$(base64 -d \u003c\u003c\u003c''',base64(parameters('clusterMdsdNamespace')),''')\n','CLUSTERPARENTDOMAINNAME=$(base64 -d \u003c\u003c\u003c''',base64(parameters('clusterParentDomainName')),''')\n','DATABASEACCOUNTNAME=$(base64 -d \u003c\u003c\u003c''',base64(parameters('databaseAccountName')),''')\n','DBTOKENCLIENTID=$(base64 -d \u003c\u003c\u003c''',base64(parameters('dbtokenClientId')),''')\n','FLUENTBITIMAGE=$(base64 -d \u003c\u003c\u003c''',base64(parameters('fluentbitImage')),''')\n','FPCLIENTID=$(base64 -d \u003c\u003c\u003c''',base64(parameters('fpClientId')),''')\n','FPSERVICEPRINCIPALID=$(base64 -d \u003c\u003c\u003c''',base64(parameters('fpServicePrincipalId')),''')\n','GATEWAYDOMAINS=$(base64 -d \u003c\u003c\u003c''',base64(parameters('gatewayDomains')),''')\n','GATEWAYRESOURCEGROUPNAME=$(base64 -d \u003c\u003c\u003c''',base64(parameters('gatewayResourceGroupName')),''')\n','GATEWAYSERVICEPRINCIPALID=$(base64 -d \u003c\u003c\u003c''',base64(parameters('gatewayServicePrincipalId')),''')\n','KEYVAULTDNSSUFFIX=$(base64 -d \u003c\u003c\u003c''',base64(parameters('keyvaultDNSSuffix')),''')\n','KEYVAULTPREFIX=$(base64 -d \u003c\u003c\u003c''',base64(parameters('keyvaultPrefix')),''')\n','MDMFRONTENDURL=$(base64 -d \u003c\u003c\u003c''',base64(parameters('mdmFrontendUrl')),''')\n','MDSDENVIRONMENT=$(base64 -d \u003c\u003c\u003c''',base64(parameters('mdsdEnvironment')),''')\n','PORTALACCESSGROUPIDS=$(base64 -d \u003c\u003c\u003c''',base64(parameters('portalAccessGroupIds')),''')\n','PORTALCLIENTID=$(base64 -d \u003c\u003c\u003c''',base64(parameters('portalClientId')),''')\n','PORTALELEVATEDGROUPIDS=$(base64 -d \u003c\u003c\u003c''',base64(parameters('portalElevatedGroupIds')),''')\n','RPFEATURES=$(base64 -d \u003c\u003c\u003c''',base64(parameters('rpFeatures')),''')\n','RPIMAGE=$(base64 -d \u003c\u003c\u003c''',base64(parameters('rpImage')),''')\n','RPMDMACCOUNT=$(base64 -d \u003c\u003c\u003c''',base64(parameters('rpMdmAccount')),''')\n','RPMDSDACCOUNT=$(base64 -d \u003c\u003c\u003c''',base64(parameters('rpMdsdAccount')),''')\n','RPMDSDCONFIGVERSION=$(base64 -d \u003c\u003c\u003c''',base64(parameters('rpMdsdConfigVersion')),''')\n','RPMDSDNAMESPACE=$(base64 -d \u003c\u003c\u003c''',base64(parameters('rpMdsdNamespace')),''')\n','RPPARENTDOMAINNAME=$(base64 -d \u003c\u003c\u003c''',base64(parameters('rpParentDomainName')),''')\n','CLUSTERSINSTALLVIAHIVE=$(base64 -d \u003c\u003c\u003c''',base64(parameters('clustersInstallViaHive')),''')\n','CLUSTERSADOPTBYHIVE=$(base64 -d \u003c\u003c\u003c''',base64(parameters('clustersAdoptByHive')),''')\n','CLUSTERDEFAULTINSTALLERPULLSPEC=$(base64 -d \u003c\u003c\u003c''',base64(parameters('clusterDefaultInstallerPullspec')),''')\n','USECHECKACCESS=$(base64 -d \u003c\u003c\u003c''',base64(parameters('useCheckAccess')),''')\n','ADMINAPICABUNDLE=''',parameters('adminApiCaBundle'),'''\n','ARMAPICABUNDLE=''',parameters('armApiCaBundle'),'''\n','MDMIMAGE=''/genevamdm:2.2024.328.1744-c5fb79-20240328t1935''\n','LOCATION=$(base64 -d \u003c\u003c\u003c''',base64(resourceGroup().location),''')\n','SUBSCRIPTIONID=$(base64 -d \u003c\u003c\u003c''',base64(subscription().subscriptionId),''')\n','RESOURCEGROUPNAME=$(base64 -d \u003c\u003c\u003c''',base64(resourceGroup().name),''')\n','\n',base64ToString('IyEvYmluL2Jhc2gKCmVjaG8gInNldHRpbmcgc3NoIHBhc3N3b3JkIGF1dGhlbnRpY2F0aW9uIgojIFdlIG5lZWQgdG8gbWFudWFsbHkgc2V0IFBhc3N3b3JkQXV0aGVudGljYXRpb24gdG8gdHJ1ZSBpbiBvcmRlciBmb3IgdGhlIFZNU1MgQWNjZXNzIEpJVCB0byB3b3JrCnNlZCAtaSAncy9QYXNzd29yZEF1dGhlbnRpY2F0aW9uIG5vL1Bhc3N3b3JkQXV0aGVudGljYXRpb24geWVzL2cnIC9ldGMvc3NoL3NzaGRfY29uZmlnCnN5c3RlbWN0bCByZWxvYWQgc3NoZC5zZXJ2aWNlCgojQWRkaW5nIHJldHJ5IGxvZ2ljIHRvIHl1bSBjb21tYW5kcyBpbiBvcmRlciB0byBhdm9pZCBzdGFsbGluZyBvdXQgb24gcmVzb3VyY2UgbG9ja3MKZWNobyAicnVubmluZyBSSFVJIGZpeCIKZm9yIGF0dGVtcHQgaW4gezEuLjYwfTsgZG8KICB5dW0gdXBkYXRlIC15IC0tZGlzYWJsZXJlcG89JyonIC0tZW5hYmxlcmVwbz0ncmh1aS1taWNyb3NvZnQtYXp1cmUqJyAmJiBicmVhawogIGlmIFtbICR7YXR0ZW1wdH0gLWx0IDYwIF1dOyB0aGVuIHNsZWVwIDMwOyBlbHNlIGV4aXQgMTsgZmkKZG9uZQoKZWNobyAicnVubmluZyB5dW0gdXBkYXRlIgpmb3IgYXR0ZW1wdCBpbiB7MS4uNjB9OyBkbwogIHl1bSAteSAteCBXQUxpbnV4QWdlbnQgLXggV0FMaW51eEFnZW50LXVkZXYgdXBkYXRlIC0tYWxsb3dlcmFzaW5nICYmIGJyZWFrCiAgaWYgW1sgJHthdHRlbXB0fSAtbHQgNjAgXV07IHRoZW4gc2xlZXAgMzA7IGVsc2UgZXhpdCAxOyBmaQpkb25lCgplY2hvICJleHRlbmRpbmcgcGFydGl0aW9uIHRhYmxlIgojIExpbnV4IGJsb2NrIGRldmljZXMgYXJlIGluY29uc2lzdGVudGx5IG5hbWVkCiMgaXQncyBkaWZmaWN1bHQgdG8gdGllIHRoZSBsdm0gcHYgdG8gdGhlIHBoeXNpY2FsIGRpc2sgdXNpbmcgL2Rldi9kaXNrIGZpbGVzLCB3aGljaCBpcyB3aHkgbHZzIGlzIHVzZWQgaGVyZQpwaHlzaWNhbERpc2s9IiQobHZzIC1vIGRldmljZXMgLWEgfCBoZWFkIC1uMiB8IHRhaWwgLW4xIHwgY3V0IC1kICcgJyAtZiAzIHwgY3V0IC1kIFwoIC1mIDEgfCB0ciAtZCAnWzpkaWdpdDpdJykiCmdyb3dwYXJ0ICIkcGh5c2ljYWxEaXNrIiAyCgplY2hvICJleHRlbmRpbmcgZmlsZXN5c3RlbXMiCmx2ZXh0ZW5kIC1sICsyMCVGUkVFIC9kZXYvcm9vdHZnL3Jvb3Rsdgp4ZnNfZ3Jvd2ZzIC8KCmx2ZXh0ZW5kIC1sICsxMDAlRlJFRSAvZGV2L3Jvb3R2Zy92YXJsdgp4ZnNfZ3Jvd2ZzIC92YXIKCmVjaG8gImltcG9ydGluZyBycG0gcmVwb3NpdG9yaWVzIgpycG0gLS1pbXBvcnQgaHR0cHM6Ly9kbC5mZWRvcmFwcm9qZWN0Lm9yZy9wdWIvZXBlbC9SUE0tR1BHLUtFWS1FUEVMLTgKcnBtIC0taW1wb3J0IGh0dHBzOi8vcGFja2FnZXMubWljcm9zb2Z0LmNvbS9rZXlzL21pY3Jvc29mdC5hc2MKCmZvciBhdHRlbXB0IGluIHsxLi42MH07IGRvCiAgeXVtIC15IGluc3RhbGwgaHR0cHM6Ly9kbC5mZWRvcmFwcm9qZWN0Lm9yZy9wdWIvZXBlbC9lcGVsLXJlbGVhc2UtbGF0ZXN0LTgubm9hcmNoLnJwbSAmJiBicmVhawogIGlmIFtbICR7YXR0ZW1wdH0gLWx0IDYwIF1dOyB0aGVuIHNsZWVwIDMwOyBlbHNlIGV4aXQgMTsgZmkKZG9uZQoKZWNobyAiY29uZmlndXJpbmcgbG9ncm90YXRlIgpjYXQgPi9ldGMvbG9ncm90YXRlLmNvbmYgPDwnRU9GJwojIHNlZSAibWFuIGxvZ3JvdGF0ZSIgZm9yIGRldGFpbHMKIyByb3RhdGUgbG9nIGZpbGVzIHdlZWtseQp3ZWVrbHkKCiMga2VlcCAyIHdlZWtzIHdvcnRoIG9mIGJhY2tsb2dzCnJvdGF0ZSAyCgojIGNyZWF0ZSBuZXcgKGVtcHR5KSBsb2cgZmlsZXMgYWZ0ZXIgcm90YXRpbmcgb2xkIG9uZXMKY3JlYXRlCgojIHVzZSBkYXRlIGFzIGEgc3VmZml4IG9mIHRoZSByb3RhdGVkIGZpbGUKZGF0ZWV4dAoKIyB1bmNvbW1lbnQgdGhpcyBpZiB5b3Ugd2FudCB5b3VyIGxvZyBmaWxlcyBjb21wcmVzc2VkCmNvbXByZXNzCgojIFJQTSBwYWNrYWdlcyBkcm9wIGxvZyByb3RhdGlvbiBpbmZvcm1hdGlvbiBpbnRvIHRoaXMgZGlyZWN0b3J5CmluY2x1ZGUgL2V0Yy9sb2dyb3RhdGUuZAoKIyBubyBwYWNrYWdlcyBvd24gd3RtcCBhbmQgYnRtcCAtLSB3ZSdsbCByb3RhdGUgdGhlbSBoZXJlCi92YXIvbG9nL3d0bXAgewogICAgbW9udGhseQogICAgY3JlYXRlIDA2NjQgcm9vdCB1dG1wCiAgICAgICAgbWluc2l6ZSAxTQogICAgcm90YXRlIDEKfQoKL3Zhci9sb2cvYnRtcCB7CiAgICBtaXNzaW5nb2sKICAgIG1vbnRobHkKICAgIGNyZWF0ZSAwNjAwIHJvb3QgdXRtcAogICAgcm90YXRlIDEKfQpFT0YKCmVjaG8gImNvbmZpZ3VyaW5nIHl1bSByZXBvc2l0b3J5IGFuZCBydW5uaW5nIHl1bSB1cGRhdGUiCmNhdCA+L2V0Yy95dW0ucmVwb3MuZC9henVyZS5yZXBvIDw8J0VPRicKW2F6dXJlLWNsaV0KbmFtZT1henVyZS1jbGkKYmFzZXVybD1odHRwczovL3BhY2thZ2VzLm1pY3Jvc29mdC5jb20veXVtcmVwb3MvYXp1cmUtY2xpCmVuYWJsZWQ9eWVzCmdwZ2NoZWNrPXllcwoKW2F6dXJlY29yZV0KbmFtZT1henVyZWNvcmUKYmFzZXVybD1odHRwczovL3BhY2thZ2VzLm1pY3Jvc29mdC5jb20veXVtcmVwb3MvYXp1cmVjb3JlCmVuYWJsZWQ9eWVzCmdwZ2NoZWNrPW5vCkVPRgoKc2VtYW5hZ2UgZmNvbnRleHQgLWEgLXQgdmFyX2xvZ190ICIvdmFyL2xvZy9qb3VybmFsKC8uKik/Igpta2RpciAtcCAvdmFyL2xvZy9qb3VybmFsCgpmb3IgYXR0ZW1wdCBpbiB7MS4uNjB9OyBkbwp5dW0gLXkgaW5zdGFsbCBjbGFtYXYgYXpzZWMtY2xhbWF2IGF6c2VjLW1vbml0b3IgYXp1cmUtY2xpIGF6dXJlLW1kc2QgYXp1cmUtc2VjdXJpdHkgcG9kbWFuIHBvZG1hbi1kb2NrZXIgb3BlbnNzbC1wZXJsIHB5dGhvbjMgJiYgYnJlYWsKICAjIGhhY2sgLSB3ZSBhcmUgaW5zdGFsbGluZyBweXRob24zIG9uIGhvc3RzIGR1ZSB0byBhbiBpc3N1ZSB3aXRoIEF6dXJlIExpbnV4IEV4dGVuc2lvbnMgaHR0cHM6Ly9naXRodWIuY29tL0F6dXJlL2F6dXJlLWxpbnV4LWV4dGVuc2lvbnMvcHVsbC8xNTA1CiAgaWYgW1sgJHthdHRlbXB0fSAtbHQgNjAgXV07IHRoZW4gc2xlZXAgMzA7IGVsc2UgZXhpdCAxOyBmaQpkb25lCgojIGh0dHBzOi8vYWNjZXNzLnJlZGhhdC5jb20vc2VjdXJpdHkvY3ZlL2N2ZS0yMDIwLTEzNDAxCmVjaG8gImFwcGx5aW5nIGZpcmV3YWxsIHJ1bGVzIgpjYXQgPi9ldGMvc3lzY3RsLmQvMDItZGlzYWJsZS1hY2NlcHQtcmEuY29uZiA8PCdFT0YnCm5ldC5pcHY2LmNvbmYuYWxsLmFjY2VwdF9yYT0wCkVPRgoKY2F0ID4vZXRjL3N5c2N0bC5kLzAxLWRpc2FibGUtY29yZS5jb25mIDw8J0VPRicKa2VybmVsLmNvcmVfcGF0dGVybiA9IHwvYmluL3RydWUKRU9GCnN5c2N0bCAtLXN5c3RlbQoKZmlyZXdhbGwtY21kIC0tYWRkLXBvcnQ9NDQzL3RjcCAtLXBlcm1hbmVudApmaXJld2FsbC1jbWQgLS1hZGQtcG9ydD00NDQvdGNwIC0tcGVybWFuZW50CmZpcmV3YWxsLWNtZCAtLWFkZC1wb3J0PTQ0NS90Y3AgLS1wZXJtYW5lbnQKZmlyZXdhbGwtY21kIC0tYWRkLXBvcnQ9MjIyMi90Y3AgLS1wZXJtYW5lbnQKCmV4cG9ydCBBWlVSRV9DTE9VRF9OQU1FPSRBWlVSRUNMT1VETkFNRQoKZWNobyAibG9nZ2luZyBpbnRvIHByb2QgYWNyIgpheiBsb2dpbiAtaSAtLWFsbG93LW5vLXN1YnNjcmlwdGlvbnMKCiMgU3VwcHJlc3MgZW11bGF0aW9uIG91dHB1dCBmb3IgcG9kbWFuIGluc3RlYWQgb2YgZG9ja2VyIGZvciBheiBhY3IgY29tcGF0YWJpbGl0eQpta2RpciAtcCAvZXRjL2NvbnRhaW5lcnMvCnRvdWNoIC9ldGMvY29udGFpbmVycy9ub2RvY2tlcgoKbWtkaXIgLXAgL3Jvb3QvLmRvY2tlcgpSRUdJU1RSWV9BVVRIX0ZJTEU9L3Jvb3QvLmRvY2tlci9jb25maWcuanNvbiBheiBhY3IgbG9naW4gLS1uYW1lICIkKHNlZCAtZSAnc3wuKi98fCcgPDw8IiRBQ1JSRVNPVVJDRUlEIikiCgpNRE1JTUFHRT0iJHtSUElNQUdFJSUvKn0vJHtNRE1JTUFHRSMjKi99Igpkb2NrZXIgcHVsbCAiJE1ETUlNQUdFIgpkb2NrZXIgcHVsbCAiJFJQSU1BR0UiCmRvY2tlciBwdWxsICIkRkxVRU5UQklUSU1BR0UiCgpheiBsb2dvdXQKCmVjaG8gImNvbmZpZ3VyaW5nIGZsdWVudGJpdCBzZXJ2aWNlIgpta2RpciAtcCAvZXRjL2ZsdWVudGJpdC8KbWtkaXIgLXAgL3Zhci9saWIvZmx1ZW50CgpjYXQgPi9ldGMvZmx1ZW50Yml0L2ZsdWVudGJpdC5jb25mIDw8J0VPRicKW0lOUFVUXQoJTmFtZSBzeXN0ZW1kCglUYWcgam91cm5hbGQKCVN5c3RlbWRfRmlsdGVyIF9DT01NPWFybwoJREIgL3Zhci9saWIvZmx1ZW50L2pvdXJuYWxkYgoKW0ZJTFRFUl0KCU5hbWUgbW9kaWZ5CglNYXRjaCBqb3VybmFsZAoJUmVtb3ZlX3dpbGRjYXJkIF8KCVJlbW92ZSBUSU1FU1RBTVAKCltGSUxURVJdCglOYW1lIHJld3JpdGVfdGFnCglNYXRjaCBqb3VybmFsZAoJUnVsZSAkTE9HS0lORCBhc3luY3FvcyBhc3luY3FvcyB0cnVlCgpbRklMVEVSXQoJTmFtZSBtb2RpZnkKCU1hdGNoIGFzeW5jcW9zCglSZW1vdmUgQ0xJRU5UX1BSSU5DSVBBTF9OQU1FCglSZW1vdmUgRklMRQoJUmVtb3ZlIENPTVBPTkVOVAoKW0ZJTFRFUl0KCU5hbWUgcmV3cml0ZV90YWcKCU1hdGNoIGpvdXJuYWxkCglSdWxlICRMT0dLSU5EIGlmeGF1ZGl0IGlmeGF1ZGl0IGZhbHNlCgpbT1VUUFVUXQoJTmFtZSBmb3J3YXJkCglNYXRjaCAqCglQb3J0IDI5MjMwCkVPRgoKZWNobyAiRkxVRU5UQklUSU1BR0U9JEZMVUVOVEJJVElNQUdFIiA+L2V0Yy9zeXNjb25maWcvZmx1ZW50Yml0CgpjYXQgPi9ldGMvc3lzdGVtZC9zeXN0ZW0vZmx1ZW50Yml0LnNlcnZpY2UgPDwnRU9GJwpbVW5pdF0KQWZ0ZXI9bmV0d29yay1vbmxpbmUudGFyZ2V0CldhbnRzPW5ldHdvcmstb25saW5lLnRhcmdldApTdGFydExpbWl0SW50ZXJ2YWxTZWM9MAoKW1NlcnZpY2VdClJlc3RhcnRTZWM9MXMKRW52aXJvbm1lbnRGaWxlPS9ldGMvc3lzY29uZmlnL2ZsdWVudGJpdApFeGVjU3RhcnRQcmU9LS91c3IvYmluL2RvY2tlciBybSAtZiAlTgpFeGVjU3RhcnQ9L3Vzci9iaW4vZG9ja2VyIHJ1biBcCiAgLS1zZWN1cml0eS1vcHQgbGFiZWw9ZGlzYWJsZSBcCiAgLS1lbnRyeXBvaW50IC9vcHQvdGQtYWdlbnQtYml0L2Jpbi90ZC1hZ2VudC1iaXQgXAogIC0tbmV0PWhvc3QgXAogIC0taG9zdG5hbWUgJUggXAogIC0tbmFtZSAlTiBcCiAgLS1ybSBcCiAgLS1jYXAtZHJvcCBuZXRfcmF3IFwKICAtdiAvZXRjL2ZsdWVudGJpdC9mbHVlbnRiaXQuY29uZjovZXRjL2ZsdWVudGJpdC9mbHVlbnRiaXQuY29uZiBcCiAgLXYgL3Zhci9saWIvZmx1ZW50Oi92YXIvbGliL2ZsdWVudDp6IFwKICAtdiAvdmFyL2xvZy9qb3VybmFsOi92YXIvbG9nL2pvdXJuYWw6cm8gXAogIC12IC9ldGMvbWFjaGluZS1pZDovZXRjL21hY2hpbmUtaWQ6cm8gXAogICRGTFVFTlRCSVRJTUFHRSBcCiAgLWMgL2V0Yy9mbHVlbnRiaXQvZmx1ZW50Yml0LmNvbmYKCkV4ZWNTdG9wPS91c3IvYmluL2RvY2tlciBzdG9wICVOClJlc3RhcnQ9YWx3YXlzClJlc3RhcnRTZWM9NQpTdGFydExpbWl0SW50ZXJ2YWw9MAoKW0luc3RhbGxdCldhbnRlZEJ5PW11bHRpLXVzZXIudGFyZ2V0CkVPRgoKbWtkaXIgL2V0Yy9hcm8tcnAKYmFzZTY0IC1kIDw8PCIkQURNSU5BUElDQUJVTkRMRSIgPi9ldGMvYXJvLXJwL2FkbWluLWNhLWJ1bmRsZS5wZW0KaWYgW1sgLW4gIiRBUk1BUElDQUJVTkRMRSIgXV07IHRoZW4KICBiYXNlNjQgLWQgPDw8IiRBUk1BUElDQUJVTkRMRSIgPi9ldGMvYXJvLXJwL2FybS1jYS1idW5kbGUucGVtCmZpCmNob3duIC1SIDEwMDA6MTAwMCAvZXRjL2Fyby1ycAoKZWNobyAiY29uZmlndXJpbmcgbWRtIHNlcnZpY2UiCmNhdCA+L2V0Yy9zeXNjb25maWcvbWRtIDw8RU9GCk1ETUZST05URU5EVVJMPSckTURNRlJPTlRFTkRVUkwnCk1ETUlNQUdFPSckTURNSU1BR0UnCk1ETVNPVVJDRUVOVklST05NRU5UPSckTE9DQVRJT04nCk1ETVNPVVJDRVJPTEU9cnAKTURNU09VUkNFUk9MRUlOU1RBTkNFPSckKGhvc3RuYW1lKScKRU9GCgpta2RpciAvdmFyL2V0dwpjYXQgPi9ldGMvc3lzdGVtZC9zeXN0ZW0vbWRtLnNlcnZpY2UgPDwnRU9GJwpbVW5pdF0KQWZ0ZXI9bmV0d29yay1vbmxpbmUudGFyZ2V0CldhbnRzPW5ldHdvcmstb25saW5lLnRhcmdldAoKW1NlcnZpY2VdCkVudmlyb25tZW50RmlsZT0vZXRjL3N5c2NvbmZpZy9tZG0KRXhlY1N0YXJ0UHJlPS0vdXNyL2Jpbi9kb2NrZXIgcm0gLWYgJU4KRXhlY1N0YXJ0PS91c3IvYmluL2RvY2tlciBydW4gXAogIC0tZW50cnlwb2ludCAvdXNyL3NiaW4vTWV0cmljc0V4dGVuc2lvbiBcCiAgLS1ob3N0bmFtZSAlSCBcCiAgLS1uYW1lICVOIFwKICAtLXJtIFwKICAtLWNhcC1kcm9wIG5ldF9yYXcgXAogIC1tIDJnIFwKICAtdiAvZXRjL21kbS5wZW06L2V0Yy9tZG0ucGVtIFwKICAtdiAvdmFyL2V0dzovdmFyL2V0dzp6IFwKICAkTURNSU1BR0UgXAogIC1DZXJ0RmlsZSAvZXRjL21kbS5wZW0gXAogIC1Gcm9udEVuZFVybCAkTURNRlJPTlRFTkRVUkwgXAogIC1Mb2dnZXIgQ29uc29sZSBcCiAgLUxvZ0xldmVsIFdhcm5pbmcgXAogIC1Qcml2YXRlS2V5RmlsZSAvZXRjL21kbS5wZW0gXAogIC1Tb3VyY2VFbnZpcm9ubWVudCAkTURNU09VUkNFRU5WSVJPTk1FTlQgXAogIC1Tb3VyY2VSb2xlICRNRE1TT1VSQ0VST0xFIFwKICAtU291cmNlUm9sZUluc3RhbmNlICRNRE1TT1VSQ0VST0xFSU5TVEFOQ0UKRXhlY1N0b3A9L3Vzci9iaW4vZG9ja2VyIHN0b3AgJU4KUmVzdGFydD1hbHdheXMKUmVzdGFydFNlYz0xClN0YXJ0TGltaXRJbnRlcnZhbD0wCgpbSW5zdGFsbF0KV2FudGVkQnk9bXVsdGktdXNlci50YXJnZXQKRU9GCgplY2hvICJjb25maWd1cmluZyBhcm8tcnAgc2VydmljZSIKY2F0ID4vZXRjL3N5c2NvbmZpZy9hcm8tcnAgPDxFT0YKQUNSX1JFU09VUkNFX0lEPSckQUNSUkVTT1VSQ0VJRCcKQURNSU5fQVBJX0NMSUVOVF9DRVJUX0NPTU1PTl9OQU1FPSckQURNSU5BUElDTElFTlRDRVJUQ09NTU9OTkFNRScKQVJNX0FQSV9DTElFTlRfQ0VSVF9DT01NT05fTkFNRT0nJEFSTUFQSUNMSUVOVENFUlRDT01NT05OQU1FJwpBWlVSRV9BUk1fQ0xJRU5UX0lEPSckQVJNQ0xJRU5USUQnCkFaVVJFX0ZQX0NMSUVOVF9JRD0nJEZQQ0xJRU5USUQnCkFaVVJFX0ZQX1NFUlZJQ0VfUFJJTkNJUEFMX0lEPSckRlBTRVJWSUNFUFJJTkNJUEFMSUQnCkNMVVNURVJfTURNX0FDQ09VTlQ9JyRDTFVTVEVSTURNQUNDT1VOVCcKQ0xVU1RFUl9NRE1fTkFNRVNQQUNFPVJQCkNMVVNURVJfTURTRF9BQ0NPVU5UPSckQ0xVU1RFUk1EU0RBQ0NPVU5UJwpDTFVTVEVSX01EU0RfQ09ORklHX1ZFUlNJT049JyRDTFVTVEVSTURTRENPTkZJR1ZFUlNJT04nCkNMVVNURVJfTURTRF9OQU1FU1BBQ0U9JyRDTFVTVEVSTURTRE5BTUVTUEFDRScKREFUQUJBU0VfQUNDT1VOVF9OQU1FPSckREFUQUJBU0VBQ0NPVU5UTkFNRScKRE9NQUlOX05BTUU9JyRMT0NBVElPTi4kQ0xVU1RFUlBBUkVOVERPTUFJTk5BTUUnCkdBVEVXQVlfRE9NQUlOUz0nJEdBVEVXQVlET01BSU5TJwpHQVRFV0FZX1JFU09VUkNFR1JPVVA9JyRHQVRFV0FZUkVTT1VSQ0VHUk9VUE5BTUUnCktFWVZBVUxUX1BSRUZJWD0nJEtFWVZBVUxUUFJFRklYJwpNRE1fQUNDT1VOVD0nJFJQTURNQUNDT1VOVCcKTURNX05BTUVTUEFDRT1SUApNRFNEX0VOVklST05NRU5UPSckTURTREVOVklST05NRU5UJwpSUF9GRUFUVVJFUz0nJFJQRkVBVFVSRVMnClJQSU1BR0U9JyRSUElNQUdFJwpBUk9fSU5TVEFMTF9WSUFfSElWRT0nJENMVVNURVJTSU5TVEFMTFZJQUhJVkUnCkFST19ISVZFX0RFRkFVTFRfSU5TVEFMTEVSX1BVTExTUEVDPSckQ0xVU1RFUkRFRkFVTFRJTlNUQUxMRVJQVUxMU1BFQycKQVJPX0FET1BUX0JZX0hJVkU9JyRDTFVTVEVSU0FET1BUQllISVZFJwpVU0VfQ0hFQ0tBQ0NFU1M9JyRVU0VDSEVDS0FDQ0VTUycKRU9GCgpjYXQgPi9ldGMvc3lzdGVtZC9zeXN0ZW0vYXJvLXJwLnNlcnZpY2UgPDwnRU9GJwpbVW5pdF0KQWZ0ZXI9bmV0d29yay1vbmxpbmUudGFyZ2V0CldhbnRzPW5ldHdvcmstb25saW5lLnRhcmdldAoKW1NlcnZpY2VdCkVudmlyb25tZW50RmlsZT0vZXRjL3N5c2NvbmZpZy9hcm8tcnAKRXhlY1N0YXJ0UHJlPS0vdXNyL2Jpbi9kb2NrZXIgcm0gLWYgJU4KRXhlY1N0YXJ0PS91c3IvYmluL2RvY2tlciBydW4gXAogIC0taG9zdG5hbWUgJUggXAogIC0tbmFtZSAlTiBcCiAgLS1ybSBcCiAgLS1jYXAtZHJvcCBuZXRfcmF3IFwKICAtZSBBQ1JfUkVTT1VSQ0VfSUQgXAogIC1lIEFETUlOX0FQSV9DTElFTlRfQ0VSVF9DT01NT05fTkFNRSBcCiAgLWUgQVJNX0FQSV9DTElFTlRfQ0VSVF9DT01NT05fTkFNRSBcCiAgLWUgQVpVUkVfQVJNX0NMSUVOVF9JRCBcCiAgLWUgQVpVUkVfRlBfQ0xJRU5UX0lEIFwKICAtZSBDTFVTVEVSX01ETV9BQ0NPVU5UIFwKICAtZSBDTFVTVEVSX01ETV9OQU1FU1BBQ0UgXAogIC1lIENMVVNURVJfTURTRF9BQ0NPVU5UIFwKICAtZSBDTFVTVEVSX01EU0RfQ09ORklHX1ZFUlNJT04gXAogIC1lIENMVVNURVJfTURTRF9OQU1FU1BBQ0UgXAogIC1lIERBVEFCQVNFX0FDQ09VTlRfTkFNRSBcCiAgLWUgRE9NQUlOX05BTUUgXAogIC1lIEdBVEVXQVlfRE9NQUlOUyBcCiAgLWUgR0FURVdBWV9SRVNPVVJDRUdST1VQIFwKICAtZSBLRVlWQVVMVF9QUkVGSVggXAogIC1lIE1ETV9BQ0NPVU5UIFwKICAtZSBNRE1fTkFNRVNQQUNFIFwKICAtZSBNRFNEX0VOVklST05NRU5UIFwKICAtZSBSUF9GRUFUVVJFUyBcCiAgLWUgQVJPX0lOU1RBTExfVklBX0hJVkUgXAogIC1lIEFST19ISVZFX0RFRkFVTFRfSU5TVEFMTEVSX1BVTExTUEVDIFwKICAtZSBBUk9fQURPUFRfQllfSElWRSBcCiAgLWUgVVNFX0NIRUNLQUNDRVNTIFwKICAtbSAyZyBcCiAgLXAgNDQzOjg0NDMgXAogIC12IC9ldGMvYXJvLXJwOi9ldGMvYXJvLXJwIFwKICAtdiAvcnVuL3N5c3RlbWQvam91cm5hbDovcnVuL3N5c3RlbWQvam91cm5hbCBcCiAgLXYgL3Zhci9ldHc6L3Zhci9ldHc6eiBcCiAgJFJQSU1BR0UgXAogIHJwCkV4ZWNTdG9wPS91c3IvYmluL2RvY2tlciBzdG9wIC10IDM2MDAgJU4KVGltZW91dFN0b3BTZWM9MzYwMApSZXN0YXJ0PWFsd2F5cwpSZXN0YXJ0U2VjPTEKU3RhcnRMaW1pdEludGVydmFsPTAKCltJbnN0YWxsXQpXYW50ZWRCeT1tdWx0aS11c2VyLnRhcmdldApFT0YKCmVjaG8gImNvbmZpZ3VyaW5nIGFyby1kYnRva2VuIHNlcnZpY2UiCmNhdCA+L2V0Yy9zeXNjb25maWcvYXJvLWRidG9rZW4gPDxFT0YKREFUQUJBU0VfQUNDT1VOVF9OQU1FPSckREFUQUJBU0VBQ0NPVU5UTkFNRScKQVpVUkVfREJUT0tFTl9DTElFTlRfSUQ9JyREQlRPS0VOQ0xJRU5USUQnCkFaVVJFX0dBVEVXQVlfU0VSVklDRV9QUklOQ0lQQUxfSUQ9JyRHQVRFV0FZU0VSVklDRVBSSU5DSVBBTElEJwpLRVlWQVVMVF9QUkVGSVg9JyRLRVlWQVVMVFBSRUZJWCcKTURNX0FDQ09VTlQ9JyRSUE1ETUFDQ09VTlQnCk1ETV9OQU1FU1BBQ0U9REJUb2tlbgpSUElNQUdFPSckUlBJTUFHRScKRU9GCgpjYXQgPi9ldGMvc3lzdGVtZC9zeXN0ZW0vYXJvLWRidG9rZW4uc2VydmljZSA8PCdFT0YnCltVbml0XQpBZnRlcj1uZXR3b3JrLW9ubGluZS50YXJnZXQKV2FudHM9bmV0d29yay1vbmxpbmUudGFyZ2V0CgpbU2VydmljZV0KRW52aXJvbm1lbnRGaWxlPS9ldGMvc3lzY29uZmlnL2Fyby1kYnRva2VuCkV4ZWNTdGFydFByZT0tL3Vzci9iaW4vZG9ja2VyIHJtIC1mICVOCkV4ZWNTdGFydD0vdXNyL2Jpbi9kb2NrZXIgcnVuIFwKICAtLWhvc3RuYW1lICVIIFwKICAtLW5hbWUgJU4gXAogIC0tcm0gXAogIC0tY2FwLWRyb3AgbmV0X3JhdyBcCiAgLWUgQVpVUkVfR0FURVdBWV9TRVJWSUNFX1BSSU5DSVBBTF9JRCBcCiAgLWUgREFUQUJBU0VfQUNDT1VOVF9OQU1FIFwKICAtZSBBWlVSRV9EQlRPS0VOX0NMSUVOVF9JRCBcCiAgLWUgS0VZVkFVTFRfUFJFRklYIFwKICAtZSBNRE1fQUNDT1VOVCBcCiAgLWUgTURNX05BTUVTUEFDRSBcCiAgLW0gMmcgXAogIC1wIDQ0NTo4NDQ1IFwKICAtdiAvcnVuL3N5c3RlbWQvam91cm5hbDovcnVuL3N5c3RlbWQvam91cm5hbCBcCiAgLXYgL3Zhci9ldHc6L3Zhci9ldHc6eiBcCiAgJFJQSU1BR0UgXAogIGRidG9rZW4KRXhlY1N0b3A9L3Vzci9iaW4vZG9ja2VyIHN0b3AgLXQgMzYwMCAlTgpUaW1lb3V0U3RvcFNlYz0zNjAwClJlc3RhcnQ9YWx3YXlzClJlc3RhcnRTZWM9MQpTdGFydExpbWl0SW50ZXJ2YWw9MAoKW0luc3RhbGxdCldhbnRlZEJ5PW11bHRpLXVzZXIudGFyZ2V0CkVPRgoKIyBET01BSU5fTkFNRSwgQ0xVU1RFUl9NRFNEX0FDQ09VTlQsIENMVVNURVJfTURTRF9DT05GSUdfVkVSU0lPTiwgR0FURVdBWV9ET01BSU5TLCBHQVRFV0FZX1JFU09VUkNFR1JPVVAsIE1EU0RfRU5WSVJPTk1FTlQgQ0xVU1RFUl9NRFNEX05BTUVTUEFDRQojIGFyZSBub3QgdXNlZCwgYnV0IGNhbid0IGVhc2lseSBiZSByZWZhY3RvcmVkIG91dC4gU2hvdWxkIGJlIHJldmlzaXRlZCBpbiB0aGUgZnV0dXJlLgplY2hvICJjb25maWd1cmluZyBhcm8tbW9uaXRvciBzZXJ2aWNlIgpjYXQgPi9ldGMvc3lzY29uZmlnL2Fyby1tb25pdG9yIDw8RU9GCkFaVVJFX0ZQX0NMSUVOVF9JRD0nJEZQQ0xJRU5USUQnCkRPTUFJTl9OQU1FPSckTE9DQVRJT04uJENMVVNURVJQQVJFTlRET01BSU5OQU1FJwpDTFVTVEVSX01EU0RfQUNDT1VOVD0nJENMVVNURVJNRFNEQUNDT1VOVCcKQ0xVU1RFUl9NRFNEX0NPTkZJR19WRVJTSU9OPSckQ0xVU1RFUk1EU0RDT05GSUdWRVJTSU9OJwpHQVRFV0FZX0RPTUFJTlM9JyRHQVRFV0FZRE9NQUlOUycKR0FURVdBWV9SRVNPVVJDRUdST1VQPSckR0FURVdBWVJFU09VUkNFR1JPVVBOQU1FJwpNRFNEX0VOVklST05NRU5UPSckTURTREVOVklST05NRU5UJwpDTFVTVEVSX01EU0RfTkFNRVNQQUNFPSckQ0xVU1RFUk1EU0ROQU1FU1BBQ0UnCkNMVVNURVJfTURNX0FDQ09VTlQ9JyRDTFVTVEVSTURNQUNDT1VOVCcKQ0xVU1RFUl9NRE1fTkFNRVNQQUNFPUJCTQpEQVRBQkFTRV9BQ0NPVU5UX05BTUU9JyREQVRBQkFTRUFDQ09VTlROQU1FJwpLRVlWQVVMVF9QUkVGSVg9JyRLRVlWQVVMVFBSRUZJWCcKTURNX0FDQ09VTlQ9JyRSUE1ETUFDQ09VTlQnCk1ETV9OQU1FU1BBQ0U9QkJNClJQSU1BR0U9JyRSUElNQUdFJwpFT0YKCmNhdCA+L2V0Yy9zeXN0ZW1kL3N5c3RlbS9hcm8tbW9uaXRvci5zZXJ2aWNlIDw8J0VPRicKW1VuaXRdCkFmdGVyPW5ldHdvcmstb25saW5lLnRhcmdldApXYW50cz1uZXR3b3JrLW9ubGluZS50YXJnZXQKCltTZXJ2aWNlXQpFbnZpcm9ubWVudEZpbGU9L2V0Yy9zeXNjb25maWcvYXJvLW1vbml0b3IKRXhlY1N0YXJ0UHJlPS0vdXNyL2Jpbi9kb2NrZXIgcm0gLWYgJU4KRXhlY1N0YXJ0PS91c3IvYmluL2RvY2tlciBydW4gXAogIC0taG9zdG5hbWUgJUggXAogIC0tbmFtZSAlTiBcCiAgLS1ybSBcCiAgLS1jYXAtZHJvcCBuZXRfcmF3IFwKICAtZSBBWlVSRV9GUF9DTElFTlRfSUQgXAogIC1lIERPTUFJTl9OQU1FIFwKICAtZSBDTFVTVEVSX01EU0RfQUNDT1VOVCBcCiAgLWUgQ0xVU1RFUl9NRFNEX0NPTkZJR19WRVJTSU9OIFwKICAtZSBHQVRFV0FZX0RPTUFJTlMgXAogIC1lIEdBVEVXQVlfUkVTT1VSQ0VHUk9VUCBcCiAgLWUgTURTRF9FTlZJUk9OTUVOVCBcCiAgLWUgQ0xVU1RFUl9NRFNEX05BTUVTUEFDRSBcCiAgLWUgQ0xVU1RFUl9NRE1fQUNDT1VOVCBcCiAgLWUgQ0xVU1RFUl9NRE1fTkFNRVNQQUNFIFwKICAtZSBEQVRBQkFTRV9BQ0NPVU5UX05BTUUgXAogIC1lIEtFWVZBVUxUX1BSRUZJWCBcCiAgLWUgTURNX0FDQ09VTlQgXAogIC1lIE1ETV9OQU1FU1BBQ0UgXAogIC1tIDIuNWcgXAogIC12IC9ydW4vc3lzdGVtZC9qb3VybmFsOi9ydW4vc3lzdGVtZC9qb3VybmFsIFwKICAtdiAvdmFyL2V0dzovdmFyL2V0dzp6IFwKICAkUlBJTUFHRSBcCiAgbW9uaXRvcgpSZXN0YXJ0PWFsd2F5cwpSZXN0YXJ0U2VjPTEKU3RhcnRMaW1pdEludGVydmFsPTAKCltJbnN0YWxsXQpXYW50ZWRCeT1tdWx0aS11c2VyLnRhcmdldApFT0YKCmVjaG8gImNvbmZpZ3VyaW5nIGFyby1wb3J0YWwgc2VydmljZSIKY2F0ID4vZXRjL3N5c2NvbmZpZy9hcm8tcG9ydGFsIDw8RU9GCkFaVVJFX1BPUlRBTF9BQ0NFU1NfR1JPVVBfSURTPSckUE9SVEFMQUNDRVNTR1JPVVBJRFMnCkFaVVJFX1BPUlRBTF9DTElFTlRfSUQ9JyRQT1JUQUxDTElFTlRJRCcKQVpVUkVfUE9SVEFMX0VMRVZBVEVEX0dST1VQX0lEUz0nJFBPUlRBTEVMRVZBVEVER1JPVVBJRFMnCkRBVEFCQVNFX0FDQ09VTlRfTkFNRT0nJERBVEFCQVNFQUNDT1VOVE5BTUUnCktFWVZBVUxUX1BSRUZJWD0nJEtFWVZBVUxUUFJFRklYJwpNRE1fQUNDT1VOVD0nJFJQTURNQUNDT1VOVCcKTURNX05BTUVTUEFDRT1Qb3J0YWwKUE9SVEFMX0hPU1ROQU1FPSckTE9DQVRJT04uYWRtaW4uJFJQUEFSRU5URE9NQUlOTkFNRScKUlBJTUFHRT0nJFJQSU1BR0UnCkVPRgoKY2F0ID4vZXRjL3N5c3RlbWQvc3lzdGVtL2Fyby1wb3J0YWwuc2VydmljZSA8PCdFT0YnCltVbml0XQpBZnRlcj1uZXR3b3JrLW9ubGluZS50YXJnZXQKV2FudHM9bmV0d29yay1vbmxpbmUudGFyZ2V0ClN0YXJ0TGltaXRJbnRlcnZhbD0wCgpbU2VydmljZV0KRW52aXJvbm1lbnRGaWxlPS9ldGMvc3lzY29uZmlnL2Fyby1wb3J0YWwKRXhlY1N0YXJ0UHJlPS0vdXNyL2Jpbi9kb2NrZXIgcm0gLWYgJU4KRXhlY1N0YXJ0PS91c3IvYmluL2RvY2tlciBydW4gXAogIC0taG9zdG5hbWUgJUggXAogIC0tbmFtZSAlTiBcCiAgLS1ybSBcCiAgLS1jYXAtZHJvcCBuZXRfcmF3IFwKICAtZSBBWlVSRV9QT1JUQUxfQUNDRVNTX0dST1VQX0lEUyBcCiAgLWUgQVpVUkVfUE9SVEFMX0NMSUVOVF9JRCBcCiAgLWUgQVpVUkVfUE9SVEFMX0VMRVZBVEVEX0dST1VQX0lEUyBcCiAgLWUgREFUQUJBU0VfQUNDT1VOVF9OQU1FIFwKICAtZSBLRVlWQVVMVF9QUkVGSVggXAogIC1lIE1ETV9BQ0NPVU5UIFwKICAtZSBNRE1fTkFNRVNQQUNFIFwKICAtZSBQT1JUQUxfSE9TVE5BTUUgXAogIC1tIDJnIFwKICAtcCA0NDQ6ODQ0NCBcCiAgLXAgMjIyMjoyMjIyIFwKICAtdiAvcnVuL3N5c3RlbWQvam91cm5hbDovcnVuL3N5c3RlbWQvam91cm5hbCBcCiAgLXYgL3Zhci9ldHc6L3Zhci9ldHc6eiBcCiAgJFJQSU1BR0UgXAogIHBvcnRhbApSZXN0YXJ0PWFsd2F5cwpSZXN0YXJ0U2VjPTEKCltJbnN0YWxsXQpXYW50ZWRCeT1tdWx0aS11c2VyLnRhcmdldApFT0YKCmVjaG8gImNvbmZpZ3VyaW5nIG1kc2QgYW5kIG1kbSBzZXJ2aWNlcyIKY2hjb24gLVIgc3lzdGVtX3U6b2JqZWN0X3I6dmFyX2xvZ190OnMwIC92YXIvb3B0L21pY3Jvc29mdC9saW51eG1vbmFnZW50Cgpta2RpciAtcCAvdmFyL2xpYi93YWFnZW50L01pY3Jvc29mdC5BenVyZS5LZXlWYXVsdC5TdG9yZQoKZm9yIHZhciBpbiAibWRzZCIgIm1kbSI7IGRvCmNhdCA+L2V0Yy9zeXN0ZW1kL3N5c3RlbS9kb3dubG9hZC0kdmFyLWNyZWRlbnRpYWxzLnNlcnZpY2UgPDxFT0YKW1VuaXRdCkRlc2NyaXB0aW9uPVBlcmlvZGljICR2YXIgY3JlZGVudGlhbHMgcmVmcmVzaAoKW1NlcnZpY2VdClR5cGU9b25lc2hvdApFeGVjU3RhcnQ9L3Vzci9sb2NhbC9iaW4vZG93bmxvYWQtY3JlZGVudGlhbHMuc2ggJHZhcgpFT0YKCmNhdCA+L2V0Yy9zeXN0ZW1kL3N5c3RlbS9kb3dubG9hZC0kdmFyLWNyZWRlbnRpYWxzLnRpbWVyIDw8RU9GCltVbml0XQpEZXNjcmlwdGlvbj1QZXJpb2RpYyAkdmFyIGNyZWRlbnRpYWxzIHJlZnJlc2gKQWZ0ZXI9bmV0d29yay1vbmxpbmUudGFyZ2V0CldhbnRzPW5ldHdvcmstb25saW5lLnRhcmdldAoKW1RpbWVyXQpPbkJvb3RTZWM9MG1pbgpPbkNhbGVuZGFyPTAvMTI6MDA6MDAKQWNjdXJhY3lTZWM9NXMKCltJbnN0YWxsXQpXYW50ZWRCeT10aW1lcnMudGFyZ2V0CkVPRgpkb25lCgpjYXQgPi91c3IvbG9jYWwvYmluL2Rvd25sb2FkLWNyZWRlbnRpYWxzLnNoIDw8RU9GCiMhL2Jpbi9iYXNoCnNldCAtZXUKCkNPTVBPTkVOVD0iXCQxIgplY2hvICJEb3dubG9hZCBcJENPTVBPTkVOVCBjcmVkZW50aWFscyIKClRFTVBfRElSPVwkKG1rdGVtcCAtZCkKZXhwb3J0IEFaVVJFX0NPTkZJR19ESVI9XCQobWt0ZW1wIC1kKQoKZWNobyAiTG9nZ2luZyBpbnRvIEF6dXJlLi4uIgpSRVRSSUVTPTMKd2hpbGUgWyAiXCRSRVRSSUVTIiAtZ3QgMCBdOyBkbwogICAgaWYgYXogbG9naW4gLWkgLS1hbGxvdy1uby1zdWJzY3JpcHRpb25zCiAgICB0aGVuCiAgICAgICAgZWNobyAiYXogbG9naW4gc3VjY2Vzc2Z1bCIKICAgICAgICBicmVhawogICAgZWxzZQogICAgICAgIGVjaG8gImF6IGxvZ2luIGZhaWxlZC4gUmV0cnlpbmcuLi4iCiAgICAgICAgbGV0IFJFVFJJRVMtPTEKICAgICAgICBzbGVlcCA1CiAgICBmaQpkb25lCgp0cmFwICJjbGVhbnVwIiBFWElUCgpjbGVhbnVwKCkgewogIGF6IGxvZ291dAogIFtbICJcJFRFTVBfRElSIiA9fiAvdG1wLy4rIF1dICYmIHJtIC1yZiBcJFRFTVBfRElSCiAgW1sgIlwkQVpVUkVfQ09ORklHX0RJUiIgPX4gL3RtcC8uKyBdXSAmJiBybSAtcmYgXCRBWlVSRV9DT05GSUdfRElSCn0KCmlmIFsgIlwkQ09NUE9ORU5UIiA9ICJtZG0iIF07IHRoZW4KICBDVVJSRU5UX0NFUlRfRklMRT0iL2V0Yy9tZG0ucGVtIgplbGlmIFsgIlwkQ09NUE9ORU5UIiA9ICJtZHNkIiBdOyB0aGVuCiAgQ1VSUkVOVF9DRVJUX0ZJTEU9Ii92YXIvbGliL3dhYWdlbnQvTWljcm9zb2Z0LkF6dXJlLktleVZhdWx0LlN0b3JlL21kc2QucGVtIgplbHNlCiAgZWNobyBJbnZhbGlkIHVzYWdlICYmIGV4aXQgMQpmaQoKU0VDUkVUX05BTUU9InJwLVwke0NPTVBPTkVOVH0iCk5FV19DRVJUX0ZJTEU9IlwkVEVNUF9ESVIvXCRDT01QT05FTlQucGVtIgpmb3IgYXR0ZW1wdCBpbiB7MS4uNX07IGRvCiAgYXoga2V5dmF1bHQgc2VjcmV0IGRvd25sb2FkIC0tZmlsZSBcJE5FV19DRVJUX0ZJTEUgLS1pZCAiaHR0cHM6Ly8kS0VZVkFVTFRQUkVGSVgtc3ZjLiRLRVlWQVVMVEROU1NVRkZJWC9zZWNyZXRzL1wkU0VDUkVUX05BTUUiICYmIGJyZWFrCiAgaWYgW1sgXCRhdHRlbXB0IC1sdCA1IF1dOyB0aGVuIHNsZWVwIDEwOyBlbHNlIGV4aXQgMTsgZmkKZG9uZQoKaWYgWyAtZiBcJE5FV19DRVJUX0ZJTEUgXTsgdGhlbgogIGlmIFsgIlwkQ09NUE9ORU5UIiA9ICJtZHNkIiBdOyB0aGVuCiAgICBjaG93biBzeXNsb2c6c3lzbG9nIFwkTkVXX0NFUlRfRklMRQogIGVsc2UKICAgIHNlZCAtaSAtbmUgJzEsL0VORCBDRVJUSUZJQ0FURS8gcCcgXCRORVdfQ0VSVF9GSUxFCiAgZmkKCiAgbmV3X2NlcnRfc249IlwkKG9wZW5zc2wgeDUwOSAtaW4gIlwkTkVXX0NFUlRfRklMRSIgLW5vb3V0IC1zZXJpYWwgfCBhd2sgLUY9ICd7cHJpbnQgXCQyfScpIgogIGN1cnJlbnRfY2VydF9zbj0iXCQob3BlbnNzbCB4NTA5IC1pbiAiXCRDVVJSRU5UX0NFUlRfRklMRSIgLW5vb3V0IC1zZXJpYWwgfCBhd2sgLUY9ICd7cHJpbnQgXCQyfScpIgogIGlmIFtbICEgLXogXCRuZXdfY2VydF9zbiBdXSAmJiBbWyBcJG5ld19jZXJ0X3NuICE9ICJcJGN1cnJlbnRfY2VydF9zbiIgXV07IHRoZW4KICAgIGVjaG8gdXBkYXRpbmcgY2VydGlmaWNhdGUgZm9yIFwkQ09NUE9ORU5UCiAgICBjaG1vZCAwNjAwIFwkTkVXX0NFUlRfRklMRQogICAgbXYgXCRORVdfQ0VSVF9GSUxFIFwkQ1VSUkVOVF9DRVJUX0ZJTEUKICBmaQplbHNlCiAgZWNobyBGYWlsZWQgdG8gcmVmcmVzaCBjZXJ0aWZpY2F0ZSBmb3IgXCRDT01QT05FTlQgJiYgZXhpdCAxCmZpCkVPRgoKY2htb2QgdSt4IC91c3IvbG9jYWwvYmluL2Rvd25sb2FkLWNyZWRlbnRpYWxzLnNoCgpzeXN0ZW1jdGwgZW5hYmxlIGRvd25sb2FkLW1kc2QtY3JlZGVudGlhbHMudGltZXIKc3lzdGVtY3RsIGVuYWJsZSBkb3dubG9hZC1tZG0tY3JlZGVudGlhbHMudGltZXIKCi91c3IvbG9jYWwvYmluL2Rvd25sb2FkLWNyZWRlbnRpYWxzLnNoIG1kc2QKL3Vzci9sb2NhbC9iaW4vZG93bmxvYWQtY3JlZGVudGlhbHMuc2ggbWRtCk1EU0RDRVJUSUZJQ0FURVNBTj0kKG9wZW5zc2wgeDUwOSAtaW4gL3Zhci9saWIvd2FhZ2VudC9NaWNyb3NvZnQuQXp1cmUuS2V5VmF1bHQuU3RvcmUvbWRzZC5wZW0gLW5vb3V0IC1zdWJqZWN0IHwgc2VkIC1lICdzLy4qQ04gPSAvLycpCgpjYXQgPi9ldGMvc3lzdGVtZC9zeXN0ZW0vd2F0Y2gtbWRtLWNyZWRlbnRpYWxzLnNlcnZpY2UgPDxFT0YKW1VuaXRdCkRlc2NyaXB0aW9uPVdhdGNoIGZvciBjaGFuZ2VzIGluIG1kbS5wZW0gYW5kIHJlc3RhcnRzIHRoZSBtZG0gc2VydmljZQoKW1NlcnZpY2VdClR5cGU9b25lc2hvdApFeGVjU3RhcnQ9L3Vzci9iaW4vc3lzdGVtY3RsIHJlc3RhcnQgbWRtLnNlcnZpY2UKCltJbnN0YWxsXQpXYW50ZWRCeT1tdWx0aS11c2VyLnRhcmdldApFT0YKCmNhdCA+L2V0Yy9zeXN0ZW1kL3N5c3RlbS93YXRjaC1tZG0tY3JlZGVudGlhbHMucGF0aCA8PEVPRgpbUGF0aF0KUGF0aE1vZGlmaWVkPS9ldGMvbWRtLnBlbQoKW0luc3RhbGxdCldhbnRlZEJ5PW11bHRpLXVzZXIudGFyZ2V0CkVPRgoKc3lzdGVtY3RsIGVuYWJsZSB3YXRjaC1tZG0tY3JlZGVudGlhbHMucGF0aApzeXN0ZW1jdGwgc3RhcnQgd2F0Y2gtbWRtLWNyZWRlbnRpYWxzLnBhdGgKCm1rZGlyIC9ldGMvc3lzdGVtZC9zeXN0ZW0vbWRzZC5zZXJ2aWNlLmQKY2F0ID4vZXRjL3N5c3RlbWQvc3lzdGVtL21kc2Quc2VydmljZS5kL292ZXJyaWRlLmNvbmYgPDwnRU9GJwpbVW5pdF0KQWZ0ZXI9bmV0d29yay1vbmxpbmUudGFyZ2V0CkVPRgoKY2F0ID4vZXRjL2RlZmF1bHQvbWRzZCA8PEVPRgpNRFNEX1JPTEVfUFJFRklYPS92YXIvcnVuL21kc2QvZGVmYXVsdApNRFNEX09QVElPTlM9Ii1BIC1kIC1yIFwkTURTRF9ST0xFX1BSRUZJWCIKCmV4cG9ydCBNT05JVE9SSU5HX0dDU19FTlZJUk9OTUVOVD0nJE1EU0RFTlZJUk9OTUVOVCcKZXhwb3J0IE1PTklUT1JJTkdfR0NTX0FDQ09VTlQ9JyRSUE1EU0RBQ0NPVU5UJwpleHBvcnQgTU9OSVRPUklOR19HQ1NfUkVHSU9OPSckTE9DQVRJT04nCmV4cG9ydCBNT05JVE9SSU5HX0dDU19BVVRIX0lEX1RZUEU9QXV0aEtleVZhdWx0CmV4cG9ydCBNT05JVE9SSU5HX0dDU19BVVRIX0lEPSckTURTRENFUlRJRklDQVRFU0FOJwpleHBvcnQgTU9OSVRPUklOR19HQ1NfTkFNRVNQQUNFPSckUlBNRFNETkFNRVNQQUNFJwpleHBvcnQgTU9OSVRPUklOR19DT05GSUdfVkVSU0lPTj0nJFJQTURTRENPTkZJR1ZFUlNJT04nCmV4cG9ydCBNT05JVE9SSU5HX1VTRV9HRU5FVkFfQ09ORklHX1NFUlZJQ0U9dHJ1ZQoKZXhwb3J0IE1PTklUT1JJTkdfVEVOQU5UPSckTE9DQVRJT04nCmV4cG9ydCBNT05JVE9SSU5HX1JPTEU9cnAKZXhwb3J0IE1PTklUT1JJTkdfUk9MRV9JTlNUQU5DRT0nJChob3N0bmFtZSknCgpleHBvcnQgTURTRF9NU0dQQUNLX1NPUlRfQ09MVU1OUz0xCkVPRgoKIyBzZXR0aW5nIE1PTklUT1JJTkdfR0NTX0FVVEhfSURfVFlQRT1BdXRoS2V5VmF1bHQgc2VlbXMgdG8gaGF2ZSBjYXVzZWQgbWRzZCBub3QKIyB0byBob25vdXIgU1NMX0NFUlRfRklMRSBhbnkgbW9yZSwgaGVhdmVuIG9ubHkga25vd3Mgd2h5Lgpta2RpciAtcCAvdXNyL2xpYi9zc2wvY2VydHMKY3NwbGl0IC1mIC91c3IvbGliL3NzbC9jZXJ0cy9jZXJ0LSAtYiAlMDNkLnBlbSAvZXRjL3BraS90bHMvY2VydHMvY2EtYnVuZGxlLmNydCAvXiQvMSB7Kn0gPi9kZXYvbnVsbApjX3JlaGFzaCAvdXNyL2xpYi9zc2wvY2VydHMKCiMgd2UgbGVhdmUgY2xpZW50SWQgYmxhbmsgYXMgbG9uZyBhcyBvbmx5IDEgbWFuYWdlZCBpZGVudGl0eSBhc3NpZ25lZCB0byB2bXNzCiMgaWYgd2UgaGF2ZSBtb3JlIHRoYW4gMSwgd2Ugd2lsbCBuZWVkIHRvIHBvcHVsYXRlIHdpdGggY2xpZW50SWQgdXNlZCBmb3Igb2ZmLW5vZGUgc2Nhbm5pbmcKY2F0ID4vZXRjL2RlZmF1bHQvdnNhLW5vZGVzY2FuLWFnZW50LmNvbmZpZyA8PEVPRgp7CiAgICAiTmljZSI6IDE5LAogICAgIlRpbWVvdXQiOiAxMDgwMCwKICAgICJDbGllbnRJZCI6ICIiLAogICAgIlRlbmFudElkIjogIiRBWlVSRVNFQ1BBQ0tWU0FURU5BTlRJRCIsCiAgICAiUXVhbHlzU3RvcmVCYXNlVXJsIjogIiRBWlVSRVNFQ1BBQ0tRVUFMWVNVUkwiLAogICAgIlByb2Nlc3NUaW1lb3V0IjogMzAwLAogICAgIkNvbW1hbmREZWxheSI6IDAKICB9CkVPRgoKZWNobyAiZW5hYmxpbmcgYXJvIHNlcnZpY2VzIgpmb3Igc2VydmljZSBpbiBhcm8tZGJ0b2tlbiBhcm8tbW9uaXRvciBhcm8tcG9ydGFsIGFyby1ycCBhdW9tcyBhenNlY2QgYXpzZWNtb25kIG1kc2QgbWRtIGNocm9ueWQgZmx1ZW50Yml0OyBkbwogIHN5c3RlbWN0bCBlbmFibGUgJHNlcnZpY2Uuc2VydmljZQpkb25lCgpmb3Igc2NhbiBpbiBiYXNlbGluZSBjbGFtYXYgc29mdHdhcmU7IGRvCiAgL3Vzci9sb2NhbC9iaW4vYXpzZWNkIGNvbmZpZyAtcyAkc2NhbiAtZCBQMUQKZG9uZQoKZWNobyAicmVib290aW5nIgpyZXN0b3JlY29uIC1SRiAvdmFyL2xvZy8qCihzbGVlcCAzMDsgcmVib290KSAmCg==')))]" + "script": "[base64(concat(base64ToString('c2V0IC1leAoK'),'ACRRESOURCEID=$(base64 -d \u003c\u003c\u003c''',base64(parameters('acrResourceId')),''')\n','ADMINAPICLIENTCERTCOMMONNAME=$(base64 -d \u003c\u003c\u003c''',base64(parameters('adminApiClientCertCommonName')),''')\n','ARMAPICLIENTCERTCOMMONNAME=$(base64 -d \u003c\u003c\u003c''',base64(parameters('armApiClientCertCommonName')),''')\n','ARMCLIENTID=$(base64 -d \u003c\u003c\u003c''',base64(parameters('armClientId')),''')\n','AZURECLOUDNAME=$(base64 -d \u003c\u003c\u003c''',base64(parameters('azureCloudName')),''')\n','AZURESECPACKQUALYSURL=$(base64 -d \u003c\u003c\u003c''',base64(parameters('azureSecPackQualysUrl')),''')\n','AZURESECPACKVSATENANTID=$(base64 -d \u003c\u003c\u003c''',base64(parameters('azureSecPackVSATenantId')),''')\n','CLUSTERMDMACCOUNT=$(base64 -d \u003c\u003c\u003c''',base64(parameters('clusterMdmAccount')),''')\n','CLUSTERMDSDACCOUNT=$(base64 -d \u003c\u003c\u003c''',base64(parameters('clusterMdsdAccount')),''')\n','CLUSTERMDSDCONFIGVERSION=$(base64 -d \u003c\u003c\u003c''',base64(parameters('clusterMdsdConfigVersion')),''')\n','CLUSTERMDSDNAMESPACE=$(base64 -d \u003c\u003c\u003c''',base64(parameters('clusterMdsdNamespace')),''')\n','CLUSTERPARENTDOMAINNAME=$(base64 -d \u003c\u003c\u003c''',base64(parameters('clusterParentDomainName')),''')\n','DATABASEACCOUNTNAME=$(base64 -d \u003c\u003c\u003c''',base64(parameters('databaseAccountName')),''')\n','DBTOKENCLIENTID=$(base64 -d \u003c\u003c\u003c''',base64(parameters('dbtokenClientId')),''')\n','FLUENTBITIMAGE=$(base64 -d \u003c\u003c\u003c''',base64(parameters('fluentbitImage')),''')\n','FPCLIENTID=$(base64 -d \u003c\u003c\u003c''',base64(parameters('fpClientId')),''')\n','FPSERVICEPRINCIPALID=$(base64 -d \u003c\u003c\u003c''',base64(parameters('fpServicePrincipalId')),''')\n','GATEWAYDOMAINS=$(base64 -d \u003c\u003c\u003c''',base64(parameters('gatewayDomains')),''')\n','GATEWAYRESOURCEGROUPNAME=$(base64 -d \u003c\u003c\u003c''',base64(parameters('gatewayResourceGroupName')),''')\n','GATEWAYSERVICEPRINCIPALID=$(base64 -d \u003c\u003c\u003c''',base64(parameters('gatewayServicePrincipalId')),''')\n','KEYVAULTDNSSUFFIX=$(base64 -d \u003c\u003c\u003c''',base64(parameters('keyvaultDNSSuffix')),''')\n','KEYVAULTPREFIX=$(base64 -d \u003c\u003c\u003c''',base64(parameters('keyvaultPrefix')),''')\n','MDMFRONTENDURL=$(base64 -d \u003c\u003c\u003c''',base64(parameters('mdmFrontendUrl')),''')\n','MDSDENVIRONMENT=$(base64 -d \u003c\u003c\u003c''',base64(parameters('mdsdEnvironment')),''')\n','PORTALACCESSGROUPIDS=$(base64 -d \u003c\u003c\u003c''',base64(parameters('portalAccessGroupIds')),''')\n','PORTALCLIENTID=$(base64 -d \u003c\u003c\u003c''',base64(parameters('portalClientId')),''')\n','PORTALELEVATEDGROUPIDS=$(base64 -d \u003c\u003c\u003c''',base64(parameters('portalElevatedGroupIds')),''')\n','RPFEATURES=$(base64 -d \u003c\u003c\u003c''',base64(parameters('rpFeatures')),''')\n','RPIMAGE=$(base64 -d \u003c\u003c\u003c''',base64(parameters('rpImage')),''')\n','RPMDMACCOUNT=$(base64 -d \u003c\u003c\u003c''',base64(parameters('rpMdmAccount')),''')\n','RPMDSDACCOUNT=$(base64 -d \u003c\u003c\u003c''',base64(parameters('rpMdsdAccount')),''')\n','RPMDSDCONFIGVERSION=$(base64 -d \u003c\u003c\u003c''',base64(parameters('rpMdsdConfigVersion')),''')\n','RPMDSDNAMESPACE=$(base64 -d \u003c\u003c\u003c''',base64(parameters('rpMdsdNamespace')),''')\n','RPPARENTDOMAINNAME=$(base64 -d \u003c\u003c\u003c''',base64(parameters('rpParentDomainName')),''')\n','CLUSTERSINSTALLVIAHIVE=$(base64 -d \u003c\u003c\u003c''',base64(parameters('clustersInstallViaHive')),''')\n','CLUSTERSADOPTBYHIVE=$(base64 -d \u003c\u003c\u003c''',base64(parameters('clustersAdoptByHive')),''')\n','CLUSTERDEFAULTINSTALLERPULLSPEC=$(base64 -d \u003c\u003c\u003c''',base64(parameters('clusterDefaultInstallerPullspec')),''')\n','USECHECKACCESS=$(base64 -d \u003c\u003c\u003c''',base64(parameters('useCheckAccess')),''')\n','ADMINAPICABUNDLE=''',parameters('adminApiCaBundle'),'''\n','ARMAPICABUNDLE=''',parameters('armApiCaBundle'),'''\n','MDMIMAGE=''/distroless/genevamdm:2.2024.517.533-b73893-20240522t0954@sha256:939df9d7b6660874697f8ebed1fe56504f86d92f99801a9dc6fd98e9176d3f75''\n','LOCATION=$(base64 -d \u003c\u003c\u003c''',base64(resourceGroup().location),''')\n','SUBSCRIPTIONID=$(base64 -d \u003c\u003c\u003c''',base64(subscription().subscriptionId),''')\n','RESOURCEGROUPNAME=$(base64 -d \u003c\u003c\u003c''',base64(resourceGroup().name),''')\n','\n',base64ToString('IyEvYmluL2Jhc2gKCmVjaG8gInNldHRpbmcgc3NoIHBhc3N3b3JkIGF1dGhlbnRpY2F0aW9uIgojIFdlIG5lZWQgdG8gbWFudWFsbHkgc2V0IFBhc3N3b3JkQXV0aGVudGljYXRpb24gdG8gdHJ1ZSBpbiBvcmRlciBmb3IgdGhlIFZNU1MgQWNjZXNzIEpJVCB0byB3b3JrCnNlZCAtaSAncy9QYXNzd29yZEF1dGhlbnRpY2F0aW9uIG5vL1Bhc3N3b3JkQXV0aGVudGljYXRpb24geWVzL2cnIC9ldGMvc3NoL3NzaGRfY29uZmlnCnN5c3RlbWN0bCByZWxvYWQgc3NoZC5zZXJ2aWNlCgojQWRkaW5nIHJldHJ5IGxvZ2ljIHRvIHl1bSBjb21tYW5kcyBpbiBvcmRlciB0byBhdm9pZCBzdGFsbGluZyBvdXQgb24gcmVzb3VyY2UgbG9ja3MKZWNobyAicnVubmluZyBSSFVJIGZpeCIKZm9yIGF0dGVtcHQgaW4gezEuLjYwfTsgZG8KICB5dW0gdXBkYXRlIC15IC0tZGlzYWJsZXJlcG89JyonIC0tZW5hYmxlcmVwbz0ncmh1aS1taWNyb3NvZnQtYXp1cmUqJyAmJiBicmVhawogIGlmIFtbICR7YXR0ZW1wdH0gLWx0IDYwIF1dOyB0aGVuIHNsZWVwIDMwOyBlbHNlIGV4aXQgMTsgZmkKZG9uZQoKZWNobyAicnVubmluZyB5dW0gdXBkYXRlIgpmb3IgYXR0ZW1wdCBpbiB7MS4uNjB9OyBkbwogIHl1bSAteSAteCBXQUxpbnV4QWdlbnQgLXggV0FMaW51eEFnZW50LXVkZXYgdXBkYXRlIC0tYWxsb3dlcmFzaW5nICYmIGJyZWFrCiAgaWYgW1sgJHthdHRlbXB0fSAtbHQgNjAgXV07IHRoZW4gc2xlZXAgMzA7IGVsc2UgZXhpdCAxOyBmaQpkb25lCgplY2hvICJleHRlbmRpbmcgcGFydGl0aW9uIHRhYmxlIgojIExpbnV4IGJsb2NrIGRldmljZXMgYXJlIGluY29uc2lzdGVudGx5IG5hbWVkCiMgaXQncyBkaWZmaWN1bHQgdG8gdGllIHRoZSBsdm0gcHYgdG8gdGhlIHBoeXNpY2FsIGRpc2sgdXNpbmcgL2Rldi9kaXNrIGZpbGVzLCB3aGljaCBpcyB3aHkgbHZzIGlzIHVzZWQgaGVyZQpwaHlzaWNhbERpc2s9IiQobHZzIC1vIGRldmljZXMgLWEgfCBoZWFkIC1uMiB8IHRhaWwgLW4xIHwgY3V0IC1kICcgJyAtZiAzIHwgY3V0IC1kIFwoIC1mIDEgfCB0ciAtZCAnWzpkaWdpdDpdJykiCmdyb3dwYXJ0ICIkcGh5c2ljYWxEaXNrIiAyCgplY2hvICJleHRlbmRpbmcgZmlsZXN5c3RlbXMiCmx2ZXh0ZW5kIC1sICsyMCVGUkVFIC9kZXYvcm9vdHZnL3Jvb3Rsdgp4ZnNfZ3Jvd2ZzIC8KCmx2ZXh0ZW5kIC1sICsxMDAlRlJFRSAvZGV2L3Jvb3R2Zy92YXJsdgp4ZnNfZ3Jvd2ZzIC92YXIKCmVjaG8gImltcG9ydGluZyBycG0gcmVwb3NpdG9yaWVzIgpycG0gLS1pbXBvcnQgaHR0cHM6Ly9kbC5mZWRvcmFwcm9qZWN0Lm9yZy9wdWIvZXBlbC9SUE0tR1BHLUtFWS1FUEVMLTgKcnBtIC0taW1wb3J0IGh0dHBzOi8vcGFja2FnZXMubWljcm9zb2Z0LmNvbS9rZXlzL21pY3Jvc29mdC5hc2MKCmZvciBhdHRlbXB0IGluIHsxLi42MH07IGRvCiAgeXVtIC15IGluc3RhbGwgaHR0cHM6Ly9kbC5mZWRvcmFwcm9qZWN0Lm9yZy9wdWIvZXBlbC9lcGVsLXJlbGVhc2UtbGF0ZXN0LTgubm9hcmNoLnJwbSAmJiBicmVhawogIGlmIFtbICR7YXR0ZW1wdH0gLWx0IDYwIF1dOyB0aGVuIHNsZWVwIDMwOyBlbHNlIGV4aXQgMTsgZmkKZG9uZQoKZWNobyAiY29uZmlndXJpbmcgbG9ncm90YXRlIgpjYXQgPi9ldGMvbG9ncm90YXRlLmNvbmYgPDwnRU9GJwojIHNlZSAibWFuIGxvZ3JvdGF0ZSIgZm9yIGRldGFpbHMKIyByb3RhdGUgbG9nIGZpbGVzIHdlZWtseQp3ZWVrbHkKCiMga2VlcCAyIHdlZWtzIHdvcnRoIG9mIGJhY2tsb2dzCnJvdGF0ZSAyCgojIGNyZWF0ZSBuZXcgKGVtcHR5KSBsb2cgZmlsZXMgYWZ0ZXIgcm90YXRpbmcgb2xkIG9uZXMKY3JlYXRlCgojIHVzZSBkYXRlIGFzIGEgc3VmZml4IG9mIHRoZSByb3RhdGVkIGZpbGUKZGF0ZWV4dAoKIyB1bmNvbW1lbnQgdGhpcyBpZiB5b3Ugd2FudCB5b3VyIGxvZyBmaWxlcyBjb21wcmVzc2VkCmNvbXByZXNzCgojIFJQTSBwYWNrYWdlcyBkcm9wIGxvZyByb3RhdGlvbiBpbmZvcm1hdGlvbiBpbnRvIHRoaXMgZGlyZWN0b3J5CmluY2x1ZGUgL2V0Yy9sb2dyb3RhdGUuZAoKIyBubyBwYWNrYWdlcyBvd24gd3RtcCBhbmQgYnRtcCAtLSB3ZSdsbCByb3RhdGUgdGhlbSBoZXJlCi92YXIvbG9nL3d0bXAgewogICAgbW9udGhseQogICAgY3JlYXRlIDA2NjQgcm9vdCB1dG1wCiAgICAgICAgbWluc2l6ZSAxTQogICAgcm90YXRlIDEKfQoKL3Zhci9sb2cvYnRtcCB7CiAgICBtaXNzaW5nb2sKICAgIG1vbnRobHkKICAgIGNyZWF0ZSAwNjAwIHJvb3QgdXRtcAogICAgcm90YXRlIDEKfQpFT0YKCmVjaG8gImNvbmZpZ3VyaW5nIHl1bSByZXBvc2l0b3J5IGFuZCBydW5uaW5nIHl1bSB1cGRhdGUiCmNhdCA+L2V0Yy95dW0ucmVwb3MuZC9henVyZS5yZXBvIDw8J0VPRicKW2F6dXJlLWNsaV0KbmFtZT1henVyZS1jbGkKYmFzZXVybD1odHRwczovL3BhY2thZ2VzLm1pY3Jvc29mdC5jb20veXVtcmVwb3MvYXp1cmUtY2xpCmVuYWJsZWQ9eWVzCmdwZ2NoZWNrPXllcwoKW2F6dXJlY29yZV0KbmFtZT1henVyZWNvcmUKYmFzZXVybD1odHRwczovL3BhY2thZ2VzLm1pY3Jvc29mdC5jb20veXVtcmVwb3MvYXp1cmVjb3JlCmVuYWJsZWQ9eWVzCmdwZ2NoZWNrPW5vCkVPRgoKc2VtYW5hZ2UgZmNvbnRleHQgLWEgLXQgdmFyX2xvZ190ICIvdmFyL2xvZy9qb3VybmFsKC8uKik/Igpta2RpciAtcCAvdmFyL2xvZy9qb3VybmFsCgpmb3IgYXR0ZW1wdCBpbiB7MS4uNjB9OyBkbwp5dW0gLXkgaW5zdGFsbCBjbGFtYXYgYXpzZWMtY2xhbWF2IGF6c2VjLW1vbml0b3IgYXp1cmUtY2xpIGF6dXJlLW1kc2QgYXp1cmUtc2VjdXJpdHkgcG9kbWFuIHBvZG1hbi1kb2NrZXIgb3BlbnNzbC1wZXJsIHB5dGhvbjMgJiYgYnJlYWsKICAjIGhhY2sgLSB3ZSBhcmUgaW5zdGFsbGluZyBweXRob24zIG9uIGhvc3RzIGR1ZSB0byBhbiBpc3N1ZSB3aXRoIEF6dXJlIExpbnV4IEV4dGVuc2lvbnMgaHR0cHM6Ly9naXRodWIuY29tL0F6dXJlL2F6dXJlLWxpbnV4LWV4dGVuc2lvbnMvcHVsbC8xNTA1CiAgaWYgW1sgJHthdHRlbXB0fSAtbHQgNjAgXV07IHRoZW4gc2xlZXAgMzA7IGVsc2UgZXhpdCAxOyBmaQpkb25lCgojIGh0dHBzOi8vYWNjZXNzLnJlZGhhdC5jb20vc2VjdXJpdHkvY3ZlL2N2ZS0yMDIwLTEzNDAxCmVjaG8gImFwcGx5aW5nIGZpcmV3YWxsIHJ1bGVzIgpjYXQgPi9ldGMvc3lzY3RsLmQvMDItZGlzYWJsZS1hY2NlcHQtcmEuY29uZiA8PCdFT0YnCm5ldC5pcHY2LmNvbmYuYWxsLmFjY2VwdF9yYT0wCkVPRgoKY2F0ID4vZXRjL3N5c2N0bC5kLzAxLWRpc2FibGUtY29yZS5jb25mIDw8J0VPRicKa2VybmVsLmNvcmVfcGF0dGVybiA9IHwvYmluL3RydWUKRU9GCnN5c2N0bCAtLXN5c3RlbQoKZmlyZXdhbGwtY21kIC0tYWRkLXBvcnQ9NDQzL3RjcCAtLXBlcm1hbmVudApmaXJld2FsbC1jbWQgLS1hZGQtcG9ydD00NDQvdGNwIC0tcGVybWFuZW50CmZpcmV3YWxsLWNtZCAtLWFkZC1wb3J0PTQ0NS90Y3AgLS1wZXJtYW5lbnQKZmlyZXdhbGwtY21kIC0tYWRkLXBvcnQ9MjIyMi90Y3AgLS1wZXJtYW5lbnQKCmV4cG9ydCBBWlVSRV9DTE9VRF9OQU1FPSRBWlVSRUNMT1VETkFNRQoKZWNobyAibG9nZ2luZyBpbnRvIHByb2QgYWNyIgpheiBsb2dpbiAtaSAtLWFsbG93LW5vLXN1YnNjcmlwdGlvbnMKCiMgU3VwcHJlc3MgZW11bGF0aW9uIG91dHB1dCBmb3IgcG9kbWFuIGluc3RlYWQgb2YgZG9ja2VyIGZvciBheiBhY3IgY29tcGF0YWJpbGl0eQpta2RpciAtcCAvZXRjL2NvbnRhaW5lcnMvCnRvdWNoIC9ldGMvY29udGFpbmVycy9ub2RvY2tlcgoKbWtkaXIgLXAgL3Jvb3QvLmRvY2tlcgpSRUdJU1RSWV9BVVRIX0ZJTEU9L3Jvb3QvLmRvY2tlci9jb25maWcuanNvbiBheiBhY3IgbG9naW4gLS1uYW1lICIkKHNlZCAtZSAnc3wuKi98fCcgPDw8IiRBQ1JSRVNPVVJDRUlEIikiCgpNRE1JTUFHRT0iJHtSUElNQUdFJSUvKn0vJHtNRE1JTUFHRSMqL30iCmRvY2tlciBwdWxsICIkTURNSU1BR0UiCmRvY2tlciBwdWxsICIkUlBJTUFHRSIKZG9ja2VyIHB1bGwgIiRGTFVFTlRCSVRJTUFHRSIKCmF6IGxvZ291dAoKZWNobyAiY29uZmlndXJpbmcgZmx1ZW50Yml0IHNlcnZpY2UiCm1rZGlyIC1wIC9ldGMvZmx1ZW50Yml0Lwpta2RpciAtcCAvdmFyL2xpYi9mbHVlbnQKCmNhdCA+L2V0Yy9mbHVlbnRiaXQvZmx1ZW50Yml0LmNvbmYgPDwnRU9GJwpbSU5QVVRdCglOYW1lIHN5c3RlbWQKCVRhZyBqb3VybmFsZAoJU3lzdGVtZF9GaWx0ZXIgX0NPTU09YXJvCglEQiAvdmFyL2xpYi9mbHVlbnQvam91cm5hbGRiCgpbRklMVEVSXQoJTmFtZSBtb2RpZnkKCU1hdGNoIGpvdXJuYWxkCglSZW1vdmVfd2lsZGNhcmQgXwoJUmVtb3ZlIFRJTUVTVEFNUAoKW0ZJTFRFUl0KCU5hbWUgcmV3cml0ZV90YWcKCU1hdGNoIGpvdXJuYWxkCglSdWxlICRMT0dLSU5EIGFzeW5jcW9zIGFzeW5jcW9zIHRydWUKCltGSUxURVJdCglOYW1lIG1vZGlmeQoJTWF0Y2ggYXN5bmNxb3MKCVJlbW92ZSBDTElFTlRfUFJJTkNJUEFMX05BTUUKCVJlbW92ZSBGSUxFCglSZW1vdmUgQ09NUE9ORU5UCgpbRklMVEVSXQoJTmFtZSByZXdyaXRlX3RhZwoJTWF0Y2ggam91cm5hbGQKCVJ1bGUgJExPR0tJTkQgaWZ4YXVkaXQgaWZ4YXVkaXQgZmFsc2UKCltPVVRQVVRdCglOYW1lIGZvcndhcmQKCU1hdGNoICoKCVBvcnQgMjkyMzAKRU9GCgplY2hvICJGTFVFTlRCSVRJTUFHRT0kRkxVRU5UQklUSU1BR0UiID4vZXRjL3N5c2NvbmZpZy9mbHVlbnRiaXQKCmNhdCA+L2V0Yy9zeXN0ZW1kL3N5c3RlbS9mbHVlbnRiaXQuc2VydmljZSA8PCdFT0YnCltVbml0XQpBZnRlcj1uZXR3b3JrLW9ubGluZS50YXJnZXQKV2FudHM9bmV0d29yay1vbmxpbmUudGFyZ2V0ClN0YXJ0TGltaXRJbnRlcnZhbFNlYz0wCgpbU2VydmljZV0KUmVzdGFydFNlYz0xcwpFbnZpcm9ubWVudEZpbGU9L2V0Yy9zeXNjb25maWcvZmx1ZW50Yml0CkV4ZWNTdGFydFByZT0tL3Vzci9iaW4vZG9ja2VyIHJtIC1mICVOCkV4ZWNTdGFydD0vdXNyL2Jpbi9kb2NrZXIgcnVuIFwKICAtLXNlY3VyaXR5LW9wdCBsYWJlbD1kaXNhYmxlIFwKICAtLWVudHJ5cG9pbnQgL29wdC90ZC1hZ2VudC1iaXQvYmluL3RkLWFnZW50LWJpdCBcCiAgLS1uZXQ9aG9zdCBcCiAgLS1ob3N0bmFtZSAlSCBcCiAgLS1uYW1lICVOIFwKICAtLXJtIFwKICAtLWNhcC1kcm9wIG5ldF9yYXcgXAogIC12IC9ldGMvZmx1ZW50Yml0L2ZsdWVudGJpdC5jb25mOi9ldGMvZmx1ZW50Yml0L2ZsdWVudGJpdC5jb25mIFwKICAtdiAvdmFyL2xpYi9mbHVlbnQ6L3Zhci9saWIvZmx1ZW50OnogXAogIC12IC92YXIvbG9nL2pvdXJuYWw6L3Zhci9sb2cvam91cm5hbDpybyBcCiAgLXYgL2V0Yy9tYWNoaW5lLWlkOi9ldGMvbWFjaGluZS1pZDpybyBcCiAgJEZMVUVOVEJJVElNQUdFIFwKICAtYyAvZXRjL2ZsdWVudGJpdC9mbHVlbnRiaXQuY29uZgoKRXhlY1N0b3A9L3Vzci9iaW4vZG9ja2VyIHN0b3AgJU4KUmVzdGFydD1hbHdheXMKUmVzdGFydFNlYz01ClN0YXJ0TGltaXRJbnRlcnZhbD0wCgpbSW5zdGFsbF0KV2FudGVkQnk9bXVsdGktdXNlci50YXJnZXQKRU9GCgpta2RpciAvZXRjL2Fyby1ycApiYXNlNjQgLWQgPDw8IiRBRE1JTkFQSUNBQlVORExFIiA+L2V0Yy9hcm8tcnAvYWRtaW4tY2EtYnVuZGxlLnBlbQppZiBbWyAtbiAiJEFSTUFQSUNBQlVORExFIiBdXTsgdGhlbgogIGJhc2U2NCAtZCA8PDwiJEFSTUFQSUNBQlVORExFIiA+L2V0Yy9hcm8tcnAvYXJtLWNhLWJ1bmRsZS5wZW0KZmkKY2hvd24gLVIgMTAwMDoxMDAwIC9ldGMvYXJvLXJwCgplY2hvICJjb25maWd1cmluZyBtZG0gc2VydmljZSIKY2F0ID4vZXRjL3N5c2NvbmZpZy9tZG0gPDxFT0YKTURNRlJPTlRFTkRVUkw9JyRNRE1GUk9OVEVORFVSTCcKTURNSU1BR0U9JyRNRE1JTUFHRScKTURNU09VUkNFRU5WSVJPTk1FTlQ9JyRMT0NBVElPTicKTURNU09VUkNFUk9MRT1ycApNRE1TT1VSQ0VST0xFSU5TVEFOQ0U9JyQoaG9zdG5hbWUpJwpFT0YKCm1rZGlyIC92YXIvZXR3CmNhdCA+L2V0Yy9zeXN0ZW1kL3N5c3RlbS9tZG0uc2VydmljZSA8PCdFT0YnCltVbml0XQpBZnRlcj1uZXR3b3JrLW9ubGluZS50YXJnZXQKV2FudHM9bmV0d29yay1vbmxpbmUudGFyZ2V0CgpbU2VydmljZV0KRW52aXJvbm1lbnRGaWxlPS9ldGMvc3lzY29uZmlnL21kbQpFeGVjU3RhcnRQcmU9LS91c3IvYmluL2RvY2tlciBybSAtZiAlTgpFeGVjU3RhcnQ9L3Vzci9iaW4vZG9ja2VyIHJ1biBcCiAgLS1lbnRyeXBvaW50IC91c3Ivc2Jpbi9NZXRyaWNzRXh0ZW5zaW9uIFwKICAtLWhvc3RuYW1lICVIIFwKICAtLW5hbWUgJU4gXAogIC0tcm0gXAogIC0tY2FwLWRyb3AgbmV0X3JhdyBcCiAgLW0gMmcgXAogIC12IC9ldGMvbWRtLnBlbTovZXRjL21kbS5wZW0gXAogIC12IC92YXIvZXR3Oi92YXIvZXR3OnogXAogICRNRE1JTUFHRSBcCiAgLUNlcnRGaWxlIC9ldGMvbWRtLnBlbSBcCiAgLUZyb250RW5kVXJsICRNRE1GUk9OVEVORFVSTCBcCiAgLUxvZ2dlciBDb25zb2xlIFwKICAtTG9nTGV2ZWwgV2FybmluZyBcCiAgLVByaXZhdGVLZXlGaWxlIC9ldGMvbWRtLnBlbSBcCiAgLVNvdXJjZUVudmlyb25tZW50ICRNRE1TT1VSQ0VFTlZJUk9OTUVOVCBcCiAgLVNvdXJjZVJvbGUgJE1ETVNPVVJDRVJPTEUgXAogIC1Tb3VyY2VSb2xlSW5zdGFuY2UgJE1ETVNPVVJDRVJPTEVJTlNUQU5DRQpFeGVjU3RvcD0vdXNyL2Jpbi9kb2NrZXIgc3RvcCAlTgpSZXN0YXJ0PWFsd2F5cwpSZXN0YXJ0U2VjPTEKU3RhcnRMaW1pdEludGVydmFsPTAKCltJbnN0YWxsXQpXYW50ZWRCeT1tdWx0aS11c2VyLnRhcmdldApFT0YKCmVjaG8gImNvbmZpZ3VyaW5nIGFyby1ycCBzZXJ2aWNlIgpjYXQgPi9ldGMvc3lzY29uZmlnL2Fyby1ycCA8PEVPRgpBQ1JfUkVTT1VSQ0VfSUQ9JyRBQ1JSRVNPVVJDRUlEJwpBRE1JTl9BUElfQ0xJRU5UX0NFUlRfQ09NTU9OX05BTUU9JyRBRE1JTkFQSUNMSUVOVENFUlRDT01NT05OQU1FJwpBUk1fQVBJX0NMSUVOVF9DRVJUX0NPTU1PTl9OQU1FPSckQVJNQVBJQ0xJRU5UQ0VSVENPTU1PTk5BTUUnCkFaVVJFX0FSTV9DTElFTlRfSUQ9JyRBUk1DTElFTlRJRCcKQVpVUkVfRlBfQ0xJRU5UX0lEPSckRlBDTElFTlRJRCcKQVpVUkVfRlBfU0VSVklDRV9QUklOQ0lQQUxfSUQ9JyRGUFNFUlZJQ0VQUklOQ0lQQUxJRCcKQ0xVU1RFUl9NRE1fQUNDT1VOVD0nJENMVVNURVJNRE1BQ0NPVU5UJwpDTFVTVEVSX01ETV9OQU1FU1BBQ0U9UlAKQ0xVU1RFUl9NRFNEX0FDQ09VTlQ9JyRDTFVTVEVSTURTREFDQ09VTlQnCkNMVVNURVJfTURTRF9DT05GSUdfVkVSU0lPTj0nJENMVVNURVJNRFNEQ09ORklHVkVSU0lPTicKQ0xVU1RFUl9NRFNEX05BTUVTUEFDRT0nJENMVVNURVJNRFNETkFNRVNQQUNFJwpEQVRBQkFTRV9BQ0NPVU5UX05BTUU9JyREQVRBQkFTRUFDQ09VTlROQU1FJwpET01BSU5fTkFNRT0nJExPQ0FUSU9OLiRDTFVTVEVSUEFSRU5URE9NQUlOTkFNRScKR0FURVdBWV9ET01BSU5TPSckR0FURVdBWURPTUFJTlMnCkdBVEVXQVlfUkVTT1VSQ0VHUk9VUD0nJEdBVEVXQVlSRVNPVVJDRUdST1VQTkFNRScKS0VZVkFVTFRfUFJFRklYPSckS0VZVkFVTFRQUkVGSVgnCk1ETV9BQ0NPVU5UPSckUlBNRE1BQ0NPVU5UJwpNRE1fTkFNRVNQQUNFPVJQCk1EU0RfRU5WSVJPTk1FTlQ9JyRNRFNERU5WSVJPTk1FTlQnClJQX0ZFQVRVUkVTPSckUlBGRUFUVVJFUycKUlBJTUFHRT0nJFJQSU1BR0UnCkFST19JTlNUQUxMX1ZJQV9ISVZFPSckQ0xVU1RFUlNJTlNUQUxMVklBSElWRScKQVJPX0hJVkVfREVGQVVMVF9JTlNUQUxMRVJfUFVMTFNQRUM9JyRDTFVTVEVSREVGQVVMVElOU1RBTExFUlBVTExTUEVDJwpBUk9fQURPUFRfQllfSElWRT0nJENMVVNURVJTQURPUFRCWUhJVkUnClVTRV9DSEVDS0FDQ0VTUz0nJFVTRUNIRUNLQUNDRVNTJwpFT0YKCmNhdCA+L2V0Yy9zeXN0ZW1kL3N5c3RlbS9hcm8tcnAuc2VydmljZSA8PCdFT0YnCltVbml0XQpBZnRlcj1uZXR3b3JrLW9ubGluZS50YXJnZXQKV2FudHM9bmV0d29yay1vbmxpbmUudGFyZ2V0CgpbU2VydmljZV0KRW52aXJvbm1lbnRGaWxlPS9ldGMvc3lzY29uZmlnL2Fyby1ycApFeGVjU3RhcnRQcmU9LS91c3IvYmluL2RvY2tlciBybSAtZiAlTgpFeGVjU3RhcnQ9L3Vzci9iaW4vZG9ja2VyIHJ1biBcCiAgLS1ob3N0bmFtZSAlSCBcCiAgLS1uYW1lICVOIFwKICAtLXJtIFwKICAtLWNhcC1kcm9wIG5ldF9yYXcgXAogIC1lIEFDUl9SRVNPVVJDRV9JRCBcCiAgLWUgQURNSU5fQVBJX0NMSUVOVF9DRVJUX0NPTU1PTl9OQU1FIFwKICAtZSBBUk1fQVBJX0NMSUVOVF9DRVJUX0NPTU1PTl9OQU1FIFwKICAtZSBBWlVSRV9BUk1fQ0xJRU5UX0lEIFwKICAtZSBBWlVSRV9GUF9DTElFTlRfSUQgXAogIC1lIENMVVNURVJfTURNX0FDQ09VTlQgXAogIC1lIENMVVNURVJfTURNX05BTUVTUEFDRSBcCiAgLWUgQ0xVU1RFUl9NRFNEX0FDQ09VTlQgXAogIC1lIENMVVNURVJfTURTRF9DT05GSUdfVkVSU0lPTiBcCiAgLWUgQ0xVU1RFUl9NRFNEX05BTUVTUEFDRSBcCiAgLWUgREFUQUJBU0VfQUNDT1VOVF9OQU1FIFwKICAtZSBET01BSU5fTkFNRSBcCiAgLWUgR0FURVdBWV9ET01BSU5TIFwKICAtZSBHQVRFV0FZX1JFU09VUkNFR1JPVVAgXAogIC1lIEtFWVZBVUxUX1BSRUZJWCBcCiAgLWUgTURNX0FDQ09VTlQgXAogIC1lIE1ETV9OQU1FU1BBQ0UgXAogIC1lIE1EU0RfRU5WSVJPTk1FTlQgXAogIC1lIFJQX0ZFQVRVUkVTIFwKICAtZSBBUk9fSU5TVEFMTF9WSUFfSElWRSBcCiAgLWUgQVJPX0hJVkVfREVGQVVMVF9JTlNUQUxMRVJfUFVMTFNQRUMgXAogIC1lIEFST19BRE9QVF9CWV9ISVZFIFwKICAtZSBVU0VfQ0hFQ0tBQ0NFU1MgXAogIC1tIDJnIFwKICAtcCA0NDM6ODQ0MyBcCiAgLXYgL2V0Yy9hcm8tcnA6L2V0Yy9hcm8tcnAgXAogIC12IC9ydW4vc3lzdGVtZC9qb3VybmFsOi9ydW4vc3lzdGVtZC9qb3VybmFsIFwKICAtdiAvdmFyL2V0dzovdmFyL2V0dzp6IFwKICAkUlBJTUFHRSBcCiAgcnAKRXhlY1N0b3A9L3Vzci9iaW4vZG9ja2VyIHN0b3AgLXQgMzYwMCAlTgpUaW1lb3V0U3RvcFNlYz0zNjAwClJlc3RhcnQ9YWx3YXlzClJlc3RhcnRTZWM9MQpTdGFydExpbWl0SW50ZXJ2YWw9MAoKW0luc3RhbGxdCldhbnRlZEJ5PW11bHRpLXVzZXIudGFyZ2V0CkVPRgoKZWNobyAiY29uZmlndXJpbmcgYXJvLWRidG9rZW4gc2VydmljZSIKY2F0ID4vZXRjL3N5c2NvbmZpZy9hcm8tZGJ0b2tlbiA8PEVPRgpEQVRBQkFTRV9BQ0NPVU5UX05BTUU9JyREQVRBQkFTRUFDQ09VTlROQU1FJwpBWlVSRV9EQlRPS0VOX0NMSUVOVF9JRD0nJERCVE9LRU5DTElFTlRJRCcKQVpVUkVfR0FURVdBWV9TRVJWSUNFX1BSSU5DSVBBTF9JRD0nJEdBVEVXQVlTRVJWSUNFUFJJTkNJUEFMSUQnCktFWVZBVUxUX1BSRUZJWD0nJEtFWVZBVUxUUFJFRklYJwpNRE1fQUNDT1VOVD0nJFJQTURNQUNDT1VOVCcKTURNX05BTUVTUEFDRT1EQlRva2VuClJQSU1BR0U9JyRSUElNQUdFJwpFT0YKCmNhdCA+L2V0Yy9zeXN0ZW1kL3N5c3RlbS9hcm8tZGJ0b2tlbi5zZXJ2aWNlIDw8J0VPRicKW1VuaXRdCkFmdGVyPW5ldHdvcmstb25saW5lLnRhcmdldApXYW50cz1uZXR3b3JrLW9ubGluZS50YXJnZXQKCltTZXJ2aWNlXQpFbnZpcm9ubWVudEZpbGU9L2V0Yy9zeXNjb25maWcvYXJvLWRidG9rZW4KRXhlY1N0YXJ0UHJlPS0vdXNyL2Jpbi9kb2NrZXIgcm0gLWYgJU4KRXhlY1N0YXJ0PS91c3IvYmluL2RvY2tlciBydW4gXAogIC0taG9zdG5hbWUgJUggXAogIC0tbmFtZSAlTiBcCiAgLS1ybSBcCiAgLS1jYXAtZHJvcCBuZXRfcmF3IFwKICAtZSBBWlVSRV9HQVRFV0FZX1NFUlZJQ0VfUFJJTkNJUEFMX0lEIFwKICAtZSBEQVRBQkFTRV9BQ0NPVU5UX05BTUUgXAogIC1lIEFaVVJFX0RCVE9LRU5fQ0xJRU5UX0lEIFwKICAtZSBLRVlWQVVMVF9QUkVGSVggXAogIC1lIE1ETV9BQ0NPVU5UIFwKICAtZSBNRE1fTkFNRVNQQUNFIFwKICAtbSAyZyBcCiAgLXAgNDQ1Ojg0NDUgXAogIC12IC9ydW4vc3lzdGVtZC9qb3VybmFsOi9ydW4vc3lzdGVtZC9qb3VybmFsIFwKICAtdiAvdmFyL2V0dzovdmFyL2V0dzp6IFwKICAkUlBJTUFHRSBcCiAgZGJ0b2tlbgpFeGVjU3RvcD0vdXNyL2Jpbi9kb2NrZXIgc3RvcCAtdCAzNjAwICVOClRpbWVvdXRTdG9wU2VjPTM2MDAKUmVzdGFydD1hbHdheXMKUmVzdGFydFNlYz0xClN0YXJ0TGltaXRJbnRlcnZhbD0wCgpbSW5zdGFsbF0KV2FudGVkQnk9bXVsdGktdXNlci50YXJnZXQKRU9GCgojIERPTUFJTl9OQU1FLCBDTFVTVEVSX01EU0RfQUNDT1VOVCwgQ0xVU1RFUl9NRFNEX0NPTkZJR19WRVJTSU9OLCBHQVRFV0FZX0RPTUFJTlMsIEdBVEVXQVlfUkVTT1VSQ0VHUk9VUCwgTURTRF9FTlZJUk9OTUVOVCBDTFVTVEVSX01EU0RfTkFNRVNQQUNFCiMgYXJlIG5vdCB1c2VkLCBidXQgY2FuJ3QgZWFzaWx5IGJlIHJlZmFjdG9yZWQgb3V0LiBTaG91bGQgYmUgcmV2aXNpdGVkIGluIHRoZSBmdXR1cmUuCmVjaG8gImNvbmZpZ3VyaW5nIGFyby1tb25pdG9yIHNlcnZpY2UiCmNhdCA+L2V0Yy9zeXNjb25maWcvYXJvLW1vbml0b3IgPDxFT0YKQVpVUkVfRlBfQ0xJRU5UX0lEPSckRlBDTElFTlRJRCcKRE9NQUlOX05BTUU9JyRMT0NBVElPTi4kQ0xVU1RFUlBBUkVOVERPTUFJTk5BTUUnCkNMVVNURVJfTURTRF9BQ0NPVU5UPSckQ0xVU1RFUk1EU0RBQ0NPVU5UJwpDTFVTVEVSX01EU0RfQ09ORklHX1ZFUlNJT049JyRDTFVTVEVSTURTRENPTkZJR1ZFUlNJT04nCkdBVEVXQVlfRE9NQUlOUz0nJEdBVEVXQVlET01BSU5TJwpHQVRFV0FZX1JFU09VUkNFR1JPVVA9JyRHQVRFV0FZUkVTT1VSQ0VHUk9VUE5BTUUnCk1EU0RfRU5WSVJPTk1FTlQ9JyRNRFNERU5WSVJPTk1FTlQnCkNMVVNURVJfTURTRF9OQU1FU1BBQ0U9JyRDTFVTVEVSTURTRE5BTUVTUEFDRScKQ0xVU1RFUl9NRE1fQUNDT1VOVD0nJENMVVNURVJNRE1BQ0NPVU5UJwpDTFVTVEVSX01ETV9OQU1FU1BBQ0U9QkJNCkRBVEFCQVNFX0FDQ09VTlRfTkFNRT0nJERBVEFCQVNFQUNDT1VOVE5BTUUnCktFWVZBVUxUX1BSRUZJWD0nJEtFWVZBVUxUUFJFRklYJwpNRE1fQUNDT1VOVD0nJFJQTURNQUNDT1VOVCcKTURNX05BTUVTUEFDRT1CQk0KUlBJTUFHRT0nJFJQSU1BR0UnCkVPRgoKY2F0ID4vZXRjL3N5c3RlbWQvc3lzdGVtL2Fyby1tb25pdG9yLnNlcnZpY2UgPDwnRU9GJwpbVW5pdF0KQWZ0ZXI9bmV0d29yay1vbmxpbmUudGFyZ2V0CldhbnRzPW5ldHdvcmstb25saW5lLnRhcmdldAoKW1NlcnZpY2VdCkVudmlyb25tZW50RmlsZT0vZXRjL3N5c2NvbmZpZy9hcm8tbW9uaXRvcgpFeGVjU3RhcnRQcmU9LS91c3IvYmluL2RvY2tlciBybSAtZiAlTgpFeGVjU3RhcnQ9L3Vzci9iaW4vZG9ja2VyIHJ1biBcCiAgLS1ob3N0bmFtZSAlSCBcCiAgLS1uYW1lICVOIFwKICAtLXJtIFwKICAtLWNhcC1kcm9wIG5ldF9yYXcgXAogIC1lIEFaVVJFX0ZQX0NMSUVOVF9JRCBcCiAgLWUgRE9NQUlOX05BTUUgXAogIC1lIENMVVNURVJfTURTRF9BQ0NPVU5UIFwKICAtZSBDTFVTVEVSX01EU0RfQ09ORklHX1ZFUlNJT04gXAogIC1lIEdBVEVXQVlfRE9NQUlOUyBcCiAgLWUgR0FURVdBWV9SRVNPVVJDRUdST1VQIFwKICAtZSBNRFNEX0VOVklST05NRU5UIFwKICAtZSBDTFVTVEVSX01EU0RfTkFNRVNQQUNFIFwKICAtZSBDTFVTVEVSX01ETV9BQ0NPVU5UIFwKICAtZSBDTFVTVEVSX01ETV9OQU1FU1BBQ0UgXAogIC1lIERBVEFCQVNFX0FDQ09VTlRfTkFNRSBcCiAgLWUgS0VZVkFVTFRfUFJFRklYIFwKICAtZSBNRE1fQUNDT1VOVCBcCiAgLWUgTURNX05BTUVTUEFDRSBcCiAgLW0gMi41ZyBcCiAgLXYgL3J1bi9zeXN0ZW1kL2pvdXJuYWw6L3J1bi9zeXN0ZW1kL2pvdXJuYWwgXAogIC12IC92YXIvZXR3Oi92YXIvZXR3OnogXAogICRSUElNQUdFIFwKICBtb25pdG9yClJlc3RhcnQ9YWx3YXlzClJlc3RhcnRTZWM9MQpTdGFydExpbWl0SW50ZXJ2YWw9MAoKW0luc3RhbGxdCldhbnRlZEJ5PW11bHRpLXVzZXIudGFyZ2V0CkVPRgoKZWNobyAiY29uZmlndXJpbmcgYXJvLXBvcnRhbCBzZXJ2aWNlIgpjYXQgPi9ldGMvc3lzY29uZmlnL2Fyby1wb3J0YWwgPDxFT0YKQVpVUkVfUE9SVEFMX0FDQ0VTU19HUk9VUF9JRFM9JyRQT1JUQUxBQ0NFU1NHUk9VUElEUycKQVpVUkVfUE9SVEFMX0NMSUVOVF9JRD0nJFBPUlRBTENMSUVOVElEJwpBWlVSRV9QT1JUQUxfRUxFVkFURURfR1JPVVBfSURTPSckUE9SVEFMRUxFVkFURURHUk9VUElEUycKREFUQUJBU0VfQUNDT1VOVF9OQU1FPSckREFUQUJBU0VBQ0NPVU5UTkFNRScKS0VZVkFVTFRfUFJFRklYPSckS0VZVkFVTFRQUkVGSVgnCk1ETV9BQ0NPVU5UPSckUlBNRE1BQ0NPVU5UJwpNRE1fTkFNRVNQQUNFPVBvcnRhbApQT1JUQUxfSE9TVE5BTUU9JyRMT0NBVElPTi5hZG1pbi4kUlBQQVJFTlRET01BSU5OQU1FJwpSUElNQUdFPSckUlBJTUFHRScKRU9GCgpjYXQgPi9ldGMvc3lzdGVtZC9zeXN0ZW0vYXJvLXBvcnRhbC5zZXJ2aWNlIDw8J0VPRicKW1VuaXRdCkFmdGVyPW5ldHdvcmstb25saW5lLnRhcmdldApXYW50cz1uZXR3b3JrLW9ubGluZS50YXJnZXQKU3RhcnRMaW1pdEludGVydmFsPTAKCltTZXJ2aWNlXQpFbnZpcm9ubWVudEZpbGU9L2V0Yy9zeXNjb25maWcvYXJvLXBvcnRhbApFeGVjU3RhcnRQcmU9LS91c3IvYmluL2RvY2tlciBybSAtZiAlTgpFeGVjU3RhcnQ9L3Vzci9iaW4vZG9ja2VyIHJ1biBcCiAgLS1ob3N0bmFtZSAlSCBcCiAgLS1uYW1lICVOIFwKICAtLXJtIFwKICAtLWNhcC1kcm9wIG5ldF9yYXcgXAogIC1lIEFaVVJFX1BPUlRBTF9BQ0NFU1NfR1JPVVBfSURTIFwKICAtZSBBWlVSRV9QT1JUQUxfQ0xJRU5UX0lEIFwKICAtZSBBWlVSRV9QT1JUQUxfRUxFVkFURURfR1JPVVBfSURTIFwKICAtZSBEQVRBQkFTRV9BQ0NPVU5UX05BTUUgXAogIC1lIEtFWVZBVUxUX1BSRUZJWCBcCiAgLWUgTURNX0FDQ09VTlQgXAogIC1lIE1ETV9OQU1FU1BBQ0UgXAogIC1lIFBPUlRBTF9IT1NUTkFNRSBcCiAgLW0gMmcgXAogIC1wIDQ0NDo4NDQ0IFwKICAtcCAyMjIyOjIyMjIgXAogIC12IC9ydW4vc3lzdGVtZC9qb3VybmFsOi9ydW4vc3lzdGVtZC9qb3VybmFsIFwKICAtdiAvdmFyL2V0dzovdmFyL2V0dzp6IFwKICAkUlBJTUFHRSBcCiAgcG9ydGFsClJlc3RhcnQ9YWx3YXlzClJlc3RhcnRTZWM9MQoKW0luc3RhbGxdCldhbnRlZEJ5PW11bHRpLXVzZXIudGFyZ2V0CkVPRgoKZWNobyAiY29uZmlndXJpbmcgbWRzZCBhbmQgbWRtIHNlcnZpY2VzIgpjaGNvbiAtUiBzeXN0ZW1fdTpvYmplY3Rfcjp2YXJfbG9nX3Q6czAgL3Zhci9vcHQvbWljcm9zb2Z0L2xpbnV4bW9uYWdlbnQKCm1rZGlyIC1wIC92YXIvbGliL3dhYWdlbnQvTWljcm9zb2Z0LkF6dXJlLktleVZhdWx0LlN0b3JlCgpmb3IgdmFyIGluICJtZHNkIiAibWRtIjsgZG8KY2F0ID4vZXRjL3N5c3RlbWQvc3lzdGVtL2Rvd25sb2FkLSR2YXItY3JlZGVudGlhbHMuc2VydmljZSA8PEVPRgpbVW5pdF0KRGVzY3JpcHRpb249UGVyaW9kaWMgJHZhciBjcmVkZW50aWFscyByZWZyZXNoCgpbU2VydmljZV0KVHlwZT1vbmVzaG90CkV4ZWNTdGFydD0vdXNyL2xvY2FsL2Jpbi9kb3dubG9hZC1jcmVkZW50aWFscy5zaCAkdmFyCkVPRgoKY2F0ID4vZXRjL3N5c3RlbWQvc3lzdGVtL2Rvd25sb2FkLSR2YXItY3JlZGVudGlhbHMudGltZXIgPDxFT0YKW1VuaXRdCkRlc2NyaXB0aW9uPVBlcmlvZGljICR2YXIgY3JlZGVudGlhbHMgcmVmcmVzaApBZnRlcj1uZXR3b3JrLW9ubGluZS50YXJnZXQKV2FudHM9bmV0d29yay1vbmxpbmUudGFyZ2V0CgpbVGltZXJdCk9uQm9vdFNlYz0wbWluCk9uQ2FsZW5kYXI9MC8xMjowMDowMApBY2N1cmFjeVNlYz01cwoKW0luc3RhbGxdCldhbnRlZEJ5PXRpbWVycy50YXJnZXQKRU9GCmRvbmUKCmNhdCA+L3Vzci9sb2NhbC9iaW4vZG93bmxvYWQtY3JlZGVudGlhbHMuc2ggPDxFT0YKIyEvYmluL2Jhc2gKc2V0IC1ldQoKQ09NUE9ORU5UPSJcJDEiCmVjaG8gIkRvd25sb2FkIFwkQ09NUE9ORU5UIGNyZWRlbnRpYWxzIgoKVEVNUF9ESVI9XCQobWt0ZW1wIC1kKQpleHBvcnQgQVpVUkVfQ09ORklHX0RJUj1cJChta3RlbXAgLWQpCgplY2hvICJMb2dnaW5nIGludG8gQXp1cmUuLi4iClJFVFJJRVM9Mwp3aGlsZSBbICJcJFJFVFJJRVMiIC1ndCAwIF07IGRvCiAgICBpZiBheiBsb2dpbiAtaSAtLWFsbG93LW5vLXN1YnNjcmlwdGlvbnMKICAgIHRoZW4KICAgICAgICBlY2hvICJheiBsb2dpbiBzdWNjZXNzZnVsIgogICAgICAgIGJyZWFrCiAgICBlbHNlCiAgICAgICAgZWNobyAiYXogbG9naW4gZmFpbGVkLiBSZXRyeWluZy4uLiIKICAgICAgICBsZXQgUkVUUklFUy09MQogICAgICAgIHNsZWVwIDUKICAgIGZpCmRvbmUKCnRyYXAgImNsZWFudXAiIEVYSVQKCmNsZWFudXAoKSB7CiAgYXogbG9nb3V0CiAgW1sgIlwkVEVNUF9ESVIiID1+IC90bXAvLisgXV0gJiYgcm0gLXJmIFwkVEVNUF9ESVIKICBbWyAiXCRBWlVSRV9DT05GSUdfRElSIiA9fiAvdG1wLy4rIF1dICYmIHJtIC1yZiBcJEFaVVJFX0NPTkZJR19ESVIKfQoKaWYgWyAiXCRDT01QT05FTlQiID0gIm1kbSIgXTsgdGhlbgogIENVUlJFTlRfQ0VSVF9GSUxFPSIvZXRjL21kbS5wZW0iCmVsaWYgWyAiXCRDT01QT05FTlQiID0gIm1kc2QiIF07IHRoZW4KICBDVVJSRU5UX0NFUlRfRklMRT0iL3Zhci9saWIvd2FhZ2VudC9NaWNyb3NvZnQuQXp1cmUuS2V5VmF1bHQuU3RvcmUvbWRzZC5wZW0iCmVsc2UKICBlY2hvIEludmFsaWQgdXNhZ2UgJiYgZXhpdCAxCmZpCgpTRUNSRVRfTkFNRT0icnAtXCR7Q09NUE9ORU5UfSIKTkVXX0NFUlRfRklMRT0iXCRURU1QX0RJUi9cJENPTVBPTkVOVC5wZW0iCmZvciBhdHRlbXB0IGluIHsxLi41fTsgZG8KICBheiBrZXl2YXVsdCBzZWNyZXQgZG93bmxvYWQgLS1maWxlIFwkTkVXX0NFUlRfRklMRSAtLWlkICJodHRwczovLyRLRVlWQVVMVFBSRUZJWC1zdmMuJEtFWVZBVUxURE5TU1VGRklYL3NlY3JldHMvXCRTRUNSRVRfTkFNRSIgJiYgYnJlYWsKICBpZiBbWyBcJGF0dGVtcHQgLWx0IDUgXV07IHRoZW4gc2xlZXAgMTA7IGVsc2UgZXhpdCAxOyBmaQpkb25lCgppZiBbIC1mIFwkTkVXX0NFUlRfRklMRSBdOyB0aGVuCiAgaWYgWyAiXCRDT01QT05FTlQiID0gIm1kc2QiIF07IHRoZW4KICAgIGNob3duIHN5c2xvZzpzeXNsb2cgXCRORVdfQ0VSVF9GSUxFCiAgZWxzZQogICAgc2VkIC1pIC1uZSAnMSwvRU5EIENFUlRJRklDQVRFLyBwJyBcJE5FV19DRVJUX0ZJTEUKICBmaQoKICBuZXdfY2VydF9zbj0iXCQob3BlbnNzbCB4NTA5IC1pbiAiXCRORVdfQ0VSVF9GSUxFIiAtbm9vdXQgLXNlcmlhbCB8IGF3ayAtRj0gJ3twcmludCBcJDJ9JykiCiAgY3VycmVudF9jZXJ0X3NuPSJcJChvcGVuc3NsIHg1MDkgLWluICJcJENVUlJFTlRfQ0VSVF9GSUxFIiAtbm9vdXQgLXNlcmlhbCB8IGF3ayAtRj0gJ3twcmludCBcJDJ9JykiCiAgaWYgW1sgISAteiBcJG5ld19jZXJ0X3NuIF1dICYmIFtbIFwkbmV3X2NlcnRfc24gIT0gIlwkY3VycmVudF9jZXJ0X3NuIiBdXTsgdGhlbgogICAgZWNobyB1cGRhdGluZyBjZXJ0aWZpY2F0ZSBmb3IgXCRDT01QT05FTlQKICAgIGNobW9kIDA2MDAgXCRORVdfQ0VSVF9GSUxFCiAgICBtdiBcJE5FV19DRVJUX0ZJTEUgXCRDVVJSRU5UX0NFUlRfRklMRQogIGZpCmVsc2UKICBlY2hvIEZhaWxlZCB0byByZWZyZXNoIGNlcnRpZmljYXRlIGZvciBcJENPTVBPTkVOVCAmJiBleGl0IDEKZmkKRU9GCgpjaG1vZCB1K3ggL3Vzci9sb2NhbC9iaW4vZG93bmxvYWQtY3JlZGVudGlhbHMuc2gKCnN5c3RlbWN0bCBlbmFibGUgZG93bmxvYWQtbWRzZC1jcmVkZW50aWFscy50aW1lcgpzeXN0ZW1jdGwgZW5hYmxlIGRvd25sb2FkLW1kbS1jcmVkZW50aWFscy50aW1lcgoKL3Vzci9sb2NhbC9iaW4vZG93bmxvYWQtY3JlZGVudGlhbHMuc2ggbWRzZAovdXNyL2xvY2FsL2Jpbi9kb3dubG9hZC1jcmVkZW50aWFscy5zaCBtZG0KTURTRENFUlRJRklDQVRFU0FOPSQob3BlbnNzbCB4NTA5IC1pbiAvdmFyL2xpYi93YWFnZW50L01pY3Jvc29mdC5BenVyZS5LZXlWYXVsdC5TdG9yZS9tZHNkLnBlbSAtbm9vdXQgLXN1YmplY3QgfCBzZWQgLWUgJ3MvLipDTiA9IC8vJykKCmNhdCA+L2V0Yy9zeXN0ZW1kL3N5c3RlbS93YXRjaC1tZG0tY3JlZGVudGlhbHMuc2VydmljZSA8PEVPRgpbVW5pdF0KRGVzY3JpcHRpb249V2F0Y2ggZm9yIGNoYW5nZXMgaW4gbWRtLnBlbSBhbmQgcmVzdGFydHMgdGhlIG1kbSBzZXJ2aWNlCgpbU2VydmljZV0KVHlwZT1vbmVzaG90CkV4ZWNTdGFydD0vdXNyL2Jpbi9zeXN0ZW1jdGwgcmVzdGFydCBtZG0uc2VydmljZQoKW0luc3RhbGxdCldhbnRlZEJ5PW11bHRpLXVzZXIudGFyZ2V0CkVPRgoKY2F0ID4vZXRjL3N5c3RlbWQvc3lzdGVtL3dhdGNoLW1kbS1jcmVkZW50aWFscy5wYXRoIDw8RU9GCltQYXRoXQpQYXRoTW9kaWZpZWQ9L2V0Yy9tZG0ucGVtCgpbSW5zdGFsbF0KV2FudGVkQnk9bXVsdGktdXNlci50YXJnZXQKRU9GCgpzeXN0ZW1jdGwgZW5hYmxlIHdhdGNoLW1kbS1jcmVkZW50aWFscy5wYXRoCnN5c3RlbWN0bCBzdGFydCB3YXRjaC1tZG0tY3JlZGVudGlhbHMucGF0aAoKbWtkaXIgL2V0Yy9zeXN0ZW1kL3N5c3RlbS9tZHNkLnNlcnZpY2UuZApjYXQgPi9ldGMvc3lzdGVtZC9zeXN0ZW0vbWRzZC5zZXJ2aWNlLmQvb3ZlcnJpZGUuY29uZiA8PCdFT0YnCltVbml0XQpBZnRlcj1uZXR3b3JrLW9ubGluZS50YXJnZXQKRU9GCgpjYXQgPi9ldGMvZGVmYXVsdC9tZHNkIDw8RU9GCk1EU0RfUk9MRV9QUkVGSVg9L3Zhci9ydW4vbWRzZC9kZWZhdWx0Ck1EU0RfT1BUSU9OUz0iLUEgLWQgLXIgXCRNRFNEX1JPTEVfUFJFRklYIgoKZXhwb3J0IE1PTklUT1JJTkdfR0NTX0VOVklST05NRU5UPSckTURTREVOVklST05NRU5UJwpleHBvcnQgTU9OSVRPUklOR19HQ1NfQUNDT1VOVD0nJFJQTURTREFDQ09VTlQnCmV4cG9ydCBNT05JVE9SSU5HX0dDU19SRUdJT049JyRMT0NBVElPTicKZXhwb3J0IE1PTklUT1JJTkdfR0NTX0FVVEhfSURfVFlQRT1BdXRoS2V5VmF1bHQKZXhwb3J0IE1PTklUT1JJTkdfR0NTX0FVVEhfSUQ9JyRNRFNEQ0VSVElGSUNBVEVTQU4nCmV4cG9ydCBNT05JVE9SSU5HX0dDU19OQU1FU1BBQ0U9JyRSUE1EU0ROQU1FU1BBQ0UnCmV4cG9ydCBNT05JVE9SSU5HX0NPTkZJR19WRVJTSU9OPSckUlBNRFNEQ09ORklHVkVSU0lPTicKZXhwb3J0IE1PTklUT1JJTkdfVVNFX0dFTkVWQV9DT05GSUdfU0VSVklDRT10cnVlCgpleHBvcnQgTU9OSVRPUklOR19URU5BTlQ9JyRMT0NBVElPTicKZXhwb3J0IE1PTklUT1JJTkdfUk9MRT1ycApleHBvcnQgTU9OSVRPUklOR19ST0xFX0lOU1RBTkNFPSckKGhvc3RuYW1lKScKCmV4cG9ydCBNRFNEX01TR1BBQ0tfU09SVF9DT0xVTU5TPTEKRU9GCgojIHNldHRpbmcgTU9OSVRPUklOR19HQ1NfQVVUSF9JRF9UWVBFPUF1dGhLZXlWYXVsdCBzZWVtcyB0byBoYXZlIGNhdXNlZCBtZHNkIG5vdAojIHRvIGhvbm91ciBTU0xfQ0VSVF9GSUxFIGFueSBtb3JlLCBoZWF2ZW4gb25seSBrbm93cyB3aHkuCm1rZGlyIC1wIC91c3IvbGliL3NzbC9jZXJ0cwpjc3BsaXQgLWYgL3Vzci9saWIvc3NsL2NlcnRzL2NlcnQtIC1iICUwM2QucGVtIC9ldGMvcGtpL3Rscy9jZXJ0cy9jYS1idW5kbGUuY3J0IC9eJC8xIHsqfSA+L2Rldi9udWxsCmNfcmVoYXNoIC91c3IvbGliL3NzbC9jZXJ0cwoKIyB3ZSBsZWF2ZSBjbGllbnRJZCBibGFuayBhcyBsb25nIGFzIG9ubHkgMSBtYW5hZ2VkIGlkZW50aXR5IGFzc2lnbmVkIHRvIHZtc3MKIyBpZiB3ZSBoYXZlIG1vcmUgdGhhbiAxLCB3ZSB3aWxsIG5lZWQgdG8gcG9wdWxhdGUgd2l0aCBjbGllbnRJZCB1c2VkIGZvciBvZmYtbm9kZSBzY2FubmluZwpjYXQgPi9ldGMvZGVmYXVsdC92c2Etbm9kZXNjYW4tYWdlbnQuY29uZmlnIDw8RU9GCnsKICAgICJOaWNlIjogMTksCiAgICAiVGltZW91dCI6IDEwODAwLAogICAgIkNsaWVudElkIjogIiIsCiAgICAiVGVuYW50SWQiOiAiJEFaVVJFU0VDUEFDS1ZTQVRFTkFOVElEIiwKICAgICJRdWFseXNTdG9yZUJhc2VVcmwiOiAiJEFaVVJFU0VDUEFDS1FVQUxZU1VSTCIsCiAgICAiUHJvY2Vzc1RpbWVvdXQiOiAzMDAsCiAgICAiQ29tbWFuZERlbGF5IjogMAogIH0KRU9GCgplY2hvICJlbmFibGluZyBhcm8gc2VydmljZXMiCmZvciBzZXJ2aWNlIGluIGFyby1kYnRva2VuIGFyby1tb25pdG9yIGFyby1wb3J0YWwgYXJvLXJwIGF1b21zIGF6c2VjZCBhenNlY21vbmQgbWRzZCBtZG0gY2hyb255ZCBmbHVlbnRiaXQ7IGRvCiAgc3lzdGVtY3RsIGVuYWJsZSAkc2VydmljZS5zZXJ2aWNlCmRvbmUKCmZvciBzY2FuIGluIGJhc2VsaW5lIGNsYW1hdiBzb2Z0d2FyZTsgZG8KICAvdXNyL2xvY2FsL2Jpbi9henNlY2QgY29uZmlnIC1zICRzY2FuIC1kIFAxRApkb25lCgplY2hvICJyZWJvb3RpbmciCnJlc3RvcmVjb24gLVJGIC92YXIvbG9nLyoKKHNsZWVwIDMwOyByZWJvb3QpICYK')))]" } } } @@ -535,15 +531,6 @@ "[resourceId('Microsoft.Storage/storageAccounts', substring(parameters('storageAccountDomain'), 0, indexOf(parameters('storageAccountDomain'), '.')))]" ] }, - { - "sku": { - "name": "Standard_LRS" - }, - "location": "[resourceGroup().location]", - "name": "[substring(parameters('storageAccountDomain'), 0, indexOf(parameters('storageAccountDomain'), '.'))]", - "type": "Microsoft.Storage/storageAccounts", - "apiVersion": "2019-06-01" - }, { "properties": { "severity": 2, diff --git a/pkg/deploy/generator/resources_gateway.go b/pkg/deploy/generator/resources_gateway.go index 662df3de5a4..032f1863fb3 100644 --- a/pkg/deploy/generator/resources_gateway.go +++ b/pkg/deploy/generator/resources_gateway.go @@ -346,8 +346,7 @@ func (g *generator) gatewayVMSS() *arm.Resource { }, DiagnosticsProfile: &mgmtcompute.DiagnosticsProfile{ BootDiagnostics: &mgmtcompute.BootDiagnostics{ - Enabled: to.BoolPtr(true), - StorageURI: to.StringPtr("[concat('https://', parameters('gatewayStorageAccountDomain'), '/')]"), + Enabled: to.BoolPtr(true), }, }, }, @@ -419,7 +418,3 @@ func (g *generator) gatewayRBAC() []*arm.Resource { ), } } - -func (g *generator) gatewayStorageAccount() *arm.Resource { - return g.storageAccount("[substring(parameters('gatewayStorageAccountDomain'), 0, indexOf(parameters('gatewayStorageAccountDomain'), '.'))]", nil, nil) -} diff --git a/pkg/deploy/generator/resources_rp.go b/pkg/deploy/generator/resources_rp.go index 3d961e70dff..809b7830f74 100644 --- a/pkg/deploy/generator/resources_rp.go +++ b/pkg/deploy/generator/resources_rp.go @@ -634,8 +634,7 @@ func (g *generator) rpVMSS() *arm.Resource { }, DiagnosticsProfile: &mgmtcompute.DiagnosticsProfile{ BootDiagnostics: &mgmtcompute.BootDiagnostics{ - Enabled: to.BoolPtr(true), - StorageURI: to.StringPtr("[concat('https://', parameters('storageAccountDomain'), '/')]"), + Enabled: to.BoolPtr(true), }, }, }, @@ -1559,7 +1558,3 @@ func (g *generator) rpVersionStorageAccount() []*arm.Resource { }, } } - -func (g *generator) rpStorageAccount() *arm.Resource { - return g.storageAccount("[substring(parameters('storageAccountDomain'), 0, indexOf(parameters('storageAccountDomain'), '.'))]", nil, nil) -} diff --git a/pkg/deploy/generator/scripts/gatewayVMSS.sh b/pkg/deploy/generator/scripts/gatewayVMSS.sh index 2f9b09efea1..464ca60cd9d 100644 --- a/pkg/deploy/generator/scripts/gatewayVMSS.sh +++ b/pkg/deploy/generator/scripts/gatewayVMSS.sh @@ -150,7 +150,7 @@ touch /etc/containers/nodocker mkdir -p /root/.docker REGISTRY_AUTH_FILE=/root/.docker/config.json az acr login --name "$(sed -e 's|.*/||' <<<"$ACRRESOURCEID")" -MDMIMAGE="${RPIMAGE%%/*}/${MDMIMAGE##*/}" +MDMIMAGE="${RPIMAGE%%/*}/${MDMIMAGE#*/}" docker pull "$MDMIMAGE" docker pull "$RPIMAGE" docker pull "$FLUENTBITIMAGE" diff --git a/pkg/deploy/generator/scripts/rpVMSS.sh b/pkg/deploy/generator/scripts/rpVMSS.sh index 78149f30451..cc654dc87f0 100644 --- a/pkg/deploy/generator/scripts/rpVMSS.sh +++ b/pkg/deploy/generator/scripts/rpVMSS.sh @@ -129,7 +129,7 @@ touch /etc/containers/nodocker mkdir -p /root/.docker REGISTRY_AUTH_FILE=/root/.docker/config.json az acr login --name "$(sed -e 's|.*/||' <<<"$ACRRESOURCEID")" -MDMIMAGE="${RPIMAGE%%/*}/${MDMIMAGE##*/}" +MDMIMAGE="${RPIMAGE%%/*}/${MDMIMAGE#*/}" docker pull "$MDMIMAGE" docker pull "$RPIMAGE" docker pull "$FLUENTBITIMAGE" diff --git a/pkg/deploy/generator/templates_gateway.go b/pkg/deploy/generator/templates_gateway.go index 958aef22554..f7ed8a2e04a 100644 --- a/pkg/deploy/generator/templates_gateway.go +++ b/pkg/deploy/generator/templates_gateway.go @@ -34,7 +34,6 @@ func (g *generator) gatewayTemplate() *arm.Template { "gatewayFeatures", "gatewayMdsdConfigVersion", "gatewayServicePrincipalId", - "gatewayStorageAccountDomain", "gatewayVmSize", "gatewayVmssCapacity", "keyvaultDNSSuffix", @@ -84,7 +83,6 @@ func (g *generator) gatewayTemplate() *arm.Template { } t.Resources = append(t.Resources, - g.gatewayStorageAccount(), g.gatewayLB(), g.gatewayPLS(), g.gatewayVMSS(), diff --git a/pkg/deploy/generator/templates_rp.go b/pkg/deploy/generator/templates_rp.go index d5506af6441..3d33f84da0b 100644 --- a/pkg/deploy/generator/templates_rp.go +++ b/pkg/deploy/generator/templates_rp.go @@ -73,7 +73,6 @@ func (g *generator) rpTemplate() *arm.Template { "rpParentDomainName", "rpVmssCapacity", "sshPublicKey", - "storageAccountDomain", "subscriptionResourceGroupName", "vmSize", "vmssCleanupEnabled", @@ -168,7 +167,6 @@ func (g *generator) rpTemplate() *arm.Template { g.rpLB(), g.rpLBInternal(), g.rpVMSS(), - g.rpStorageAccount(), g.rpLBAlert(30.0, 2, "rp-availability-alert", "PT5M", "PT15M", "DipAvailability"), // triggers on all 3 RPs being down for 10min, can't be >=0.3 due to deploys going down to 32% at times. g.rpLBAlert(67.0, 3, "rp-degraded-alert", "PT15M", "PT6H", "DipAvailability"), // 1/3 backend down for 1h or 2/3 down for 3h in the last 6h g.rpLBAlert(33.0, 2, "rp-vnet-alert", "PT5M", "PT5M", "VipAvailability")) // this will trigger only if the Azure network infrastructure between the loadBalancers and VMs is down for 3.5min diff --git a/pkg/frontend/admin_hive_clusterdeployment_get_test.go b/pkg/frontend/admin_hive_clusterdeployment_get_test.go index cbaa7063fa6..4e40e4516d8 100644 --- a/pkg/frontend/admin_hive_clusterdeployment_get_test.go +++ b/pkg/frontend/admin_hive_clusterdeployment_get_test.go @@ -90,10 +90,10 @@ func Test_getAdminHiveClusterDeployment(t *testing.T) { clusterManager := mock_hive.NewMockClusterManager(controller) clusterManager.EXPECT().GetClusterDeployment(gomock.Any(), gomock.Any()).Return(&clusterDeployment, nil).Times(tt.expectedGetClusterDeploymentCallCount) f, err = NewFrontend(ctx, ti.audit, ti.log, _env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, - ti.subscriptionsDatabase, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, clusterManager, nil, nil, nil) + ti.subscriptionsDatabase, nil, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, clusterManager, nil, nil, nil) } else { f, err = NewFrontend(ctx, ti.audit, ti.log, _env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, - ti.subscriptionsDatabase, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, nil, nil) + ti.subscriptionsDatabase, nil, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, nil, nil) } if err != nil { diff --git a/pkg/frontend/admin_openshiftcluster_approvecsr_test.go b/pkg/frontend/admin_openshiftcluster_approvecsr_test.go index 3bef4853bad..53b76de1753 100644 --- a/pkg/frontend/admin_openshiftcluster_approvecsr_test.go +++ b/pkg/frontend/admin_openshiftcluster_approvecsr_test.go @@ -91,7 +91,7 @@ func TestAdminApproveCSR(t *testing.T) { t.Fatal(err) } - f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, func(*logrus.Entry, env.Interface, *api.OpenShiftCluster) (adminactions.KubeActions, error) { + f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, nil, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, func(*logrus.Entry, env.Interface, *api.OpenShiftCluster) (adminactions.KubeActions, error) { return k, nil }, nil, nil) diff --git a/pkg/frontend/admin_openshiftcluster_cordonnode_test.go b/pkg/frontend/admin_openshiftcluster_cordonnode_test.go index d524af4a757..52097ef559c 100644 --- a/pkg/frontend/admin_openshiftcluster_cordonnode_test.go +++ b/pkg/frontend/admin_openshiftcluster_cordonnode_test.go @@ -152,7 +152,7 @@ func TestAdminCordonUncordonNode(t *testing.T) { t.Fatal(err) } - f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, func(*logrus.Entry, env.Interface, *api.OpenShiftCluster) (adminactions.KubeActions, error) { + f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, nil, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, func(*logrus.Entry, env.Interface, *api.OpenShiftCluster) (adminactions.KubeActions, error) { return k, nil }, nil, nil) diff --git a/pkg/frontend/admin_openshiftcluster_delete_managedresource_test.go b/pkg/frontend/admin_openshiftcluster_delete_managedresource_test.go index 4215af7ceaa..3fd49d5b22a 100644 --- a/pkg/frontend/admin_openshiftcluster_delete_managedresource_test.go +++ b/pkg/frontend/admin_openshiftcluster_delete_managedresource_test.go @@ -113,7 +113,7 @@ func TestAdminDeleteManagedResource(t *testing.T) { a := mock_adminactions.NewMockAzureActions(ti.controller) tt.mocks(tt, a) - f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, func(*logrus.Entry, env.Interface, *api.OpenShiftCluster, *api.SubscriptionDocument) (adminactions.AzureActions, error) { + f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, nil, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, func(*logrus.Entry, env.Interface, *api.OpenShiftCluster, *api.SubscriptionDocument) (adminactions.AzureActions, error) { return a, nil }, nil) diff --git a/pkg/frontend/admin_openshiftcluster_drainnode_test.go b/pkg/frontend/admin_openshiftcluster_drainnode_test.go index b7d94da3587..c23a6b968b8 100644 --- a/pkg/frontend/admin_openshiftcluster_drainnode_test.go +++ b/pkg/frontend/admin_openshiftcluster_drainnode_test.go @@ -84,7 +84,7 @@ func TestAdminDrainNode(t *testing.T) { t.Fatal(err) } - f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, func(*logrus.Entry, env.Interface, *api.OpenShiftCluster) (adminactions.KubeActions, error) { + f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, nil, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, func(*logrus.Entry, env.Interface, *api.OpenShiftCluster) (adminactions.KubeActions, error) { return k, nil }, nil, nil) diff --git a/pkg/frontend/admin_openshiftcluster_etcdcertificaterenew_test.go b/pkg/frontend/admin_openshiftcluster_etcdcertificaterenew_test.go index a26cf2d1d2d..ab58f9615cd 100644 --- a/pkg/frontend/admin_openshiftcluster_etcdcertificaterenew_test.go +++ b/pkg/frontend/admin_openshiftcluster_etcdcertificaterenew_test.go @@ -536,6 +536,7 @@ func TestAdminEtcdCertificateRenew(t *testing.T) { ti.openShiftClustersDatabase, ti.subscriptionsDatabase, nil, + nil, api.APIs, &noop.Noop{}, &noop.Noop{}, @@ -753,6 +754,7 @@ func TestAdminEtcdCertificateRecovery(t *testing.T) { ti.openShiftClustersDatabase, ti.subscriptionsDatabase, nil, + nil, api.APIs, &noop.Noop{}, &noop.Noop{}, diff --git a/pkg/frontend/admin_openshiftcluster_etcdrecovery_test.go b/pkg/frontend/admin_openshiftcluster_etcdrecovery_test.go index f2f405340ef..2e15ccb7e35 100644 --- a/pkg/frontend/admin_openshiftcluster_etcdrecovery_test.go +++ b/pkg/frontend/admin_openshiftcluster_etcdrecovery_test.go @@ -167,6 +167,7 @@ func TestAdminEtcdRecovery(t *testing.T) { ti.openShiftClustersDatabase, ti.subscriptionsDatabase, nil, + nil, api.APIs, &noop.Noop{}, &noop.Noop{}, diff --git a/pkg/frontend/admin_openshiftcluster_kubernetesobjects_test.go b/pkg/frontend/admin_openshiftcluster_kubernetesobjects_test.go index 604f064b31c..b0b2e06bd75 100644 --- a/pkg/frontend/admin_openshiftcluster_kubernetesobjects_test.go +++ b/pkg/frontend/admin_openshiftcluster_kubernetesobjects_test.go @@ -262,7 +262,7 @@ func TestAdminKubernetesObjectsGetAndDelete(t *testing.T) { t.Fatal(err) } - f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, func(*logrus.Entry, env.Interface, *api.OpenShiftCluster) (adminactions.KubeActions, error) { + f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, nil, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, func(*logrus.Entry, env.Interface, *api.OpenShiftCluster) (adminactions.KubeActions, error) { return k, nil }, nil, nil) if err != nil { @@ -411,7 +411,7 @@ func TestAdminPostKubernetesObjects(t *testing.T) { t.Fatal(err) } - f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, func(*logrus.Entry, env.Interface, *api.OpenShiftCluster) (adminactions.KubeActions, error) { + f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, nil, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, func(*logrus.Entry, env.Interface, *api.OpenShiftCluster) (adminactions.KubeActions, error) { return k, nil }, nil, nil) if err != nil { diff --git a/pkg/frontend/admin_openshiftcluster_kubernetespods_logs_test.go b/pkg/frontend/admin_openshiftcluster_kubernetespods_logs_test.go index 91b84279719..8dd8c53b563 100644 --- a/pkg/frontend/admin_openshiftcluster_kubernetespods_logs_test.go +++ b/pkg/frontend/admin_openshiftcluster_kubernetespods_logs_test.go @@ -125,7 +125,7 @@ func TestAdminKubernetesGetPodLogs(t *testing.T) { t.Fatal(err) } - f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, func(*logrus.Entry, env.Interface, *api.OpenShiftCluster) (adminactions.KubeActions, error) { + f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, nil, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, func(*logrus.Entry, env.Interface, *api.OpenShiftCluster) (adminactions.KubeActions, error) { return k, nil }, nil, nil) if err != nil { diff --git a/pkg/frontend/admin_openshiftcluster_list_test.go b/pkg/frontend/admin_openshiftcluster_list_test.go index c39934e30ef..27922d2a433 100644 --- a/pkg/frontend/admin_openshiftcluster_list_test.go +++ b/pkg/frontend/admin_openshiftcluster_list_test.go @@ -124,7 +124,7 @@ func TestAdminListOpenShiftCluster(t *testing.T) { ti.openShiftClustersClient.SetError(tt.throwsError) } - f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, aead, nil, nil, nil, ti.enricher) + f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, nil, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, aead, nil, nil, nil, ti.enricher) if err != nil { t.Fatal(err) } diff --git a/pkg/frontend/admin_openshiftcluster_redeployvm_test.go b/pkg/frontend/admin_openshiftcluster_redeployvm_test.go index fef1bdcf051..818dcea16e7 100644 --- a/pkg/frontend/admin_openshiftcluster_redeployvm_test.go +++ b/pkg/frontend/admin_openshiftcluster_redeployvm_test.go @@ -84,7 +84,7 @@ func TestAdminRedeployVM(t *testing.T) { t.Fatal(err) } - f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, func(*logrus.Entry, env.Interface, *api.OpenShiftCluster, *api.SubscriptionDocument) (adminactions.AzureActions, error) { + f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, nil, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, func(*logrus.Entry, env.Interface, *api.OpenShiftCluster, *api.SubscriptionDocument) (adminactions.AzureActions, error) { return a, nil }, nil) diff --git a/pkg/frontend/admin_openshiftcluster_resources_list_test.go b/pkg/frontend/admin_openshiftcluster_resources_list_test.go index 0fdc9a64f8f..bb56f0bebd3 100644 --- a/pkg/frontend/admin_openshiftcluster_resources_list_test.go +++ b/pkg/frontend/admin_openshiftcluster_resources_list_test.go @@ -93,7 +93,7 @@ func TestAdminListResourcesList(t *testing.T) { t.Fatal(err) } - f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, func(*logrus.Entry, env.Interface, *api.OpenShiftCluster, *api.SubscriptionDocument) (adminactions.AzureActions, error) { + f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, nil, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, func(*logrus.Entry, env.Interface, *api.OpenShiftCluster, *api.SubscriptionDocument) (adminactions.AzureActions, error) { return a, nil }, nil) mockResponder := mock_frontend.NewMockStreamResponder(ti.controller) diff --git a/pkg/frontend/admin_openshiftcluster_startvm_test.go b/pkg/frontend/admin_openshiftcluster_startvm_test.go index cf0eadb9f01..e7a91708bd4 100644 --- a/pkg/frontend/admin_openshiftcluster_startvm_test.go +++ b/pkg/frontend/admin_openshiftcluster_startvm_test.go @@ -84,7 +84,7 @@ func TestAdminStartVM(t *testing.T) { t.Fatal(err) } - f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, func(*logrus.Entry, env.Interface, *api.OpenShiftCluster, *api.SubscriptionDocument) (adminactions.AzureActions, error) { + f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, nil, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, func(*logrus.Entry, env.Interface, *api.OpenShiftCluster, *api.SubscriptionDocument) (adminactions.AzureActions, error) { return a, nil }, nil) diff --git a/pkg/frontend/admin_openshiftcluster_stopvm_test.go b/pkg/frontend/admin_openshiftcluster_stopvm_test.go index 77fcd351d28..bb35093ed84 100644 --- a/pkg/frontend/admin_openshiftcluster_stopvm_test.go +++ b/pkg/frontend/admin_openshiftcluster_stopvm_test.go @@ -86,7 +86,7 @@ func TestAdminStopVM(t *testing.T) { t.Fatal(err) } - f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, func(*logrus.Entry, env.Interface, *api.OpenShiftCluster, *api.SubscriptionDocument) (adminactions.AzureActions, error) { + f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, nil, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, func(*logrus.Entry, env.Interface, *api.OpenShiftCluster, *api.SubscriptionDocument) (adminactions.AzureActions, error) { return a, nil }, nil) diff --git a/pkg/frontend/admin_openshiftcluster_vmresize_test.go b/pkg/frontend/admin_openshiftcluster_vmresize_test.go index 0d5946f31eb..6eaa65f9acd 100644 --- a/pkg/frontend/admin_openshiftcluster_vmresize_test.go +++ b/pkg/frontend/admin_openshiftcluster_vmresize_test.go @@ -202,7 +202,7 @@ func TestAdminVMResize(t *testing.T) { t.Fatal(err) } - f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, + f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, nil, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, func(*logrus.Entry, env.Interface, *api.OpenShiftCluster, *api.SubscriptionDocument) (adminactions.AzureActions, error) { return a, nil }, nil) diff --git a/pkg/frontend/admin_openshiftcluster_vmsizelist_test.go b/pkg/frontend/admin_openshiftcluster_vmsizelist_test.go index 77fa4ea51b8..1d7fda2cabd 100644 --- a/pkg/frontend/admin_openshiftcluster_vmsizelist_test.go +++ b/pkg/frontend/admin_openshiftcluster_vmsizelist_test.go @@ -136,7 +136,7 @@ func TestAdminListVMSizeList(t *testing.T) { t.Fatal(err) } - f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, func(*logrus.Entry, env.Interface, *api.OpenShiftCluster, *api.SubscriptionDocument) (adminactions.AzureActions, error) { + f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, nil, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, func(*logrus.Entry, env.Interface, *api.OpenShiftCluster, *api.SubscriptionDocument) (adminactions.AzureActions, error) { return a, nil }, nil) diff --git a/pkg/frontend/admin_openshiftversion_list_test.go b/pkg/frontend/admin_openshiftversion_list_test.go index c851c806bb4..ff18ac7aebf 100644 --- a/pkg/frontend/admin_openshiftversion_list_test.go +++ b/pkg/frontend/admin_openshiftversion_list_test.go @@ -110,7 +110,7 @@ func TestOpenShiftVersionList(t *testing.T) { t.Fatal(err) } - f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, nil, nil, nil, nil, ti.openShiftVersionsDatabase, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, nil, nil) + f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, nil, nil, nil, nil, ti.openShiftVersionsDatabase, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, nil, nil) if err != nil { t.Fatal(err) diff --git a/pkg/frontend/admin_openshiftversion_put_test.go b/pkg/frontend/admin_openshiftversion_put_test.go index e92e10cb909..efef5a07b6a 100644 --- a/pkg/frontend/admin_openshiftversion_put_test.go +++ b/pkg/frontend/admin_openshiftversion_put_test.go @@ -266,7 +266,7 @@ func TestOpenShiftVersionPut(t *testing.T) { t.Fatal(err) } - f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, nil, nil, nil, nil, ti.openShiftVersionsDatabase, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, nil, nil) + f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, nil, nil, nil, nil, ti.openShiftVersionsDatabase, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, nil, nil) if err != nil { t.Fatal(err) } diff --git a/pkg/frontend/admin_platformworkloadidentityroleset_list.go b/pkg/frontend/admin_platformworkloadidentityroleset_list.go new file mode 100644 index 00000000000..855b808e2e1 --- /dev/null +++ b/pkg/frontend/admin_platformworkloadidentityroleset_list.go @@ -0,0 +1,47 @@ +package frontend + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "encoding/json" + "net/http" + "path/filepath" + "sort" + + "github.com/sirupsen/logrus" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/api/admin" + "github.com/Azure/ARO-RP/pkg/frontend/middleware" + "github.com/Azure/ARO-RP/pkg/util/version" +) + +func (f *frontend) getAdminPlatformWorkloadIdentityRoleSets(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + log := ctx.Value(middleware.ContextKeyLog).(*logrus.Entry) + r.URL.Path = filepath.Dir(r.URL.Path) + + converter := f.apis[admin.APIVersion].PlatformWorkloadIdentityRoleSetConverter + + docs, err := f.dbPlatformWorkloadIdentityRoleSets.ListAll(ctx) + if err != nil { + log.Error(err) + api.WriteError(w, http.StatusInternalServerError, api.CloudErrorCodeInternalServerError, "", "Internal server error.") + return + } + + var roleSets []*api.PlatformWorkloadIdentityRoleSet + if docs != nil { + for _, doc := range docs.PlatformWorkloadIdentityRoleSetDocuments { + roleSets = append(roleSets, doc.PlatformWorkloadIdentityRoleSet) + } + } + + sort.Slice(roleSets, func(i, j int) bool { + return version.CreateSemverFromMinorVersionString(roleSets[i].Properties.OpenShiftVersion).LessThan(*version.CreateSemverFromMinorVersionString(roleSets[j].Properties.OpenShiftVersion)) + }) + + b, err := json.MarshalIndent(converter.ToExternalList(roleSets), "", " ") + adminReply(log, w, nil, b, err) +} diff --git a/pkg/frontend/admin_platformworkloadidentityroleset_list_test.go b/pkg/frontend/admin_platformworkloadidentityroleset_list_test.go new file mode 100644 index 00000000000..4f92a3be110 --- /dev/null +++ b/pkg/frontend/admin_platformworkloadidentityroleset_list_test.go @@ -0,0 +1,281 @@ +package frontend + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "errors" + "net/http" + "sort" + "testing" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/api/admin" + "github.com/Azure/ARO-RP/pkg/database/cosmosdb" + "github.com/Azure/ARO-RP/pkg/metrics/noop" + "github.com/Azure/ARO-RP/pkg/util/version" + testdatabase "github.com/Azure/ARO-RP/test/database" +) + +func TestPlatformWorkloadIdentityRoleSetList(t *testing.T) { + ctx := context.Background() + + type test struct { + name string + fixture func(f *testdatabase.Fixture) + cosmosdb func(c *cosmosdb.FakePlatformWorkloadIdentityRoleSetDocumentClient) + wantStatusCode int + wantResponse *admin.PlatformWorkloadIdentityRoleSetList + wantError string + } + + for _, tt := range []*test{ + { + name: "GET request returns empty result with StatusOK", + fixture: func(f *testdatabase.Fixture) {}, + wantStatusCode: http.StatusOK, + wantResponse: &admin.PlatformWorkloadIdentityRoleSetList{ + PlatformWorkloadIdentityRoleSets: []*admin.PlatformWorkloadIdentityRoleSet{}, + }, + }, + { + name: "GET request returns non-empty result with StatusOK", + fixture: func(f *testdatabase.Fixture) { + f.AddPlatformWorkloadIdentityRoleSetDocuments( + &api.PlatformWorkloadIdentityRoleSetDocument{ + PlatformWorkloadIdentityRoleSet: &api.PlatformWorkloadIdentityRoleSet{ + Properties: api.PlatformWorkloadIdentityRoleSetProperties{ + OpenShiftVersion: "4.14", + PlatformWorkloadIdentityRoles: []api.PlatformWorkloadIdentityRole{ + { + OperatorName: "CloudControllerManager", + RoleDefinitionName: "Azure RedHat OpenShift Cloud Controller Manager Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/a1f96423-95ce-4224-ab27-4e3dc72facd4", + ServiceAccounts: []string{ + "openshift-cloud-controller-manager:cloud-controller-manager", + }, + }, + { + OperatorName: "ClusterIngressOperator", + RoleDefinitionName: "Azure RedHat OpenShift Cluster Ingress Operator Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/0336e1d3-7a87-462b-b6db-342b63f7802c", + ServiceAccounts: []string{ + "openshift-ingress-operator:ingress-operator", + }, + }, + }, + }, + }, + }, + &api.PlatformWorkloadIdentityRoleSetDocument{ + PlatformWorkloadIdentityRoleSet: &api.PlatformWorkloadIdentityRoleSet{ + Properties: api.PlatformWorkloadIdentityRoleSetProperties{ + OpenShiftVersion: "4.15", + PlatformWorkloadIdentityRoles: []api.PlatformWorkloadIdentityRole{ + { + OperatorName: "CloudControllerManager", + RoleDefinitionName: "Azure RedHat OpenShift Cloud Controller Manager Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/a1f96423-95ce-4224-ab27-4e3dc72facd4", + ServiceAccounts: []string{ + "openshift-cloud-controller-manager:cloud-controller-manager", + }, + }, + }, + }, + }, + }, + ) + }, + wantStatusCode: http.StatusOK, + wantResponse: &admin.PlatformWorkloadIdentityRoleSetList{ + PlatformWorkloadIdentityRoleSets: []*admin.PlatformWorkloadIdentityRoleSet{ + { + Properties: admin.PlatformWorkloadIdentityRoleSetProperties{ + OpenShiftVersion: "4.14", + PlatformWorkloadIdentityRoles: []admin.PlatformWorkloadIdentityRole{ + { + OperatorName: "CloudControllerManager", + RoleDefinitionName: "Azure RedHat OpenShift Cloud Controller Manager Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/a1f96423-95ce-4224-ab27-4e3dc72facd4", + ServiceAccounts: []string{ + "openshift-cloud-controller-manager:cloud-controller-manager", + }, + }, + { + OperatorName: "ClusterIngressOperator", + RoleDefinitionName: "Azure RedHat OpenShift Cluster Ingress Operator Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/0336e1d3-7a87-462b-b6db-342b63f7802c", + ServiceAccounts: []string{ + "openshift-ingress-operator:ingress-operator", + }, + }, + }, + }, + }, + { + Properties: admin.PlatformWorkloadIdentityRoleSetProperties{ + OpenShiftVersion: "4.15", + PlatformWorkloadIdentityRoles: []admin.PlatformWorkloadIdentityRole{ + { + OperatorName: "CloudControllerManager", + RoleDefinitionName: "Azure RedHat OpenShift Cloud Controller Manager Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/a1f96423-95ce-4224-ab27-4e3dc72facd4", + ServiceAccounts: []string{ + "openshift-cloud-controller-manager:cloud-controller-manager", + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "GET request with StatusOK returns results in correct order even if Cosmos DB returns them in a different order", + fixture: func(f *testdatabase.Fixture) { + f.AddPlatformWorkloadIdentityRoleSetDocuments( + &api.PlatformWorkloadIdentityRoleSetDocument{ + PlatformWorkloadIdentityRoleSet: &api.PlatformWorkloadIdentityRoleSet{ + Properties: api.PlatformWorkloadIdentityRoleSetProperties{ + OpenShiftVersion: "4.14", + PlatformWorkloadIdentityRoles: []api.PlatformWorkloadIdentityRole{ + { + OperatorName: "CloudControllerManager", + RoleDefinitionName: "Azure RedHat OpenShift Cloud Controller Manager Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/a1f96423-95ce-4224-ab27-4e3dc72facd4", + ServiceAccounts: []string{ + "openshift-cloud-controller-manager:cloud-controller-manager", + }, + }, + { + OperatorName: "ClusterIngressOperator", + RoleDefinitionName: "Azure RedHat OpenShift Cluster Ingress Operator Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/0336e1d3-7a87-462b-b6db-342b63f7802c", + ServiceAccounts: []string{ + "openshift-ingress-operator:ingress-operator", + }, + }, + }, + }, + }, + }, + &api.PlatformWorkloadIdentityRoleSetDocument{ + PlatformWorkloadIdentityRoleSet: &api.PlatformWorkloadIdentityRoleSet{ + Properties: api.PlatformWorkloadIdentityRoleSetProperties{ + OpenShiftVersion: "4.15", + PlatformWorkloadIdentityRoles: []api.PlatformWorkloadIdentityRole{ + { + OperatorName: "CloudControllerManager", + RoleDefinitionName: "Azure RedHat OpenShift Cloud Controller Manager Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/a1f96423-95ce-4224-ab27-4e3dc72facd4", + ServiceAccounts: []string{ + "openshift-cloud-controller-manager:cloud-controller-manager", + }, + }, + }, + }, + }, + }, + ) + }, + cosmosdb: func(c *cosmosdb.FakePlatformWorkloadIdentityRoleSetDocumentClient) { + // Sort the documents in descending order rather than ascending order, which + // is the order we expect to see in the response. + c.SetSorter(func(roleSets []*api.PlatformWorkloadIdentityRoleSetDocument) { + sort.Slice(roleSets, func(i, j int) bool { + return version.CreateSemverFromMinorVersionString(roleSets[j].PlatformWorkloadIdentityRoleSet.Properties.OpenShiftVersion).LessThan(*version.CreateSemverFromMinorVersionString(roleSets[i].PlatformWorkloadIdentityRoleSet.Properties.OpenShiftVersion)) + }) + }) + }, + wantStatusCode: http.StatusOK, + wantResponse: &admin.PlatformWorkloadIdentityRoleSetList{ + PlatformWorkloadIdentityRoleSets: []*admin.PlatformWorkloadIdentityRoleSet{ + { + Properties: admin.PlatformWorkloadIdentityRoleSetProperties{ + OpenShiftVersion: "4.14", + PlatformWorkloadIdentityRoles: []admin.PlatformWorkloadIdentityRole{ + { + OperatorName: "CloudControllerManager", + RoleDefinitionName: "Azure RedHat OpenShift Cloud Controller Manager Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/a1f96423-95ce-4224-ab27-4e3dc72facd4", + ServiceAccounts: []string{ + "openshift-cloud-controller-manager:cloud-controller-manager", + }, + }, + { + OperatorName: "ClusterIngressOperator", + RoleDefinitionName: "Azure RedHat OpenShift Cluster Ingress Operator Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/0336e1d3-7a87-462b-b6db-342b63f7802c", + ServiceAccounts: []string{ + "openshift-ingress-operator:ingress-operator", + }, + }, + }, + }, + }, + { + Properties: admin.PlatformWorkloadIdentityRoleSetProperties{ + OpenShiftVersion: "4.15", + PlatformWorkloadIdentityRoles: []admin.PlatformWorkloadIdentityRole{ + { + OperatorName: "CloudControllerManager", + RoleDefinitionName: "Azure RedHat OpenShift Cloud Controller Manager Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/a1f96423-95ce-4224-ab27-4e3dc72facd4", + ServiceAccounts: []string{ + "openshift-cloud-controller-manager:cloud-controller-manager", + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "GET request results in StatusInternalServerError due to issues with Cosmos DB", + fixture: func(f *testdatabase.Fixture) {}, + cosmosdb: func(c *cosmosdb.FakePlatformWorkloadIdentityRoleSetDocumentClient) { + c.SetError(errors.New("Well shoot, Cosmos DB isn't working!")) + }, + wantStatusCode: http.StatusInternalServerError, + wantResponse: &admin.PlatformWorkloadIdentityRoleSetList{ + PlatformWorkloadIdentityRoleSets: []*admin.PlatformWorkloadIdentityRoleSet{}, + }, + wantError: api.NewCloudError(http.StatusInternalServerError, api.CloudErrorCodeInternalServerError, "", "Internal server error.").Error(), + }, + } { + t.Run(tt.name, func(t *testing.T) { + ti := newTestInfra(t).WithPlatformWorkloadIdentityRoleSets() + defer ti.done() + + if tt.cosmosdb != nil { + tt.cosmosdb(ti.platformWorkloadIdentityRoleSetsClient) + } + + err := ti.buildFixtures(tt.fixture) + if err != nil { + t.Fatal(err) + } + + f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, nil, nil, nil, nil, nil, ti.platformWorkloadIdentityRoleSetsDatabase, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, nil, nil) + + if err != nil { + t.Fatal(err) + } + + go f.Run(ctx, nil, nil) + + resp, b, err := ti.request(http.MethodGet, "https://server/admin/platformworkloadidentityrolesets", + nil, nil) + if err != nil { + t.Fatal(err) + } + + err = validateResponse(resp, b, tt.wantStatusCode, tt.wantError, tt.wantResponse) + if err != nil { + t.Error(err) + } + }) + } +} diff --git a/pkg/frontend/admin_platformworkloadidentityroleset_put.go b/pkg/frontend/admin_platformworkloadidentityroleset_put.go new file mode 100644 index 00000000000..8b11641b4da --- /dev/null +++ b/pkg/frontend/admin_platformworkloadidentityroleset_put.go @@ -0,0 +1,96 @@ +package frontend + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "encoding/json" + "net/http" + "path/filepath" + + "github.com/sirupsen/logrus" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/api/admin" + "github.com/Azure/ARO-RP/pkg/frontend/middleware" +) + +func (f *frontend) putAdminPlatformWorkloadIdentityRoleSet(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + log := ctx.Value(middleware.ContextKeyLog).(*logrus.Entry) + r.URL.Path = filepath.Dir(r.URL.Path) + + converter := f.apis[admin.APIVersion].PlatformWorkloadIdentityRoleSetConverter + staticValidator := f.apis[admin.APIVersion].PlatformWorkloadIdentityRoleSetStaticValidator + + body := r.Context().Value(middleware.ContextKeyBody).([]byte) + if len(body) == 0 || !json.Valid(body) { + api.WriteError(w, http.StatusBadRequest, api.CloudErrorCodeInvalidRequestContent, "", "The request content was invalid and could not be deserialized.") + return + } + + var ext *admin.PlatformWorkloadIdentityRoleSet + err := json.Unmarshal(body, &ext) + if err != nil { + api.WriteError(w, http.StatusBadRequest, api.CloudErrorCodeInvalidRequestContent, "", "The request content could not be deserialized: "+err.Error()) + return + } + + docs, err := f.dbPlatformWorkloadIdentityRoleSets.ListAll(ctx) + if err != nil { + log.Error(err) + api.WriteError(w, http.StatusInternalServerError, api.CloudErrorCodeInternalServerError, "", "Internal server error.") + return + } + + var roleSetDoc *api.PlatformWorkloadIdentityRoleSetDocument + if docs != nil { + for _, doc := range docs.PlatformWorkloadIdentityRoleSetDocuments { + if doc.PlatformWorkloadIdentityRoleSet.Properties.OpenShiftVersion == ext.Properties.OpenShiftVersion { + roleSetDoc = doc + break + } + } + } + + isCreate := roleSetDoc == nil + if isCreate { + err = staticValidator.Static(ext, nil) + roleSetDoc = &api.PlatformWorkloadIdentityRoleSetDocument{ + ID: f.dbPlatformWorkloadIdentityRoleSets.NewUUID(), + PlatformWorkloadIdentityRoleSet: &api.PlatformWorkloadIdentityRoleSet{}, + } + } else { + err = staticValidator.Static(ext, roleSetDoc.PlatformWorkloadIdentityRoleSet) + } + if err != nil { + adminReply(log, w, nil, []byte{}, err) + return + } + + converter.ToInternal(ext, roleSetDoc.PlatformWorkloadIdentityRoleSet) + + if isCreate { + roleSetDoc, err = f.dbPlatformWorkloadIdentityRoleSets.Create(ctx, roleSetDoc) + if err != nil { + log.Error(err) + api.WriteError(w, http.StatusInternalServerError, api.CloudErrorCodeInternalServerError, "", "Internal server error.") + return + } + } else { + roleSetDoc, err = f.dbPlatformWorkloadIdentityRoleSets.Update(ctx, roleSetDoc) + if err != nil { + log.Error(err) + api.WriteError(w, http.StatusInternalServerError, api.CloudErrorCodeInternalServerError, "", "Internal server error.") + return + } + } + + b, err := json.MarshalIndent(converter.ToExternal(roleSetDoc.PlatformWorkloadIdentityRoleSet), "", " ") + if err == nil { + if isCreate { + err = statusCodeError(http.StatusCreated) + } + } + adminReply(log, w, nil, b, err) +} diff --git a/pkg/frontend/admin_platformworkloadidentityroleset_put_test.go b/pkg/frontend/admin_platformworkloadidentityroleset_put_test.go new file mode 100644 index 00000000000..0c632dc93e8 --- /dev/null +++ b/pkg/frontend/admin_platformworkloadidentityroleset_put_test.go @@ -0,0 +1,788 @@ +package frontend + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "net/http" + "testing" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/api/admin" + "github.com/Azure/ARO-RP/pkg/metrics/noop" + testdatabase "github.com/Azure/ARO-RP/test/database" +) + +func TestPlatformWorkloadIdentityRoleSetPut(t *testing.T) { + ctx := context.Background() + + type test struct { + name string + fixture func(f *testdatabase.Fixture) + body *admin.PlatformWorkloadIdentityRoleSet + wantStatusCode int + wantResponse *admin.PlatformWorkloadIdentityRoleSet + wantError string + wantDocuments []*api.PlatformWorkloadIdentityRoleSetDocument + } + + for _, tt := range []*test{ + { + name: "PUT to update an existing entry updates it in-place and results in StatusOK", + fixture: func(f *testdatabase.Fixture) { + f.AddPlatformWorkloadIdentityRoleSetDocuments( + &api.PlatformWorkloadIdentityRoleSetDocument{ + PlatformWorkloadIdentityRoleSet: &api.PlatformWorkloadIdentityRoleSet{ + Properties: api.PlatformWorkloadIdentityRoleSetProperties{ + OpenShiftVersion: "4.14", + PlatformWorkloadIdentityRoles: []api.PlatformWorkloadIdentityRole{ + { + OperatorName: "CloudControllerManager", + RoleDefinitionName: "Azure RedHat OpenShift Cloud Controller Manager Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/a1f96423-95ce-4224-ab27-4e3dc72facd4", + ServiceAccounts: []string{ + "openshift-cloud-controller-manager:cloud-controller-manager", + }, + }, + { + OperatorName: "ClusterIngressOperator", + RoleDefinitionName: "Azure RedHat OpenShift Cluster Ingress Operator Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/0336e1d3-7a87-462b-b6db-342b63f7802c", + ServiceAccounts: []string{ + "openshift-ingress-operator:ingress-operator", + }, + }, + }, + }, + }, + }, + ) + }, + body: &admin.PlatformWorkloadIdentityRoleSet{ + Properties: admin.PlatformWorkloadIdentityRoleSetProperties{ + OpenShiftVersion: "4.14", + PlatformWorkloadIdentityRoles: []admin.PlatformWorkloadIdentityRole{ + { + OperatorName: "CloudControllerManager", + RoleDefinitionName: "Azure RedHat OpenShift Cloud Controller Manager Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/a1f96423-95ce-4224-ab27-4e3dc72facd4", + ServiceAccounts: []string{ + "openshift-cloud-controller-manager:cloud-controller-manager", + }, + }, + }, + }, + }, + wantStatusCode: http.StatusOK, + wantResponse: &admin.PlatformWorkloadIdentityRoleSet{ + Properties: admin.PlatformWorkloadIdentityRoleSetProperties{ + OpenShiftVersion: "4.14", + PlatformWorkloadIdentityRoles: []admin.PlatformWorkloadIdentityRole{ + { + OperatorName: "CloudControllerManager", + RoleDefinitionName: "Azure RedHat OpenShift Cloud Controller Manager Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/a1f96423-95ce-4224-ab27-4e3dc72facd4", + ServiceAccounts: []string{ + "openshift-cloud-controller-manager:cloud-controller-manager", + }, + }, + }, + }, + }, + wantDocuments: []*api.PlatformWorkloadIdentityRoleSetDocument{ + { + ID: "08080808-0808-0808-0808-080808080001", + PlatformWorkloadIdentityRoleSet: &api.PlatformWorkloadIdentityRoleSet{ + Properties: api.PlatformWorkloadIdentityRoleSetProperties{ + OpenShiftVersion: "4.14", + PlatformWorkloadIdentityRoles: []api.PlatformWorkloadIdentityRole{ + { + OperatorName: "CloudControllerManager", + RoleDefinitionName: "Azure RedHat OpenShift Cloud Controller Manager Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/a1f96423-95ce-4224-ab27-4e3dc72facd4", + ServiceAccounts: []string{ + "openshift-cloud-controller-manager:cloud-controller-manager", + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "PUT to add a new entry creates it successfully and results in StatusOK", + fixture: func(f *testdatabase.Fixture) { + f.AddPlatformWorkloadIdentityRoleSetDocuments( + &api.PlatformWorkloadIdentityRoleSetDocument{ + PlatformWorkloadIdentityRoleSet: &api.PlatformWorkloadIdentityRoleSet{ + Properties: api.PlatformWorkloadIdentityRoleSetProperties{ + OpenShiftVersion: "4.14", + PlatformWorkloadIdentityRoles: []api.PlatformWorkloadIdentityRole{ + { + OperatorName: "CloudControllerManager", + RoleDefinitionName: "Azure RedHat OpenShift Cloud Controller Manager Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/a1f96423-95ce-4224-ab27-4e3dc72facd4", + ServiceAccounts: []string{ + "openshift-cloud-controller-manager:cloud-controller-manager", + }, + }, + { + OperatorName: "ClusterIngressOperator", + RoleDefinitionName: "Azure RedHat OpenShift Cluster Ingress Operator Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/0336e1d3-7a87-462b-b6db-342b63f7802c", + ServiceAccounts: []string{ + "openshift-ingress-operator:ingress-operator", + }, + }, + }, + }, + }, + }, + ) + }, + body: &admin.PlatformWorkloadIdentityRoleSet{ + Properties: admin.PlatformWorkloadIdentityRoleSetProperties{ + OpenShiftVersion: "4.15", + PlatformWorkloadIdentityRoles: []admin.PlatformWorkloadIdentityRole{ + { + OperatorName: "ClusterIngressOperator", + RoleDefinitionName: "Azure RedHat OpenShift Cluster Ingress Operator Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/0336e1d3-7a87-462b-b6db-342b63f7802c", + ServiceAccounts: []string{ + "openshift-ingress-operator:ingress-operator", + }, + }, + }, + }, + }, + wantStatusCode: http.StatusCreated, + wantResponse: &admin.PlatformWorkloadIdentityRoleSet{ + Properties: admin.PlatformWorkloadIdentityRoleSetProperties{ + OpenShiftVersion: "4.15", + PlatformWorkloadIdentityRoles: []admin.PlatformWorkloadIdentityRole{ + { + OperatorName: "ClusterIngressOperator", + RoleDefinitionName: "Azure RedHat OpenShift Cluster Ingress Operator Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/0336e1d3-7a87-462b-b6db-342b63f7802c", + ServiceAccounts: []string{ + "openshift-ingress-operator:ingress-operator", + }, + }, + }, + }, + }, + wantDocuments: []*api.PlatformWorkloadIdentityRoleSetDocument{ + { + ID: "08080808-0808-0808-0808-080808080001", + PlatformWorkloadIdentityRoleSet: &api.PlatformWorkloadIdentityRoleSet{ + Properties: api.PlatformWorkloadIdentityRoleSetProperties{ + OpenShiftVersion: "4.14", + PlatformWorkloadIdentityRoles: []api.PlatformWorkloadIdentityRole{ + { + OperatorName: "CloudControllerManager", + RoleDefinitionName: "Azure RedHat OpenShift Cloud Controller Manager Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/a1f96423-95ce-4224-ab27-4e3dc72facd4", + ServiceAccounts: []string{ + "openshift-cloud-controller-manager:cloud-controller-manager", + }, + }, + { + OperatorName: "ClusterIngressOperator", + RoleDefinitionName: "Azure RedHat OpenShift Cluster Ingress Operator Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/0336e1d3-7a87-462b-b6db-342b63f7802c", + ServiceAccounts: []string{ + "openshift-ingress-operator:ingress-operator", + }, + }, + }, + }, + }, + }, + { + ID: "08080808-0808-0808-0808-080808080002", + PlatformWorkloadIdentityRoleSet: &api.PlatformWorkloadIdentityRoleSet{ + Properties: api.PlatformWorkloadIdentityRoleSetProperties{ + OpenShiftVersion: "4.15", + PlatformWorkloadIdentityRoles: []api.PlatformWorkloadIdentityRole{ + { + OperatorName: "ClusterIngressOperator", + RoleDefinitionName: "Azure RedHat OpenShift Cluster Ingress Operator Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/0336e1d3-7a87-462b-b6db-342b63f7802c", + ServiceAccounts: []string{ + "openshift-ingress-operator:ingress-operator", + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "PUT with missing request body results in StatusBadRequest", + fixture: func(f *testdatabase.Fixture) {}, + body: &admin.PlatformWorkloadIdentityRoleSet{}, + wantStatusCode: http.StatusBadRequest, + wantError: "400: InvalidParameter: properties.openShiftVersion: Must be provided", + wantDocuments: []*api.PlatformWorkloadIdentityRoleSetDocument{}, + }, + { + name: "PUT with missing OpenShiftVersion results in StatusBadRequest", + fixture: func(f *testdatabase.Fixture) { + f.AddPlatformWorkloadIdentityRoleSetDocuments( + &api.PlatformWorkloadIdentityRoleSetDocument{ + PlatformWorkloadIdentityRoleSet: &api.PlatformWorkloadIdentityRoleSet{ + Properties: api.PlatformWorkloadIdentityRoleSetProperties{ + OpenShiftVersion: "4.14", + PlatformWorkloadIdentityRoles: []api.PlatformWorkloadIdentityRole{ + { + OperatorName: "CloudControllerManager", + RoleDefinitionName: "Azure RedHat OpenShift Cloud Controller Manager Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/a1f96423-95ce-4224-ab27-4e3dc72facd4", + ServiceAccounts: []string{ + "openshift-cloud-controller-manager:cloud-controller-manager", + }, + }, + { + OperatorName: "ClusterIngressOperator", + RoleDefinitionName: "Azure RedHat OpenShift Cluster Ingress Operator Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/0336e1d3-7a87-462b-b6db-342b63f7802c", + ServiceAccounts: []string{ + "openshift-ingress-operator:ingress-operator", + }, + }, + }, + }, + }, + }, + ) + }, + body: &admin.PlatformWorkloadIdentityRoleSet{ + Properties: admin.PlatformWorkloadIdentityRoleSetProperties{ + PlatformWorkloadIdentityRoles: []admin.PlatformWorkloadIdentityRole{ + { + OperatorName: "CloudControllerManager", + RoleDefinitionName: "Azure RedHat OpenShift Cloud Controller Manager Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/a1f96423-95ce-4224-ab27-4e3dc72facd4", + ServiceAccounts: []string{ + "openshift-cloud-controller-manager:cloud-controller-manager", + }, + }, + }, + }, + }, + wantStatusCode: http.StatusBadRequest, + wantError: "400: InvalidParameter: properties.openShiftVersion: Must be provided", + wantDocuments: []*api.PlatformWorkloadIdentityRoleSetDocument{ + { + ID: "08080808-0808-0808-0808-080808080001", + PlatformWorkloadIdentityRoleSet: &api.PlatformWorkloadIdentityRoleSet{ + Properties: api.PlatformWorkloadIdentityRoleSetProperties{ + OpenShiftVersion: "4.14", + PlatformWorkloadIdentityRoles: []api.PlatformWorkloadIdentityRole{ + { + OperatorName: "CloudControllerManager", + RoleDefinitionName: "Azure RedHat OpenShift Cloud Controller Manager Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/a1f96423-95ce-4224-ab27-4e3dc72facd4", + ServiceAccounts: []string{ + "openshift-cloud-controller-manager:cloud-controller-manager", + }, + }, + { + OperatorName: "ClusterIngressOperator", + RoleDefinitionName: "Azure RedHat OpenShift Cluster Ingress Operator Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/0336e1d3-7a87-462b-b6db-342b63f7802c", + ServiceAccounts: []string{ + "openshift-ingress-operator:ingress-operator", + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "PUT with missing PlatformWorkloadIdentityRoles results in StatusBadRequest", + fixture: func(f *testdatabase.Fixture) { + f.AddPlatformWorkloadIdentityRoleSetDocuments( + &api.PlatformWorkloadIdentityRoleSetDocument{ + PlatformWorkloadIdentityRoleSet: &api.PlatformWorkloadIdentityRoleSet{ + Properties: api.PlatformWorkloadIdentityRoleSetProperties{ + OpenShiftVersion: "4.14", + PlatformWorkloadIdentityRoles: []api.PlatformWorkloadIdentityRole{ + { + OperatorName: "CloudControllerManager", + RoleDefinitionName: "Azure RedHat OpenShift Cloud Controller Manager Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/a1f96423-95ce-4224-ab27-4e3dc72facd4", + ServiceAccounts: []string{ + "openshift-cloud-controller-manager:cloud-controller-manager", + }, + }, + { + OperatorName: "ClusterIngressOperator", + RoleDefinitionName: "Azure RedHat OpenShift Cluster Ingress Operator Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/0336e1d3-7a87-462b-b6db-342b63f7802c", + ServiceAccounts: []string{ + "openshift-ingress-operator:ingress-operator", + }, + }, + }, + }, + }, + }, + ) + }, + body: &admin.PlatformWorkloadIdentityRoleSet{ + Properties: admin.PlatformWorkloadIdentityRoleSetProperties{ + OpenShiftVersion: "4.14", + }, + }, + wantStatusCode: http.StatusBadRequest, + wantError: "400: InvalidParameter: properties.platformWorkloadIdentityRoles: Must be provided and must be non-empty", + wantDocuments: []*api.PlatformWorkloadIdentityRoleSetDocument{ + { + ID: "08080808-0808-0808-0808-080808080001", + PlatformWorkloadIdentityRoleSet: &api.PlatformWorkloadIdentityRoleSet{ + Properties: api.PlatformWorkloadIdentityRoleSetProperties{ + OpenShiftVersion: "4.14", + PlatformWorkloadIdentityRoles: []api.PlatformWorkloadIdentityRole{ + { + OperatorName: "CloudControllerManager", + RoleDefinitionName: "Azure RedHat OpenShift Cloud Controller Manager Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/a1f96423-95ce-4224-ab27-4e3dc72facd4", + ServiceAccounts: []string{ + "openshift-cloud-controller-manager:cloud-controller-manager", + }, + }, + { + OperatorName: "ClusterIngressOperator", + RoleDefinitionName: "Azure RedHat OpenShift Cluster Ingress Operator Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/0336e1d3-7a87-462b-b6db-342b63f7802c", + ServiceAccounts: []string{ + "openshift-ingress-operator:ingress-operator", + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "PUT with missing PlatformWorkloadIdentityRole.OperatorName results in StatusBadRequest", + fixture: func(f *testdatabase.Fixture) { + f.AddPlatformWorkloadIdentityRoleSetDocuments( + &api.PlatformWorkloadIdentityRoleSetDocument{ + PlatformWorkloadIdentityRoleSet: &api.PlatformWorkloadIdentityRoleSet{ + Properties: api.PlatformWorkloadIdentityRoleSetProperties{ + OpenShiftVersion: "4.14", + PlatformWorkloadIdentityRoles: []api.PlatformWorkloadIdentityRole{ + { + OperatorName: "CloudControllerManager", + RoleDefinitionName: "Azure RedHat OpenShift Cloud Controller Manager Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/a1f96423-95ce-4224-ab27-4e3dc72facd4", + ServiceAccounts: []string{ + "openshift-cloud-controller-manager:cloud-controller-manager", + }, + }, + { + OperatorName: "ClusterIngressOperator", + RoleDefinitionName: "Azure RedHat OpenShift Cluster Ingress Operator Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/0336e1d3-7a87-462b-b6db-342b63f7802c", + ServiceAccounts: []string{ + "openshift-ingress-operator:ingress-operator", + }, + }, + }, + }, + }, + }, + ) + }, + body: &admin.PlatformWorkloadIdentityRoleSet{ + Properties: admin.PlatformWorkloadIdentityRoleSetProperties{ + OpenShiftVersion: "4.14", + PlatformWorkloadIdentityRoles: []admin.PlatformWorkloadIdentityRole{ + { + RoleDefinitionName: "Azure RedHat OpenShift Cluster Ingress Operator Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/0336e1d3-7a87-462b-b6db-342b63f7802c", + ServiceAccounts: []string{ + "openshift-ingress-operator:ingress-operator", + }, + }, + }, + }, + }, + wantStatusCode: http.StatusBadRequest, + wantError: "400: InvalidParameter: properties.platformWorkloadIdentityRoles[0].operatorName: Must be provided", + wantDocuments: []*api.PlatformWorkloadIdentityRoleSetDocument{ + { + ID: "08080808-0808-0808-0808-080808080001", + PlatformWorkloadIdentityRoleSet: &api.PlatformWorkloadIdentityRoleSet{ + Properties: api.PlatformWorkloadIdentityRoleSetProperties{ + OpenShiftVersion: "4.14", + PlatformWorkloadIdentityRoles: []api.PlatformWorkloadIdentityRole{ + { + OperatorName: "CloudControllerManager", + RoleDefinitionName: "Azure RedHat OpenShift Cloud Controller Manager Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/a1f96423-95ce-4224-ab27-4e3dc72facd4", + ServiceAccounts: []string{ + "openshift-cloud-controller-manager:cloud-controller-manager", + }, + }, + { + OperatorName: "ClusterIngressOperator", + RoleDefinitionName: "Azure RedHat OpenShift Cluster Ingress Operator Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/0336e1d3-7a87-462b-b6db-342b63f7802c", + ServiceAccounts: []string{ + "openshift-ingress-operator:ingress-operator", + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "PUT with missing PlatformWorkloadIdentityRole.RoleDefinitionName results in StatusBadRequest", + fixture: func(f *testdatabase.Fixture) { + f.AddPlatformWorkloadIdentityRoleSetDocuments( + &api.PlatformWorkloadIdentityRoleSetDocument{ + PlatformWorkloadIdentityRoleSet: &api.PlatformWorkloadIdentityRoleSet{ + Properties: api.PlatformWorkloadIdentityRoleSetProperties{ + OpenShiftVersion: "4.14", + PlatformWorkloadIdentityRoles: []api.PlatformWorkloadIdentityRole{ + { + OperatorName: "CloudControllerManager", + RoleDefinitionName: "Azure RedHat OpenShift Cloud Controller Manager Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/a1f96423-95ce-4224-ab27-4e3dc72facd4", + ServiceAccounts: []string{ + "openshift-cloud-controller-manager:cloud-controller-manager", + }, + }, + { + OperatorName: "ClusterIngressOperator", + RoleDefinitionName: "Azure RedHat OpenShift Cluster Ingress Operator Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/0336e1d3-7a87-462b-b6db-342b63f7802c", + ServiceAccounts: []string{ + "openshift-ingress-operator:ingress-operator", + }, + }, + }, + }, + }, + }, + ) + }, + body: &admin.PlatformWorkloadIdentityRoleSet{ + Properties: admin.PlatformWorkloadIdentityRoleSetProperties{ + OpenShiftVersion: "4.14", + PlatformWorkloadIdentityRoles: []admin.PlatformWorkloadIdentityRole{ + { + OperatorName: "ClusterIngressOperator", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/0336e1d3-7a87-462b-b6db-342b63f7802c", + ServiceAccounts: []string{ + "openshift-ingress-operator:ingress-operator", + }, + }, + }, + }, + }, + wantStatusCode: http.StatusBadRequest, + wantError: "400: InvalidParameter: properties.platformWorkloadIdentityRoles[0].roleDefinitionName: Must be provided", + wantDocuments: []*api.PlatformWorkloadIdentityRoleSetDocument{ + { + ID: "08080808-0808-0808-0808-080808080001", + PlatformWorkloadIdentityRoleSet: &api.PlatformWorkloadIdentityRoleSet{ + Properties: api.PlatformWorkloadIdentityRoleSetProperties{ + OpenShiftVersion: "4.14", + PlatformWorkloadIdentityRoles: []api.PlatformWorkloadIdentityRole{ + { + OperatorName: "CloudControllerManager", + RoleDefinitionName: "Azure RedHat OpenShift Cloud Controller Manager Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/a1f96423-95ce-4224-ab27-4e3dc72facd4", + ServiceAccounts: []string{ + "openshift-cloud-controller-manager:cloud-controller-manager", + }, + }, + { + OperatorName: "ClusterIngressOperator", + RoleDefinitionName: "Azure RedHat OpenShift Cluster Ingress Operator Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/0336e1d3-7a87-462b-b6db-342b63f7802c", + ServiceAccounts: []string{ + "openshift-ingress-operator:ingress-operator", + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "PUT with missing PlatformWorkloadIdentityRole.RoleDefinitionID results in StatusBadRequest", + fixture: func(f *testdatabase.Fixture) { + f.AddPlatformWorkloadIdentityRoleSetDocuments( + &api.PlatformWorkloadIdentityRoleSetDocument{ + PlatformWorkloadIdentityRoleSet: &api.PlatformWorkloadIdentityRoleSet{ + Properties: api.PlatformWorkloadIdentityRoleSetProperties{ + OpenShiftVersion: "4.14", + PlatformWorkloadIdentityRoles: []api.PlatformWorkloadIdentityRole{ + { + OperatorName: "CloudControllerManager", + RoleDefinitionName: "Azure RedHat OpenShift Cloud Controller Manager Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/a1f96423-95ce-4224-ab27-4e3dc72facd4", + ServiceAccounts: []string{ + "openshift-cloud-controller-manager:cloud-controller-manager", + }, + }, + { + OperatorName: "ClusterIngressOperator", + RoleDefinitionName: "Azure RedHat OpenShift Cluster Ingress Operator Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/0336e1d3-7a87-462b-b6db-342b63f7802c", + ServiceAccounts: []string{ + "openshift-ingress-operator:ingress-operator", + }, + }, + }, + }, + }, + }, + ) + }, + body: &admin.PlatformWorkloadIdentityRoleSet{ + Properties: admin.PlatformWorkloadIdentityRoleSetProperties{ + OpenShiftVersion: "4.14", + PlatformWorkloadIdentityRoles: []admin.PlatformWorkloadIdentityRole{ + { + OperatorName: "ClusterIngressOperator", + RoleDefinitionName: "Azure RedHat OpenShift Cluster Ingress Operator Role", + ServiceAccounts: []string{ + "openshift-ingress-operator:ingress-operator", + }, + }, + }, + }, + }, + wantStatusCode: http.StatusBadRequest, + wantError: "400: InvalidParameter: properties.platformWorkloadIdentityRoles[0].roleDefinitionId: Must be provided", + wantDocuments: []*api.PlatformWorkloadIdentityRoleSetDocument{ + { + ID: "08080808-0808-0808-0808-080808080001", + PlatformWorkloadIdentityRoleSet: &api.PlatformWorkloadIdentityRoleSet{ + Properties: api.PlatformWorkloadIdentityRoleSetProperties{ + OpenShiftVersion: "4.14", + PlatformWorkloadIdentityRoles: []api.PlatformWorkloadIdentityRole{ + { + OperatorName: "CloudControllerManager", + RoleDefinitionName: "Azure RedHat OpenShift Cloud Controller Manager Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/a1f96423-95ce-4224-ab27-4e3dc72facd4", + ServiceAccounts: []string{ + "openshift-cloud-controller-manager:cloud-controller-manager", + }, + }, + { + OperatorName: "ClusterIngressOperator", + RoleDefinitionName: "Azure RedHat OpenShift Cluster Ingress Operator Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/0336e1d3-7a87-462b-b6db-342b63f7802c", + ServiceAccounts: []string{ + "openshift-ingress-operator:ingress-operator", + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "PUT with missing PlatformWorkloadIdentityRole.ServiceAccounts results in StatusBadRequest", + fixture: func(f *testdatabase.Fixture) { + f.AddPlatformWorkloadIdentityRoleSetDocuments( + &api.PlatformWorkloadIdentityRoleSetDocument{ + PlatformWorkloadIdentityRoleSet: &api.PlatformWorkloadIdentityRoleSet{ + Properties: api.PlatformWorkloadIdentityRoleSetProperties{ + OpenShiftVersion: "4.14", + PlatformWorkloadIdentityRoles: []api.PlatformWorkloadIdentityRole{ + { + OperatorName: "CloudControllerManager", + RoleDefinitionName: "Azure RedHat OpenShift Cloud Controller Manager Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/a1f96423-95ce-4224-ab27-4e3dc72facd4", + ServiceAccounts: []string{ + "openshift-cloud-controller-manager:cloud-controller-manager", + }, + }, + { + OperatorName: "ClusterIngressOperator", + RoleDefinitionName: "Azure RedHat OpenShift Cluster Ingress Operator Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/0336e1d3-7a87-462b-b6db-342b63f7802c", + ServiceAccounts: []string{ + "openshift-ingress-operator:ingress-operator", + }, + }, + }, + }, + }, + }, + ) + }, + body: &admin.PlatformWorkloadIdentityRoleSet{ + Properties: admin.PlatformWorkloadIdentityRoleSetProperties{ + OpenShiftVersion: "4.14", + PlatformWorkloadIdentityRoles: []admin.PlatformWorkloadIdentityRole{ + { + OperatorName: "ClusterIngressOperator", + RoleDefinitionName: "Azure RedHat OpenShift Cluster Ingress Operator Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/a1f96423-95ce-4224-ab27-4e3dc72facd4", + }, + }, + }, + }, + wantStatusCode: http.StatusBadRequest, + wantError: "400: InvalidParameter: properties.platformWorkloadIdentityRoles[0].serviceAccounts: Must be provided", + wantDocuments: []*api.PlatformWorkloadIdentityRoleSetDocument{ + { + ID: "08080808-0808-0808-0808-080808080001", + PlatformWorkloadIdentityRoleSet: &api.PlatformWorkloadIdentityRoleSet{ + Properties: api.PlatformWorkloadIdentityRoleSetProperties{ + OpenShiftVersion: "4.14", + PlatformWorkloadIdentityRoles: []api.PlatformWorkloadIdentityRole{ + { + OperatorName: "CloudControllerManager", + RoleDefinitionName: "Azure RedHat OpenShift Cloud Controller Manager Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/a1f96423-95ce-4224-ab27-4e3dc72facd4", + ServiceAccounts: []string{ + "openshift-cloud-controller-manager:cloud-controller-manager", + }, + }, + { + OperatorName: "ClusterIngressOperator", + RoleDefinitionName: "Azure RedHat OpenShift Cluster Ingress Operator Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/0336e1d3-7a87-462b-b6db-342b63f7802c", + ServiceAccounts: []string{ + "openshift-ingress-operator:ingress-operator", + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "PUT with missing PlatformWorkloadIdentityRole.RoleDefinitionId and PlatformWorkloadIdentityRole.ServiceAccounts results in StatusBadRequest - tests the case where multiple attributes are missing and error message consists of messages about multiple missing properties joined together", + fixture: func(f *testdatabase.Fixture) { + f.AddPlatformWorkloadIdentityRoleSetDocuments( + &api.PlatformWorkloadIdentityRoleSetDocument{ + PlatformWorkloadIdentityRoleSet: &api.PlatformWorkloadIdentityRoleSet{ + Properties: api.PlatformWorkloadIdentityRoleSetProperties{ + OpenShiftVersion: "4.14", + PlatformWorkloadIdentityRoles: []api.PlatformWorkloadIdentityRole{ + { + OperatorName: "CloudControllerManager", + RoleDefinitionName: "Azure RedHat OpenShift Cloud Controller Manager Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/a1f96423-95ce-4224-ab27-4e3dc72facd4", + ServiceAccounts: []string{ + "openshift-cloud-controller-manager:cloud-controller-manager", + }, + }, + { + OperatorName: "ClusterIngressOperator", + RoleDefinitionName: "Azure RedHat OpenShift Cluster Ingress Operator Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/0336e1d3-7a87-462b-b6db-342b63f7802c", + ServiceAccounts: []string{ + "openshift-ingress-operator:ingress-operator", + }, + }, + }, + }, + }, + }, + ) + }, + body: &admin.PlatformWorkloadIdentityRoleSet{ + Properties: admin.PlatformWorkloadIdentityRoleSetProperties{ + OpenShiftVersion: "4.14", + PlatformWorkloadIdentityRoles: []admin.PlatformWorkloadIdentityRole{ + { + OperatorName: "ClusterIngressOperator", + RoleDefinitionName: "Azure RedHat OpenShift Cluster Ingress Operator Role", + }, + }, + }, + }, + wantStatusCode: http.StatusBadRequest, + wantError: "400: InvalidParameter: properties.platformWorkloadIdentityRoles[0].roleDefinitionId, properties.platformWorkloadIdentityRoles[0].serviceAccounts: Must be provided", + wantDocuments: []*api.PlatformWorkloadIdentityRoleSetDocument{ + { + ID: "08080808-0808-0808-0808-080808080001", + PlatformWorkloadIdentityRoleSet: &api.PlatformWorkloadIdentityRoleSet{ + Properties: api.PlatformWorkloadIdentityRoleSetProperties{ + OpenShiftVersion: "4.14", + PlatformWorkloadIdentityRoles: []api.PlatformWorkloadIdentityRole{ + { + OperatorName: "CloudControllerManager", + RoleDefinitionName: "Azure RedHat OpenShift Cloud Controller Manager Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/a1f96423-95ce-4224-ab27-4e3dc72facd4", + ServiceAccounts: []string{ + "openshift-cloud-controller-manager:cloud-controller-manager", + }, + }, + { + OperatorName: "ClusterIngressOperator", + RoleDefinitionName: "Azure RedHat OpenShift Cluster Ingress Operator Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/0336e1d3-7a87-462b-b6db-342b63f7802c", + ServiceAccounts: []string{ + "openshift-ingress-operator:ingress-operator", + }, + }, + }, + }, + }, + }, + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + ti := newTestInfra(t).WithPlatformWorkloadIdentityRoleSets() + + defer ti.done() + + err := ti.buildFixtures(tt.fixture) + if err != nil { + t.Fatal(err) + } + + f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, nil, nil, nil, nil, nil, ti.platformWorkloadIdentityRoleSetsDatabase, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + + go f.Run(ctx, nil, nil) + + resp, b, err := ti.request(http.MethodPut, "https://server/admin/platformworkloadidentityrolesets", + http.Header{ + "Content-Type": []string{"application/json"}, + }, tt.body) + if err != nil { + t.Fatal(err) + } + + err = validateResponse(resp, b, tt.wantStatusCode, tt.wantError, tt.wantResponse) + if err != nil { + t.Error(err) + } + + if tt.wantDocuments != nil { + ti.checker.AddPlatformWorkloadIdentityRoleSetDocuments(tt.wantDocuments...) + for _, err := range ti.checker.CheckPlatformWorkloadIdentityRoleSets(ti.platformWorkloadIdentityRoleSetsClient) { + t.Error(err) + } + } + }) + } +} diff --git a/pkg/frontend/asyncoperationresult_get_test.go b/pkg/frontend/asyncoperationresult_get_test.go index 516fcac902a..a48eefad21b 100644 --- a/pkg/frontend/asyncoperationresult_get_test.go +++ b/pkg/frontend/asyncoperationresult_get_test.go @@ -136,7 +136,7 @@ func TestGetAsyncOperationResult(t *testing.T) { t.Fatal(err) } - f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, nil, nil) + f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, nil, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, nil, nil) if err != nil { t.Fatal(err) } diff --git a/pkg/frontend/asyncoperationsstatus_get_test.go b/pkg/frontend/asyncoperationsstatus_get_test.go index f6a6d3a9a1f..9533d02db1b 100644 --- a/pkg/frontend/asyncoperationsstatus_get_test.go +++ b/pkg/frontend/asyncoperationsstatus_get_test.go @@ -183,7 +183,7 @@ func TestGetAsyncOperationsStatus(t *testing.T) { ti.asyncOperationsClient.SetError(tt.dbError) } - f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, nil, nil) + f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, nil, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, nil, nil) if err != nil { t.Fatal(err) } diff --git a/pkg/frontend/changefeed.go b/pkg/frontend/changefeed.go index 4291e1c9aa0..5bb5c32017e 100644 --- a/pkg/frontend/changefeed.go +++ b/pkg/frontend/changefeed.go @@ -12,7 +12,7 @@ import ( "github.com/Azure/ARO-RP/pkg/util/recover" ) -func (f *frontend) changefeed(ctx context.Context) { +func (f *frontend) changefeedOcpVersions(ctx context.Context) { defer recover.Panic(f.baseLog) // f.dbOpenShiftVersions will be nil when running unit tests. Return here to avoid nil pointer panic @@ -20,15 +20,30 @@ func (f *frontend) changefeed(ctx context.Context) { return } - frontendIterator := f.dbOpenShiftVersions.ChangeFeed() + ocpVersionsIterator := f.dbOpenShiftVersions.ChangeFeed() t := time.NewTicker(10 * time.Second) defer t.Stop() - f.updateFromIterator(ctx, t, frontendIterator) + f.updateFromIteratorOcpVersions(ctx, t, ocpVersionsIterator) } -func (f *frontend) updateFromIterator(ctx context.Context, ticker *time.Ticker, frontendIterator cosmosdb.OpenShiftVersionDocumentIterator) { +func (f *frontend) changefeedRoleSets(ctx context.Context) { + defer recover.Panic(f.baseLog) + + if f.dbPlatformWorkloadIdentityRoleSets == nil { + return + } + + roleSetsIterator := f.dbPlatformWorkloadIdentityRoleSets.ChangeFeed() + + t := time.NewTicker(10 * time.Second) + defer t.Stop() + + f.updateFromIteratorRoleSets(ctx, t, roleSetsIterator) +} + +func (f *frontend) updateFromIteratorOcpVersions(ctx context.Context, ticker *time.Ticker, frontendIterator cosmosdb.OpenShiftVersionDocumentIterator) { for { successful := true @@ -47,7 +62,7 @@ func (f *frontend) updateFromIterator(ctx context.Context, ticker *time.Ticker, } if successful { - f.lastChangefeed.Store(time.Now()) + f.lastOcpVersionsChangefeed.Store(time.Now()) } select { @@ -60,8 +75,8 @@ func (f *frontend) updateFromIterator(ctx context.Context, ticker *time.Ticker, // updateOcpVersions adds enabled versions to the frontend cache func (f *frontend) updateOcpVersions(docs []*api.OpenShiftVersionDocument) { - f.mu.Lock() - defer f.mu.Unlock() + f.ocpVersionsMu.Lock() + defer f.ocpVersionsMu.Unlock() for _, doc := range docs { if doc.OpenShiftVersion.Deleting || !doc.OpenShiftVersion.Properties.Enabled { @@ -75,3 +90,47 @@ func (f *frontend) updateOcpVersions(docs []*api.OpenShiftVersionDocument) { } } } + +func (f *frontend) updateFromIteratorRoleSets(ctx context.Context, ticker *time.Ticker, frontendIterator cosmosdb.PlatformWorkloadIdentityRoleSetDocumentIterator) { + for { + successful := true + + for { + docs, err := frontendIterator.Next(ctx, -1) + if err != nil { + successful = false + f.baseLog.Error(err) + break + } + if docs == nil { + break + } + + f.updatePlatformWorkloadIdentityRoleSets(docs.PlatformWorkloadIdentityRoleSetDocuments) + } + + if successful { + f.lastPlatformWorkloadIdentityRoleSetsChangefeed.Store(time.Now()) + } + + select { + case <-ticker.C: + case <-ctx.Done(): + return + } + } +} + +func (f *frontend) updatePlatformWorkloadIdentityRoleSets(docs []*api.PlatformWorkloadIdentityRoleSetDocument) { + f.platformWorkloadIdentityRoleSetsMu.Lock() + defer f.platformWorkloadIdentityRoleSetsMu.Unlock() + + for _, doc := range docs { + if doc.PlatformWorkloadIdentityRoleSet.Deleting { + // https://docs.microsoft.com/en-us/azure/cosmos-db/change-feed-design-patterns#deletes + delete(f.availablePlatformWorkloadIdentityRoleSets, doc.PlatformWorkloadIdentityRoleSet.Properties.OpenShiftVersion) + } else { + f.availablePlatformWorkloadIdentityRoleSets[doc.PlatformWorkloadIdentityRoleSet.Properties.OpenShiftVersion] = doc.PlatformWorkloadIdentityRoleSet + } + } +} diff --git a/pkg/frontend/changefeed_test.go b/pkg/frontend/changefeed_test.go index fb56d28c464..6e572d8cd86 100644 --- a/pkg/frontend/changefeed_test.go +++ b/pkg/frontend/changefeed_test.go @@ -14,7 +14,7 @@ import ( "github.com/Azure/ARO-RP/pkg/util/cmp" ) -func TestUpdateFromIterator(t *testing.T) { +func TestUpdateFromIteratorOcpVersions(t *testing.T) { for _, tt := range []struct { name string docsInIterator []*api.OpenShiftVersionDocument @@ -22,7 +22,7 @@ func TestUpdateFromIterator(t *testing.T) { wantVersions map[string]*api.OpenShiftVersion }{ { - name: "add to empty", + name: "Add a new doc from the changefeed to an empty frontend cache", docsInIterator: []*api.OpenShiftVersionDocument{ { OpenShiftVersion: &api.OpenShiftVersion{ @@ -44,7 +44,7 @@ func TestUpdateFromIterator(t *testing.T) { }, }, { - name: "do nothing", + name: "Docs in changefeed match docs in frontend cache - no changes needed", docsInIterator: []*api.OpenShiftVersionDocument{ { OpenShiftVersion: &api.OpenShiftVersion{ @@ -73,7 +73,7 @@ func TestUpdateFromIterator(t *testing.T) { }, }, { - name: "add to not empty", + name: "Add a new doc from the iterator to a non-empty frontend cache", docsInIterator: []*api.OpenShiftVersionDocument{ { OpenShiftVersion: &api.OpenShiftVersion{ @@ -108,7 +108,7 @@ func TestUpdateFromIterator(t *testing.T) { }, }, { - name: "remove existing", + name: "A doc present in the frontend cache is marked deleting in the changefeed - remove it from the cache", docsInIterator: []*api.OpenShiftVersionDocument{ { OpenShiftVersion: &api.OpenShiftVersion{ @@ -152,7 +152,7 @@ func TestUpdateFromIterator(t *testing.T) { }, }, { - name: "remove disabled versions", + name: "A doc present in the frontend cache is marked disabled in the changefeed - remove it from the cache", docsInIterator: []*api.OpenShiftVersionDocument{ { OpenShiftVersion: &api.OpenShiftVersion{ @@ -184,7 +184,7 @@ func TestUpdateFromIterator(t *testing.T) { fakeIterator := cosmosdb.NewFakeOpenShiftVersionDocumentIterator(tt.docsInIterator, 0) - go frontend.updateFromIterator(ctx, ticker, fakeIterator) + go frontend.updateFromIteratorOcpVersions(ctx, ticker, fakeIterator) time.Sleep(time.Second) cancel() @@ -194,3 +194,345 @@ func TestUpdateFromIterator(t *testing.T) { }) } } + +func TestUpdateFromIteratorRoleSets(t *testing.T) { + for _, tt := range []struct { + name string + docsInIterator []*api.PlatformWorkloadIdentityRoleSetDocument + roleSets map[string]*api.PlatformWorkloadIdentityRoleSet + wantRoleSets map[string]*api.PlatformWorkloadIdentityRoleSet + }{ + { + name: "add to empty", + docsInIterator: []*api.PlatformWorkloadIdentityRoleSetDocument{ + { + PlatformWorkloadIdentityRoleSet: &api.PlatformWorkloadIdentityRoleSet{ + Properties: api.PlatformWorkloadIdentityRoleSetProperties{ + OpenShiftVersion: "4.14", + PlatformWorkloadIdentityRoles: []api.PlatformWorkloadIdentityRole{ + { + OperatorName: "CloudControllerManager", + RoleDefinitionName: "Azure RedHat OpenShift Cloud Controller Manager Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/a1f96423-95ce-4224-ab27-4e3dc72facd4", + ServiceAccounts: []string{ + "openshift-cloud-controller-manager:cloud-controller-manager", + }, + }, + }, + }, + }, + }, + }, + roleSets: map[string]*api.PlatformWorkloadIdentityRoleSet{}, + wantRoleSets: map[string]*api.PlatformWorkloadIdentityRoleSet{ + "4.14": { + Properties: api.PlatformWorkloadIdentityRoleSetProperties{ + OpenShiftVersion: "4.14", + PlatformWorkloadIdentityRoles: []api.PlatformWorkloadIdentityRole{ + { + OperatorName: "CloudControllerManager", + RoleDefinitionName: "Azure RedHat OpenShift Cloud Controller Manager Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/a1f96423-95ce-4224-ab27-4e3dc72facd4", + ServiceAccounts: []string{ + "openshift-cloud-controller-manager:cloud-controller-manager", + }, + }, + }, + }, + }, + }, + }, + { + name: "do nothing", + docsInIterator: []*api.PlatformWorkloadIdentityRoleSetDocument{ + { + PlatformWorkloadIdentityRoleSet: &api.PlatformWorkloadIdentityRoleSet{ + Properties: api.PlatformWorkloadIdentityRoleSetProperties{ + OpenShiftVersion: "4.14", + PlatformWorkloadIdentityRoles: []api.PlatformWorkloadIdentityRole{ + { + OperatorName: "CloudControllerManager", + RoleDefinitionName: "Azure RedHat OpenShift Cloud Controller Manager Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/a1f96423-95ce-4224-ab27-4e3dc72facd4", + ServiceAccounts: []string{ + "openshift-cloud-controller-manager:cloud-controller-manager", + }, + }, + }, + }, + }, + }, + }, + roleSets: map[string]*api.PlatformWorkloadIdentityRoleSet{ + "4.14": { + Properties: api.PlatformWorkloadIdentityRoleSetProperties{ + OpenShiftVersion: "4.14", + PlatformWorkloadIdentityRoles: []api.PlatformWorkloadIdentityRole{ + { + OperatorName: "CloudControllerManager", + RoleDefinitionName: "Azure RedHat OpenShift Cloud Controller Manager Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/a1f96423-95ce-4224-ab27-4e3dc72facd4", + ServiceAccounts: []string{ + "openshift-cloud-controller-manager:cloud-controller-manager", + }, + }, + }, + }, + }, + }, + wantRoleSets: map[string]*api.PlatformWorkloadIdentityRoleSet{ + "4.14": { + Properties: api.PlatformWorkloadIdentityRoleSetProperties{ + OpenShiftVersion: "4.14", + PlatformWorkloadIdentityRoles: []api.PlatformWorkloadIdentityRole{ + { + OperatorName: "CloudControllerManager", + RoleDefinitionName: "Azure RedHat OpenShift Cloud Controller Manager Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/a1f96423-95ce-4224-ab27-4e3dc72facd4", + ServiceAccounts: []string{ + "openshift-cloud-controller-manager:cloud-controller-manager", + }, + }, + }, + }, + }, + }, + }, + { + name: "add to not empty", + docsInIterator: []*api.PlatformWorkloadIdentityRoleSetDocument{ + { + PlatformWorkloadIdentityRoleSet: &api.PlatformWorkloadIdentityRoleSet{ + Properties: api.PlatformWorkloadIdentityRoleSetProperties{ + OpenShiftVersion: "4.14", + PlatformWorkloadIdentityRoles: []api.PlatformWorkloadIdentityRole{ + { + OperatorName: "CloudControllerManager", + RoleDefinitionName: "Azure RedHat OpenShift Cloud Controller Manager Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/a1f96423-95ce-4224-ab27-4e3dc72facd4", + ServiceAccounts: []string{ + "openshift-cloud-controller-manager:cloud-controller-manager", + }, + }, + }, + }, + }, + }, + { + PlatformWorkloadIdentityRoleSet: &api.PlatformWorkloadIdentityRoleSet{ + Properties: api.PlatformWorkloadIdentityRoleSetProperties{ + OpenShiftVersion: "4.15", + PlatformWorkloadIdentityRoles: []api.PlatformWorkloadIdentityRole{ + { + OperatorName: "CloudControllerManager", + RoleDefinitionName: "Azure RedHat OpenShift Cloud Controller Manager Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/a1f96423-95ce-4224-ab27-4e3dc72facd4", + ServiceAccounts: []string{ + "openshift-cloud-controller-manager:cloud-controller-manager", + }, + }, + { + OperatorName: "ClusterIngressOperator", + RoleDefinitionName: "Azure RedHat OpenShift Cluster Ingress Operator Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/0336e1d3-7a87-462b-b6db-342b63f7802c", + ServiceAccounts: []string{ + "openshift-ingress-operator:ingress-operator", + }, + }, + }, + }, + }, + }, + }, + roleSets: map[string]*api.PlatformWorkloadIdentityRoleSet{ + "4.14": { + Properties: api.PlatformWorkloadIdentityRoleSetProperties{ + OpenShiftVersion: "4.14", + PlatformWorkloadIdentityRoles: []api.PlatformWorkloadIdentityRole{ + { + OperatorName: "CloudControllerManager", + RoleDefinitionName: "Azure RedHat OpenShift Cloud Controller Manager Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/a1f96423-95ce-4224-ab27-4e3dc72facd4", + ServiceAccounts: []string{ + "openshift-cloud-controller-manager:cloud-controller-manager", + }, + }, + }, + }, + }, + }, + wantRoleSets: map[string]*api.PlatformWorkloadIdentityRoleSet{ + "4.14": { + Properties: api.PlatformWorkloadIdentityRoleSetProperties{ + OpenShiftVersion: "4.14", + PlatformWorkloadIdentityRoles: []api.PlatformWorkloadIdentityRole{ + { + OperatorName: "CloudControllerManager", + RoleDefinitionName: "Azure RedHat OpenShift Cloud Controller Manager Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/a1f96423-95ce-4224-ab27-4e3dc72facd4", + ServiceAccounts: []string{ + "openshift-cloud-controller-manager:cloud-controller-manager", + }, + }, + }, + }, + }, + "4.15": { + Properties: api.PlatformWorkloadIdentityRoleSetProperties{ + OpenShiftVersion: "4.15", + PlatformWorkloadIdentityRoles: []api.PlatformWorkloadIdentityRole{ + { + OperatorName: "CloudControllerManager", + RoleDefinitionName: "Azure RedHat OpenShift Cloud Controller Manager Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/a1f96423-95ce-4224-ab27-4e3dc72facd4", + ServiceAccounts: []string{ + "openshift-cloud-controller-manager:cloud-controller-manager", + }, + }, + { + OperatorName: "ClusterIngressOperator", + RoleDefinitionName: "Azure RedHat OpenShift Cluster Ingress Operator Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/0336e1d3-7a87-462b-b6db-342b63f7802c", + ServiceAccounts: []string{ + "openshift-ingress-operator:ingress-operator", + }, + }, + }, + }, + }, + }, + }, + { + name: "remove existing", + docsInIterator: []*api.PlatformWorkloadIdentityRoleSetDocument{ + { + PlatformWorkloadIdentityRoleSet: &api.PlatformWorkloadIdentityRoleSet{ + Properties: api.PlatformWorkloadIdentityRoleSetProperties{ + OpenShiftVersion: "4.14", + PlatformWorkloadIdentityRoles: []api.PlatformWorkloadIdentityRole{ + { + OperatorName: "CloudControllerManager", + RoleDefinitionName: "Azure RedHat OpenShift Cloud Controller Manager Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/a1f96423-95ce-4224-ab27-4e3dc72facd4", + ServiceAccounts: []string{ + "openshift-cloud-controller-manager:cloud-controller-manager", + }, + }, + }, + }, + Deleting: true, + }, + }, + { + PlatformWorkloadIdentityRoleSet: &api.PlatformWorkloadIdentityRoleSet{ + Properties: api.PlatformWorkloadIdentityRoleSetProperties{ + OpenShiftVersion: "4.15", + PlatformWorkloadIdentityRoles: []api.PlatformWorkloadIdentityRole{ + { + OperatorName: "CloudControllerManager", + RoleDefinitionName: "Azure RedHat OpenShift Cloud Controller Manager Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/a1f96423-95ce-4224-ab27-4e3dc72facd4", + ServiceAccounts: []string{ + "openshift-cloud-controller-manager:cloud-controller-manager", + }, + }, + { + OperatorName: "ClusterIngressOperator", + RoleDefinitionName: "Azure RedHat OpenShift Cluster Ingress Operator Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/0336e1d3-7a87-462b-b6db-342b63f7802c", + ServiceAccounts: []string{ + "openshift-ingress-operator:ingress-operator", + }, + }, + }, + }, + }, + }, + }, + roleSets: map[string]*api.PlatformWorkloadIdentityRoleSet{ + "4.14": { + Properties: api.PlatformWorkloadIdentityRoleSetProperties{ + OpenShiftVersion: "4.14", + PlatformWorkloadIdentityRoles: []api.PlatformWorkloadIdentityRole{ + { + OperatorName: "CloudControllerManager", + RoleDefinitionName: "Azure RedHat OpenShift Cloud Controller Manager Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/a1f96423-95ce-4224-ab27-4e3dc72facd4", + ServiceAccounts: []string{ + "openshift-cloud-controller-manager:cloud-controller-manager", + }, + }, + }, + }, + }, + "4.15": { + Properties: api.PlatformWorkloadIdentityRoleSetProperties{ + OpenShiftVersion: "4.15", + PlatformWorkloadIdentityRoles: []api.PlatformWorkloadIdentityRole{ + { + OperatorName: "CloudControllerManager", + RoleDefinitionName: "Azure RedHat OpenShift Cloud Controller Manager Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/a1f96423-95ce-4224-ab27-4e3dc72facd4", + ServiceAccounts: []string{ + "openshift-cloud-controller-manager:cloud-controller-manager", + }, + }, + { + OperatorName: "ClusterIngressOperator", + RoleDefinitionName: "Azure RedHat OpenShift Cluster Ingress Operator Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/0336e1d3-7a87-462b-b6db-342b63f7802c", + ServiceAccounts: []string{ + "openshift-ingress-operator:ingress-operator", + }, + }, + }, + }, + }, + }, + wantRoleSets: map[string]*api.PlatformWorkloadIdentityRoleSet{ + "4.15": { + Properties: api.PlatformWorkloadIdentityRoleSetProperties{ + OpenShiftVersion: "4.15", + PlatformWorkloadIdentityRoles: []api.PlatformWorkloadIdentityRole{ + { + OperatorName: "CloudControllerManager", + RoleDefinitionName: "Azure RedHat OpenShift Cloud Controller Manager Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/a1f96423-95ce-4224-ab27-4e3dc72facd4", + ServiceAccounts: []string{ + "openshift-cloud-controller-manager:cloud-controller-manager", + }, + }, + { + OperatorName: "ClusterIngressOperator", + RoleDefinitionName: "Azure RedHat OpenShift Cluster Ingress Operator Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/0336e1d3-7a87-462b-b6db-342b63f7802c", + ServiceAccounts: []string{ + "openshift-ingress-operator:ingress-operator", + }, + }, + }, + }, + }, + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + ticker := time.NewTicker(1) + ctx, cancel := context.WithCancel(context.TODO()) + + frontend := frontend{ + availablePlatformWorkloadIdentityRoleSets: tt.roleSets, + } + + fakeIterator := cosmosdb.NewFakePlatformWorkloadIdentityRoleSetDocumentIterator(tt.docsInIterator, 0) + + go frontend.updateFromIteratorRoleSets(ctx, ticker, fakeIterator) + time.Sleep(time.Second) + cancel() + + if !reflect.DeepEqual(frontend.availablePlatformWorkloadIdentityRoleSets, tt.wantRoleSets) { + t.Error(cmp.Diff(frontend.availablePlatformWorkloadIdentityRoleSets, tt.wantRoleSets)) + } + }) + } +} diff --git a/pkg/frontend/clustermanager_delete_test.go b/pkg/frontend/clustermanager_delete_test.go index 7c85ef3e453..91d83e6d8a2 100644 --- a/pkg/frontend/clustermanager_delete_test.go +++ b/pkg/frontend/clustermanager_delete_test.go @@ -120,7 +120,7 @@ func TestDeleteClusterManagerConfiguration(t *testing.T) { t.Fatal(err) } - f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, nil, ti.clusterManagerDatabase, nil, ti.subscriptionsDatabase, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, nil, nil) + f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, nil, ti.clusterManagerDatabase, nil, ti.subscriptionsDatabase, nil, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, nil, nil) if err != nil { t.Fatal(err) } diff --git a/pkg/frontend/clustermanager_get_test.go b/pkg/frontend/clustermanager_get_test.go index eaf73d78d01..0f38667ab22 100644 --- a/pkg/frontend/clustermanager_get_test.go +++ b/pkg/frontend/clustermanager_get_test.go @@ -123,7 +123,7 @@ func TestGetClusterManagerConfiguration(t *testing.T) { t.Fatal(err) } - f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, nil, ti.clusterManagerDatabase, nil, nil, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, nil, nil) + f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, nil, ti.clusterManagerDatabase, nil, nil, nil, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, nil, nil) if err != nil { t.Fatal(err) } diff --git a/pkg/frontend/clustermanager_putorpatch_test.go b/pkg/frontend/clustermanager_putorpatch_test.go index 7708c7e459e..9bd73948a57 100644 --- a/pkg/frontend/clustermanager_putorpatch_test.go +++ b/pkg/frontend/clustermanager_putorpatch_test.go @@ -201,7 +201,7 @@ func TestPutOrPatchClusterManagerConfiguration(t *testing.T) { t.Fatal(err) } - f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, nil, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, nil, nil) + f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, nil, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, nil, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, nil, nil) if err != nil { t.Fatal(err) } diff --git a/pkg/frontend/fixetcd_test.go b/pkg/frontend/fixetcd_test.go index b25b9a482b1..43ad982bbb4 100644 --- a/pkg/frontend/fixetcd_test.go +++ b/pkg/frontend/fixetcd_test.go @@ -523,6 +523,7 @@ func TestFixEtcd(t *testing.T) { ti.openShiftClustersDatabase, ti.subscriptionsDatabase, nil, + nil, api.APIs, &noop.Noop{}, &noop.Noop{}, diff --git a/pkg/frontend/frontend.go b/pkg/frontend/frontend.go index 5d300085111..9e62d3000cb 100644 --- a/pkg/frontend/frontend.go +++ b/pkg/frontend/frontend.go @@ -57,18 +57,22 @@ type frontend struct { apiVersionMiddleware middleware.ApiVersionValidator maintenanceMiddleware middleware.MaintenanceMiddleware - dbAsyncOperations database.AsyncOperations - dbClusterManagerConfiguration database.ClusterManagerConfigurations - dbOpenShiftClusters database.OpenShiftClusters - dbSubscriptions database.Subscriptions - dbOpenShiftVersions database.OpenShiftVersions - - defaultOcpVersion string // always enabled - enabledOcpVersions map[string]*api.OpenShiftVersion - apis map[string]*api.Version - - lastChangefeed atomic.Value //time.Time - mu sync.RWMutex + dbAsyncOperations database.AsyncOperations + dbClusterManagerConfiguration database.ClusterManagerConfigurations + dbOpenShiftClusters database.OpenShiftClusters + dbSubscriptions database.Subscriptions + dbOpenShiftVersions database.OpenShiftVersions + dbPlatformWorkloadIdentityRoleSets database.PlatformWorkloadIdentityRoleSets + + defaultOcpVersion string // always enabled + enabledOcpVersions map[string]*api.OpenShiftVersion + availablePlatformWorkloadIdentityRoleSets map[string]*api.PlatformWorkloadIdentityRoleSet + apis map[string]*api.Version + + lastOcpVersionsChangefeed atomic.Value //time.Time + lastPlatformWorkloadIdentityRoleSetsChangefeed atomic.Value + ocpVersionsMu sync.RWMutex + platformWorkloadIdentityRoleSetsMu sync.RWMutex aead encryption.AEAD @@ -107,6 +111,7 @@ type Runnable interface { Run(context.Context, <-chan struct{}, chan<- struct{}) } +// TODO: Get the number of function parameters under control :D // NewFrontend returns a new runnable frontend func NewFrontend(ctx context.Context, auditLog *logrus.Entry, @@ -117,6 +122,7 @@ func NewFrontend(ctx context.Context, dbOpenShiftClusters database.OpenShiftClusters, dbSubscriptions database.Subscriptions, dbOpenShiftVersions database.OpenShiftVersions, + dbPlatformWorkloadIdentityRoleSets database.PlatformWorkloadIdentityRoleSets, apis map[string]*api.Version, m metrics.Emitter, clusterm metrics.Emitter, @@ -148,18 +154,19 @@ func NewFrontend(ctx context.Context, AdminAuth: _env.AdminClientAuthorizer(), ArmAuth: _env.ArmClientAuthorizer(), }, - dbAsyncOperations: dbAsyncOperations, - dbClusterManagerConfiguration: dbClusterManagerConfiguration, - dbOpenShiftClusters: dbOpenShiftClusters, - dbSubscriptions: dbSubscriptions, - dbOpenShiftVersions: dbOpenShiftVersions, - apis: apis, - m: middleware.MetricsMiddleware{Emitter: m}, - maintenanceMiddleware: middleware.MaintenanceMiddleware{Emitter: clusterm}, - aead: aead, - hiveClusterManager: hiveClusterManager, - kubeActionsFactory: kubeActionsFactory, - azureActionsFactory: azureActionsFactory, + dbAsyncOperations: dbAsyncOperations, + dbClusterManagerConfiguration: dbClusterManagerConfiguration, + dbOpenShiftClusters: dbOpenShiftClusters, + dbSubscriptions: dbSubscriptions, + dbOpenShiftVersions: dbOpenShiftVersions, + dbPlatformWorkloadIdentityRoleSets: dbPlatformWorkloadIdentityRoleSets, + apis: apis, + m: middleware.MetricsMiddleware{Emitter: m}, + maintenanceMiddleware: middleware.MaintenanceMiddleware{Emitter: clusterm}, + aead: aead, + hiveClusterManager: hiveClusterManager, + kubeActionsFactory: kubeActionsFactory, + azureActionsFactory: azureActionsFactory, quotaValidator: quotaValidator{}, skuValidator: skuValidator{}, @@ -167,7 +174,8 @@ func NewFrontend(ctx context.Context, clusterEnricher: enricher, - enabledOcpVersions: map[string]*api.OpenShiftVersion{}, + enabledOcpVersions: map[string]*api.OpenShiftVersion{}, + availablePlatformWorkloadIdentityRoleSets: map[string]*api.PlatformWorkloadIdentityRoleSet{}, bucketAllocator: &bucket.Random{}, @@ -277,6 +285,8 @@ func (f *frontend) chiAuthenticatedRoutes(router chi.Router) { r.Get("/operationresults/{operationId}", f.getAsyncOperationResult) r.Get("/openshiftversions", f.listInstallVersions) + + r.Get("/platformworkloadidentityrolesets", f.listPlatformWorkloadIdentityRoleSets) }) }) }) @@ -288,6 +298,10 @@ func (f *frontend) chiAuthenticatedRoutes(router chi.Router) { r.Get("/", f.getAdminOpenShiftVersions) r.Put("/", f.putAdminOpenShiftVersion) }) + r.Route("/platformworkloadidentityrolesets", func(r chi.Router) { + r.Get("/", f.getAdminPlatformWorkloadIdentityRoleSets) + r.Put("/", f.putAdminPlatformWorkloadIdentityRoleSet) + }) r.Get("/supportedvmsizes", f.supportedvmsizes) r.Route("/subscriptions/{subscriptionId}", func(r chi.Router) { @@ -372,7 +386,8 @@ func (f *frontend) setupRouter() chi.Router { func (f *frontend) Run(ctx context.Context, stop <-chan struct{}, done chan<- struct{}) { defer recover.Panic(f.baseLog) - go f.changefeed(ctx) + go f.changefeedOcpVersions(ctx) + go f.changefeedRoleSets(ctx) if stop != nil { go func() { diff --git a/pkg/frontend/openshiftcluster_applensdetectors_test.go b/pkg/frontend/openshiftcluster_applensdetectors_test.go index fc3bc445548..b1665012e32 100644 --- a/pkg/frontend/openshiftcluster_applensdetectors_test.go +++ b/pkg/frontend/openshiftcluster_applensdetectors_test.go @@ -97,7 +97,7 @@ func TestAppLensDetectors(t *testing.T) { t.Fatal(err) } - f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, func(*logrus.Entry, env.Interface, *api.OpenShiftCluster, *api.SubscriptionDocument) (adminactions.AzureActions, error) { + f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, nil, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, func(*logrus.Entry, env.Interface, *api.OpenShiftCluster, *api.SubscriptionDocument) (adminactions.AzureActions, error) { return a, nil }, nil) diff --git a/pkg/frontend/openshiftcluster_delete_test.go b/pkg/frontend/openshiftcluster_delete_test.go index e9774b58b76..08883437ff1 100644 --- a/pkg/frontend/openshiftcluster_delete_test.go +++ b/pkg/frontend/openshiftcluster_delete_test.go @@ -114,7 +114,7 @@ func TestDeleteOpenShiftCluster(t *testing.T) { ti.subscriptionsClient.SetError(tt.dbError) } - f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, nil, nil) + f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, nil, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, nil, nil) if err != nil { t.Fatal(err) } diff --git a/pkg/frontend/openshiftcluster_get_test.go b/pkg/frontend/openshiftcluster_get_test.go index 65fff16875c..1ac87c137e5 100644 --- a/pkg/frontend/openshiftcluster_get_test.go +++ b/pkg/frontend/openshiftcluster_get_test.go @@ -95,7 +95,7 @@ func TestGetOpenShiftCluster(t *testing.T) { ti.openShiftClustersClient.SetError(tt.dbError) } - f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, nil, ti.enricher) + f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, nil, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, nil, ti.enricher) if err != nil { t.Fatal(err) } diff --git a/pkg/frontend/openshiftcluster_list_test.go b/pkg/frontend/openshiftcluster_list_test.go index c22c16c99f7..9fc49f4bda7 100644 --- a/pkg/frontend/openshiftcluster_list_test.go +++ b/pkg/frontend/openshiftcluster_list_test.go @@ -204,7 +204,7 @@ func TestListOpenShiftCluster(t *testing.T) { aead := testdatabase.NewFakeAEAD() - f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, aead, nil, nil, nil, ti.enricher) + f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, nil, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, aead, nil, nil, nil, ti.enricher) if err != nil { t.Fatal(err) } diff --git a/pkg/frontend/openshiftcluster_preflightvalidation_test.go b/pkg/frontend/openshiftcluster_preflightvalidation_test.go index 8f22653364c..0eb6075f40d 100644 --- a/pkg/frontend/openshiftcluster_preflightvalidation_test.go +++ b/pkg/frontend/openshiftcluster_preflightvalidation_test.go @@ -398,16 +398,24 @@ func TestPreflightValidation(t *testing.T) { t.Fatal(err) } - f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, ti.openShiftVersionsDatabase, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, nil, nil) + f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, ti.openShiftVersionsDatabase, ti.platformWorkloadIdentityRoleSetsDatabase, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, nil, nil) if err != nil { t.Fatal(err) } oc := tt.preflightRequest() go f.Run(ctx, nil, nil) - f.mu.Lock() - f.enabledOcpVersions = tt.changeFeed - f.mu.Unlock() + + f.ocpVersionsMu.Lock() + f.defaultOcpVersion = "4.10.0" + f.enabledOcpVersions = map[string]*api.OpenShiftVersion{ + f.defaultOcpVersion: { + Properties: api.OpenShiftVersionProperties{ + Version: f.defaultOcpVersion, + }, + }, + } + f.ocpVersionsMu.Unlock() headers := http.Header{ "Content-Type": []string{"application/json"}, diff --git a/pkg/frontend/openshiftcluster_putorpatch.go b/pkg/frontend/openshiftcluster_putorpatch.go index 1fa6f26fdca..5c630897853 100644 --- a/pkg/frontend/openshiftcluster_putorpatch.go +++ b/pkg/frontend/openshiftcluster_putorpatch.go @@ -25,6 +25,8 @@ import ( "github.com/Azure/ARO-RP/pkg/util/version" ) +var errMissingIdentityURL error = fmt.Errorf("identityURL not provided but required for workload identity cluster") + func (f *frontend) putOrPatchOpenShiftCluster(w http.ResponseWriter, r *http.Request) { ctx := r.Context() log := ctx.Value(middleware.ContextKeyLog).(*logrus.Entry) @@ -41,10 +43,12 @@ func (f *frontend) putOrPatchOpenShiftCluster(w http.ResponseWriter, r *http.Req subId := chi.URLParam(r, "subscriptionId") resourceProviderNamespace := chi.URLParam(r, "resourceProviderNamespace") + identityURL := r.Header.Get("x-ms-identity-url") + apiVersion := r.URL.Query().Get(api.APIVersionKey) err := cosmosdb.RetryOnPreconditionFailed(func() error { var err error - b, err = f._putOrPatchOpenShiftCluster(ctx, log, body, correlationData, systemData, r.URL.Path, originalPath, r.Method, referer, &header, f.apis[apiVersion].OpenShiftClusterConverter, f.apis[apiVersion].OpenShiftClusterStaticValidator, subId, resourceProviderNamespace, apiVersion) + b, err = f._putOrPatchOpenShiftCluster(ctx, log, body, correlationData, systemData, r.URL.Path, originalPath, r.Method, referer, &header, f.apis[apiVersion].OpenShiftClusterConverter, f.apis[apiVersion].OpenShiftClusterStaticValidator, subId, resourceProviderNamespace, apiVersion, identityURL) return err }) @@ -52,7 +56,7 @@ func (f *frontend) putOrPatchOpenShiftCluster(w http.ResponseWriter, r *http.Req reply(log, w, header, b, err) } -func (f *frontend) _putOrPatchOpenShiftCluster(ctx context.Context, log *logrus.Entry, body []byte, correlationData *api.CorrelationData, systemData *api.SystemData, path, originalPath, method, referer string, header *http.Header, converter api.OpenShiftClusterConverter, staticValidator api.OpenShiftClusterStaticValidator, subId, resourceProviderNamespace string, apiVersion string) ([]byte, error) { +func (f *frontend) _putOrPatchOpenShiftCluster(ctx context.Context, log *logrus.Entry, body []byte, correlationData *api.CorrelationData, systemData *api.SystemData, path, originalPath, method, referer string, header *http.Header, converter api.OpenShiftClusterConverter, staticValidator api.OpenShiftClusterStaticValidator, subId, resourceProviderNamespace string, apiVersion string, identityURL string) ([]byte, error) { subscription, err := f.validateSubscriptionState(ctx, path, api.SubscriptionStateRegistered) if err != nil { return nil, err @@ -86,11 +90,17 @@ func (f *frontend) _putOrPatchOpenShiftCluster(ctx context.Context, log *logrus. }, }, } + if !f.env.IsLocalDevelopmentMode() /* not local dev or CI */ { doc.OpenShiftCluster.Properties.FeatureProfile.GatewayEnabled = true } } + err = validateIdentityUrl(doc.OpenShiftCluster, identityURL, isCreate) + if err != nil { + return nil, err + } + doc.CorrelationData = correlationData err = validateTerminalProvisioningState(doc.OpenShiftCluster.Properties.ProvisioningState) @@ -288,6 +298,24 @@ func enrichClusterSystemData(doc *api.OpenShiftClusterDocument, systemData *api. } } +func validateIdentityUrl(cluster *api.OpenShiftCluster, identityURL string, isCreate bool) error { + // Don't persist identity URL in non-wimi clusters + if cluster.Properties.ServicePrincipalProfile != nil || cluster.Identity == nil { + return nil + } + + if identityURL == "" { + if isCreate { + return errMissingIdentityURL + } + return nil + } + + cluster.Identity.IdentityURL = identityURL + + return nil +} + func (f *frontend) ValidateNewCluster(ctx context.Context, subscription *api.SubscriptionDocument, cluster *api.OpenShiftCluster, staticValidator api.OpenShiftClusterStaticValidator, ext interface{}, path string) error { err := staticValidator.Static(ext, nil, f.env.Location(), f.env.Domain(), f.env.FeatureIsSet(env.FeatureRequireD2sV3Workers), path) if err != nil { diff --git a/pkg/frontend/openshiftcluster_putorpatch_test.go b/pkg/frontend/openshiftcluster_putorpatch_test.go index f7c10ade968..15ee5181237 100644 --- a/pkg/frontend/openshiftcluster_putorpatch_test.go +++ b/pkg/frontend/openshiftcluster_putorpatch_test.go @@ -1727,7 +1727,7 @@ func TestPutOrPatchOpenShiftClusterAdminAPI(t *testing.T) { t.Fatal(err) } - f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, nil, apis, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, nil, ti.enricher) + f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, nil, nil, apis, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, nil, ti.enricher) if err != nil { t.Fatal(err) } @@ -2800,7 +2800,7 @@ func TestPutOrPatchOpenShiftCluster(t *testing.T) { t.Fatal(err) } - f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, ti.openShiftVersionsDatabase, apis, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, nil, ti.enricher) + f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, ti.openShiftVersionsDatabase, ti.platformWorkloadIdentityRoleSetsDatabase, apis, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, nil, ti.enricher) if err != nil { t.Fatal(err) } @@ -2817,14 +2817,14 @@ func TestPutOrPatchOpenShiftCluster(t *testing.T) { } go f.Run(ctx, nil, nil) - f.mu.Lock() + f.ocpVersionsMu.Lock() f.enabledOcpVersions = tt.changeFeed for key, doc := range tt.changeFeed { if doc.Properties.Default { f.defaultOcpVersion = key } } - f.mu.Unlock() + f.ocpVersionsMu.Unlock() oc := &v20200430.OpenShiftCluster{} if tt.request != nil { @@ -3133,7 +3133,7 @@ func TestPutOrPatchOpenShiftClusterValidated(t *testing.T) { t.Fatal(err) } - f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, ti.openShiftVersionsDatabase, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, nil, ti.enricher) + f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, ti.openShiftVersionsDatabase, ti.platformWorkloadIdentityRoleSetsDatabase, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, nil, ti.enricher) if err != nil { t.Fatal(err) } @@ -3305,3 +3305,84 @@ func TestEnrichClusterSystemData(t *testing.T) { }) } } + +func TestValidateIdentityUrl(t *testing.T) { + for _, tt := range []struct { + name string + identityURL string + cluster *api.OpenShiftCluster + expected *api.OpenShiftCluster + isCreate bool + wantError error + }{ + { + name: "identity URL is empty, is not wi/mi cluster create", + identityURL: "", + cluster: &api.OpenShiftCluster{}, + expected: &api.OpenShiftCluster{}, + isCreate: false, + }, + { + name: "identity URL is empty, is wi/mi cluster create", + identityURL: "", + cluster: &api.OpenShiftCluster{}, + expected: &api.OpenShiftCluster{}, + isCreate: true, + wantError: errMissingIdentityURL, + }, + { + name: "cluster is not wi/mi, identityURL passed", + identityURL: "http://foo.bar", + cluster: &api.OpenShiftCluster{ + Properties: api.OpenShiftClusterProperties{ + ServicePrincipalProfile: &api.ServicePrincipalProfile{}, + }, + }, + expected: &api.OpenShiftCluster{ + Properties: api.OpenShiftClusterProperties{ + ServicePrincipalProfile: &api.ServicePrincipalProfile{}, + }, + }, + isCreate: true, + }, + { + name: "cluster is not wi/mi, identityURL not passed", + identityURL: "", + cluster: &api.OpenShiftCluster{ + Properties: api.OpenShiftClusterProperties{ + ServicePrincipalProfile: &api.ServicePrincipalProfile{}, + }, + }, + expected: &api.OpenShiftCluster{ + Properties: api.OpenShiftClusterProperties{ + ServicePrincipalProfile: &api.ServicePrincipalProfile{}, + }, + }, + isCreate: true, + }, + { + name: "pass - identity URL passed on wi/mi cluster", + cluster: &api.OpenShiftCluster{ + Identity: &api.Identity{}, + }, + identityURL: "http://foo.bar", + expected: &api.OpenShiftCluster{ + Identity: &api.Identity{ + IdentityURL: "http://foo.bar", + }, + }, + isCreate: true, + }, + } { + t.Run(tt.name, func(t *testing.T) { + err := validateIdentityUrl(tt.cluster, tt.identityURL, tt.isCreate) + if err != nil && err != tt.wantError { + t.Error(cmp.Diff(err, tt.wantError)) + } + + if !reflect.DeepEqual(tt.cluster, tt.expected) { + t.Error(cmp.Diff(tt.cluster, tt.expected)) + } + }) + } +} diff --git a/pkg/frontend/openshiftclustercredentials_post_test.go b/pkg/frontend/openshiftclustercredentials_post_test.go index 54f445928fd..14400926f3f 100644 --- a/pkg/frontend/openshiftclustercredentials_post_test.go +++ b/pkg/frontend/openshiftclustercredentials_post_test.go @@ -267,7 +267,7 @@ func TestPostOpenShiftClusterCredentials(t *testing.T) { t.Fatal(err) } - f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, nil, apis, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, nil, nil) + f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, nil, nil, apis, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, nil, nil) if err != nil { t.Fatal(err) } diff --git a/pkg/frontend/openshiftclusterkubeconfigcredentials_post_test.go b/pkg/frontend/openshiftclusterkubeconfigcredentials_post_test.go index c233024ba3e..f8461d56679 100644 --- a/pkg/frontend/openshiftclusterkubeconfigcredentials_post_test.go +++ b/pkg/frontend/openshiftclusterkubeconfigcredentials_post_test.go @@ -242,7 +242,7 @@ func TestPostOpenShiftClusterKubeConfigCredentials(t *testing.T) { t.Fatal(err) } - f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, nil, apis, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, nil, nil) + f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, nil, nil, apis, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, nil, nil) if err != nil { t.Fatal(err) } diff --git a/pkg/frontend/openshiftversions_list.go b/pkg/frontend/openshiftversions_list.go index c3de112b302..9306dc628fd 100644 --- a/pkg/frontend/openshiftversions_list.go +++ b/pkg/frontend/openshiftversions_list.go @@ -35,11 +35,11 @@ func (f *frontend) listInstallVersions(w http.ResponseWriter, r *http.Request) { func (f *frontend) getEnabledInstallVersions(ctx context.Context) []*api.OpenShiftVersion { versions := make([]*api.OpenShiftVersion, 0) - f.mu.RLock() + f.ocpVersionsMu.RLock() for _, v := range f.enabledOcpVersions { versions = append(versions, v) } - f.mu.RUnlock() + f.ocpVersionsMu.RUnlock() return versions } diff --git a/pkg/frontend/openshiftversions_list_test.go b/pkg/frontend/openshiftversions_list_test.go index 3f1413c9003..11b90ed9db7 100644 --- a/pkg/frontend/openshiftversions_list_test.go +++ b/pkg/frontend/openshiftversions_list_test.go @@ -78,21 +78,21 @@ func TestListInstallVersions(t *testing.T) { ti := newTestInfra(t).WithSubscriptions().WithOpenShiftVersions() defer ti.done() - frontend, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, nil, nil, nil, nil, ti.openShiftVersionsDatabase, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, nil, nil) + frontend, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, nil, nil, nil, nil, ti.openShiftVersionsDatabase, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, nil, nil) if err != nil { t.Fatal(err) } go frontend.Run(ctx, nil, nil) - frontend.mu.Lock() + frontend.ocpVersionsMu.Lock() frontend.enabledOcpVersions = tt.changeFeed for key, doc := range tt.changeFeed { if doc.Properties.Enabled { frontend.defaultOcpVersion = key } } - frontend.mu.Unlock() + frontend.ocpVersionsMu.Unlock() resp, b, err := ti.request(method, fmt.Sprintf("https://server/subscriptions/%s/providers/Microsoft.RedHatOpenShift/locations/%s/openshiftversions?api-version=%s", mockSubID, ti.env.Location(), tt.apiVersion), diff --git a/pkg/frontend/platformworkloadidentityrolesets_list.go b/pkg/frontend/platformworkloadidentityrolesets_list.go new file mode 100644 index 00000000000..c03d7de4062 --- /dev/null +++ b/pkg/frontend/platformworkloadidentityrolesets_list.go @@ -0,0 +1,45 @@ +package frontend + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "encoding/json" + "net/http" + + "github.com/go-chi/chi/v5" + "github.com/sirupsen/logrus" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/frontend/middleware" +) + +func (f *frontend) listPlatformWorkloadIdentityRoleSets(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + log := ctx.Value(middleware.ContextKeyLog).(*logrus.Entry) + apiVersion := r.URL.Query().Get(api.APIVersionKey) + resourceProviderNamespace := chi.URLParam(r, "resourceProviderNamespace") + if f.apis[apiVersion].PlatformWorkloadIdentityRoleSetConverter == nil { + api.WriteError(w, http.StatusBadRequest, api.CloudErrorCodeInvalidResourceType, "", "The endpoint could not be found in the namespace '%s' for api version '%s'.", resourceProviderNamespace, apiVersion) + return + } + + roleSets := f.getAvailablePlatformWorkloadIdentityRoleSets(ctx) + converter := f.apis[apiVersion].PlatformWorkloadIdentityRoleSetConverter + + b, err := json.MarshalIndent(converter.ToExternalList(roleSets), "", " ") + reply(log, w, nil, b, err) +} + +func (f *frontend) getAvailablePlatformWorkloadIdentityRoleSets(ctx context.Context) []*api.PlatformWorkloadIdentityRoleSet { + roleSets := make([]*api.PlatformWorkloadIdentityRoleSet, 0) + + f.platformWorkloadIdentityRoleSetsMu.RLock() + for _, pwirs := range f.availablePlatformWorkloadIdentityRoleSets { + roleSets = append(roleSets, pwirs) + } + f.platformWorkloadIdentityRoleSetsMu.RUnlock() + + return roleSets +} diff --git a/pkg/frontend/platformworkloadidentityrolesets_list_test.go b/pkg/frontend/platformworkloadidentityrolesets_list_test.go new file mode 100644 index 00000000000..1ab45d9317f --- /dev/null +++ b/pkg/frontend/platformworkloadidentityrolesets_list_test.go @@ -0,0 +1,176 @@ +package frontend + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "sort" + "testing" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/api/v20240812preview" + "github.com/Azure/ARO-RP/pkg/metrics/noop" + "github.com/Azure/ARO-RP/pkg/util/version" +) + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +func TestListPlatformWorkloadIdentityRoleSets(t *testing.T) { + mockSubID := "00000000-0000-0000-0000-000000000000" + method := http.MethodGet + ctx := context.Background() + + for _, tt := range []struct { + name string + changeFeed map[string]*api.PlatformWorkloadIdentityRoleSet + apiVersion string + wantStatusCode int + wantResponse v20240812preview.PlatformWorkloadIdentityRoleSetList + wantError string + }{ + { + name: "GET request results in StatusOK", + changeFeed: map[string]*api.PlatformWorkloadIdentityRoleSet{ + "4.14": { + Properties: api.PlatformWorkloadIdentityRoleSetProperties{ + OpenShiftVersion: "4.14", + PlatformWorkloadIdentityRoles: []api.PlatformWorkloadIdentityRole{ + { + OperatorName: "CloudControllerManager", + RoleDefinitionName: "Azure RedHat OpenShift Cloud Controller Manager Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/a1f96423-95ce-4224-ab27-4e3dc72facd4", + ServiceAccounts: []string{ + "openshift-cloud-controller-manager:cloud-controller-manager", + }, + }, + }, + }, + }, + "4.15": { + Properties: api.PlatformWorkloadIdentityRoleSetProperties{ + OpenShiftVersion: "4.15", + PlatformWorkloadIdentityRoles: []api.PlatformWorkloadIdentityRole{ + { + OperatorName: "CloudControllerManager", + RoleDefinitionName: "Azure RedHat OpenShift Cloud Controller Manager Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/a1f96423-95ce-4224-ab27-4e3dc72facd4", + ServiceAccounts: []string{ + "openshift-cloud-controller-manager:cloud-controller-manager", + }, + }, + { + OperatorName: "ClusterIngressOperator", + RoleDefinitionName: "Azure RedHat OpenShift Cluster Ingress Operator Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/0336e1d3-7a87-462b-b6db-342b63f7802c", + ServiceAccounts: []string{ + "openshift-ingress-operator:ingress-operator", + }, + }, + }, + }, + }, + }, + apiVersion: "2024-08-12-preview", + wantStatusCode: 200, + wantResponse: v20240812preview.PlatformWorkloadIdentityRoleSetList{ + PlatformWorkloadIdentityRoleSets: []*v20240812preview.PlatformWorkloadIdentityRoleSet{ + { + Properties: v20240812preview.PlatformWorkloadIdentityRoleSetProperties{ + OpenShiftVersion: "4.14", + PlatformWorkloadIdentityRoles: []v20240812preview.PlatformWorkloadIdentityRole{ + { + OperatorName: "CloudControllerManager", + RoleDefinitionName: "Azure RedHat OpenShift Cloud Controller Manager Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/a1f96423-95ce-4224-ab27-4e3dc72facd4", + }, + }, + }, + }, + { + Properties: v20240812preview.PlatformWorkloadIdentityRoleSetProperties{ + OpenShiftVersion: "4.15", + PlatformWorkloadIdentityRoles: []v20240812preview.PlatformWorkloadIdentityRole{ + { + OperatorName: "CloudControllerManager", + RoleDefinitionName: "Azure RedHat OpenShift Cloud Controller Manager Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/a1f96423-95ce-4224-ab27-4e3dc72facd4", + }, + { + OperatorName: "ClusterIngressOperator", + RoleDefinitionName: "Azure RedHat OpenShift Cluster Ingress Operator Role", + RoleDefinitionID: "/providers/Microsoft.Authorization/roleDefinitions/0336e1d3-7a87-462b-b6db-342b63f7802c", + }, + }, + }, + }, + }, + }, + }, + { + name: "GET request with non-existent API version results in StatusBadRequest", + apiVersion: "invalid", + wantStatusCode: http.StatusBadRequest, + wantError: "400: InvalidResourceType: : The resource type '' could not be found in the namespace 'microsoft.redhatopenshift' for api version 'invalid'.", + }, + { + name: "GET request with old API version that doesn't support MIWI results in StatusBadRequest", + apiVersion: "2022-09-04", + wantStatusCode: http.StatusBadRequest, + wantError: "400: InvalidResourceType: : The endpoint could not be found in the namespace 'microsoft.redhatopenshift' for api version '2022-09-04'.", + }, + } { + t.Run(tt.name, func(t *testing.T) { + ti := newTestInfra(t).WithSubscriptions().WithPlatformWorkloadIdentityRoleSets() + defer ti.done() + + frontend, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, nil, nil, nil, nil, nil, ti.platformWorkloadIdentityRoleSetsDatabase, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + + go frontend.Run(ctx, nil, nil) + + frontend.platformWorkloadIdentityRoleSetsMu.Lock() + frontend.availablePlatformWorkloadIdentityRoleSets = tt.changeFeed + frontend.platformWorkloadIdentityRoleSetsMu.Unlock() + + resp, b, err := ti.request(method, + fmt.Sprintf("https://server/subscriptions/%s/providers/Microsoft.RedHatOpenShift/locations/%s/platformworkloadidentityrolesets?api-version=%s", mockSubID, ti.env.Location(), tt.apiVersion), + nil, nil) + if err != nil { + t.Fatal(err) + } + + // sort the response as the version order might be changed + if b != nil && resp.StatusCode == http.StatusOK { + var r v20240812preview.PlatformWorkloadIdentityRoleSetList + if err = json.Unmarshal(b, &r); err != nil { + t.Error(err) + } + + sort.Slice(r.PlatformWorkloadIdentityRoleSets, func(i, j int) bool { + return version.CreateSemverFromMinorVersionString(r.PlatformWorkloadIdentityRoleSets[i].Properties.OpenShiftVersion).LessThan(*version.CreateSemverFromMinorVersionString(r.PlatformWorkloadIdentityRoleSets[j].Properties.OpenShiftVersion)) + }) + + b, err = json.Marshal(r) + if err != nil { + t.Error(err) + } + } + + // marshal the expected response into a []byte otherwise + // it will compare zero values to omitempty json tags + want, err := json.Marshal(tt.wantResponse) + if err != nil { + t.Error(err) + } + + err = validateResponse(resp, b, tt.wantStatusCode, tt.wantError, want) + if err != nil { + t.Error(err) + } + }) + } +} diff --git a/pkg/frontend/ready_get.go b/pkg/frontend/ready_get.go index 965c898161b..b62a6a7ff79 100644 --- a/pkg/frontend/ready_get.go +++ b/pkg/frontend/ready_get.go @@ -21,9 +21,10 @@ func (f *frontend) checkReady() bool { return false } - _, ok := f.lastChangefeed.Load().(time.Time) + _, okOcpVersions := f.lastOcpVersionsChangefeed.Load().(time.Time) + _, okPlatformWorkloadIdentityRoleSets := f.lastPlatformWorkloadIdentityRoleSetsChangefeed.Load().(time.Time) - return ok && + return okOcpVersions && okPlatformWorkloadIdentityRoleSets && f.ready.Load().(bool) && f.env.ArmClientAuthorizer().IsReady() && f.env.AdminClientAuthorizer().IsReady() diff --git a/pkg/frontend/security_test.go b/pkg/frontend/security_test.go index ac47d43e28d..1f501074234 100644 --- a/pkg/frontend/security_test.go +++ b/pkg/frontend/security_test.go @@ -77,14 +77,15 @@ func TestSecurity(t *testing.T) { log := logrus.NewEntry(logrus.StandardLogger()) auditHook, auditEntry := testlog.NewAudit() - f, err := NewFrontend(ctx, auditEntry, log, _env, nil, nil, nil, nil, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, nil, nil) + f, err := NewFrontend(ctx, auditEntry, log, _env, nil, nil, nil, nil, nil, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, nil, nil) if err != nil { t.Fatal(err) } // enable /healthz to return 200 f.startTime = time.Time{} - f.lastChangefeed.Store(time.Time{}) + f.lastOcpVersionsChangefeed.Store(time.Time{}) + f.lastPlatformWorkloadIdentityRoleSetsChangefeed.Store(time.Time{}) go f.Run(ctx, nil, nil) diff --git a/pkg/frontend/shared_test.go b/pkg/frontend/shared_test.go index bb37ab22929..c7ffb6523ba 100644 --- a/pkg/frontend/shared_test.go +++ b/pkg/frontend/shared_test.go @@ -68,18 +68,20 @@ type testInfra struct { fixture *testdatabase.Fixture checker *testdatabase.Checker - openShiftClustersClient *cosmosdb.FakeOpenShiftClusterDocumentClient - openShiftClustersDatabase database.OpenShiftClusters - asyncOperationsClient *cosmosdb.FakeAsyncOperationDocumentClient - asyncOperationsDatabase database.AsyncOperations - billingClient *cosmosdb.FakeBillingDocumentClient - billingDatabase database.Billing - clusterManagerClient *cosmosdb.FakeClusterManagerConfigurationDocumentClient - clusterManagerDatabase database.ClusterManagerConfigurations - subscriptionsClient *cosmosdb.FakeSubscriptionDocumentClient - subscriptionsDatabase database.Subscriptions - openShiftVersionsClient *cosmosdb.FakeOpenShiftVersionDocumentClient - openShiftVersionsDatabase database.OpenShiftVersions + openShiftClustersClient *cosmosdb.FakeOpenShiftClusterDocumentClient + openShiftClustersDatabase database.OpenShiftClusters + asyncOperationsClient *cosmosdb.FakeAsyncOperationDocumentClient + asyncOperationsDatabase database.AsyncOperations + billingClient *cosmosdb.FakeBillingDocumentClient + billingDatabase database.Billing + clusterManagerClient *cosmosdb.FakeClusterManagerConfigurationDocumentClient + clusterManagerDatabase database.ClusterManagerConfigurations + subscriptionsClient *cosmosdb.FakeSubscriptionDocumentClient + subscriptionsDatabase database.Subscriptions + openShiftVersionsClient *cosmosdb.FakeOpenShiftVersionDocumentClient + openShiftVersionsDatabase database.OpenShiftVersions + platformWorkloadIdentityRoleSetsClient *cosmosdb.FakePlatformWorkloadIdentityRoleSetDocumentClient + platformWorkloadIdentityRoleSetsDatabase database.PlatformWorkloadIdentityRoleSets } func newTestInfra(t *testing.T) *testInfra { @@ -179,6 +181,13 @@ func (ti *testInfra) WithOpenShiftVersions() *testInfra { return ti } +func (ti *testInfra) WithPlatformWorkloadIdentityRoleSets() *testInfra { + uuid := deterministicuuid.NewTestUUIDGenerator(8) + ti.platformWorkloadIdentityRoleSetsDatabase, ti.platformWorkloadIdentityRoleSetsClient = testdatabase.NewFakePlatformWorkloadIdentityRoleSets(uuid) + ti.fixture.WithPlatformWorkloadIdentityRoleSets(ti.platformWorkloadIdentityRoleSetsDatabase, uuid) + return ti +} + func (ti *testInfra) WithClusterManagerConfigurations() *testInfra { ti.clusterManagerDatabase, ti.clusterManagerClient = testdatabase.NewFakeClusterManager() ti.fixture.WithClusterManagerConfigurations(ti.clusterManagerDatabase) diff --git a/pkg/frontend/subscriptions_put_test.go b/pkg/frontend/subscriptions_put_test.go index cd236ba816a..7be248dbd9b 100644 --- a/pkg/frontend/subscriptions_put_test.go +++ b/pkg/frontend/subscriptions_put_test.go @@ -244,7 +244,7 @@ func TestPutSubscription(t *testing.T) { t.Fatal(err) } - f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, nil, nil) + f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, nil, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, nil, nil) if err != nil { t.Fatal(err) } diff --git a/pkg/frontend/validate.go b/pkg/frontend/validate.go index 7167c1add14..0fdf2bef725 100644 --- a/pkg/frontend/validate.go +++ b/pkg/frontend/validate.go @@ -203,14 +203,14 @@ func validateAdminMasterVMSize(vmSize string) error { // validateInstallVersion validates the install version set in the clusterprofile.version // TODO convert this into static validation instead of this receiver function in the validation for frontend. func (f *frontend) validateInstallVersion(ctx context.Context, oc *api.OpenShiftCluster) error { - f.mu.RLock() + f.ocpVersionsMu.RLock() // If this request is from an older API or the user did not specify // the version to install, use the default version. if oc.Properties.ClusterProfile.Version == "" { oc.Properties.ClusterProfile.Version = f.defaultOcpVersion } _, ok := f.enabledOcpVersions[oc.Properties.ClusterProfile.Version] - f.mu.RUnlock() + f.ocpVersionsMu.RUnlock() if !ok || !validate.RxInstallVersion.MatchString(oc.Properties.ClusterProfile.Version) { return api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidParameter, "properties.clusterProfile.version", "The requested OpenShift version '%s' is invalid.", oc.Properties.ClusterProfile.Version) diff --git a/pkg/operator/controllers/guardrails/guardrails_controller_test.go b/pkg/operator/controllers/guardrails/guardrails_controller_test.go index ceb3b56051a..33408bcfb05 100644 --- a/pkg/operator/controllers/guardrails/guardrails_controller_test.go +++ b/pkg/operator/controllers/guardrails/guardrails_controller_test.go @@ -85,7 +85,7 @@ func TestGuardRailsReconciler(t *testing.T) { }, mocks: func(md *mock_deployer.MockDeployer, cluster *arov1alpha1.Cluster) { expectedConfig := &config.GuardRailsDeploymentConfig{ - Pullspec: "acrtest.example.com/gatekeeper:v3.11.1", + Pullspec: "acrtest.example.com/gatekeeper:v3.15.1", Namespace: "openshift-azure-guardrails", ManagerRequestsCPU: "100m", ManagerLimitCPU: "1000m", diff --git a/pkg/operator/controllers/guardrails/staticresources/crd_gk_config_configs.yaml b/pkg/operator/controllers/guardrails/staticresources/crd_gk_config_configs.yaml index 57826ac09aa..269ca95f9a2 100644 --- a/pkg/operator/controllers/guardrails/staticresources/crd_gk_config_configs.yaml +++ b/pkg/operator/controllers/guardrails/staticresources/crd_gk_config_configs.yaml @@ -39,7 +39,7 @@ spec: excludedNamespaces: items: description: 'A string that supports globbing at its front or end. Ex: "kube-*" will match "kube-system" or "kube-public", "*-system" will match "kube-system" or "gatekeeper-system". The asterisk is required for wildcard matching.' - pattern: ^(\*|\*-)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\*|-\*)?$ + pattern: ^(\*|\*-)?[a-z0-9]([-:a-z0-9]*[a-z0-9])?(\*|-\*)?$ type: string type: array processes: diff --git a/pkg/operator/controllers/guardrails/staticresources/crd_gk_expansion_expansiontemplate.yaml b/pkg/operator/controllers/guardrails/staticresources/crd_gk_expansion_expansiontemplate.yaml index 042249cf102..07ab319b679 100644 --- a/pkg/operator/controllers/guardrails/staticresources/crd_gk_expansion_expansiontemplate.yaml +++ b/pkg/operator/controllers/guardrails/staticresources/crd_gk_expansion_expansiontemplate.yaml @@ -28,6 +28,10 @@ spec: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: + properties: + name: + maxLength: 63 + type: string type: object spec: description: ExpansionTemplateSpec defines the desired state of ExpansionTemplate. @@ -68,6 +72,133 @@ spec: description: TemplateSource specifies the source field on the generator resource to use as the base for expanded resource. For Pod-creating generators, this is usually spec.template type: string type: object + status: + description: ExpansionTemplateStatus defines the observed state of ExpansionTemplate. + properties: + byPod: + items: + description: ExpansionTemplatePodStatusStatus defines the observed state of ExpansionTemplatePodStatus. + properties: + errors: + items: + properties: + message: + type: string + type: + type: string + required: + - message + type: object + type: array + id: + description: 'Important: Run "make" to regenerate code after modifying this file' + type: string + observedGeneration: + format: int64 + type: integer + operations: + items: + type: string + type: array + templateUID: + description: UID is a type that holds unique ID values, including UUIDs. Because we don't ONLY use UUIDs, this is an alias to string. Being a type captures intent and helps make sure that UIDs and names do not get conflated. + type: string + type: object + type: array + type: object type: object served: true storage: true + subresources: + status: {} + - name: v1beta1 + schema: + openAPIV3Schema: + description: ExpansionTemplate is the Schema for the ExpansionTemplate API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ExpansionTemplateSpec defines the desired state of ExpansionTemplate. + properties: + applyTo: + description: ApplyTo lists the specific groups, versions and kinds of generator resources which will be expanded. + items: + description: ApplyTo determines what GVKs items the mutation should apply to. Globs are not allowed. + properties: + groups: + items: + type: string + type: array + kinds: + items: + type: string + type: array + versions: + items: + type: string + type: array + type: object + type: array + enforcementAction: + description: EnforcementAction specifies the enforcement action to be used for resources matching the ExpansionTemplate. Specifying an empty value will use the enforcement action specified by the Constraint in violation. + type: string + generatedGVK: + description: GeneratedGVK specifies the GVK of the resources which the generator resource creates. + properties: + group: + type: string + kind: + type: string + version: + type: string + type: object + templateSource: + description: TemplateSource specifies the source field on the generator resource to use as the base for expanded resource. For Pod-creating generators, this is usually spec.template + type: string + type: object + status: + description: ExpansionTemplateStatus defines the observed state of ExpansionTemplate. + properties: + byPod: + items: + description: ExpansionTemplatePodStatusStatus defines the observed state of ExpansionTemplatePodStatus. + properties: + errors: + items: + properties: + message: + type: string + type: + type: string + required: + - message + type: object + type: array + id: + description: 'Important: Run "make" to regenerate code after modifying this file' + type: string + observedGeneration: + format: int64 + type: integer + operations: + items: + type: string + type: array + templateUID: + description: UID is a type that holds unique ID values, including UUIDs. Because we don't ONLY use UUIDs, this is an alias to string. Being a type captures intent and helps make sure that UIDs and names do not get conflated. + type: string + type: object + type: array + type: object + type: object + served: true + storage: false + subresources: + status: {} diff --git a/pkg/operator/controllers/guardrails/staticresources/crd_gk_externaldata_providers.yaml b/pkg/operator/controllers/guardrails/staticresources/crd_gk_externaldata_providers.yaml index 0deb6f630b1..177afbb6780 100644 --- a/pkg/operator/controllers/guardrails/staticresources/crd_gk_externaldata_providers.yaml +++ b/pkg/operator/controllers/guardrails/staticresources/crd_gk_externaldata_providers.yaml @@ -41,12 +41,12 @@ spec: description: Timeout is the timeout when querying the provider. type: integer url: - description: URL is the url for the provider. URL is prefixed with http:// or https://. + description: URL is the url for the provider. URL is prefixed with https://. type: string type: object type: object served: true - storage: true + storage: false - name: v1beta1 schema: openAPIV3Schema: @@ -70,9 +70,9 @@ spec: description: Timeout is the timeout when querying the provider. type: integer url: - description: URL is the url for the provider. URL is prefixed with http:// or https://. + description: URL is the url for the provider. URL is prefixed with https://. type: string type: object type: object served: true - storage: false + storage: true diff --git a/pkg/operator/controllers/guardrails/staticresources/crd_gk_mutations_assign.yaml b/pkg/operator/controllers/guardrails/staticresources/crd_gk_mutations_assign.yaml index ce98648baff..0221a194812 100644 --- a/pkg/operator/controllers/guardrails/staticresources/crd_gk_mutations_assign.yaml +++ b/pkg/operator/controllers/guardrails/staticresources/crd_gk_mutations_assign.yaml @@ -65,7 +65,7 @@ spec: description: 'ExcludedNamespaces is a list of namespace names. If defined, a constraint only applies to resources not in a listed namespace. ExcludedNamespaces also supports a prefix or suffix based glob. For example, `excludedNamespaces: [kube-*]` matches both `kube-system` and `kube-public`, and `excludedNamespaces: [*-system]` matches both `kube-system` and `gatekeeper-system`.' items: description: 'A string that supports globbing at its front or end. Ex: "kube-*" will match "kube-system" or "kube-public", "*-system" will match "kube-system" or "gatekeeper-system". The asterisk is required for wildcard matching.' - pattern: ^(\*|\*-)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\*|-\*)?$ + pattern: ^(\*|\*-)?[a-z0-9]([-:a-z0-9]*[a-z0-9])?(\*|-\*)?$ type: string type: array kinds: @@ -115,7 +115,7 @@ spec: type: object name: description: 'Name is the name of an object. If defined, it will match against objects with the specified name. Name also supports a prefix or suffix glob. For example, `name: pod-*` would match both `pod-a` and `pod-b`, and `name: *-pod` would match both `a-pod` and `b-pod`.' - pattern: ^(\*|\*-)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\*|-\*)?$ + pattern: ^(\*|\*-)?[a-z0-9]([-:a-z0-9]*[a-z0-9])?(\*|-\*)?$ type: string namespaceSelector: description: NamespaceSelector is a label selector against an object's containing namespace or the object itself, if the object is a namespace. @@ -151,7 +151,7 @@ spec: description: 'Namespaces is a list of namespace names. If defined, a constraint only applies to resources in a listed namespace. Namespaces also supports a prefix or suffix based glob. For example, `namespaces: [kube-*]` matches both `kube-system` and `kube-public`, and `namespaces: [*-system]` matches both `kube-system` and `gatekeeper-system`.' items: description: 'A string that supports globbing at its front or end. Ex: "kube-*" will match "kube-system" or "kube-public", "*-system" will match "kube-system" or "gatekeeper-system". The asterisk is required for wildcard matching.' - pattern: ^(\*|\*-)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\*|-\*)?$ + pattern: ^(\*|\*-)?[a-z0-9]([-:a-z0-9]*[a-z0-9])?(\*|-\*)?$ type: string type: array scope: @@ -310,7 +310,7 @@ spec: description: 'ExcludedNamespaces is a list of namespace names. If defined, a constraint only applies to resources not in a listed namespace. ExcludedNamespaces also supports a prefix or suffix based glob. For example, `excludedNamespaces: [kube-*]` matches both `kube-system` and `kube-public`, and `excludedNamespaces: [*-system]` matches both `kube-system` and `gatekeeper-system`.' items: description: 'A string that supports globbing at its front or end. Ex: "kube-*" will match "kube-system" or "kube-public", "*-system" will match "kube-system" or "gatekeeper-system". The asterisk is required for wildcard matching.' - pattern: ^(\*|\*-)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\*|-\*)?$ + pattern: ^(\*|\*-)?[a-z0-9]([-:a-z0-9]*[a-z0-9])?(\*|-\*)?$ type: string type: array kinds: @@ -360,7 +360,7 @@ spec: type: object name: description: 'Name is the name of an object. If defined, it will match against objects with the specified name. Name also supports a prefix or suffix glob. For example, `name: pod-*` would match both `pod-a` and `pod-b`, and `name: *-pod` would match both `a-pod` and `b-pod`.' - pattern: ^(\*|\*-)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\*|-\*)?$ + pattern: ^(\*|\*-)?[a-z0-9]([-:a-z0-9]*[a-z0-9])?(\*|-\*)?$ type: string namespaceSelector: description: NamespaceSelector is a label selector against an object's containing namespace or the object itself, if the object is a namespace. @@ -396,7 +396,7 @@ spec: description: 'Namespaces is a list of namespace names. If defined, a constraint only applies to resources in a listed namespace. Namespaces also supports a prefix or suffix based glob. For example, `namespaces: [kube-*]` matches both `kube-system` and `kube-public`, and `namespaces: [*-system]` matches both `kube-system` and `gatekeeper-system`.' items: description: 'A string that supports globbing at its front or end. Ex: "kube-*" will match "kube-system" or "kube-public", "*-system" will match "kube-system" or "gatekeeper-system". The asterisk is required for wildcard matching.' - pattern: ^(\*|\*-)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\*|-\*)?$ + pattern: ^(\*|\*-)?[a-z0-9]([-:a-z0-9]*[a-z0-9])?(\*|-\*)?$ type: string type: array scope: @@ -555,7 +555,7 @@ spec: description: 'ExcludedNamespaces is a list of namespace names. If defined, a constraint only applies to resources not in a listed namespace. ExcludedNamespaces also supports a prefix or suffix based glob. For example, `excludedNamespaces: [kube-*]` matches both `kube-system` and `kube-public`, and `excludedNamespaces: [*-system]` matches both `kube-system` and `gatekeeper-system`.' items: description: 'A string that supports globbing at its front or end. Ex: "kube-*" will match "kube-system" or "kube-public", "*-system" will match "kube-system" or "gatekeeper-system". The asterisk is required for wildcard matching.' - pattern: ^(\*|\*-)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\*|-\*)?$ + pattern: ^(\*|\*-)?[a-z0-9]([-:a-z0-9]*[a-z0-9])?(\*|-\*)?$ type: string type: array kinds: @@ -605,7 +605,7 @@ spec: type: object name: description: 'Name is the name of an object. If defined, it will match against objects with the specified name. Name also supports a prefix or suffix glob. For example, `name: pod-*` would match both `pod-a` and `pod-b`, and `name: *-pod` would match both `a-pod` and `b-pod`.' - pattern: ^(\*|\*-)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\*|-\*)?$ + pattern: ^(\*|\*-)?[a-z0-9]([-:a-z0-9]*[a-z0-9])?(\*|-\*)?$ type: string namespaceSelector: description: NamespaceSelector is a label selector against an object's containing namespace or the object itself, if the object is a namespace. @@ -641,7 +641,7 @@ spec: description: 'Namespaces is a list of namespace names. If defined, a constraint only applies to resources in a listed namespace. Namespaces also supports a prefix or suffix based glob. For example, `namespaces: [kube-*]` matches both `kube-system` and `kube-public`, and `namespaces: [*-system]` matches both `kube-system` and `gatekeeper-system`.' items: description: 'A string that supports globbing at its front or end. Ex: "kube-*" will match "kube-system" or "kube-public", "*-system" will match "kube-system" or "gatekeeper-system". The asterisk is required for wildcard matching.' - pattern: ^(\*|\*-)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\*|-\*)?$ + pattern: ^(\*|\*-)?[a-z0-9]([-:a-z0-9]*[a-z0-9])?(\*|-\*)?$ type: string type: array scope: diff --git a/pkg/operator/controllers/guardrails/staticresources/crd_gk_mutations_assignimage.yaml b/pkg/operator/controllers/guardrails/staticresources/crd_gk_mutations_assignimage.yaml new file mode 100644 index 00000000000..676f32b524a --- /dev/null +++ b/pkg/operator/controllers/guardrails/staticresources/crd_gk_mutations_assignimage.yaml @@ -0,0 +1,237 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.10.0 + labels: + gatekeeper.sh/system: "yes" + name: assignimage.mutations.gatekeeper.sh +spec: + group: mutations.gatekeeper.sh + names: + kind: AssignImage + listKind: AssignImageList + plural: assignimage + singular: assignimage + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: AssignImage is the Schema for the assignimage API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + properties: + name: + maxLength: 63 + type: string + type: object + spec: + description: AssignImageSpec defines the desired state of AssignImage. + properties: + applyTo: + description: ApplyTo lists the specific groups, versions and kinds a mutation will be applied to. This is necessary because every mutation implies part of an object schema and object schemas are associated with specific GVKs. + items: + description: ApplyTo determines what GVKs items the mutation should apply to. Globs are not allowed. + properties: + groups: + items: + type: string + type: array + kinds: + items: + type: string + type: array + versions: + items: + type: string + type: array + type: object + type: array + location: + description: 'Location describes the path to be mutated, for example: `spec.containers[name: main].image`.' + type: string + match: + description: Match allows the user to limit which resources get mutated. Individual match criteria are AND-ed together. An undefined match criteria matches everything. + properties: + excludedNamespaces: + description: 'ExcludedNamespaces is a list of namespace names. If defined, a constraint only applies to resources not in a listed namespace. ExcludedNamespaces also supports a prefix or suffix based glob. For example, `excludedNamespaces: [kube-*]` matches both `kube-system` and `kube-public`, and `excludedNamespaces: [*-system]` matches both `kube-system` and `gatekeeper-system`.' + items: + description: 'A string that supports globbing at its front or end. Ex: "kube-*" will match "kube-system" or "kube-public", "*-system" will match "kube-system" or "gatekeeper-system". The asterisk is required for wildcard matching.' + pattern: ^(\*|\*-)?[a-z0-9]([-:a-z0-9]*[a-z0-9])?(\*|-\*)?$ + type: string + type: array + kinds: + items: + description: Kinds accepts a list of objects with apiGroups and kinds fields that list the groups/kinds of objects to which the mutation will apply. If multiple groups/kinds objects are specified, only one match is needed for the resource to be in scope. + properties: + apiGroups: + description: APIGroups is the API groups the resources belong to. '*' is all groups. If '*' is present, the length of the slice must be one. Required. + items: + type: string + type: array + kinds: + items: + type: string + type: array + type: object + type: array + labelSelector: + description: 'LabelSelector is the combination of two optional fields: `matchLabels` and `matchExpressions`. These two fields provide different methods of selecting or excluding k8s objects based on the label keys and values included in object metadata. All selection expressions from both sections are ANDed to determine if an object meets the cumulative requirements of the selector.' + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + name: + description: 'Name is the name of an object. If defined, it will match against objects with the specified name. Name also supports a prefix or suffix glob. For example, `name: pod-*` would match both `pod-a` and `pod-b`, and `name: *-pod` would match both `a-pod` and `b-pod`.' + pattern: ^(\*|\*-)?[a-z0-9]([-:a-z0-9]*[a-z0-9])?(\*|-\*)?$ + type: string + namespaceSelector: + description: NamespaceSelector is a label selector against an object's containing namespace or the object itself, if the object is a namespace. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + namespaces: + description: 'Namespaces is a list of namespace names. If defined, a constraint only applies to resources in a listed namespace. Namespaces also supports a prefix or suffix based glob. For example, `namespaces: [kube-*]` matches both `kube-system` and `kube-public`, and `namespaces: [*-system]` matches both `kube-system` and `gatekeeper-system`.' + items: + description: 'A string that supports globbing at its front or end. Ex: "kube-*" will match "kube-system" or "kube-public", "*-system" will match "kube-system" or "gatekeeper-system". The asterisk is required for wildcard matching.' + pattern: ^(\*|\*-)?[a-z0-9]([-:a-z0-9]*[a-z0-9])?(\*|-\*)?$ + type: string + type: array + scope: + description: Scope determines if cluster-scoped and/or namespaced-scoped resources are matched. Accepts `*`, `Cluster`, or `Namespaced`. (defaults to `*`) + type: string + source: + description: Source determines whether generated or original resources are matched. Accepts `Generated`|`Original`|`All` (defaults to `All`). A value of `Generated` will only match generated resources, while `Original` will only match regular resources. + enum: + - All + - Generated + - Original + type: string + type: object + parameters: + description: Parameters define the behavior of the mutator. + properties: + assignDomain: + description: AssignDomain sets the domain component on an image string. The trailing slash should not be included. + type: string + assignPath: + description: AssignPath sets the domain component on an image string. + type: string + assignTag: + description: AssignImage sets the image component on an image string. It must start with a `:` or `@`. + type: string + pathTests: + items: + description: "PathTest allows the user to customize how the mutation works if parent paths are missing. It traverses the list in order. All sub paths are tested against the provided condition, if the test fails, the mutation is not applied. All `subPath` entries must be a prefix of `location`. Any glob characters will take on the same value as was used to expand the matching glob in `location`. \n Available Tests: * MustExist - the path must exist or do not mutate * MustNotExist - the path must not exist or do not mutate." + properties: + condition: + description: Condition describes whether the path either MustExist or MustNotExist in the original object + enum: + - MustExist + - MustNotExist + type: string + subPath: + type: string + type: object + type: array + type: object + type: object + status: + description: AssignImageStatus defines the observed state of AssignImage. + properties: + byPod: + items: + description: MutatorPodStatusStatus defines the observed state of MutatorPodStatus. + properties: + enforced: + type: boolean + errors: + items: + description: MutatorError represents a single error caught while adding a mutator to a system. + properties: + message: + type: string + type: + description: Type indicates a specific class of error for use by controller code. If not present, the error should be treated as not matching any known type. + type: string + required: + - message + type: object + type: array + id: + type: string + mutatorUID: + description: Storing the mutator UID allows us to detect drift, such as when a mutator has been recreated after its CRD was deleted out from under it, interrupting the watch + type: string + observedGeneration: + format: int64 + type: integer + operations: + items: + type: string + type: array + type: object + type: array + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/pkg/operator/controllers/guardrails/staticresources/crd_gk_mutations_assignmetadata.yaml b/pkg/operator/controllers/guardrails/staticresources/crd_gk_mutations_assignmetadata.yaml index 3a63eef3cb3..65c17ed3ae1 100644 --- a/pkg/operator/controllers/guardrails/staticresources/crd_gk_mutations_assignmetadata.yaml +++ b/pkg/operator/controllers/guardrails/staticresources/crd_gk_mutations_assignmetadata.yaml @@ -39,13 +39,13 @@ spec: location: type: string match: - description: Match selects objects to apply mutations to. + description: Match selects which objects are in scope. properties: excludedNamespaces: description: 'ExcludedNamespaces is a list of namespace names. If defined, a constraint only applies to resources not in a listed namespace. ExcludedNamespaces also supports a prefix or suffix based glob. For example, `excludedNamespaces: [kube-*]` matches both `kube-system` and `kube-public`, and `excludedNamespaces: [*-system]` matches both `kube-system` and `gatekeeper-system`.' items: description: 'A string that supports globbing at its front or end. Ex: "kube-*" will match "kube-system" or "kube-public", "*-system" will match "kube-system" or "gatekeeper-system". The asterisk is required for wildcard matching.' - pattern: ^(\*|\*-)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\*|-\*)?$ + pattern: ^(\*|\*-)?[a-z0-9]([-:a-z0-9]*[a-z0-9])?(\*|-\*)?$ type: string type: array kinds: @@ -95,7 +95,7 @@ spec: type: object name: description: 'Name is the name of an object. If defined, it will match against objects with the specified name. Name also supports a prefix or suffix glob. For example, `name: pod-*` would match both `pod-a` and `pod-b`, and `name: *-pod` would match both `a-pod` and `b-pod`.' - pattern: ^(\*|\*-)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\*|-\*)?$ + pattern: ^(\*|\*-)?[a-z0-9]([-:a-z0-9]*[a-z0-9])?(\*|-\*)?$ type: string namespaceSelector: description: NamespaceSelector is a label selector against an object's containing namespace or the object itself, if the object is a namespace. @@ -131,7 +131,7 @@ spec: description: 'Namespaces is a list of namespace names. If defined, a constraint only applies to resources in a listed namespace. Namespaces also supports a prefix or suffix based glob. For example, `namespaces: [kube-*]` matches both `kube-system` and `kube-public`, and `namespaces: [*-system]` matches both `kube-system` and `gatekeeper-system`.' items: description: 'A string that supports globbing at its front or end. Ex: "kube-*" will match "kube-system" or "kube-public", "*-system" will match "kube-system" or "gatekeeper-system". The asterisk is required for wildcard matching.' - pattern: ^(\*|\*-)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\*|-\*)?$ + pattern: ^(\*|\*-)?[a-z0-9]([-:a-z0-9]*[a-z0-9])?(\*|-\*)?$ type: string type: array scope: @@ -250,13 +250,13 @@ spec: location: type: string match: - description: Match selects objects to apply mutations to. + description: Match selects which objects are in scope. properties: excludedNamespaces: description: 'ExcludedNamespaces is a list of namespace names. If defined, a constraint only applies to resources not in a listed namespace. ExcludedNamespaces also supports a prefix or suffix based glob. For example, `excludedNamespaces: [kube-*]` matches both `kube-system` and `kube-public`, and `excludedNamespaces: [*-system]` matches both `kube-system` and `gatekeeper-system`.' items: description: 'A string that supports globbing at its front or end. Ex: "kube-*" will match "kube-system" or "kube-public", "*-system" will match "kube-system" or "gatekeeper-system". The asterisk is required for wildcard matching.' - pattern: ^(\*|\*-)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\*|-\*)?$ + pattern: ^(\*|\*-)?[a-z0-9]([-:a-z0-9]*[a-z0-9])?(\*|-\*)?$ type: string type: array kinds: @@ -306,7 +306,7 @@ spec: type: object name: description: 'Name is the name of an object. If defined, it will match against objects with the specified name. Name also supports a prefix or suffix glob. For example, `name: pod-*` would match both `pod-a` and `pod-b`, and `name: *-pod` would match both `a-pod` and `b-pod`.' - pattern: ^(\*|\*-)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\*|-\*)?$ + pattern: ^(\*|\*-)?[a-z0-9]([-:a-z0-9]*[a-z0-9])?(\*|-\*)?$ type: string namespaceSelector: description: NamespaceSelector is a label selector against an object's containing namespace or the object itself, if the object is a namespace. @@ -342,7 +342,7 @@ spec: description: 'Namespaces is a list of namespace names. If defined, a constraint only applies to resources in a listed namespace. Namespaces also supports a prefix or suffix based glob. For example, `namespaces: [kube-*]` matches both `kube-system` and `kube-public`, and `namespaces: [*-system]` matches both `kube-system` and `gatekeeper-system`.' items: description: 'A string that supports globbing at its front or end. Ex: "kube-*" will match "kube-system" or "kube-public", "*-system" will match "kube-system" or "gatekeeper-system". The asterisk is required for wildcard matching.' - pattern: ^(\*|\*-)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\*|-\*)?$ + pattern: ^(\*|\*-)?[a-z0-9]([-:a-z0-9]*[a-z0-9])?(\*|-\*)?$ type: string type: array scope: @@ -461,13 +461,13 @@ spec: location: type: string match: - description: Match selects objects to apply mutations to. + description: Match selects which objects are in scope. properties: excludedNamespaces: description: 'ExcludedNamespaces is a list of namespace names. If defined, a constraint only applies to resources not in a listed namespace. ExcludedNamespaces also supports a prefix or suffix based glob. For example, `excludedNamespaces: [kube-*]` matches both `kube-system` and `kube-public`, and `excludedNamespaces: [*-system]` matches both `kube-system` and `gatekeeper-system`.' items: description: 'A string that supports globbing at its front or end. Ex: "kube-*" will match "kube-system" or "kube-public", "*-system" will match "kube-system" or "gatekeeper-system". The asterisk is required for wildcard matching.' - pattern: ^(\*|\*-)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\*|-\*)?$ + pattern: ^(\*|\*-)?[a-z0-9]([-:a-z0-9]*[a-z0-9])?(\*|-\*)?$ type: string type: array kinds: @@ -517,7 +517,7 @@ spec: type: object name: description: 'Name is the name of an object. If defined, it will match against objects with the specified name. Name also supports a prefix or suffix glob. For example, `name: pod-*` would match both `pod-a` and `pod-b`, and `name: *-pod` would match both `a-pod` and `b-pod`.' - pattern: ^(\*|\*-)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\*|-\*)?$ + pattern: ^(\*|\*-)?[a-z0-9]([-:a-z0-9]*[a-z0-9])?(\*|-\*)?$ type: string namespaceSelector: description: NamespaceSelector is a label selector against an object's containing namespace or the object itself, if the object is a namespace. @@ -553,7 +553,7 @@ spec: description: 'Namespaces is a list of namespace names. If defined, a constraint only applies to resources in a listed namespace. Namespaces also supports a prefix or suffix based glob. For example, `namespaces: [kube-*]` matches both `kube-system` and `kube-public`, and `namespaces: [*-system]` matches both `kube-system` and `gatekeeper-system`.' items: description: 'A string that supports globbing at its front or end. Ex: "kube-*" will match "kube-system" or "kube-public", "*-system" will match "kube-system" or "gatekeeper-system". The asterisk is required for wildcard matching.' - pattern: ^(\*|\*-)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\*|-\*)?$ + pattern: ^(\*|\*-)?[a-z0-9]([-:a-z0-9]*[a-z0-9])?(\*|-\*)?$ type: string type: array scope: diff --git a/pkg/operator/controllers/guardrails/staticresources/crd_gk_mutations_modifyset.yaml b/pkg/operator/controllers/guardrails/staticresources/crd_gk_mutations_modifyset.yaml index 1bb1933366d..46574fd369f 100644 --- a/pkg/operator/controllers/guardrails/staticresources/crd_gk_mutations_modifyset.yaml +++ b/pkg/operator/controllers/guardrails/staticresources/crd_gk_mutations_modifyset.yaml @@ -65,7 +65,7 @@ spec: description: 'ExcludedNamespaces is a list of namespace names. If defined, a constraint only applies to resources not in a listed namespace. ExcludedNamespaces also supports a prefix or suffix based glob. For example, `excludedNamespaces: [kube-*]` matches both `kube-system` and `kube-public`, and `excludedNamespaces: [*-system]` matches both `kube-system` and `gatekeeper-system`.' items: description: 'A string that supports globbing at its front or end. Ex: "kube-*" will match "kube-system" or "kube-public", "*-system" will match "kube-system" or "gatekeeper-system". The asterisk is required for wildcard matching.' - pattern: ^(\*|\*-)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\*|-\*)?$ + pattern: ^(\*|\*-)?[a-z0-9]([-:a-z0-9]*[a-z0-9])?(\*|-\*)?$ type: string type: array kinds: @@ -115,7 +115,7 @@ spec: type: object name: description: 'Name is the name of an object. If defined, it will match against objects with the specified name. Name also supports a prefix or suffix glob. For example, `name: pod-*` would match both `pod-a` and `pod-b`, and `name: *-pod` would match both `a-pod` and `b-pod`.' - pattern: ^(\*|\*-)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\*|-\*)?$ + pattern: ^(\*|\*-)?[a-z0-9]([-:a-z0-9]*[a-z0-9])?(\*|-\*)?$ type: string namespaceSelector: description: NamespaceSelector is a label selector against an object's containing namespace or the object itself, if the object is a namespace. @@ -151,7 +151,7 @@ spec: description: 'Namespaces is a list of namespace names. If defined, a constraint only applies to resources in a listed namespace. Namespaces also supports a prefix or suffix based glob. For example, `namespaces: [kube-*]` matches both `kube-system` and `kube-public`, and `namespaces: [*-system]` matches both `kube-system` and `gatekeeper-system`.' items: description: 'A string that supports globbing at its front or end. Ex: "kube-*" will match "kube-system" or "kube-public", "*-system" will match "kube-system" or "gatekeeper-system". The asterisk is required for wildcard matching.' - pattern: ^(\*|\*-)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\*|-\*)?$ + pattern: ^(\*|\*-)?[a-z0-9]([-:a-z0-9]*[a-z0-9])?(\*|-\*)?$ type: string type: array scope: @@ -283,7 +283,7 @@ spec: description: 'ExcludedNamespaces is a list of namespace names. If defined, a constraint only applies to resources not in a listed namespace. ExcludedNamespaces also supports a prefix or suffix based glob. For example, `excludedNamespaces: [kube-*]` matches both `kube-system` and `kube-public`, and `excludedNamespaces: [*-system]` matches both `kube-system` and `gatekeeper-system`.' items: description: 'A string that supports globbing at its front or end. Ex: "kube-*" will match "kube-system" or "kube-public", "*-system" will match "kube-system" or "gatekeeper-system". The asterisk is required for wildcard matching.' - pattern: ^(\*|\*-)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\*|-\*)?$ + pattern: ^(\*|\*-)?[a-z0-9]([-:a-z0-9]*[a-z0-9])?(\*|-\*)?$ type: string type: array kinds: @@ -333,7 +333,7 @@ spec: type: object name: description: 'Name is the name of an object. If defined, it will match against objects with the specified name. Name also supports a prefix or suffix glob. For example, `name: pod-*` would match both `pod-a` and `pod-b`, and `name: *-pod` would match both `a-pod` and `b-pod`.' - pattern: ^(\*|\*-)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\*|-\*)?$ + pattern: ^(\*|\*-)?[a-z0-9]([-:a-z0-9]*[a-z0-9])?(\*|-\*)?$ type: string namespaceSelector: description: NamespaceSelector is a label selector against an object's containing namespace or the object itself, if the object is a namespace. @@ -369,7 +369,7 @@ spec: description: 'Namespaces is a list of namespace names. If defined, a constraint only applies to resources in a listed namespace. Namespaces also supports a prefix or suffix based glob. For example, `namespaces: [kube-*]` matches both `kube-system` and `kube-public`, and `namespaces: [*-system]` matches both `kube-system` and `gatekeeper-system`.' items: description: 'A string that supports globbing at its front or end. Ex: "kube-*" will match "kube-system" or "kube-public", "*-system" will match "kube-system" or "gatekeeper-system". The asterisk is required for wildcard matching.' - pattern: ^(\*|\*-)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\*|-\*)?$ + pattern: ^(\*|\*-)?[a-z0-9]([-:a-z0-9]*[a-z0-9])?(\*|-\*)?$ type: string type: array scope: @@ -501,7 +501,7 @@ spec: description: 'ExcludedNamespaces is a list of namespace names. If defined, a constraint only applies to resources not in a listed namespace. ExcludedNamespaces also supports a prefix or suffix based glob. For example, `excludedNamespaces: [kube-*]` matches both `kube-system` and `kube-public`, and `excludedNamespaces: [*-system]` matches both `kube-system` and `gatekeeper-system`.' items: description: 'A string that supports globbing at its front or end. Ex: "kube-*" will match "kube-system" or "kube-public", "*-system" will match "kube-system" or "gatekeeper-system". The asterisk is required for wildcard matching.' - pattern: ^(\*|\*-)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\*|-\*)?$ + pattern: ^(\*|\*-)?[a-z0-9]([-:a-z0-9]*[a-z0-9])?(\*|-\*)?$ type: string type: array kinds: @@ -551,7 +551,7 @@ spec: type: object name: description: 'Name is the name of an object. If defined, it will match against objects with the specified name. Name also supports a prefix or suffix glob. For example, `name: pod-*` would match both `pod-a` and `pod-b`, and `name: *-pod` would match both `a-pod` and `b-pod`.' - pattern: ^(\*|\*-)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\*|-\*)?$ + pattern: ^(\*|\*-)?[a-z0-9]([-:a-z0-9]*[a-z0-9])?(\*|-\*)?$ type: string namespaceSelector: description: NamespaceSelector is a label selector against an object's containing namespace or the object itself, if the object is a namespace. @@ -587,7 +587,7 @@ spec: description: 'Namespaces is a list of namespace names. If defined, a constraint only applies to resources in a listed namespace. Namespaces also supports a prefix or suffix based glob. For example, `namespaces: [kube-*]` matches both `kube-system` and `kube-public`, and `namespaces: [*-system]` matches both `kube-system` and `gatekeeper-system`.' items: description: 'A string that supports globbing at its front or end. Ex: "kube-*" will match "kube-system" or "kube-public", "*-system" will match "kube-system" or "gatekeeper-system". The asterisk is required for wildcard matching.' - pattern: ^(\*|\*-)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\*|-\*)?$ + pattern: ^(\*|\*-)?[a-z0-9]([-:a-z0-9]*[a-z0-9])?(\*|-\*)?$ type: string type: array scope: diff --git a/pkg/operator/controllers/guardrails/staticresources/crd_gk_status_expansiontemplatepodstatuses.yaml b/pkg/operator/controllers/guardrails/staticresources/crd_gk_status_expansiontemplatepodstatuses.yaml new file mode 100644 index 00000000000..a00bdd53536 --- /dev/null +++ b/pkg/operator/controllers/guardrails/staticresources/crd_gk_status_expansiontemplatepodstatuses.yaml @@ -0,0 +1,62 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.10.0 + labels: + gatekeeper.sh/system: "yes" + name: expansiontemplatepodstatuses.status.gatekeeper.sh +spec: + group: status.gatekeeper.sh + names: + kind: ExpansionTemplatePodStatus + listKind: ExpansionTemplatePodStatusList + plural: expansiontemplatepodstatuses + singular: expansiontemplatepodstatus + preserveUnknownFields: false + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: ExpansionTemplatePodStatus is the Schema for the expansiontemplatepodstatuses API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + status: + description: ExpansionTemplatePodStatusStatus defines the observed state of ExpansionTemplatePodStatus. + properties: + errors: + items: + properties: + message: + type: string + type: + type: string + required: + - message + type: object + type: array + id: + description: 'Important: Run "make" to regenerate code after modifying this file' + type: string + observedGeneration: + format: int64 + type: integer + operations: + items: + type: string + type: array + templateUID: + description: UID is a type that holds unique ID values, including UUIDs. Because we don't ONLY use UUIDs, this is an alias to string. Being a type captures intent and helps make sure that UIDs and names do not get conflated. + type: string + type: object + type: object + served: true + storage: true diff --git a/pkg/operator/controllers/guardrails/staticresources/crd_gk_syncset_syncsets.yaml b/pkg/operator/controllers/guardrails/staticresources/crd_gk_syncset_syncsets.yaml new file mode 100644 index 00000000000..e2bc4c53913 --- /dev/null +++ b/pkg/operator/controllers/guardrails/staticresources/crd_gk_syncset_syncsets.yaml @@ -0,0 +1,52 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.10.0 + labels: + gatekeeper.sh/system: "yes" + name: syncsets.syncset.gatekeeper.sh +spec: + group: syncset.gatekeeper.sh + names: + kind: SyncSet + listKind: SyncSetList + plural: syncsets + singular: syncset + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: SyncSet defines which resources Gatekeeper will cache. The union of all SyncSets plus the syncOnly field of Gatekeeper's Config resource defines the sets of resources that will be synced. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + properties: + name: + maxLength: 63 + type: string + type: object + spec: + properties: + gvks: + items: + properties: + group: + type: string + kind: + type: string + version: + type: string + type: object + type: array + type: object + type: object + served: true + storage: true diff --git a/pkg/operator/controllers/guardrails/staticresources/crd_gk_templates_constrainttemplates.yaml b/pkg/operator/controllers/guardrails/staticresources/crd_gk_templates_constrainttemplates.yaml index a4da4e9e90f..afc89d03bdf 100644 --- a/pkg/operator/controllers/guardrails/staticresources/crd_gk_templates_constrainttemplates.yaml +++ b/pkg/operator/controllers/guardrails/staticresources/crd_gk_templates_constrainttemplates.yaml @@ -61,6 +61,24 @@ spec: targets: items: properties: + code: + description: The source code options for the constraint template. "Rego" can only be specified in one place (either here or in the "rego" field) + items: + properties: + engine: + description: 'The engine used to evaluate the code. Example: "Rego". Required.' + type: string + source: + description: The source code for the template. Required. + x-kubernetes-preserve-unknown-fields: true + required: + - engine + - source + type: object + type: array + x-kubernetes-list-map-keys: + - engine + x-kubernetes-list-type: map libs: items: type: string @@ -156,6 +174,24 @@ spec: targets: items: properties: + code: + description: The source code options for the constraint template. "Rego" can only be specified in one place (either here or in the "rego" field) + items: + properties: + engine: + description: 'The engine used to evaluate the code. Example: "Rego". Required.' + type: string + source: + description: The source code for the template. Required. + x-kubernetes-preserve-unknown-fields: true + required: + - engine + - source + type: object + type: array + x-kubernetes-list-map-keys: + - engine + x-kubernetes-list-type: map libs: items: type: string @@ -251,6 +287,24 @@ spec: targets: items: properties: + code: + description: The source code options for the constraint template. "Rego" can only be specified in one place (either here or in the "rego" field) + items: + properties: + engine: + description: 'The engine used to evaluate the code. Example: "Rego". Required.' + type: string + source: + description: The source code for the template. Required. + x-kubernetes-preserve-unknown-fields: true + required: + - engine + - source + type: object + type: array + x-kubernetes-list-map-keys: + - engine + x-kubernetes-list-type: map libs: items: type: string diff --git a/pkg/operator/controllers/guardrails/staticresources/gk_audit_controller_deployment.yaml b/pkg/operator/controllers/guardrails/staticresources/gk_audit_controller_deployment.yaml index 3431ee49697..32a9d6b1ae6 100644 --- a/pkg/operator/controllers/guardrails/staticresources/gk_audit_controller_deployment.yaml +++ b/pkg/operator/controllers/guardrails/staticresources/gk_audit_controller_deployment.yaml @@ -69,6 +69,8 @@ spec: fieldPath: metadata.namespace - name: CONTAINER_NAME value: manager + - name: OTEL_RESOURCE_ATTRIBUTES + value: k8s.pod.name=$(POD_NAME),k8s.namespace.name=$(NAMESPACE),k8s.container.name=$(CONTAINER_NAME) image: {{.Pullspec}} imagePullPolicy: Always livenessProbe: diff --git a/pkg/operator/controllers/guardrails/staticresources/gk_cluster_role.yaml b/pkg/operator/controllers/guardrails/staticresources/gk_cluster_role.yaml index ad28f71eb89..8c49df91a79 100644 --- a/pkg/operator/controllers/guardrails/staticresources/gk_cluster_role.yaml +++ b/pkg/operator/controllers/guardrails/staticresources/gk_cluster_role.yaml @@ -6,6 +6,13 @@ metadata: gatekeeper.sh/system: "yes" name: gatekeeper-manager-role rules: +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch - apiGroups: - '*' resources: @@ -70,6 +77,18 @@ rules: - patch - update - watch +- apiGroups: + - expansion.gatekeeper.sh + resources: + - '*' + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - externaldata.gatekeeper.sh resources: diff --git a/pkg/operator/controllers/guardrails/staticresources/gk_controller_manager_deployment.yaml b/pkg/operator/controllers/guardrails/staticresources/gk_controller_manager_deployment.yaml index c4e1ede2e07..2098cf2282b 100644 --- a/pkg/operator/controllers/guardrails/staticresources/gk_controller_manager_deployment.yaml +++ b/pkg/operator/controllers/guardrails/staticresources/gk_controller_manager_deployment.yaml @@ -78,6 +78,8 @@ spec: fieldPath: metadata.namespace - name: CONTAINER_NAME value: manager + - name: OTEL_RESOURCE_ATTRIBUTES + value: k8s.pod.name=$(POD_NAME),k8s.namespace.name=$(NAMESPACE),k8s.container.name=$(CONTAINER_NAME) image: {{.Pullspec}} imagePullPolicy: Always livenessProbe: diff --git a/pkg/operator/controllers/guardrails/template_test.go b/pkg/operator/controllers/guardrails/template_test.go index 682ae2a24a3..c6995f67a01 100644 --- a/pkg/operator/controllers/guardrails/template_test.go +++ b/pkg/operator/controllers/guardrails/template_test.go @@ -66,7 +66,7 @@ func TestDeployCreateOrUpdateCorrectKinds(t *testing.T) { expectedKinds := map[string]int{ "ClusterRole": 1, "ClusterRoleBinding": 1, - "CustomResourceDefinition": 10, + "CustomResourceDefinition": 13, "Deployment": 2, "Namespace": 1, "Role": 1, diff --git a/pkg/operator/deploy/deploy.go b/pkg/operator/deploy/deploy.go index ab18e2505a4..1fbc93a7cd0 100644 --- a/pkg/operator/deploy/deploy.go +++ b/pkg/operator/deploy/deploy.go @@ -14,6 +14,7 @@ import ( "time" "github.com/hashicorp/go-multierror" + operatorclient "github.com/openshift/client-go/operator/clientset/versioned" "github.com/sirupsen/logrus" corev1 "k8s.io/api/core/v1" extensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" @@ -56,6 +57,7 @@ type Operator interface { Restart(context.Context, []string) error IsRunningDesiredVersion(context.Context) (bool, error) RenewMDSDCertificate(context.Context) error + EnsureUpgradeAnnotation(context.Context) error } type operator struct { @@ -67,10 +69,11 @@ type operator struct { client client.Client extensionscli extensionsclient.Interface kubernetescli kubernetes.Interface + operatorcli operatorclient.Interface dh dynamichelper.Interface } -func New(log *logrus.Entry, env env.Interface, oc *api.OpenShiftCluster, arocli aroclient.Interface, client client.Client, extensionscli extensionsclient.Interface, kubernetescli kubernetes.Interface) (Operator, error) { +func New(log *logrus.Entry, env env.Interface, oc *api.OpenShiftCluster, arocli aroclient.Interface, client client.Client, extensionscli extensionsclient.Interface, kubernetescli kubernetes.Interface, operatorcli operatorclient.Interface) (Operator, error) { restConfig, err := restconfig.RestConfig(env, oc) if err != nil { return nil, err @@ -89,6 +92,7 @@ func New(log *logrus.Entry, env env.Interface, oc *api.OpenShiftCluster, arocli client: client, extensionscli: extensionscli, kubernetescli: kubernetescli, + operatorcli: operatorcli, dh: dh, }, nil } @@ -432,6 +436,34 @@ func (o *operator) RenewMDSDCertificate(ctx context.Context) error { return nil } +func (o *operator) EnsureUpgradeAnnotation(ctx context.Context) error { + if o.oc.Properties.PlatformWorkloadIdentityProfile == nil || + o.oc.Properties.ServicePrincipalProfile != nil { + return nil + } + + upgradeableTo := string(*o.oc.Properties.PlatformWorkloadIdentityProfile.UpgradeableTo) + upgradeableAnnotation := "cloudcredential.openshift.io/upgradeable-to" + + cloudcredentialobject, err := o.operatorcli.OperatorV1().CloudCredentials().Get(ctx, "cluster", metav1.GetOptions{}) + if err != nil { + return err + } + + if cloudcredentialobject.Annotations == nil { + cloudcredentialobject.Annotations = map[string]string{} + } + + cloudcredentialobject.Annotations[upgradeableAnnotation] = upgradeableTo + + _, err = o.operatorcli.OperatorV1().CloudCredentials().Update(ctx, cloudcredentialobject, metav1.UpdateOptions{}) + if err != nil { + return err + } + + return nil +} + func (o *operator) IsReady(ctx context.Context) (bool, error) { ok, err := ready.CheckDeploymentIsReady(ctx, o.kubernetescli.AppsV1().Deployments(pkgoperator.Namespace), "aro-operator-master")() o.log.Infof("deployment %q ok status is: %v, err is: %v", "aro-operator-master", ok, err) diff --git a/pkg/operator/deploy/deploy_test.go b/pkg/operator/deploy/deploy_test.go index 6002e65dd7d..44f5e257a63 100644 --- a/pkg/operator/deploy/deploy_test.go +++ b/pkg/operator/deploy/deploy_test.go @@ -10,6 +10,8 @@ import ( "github.com/golang/mock/gomock" configv1 "github.com/openshift/api/config/v1" + operatorv1 "github.com/openshift/api/operator/v1" + operatorfake "github.com/openshift/client-go/operator/clientset/versioned/fake" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -461,3 +463,122 @@ func TestCheckPodImageVersion(t *testing.T) { }) } } + +func TestTestEnsureUpgradeAnnotation(t *testing.T) { + UpgradeableTo1 := api.UpgradeableTo("4.14.59") + + for _, tt := range []struct { + name string + cluster api.OpenShiftClusterProperties + annotation map[string]string + wantAnnotation map[string]string + wantErr string + cloudCredentialsName string + }{ + { + name: "nil PlatformWorkloadIdentityProfile, no version persisted in cluster document", + }, + { + name: "non-nil ServicePrincipalProfile, no version persisted in cluster document", + cluster: api.OpenShiftClusterProperties{ + ServicePrincipalProfile: &api.ServicePrincipalProfile{ + ClientID: "", + ClientSecret: "", + }, + PlatformWorkloadIdentityProfile: &api.PlatformWorkloadIdentityProfile{}, + }, + }, + { + name: "no version persisted in cluster document, persist it", + cluster: api.OpenShiftClusterProperties{ + PlatformWorkloadIdentityProfile: &api.PlatformWorkloadIdentityProfile{ + UpgradeableTo: &UpgradeableTo1, + }, + }, + annotation: nil, + wantAnnotation: map[string]string{ + "cloudcredential.openshift.io/upgradeable-to": "4.14.59", + }, + }, + { + name: "cloud credential 'cluster' doesn't exist", + cluster: api.OpenShiftClusterProperties{ + PlatformWorkloadIdentityProfile: &api.PlatformWorkloadIdentityProfile{ + UpgradeableTo: &UpgradeableTo1, + }, + }, + cloudCredentialsName: "oh_no", + annotation: nil, + wantAnnotation: nil, + wantErr: `cloudcredentials.operator.openshift.io "cluster" not found`, + }, + { + name: "version persisted in cluster document, replace it", + cluster: api.OpenShiftClusterProperties{ + PlatformWorkloadIdentityProfile: &api.PlatformWorkloadIdentityProfile{ + UpgradeableTo: &UpgradeableTo1, + }, + }, + annotation: map[string]string{ + "cloudcredential.openshift.io/upgradeable-to": "4.14.02", + }, + wantAnnotation: map[string]string{ + "cloudcredential.openshift.io/upgradeable-to": "4.14.59", + }, + }, + { + name: "annotations exist, append the upgradeable annotation", + cluster: api.OpenShiftClusterProperties{ + PlatformWorkloadIdentityProfile: &api.PlatformWorkloadIdentityProfile{ + UpgradeableTo: &UpgradeableTo1, + }, + }, + annotation: map[string]string{ + "foo": "bar", + }, + wantAnnotation: map[string]string{ + "foo": "bar", + "cloudcredential.openshift.io/upgradeable-to": "4.14.59", + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + controller := gomock.NewController(t) + defer controller.Finish() + + env := mock_env.NewMockInterface(controller) + + oc := &api.OpenShiftCluster{ + Properties: tt.cluster, + } + + if tt.cloudCredentialsName == "" { + tt.cloudCredentialsName = "cluster" + } + + cloudcredentialobject := &operatorv1.CloudCredential{ + ObjectMeta: metav1.ObjectMeta{ + Name: tt.cloudCredentialsName, + Annotations: tt.annotation, + }, + } + + o := operator{ + oc: oc, + env: env, + operatorcli: operatorfake.NewSimpleClientset(cloudcredentialobject), + } + + err := o.EnsureUpgradeAnnotation(ctx) + utilerror.AssertErrorMessage(t, err, tt.wantErr) + result, _ := o.operatorcli.OperatorV1().CloudCredentials().List(ctx, metav1.ListOptions{}) + for _, v := range result.Items { + actualAnnotations := v.ObjectMeta.Annotations + if !reflect.DeepEqual(actualAnnotations, tt.wantAnnotation) { + t.Errorf("actual annotation: %v, wanted %v", tt.annotation, tt.wantAnnotation) + } + } + }) + } +} diff --git a/pkg/portal/assets/v2/build/asset-manifest.json b/pkg/portal/assets/v2/build/asset-manifest.json index 730266b73ad..7c3ca423311 100644 --- a/pkg/portal/assets/v2/build/asset-manifest.json +++ b/pkg/portal/assets/v2/build/asset-manifest.json @@ -1,10 +1,10 @@ { "files": { - "main.js": "/static/js/main.662aea13.js", + "main.js": "/static/js/main.cef1fecf.js", "index.html": "/index.html", - "main.662aea13.js.map": "/static/js/main.662aea13.js.map" + "main.cef1fecf.js.map": "/static/js/main.cef1fecf.js.map" }, "entrypoints": [ - "static/js/main.662aea13.js" + "static/js/main.cef1fecf.js" ] } \ No newline at end of file diff --git a/pkg/portal/assets/v2/build/index.html b/pkg/portal/assets/v2/build/index.html index cf20b15943b..0706afad625 100644 --- a/pkg/portal/assets/v2/build/index.html +++ b/pkg/portal/assets/v2/build/index.html @@ -1 +1 @@ -ARO Portal
\ No newline at end of file +ARO Portal
\ No newline at end of file diff --git a/pkg/portal/assets/v2/build/static/js/main.662aea13.js b/pkg/portal/assets/v2/build/static/js/main.662aea13.js deleted file mode 100644 index 98214742a71..00000000000 --- a/pkg/portal/assets/v2/build/static/js/main.662aea13.js +++ /dev/null @@ -1,3 +0,0 @@ -/*! For license information please see main.662aea13.js.LICENSE.txt */ -(()=>{var e={9:(e,t)=>{"use strict";t.byteLength=function(e){var t=l(e),n=t[0],r=t[1];return 3*(n+r)/4-r},t.toByteArray=function(e){var t,n,i=l(e),a=i[0],s=i[1],u=new o(function(e,t,n){return 3*(t+n)/4-n}(0,a,s)),c=0,d=s>0?a-4:a;for(n=0;n>16&255,u[c++]=t>>8&255,u[c++]=255&t;2===s&&(t=r[e.charCodeAt(n)]<<2|r[e.charCodeAt(n+1)]>>4,u[c++]=255&t);1===s&&(t=r[e.charCodeAt(n)]<<10|r[e.charCodeAt(n+1)]<<4|r[e.charCodeAt(n+2)]>>2,u[c++]=t>>8&255,u[c++]=255&t);return u},t.fromByteArray=function(e){for(var t,r=e.length,o=r%3,i=[],a=16383,s=0,l=r-o;sl?l:s+a));1===o?(t=e[r-1],i.push(n[t>>2]+n[t<<4&63]+"==")):2===o&&(t=(e[r-2]<<8)+e[r-1],i.push(n[t>>10]+n[t>>4&63]+n[t<<2&63]+"="));return i.join("")};for(var n=[],r=[],o="undefined"!==typeof Uint8Array?Uint8Array:Array,i="ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/",a=0,s=i.length;a0)throw new Error("Invalid string. Length must be a multiple of 4");var n=e.indexOf("=");return-1===n&&(n=t),[n,n===t?0:4-n%4]}function u(e,t,r){for(var o,i,a=[],s=t;s>18&63]+n[i>>12&63]+n[i>>6&63]+n[63&i]);return a.join("")}r["-".charCodeAt(0)]=62,r["_".charCodeAt(0)]=63},778:(e,t,n)=>{"use strict";const r=n(9),o=n(38),i="function"===typeof Symbol&&"function"===typeof Symbol.for?Symbol.for("nodejs.util.inspect.custom"):null;t.Buffer=l,t.SlowBuffer=function(e){+e!=e&&(e=0);return l.alloc(+e)},t.INSPECT_MAX_BYTES=50;const a=2147483647;function s(e){if(e>a)throw new RangeError('The value "'+e+'" is invalid for option "size"');const t=new Uint8Array(e);return Object.setPrototypeOf(t,l.prototype),t}function l(e,t,n){if("number"===typeof e){if("string"===typeof t)throw new TypeError('The "string" argument must be of type string. Received type number');return d(e)}return u(e,t,n)}function u(e,t,n){if("string"===typeof e)return function(e,t){"string"===typeof t&&""!==t||(t="utf8");if(!l.isEncoding(t))throw new TypeError("Unknown encoding: "+t);const n=0|m(e,t);let r=s(n);const o=r.write(e,t);o!==n&&(r=r.slice(0,o));return r}(e,t);if(ArrayBuffer.isView(e))return function(e){if($(e,Uint8Array)){const t=new Uint8Array(e);return h(t.buffer,t.byteOffset,t.byteLength)}return p(e)}(e);if(null==e)throw new TypeError("The first argument must be one of type string, Buffer, ArrayBuffer, Array, or Array-like Object. Received type "+typeof e);if($(e,ArrayBuffer)||e&&$(e.buffer,ArrayBuffer))return h(e,t,n);if("undefined"!==typeof SharedArrayBuffer&&($(e,SharedArrayBuffer)||e&&$(e.buffer,SharedArrayBuffer)))return h(e,t,n);if("number"===typeof e)throw new TypeError('The "value" argument must not be of type number. Received type number');const r=e.valueOf&&e.valueOf();if(null!=r&&r!==e)return l.from(r,t,n);const o=function(e){if(l.isBuffer(e)){const t=0|f(e.length),n=s(t);return 0===n.length||e.copy(n,0,0,t),n}if(void 0!==e.length)return"number"!==typeof e.length||Z(e.length)?s(0):p(e);if("Buffer"===e.type&&Array.isArray(e.data))return p(e.data)}(e);if(o)return o;if("undefined"!==typeof Symbol&&null!=Symbol.toPrimitive&&"function"===typeof e[Symbol.toPrimitive])return l.from(e[Symbol.toPrimitive]("string"),t,n);throw new TypeError("The first argument must be one of type string, Buffer, ArrayBuffer, Array, or Array-like Object. Received type "+typeof e)}function c(e){if("number"!==typeof e)throw new TypeError('"size" argument must be of type number');if(e<0)throw new RangeError('The value "'+e+'" is invalid for option "size"')}function d(e){return c(e),s(e<0?0:0|f(e))}function p(e){const t=e.length<0?0:0|f(e.length),n=s(t);for(let r=0;r=a)throw new RangeError("Attempt to allocate Buffer larger than maximum size: 0x"+a.toString(16)+" bytes");return 0|e}function m(e,t){if(l.isBuffer(e))return e.length;if(ArrayBuffer.isView(e)||$(e,ArrayBuffer))return e.byteLength;if("string"!==typeof e)throw new TypeError('The "string" argument must be one of type string, Buffer, or ArrayBuffer. Received type '+typeof e);const n=e.length,r=arguments.length>2&&!0===arguments[2];if(!r&&0===n)return 0;let o=!1;for(;;)switch(t){case"ascii":case"latin1":case"binary":return n;case"utf8":case"utf-8":return K(e).length;case"ucs2":case"ucs-2":case"utf16le":case"utf-16le":return 2*n;case"hex":return n>>>1;case"base64":return Y(e).length;default:if(o)return r?-1:K(e).length;t=(""+t).toLowerCase(),o=!0}}function g(e,t,n){let r=!1;if((void 0===t||t<0)&&(t=0),t>this.length)return"";if((void 0===n||n>this.length)&&(n=this.length),n<=0)return"";if((n>>>=0)<=(t>>>=0))return"";for(e||(e="utf8");;)switch(e){case"hex":return T(this,t,n);case"utf8":case"utf-8":return D(this,t,n);case"ascii":return L(this,t,n);case"latin1":case"binary":return I(this,t,n);case"base64":return k(this,t,n);case"ucs2":case"ucs-2":case"utf16le":case"utf-16le":return M(this,t,n);default:if(r)throw new TypeError("Unknown encoding: "+e);e=(e+"").toLowerCase(),r=!0}}function v(e,t,n){const r=e[t];e[t]=e[n],e[n]=r}function y(e,t,n,r,o){if(0===e.length)return-1;if("string"===typeof n?(r=n,n=0):n>2147483647?n=2147483647:n<-2147483648&&(n=-2147483648),Z(n=+n)&&(n=o?0:e.length-1),n<0&&(n=e.length+n),n>=e.length){if(o)return-1;n=e.length-1}else if(n<0){if(!o)return-1;n=0}if("string"===typeof t&&(t=l.from(t,r)),l.isBuffer(t))return 0===t.length?-1:b(e,t,n,r,o);if("number"===typeof t)return t&=255,"function"===typeof Uint8Array.prototype.indexOf?o?Uint8Array.prototype.indexOf.call(e,t,n):Uint8Array.prototype.lastIndexOf.call(e,t,n):b(e,[t],n,r,o);throw new TypeError("val must be string, number or Buffer")}function b(e,t,n,r,o){let i,a=1,s=e.length,l=t.length;if(void 0!==r&&("ucs2"===(r=String(r).toLowerCase())||"ucs-2"===r||"utf16le"===r||"utf-16le"===r)){if(e.length<2||t.length<2)return-1;a=2,s/=2,l/=2,n/=2}function u(e,t){return 1===a?e[t]:e.readUInt16BE(t*a)}if(o){let r=-1;for(i=n;is&&(n=s-l),i=n;i>=0;i--){let n=!0;for(let r=0;ro&&(r=o):r=o;const i=t.length;let a;for(r>i/2&&(r=i/2),a=0;a>8,o=n%256,i.push(o),i.push(r);return i}(t,e.length-n),e,n,r)}function k(e,t,n){return 0===t&&n===e.length?r.fromByteArray(e):r.fromByteArray(e.slice(t,n))}function D(e,t,n){n=Math.min(e.length,n);const r=[];let o=t;for(;o239?4:t>223?3:t>191?2:1;if(o+a<=n){let n,r,s,l;switch(a){case 1:t<128&&(i=t);break;case 2:n=e[o+1],128===(192&n)&&(l=(31&t)<<6|63&n,l>127&&(i=l));break;case 3:n=e[o+1],r=e[o+2],128===(192&n)&&128===(192&r)&&(l=(15&t)<<12|(63&n)<<6|63&r,l>2047&&(l<55296||l>57343)&&(i=l));break;case 4:n=e[o+1],r=e[o+2],s=e[o+3],128===(192&n)&&128===(192&r)&&128===(192&s)&&(l=(15&t)<<18|(63&n)<<12|(63&r)<<6|63&s,l>65535&&l<1114112&&(i=l))}}null===i?(i=65533,a=1):i>65535&&(i-=65536,r.push(i>>>10&1023|55296),i=56320|1023&i),r.push(i),o+=a}return function(e){const t=e.length;if(t<=E)return String.fromCharCode.apply(String,e);let n="",r=0;for(;rr.length?(l.isBuffer(t)||(t=l.from(t)),t.copy(r,o)):Uint8Array.prototype.set.call(r,t,o);else{if(!l.isBuffer(t))throw new TypeError('"list" argument must be an Array of Buffers');t.copy(r,o)}o+=t.length}return r},l.byteLength=m,l.prototype._isBuffer=!0,l.prototype.swap16=function(){const e=this.length;if(e%2!==0)throw new RangeError("Buffer size must be a multiple of 16-bits");for(let t=0;tn&&(e+=" ... "),""},i&&(l.prototype[i]=l.prototype.inspect),l.prototype.compare=function(e,t,n,r,o){if($(e,Uint8Array)&&(e=l.from(e,e.offset,e.byteLength)),!l.isBuffer(e))throw new TypeError('The "target" argument must be one of type Buffer or Uint8Array. Received type '+typeof e);if(void 0===t&&(t=0),void 0===n&&(n=e?e.length:0),void 0===r&&(r=0),void 0===o&&(o=this.length),t<0||n>e.length||r<0||o>this.length)throw new RangeError("out of range index");if(r>=o&&t>=n)return 0;if(r>=o)return-1;if(t>=n)return 1;if(this===e)return 0;let i=(o>>>=0)-(r>>>=0),a=(n>>>=0)-(t>>>=0);const s=Math.min(i,a),u=this.slice(r,o),c=e.slice(t,n);for(let l=0;l>>=0,isFinite(n)?(n>>>=0,void 0===r&&(r="utf8")):(r=n,n=void 0)}const o=this.length-t;if((void 0===n||n>o)&&(n=o),e.length>0&&(n<0||t<0)||t>this.length)throw new RangeError("Attempt to write outside buffer bounds");r||(r="utf8");let i=!1;for(;;)switch(r){case"hex":return C(this,e,t,n);case"utf8":case"utf-8":return _(this,e,t,n);case"ascii":case"latin1":case"binary":return x(this,e,t,n);case"base64":return S(this,e,t,n);case"ucs2":case"ucs-2":case"utf16le":case"utf-16le":return w(this,e,t,n);default:if(i)throw new TypeError("Unknown encoding: "+r);r=(""+r).toLowerCase(),i=!0}},l.prototype.toJSON=function(){return{type:"Buffer",data:Array.prototype.slice.call(this._arr||this,0)}};const E=4096;function L(e,t,n){let r="";n=Math.min(e.length,n);for(let o=t;or)&&(n=r);let o="";for(let i=t;in)throw new RangeError("Trying to access beyond buffer length")}function P(e,t,n,r,o,i){if(!l.isBuffer(e))throw new TypeError('"buffer" argument must be a Buffer instance');if(t>o||te.length)throw new RangeError("Index out of range")}function N(e,t,n,r,o){j(t,r,o,e,n,7);let i=Number(t&BigInt(4294967295));e[n++]=i,i>>=8,e[n++]=i,i>>=8,e[n++]=i,i>>=8,e[n++]=i;let a=Number(t>>BigInt(32)&BigInt(4294967295));return e[n++]=a,a>>=8,e[n++]=a,a>>=8,e[n++]=a,a>>=8,e[n++]=a,n}function A(e,t,n,r,o){j(t,r,o,e,n,7);let i=Number(t&BigInt(4294967295));e[n+7]=i,i>>=8,e[n+6]=i,i>>=8,e[n+5]=i,i>>=8,e[n+4]=i;let a=Number(t>>BigInt(32)&BigInt(4294967295));return e[n+3]=a,a>>=8,e[n+2]=a,a>>=8,e[n+1]=a,a>>=8,e[n]=a,n+8}function F(e,t,n,r,o,i){if(n+r>e.length)throw new RangeError("Index out of range");if(n<0)throw new RangeError("Index out of range")}function B(e,t,n,r,i){return t=+t,n>>>=0,i||F(e,0,n,4),o.write(e,t,n,r,23,4),n+4}function O(e,t,n,r,i){return t=+t,n>>>=0,i||F(e,0,n,8),o.write(e,t,n,r,52,8),n+8}l.prototype.slice=function(e,t){const n=this.length;(e=~~e)<0?(e+=n)<0&&(e=0):e>n&&(e=n),(t=void 0===t?n:~~t)<0?(t+=n)<0&&(t=0):t>n&&(t=n),t>>=0,t>>>=0,n||R(e,t,this.length);let r=this[e],o=1,i=0;for(;++i>>=0,t>>>=0,n||R(e,t,this.length);let r=this[e+--t],o=1;for(;t>0&&(o*=256);)r+=this[e+--t]*o;return r},l.prototype.readUint8=l.prototype.readUInt8=function(e,t){return e>>>=0,t||R(e,1,this.length),this[e]},l.prototype.readUint16LE=l.prototype.readUInt16LE=function(e,t){return e>>>=0,t||R(e,2,this.length),this[e]|this[e+1]<<8},l.prototype.readUint16BE=l.prototype.readUInt16BE=function(e,t){return e>>>=0,t||R(e,2,this.length),this[e]<<8|this[e+1]},l.prototype.readUint32LE=l.prototype.readUInt32LE=function(e,t){return e>>>=0,t||R(e,4,this.length),(this[e]|this[e+1]<<8|this[e+2]<<16)+16777216*this[e+3]},l.prototype.readUint32BE=l.prototype.readUInt32BE=function(e,t){return e>>>=0,t||R(e,4,this.length),16777216*this[e]+(this[e+1]<<16|this[e+2]<<8|this[e+3])},l.prototype.readBigUInt64LE=Q((function(e){U(e>>>=0,"offset");const t=this[e],n=this[e+7];void 0!==t&&void 0!==n||V(e,this.length-8);const r=t+256*this[++e]+65536*this[++e]+this[++e]*2**24,o=this[++e]+256*this[++e]+65536*this[++e]+n*2**24;return BigInt(r)+(BigInt(o)<>>=0,"offset");const t=this[e],n=this[e+7];void 0!==t&&void 0!==n||V(e,this.length-8);const r=t*2**24+65536*this[++e]+256*this[++e]+this[++e],o=this[++e]*2**24+65536*this[++e]+256*this[++e]+n;return(BigInt(r)<>>=0,t>>>=0,n||R(e,t,this.length);let r=this[e],o=1,i=0;for(;++i=o&&(r-=Math.pow(2,8*t)),r},l.prototype.readIntBE=function(e,t,n){e>>>=0,t>>>=0,n||R(e,t,this.length);let r=t,o=1,i=this[e+--r];for(;r>0&&(o*=256);)i+=this[e+--r]*o;return o*=128,i>=o&&(i-=Math.pow(2,8*t)),i},l.prototype.readInt8=function(e,t){return e>>>=0,t||R(e,1,this.length),128&this[e]?-1*(255-this[e]+1):this[e]},l.prototype.readInt16LE=function(e,t){e>>>=0,t||R(e,2,this.length);const n=this[e]|this[e+1]<<8;return 32768&n?4294901760|n:n},l.prototype.readInt16BE=function(e,t){e>>>=0,t||R(e,2,this.length);const n=this[e+1]|this[e]<<8;return 32768&n?4294901760|n:n},l.prototype.readInt32LE=function(e,t){return e>>>=0,t||R(e,4,this.length),this[e]|this[e+1]<<8|this[e+2]<<16|this[e+3]<<24},l.prototype.readInt32BE=function(e,t){return e>>>=0,t||R(e,4,this.length),this[e]<<24|this[e+1]<<16|this[e+2]<<8|this[e+3]},l.prototype.readBigInt64LE=Q((function(e){U(e>>>=0,"offset");const t=this[e],n=this[e+7];void 0!==t&&void 0!==n||V(e,this.length-8);const r=this[e+4]+256*this[e+5]+65536*this[e+6]+(n<<24);return(BigInt(r)<>>=0,"offset");const t=this[e],n=this[e+7];void 0!==t&&void 0!==n||V(e,this.length-8);const r=(t<<24)+65536*this[++e]+256*this[++e]+this[++e];return(BigInt(r)<>>=0,t||R(e,4,this.length),o.read(this,e,!0,23,4)},l.prototype.readFloatBE=function(e,t){return e>>>=0,t||R(e,4,this.length),o.read(this,e,!1,23,4)},l.prototype.readDoubleLE=function(e,t){return e>>>=0,t||R(e,8,this.length),o.read(this,e,!0,52,8)},l.prototype.readDoubleBE=function(e,t){return e>>>=0,t||R(e,8,this.length),o.read(this,e,!1,52,8)},l.prototype.writeUintLE=l.prototype.writeUIntLE=function(e,t,n,r){if(e=+e,t>>>=0,n>>>=0,!r){P(this,e,t,n,Math.pow(2,8*n)-1,0)}let o=1,i=0;for(this[t]=255&e;++i>>=0,n>>>=0,!r){P(this,e,t,n,Math.pow(2,8*n)-1,0)}let o=n-1,i=1;for(this[t+o]=255&e;--o>=0&&(i*=256);)this[t+o]=e/i&255;return t+n},l.prototype.writeUint8=l.prototype.writeUInt8=function(e,t,n){return e=+e,t>>>=0,n||P(this,e,t,1,255,0),this[t]=255&e,t+1},l.prototype.writeUint16LE=l.prototype.writeUInt16LE=function(e,t,n){return e=+e,t>>>=0,n||P(this,e,t,2,65535,0),this[t]=255&e,this[t+1]=e>>>8,t+2},l.prototype.writeUint16BE=l.prototype.writeUInt16BE=function(e,t,n){return e=+e,t>>>=0,n||P(this,e,t,2,65535,0),this[t]=e>>>8,this[t+1]=255&e,t+2},l.prototype.writeUint32LE=l.prototype.writeUInt32LE=function(e,t,n){return e=+e,t>>>=0,n||P(this,e,t,4,4294967295,0),this[t+3]=e>>>24,this[t+2]=e>>>16,this[t+1]=e>>>8,this[t]=255&e,t+4},l.prototype.writeUint32BE=l.prototype.writeUInt32BE=function(e,t,n){return e=+e,t>>>=0,n||P(this,e,t,4,4294967295,0),this[t]=e>>>24,this[t+1]=e>>>16,this[t+2]=e>>>8,this[t+3]=255&e,t+4},l.prototype.writeBigUInt64LE=Q((function(e,t=0){return N(this,e,t,BigInt(0),BigInt("0xffffffffffffffff"))})),l.prototype.writeBigUInt64BE=Q((function(e,t=0){return A(this,e,t,BigInt(0),BigInt("0xffffffffffffffff"))})),l.prototype.writeIntLE=function(e,t,n,r){if(e=+e,t>>>=0,!r){const r=Math.pow(2,8*n-1);P(this,e,t,n,r-1,-r)}let o=0,i=1,a=0;for(this[t]=255&e;++o>0)-a&255;return t+n},l.prototype.writeIntBE=function(e,t,n,r){if(e=+e,t>>>=0,!r){const r=Math.pow(2,8*n-1);P(this,e,t,n,r-1,-r)}let o=n-1,i=1,a=0;for(this[t+o]=255&e;--o>=0&&(i*=256);)e<0&&0===a&&0!==this[t+o+1]&&(a=1),this[t+o]=(e/i>>0)-a&255;return t+n},l.prototype.writeInt8=function(e,t,n){return e=+e,t>>>=0,n||P(this,e,t,1,127,-128),e<0&&(e=255+e+1),this[t]=255&e,t+1},l.prototype.writeInt16LE=function(e,t,n){return e=+e,t>>>=0,n||P(this,e,t,2,32767,-32768),this[t]=255&e,this[t+1]=e>>>8,t+2},l.prototype.writeInt16BE=function(e,t,n){return e=+e,t>>>=0,n||P(this,e,t,2,32767,-32768),this[t]=e>>>8,this[t+1]=255&e,t+2},l.prototype.writeInt32LE=function(e,t,n){return e=+e,t>>>=0,n||P(this,e,t,4,2147483647,-2147483648),this[t]=255&e,this[t+1]=e>>>8,this[t+2]=e>>>16,this[t+3]=e>>>24,t+4},l.prototype.writeInt32BE=function(e,t,n){return e=+e,t>>>=0,n||P(this,e,t,4,2147483647,-2147483648),e<0&&(e=4294967295+e+1),this[t]=e>>>24,this[t+1]=e>>>16,this[t+2]=e>>>8,this[t+3]=255&e,t+4},l.prototype.writeBigInt64LE=Q((function(e,t=0){return N(this,e,t,-BigInt("0x8000000000000000"),BigInt("0x7fffffffffffffff"))})),l.prototype.writeBigInt64BE=Q((function(e,t=0){return A(this,e,t,-BigInt("0x8000000000000000"),BigInt("0x7fffffffffffffff"))})),l.prototype.writeFloatLE=function(e,t,n){return B(this,e,t,!0,n)},l.prototype.writeFloatBE=function(e,t,n){return B(this,e,t,!1,n)},l.prototype.writeDoubleLE=function(e,t,n){return O(this,e,t,!0,n)},l.prototype.writeDoubleBE=function(e,t,n){return O(this,e,t,!1,n)},l.prototype.copy=function(e,t,n,r){if(!l.isBuffer(e))throw new TypeError("argument should be a Buffer");if(n||(n=0),r||0===r||(r=this.length),t>=e.length&&(t=e.length),t||(t=0),r>0&&r=this.length)throw new RangeError("Index out of range");if(r<0)throw new RangeError("sourceEnd out of bounds");r>this.length&&(r=this.length),e.length-t>>=0,n=void 0===n?this.length:n>>>0,e||(e=0),"number"===typeof e)for(o=t;o=r+4;n-=3)t=`_${e.slice(n-3,n)}${t}`;return`${e.slice(0,n)}${t}`}function j(e,t,n,r,o,i){if(e>n||e3?0===t||t===BigInt(0)?`>= 0${r} and < 2${r} ** ${8*(i+1)}${r}`:`>= -(2${r} ** ${8*(i+1)-1}${r}) and < 2 ** ${8*(i+1)-1}${r}`:`>= ${t}${r} and <= ${n}${r}`,new H.ERR_OUT_OF_RANGE("value",o,e)}!function(e,t,n){U(t,"offset"),void 0!==e[t]&&void 0!==e[t+n]||V(t,e.length-(n+1))}(r,o,i)}function U(e,t){if("number"!==typeof e)throw new H.ERR_INVALID_ARG_TYPE(t,"number",e)}function V(e,t,n){if(Math.floor(e)!==e)throw U(e,n),new H.ERR_OUT_OF_RANGE(n||"offset","an integer",e);if(t<0)throw new H.ERR_BUFFER_OUT_OF_BOUNDS;throw new H.ERR_OUT_OF_RANGE(n||"offset",`>= ${n?1:0} and <= ${t}`,e)}W("ERR_BUFFER_OUT_OF_BOUNDS",(function(e){return e?`${e} is outside of buffer bounds`:"Attempt to access memory outside buffer bounds"}),RangeError),W("ERR_INVALID_ARG_TYPE",(function(e,t){return`The "${e}" argument must be of type number. Received type ${typeof t}`}),TypeError),W("ERR_OUT_OF_RANGE",(function(e,t,n){let r=`The value of "${e}" is out of range.`,o=n;return Number.isInteger(n)&&Math.abs(n)>2**32?o=z(String(n)):"bigint"===typeof n&&(o=String(n),(n>BigInt(2)**BigInt(32)||n<-(BigInt(2)**BigInt(32)))&&(o=z(o)),o+="n"),r+=` It must be ${t}. Received ${o}`,r}),RangeError);const G=/[^+/0-9A-Za-z-_]/g;function K(e,t){let n;t=t||1/0;const r=e.length;let o=null;const i=[];for(let a=0;a55295&&n<57344){if(!o){if(n>56319){(t-=3)>-1&&i.push(239,191,189);continue}if(a+1===r){(t-=3)>-1&&i.push(239,191,189);continue}o=n;continue}if(n<56320){(t-=3)>-1&&i.push(239,191,189),o=n;continue}n=65536+(o-55296<<10|n-56320)}else o&&(t-=3)>-1&&i.push(239,191,189);if(o=null,n<128){if((t-=1)<0)break;i.push(n)}else if(n<2048){if((t-=2)<0)break;i.push(n>>6|192,63&n|128)}else if(n<65536){if((t-=3)<0)break;i.push(n>>12|224,n>>6&63|128,63&n|128)}else{if(!(n<1114112))throw new Error("Invalid code point");if((t-=4)<0)break;i.push(n>>18|240,n>>12&63|128,n>>6&63|128,63&n|128)}}return i}function Y(e){return r.toByteArray(function(e){if((e=(e=e.split("=")[0]).trim().replace(G,"")).length<2)return"";for(;e.length%4!==0;)e+="=";return e}(e))}function q(e,t,n,r){let o;for(o=0;o=t.length||o>=e.length);++o)t[o+n]=e[o];return o}function $(e,t){return e instanceof t||null!=e&&null!=e.constructor&&null!=e.constructor.name&&e.constructor.name===t.name}function Z(e){return e!==e}const X=function(){const e="0123456789abcdef",t=new Array(256);for(let n=0;n<16;++n){const r=16*n;for(let o=0;o<16;++o)t[r+o]=e[n]+e[o]}return t}();function Q(e){return"undefined"===typeof BigInt?J:e}function J(){throw new Error("BigInt not supported")}},207:(e,t,n)=>{"use strict";e.exports=function(e,t){var n=t||{},o=n.type||"attachment",i=function(e,t){if(void 0===e)return;var n={};if("string"!==typeof e)throw new TypeError("filename must be a string");void 0===t&&(t=!0);if("string"!==typeof t&&"boolean"!==typeof t)throw new TypeError("fallback must be a string or boolean");if("string"===typeof t&&l.test(t))throw new TypeError("fallback must be ISO-8859-1 string");var o=r(e),i=p.test(o),s="string"!==typeof t?t&&v(o):r(t),u="string"===typeof s&&s!==o;(u||!i||a.test(o))&&(n["filename*"]=o);(i||u)&&(n.filename=u?s:o);return n}(e,n.fallback);return function(e){var t=e.parameters,n=e.type;if(!n||"string"!==typeof n||!h.test(n))throw new TypeError("invalid type");var r=String(n).toLowerCase();if(t&&"object"===typeof t)for(var o,i=Object.keys(t).sort(),a=0;a?@[\\\]{}\x7f]/g,a=/%[0-9A-Fa-f]{2}/,s=/%([0-9A-Fa-f]{2})/g,l=/[^\x20-\x7e\xa0-\xff]/g,u=/\\([\u0000-\u007f])/g,c=/([\\"])/g,d=/;[\x09\x20]*([!#$%&'*+.0-9A-Z^_`a-z|~-]+)[\x09\x20]*=[\x09\x20]*("(?:[\x20!\x23-\x5b\x5d-\x7e\x80-\xff]|\\[\x20-\x7e])*"|[!#$%&'*+.0-9A-Z^_`a-z|~-]+)[\x09\x20]*/g,p=/^[\x20-\x7e\x80-\xff]+$/,h=/^[!#$%&'*+.0-9A-Z^_`a-z|~-]+$/,f=/^([A-Za-z0-9!#$%&+\-^_`{}~]+)'(?:[A-Za-z]{2,3}(?:-[A-Za-z]{3}){0,3}|[A-Za-z]{4,8}|)'((?:%[0-9A-Fa-f]{2}|[A-Za-z0-9!#$&+.^_`|~-])+)$/,m=/^([!#$%&'*+.0-9A-Z^_`a-z|~-]+)[\x09\x20]*(?:$|;)/;function g(e){var t=f.exec(e);if(!t)throw new TypeError("invalid extended field value");var n,r=t[1].toLowerCase(),i=t[2].replace(s,y);switch(r){case"iso-8859-1":n=v(i);break;case"utf-8":n=o.from(i,"binary").toString("utf8");break;default:throw new TypeError("unsupported charset in extended field")}return n}function v(e){return String(e).replace(l,"?")}function y(e,t){return String.fromCharCode(parseInt(t,16))}function b(e){return"%"+String(e).charCodeAt(0).toString(16).toUpperCase()}function C(e){return'"'+String(e).replace(c,"\\$1")+'"'}function _(e){var t=String(e);return"UTF-8''"+encodeURIComponent(t).replace(i,b)}function x(e,t){this.type=e,this.parameters=t}},38:(e,t)=>{t.read=function(e,t,n,r,o){var i,a,s=8*o-r-1,l=(1<>1,c=-7,d=n?o-1:0,p=n?-1:1,h=e[t+d];for(d+=p,i=h&(1<<-c)-1,h>>=-c,c+=s;c>0;i=256*i+e[t+d],d+=p,c-=8);for(a=i&(1<<-c)-1,i>>=-c,c+=r;c>0;a=256*a+e[t+d],d+=p,c-=8);if(0===i)i=1-u;else{if(i===l)return a?NaN:1/0*(h?-1:1);a+=Math.pow(2,r),i-=u}return(h?-1:1)*a*Math.pow(2,i-r)},t.write=function(e,t,n,r,o,i){var a,s,l,u=8*i-o-1,c=(1<>1,p=23===o?Math.pow(2,-24)-Math.pow(2,-77):0,h=r?0:i-1,f=r?1:-1,m=t<0||0===t&&1/t<0?1:0;for(t=Math.abs(t),isNaN(t)||t===1/0?(s=isNaN(t)?1:0,a=c):(a=Math.floor(Math.log(t)/Math.LN2),t*(l=Math.pow(2,-a))<1&&(a--,l*=2),(t+=a+d>=1?p/l:p*Math.pow(2,1-d))*l>=2&&(a++,l/=2),a+d>=c?(s=0,a=c):a+d>=1?(s=(t*l-1)*Math.pow(2,o),a+=d):(s=t*Math.pow(2,d-1)*Math.pow(2,o),a=0));o>=8;e[n+h]=255&s,h+=f,s/=256,o-=8);for(a=a<0;e[n+h]=255&a,h+=f,a/=256,u-=8);e[n+h-f]|=128*m}},982:(e,t,n)=>{"use strict";var r=n(426);function o(e){if("string"!==typeof e)throw new TypeError("Path must be a string. Received "+JSON.stringify(e))}function i(e,t){for(var n,r="",o=0,i=-1,a=0,s=0;s<=e.length;++s){if(s2){var l=r.lastIndexOf("/");if(l!==r.length-1){-1===l?(r="",o=0):o=(r=r.slice(0,l)).length-1-r.lastIndexOf("/"),i=s,a=0;continue}}else if(2===r.length||1===r.length){r="",o=0,i=s,a=0;continue}t&&(r.length>0?r+="/..":r="..",o=2)}else r.length>0?r+="/"+e.slice(i+1,s):r=e.slice(i+1,s),o=s-i-1;i=s,a=0}else 46===n&&-1!==a?++a:a=-1}return r}var a={resolve:function(){for(var e,t="",n=!1,a=arguments.length-1;a>=-1&&!n;a--){var s;a>=0?s=arguments[a]:(void 0===e&&(e=r.cwd()),s=e),o(s),0!==s.length&&(t=s+"/"+t,n=47===s.charCodeAt(0))}return t=i(t,!n),n?t.length>0?"/"+t:"/":t.length>0?t:"."},normalize:function(e){if(o(e),0===e.length)return".";var t=47===e.charCodeAt(0),n=47===e.charCodeAt(e.length-1);return 0!==(e=i(e,!t)).length||t||(e="."),e.length>0&&n&&(e+="/"),t?"/"+e:e},isAbsolute:function(e){return o(e),e.length>0&&47===e.charCodeAt(0)},join:function(){if(0===arguments.length)return".";for(var e,t=0;t0&&(void 0===e?e=n:e+="/"+n)}return void 0===e?".":a.normalize(e)},relative:function(e,t){if(o(e),o(t),e===t)return"";if((e=a.resolve(e))===(t=a.resolve(t)))return"";for(var n=1;nu){if(47===t.charCodeAt(s+d))return t.slice(s+d+1);if(0===d)return t.slice(s+d)}else i>u&&(47===e.charCodeAt(n+d)?c=d:0===d&&(c=0));break}var p=e.charCodeAt(n+d);if(p!==t.charCodeAt(s+d))break;47===p&&(c=d)}var h="";for(d=n+c+1;d<=r;++d)d!==r&&47!==e.charCodeAt(d)||(0===h.length?h+="..":h+="/..");return h.length>0?h+t.slice(s+c):(s+=c,47===t.charCodeAt(s)&&++s,t.slice(s))},_makeLong:function(e){return e},dirname:function(e){if(o(e),0===e.length)return".";for(var t=e.charCodeAt(0),n=47===t,r=-1,i=!0,a=e.length-1;a>=1;--a)if(47===(t=e.charCodeAt(a))){if(!i){r=a;break}}else i=!1;return-1===r?n?"/":".":n&&1===r?"//":e.slice(0,r)},basename:function(e,t){if(void 0!==t&&"string"!==typeof t)throw new TypeError('"ext" argument must be a string');o(e);var n,r=0,i=-1,a=!0;if(void 0!==t&&t.length>0&&t.length<=e.length){if(t.length===e.length&&t===e)return"";var s=t.length-1,l=-1;for(n=e.length-1;n>=0;--n){var u=e.charCodeAt(n);if(47===u){if(!a){r=n+1;break}}else-1===l&&(a=!1,l=n+1),s>=0&&(u===t.charCodeAt(s)?-1===--s&&(i=n):(s=-1,i=l))}return r===i?i=l:-1===i&&(i=e.length),e.slice(r,i)}for(n=e.length-1;n>=0;--n)if(47===e.charCodeAt(n)){if(!a){r=n+1;break}}else-1===i&&(a=!1,i=n+1);return-1===i?"":e.slice(r,i)},extname:function(e){o(e);for(var t=-1,n=0,r=-1,i=!0,a=0,s=e.length-1;s>=0;--s){var l=e.charCodeAt(s);if(47!==l)-1===r&&(i=!1,r=s+1),46===l?-1===t?t=s:1!==a&&(a=1):-1!==t&&(a=-1);else if(!i){n=s+1;break}}return-1===t||-1===r||0===a||1===a&&t===r-1&&t===n+1?"":e.slice(t,r)},format:function(e){if(null===e||"object"!==typeof e)throw new TypeError('The "pathObject" argument must be of type Object. Received type '+typeof e);return function(e,t){var n=t.dir||t.root,r=t.base||(t.name||"")+(t.ext||"");return n?n===t.root?n+r:n+e+r:r}("/",e)},parse:function(e){o(e);var t={root:"",dir:"",base:"",ext:"",name:""};if(0===e.length)return t;var n,r=e.charCodeAt(0),i=47===r;i?(t.root="/",n=1):n=0;for(var a=-1,s=0,l=-1,u=!0,c=e.length-1,d=0;c>=n;--c)if(47!==(r=e.charCodeAt(c)))-1===l&&(u=!1,l=c+1),46===r?-1===a?a=c:1!==d&&(d=1):-1!==a&&(d=-1);else if(!u){s=c+1;break}return-1===a||-1===l||0===d||1===d&&a===l-1&&a===s+1?-1!==l&&(t.base=t.name=0===s&&i?e.slice(1,l):e.slice(s,l)):(0===s&&i?(t.name=e.slice(1,a),t.base=e.slice(1,l)):(t.name=e.slice(s,a),t.base=e.slice(s,l)),t.ext=e.slice(a,l)),s>0?t.dir=e.slice(0,s-1):i&&(t.dir="/"),t},sep:"/",delimiter:":",win32:null,posix:null};a.posix=a,e.exports=a},426:e=>{var t,n,r=e.exports={};function o(){throw new Error("setTimeout has not been defined")}function i(){throw new Error("clearTimeout has not been defined")}function a(e){if(t===setTimeout)return setTimeout(e,0);if((t===o||!t)&&setTimeout)return t=setTimeout,setTimeout(e,0);try{return t(e,0)}catch(n){try{return t.call(null,e,0)}catch(n){return t.call(this,e,0)}}}!function(){try{t="function"===typeof setTimeout?setTimeout:o}catch(e){t=o}try{n="function"===typeof clearTimeout?clearTimeout:i}catch(e){n=i}}();var s,l=[],u=!1,c=-1;function d(){u&&s&&(u=!1,s.length?l=s.concat(l):c=-1,l.length&&p())}function p(){if(!u){var e=a(d);u=!0;for(var t=l.length;t;){for(s=l,l=[];++c1)for(var n=1;n{"use strict";var r=n(791),o=n(296);function i(e){for(var t="https://reactjs.org/docs/error-decoder.html?invariant="+e,n=1;n