diff --git a/.pipelines/e2e.yml b/.pipelines/e2e.yml index cc7cc47feab..1f58b07f8bc 100644 --- a/.pipelines/e2e.yml +++ b/.pipelines/e2e.yml @@ -70,6 +70,8 @@ jobs: - script: | export CI=true + # Tell the E2E binary to run the MIMO tests + export ARO_E2E_MIMO=true . secrets/env . ./hack/e2e/run-rp-and-e2e.sh @@ -84,6 +86,9 @@ jobs: run_selenium validate_selenium_running + run_mimo_actuator + validate_mimo_actuator_running + run_rp validate_rp_running @@ -128,6 +133,7 @@ jobs: delete_e2e_cluster kill_rp + kill_mimo_actuator kill_selenium kill_podman kill_vpn diff --git a/Makefile b/Makefile index 36c95bff746..219a36c0abb 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ SHELL = /bin/bash TAG ?= $(shell git describe --exact-match 2>/dev/null) COMMIT = $(shell git rev-parse --short=7 HEAD)$(shell [[ $$(git status --porcelain) = "" ]] || echo -dirty) ARO_IMAGE_BASE = ${RP_IMAGE_ACR}.azurecr.io/aro -E2E_FLAGS ?= -test.v --ginkgo.v --ginkgo.timeout 180m --ginkgo.flake-attempts=2 --ginkgo.junit-report=e2e-report.xml +E2E_FLAGS ?= -test.v --ginkgo.vv --ginkgo.timeout 180m --ginkgo.flake-attempts=2 --ginkgo.junit-report=e2e-report.xml E2E_LABEL ?= !smoke&&!regressiontest GO_FLAGS ?= -tags=containers_image_openpgp,exclude_graphdriver_btrfs,exclude_graphdriver_devicemapper @@ -67,7 +67,7 @@ aro: check-release generate .PHONY: runlocal-rp runlocal-rp: - go run -ldflags "-X github.com/Azure/ARO-RP/pkg/util/version.GitCommit=$(VERSION)" ./cmd/aro rp + go run -ldflags "-X github.com/Azure/ARO-RP/pkg/util/version.GitCommit=$(VERSION)" ./cmd/aro ${ARO_CMD_ARGS} rp .PHONY: az az: pyenv @@ -196,7 +196,11 @@ proxy: .PHONY: runlocal-portal runlocal-portal: - go run -ldflags "-X github.com/Azure/ARO-RP/pkg/util/version.GitCommit=$(VERSION)" ./cmd/aro portal + go run -ldflags "-X github.com/Azure/ARO-RP/pkg/util/version.GitCommit=$(VERSION)" ./cmd/aro ${ARO_CMD_ARGS} portal + +.PHONY: runlocal-actuator +runlocal-actuator: + go run -ldflags "-X github.com/Azure/ARO-RP/pkg/util/version.GitCommit=$(VERSION)" ./cmd/aro ${ARO_CMD_ARGS} mimo-actuator .PHONY: build-portal build-portal: diff --git a/cmd/aro/main.go b/cmd/aro/main.go index e629e8202ab..eea63e22b06 100644 --- a/cmd/aro/main.go +++ b/cmd/aro/main.go @@ -28,6 +28,7 @@ func usage() { fmt.Fprintf(flag.CommandLine.Output(), " %s operator {master,worker}\n", os.Args[0]) fmt.Fprintf(flag.CommandLine.Output(), " %s update-versions\n", os.Args[0]) fmt.Fprintf(flag.CommandLine.Output(), " %s update-role-sets\n", os.Args[0]) + fmt.Fprintf(flag.CommandLine.Output(), " %s mimo-actuator\n", os.Args[0]) flag.PrintDefaults() } @@ -74,6 +75,9 @@ func main() { case "update-role-sets": checkArgs(1) err = updatePlatformWorkloadIdentityRoleSets(ctx, log) + case "mimo-actuator": + checkArgs(1) + err = mimoActuator(ctx, log) default: usage() os.Exit(2) diff --git a/cmd/aro/mimoactuator.go b/cmd/aro/mimoactuator.go new file mode 100644 index 00000000000..1136f3fb572 --- /dev/null +++ b/cmd/aro/mimoactuator.go @@ -0,0 +1,98 @@ +package main + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "os" + "os/signal" + "syscall" + + "github.com/sirupsen/logrus" + + "github.com/Azure/ARO-RP/pkg/database" + "github.com/Azure/ARO-RP/pkg/env" + "github.com/Azure/ARO-RP/pkg/metrics/statsd" + "github.com/Azure/ARO-RP/pkg/metrics/statsd/golang" + "github.com/Azure/ARO-RP/pkg/mimo/actuator" + "github.com/Azure/ARO-RP/pkg/mimo/tasks" + "github.com/Azure/ARO-RP/pkg/proxy" + "github.com/Azure/ARO-RP/pkg/util/service" +) + +func mimoActuator(ctx context.Context, log *logrus.Entry) error { + stop := make(chan struct{}) + + _env, err := env.NewEnv(ctx, log, env.COMPONENT_MIMO_ACTUATOR) + if err != nil { + return err + } + + var keys []string + if _env.IsLocalDevelopmentMode() { + keys = []string{} + } else { + keys = []string{ + "MDM_ACCOUNT", + "MDM_NAMESPACE", + } + } + + if err = env.ValidateVars(keys...); err != nil { + return err + } + + m := statsd.New(ctx, log.WithField("component", "actuator"), _env, os.Getenv("MDM_ACCOUNT"), os.Getenv("MDM_NAMESPACE"), os.Getenv("MDM_STATSD_SOCKET")) + + g, err := golang.NewMetrics(_env.Logger(), m) + if err != nil { + return err + } + go g.Run() + + dbc, err := service.NewDatabase(ctx, _env, log, m, true) + if err != nil { + return err + } + + dbName, err := service.DBName(_env.IsLocalDevelopmentMode()) + if err != nil { + return err + } + + clusters, err := database.NewOpenShiftClusters(ctx, dbc, dbName) + if err != nil { + return err + } + + manifests, err := database.NewMaintenanceManifests(ctx, dbc, dbName) + if err != nil { + return err + } + + dbg := database.NewDBGroup(). + WithOpenShiftClusters(clusters). + WithMaintenanceManifests(manifests) + + dialer, err := proxy.NewDialer(_env.IsLocalDevelopmentMode()) + if err != nil { + return err + } + + a := actuator.NewService(_env, _env.Logger(), dialer, dbg, m) + a.SetMaintenanceTasks(tasks.DEFAULT_MAINTENANCE_SETS) + + sigterm := make(chan os.Signal, 1) + done := make(chan struct{}) + signal.Notify(sigterm, syscall.SIGTERM) + + go a.Run(ctx, stop, done) + + <-sigterm + log.Print("received SIGTERM") + close(stop) + <-done + + return nil +} diff --git a/cmd/aro/rp.go b/cmd/aro/rp.go index 1dd9f6e3335..5e8227a68ff 100644 --- a/cmd/aro/rp.go +++ b/cmd/aro/rp.go @@ -172,6 +172,15 @@ func rp(ctx context.Context, log, audit *logrus.Entry) error { WithPlatformWorkloadIdentityRoleSets(dbPlatformWorkloadIdentityRoleSets). WithSubscriptions(dbSubscriptions) + // MIMO only activated in development for now + if _env.IsLocalDevelopmentMode() { + dbMaintenanceManifests, err := database.NewMaintenanceManifests(ctx, dbc, dbName) + if err != nil { + return err + } + dbg.WithMaintenanceManifests(dbMaintenanceManifests) + } + f, err := frontend.NewFrontend(ctx, audit, log.WithField("component", "frontend"), _env, dbg, api.APIs, metrics, clusterm, feAead, hiveClusterManager, adminactions.NewKubeActions, adminactions.NewAzureActions, adminactions.NewAppLensActions, clusterdata.NewParallelEnricher(metrics, _env)) if err != nil { return err diff --git a/docs/mimo/README.md b/docs/mimo/README.md new file mode 100644 index 00000000000..8d03ab961c4 --- /dev/null +++ b/docs/mimo/README.md @@ -0,0 +1,22 @@ +# MIMO Documentation + +The Managed Infrastructure Maintenance Operator, or MIMO, is a component of the Azure Red Hat OpenShift Resource Provider (ARO-RP) which is responsible for automated maintenance of clusters provisioned by the platform. +MIMO specifically focuses on "managed infrastructure", the parts of ARO that are deployed and maintained by the RP and ARO Operator instead of by OCP (in-cluster) or Hive (out-of-cluster). + +MIMO consists of two main components, the [Actuator](./actuator.md) and the [Scheduler](./scheduler.md). It is primarily interfaced with via the [Admin API](./admin-api.md). + +## A Primer On MIMO + +The smallest thing that you can tell MIMO to run is a **Task** (see [`pkg/mimo/tasks/`](../../pkg/mimo/tasks/)). +A Task is composed of reusable **Steps** (see [`pkg/mimo/steps/`](../../pkg/mimo/steps/)), reusing the framework utilised by AdminUpdate/Update/Install methods in `pkg/cluster/`. +A Task only runs in the scope of a singular cluster. +These steps are run in sequence and can return either **Terminal** errors (causing the ran Task to fail and not be retried) or **Transient** errors (which indicates that the Task can be retried later). + +Tasks are executed by the **Actuator** by way of creation of a **Maintenance Manifest**. +This Manifest is created with the cluster ID (which is elided from the cluster-scoped Admin APIs), the Task ID (which is currently a UUID), and optional priority, "start after", and "start before" times which are filled in with defaults if not provided. +The Actuator will treat these Maintenance Manifests as a work queue, taking ones which are past their "start after" time and executing them in order of earliest start-after and priority. +After running each, a state will be written into the Manifest (with optional free-form status text) with the result of the ran Task. +Manifests past their start-before times are marked as having a "timed out" state and not ran. + +Currently, Manifests are created by the Admin API. +In the future, the Scheduler will create some these Manifests depending on cluster state/version and wall-clock time, providing the ability to perform tasks like rotations of secrets autonomously. diff --git a/docs/mimo/actuator.md b/docs/mimo/actuator.md new file mode 100644 index 00000000000..5950662890e --- /dev/null +++ b/docs/mimo/actuator.md @@ -0,0 +1,30 @@ +# Managed Infrastructure Maintenance Operator: Actuator + +The Actuator is the MIMO component that performs execution of tasks. +The process of running tasks looks like this: + +```mermaid +graph TD; + START((Start))-->QUERY; + QUERY[Fetch all State = Pending] -->SORT; + SORT[Sort tasks by RUNAFTER and PRIORITY]-->ITERATE[Iterate over tasks]; + ITERATE-- Per Task -->ISEXPIRED; + subgraph PerTask[ ] + ISEXPIRED{{Is RUNBEFORE > now?}}-- Yes --> STATETIMEDOUT([State = TimedOut]) --> CONTINUE[Continue]; + ISEXPIRED-- No --> DEQUEUECLUSTER; + DEQUEUECLUSTER[Claim lease on OpenShiftClusterDocument] --> DEQUEUE; + DEQUEUE[Actuator dequeues task]--> ISRETRYLIMIT; + ISRETRYLIMIT{{Have we retried the task too many times?}} -- Yes --> STATETIMEDOUT; + ISRETRYLIMIT -- No -->STATEINPROGRESS; + STATEINPROGRESS([State = InProgress]) -->RUN[[Task is run]]; + RUN -- Success --> SUCCESS + RUN-- Terminal Error-->TERMINALERROR; + RUN-- Transient Error-->TRANSIENTERROR; + SUCCESS([State = Completed])-->DELEASECLUSTER + TERMINALERROR([State = Failed])-->DELEASECLUSTER; + TRANSIENTERROR([State = Pending])-->DELEASECLUSTER; + DELEASECLUSTER[Release Lease on OpenShiftClusterDocument] -->CONTINUE; + end + CONTINUE-->ITERATE; + ITERATE-- Finished -->END; +``` diff --git a/docs/mimo/admin-api.md b/docs/mimo/admin-api.md new file mode 100644 index 00000000000..e69de29bb2d diff --git a/docs/mimo/scheduler.md b/docs/mimo/scheduler.md new file mode 100644 index 00000000000..8a457798a25 --- /dev/null +++ b/docs/mimo/scheduler.md @@ -0,0 +1,3 @@ +# MIMO Scheduler + +The MIMO Scheduler is a planned component, but is not yet implemented. diff --git a/docs/mimo/writing-tasks.md b/docs/mimo/writing-tasks.md new file mode 100644 index 00000000000..91861535fd2 --- /dev/null +++ b/docs/mimo/writing-tasks.md @@ -0,0 +1 @@ +# Writing MIMO Tasks diff --git a/go.mod b/go.mod index 3221751d4c2..2d967e4cf63 100644 --- a/go.mod +++ b/go.mod @@ -77,6 +77,7 @@ require ( github.com/vincent-petithory/dataurl v1.0.0 go.uber.org/mock v0.4.0 golang.org/x/crypto v0.28.0 + golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 golang.org/x/net v0.30.0 golang.org/x/oauth2 v0.18.0 golang.org/x/sync v0.8.0 @@ -260,7 +261,6 @@ require ( go.opentelemetry.io/otel/metric v1.22.0 // indirect go.opentelemetry.io/otel/trace v1.22.0 // indirect go.starlark.net v0.0.0-20220328144851-d1966c6b9fcd // indirect - golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect golang.org/x/mod v0.17.0 // indirect golang.org/x/sys v0.26.0 // indirect golang.org/x/term v0.25.0 // indirect diff --git a/hack/e2e/run-rp-and-e2e.sh b/hack/e2e/run-rp-and-e2e.sh index c20f67c4c6f..a802637db25 100755 --- a/hack/e2e/run-rp-and-e2e.sh +++ b/hack/e2e/run-rp-and-e2e.sh @@ -91,6 +91,43 @@ kill_portal() { wait $rppid } +run_mimo_actuator() { + echo "########## 🚀 Run MIMO Actuator in background ##########" + export AZURE_ENVIRONMENT=AzurePublicCloud + ./aro mimo-actuator & +} + +kill_mimo_actuator() { + echo "########## Kill the MIMO Actuator running in background ##########" + rppid=$(lsof -t -i :8445) + kill $rppid + wait $rppid +} + +validate_mimo_actuator_running() { + echo "########## ？Checking MIMO Actuator Status ##########" + ELAPSED=0 + while true; do + sleep 5 + http_code=$(curl -k -s -o /dev/null -w '%{http_code}' http://localhost:8445/healthz/ready) + case $http_code in + "200") + echo "########## ✅ ARO MIMO Actuator Running ##########" + break + ;; + *) + echo "Attempt $ELAPSED - local MIMO Actuator is NOT up. Code : $http_code, waiting" + sleep 2 + # after 40 secs return exit 1 to not block ci + ELAPSED=$((ELAPSED + 1)) + if [ $ELAPSED -eq 20 ]; then + exit 1 + fi + ;; + esac + done +} + run_vpn() { echo "########## 🚀 Run OpenVPN in background ##########" echo "Using Secret secrets/$VPN" diff --git a/pkg/api/admin/mimo.go b/pkg/api/admin/mimo.go new file mode 100644 index 00000000000..b57c5664766 --- /dev/null +++ b/pkg/api/admin/mimo.go @@ -0,0 +1,40 @@ +package admin + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +type MaintenanceManifestState string + +const ( + MaintenanceManifestStatePending MaintenanceManifestState = "Pending" + MaintenanceManifestStateInProgress MaintenanceManifestState = "InProgress" + MaintenanceManifestStateCompleted MaintenanceManifestState = "Completed" + MaintenanceManifestStateFailed MaintenanceManifestState = "Failed" + MaintenanceManifestStateTimedOut MaintenanceManifestState = "TimedOut" + MaintenanceManifestStateCancelled MaintenanceManifestState = "Cancelled" +) + +type MaintenanceManifest struct { + // The ID for the resource. + ID string `json:"id,omitempty"` + + State MaintenanceManifestState `json:"state,omitempty"` + StatusText string `json:"statusText,omitempty"` + + MaintenanceTaskID string `json:"maintenanceTaskID,omitempty"` + Priority int `json:"priority,omitempty"` + + // RunAfter defines the earliest that this manifest should start running + RunAfter int `json:"runAfter,omitempty"` + // RunBefore defines the latest that this manifest should start running + RunBefore int `json:"runBefore,omitempty"` +} + +// MaintenanceManifestList represents a list of MaintenanceManifests. +type MaintenanceManifestList struct { + // The list of MaintenanceManifests. + MaintenanceManifests []*MaintenanceManifest `json:"value"` + + // The link used to get the next page of operations. + NextLink string `json:"nextLink,omitempty"` +} diff --git a/pkg/api/admin/mimo_convert.go b/pkg/api/admin/mimo_convert.go new file mode 100644 index 00000000000..3bc1b468554 --- /dev/null +++ b/pkg/api/admin/mimo_convert.go @@ -0,0 +1,50 @@ +package admin + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "github.com/Azure/ARO-RP/pkg/api" +) + +type maintenanceManifestConverter struct{} + +func (m maintenanceManifestConverter) ToExternal(d *api.MaintenanceManifestDocument) interface{} { + return &MaintenanceManifest{ + ID: d.ID, + + State: MaintenanceManifestState(d.MaintenanceManifest.State), + StatusText: d.MaintenanceManifest.StatusText, + + MaintenanceTaskID: d.MaintenanceManifest.MaintenanceTaskID, + Priority: d.MaintenanceManifest.Priority, + + RunAfter: d.MaintenanceManifest.RunAfter, + RunBefore: d.MaintenanceManifest.RunBefore, + } +} + +func (m maintenanceManifestConverter) ToExternalList(docs []*api.MaintenanceManifestDocument, nextLink string) interface{} { + l := &MaintenanceManifestList{ + MaintenanceManifests: make([]*MaintenanceManifest, 0, len(docs)), + NextLink: nextLink, + } + + for _, doc := range docs { + l.MaintenanceManifests = append(l.MaintenanceManifests, m.ToExternal(doc).(*MaintenanceManifest)) + } + + return l +} + +func (m maintenanceManifestConverter) ToInternal(_i interface{}, out *api.MaintenanceManifestDocument) { + i := _i.(*MaintenanceManifest) + + out.ID = i.ID + out.MaintenanceManifest.MaintenanceTaskID = i.MaintenanceTaskID + out.MaintenanceManifest.Priority = i.Priority + out.MaintenanceManifest.RunAfter = i.RunAfter + out.MaintenanceManifest.RunBefore = i.RunBefore + out.MaintenanceManifest.State = api.MaintenanceManifestState(i.State) + out.MaintenanceManifest.StatusText = i.StatusText +} diff --git a/pkg/api/admin/mimo_validatestatic.go b/pkg/api/admin/mimo_validatestatic.go new file mode 100644 index 00000000000..ca052458631 --- /dev/null +++ b/pkg/api/admin/mimo_validatestatic.go @@ -0,0 +1,59 @@ +package admin + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "net/http" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/api/util/immutable" +) + +type maintenanceManifestStaticValidator struct{} + +// Validate validates an OpenShift cluster +func (sv maintenanceManifestStaticValidator) Static(_new interface{}, _current *api.MaintenanceManifestDocument) error { + new := _new.(*MaintenanceManifest) + + var current *MaintenanceManifest + if _current != nil { + current = (&maintenanceManifestConverter{}).ToExternal(_current).(*MaintenanceManifest) + } + + err := sv.validate(new) + if err != nil { + return err + } + + if current == nil { + return nil + } + + return sv.validateDelta(new, current) +} + +func (sv maintenanceManifestStaticValidator) validate(new *MaintenanceManifest) error { + if new.MaintenanceTaskID == "" { + return api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidParameter, "maintenanceTaskID", "Must be provided") + } + + if new.RunAfter == 0 { + return api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidParameter, "runAfter", "Must be provided") + } + + if new.RunBefore == 0 { + return api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidParameter, "runBefore", "Must be provided") + } + + return nil +} + +func (sv maintenanceManifestStaticValidator) validateDelta(new, current *MaintenanceManifest) error { + err := immutable.Validate("", new, current) + if err != nil { + err := err.(*immutable.ValidationError) + return api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodePropertyChangeNotAllowed, err.Target, err.Message) + } + return nil +} diff --git a/pkg/api/admin/openshiftcluster.go b/pkg/api/admin/openshiftcluster.go index 2a209eb4bb5..7bbd02e62cd 100644 --- a/pkg/api/admin/openshiftcluster.go +++ b/pkg/api/admin/openshiftcluster.go @@ -5,6 +5,8 @@ package admin import ( "time" + + "github.com/Azure/go-autorest/autorest/date" ) // OpenShiftClusterList represents a list of OpenShift clusters. @@ -18,13 +20,14 @@ type OpenShiftClusterList struct { // OpenShiftCluster represents an Azure Red Hat OpenShift cluster. type OpenShiftCluster struct { - ID string `json:"id,omitempty" mutable:"case"` - Name string `json:"name,omitempty" mutable:"case"` - Type string `json:"type,omitempty" mutable:"case"` - Location string `json:"location,omitempty"` - Tags map[string]string `json:"tags,omitempty"` - Properties OpenShiftClusterProperties `json:"properties,omitempty"` - Identity *Identity `json:"identity,omitempty"` + ID string `json:"id,omitempty" mutable:"case"` + Name string `json:"name,omitempty" mutable:"case"` + Type string `json:"type,omitempty" mutable:"case"` + Location string `json:"location,omitempty"` + Tags map[string]string `json:"tags,omitempty"` + Properties OpenShiftClusterProperties `json:"properties,omitempty"` + Identity *Identity `json:"identity,omitempty"` + OperatorFlagsMergeStrategy string `json:"operatorFlagsMergeStrategy,omitempty" mutable:"true"` } // OpenShiftClusterProperties represents an OpenShift cluster's properties. @@ -466,8 +469,9 @@ const ( // RegistryProfile represents a registry profile type RegistryProfile struct { - Name string `json:"name,omitempty"` - Username string `json:"username,omitempty"` + Name string `json:"name,omitempty"` + Username string `json:"username,omitempty"` + IssueDate *date.Time `json:"issueDate,omitempty"` } // ArchitectureVersion represents an architecture version diff --git a/pkg/api/admin/openshiftcluster_validatestatic.go b/pkg/api/admin/openshiftcluster_validatestatic.go index fe0d7d8d420..ea8fbf8aff9 100644 --- a/pkg/api/admin/openshiftcluster_validatestatic.go +++ b/pkg/api/admin/openshiftcluster_validatestatic.go @@ -39,6 +39,7 @@ func validateMaintenanceTask(task MaintenanceTask) error { task == MaintenanceTaskRenewCerts || task == MaintenanceTaskPending || task == MaintenanceTaskNone || + task == MaintenanceTaskSyncClusterObject || task == MaintenanceTaskCustomerActionNeeded) { return api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidParameter, "properties.maintenanceTask", "Invalid enum parameter.") } diff --git a/pkg/api/admin/openshiftcluster_validatestatic_test.go b/pkg/api/admin/openshiftcluster_validatestatic_test.go index 3088f319a4a..08ccb0eb9f1 100644 --- a/pkg/api/admin/openshiftcluster_validatestatic_test.go +++ b/pkg/api/admin/openshiftcluster_validatestatic_test.go @@ -10,6 +10,8 @@ import ( "testing" "time" + "github.com/Azure/go-autorest/autorest/date" + "github.com/Azure/ARO-RP/pkg/api" "github.com/Azure/ARO-RP/pkg/util/uuid" "github.com/Azure/ARO-RP/test/validate" @@ -624,11 +626,11 @@ func TestOpenShiftClusterStaticValidateDelta(t *testing.T) { wantErr: "400: PropertyChangeNotAllowed: properties.provisionedBy: Changing property 'properties.provisionedBy' is not allowed.", }, { - name: "registryProfiles change is not allowed", + name: "registryProfiles username change is not allowed", oc: func() *OpenShiftCluster { return &OpenShiftCluster{ Properties: OpenShiftClusterProperties{ - RegistryProfiles: []RegistryProfile{{Name: "test", Username: "testuser"}}, + RegistryProfiles: []RegistryProfile{{Name: "test", Username: "testuser", IssueDate: toDate(time.Now())}}, }, } }, @@ -637,6 +639,20 @@ func TestOpenShiftClusterStaticValidateDelta(t *testing.T) { }, wantErr: "400: PropertyChangeNotAllowed: properties.registryProfiles: Changing property 'properties.registryProfiles' is not allowed.", }, + { + name: "registryProfiles expiry change is not allowed", + oc: func() *OpenShiftCluster { + return &OpenShiftCluster{ + Properties: OpenShiftClusterProperties{ + RegistryProfiles: []RegistryProfile{{Name: "test", Username: "testuser", IssueDate: toDate(time.Now())}}, + }, + } + }, + modify: func(oc *OpenShiftCluster) { + oc.Properties.RegistryProfiles[0].IssueDate = toDate(time.Now().UTC().Add(time.Hour * 24 * 30)) + }, + wantErr: "400: PropertyChangeNotAllowed: properties.registryProfiles: Changing property 'properties.registryProfiles' is not allowed.", + }, { name: "maintenanceTask change to Everything is allowed", oc: func() *OpenShiftCluster { @@ -766,3 +782,7 @@ func TestOpenShiftClusterStaticValidateDelta(t *testing.T) { }) } } + +func toDate(t time.Time) *date.Time { + return &date.Time{Time: t} +} diff --git a/pkg/api/admin/operatorflags.go b/pkg/api/admin/operatorflags.go new file mode 100644 index 00000000000..88d5e0873a6 --- /dev/null +++ b/pkg/api/admin/operatorflags.go @@ -0,0 +1,55 @@ +package admin + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "encoding/json" + "net/http" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/operator" +) + +const ( + OperatorFlagsMergeStrategyMerge string = "merge" + OperatorFlagsMergeStrategyReset string = "reset" +) + +// When a cluster is edited via the PATCH Cluster Geneva Action (aka an Admin Update) +// the flags given are treated according to the provided Update Strategy, +// provided in operatorFlagsMergeStrategy + +// merge (default): The provided cluster flags are laid on top of the cluster’s existing flags. +// reset: The provided cluster flags are laid on top of the default cluster flags, +// essentially ‘resetting’ the flags if no new flags are provided. +func OperatorFlagsMergeStrategy(oc *api.OpenShiftCluster, body []byte) error { + payload := &OpenShiftCluster{} + + err := json.Unmarshal(body, &payload) + if err != nil { + return api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidRequestContent, "", "The request content was invalid and could not be deserialized: %q.", err) + } + + // if it's empty, use the default of merge, which is performed by + // deserialising the body JSON later + if payload.OperatorFlagsMergeStrategy == "" { + return nil + } + + // return error if OperatorFlagsMergeStrategy is not merge or reset, default is merge + if payload.OperatorFlagsMergeStrategy != OperatorFlagsMergeStrategyMerge && + payload.OperatorFlagsMergeStrategy != OperatorFlagsMergeStrategyReset { + return api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidParameter, "", "invalid operatorFlagsMergeStrategy '%s', can only be 'merge' or 'reset'", payload.OperatorFlagsMergeStrategy) + } + + // return nil, if OperatorFlagsMergeStrategy is merge and payload has not operatorFlags + // return operatorFlags of payload, if OperatorFlagsMergeStrategy is merge and payload has operatorFlags + // return defaultOperatorFlags, if OperatorFlagsMergeStrategy is reset and payload has not operatorFlags + // return defaultOperatorFlags + operatorFlags of payload, if OperatorFlagsMergeStrategy is reset and payload has operatorFlags + if payload.OperatorFlagsMergeStrategy == OperatorFlagsMergeStrategyReset { + oc.Properties.OperatorFlags = operator.DefaultOperatorFlags() + } + + return nil +} diff --git a/pkg/api/admin/operatorflags_test.go b/pkg/api/admin/operatorflags_test.go new file mode 100644 index 00000000000..54947f9fea4 --- /dev/null +++ b/pkg/api/admin/operatorflags_test.go @@ -0,0 +1,60 @@ +package admin + +import ( + "testing" + + "github.com/Azure/ARO-RP/pkg/api" + utilerror "github.com/Azure/ARO-RP/test/util/error" +) + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +func TestOperatorFlagsMergeStrategy(t *testing.T) { + tests := []struct { + name string + oc *api.OpenShiftCluster + wantOc *api.OpenShiftCluster + body []byte + wantErr string + }{ + { + name: "invalid_json", + oc: nil, + body: []byte(`{{}`), + wantErr: `400: InvalidRequestContent: : The request content was invalid and could not be deserialized: "invalid character '{' looking for beginning of object key string".`, + }, + { + name: "OperatorFlagsMergeStrategy_is_not_merge_or_reset", + oc: nil, + body: []byte(`{"operatorFlagsMergeStrategy": "xyz"}`), + wantErr: `400: InvalidParameter: : invalid operatorFlagsMergeStrategy 'xyz', can only be 'merge' or 'reset'`, + }, + { + name: "OperatorFlagsMergeStrategy_payload_is_empty", + oc: &api.OpenShiftCluster{ + Properties: api.OpenShiftClusterProperties{ + OperatorFlags: api.OperatorFlags{"aro.feature1.enabled": "false"}, + }, + }, + body: []byte(`{"operatorflagsmergestrategy":"merge"}`), + wantErr: "", + }, + { + name: "OperatorFlagsMergeStrategy_reset", + oc: &api.OpenShiftCluster{ + Properties: api.OpenShiftClusterProperties{ + OperatorFlags: api.OperatorFlags{"aro.feature1.enabled": "false"}, + }, + }, + body: []byte(`{"operatorflagsmergestrategy":"reset"}`), + wantErr: "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := OperatorFlagsMergeStrategy(tt.oc, tt.body) + utilerror.AssertErrorMessage(t, err, tt.wantErr) + }) + } +} diff --git a/pkg/api/admin/register.go b/pkg/api/admin/register.go index 4a0613c276e..028f7dd2bcb 100644 --- a/pkg/api/admin/register.go +++ b/pkg/api/admin/register.go @@ -18,5 +18,7 @@ func init() { OpenShiftVersionStaticValidator: openShiftVersionStaticValidator{}, PlatformWorkloadIdentityRoleSetConverter: platformWorkloadIdentityRoleSetConverter{}, PlatformWorkloadIdentityRoleSetStaticValidator: platformWorkloadIdentityRoleSetStaticValidator{}, + MaintenanceManifestConverter: maintenanceManifestConverter{}, + MaintenanceManifestStaticValidator: maintenanceManifestStaticValidator{}, } } diff --git a/pkg/api/mimo.go b/pkg/api/mimo.go new file mode 100644 index 00000000000..b8e983b2992 --- /dev/null +++ b/pkg/api/mimo.go @@ -0,0 +1,32 @@ +package api + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +type MaintenanceManifestState string + +const ( + MaintenanceManifestStatePending MaintenanceManifestState = "Pending" + MaintenanceManifestStateInProgress MaintenanceManifestState = "InProgress" + MaintenanceManifestStateCompleted MaintenanceManifestState = "Completed" + MaintenanceManifestStateFailed MaintenanceManifestState = "Failed" + MaintenanceManifestStateTimedOut MaintenanceManifestState = "TimedOut" + MaintenanceManifestStateCancelled MaintenanceManifestState = "Cancelled" +) + +// MaintenanceManifest represents an instance of a MaintenanceTask running on a +// given cluster. +type MaintenanceManifest struct { + MissingFields + + State MaintenanceManifestState `json:"state,omitempty"` + StatusText string `json:"statusText,omitempty"` + + MaintenanceTaskID string `json:"maintenanceTaskID,omitempty"` + Priority int `json:"priority,omitempty"` + + // RunAfter defines the earliest that this manifest should start running + RunAfter int `json:"runAfter,omitempty"` + // RunBefore defines the latest that this manifest should start running + RunBefore int `json:"runBefore,omitempty"` +} diff --git a/pkg/api/mimodocument.go b/pkg/api/mimodocument.go new file mode 100644 index 00000000000..141b75f412c --- /dev/null +++ b/pkg/api/mimodocument.go @@ -0,0 +1,39 @@ +package api + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +type MaintenanceManifestDocuments struct { + Count int `json:"_count,omitempty"` + ResourceID string `json:"_rid,omitempty"` + MaintenanceManifestDocuments []*MaintenanceManifestDocument `json:"Documents,omitempty"` +} + +func (e *MaintenanceManifestDocuments) String() string { + return encodeJSON(e) +} + +type MaintenanceManifestDocument struct { + MissingFields + + ID string `json:"id,omitempty"` + ResourceID string `json:"_rid,omitempty"` + Timestamp int `json:"_ts,omitempty"` + Self string `json:"_self,omitempty"` + ETag string `json:"_etag,omitempty" deep:"-"` + Attachments string `json:"_attachments,omitempty"` + TTL int `json:"ttl,omitempty"` + LSN int `json:"_lsn,omitempty"` + Metadata map[string]interface{} `json:"_metadata,omitempty"` + + ClusterResourceID string `json:"clusterResourceID,omitempty"` + MaintenanceManifest MaintenanceManifest `json:"maintenanceManifest,omitempty"` + + LeaseOwner string `json:"leaseOwner,omitempty" deep:"-"` + LeaseExpires int `json:"leaseExpires,omitempty" deep:"-"` + Dequeues int `json:"dequeues,omitempty"` +} + +func (e *MaintenanceManifestDocument) String() string { + return encodeJSON(e) +} diff --git a/pkg/api/openshiftcluster.go b/pkg/api/openshiftcluster.go index 4b47190ff0c..b0ee2ad472b 100644 --- a/pkg/api/openshiftcluster.go +++ b/pkg/api/openshiftcluster.go @@ -9,6 +9,7 @@ import ( "time" "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" + "github.com/Azure/go-autorest/autorest/date" ) // OpenShiftCluster represents an OpenShift cluster @@ -211,6 +212,7 @@ const ( ProvisioningStateUpdating ProvisioningState = "Updating" ProvisioningStateAdminUpdating ProvisioningState = "AdminUpdating" ProvisioningStateCanceled ProvisioningState = "Canceled" + ProvisioningStateMaintenance ProvisioningState = "Maintenance" ProvisioningStateDeleting ProvisioningState = "Deleting" ProvisioningStateSucceeded ProvisioningState = "Succeeded" ProvisioningStateFailed ProvisioningState = "Failed" @@ -260,6 +262,7 @@ func (t MaintenanceTask) IsMaintenanceOngoingTask() bool { result := (t == MaintenanceTaskEverything) || (t == MaintenanceTaskOperator) || (t == MaintenanceTaskRenewCerts) || + (t == MaintenanceTaskSyncClusterObject) || (t == "") return result } @@ -768,9 +771,10 @@ type IngressProfile struct { type RegistryProfile struct { MissingFields - Name string `json:"name,omitempty"` - Username string `json:"username,omitempty"` - Password SecureString `json:"password,omitempty"` + Name string `json:"name,omitempty"` + Username string `json:"username,omitempty"` + Password SecureString `json:"password,omitempty"` + IssueDate *date.Time `json:"issueDate,omitempty"` } // Install represents an install process diff --git a/pkg/api/register.go b/pkg/api/register.go index 5e91693909b..59fa1ebcda8 100644 --- a/pkg/api/register.go +++ b/pkg/api/register.go @@ -71,6 +71,16 @@ type SecretConverter interface { ToInternal(interface{}, *Secret) } +type MaintenanceManifestConverter interface { + ToExternal(*MaintenanceManifestDocument) interface{} + ToExternalList([]*MaintenanceManifestDocument, string) interface{} + ToInternal(interface{}, *MaintenanceManifestDocument) +} + +type MaintenanceManifestStaticValidator interface { + Static(interface{}, *MaintenanceManifestDocument) error +} + // Version is a set of endpoints implemented by each API version type Version struct { OpenShiftClusterConverter OpenShiftClusterConverter @@ -87,6 +97,8 @@ type Version struct { SyncIdentityProviderConverter SyncIdentityProviderConverter SecretConverter SecretConverter ClusterManagerStaticValidator ClusterManagerStaticValidator + MaintenanceManifestConverter MaintenanceManifestConverter + MaintenanceManifestStaticValidator MaintenanceManifestStaticValidator } // APIs is the map of registered API versions diff --git a/pkg/cluster/acrtoken.go b/pkg/cluster/acrtoken.go index 9ffc25e65ed..ba7c4e8d9b7 100644 --- a/pkg/cluster/acrtoken.go +++ b/pkg/cluster/acrtoken.go @@ -7,6 +7,7 @@ import ( "context" "time" + "github.com/Azure/go-autorest/autorest/date" corev1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -57,6 +58,7 @@ func (m *manager) ensureACRToken(ctx context.Context) error { } rp.Password = api.SecureString(password) + rp.IssueDate = &date.Time{Time: time.Now().UTC()} m.doc, err = m.db.PatchWithLease(ctx, m.doc.Key, func(doc *api.OpenShiftClusterDocument) error { token.PutRegistryProfile(doc.OpenShiftCluster, rp) diff --git a/pkg/cluster/condition.go b/pkg/cluster/condition.go index a5f1fbd9140..6a1744822db 100644 --- a/pkg/cluster/condition.go +++ b/pkg/cluster/condition.go @@ -12,6 +12,8 @@ import ( corev1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/Azure/ARO-RP/pkg/util/clusteroperators" ) const minimumWorkerNodes = 2 @@ -24,7 +26,7 @@ func (m *manager) apiServersReady(ctx context.Context) (bool, error) { if err != nil { return false, nil } - return isOperatorAvailable(apiserver), nil + return clusteroperators.IsOperatorAvailable(apiserver), nil } func (m *manager) minimumWorkerNodesReady(ctx context.Context) (bool, error) { @@ -57,7 +59,7 @@ func (m *manager) operatorConsoleReady(ctx context.Context) (bool, error) { if err != nil { return false, nil } - return isOperatorAvailable(consoleOperator), nil + return clusteroperators.IsOperatorAvailable(consoleOperator), nil } func (m *manager) clusterVersionReady(ctx context.Context) (bool, error) { @@ -77,15 +79,7 @@ func (m *manager) ingressControllerReady(ctx context.Context) (bool, error) { if err != nil { return false, nil } - return isOperatorAvailable(ingressOperator), nil -} - -func isOperatorAvailable(operator *configv1.ClusterOperator) bool { - m := make(map[configv1.ClusterStatusConditionType]configv1.ConditionStatus, len(operator.Status.Conditions)) - for _, cond := range operator.Status.Conditions { - m[cond.Type] = cond.Status - } - return m[configv1.OperatorAvailable] == configv1.ConditionTrue && m[configv1.OperatorProgressing] == configv1.ConditionFalse + return clusteroperators.IsOperatorAvailable(ingressOperator), nil } // aroCredentialsRequestReconciled evaluates whether the openshift-azure-operator CredentialsRequest has recently been reconciled and returns true diff --git a/pkg/cluster/condition_test.go b/pkg/cluster/condition_test.go index a0515659204..c035bce48ca 100644 --- a/pkg/cluster/condition_test.go +++ b/pkg/cluster/condition_test.go @@ -63,59 +63,6 @@ func TestOperatorConsoleExists(t *testing.T) { } } -func TestIsOperatorAvailable(t *testing.T) { - for _, tt := range []struct { - name string - availableCondition configv1.ConditionStatus - progressingCondition configv1.ConditionStatus - want bool - }{ - { - name: "Available && Progressing; not available", - availableCondition: configv1.ConditionTrue, - progressingCondition: configv1.ConditionTrue, - }, - { - name: "Available && !Progressing; available", - availableCondition: configv1.ConditionTrue, - progressingCondition: configv1.ConditionFalse, - want: true, - }, - { - name: "!Available && Progressing; not available", - availableCondition: configv1.ConditionFalse, - progressingCondition: configv1.ConditionTrue, - }, - { - name: "!Available && !Progressing; not available", - availableCondition: configv1.ConditionFalse, - progressingCondition: configv1.ConditionFalse, - }, - } { - operator := &configv1.ClusterOperator{ - ObjectMeta: metav1.ObjectMeta{ - Name: "name", - }, - Status: configv1.ClusterOperatorStatus{ - Conditions: []configv1.ClusterOperatorStatusCondition{ - { - Type: configv1.OperatorAvailable, - Status: tt.availableCondition, - }, - { - Type: configv1.OperatorProgressing, - Status: tt.progressingCondition, - }, - }, - }, - } - available := isOperatorAvailable(operator) - if available != tt.want { - t.Error(available) - } - } -} - func TestMinimumWorkerNodesReady(t *testing.T) { ctx := context.Background() diff --git a/pkg/database/cosmosdb/generate.go b/pkg/database/cosmosdb/generate.go index 56bf0816e78..b456c1819b5 100644 --- a/pkg/database/cosmosdb/generate.go +++ b/pkg/database/cosmosdb/generate.go @@ -3,7 +3,7 @@ package cosmosdb // Copyright (c) Microsoft Corporation. // Licensed under the Apache License 2.0. -//go:generate gencosmosdb github.com/Azure/ARO-RP/pkg/api,AsyncOperationDocument github.com/Azure/ARO-RP/pkg/api,BillingDocument github.com/Azure/ARO-RP/pkg/api,GatewayDocument github.com/Azure/ARO-RP/pkg/api,MonitorDocument github.com/Azure/ARO-RP/pkg/api,OpenShiftClusterDocument github.com/Azure/ARO-RP/pkg/api,SubscriptionDocument github.com/Azure/ARO-RP/pkg/api,OpenShiftVersionDocument github.com/Azure/ARO-RP/pkg/api,ClusterManagerConfigurationDocument github.com/Azure/ARO-RP/pkg/api,PlatformWorkloadIdentityRoleSetDocument +//go:generate gencosmosdb github.com/Azure/ARO-RP/pkg/api,AsyncOperationDocument github.com/Azure/ARO-RP/pkg/api,BillingDocument github.com/Azure/ARO-RP/pkg/api,GatewayDocument github.com/Azure/ARO-RP/pkg/api,MonitorDocument github.com/Azure/ARO-RP/pkg/api,OpenShiftClusterDocument github.com/Azure/ARO-RP/pkg/api,SubscriptionDocument github.com/Azure/ARO-RP/pkg/api,OpenShiftVersionDocument github.com/Azure/ARO-RP/pkg/api,ClusterManagerConfigurationDocument github.com/Azure/ARO-RP/pkg/api,PlatformWorkloadIdentityRoleSetDocument github.com/Azure/ARO-RP/pkg/api,MaintenanceManifestDocument //go:generate goimports -local=github.com/Azure/ARO-RP -e -w ./ //go:generate mockgen -destination=../../util/mocks/$GOPACKAGE/$GOPACKAGE.go github.com/Azure/ARO-RP/pkg/database/$GOPACKAGE PermissionClient //go:generate goimports -local=github.com/Azure/ARO-RP -e -w ../../util/mocks/$GOPACKAGE/$GOPACKAGE.go diff --git a/pkg/database/cosmosdb/zz_generated_maintenancemanifestdocument.go b/pkg/database/cosmosdb/zz_generated_maintenancemanifestdocument.go new file mode 100644 index 00000000000..1aa237d5d06 --- /dev/null +++ b/pkg/database/cosmosdb/zz_generated_maintenancemanifestdocument.go @@ -0,0 +1,313 @@ +// Code generated by github.com/jewzaam/go-cosmosdb, DO NOT EDIT. + +package cosmosdb + +import ( + "context" + "net/http" + "strconv" + "strings" + + pkg "github.com/Azure/ARO-RP/pkg/api" +) + +type maintenanceManifestDocumentClient struct { + *databaseClient + path string +} + +// MaintenanceManifestDocumentClient is a maintenanceManifestDocument client +type MaintenanceManifestDocumentClient interface { + Create(context.Context, string, *pkg.MaintenanceManifestDocument, *Options) (*pkg.MaintenanceManifestDocument, error) + List(*Options) MaintenanceManifestDocumentIterator + ListAll(context.Context, *Options) (*pkg.MaintenanceManifestDocuments, error) + Get(context.Context, string, string, *Options) (*pkg.MaintenanceManifestDocument, error) + Replace(context.Context, string, *pkg.MaintenanceManifestDocument, *Options) (*pkg.MaintenanceManifestDocument, error) + Delete(context.Context, string, *pkg.MaintenanceManifestDocument, *Options) error + Query(string, *Query, *Options) MaintenanceManifestDocumentRawIterator + QueryAll(context.Context, string, *Query, *Options) (*pkg.MaintenanceManifestDocuments, error) + ChangeFeed(*Options) MaintenanceManifestDocumentIterator +} + +type maintenanceManifestDocumentChangeFeedIterator struct { + *maintenanceManifestDocumentClient + continuation string + options *Options +} + +type maintenanceManifestDocumentListIterator struct { + *maintenanceManifestDocumentClient + continuation string + done bool + options *Options +} + +type maintenanceManifestDocumentQueryIterator struct { + *maintenanceManifestDocumentClient + partitionkey string + query *Query + continuation string + done bool + options *Options +} + +// MaintenanceManifestDocumentIterator is a maintenanceManifestDocument iterator +type MaintenanceManifestDocumentIterator interface { + Next(context.Context, int) (*pkg.MaintenanceManifestDocuments, error) + Continuation() string +} + +// MaintenanceManifestDocumentRawIterator is a maintenanceManifestDocument raw iterator +type MaintenanceManifestDocumentRawIterator interface { + MaintenanceManifestDocumentIterator + NextRaw(context.Context, int, interface{}) error +} + +// NewMaintenanceManifestDocumentClient returns a new maintenanceManifestDocument client +func NewMaintenanceManifestDocumentClient(collc CollectionClient, collid string) MaintenanceManifestDocumentClient { + return &maintenanceManifestDocumentClient{ + databaseClient: collc.(*collectionClient).databaseClient, + path: collc.(*collectionClient).path + "/colls/" + collid, + } +} + +func (c *maintenanceManifestDocumentClient) all(ctx context.Context, i MaintenanceManifestDocumentIterator) (*pkg.MaintenanceManifestDocuments, error) { + allmaintenanceManifestDocuments := &pkg.MaintenanceManifestDocuments{} + + for { + maintenanceManifestDocuments, err := i.Next(ctx, -1) + if err != nil { + return nil, err + } + if maintenanceManifestDocuments == nil { + break + } + + allmaintenanceManifestDocuments.Count += maintenanceManifestDocuments.Count + allmaintenanceManifestDocuments.ResourceID = maintenanceManifestDocuments.ResourceID + allmaintenanceManifestDocuments.MaintenanceManifestDocuments = append(allmaintenanceManifestDocuments.MaintenanceManifestDocuments, maintenanceManifestDocuments.MaintenanceManifestDocuments...) + } + + return allmaintenanceManifestDocuments, nil +} + +func (c *maintenanceManifestDocumentClient) Create(ctx context.Context, partitionkey string, newmaintenanceManifestDocument *pkg.MaintenanceManifestDocument, options *Options) (maintenanceManifestDocument *pkg.MaintenanceManifestDocument, err error) { + headers := http.Header{} + headers.Set("X-Ms-Documentdb-Partitionkey", `["`+partitionkey+`"]`) + + if options == nil { + options = &Options{} + } + options.NoETag = true + + err = c.setOptions(options, newmaintenanceManifestDocument, headers) + if err != nil { + return + } + + err = c.do(ctx, http.MethodPost, c.path+"/docs", "docs", c.path, http.StatusCreated, &newmaintenanceManifestDocument, &maintenanceManifestDocument, headers) + return +} + +func (c *maintenanceManifestDocumentClient) List(options *Options) MaintenanceManifestDocumentIterator { + continuation := "" + if options != nil { + continuation = options.Continuation + } + + return &maintenanceManifestDocumentListIterator{maintenanceManifestDocumentClient: c, options: options, continuation: continuation} +} + +func (c *maintenanceManifestDocumentClient) ListAll(ctx context.Context, options *Options) (*pkg.MaintenanceManifestDocuments, error) { + return c.all(ctx, c.List(options)) +} + +func (c *maintenanceManifestDocumentClient) Get(ctx context.Context, partitionkey, maintenanceManifestDocumentid string, options *Options) (maintenanceManifestDocument *pkg.MaintenanceManifestDocument, err error) { + headers := http.Header{} + headers.Set("X-Ms-Documentdb-Partitionkey", `["`+partitionkey+`"]`) + + err = c.setOptions(options, nil, headers) + if err != nil { + return + } + + err = c.do(ctx, http.MethodGet, c.path+"/docs/"+maintenanceManifestDocumentid, "docs", c.path+"/docs/"+maintenanceManifestDocumentid, http.StatusOK, nil, &maintenanceManifestDocument, headers) + return +} + +func (c *maintenanceManifestDocumentClient) Replace(ctx context.Context, partitionkey string, newmaintenanceManifestDocument *pkg.MaintenanceManifestDocument, options *Options) (maintenanceManifestDocument *pkg.MaintenanceManifestDocument, err error) { + headers := http.Header{} + headers.Set("X-Ms-Documentdb-Partitionkey", `["`+partitionkey+`"]`) + + err = c.setOptions(options, newmaintenanceManifestDocument, headers) + if err != nil { + return + } + + err = c.do(ctx, http.MethodPut, c.path+"/docs/"+newmaintenanceManifestDocument.ID, "docs", c.path+"/docs/"+newmaintenanceManifestDocument.ID, http.StatusOK, &newmaintenanceManifestDocument, &maintenanceManifestDocument, headers) + return +} + +func (c *maintenanceManifestDocumentClient) Delete(ctx context.Context, partitionkey string, maintenanceManifestDocument *pkg.MaintenanceManifestDocument, options *Options) (err error) { + headers := http.Header{} + headers.Set("X-Ms-Documentdb-Partitionkey", `["`+partitionkey+`"]`) + + err = c.setOptions(options, maintenanceManifestDocument, headers) + if err != nil { + return + } + + err = c.do(ctx, http.MethodDelete, c.path+"/docs/"+maintenanceManifestDocument.ID, "docs", c.path+"/docs/"+maintenanceManifestDocument.ID, http.StatusNoContent, nil, nil, headers) + return +} + +func (c *maintenanceManifestDocumentClient) Query(partitionkey string, query *Query, options *Options) MaintenanceManifestDocumentRawIterator { + continuation := "" + if options != nil { + continuation = options.Continuation + } + + return &maintenanceManifestDocumentQueryIterator{maintenanceManifestDocumentClient: c, partitionkey: partitionkey, query: query, options: options, continuation: continuation} +} + +func (c *maintenanceManifestDocumentClient) QueryAll(ctx context.Context, partitionkey string, query *Query, options *Options) (*pkg.MaintenanceManifestDocuments, error) { + return c.all(ctx, c.Query(partitionkey, query, options)) +} + +func (c *maintenanceManifestDocumentClient) ChangeFeed(options *Options) MaintenanceManifestDocumentIterator { + continuation := "" + if options != nil { + continuation = options.Continuation + } + + return &maintenanceManifestDocumentChangeFeedIterator{maintenanceManifestDocumentClient: c, options: options, continuation: continuation} +} + +func (c *maintenanceManifestDocumentClient) setOptions(options *Options, maintenanceManifestDocument *pkg.MaintenanceManifestDocument, headers http.Header) error { + if options == nil { + return nil + } + + if maintenanceManifestDocument != nil && !options.NoETag { + if maintenanceManifestDocument.ETag == "" { + return ErrETagRequired + } + headers.Set("If-Match", maintenanceManifestDocument.ETag) + } + if len(options.PreTriggers) > 0 { + headers.Set("X-Ms-Documentdb-Pre-Trigger-Include", strings.Join(options.PreTriggers, ",")) + } + if len(options.PostTriggers) > 0 { + headers.Set("X-Ms-Documentdb-Post-Trigger-Include", strings.Join(options.PostTriggers, ",")) + } + if len(options.PartitionKeyRangeID) > 0 { + headers.Set("X-Ms-Documentdb-PartitionKeyRangeID", options.PartitionKeyRangeID) + } + + return nil +} + +func (i *maintenanceManifestDocumentChangeFeedIterator) Next(ctx context.Context, maxItemCount int) (maintenanceManifestDocuments *pkg.MaintenanceManifestDocuments, err error) { + headers := http.Header{} + headers.Set("A-IM", "Incremental feed") + + headers.Set("X-Ms-Max-Item-Count", strconv.Itoa(maxItemCount)) + if i.continuation != "" { + headers.Set("If-None-Match", i.continuation) + } + + err = i.setOptions(i.options, nil, headers) + if err != nil { + return + } + + err = i.do(ctx, http.MethodGet, i.path+"/docs", "docs", i.path, http.StatusOK, nil, &maintenanceManifestDocuments, headers) + if IsErrorStatusCode(err, http.StatusNotModified) { + err = nil + } + if err != nil { + return + } + + i.continuation = headers.Get("Etag") + + return +} + +func (i *maintenanceManifestDocumentChangeFeedIterator) Continuation() string { + return i.continuation +} + +func (i *maintenanceManifestDocumentListIterator) Next(ctx context.Context, maxItemCount int) (maintenanceManifestDocuments *pkg.MaintenanceManifestDocuments, err error) { + if i.done { + return + } + + headers := http.Header{} + headers.Set("X-Ms-Max-Item-Count", strconv.Itoa(maxItemCount)) + if i.continuation != "" { + headers.Set("X-Ms-Continuation", i.continuation) + } + + err = i.setOptions(i.options, nil, headers) + if err != nil { + return + } + + err = i.do(ctx, http.MethodGet, i.path+"/docs", "docs", i.path, http.StatusOK, nil, &maintenanceManifestDocuments, headers) + if err != nil { + return + } + + i.continuation = headers.Get("X-Ms-Continuation") + i.done = i.continuation == "" + + return +} + +func (i *maintenanceManifestDocumentListIterator) Continuation() string { + return i.continuation +} + +func (i *maintenanceManifestDocumentQueryIterator) Next(ctx context.Context, maxItemCount int) (maintenanceManifestDocuments *pkg.MaintenanceManifestDocuments, err error) { + err = i.NextRaw(ctx, maxItemCount, &maintenanceManifestDocuments) + return +} + +func (i *maintenanceManifestDocumentQueryIterator) NextRaw(ctx context.Context, maxItemCount int, raw interface{}) (err error) { + if i.done { + return + } + + headers := http.Header{} + headers.Set("X-Ms-Max-Item-Count", strconv.Itoa(maxItemCount)) + headers.Set("X-Ms-Documentdb-Isquery", "True") + headers.Set("Content-Type", "application/query+json") + if i.partitionkey != "" { + headers.Set("X-Ms-Documentdb-Partitionkey", `["`+i.partitionkey+`"]`) + } else { + headers.Set("X-Ms-Documentdb-Query-Enablecrosspartition", "True") + } + if i.continuation != "" { + headers.Set("X-Ms-Continuation", i.continuation) + } + + err = i.setOptions(i.options, nil, headers) + if err != nil { + return + } + + err = i.do(ctx, http.MethodPost, i.path+"/docs", "docs", i.path, http.StatusOK, &i.query, &raw, headers) + if err != nil { + return + } + + i.continuation = headers.Get("X-Ms-Continuation") + i.done = i.continuation == "" + + return +} + +func (i *maintenanceManifestDocumentQueryIterator) Continuation() string { + return i.continuation +} diff --git a/pkg/database/cosmosdb/zz_generated_maintenancemanifestdocument_fake.go b/pkg/database/cosmosdb/zz_generated_maintenancemanifestdocument_fake.go new file mode 100644 index 00000000000..b6a6a20413f --- /dev/null +++ b/pkg/database/cosmosdb/zz_generated_maintenancemanifestdocument_fake.go @@ -0,0 +1,361 @@ +// Code generated by github.com/jewzaam/go-cosmosdb, DO NOT EDIT. + +package cosmosdb + +import ( + "context" + "fmt" + "net/http" + "sync" + + "github.com/ugorji/go/codec" + + pkg "github.com/Azure/ARO-RP/pkg/api" +) + +type fakeMaintenanceManifestDocumentTriggerHandler func(context.Context, *pkg.MaintenanceManifestDocument) error +type fakeMaintenanceManifestDocumentQueryHandler func(MaintenanceManifestDocumentClient, *Query, *Options) MaintenanceManifestDocumentRawIterator + +var _ MaintenanceManifestDocumentClient = &FakeMaintenanceManifestDocumentClient{} + +// NewFakeMaintenanceManifestDocumentClient returns a FakeMaintenanceManifestDocumentClient +func NewFakeMaintenanceManifestDocumentClient(h *codec.JsonHandle) *FakeMaintenanceManifestDocumentClient { + return &FakeMaintenanceManifestDocumentClient{ + jsonHandle: h, + maintenanceManifestDocuments: make(map[string]*pkg.MaintenanceManifestDocument), + triggerHandlers: make(map[string]fakeMaintenanceManifestDocumentTriggerHandler), + queryHandlers: make(map[string]fakeMaintenanceManifestDocumentQueryHandler), + } +} + +// FakeMaintenanceManifestDocumentClient is a FakeMaintenanceManifestDocumentClient +type FakeMaintenanceManifestDocumentClient struct { + lock sync.RWMutex + jsonHandle *codec.JsonHandle + maintenanceManifestDocuments map[string]*pkg.MaintenanceManifestDocument + triggerHandlers map[string]fakeMaintenanceManifestDocumentTriggerHandler + queryHandlers map[string]fakeMaintenanceManifestDocumentQueryHandler + sorter func([]*pkg.MaintenanceManifestDocument) + etag int + + // returns true if documents conflict + conflictChecker func(*pkg.MaintenanceManifestDocument, *pkg.MaintenanceManifestDocument) bool + + // err, if not nil, is an error to return when attempting to communicate + // with this Client + err error +} + +// SetError sets or unsets an error that will be returned on any +// FakeMaintenanceManifestDocumentClient method invocation +func (c *FakeMaintenanceManifestDocumentClient) SetError(err error) { + c.lock.Lock() + defer c.lock.Unlock() + + c.err = err +} + +// SetSorter sets or unsets a sorter function which will be used to sort values +// returned by List() for test stability +func (c *FakeMaintenanceManifestDocumentClient) SetSorter(sorter func([]*pkg.MaintenanceManifestDocument)) { + c.lock.Lock() + defer c.lock.Unlock() + + c.sorter = sorter +} + +// SetConflictChecker sets or unsets a function which can be used to validate +// additional unique keys in a MaintenanceManifestDocument +func (c *FakeMaintenanceManifestDocumentClient) SetConflictChecker(conflictChecker func(*pkg.MaintenanceManifestDocument, *pkg.MaintenanceManifestDocument) bool) { + c.lock.Lock() + defer c.lock.Unlock() + + c.conflictChecker = conflictChecker +} + +// SetTriggerHandler sets or unsets a trigger handler +func (c *FakeMaintenanceManifestDocumentClient) SetTriggerHandler(triggerName string, trigger fakeMaintenanceManifestDocumentTriggerHandler) { + c.lock.Lock() + defer c.lock.Unlock() + + c.triggerHandlers[triggerName] = trigger +} + +// SetQueryHandler sets or unsets a query handler +func (c *FakeMaintenanceManifestDocumentClient) SetQueryHandler(queryName string, query fakeMaintenanceManifestDocumentQueryHandler) { + c.lock.Lock() + defer c.lock.Unlock() + + c.queryHandlers[queryName] = query +} + +func (c *FakeMaintenanceManifestDocumentClient) deepCopy(maintenanceManifestDocument *pkg.MaintenanceManifestDocument) (*pkg.MaintenanceManifestDocument, error) { + var b []byte + err := codec.NewEncoderBytes(&b, c.jsonHandle).Encode(maintenanceManifestDocument) + if err != nil { + return nil, err + } + + maintenanceManifestDocument = nil + err = codec.NewDecoderBytes(b, c.jsonHandle).Decode(&maintenanceManifestDocument) + if err != nil { + return nil, err + } + + return maintenanceManifestDocument, nil +} + +func (c *FakeMaintenanceManifestDocumentClient) apply(ctx context.Context, partitionkey string, maintenanceManifestDocument *pkg.MaintenanceManifestDocument, options *Options, isCreate bool) (*pkg.MaintenanceManifestDocument, error) { + c.lock.Lock() + defer c.lock.Unlock() + + if c.err != nil { + return nil, c.err + } + + maintenanceManifestDocument, err := c.deepCopy(maintenanceManifestDocument) // copy now because pretriggers can mutate maintenanceManifestDocument + if err != nil { + return nil, err + } + + if options != nil { + err := c.processPreTriggers(ctx, maintenanceManifestDocument, options) + if err != nil { + return nil, err + } + } + + existingMaintenanceManifestDocument, exists := c.maintenanceManifestDocuments[maintenanceManifestDocument.ID] + if isCreate && exists { + return nil, &Error{ + StatusCode: http.StatusConflict, + Message: "Entity with the specified id already exists in the system", + } + } + if !isCreate { + if !exists { + return nil, &Error{StatusCode: http.StatusNotFound} + } + + if maintenanceManifestDocument.ETag != existingMaintenanceManifestDocument.ETag { + return nil, &Error{StatusCode: http.StatusPreconditionFailed} + } + } + + if c.conflictChecker != nil { + for _, maintenanceManifestDocumentToCheck := range c.maintenanceManifestDocuments { + if c.conflictChecker(maintenanceManifestDocumentToCheck, maintenanceManifestDocument) { + return nil, &Error{ + StatusCode: http.StatusConflict, + Message: "Entity with the specified id already exists in the system", + } + } + } + } + + maintenanceManifestDocument.ETag = fmt.Sprint(c.etag) + c.etag++ + + c.maintenanceManifestDocuments[maintenanceManifestDocument.ID] = maintenanceManifestDocument + + return c.deepCopy(maintenanceManifestDocument) +} + +// Create creates a MaintenanceManifestDocument in the database +func (c *FakeMaintenanceManifestDocumentClient) Create(ctx context.Context, partitionkey string, maintenanceManifestDocument *pkg.MaintenanceManifestDocument, options *Options) (*pkg.MaintenanceManifestDocument, error) { + return c.apply(ctx, partitionkey, maintenanceManifestDocument, options, true) +} + +// Replace replaces a MaintenanceManifestDocument in the database +func (c *FakeMaintenanceManifestDocumentClient) Replace(ctx context.Context, partitionkey string, maintenanceManifestDocument *pkg.MaintenanceManifestDocument, options *Options) (*pkg.MaintenanceManifestDocument, error) { + return c.apply(ctx, partitionkey, maintenanceManifestDocument, options, false) +} + +// List returns a MaintenanceManifestDocumentIterator to list all MaintenanceManifestDocuments in the database +func (c *FakeMaintenanceManifestDocumentClient) List(*Options) MaintenanceManifestDocumentIterator { + c.lock.RLock() + defer c.lock.RUnlock() + + if c.err != nil { + return NewFakeMaintenanceManifestDocumentErroringRawIterator(c.err) + } + + maintenanceManifestDocuments := make([]*pkg.MaintenanceManifestDocument, 0, len(c.maintenanceManifestDocuments)) + for _, maintenanceManifestDocument := range c.maintenanceManifestDocuments { + maintenanceManifestDocument, err := c.deepCopy(maintenanceManifestDocument) + if err != nil { + return NewFakeMaintenanceManifestDocumentErroringRawIterator(err) + } + maintenanceManifestDocuments = append(maintenanceManifestDocuments, maintenanceManifestDocument) + } + + if c.sorter != nil { + c.sorter(maintenanceManifestDocuments) + } + + return NewFakeMaintenanceManifestDocumentIterator(maintenanceManifestDocuments, 0) +} + +// ListAll lists all MaintenanceManifestDocuments in the database +func (c *FakeMaintenanceManifestDocumentClient) ListAll(ctx context.Context, options *Options) (*pkg.MaintenanceManifestDocuments, error) { + iter := c.List(options) + return iter.Next(ctx, -1) +} + +// Get gets a MaintenanceManifestDocument from the database +func (c *FakeMaintenanceManifestDocumentClient) Get(ctx context.Context, partitionkey string, id string, options *Options) (*pkg.MaintenanceManifestDocument, error) { + c.lock.RLock() + defer c.lock.RUnlock() + + if c.err != nil { + return nil, c.err + } + + maintenanceManifestDocument, exists := c.maintenanceManifestDocuments[id] + if !exists { + return nil, &Error{StatusCode: http.StatusNotFound} + } + + return c.deepCopy(maintenanceManifestDocument) +} + +// Delete deletes a MaintenanceManifestDocument from the database +func (c *FakeMaintenanceManifestDocumentClient) Delete(ctx context.Context, partitionKey string, maintenanceManifestDocument *pkg.MaintenanceManifestDocument, options *Options) error { + c.lock.Lock() + defer c.lock.Unlock() + + if c.err != nil { + return c.err + } + + _, exists := c.maintenanceManifestDocuments[maintenanceManifestDocument.ID] + if !exists { + return &Error{StatusCode: http.StatusNotFound} + } + + delete(c.maintenanceManifestDocuments, maintenanceManifestDocument.ID) + return nil +} + +// ChangeFeed is unimplemented +func (c *FakeMaintenanceManifestDocumentClient) ChangeFeed(*Options) MaintenanceManifestDocumentIterator { + c.lock.RLock() + defer c.lock.RUnlock() + + if c.err != nil { + return NewFakeMaintenanceManifestDocumentErroringRawIterator(c.err) + } + + return NewFakeMaintenanceManifestDocumentErroringRawIterator(ErrNotImplemented) +} + +func (c *FakeMaintenanceManifestDocumentClient) processPreTriggers(ctx context.Context, maintenanceManifestDocument *pkg.MaintenanceManifestDocument, options *Options) error { + for _, triggerName := range options.PreTriggers { + if triggerHandler := c.triggerHandlers[triggerName]; triggerHandler != nil { + c.lock.Unlock() + err := triggerHandler(ctx, maintenanceManifestDocument) + c.lock.Lock() + if err != nil { + return err + } + } else { + return ErrNotImplemented + } + } + + return nil +} + +// Query calls a query handler to implement database querying +func (c *FakeMaintenanceManifestDocumentClient) Query(name string, query *Query, options *Options) MaintenanceManifestDocumentRawIterator { + c.lock.RLock() + defer c.lock.RUnlock() + + if c.err != nil { + return NewFakeMaintenanceManifestDocumentErroringRawIterator(c.err) + } + + if queryHandler := c.queryHandlers[query.Query]; queryHandler != nil { + c.lock.RUnlock() + i := queryHandler(c, query, options) + c.lock.RLock() + return i + } + + return NewFakeMaintenanceManifestDocumentErroringRawIterator(ErrNotImplemented) +} + +// QueryAll calls a query handler to implement database querying +func (c *FakeMaintenanceManifestDocumentClient) QueryAll(ctx context.Context, partitionkey string, query *Query, options *Options) (*pkg.MaintenanceManifestDocuments, error) { + iter := c.Query("", query, options) + return iter.Next(ctx, -1) +} + +func NewFakeMaintenanceManifestDocumentIterator(maintenanceManifestDocuments []*pkg.MaintenanceManifestDocument, continuation int) MaintenanceManifestDocumentRawIterator { + return &fakeMaintenanceManifestDocumentIterator{maintenanceManifestDocuments: maintenanceManifestDocuments, continuation: continuation} +} + +type fakeMaintenanceManifestDocumentIterator struct { + maintenanceManifestDocuments []*pkg.MaintenanceManifestDocument + continuation int + done bool +} + +func (i *fakeMaintenanceManifestDocumentIterator) NextRaw(ctx context.Context, maxItemCount int, out interface{}) error { + return ErrNotImplemented +} + +func (i *fakeMaintenanceManifestDocumentIterator) Next(ctx context.Context, maxItemCount int) (*pkg.MaintenanceManifestDocuments, error) { + if i.done { + return nil, nil + } + + var maintenanceManifestDocuments []*pkg.MaintenanceManifestDocument + if maxItemCount == -1 { + maintenanceManifestDocuments = i.maintenanceManifestDocuments[i.continuation:] + i.continuation = len(i.maintenanceManifestDocuments) + i.done = true + } else { + max := i.continuation + maxItemCount + if max > len(i.maintenanceManifestDocuments) { + max = len(i.maintenanceManifestDocuments) + } + maintenanceManifestDocuments = i.maintenanceManifestDocuments[i.continuation:max] + i.continuation += max + i.done = i.Continuation() == "" + } + + return &pkg.MaintenanceManifestDocuments{ + MaintenanceManifestDocuments: maintenanceManifestDocuments, + Count: len(maintenanceManifestDocuments), + }, nil +} + +func (i *fakeMaintenanceManifestDocumentIterator) Continuation() string { + if i.continuation >= len(i.maintenanceManifestDocuments) { + return "" + } + return fmt.Sprintf("%d", i.continuation) +} + +// NewFakeMaintenanceManifestDocumentErroringRawIterator returns a MaintenanceManifestDocumentRawIterator which +// whose methods return the given error +func NewFakeMaintenanceManifestDocumentErroringRawIterator(err error) MaintenanceManifestDocumentRawIterator { + return &fakeMaintenanceManifestDocumentErroringRawIterator{err: err} +} + +type fakeMaintenanceManifestDocumentErroringRawIterator struct { + err error +} + +func (i *fakeMaintenanceManifestDocumentErroringRawIterator) Next(ctx context.Context, maxItemCount int) (*pkg.MaintenanceManifestDocuments, error) { + return nil, i.err +} + +func (i *fakeMaintenanceManifestDocumentErroringRawIterator) NextRaw(context.Context, int, interface{}) error { + return i.err +} + +func (i *fakeMaintenanceManifestDocumentErroringRawIterator) Continuation() string { + return "" +} diff --git a/pkg/database/database.go b/pkg/database/database.go index 36f8cb93652..b63cce2b53a 100644 --- a/pkg/database/database.go +++ b/pkg/database/database.go @@ -37,6 +37,7 @@ const ( collPlatformWorkloadIdentityRoleSet = "PlatformWorkloadIdentityRoleSets" collPortal = "Portal" collSubscriptions = "Subscriptions" + collMaintenanceManifests = "MaintenanceManifests" ) func NewDatabaseClient(log *logrus.Entry, _env env.Core, authorizer cosmosdb.Authorizer, m metrics.Emitter, aead encryption.AEAD, databaseAccountName string) (cosmosdb.DatabaseClient, error) { diff --git a/pkg/database/dbgroup.go b/pkg/database/dbgroup.go index fa389d78aaa..1b4b8bebc25 100644 --- a/pkg/database/dbgroup.go +++ b/pkg/database/dbgroup.go @@ -37,6 +37,10 @@ type DatabaseGroupWithPortal interface { Portal() (Portal, error) } +type DatabaseGroupWithMaintenanceManifests interface { + MaintenanceManifests() (MaintenanceManifests, error) +} + type DatabaseGroup interface { DatabaseGroupWithOpenShiftClusters DatabaseGroupWithSubscriptions @@ -46,6 +50,7 @@ type DatabaseGroup interface { DatabaseGroupWithAsyncOperations DatabaseGroupWithBilling DatabaseGroupWithPortal + DatabaseGroupWithMaintenanceManifests WithOpenShiftClusters(db OpenShiftClusters) DatabaseGroup WithSubscriptions(db Subscriptions) DatabaseGroup @@ -55,6 +60,7 @@ type DatabaseGroup interface { WithAsyncOperations(db AsyncOperations) DatabaseGroup WithBilling(db Billing) DatabaseGroup WithPortal(db Portal) DatabaseGroup + WithMaintenanceManifests(db MaintenanceManifests) DatabaseGroup } type dbGroup struct { @@ -66,6 +72,7 @@ type dbGroup struct { asyncOperations AsyncOperations billing Billing portal Portal + maintenanceManifests MaintenanceManifests } func (d *dbGroup) OpenShiftClusters() (OpenShiftClusters, error) { @@ -164,6 +171,18 @@ func (d *dbGroup) WithPortal(db Portal) DatabaseGroup { return d } +func (d *dbGroup) MaintenanceManifests() (MaintenanceManifests, error) { + if d.maintenanceManifests == nil { + return nil, errors.New("no MaintenanceManifests defined") + } + return d.maintenanceManifests, nil +} + +func (d *dbGroup) WithMaintenanceManifests(db MaintenanceManifests) DatabaseGroup { + d.maintenanceManifests = db + return d +} + func NewDBGroup() DatabaseGroup { return &dbGroup{} } diff --git a/pkg/database/mimo.go b/pkg/database/mimo.go new file mode 100644 index 00000000000..387342f05c1 --- /dev/null +++ b/pkg/database/mimo.go @@ -0,0 +1,240 @@ +package database + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "fmt" + "net/http" + "strings" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/database/cosmosdb" + "github.com/Azure/ARO-RP/pkg/util/uuid" +) + +const ( + MaintenanceManifestDequeueQueryForCluster = `SELECT * FROM MaintenanceManifests doc WHERE doc.maintenanceManifest.state IN ("Pending") AND doc.clusterResourceID = @clusterResourceID` + MaintenanceManifestQueryForCluster = `SELECT * FROM MaintenanceManifests doc WHERE doc.clusterResourceID = @clusterResourceID` + MaintenanceManifestQueueLengthQuery = `SELECT VALUE COUNT(1) FROM MaintenanceManifests doc WHERE doc.maintenanceManifest.state IN ("Pending") AND (doc.leaseExpires ?? 0) < GetCurrentTimestamp() / 1000` +) + +type MaintenanceManifestDocumentMutator func(*api.MaintenanceManifestDocument) error + +type maintenanceManifests struct { + c cosmosdb.MaintenanceManifestDocumentClient + collc cosmosdb.CollectionClient + uuid string + uuidGenerator uuid.Generator +} + +type MaintenanceManifests interface { + Create(context.Context, *api.MaintenanceManifestDocument) (*api.MaintenanceManifestDocument, error) + GetByClusterResourceID(ctx context.Context, clusterResourceID string, continuation string) (cosmosdb.MaintenanceManifestDocumentIterator, error) + GetQueuedByClusterResourceID(ctx context.Context, clusterResourceID string, continuation string) (cosmosdb.MaintenanceManifestDocumentIterator, error) + Patch(context.Context, string, string, MaintenanceManifestDocumentMutator) (*api.MaintenanceManifestDocument, error) + PatchWithLease(context.Context, string, string, MaintenanceManifestDocumentMutator) (*api.MaintenanceManifestDocument, error) + Lease(ctx context.Context, clusterResourceID string, id string) (*api.MaintenanceManifestDocument, error) + EndLease(context.Context, string, string, api.MaintenanceManifestState, *string) (*api.MaintenanceManifestDocument, error) + Get(context.Context, string, string) (*api.MaintenanceManifestDocument, error) + Delete(context.Context, string, string) error + + NewUUID() string +} + +func NewMaintenanceManifests(ctx context.Context, dbc cosmosdb.DatabaseClient, dbName string) (MaintenanceManifests, error) { + collc := cosmosdb.NewCollectionClient(dbc, dbName) + + documentClient := cosmosdb.NewMaintenanceManifestDocumentClient(collc, collMaintenanceManifests) + return NewMaintenanceManifestsWithProvidedClient(documentClient, collc, uuid.DefaultGenerator.Generate(), uuid.DefaultGenerator), nil +} + +func NewMaintenanceManifestsWithProvidedClient(client cosmosdb.MaintenanceManifestDocumentClient, collectionClient cosmosdb.CollectionClient, uuid string, uuidGenerator uuid.Generator) MaintenanceManifests { + return &maintenanceManifests{ + c: client, + uuid: uuid, + collc: collectionClient, + uuidGenerator: uuidGenerator, + } +} + +func (c *maintenanceManifests) NewUUID() string { + return c.uuidGenerator.Generate() +} + +func (c *maintenanceManifests) Create(ctx context.Context, doc *api.MaintenanceManifestDocument) (*api.MaintenanceManifestDocument, error) { + if doc.ID != strings.ToLower(doc.ID) { + return nil, fmt.Errorf("id %q is not lower case", doc.ID) + } + + doc, err := c.c.Create(ctx, doc.ClusterResourceID, doc, nil) + + if err, ok := err.(*cosmosdb.Error); ok && err.StatusCode == http.StatusConflict { + err.StatusCode = http.StatusPreconditionFailed + } + + return doc, err +} + +func (c *maintenanceManifests) Get(ctx context.Context, clusterResourceID string, id string) (*api.MaintenanceManifestDocument, error) { + if id != strings.ToLower(id) { + return nil, fmt.Errorf("id %q is not lower case", id) + } + + return c.c.Get(ctx, clusterResourceID, id, nil) +} + +// QueueLength returns maintenanceManifests un-queued document count. +// If error occurs, 0 is returned with error message +func (c *maintenanceManifests) QueueLength(ctx context.Context, collid string) (int, error) { + partitions, err := c.collc.PartitionKeyRanges(ctx, collid) + if err != nil { + return 0, err + } + + var countTotal int + for _, r := range partitions.PartitionKeyRanges { + result := c.c.Query("", &cosmosdb.Query{ + Query: MaintenanceManifestQueueLengthQuery, + }, &cosmosdb.Options{ + PartitionKeyRangeID: r.ID, + }) + // because we aggregate count we don't expect pagination in this query result, + // so we gonna call Next() only once per partition. + var data struct { + api.MissingFields + Document []int `json:"Documents,omitempty"` + } + err := result.NextRaw(ctx, -1, &data) + if err != nil { + return 0, err + } + + countTotal = countTotal + data.Document[0] + } + return countTotal, nil +} + +func (c *maintenanceManifests) Patch(ctx context.Context, clusterResourceID string, id string, f MaintenanceManifestDocumentMutator) (*api.MaintenanceManifestDocument, error) { + return c.patch(ctx, clusterResourceID, id, f, nil) +} + +func (c *maintenanceManifests) patch(ctx context.Context, clusterResourceID string, id string, f MaintenanceManifestDocumentMutator, options *cosmosdb.Options) (*api.MaintenanceManifestDocument, error) { + var doc *api.MaintenanceManifestDocument + + err := cosmosdb.RetryOnPreconditionFailed(func() (err error) { + doc, err = c.Get(ctx, clusterResourceID, id) + if err != nil { + return + } + + err = f(doc) + if err != nil { + return + } + + doc, err = c.update(ctx, doc, options) + return + }) + + return doc, err +} + +// PatchWithLease performs a patch on the cluster that verifies the lease is +// being held by this client before applying. +func (c *maintenanceManifests) PatchWithLease(ctx context.Context, clusterResourceID string, id string, f MaintenanceManifestDocumentMutator) (*api.MaintenanceManifestDocument, error) { + return c.patchWithLease(ctx, clusterResourceID, id, f, &cosmosdb.Options{PreTriggers: []string{"renewLease"}}) +} + +func (c *maintenanceManifests) patchWithLease(ctx context.Context, clusterResourceID string, id string, f MaintenanceManifestDocumentMutator, options *cosmosdb.Options) (*api.MaintenanceManifestDocument, error) { + return c.patch(ctx, clusterResourceID, id, func(doc *api.MaintenanceManifestDocument) error { + if doc.LeaseOwner != c.uuid { + return fmt.Errorf("lost lease") + } + + return f(doc) + }, options) +} + +func (c *maintenanceManifests) update(ctx context.Context, doc *api.MaintenanceManifestDocument, options *cosmosdb.Options) (*api.MaintenanceManifestDocument, error) { + if doc.ID != strings.ToLower(doc.ID) { + return nil, fmt.Errorf("id %q is not lower case", doc.ID) + } + + return c.c.Replace(ctx, doc.ClusterResourceID, doc, options) +} + +func (c *maintenanceManifests) ChangeFeed() cosmosdb.MaintenanceManifestDocumentIterator { + return c.c.ChangeFeed(nil) +} + +func (c *maintenanceManifests) GetByClusterResourceID(ctx context.Context, clusterResourceID string, continuation string) (cosmosdb.MaintenanceManifestDocumentIterator, error) { + if clusterResourceID != strings.ToLower(clusterResourceID) { + return nil, fmt.Errorf("clusterResourceID %q is not lower case", clusterResourceID) + } + + return c.c.Query("", &cosmosdb.Query{ + Query: MaintenanceManifestQueryForCluster, + Parameters: []cosmosdb.Parameter{ + { + Name: "@clusterResourceID", + Value: clusterResourceID, + }, + }, + }, &cosmosdb.Options{Continuation: continuation}), nil +} + +func (c *maintenanceManifests) GetQueuedByClusterResourceID(ctx context.Context, clusterResourceID string, continuation string) (cosmosdb.MaintenanceManifestDocumentIterator, error) { + if clusterResourceID != strings.ToLower(clusterResourceID) { + return nil, fmt.Errorf("clusterResourceID %q is not lower case", clusterResourceID) + } + + return c.c.Query("", &cosmosdb.Query{ + Query: MaintenanceManifestDequeueQueryForCluster, + Parameters: []cosmosdb.Parameter{ + { + Name: "@clusterResourceID", + Value: clusterResourceID, + }, + }, + }, &cosmosdb.Options{Continuation: continuation}), nil +} + +func (c *maintenanceManifests) EndLease(ctx context.Context, clusterResourceID string, id string, provisioningState api.MaintenanceManifestState, statusString *string) (*api.MaintenanceManifestDocument, error) { + return c.patchWithLease(ctx, clusterResourceID, id, func(doc *api.MaintenanceManifestDocument) error { + doc.MaintenanceManifest.State = provisioningState + if statusString != nil { + doc.MaintenanceManifest.StatusText = *statusString + } + + doc.LeaseOwner = "" + doc.LeaseExpires = 0 + + if provisioningState != api.MaintenanceManifestStateFailed { + doc.Dequeues = 0 + } + return nil + }, nil) +} + +// Lease performs the initial lease/dequeue on the document. +func (c *maintenanceManifests) Lease(ctx context.Context, clusterResourceID string, id string) (*api.MaintenanceManifestDocument, error) { + if clusterResourceID != strings.ToLower(clusterResourceID) { + return nil, fmt.Errorf("clusterID %q is not lower case", clusterResourceID) + } + + return c.patch(ctx, clusterResourceID, id, func(doc *api.MaintenanceManifestDocument) error { + doc.LeaseOwner = c.uuid + doc.Dequeues++ + return nil + }, &cosmosdb.Options{PreTriggers: []string{"renewLease"}}) +} + +func (c *maintenanceManifests) Delete(ctx context.Context, clusterResourceID string, id string) error { + if clusterResourceID != strings.ToLower(clusterResourceID) { + return fmt.Errorf("clusterID %q is not lower case", clusterResourceID) + } + + return c.c.Delete(ctx, clusterResourceID, &api.MaintenanceManifestDocument{ID: id}, nil) +} diff --git a/pkg/database/openshiftclusters.go b/pkg/database/openshiftclusters.go index a41536371fe..409e4c4f0a0 100644 --- a/pkg/database/openshiftclusters.go +++ b/pkg/database/openshiftclusters.go @@ -17,12 +17,13 @@ import ( ) const ( - OpenShiftClustersDequeueQuery = `SELECT * FROM OpenShiftClusters doc WHERE doc.openShiftCluster.properties.provisioningState IN ("Creating", "Deleting", "Updating", "AdminUpdating") AND (doc.leaseExpires ?? 0) < GetCurrentTimestamp() / 1000` - OpenShiftClustersQueueLengthQuery = `SELECT VALUE COUNT(1) FROM OpenShiftClusters doc WHERE doc.openShiftCluster.properties.provisioningState IN ("Creating", "Deleting", "Updating", "AdminUpdating") AND (doc.leaseExpires ?? 0) < GetCurrentTimestamp() / 1000` - OpenShiftClustersGetQuery = `SELECT * FROM OpenShiftClusters doc WHERE doc.key = @key` - OpenshiftClustersPrefixQuery = `SELECT * FROM OpenShiftClusters doc WHERE STARTSWITH(doc.key, @prefix)` - OpenshiftClustersClientIdQuery = `SELECT * FROM OpenShiftClusters doc WHERE doc.clientIdKey = @clientID` - OpenshiftClustersResourceGroupQuery = `SELECT * FROM OpenShiftClusters doc WHERE doc.clusterResourceGroupIdKey = @resourceGroupID` + OpenShiftClustersDequeueQuery = `SELECT * FROM OpenShiftClusters doc WHERE doc.openShiftCluster.properties.provisioningState IN ("Creating", "Deleting", "Updating", "AdminUpdating") AND (doc.leaseExpires ?? 0) < GetCurrentTimestamp() / 1000` + OpenShiftClustersQueueLengthQuery = `SELECT VALUE COUNT(1) FROM OpenShiftClusters doc WHERE doc.openShiftCluster.properties.provisioningState IN ("Creating", "Deleting", "Updating", "AdminUpdating") AND (doc.leaseExpires ?? 0) < GetCurrentTimestamp() / 1000` + OpenShiftClustersGetQuery = `SELECT * FROM OpenShiftClusters doc WHERE doc.key = @key` + OpenshiftClustersPrefixQuery = `SELECT * FROM OpenShiftClusters doc WHERE STARTSWITH(doc.key, @prefix)` + OpenshiftClustersClientIdQuery = `SELECT * FROM OpenShiftClusters doc WHERE doc.clientIdKey = @clientID` + OpenshiftClustersResourceGroupQuery = `SELECT * FROM OpenShiftClusters doc WHERE doc.clusterResourceGroupIdKey = @resourceGroupID` + OpenshiftClustersClusterResourceIDOnlyQuery = `SELECT doc.id, doc.key FROM OpenShiftClusters doc WHERE doc.openShiftCluster.properties.provisioningState NOT IN ("Creating", "Deleting")` ) type OpenShiftClusterDocumentMutator func(*api.OpenShiftClusterDocument) error @@ -52,6 +53,8 @@ type OpenShiftClusters interface { EndLease(context.Context, string, api.ProvisioningState, api.ProvisioningState, *string) (*api.OpenShiftClusterDocument, error) GetByClientID(ctx context.Context, partitionKey, clientID string) (*api.OpenShiftClusterDocuments, error) GetByClusterResourceGroupID(ctx context.Context, partitionKey, resourceGroupID string) (*api.OpenShiftClusterDocuments, error) + GetAllResourceIDs(ctx context.Context, continuation string) (cosmosdb.OpenShiftClusterDocumentIterator, error) + DoDequeue(ctx context.Context, doc *api.OpenShiftClusterDocument) (*api.OpenShiftClusterDocument, error) NewUUID() string } @@ -266,9 +269,7 @@ func (c *openShiftClusters) Dequeue(ctx context.Context) (*api.OpenShiftClusterD } for _, doc := range docs.OpenShiftClusterDocuments { - doc.LeaseOwner = c.uuid - doc.Dequeues++ - doc, err = c.update(ctx, doc, &cosmosdb.Options{PreTriggers: []string{"renewLease"}}) + doc, err = c.DoDequeue(ctx, doc) if cosmosdb.IsErrorStatusCode(err, http.StatusPreconditionFailed) { // someone else got there first continue } @@ -277,6 +278,12 @@ func (c *openShiftClusters) Dequeue(ctx context.Context) (*api.OpenShiftClusterD } } +func (c *openShiftClusters) DoDequeue(ctx context.Context, doc *api.OpenShiftClusterDocument) (*api.OpenShiftClusterDocument, error) { + doc.LeaseOwner = c.uuid + doc.Dequeues++ + return c.update(ctx, doc, &cosmosdb.Options{PreTriggers: []string{"renewLease"}}) +} + func (c *openShiftClusters) Lease(ctx context.Context, key string) (*api.OpenShiftClusterDocument, error) { return c.patchWithLease(ctx, key, func(doc *api.OpenShiftClusterDocument) error { return nil @@ -347,3 +354,13 @@ func (c *openShiftClusters) GetByClusterResourceGroupID(ctx context.Context, par } return docs, nil } + +func (c *openShiftClusters) GetAllResourceIDs(ctx context.Context, continuation string) (cosmosdb.OpenShiftClusterDocumentIterator, error) { + return c.c.Query( + "", + &cosmosdb.Query{ + Query: OpenshiftClustersClusterResourceIDOnlyQuery, + }, + &cosmosdb.Options{Continuation: continuation}, + ), nil +} diff --git a/pkg/deploy/assets/databases-development.json b/pkg/deploy/assets/databases-development.json index 191670b7f6f..dd4daa9375c 100644 --- a/pkg/deploy/assets/databases-development.json +++ b/pkg/deploy/assets/databases-development.json @@ -368,6 +368,46 @@ } }, "type": "Microsoft.DocumentDB/databaseAccounts/sqlDatabases/containers/triggers" + }, + { + "apiVersion": "2023-04-15", + "dependsOn": [ + "[resourceId('Microsoft.DocumentDB/databaseAccounts/sqlDatabases', parameters('databaseAccountName'), parameters('databaseName'))]" + ], + "location": "[resourceGroup().location]", + "name": "[concat(parameters('databaseAccountName'), '/', parameters('databaseName'), '/MaintenanceManifests')]", + "properties": { + "options": {}, + "resource": { + "defaultTtl": -1, + "id": "MaintenanceManifests", + "partitionKey": { + "kind": "Hash", + "paths": [ + "/clusterResourceID" + ] + } + } + }, + "type": "Microsoft.DocumentDB/databaseAccounts/sqlDatabases/containers" + }, + { + "apiVersion": "2023-04-15", + "dependsOn": [ + "[resourceId('Microsoft.DocumentDB/databaseAccounts/sqlDatabases', parameters('databaseAccountName'), parameters('databaseName'))]", + "[resourceId('Microsoft.DocumentDB/databaseAccounts/sqlDatabases/containers', parameters('databaseAccountName'), parameters('databaseName'), 'MaintenanceManifests')]" + ], + "location": "[resourceGroup().location]", + "name": "[concat(parameters('databaseAccountName'), '/', parameters('databaseName'), '/MaintenanceManifests/renewLease')]", + "properties": { + "resource": { + "body": "function trigger() {\n\t\t\t\tvar request = getContext().getRequest();\n\t\t\t\tvar body = request.getBody();\n\t\t\t\tvar date = new Date();\n\t\t\t\tbody[\"leaseExpires\"] = Math.floor(date.getTime() / 1000) + 60;\n\t\t\t\trequest.setBody(body);\n\t\t\t}", + "id": "renewLease", + "triggerOperation": "All", + "triggerType": "Pre" + } + }, + "type": "Microsoft.DocumentDB/databaseAccounts/sqlDatabases/containers/triggers" } ] } diff --git a/pkg/deploy/generator/resources_rp.go b/pkg/deploy/generator/resources_rp.go index 4eed41ea00a..e630e4b31d5 100644 --- a/pkg/deploy/generator/resources_rp.go +++ b/pkg/deploy/generator/resources_rp.go @@ -1023,6 +1023,34 @@ func (g *generator) database(databaseName string, addDependsOn bool) []*arm.Reso Type: "Microsoft.DocumentDB/databaseAccounts/sqlDatabases", } + mimo := &arm.Resource{ + Resource: &sdkcosmos.SQLContainerCreateUpdateParameters{ + Properties: &sdkcosmos.SQLContainerCreateUpdateProperties{ + Resource: &sdkcosmos.SQLContainerResource{ + ID: to.StringPtr("MaintenanceManifests"), + PartitionKey: &sdkcosmos.ContainerPartitionKey{ + Paths: []*string{ + to.StringPtr("/clusterResourceID"), + }, + Kind: &hashPartitionKey, + }, + DefaultTTL: to.Int32Ptr(-1), + }, + Options: &sdkcosmos.CreateUpdateOptions{ + Throughput: to.Int32Ptr(cosmosDbGatewayProvisionedThroughputHack), + }, + }, + Name: to.StringPtr("[concat(parameters('databaseAccountName'), '/', " + databaseName + ", '/MaintenanceManifests')]"), + Type: to.StringPtr("Microsoft.DocumentDB/databaseAccounts/sqlDatabases/containers"), + Location: to.StringPtr("[resourceGroup().location]"), + }, + APIVersion: azureclient.APIVersion("Microsoft.DocumentDB"), + DependsOn: []string{ + "[resourceId('Microsoft.DocumentDB/databaseAccounts/sqlDatabases', parameters('databaseAccountName'), " + databaseName + ")]", + }, + Type: "Microsoft.DocumentDB/databaseAccounts/sqlDatabases", + } + if !g.production { database.Resource.(*sdkcosmos.SQLDatabaseCreateUpdateParameters).Properties.Options = &sdkcosmos.CreateUpdateOptions{ AutoscaleSettings: &sdkcosmos.AutoscaleSettings{ @@ -1031,6 +1059,7 @@ func (g *generator) database(databaseName string, addDependsOn bool) []*arm.Reso } portal.Resource.(*sdkcosmos.SQLContainerCreateUpdateParameters).Properties.Options = &sdkcosmos.CreateUpdateOptions{} gateway.Resource.(*sdkcosmos.SQLContainerCreateUpdateParameters).Properties.Options = &sdkcosmos.CreateUpdateOptions{} + mimo.Resource.(*sdkcosmos.SQLContainerCreateUpdateParameters).Properties.Options = &sdkcosmos.CreateUpdateOptions{} } rs := []*arm.Resource{ @@ -1268,6 +1297,15 @@ func (g *generator) database(databaseName string, addDependsOn bool) []*arm.Reso g.rpCosmosDBTriggers(databaseName, "Monitors", "renewLease", renewLeaseTriggerFunction, sdkcosmos.TriggerTypePre, sdkcosmos.TriggerOperationAll), ) + // Don't deploy the MIMO databases in production yet + if !g.production { + rs = append(rs, + mimo, + // MIMO DB triggers + g.rpCosmosDBTriggers(databaseName, "MaintenanceManifests", "renewLease", renewLeaseTriggerFunction, sdkcosmos.TriggerTypePre, sdkcosmos.TriggerOperationAll), + ) + } + if addDependsOn { for i := range rs { rs[i].DependsOn = append(rs[i].DependsOn, diff --git a/pkg/env/core.go b/pkg/env/core.go index 8ef466616fc..ec5258b1967 100644 --- a/pkg/env/core.go +++ b/pkg/env/core.go @@ -29,6 +29,8 @@ const ( COMPONENT_UPDATE_ROLE_SETS ServiceComponent = "UPDATE_ROLE_SETS" COMPONENT_DEPLOY ServiceComponent = "DEPLOY" COMPONENT_TOOLING ServiceComponent = "TOOLING" + COMPONENT_MIMO_SCHEDULER ServiceComponent = "MIMO_SCHEDULER" + COMPONENT_MIMO_ACTUATOR ServiceComponent = "MIMO_ACTUATOR" ) // Core collects basic configuration information which is expected to be diff --git a/pkg/env/dev.go b/pkg/env/dev.go index 1dee4514c63..d4f0cf13752 100644 --- a/pkg/env/dev.go +++ b/pkg/env/dev.go @@ -77,6 +77,9 @@ func (d *dev) AROOperatorImage() string { } func (d *dev) Listen() (net.Listener, error) { + if d.Component() == string(COMPONENT_MIMO_ACTUATOR) { + return net.Listen("tcp", ":8445") + } return net.Listen("tcp", ":8443") } diff --git a/pkg/frontend/admin_mimo_manifest_cancel.go b/pkg/frontend/admin_mimo_manifest_cancel.go new file mode 100644 index 00000000000..a453a4824cc --- /dev/null +++ b/pkg/frontend/admin_mimo_manifest_cancel.go @@ -0,0 +1,79 @@ +package frontend + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + + "github.com/go-chi/chi/v5" + "github.com/sirupsen/logrus" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/api/admin" + "github.com/Azure/ARO-RP/pkg/database/cosmosdb" + "github.com/Azure/ARO-RP/pkg/frontend/middleware" +) + +func (f *frontend) postAdminMaintManifestCancel(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + log := ctx.Value(middleware.ContextKeyLog).(*logrus.Entry) + resourceID := resourceIdFromURLParams(r) + b, err := f._postAdminMaintManifestCancel(ctx, r, resourceID) + + if cloudErr, ok := err.(*api.CloudError); ok { + api.WriteCloudError(w, cloudErr) + return + } + + adminReply(log, w, nil, b, err) +} + +func (f *frontend) _postAdminMaintManifestCancel(ctx context.Context, r *http.Request, resourceID string) ([]byte, error) { + manifestId := chi.URLParam(r, "manifestId") + + converter := f.apis[admin.APIVersion].MaintenanceManifestConverter + + dbOpenShiftClusters, err := f.dbGroup.OpenShiftClusters() + if err != nil { + return nil, api.NewCloudError(http.StatusInternalServerError, api.CloudErrorCodeInternalServerError, "", err.Error()) + } + + dbMaintenanceManifests, err := f.dbGroup.MaintenanceManifests() + if err != nil { + return nil, api.NewCloudError(http.StatusInternalServerError, api.CloudErrorCodeInternalServerError, "", err.Error()) + } + + doc, err := dbOpenShiftClusters.Get(ctx, resourceID) + if err != nil { + return nil, api.NewCloudError(http.StatusNotFound, api.CloudErrorCodeNotFound, "", fmt.Sprintf("cluster not found: %s", err.Error())) + } + + if doc.OpenShiftCluster.Properties.ProvisioningState == api.ProvisioningStateDeleting { + return nil, api.NewCloudError(http.StatusNotFound, api.CloudErrorCodeNotFound, "", "cluster being deleted") + } + + modifiedDoc, err := dbMaintenanceManifests.Patch(ctx, resourceID, manifestId, func(mmd *api.MaintenanceManifestDocument) error { + if mmd.MaintenanceManifest.State != api.MaintenanceManifestStatePending { + return api.NewCloudError(http.StatusNotAcceptable, api.CloudErrorCodePropertyChangeNotAllowed, "", fmt.Sprintf("cannot cancel task in state %s", mmd.MaintenanceManifest.State)) + } + + mmd.MaintenanceManifest.State = api.MaintenanceManifestStateCancelled + return nil + }) + if err != nil { + cloudErr, ok := err.(*api.CloudError) + if ok { + return nil, cloudErr + } else if cosmosdb.IsErrorStatusCode(err, http.StatusNotFound) { + return nil, api.NewCloudError(http.StatusNotFound, api.CloudErrorCodeNotFound, "", fmt.Sprintf("manifest not found: %s", err.Error())) + } else { + return nil, api.NewCloudError(http.StatusInternalServerError, api.CloudErrorCodeInternalServerError, "", err.Error()) + } + } + + return json.MarshalIndent(converter.ToExternal(modifiedDoc), "", " ") +} diff --git a/pkg/frontend/admin_mimo_manifest_cancel_test.go b/pkg/frontend/admin_mimo_manifest_cancel_test.go new file mode 100644 index 00000000000..ae8536a4fc6 --- /dev/null +++ b/pkg/frontend/admin_mimo_manifest_cancel_test.go @@ -0,0 +1,205 @@ +package frontend + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "fmt" + "net/http" + "strings" + "testing" + "time" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/api/admin" + "github.com/Azure/ARO-RP/pkg/metrics/noop" + testdatabase "github.com/Azure/ARO-RP/test/database" +) + +func TestMIMOCancelManifest(t *testing.T) { + mockSubID := "00000000-0000-0000-0000-000000000000" + mockTenantID := "00000000-0000-0000-0000-000000000000" + resourceID := fmt.Sprintf("/subscriptions/%s/resourcegroups/resourceGroup/providers/Microsoft.RedHatOpenShift/openShiftClusters/resourceName", mockSubID) + ctx := context.Background() + + type test struct { + name string + fixtures func(f *testdatabase.Fixture) + wantStatusCode int + wantResponse *admin.MaintenanceManifest + wantResult func(f *testdatabase.Checker) + wantError string + } + + for _, tt := range []*test{ + { + name: "no cluster", + wantError: "404: NotFound: : cluster not found: 404 : ", + fixtures: func(f *testdatabase.Fixture) {}, + wantStatusCode: http.StatusNotFound, + }, + { + name: "cluster being deleted", + fixtures: func(f *testdatabase.Fixture) { + f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{ + Key: strings.ToLower(resourceID), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: resourceID, + Name: "resourceName", + Type: "Microsoft.RedHatOpenShift/openshiftClusters", + Properties: api.OpenShiftClusterProperties{ + ProvisioningState: api.ProvisioningStateDeleting, + }, + }, + }) + }, + wantError: "404: NotFound: : cluster being deleted", + wantStatusCode: http.StatusNotFound, + }, + { + name: "no item", + fixtures: func(f *testdatabase.Fixture) { + f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{ + Key: strings.ToLower(resourceID), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: resourceID, + Name: "resourceName", + Type: "Microsoft.RedHatOpenShift/openshiftClusters", + }, + }) + }, + wantError: "404: NotFound: : manifest not found: 404 : ", + wantStatusCode: http.StatusNotFound, + }, + { + name: "cancel pending", + fixtures: func(f *testdatabase.Fixture) { + f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{ + Key: strings.ToLower(resourceID), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: resourceID, + Name: "resourceName", + Type: "Microsoft.RedHatOpenShift/openshiftClusters", + }, + }) + f.AddMaintenanceManifestDocuments(&api.MaintenanceManifestDocument{ + ClusterResourceID: strings.ToLower(resourceID), + MaintenanceManifest: api.MaintenanceManifest{ + MaintenanceTaskID: "exampletask", + State: api.MaintenanceManifestStatePending, + RunAfter: 1, + RunBefore: 1, + }, + }) + }, + wantResult: func(c *testdatabase.Checker) { + c.AddMaintenanceManifestDocuments(&api.MaintenanceManifestDocument{ + ID: "07070707-0707-0707-0707-070707070001", + ClusterResourceID: strings.ToLower(resourceID), + MaintenanceManifest: api.MaintenanceManifest{ + MaintenanceTaskID: "exampletask", + State: api.MaintenanceManifestStateCancelled, + RunAfter: 1, + RunBefore: 1, + }, + }) + }, + wantResponse: &admin.MaintenanceManifest{ + ID: "07070707-0707-0707-0707-070707070001", + MaintenanceTaskID: "exampletask", + State: admin.MaintenanceManifestStateCancelled, + Priority: 0, + RunAfter: 1, + RunBefore: 1, + }, + wantStatusCode: http.StatusOK, + }, + { + name: "cannot cancel failed", + fixtures: func(f *testdatabase.Fixture) { + f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{ + Key: strings.ToLower(resourceID), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: resourceID, + Name: "resourceName", + Type: "Microsoft.RedHatOpenShift/openshiftClusters", + }, + }) + f.AddMaintenanceManifestDocuments(&api.MaintenanceManifestDocument{ + ClusterResourceID: strings.ToLower(resourceID), + MaintenanceManifest: api.MaintenanceManifest{ + MaintenanceTaskID: "exampletask", + State: api.MaintenanceManifestStateFailed, + RunAfter: 1, + RunBefore: 1, + }, + }) + }, + wantResult: func(c *testdatabase.Checker) { + c.AddMaintenanceManifestDocuments(&api.MaintenanceManifestDocument{ + ID: "07070707-0707-0707-0707-070707070001", + ClusterResourceID: strings.ToLower(resourceID), + MaintenanceManifest: api.MaintenanceManifest{ + MaintenanceTaskID: "exampletask", + State: api.MaintenanceManifestStateFailed, + RunAfter: 1, + RunBefore: 1, + }, + }) + }, + wantError: "406: PropertyChangeNotAllowed: : cannot cancel task in state Failed", + wantStatusCode: http.StatusNotAcceptable, + }, + } { + t.Run(tt.name, func(t *testing.T) { + now := func() time.Time { return time.Unix(1000, 0) } + + ti := newTestInfra(t).WithOpenShiftClusters().WithSubscriptions().WithMaintenanceManifests(now) + defer ti.done() + + ti.fixture.AddSubscriptionDocuments(&api.SubscriptionDocument{ + ID: mockSubID, + Subscription: &api.Subscription{ + State: api.SubscriptionStateRegistered, + Properties: &api.SubscriptionProperties{ + TenantID: mockTenantID, + }, + }, + }) + + err := ti.buildFixtures(tt.fixtures) + if err != nil { + t.Fatal(err) + } + + if tt.wantResult != nil { + tt.wantResult(ti.checker) + } + + f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.dbGroup, api.APIs, &noop.Noop{}, &noop.Noop{}, testdatabase.NewFakeAEAD(), nil, nil, nil, nil, nil) + + if err != nil { + t.Fatal(err) + } + + go f.Run(ctx, nil, nil) + + resp, b, err := ti.request(http.MethodPost, + fmt.Sprintf("https://server/admin%s/maintenancemanifests/07070707-0707-0707-0707-070707070001/cancel", resourceID), + nil, nil) + if err != nil { + t.Fatal(err) + } + + err = validateResponse(resp, b, tt.wantStatusCode, tt.wantError, tt.wantResponse) + if err != nil { + t.Error(err) + } + + for _, err := range ti.checker.CheckMaintenanceManifests(ti.maintenanceManifestsClient) { + t.Error(err) + } + }) + } +} diff --git a/pkg/frontend/admin_mimo_manifest_create.go b/pkg/frontend/admin_mimo_manifest_create.go new file mode 100644 index 00000000000..cf055d0062e --- /dev/null +++ b/pkg/frontend/admin_mimo_manifest_create.go @@ -0,0 +1,98 @@ +package frontend + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "time" + + "github.com/sirupsen/logrus" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/api/admin" + "github.com/Azure/ARO-RP/pkg/frontend/middleware" +) + +func (f *frontend) putAdminMaintManifestCreate(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + log := ctx.Value(middleware.ContextKeyLog).(*logrus.Entry) + resourceID := resourceIdFromURLParams(r) + b, err := f._putAdminMaintManifestCreate(ctx, r, resourceID) + + if cloudErr, ok := err.(*api.CloudError); ok { + api.WriteCloudError(w, cloudErr) + return + } + + err = statusCodeError(http.StatusCreated) + adminReply(log, w, nil, b, err) +} + +func (f *frontend) _putAdminMaintManifestCreate(ctx context.Context, r *http.Request, resourceID string) ([]byte, error) { + converter := f.apis[admin.APIVersion].MaintenanceManifestConverter + validator := f.apis[admin.APIVersion].MaintenanceManifestStaticValidator + + body := r.Context().Value(middleware.ContextKeyBody).([]byte) + if len(body) == 0 || !json.Valid(body) { + return nil, api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidRequestContent, "", "The request content was invalid and could not be deserialized.") + } + + dbOpenShiftClusters, err := f.dbGroup.OpenShiftClusters() + if err != nil { + return nil, api.NewCloudError(http.StatusInternalServerError, api.CloudErrorCodeInternalServerError, "", err.Error()) + } + + dbMaintenanceManifests, err := f.dbGroup.MaintenanceManifests() + if err != nil { + return nil, api.NewCloudError(http.StatusInternalServerError, api.CloudErrorCodeInternalServerError, "", err.Error()) + } + + doc, err := dbOpenShiftClusters.Get(ctx, resourceID) + if err != nil { + return nil, api.NewCloudError(http.StatusNotFound, api.CloudErrorCodeNotFound, "", fmt.Sprintf("cluster not found: %s", err.Error())) + } + + if doc.OpenShiftCluster.Properties.ProvisioningState == api.ProvisioningStateDeleting { + return nil, api.NewCloudError(http.StatusNotFound, api.CloudErrorCodeNotFound, "", "cluster being deleted") + } + + var ext *admin.MaintenanceManifest + err = json.Unmarshal(body, &ext) + if err != nil { + return nil, api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidRequestContent, "", "The request content could not be deserialized: "+err.Error()) + } + + // fill in some defaults + ext.ID = dbMaintenanceManifests.NewUUID() + ext.State = admin.MaintenanceManifestStatePending + + if ext.RunAfter == 0 { + ext.RunAfter = int(f.now().Unix()) + } + + // add a 7d timeout by default + if ext.RunBefore == 0 { + ext.RunBefore = int(f.now().Add(time.Hour * 7 * 24).Unix()) + } + + err = validator.Static(ext, nil) + if err != nil { + return nil, err + } + + manifestDoc := &api.MaintenanceManifestDocument{ + ClusterResourceID: resourceID, + } + converter.ToInternal(ext, manifestDoc) + + savedDoc, err := dbMaintenanceManifests.Create(ctx, manifestDoc) + if err != nil { + return nil, api.NewCloudError(http.StatusInternalServerError, api.CloudErrorCodeInternalServerError, "", err.Error()) + } + + return json.MarshalIndent(converter.ToExternal(savedDoc), "", " ") +} diff --git a/pkg/frontend/admin_mimo_manifest_create_test.go b/pkg/frontend/admin_mimo_manifest_create_test.go new file mode 100644 index 00000000000..d309e3d8c71 --- /dev/null +++ b/pkg/frontend/admin_mimo_manifest_create_test.go @@ -0,0 +1,208 @@ +package frontend + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "fmt" + "net/http" + "strings" + "testing" + "time" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/api/admin" + "github.com/Azure/ARO-RP/pkg/metrics/noop" + testdatabase "github.com/Azure/ARO-RP/test/database" +) + +func TestMIMOCreateManifest(t *testing.T) { + mockSubID := "00000000-0000-0000-0000-000000000000" + mockTenantID := "00000000-0000-0000-0000-000000000000" + resourceID := fmt.Sprintf("/subscriptions/%s/resourcegroups/resourceGroup/providers/Microsoft.RedHatOpenShift/openShiftClusters/resourceName", mockSubID) + ctx := context.Background() + + type test struct { + name string + fixtures func(f *testdatabase.Fixture) + body *admin.MaintenanceManifest + wantStatusCode int + wantResponse *admin.MaintenanceManifest + wantResult func(f *testdatabase.Checker) + wantError string + } + + for _, tt := range []*test{ + { + name: "no cluster", + wantError: "404: NotFound: : cluster not found: 404 : ", + fixtures: func(f *testdatabase.Fixture) {}, + wantStatusCode: http.StatusNotFound, + }, + { + name: "cluster being deleted", + fixtures: func(f *testdatabase.Fixture) { + f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{ + Key: strings.ToLower(resourceID), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: resourceID, + Name: "resourceName", + Type: "Microsoft.RedHatOpenShift/openshiftClusters", + Properties: api.OpenShiftClusterProperties{ + ProvisioningState: api.ProvisioningStateDeleting, + }, + }, + }) + }, + wantError: "404: NotFound: : cluster being deleted", + wantStatusCode: http.StatusNotFound, + }, + { + name: "invalid", + fixtures: func(f *testdatabase.Fixture) { + f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{ + Key: strings.ToLower(resourceID), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: resourceID, + Name: "resourceName", + Type: "Microsoft.RedHatOpenShift/openshiftClusters", + }, + }) + }, + body: &admin.MaintenanceManifest{}, + wantError: "400: InvalidParameter: maintenanceTaskID: Must be provided", + wantStatusCode: http.StatusBadRequest, + }, + + { + name: "good", + fixtures: func(f *testdatabase.Fixture) { + f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{ + Key: strings.ToLower(resourceID), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: resourceID, + Name: "resourceName", + Type: "Microsoft.RedHatOpenShift/openshiftClusters", + }, + }) + }, + body: &admin.MaintenanceManifest{ + MaintenanceTaskID: "exampletask", + State: admin.MaintenanceManifestStatePending, + RunAfter: 1, + RunBefore: 1, + }, + wantResult: func(c *testdatabase.Checker) { + c.AddMaintenanceManifestDocuments(&api.MaintenanceManifestDocument{ + ID: "07070707-0707-0707-0707-070707070001", + ClusterResourceID: strings.ToLower(resourceID), + MaintenanceManifest: api.MaintenanceManifest{ + MaintenanceTaskID: "exampletask", + State: api.MaintenanceManifestStatePending, + RunAfter: 1, + RunBefore: 1, + }, + }) + }, + wantResponse: &admin.MaintenanceManifest{ + ID: "07070707-0707-0707-0707-070707070001", + MaintenanceTaskID: "exampletask", + State: admin.MaintenanceManifestStatePending, + RunAfter: 1, + RunBefore: 1, + }, + wantStatusCode: http.StatusCreated, + }, + { + name: "default set to pending", + fixtures: func(f *testdatabase.Fixture) { + f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{ + Key: strings.ToLower(resourceID), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: resourceID, + Name: "resourceName", + Type: "Microsoft.RedHatOpenShift/openshiftClusters", + }, + }) + }, + body: &admin.MaintenanceManifest{ + MaintenanceTaskID: "exampletask", + RunAfter: 1, + RunBefore: 1, + }, + wantResult: func(c *testdatabase.Checker) { + c.AddMaintenanceManifestDocuments(&api.MaintenanceManifestDocument{ + ID: "07070707-0707-0707-0707-070707070001", + ClusterResourceID: strings.ToLower(resourceID), + MaintenanceManifest: api.MaintenanceManifest{ + MaintenanceTaskID: "exampletask", + State: api.MaintenanceManifestStatePending, + RunAfter: 1, + RunBefore: 1, + }, + }) + }, + wantResponse: &admin.MaintenanceManifest{ + ID: "07070707-0707-0707-0707-070707070001", + MaintenanceTaskID: "exampletask", + State: admin.MaintenanceManifestStatePending, + RunAfter: 1, + RunBefore: 1, + }, + wantStatusCode: http.StatusCreated, + }, + } { + t.Run(tt.name, func(t *testing.T) { + now := func() time.Time { return time.Unix(1000, 0) } + + ti := newTestInfra(t).WithOpenShiftClusters().WithSubscriptions().WithMaintenanceManifests(now) + defer ti.done() + + ti.fixture.AddSubscriptionDocuments(&api.SubscriptionDocument{ + ID: mockSubID, + Subscription: &api.Subscription{ + State: api.SubscriptionStateRegistered, + Properties: &api.SubscriptionProperties{ + TenantID: mockTenantID, + }, + }, + }) + + err := ti.buildFixtures(tt.fixtures) + if err != nil { + t.Fatal(err) + } + + if tt.wantResult != nil { + tt.wantResult(ti.checker) + } + + f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.dbGroup, api.APIs, &noop.Noop{}, &noop.Noop{}, testdatabase.NewFakeAEAD(), nil, nil, nil, nil, nil) + + if err != nil { + t.Fatal(err) + } + + go f.Run(ctx, nil, nil) + + resp, b, err := ti.request(http.MethodPut, + fmt.Sprintf("https://server/admin%s/maintenancemanifests", resourceID), + http.Header{ + "Content-Type": []string{"application/json"}, + }, tt.body) + if err != nil { + t.Fatal(err) + } + + err = validateResponse(resp, b, tt.wantStatusCode, tt.wantError, tt.wantResponse) + if err != nil { + t.Error(err) + } + + for _, err := range ti.checker.CheckMaintenanceManifests(ti.maintenanceManifestsClient) { + t.Error(err) + } + }) + } +} diff --git a/pkg/frontend/admin_mimo_manifest_delete.go b/pkg/frontend/admin_mimo_manifest_delete.go new file mode 100644 index 00000000000..73e62a46b11 --- /dev/null +++ b/pkg/frontend/admin_mimo_manifest_delete.go @@ -0,0 +1,69 @@ +package frontend + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + + "github.com/go-chi/chi/v5" + "github.com/sirupsen/logrus" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/database/cosmosdb" + "github.com/Azure/ARO-RP/pkg/frontend/middleware" +) + +func (f *frontend) deleteAdminMaintManifest(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + log := ctx.Value(middleware.ContextKeyLog).(*logrus.Entry) + resourceID := resourceIdFromURLParams(r) + b, err := f._deleteAdminMaintManifest(ctx, r, resourceID) + + if cloudErr, ok := err.(*api.CloudError); ok { + api.WriteCloudError(w, cloudErr) + return + } + + adminReply(log, w, nil, b, err) +} + +func (f *frontend) _deleteAdminMaintManifest(ctx context.Context, r *http.Request, resourceID string) ([]byte, error) { + manifestId := chi.URLParam(r, "manifestId") + + dbOpenShiftClusters, err := f.dbGroup.OpenShiftClusters() + if err != nil { + return nil, api.NewCloudError(http.StatusInternalServerError, api.CloudErrorCodeInternalServerError, "", err.Error()) + } + + dbMaintenanceManifests, err := f.dbGroup.MaintenanceManifests() + if err != nil { + return nil, api.NewCloudError(http.StatusInternalServerError, api.CloudErrorCodeInternalServerError, "", err.Error()) + } + + doc, err := dbOpenShiftClusters.Get(ctx, resourceID) + if err != nil { + return nil, api.NewCloudError(http.StatusNotFound, api.CloudErrorCodeNotFound, "", fmt.Sprintf("cluster not found: %s", err.Error())) + } + + if doc.OpenShiftCluster.Properties.ProvisioningState == api.ProvisioningStateDeleting { + return nil, api.NewCloudError(http.StatusNotFound, api.CloudErrorCodeNotFound, "", "cluster being deleted") + } + + err = dbMaintenanceManifests.Delete(ctx, resourceID, manifestId) + if err != nil { + cloudErr, ok := err.(*api.CloudError) + if ok { + return nil, cloudErr + } else if cosmosdb.IsErrorStatusCode(err, http.StatusNotFound) { + return nil, api.NewCloudError(http.StatusNotFound, api.CloudErrorCodeNotFound, "", fmt.Sprintf("manifest not found: %s", err.Error())) + } else { + return nil, api.NewCloudError(http.StatusInternalServerError, api.CloudErrorCodeInternalServerError, "", err.Error()) + } + } + + return json.MarshalIndent(map[string]string{}, "", " ") +} diff --git a/pkg/frontend/admin_mimo_manifest_delete_test.go b/pkg/frontend/admin_mimo_manifest_delete_test.go new file mode 100644 index 00000000000..221d02cabca --- /dev/null +++ b/pkg/frontend/admin_mimo_manifest_delete_test.go @@ -0,0 +1,151 @@ +package frontend + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "fmt" + "net/http" + "strings" + "testing" + "time" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/metrics/noop" + testdatabase "github.com/Azure/ARO-RP/test/database" +) + +func TestMIMODeleteManifest(t *testing.T) { + mockSubID := "00000000-0000-0000-0000-000000000000" + mockTenantID := "00000000-0000-0000-0000-000000000000" + resourceID := fmt.Sprintf("/subscriptions/%s/resourcegroups/resourceGroup/providers/Microsoft.RedHatOpenShift/openShiftClusters/resourceName", mockSubID) + ctx := context.Background() + + type test struct { + name string + fixtures func(f *testdatabase.Fixture) + wantStatusCode int + wantResponse []byte + wantResult func(f *testdatabase.Checker) + wantError string + } + + for _, tt := range []*test{ + { + name: "no cluster", + wantError: "404: NotFound: : cluster not found: 404 : ", + fixtures: func(f *testdatabase.Fixture) {}, + wantStatusCode: http.StatusNotFound, + }, + { + name: "cluster being deleted", + fixtures: func(f *testdatabase.Fixture) { + f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{ + Key: strings.ToLower(resourceID), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: resourceID, + Name: "resourceName", + Type: "Microsoft.RedHatOpenShift/openshiftClusters", + Properties: api.OpenShiftClusterProperties{ + ProvisioningState: api.ProvisioningStateDeleting, + }, + }, + }) + }, + wantError: "404: NotFound: : cluster being deleted", + wantStatusCode: http.StatusNotFound, + }, + { + name: "no item", + fixtures: func(f *testdatabase.Fixture) { + f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{ + Key: strings.ToLower(resourceID), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: resourceID, + Name: "resourceName", + Type: "Microsoft.RedHatOpenShift/openshiftClusters", + }, + }) + }, + wantError: "404: NotFound: : manifest not found: 404 : ", + wantStatusCode: http.StatusNotFound, + }, + { + name: "delete successfully", + fixtures: func(f *testdatabase.Fixture) { + f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{ + Key: strings.ToLower(resourceID), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: resourceID, + Name: "resourceName", + Type: "Microsoft.RedHatOpenShift/openshiftClusters", + }, + }) + f.AddMaintenanceManifestDocuments(&api.MaintenanceManifestDocument{ + ClusterResourceID: strings.ToLower(resourceID), + MaintenanceManifest: api.MaintenanceManifest{ + MaintenanceTaskID: "exampletask", + State: api.MaintenanceManifestStatePending, + RunAfter: 1, + RunBefore: 1, + }, + }) + }, + wantResult: func(c *testdatabase.Checker) { + }, + wantResponse: []byte("{}\n"), + wantStatusCode: http.StatusOK, + }, + } { + t.Run(tt.name, func(t *testing.T) { + now := func() time.Time { return time.Unix(1000, 0) } + + ti := newTestInfra(t).WithOpenShiftClusters().WithSubscriptions().WithMaintenanceManifests(now) + defer ti.done() + + ti.fixture.AddSubscriptionDocuments(&api.SubscriptionDocument{ + ID: mockSubID, + Subscription: &api.Subscription{ + State: api.SubscriptionStateRegistered, + Properties: &api.SubscriptionProperties{ + TenantID: mockTenantID, + }, + }, + }) + + err := ti.buildFixtures(tt.fixtures) + if err != nil { + t.Fatal(err) + } + + if tt.wantResult != nil { + tt.wantResult(ti.checker) + } + + f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.dbGroup, api.APIs, &noop.Noop{}, &noop.Noop{}, testdatabase.NewFakeAEAD(), nil, nil, nil, nil, nil) + + if err != nil { + t.Fatal(err) + } + + go f.Run(ctx, nil, nil) + + resp, b, err := ti.request(http.MethodDelete, + fmt.Sprintf("https://server/admin%s/maintenancemanifests/07070707-0707-0707-0707-070707070001", resourceID), + nil, nil) + if err != nil { + t.Fatal(err) + } + + err = validateResponse(resp, b, tt.wantStatusCode, tt.wantError, tt.wantResponse) + if err != nil { + t.Error(err) + } + + for _, err := range ti.checker.CheckMaintenanceManifests(ti.maintenanceManifestsClient) { + t.Error(err) + } + }) + } +} diff --git a/pkg/frontend/admin_mimo_manifest_get.go b/pkg/frontend/admin_mimo_manifest_get.go new file mode 100644 index 00000000000..d8de769cc7e --- /dev/null +++ b/pkg/frontend/admin_mimo_manifest_get.go @@ -0,0 +1,64 @@ +package frontend + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + + "github.com/go-chi/chi/v5" + "github.com/sirupsen/logrus" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/api/admin" + "github.com/Azure/ARO-RP/pkg/frontend/middleware" +) + +func (f *frontend) getSingleAdminMaintManifest(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + log := ctx.Value(middleware.ContextKeyLog).(*logrus.Entry) + resourceID := resourceIdFromURLParams(r) + b, err := f._getSingleAdminMaintManifest(ctx, r, resourceID) + + if cloudErr, ok := err.(*api.CloudError); ok { + api.WriteCloudError(w, cloudErr) + return + } + + adminReply(log, w, nil, b, err) +} + +func (f *frontend) _getSingleAdminMaintManifest(ctx context.Context, r *http.Request, resourceID string) ([]byte, error) { + manifestId := chi.URLParam(r, "manifestId") + + converter := f.apis[admin.APIVersion].MaintenanceManifestConverter + + dbOpenShiftClusters, err := f.dbGroup.OpenShiftClusters() + if err != nil { + return nil, api.NewCloudError(http.StatusInternalServerError, api.CloudErrorCodeInternalServerError, "", err.Error()) + } + + dbMaintenanceManifests, err := f.dbGroup.MaintenanceManifests() + if err != nil { + return nil, api.NewCloudError(http.StatusInternalServerError, api.CloudErrorCodeInternalServerError, "", err.Error()) + } + + doc, err := dbOpenShiftClusters.Get(ctx, resourceID) + if err != nil { + return nil, api.NewCloudError(http.StatusNotFound, api.CloudErrorCodeNotFound, "", fmt.Sprintf("cluster not found: %s", err.Error())) + } + + if doc.OpenShiftCluster.Properties.ProvisioningState == api.ProvisioningStateDeleting { + return nil, api.NewCloudError(http.StatusNotFound, api.CloudErrorCodeNotFound, "", "cluster being deleted") + } + + manifest, err := dbMaintenanceManifests.Get(ctx, resourceID, manifestId) + if err != nil { + return nil, api.NewCloudError(http.StatusNotFound, api.CloudErrorCodeNotFound, "", fmt.Sprintf("manifest not found: %s", err.Error())) + } + + return json.MarshalIndent(converter.ToExternal(manifest), "", " ") +} diff --git a/pkg/frontend/admin_mimo_manifest_get_test.go b/pkg/frontend/admin_mimo_manifest_get_test.go new file mode 100644 index 00000000000..ddad3d36f9f --- /dev/null +++ b/pkg/frontend/admin_mimo_manifest_get_test.go @@ -0,0 +1,155 @@ +package frontend + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "fmt" + "net/http" + "strings" + "testing" + "time" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/api/admin" + "github.com/Azure/ARO-RP/pkg/metrics/noop" + testdatabase "github.com/Azure/ARO-RP/test/database" +) + +func TestMIMOGet(t *testing.T) { + mockSubID := "00000000-0000-0000-0000-000000000000" + mockTenantID := "00000000-0000-0000-0000-000000000000" + resourceID := fmt.Sprintf("/subscriptions/%s/resourcegroups/resourceGroup/providers/Microsoft.RedHatOpenShift/openShiftClusters/resourceName", mockSubID) + ctx := context.Background() + + type test struct { + name string + fixtures func(f *testdatabase.Fixture) + wantStatusCode int + wantResponse *admin.MaintenanceManifest + wantError string + } + + for _, tt := range []*test{ + { + name: "no cluster", + wantError: "404: NotFound: : cluster not found: 404 : ", + fixtures: func(f *testdatabase.Fixture) {}, + wantStatusCode: http.StatusNotFound, + }, + { + name: "cluster being deleted", + fixtures: func(f *testdatabase.Fixture) { + f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{ + Key: strings.ToLower(resourceID), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: resourceID, + Name: "resourceName", + Type: "Microsoft.RedHatOpenShift/openshiftClusters", + Properties: api.OpenShiftClusterProperties{ + ProvisioningState: api.ProvisioningStateDeleting, + }, + }, + }) + }, + wantError: "404: NotFound: : cluster being deleted", + wantStatusCode: http.StatusNotFound, + }, + { + name: "no item", + fixtures: func(f *testdatabase.Fixture) { + f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{ + Key: strings.ToLower(resourceID), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: resourceID, + Name: "resourceName", + Type: "Microsoft.RedHatOpenShift/openshiftClusters", + }, + }) + }, + wantError: "404: NotFound: : manifest not found: 404 : ", + wantStatusCode: http.StatusNotFound, + }, + { + name: "get entry", + fixtures: func(f *testdatabase.Fixture) { + f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{ + Key: strings.ToLower(resourceID), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: resourceID, + Name: "resourceName", + Type: "Microsoft.RedHatOpenShift/openshiftClusters", + }, + }) + f.AddMaintenanceManifestDocuments(&api.MaintenanceManifestDocument{ + ClusterResourceID: strings.ToLower(resourceID), + MaintenanceManifest: api.MaintenanceManifest{ + MaintenanceTaskID: "exampletask", + State: api.MaintenanceManifestStatePending, + RunAfter: 1, + RunBefore: 1, + }, + }) + }, + wantResponse: &admin.MaintenanceManifest{ + ID: "07070707-0707-0707-0707-070707070001", + MaintenanceTaskID: "exampletask", + State: admin.MaintenanceManifestStatePending, + Priority: 0, + RunAfter: 1, + RunBefore: 1, + }, + wantStatusCode: http.StatusOK, + }, + } { + t.Run(tt.name, func(t *testing.T) { + now := func() time.Time { return time.Unix(1000, 0) } + + ti := newTestInfra(t).WithOpenShiftClusters().WithSubscriptions().WithMaintenanceManifests(now) + defer ti.done() + + ti.fixture.AddSubscriptionDocuments(&api.SubscriptionDocument{ + ID: mockSubID, + Subscription: &api.Subscription{ + State: api.SubscriptionStateRegistered, + Properties: &api.SubscriptionProperties{ + TenantID: mockTenantID, + }, + }, + }) + + err := ti.buildFixtures(tt.fixtures) + if err != nil { + t.Fatal(err) + } + + a, err := ti.openShiftClustersClient.ListAll(ctx, nil) + if err != nil { + t.Fatal(err) + } + + fmt.Print(a) + + f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.dbGroup, api.APIs, &noop.Noop{}, &noop.Noop{}, testdatabase.NewFakeAEAD(), nil, nil, nil, nil, nil) + + if err != nil { + t.Fatal(err) + } + + go f.Run(ctx, nil, nil) + + resp, b, err := ti.request(http.MethodGet, + fmt.Sprintf("https://server/admin%s/maintenancemanifests/07070707-0707-0707-0707-070707070001", resourceID), + nil, nil) + if err != nil { + t.Fatal(err) + } + + err = validateResponse(resp, b, tt.wantStatusCode, tt.wantError, tt.wantResponse) + if err != nil { + t.Error(err) + } + }) + } +} diff --git a/pkg/frontend/admin_mimo_manifest_list.go b/pkg/frontend/admin_mimo_manifest_list.go new file mode 100644 index 00000000000..48c222e8534 --- /dev/null +++ b/pkg/frontend/admin_mimo_manifest_list.go @@ -0,0 +1,96 @@ +package frontend + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "encoding/json" + "fmt" + "math" + "net/http" + "strconv" + + "github.com/sirupsen/logrus" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/api/admin" + "github.com/Azure/ARO-RP/pkg/frontend/middleware" +) + +func (f *frontend) getAdminMaintManifests(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + log := ctx.Value(middleware.ContextKeyLog).(*logrus.Entry) + resourceID := resourceIdFromURLParams(r) + b, err := f._getAdminMaintManifests(ctx, r, resourceID) + + if cloudErr, ok := err.(*api.CloudError); ok { + api.WriteCloudError(w, cloudErr) + return + } + + adminReply(log, w, nil, b, err) +} + +func (f *frontend) _getAdminMaintManifests(ctx context.Context, r *http.Request, resourceID string) ([]byte, error) { + limitstr := r.URL.Query().Get("limit") + limit, err := strconv.Atoi(limitstr) + if err != nil { + limit = 100 + } + + converter := f.apis[admin.APIVersion].MaintenanceManifestConverter + + dbOpenShiftClusters, err := f.dbGroup.OpenShiftClusters() + if err != nil { + return nil, api.NewCloudError(http.StatusInternalServerError, api.CloudErrorCodeInternalServerError, "", err.Error()) + } + + dbMaintenanceManifests, err := f.dbGroup.MaintenanceManifests() + if err != nil { + return nil, api.NewCloudError(http.StatusInternalServerError, api.CloudErrorCodeInternalServerError, "", err.Error()) + } + + doc, err := dbOpenShiftClusters.Get(ctx, resourceID) + if err != nil { + return nil, api.NewCloudError(http.StatusNotFound, api.CloudErrorCodeNotFound, "", fmt.Sprintf("cluster not found: %s", err.Error())) + } + + if doc.OpenShiftCluster.Properties.ProvisioningState == api.ProvisioningStateDeleting { + return nil, api.NewCloudError(http.StatusNotFound, api.CloudErrorCodeNotFound, "", "cluster being deleted") + } + + skipToken, err := f.parseSkipToken(r.URL.String()) + if err != nil { + return nil, err + } + + i, err := dbMaintenanceManifests.GetByClusterResourceID(ctx, resourceID, skipToken) + if err != nil { + return nil, api.NewCloudError(http.StatusInternalServerError, api.CloudErrorCodeInternalServerError, "", err.Error()) + } + + docList := make([]*api.MaintenanceManifestDocument, 0) + for { + docs, err := i.Next(ctx, int(math.Min(float64(limit), 10))) + if err != nil { + return nil, api.NewCloudError(http.StatusInternalServerError, api.CloudErrorCodeInternalServerError, "", fmt.Errorf("failed reading next manifest document: %w", err).Error()) + } + if docs == nil { + break + } + + docList = append(docList, docs.MaintenanceManifestDocuments...) + + if len(docList) >= limit { + break + } + } + + nextLink, err := f.buildNextLink(r.Header.Get("Referer"), i.Continuation()) + if err != nil { + return nil, err + } + + return json.MarshalIndent(converter.ToExternalList(docList, nextLink), "", " ") +} diff --git a/pkg/frontend/admin_mimo_manifest_list_test.go b/pkg/frontend/admin_mimo_manifest_list_test.go new file mode 100644 index 00000000000..4c7664e5f4d --- /dev/null +++ b/pkg/frontend/admin_mimo_manifest_list_test.go @@ -0,0 +1,210 @@ +package frontend + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "encoding/base64" + "fmt" + "net/http" + "net/url" + "strings" + "testing" + "time" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/api/admin" + "github.com/Azure/ARO-RP/pkg/metrics/noop" + testdatabase "github.com/Azure/ARO-RP/test/database" +) + +func TestMIMOListManifests(t *testing.T) { + mockSubID := "00000000-0000-0000-0000-000000000000" + mockTenantID := "00000000-0000-0000-0000-000000000000" + resourceID := fmt.Sprintf("/subscriptions/%s/resourcegroups/resourceGroup/providers/Microsoft.RedHatOpenShift/openShiftClusters/resourceName", mockSubID) + ctx := context.Background() + + type test struct { + name string + fixtures func(f *testdatabase.Fixture) + limit int + wantStatusCode int + wantResponse *admin.MaintenanceManifestList + wantError string + } + + for _, tt := range []*test{ + { + name: "no entries", + fixtures: func(f *testdatabase.Fixture) { + f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{ + Key: strings.ToLower(resourceID), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: resourceID, + Name: "resourceName", + Type: "Microsoft.RedHatOpenShift/openshiftClusters", + }, + }) + }, + wantResponse: &admin.MaintenanceManifestList{ + MaintenanceManifests: []*admin.MaintenanceManifest{}, + }, + wantStatusCode: http.StatusOK, + }, + { + name: "single entry", + fixtures: func(f *testdatabase.Fixture) { + f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{ + Key: strings.ToLower(resourceID), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: resourceID, + Name: "resourceName", + Type: "Microsoft.RedHatOpenShift/openshiftClusters", + }, + }) + f.AddMaintenanceManifestDocuments(&api.MaintenanceManifestDocument{ + ClusterResourceID: strings.ToLower(resourceID), + MaintenanceManifest: api.MaintenanceManifest{ + MaintenanceTaskID: "exampletask", + State: api.MaintenanceManifestStatePending, + RunAfter: 1, + RunBefore: 1, + }, + }) + }, + wantResponse: &admin.MaintenanceManifestList{ + MaintenanceManifests: []*admin.MaintenanceManifest{ + { + ID: "07070707-0707-0707-0707-070707070001", + MaintenanceTaskID: "exampletask", + State: admin.MaintenanceManifestStatePending, + Priority: 0, + RunAfter: 1, + RunBefore: 1, + }, + }, + }, + wantStatusCode: http.StatusOK, + }, + { + name: "limit over", + limit: 1, + fixtures: func(f *testdatabase.Fixture) { + f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{ + Key: strings.ToLower(resourceID), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: resourceID, + Name: "resourceName", + Type: "Microsoft.RedHatOpenShift/openshiftClusters", + }, + }) + f.AddMaintenanceManifestDocuments(&api.MaintenanceManifestDocument{ + ClusterResourceID: strings.ToLower(resourceID), + MaintenanceManifest: api.MaintenanceManifest{ + MaintenanceTaskID: "exampletask", + State: api.MaintenanceManifestStatePending, + RunAfter: 1, + RunBefore: 1, + }, + }) + f.AddMaintenanceManifestDocuments(&api.MaintenanceManifestDocument{ + ClusterResourceID: strings.ToLower(resourceID), + MaintenanceManifest: api.MaintenanceManifest{ + MaintenanceTaskID: "exampleset2", + State: api.MaintenanceManifestStatePending, + RunAfter: 1, + RunBefore: 1, + }, + }) + }, + wantResponse: &admin.MaintenanceManifestList{ + NextLink: "https://mockrefererhost/?%24skipToken=" + url.QueryEscape(base64.StdEncoding.EncodeToString([]byte("FAKE1"))), + MaintenanceManifests: []*admin.MaintenanceManifest{ + { + ID: "07070707-0707-0707-0707-070707070001", + MaintenanceTaskID: "exampletask", + State: admin.MaintenanceManifestStatePending, + Priority: 0, + RunAfter: 1, + RunBefore: 1, + }, + }, + }, + wantStatusCode: http.StatusOK, + }, + { + name: "cluster being deleted", + fixtures: func(f *testdatabase.Fixture) { + f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{ + Key: strings.ToLower(resourceID), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: resourceID, + Name: "resourceName", + Type: "Microsoft.RedHatOpenShift/openshiftClusters", + Properties: api.OpenShiftClusterProperties{ + ProvisioningState: api.ProvisioningStateDeleting, + }, + }, + }) + }, + wantError: "404: NotFound: : cluster being deleted", + wantStatusCode: http.StatusNotFound, + }, + { + name: "missing cluster", + wantError: "404: NotFound: : cluster not found: 404 : ", + wantStatusCode: http.StatusNotFound, + }, + } { + t.Run(tt.name, func(t *testing.T) { + now := func() time.Time { return time.Unix(1000, 0) } + + ti := newTestInfra(t).WithOpenShiftClusters().WithSubscriptions().WithMaintenanceManifests(now) + defer ti.done() + + ti.fixture.AddSubscriptionDocuments(&api.SubscriptionDocument{ + ID: mockSubID, + Subscription: &api.Subscription{ + State: api.SubscriptionStateRegistered, + Properties: &api.SubscriptionProperties{ + TenantID: mockTenantID, + }, + }, + }) + + err := ti.buildFixtures(tt.fixtures) + if err != nil { + t.Fatal(err) + } + + f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.dbGroup, api.APIs, &noop.Noop{}, &noop.Noop{}, testdatabase.NewFakeAEAD(), nil, nil, nil, nil, nil) + + if err != nil { + t.Fatal(err) + } + + go f.Run(ctx, nil, nil) + + if tt.limit == 0 { + tt.limit = 100 + } + + fmt.Printf("limit: %d", tt.limit) + + resp, b, err := ti.request(http.MethodGet, + fmt.Sprintf("https://server/admin%s/maintenancemanifests?limit=%d", resourceID, tt.limit), + http.Header{ + "Referer": []string{"https://mockrefererhost/"}, + }, nil) + if err != nil { + t.Fatal(err) + } + + err = validateResponse(resp, b, tt.wantStatusCode, tt.wantError, tt.wantResponse) + if err != nil { + t.Error(err) + } + }) + } +} diff --git a/pkg/frontend/frontend.go b/pkg/frontend/frontend.go index 83e9d97b660..d24938a3efa 100644 --- a/pkg/frontend/frontend.go +++ b/pkg/frontend/frontend.go @@ -10,10 +10,12 @@ import ( "log" "net" "net/http" + "strings" "sync" "sync/atomic" "time" + "github.com/Azure/go-autorest/autorest/azure" "github.com/go-chi/chi/v5" chiMiddlewares "github.com/go-chi/chi/v5/middleware" "github.com/sirupsen/logrus" @@ -48,6 +50,7 @@ type frontendDBs interface { database.DatabaseGroupWithAsyncOperations database.DatabaseGroupWithSubscriptions database.DatabaseGroupWithPlatformWorkloadIdentityRoleSets + database.DatabaseGroupWithMaintenanceManifests } type kubeActionsFactory func(*logrus.Entry, env.Interface, *api.OpenShiftCluster) (adminactions.KubeActions, error) @@ -326,6 +329,17 @@ func (f *frontend) chiAuthenticatedRoutes(router chi.Router) { r.With(f.maintenanceMiddleware.UnplannedMaintenanceSignal).Post("/etcdcertificaterenew", f.postAdminOpenShiftClusterEtcdCertificateRenew) r.With(f.maintenanceMiddleware.UnplannedMaintenanceSignal).Post("/deletemanagedresource", f.postAdminOpenShiftDeleteManagedResource) + + // MIMO + r.Route("/maintenancemanifests", func(r chi.Router) { + r.Get("/", f.getAdminMaintManifests) + r.Put("/", f.putAdminMaintManifestCreate) + r.Route("/{manifestId}", func(r chi.Router) { + r.Get("/", f.getSingleAdminMaintManifest) + r.Delete("/", f.deleteAdminMaintManifest) + r.Post("/cancel", f.postAdminMaintManifestCancel) + }) + }) }) }) @@ -493,3 +507,21 @@ func frontendOperationResultLog(log *logrus.Entry, method string, err error) { log = log.WithField("errorDetails", err.Error()) log.Info("front end operation failed") } + +// resourceIdFromURLParams returns an Azure Resource ID built out of the +// individual parameters of the URL. +func resourceIdFromURLParams(r *http.Request) string { + subID, resType, resProvider, resName, resGroupName := chi.URLParam(r, "subscriptionId"), + chi.URLParam(r, "resourceType"), + chi.URLParam(r, "resourceProviderNamespace"), + chi.URLParam(r, "resourceName"), + chi.URLParam(r, "resourceGroupName") + + return strings.ToLower(azure.Resource{ + SubscriptionID: subID, + ResourceGroup: resGroupName, + ResourceType: resType, + ResourceName: resName, + Provider: resProvider, + }.String()) +} diff --git a/pkg/frontend/openshiftcluster_putorpatch.go b/pkg/frontend/openshiftcluster_putorpatch.go index e802b5f3518..f5e9fdee896 100644 --- a/pkg/frontend/openshiftcluster_putorpatch.go +++ b/pkg/frontend/openshiftcluster_putorpatch.go @@ -198,6 +198,16 @@ func (f *frontend) _putOrPatchOpenShiftCluster(ctx context.Context, log *logrus. // provide single field json to be updated in the database. // Patch should be used for updating individual fields of the document. case http.MethodPatch: + if putOrPatchClusterParameters.apiVersion == admin.APIVersion { + // OperatorFlagsMergeStrategy==reset will place the default flags in + // the external object and then merge in the body's flags when the + // request is unmarshaled below. + err = admin.OperatorFlagsMergeStrategy(doc.OpenShiftCluster, putOrPatchClusterParameters.body) + if err != nil { + // OperatorFlagsMergeStrategy returns CloudErrors + return nil, err + } + } ext = putOrPatchClusterParameters.converter.ToExternal(doc.OpenShiftCluster) } diff --git a/pkg/frontend/openshiftcluster_putorpatch_test.go b/pkg/frontend/openshiftcluster_putorpatch_test.go index d7e32ae7d9a..5dd0d09caf0 100644 --- a/pkg/frontend/openshiftcluster_putorpatch_test.go +++ b/pkg/frontend/openshiftcluster_putorpatch_test.go @@ -58,7 +58,7 @@ func TestPutOrPatchOpenShiftClusterAdminAPI(t *testing.T) { wantStatusCode int wantEnriched []string wantDocuments func(*testdatabase.Checker) - wantResponse *admin.OpenShiftCluster + wantResponse func() *admin.OpenShiftCluster wantAsync bool wantError string wantSystemDataEnriched bool @@ -133,33 +133,34 @@ func TestPutOrPatchOpenShiftClusterAdminAPI(t *testing.T) { }, wantAsync: true, wantStatusCode: http.StatusOK, - wantResponse: &admin.OpenShiftCluster{ - ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), - Type: "Microsoft.RedHatOpenShift/openShiftClusters", - Tags: map[string]string{"tag": "will-be-kept"}, - Properties: admin.OpenShiftClusterProperties{ - ProvisioningState: admin.ProvisioningStateAdminUpdating, - LastProvisioningState: admin.ProvisioningStateSucceeded, - ClusterProfile: admin.ClusterProfile{ - FipsValidatedModules: admin.FipsValidatedModulesDisabled, - }, - MaintenanceTask: admin.MaintenanceTaskEverything, - NetworkProfile: admin.NetworkProfile{ - OutboundType: admin.OutboundTypeLoadbalancer, - LoadBalancerProfile: &admin.LoadBalancerProfile{ - ManagedOutboundIPs: &admin.ManagedOutboundIPs{ - Count: 1, + wantResponse: func() *admin.OpenShiftCluster { + return &admin.OpenShiftCluster{ + ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), + Type: "Microsoft.RedHatOpenShift/openShiftClusters", + Tags: map[string]string{"tag": "will-be-kept"}, + Properties: admin.OpenShiftClusterProperties{ + ProvisioningState: admin.ProvisioningStateAdminUpdating, + LastProvisioningState: admin.ProvisioningStateSucceeded, + ClusterProfile: admin.ClusterProfile{ + FipsValidatedModules: admin.FipsValidatedModulesDisabled, + }, + MaintenanceTask: admin.MaintenanceTaskEverything, + NetworkProfile: admin.NetworkProfile{ + OutboundType: admin.OutboundTypeLoadbalancer, + LoadBalancerProfile: &admin.LoadBalancerProfile{ + ManagedOutboundIPs: &admin.ManagedOutboundIPs{ + Count: 1, + }, }, }, + MasterProfile: admin.MasterProfile{ + EncryptionAtHost: admin.EncryptionAtHostDisabled, + }, + OperatorFlags: admin.OperatorFlags{"testFlag": "true"}, + MaintenanceState: admin.MaintenanceStateUnplanned, }, - MasterProfile: admin.MasterProfile{ - EncryptionAtHost: admin.EncryptionAtHostDisabled, - }, - OperatorFlags: admin.OperatorFlags{"testFlag": "true"}, - MaintenanceState: admin.MaintenanceStateUnplanned, - }, - }, - }, + } + }}, { name: "patch with flags merges the flags together", request: func(oc *admin.OpenShiftCluster) { @@ -234,33 +235,34 @@ func TestPutOrPatchOpenShiftClusterAdminAPI(t *testing.T) { }, wantAsync: true, wantStatusCode: http.StatusOK, - wantResponse: &admin.OpenShiftCluster{ - ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), - Type: "Microsoft.RedHatOpenShift/openShiftClusters", - Tags: map[string]string{"tag": "will-be-kept"}, - Properties: admin.OpenShiftClusterProperties{ - ProvisioningState: admin.ProvisioningStateAdminUpdating, - LastProvisioningState: admin.ProvisioningStateSucceeded, - MaintenanceTask: admin.MaintenanceTaskOperator, - NetworkProfile: admin.NetworkProfile{ - OutboundType: admin.OutboundTypeLoadbalancer, - LoadBalancerProfile: &admin.LoadBalancerProfile{ - ManagedOutboundIPs: &admin.ManagedOutboundIPs{ - Count: 1, + wantResponse: func() *admin.OpenShiftCluster { + return &admin.OpenShiftCluster{ + ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), + Type: "Microsoft.RedHatOpenShift/openShiftClusters", + Tags: map[string]string{"tag": "will-be-kept"}, + Properties: admin.OpenShiftClusterProperties{ + ProvisioningState: admin.ProvisioningStateAdminUpdating, + LastProvisioningState: admin.ProvisioningStateSucceeded, + MaintenanceTask: admin.MaintenanceTaskOperator, + NetworkProfile: admin.NetworkProfile{ + OutboundType: admin.OutboundTypeLoadbalancer, + LoadBalancerProfile: &admin.LoadBalancerProfile{ + ManagedOutboundIPs: &admin.ManagedOutboundIPs{ + Count: 1, + }, }, }, + ClusterProfile: admin.ClusterProfile{ + FipsValidatedModules: admin.FipsValidatedModulesDisabled, + }, + MasterProfile: admin.MasterProfile{ + EncryptionAtHost: admin.EncryptionAtHostDisabled, + }, + OperatorFlags: admin.OperatorFlags{"exploding-flag": "true", "overwrittenFlag": "true", "testFlag": "true"}, + MaintenanceState: admin.MaintenanceStateUnplanned, }, - ClusterProfile: admin.ClusterProfile{ - FipsValidatedModules: admin.FipsValidatedModulesDisabled, - }, - MasterProfile: admin.MasterProfile{ - EncryptionAtHost: admin.EncryptionAtHostDisabled, - }, - OperatorFlags: admin.OperatorFlags{"exploding-flag": "true", "overwrittenFlag": "true", "testFlag": "true"}, - MaintenanceState: admin.MaintenanceStateUnplanned, - }, - }, - }, + } + }}, { name: "patch an existing cluster with no flags in db will use defaults", request: func(oc *admin.OpenShiftCluster) { @@ -331,33 +333,34 @@ func TestPutOrPatchOpenShiftClusterAdminAPI(t *testing.T) { }, wantAsync: true, wantStatusCode: http.StatusOK, - wantResponse: &admin.OpenShiftCluster{ - ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), - Type: "Microsoft.RedHatOpenShift/openShiftClusters", - Tags: map[string]string{"tag": "will-be-kept"}, - Properties: admin.OpenShiftClusterProperties{ - ProvisioningState: admin.ProvisioningStateAdminUpdating, - LastProvisioningState: admin.ProvisioningStateSucceeded, - MaintenanceTask: admin.MaintenanceTaskEverything, - NetworkProfile: admin.NetworkProfile{ - OutboundType: admin.OutboundTypeLoadbalancer, - LoadBalancerProfile: &admin.LoadBalancerProfile{ - ManagedOutboundIPs: &admin.ManagedOutboundIPs{ - Count: 1, + wantResponse: func() *admin.OpenShiftCluster { + return &admin.OpenShiftCluster{ + ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), + Type: "Microsoft.RedHatOpenShift/openShiftClusters", + Tags: map[string]string{"tag": "will-be-kept"}, + Properties: admin.OpenShiftClusterProperties{ + ProvisioningState: admin.ProvisioningStateAdminUpdating, + LastProvisioningState: admin.ProvisioningStateSucceeded, + MaintenanceTask: admin.MaintenanceTaskEverything, + NetworkProfile: admin.NetworkProfile{ + OutboundType: admin.OutboundTypeLoadbalancer, + LoadBalancerProfile: &admin.LoadBalancerProfile{ + ManagedOutboundIPs: &admin.ManagedOutboundIPs{ + Count: 1, + }, }, }, + MasterProfile: admin.MasterProfile{ + EncryptionAtHost: admin.EncryptionAtHostDisabled, + }, + ClusterProfile: admin.ClusterProfile{ + FipsValidatedModules: admin.FipsValidatedModulesDisabled, + }, + OperatorFlags: admin.OperatorFlags(operator.DefaultOperatorFlags()), + MaintenanceState: admin.MaintenanceStateUnplanned, }, - MasterProfile: admin.MasterProfile{ - EncryptionAtHost: admin.EncryptionAtHostDisabled, - }, - ClusterProfile: admin.ClusterProfile{ - FipsValidatedModules: admin.FipsValidatedModulesDisabled, - }, - OperatorFlags: admin.OperatorFlags(operator.DefaultOperatorFlags()), - MaintenanceState: admin.MaintenanceStateUnplanned, - }, - }, - }, + } + }}, { name: "patch with operator update request", request: func(oc *admin.OpenShiftCluster) { @@ -426,31 +429,145 @@ func TestPutOrPatchOpenShiftClusterAdminAPI(t *testing.T) { }, wantAsync: true, wantStatusCode: http.StatusOK, - wantResponse: &admin.OpenShiftCluster{ - ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), - Type: "Microsoft.RedHatOpenShift/openShiftClusters", - Tags: map[string]string{"tag": "will-be-kept"}, - Properties: admin.OpenShiftClusterProperties{ - ProvisioningState: admin.ProvisioningStateAdminUpdating, - LastProvisioningState: admin.ProvisioningStateSucceeded, - ClusterProfile: admin.ClusterProfile{ - FipsValidatedModules: admin.FipsValidatedModulesDisabled, - }, - MaintenanceTask: admin.MaintenanceTaskOperator, - NetworkProfile: admin.NetworkProfile{ - OutboundType: admin.OutboundTypeLoadbalancer, - LoadBalancerProfile: &admin.LoadBalancerProfile{ - ManagedOutboundIPs: &admin.ManagedOutboundIPs{ - Count: 1, + wantResponse: func() *admin.OpenShiftCluster { + return &admin.OpenShiftCluster{ + ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), + Type: "Microsoft.RedHatOpenShift/openShiftClusters", + Tags: map[string]string{"tag": "will-be-kept"}, + Properties: admin.OpenShiftClusterProperties{ + ProvisioningState: admin.ProvisioningStateAdminUpdating, + LastProvisioningState: admin.ProvisioningStateSucceeded, + ClusterProfile: admin.ClusterProfile{ + FipsValidatedModules: admin.FipsValidatedModulesDisabled, + }, + MaintenanceTask: admin.MaintenanceTaskOperator, + NetworkProfile: admin.NetworkProfile{ + OutboundType: admin.OutboundTypeLoadbalancer, + LoadBalancerProfile: &admin.LoadBalancerProfile{ + ManagedOutboundIPs: &admin.ManagedOutboundIPs{ + Count: 1, + }, }, }, + MasterProfile: admin.MasterProfile{ + EncryptionAtHost: admin.EncryptionAtHostDisabled, + }, + OperatorFlags: admin.OperatorFlags(operator.DefaultOperatorFlags()), + MaintenanceState: admin.MaintenanceStateUnplanned, }, - MasterProfile: admin.MasterProfile{ - EncryptionAtHost: admin.EncryptionAtHostDisabled, + } + }, + }, + { + name: "patch with OperatorFlagsMergeStrategy=reset will reset flags to defaults and merge in request flags", + request: func(oc *admin.OpenShiftCluster) { + oc.OperatorFlagsMergeStrategy = admin.OperatorFlagsMergeStrategyReset + oc.Properties.MaintenanceTask = admin.MaintenanceTaskOperator + oc.Properties.OperatorFlags = admin.OperatorFlags{"exploding-flag": "true", "overwrittenFlag": "true"} + }, + isPatch: true, + fixture: func(f *testdatabase.Fixture) { + f.AddSubscriptionDocuments(&api.SubscriptionDocument{ + ID: mockSubID, + Subscription: &api.Subscription{ + State: api.SubscriptionStateRegistered, }, - OperatorFlags: admin.OperatorFlags(operator.DefaultOperatorFlags()), - MaintenanceState: admin.MaintenanceStateUnplanned, - }, + }) + f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{ + Key: strings.ToLower(testdatabase.GetResourcePath(mockSubID, "resourceName")), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), + Type: "Microsoft.RedHatOpenShift/openShiftClusters", + + Tags: map[string]string{"tag": "will-be-kept"}, + Properties: api.OpenShiftClusterProperties{ + ClusterProfile: api.ClusterProfile{ + FipsValidatedModules: api.FipsValidatedModulesDisabled, + }, + ProvisioningState: api.ProvisioningStateSucceeded, + OperatorFlags: api.OperatorFlags{"testFlag": "true", "overwrittenFlag": "false"}, + }, + }, + }) + }, + wantSystemDataEnriched: true, + wantEnriched: []string{testdatabase.GetResourcePath(mockSubID, "resourceName")}, + wantDocuments: func(c *testdatabase.Checker) { + expectedFlags := operator.DefaultOperatorFlags() + expectedFlags["exploding-flag"] = "true" + expectedFlags["overwrittenFlag"] = "true" + + c.AddAsyncOperationDocuments(&api.AsyncOperationDocument{ + OpenShiftClusterKey: strings.ToLower(testdatabase.GetResourcePath(mockSubID, "resourceName")), + AsyncOperation: &api.AsyncOperation{ + InitialProvisioningState: api.ProvisioningStateAdminUpdating, + ProvisioningState: api.ProvisioningStateAdminUpdating, + }, + }) + c.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{ + Key: strings.ToLower(testdatabase.GetResourcePath(mockSubID, "resourceName")), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), + Type: "Microsoft.RedHatOpenShift/openShiftClusters", + Tags: map[string]string{"tag": "will-be-kept"}, + Properties: api.OpenShiftClusterProperties{ + ProvisioningState: api.ProvisioningStateAdminUpdating, + LastProvisioningState: api.ProvisioningStateSucceeded, + MaintenanceTask: api.MaintenanceTaskOperator, + ClusterProfile: api.ClusterProfile{ + FipsValidatedModules: api.FipsValidatedModulesDisabled, + }, + NetworkProfile: api.NetworkProfile{ + OutboundType: api.OutboundTypeLoadbalancer, + PreconfiguredNSG: api.PreconfiguredNSGDisabled, + LoadBalancerProfile: &api.LoadBalancerProfile{ + ManagedOutboundIPs: &api.ManagedOutboundIPs{ + Count: 1, + }, + }, + }, + MasterProfile: api.MasterProfile{ + EncryptionAtHost: api.EncryptionAtHostDisabled, + }, + OperatorFlags: api.OperatorFlags(expectedFlags), + MaintenanceState: api.MaintenanceStateUnplanned, + }, + }, + }) + }, + wantAsync: true, + wantStatusCode: http.StatusOK, + wantResponse: func() *admin.OpenShiftCluster { + expectedFlags := operator.DefaultOperatorFlags() + expectedFlags["exploding-flag"] = "true" + expectedFlags["overwrittenFlag"] = "true" + + return &admin.OpenShiftCluster{ + ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), + Type: "Microsoft.RedHatOpenShift/openShiftClusters", + Tags: map[string]string{"tag": "will-be-kept"}, + Properties: admin.OpenShiftClusterProperties{ + ProvisioningState: admin.ProvisioningStateAdminUpdating, + LastProvisioningState: admin.ProvisioningStateSucceeded, + MaintenanceTask: admin.MaintenanceTaskOperator, + NetworkProfile: admin.NetworkProfile{ + OutboundType: admin.OutboundTypeLoadbalancer, + LoadBalancerProfile: &admin.LoadBalancerProfile{ + ManagedOutboundIPs: &admin.ManagedOutboundIPs{ + Count: 1, + }, + }, + }, + ClusterProfile: admin.ClusterProfile{ + FipsValidatedModules: admin.FipsValidatedModulesDisabled, + }, + MasterProfile: admin.MasterProfile{ + EncryptionAtHost: admin.EncryptionAtHostDisabled, + }, + OperatorFlags: admin.OperatorFlags(expectedFlags), + MaintenanceState: admin.MaintenanceStateUnplanned, + }, + } }, }, { @@ -524,31 +641,33 @@ func TestPutOrPatchOpenShiftClusterAdminAPI(t *testing.T) { }, wantAsync: true, wantStatusCode: http.StatusOK, - wantResponse: &admin.OpenShiftCluster{ - ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), - Type: "Microsoft.RedHatOpenShift/openShiftClusters", - Tags: map[string]string{"tag": "will-be-kept"}, - Properties: admin.OpenShiftClusterProperties{ - ProvisioningState: admin.ProvisioningStateAdminUpdating, - LastProvisioningState: admin.ProvisioningStateSucceeded, - ClusterProfile: admin.ClusterProfile{ - FipsValidatedModules: admin.FipsValidatedModulesDisabled, - }, - MaintenanceTask: admin.MaintenanceTaskOperator, - NetworkProfile: admin.NetworkProfile{ - OutboundType: admin.OutboundTypeLoadbalancer, - LoadBalancerProfile: &admin.LoadBalancerProfile{ - ManagedOutboundIPs: &admin.ManagedOutboundIPs{ - Count: 1, + wantResponse: func() *admin.OpenShiftCluster { + return &admin.OpenShiftCluster{ + ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), + Type: "Microsoft.RedHatOpenShift/openShiftClusters", + Tags: map[string]string{"tag": "will-be-kept"}, + Properties: admin.OpenShiftClusterProperties{ + ProvisioningState: admin.ProvisioningStateAdminUpdating, + LastProvisioningState: admin.ProvisioningStateSucceeded, + ClusterProfile: admin.ClusterProfile{ + FipsValidatedModules: admin.FipsValidatedModulesDisabled, + }, + MaintenanceTask: admin.MaintenanceTaskOperator, + NetworkProfile: admin.NetworkProfile{ + OutboundType: admin.OutboundTypeLoadbalancer, + LoadBalancerProfile: &admin.LoadBalancerProfile{ + ManagedOutboundIPs: &admin.ManagedOutboundIPs{ + Count: 1, + }, }, }, + MasterProfile: admin.MasterProfile{ + EncryptionAtHost: admin.EncryptionAtHostDisabled, + }, + OperatorFlags: admin.OperatorFlags{"testFlag": "true"}, + MaintenanceState: admin.MaintenanceStateUnplanned, }, - MasterProfile: admin.MasterProfile{ - EncryptionAtHost: admin.EncryptionAtHostDisabled, - }, - OperatorFlags: admin.OperatorFlags{"testFlag": "true"}, - MaintenanceState: admin.MaintenanceStateUnplanned, - }, + } }, }, { @@ -631,6 +750,7 @@ func TestPutOrPatchOpenShiftClusterAdminAPI(t *testing.T) { wantAsync: false, wantStatusCode: http.StatusBadRequest, wantError: `400: PropertyChangeNotAllowed: properties.registryProfiles: Changing property 'properties.registryProfiles' is not allowed.`, + wantResponse: func() *admin.OpenShiftCluster { return nil }, }, { name: "patch an empty maintenance state cluster with maintenance pending request", @@ -702,31 +822,33 @@ func TestPutOrPatchOpenShiftClusterAdminAPI(t *testing.T) { }, wantAsync: true, wantStatusCode: http.StatusOK, - wantResponse: &admin.OpenShiftCluster{ - ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), - Type: "Microsoft.RedHatOpenShift/openShiftClusters", - Tags: map[string]string{"tag": "will-be-kept"}, - Properties: admin.OpenShiftClusterProperties{ - ProvisioningState: admin.ProvisioningStateSucceeded, - LastProvisioningState: admin.ProvisioningStateSucceeded, - ClusterProfile: admin.ClusterProfile{ - FipsValidatedModules: admin.FipsValidatedModulesDisabled, - }, - MaintenanceTask: "", - NetworkProfile: admin.NetworkProfile{ - OutboundType: admin.OutboundTypeLoadbalancer, - LoadBalancerProfile: &admin.LoadBalancerProfile{ - ManagedOutboundIPs: &admin.ManagedOutboundIPs{ - Count: 1, + wantResponse: func() *admin.OpenShiftCluster { + return &admin.OpenShiftCluster{ + ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), + Type: "Microsoft.RedHatOpenShift/openShiftClusters", + Tags: map[string]string{"tag": "will-be-kept"}, + Properties: admin.OpenShiftClusterProperties{ + ProvisioningState: admin.ProvisioningStateSucceeded, + LastProvisioningState: admin.ProvisioningStateSucceeded, + ClusterProfile: admin.ClusterProfile{ + FipsValidatedModules: admin.FipsValidatedModulesDisabled, + }, + MaintenanceTask: "", + NetworkProfile: admin.NetworkProfile{ + OutboundType: admin.OutboundTypeLoadbalancer, + LoadBalancerProfile: &admin.LoadBalancerProfile{ + ManagedOutboundIPs: &admin.ManagedOutboundIPs{ + Count: 1, + }, }, }, + MaintenanceState: admin.MaintenanceStatePending, + MasterProfile: admin.MasterProfile{ + EncryptionAtHost: admin.EncryptionAtHostDisabled, + }, + OperatorFlags: admin.OperatorFlags(operator.DefaultOperatorFlags()), }, - MaintenanceState: admin.MaintenanceStatePending, - MasterProfile: admin.MasterProfile{ - EncryptionAtHost: admin.EncryptionAtHostDisabled, - }, - OperatorFlags: admin.OperatorFlags(operator.DefaultOperatorFlags()), - }, + } }, }, { @@ -800,31 +922,33 @@ func TestPutOrPatchOpenShiftClusterAdminAPI(t *testing.T) { }, wantAsync: true, wantStatusCode: http.StatusOK, - wantResponse: &admin.OpenShiftCluster{ - ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), - Type: "Microsoft.RedHatOpenShift/openShiftClusters", - Tags: map[string]string{"tag": "will-be-kept"}, - Properties: admin.OpenShiftClusterProperties{ - ProvisioningState: admin.ProvisioningStateSucceeded, - LastProvisioningState: admin.ProvisioningStateSucceeded, - ClusterProfile: admin.ClusterProfile{ - FipsValidatedModules: admin.FipsValidatedModulesDisabled, - }, - MaintenanceTask: "", - NetworkProfile: admin.NetworkProfile{ - OutboundType: admin.OutboundTypeLoadbalancer, - LoadBalancerProfile: &admin.LoadBalancerProfile{ - ManagedOutboundIPs: &admin.ManagedOutboundIPs{ - Count: 1, + wantResponse: func() *admin.OpenShiftCluster { + return &admin.OpenShiftCluster{ + ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), + Type: "Microsoft.RedHatOpenShift/openShiftClusters", + Tags: map[string]string{"tag": "will-be-kept"}, + Properties: admin.OpenShiftClusterProperties{ + ProvisioningState: admin.ProvisioningStateSucceeded, + LastProvisioningState: admin.ProvisioningStateSucceeded, + ClusterProfile: admin.ClusterProfile{ + FipsValidatedModules: admin.FipsValidatedModulesDisabled, + }, + MaintenanceTask: "", + NetworkProfile: admin.NetworkProfile{ + OutboundType: admin.OutboundTypeLoadbalancer, + LoadBalancerProfile: &admin.LoadBalancerProfile{ + ManagedOutboundIPs: &admin.ManagedOutboundIPs{ + Count: 1, + }, }, }, + MaintenanceState: admin.MaintenanceStatePending, + MasterProfile: admin.MasterProfile{ + EncryptionAtHost: admin.EncryptionAtHostDisabled, + }, + OperatorFlags: admin.OperatorFlags(operator.DefaultOperatorFlags()), }, - MaintenanceState: admin.MaintenanceStatePending, - MasterProfile: admin.MasterProfile{ - EncryptionAtHost: admin.EncryptionAtHostDisabled, - }, - OperatorFlags: admin.OperatorFlags(operator.DefaultOperatorFlags()), - }, + } }, }, { @@ -897,31 +1021,33 @@ func TestPutOrPatchOpenShiftClusterAdminAPI(t *testing.T) { }, wantAsync: true, wantStatusCode: http.StatusOK, - wantResponse: &admin.OpenShiftCluster{ - ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), - Type: "Microsoft.RedHatOpenShift/openShiftClusters", - Tags: map[string]string{"tag": "will-be-kept"}, - Properties: admin.OpenShiftClusterProperties{ - ProvisioningState: admin.ProvisioningStateAdminUpdating, - LastProvisioningState: admin.ProvisioningStateSucceeded, - ClusterProfile: admin.ClusterProfile{ - FipsValidatedModules: admin.FipsValidatedModulesDisabled, - }, - MaintenanceTask: admin.MaintenanceTaskEverything, - NetworkProfile: admin.NetworkProfile{ - OutboundType: admin.OutboundTypeLoadbalancer, - LoadBalancerProfile: &admin.LoadBalancerProfile{ - ManagedOutboundIPs: &admin.ManagedOutboundIPs{ - Count: 1, + wantResponse: func() *admin.OpenShiftCluster { + return &admin.OpenShiftCluster{ + ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), + Type: "Microsoft.RedHatOpenShift/openShiftClusters", + Tags: map[string]string{"tag": "will-be-kept"}, + Properties: admin.OpenShiftClusterProperties{ + ProvisioningState: admin.ProvisioningStateAdminUpdating, + LastProvisioningState: admin.ProvisioningStateSucceeded, + ClusterProfile: admin.ClusterProfile{ + FipsValidatedModules: admin.FipsValidatedModulesDisabled, + }, + MaintenanceTask: admin.MaintenanceTaskEverything, + NetworkProfile: admin.NetworkProfile{ + OutboundType: admin.OutboundTypeLoadbalancer, + LoadBalancerProfile: &admin.LoadBalancerProfile{ + ManagedOutboundIPs: &admin.ManagedOutboundIPs{ + Count: 1, + }, }, }, + MasterProfile: admin.MasterProfile{ + EncryptionAtHost: admin.EncryptionAtHostDisabled, + }, + OperatorFlags: admin.OperatorFlags(operator.DefaultOperatorFlags()), + MaintenanceState: admin.MaintenanceStatePlanned, }, - MasterProfile: admin.MasterProfile{ - EncryptionAtHost: admin.EncryptionAtHostDisabled, - }, - OperatorFlags: admin.OperatorFlags(operator.DefaultOperatorFlags()), - MaintenanceState: admin.MaintenanceStatePlanned, - }, + } }, }, { @@ -995,31 +1121,33 @@ func TestPutOrPatchOpenShiftClusterAdminAPI(t *testing.T) { }, wantAsync: true, wantStatusCode: http.StatusOK, - wantResponse: &admin.OpenShiftCluster{ - ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), - Type: "Microsoft.RedHatOpenShift/openShiftClusters", - Tags: map[string]string{"tag": "will-be-kept"}, - Properties: admin.OpenShiftClusterProperties{ - ProvisioningState: admin.ProvisioningStateSucceeded, - LastProvisioningState: admin.ProvisioningStateSucceeded, - ClusterProfile: admin.ClusterProfile{ - FipsValidatedModules: admin.FipsValidatedModulesDisabled, - }, - MaintenanceTask: "", - NetworkProfile: admin.NetworkProfile{ - OutboundType: admin.OutboundTypeLoadbalancer, - LoadBalancerProfile: &admin.LoadBalancerProfile{ - ManagedOutboundIPs: &admin.ManagedOutboundIPs{ - Count: 1, + wantResponse: func() *admin.OpenShiftCluster { + return &admin.OpenShiftCluster{ + ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), + Type: "Microsoft.RedHatOpenShift/openShiftClusters", + Tags: map[string]string{"tag": "will-be-kept"}, + Properties: admin.OpenShiftClusterProperties{ + ProvisioningState: admin.ProvisioningStateSucceeded, + LastProvisioningState: admin.ProvisioningStateSucceeded, + ClusterProfile: admin.ClusterProfile{ + FipsValidatedModules: admin.FipsValidatedModulesDisabled, + }, + MaintenanceTask: "", + NetworkProfile: admin.NetworkProfile{ + OutboundType: admin.OutboundTypeLoadbalancer, + LoadBalancerProfile: &admin.LoadBalancerProfile{ + ManagedOutboundIPs: &admin.ManagedOutboundIPs{ + Count: 1, + }, }, }, + MaintenanceState: admin.MaintenanceStateNone, + MasterProfile: admin.MasterProfile{ + EncryptionAtHost: admin.EncryptionAtHostDisabled, + }, + OperatorFlags: admin.OperatorFlags(operator.DefaultOperatorFlags()), }, - MaintenanceState: admin.MaintenanceStateNone, - MasterProfile: admin.MasterProfile{ - EncryptionAtHost: admin.EncryptionAtHostDisabled, - }, - OperatorFlags: admin.OperatorFlags(operator.DefaultOperatorFlags()), - }, + } }, }, { @@ -1093,31 +1221,33 @@ func TestPutOrPatchOpenShiftClusterAdminAPI(t *testing.T) { }, wantAsync: true, wantStatusCode: http.StatusOK, - wantResponse: &admin.OpenShiftCluster{ - ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), - Type: "Microsoft.RedHatOpenShift/openShiftClusters", - Tags: map[string]string{"tag": "will-be-kept"}, - Properties: admin.OpenShiftClusterProperties{ - ProvisioningState: admin.ProvisioningStateSucceeded, - LastProvisioningState: admin.ProvisioningStateSucceeded, - ClusterProfile: admin.ClusterProfile{ - FipsValidatedModules: admin.FipsValidatedModulesDisabled, - }, - MaintenanceTask: "", - NetworkProfile: admin.NetworkProfile{ - OutboundType: admin.OutboundTypeLoadbalancer, - LoadBalancerProfile: &admin.LoadBalancerProfile{ - ManagedOutboundIPs: &admin.ManagedOutboundIPs{ - Count: 1, + wantResponse: func() *admin.OpenShiftCluster { + return &admin.OpenShiftCluster{ + ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), + Type: "Microsoft.RedHatOpenShift/openShiftClusters", + Tags: map[string]string{"tag": "will-be-kept"}, + Properties: admin.OpenShiftClusterProperties{ + ProvisioningState: admin.ProvisioningStateSucceeded, + LastProvisioningState: admin.ProvisioningStateSucceeded, + ClusterProfile: admin.ClusterProfile{ + FipsValidatedModules: admin.FipsValidatedModulesDisabled, + }, + MaintenanceTask: "", + NetworkProfile: admin.NetworkProfile{ + OutboundType: admin.OutboundTypeLoadbalancer, + LoadBalancerProfile: &admin.LoadBalancerProfile{ + ManagedOutboundIPs: &admin.ManagedOutboundIPs{ + Count: 1, + }, }, }, + MaintenanceState: admin.MaintenanceStateNone, + MasterProfile: admin.MasterProfile{ + EncryptionAtHost: admin.EncryptionAtHostDisabled, + }, + OperatorFlags: admin.OperatorFlags(operator.DefaultOperatorFlags()), }, - MaintenanceState: admin.MaintenanceStateNone, - MasterProfile: admin.MasterProfile{ - EncryptionAtHost: admin.EncryptionAtHostDisabled, - }, - OperatorFlags: admin.OperatorFlags(operator.DefaultOperatorFlags()), - }, + } }, }, { @@ -1189,31 +1319,33 @@ func TestPutOrPatchOpenShiftClusterAdminAPI(t *testing.T) { }, wantAsync: true, wantStatusCode: http.StatusOK, - wantResponse: &admin.OpenShiftCluster{ - ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), - Type: "Microsoft.RedHatOpenShift/openShiftClusters", - Tags: map[string]string{"tag": "will-be-kept"}, - Properties: admin.OpenShiftClusterProperties{ - ProvisioningState: admin.ProvisioningStateAdminUpdating, - LastProvisioningState: admin.ProvisioningStateSucceeded, - ClusterProfile: admin.ClusterProfile{ - FipsValidatedModules: admin.FipsValidatedModulesDisabled, - }, - MaintenanceTask: admin.MaintenanceTaskEverything, - NetworkProfile: admin.NetworkProfile{ - OutboundType: admin.OutboundTypeLoadbalancer, - LoadBalancerProfile: &admin.LoadBalancerProfile{ - ManagedOutboundIPs: &admin.ManagedOutboundIPs{ - Count: 1, + wantResponse: func() *admin.OpenShiftCluster { + return &admin.OpenShiftCluster{ + ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), + Type: "Microsoft.RedHatOpenShift/openShiftClusters", + Tags: map[string]string{"tag": "will-be-kept"}, + Properties: admin.OpenShiftClusterProperties{ + ProvisioningState: admin.ProvisioningStateAdminUpdating, + LastProvisioningState: admin.ProvisioningStateSucceeded, + ClusterProfile: admin.ClusterProfile{ + FipsValidatedModules: admin.FipsValidatedModulesDisabled, + }, + MaintenanceTask: admin.MaintenanceTaskEverything, + NetworkProfile: admin.NetworkProfile{ + OutboundType: admin.OutboundTypeLoadbalancer, + LoadBalancerProfile: &admin.LoadBalancerProfile{ + ManagedOutboundIPs: &admin.ManagedOutboundIPs{ + Count: 1, + }, }, }, + MasterProfile: admin.MasterProfile{ + EncryptionAtHost: admin.EncryptionAtHostDisabled, + }, + OperatorFlags: admin.OperatorFlags{"testFlag": "true"}, + MaintenanceState: admin.MaintenanceStateUnplanned, }, - MasterProfile: admin.MasterProfile{ - EncryptionAtHost: admin.EncryptionAtHostDisabled, - }, - OperatorFlags: admin.OperatorFlags{"testFlag": "true"}, - MaintenanceState: admin.MaintenanceStateUnplanned, - }, + } }, }, { @@ -1289,31 +1421,33 @@ func TestPutOrPatchOpenShiftClusterAdminAPI(t *testing.T) { }, wantAsync: true, wantStatusCode: http.StatusOK, - wantResponse: &admin.OpenShiftCluster{ - ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), - Type: "Microsoft.RedHatOpenShift/openShiftClusters", - Tags: map[string]string{"tag": "will-be-kept"}, - Properties: admin.OpenShiftClusterProperties{ - ProvisioningState: admin.ProvisioningStateSucceeded, - LastProvisioningState: admin.ProvisioningStateSucceeded, - ClusterProfile: admin.ClusterProfile{ - FipsValidatedModules: admin.FipsValidatedModulesDisabled, - }, - NetworkProfile: admin.NetworkProfile{ - OutboundType: admin.OutboundTypeLoadbalancer, - LoadBalancerProfile: &admin.LoadBalancerProfile{ - ManagedOutboundIPs: &admin.ManagedOutboundIPs{ - Count: 1, + wantResponse: func() *admin.OpenShiftCluster { + return &admin.OpenShiftCluster{ + ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), + Type: "Microsoft.RedHatOpenShift/openShiftClusters", + Tags: map[string]string{"tag": "will-be-kept"}, + Properties: admin.OpenShiftClusterProperties{ + ProvisioningState: admin.ProvisioningStateSucceeded, + LastProvisioningState: admin.ProvisioningStateSucceeded, + ClusterProfile: admin.ClusterProfile{ + FipsValidatedModules: admin.FipsValidatedModulesDisabled, + }, + NetworkProfile: admin.NetworkProfile{ + OutboundType: admin.OutboundTypeLoadbalancer, + LoadBalancerProfile: &admin.LoadBalancerProfile{ + ManagedOutboundIPs: &admin.ManagedOutboundIPs{ + Count: 1, + }, }, }, + MasterProfile: admin.MasterProfile{ + EncryptionAtHost: admin.EncryptionAtHostDisabled, + }, + OperatorFlags: admin.OperatorFlags{"testFlag": "true"}, + MaintenanceState: admin.MaintenanceStateCustomerActionNeeded, + LastAdminUpdateError: "error", }, - MasterProfile: admin.MasterProfile{ - EncryptionAtHost: admin.EncryptionAtHostDisabled, - }, - OperatorFlags: admin.OperatorFlags{"testFlag": "true"}, - MaintenanceState: admin.MaintenanceStateCustomerActionNeeded, - LastAdminUpdateError: "error", - }, + } }, }, { @@ -1389,31 +1523,33 @@ func TestPutOrPatchOpenShiftClusterAdminAPI(t *testing.T) { }, wantAsync: true, wantStatusCode: http.StatusOK, - wantResponse: &admin.OpenShiftCluster{ - ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), - Type: "Microsoft.RedHatOpenShift/openShiftClusters", - Tags: map[string]string{"tag": "will-be-kept"}, - Properties: admin.OpenShiftClusterProperties{ - ProvisioningState: admin.ProvisioningStateSucceeded, - FailedProvisioningState: admin.ProvisioningStateUpdating, - ClusterProfile: admin.ClusterProfile{ - FipsValidatedModules: admin.FipsValidatedModulesDisabled, - }, - NetworkProfile: admin.NetworkProfile{ - OutboundType: admin.OutboundTypeLoadbalancer, - LoadBalancerProfile: &admin.LoadBalancerProfile{ - ManagedOutboundIPs: &admin.ManagedOutboundIPs{ - Count: 1, + wantResponse: func() *admin.OpenShiftCluster { + return &admin.OpenShiftCluster{ + ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), + Type: "Microsoft.RedHatOpenShift/openShiftClusters", + Tags: map[string]string{"tag": "will-be-kept"}, + Properties: admin.OpenShiftClusterProperties{ + ProvisioningState: admin.ProvisioningStateSucceeded, + FailedProvisioningState: admin.ProvisioningStateUpdating, + ClusterProfile: admin.ClusterProfile{ + FipsValidatedModules: admin.FipsValidatedModulesDisabled, + }, + NetworkProfile: admin.NetworkProfile{ + OutboundType: admin.OutboundTypeLoadbalancer, + LoadBalancerProfile: &admin.LoadBalancerProfile{ + ManagedOutboundIPs: &admin.ManagedOutboundIPs{ + Count: 1, + }, }, }, + MasterProfile: admin.MasterProfile{ + EncryptionAtHost: admin.EncryptionAtHostDisabled, + }, + OperatorFlags: admin.OperatorFlags{"testFlag": "true"}, + MaintenanceState: admin.MaintenanceStateCustomerActionNeeded, + LastAdminUpdateError: "error", }, - MasterProfile: admin.MasterProfile{ - EncryptionAtHost: admin.EncryptionAtHostDisabled, - }, - OperatorFlags: admin.OperatorFlags{"testFlag": "true"}, - MaintenanceState: admin.MaintenanceStateCustomerActionNeeded, - LastAdminUpdateError: "error", - }, + } }, }, { @@ -1489,31 +1625,33 @@ func TestPutOrPatchOpenShiftClusterAdminAPI(t *testing.T) { }, wantAsync: true, wantStatusCode: http.StatusOK, - wantResponse: &admin.OpenShiftCluster{ - ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), - Type: "Microsoft.RedHatOpenShift/openShiftClusters", - Tags: map[string]string{"tag": "will-be-kept"}, - Properties: admin.OpenShiftClusterProperties{ - ProvisioningState: admin.ProvisioningStateSucceeded, - FailedProvisioningState: admin.ProvisioningStateUpdating, - ClusterProfile: admin.ClusterProfile{ - FipsValidatedModules: admin.FipsValidatedModulesDisabled, - }, - NetworkProfile: admin.NetworkProfile{ - OutboundType: admin.OutboundTypeLoadbalancer, - LoadBalancerProfile: &admin.LoadBalancerProfile{ - ManagedOutboundIPs: &admin.ManagedOutboundIPs{ - Count: 1, + wantResponse: func() *admin.OpenShiftCluster { + return &admin.OpenShiftCluster{ + ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), + Type: "Microsoft.RedHatOpenShift/openShiftClusters", + Tags: map[string]string{"tag": "will-be-kept"}, + Properties: admin.OpenShiftClusterProperties{ + ProvisioningState: admin.ProvisioningStateSucceeded, + FailedProvisioningState: admin.ProvisioningStateUpdating, + ClusterProfile: admin.ClusterProfile{ + FipsValidatedModules: admin.FipsValidatedModulesDisabled, + }, + NetworkProfile: admin.NetworkProfile{ + OutboundType: admin.OutboundTypeLoadbalancer, + LoadBalancerProfile: &admin.LoadBalancerProfile{ + ManagedOutboundIPs: &admin.ManagedOutboundIPs{ + Count: 1, + }, }, }, + MasterProfile: admin.MasterProfile{ + EncryptionAtHost: admin.EncryptionAtHostDisabled, + }, + OperatorFlags: admin.OperatorFlags{"testFlag": "true"}, + MaintenanceState: admin.MaintenanceStateCustomerActionNeeded, + LastAdminUpdateError: "error", }, - MasterProfile: admin.MasterProfile{ - EncryptionAtHost: admin.EncryptionAtHostDisabled, - }, - OperatorFlags: admin.OperatorFlags{"testFlag": "true"}, - MaintenanceState: admin.MaintenanceStateCustomerActionNeeded, - LastAdminUpdateError: "error", - }, + } }, }, { @@ -1589,31 +1727,33 @@ func TestPutOrPatchOpenShiftClusterAdminAPI(t *testing.T) { }, wantAsync: true, wantStatusCode: http.StatusOK, - wantResponse: &admin.OpenShiftCluster{ - ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), - Type: "Microsoft.RedHatOpenShift/openShiftClusters", - Tags: map[string]string{"tag": "will-be-kept"}, - Properties: admin.OpenShiftClusterProperties{ - ProvisioningState: admin.ProvisioningStateSucceeded, - FailedProvisioningState: admin.ProvisioningStateUpdating, - ClusterProfile: admin.ClusterProfile{ - FipsValidatedModules: admin.FipsValidatedModulesDisabled, - }, - NetworkProfile: admin.NetworkProfile{ - OutboundType: admin.OutboundTypeLoadbalancer, - LoadBalancerProfile: &admin.LoadBalancerProfile{ - ManagedOutboundIPs: &admin.ManagedOutboundIPs{ - Count: 1, + wantResponse: func() *admin.OpenShiftCluster { + return &admin.OpenShiftCluster{ + ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), + Type: "Microsoft.RedHatOpenShift/openShiftClusters", + Tags: map[string]string{"tag": "will-be-kept"}, + Properties: admin.OpenShiftClusterProperties{ + ProvisioningState: admin.ProvisioningStateSucceeded, + FailedProvisioningState: admin.ProvisioningStateUpdating, + ClusterProfile: admin.ClusterProfile{ + FipsValidatedModules: admin.FipsValidatedModulesDisabled, + }, + NetworkProfile: admin.NetworkProfile{ + OutboundType: admin.OutboundTypeLoadbalancer, + LoadBalancerProfile: &admin.LoadBalancerProfile{ + ManagedOutboundIPs: &admin.ManagedOutboundIPs{ + Count: 1, + }, }, }, + MasterProfile: admin.MasterProfile{ + EncryptionAtHost: admin.EncryptionAtHostDisabled, + }, + OperatorFlags: admin.OperatorFlags{"testFlag": "true"}, + MaintenanceState: admin.MaintenanceStateNone, + LastAdminUpdateError: "error", }, - MasterProfile: admin.MasterProfile{ - EncryptionAtHost: admin.EncryptionAtHostDisabled, - }, - OperatorFlags: admin.OperatorFlags{"testFlag": "true"}, - MaintenanceState: admin.MaintenanceStateNone, - LastAdminUpdateError: "error", - }, + } }, }, { @@ -1689,32 +1829,34 @@ func TestPutOrPatchOpenShiftClusterAdminAPI(t *testing.T) { }, wantAsync: true, wantStatusCode: http.StatusOK, - wantResponse: &admin.OpenShiftCluster{ - ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), - Type: "Microsoft.RedHatOpenShift/openShiftClusters", - Tags: map[string]string{"tag": "will-be-kept"}, - Properties: admin.OpenShiftClusterProperties{ - ProvisioningState: admin.ProvisioningStateAdminUpdating, - LastProvisioningState: admin.ProvisioningStateSucceeded, - FailedProvisioningState: admin.ProvisioningStateUpdating, - ClusterProfile: admin.ClusterProfile{ - FipsValidatedModules: admin.FipsValidatedModulesDisabled, - }, - NetworkProfile: admin.NetworkProfile{ - OutboundType: admin.OutboundTypeLoadbalancer, - LoadBalancerProfile: &admin.LoadBalancerProfile{ - ManagedOutboundIPs: &admin.ManagedOutboundIPs{ - Count: 1, + wantResponse: func() *admin.OpenShiftCluster { + return &admin.OpenShiftCluster{ + ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), + Type: "Microsoft.RedHatOpenShift/openShiftClusters", + Tags: map[string]string{"tag": "will-be-kept"}, + Properties: admin.OpenShiftClusterProperties{ + ProvisioningState: admin.ProvisioningStateAdminUpdating, + LastProvisioningState: admin.ProvisioningStateSucceeded, + FailedProvisioningState: admin.ProvisioningStateUpdating, + ClusterProfile: admin.ClusterProfile{ + FipsValidatedModules: admin.FipsValidatedModulesDisabled, + }, + NetworkProfile: admin.NetworkProfile{ + OutboundType: admin.OutboundTypeLoadbalancer, + LoadBalancerProfile: &admin.LoadBalancerProfile{ + ManagedOutboundIPs: &admin.ManagedOutboundIPs{ + Count: 1, + }, }, }, + MasterProfile: admin.MasterProfile{ + EncryptionAtHost: admin.EncryptionAtHostDisabled, + }, + OperatorFlags: admin.OperatorFlags{"testFlag": "true"}, + MaintenanceState: admin.MaintenanceStateUnplanned, + MaintenanceTask: admin.MaintenanceTaskEverything, }, - MasterProfile: admin.MasterProfile{ - EncryptionAtHost: admin.EncryptionAtHostDisabled, - }, - OperatorFlags: admin.OperatorFlags{"testFlag": "true"}, - MaintenanceState: admin.MaintenanceStateUnplanned, - MaintenanceTask: admin.MaintenanceTaskEverything, - }, + } }, }, } { @@ -1772,7 +1914,7 @@ func TestPutOrPatchOpenShiftClusterAdminAPI(t *testing.T) { t.Error(azureAsyncOperation) } } - err = validateResponse(resp, b, tt.wantStatusCode, tt.wantError, tt.wantResponse) + err = validateResponse(resp, b, tt.wantStatusCode, tt.wantError, tt.wantResponse()) if err != nil { t.Error(err) } diff --git a/pkg/frontend/shared_test.go b/pkg/frontend/shared_test.go index 0bd32983bef..4b7e0240d27 100644 --- a/pkg/frontend/shared_test.go +++ b/pkg/frontend/shared_test.go @@ -15,6 +15,7 @@ import ( "net/http" "reflect" "testing" + "time" "github.com/go-test/deep" "github.com/sirupsen/logrus" @@ -83,6 +84,8 @@ type testInfra struct { openShiftVersionsDatabase database.OpenShiftVersions platformWorkloadIdentityRoleSetsClient *cosmosdb.FakePlatformWorkloadIdentityRoleSetDocumentClient platformWorkloadIdentityRoleSetsDatabase database.PlatformWorkloadIdentityRoleSets + maintenanceManifestsClient *cosmosdb.FakeMaintenanceManifestDocumentClient + maintenanceManifestsDatabase database.MaintenanceManifests } func newTestInfra(t *testing.T) *testInfra { @@ -204,6 +207,13 @@ func (ti *testInfra) WithClusterManagerConfigurations() *testInfra { return ti } +func (ti *testInfra) WithMaintenanceManifests(now func() time.Time) *testInfra { + ti.maintenanceManifestsDatabase, ti.maintenanceManifestsClient = testdatabase.NewFakeMaintenanceManifests(now) + ti.fixture.WithMaintenanceManifests(ti.maintenanceManifestsDatabase) + ti.dbGroup.WithMaintenanceManifests(ti.maintenanceManifestsDatabase) + return ti +} + func (ti *testInfra) done() { ti.controller.Finish() ti.cli.CloseIdleConnections() @@ -254,7 +264,7 @@ func (ti *testInfra) request(method, url string, header http.Header, in interfac func validateResponse(resp *http.Response, b []byte, wantStatusCode int, wantError string, wantResponse interface{}) error { if resp.StatusCode != wantStatusCode { - return fmt.Errorf("unexpected status code %d, wanted %d", resp.StatusCode, wantStatusCode) + return fmt.Errorf("unexpected status code %d, wanted %d: %s", resp.StatusCode, wantStatusCode, string(b)) } if wantError != "" { @@ -264,8 +274,8 @@ func validateResponse(resp *http.Response, b []byte, wantStatusCode int, wantErr return err } - if cloudErr.Error() != wantError { - return fmt.Errorf("unexpected error %s, wanted %s", cloudErr.Error(), wantError) + if diff := deep.Equal(cloudErr.Error(), wantError); diff != nil { + return fmt.Errorf("unexpected error %s, wanted %s (%s)", cloudErr.Error(), wantError, diff) } return nil diff --git a/pkg/mimo/actuator/actuator_test.go b/pkg/mimo/actuator/actuator_test.go new file mode 100644 index 00000000000..1cfc82a9844 --- /dev/null +++ b/pkg/mimo/actuator/actuator_test.go @@ -0,0 +1,321 @@ +package actuator + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/sirupsen/logrus" + "go.uber.org/mock/gomock" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/database" + "github.com/Azure/ARO-RP/pkg/database/cosmosdb" + "github.com/Azure/ARO-RP/pkg/env" + "github.com/Azure/ARO-RP/pkg/mimo/tasks" + "github.com/Azure/ARO-RP/pkg/util/mimo" + mock_env "github.com/Azure/ARO-RP/pkg/util/mocks/env" + testdatabase "github.com/Azure/ARO-RP/test/database" + testlog "github.com/Azure/ARO-RP/test/util/log" +) + +var _ = Describe("MIMO Actuator", Ordered, func() { + var fixtures *testdatabase.Fixture + var checker *testdatabase.Checker + var manifests database.MaintenanceManifests + var manifestsClient *cosmosdb.FakeMaintenanceManifestDocumentClient + var clusters database.OpenShiftClusters + //var clustersClient cosmosdb.OpenShiftClusterDocumentClient + + var a Actuator + + var ctx context.Context + var cancel context.CancelFunc + + //var hook *test.Hook + var log *logrus.Entry + var _env env.Interface + + var controller *gomock.Controller + + mockSubID := "00000000-0000-0000-0000-000000000000" + clusterResourceID := fmt.Sprintf("/subscriptions/%s/resourcegroups/resourceGroup/providers/Microsoft.RedHatOpenShift/openShiftClusters/resourceName", mockSubID) + + AfterAll(func() { + if cancel != nil { + cancel() + } + + if controller != nil { + controller.Finish() + } + }) + + BeforeAll(func() { + controller = gomock.NewController(nil) + _env = mock_env.NewMockInterface(controller) + + ctx, cancel = context.WithCancel(context.Background()) + + _, log = testlog.New() + + fixtures = testdatabase.NewFixture() + checker = testdatabase.NewChecker() + }) + + BeforeEach(func() { + now := func() time.Time { return time.Unix(120, 0) } + manifests, manifestsClient = testdatabase.NewFakeMaintenanceManifests(now) + clusters, _ = testdatabase.NewFakeOpenShiftClusters() + + a = &actuator{ + log: log, + env: _env, + + clusterResourceID: strings.ToLower(clusterResourceID), + + mmf: manifests, + oc: clusters, + + tasks: map[string]tasks.MaintenanceTask{}, + now: now, + } + }) + + JustBeforeEach(func() { + err := fixtures.WithOpenShiftClusters(clusters).WithMaintenanceManifests(manifests).Create() + Expect(err).ToNot(HaveOccurred()) + }) + + When("old manifest", func() { + var manifestID string + + BeforeEach(func() { + fixtures.Clear() + fixtures.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{ + Key: strings.ToLower(clusterResourceID), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: clusterResourceID, + }, + }) + + manifestID = manifests.NewUUID() + fixtures.AddMaintenanceManifestDocuments(&api.MaintenanceManifestDocument{ + ID: manifestID, + ClusterResourceID: strings.ToLower(clusterResourceID), + MaintenanceManifest: api.MaintenanceManifest{ + State: api.MaintenanceManifestStatePending, + RunBefore: 60, + RunAfter: 0, + }, + }) + + checker.Clear() + checker.AddMaintenanceManifestDocuments(&api.MaintenanceManifestDocument{ + ID: manifestID, + ClusterResourceID: strings.ToLower(clusterResourceID), + MaintenanceManifest: api.MaintenanceManifest{ + State: api.MaintenanceManifestStateTimedOut, + StatusText: "timed out at 1970-01-01 00:02:00 +0000 UTC", + RunBefore: 60, + RunAfter: 0, + }, + }) + }) + + It("expires them", func() { + didWork, err := a.Process(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(didWork).To(BeFalse()) + + errs := checker.CheckMaintenanceManifests(manifestsClient) + Expect(errs).To(BeNil(), fmt.Sprintf("%v", errs)) + }) + }) + + When("new manifest", func() { + var manifestID string + + BeforeEach(func() { + fixtures.Clear() + fixtures.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{ + Key: strings.ToLower(clusterResourceID), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: clusterResourceID, + }, + }) + + manifestID = manifests.NewUUID() + fixtures.AddMaintenanceManifestDocuments(&api.MaintenanceManifestDocument{ + ID: manifestID, + ClusterResourceID: strings.ToLower(clusterResourceID), + MaintenanceManifest: api.MaintenanceManifest{ + State: api.MaintenanceManifestStatePending, + MaintenanceTaskID: "0", + RunBefore: 600, + RunAfter: 0, + }, + }) + + checker.Clear() + checker.AddMaintenanceManifestDocuments(&api.MaintenanceManifestDocument{ + ID: manifestID, + ClusterResourceID: strings.ToLower(clusterResourceID), + MaintenanceManifest: api.MaintenanceManifest{ + State: api.MaintenanceManifestStateCompleted, + MaintenanceTaskID: "0", + StatusText: "done", + RunBefore: 600, + RunAfter: 0, + }, + }) + }) + + It("runs them", func() { + a.AddMaintenanceTasks(map[string]tasks.MaintenanceTask{ + "0": func(th mimo.TaskContext, mmd *api.MaintenanceManifestDocument, oscd *api.OpenShiftClusterDocument) (api.MaintenanceManifestState, string) { + return api.MaintenanceManifestStateCompleted, "done" + }, + }) + + didWork, err := a.Process(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(didWork).To(BeTrue()) + + errs := checker.CheckMaintenanceManifests(manifestsClient) + Expect(errs).To(BeNil(), fmt.Sprintf("%v", errs)) + }) + }) + + When("new manifests", func() { + var manifestIDs []string + + BeforeEach(func() { + fixtures.Clear() + fixtures.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{ + Key: strings.ToLower(clusterResourceID), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: clusterResourceID, + }, + }) + + manifestIDs = []string{manifests.NewUUID(), manifests.NewUUID(), manifests.NewUUID()} + fixtures.AddMaintenanceManifestDocuments(&api.MaintenanceManifestDocument{ + ID: manifestIDs[0], + ClusterResourceID: strings.ToLower(clusterResourceID), + MaintenanceManifest: api.MaintenanceManifest{ + State: api.MaintenanceManifestStatePending, + MaintenanceTaskID: "0", + RunBefore: 600, + RunAfter: 0, + Priority: 2, + }, + }, + &api.MaintenanceManifestDocument{ + ID: manifestIDs[1], + ClusterResourceID: strings.ToLower(clusterResourceID), + MaintenanceManifest: api.MaintenanceManifest{ + State: api.MaintenanceManifestStatePending, + MaintenanceTaskID: "1", + RunBefore: 600, + RunAfter: 0, + Priority: 1, + }, + }, + &api.MaintenanceManifestDocument{ + ID: manifestIDs[2], + ClusterResourceID: strings.ToLower(clusterResourceID), + MaintenanceManifest: api.MaintenanceManifest{ + State: api.MaintenanceManifestStatePending, + MaintenanceTaskID: "2", + RunBefore: 600, + RunAfter: 1, + Priority: 0, + }, + }) + + checker.Clear() + checker.AddMaintenanceManifestDocuments(&api.MaintenanceManifestDocument{ + ID: manifestIDs[0], + ClusterResourceID: strings.ToLower(clusterResourceID), + MaintenanceManifest: api.MaintenanceManifest{ + State: api.MaintenanceManifestStateCompleted, + MaintenanceTaskID: "0", + StatusText: "done", + RunBefore: 600, + RunAfter: 0, + Priority: 2, + }, + }, + &api.MaintenanceManifestDocument{ + ID: manifestIDs[1], + ClusterResourceID: strings.ToLower(clusterResourceID), + MaintenanceManifest: api.MaintenanceManifest{ + State: api.MaintenanceManifestStateCompleted, + MaintenanceTaskID: "1", + StatusText: "done", + RunBefore: 600, + RunAfter: 0, + Priority: 1, + }, + }, + &api.MaintenanceManifestDocument{ + ID: manifestIDs[2], + ClusterResourceID: strings.ToLower(clusterResourceID), + MaintenanceManifest: api.MaintenanceManifest{ + State: api.MaintenanceManifestStateCompleted, + MaintenanceTaskID: "2", + StatusText: "done", + RunBefore: 600, + RunAfter: 1, + Priority: 0, + }, + }) + }) + + It("runs them", func() { + ordering := []string{} + + a.AddMaintenanceTasks(map[string]tasks.MaintenanceTask{ + "0": func(th mimo.TaskContext, mmd *api.MaintenanceManifestDocument, oscd *api.OpenShiftClusterDocument) (api.MaintenanceManifestState, string) { + ordering = append(ordering, "0") + return api.MaintenanceManifestStateCompleted, "done" + }, + "1": func(th mimo.TaskContext, mmd *api.MaintenanceManifestDocument, oscd *api.OpenShiftClusterDocument) (api.MaintenanceManifestState, string) { + ordering = append(ordering, "1") + return api.MaintenanceManifestStateCompleted, "done" + }, + "2": func(th mimo.TaskContext, mmd *api.MaintenanceManifestDocument, oscd *api.OpenShiftClusterDocument) (api.MaintenanceManifestState, string) { + ordering = append(ordering, "2") + return api.MaintenanceManifestStateCompleted, "done" + }, + }) + + didWork, err := a.Process(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(didWork).To(BeTrue()) + + // We expect 1 (start time of 0, but higher priority), then 0 (start + // time of 0, lower priority), then 2 (start time of 1, then highest + // priority) + Expect(ordering).To(BeEquivalentTo([]string{"1", "0", "2"})) + + errs := checker.CheckMaintenanceManifests(manifestsClient) + Expect(errs).To(BeNil(), fmt.Sprintf("%v", errs)) + }) + }) + +}) + +func TestActuator(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Actuator Suite") +} diff --git a/pkg/mimo/actuator/manager.go b/pkg/mimo/actuator/manager.go new file mode 100644 index 00000000000..3b8aae922e8 --- /dev/null +++ b/pkg/mimo/actuator/manager.go @@ -0,0 +1,192 @@ +package actuator + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "fmt" + "sort" + "strings" + "time" + + "github.com/Azure/go-autorest/autorest/to" + "github.com/sirupsen/logrus" + "golang.org/x/exp/maps" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/database" + "github.com/Azure/ARO-RP/pkg/env" + "github.com/Azure/ARO-RP/pkg/mimo/tasks" +) + +const maxDequeueCount = 5 + +type Actuator interface { + Process(context.Context) (bool, error) + AddMaintenanceTasks(map[string]tasks.MaintenanceTask) +} + +type actuator struct { + env env.Interface + log *logrus.Entry + now func() time.Time + + clusterResourceID string + + oc database.OpenShiftClusters + mmf database.MaintenanceManifests + + tasks map[string]tasks.MaintenanceTask +} + +func NewActuator( + ctx context.Context, + _env env.Interface, + log *logrus.Entry, + clusterResourceID string, + oc database.OpenShiftClusters, + mmf database.MaintenanceManifests, + now func() time.Time) (Actuator, error) { + a := &actuator{ + env: _env, + log: log, + clusterResourceID: strings.ToLower(clusterResourceID), + oc: oc, + mmf: mmf, + tasks: make(map[string]tasks.MaintenanceTask), + + now: now, + } + + return a, nil +} + +func (a *actuator) AddMaintenanceTasks(tasks map[string]tasks.MaintenanceTask) { + maps.Copy(a.tasks, tasks) +} + +func (a *actuator) Process(ctx context.Context) (bool, error) { + // Get the manifests for this cluster which need to be worked + i, err := a.mmf.GetQueuedByClusterResourceID(ctx, a.clusterResourceID, "") + if err != nil { + return false, fmt.Errorf("failed getting manifests: %w", err) + } + + docList := make([]*api.MaintenanceManifestDocument, 0) + for { + docs, err := i.Next(ctx, -1) + if err != nil { + return false, fmt.Errorf("failed reading next manifest document: %w", err) + } + if docs == nil { + break + } + + docList = append(docList, docs.MaintenanceManifestDocuments...) + } + + manifestsToAction := make([]*api.MaintenanceManifestDocument, 0) + + // Order manifests in order of RunAfter, and then Priority for ones with the + // same RunAfter. + sort.SliceStable(docList, func(i, j int) bool { + if docList[i].MaintenanceManifest.RunAfter == docList[j].MaintenanceManifest.RunAfter { + return docList[i].MaintenanceManifest.Priority < docList[j].MaintenanceManifest.Priority + } + + return docList[i].MaintenanceManifest.RunAfter < docList[j].MaintenanceManifest.RunAfter + }) + + evaluationTime := a.now() + + // Check for manifests that have timed out first + for _, doc := range docList { + if evaluationTime.After(time.Unix(int64(doc.MaintenanceManifest.RunBefore), 0)) { + // timed out, mark as such + a.log.Infof("marking %v as outdated: %v older than %v", doc.ID, doc.MaintenanceManifest.RunBefore, evaluationTime.UTC()) + + _, err := a.mmf.Patch(ctx, a.clusterResourceID, doc.ID, func(d *api.MaintenanceManifestDocument) error { + d.MaintenanceManifest.State = api.MaintenanceManifestStateTimedOut + d.MaintenanceManifest.StatusText = fmt.Sprintf("timed out at %s", evaluationTime.UTC()) + return nil + }) + if err != nil { + a.log.Error(err) + } + } else { + // not timed out, do something about it + manifestsToAction = append(manifestsToAction, doc) + } + } + + // Nothing to do, don't dequeue + if len(manifestsToAction) == 0 { + return false, nil + } + + // Dequeue the document + oc, err := a.oc.Get(ctx, a.clusterResourceID) + if err != nil { + return false, fmt.Errorf("failed getting cluster document: %w", err) + } + + oc, err = a.oc.DoDequeue(ctx, oc) + if err != nil { + return false, fmt.Errorf("failed dequeuing cluster document: %w", err) // This will include StatusPreconditionFaileds + } + + taskContext := newTaskContext(ctx, a.env, a.log, oc) + + // Execute on the manifests we want to action + for _, doc := range manifestsToAction { + // here + f, ok := a.tasks[doc.MaintenanceManifest.MaintenanceTaskID] + if !ok { + a.log.Infof("not found %v", doc.MaintenanceManifest.MaintenanceTaskID) + continue + } + + // Attempt a dequeue + doc, err = a.mmf.Lease(ctx, a.clusterResourceID, doc.ID) + if err != nil { + // log and continue to the next task if it doesn't work + a.log.Error(err) + continue + } + + // if we've tried too many times, give up + if doc.Dequeues > maxDequeueCount { + err := fmt.Errorf("dequeued %d times, failing", doc.Dequeues) + _, leaseErr := a.mmf.EndLease(ctx, doc.ClusterResourceID, doc.ID, api.MaintenanceManifestStateTimedOut, to.StringPtr(err.Error())) + if leaseErr != nil { + a.log.Error(err) + } + continue + } + + var state api.MaintenanceManifestState + var msg string + + // Perform the task with a timeout + err = taskContext.RunInTimeout(time.Minute*60, func() error { + state, msg = f(taskContext, doc, oc) + return taskContext.Err() + }) + if err != nil { + a.log.Error(err) + } + + _, err = a.mmf.EndLease(ctx, doc.ClusterResourceID, doc.ID, state, &msg) + if err != nil { + a.log.Error(err) + } + } + + // release the OpenShiftCluster + _, err = a.oc.EndLease(ctx, a.clusterResourceID, oc.OpenShiftCluster.Properties.ProvisioningState, api.ProvisioningStateMaintenance, nil) + if err != nil { + return false, fmt.Errorf("failed ending lease on cluster document: %w", err) + } + return true, nil +} diff --git a/pkg/mimo/actuator/service.go b/pkg/mimo/actuator/service.go new file mode 100644 index 00000000000..62a1133fcd8 --- /dev/null +++ b/pkg/mimo/actuator/service.go @@ -0,0 +1,314 @@ +package actuator + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "errors" + "log" + "net" + "net/http" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/sirupsen/logrus" + "golang.org/x/exp/maps" + "sigs.k8s.io/controller-runtime/pkg/healthz" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/database" + "github.com/Azure/ARO-RP/pkg/env" + "github.com/Azure/ARO-RP/pkg/metrics" + "github.com/Azure/ARO-RP/pkg/mimo/tasks" + "github.com/Azure/ARO-RP/pkg/proxy" + "github.com/Azure/ARO-RP/pkg/util/buckets" + "github.com/Azure/ARO-RP/pkg/util/heartbeat" + utillog "github.com/Azure/ARO-RP/pkg/util/log" + "github.com/Azure/ARO-RP/pkg/util/recover" +) + +type Runnable interface { + Run(context.Context, <-chan struct{}, chan<- struct{}) error +} + +type service struct { + dialer proxy.Dialer + baseLog *logrus.Entry + env env.Interface + + dbGroup actuatorDBs + + m metrics.Emitter + mu sync.RWMutex + cond *sync.Cond + stopping *atomic.Bool + workers *atomic.Int32 + + b buckets.BucketWorker + + lastChangefeed atomic.Value //time.Time + startTime time.Time + + pollTime time.Duration + now func() time.Time + + tasks map[string]tasks.MaintenanceTask + + serveHealthz bool +} + +type actuatorDBs interface { + database.DatabaseGroupWithOpenShiftClusters + database.DatabaseGroupWithMaintenanceManifests +} + +func NewService(env env.Interface, log *logrus.Entry, dialer proxy.Dialer, dbg actuatorDBs, m metrics.Emitter) *service { + s := &service{ + env: env, + baseLog: log, + dialer: dialer, + + dbGroup: dbg, + + m: m, + stopping: &atomic.Bool{}, + workers: &atomic.Int32{}, + + startTime: time.Now(), + now: time.Now, + pollTime: time.Minute, + + serveHealthz: true, + } + + s.b = buckets.NewBucketWorker(log, s.worker, &s.mu) + + return s +} + +func (s *service) SetMaintenanceTasks(tasks map[string]tasks.MaintenanceTask) { + s.tasks = tasks +} + +func (s *service) Run(ctx context.Context, stop <-chan struct{}, done chan<- struct{}) error { + defer recover.Panic(s.baseLog) + + // Only enable the healthz endpoint if configured (disabled in unit tests) + if s.serveHealthz { + c := &healthz.Handler{ + Checks: map[string]healthz.Checker{ + "ready": func(h *http.Request) error { + if !s.checkReady() { + return errors.New("not ready") + } + return nil + }, + }, + } + + m := http.NewServeMux() + m.Handle("/healthz", http.StripPrefix("/healthz", c)) + // Handle healthz subpaths + m.Handle("/healthz/", http.StripPrefix("/healthz", c)) + + h := &http.Server{ + Handler: m, + ErrorLog: log.New(s.baseLog.Writer(), "", 0), + BaseContext: func(net.Listener) context.Context { return ctx }, + } + + listener, err := s.env.Listen() + if err != nil { + return err + } + + go func() { + err := h.Serve(listener) + if err != http.ErrServerClosed { + s.baseLog.Error(err) + } + }() + } + + t := time.NewTicker(10 * time.Second) + defer t.Stop() + + if stop != nil { + go func() { + defer recover.Panic(s.baseLog) + + <-stop + s.baseLog.Print("stopping") + s.stopping.Store(true) + s.cond.Signal() + }() + } + go heartbeat.EmitHeartbeat(s.baseLog, s.m, "actuator.heartbeat", nil, s.checkReady) + + lastGotDocs := make(map[string]*api.OpenShiftClusterDocument) + + for { + if s.stopping.Load() { + break + } + + old, err := s.poll(ctx, lastGotDocs) + if err != nil { + s.baseLog.Error(err) + } else { + lastGotDocs = old + } + + <-t.C + } + + if !s.env.FeatureIsSet(env.FeatureDisableReadinessDelay) { + s.waitForWorkerCompletion() + } + s.baseLog.Print("exiting") + close(done) + return nil +} + +// Temporary method of updating without the changefeed -- the reason why is +// complicated +func (s *service) poll(ctx context.Context, oldDocs map[string]*api.OpenShiftClusterDocument) (map[string]*api.OpenShiftClusterDocument, error) { + dbOpenShiftClusters, err := s.dbGroup.OpenShiftClusters() + if err != nil { + return nil, err + } + + // Fetch all of the cluster UUIDs + i, err := dbOpenShiftClusters.GetAllResourceIDs(ctx, "") + if err != nil { + return nil, err + } + + docs := make([]*api.OpenShiftClusterDocument, 0) + + for { + d, err := i.Next(ctx, -1) + if err != nil { + return nil, err + } + if d == nil { + break + } + + docs = append(docs, d.OpenShiftClusterDocuments...) + } + + s.baseLog.Debugf("fetched %d clusters from CosmosDB", len(docs)) + + docMap := make(map[string]*api.OpenShiftClusterDocument) + for _, d := range docs { + docMap[strings.ToLower(d.Key)] = d + } + + // remove docs that don't exist in the new set (removed clusters) + for _, oldCluster := range maps.Keys(oldDocs) { + _, ok := docMap[strings.ToLower(oldCluster)] + if !ok { + s.b.DeleteDoc(oldDocs[oldCluster]) + s.baseLog.Debugf("removed %s from buckets", oldCluster) + } + } + + s.baseLog.Debugf("updating %d clusters", len(docMap)) + + for _, cluster := range maps.Values(docMap) { + s.b.UpsertDoc(cluster) + } + + // Store when we last fetched the clusters + s.lastChangefeed.Store(s.now()) + + return docMap, nil +} + +func (s *service) waitForWorkerCompletion() { + s.mu.Lock() + for s.workers.Load() > 0 { + s.cond.Wait() + } + s.mu.Unlock() +} + +func (s *service) checkReady() bool { + lastChangefeedTime, ok := s.lastChangefeed.Load().(time.Time) + if !ok { + return false + } + + if s.env.IsLocalDevelopmentMode() { + return (time.Since(lastChangefeedTime) < time.Minute) // did we update our list of clusters recently? + } else { + return (time.Since(lastChangefeedTime) < time.Minute) && // did we update our list of clusters recently? + (time.Since(s.startTime) > 2*time.Minute) // are we running for at least 2 minutes? + } +} + +func (s *service) worker(stop <-chan struct{}, delay time.Duration, id string) { + defer recover.Panic(s.baseLog) + + log := utillog.EnrichWithResourceID(s.baseLog, id) + log.Debugf("starting worker for %s in %s...", id, delay.String()) + + // Wait for a randomised delay before starting + time.Sleep(delay) + + dbOpenShiftClusters, err := s.dbGroup.OpenShiftClusters() + if err != nil { + log.Error(err) + return + } + + dbMaintenanceManifests, err := s.dbGroup.MaintenanceManifests() + if err != nil { + log.Error(err) + return + } + + a, err := NewActuator(context.Background(), s.env, log, id, dbOpenShiftClusters, dbMaintenanceManifests, s.now) + if err != nil { + log.Error(err) + return + } + + // load in the tasks for the Actuator from the controller + a.AddMaintenanceTasks(s.tasks) + + t := time.NewTicker(s.pollTime) + defer t.Stop() + +out: + for { + if s.stopping.Load() { + break + } + + func() { + s.workers.Add(1) + s.m.EmitGauge("mimo.actuator.workers.active.count", int64(s.workers.Load()), nil) + + defer func() { + s.workers.Add(-1) + s.m.EmitGauge("mimo.actuator.workers.active.count", int64(s.workers.Load()), nil) + }() + + _, err := a.Process(context.Background()) + if err != nil { + log.Error(err) + } + }() + + select { + case <-t.C: + case <-stop: + break out + } + } +} diff --git a/pkg/mimo/actuator/service_test.go b/pkg/mimo/actuator/service_test.go new file mode 100644 index 00000000000..0d686271f02 --- /dev/null +++ b/pkg/mimo/actuator/service_test.go @@ -0,0 +1,252 @@ +package actuator + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "fmt" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/sirupsen/logrus" + "go.uber.org/mock/gomock" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/database" + "github.com/Azure/ARO-RP/pkg/database/cosmosdb" + "github.com/Azure/ARO-RP/pkg/env" + "github.com/Azure/ARO-RP/pkg/metrics" + "github.com/Azure/ARO-RP/pkg/mimo/tasks" + "github.com/Azure/ARO-RP/pkg/util/mimo" + mock_env "github.com/Azure/ARO-RP/pkg/util/mocks/env" + testdatabase "github.com/Azure/ARO-RP/test/database" + testlog "github.com/Azure/ARO-RP/test/util/log" +) + +type fakeMetricsEmitter struct { + Metrics map[string]int64 +} + +func newfakeMetricsEmitter() *fakeMetricsEmitter { + m := make(map[string]int64) + return &fakeMetricsEmitter{ + Metrics: m, + } +} + +func (e *fakeMetricsEmitter) EmitGauge(metricName string, metricValue int64, dimensions map[string]string) { + e.Metrics[metricName] = metricValue +} + +func (e *fakeMetricsEmitter) EmitFloat(metricName string, metricValue float64, dimensions map[string]string) { +} + +var _ = Describe("MIMO Actuator Service", Ordered, func() { + var fixtures *testdatabase.Fixture + var checker *testdatabase.Checker + var manifests database.MaintenanceManifests + var manifestsClient *cosmosdb.FakeMaintenanceManifestDocumentClient + var clusters database.OpenShiftClusters + //var clustersClient cosmosdb.OpenShiftClusterDocumentClient + var m metrics.Emitter + + var svc *service + + var ctx context.Context + var cancel context.CancelFunc + + //var hook *test.Hook + var log *logrus.Entry + var _env env.Interface + + var controller *gomock.Controller + + mockSubID := "00000000-0000-0000-0000-000000000000" + clusterResourceID := fmt.Sprintf("/subscriptions/%s/resourcegroups/resourceGroup/providers/Microsoft.RedHatOpenShift/openShiftClusters/resourceName", mockSubID) + + AfterAll(func() { + if cancel != nil { + cancel() + } + + if controller != nil { + controller.Finish() + } + }) + + BeforeAll(func() { + controller = gomock.NewController(nil) + _env = mock_env.NewMockInterface(controller) + + ctx, cancel = context.WithCancel(context.Background()) + + _, log = testlog.New() + + m = newfakeMetricsEmitter() + + fixtures = testdatabase.NewFixture() + checker = testdatabase.NewChecker() + }) + + BeforeEach(func() { + now := func() time.Time { return time.Unix(120, 0) } + manifests, manifestsClient = testdatabase.NewFakeMaintenanceManifests(now) + clusters, _ = testdatabase.NewFakeOpenShiftClusters() + dbg := database.NewDBGroup().WithMaintenanceManifests(manifests).WithOpenShiftClusters(clusters) + + svc = NewService(_env, log, nil, dbg, m) + svc.now = now + svc.serveHealthz = false + }) + + JustBeforeEach(func() { + err := fixtures.WithOpenShiftClusters(clusters).WithMaintenanceManifests(manifests).Create() + Expect(err).ToNot(HaveOccurred()) + }) + + When("clusters are polled", func() { + BeforeEach(func() { + fixtures.Clear() + fixtures.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{ + Key: strings.ToLower(clusterResourceID), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: clusterResourceID, + }, + }) + }) + + AfterAll(func() { + svc.b.Stop() + }) + + It("updates the available clusters", func() { + lastGotDocs := make(map[string]*api.OpenShiftClusterDocument) + + newOld, err := svc.poll(ctx, lastGotDocs) + Expect(err).ToNot(HaveOccurred()) + + Expect(newOld).To(HaveLen(1)) + }) + + It("removes clusters if they are not in the doc", func() { + svc.b.UpsertDoc(&api.OpenShiftClusterDocument{Key: clusterResourceID + "2"}) + + lastGotDocs := make(map[string]*api.OpenShiftClusterDocument) + lastGotDocs[clusterResourceID+"2"] = &api.OpenShiftClusterDocument{Key: clusterResourceID + "2"} + + newOld, err := svc.poll(ctx, lastGotDocs) + Expect(err).ToNot(HaveOccurred()) + + Expect(newOld).To(HaveLen(1)) + }) + }) + + When("maintenance needs to occur", func() { + var manifestID string + + BeforeEach(func() { + fixtures.Clear() + fixtures.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{ + Key: strings.ToLower(clusterResourceID), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: clusterResourceID, + Properties: api.OpenShiftClusterProperties{ + ProvisioningState: api.ProvisioningStateSucceeded, + }, + }, + }) + + manifestID = manifests.NewUUID() + manifestID2 := manifests.NewUUID() + fixtures.AddMaintenanceManifestDocuments( + &api.MaintenanceManifestDocument{ + ID: manifestID, + ClusterResourceID: strings.ToLower(clusterResourceID), + MaintenanceManifest: api.MaintenanceManifest{ + State: api.MaintenanceManifestStatePending, + RunBefore: 60, + RunAfter: 0, + }, + }, + &api.MaintenanceManifestDocument{ + ID: manifestID2, + ClusterResourceID: strings.ToLower(clusterResourceID), + MaintenanceManifest: api.MaintenanceManifest{ + State: api.MaintenanceManifestStatePending, + RunBefore: 300, + RunAfter: 0, + MaintenanceTaskID: "0000-0000-0001", + }, + }) + + checker.Clear() + checker.AddMaintenanceManifestDocuments( + &api.MaintenanceManifestDocument{ + ID: manifestID, + ClusterResourceID: strings.ToLower(clusterResourceID), + MaintenanceManifest: api.MaintenanceManifest{ + State: api.MaintenanceManifestStateTimedOut, + StatusText: "timed out at 1970-01-01 00:02:00 +0000 UTC", + RunBefore: 60, + RunAfter: 0, + }, + }, + &api.MaintenanceManifestDocument{ + ID: manifestID2, + ClusterResourceID: strings.ToLower(clusterResourceID), + MaintenanceManifest: api.MaintenanceManifest{ + State: api.MaintenanceManifestStateCompleted, + StatusText: "ok", + RunBefore: 300, + RunAfter: 0, + MaintenanceTaskID: "0000-0000-0001", + }, + }, + ) + }) + + It("expires them", func() { + // run once + done := make(chan struct{}) + svc.pollTime = time.Second + + svc.SetMaintenanceTasks(map[string]tasks.MaintenanceTask{ + "0000-0000-0001": func(th mimo.TaskContext, mmd *api.MaintenanceManifestDocument, oscd *api.OpenShiftClusterDocument) (api.MaintenanceManifestState, string) { + svc.stopping.Store(true) + return api.MaintenanceManifestStateCompleted, "ok" + }, + }) + + svc.worker(done, 0*time.Second, clusterResourceID) + + errs := checker.CheckMaintenanceManifests(manifestsClient) + Expect(errs).To(BeNil(), fmt.Sprintf("%v", errs)) + }) + + It("loads the full cluster document", func() { + // run once + done := make(chan struct{}) + svc.pollTime = time.Second + + svc.SetMaintenanceTasks(map[string]tasks.MaintenanceTask{ + "0000-0000-0001": func(th mimo.TaskContext, mmd *api.MaintenanceManifestDocument, oscd *api.OpenShiftClusterDocument) (api.MaintenanceManifestState, string) { + // ProvisioningState is in the full document, not just the + // ClusterResourceID only as in the bucket worker + Expect(oscd.OpenShiftCluster.Properties.ProvisioningState).To(Equal(api.ProvisioningStateSucceeded)) + + svc.stopping.Store(true) + return api.MaintenanceManifestStateCompleted, "ok" + }, + }) + + svc.worker(done, 0*time.Second, clusterResourceID) + + errs := checker.CheckMaintenanceManifests(manifestsClient) + Expect(errs).To(BeNil(), fmt.Sprintf("%v", errs)) + }) + }) +}) diff --git a/pkg/mimo/actuator/task.go b/pkg/mimo/actuator/task.go new file mode 100644 index 00000000000..6392acf7767 --- /dev/null +++ b/pkg/mimo/actuator/task.go @@ -0,0 +1,143 @@ +package actuator + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "time" + + "github.com/Azure/go-autorest/autorest" + "github.com/sirupsen/logrus" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/env" + "github.com/Azure/ARO-RP/pkg/util/clienthelper" + "github.com/Azure/ARO-RP/pkg/util/mimo" + "github.com/Azure/ARO-RP/pkg/util/restconfig" +) + +type th struct { + originalCtx context.Context + ctx context.Context + + env env.Interface + log *logrus.Entry + + resultMessage string + + oc *api.OpenShiftClusterDocument + + _ch clienthelper.Interface +} + +// force interface checking +var _ mimo.TaskContext = &th{} + +func newTaskContext(ctx context.Context, env env.Interface, log *logrus.Entry, oc *api.OpenShiftClusterDocument) *th { + return &th{ + originalCtx: ctx, + ctx: ctx, + env: env, + log: log, + oc: oc, + _ch: nil, + } +} + +func (t *th) RunInTimeout(timeout time.Duration, f func() error) error { + newctx, cancel := context.WithTimeout(t.originalCtx, timeout) + t.ctx = newctx + defer func() { + cancel() + t.ctx = t.originalCtx + }() + return f() +} + +// context stuff +func (t *th) Deadline() (time.Time, bool) { + return t.ctx.Deadline() +} + +func (t *th) Done() <-chan struct{} { + return t.ctx.Done() +} + +func (t *th) Err() error { + return t.ctx.Err() +} + +func (t *th) Value(key any) any { + return t.ctx.Value(key) +} + +func (t *th) Environment() env.Interface { + return t.env +} + +func (t *th) ClientHelper() (clienthelper.Interface, error) { + if t._ch != nil { + return t._ch, nil + } + + restConfig, err := restconfig.RestConfig(t.env, t.oc.OpenShiftCluster) + if err != nil { + return nil, err + } + + mapper, err := apiutil.NewDynamicRESTMapper(restConfig, apiutil.WithLazyDiscovery) + if err != nil { + return nil, err + } + + client, err := client.New(restConfig, client.Options{ + Mapper: mapper, + }) + if err != nil { + return nil, err + } + + t._ch = clienthelper.NewWithClient(t.log, client) + return t._ch, nil +} + +func (t *th) Log() *logrus.Entry { + return t.log +} + +func (t *th) Now() time.Time { + return time.Now() +} + +func (t *th) SetResultMessage(msg string) { + t.resultMessage = msg +} + +func (t *th) GetResultMessage() string { + return t.resultMessage +} + +func (t *th) GetClusterUUID() string { + return t.oc.ID +} + +func (t *th) GetOpenShiftClusterProperties() api.OpenShiftClusterProperties { + return t.oc.OpenShiftCluster.Properties +} + +// localFpAuthorizer implements mimo.TaskContext. +func (t *th) LocalFpAuthorizer() (autorest.Authorizer, error) { + localFPAuthorizer, err := t.env.FPAuthorizer(t.env.TenantID(), t.env.Environment().ResourceManagerScope) + if err != nil { + return nil, err + } + return localFPAuthorizer, nil +} + +// GetOpenshiftClusterDocument implements mimo.TaskContext. +func (t *th) GetOpenshiftClusterDocument() *api.OpenShiftClusterDocument { + return t.oc +} diff --git a/pkg/mimo/const.go b/pkg/mimo/const.go new file mode 100644 index 00000000000..7dea33e1c8e --- /dev/null +++ b/pkg/mimo/const.go @@ -0,0 +1,10 @@ +package mimo + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +const ( + TLS_CERT_ROTATION_ID = "9b741734-6505-447f-8510-85eb0ae561a2" + OPERATOR_FLAGS_UPDATE_ID = "b41749fc-af26-4ab7-b5a1-e03f3ee4cba6" + ACR_TOKEN_CHECKER_ID = "082978ce-3700-4972-835f-53d48658d291" +) diff --git a/pkg/mimo/steps/cluster/acrtoken_checker.go b/pkg/mimo/steps/cluster/acrtoken_checker.go new file mode 100644 index 00000000000..9f17733058d --- /dev/null +++ b/pkg/mimo/steps/cluster/acrtoken_checker.go @@ -0,0 +1,63 @@ +package cluster + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/Azure/ARO-RP/pkg/util/acrtoken" + "github.com/Azure/ARO-RP/pkg/util/mimo" +) + +const ( + daysValid = 90 + daysShouldRotate = 45 +) + +// EnsureACRTokenIsValid checks the expiry date of the Azure Container Registry (ACR) Token from the RegistryProfile. +// It returns an error if the expiry date is past the date now or if there is no registry profile found. +func EnsureACRTokenIsValid(ctx context.Context) error { + th, err := mimo.GetTaskContext(ctx) + if err != nil { + return mimo.TerminalError(err) + } + + env := th.Environment() + localFpAuthorizer, err := th.LocalFpAuthorizer() + if err != nil { + return mimo.TerminalError(err) + } + + manager, err := acrtoken.NewManager(env, localFpAuthorizer) + if err != nil { + return err + } + + registryProfiles := th.GetOpenShiftClusterProperties().RegistryProfiles + rp := manager.GetRegistryProfileFromSlice(registryProfiles) + if rp != nil { + var now = time.Now().UTC() + issueDate := rp.IssueDate + + if issueDate == nil { + return mimo.TerminalError(errors.New("no issue date detected, please rotate token")) + } + + daysInterval := int32(now.Sub(issueDate.Time).Hours() / 24) + + switch { + case daysInterval > daysValid: + return mimo.TerminalError(fmt.Errorf("azure container registry (acr) token is not valid, %d days have passed", daysInterval)) + case daysInterval >= daysShouldRotate: + return mimo.TerminalError(fmt.Errorf("%d days have passed since azure container registry (acr) token was issued, please rotate the token now", daysInterval)) + default: + th.SetResultMessage("azure container registry (acr) token is valid") + } + } + + return mimo.TerminalError(errors.New("no registry profile detected")) +} diff --git a/pkg/mimo/steps/cluster/acrtoken_checker_test.go b/pkg/mimo/steps/cluster/acrtoken_checker_test.go new file mode 100644 index 00000000000..8a389150d6e --- /dev/null +++ b/pkg/mimo/steps/cluster/acrtoken_checker_test.go @@ -0,0 +1,145 @@ +package cluster + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "testing" + "time" + + . "github.com/onsi/gomega" + + "github.com/Azure/go-autorest/autorest/date" + "go.uber.org/mock/gomock" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/util/azureclient" + "github.com/Azure/ARO-RP/pkg/util/clienthelper" + mock_env "github.com/Azure/ARO-RP/pkg/util/mocks/env" + testtasks "github.com/Azure/ARO-RP/test/mimo/tasks" + testclienthelper "github.com/Azure/ARO-RP/test/util/clienthelper" + utilerror "github.com/Azure/ARO-RP/test/util/error" + testlog "github.com/Azure/ARO-RP/test/util/log" +) + +const ( + registryResourceID = "/subscriptions/93aeba23-2f76-4307-be82-02921df010cf/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arointsvc" + clusterUUID = "512a50c8-2a43-4c2a-8fd9-a5539475df2a" + publicACR = "arosvc.azurecr.io" + intACR = "arointsvc.azurecr.io" + user = "testuser" +) + +func TestEnsureACRToken(t *testing.T) { + ctx := context.Background() + + for _, tt := range []struct { + name string + azureEnv azureclient.AROEnvironment + oc func() *api.OpenShiftCluster + wantErr string + }{ + { + name: "not found", + azureEnv: azureclient.PublicCloud, + oc: func() *api.OpenShiftCluster { + return &api.OpenShiftCluster{ + Properties: api.OpenShiftClusterProperties{}, + } + }, + wantErr: "TerminalError: no registry profile detected", + }, + { + name: "No issue date", + azureEnv: azureclient.PublicCloud, + oc: func() *api.OpenShiftCluster { + return &api.OpenShiftCluster{ + Properties: api.OpenShiftClusterProperties{ + RegistryProfiles: []*api.RegistryProfile{ + { + Name: intACR, + Username: user, + IssueDate: nil, + }, + }, + }, + } + }, + wantErr: "TerminalError: no issue date detected, please rotate token", + }, + { + name: "Expired", + azureEnv: azureclient.PublicCloud, + oc: func() *api.OpenShiftCluster { + return &api.OpenShiftCluster{ + Properties: api.OpenShiftClusterProperties{ + RegistryProfiles: []*api.RegistryProfile{ + { + Name: publicACR, + Username: user, + IssueDate: &date.Time{Time: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)}, + }, + { + Name: intACR, + Username: user, + IssueDate: &date.Time{Time: time.Now().UTC().AddDate(0, 0, -100)}, + }, + }, + }, + } + }, + wantErr: "TerminalError: azure container registry (acr) token is not valid, 100 days have passed", + }, + { + name: "Should rotate token", + azureEnv: azureclient.PublicCloud, + oc: func() *api.OpenShiftCluster { + return &api.OpenShiftCluster{ + Properties: api.OpenShiftClusterProperties{ + RegistryProfiles: []*api.RegistryProfile{ + { + Name: publicACR, + Username: user, + IssueDate: &date.Time{Time: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)}, + }, + { + Name: intACR, + Username: user, + IssueDate: &date.Time{Time: time.Now().UTC().AddDate(0, 0, -50)}, + }, + }, + }, + } + }, + wantErr: "TerminalError: 50 days have passed since azure container registry (acr) token was issued, please rotate the token now", + }, + } { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + controller := gomock.NewController(t) + _env := mock_env.NewMockInterface(controller) + _env.EXPECT().ACRResourceID().AnyTimes().Return(registryResourceID) + _env.EXPECT().Environment().AnyTimes().Return(&tt.azureEnv) + _, log := testlog.New() + + builder := fake.NewClientBuilder() + ch := clienthelper.NewWithClient(log, testclienthelper.NewHookingClient(builder.Build())) + tc := testtasks.NewFakeTestContext( + ctx, _env, log, func() time.Time { return time.Unix(100, 0) }, + testtasks.WithClientHelper(ch), + testtasks.WithOpenShiftClusterProperties(clusterUUID, tt.oc().Properties), + ) + + err := EnsureACRTokenIsValid(tc) + if tt.wantErr != "" && err != nil { + utilerror.AssertErrorMessage(t, err, tt.wantErr) + } else if tt.wantErr != "" && err == nil { + t.Errorf("wanted error %s", tt.wantErr) + } else if tt.wantErr == "" { + g.Expect(err).ToNot(HaveOccurred()) + } + }) + } +} diff --git a/pkg/mimo/steps/cluster/apiserver_is_up.go b/pkg/mimo/steps/cluster/apiserver_is_up.go new file mode 100644 index 00000000000..eecefa7e434 --- /dev/null +++ b/pkg/mimo/steps/cluster/apiserver_is_up.go @@ -0,0 +1,46 @@ +package cluster + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "errors" + + configv1 "github.com/openshift/api/config/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + + "github.com/Azure/ARO-RP/pkg/util/clusteroperators" + "github.com/Azure/ARO-RP/pkg/util/mimo" +) + +func EnsureAPIServerIsUp(ctx context.Context) error { + th, err := mimo.GetTaskContext(ctx) + if err != nil { + return err + } + + ch, err := th.ClientHelper() + if err != nil { + return mimo.TerminalError(err) + } + + co := &configv1.ClusterOperator{} + + err = ch.GetOne(ctx, types.NamespacedName{Name: "kube-apiserver"}, co) + if err != nil { + // 404 on kube-apiserver is likely terminal + if kerrors.IsNotFound(err) { + return mimo.TerminalError(err) + } + + return mimo.TransientError(err) + } + + available := clusteroperators.IsOperatorAvailable(co) + if !available { + return mimo.TransientError(errors.New(clusteroperators.OperatorStatusText(co))) + } + return nil +} diff --git a/pkg/mimo/steps/cluster/apiserver_is_up_test.go b/pkg/mimo/steps/cluster/apiserver_is_up_test.go new file mode 100644 index 00000000000..c89544766a1 --- /dev/null +++ b/pkg/mimo/steps/cluster/apiserver_is_up_test.go @@ -0,0 +1,108 @@ +package cluster + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "testing" + "time" + + . "github.com/onsi/gomega" + + configv1 "github.com/openshift/api/config/v1" + "go.uber.org/mock/gomock" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/Azure/ARO-RP/pkg/util/clienthelper" + mock_env "github.com/Azure/ARO-RP/pkg/util/mocks/env" + testtasks "github.com/Azure/ARO-RP/test/mimo/tasks" + testclienthelper "github.com/Azure/ARO-RP/test/util/clienthelper" + testlog "github.com/Azure/ARO-RP/test/util/log" +) + +func TestAPIServerIsUp(t *testing.T) { + ctx := context.Background() + + for _, tt := range []struct { + name string + objects []runtime.Object + wantErr string + }{ + { + name: "not found", + objects: []runtime.Object{}, + wantErr: `TerminalError: clusteroperators.config.openshift.io "kube-apiserver" not found`, + }, + { + name: "not ready", + objects: []runtime.Object{ + &configv1.ClusterOperator{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kube-apiserver", + }, + Status: configv1.ClusterOperatorStatus{ + Conditions: []configv1.ClusterOperatorStatusCondition{ + { + Type: configv1.OperatorAvailable, + Status: configv1.ConditionFalse, + }, + { + Type: configv1.OperatorProgressing, + Status: configv1.ConditionTrue, + }, + }, + }, + }, + }, + wantErr: `TransientError: kube-apiserver Available=False, Progressing=True`, + }, + { + name: "ready", + objects: []runtime.Object{ + &configv1.ClusterOperator{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kube-apiserver", + }, + Status: configv1.ClusterOperatorStatus{ + Conditions: []configv1.ClusterOperatorStatusCondition{ + { + Type: configv1.OperatorAvailable, + Status: configv1.ConditionTrue, + }, + { + Type: configv1.OperatorProgressing, + Status: configv1.ConditionFalse, + }, + }, + }, + }, + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + controller := gomock.NewController(t) + _env := mock_env.NewMockInterface(controller) + _, log := testlog.New() + + builder := fake.NewClientBuilder().WithRuntimeObjects(tt.objects...) + ch := clienthelper.NewWithClient(log, testclienthelper.NewHookingClient(builder.Build())) + tc := testtasks.NewFakeTestContext( + ctx, _env, log, func() time.Time { return time.Unix(100, 0) }, + testtasks.WithClientHelper(ch), + ) + + err := EnsureAPIServerIsUp(tc) + if tt.wantErr != "" && err != nil { + g.Expect(err).To(MatchError(tt.wantErr)) + } else if tt.wantErr != "" && err == nil { + t.Errorf("wanted error %s", tt.wantErr) + } else if tt.wantErr == "" { + g.Expect(err).ToNot(HaveOccurred()) + } + }) + } +} diff --git a/pkg/mimo/steps/cluster/operatorflags.go b/pkg/mimo/steps/cluster/operatorflags.go new file mode 100644 index 00000000000..5c9b01eb9fb --- /dev/null +++ b/pkg/mimo/steps/cluster/operatorflags.go @@ -0,0 +1,56 @@ +package cluster + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + + kerrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" + + arov1alpha1 "github.com/Azure/ARO-RP/pkg/operator/apis/aro.openshift.io/v1alpha1" + "github.com/Azure/ARO-RP/pkg/util/mimo" +) + +// UpdateClusterOperatorFlags updates the OperatorFlags object in the ARO +// Cluster document. +func UpdateClusterOperatorFlags(ctx context.Context) error { + th, err := mimo.GetTaskContext(ctx) + if err != nil { + return mimo.TerminalError(err) + } + + props := th.GetOpenShiftClusterProperties() + + ch, err := th.ClientHelper() + if err != nil { + return mimo.TerminalError(err) + } + + return retry.RetryOnConflict(retry.DefaultRetry, func() error { + clusterObj := &arov1alpha1.Cluster{} + + err = ch.GetOne(ctx, types.NamespacedName{Name: arov1alpha1.SingletonClusterName}, clusterObj) + if err != nil { + if kerrors.IsNotFound(err) { + // cluster doc being gone is unrecoverable + return mimo.TerminalError(err) + } + return mimo.TransientError(err) + } + + clusterObj.Spec.OperatorFlags = arov1alpha1.OperatorFlags(props.OperatorFlags) + + err = ch.Update(ctx, clusterObj) + if err != nil { + if kerrors.IsConflict(err) { + return err + } else { + return mimo.TransientError(err) + } + } + return nil + }) +} diff --git a/pkg/mimo/steps/cluster/operatorflags_test.go b/pkg/mimo/steps/cluster/operatorflags_test.go new file mode 100644 index 00000000000..f9b29f5fa3f --- /dev/null +++ b/pkg/mimo/steps/cluster/operatorflags_test.go @@ -0,0 +1,127 @@ +package cluster + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "testing" + "time" + + . "github.com/onsi/gomega" + + "github.com/go-test/deep" + "go.uber.org/mock/gomock" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/Azure/ARO-RP/pkg/api" + arov1alpha1 "github.com/Azure/ARO-RP/pkg/operator/apis/aro.openshift.io/v1alpha1" + "github.com/Azure/ARO-RP/pkg/util/clienthelper" + mock_env "github.com/Azure/ARO-RP/pkg/util/mocks/env" + testtasks "github.com/Azure/ARO-RP/test/mimo/tasks" + testclienthelper "github.com/Azure/ARO-RP/test/util/clienthelper" + testlog "github.com/Azure/ARO-RP/test/util/log" +) + +func TestOperatorFlags(t *testing.T) { + ctx := context.Background() + + for _, tt := range []struct { + name string + objects []runtime.Object + wantObjects []runtime.Object + wantErr string + }{ + { + name: "not found", + objects: []runtime.Object{}, + wantErr: `TerminalError: clusters.aro.openshift.io "cluster" not found`, + }, + { + name: "not ready", + objects: []runtime.Object{ + &arov1alpha1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: arov1alpha1.SingletonClusterName, + ResourceVersion: "1000", + }, + Spec: arov1alpha1.ClusterSpec{ + OperatorFlags: arov1alpha1.OperatorFlags{ + "foo": "bar", + }, + }, + }, + }, + wantObjects: []runtime.Object{ + &arov1alpha1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: arov1alpha1.SingletonClusterName, + ResourceVersion: "1001", + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Cluster", + APIVersion: arov1alpha1.SchemeGroupVersion.String(), + }, + Spec: arov1alpha1.ClusterSpec{ + OperatorFlags: arov1alpha1.OperatorFlags{ + "foo": "baz", + "gaz": "data", + }, + }, + }, + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + controller := gomock.NewController(t) + _env := mock_env.NewMockInterface(controller) + _, log := testlog.New() + + ocDoc := &api.OpenShiftClusterDocument{ + ID: "0000", + OpenShiftCluster: &api.OpenShiftCluster{ + Properties: api.OpenShiftClusterProperties{ + OperatorFlags: api.OperatorFlags{ + "foo": "baz", + "gaz": "data", + }, + }, + }, + } + + builder := fake.NewClientBuilder().WithRuntimeObjects(tt.objects...) + ch := clienthelper.NewWithClient(log, testclienthelper.NewHookingClient(builder.Build())) + tc := testtasks.NewFakeTestContext( + ctx, _env, log, func() time.Time { return time.Unix(100, 0) }, + testtasks.WithClientHelper(ch), testtasks.WithOpenShiftClusterDocument(ocDoc), + ) + + err := UpdateClusterOperatorFlags(tc) + if tt.wantErr != "" && err != nil { + g.Expect(err).To(MatchError(tt.wantErr)) + } else if tt.wantErr != "" && err == nil { + t.Errorf("wanted error %s", tt.wantErr) + } else if tt.wantErr == "" { + g.Expect(err).ToNot(HaveOccurred()) + } + + if len(tt.wantObjects) > 0 { + for _, i := range tt.wantObjects { + o, err := scheme.Scheme.New(i.GetObjectKind().GroupVersionKind()) + g.Expect(err).ToNot(HaveOccurred()) + + err = ch.GetOne(ctx, client.ObjectKeyFromObject(i.(client.Object)), o) + g.Expect(err).ToNot(HaveOccurred()) + + r := deep.Equal(i, o) + g.Expect(r).To(BeEmpty()) + } + } + }) + } +} diff --git a/pkg/mimo/steps/cluster/tls.go b/pkg/mimo/steps/cluster/tls.go new file mode 100644 index 00000000000..5501077ed5b --- /dev/null +++ b/pkg/mimo/steps/cluster/tls.go @@ -0,0 +1,103 @@ +package cluster + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + + configv1 "github.com/openshift/api/config/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" + + "github.com/Azure/ARO-RP/pkg/cluster" + "github.com/Azure/ARO-RP/pkg/util/dns" + "github.com/Azure/ARO-RP/pkg/util/mimo" +) + +func RotateAPIServerCertificate(ctx context.Context) error { + th, err := mimo.GetTaskContext(ctx) + if err != nil { + return mimo.TerminalError(err) + } + + ch, err := th.ClientHelper() + if err != nil { + return mimo.TerminalError(err) + } + + env := th.Environment() + secretName := th.GetClusterUUID() + "-apiserver" + + for _, namespace := range []string{"openshift-config", "openshift-azure-operator"} { + err = cluster.EnsureTLSSecretFromKeyvault( + ctx, env.ClusterKeyvault(), ch, types.NamespacedName{Namespace: namespace, Name: secretName}, secretName, + ) + if err != nil { + return err + } + } + + return nil +} + +func EnsureAPIServerServingCertificateConfiguration(ctx context.Context) error { + th, err := mimo.GetTaskContext(ctx) + if err != nil { + return mimo.TerminalError(err) + } + + ch, err := th.ClientHelper() + if err != nil { + return mimo.TerminalError(err) + } + + env := th.Environment() + clusterProperties := th.GetOpenShiftClusterProperties() + + managedDomain, err := dns.ManagedDomain(env, clusterProperties.ClusterProfile.Domain) + if err != nil { + // if it fails the belt&braces check then not much we can do + return mimo.TerminalError(err) + } + + if managedDomain == "" { + th.SetResultMessage("apiserver certificate is not managed") + return nil + } + + return retry.RetryOnConflict(retry.DefaultRetry, func() error { + apiserver := &configv1.APIServer{} + + err := ch.GetOne(ctx, types.NamespacedName{Name: "cluster"}, apiserver) + if err != nil { + if kerrors.IsNotFound(err) { + // apiserver not being found is probably unrecoverable + return mimo.TerminalError(err) + } + return mimo.TransientError(err) + } + + apiserver.Spec.ServingCerts.NamedCertificates = []configv1.APIServerNamedServingCert{ + { + Names: []string{ + "api." + managedDomain, + }, + ServingCertificate: configv1.SecretNameReference{ + Name: th.GetClusterUUID() + "-apiserver", + }, + }, + } + + err = ch.Update(ctx, apiserver) + if err != nil { + if kerrors.IsConflict(err) { + return err + } else { + return mimo.TransientError(err) + } + } + return nil + }) +} diff --git a/pkg/mimo/steps/cluster/tls_test.go b/pkg/mimo/steps/cluster/tls_test.go new file mode 100644 index 00000000000..64b26995264 --- /dev/null +++ b/pkg/mimo/steps/cluster/tls_test.go @@ -0,0 +1,106 @@ +package cluster + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "testing" + "time" + + . "github.com/onsi/gomega" + + configv1 "github.com/openshift/api/config/v1" + "go.uber.org/mock/gomock" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/util/clienthelper" + mock_env "github.com/Azure/ARO-RP/pkg/util/mocks/env" + testtasks "github.com/Azure/ARO-RP/test/mimo/tasks" + testlog "github.com/Azure/ARO-RP/test/util/log" +) + +func TestConfigureAPIServerCertificates(t *testing.T) { + ctx := context.Background() + clusterUUID := "512a50c8-2a43-4c2a-8fd9-a5539475df2a" + + for _, tt := range []struct { + name string + objects []runtime.Object + check func(clienthelper.Interface, Gomega) error + wantErr string + }{ + { + name: "not found", + objects: []runtime.Object{}, + wantErr: `TerminalError: apiservers.config.openshift.io "cluster" not found`, + }, + { + name: "secrets referenced", + objects: []runtime.Object{ + &configv1.APIServer{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster", + }, + Spec: configv1.APIServerSpec{}, + }, + }, + check: func(i clienthelper.Interface, g Gomega) error { + apiserver := &configv1.APIServer{} + err := i.GetOne(ctx, types.NamespacedName{Name: "cluster"}, apiserver) + if err != nil { + return err + } + + g.Expect(apiserver.Spec.ServingCerts.NamedCertificates).To(Equal([]configv1.APIServerNamedServingCert{ + { + Names: []string{"api.something.example.com"}, + ServingCertificate: configv1.SecretNameReference{ + Name: "512a50c8-2a43-4c2a-8fd9-a5539475df2a-apiserver", + }, + }, + })) + + return nil + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + controller := gomock.NewController(t) + _env := mock_env.NewMockInterface(controller) + _env.EXPECT().Domain().AnyTimes().Return("example.com") + + _, log := testlog.New() + + builder := fake.NewClientBuilder().WithRuntimeObjects(tt.objects...) + ch := clienthelper.NewWithClient(log, builder.Build()) + tc := testtasks.NewFakeTestContext( + ctx, _env, log, func() time.Time { return time.Unix(100, 0) }, + testtasks.WithClientHelper(ch), + testtasks.WithOpenShiftClusterProperties(clusterUUID, api.OpenShiftClusterProperties{ + ClusterProfile: api.ClusterProfile{ + Domain: "something", + }, + }), + ) + + err := EnsureAPIServerServingCertificateConfiguration(tc) + if tt.wantErr != "" && err != nil { + g.Expect(err).To(MatchError(tt.wantErr)) + } else if tt.wantErr != "" && err == nil { + t.Errorf("wanted error %s", tt.wantErr) + } else if tt.wantErr == "" { + g.Expect(err).ToNot(HaveOccurred()) + } + + if tt.check != nil { + g.Expect(tt.check(ch, g)).ToNot(HaveOccurred()) + } + }) + } +} diff --git a/pkg/mimo/steps/example/task.go b/pkg/mimo/steps/example/task.go new file mode 100644 index 00000000000..3dc832029fd --- /dev/null +++ b/pkg/mimo/steps/example/task.go @@ -0,0 +1,37 @@ +package example + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "fmt" + + configv1 "github.com/openshift/api/config/v1" + "k8s.io/apimachinery/pkg/types" + + "github.com/Azure/ARO-RP/pkg/util/mimo" +) + +func ReportClusterVersion(ctx context.Context) error { + th, err := mimo.GetTaskContext(ctx) + if err != nil { + return err + } + + ch, err := th.ClientHelper() + if err != nil { + return err + } + + cv := &configv1.ClusterVersion{} + + err = ch.GetOne(ctx, types.NamespacedName{Name: "version"}, cv) + if err != nil { + return fmt.Errorf("unable to get ClusterVersion: %w", err) + } + + th.SetResultMessage(fmt.Sprintf("cluster version is: %s", cv.Status.History[0].Version)) + + return nil +} diff --git a/pkg/mimo/steps/example/task_test.go b/pkg/mimo/steps/example/task_test.go new file mode 100644 index 00000000000..af1e07fa8e2 --- /dev/null +++ b/pkg/mimo/steps/example/task_test.go @@ -0,0 +1,56 @@ +package example + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "testing" + "time" + + . "github.com/onsi/gomega" + + configv1 "github.com/openshift/api/config/v1" + "go.uber.org/mock/gomock" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/Azure/ARO-RP/pkg/util/clienthelper" + mock_env "github.com/Azure/ARO-RP/pkg/util/mocks/env" + testtasks "github.com/Azure/ARO-RP/test/mimo/tasks" + testclienthelper "github.com/Azure/ARO-RP/test/util/clienthelper" + testlog "github.com/Azure/ARO-RP/test/util/log" +) + +func TestTask(t *testing.T) { + RegisterTestingT(t) + ctx := context.Background() + + controller := gomock.NewController(t) + _env := mock_env.NewMockInterface(controller) + _, log := testlog.New() + + builder := fake.NewClientBuilder().WithRuntimeObjects( + &configv1.ClusterVersion{ + ObjectMeta: metav1.ObjectMeta{ + Name: "version", + }, + Status: configv1.ClusterVersionStatus{ + History: []configv1.UpdateHistory{ + { + State: configv1.CompletedUpdate, + Version: "4.99.123", + }, + }, + }, + }, + ) + ch := clienthelper.NewWithClient(log, testclienthelper.NewHookingClient(builder.Build())) + tc := testtasks.NewFakeTestContext( + ctx, _env, log, func() time.Time { return time.Unix(100, 0) }, + testtasks.WithClientHelper(ch), + ) + err := ReportClusterVersion(tc) + Expect(err).ToNot(HaveOccurred()) + Expect(tc.GetResultMessage()).To(Equal("cluster version is: 4.99.123")) +} diff --git a/pkg/mimo/tasks/acrtoken.go b/pkg/mimo/tasks/acrtoken.go new file mode 100644 index 00000000000..1f30a9f9e35 --- /dev/null +++ b/pkg/mimo/tasks/acrtoken.go @@ -0,0 +1,19 @@ +package tasks + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/mimo/steps/cluster" + "github.com/Azure/ARO-RP/pkg/util/mimo" + "github.com/Azure/ARO-RP/pkg/util/steps" +) + +func ACRTokenChecker(t mimo.TaskContext, doc *api.MaintenanceManifestDocument, oc *api.OpenShiftClusterDocument) (api.MaintenanceManifestState, string) { + s := []steps.Step{ + steps.Action(cluster.EnsureACRTokenIsValid), + } + + return run(t, s) +} diff --git a/pkg/mimo/tasks/interfaces.go b/pkg/mimo/tasks/interfaces.go new file mode 100644 index 00000000000..13b71d1f19d --- /dev/null +++ b/pkg/mimo/tasks/interfaces.go @@ -0,0 +1,11 @@ +package tasks + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/util/mimo" +) + +type MaintenanceTask func(mimo.TaskContext, *api.MaintenanceManifestDocument, *api.OpenShiftClusterDocument) (api.MaintenanceManifestState, string) diff --git a/pkg/mimo/tasks/operatorflags.go b/pkg/mimo/tasks/operatorflags.go new file mode 100644 index 00000000000..fc256a5d0fa --- /dev/null +++ b/pkg/mimo/tasks/operatorflags.go @@ -0,0 +1,21 @@ +package tasks + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/mimo/steps/cluster" + "github.com/Azure/ARO-RP/pkg/util/mimo" + "github.com/Azure/ARO-RP/pkg/util/steps" +) + +func UpdateOperatorFlags(t mimo.TaskContext, doc *api.MaintenanceManifestDocument, oc *api.OpenShiftClusterDocument) (api.MaintenanceManifestState, string) { + s := []steps.Step{ + steps.Action(cluster.EnsureAPIServerIsUp), + + steps.Action(cluster.UpdateClusterOperatorFlags), + } + + return run(t, s) +} diff --git a/pkg/mimo/tasks/sets.go b/pkg/mimo/tasks/sets.go new file mode 100644 index 00000000000..b38fca9e218 --- /dev/null +++ b/pkg/mimo/tasks/sets.go @@ -0,0 +1,34 @@ +package tasks + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "time" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/mimo" + utilmimo "github.com/Azure/ARO-RP/pkg/util/mimo" + "github.com/Azure/ARO-RP/pkg/util/steps" +) + +const DEFAULT_POLL_TIME = time.Second * 10 +const DEFAULT_TIMEOUT_DURATION = time.Minute * 20 + +var DEFAULT_MAINTENANCE_SETS = map[string]MaintenanceTask{ + mimo.TLS_CERT_ROTATION_ID: TLSCertRotation, + mimo.ACR_TOKEN_CHECKER_ID: ACRTokenChecker, + mimo.OPERATOR_FLAGS_UPDATE_ID: UpdateOperatorFlags, +} + +func run(t utilmimo.TaskContext, s []steps.Step) (api.MaintenanceManifestState, string) { + _, err := steps.Run(t, t.Log(), DEFAULT_POLL_TIME, s, t.Now) + + if err != nil { + if utilmimo.IsRetryableError(err) { + return api.MaintenanceManifestStatePending, err.Error() + } + return api.MaintenanceManifestStateFailed, err.Error() + } + return api.MaintenanceManifestStateCompleted, t.GetResultMessage() +} diff --git a/pkg/mimo/tasks/tls_cert_rotate.go b/pkg/mimo/tasks/tls_cert_rotate.go new file mode 100644 index 00000000000..61e536dbacd --- /dev/null +++ b/pkg/mimo/tasks/tls_cert_rotate.go @@ -0,0 +1,22 @@ +package tasks + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/mimo/steps/cluster" + "github.com/Azure/ARO-RP/pkg/util/mimo" + "github.com/Azure/ARO-RP/pkg/util/steps" +) + +func TLSCertRotation(t mimo.TaskContext, doc *api.MaintenanceManifestDocument, oc *api.OpenShiftClusterDocument) (api.MaintenanceManifestState, string) { + s := []steps.Step{ + steps.Action(cluster.EnsureAPIServerIsUp), + + steps.Action(cluster.RotateAPIServerCertificate), + steps.Action(cluster.EnsureAPIServerServingCertificateConfiguration), + } + + return run(t, s) +} diff --git a/pkg/util/acrtoken/acrtoken.go b/pkg/util/acrtoken/acrtoken.go index b1c87b3ba0d..762d97a8693 100644 --- a/pkg/util/acrtoken/acrtoken.go +++ b/pkg/util/acrtoken/acrtoken.go @@ -7,10 +7,12 @@ import ( "context" "fmt" "net/http" + "time" mgmtcontainerregistry "github.com/Azure/azure-sdk-for-go/services/preview/containerregistry/mgmt/2020-11-01-preview/containerregistry" "github.com/Azure/go-autorest/autorest" "github.com/Azure/go-autorest/autorest/azure" + "github.com/Azure/go-autorest/autorest/date" "github.com/Azure/go-autorest/autorest/to" "github.com/Azure/ARO-RP/pkg/api" @@ -21,6 +23,7 @@ import ( type Manager interface { GetRegistryProfile(oc *api.OpenShiftCluster) *api.RegistryProfile + GetRegistryProfileFromSlice(oc []*api.RegistryProfile) *api.RegistryProfile NewRegistryProfile(oc *api.OpenShiftCluster) *api.RegistryProfile PutRegistryProfile(oc *api.OpenShiftCluster, rp *api.RegistryProfile) EnsureTokenAndPassword(ctx context.Context, rp *api.RegistryProfile) (string, error) @@ -36,6 +39,11 @@ type manager struct { registries containerregistry.RegistriesClient } +const ( + HoursInADay = 24 + ACRTokenLifeInDays = 90 +) + func NewManager(env env.Interface, localFPAuthorizer autorest.Authorizer) (Manager, error) { r, err := azure.ParseResourceID(env.ACRResourceID()) if err != nil { @@ -63,10 +71,21 @@ func (m *manager) GetRegistryProfile(oc *api.OpenShiftCluster) *api.RegistryProf return nil } +func (m *manager) GetRegistryProfileFromSlice(registryProfiles []*api.RegistryProfile) *api.RegistryProfile { + for _, rp := range registryProfiles { + if rp.Name == fmt.Sprintf("%s.%s", m.r.ResourceName, m.env.Environment().ContainerRegistryDNSSuffix) { + return rp + } + } + + return nil +} + func (m *manager) NewRegistryProfile(oc *api.OpenShiftCluster) *api.RegistryProfile { return &api.RegistryProfile{ - Name: fmt.Sprintf("%s.%s", m.r.ResourceName, m.env.Environment().ContainerRegistryDNSSuffix), - Username: "token-" + uuid.DefaultGenerator.Generate(), + Name: fmt.Sprintf("%s.%s", m.r.ResourceName, m.env.Environment().ContainerRegistryDNSSuffix), + Username: "token-" + uuid.DefaultGenerator.Generate(), + IssueDate: &date.Time{Time: time.Now().UTC()}, } } diff --git a/pkg/util/acrtoken/acrtoken_test.go b/pkg/util/acrtoken/acrtoken_test.go index 05639752a68..1864ce3b8f0 100644 --- a/pkg/util/acrtoken/acrtoken_test.go +++ b/pkg/util/acrtoken/acrtoken_test.go @@ -67,7 +67,7 @@ func TestEnsureTokenAndPassword(t *testing.T) { tokens: tokens, } - password, err := m.EnsureTokenAndPassword(ctx, &api.RegistryProfile{Username: tokenName}) + password, err := m.EnsureTokenAndPassword(ctx, &api.RegistryProfile{Username: tokenName, IssueDate: &date.Time{Time: time.Now().AddDate(0, 0, -50)}}) if err != nil { t.Fatal(err) } diff --git a/pkg/util/buckets/buckets.go b/pkg/util/buckets/buckets.go new file mode 100644 index 00000000000..128960e21c5 --- /dev/null +++ b/pkg/util/buckets/buckets.go @@ -0,0 +1,60 @@ +package buckets + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "strings" + "sync" + "time" + + "github.com/sirupsen/logrus" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/util/bucket" +) + +type WorkerFunc func(<-chan struct{}, time.Duration, string) + +type monitor struct { + baseLog *logrus.Entry + + bucketCount int + buckets map[int]struct{} + + mu *sync.RWMutex + docs map[string]*cacheDoc + + worker WorkerFunc +} + +type BucketWorker interface { + Stop() + + Doc(string) *api.OpenShiftClusterDocument + DeleteDoc(*api.OpenShiftClusterDocument) + UpsertDoc(*api.OpenShiftClusterDocument) +} + +func NewBucketWorker(log *logrus.Entry, worker WorkerFunc, mu *sync.RWMutex) *monitor { + return &monitor{ + baseLog: log, + + worker: worker, + docs: map[string]*cacheDoc{}, + + buckets: map[int]struct{}{}, + bucketCount: bucket.Buckets, + + mu: mu, + } +} + +func (mon *monitor) Doc(id string) *api.OpenShiftClusterDocument { + id = strings.ToLower(id) + v := mon.docs[id] + if v == nil { + return nil + } + return v.doc +} diff --git a/pkg/util/buckets/cache.go b/pkg/util/buckets/cache.go new file mode 100644 index 00000000000..603bcf4c62d --- /dev/null +++ b/pkg/util/buckets/cache.go @@ -0,0 +1,86 @@ +package buckets + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "math/rand" + "strings" + "time" + + "github.com/Azure/ARO-RP/pkg/api" +) + +type cacheDoc struct { + doc *api.OpenShiftClusterDocument + stop chan<- struct{} +} + +// deleteDoc deletes the given document from mon.docs, signalling the associated +// monitoring goroutine to stop if it exists. Caller must hold mon.mu.Lock. +func (mon *monitor) DeleteDoc(doc *api.OpenShiftClusterDocument) { + id := strings.ToLower(doc.ID) + v := mon.docs[id] + + if v != nil { + if v.stop != nil { + close(mon.docs[id].stop) + } + + delete(mon.docs, id) + } +} + +// upsertDoc inserts or updates the given document into mon.docs, starting an +// associated monitoring goroutine if the document is in a bucket owned by us. +// Caller must hold mon.mu.Lock. +func (mon *monitor) UpsertDoc(doc *api.OpenShiftClusterDocument) { + id := strings.ToLower(doc.ID) + v := mon.docs[id] + + if v == nil { + v = &cacheDoc{} + mon.docs[id] = v + } + + v.doc = doc + mon.FixDoc(doc) +} + +// fixDoc ensures that there is a monitoring goroutine for the given document +// iff it is in a bucket owned by us. Caller must hold mon.mu.Lock. +func (mon *monitor) FixDoc(doc *api.OpenShiftClusterDocument) { + id := strings.ToLower(doc.ID) + v := mon.docs[id] + + mon.baseLog.Debugf("fixing doc %s (%s)", doc.ID, doc.Key) + + // TODO: bucketing logic + //_, ours := mon.buckets[v.doc.Bucket] + ours := true + + if !ours && v.stop != nil { + mon.baseLog.Debugf("stopping channel for %s", doc.ID) + close(v.stop) + v.stop = nil + } else if ours && v.stop == nil { + ch := make(chan struct{}) + v.stop = ch + + delay := time.Duration(rand.Intn(60)) * time.Second + + go mon.worker(ch, delay, doc.Key) + } +} + +// Stop stops all workers. +func (mon *monitor) Stop() { + mon.mu.Lock() + defer mon.mu.Unlock() + for _, v := range mon.docs { + if v.stop != nil { + close(v.stop) + v.stop = nil + } + } +} diff --git a/pkg/util/clusteroperators/isavailable.go b/pkg/util/clusteroperators/isavailable.go new file mode 100644 index 00000000000..57a08dc55b8 --- /dev/null +++ b/pkg/util/clusteroperators/isavailable.go @@ -0,0 +1,28 @@ +package clusteroperators + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "fmt" + + configv1 "github.com/openshift/api/config/v1" +) + +func IsOperatorAvailable(operator *configv1.ClusterOperator) bool { + m := make(map[configv1.ClusterStatusConditionType]configv1.ConditionStatus, len(operator.Status.Conditions)) + for _, cond := range operator.Status.Conditions { + m[cond.Type] = cond.Status + } + return m[configv1.OperatorAvailable] == configv1.ConditionTrue && m[configv1.OperatorProgressing] == configv1.ConditionFalse +} + +func OperatorStatusText(operator *configv1.ClusterOperator) string { + m := make(map[configv1.ClusterStatusConditionType]configv1.ConditionStatus, len(operator.Status.Conditions)) + for _, cond := range operator.Status.Conditions { + m[cond.Type] = cond.Status + } + return fmt.Sprintf("%s %s=%s, %s=%s", operator.Name, + configv1.OperatorAvailable, m[configv1.OperatorAvailable], configv1.OperatorProgressing, m[configv1.OperatorProgressing], + ) +} diff --git a/pkg/util/clusteroperators/isavailable_test.go b/pkg/util/clusteroperators/isavailable_test.go new file mode 100644 index 00000000000..71672f799eb --- /dev/null +++ b/pkg/util/clusteroperators/isavailable_test.go @@ -0,0 +1,120 @@ +package clusteroperators + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "testing" + + configv1 "github.com/openshift/api/config/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestIsOperatorAvailable(t *testing.T) { + for _, tt := range []struct { + name string + availableCondition configv1.ConditionStatus + progressingCondition configv1.ConditionStatus + want bool + }{ + { + name: "Available && Progressing; not available", + availableCondition: configv1.ConditionTrue, + progressingCondition: configv1.ConditionTrue, + }, + { + name: "Available && !Progressing; available", + availableCondition: configv1.ConditionTrue, + progressingCondition: configv1.ConditionFalse, + want: true, + }, + { + name: "!Available && Progressing; not available", + availableCondition: configv1.ConditionFalse, + progressingCondition: configv1.ConditionTrue, + }, + { + name: "!Available && !Progressing; not available", + availableCondition: configv1.ConditionFalse, + progressingCondition: configv1.ConditionFalse, + }, + } { + operator := &configv1.ClusterOperator{ + ObjectMeta: metav1.ObjectMeta{ + Name: "name", + }, + Status: configv1.ClusterOperatorStatus{ + Conditions: []configv1.ClusterOperatorStatusCondition{ + { + Type: configv1.OperatorAvailable, + Status: tt.availableCondition, + }, + { + Type: configv1.OperatorProgressing, + Status: tt.progressingCondition, + }, + }, + }, + } + available := IsOperatorAvailable(operator) + if available != tt.want { + t.Error(available) + } + } +} + +func TestOperatorStatusText(t *testing.T) { + for _, tt := range []struct { + name string + availableCondition configv1.ConditionStatus + progressingCondition configv1.ConditionStatus + want string + }{ + { + name: "Available && Progressing; not available", + availableCondition: configv1.ConditionTrue, + progressingCondition: configv1.ConditionTrue, + want: "server Available=True, Progressing=True", + }, + { + name: "Available && !Progressing; available", + availableCondition: configv1.ConditionTrue, + progressingCondition: configv1.ConditionFalse, + want: "server Available=True, Progressing=False", + }, + { + name: "!Available && Progressing; not available", + availableCondition: configv1.ConditionFalse, + progressingCondition: configv1.ConditionTrue, + want: "server Available=False, Progressing=True", + }, + { + name: "!Available && !Progressing; not available", + availableCondition: configv1.ConditionFalse, + progressingCondition: configv1.ConditionFalse, + want: "server Available=False, Progressing=False", + }, + } { + operator := &configv1.ClusterOperator{ + ObjectMeta: metav1.ObjectMeta{ + Name: "server", + }, + Status: configv1.ClusterOperatorStatus{ + Conditions: []configv1.ClusterOperatorStatusCondition{ + { + Type: configv1.OperatorAvailable, + Status: tt.availableCondition, + }, + { + Type: configv1.OperatorProgressing, + Status: tt.progressingCondition, + }, + }, + }, + } + available := OperatorStatusText(operator) + if available != tt.want { + t.Error(available) + } + } +} diff --git a/pkg/util/mimo/errors.go b/pkg/util/mimo/errors.go new file mode 100644 index 00000000000..7ee6c2cf0a4 --- /dev/null +++ b/pkg/util/mimo/errors.go @@ -0,0 +1,57 @@ +package mimo + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import "fmt" + +type MIMOErrorVariety string + +const ( + MIMOErrorTypeTransientError MIMOErrorVariety = "TransientError" + MIMOErrorTypeTerminalError MIMOErrorVariety = "TerminalError" +) + +type MIMOError interface { + error + MIMOErrorVariety() MIMOErrorVariety +} + +type wrappedMIMOError struct { + error + variety MIMOErrorVariety +} + +func (f wrappedMIMOError) MIMOErrorVariety() MIMOErrorVariety { + return f.variety +} + +func (f wrappedMIMOError) Error() string { + return fmt.Sprintf("%s: %s", f.variety, f.error.Error()) +} + +func NewMIMOError(err error, variety MIMOErrorVariety) MIMOError { + return wrappedMIMOError{ + error: err, + variety: variety, + } +} + +func TerminalError(err error) MIMOError { + return NewMIMOError(err, MIMOErrorTypeTerminalError) +} + +func TransientError(err error) MIMOError { + return NewMIMOError(err, MIMOErrorTypeTransientError) +} + +func IsRetryableError(err error) bool { + e, ok := err.(wrappedMIMOError) + if !ok { + return false + } + if e.MIMOErrorVariety() == MIMOErrorTypeTransientError { + return true + } + return false +} diff --git a/pkg/util/mimo/taskcontext.go b/pkg/util/mimo/taskcontext.go new file mode 100644 index 00000000000..0a3ed2a8614 --- /dev/null +++ b/pkg/util/mimo/taskcontext.go @@ -0,0 +1,43 @@ +package mimo + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "fmt" + "time" + + "github.com/Azure/go-autorest/autorest" + "github.com/sirupsen/logrus" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/env" + "github.com/Azure/ARO-RP/pkg/util/clienthelper" +) + +type TaskContext interface { + context.Context + Now() time.Time + Environment() env.Interface + ClientHelper() (clienthelper.Interface, error) + Log() *logrus.Entry + LocalFpAuthorizer() (autorest.Authorizer, error) + + // OpenShiftCluster + GetClusterUUID() string + GetOpenShiftClusterProperties() api.OpenShiftClusterProperties + GetOpenshiftClusterDocument() *api.OpenShiftClusterDocument + + SetResultMessage(string) + GetResultMessage() string +} + +func GetTaskContext(c context.Context) (TaskContext, error) { + r, ok := c.(TaskContext) + if !ok { + return nil, fmt.Errorf("cannot convert %v", r) + } + + return r, nil +} diff --git a/pkg/util/service/const.go b/pkg/util/service/const.go new file mode 100644 index 00000000000..0183d581ec1 --- /dev/null +++ b/pkg/util/service/const.go @@ -0,0 +1,10 @@ +package service + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +const ( + DatabaseName = "DATABASE_NAME" + DatabaseAccountName = "DATABASE_ACCOUNT_NAME" + KeyVaultPrefix = "KEYVAULT_PREFIX" +) diff --git a/pkg/util/service/database.go b/pkg/util/service/database.go new file mode 100644 index 00000000000..6466b40a6b6 --- /dev/null +++ b/pkg/util/service/database.go @@ -0,0 +1,83 @@ +package service + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "fmt" + "os" + + "github.com/sirupsen/logrus" + + "github.com/Azure/ARO-RP/pkg/database" + "github.com/Azure/ARO-RP/pkg/database/cosmosdb" + "github.com/Azure/ARO-RP/pkg/env" + "github.com/Azure/ARO-RP/pkg/metrics" + "github.com/Azure/ARO-RP/pkg/util/encryption" + "github.com/Azure/ARO-RP/pkg/util/keyvault" +) + +// NewDatabase creates a CosmosDB database client from the environment configuration. +func NewDatabase(ctx context.Context, _env env.Core, log *logrus.Entry, m metrics.Emitter, withAEAD bool) (cosmosdb.DatabaseClient, error) { + var aead encryption.AEAD + + msiToken, err := _env.NewMSITokenCredential() + if err != nil { + return nil, err + } + + if withAEAD { + msiKVAuthorizer, err := _env.NewMSIAuthorizer(_env.Environment().KeyVaultScope) + if err != nil { + return nil, err + } + + keyVaultPrefix := os.Getenv(KeyVaultPrefix) + // TODO: should not be using the service keyvault here + serviceKeyvaultURI := keyvault.URI(_env, env.ServiceKeyvaultSuffix, keyVaultPrefix) + serviceKeyvault := keyvault.NewManager(msiKVAuthorizer, serviceKeyvaultURI) + + aead, err = encryption.NewMulti( + ctx, + serviceKeyvault, + env.EncryptionSecretV2Name, + env.EncryptionSecretName, + ) + if err != nil { + return nil, err + } + } + + dbAccountName := os.Getenv(DatabaseAccountName) + scope := []string{ + fmt.Sprintf("https://%s.%s", dbAccountName, _env.Environment().CosmosDBDNSSuffixScope), + } + + logrusEntry := log.WithField("component", "database") + + dbAuthorizer, err := database.NewTokenAuthorizer( + ctx, + logrusEntry, + msiToken, + dbAccountName, + scope, + ) + if err != nil { + return nil, err + } + + dbc, err := database.NewDatabaseClient( + logrusEntry, + _env, + dbAuthorizer, + m, + aead, + dbAccountName, + ) + if err != nil { + return nil, err + } + + return dbc, nil +} diff --git a/pkg/util/service/helpers.go b/pkg/util/service/helpers.go new file mode 100644 index 00000000000..5013849d835 --- /dev/null +++ b/pkg/util/service/helpers.go @@ -0,0 +1,23 @@ +package service + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "fmt" + "os" + + "github.com/Azure/ARO-RP/pkg/env" +) + +func DBName(isLocalDevelopmentMode bool) (string, error) { + if !isLocalDevelopmentMode { + return "ARO", nil + } + + if err := env.ValidateVars(DatabaseName); err != nil { + return "", fmt.Errorf("%v (development mode)", err.Error()) + } + + return os.Getenv(DatabaseName), nil +} diff --git a/test/database/check.go b/test/database/check.go index dd1e428a9c9..d2e34fb6997 100644 --- a/test/database/check.go +++ b/test/database/check.go @@ -27,12 +27,25 @@ type Checker struct { openShiftVersionDocuments []*api.OpenShiftVersionDocument platformWorkloadIdentityRoleSetDocuments []*api.PlatformWorkloadIdentityRoleSetDocument validationResult []*api.ValidationResult + maintenanceManifestDocuments []*api.MaintenanceManifestDocument } func NewChecker() *Checker { return &Checker{} } +func (f *Checker) Clear() { + f.openshiftClusterDocuments = []*api.OpenShiftClusterDocument{} + f.subscriptionDocuments = []*api.SubscriptionDocument{} + f.billingDocuments = []*api.BillingDocument{} + f.asyncOperationDocuments = []*api.AsyncOperationDocument{} + f.portalDocuments = []*api.PortalDocument{} + f.gatewayDocuments = []*api.GatewayDocument{} + f.openShiftVersionDocuments = []*api.OpenShiftVersionDocument{} + f.maintenanceManifestDocuments = []*api.MaintenanceManifestDocument{} + f.validationResult = []*api.ValidationResult{} +} + func (f *Checker) AddOpenShiftClusterDocuments(docs ...*api.OpenShiftClusterDocument) { for _, doc := range docs { docCopy, err := deepCopy(doc) @@ -132,6 +145,17 @@ func (f *Checker) AddValidationResult(docs ...*api.ValidationResult) { } } +func (f *Checker) AddMaintenanceManifestDocuments(docs ...*api.MaintenanceManifestDocument) { + for _, doc := range docs { + docCopy, err := deepCopy(doc) + if err != nil { + panic(err) + } + + f.maintenanceManifestDocuments = append(f.maintenanceManifestDocuments, docCopy.(*api.MaintenanceManifestDocument)) + } +} + func (f *Checker) CheckOpenShiftClusters(openShiftClusters *cosmosdb.FakeOpenShiftClusterDocumentClient) (errs []error) { ctx := context.Background() @@ -309,3 +333,27 @@ func (f *Checker) CheckPlatformWorkloadIdentityRoleSets(roleSets *cosmosdb.FakeP return errs } + +func (f *Checker) CheckMaintenanceManifests(client *cosmosdb.FakeMaintenanceManifestDocumentClient) (errs []error) { + ctx := context.Background() + + all, err := client.ListAll(ctx, nil) + if err != nil { + return []error{err} + } + + sort.Slice(all.MaintenanceManifestDocuments, func(i, j int) bool { + return all.MaintenanceManifestDocuments[i].ID < all.MaintenanceManifestDocuments[j].ID + }) + + if len(f.maintenanceManifestDocuments) != 0 && len(all.MaintenanceManifestDocuments) == len(f.maintenanceManifestDocuments) { + diff := deep.Equal(all.MaintenanceManifestDocuments, f.maintenanceManifestDocuments) + for _, i := range diff { + errs = append(errs, errors.New(i)) + } + } else if len(all.MaintenanceManifestDocuments) != 0 || len(f.maintenanceManifestDocuments) != 0 { + errs = append(errs, fmt.Errorf("document length different, %d vs %d", len(all.MaintenanceManifestDocuments), len(f.maintenanceManifestDocuments))) + } + + return errs +} diff --git a/test/database/fixtures.go b/test/database/fixtures.go index 0341e27f71a..cef398616ad 100644 --- a/test/database/fixtures.go +++ b/test/database/fixtures.go @@ -21,6 +21,7 @@ type Fixture struct { openShiftVersionDocuments []*api.OpenShiftVersionDocument platformWorkloadIdentityRoleSetDocuments []*api.PlatformWorkloadIdentityRoleSetDocument clusterManagerConfigurationDocuments []*api.ClusterManagerConfigurationDocument + maintenanceManifestDocuments []*api.MaintenanceManifestDocument openShiftClustersDatabase database.OpenShiftClusters billingDatabase database.Billing @@ -31,6 +32,7 @@ type Fixture struct { openShiftVersionsDatabase database.OpenShiftVersions platformWorkloadIdentityRoleSetsDatabase database.PlatformWorkloadIdentityRoleSets clusterManagerConfigurationsDatabase database.ClusterManagerConfigurations + maintenanceManifestsDatabase database.MaintenanceManifests openShiftVersionsUUID uuid.Generator platformWorkloadIdentityRoleSetsUUID uuid.Generator @@ -40,6 +42,18 @@ func NewFixture() *Fixture { return &Fixture{} } +func (f *Fixture) Clear() { + f.openshiftClusterDocuments = []*api.OpenShiftClusterDocument{} + f.subscriptionDocuments = []*api.SubscriptionDocument{} + f.billingDocuments = []*api.BillingDocument{} + f.asyncOperationDocuments = []*api.AsyncOperationDocument{} + f.portalDocuments = []*api.PortalDocument{} + f.gatewayDocuments = []*api.GatewayDocument{} + f.openShiftVersionDocuments = []*api.OpenShiftVersionDocument{} + f.clusterManagerConfigurationDocuments = []*api.ClusterManagerConfigurationDocument{} + f.maintenanceManifestDocuments = []*api.MaintenanceManifestDocument{} +} + func (f *Fixture) WithClusterManagerConfigurations(db database.ClusterManagerConfigurations) *Fixture { f.clusterManagerConfigurationsDatabase = db return f @@ -87,6 +101,11 @@ func (f *Fixture) WithPlatformWorkloadIdentityRoleSets(db database.PlatformWorkl return f } +func (f *Fixture) WithMaintenanceManifests(db database.MaintenanceManifests) *Fixture { + f.maintenanceManifestsDatabase = db + return f +} + func (f *Fixture) AddOpenShiftClusterDocuments(docs ...*api.OpenShiftClusterDocument) { for _, doc := range docs { docCopy, err := deepCopy(doc) @@ -186,6 +205,17 @@ func (f *Fixture) AddClusterManagerConfigurationDocuments(docs ...*api.ClusterMa } } +func (f *Fixture) AddMaintenanceManifestDocuments(docs ...*api.MaintenanceManifestDocument) { + for _, doc := range docs { + docCopy, err := deepCopy(doc) + if err != nil { + panic(err) + } + + f.maintenanceManifestDocuments = append(f.maintenanceManifestDocuments, docCopy.(*api.MaintenanceManifestDocument)) + } +} + func (f *Fixture) Create() error { ctx := context.Background() @@ -264,5 +294,15 @@ func (f *Fixture) Create() error { } } + for _, i := range f.maintenanceManifestDocuments { + if i.ID == "" { + i.ID = f.maintenanceManifestsDatabase.NewUUID() + } + _, err := f.maintenanceManifestsDatabase.Create(ctx, i) + if err != nil { + return err + } + } + return nil } diff --git a/test/database/inmemory.go b/test/database/inmemory.go index 66907cf69ed..3a8a291f83e 100644 --- a/test/database/inmemory.go +++ b/test/database/inmemory.go @@ -4,6 +4,8 @@ package database // Licensed under the Apache License 2.0. import ( + "time" + "github.com/ugorji/go/codec" "github.com/Azure/ARO-RP/pkg/database" @@ -86,3 +88,12 @@ func NewFakeClusterManager() (db database.ClusterManagerConfigurations, client * db = database.NewClusterManagerConfigurationsWithProvidedClient(client, coll, "", uuid) return db, client } + +func NewFakeMaintenanceManifests(now func() time.Time) (db database.MaintenanceManifests, client *cosmosdb.FakeMaintenanceManifestDocumentClient) { + uuid := deterministicuuid.NewTestUUIDGenerator(deterministicuuid.MAINTENANCE_MANIFESTS) + coll := &fakeCollectionClient{} + client = cosmosdb.NewFakeMaintenanceManifestDocumentClient(jsonHandle) + injectMaintenanceManifests(client, now) + db = database.NewMaintenanceManifestsWithProvidedClient(client, coll, "", uuid) + return db, client +} diff --git a/test/database/maintenancemanifests.go b/test/database/maintenancemanifests.go new file mode 100644 index 00000000000..e20387253d9 --- /dev/null +++ b/test/database/maintenancemanifests.go @@ -0,0 +1,94 @@ +package database + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "cmp" + "context" + "slices" + "strconv" + "time" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/database" + "github.com/Azure/ARO-RP/pkg/database/cosmosdb" +) + +func injectMaintenanceManifests(c *cosmosdb.FakeMaintenanceManifestDocumentClient, now func() time.Time) { + c.SetQueryHandler(database.MaintenanceManifestQueryForCluster, func(client cosmosdb.MaintenanceManifestDocumentClient, query *cosmosdb.Query, options *cosmosdb.Options) cosmosdb.MaintenanceManifestDocumentRawIterator { + return fakeMaintenanceManifestsForCluster(client, query, options, now) + }) + c.SetQueryHandler(database.MaintenanceManifestDequeueQueryForCluster, func(client cosmosdb.MaintenanceManifestDocumentClient, query *cosmosdb.Query, options *cosmosdb.Options) cosmosdb.MaintenanceManifestDocumentRawIterator { + return fakeMaintenanceManifestsDequeueForCluster(client, query, options, now) + }) + + c.SetTriggerHandler("renewLease", func(ctx context.Context, doc *api.MaintenanceManifestDocument) error { + return fakeMaintenanceManifestsRenewLeaseTrigger(ctx, doc, now) + }) +} + +func fakeMaintenanceManifestsDequeueForCluster(client cosmosdb.MaintenanceManifestDocumentClient, query *cosmosdb.Query, options *cosmosdb.Options, now func() time.Time) cosmosdb.MaintenanceManifestDocumentRawIterator { + startingIndex, err := fakeMaintenanceManifestsGetContinuation(options) + if err != nil { + return cosmosdb.NewFakeMaintenanceManifestDocumentErroringRawIterator(err) + } + + input, err := client.ListAll(context.Background(), nil) + if err != nil { + // TODO: should this never happen? + panic(err) + } + + clusterResourceID := query.Parameters[0].Value + + var results []*api.MaintenanceManifestDocument + for _, r := range input.MaintenanceManifestDocuments { + if r.ClusterResourceID == clusterResourceID && + r.MaintenanceManifest.State == api.MaintenanceManifestStatePending { + results = append(results, r) + } + } + + return cosmosdb.NewFakeMaintenanceManifestDocumentIterator(results, startingIndex) +} + +func fakeMaintenanceManifestsForCluster(client cosmosdb.MaintenanceManifestDocumentClient, query *cosmosdb.Query, options *cosmosdb.Options, now func() time.Time) cosmosdb.MaintenanceManifestDocumentRawIterator { + startingIndex, err := fakeMaintenanceManifestsGetContinuation(options) + if err != nil { + return cosmosdb.NewFakeMaintenanceManifestDocumentErroringRawIterator(err) + } + + input, err := client.ListAll(context.Background(), nil) + if err != nil { + // TODO: should this never happen? + panic(err) + } + + clusterResourceID := query.Parameters[0].Value + + var results []*api.MaintenanceManifestDocument + for _, r := range input.MaintenanceManifestDocuments { + if r.ClusterResourceID == clusterResourceID { + results = append(results, r) + } + } + + slices.SortFunc(results, func(a, b *api.MaintenanceManifestDocument) int { + return cmp.Compare(a.ID, b.ID) + }) + + return cosmosdb.NewFakeMaintenanceManifestDocumentIterator(results, startingIndex) +} + +func fakeMaintenanceManifestsRenewLeaseTrigger(ctx context.Context, doc *api.MaintenanceManifestDocument, now func() time.Time) error { + doc.LeaseExpires = int(now().Unix()) + 60 + return nil +} + +func fakeMaintenanceManifestsGetContinuation(options *cosmosdb.Options) (startingIndex int, err error) { + if options != nil && options.Continuation != "" { + startingIndex, err = strconv.Atoi(options.Continuation) + } + return +} diff --git a/test/database/openshiftclusters.go b/test/database/openshiftclusters.go index bc9c017fe58..f348f185cec 100644 --- a/test/database/openshiftclusters.go +++ b/test/database/openshiftclusters.go @@ -151,6 +151,28 @@ func openShiftClusterConflictChecker(one *api.OpenShiftClusterDocument, two *api return false } +func fakeOpenShiftClustersOnlyResourceID(client cosmosdb.OpenShiftClusterDocumentClient, query *cosmosdb.Query, options *cosmosdb.Options) cosmosdb.OpenShiftClusterDocumentRawIterator { + startingIndex, err := fakeOpenShiftClustersGetContinuation(options) + if err != nil { + return cosmosdb.NewFakeOpenShiftClusterDocumentErroringRawIterator(err) + } + + docs, err := fakeOpenShiftClustersGetAllDocuments(client) + if err != nil { + return cosmosdb.NewFakeOpenShiftClusterDocumentErroringRawIterator(err) + } + + newDocs := make([]*api.OpenShiftClusterDocument, 0) + + for _, d := range docs { + newDocs = append(newDocs, &api.OpenShiftClusterDocument{ + Key: d.Key, + }) + } + + return cosmosdb.NewFakeOpenShiftClusterDocumentIterator(newDocs, startingIndex) +} + func injectOpenShiftClusters(c *cosmosdb.FakeOpenShiftClusterDocumentClient) { c.SetQueryHandler(database.OpenShiftClustersDequeueQuery, fakeOpenShiftClustersDequeueQuery) c.SetQueryHandler(database.OpenShiftClustersQueueLengthQuery, fakeOpenShiftClustersQueueLengthQuery) @@ -158,6 +180,7 @@ func injectOpenShiftClusters(c *cosmosdb.FakeOpenShiftClusterDocumentClient) { c.SetQueryHandler(database.OpenshiftClustersClientIdQuery, fakeOpenshiftClustersMatchQuery) c.SetQueryHandler(database.OpenshiftClustersResourceGroupQuery, fakeOpenshiftClustersMatchQuery) c.SetQueryHandler(database.OpenshiftClustersPrefixQuery, fakeOpenshiftClustersPrefixQuery) + c.SetQueryHandler(database.OpenshiftClustersClusterResourceIDOnlyQuery, fakeOpenShiftClustersOnlyResourceID) c.SetTriggerHandler("renewLease", fakeOpenShiftClustersRenewLeaseTrigger) diff --git a/test/e2e/adminapi.go b/test/e2e/adminapi.go index 9b3d2375217..fea6a36178a 100644 --- a/test/e2e/adminapi.go +++ b/test/e2e/adminapi.go @@ -15,11 +15,40 @@ import ( . "github.com/onsi/gomega" + "github.com/sirupsen/logrus" + "github.com/Azure/ARO-RP/pkg/api/admin" "github.com/Azure/ARO-RP/pkg/env" ) -func adminRequest(ctx context.Context, method, path string, params url.Values, strict bool, in, out interface{}) (*http.Response, error) { +type adminReqOpts struct { + Option string + Value interface{} +} + +func logOnError(log *logrus.Entry) []adminReqOpts { + return []adminReqOpts{ + { + Option: "log", + Value: log, + }, + { + Option: "logOnError", + Value: true, + }, + } +} + +func _getAdminReqOpt(key string, opts []adminReqOpts) (interface{}, bool) { + for _, i := range opts { + if i.Option == key { + return i.Value, true + } + } + return nil, false +} + +func adminRequest(ctx context.Context, method, path string, params url.Values, strict bool, in, out interface{}, opts ...adminReqOpts) (*http.Response, error) { if !env.IsLocalDevelopmentMode() { return nil, errors.New("only development RP mode is supported") } @@ -69,7 +98,24 @@ func adminRequest(ctx context.Context, method, path string, params url.Values, s }() if out != nil && resp.Header.Get("Content-Type") == "application/json" { - decoder := json.NewDecoder(resp.Body) + body, err := io.ReadAll(resp.Body) + if err != nil { + return resp, err + } + + if resp.StatusCode == http.StatusInternalServerError { + _, ok := _getAdminReqOpt("logOnError", opts) + if ok { + log, ok := _getAdminReqOpt("log", opts) + if ok { + logger := log.(*logrus.Entry) + + logger.Errorf("Failed request, content: %s", string(body)) + } + } + } + + decoder := json.NewDecoder(bytes.NewBuffer(body)) // If strict is set, enable DisallowUnknownFields. This is used to // verify that the response doesn't contain any fields that are not // defined, namely systemData. diff --git a/test/e2e/mimo_actuator.go b/test/e2e/mimo_actuator.go new file mode 100644 index 00000000000..053c9beae32 --- /dev/null +++ b/test/e2e/mimo_actuator.go @@ -0,0 +1,111 @@ +package e2e + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "encoding/json" + "net/http" + "net/url" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/util/retry" + + "github.com/Azure/ARO-RP/pkg/api/admin" + "github.com/Azure/ARO-RP/pkg/mimo" + "github.com/Azure/ARO-RP/pkg/operator" + "github.com/Azure/ARO-RP/pkg/util/uuid" +) + +var _ = Describe("MIMO Actuator E2E Testing", func() { + BeforeEach(func() { + skipIfNotInDevelopmentEnv() + skipIfMIMOActuatorNotEnabled() + + DeferCleanup(func(ctx context.Context) { + // reset feature flags to their default values + var oc = &admin.OpenShiftCluster{} + resp, err := adminRequest(ctx, + http.MethodPatch, clusterResourceID, nil, true, + json.RawMessage("{\"operatorFlagsMergeStrategy\": \"reset\", \"properties\": {\"maintenanceTask\": \"SyncClusterObject\"}}"), oc) + Expect(err).NotTo(HaveOccurred()) + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + }) + }) + + It("Should be able to schedule and run a maintenance set via the admin API", func(ctx context.Context) { + var oc = &admin.OpenShiftCluster{} + testflag := "aro.e2e.testflag." + uuid.DefaultGenerator.Generate() + + By("set a bogus flag on the cluster") + resp, err := adminRequest(ctx, + http.MethodPatch, clusterResourceID, nil, true, + json.RawMessage("{\"properties\": {\"maintenanceTask\": \"SyncClusterObject\", \"operatorFlags\": {\""+testflag+"\": \"true\"}}}"), oc) + Expect(err).NotTo(HaveOccurred()) + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + + By("waiting for the update to complete") + Eventually(func(g Gomega, ctx context.Context) { + oc = adminGetCluster(g, ctx, clusterResourceID) + g.Expect(oc.Properties.ProvisioningState).To(Equal(admin.ProvisioningStateSucceeded)) + }).WithContext(ctx).WithTimeout(DefaultEventuallyTimeout).Should(Succeed()) + + By("check the flag is set in the cluster") + co, err := clients.AROClusters.AroV1alpha1().Clusters().Get(ctx, "cluster", metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + flag, ok := co.Spec.OperatorFlags[testflag] + Expect(ok).To(BeTrue()) + Expect(flag).To(Equal("true")) + + By("change the flag in-cluster to a wrong value") + // get the flag we want to check for + err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + co, err := clients.AROClusters.AroV1alpha1().Clusters().Get(ctx, "cluster", metav1.GetOptions{}) + if err != nil { + return err + } + + co.Spec.OperatorFlags[testflag] = operator.FlagFalse + _, err = clients.AROClusters.AroV1alpha1().Clusters().Update(ctx, co, metav1.UpdateOptions{}) + return err + }) + Expect(err).NotTo(HaveOccurred()) + + By("creating the flag update manifest via the API") + out := &admin.MaintenanceManifest{} + resp, err = adminRequest(ctx, + http.MethodPut, "/admin"+clusterResourceID+"/maintenancemanifests", + url.Values{}, true, &admin.MaintenanceManifest{ + MaintenanceTaskID: mimo.OPERATOR_FLAGS_UPDATE_ID, + }, &out, logOnError(log)...) + Expect(err).NotTo(HaveOccurred()) + Expect(resp.StatusCode).To(Equal(http.StatusCreated)) + + manifestID := out.ID + + By("waiting for the manifest run to complete") + Eventually(func(g Gomega, ctx context.Context) { + fetchedManifest := &admin.MaintenanceManifest{} + resp, err = adminRequest(ctx, + http.MethodGet, "/admin"+clusterResourceID+"/maintenancemanifests/"+manifestID, + url.Values{}, true, nil, &fetchedManifest) + + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(resp.StatusCode).To(Equal(http.StatusOK)) + g.Expect(fetchedManifest.State).To(Equal(admin.MaintenanceManifestStateCompleted)) + }).WithContext(ctx).WithTimeout(DefaultEventuallyTimeout).Should(Succeed()) + + By("checking the flag has been set back in the cluster") + co, err = clients.AROClusters.AroV1alpha1().Clusters().Get(ctx, "cluster", metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + flag, ok = co.Spec.OperatorFlags[testflag] + Expect(ok).To(BeTrue()) + Expect(flag).To(Equal("true"), "MIMO manifest has not run") + }) +}) diff --git a/test/e2e/setup.go b/test/e2e/setup.go index a337cb83b1d..cbd92ed265a 100644 --- a/test/e2e/setup.go +++ b/test/e2e/setup.go @@ -131,6 +131,12 @@ func skipIfSeleniumNotEnabled() { } } +func skipIfMIMOActuatorNotEnabled() { + if os.Getenv("ARO_E2E_MIMO") == "" { + Skip("ARO_E2E_MIMO not set, skipping MIMO e2e") + } +} + func skipIfNotHiveManagedCluster(adminAPICluster *admin.OpenShiftCluster) { if adminAPICluster.Properties.HiveProfile == (admin.HiveProfile{}) { Skip("skipping tests because this ARO cluster has not been created/adopted by Hive") diff --git a/test/mimo/tasks/testcontext.go b/test/mimo/tasks/testcontext.go new file mode 100644 index 00000000000..791f26f2ef1 --- /dev/null +++ b/test/mimo/tasks/testcontext.go @@ -0,0 +1,120 @@ +package tasks + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "fmt" + "time" + + "github.com/Azure/go-autorest/autorest" + "github.com/sirupsen/logrus" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/env" + "github.com/Azure/ARO-RP/pkg/util/clienthelper" +) + +type fakeTestContext struct { + context.Context + now func() time.Time + env env.Interface + ch clienthelper.Interface + log *logrus.Entry + + clusterUUID string + clusterResourceID string + properties api.OpenShiftClusterProperties + + resultMessage string +} + +type Option func(*fakeTestContext) + +func WithClientHelper(ch clienthelper.Interface) Option { + return func(ftc *fakeTestContext) { + ftc.ch = ch + } +} + +func WithOpenShiftClusterDocument(oc *api.OpenShiftClusterDocument) Option { + return func(ftc *fakeTestContext) { + ftc.clusterUUID = oc.ID + ftc.clusterResourceID = oc.OpenShiftCluster.ID + ftc.properties = oc.OpenShiftCluster.Properties + } +} + +func WithOpenShiftClusterProperties(uuid string, oc api.OpenShiftClusterProperties) Option { + return func(ftc *fakeTestContext) { + ftc.clusterUUID = uuid + ftc.properties = oc + } +} + +func NewFakeTestContext(ctx context.Context, env env.Interface, log *logrus.Entry, now func() time.Time, o ...Option) *fakeTestContext { + ftc := &fakeTestContext{ + Context: ctx, + env: env, + log: log, + now: now, + } + for _, i := range o { + i(ftc) + } + return ftc +} + +func (t *fakeTestContext) LocalFpAuthorizer() (autorest.Authorizer, error) { + myAuthorizer := autorest.NullAuthorizer{} + return myAuthorizer, nil +} +func (t *fakeTestContext) GetOpenshiftClusterDocument() *api.OpenShiftClusterDocument { + myCD := &api.OpenShiftClusterDocument{} + return myCD +} + +// handle + +func (t *fakeTestContext) Environment() env.Interface { + return t.env +} + +func (t *fakeTestContext) ClientHelper() (clienthelper.Interface, error) { + if t.ch == nil { + return nil, fmt.Errorf("missing clienthelper") + } + return t.ch, nil +} + +func (t *fakeTestContext) Log() *logrus.Entry { + return t.log +} + +func (t *fakeTestContext) Now() time.Time { + return t.now() +} + +// OpenShiftCluster +func (t *fakeTestContext) GetClusterUUID() string { + if t.clusterUUID == "" { + panic("didn't set up openshiftcluster in test") + } + return t.clusterUUID +} + +func (t *fakeTestContext) GetOpenShiftClusterProperties() api.OpenShiftClusterProperties { + if t.clusterUUID == "" { + panic("didn't set up openshiftcluster in test") + } + return t.properties +} + +func (t *fakeTestContext) SetResultMessage(s string) { + t.resultMessage = s +} + +func (t *fakeTestContext) GetResultMessage() string { + return t.resultMessage +} diff --git a/test/util/deterministicuuid/deterministicuuid.go b/test/util/deterministicuuid/deterministicuuid.go index f5307dd7941..702442d4998 100644 --- a/test/util/deterministicuuid/deterministicuuid.go +++ b/test/util/deterministicuuid/deterministicuuid.go @@ -19,6 +19,7 @@ const ( GATEWAY OPENSHIFT_VERSIONS CLUSTERMANAGER + MAINTENANCE_MANIFESTS ) type gen struct {