From 1cebc8688a8d026daebc81e9d282b53d854b9ca6 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Thu, 29 Feb 2024 11:39:36 +1100 Subject: [PATCH 01/67] add MIMO to dev --- pkg/api/mimo.go | 37 ++ pkg/api/mimodocument.go | 39 +++ pkg/api/openshiftcluster.go | 1 + pkg/database/cosmosdb/generate.go | 2 +- pkg/database/database.go | 1 + pkg/database/mimo.go | 213 ++++++++++++ pkg/database/openshiftclusters.go | 35 +- pkg/deploy/generator/resources_rp.go | 40 +++ pkg/env/core.go | 2 + pkg/mimo/actuator/actuator_test.go | 319 ++++++++++++++++++ pkg/mimo/actuator/manager.go | 184 ++++++++++ pkg/mimo/actuator/service.go | 242 +++++++++++++ pkg/mimo/actuator/service_test.go | 251 ++++++++++++++ pkg/mimo/actuator/task.go | 59 ++++ pkg/mimo/cmd/cli.go | 186 ++++++++++ pkg/mimo/tasks/example/task.go | 31 ++ pkg/mimo/tasks/example/task_test.go | 85 +++++ pkg/mimo/tasks/interfaces.go | 22 ++ pkg/util/buckets/buckets.go | 60 ++++ pkg/util/buckets/cache.go | 80 +++++ pkg/util/service/const.go | 10 + pkg/util/service/database.go | 83 +++++ pkg/util/service/helpers.go | 23 ++ test/database/check.go | 48 +++ test/database/fixtures.go | 40 +++ test/database/inmemory.go | 11 + test/database/maintenancemanifests.go | 64 ++++ test/database/openshiftclusters.go | 23 ++ test/mimo/tasks/testcontext.go | 38 +++ .../deterministicuuid/deterministicuuid.go | 1 + 30 files changed, 2220 insertions(+), 10 deletions(-) create mode 100644 pkg/api/mimo.go create mode 100644 pkg/api/mimodocument.go create mode 100644 pkg/database/mimo.go create mode 100644 pkg/mimo/actuator/actuator_test.go create mode 100644 pkg/mimo/actuator/manager.go create mode 100644 pkg/mimo/actuator/service.go create mode 100644 pkg/mimo/actuator/service_test.go create mode 100644 pkg/mimo/actuator/task.go create mode 100644 pkg/mimo/cmd/cli.go create mode 100644 pkg/mimo/tasks/example/task.go create mode 100644 pkg/mimo/tasks/example/task_test.go create mode 100644 pkg/mimo/tasks/interfaces.go create mode 100644 pkg/util/buckets/buckets.go create mode 100644 pkg/util/buckets/cache.go create mode 100644 pkg/util/service/const.go create mode 100644 pkg/util/service/database.go create mode 100644 pkg/util/service/helpers.go create mode 100644 test/database/maintenancemanifests.go create mode 100644 test/mimo/tasks/testcontext.go diff --git a/pkg/api/mimo.go b/pkg/api/mimo.go new file mode 100644 index 00000000000..a3ffddf0804 --- /dev/null +++ b/pkg/api/mimo.go @@ -0,0 +1,37 @@ +package api + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +type MaintenanceManifestState string + +const ( + MaintenanceManifestStatePending MaintenanceManifestState = "Pending" + MaintenanceManifestStateInProgress MaintenanceManifestState = "InProgress" + MaintenanceManifestStateCompleted MaintenanceManifestState = "Completed" + MaintenanceManifestStateFailed MaintenanceManifestState = "Failed" + MaintenanceManifestStateTimedOut MaintenanceManifestState = "TimedOut" +) + +type MaintenanceSet struct { + MissingFields + + Name string `json:"name,omitempty"` +} + +// MaintenanceManifest represents an instance of a MaintenanceSet running on a +// given cluster. +type MaintenanceManifest struct { + MissingFields + + State MaintenanceManifestState `json:"state,omitempty"` + StatusText string `json:"statusText,omitempty"` + + MaintenanceSetID string `json:"maintenanceSetID,omitempty"` + Priority int `json:"priority,omitempty"` + + // RunAfter defines the earliest that this manifest should start running + RunAfter int `json:"runAfter,omitempty"` + // RunBefore defines the latest that this manifest should start running + RunBefore int `json:"runBefore,omitempty"` +} diff --git a/pkg/api/mimodocument.go b/pkg/api/mimodocument.go new file mode 100644 index 00000000000..099d14e2a30 --- /dev/null +++ b/pkg/api/mimodocument.go @@ -0,0 +1,39 @@ +package api + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +type MaintenanceManifestDocuments struct { + Count int `json:"_count,omitempty"` + ResourceID string `json:"_rid,omitempty"` + MaintenanceManifestDocuments []*MaintenanceManifestDocument `json:"Documents,omitempty"` +} + +func (e *MaintenanceManifestDocuments) String() string { + return encodeJSON(e) +} + +type MaintenanceManifestDocument struct { + MissingFields + + ID string `json:"id,omitempty"` + ResourceID string `json:"_rid,omitempty"` + Timestamp int `json:"_ts,omitempty"` + Self string `json:"_self,omitempty"` + ETag string `json:"_etag,omitempty" deep:"-"` + Attachments string `json:"_attachments,omitempty"` + TTL int `json:"ttl,omitempty"` + LSN int `json:"_lsn,omitempty"` + Metadata map[string]interface{} `json:"_metadata,omitempty"` + + ClusterResourceID string `json:"clusterResourceID,omitempty"` + MaintenanceManifest *MaintenanceManifest `json:"maintenanceManifest,omitempty"` + + LeaseOwner string `json:"leaseOwner,omitempty" deep:"-"` + LeaseExpires int `json:"leaseExpires,omitempty" deep:"-"` + Dequeues int `json:"dequeues,omitempty"` +} + +func (e *MaintenanceManifestDocument) String() string { + return encodeJSON(e) +} diff --git a/pkg/api/openshiftcluster.go b/pkg/api/openshiftcluster.go index 4b47190ff0c..cd5aa33a1b6 100644 --- a/pkg/api/openshiftcluster.go +++ b/pkg/api/openshiftcluster.go @@ -211,6 +211,7 @@ const ( ProvisioningStateUpdating ProvisioningState = "Updating" ProvisioningStateAdminUpdating ProvisioningState = "AdminUpdating" ProvisioningStateCanceled ProvisioningState = "Canceled" + ProvisioningStateMaintenance ProvisioningState = "Maintenance" ProvisioningStateDeleting ProvisioningState = "Deleting" ProvisioningStateSucceeded ProvisioningState = "Succeeded" ProvisioningStateFailed ProvisioningState = "Failed" diff --git a/pkg/database/cosmosdb/generate.go b/pkg/database/cosmosdb/generate.go index 56bf0816e78..b456c1819b5 100644 --- a/pkg/database/cosmosdb/generate.go +++ b/pkg/database/cosmosdb/generate.go @@ -3,7 +3,7 @@ package cosmosdb // Copyright (c) Microsoft Corporation. // Licensed under the Apache License 2.0. -//go:generate gencosmosdb github.com/Azure/ARO-RP/pkg/api,AsyncOperationDocument github.com/Azure/ARO-RP/pkg/api,BillingDocument github.com/Azure/ARO-RP/pkg/api,GatewayDocument github.com/Azure/ARO-RP/pkg/api,MonitorDocument github.com/Azure/ARO-RP/pkg/api,OpenShiftClusterDocument github.com/Azure/ARO-RP/pkg/api,SubscriptionDocument github.com/Azure/ARO-RP/pkg/api,OpenShiftVersionDocument github.com/Azure/ARO-RP/pkg/api,ClusterManagerConfigurationDocument github.com/Azure/ARO-RP/pkg/api,PlatformWorkloadIdentityRoleSetDocument +//go:generate gencosmosdb github.com/Azure/ARO-RP/pkg/api,AsyncOperationDocument github.com/Azure/ARO-RP/pkg/api,BillingDocument github.com/Azure/ARO-RP/pkg/api,GatewayDocument github.com/Azure/ARO-RP/pkg/api,MonitorDocument github.com/Azure/ARO-RP/pkg/api,OpenShiftClusterDocument github.com/Azure/ARO-RP/pkg/api,SubscriptionDocument github.com/Azure/ARO-RP/pkg/api,OpenShiftVersionDocument github.com/Azure/ARO-RP/pkg/api,ClusterManagerConfigurationDocument github.com/Azure/ARO-RP/pkg/api,PlatformWorkloadIdentityRoleSetDocument github.com/Azure/ARO-RP/pkg/api,MaintenanceManifestDocument //go:generate goimports -local=github.com/Azure/ARO-RP -e -w ./ //go:generate mockgen -destination=../../util/mocks/$GOPACKAGE/$GOPACKAGE.go github.com/Azure/ARO-RP/pkg/database/$GOPACKAGE PermissionClient //go:generate goimports -local=github.com/Azure/ARO-RP -e -w ../../util/mocks/$GOPACKAGE/$GOPACKAGE.go diff --git a/pkg/database/database.go b/pkg/database/database.go index 36f8cb93652..b63cce2b53a 100644 --- a/pkg/database/database.go +++ b/pkg/database/database.go @@ -37,6 +37,7 @@ const ( collPlatformWorkloadIdentityRoleSet = "PlatformWorkloadIdentityRoleSets" collPortal = "Portal" collSubscriptions = "Subscriptions" + collMaintenanceManifests = "MaintenanceManifests" ) func NewDatabaseClient(log *logrus.Entry, _env env.Core, authorizer cosmosdb.Authorizer, m metrics.Emitter, aead encryption.AEAD, databaseAccountName string) (cosmosdb.DatabaseClient, error) { diff --git a/pkg/database/mimo.go b/pkg/database/mimo.go new file mode 100644 index 00000000000..27b5356c0d2 --- /dev/null +++ b/pkg/database/mimo.go @@ -0,0 +1,213 @@ +package database + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "fmt" + "net/http" + "strings" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/database/cosmosdb" + "github.com/Azure/ARO-RP/pkg/util/uuid" +) + +const ( + MaintenanceManifestQueryForCluster = `SELECT * FROM MaintenanceManifests doc WHERE doc.maintenanceManifest.state IN ("Pending") AND doc.clusterResourceID = @clusterID` + MaintenanceManifestQueueLengthQuery = `SELECT VALUE COUNT(1) FROM MaintenanceManifests doc WHERE doc.maintenanceManifest.state IN ("Pending") AND (doc.leaseExpires ?? 0) < GetCurrentTimestamp() / 1000` +) + +type MaintenanceManifestDocumentMutator func(*api.MaintenanceManifestDocument) error + +type maintenanceManifests struct { + c cosmosdb.MaintenanceManifestDocumentClient + collc cosmosdb.CollectionClient + uuid string + uuidGenerator uuid.Generator +} + +type MaintenanceManifests interface { + Create(context.Context, *api.MaintenanceManifestDocument) (*api.MaintenanceManifestDocument, error) + GetByClusterResourceID(context.Context, string, string) (cosmosdb.MaintenanceManifestDocumentIterator, error) + Patch(context.Context, string, string, MaintenanceManifestDocumentMutator) (*api.MaintenanceManifestDocument, error) + PatchWithLease(context.Context, string, string, MaintenanceManifestDocumentMutator) (*api.MaintenanceManifestDocument, error) + Lease(ctx context.Context, clusterResourceID string, id string) (*api.MaintenanceManifestDocument, error) + EndLease(context.Context, string, string, api.MaintenanceManifestState, *string) (*api.MaintenanceManifestDocument, error) + Get(context.Context, string, string) (*api.MaintenanceManifestDocument, error) + + NewUUID() string +} + +func NewMaintenanceManifests(ctx context.Context, dbc cosmosdb.DatabaseClient, dbName string) (MaintenanceManifests, error) { + collc := cosmosdb.NewCollectionClient(dbc, dbName) + + documentClient := cosmosdb.NewMaintenanceManifestDocumentClient(collc, collMaintenanceManifests) + return NewMaintenanceManifestsWithProvidedClient(documentClient, collc, uuid.DefaultGenerator.Generate(), uuid.DefaultGenerator), nil +} + +func NewMaintenanceManifestsWithProvidedClient(client cosmosdb.MaintenanceManifestDocumentClient, collectionClient cosmosdb.CollectionClient, uuid string, uuidGenerator uuid.Generator) MaintenanceManifests { + return &maintenanceManifests{ + c: client, + uuid: uuid, + collc: collectionClient, + uuidGenerator: uuidGenerator, + } +} + +func (c *maintenanceManifests) NewUUID() string { + return c.uuidGenerator.Generate() +} + +func (c *maintenanceManifests) Create(ctx context.Context, doc *api.MaintenanceManifestDocument) (*api.MaintenanceManifestDocument, error) { + if doc.ID != strings.ToLower(doc.ID) { + return nil, fmt.Errorf("id %q is not lower case", doc.ID) + } + + doc, err := c.c.Create(ctx, doc.ClusterResourceID, doc, nil) + + if err, ok := err.(*cosmosdb.Error); ok && err.StatusCode == http.StatusConflict { + err.StatusCode = http.StatusPreconditionFailed + } + + return doc, err +} + +func (c *maintenanceManifests) Get(ctx context.Context, clusterResourceID string, id string) (*api.MaintenanceManifestDocument, error) { + if id != strings.ToLower(id) { + return nil, fmt.Errorf("id %q is not lower case", id) + } + + return c.c.Get(ctx, clusterResourceID, id, nil) +} + +// QueueLength returns maintenanceManifests un-queued document count. +// If error occurs, 0 is returned with error message +func (c *maintenanceManifests) QueueLength(ctx context.Context, collid string) (int, error) { + partitions, err := c.collc.PartitionKeyRanges(ctx, collid) + if err != nil { + return 0, err + } + + var countTotal int + for _, r := range partitions.PartitionKeyRanges { + result := c.c.Query("", &cosmosdb.Query{ + Query: MaintenanceManifestQueueLengthQuery, + }, &cosmosdb.Options{ + PartitionKeyRangeID: r.ID, + }) + // because we aggregate count we don't expect pagination in this query result, + // so we gonna call Next() only once per partition. + var data struct { + api.MissingFields + Document []int `json:"Documents,omitempty"` + } + err := result.NextRaw(ctx, -1, &data) + if err != nil { + return 0, err + } + + countTotal = countTotal + data.Document[0] + } + return countTotal, nil +} + +func (c *maintenanceManifests) Patch(ctx context.Context, clusterResourceID string, id string, f MaintenanceManifestDocumentMutator) (*api.MaintenanceManifestDocument, error) { + return c.patch(ctx, clusterResourceID, id, f, nil) +} + +func (c *maintenanceManifests) patch(ctx context.Context, clusterResourceID string, id string, f MaintenanceManifestDocumentMutator, options *cosmosdb.Options) (*api.MaintenanceManifestDocument, error) { + var doc *api.MaintenanceManifestDocument + + err := cosmosdb.RetryOnPreconditionFailed(func() (err error) { + doc, err = c.Get(ctx, clusterResourceID, id) + if err != nil { + return + } + + err = f(doc) + if err != nil { + return + } + + doc, err = c.update(ctx, doc, options) + return + }) + + return doc, err +} + +// PatchWithLease performs a patch on the cluster that verifies the lease is +// being held by this client before applying. +func (c *maintenanceManifests) PatchWithLease(ctx context.Context, clusterResourceID string, id string, f MaintenanceManifestDocumentMutator) (*api.MaintenanceManifestDocument, error) { + return c.patchWithLease(ctx, clusterResourceID, id, f, &cosmosdb.Options{PreTriggers: []string{"renewLease"}}) +} + +func (c *maintenanceManifests) patchWithLease(ctx context.Context, clusterResourceID string, id string, f MaintenanceManifestDocumentMutator, options *cosmosdb.Options) (*api.MaintenanceManifestDocument, error) { + return c.patch(ctx, clusterResourceID, id, func(doc *api.MaintenanceManifestDocument) error { + if doc.LeaseOwner != c.uuid { + return fmt.Errorf("lost lease") + } + + return f(doc) + }, options) +} + +func (c *maintenanceManifests) update(ctx context.Context, doc *api.MaintenanceManifestDocument, options *cosmosdb.Options) (*api.MaintenanceManifestDocument, error) { + if doc.ID != strings.ToLower(doc.ID) { + return nil, fmt.Errorf("id %q is not lower case", doc.ID) + } + + return c.c.Replace(ctx, doc.ClusterResourceID, doc, options) +} + +func (c *maintenanceManifests) ChangeFeed() cosmosdb.MaintenanceManifestDocumentIterator { + return c.c.ChangeFeed(nil) +} + +func (c *maintenanceManifests) GetByClusterResourceID(ctx context.Context, clusterResourceID string, continuation string) (cosmosdb.MaintenanceManifestDocumentIterator, error) { + if clusterResourceID != strings.ToLower(clusterResourceID) { + return nil, fmt.Errorf("clusterResourceID %q is not lower case", clusterResourceID) + } + + return c.c.Query("", &cosmosdb.Query{ + Query: MaintenanceManifestQueryForCluster, + Parameters: []cosmosdb.Parameter{ + { + Name: "@clusterResourceID", + Value: clusterResourceID, + }, + }, + }, &cosmosdb.Options{Continuation: continuation}), nil +} + +func (c *maintenanceManifests) EndLease(ctx context.Context, clusterResourceID string, id string, provisioningState api.MaintenanceManifestState, statusString *string) (*api.MaintenanceManifestDocument, error) { + return c.patchWithLease(ctx, clusterResourceID, id, func(doc *api.MaintenanceManifestDocument) error { + doc.MaintenanceManifest.State = provisioningState + if statusString != nil { + doc.MaintenanceManifest.StatusText = *statusString + } + + doc.LeaseOwner = "" + doc.LeaseExpires = 0 + + if provisioningState != api.MaintenanceManifestStateFailed { + doc.Dequeues = 0 + } + return nil + }, nil) +} + +// Lease performs the initial lease/dequeue on the document. +func (c *maintenanceManifests) Lease(ctx context.Context, clusterResourceID string, id string) (*api.MaintenanceManifestDocument, error) { + if clusterResourceID != strings.ToLower(clusterResourceID) { + return nil, fmt.Errorf("clusterID %q is not lower case", clusterResourceID) + } + + return c.patchWithLease(ctx, clusterResourceID, id, func(doc *api.MaintenanceManifestDocument) error { + doc.LeaseOwner = c.uuid + doc.Dequeues++ + return nil + }, &cosmosdb.Options{PreTriggers: []string{"renewLease"}}) +} diff --git a/pkg/database/openshiftclusters.go b/pkg/database/openshiftclusters.go index a41536371fe..6c667f25d9b 100644 --- a/pkg/database/openshiftclusters.go +++ b/pkg/database/openshiftclusters.go @@ -17,12 +17,13 @@ import ( ) const ( - OpenShiftClustersDequeueQuery = `SELECT * FROM OpenShiftClusters doc WHERE doc.openShiftCluster.properties.provisioningState IN ("Creating", "Deleting", "Updating", "AdminUpdating") AND (doc.leaseExpires ?? 0) < GetCurrentTimestamp() / 1000` - OpenShiftClustersQueueLengthQuery = `SELECT VALUE COUNT(1) FROM OpenShiftClusters doc WHERE doc.openShiftCluster.properties.provisioningState IN ("Creating", "Deleting", "Updating", "AdminUpdating") AND (doc.leaseExpires ?? 0) < GetCurrentTimestamp() / 1000` - OpenShiftClustersGetQuery = `SELECT * FROM OpenShiftClusters doc WHERE doc.key = @key` - OpenshiftClustersPrefixQuery = `SELECT * FROM OpenShiftClusters doc WHERE STARTSWITH(doc.key, @prefix)` - OpenshiftClustersClientIdQuery = `SELECT * FROM OpenShiftClusters doc WHERE doc.clientIdKey = @clientID` - OpenshiftClustersResourceGroupQuery = `SELECT * FROM OpenShiftClusters doc WHERE doc.clusterResourceGroupIdKey = @resourceGroupID` + OpenShiftClustersDequeueQuery = `SELECT * FROM OpenShiftClusters doc WHERE doc.openShiftCluster.properties.provisioningState IN ("Creating", "Deleting", "Updating", "AdminUpdating") AND (doc.leaseExpires ?? 0) < GetCurrentTimestamp() / 1000` + OpenShiftClustersQueueLengthQuery = `SELECT VALUE COUNT(1) FROM OpenShiftClusters doc WHERE doc.openShiftCluster.properties.provisioningState IN ("Creating", "Deleting", "Updating", "AdminUpdating") AND (doc.leaseExpires ?? 0) < GetCurrentTimestamp() / 1000` + OpenShiftClustersGetQuery = `SELECT * FROM OpenShiftClusters doc WHERE doc.key = @key` + OpenshiftClustersPrefixQuery = `SELECT * FROM OpenShiftClusters doc WHERE STARTSWITH(doc.key, @prefix)` + OpenshiftClustersClientIdQuery = `SELECT * FROM OpenShiftClusters doc WHERE doc.clientIdKey = @clientID` + OpenshiftClustersResourceGroupQuery = `SELECT * FROM OpenShiftClusters doc WHERE doc.clusterResourceGroupIdKey = @resourceGroupID` + OpenshiftClustersClusterResourceIDOnlyQuery = `SELECT {key: doc.key} FROM OpenShiftClusters doc WHERE doc.openShiftCluster.properties.provisioningState IN ("Creating", "Deleting", "Updating", "AdminUpdating")` ) type OpenShiftClusterDocumentMutator func(*api.OpenShiftClusterDocument) error @@ -52,6 +53,8 @@ type OpenShiftClusters interface { EndLease(context.Context, string, api.ProvisioningState, api.ProvisioningState, *string) (*api.OpenShiftClusterDocument, error) GetByClientID(ctx context.Context, partitionKey, clientID string) (*api.OpenShiftClusterDocuments, error) GetByClusterResourceGroupID(ctx context.Context, partitionKey, resourceGroupID string) (*api.OpenShiftClusterDocuments, error) + GetAllResourceIDs(ctx context.Context, continuation string) (cosmosdb.OpenShiftClusterDocumentIterator, error) + DoDequeue(ctx context.Context, doc *api.OpenShiftClusterDocument) (*api.OpenShiftClusterDocument, error) NewUUID() string } @@ -266,9 +269,7 @@ func (c *openShiftClusters) Dequeue(ctx context.Context) (*api.OpenShiftClusterD } for _, doc := range docs.OpenShiftClusterDocuments { - doc.LeaseOwner = c.uuid - doc.Dequeues++ - doc, err = c.update(ctx, doc, &cosmosdb.Options{PreTriggers: []string{"renewLease"}}) + doc, err = c.DoDequeue(ctx, doc) if cosmosdb.IsErrorStatusCode(err, http.StatusPreconditionFailed) { // someone else got there first continue } @@ -277,6 +278,12 @@ func (c *openShiftClusters) Dequeue(ctx context.Context) (*api.OpenShiftClusterD } } +func (c *openShiftClusters) DoDequeue(ctx context.Context, doc *api.OpenShiftClusterDocument) (*api.OpenShiftClusterDocument, error) { + doc.LeaseOwner = c.uuid + doc.Dequeues++ + return c.update(ctx, doc, &cosmosdb.Options{PreTriggers: []string{"renewLease"}}) +} + func (c *openShiftClusters) Lease(ctx context.Context, key string) (*api.OpenShiftClusterDocument, error) { return c.patchWithLease(ctx, key, func(doc *api.OpenShiftClusterDocument) error { return nil @@ -347,3 +354,13 @@ func (c *openShiftClusters) GetByClusterResourceGroupID(ctx context.Context, par } return docs, nil } + +func (c *openShiftClusters) GetAllResourceIDs(ctx context.Context, continuation string) (cosmosdb.OpenShiftClusterDocumentIterator, error) { + return c.c.Query( + "", + &cosmosdb.Query{ + Query: OpenshiftClustersClusterResourceIDOnlyQuery, + }, + &cosmosdb.Options{Continuation: continuation}, + ), nil +} diff --git a/pkg/deploy/generator/resources_rp.go b/pkg/deploy/generator/resources_rp.go index 4eed41ea00a..1cb2941f143 100644 --- a/pkg/deploy/generator/resources_rp.go +++ b/pkg/deploy/generator/resources_rp.go @@ -1023,6 +1023,34 @@ func (g *generator) database(databaseName string, addDependsOn bool) []*arm.Reso Type: "Microsoft.DocumentDB/databaseAccounts/sqlDatabases", } + mimo := &arm.Resource{ + Resource: &sdkcosmos.SQLContainerCreateUpdateParameters{ + Properties: &sdkcosmos.SQLContainerCreateUpdateProperties{ + Resource: &sdkcosmos.SQLContainerResource{ + ID: to.StringPtr("MaintenanceManifests"), + PartitionKey: &sdkcosmos.ContainerPartitionKey{ + Paths: []*string{ + to.StringPtr("/id"), + }, + Kind: &hashPartitionKey, + }, + DefaultTTL: to.Int32Ptr(-1), + }, + Options: &sdkcosmos.CreateUpdateOptions{ + Throughput: to.Int32Ptr(cosmosDbGatewayProvisionedThroughputHack), + }, + }, + Name: to.StringPtr("[concat(parameters('databaseAccountName'), '/', " + databaseName + ", '/MaintenanceManifests')]"), + Type: to.StringPtr("Microsoft.DocumentDB/databaseAccounts/sqlDatabases/containers"), + Location: to.StringPtr("[resourceGroup().location]"), + }, + APIVersion: azureclient.APIVersion("Microsoft.DocumentDB"), + DependsOn: []string{ + "[resourceId('Microsoft.DocumentDB/databaseAccounts/sqlDatabases', parameters('databaseAccountName'), " + databaseName + ")]", + }, + Type: "Microsoft.DocumentDB/databaseAccounts/sqlDatabases", + } + if !g.production { database.Resource.(*sdkcosmos.SQLDatabaseCreateUpdateParameters).Properties.Options = &sdkcosmos.CreateUpdateOptions{ AutoscaleSettings: &sdkcosmos.AutoscaleSettings{ @@ -1031,6 +1059,7 @@ func (g *generator) database(databaseName string, addDependsOn bool) []*arm.Reso } portal.Resource.(*sdkcosmos.SQLContainerCreateUpdateParameters).Properties.Options = &sdkcosmos.CreateUpdateOptions{} gateway.Resource.(*sdkcosmos.SQLContainerCreateUpdateParameters).Properties.Options = &sdkcosmos.CreateUpdateOptions{} + mimo.Resource.(*sdkcosmos.SQLContainerCreateUpdateParameters).Properties.Options = &sdkcosmos.CreateUpdateOptions{} } rs := []*arm.Resource{ @@ -1266,8 +1295,19 @@ func (g *generator) database(databaseName string, addDependsOn bool) []*arm.Reso g.rpCosmosDBTriggers(databaseName, "OpenShiftClusters", "renewLease", renewLeaseTriggerFunction, sdkcosmos.TriggerTypePre, sdkcosmos.TriggerOperationAll), // Monitors g.rpCosmosDBTriggers(databaseName, "Monitors", "renewLease", renewLeaseTriggerFunction, sdkcosmos.TriggerTypePre, sdkcosmos.TriggerOperationAll), + // MIMO + g.rpCosmosDBTriggers(databaseName, "MaintenanceManifests", "renewLease", renewLeaseTriggerFunction, sdkcosmos.TriggerTypePre, sdkcosmos.TriggerOperationAll), ) + // Don't deploy the MIMO databases in production yet + if !g.production { + rs = append(rs, + mimo, + // MIMO DB triggers + g.rpCosmosDBTriggers(databaseName, "MaintenanceManifests", "renewLease", renewLeaseTriggerFunction, sdkcosmos.TriggerTypePre, sdkcosmos.TriggerOperationAll), + ) + } + if addDependsOn { for i := range rs { rs[i].DependsOn = append(rs[i].DependsOn, diff --git a/pkg/env/core.go b/pkg/env/core.go index 8ef466616fc..ec5258b1967 100644 --- a/pkg/env/core.go +++ b/pkg/env/core.go @@ -29,6 +29,8 @@ const ( COMPONENT_UPDATE_ROLE_SETS ServiceComponent = "UPDATE_ROLE_SETS" COMPONENT_DEPLOY ServiceComponent = "DEPLOY" COMPONENT_TOOLING ServiceComponent = "TOOLING" + COMPONENT_MIMO_SCHEDULER ServiceComponent = "MIMO_SCHEDULER" + COMPONENT_MIMO_ACTUATOR ServiceComponent = "MIMO_ACTUATOR" ) // Core collects basic configuration information which is expected to be diff --git a/pkg/mimo/actuator/actuator_test.go b/pkg/mimo/actuator/actuator_test.go new file mode 100644 index 00000000000..a1dd3dcf680 --- /dev/null +++ b/pkg/mimo/actuator/actuator_test.go @@ -0,0 +1,319 @@ +package actuator + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/golang/mock/gomock" + "github.com/sirupsen/logrus" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/database" + "github.com/Azure/ARO-RP/pkg/database/cosmosdb" + "github.com/Azure/ARO-RP/pkg/env" + "github.com/Azure/ARO-RP/pkg/mimo/tasks" + mock_env "github.com/Azure/ARO-RP/pkg/util/mocks/env" + testdatabase "github.com/Azure/ARO-RP/test/database" + testlog "github.com/Azure/ARO-RP/test/util/log" +) + +var _ = Describe("MIMO Actuator", Ordered, func() { + var fixtures *testdatabase.Fixture + var checker *testdatabase.Checker + var manifests database.MaintenanceManifests + var manifestsClient *cosmosdb.FakeMaintenanceManifestDocumentClient + var clusters database.OpenShiftClusters + //var clustersClient cosmosdb.OpenShiftClusterDocumentClient + + var a Actuator + + var ctx context.Context + var cancel context.CancelFunc + + //var hook *test.Hook + var log *logrus.Entry + var _env env.Interface + + var controller *gomock.Controller + + mockSubID := "00000000-0000-0000-0000-000000000000" + clusterResourceID := fmt.Sprintf("/subscriptions/%s/resourcegroups/resourceGroup/providers/Microsoft.RedHatOpenShift/openShiftClusters/resourceName", mockSubID) + + AfterAll(func() { + if cancel != nil { + cancel() + } + + if controller != nil { + controller.Finish() + } + }) + + BeforeAll(func() { + controller = gomock.NewController(nil) + _env = mock_env.NewMockInterface(controller) + + ctx, cancel = context.WithCancel(context.Background()) + + _, log = testlog.New() + + fixtures = testdatabase.NewFixture() + checker = testdatabase.NewChecker() + }) + + BeforeEach(func() { + now := func() time.Time { return time.Unix(120, 0) } + manifests, manifestsClient = testdatabase.NewFakeMaintenanceManifests(now) + clusters, _ = testdatabase.NewFakeOpenShiftClusters() + + a = &actuator{ + log: log, + env: _env, + + clusterResourceID: strings.ToLower(clusterResourceID), + + mmf: manifests, + oc: clusters, + + tasks: map[string]tasks.TaskFunc{}, + now: now, + } + }) + + JustBeforeEach(func() { + err := fixtures.WithOpenShiftClusters(clusters).WithMaintenanceManifests(manifests).Create() + Expect(err).ToNot(HaveOccurred()) + }) + + When("old manifest", func() { + var manifestID string + + BeforeEach(func() { + fixtures.Clear() + fixtures.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{ + Key: strings.ToLower(clusterResourceID), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: clusterResourceID, + }, + }) + + manifestID = manifests.NewUUID() + fixtures.AddMaintenanceManifestDocuments(&api.MaintenanceManifestDocument{ + ID: manifestID, + ClusterResourceID: strings.ToLower(clusterResourceID), + MaintenanceManifest: &api.MaintenanceManifest{ + State: api.MaintenanceManifestStatePending, + RunBefore: 60, + RunAfter: 0, + }, + }) + + checker.Clear() + checker.AddMaintenanceManifestDocuments(&api.MaintenanceManifestDocument{ + ID: manifestID, + ClusterResourceID: strings.ToLower(clusterResourceID), + MaintenanceManifest: &api.MaintenanceManifest{ + State: api.MaintenanceManifestStateTimedOut, + StatusText: "timed out at 1970-01-01 00:02:00 +0000 UTC", + RunBefore: 60, + RunAfter: 0, + }, + }) + }) + + It("expires them", func() { + didWork, err := a.Process(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(didWork).To(BeFalse()) + + errs := checker.CheckMaintenanceManifests(manifestsClient) + Expect(errs).To(BeNil(), fmt.Sprintf("%v", errs)) + }) + }) + + When("new manifest", func() { + var manifestID string + + BeforeEach(func() { + fixtures.Clear() + fixtures.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{ + Key: strings.ToLower(clusterResourceID), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: clusterResourceID, + }, + }) + + manifestID = manifests.NewUUID() + fixtures.AddMaintenanceManifestDocuments(&api.MaintenanceManifestDocument{ + ID: manifestID, + ClusterResourceID: strings.ToLower(clusterResourceID), + MaintenanceManifest: &api.MaintenanceManifest{ + State: api.MaintenanceManifestStatePending, + MaintenanceSetID: "0", + RunBefore: 600, + RunAfter: 0, + }, + }) + + checker.Clear() + checker.AddMaintenanceManifestDocuments(&api.MaintenanceManifestDocument{ + ID: manifestID, + ClusterResourceID: strings.ToLower(clusterResourceID), + MaintenanceManifest: &api.MaintenanceManifest{ + State: api.MaintenanceManifestStateCompleted, + MaintenanceSetID: "0", + StatusText: "done", + RunBefore: 600, + RunAfter: 0, + }, + }) + }) + + It("runs them", func() { + a.AddTask("0", func(ctx context.Context, th tasks.TaskContext, mmd *api.MaintenanceManifestDocument, oscd *api.OpenShiftClusterDocument) (api.MaintenanceManifestState, string) { + return api.MaintenanceManifestStateCompleted, "done" + }) + + didWork, err := a.Process(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(didWork).To(BeTrue()) + + errs := checker.CheckMaintenanceManifests(manifestsClient) + Expect(errs).To(BeNil(), fmt.Sprintf("%v", errs)) + }) + }) + + When("new manifests", func() { + var manifestIDs []string + + BeforeEach(func() { + fixtures.Clear() + fixtures.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{ + Key: strings.ToLower(clusterResourceID), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: clusterResourceID, + }, + }) + + manifestIDs = []string{manifests.NewUUID(), manifests.NewUUID(), manifests.NewUUID()} + fixtures.AddMaintenanceManifestDocuments(&api.MaintenanceManifestDocument{ + ID: manifestIDs[0], + ClusterResourceID: strings.ToLower(clusterResourceID), + MaintenanceManifest: &api.MaintenanceManifest{ + State: api.MaintenanceManifestStatePending, + MaintenanceSetID: "0", + RunBefore: 600, + RunAfter: 0, + Priority: 2, + }, + }, + &api.MaintenanceManifestDocument{ + ID: manifestIDs[1], + ClusterResourceID: strings.ToLower(clusterResourceID), + MaintenanceManifest: &api.MaintenanceManifest{ + State: api.MaintenanceManifestStatePending, + MaintenanceSetID: "1", + RunBefore: 600, + RunAfter: 0, + Priority: 1, + }, + }, + &api.MaintenanceManifestDocument{ + ID: manifestIDs[2], + ClusterResourceID: strings.ToLower(clusterResourceID), + MaintenanceManifest: &api.MaintenanceManifest{ + State: api.MaintenanceManifestStatePending, + MaintenanceSetID: "2", + RunBefore: 600, + RunAfter: 1, + Priority: 0, + }, + }) + + checker.Clear() + checker.AddMaintenanceManifestDocuments(&api.MaintenanceManifestDocument{ + ID: manifestIDs[0], + ClusterResourceID: strings.ToLower(clusterResourceID), + MaintenanceManifest: &api.MaintenanceManifest{ + State: api.MaintenanceManifestStateCompleted, + MaintenanceSetID: "0", + StatusText: "done", + RunBefore: 600, + RunAfter: 0, + Priority: 2, + }, + }, + &api.MaintenanceManifestDocument{ + ID: manifestIDs[1], + ClusterResourceID: strings.ToLower(clusterResourceID), + MaintenanceManifest: &api.MaintenanceManifest{ + State: api.MaintenanceManifestStateCompleted, + MaintenanceSetID: "1", + StatusText: "done", + RunBefore: 600, + RunAfter: 0, + Priority: 1, + }, + }, + &api.MaintenanceManifestDocument{ + ID: manifestIDs[2], + ClusterResourceID: strings.ToLower(clusterResourceID), + MaintenanceManifest: &api.MaintenanceManifest{ + State: api.MaintenanceManifestStateCompleted, + MaintenanceSetID: "2", + StatusText: "done", + RunBefore: 600, + RunAfter: 1, + Priority: 0, + }, + }) + }) + + It("runs them", func() { + + ordering := []string{} + + a.AddTask("0", func(ctx context.Context, th tasks.TaskContext, mmd *api.MaintenanceManifestDocument, oscd *api.OpenShiftClusterDocument) (api.MaintenanceManifestState, string) { + ordering = append(ordering, "0") + return api.MaintenanceManifestStateCompleted, "done" + }) + + a.AddTask("1", func(ctx context.Context, th tasks.TaskContext, mmd *api.MaintenanceManifestDocument, oscd *api.OpenShiftClusterDocument) (api.MaintenanceManifestState, string) { + ordering = append(ordering, "1") + return api.MaintenanceManifestStateCompleted, "done" + }) + + a.AddTask("2", func(ctx context.Context, th tasks.TaskContext, mmd *api.MaintenanceManifestDocument, oscd *api.OpenShiftClusterDocument) (api.MaintenanceManifestState, string) { + ordering = append(ordering, "2") + return api.MaintenanceManifestStateCompleted, "done" + }) + + didWork, err := a.Process(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(didWork).To(BeTrue()) + + // We expect 1 (start time of 0, but higher priority), then 0 (start + // time of 0, lower priority), then 2 (start time of 1, then highest + // priority) + Expect(ordering).To(BeEquivalentTo([]string{"1", "0", "2"})) + + errs := checker.CheckMaintenanceManifests(manifestsClient) + Expect(errs).To(BeNil(), fmt.Sprintf("%v", errs)) + }) + }) + +}) + +func TestActuator(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Actuator Suite") +} diff --git a/pkg/mimo/actuator/manager.go b/pkg/mimo/actuator/manager.go new file mode 100644 index 00000000000..48724119e84 --- /dev/null +++ b/pkg/mimo/actuator/manager.go @@ -0,0 +1,184 @@ +package actuator + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "fmt" + "sort" + "strings" + "time" + + "github.com/Azure/go-autorest/autorest/to" + "github.com/sirupsen/logrus" + "golang.org/x/exp/maps" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/database" + "github.com/Azure/ARO-RP/pkg/env" + "github.com/Azure/ARO-RP/pkg/mimo/tasks" +) + +const maxDequeueCount = 5 + +type Actuator interface { + Process(context.Context) (bool, error) + AddTask(string, tasks.TaskFunc) + AddTasks(map[string]tasks.TaskFunc) +} + +type actuator struct { + env env.Interface + log *logrus.Entry + now func() time.Time + + clusterResourceID string + + oc database.OpenShiftClusters + mmf database.MaintenanceManifests + + tasks map[string]tasks.TaskFunc +} + +func NewActuator( + ctx context.Context, + _env env.Interface, + log *logrus.Entry, + clusterResourceID string, + oc database.OpenShiftClusters, + mmf database.MaintenanceManifests, + now func() time.Time) (Actuator, error) { + a := &actuator{ + env: _env, + log: log, + clusterResourceID: strings.ToLower(clusterResourceID), + oc: oc, + mmf: mmf, + tasks: make(map[string]tasks.TaskFunc), + + now: now, + } + + return a, nil +} + +func (a *actuator) AddTask(u string, t tasks.TaskFunc) { + a.tasks[u] = t +} + +func (a *actuator) AddTasks(tasks map[string]tasks.TaskFunc) { + maps.Copy(a.tasks, tasks) +} + +func (a *actuator) Process(ctx context.Context) (bool, error) { + // Get the manifests for this cluster which need to be worked + i, err := a.mmf.GetByClusterResourceID(ctx, a.clusterResourceID, "") + if err != nil { + return false, fmt.Errorf("failed getting manifests: %w", err) + } + + docList := make([]*api.MaintenanceManifestDocument, 0) + for { + docs, err := i.Next(ctx, -1) + if err != nil { + return false, fmt.Errorf("failed reading next manifest document: %w", err) + } + if docs == nil { + break + } + + docList = append(docList, docs.MaintenanceManifestDocuments...) + } + + manifestsToAction := make([]*api.MaintenanceManifestDocument, 0) + + // Order manifests in order of RunAfter, and then Priority for ones with the + // same RunAfter. + sort.SliceStable(docList, func(i, j int) bool { + if docList[i].MaintenanceManifest.RunAfter == docList[j].MaintenanceManifest.RunAfter { + return docList[i].MaintenanceManifest.Priority < docList[j].MaintenanceManifest.Priority + } + + return docList[i].MaintenanceManifest.RunAfter < docList[j].MaintenanceManifest.RunAfter + }) + + evaluationTime := a.now() + + // Check for manifests that have timed out first + for _, doc := range docList { + if evaluationTime.After(time.Unix(int64(doc.MaintenanceManifest.RunBefore), 0)) { + // timed out, mark as such + a.log.Infof("marking %v as outdated: %v older than %v", doc.ID, doc.MaintenanceManifest.RunBefore, evaluationTime.UTC()) + + _, err := a.mmf.Patch(ctx, a.clusterResourceID, doc.ID, func(d *api.MaintenanceManifestDocument) error { + d.MaintenanceManifest.State = api.MaintenanceManifestStateTimedOut + d.MaintenanceManifest.StatusText = fmt.Sprintf("timed out at %s", evaluationTime.UTC()) + return nil + }) + if err != nil { + a.log.Error(err) + } + } else { + // not timed out, do something about it + manifestsToAction = append(manifestsToAction, doc) + } + } + + // Nothing to do, don't dequeue + if len(manifestsToAction) == 0 { + return false, nil + } + + // Dequeue the document + oc, err := a.oc.Get(ctx, a.clusterResourceID) + if err != nil { + return false, err + } + + oc, err = a.oc.DoDequeue(ctx, oc) + if err != nil { + return false, err // This will include StatusPreconditionFaileds + } + + taskContext := newTaskContext(a.env, a.log, oc) + + // Execute on the manifests we want to action + for _, doc := range manifestsToAction { + // here + f, ok := a.tasks[doc.MaintenanceManifest.MaintenanceSetID] + if !ok { + a.log.Infof("not found %v", doc.MaintenanceManifest.MaintenanceSetID) + continue + } + + // Attempt a dequeue + doc, err = a.mmf.Lease(ctx, a.clusterResourceID, doc.ID) + if err != nil { + // log and continue if it doesn't work + a.log.Error(err) + continue + } + + // if we've tried too many times, give up + if doc.Dequeues > maxDequeueCount { + err := fmt.Errorf("dequeued %d times, failing", doc.Dequeues) + _, leaseErr := a.mmf.EndLease(ctx, doc.ClusterResourceID, doc.ID, api.MaintenanceManifestStateTimedOut, to.StringPtr(err.Error())) + if leaseErr != nil { + a.log.Error(err) + } + continue + } + + // Perform the task + state, msg := f(ctx, taskContext, doc, oc) + _, err = a.mmf.EndLease(ctx, doc.ClusterResourceID, doc.ID, state, &msg) + if err != nil { + a.log.Error(err) + } + } + + // release the OpenShiftCluster + _, err = a.oc.EndLease(ctx, a.clusterResourceID, oc.OpenShiftCluster.Properties.ProvisioningState, api.ProvisioningStateMaintenance, nil) + return true, err +} diff --git a/pkg/mimo/actuator/service.go b/pkg/mimo/actuator/service.go new file mode 100644 index 00000000000..d588817cd8b --- /dev/null +++ b/pkg/mimo/actuator/service.go @@ -0,0 +1,242 @@ +package actuator + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/sirupsen/logrus" + "golang.org/x/exp/maps" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/database" + "github.com/Azure/ARO-RP/pkg/env" + "github.com/Azure/ARO-RP/pkg/metrics" + "github.com/Azure/ARO-RP/pkg/mimo/tasks" + "github.com/Azure/ARO-RP/pkg/proxy" + "github.com/Azure/ARO-RP/pkg/util/buckets" + "github.com/Azure/ARO-RP/pkg/util/heartbeat" + utillog "github.com/Azure/ARO-RP/pkg/util/log" + "github.com/Azure/ARO-RP/pkg/util/recover" +) + +type Runnable interface { + Run(context.Context, <-chan struct{}, chan<- struct{}) error +} + +type service struct { + dialer proxy.Dialer + baseLog *logrus.Entry + env env.Interface + + dbOpenShiftClusters database.OpenShiftClusters + dbMaintenanceManifests database.MaintenanceManifests + + m metrics.Emitter + mu sync.RWMutex + cond *sync.Cond + stopping *atomic.Bool + workers *atomic.Int32 + + b buckets.BucketWorker + + lastChangefeed atomic.Value //time.Time + startTime time.Time + + pollTime time.Duration + now func() time.Time + + tasks map[string]tasks.TaskFunc +} + +func NewService(env env.Interface, log *logrus.Entry, dialer proxy.Dialer, dbOpenShiftClusters database.OpenShiftClusters, dbMaintenanceManifests database.MaintenanceManifests, m metrics.Emitter) *service { + s := &service{ + env: env, + baseLog: log, + dialer: dialer, + + dbOpenShiftClusters: dbOpenShiftClusters, + dbMaintenanceManifests: dbMaintenanceManifests, + + m: m, + stopping: &atomic.Bool{}, + workers: &atomic.Int32{}, + + startTime: time.Now(), + now: time.Now, + pollTime: time.Minute, + } + + s.b = buckets.NewBucketWorker(log, s.worker, &s.mu) + + return s +} + +func (s *service) SetTasks(tasks map[string]tasks.TaskFunc) { + s.tasks = tasks +} + +func (s *service) Run(ctx context.Context, stop <-chan struct{}, done chan<- struct{}) error { + defer recover.Panic(s.baseLog) + + t := time.NewTicker(10 * time.Second) + defer t.Stop() + + if stop != nil { + go func() { + defer recover.Panic(s.baseLog) + + <-stop + s.baseLog.Print("stopping") + s.stopping.Store(true) + s.cond.Signal() + }() + } + go heartbeat.EmitHeartbeat(s.baseLog, s.m, "actuator.heartbeat", nil, s.checkReady) + + lastGotDocs := make(map[string]*api.OpenShiftClusterDocument) + + for { + if s.stopping.Load() { + break + } + + old, err := s.poll(ctx, lastGotDocs) + if err != nil { + s.baseLog.Error(err) + } else { + lastGotDocs = old + } + + <-t.C + } + + if !s.env.FeatureIsSet(env.FeatureDisableReadinessDelay) { + s.waitForWorkerCompletion() + } + s.baseLog.Print("exiting") + close(done) + return nil +} + +// Temporary method of updating without the changefeed -- the reason why is +// complicated +func (s *service) poll(ctx context.Context, oldDocs map[string]*api.OpenShiftClusterDocument) (map[string]*api.OpenShiftClusterDocument, error) { + // Fetch all of the cluster UUIDs + i, err := s.dbOpenShiftClusters.GetAllResourceIDs(ctx, "") + if err != nil { + return nil, err + } + + docs := make([]*api.OpenShiftClusterDocument, 0) + + for { + d, err := i.Next(ctx, -1) + if err != nil { + return nil, err + } + if d == nil { + break + } + + docs = append(docs, d.OpenShiftClusterDocuments...) + } + + s.baseLog.Debugf("fetched %d clusters from CosmosDB", len(docs)) + + docMap := make(map[string]*api.OpenShiftClusterDocument) + for _, d := range docs { + docMap[strings.ToLower(d.Key)] = d + } + + // remove docs that don't exist in the new set (removed clusters) + for _, oldCluster := range maps.Keys(oldDocs) { + _, ok := docMap[strings.ToLower(oldCluster)] + if !ok { + s.b.DeleteDoc(oldDocs[oldCluster]) + s.baseLog.Debugf("removed %s from buckets", oldCluster) + } + } + + s.baseLog.Debugf("updating %d clusters", len(docMap)) + + for _, cluster := range maps.Values(docMap) { + s.b.UpsertDoc(cluster) + } + + // Store when we last fetched the clusters + s.lastChangefeed.Store(s.now()) + + return docMap, nil +} + +func (s *service) waitForWorkerCompletion() { + s.mu.Lock() + for s.workers.Load() > 0 { + s.cond.Wait() + } + s.mu.Unlock() +} + +func (s *service) checkReady() bool { + lastChangefeedTime, ok := s.lastChangefeed.Load().(time.Time) + if !ok { + return false + } + return (time.Since(lastChangefeedTime) < time.Minute) && // did we update our list of clusters recently? + (time.Since(s.startTime) > 2*time.Minute) // are we running for at least 2 minutes? +} + +func (s *service) worker(stop <-chan struct{}, delay time.Duration, id string) { + defer recover.Panic(s.baseLog) + + // Wait for a randomised delay before starting + time.Sleep(delay) + + log := utillog.EnrichWithResourceID(s.baseLog, id) + + a, err := NewActuator(context.Background(), s.env, log, id, s.dbOpenShiftClusters, s.dbMaintenanceManifests, s.now) + if err != nil { + log.Error(err) + return + } + + // load in the tasks for the Actuator from the controller + a.AddTasks(s.tasks) + + t := time.NewTicker(s.pollTime) + defer t.Stop() + +out: + for { + if s.stopping.Load() { + break + } + + func() { + s.workers.Add(1) + s.m.EmitGauge("mimo.actuator.workers.active.count", int64(s.workers.Load()), nil) + + defer func() { + s.workers.Add(-1) + s.m.EmitGauge("mimo.actuator.workers.active.count", int64(s.workers.Load()), nil) + }() + + _, err := a.Process(context.Background()) + if err != nil { + log.Error(err) + } + }() + + select { + case <-t.C: + case <-stop: + break out + } + } +} diff --git a/pkg/mimo/actuator/service_test.go b/pkg/mimo/actuator/service_test.go new file mode 100644 index 00000000000..9613bb0c402 --- /dev/null +++ b/pkg/mimo/actuator/service_test.go @@ -0,0 +1,251 @@ +package actuator + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "fmt" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/golang/mock/gomock" + "github.com/sirupsen/logrus" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/database" + "github.com/Azure/ARO-RP/pkg/database/cosmosdb" + "github.com/Azure/ARO-RP/pkg/env" + "github.com/Azure/ARO-RP/pkg/metrics" + "github.com/Azure/ARO-RP/pkg/mimo/tasks" + mock_env "github.com/Azure/ARO-RP/pkg/util/mocks/env" + testdatabase "github.com/Azure/ARO-RP/test/database" + testlog "github.com/Azure/ARO-RP/test/util/log" +) + +type fakeMetricsEmitter struct { + Metrics map[string]int64 +} + +func newfakeMetricsEmitter() *fakeMetricsEmitter { + m := make(map[string]int64) + return &fakeMetricsEmitter{ + Metrics: m, + } +} + +func (e *fakeMetricsEmitter) EmitGauge(metricName string, metricValue int64, dimensions map[string]string) { + e.Metrics[metricName] = metricValue +} + +func (e *fakeMetricsEmitter) EmitFloat(metricName string, metricValue float64, dimensions map[string]string) { +} + +var _ = Describe("MIMO Actuator Service", Ordered, func() { + var fixtures *testdatabase.Fixture + var checker *testdatabase.Checker + var manifests database.MaintenanceManifests + var manifestsClient *cosmosdb.FakeMaintenanceManifestDocumentClient + var clusters database.OpenShiftClusters + //var clustersClient cosmosdb.OpenShiftClusterDocumentClient + var m metrics.Emitter + + var svc *service + + var ctx context.Context + var cancel context.CancelFunc + + //var hook *test.Hook + var log *logrus.Entry + var _env env.Interface + + var controller *gomock.Controller + + mockSubID := "00000000-0000-0000-0000-000000000000" + clusterResourceID := fmt.Sprintf("/subscriptions/%s/resourcegroups/resourceGroup/providers/Microsoft.RedHatOpenShift/openShiftClusters/resourceName", mockSubID) + + AfterAll(func() { + if cancel != nil { + cancel() + } + + if controller != nil { + controller.Finish() + } + }) + + BeforeAll(func() { + controller = gomock.NewController(nil) + _env = mock_env.NewMockInterface(controller) + + ctx, cancel = context.WithCancel(context.Background()) + + _, log = testlog.New() + + m = newfakeMetricsEmitter() + + fixtures = testdatabase.NewFixture() + checker = testdatabase.NewChecker() + }) + + BeforeEach(func() { + now := func() time.Time { return time.Unix(120, 0) } + manifests, manifestsClient = testdatabase.NewFakeMaintenanceManifests(now) + clusters, _ = testdatabase.NewFakeOpenShiftClusters() + + svc = NewService(_env, log, nil, clusters, manifests, m) + svc.now = now + + }) + + JustBeforeEach(func() { + err := fixtures.WithOpenShiftClusters(clusters).WithMaintenanceManifests(manifests).Create() + Expect(err).ToNot(HaveOccurred()) + }) + + When("clusters are polled", func() { + BeforeEach(func() { + fixtures.Clear() + fixtures.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{ + Key: strings.ToLower(clusterResourceID), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: clusterResourceID, + }, + }) + }) + + AfterAll(func() { + svc.b.Stop() + }) + + It("updates the available clusters", func() { + lastGotDocs := make(map[string]*api.OpenShiftClusterDocument) + + newOld, err := svc.poll(ctx, lastGotDocs) + Expect(err).ToNot(HaveOccurred()) + + Expect(newOld).To(HaveLen(1)) + }) + + It("removes clusters if they are not in the doc", func() { + svc.b.UpsertDoc(&api.OpenShiftClusterDocument{Key: clusterResourceID + "2"}) + + lastGotDocs := make(map[string]*api.OpenShiftClusterDocument) + lastGotDocs[clusterResourceID+"2"] = &api.OpenShiftClusterDocument{Key: clusterResourceID + "2"} + + newOld, err := svc.poll(ctx, lastGotDocs) + Expect(err).ToNot(HaveOccurred()) + + Expect(newOld).To(HaveLen(1)) + }) + }) + + When("maintenance needs to occur", func() { + var manifestID string + + BeforeEach(func() { + fixtures.Clear() + fixtures.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{ + Key: strings.ToLower(clusterResourceID), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: clusterResourceID, + Properties: api.OpenShiftClusterProperties{ + ProvisioningState: api.ProvisioningStateSucceeded, + }, + }, + }) + + manifestID = manifests.NewUUID() + manifestID2 := manifests.NewUUID() + fixtures.AddMaintenanceManifestDocuments( + &api.MaintenanceManifestDocument{ + ID: manifestID, + ClusterResourceID: strings.ToLower(clusterResourceID), + MaintenanceManifest: &api.MaintenanceManifest{ + State: api.MaintenanceManifestStatePending, + RunBefore: 60, + RunAfter: 0, + }, + }, + &api.MaintenanceManifestDocument{ + ID: manifestID2, + ClusterResourceID: strings.ToLower(clusterResourceID), + MaintenanceManifest: &api.MaintenanceManifest{ + State: api.MaintenanceManifestStatePending, + RunBefore: 300, + RunAfter: 0, + MaintenanceSetID: "0000-0000-0001", + }, + }) + + checker.Clear() + checker.AddMaintenanceManifestDocuments( + &api.MaintenanceManifestDocument{ + ID: manifestID, + ClusterResourceID: strings.ToLower(clusterResourceID), + MaintenanceManifest: &api.MaintenanceManifest{ + State: api.MaintenanceManifestStateTimedOut, + StatusText: "timed out at 1970-01-01 00:02:00 +0000 UTC", + RunBefore: 60, + RunAfter: 0, + }, + }, + &api.MaintenanceManifestDocument{ + ID: manifestID2, + ClusterResourceID: strings.ToLower(clusterResourceID), + MaintenanceManifest: &api.MaintenanceManifest{ + State: api.MaintenanceManifestStateCompleted, + StatusText: "ok", + RunBefore: 300, + RunAfter: 0, + MaintenanceSetID: "0000-0000-0001", + }, + }, + ) + }) + + It("expires them", func() { + // run once + done := make(chan struct{}) + svc.pollTime = time.Second + + svc.SetTasks(map[string]tasks.TaskFunc{ + "0000-0000-0001": func(ctx context.Context, th tasks.TaskContext, mmd *api.MaintenanceManifestDocument, oscd *api.OpenShiftClusterDocument) (api.MaintenanceManifestState, string) { + svc.stopping.Store(true) + return api.MaintenanceManifestStateCompleted, "ok" + }, + }) + + svc.worker(done, 0*time.Second, clusterResourceID) + + errs := checker.CheckMaintenanceManifests(manifestsClient) + Expect(errs).To(BeNil(), fmt.Sprintf("%v", errs)) + }) + + It("loads the full cluster document", func() { + // run once + done := make(chan struct{}) + svc.pollTime = time.Second + + svc.SetTasks(map[string]tasks.TaskFunc{ + "0000-0000-0001": func(ctx context.Context, th tasks.TaskContext, mmd *api.MaintenanceManifestDocument, oscd *api.OpenShiftClusterDocument) (api.MaintenanceManifestState, string) { + // ProvisioningState is in the full document, not just the + // ClusterResourceID only as in the bucket worker + Expect(oscd.OpenShiftCluster.Properties.ProvisioningState).To(Equal(api.ProvisioningStateSucceeded)) + + svc.stopping.Store(true) + return api.MaintenanceManifestStateCompleted, "ok" + }, + }) + + svc.worker(done, 0*time.Second, clusterResourceID) + + errs := checker.CheckMaintenanceManifests(manifestsClient) + Expect(errs).To(BeNil(), fmt.Sprintf("%v", errs)) + }) + }) + +}) diff --git a/pkg/mimo/actuator/task.go b/pkg/mimo/actuator/task.go new file mode 100644 index 00000000000..6c5b71a4222 --- /dev/null +++ b/pkg/mimo/actuator/task.go @@ -0,0 +1,59 @@ +package actuator + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "github.com/sirupsen/logrus" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/env" + "github.com/Azure/ARO-RP/pkg/mimo/tasks" + "github.com/Azure/ARO-RP/pkg/util/clienthelper" + "github.com/Azure/ARO-RP/pkg/util/restconfig" +) + +type th struct { + env env.Interface + log *logrus.Entry + + oc *api.OpenShiftClusterDocument + + _ch clienthelper.Interface +} + +func newTaskContext(env env.Interface, log *logrus.Entry, oc *api.OpenShiftClusterDocument) tasks.TaskContext { + return &th{ + env: env, + log: log, + oc: oc, + _ch: nil, + } +} + +func (t *th) Environment() env.Interface { + return t.env +} + +func (t *th) ClientHelper() (clienthelper.Interface, error) { + if t._ch != nil { + return t._ch, nil + } + + restConfig, err := restconfig.RestConfig(t.env, t.oc.OpenShiftCluster) + if err != nil { + return nil, err + } + + ch, err := clienthelper.New(t.log, restConfig) + if err != nil { + return nil, err + } + + t._ch = ch + return t._ch, nil +} + +func (t *th) Log() *logrus.Entry { + return t.log +} diff --git a/pkg/mimo/cmd/cli.go b/pkg/mimo/cmd/cli.go new file mode 100644 index 00000000000..feffce5a9be --- /dev/null +++ b/pkg/mimo/cmd/cli.go @@ -0,0 +1,186 @@ +package main + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "os" + "os/signal" + "syscall" + + "github.com/sirupsen/logrus" + "github.com/urfave/cli/v2" + + "github.com/Azure/ARO-RP/pkg/database" + "github.com/Azure/ARO-RP/pkg/env" + "github.com/Azure/ARO-RP/pkg/metrics/statsd" + "github.com/Azure/ARO-RP/pkg/metrics/statsd/golang" + "github.com/Azure/ARO-RP/pkg/mimo/actuator" + "github.com/Azure/ARO-RP/pkg/mimo/tasks" + "github.com/Azure/ARO-RP/pkg/mimo/tasks/example" + "github.com/Azure/ARO-RP/pkg/proxy" + utillog "github.com/Azure/ARO-RP/pkg/util/log" + "github.com/Azure/ARO-RP/pkg/util/service" +) + +func main() { + log := utillog.GetLogger() + developmentMode := env.IsLocalDevelopmentMode() + + app := &cli.App{ + Name: "MIMO", + Usage: "Managed Infrastructure Maintenance Operator", + Commands: []*cli.Command{ + { + Name: "scheduler", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "MDM_ACCOUNT", + EnvVars: []string{"MDM_ACCOUNT"}, + Required: !developmentMode, + }, + &cli.StringFlag{ + Name: "MDM_NAMESPACE", + EnvVars: []string{"MDM_NAMESPACE"}, + Required: !developmentMode, + }, + &cli.StringFlag{ + Name: "MDM_STATSD_SOCKET", + EnvVars: []string{"MDM_STATSD_SOCKET"}, + Required: false, + }, + }, + Action: func(ctx *cli.Context) error { + _env, err := env.NewEnv(ctx.Context, log, env.COMPONENT_MIMO_SCHEDULER) + if err != nil { + return err + } + + m := statsd.New(ctx.Context, log.WithField("component", "scheduler"), _env, os.Getenv("MDM_ACCOUNT"), os.Getenv("MDM_NAMESPACE"), os.Getenv("MDM_STATSD_SOCKET")) + + g, err := golang.NewMetrics(_env.Logger(), m) + if err != nil { + return err + } + go g.Run() + + // dbc, err := service.NewDatabase(ctx.Context, _env, log, m, service.DB_DBTOKEN_PROD_MASTERKEY_DEV, false) + // if err != nil { + // return err + // } + + // dbName, err := service.DBName(_env.IsLocalDevelopmentMode()) + // if err != nil { + // return err + // } + + // clusters, err := database.NewOpenShiftClusters(ctx.Context, dbc, dbName) + // if err != nil { + // return err + // } + + return nil + }, + }, + { + Name: "actuator", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "MDM_ACCOUNT", + EnvVars: []string{"MDM_ACCOUNT"}, + Required: !developmentMode, + }, + &cli.StringFlag{ + Name: "MDM_NAMESPACE", + EnvVars: []string{"MDM_NAMESPACE"}, + Required: !developmentMode, + }, + &cli.StringFlag{ + Name: "MDM_STATSD_SOCKET", + EnvVars: []string{"MDM_STATSD_SOCKET"}, + Required: false, + }, + &cli.StringFlag{ + Name: "loglevel", + EnvVars: []string{"ARO_LOGLEVEL"}, + DefaultText: "info", + Required: false, + }, + }, + Before: func(ctx *cli.Context) error { + log.Print("MIMO actuator initialising") + return nil + }, + Action: func(ctx *cli.Context) error { + stop := make(chan struct{}) + + l, err := logrus.ParseLevel(ctx.String("loglevel")) + if err != nil { + log.Logger.SetLevel(l) + } + + _env, err := env.NewEnv(ctx.Context, log, env.COMPONENT_MIMO_ACTUATOR) + if err != nil { + return err + } + + m := statsd.New(ctx.Context, log.WithField("component", "actuator"), _env, os.Getenv("MDM_ACCOUNT"), os.Getenv("MDM_NAMESPACE"), os.Getenv("MDM_STATSD_SOCKET")) + + g, err := golang.NewMetrics(_env.Logger(), m) + if err != nil { + return err + } + go g.Run() + + dbc, err := service.NewDatabase(ctx.Context, _env, log, m, false) + if err != nil { + return err + } + + dbName, err := service.DBName(_env.IsLocalDevelopmentMode()) + if err != nil { + return err + } + + clusters, err := database.NewOpenShiftClusters(ctx.Context, dbc, dbName) + if err != nil { + return err + } + + manifests, err := database.NewMaintenanceManifests(ctx.Context, dbc, dbName) + if err != nil { + return err + } + + dialer, err := proxy.NewDialer(_env.IsLocalDevelopmentMode()) + if err != nil { + return err + } + + a := actuator.NewService(_env, _env.Logger(), dialer, clusters, manifests, m) + + a.SetTasks(map[string]tasks.TaskFunc{ + "64d23e34-78c3-4d60-831c-6efb22f08508": example.ExampleTask, + }) + + sigterm := make(chan os.Signal, 1) + done := make(chan struct{}) + signal.Notify(sigterm, syscall.SIGTERM) + + go a.Run(ctx.Context, stop, done) + + <-sigterm + log.Print("received SIGTERM") + close(stop) + //<-done + + return nil + }, + }, + }, + } + + if err := app.Run(os.Args); err != nil { + log.Fatal(err) + } +} diff --git a/pkg/mimo/tasks/example/task.go b/pkg/mimo/tasks/example/task.go new file mode 100644 index 00000000000..b8186f2167d --- /dev/null +++ b/pkg/mimo/tasks/example/task.go @@ -0,0 +1,31 @@ +package example + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "fmt" + + configv1 "github.com/openshift/api/config/v1" + "k8s.io/apimachinery/pkg/types" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/mimo/tasks" +) + +func ExampleTask(ctx context.Context, th tasks.TaskContext, manifest *api.MaintenanceManifestDocument, oc *api.OpenShiftClusterDocument) (api.MaintenanceManifestState, string) { + ch, err := th.ClientHelper() + if err != nil { + return api.MaintenanceManifestStateFailed, err.Error() + } + + cv := &configv1.ClusterVersion{} + + err = ch.GetOne(ctx, types.NamespacedName{Name: "version"}, cv) + if err != nil { + return api.MaintenanceManifestStateFailed, fmt.Errorf("unable to get ClusterVersion: %w", err).Error() + } + + return api.MaintenanceManifestStateCompleted, fmt.Sprintf("cluster version is: %s", cv.Status.History[0].Version) +} diff --git a/pkg/mimo/tasks/example/task_test.go b/pkg/mimo/tasks/example/task_test.go new file mode 100644 index 00000000000..055d3ec9a8f --- /dev/null +++ b/pkg/mimo/tasks/example/task_test.go @@ -0,0 +1,85 @@ +package example + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "fmt" + "strings" + "testing" + + . "github.com/onsi/gomega" + + "github.com/golang/mock/gomock" + configv1 "github.com/openshift/api/config/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/util/clienthelper" + mock_env "github.com/Azure/ARO-RP/pkg/util/mocks/env" + testdatabase "github.com/Azure/ARO-RP/test/database" + testtasks "github.com/Azure/ARO-RP/test/mimo/tasks" + testclienthelper "github.com/Azure/ARO-RP/test/util/clienthelper" + testlog "github.com/Azure/ARO-RP/test/util/log" +) + +func TestTask(t *testing.T) { + RegisterTestingT(t) + ctx := context.Background() + + mockSubID := "00000000-0000-0000-0000-000000000000" + clusterResourceID := fmt.Sprintf("/subscriptions/%s/resourcegroups/resourceGroup/providers/Microsoft.RedHatOpenShift/openShiftClusters/resourceName", mockSubID) + + controller := gomock.NewController(t) + _env := mock_env.NewMockInterface(controller) + _, log := testlog.New() + + fixtures := testdatabase.NewFixture() + clusters, _ := testdatabase.NewFakeOpenShiftClusters() + + fixtures.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{ + Key: strings.ToLower(clusterResourceID), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: clusterResourceID, + }, + }) + + manifest := &api.MaintenanceManifestDocument{ + ClusterResourceID: strings.ToLower(clusterResourceID), + MaintenanceManifest: &api.MaintenanceManifest{ + State: api.MaintenanceManifestStatePending, + RunBefore: 60, + RunAfter: 0, + }, + } + + err := fixtures.WithOpenShiftClusters(clusters).Create() + Expect(err).ToNot(HaveOccurred()) + + builder := fake.NewClientBuilder().WithRuntimeObjects( + &configv1.ClusterVersion{ + ObjectMeta: metav1.ObjectMeta{ + Name: "version", + }, + Status: configv1.ClusterVersionStatus{ + History: []configv1.UpdateHistory{ + { + State: configv1.CompletedUpdate, + Version: "4.99.123", + }, + }, + }, + }, + ) + ch := clienthelper.NewWithClient(log, testclienthelper.NewHookingClient(builder.Build())) + tc := testtasks.NewFakeTestContext(_env, log, ch) + + oc, err := clusters.Get(ctx, strings.ToLower(clusterResourceID)) + Expect(err).ToNot(HaveOccurred()) + r, text := ExampleTask(ctx, tc, manifest, oc) + + Expect(r).To(Equal(api.MaintenanceManifestStateCompleted)) + Expect(text).To(Equal("cluster version is: 4.99.123")) +} diff --git a/pkg/mimo/tasks/interfaces.go b/pkg/mimo/tasks/interfaces.go new file mode 100644 index 00000000000..f52974ff258 --- /dev/null +++ b/pkg/mimo/tasks/interfaces.go @@ -0,0 +1,22 @@ +package tasks + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + + "github.com/sirupsen/logrus" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/env" + "github.com/Azure/ARO-RP/pkg/util/clienthelper" +) + +type TaskContext interface { + Environment() env.Interface + ClientHelper() (clienthelper.Interface, error) + Log() *logrus.Entry +} + +type TaskFunc func(context.Context, TaskContext, *api.MaintenanceManifestDocument, *api.OpenShiftClusterDocument) (api.MaintenanceManifestState, string) diff --git a/pkg/util/buckets/buckets.go b/pkg/util/buckets/buckets.go new file mode 100644 index 00000000000..128960e21c5 --- /dev/null +++ b/pkg/util/buckets/buckets.go @@ -0,0 +1,60 @@ +package buckets + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "strings" + "sync" + "time" + + "github.com/sirupsen/logrus" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/util/bucket" +) + +type WorkerFunc func(<-chan struct{}, time.Duration, string) + +type monitor struct { + baseLog *logrus.Entry + + bucketCount int + buckets map[int]struct{} + + mu *sync.RWMutex + docs map[string]*cacheDoc + + worker WorkerFunc +} + +type BucketWorker interface { + Stop() + + Doc(string) *api.OpenShiftClusterDocument + DeleteDoc(*api.OpenShiftClusterDocument) + UpsertDoc(*api.OpenShiftClusterDocument) +} + +func NewBucketWorker(log *logrus.Entry, worker WorkerFunc, mu *sync.RWMutex) *monitor { + return &monitor{ + baseLog: log, + + worker: worker, + docs: map[string]*cacheDoc{}, + + buckets: map[int]struct{}{}, + bucketCount: bucket.Buckets, + + mu: mu, + } +} + +func (mon *monitor) Doc(id string) *api.OpenShiftClusterDocument { + id = strings.ToLower(id) + v := mon.docs[id] + if v == nil { + return nil + } + return v.doc +} diff --git a/pkg/util/buckets/cache.go b/pkg/util/buckets/cache.go new file mode 100644 index 00000000000..12f8d09d539 --- /dev/null +++ b/pkg/util/buckets/cache.go @@ -0,0 +1,80 @@ +package buckets + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "math/rand" + "strings" + "time" + + "github.com/Azure/ARO-RP/pkg/api" +) + +type cacheDoc struct { + doc *api.OpenShiftClusterDocument + stop chan<- struct{} +} + +// deleteDoc deletes the given document from mon.docs, signalling the associated +// monitoring goroutine to stop if it exists. Caller must hold mon.mu.Lock. +func (mon *monitor) DeleteDoc(doc *api.OpenShiftClusterDocument) { + id := strings.ToLower(doc.ID) + v := mon.docs[id] + + if v != nil { + if v.stop != nil { + close(mon.docs[id].stop) + } + + delete(mon.docs, id) + } +} + +// upsertDoc inserts or updates the given document into mon.docs, starting an +// associated monitoring goroutine if the document is in a bucket owned by us. +// Caller must hold mon.mu.Lock. +func (mon *monitor) UpsertDoc(doc *api.OpenShiftClusterDocument) { + id := strings.ToLower(doc.ID) + v := mon.docs[id] + + if v == nil { + v = &cacheDoc{} + mon.docs[id] = v + } + + v.doc = doc + mon.FixDoc(doc) +} + +// fixDoc ensures that there is a monitoring goroutine for the given document +// iff it is in a bucket owned by us. Caller must hold mon.mu.Lock. +func (mon *monitor) FixDoc(doc *api.OpenShiftClusterDocument) { + id := strings.ToLower(doc.ID) + v := mon.docs[id] + _, ours := mon.buckets[v.doc.Bucket] + + if !ours && v.stop != nil { + close(v.stop) + v.stop = nil + } else if ours && v.stop == nil { + ch := make(chan struct{}) + v.stop = ch + + delay := time.Duration(rand.Intn(60)) * time.Second + + go mon.worker(ch, delay, id) + } +} + +// Stop stops all workers. +func (mon *monitor) Stop() { + mon.mu.Lock() + defer mon.mu.Unlock() + for _, v := range mon.docs { + if v.stop != nil { + close(v.stop) + v.stop = nil + } + } +} diff --git a/pkg/util/service/const.go b/pkg/util/service/const.go new file mode 100644 index 00000000000..0183d581ec1 --- /dev/null +++ b/pkg/util/service/const.go @@ -0,0 +1,10 @@ +package service + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +const ( + DatabaseName = "DATABASE_NAME" + DatabaseAccountName = "DATABASE_ACCOUNT_NAME" + KeyVaultPrefix = "KEYVAULT_PREFIX" +) diff --git a/pkg/util/service/database.go b/pkg/util/service/database.go new file mode 100644 index 00000000000..6466b40a6b6 --- /dev/null +++ b/pkg/util/service/database.go @@ -0,0 +1,83 @@ +package service + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "fmt" + "os" + + "github.com/sirupsen/logrus" + + "github.com/Azure/ARO-RP/pkg/database" + "github.com/Azure/ARO-RP/pkg/database/cosmosdb" + "github.com/Azure/ARO-RP/pkg/env" + "github.com/Azure/ARO-RP/pkg/metrics" + "github.com/Azure/ARO-RP/pkg/util/encryption" + "github.com/Azure/ARO-RP/pkg/util/keyvault" +) + +// NewDatabase creates a CosmosDB database client from the environment configuration. +func NewDatabase(ctx context.Context, _env env.Core, log *logrus.Entry, m metrics.Emitter, withAEAD bool) (cosmosdb.DatabaseClient, error) { + var aead encryption.AEAD + + msiToken, err := _env.NewMSITokenCredential() + if err != nil { + return nil, err + } + + if withAEAD { + msiKVAuthorizer, err := _env.NewMSIAuthorizer(_env.Environment().KeyVaultScope) + if err != nil { + return nil, err + } + + keyVaultPrefix := os.Getenv(KeyVaultPrefix) + // TODO: should not be using the service keyvault here + serviceKeyvaultURI := keyvault.URI(_env, env.ServiceKeyvaultSuffix, keyVaultPrefix) + serviceKeyvault := keyvault.NewManager(msiKVAuthorizer, serviceKeyvaultURI) + + aead, err = encryption.NewMulti( + ctx, + serviceKeyvault, + env.EncryptionSecretV2Name, + env.EncryptionSecretName, + ) + if err != nil { + return nil, err + } + } + + dbAccountName := os.Getenv(DatabaseAccountName) + scope := []string{ + fmt.Sprintf("https://%s.%s", dbAccountName, _env.Environment().CosmosDBDNSSuffixScope), + } + + logrusEntry := log.WithField("component", "database") + + dbAuthorizer, err := database.NewTokenAuthorizer( + ctx, + logrusEntry, + msiToken, + dbAccountName, + scope, + ) + if err != nil { + return nil, err + } + + dbc, err := database.NewDatabaseClient( + logrusEntry, + _env, + dbAuthorizer, + m, + aead, + dbAccountName, + ) + if err != nil { + return nil, err + } + + return dbc, nil +} diff --git a/pkg/util/service/helpers.go b/pkg/util/service/helpers.go new file mode 100644 index 00000000000..5013849d835 --- /dev/null +++ b/pkg/util/service/helpers.go @@ -0,0 +1,23 @@ +package service + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "fmt" + "os" + + "github.com/Azure/ARO-RP/pkg/env" +) + +func DBName(isLocalDevelopmentMode bool) (string, error) { + if !isLocalDevelopmentMode { + return "ARO", nil + } + + if err := env.ValidateVars(DatabaseName); err != nil { + return "", fmt.Errorf("%v (development mode)", err.Error()) + } + + return os.Getenv(DatabaseName), nil +} diff --git a/test/database/check.go b/test/database/check.go index dd1e428a9c9..d2e34fb6997 100644 --- a/test/database/check.go +++ b/test/database/check.go @@ -27,12 +27,25 @@ type Checker struct { openShiftVersionDocuments []*api.OpenShiftVersionDocument platformWorkloadIdentityRoleSetDocuments []*api.PlatformWorkloadIdentityRoleSetDocument validationResult []*api.ValidationResult + maintenanceManifestDocuments []*api.MaintenanceManifestDocument } func NewChecker() *Checker { return &Checker{} } +func (f *Checker) Clear() { + f.openshiftClusterDocuments = []*api.OpenShiftClusterDocument{} + f.subscriptionDocuments = []*api.SubscriptionDocument{} + f.billingDocuments = []*api.BillingDocument{} + f.asyncOperationDocuments = []*api.AsyncOperationDocument{} + f.portalDocuments = []*api.PortalDocument{} + f.gatewayDocuments = []*api.GatewayDocument{} + f.openShiftVersionDocuments = []*api.OpenShiftVersionDocument{} + f.maintenanceManifestDocuments = []*api.MaintenanceManifestDocument{} + f.validationResult = []*api.ValidationResult{} +} + func (f *Checker) AddOpenShiftClusterDocuments(docs ...*api.OpenShiftClusterDocument) { for _, doc := range docs { docCopy, err := deepCopy(doc) @@ -132,6 +145,17 @@ func (f *Checker) AddValidationResult(docs ...*api.ValidationResult) { } } +func (f *Checker) AddMaintenanceManifestDocuments(docs ...*api.MaintenanceManifestDocument) { + for _, doc := range docs { + docCopy, err := deepCopy(doc) + if err != nil { + panic(err) + } + + f.maintenanceManifestDocuments = append(f.maintenanceManifestDocuments, docCopy.(*api.MaintenanceManifestDocument)) + } +} + func (f *Checker) CheckOpenShiftClusters(openShiftClusters *cosmosdb.FakeOpenShiftClusterDocumentClient) (errs []error) { ctx := context.Background() @@ -309,3 +333,27 @@ func (f *Checker) CheckPlatformWorkloadIdentityRoleSets(roleSets *cosmosdb.FakeP return errs } + +func (f *Checker) CheckMaintenanceManifests(client *cosmosdb.FakeMaintenanceManifestDocumentClient) (errs []error) { + ctx := context.Background() + + all, err := client.ListAll(ctx, nil) + if err != nil { + return []error{err} + } + + sort.Slice(all.MaintenanceManifestDocuments, func(i, j int) bool { + return all.MaintenanceManifestDocuments[i].ID < all.MaintenanceManifestDocuments[j].ID + }) + + if len(f.maintenanceManifestDocuments) != 0 && len(all.MaintenanceManifestDocuments) == len(f.maintenanceManifestDocuments) { + diff := deep.Equal(all.MaintenanceManifestDocuments, f.maintenanceManifestDocuments) + for _, i := range diff { + errs = append(errs, errors.New(i)) + } + } else if len(all.MaintenanceManifestDocuments) != 0 || len(f.maintenanceManifestDocuments) != 0 { + errs = append(errs, fmt.Errorf("document length different, %d vs %d", len(all.MaintenanceManifestDocuments), len(f.maintenanceManifestDocuments))) + } + + return errs +} diff --git a/test/database/fixtures.go b/test/database/fixtures.go index 0341e27f71a..cef398616ad 100644 --- a/test/database/fixtures.go +++ b/test/database/fixtures.go @@ -21,6 +21,7 @@ type Fixture struct { openShiftVersionDocuments []*api.OpenShiftVersionDocument platformWorkloadIdentityRoleSetDocuments []*api.PlatformWorkloadIdentityRoleSetDocument clusterManagerConfigurationDocuments []*api.ClusterManagerConfigurationDocument + maintenanceManifestDocuments []*api.MaintenanceManifestDocument openShiftClustersDatabase database.OpenShiftClusters billingDatabase database.Billing @@ -31,6 +32,7 @@ type Fixture struct { openShiftVersionsDatabase database.OpenShiftVersions platformWorkloadIdentityRoleSetsDatabase database.PlatformWorkloadIdentityRoleSets clusterManagerConfigurationsDatabase database.ClusterManagerConfigurations + maintenanceManifestsDatabase database.MaintenanceManifests openShiftVersionsUUID uuid.Generator platformWorkloadIdentityRoleSetsUUID uuid.Generator @@ -40,6 +42,18 @@ func NewFixture() *Fixture { return &Fixture{} } +func (f *Fixture) Clear() { + f.openshiftClusterDocuments = []*api.OpenShiftClusterDocument{} + f.subscriptionDocuments = []*api.SubscriptionDocument{} + f.billingDocuments = []*api.BillingDocument{} + f.asyncOperationDocuments = []*api.AsyncOperationDocument{} + f.portalDocuments = []*api.PortalDocument{} + f.gatewayDocuments = []*api.GatewayDocument{} + f.openShiftVersionDocuments = []*api.OpenShiftVersionDocument{} + f.clusterManagerConfigurationDocuments = []*api.ClusterManagerConfigurationDocument{} + f.maintenanceManifestDocuments = []*api.MaintenanceManifestDocument{} +} + func (f *Fixture) WithClusterManagerConfigurations(db database.ClusterManagerConfigurations) *Fixture { f.clusterManagerConfigurationsDatabase = db return f @@ -87,6 +101,11 @@ func (f *Fixture) WithPlatformWorkloadIdentityRoleSets(db database.PlatformWorkl return f } +func (f *Fixture) WithMaintenanceManifests(db database.MaintenanceManifests) *Fixture { + f.maintenanceManifestsDatabase = db + return f +} + func (f *Fixture) AddOpenShiftClusterDocuments(docs ...*api.OpenShiftClusterDocument) { for _, doc := range docs { docCopy, err := deepCopy(doc) @@ -186,6 +205,17 @@ func (f *Fixture) AddClusterManagerConfigurationDocuments(docs ...*api.ClusterMa } } +func (f *Fixture) AddMaintenanceManifestDocuments(docs ...*api.MaintenanceManifestDocument) { + for _, doc := range docs { + docCopy, err := deepCopy(doc) + if err != nil { + panic(err) + } + + f.maintenanceManifestDocuments = append(f.maintenanceManifestDocuments, docCopy.(*api.MaintenanceManifestDocument)) + } +} + func (f *Fixture) Create() error { ctx := context.Background() @@ -264,5 +294,15 @@ func (f *Fixture) Create() error { } } + for _, i := range f.maintenanceManifestDocuments { + if i.ID == "" { + i.ID = f.maintenanceManifestsDatabase.NewUUID() + } + _, err := f.maintenanceManifestsDatabase.Create(ctx, i) + if err != nil { + return err + } + } + return nil } diff --git a/test/database/inmemory.go b/test/database/inmemory.go index 66907cf69ed..3a8a291f83e 100644 --- a/test/database/inmemory.go +++ b/test/database/inmemory.go @@ -4,6 +4,8 @@ package database // Licensed under the Apache License 2.0. import ( + "time" + "github.com/ugorji/go/codec" "github.com/Azure/ARO-RP/pkg/database" @@ -86,3 +88,12 @@ func NewFakeClusterManager() (db database.ClusterManagerConfigurations, client * db = database.NewClusterManagerConfigurationsWithProvidedClient(client, coll, "", uuid) return db, client } + +func NewFakeMaintenanceManifests(now func() time.Time) (db database.MaintenanceManifests, client *cosmosdb.FakeMaintenanceManifestDocumentClient) { + uuid := deterministicuuid.NewTestUUIDGenerator(deterministicuuid.MAINTENANCE_MANIFESTS) + coll := &fakeCollectionClient{} + client = cosmosdb.NewFakeMaintenanceManifestDocumentClient(jsonHandle) + injectMaintenanceManifests(client, now) + db = database.NewMaintenanceManifestsWithProvidedClient(client, coll, "", uuid) + return db, client +} diff --git a/test/database/maintenancemanifests.go b/test/database/maintenancemanifests.go new file mode 100644 index 00000000000..68e7e29ac9a --- /dev/null +++ b/test/database/maintenancemanifests.go @@ -0,0 +1,64 @@ +package database + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "fmt" + "strconv" + "time" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/database" + "github.com/Azure/ARO-RP/pkg/database/cosmosdb" +) + +func injectMaintenanceManifests(c *cosmosdb.FakeMaintenanceManifestDocumentClient, now func() time.Time) { + c.SetQueryHandler(database.MaintenanceManifestQueryForCluster, func(client cosmosdb.MaintenanceManifestDocumentClient, query *cosmosdb.Query, options *cosmosdb.Options) cosmosdb.MaintenanceManifestDocumentRawIterator { + return fakeMaintenanceManifestsForCluster(client, query, options, now) + }) + + c.SetTriggerHandler("renewLease", func(ctx context.Context, doc *api.MaintenanceManifestDocument) error { + return fakeMaintenanceManifestsRenewLeaseTrigger(ctx, doc, now) + }) +} + +func fakeMaintenanceManifestsForCluster(client cosmosdb.MaintenanceManifestDocumentClient, query *cosmosdb.Query, options *cosmosdb.Options, now func() time.Time) cosmosdb.MaintenanceManifestDocumentRawIterator { + startingIndex, err := fakeMaintenanceManifestsGetContinuation(options) + if err != nil { + return cosmosdb.NewFakeMaintenanceManifestDocumentErroringRawIterator(err) + } + + input, err := client.ListAll(context.Background(), nil) + if err != nil { + // TODO: should this never happen? + panic(err) + } + + clusterResourceID := query.Parameters[0].Value + + fmt.Print(clusterResourceID, startingIndex) + + var results []*api.MaintenanceManifestDocument + for _, r := range input.MaintenanceManifestDocuments { + if r.ClusterResourceID == clusterResourceID && + r.MaintenanceManifest.State == api.MaintenanceManifestStatePending { + results = append(results, r) + } + } + + return cosmosdb.NewFakeMaintenanceManifestDocumentIterator(results, startingIndex) +} + +func fakeMaintenanceManifestsRenewLeaseTrigger(ctx context.Context, doc *api.MaintenanceManifestDocument, now func() time.Time) error { + doc.LeaseExpires = int(now().Unix()) + 60 + return nil +} + +func fakeMaintenanceManifestsGetContinuation(options *cosmosdb.Options) (startingIndex int, err error) { + if options != nil && options.Continuation != "" { + startingIndex, err = strconv.Atoi(options.Continuation) + } + return +} diff --git a/test/database/openshiftclusters.go b/test/database/openshiftclusters.go index bc9c017fe58..f348f185cec 100644 --- a/test/database/openshiftclusters.go +++ b/test/database/openshiftclusters.go @@ -151,6 +151,28 @@ func openShiftClusterConflictChecker(one *api.OpenShiftClusterDocument, two *api return false } +func fakeOpenShiftClustersOnlyResourceID(client cosmosdb.OpenShiftClusterDocumentClient, query *cosmosdb.Query, options *cosmosdb.Options) cosmosdb.OpenShiftClusterDocumentRawIterator { + startingIndex, err := fakeOpenShiftClustersGetContinuation(options) + if err != nil { + return cosmosdb.NewFakeOpenShiftClusterDocumentErroringRawIterator(err) + } + + docs, err := fakeOpenShiftClustersGetAllDocuments(client) + if err != nil { + return cosmosdb.NewFakeOpenShiftClusterDocumentErroringRawIterator(err) + } + + newDocs := make([]*api.OpenShiftClusterDocument, 0) + + for _, d := range docs { + newDocs = append(newDocs, &api.OpenShiftClusterDocument{ + Key: d.Key, + }) + } + + return cosmosdb.NewFakeOpenShiftClusterDocumentIterator(newDocs, startingIndex) +} + func injectOpenShiftClusters(c *cosmosdb.FakeOpenShiftClusterDocumentClient) { c.SetQueryHandler(database.OpenShiftClustersDequeueQuery, fakeOpenShiftClustersDequeueQuery) c.SetQueryHandler(database.OpenShiftClustersQueueLengthQuery, fakeOpenShiftClustersQueueLengthQuery) @@ -158,6 +180,7 @@ func injectOpenShiftClusters(c *cosmosdb.FakeOpenShiftClusterDocumentClient) { c.SetQueryHandler(database.OpenshiftClustersClientIdQuery, fakeOpenshiftClustersMatchQuery) c.SetQueryHandler(database.OpenshiftClustersResourceGroupQuery, fakeOpenshiftClustersMatchQuery) c.SetQueryHandler(database.OpenshiftClustersPrefixQuery, fakeOpenshiftClustersPrefixQuery) + c.SetQueryHandler(database.OpenshiftClustersClusterResourceIDOnlyQuery, fakeOpenShiftClustersOnlyResourceID) c.SetTriggerHandler("renewLease", fakeOpenShiftClustersRenewLeaseTrigger) diff --git a/test/mimo/tasks/testcontext.go b/test/mimo/tasks/testcontext.go new file mode 100644 index 00000000000..2d7f6aea095 --- /dev/null +++ b/test/mimo/tasks/testcontext.go @@ -0,0 +1,38 @@ +package tasks + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "github.com/sirupsen/logrus" + + "github.com/Azure/ARO-RP/pkg/env" + "github.com/Azure/ARO-RP/pkg/mimo/tasks" + "github.com/Azure/ARO-RP/pkg/util/clienthelper" +) + +func NewFakeTestContext(env env.Interface, log *logrus.Entry, ch clienthelper.Interface) tasks.TaskContext { + return &fakeTestContext{ + env: env, + log: log, + ch: ch, + } +} + +type fakeTestContext struct { + env env.Interface + ch clienthelper.Interface + log *logrus.Entry +} + +func (t *fakeTestContext) Environment() env.Interface { + return t.env +} + +func (t *fakeTestContext) ClientHelper() (clienthelper.Interface, error) { + return t.ch, nil +} + +func (t *fakeTestContext) Log() *logrus.Entry { + return t.log +} diff --git a/test/util/deterministicuuid/deterministicuuid.go b/test/util/deterministicuuid/deterministicuuid.go index f5307dd7941..702442d4998 100644 --- a/test/util/deterministicuuid/deterministicuuid.go +++ b/test/util/deterministicuuid/deterministicuuid.go @@ -19,6 +19,7 @@ const ( GATEWAY OPENSHIFT_VERSIONS CLUSTERMANAGER + MAINTENANCE_MANIFESTS ) type gen struct { From c733dd13d4bd057136be4f6001ce943b5e11c825 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Mon, 15 Jul 2024 13:33:41 +1000 Subject: [PATCH 02/67] initial maint API --- pkg/frontend/admin_mimo_list.go | 57 +++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 pkg/frontend/admin_mimo_list.go diff --git a/pkg/frontend/admin_mimo_list.go b/pkg/frontend/admin_mimo_list.go new file mode 100644 index 00000000000..4a8308150bf --- /dev/null +++ b/pkg/frontend/admin_mimo_list.go @@ -0,0 +1,57 @@ +package frontend + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "net/http" + "path/filepath" + "strings" + + "github.com/sirupsen/logrus" + "github.com/ugorji/go/codec" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/frontend/middleware" +) + +func (f *frontend) getAdminMaintManifests(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + log := ctx.Value(middleware.ContextKeyLog).(*logrus.Entry) + resourceID := strings.TrimPrefix(filepath.Dir(r.URL.Path), "/admin") + b, err := f._getAdminMaintManifests(ctx, resourceID) + + if cloudErr, ok := err.(*api.CloudError); ok { + api.WriteCloudError(w, cloudErr) + return + } + + adminReply(log, w, nil, b, err) +} + +func (f *frontend) _getAdminMaintManifests(ctx context.Context, resourceID string) ([]byte, error) { + doc, err := f.dbOpenShiftClusters.Get(ctx, resourceID) + if err != nil { + return nil, api.NewCloudError(http.StatusNotFound, api.CloudErrorCodeNotFound, "", "cluster not found") + } + + f.dbOpenShiftVersions + + if doc.OpenShiftCluster.Properties.HiveProfile.Namespace == "" { + return nil, api.NewCloudError(http.StatusNoContent, api.CloudErrorCodeResourceNotFound, "", "cluster is not managed by hive") + } + + cd, err := f.hiveClusterManager.GetClusterDeployment(ctx, doc) + if err != nil { + return nil, api.NewCloudError(http.StatusNotFound, api.CloudErrorCodeNotFound, "", "cluster deployment not found") + } + + var b []byte + err = codec.NewEncoderBytes(&b, &codec.JsonHandle{}).Encode(cd) + if err != nil { + return nil, api.NewCloudError(http.StatusInternalServerError, api.CloudErrorCodeInternalServerError, "", "unable to marshal response") + } + + return b, nil +} From 733dabd710c6895b3953598be3f9a739589213d1 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Thu, 18 Jul 2024 13:17:47 +1000 Subject: [PATCH 03/67] update mimo DB to have a fetch which isn't just pending --- pkg/database/mimo.go | 24 +++++++++++++++++--- pkg/mimo/actuator/manager.go | 2 +- test/database/maintenancemanifests.go | 32 +++++++++++++++++++++++---- 3 files changed, 50 insertions(+), 8 deletions(-) diff --git a/pkg/database/mimo.go b/pkg/database/mimo.go index 27b5356c0d2..582965c98cb 100644 --- a/pkg/database/mimo.go +++ b/pkg/database/mimo.go @@ -15,8 +15,9 @@ import ( ) const ( - MaintenanceManifestQueryForCluster = `SELECT * FROM MaintenanceManifests doc WHERE doc.maintenanceManifest.state IN ("Pending") AND doc.clusterResourceID = @clusterID` - MaintenanceManifestQueueLengthQuery = `SELECT VALUE COUNT(1) FROM MaintenanceManifests doc WHERE doc.maintenanceManifest.state IN ("Pending") AND (doc.leaseExpires ?? 0) < GetCurrentTimestamp() / 1000` + MaintenanceManifestDequeueQueryForCluster = `SELECT * FROM MaintenanceManifests doc WHERE doc.maintenanceManifest.state IN ("Pending") AND doc.clusterResourceID = @clusterID` + MaintenanceManifestQueryForCluster = `SELECT * FROM MaintenanceManifests doc WHERE doc.clusterResourceID = @clusterID` + MaintenanceManifestQueueLengthQuery = `SELECT VALUE COUNT(1) FROM MaintenanceManifests doc WHERE doc.maintenanceManifest.state IN ("Pending") AND (doc.leaseExpires ?? 0) < GetCurrentTimestamp() / 1000` ) type MaintenanceManifestDocumentMutator func(*api.MaintenanceManifestDocument) error @@ -30,7 +31,8 @@ type maintenanceManifests struct { type MaintenanceManifests interface { Create(context.Context, *api.MaintenanceManifestDocument) (*api.MaintenanceManifestDocument, error) - GetByClusterResourceID(context.Context, string, string) (cosmosdb.MaintenanceManifestDocumentIterator, error) + GetByClusterResourceID(ctx context.Context, clusterResourceID string, continuation string) (cosmosdb.MaintenanceManifestDocumentIterator, error) + GetQueuedByClusterResourceID(ctx context.Context, clusterResourceID string, continuation string) (cosmosdb.MaintenanceManifestDocumentIterator, error) Patch(context.Context, string, string, MaintenanceManifestDocumentMutator) (*api.MaintenanceManifestDocument, error) PatchWithLease(context.Context, string, string, MaintenanceManifestDocumentMutator) (*api.MaintenanceManifestDocument, error) Lease(ctx context.Context, clusterResourceID string, id string) (*api.MaintenanceManifestDocument, error) @@ -182,6 +184,22 @@ func (c *maintenanceManifests) GetByClusterResourceID(ctx context.Context, clust }, &cosmosdb.Options{Continuation: continuation}), nil } +func (c *maintenanceManifests) GetQueuedByClusterResourceID(ctx context.Context, clusterResourceID string, continuation string) (cosmosdb.MaintenanceManifestDocumentIterator, error) { + if clusterResourceID != strings.ToLower(clusterResourceID) { + return nil, fmt.Errorf("clusterResourceID %q is not lower case", clusterResourceID) + } + + return c.c.Query("", &cosmosdb.Query{ + Query: MaintenanceManifestDequeueQueryForCluster, + Parameters: []cosmosdb.Parameter{ + { + Name: "@clusterResourceID", + Value: clusterResourceID, + }, + }, + }, &cosmosdb.Options{Continuation: continuation}), nil +} + func (c *maintenanceManifests) EndLease(ctx context.Context, clusterResourceID string, id string, provisioningState api.MaintenanceManifestState, statusString *string) (*api.MaintenanceManifestDocument, error) { return c.patchWithLease(ctx, clusterResourceID, id, func(doc *api.MaintenanceManifestDocument) error { doc.MaintenanceManifest.State = provisioningState diff --git a/pkg/mimo/actuator/manager.go b/pkg/mimo/actuator/manager.go index 48724119e84..16531b96d9f 100644 --- a/pkg/mimo/actuator/manager.go +++ b/pkg/mimo/actuator/manager.go @@ -73,7 +73,7 @@ func (a *actuator) AddTasks(tasks map[string]tasks.TaskFunc) { func (a *actuator) Process(ctx context.Context) (bool, error) { // Get the manifests for this cluster which need to be worked - i, err := a.mmf.GetByClusterResourceID(ctx, a.clusterResourceID, "") + i, err := a.mmf.GetQueuedByClusterResourceID(ctx, a.clusterResourceID, "") if err != nil { return false, fmt.Errorf("failed getting manifests: %w", err) } diff --git a/test/database/maintenancemanifests.go b/test/database/maintenancemanifests.go index 68e7e29ac9a..e6e511214ab 100644 --- a/test/database/maintenancemanifests.go +++ b/test/database/maintenancemanifests.go @@ -5,7 +5,6 @@ package database import ( "context" - "fmt" "strconv" "time" @@ -18,13 +17,16 @@ func injectMaintenanceManifests(c *cosmosdb.FakeMaintenanceManifestDocumentClien c.SetQueryHandler(database.MaintenanceManifestQueryForCluster, func(client cosmosdb.MaintenanceManifestDocumentClient, query *cosmosdb.Query, options *cosmosdb.Options) cosmosdb.MaintenanceManifestDocumentRawIterator { return fakeMaintenanceManifestsForCluster(client, query, options, now) }) + c.SetQueryHandler(database.MaintenanceManifestDequeueQueryForCluster, func(client cosmosdb.MaintenanceManifestDocumentClient, query *cosmosdb.Query, options *cosmosdb.Options) cosmosdb.MaintenanceManifestDocumentRawIterator { + return fakeMaintenanceManifestsDequeueForCluster(client, query, options, now) + }) c.SetTriggerHandler("renewLease", func(ctx context.Context, doc *api.MaintenanceManifestDocument) error { return fakeMaintenanceManifestsRenewLeaseTrigger(ctx, doc, now) }) } -func fakeMaintenanceManifestsForCluster(client cosmosdb.MaintenanceManifestDocumentClient, query *cosmosdb.Query, options *cosmosdb.Options, now func() time.Time) cosmosdb.MaintenanceManifestDocumentRawIterator { +func fakeMaintenanceManifestsDequeueForCluster(client cosmosdb.MaintenanceManifestDocumentClient, query *cosmosdb.Query, options *cosmosdb.Options, now func() time.Time) cosmosdb.MaintenanceManifestDocumentRawIterator { startingIndex, err := fakeMaintenanceManifestsGetContinuation(options) if err != nil { return cosmosdb.NewFakeMaintenanceManifestDocumentErroringRawIterator(err) @@ -38,8 +40,6 @@ func fakeMaintenanceManifestsForCluster(client cosmosdb.MaintenanceManifestDocum clusterResourceID := query.Parameters[0].Value - fmt.Print(clusterResourceID, startingIndex) - var results []*api.MaintenanceManifestDocument for _, r := range input.MaintenanceManifestDocuments { if r.ClusterResourceID == clusterResourceID && @@ -51,6 +51,30 @@ func fakeMaintenanceManifestsForCluster(client cosmosdb.MaintenanceManifestDocum return cosmosdb.NewFakeMaintenanceManifestDocumentIterator(results, startingIndex) } +func fakeMaintenanceManifestsForCluster(client cosmosdb.MaintenanceManifestDocumentClient, query *cosmosdb.Query, options *cosmosdb.Options, now func() time.Time) cosmosdb.MaintenanceManifestDocumentRawIterator { + startingIndex, err := fakeMaintenanceManifestsGetContinuation(options) + if err != nil { + return cosmosdb.NewFakeMaintenanceManifestDocumentErroringRawIterator(err) + } + + input, err := client.ListAll(context.Background(), nil) + if err != nil { + // TODO: should this never happen? + panic(err) + } + + clusterResourceID := query.Parameters[0].Value + + var results []*api.MaintenanceManifestDocument + for _, r := range input.MaintenanceManifestDocuments { + if r.ClusterResourceID == clusterResourceID { + results = append(results, r) + } + } + + return cosmosdb.NewFakeMaintenanceManifestDocumentIterator(results, startingIndex) +} + func fakeMaintenanceManifestsRenewLeaseTrigger(ctx context.Context, doc *api.MaintenanceManifestDocument, now func() time.Time) error { doc.LeaseExpires = int(now().Unix()) + 60 return nil From 0c97cf618fc65d0d3cd6657b67a00bddc3e311a7 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Thu, 18 Jul 2024 13:18:19 +1000 Subject: [PATCH 04/67] update db fakes for maintmanifests --- pkg/database/dbgroup.go | 19 +++++++++++++++++++ pkg/frontend/shared_test.go | 16 +++++++++++++--- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/pkg/database/dbgroup.go b/pkg/database/dbgroup.go index fa389d78aaa..1b4b8bebc25 100644 --- a/pkg/database/dbgroup.go +++ b/pkg/database/dbgroup.go @@ -37,6 +37,10 @@ type DatabaseGroupWithPortal interface { Portal() (Portal, error) } +type DatabaseGroupWithMaintenanceManifests interface { + MaintenanceManifests() (MaintenanceManifests, error) +} + type DatabaseGroup interface { DatabaseGroupWithOpenShiftClusters DatabaseGroupWithSubscriptions @@ -46,6 +50,7 @@ type DatabaseGroup interface { DatabaseGroupWithAsyncOperations DatabaseGroupWithBilling DatabaseGroupWithPortal + DatabaseGroupWithMaintenanceManifests WithOpenShiftClusters(db OpenShiftClusters) DatabaseGroup WithSubscriptions(db Subscriptions) DatabaseGroup @@ -55,6 +60,7 @@ type DatabaseGroup interface { WithAsyncOperations(db AsyncOperations) DatabaseGroup WithBilling(db Billing) DatabaseGroup WithPortal(db Portal) DatabaseGroup + WithMaintenanceManifests(db MaintenanceManifests) DatabaseGroup } type dbGroup struct { @@ -66,6 +72,7 @@ type dbGroup struct { asyncOperations AsyncOperations billing Billing portal Portal + maintenanceManifests MaintenanceManifests } func (d *dbGroup) OpenShiftClusters() (OpenShiftClusters, error) { @@ -164,6 +171,18 @@ func (d *dbGroup) WithPortal(db Portal) DatabaseGroup { return d } +func (d *dbGroup) MaintenanceManifests() (MaintenanceManifests, error) { + if d.maintenanceManifests == nil { + return nil, errors.New("no MaintenanceManifests defined") + } + return d.maintenanceManifests, nil +} + +func (d *dbGroup) WithMaintenanceManifests(db MaintenanceManifests) DatabaseGroup { + d.maintenanceManifests = db + return d +} + func NewDBGroup() DatabaseGroup { return &dbGroup{} } diff --git a/pkg/frontend/shared_test.go b/pkg/frontend/shared_test.go index 0bd32983bef..4b7e0240d27 100644 --- a/pkg/frontend/shared_test.go +++ b/pkg/frontend/shared_test.go @@ -15,6 +15,7 @@ import ( "net/http" "reflect" "testing" + "time" "github.com/go-test/deep" "github.com/sirupsen/logrus" @@ -83,6 +84,8 @@ type testInfra struct { openShiftVersionsDatabase database.OpenShiftVersions platformWorkloadIdentityRoleSetsClient *cosmosdb.FakePlatformWorkloadIdentityRoleSetDocumentClient platformWorkloadIdentityRoleSetsDatabase database.PlatformWorkloadIdentityRoleSets + maintenanceManifestsClient *cosmosdb.FakeMaintenanceManifestDocumentClient + maintenanceManifestsDatabase database.MaintenanceManifests } func newTestInfra(t *testing.T) *testInfra { @@ -204,6 +207,13 @@ func (ti *testInfra) WithClusterManagerConfigurations() *testInfra { return ti } +func (ti *testInfra) WithMaintenanceManifests(now func() time.Time) *testInfra { + ti.maintenanceManifestsDatabase, ti.maintenanceManifestsClient = testdatabase.NewFakeMaintenanceManifests(now) + ti.fixture.WithMaintenanceManifests(ti.maintenanceManifestsDatabase) + ti.dbGroup.WithMaintenanceManifests(ti.maintenanceManifestsDatabase) + return ti +} + func (ti *testInfra) done() { ti.controller.Finish() ti.cli.CloseIdleConnections() @@ -254,7 +264,7 @@ func (ti *testInfra) request(method, url string, header http.Header, in interfac func validateResponse(resp *http.Response, b []byte, wantStatusCode int, wantError string, wantResponse interface{}) error { if resp.StatusCode != wantStatusCode { - return fmt.Errorf("unexpected status code %d, wanted %d", resp.StatusCode, wantStatusCode) + return fmt.Errorf("unexpected status code %d, wanted %d: %s", resp.StatusCode, wantStatusCode, string(b)) } if wantError != "" { @@ -264,8 +274,8 @@ func validateResponse(resp *http.Response, b []byte, wantStatusCode int, wantErr return err } - if cloudErr.Error() != wantError { - return fmt.Errorf("unexpected error %s, wanted %s", cloudErr.Error(), wantError) + if diff := deep.Equal(cloudErr.Error(), wantError); diff != nil { + return fmt.Errorf("unexpected error %s, wanted %s (%s)", cloudErr.Error(), wantError, diff) } return nil From 2b5986441db691337807eef6458b63128fcf9b5e Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Thu, 18 Jul 2024 13:18:34 +1000 Subject: [PATCH 05/67] mimo API conversions for admin API --- pkg/api/admin/mimo.go | 39 +++++++++++++++++++++++++++++++++++ pkg/api/admin/mimo_convert.go | 33 +++++++++++++++++++++++++++++ pkg/api/admin/register.go | 1 + pkg/api/register.go | 6 ++++++ 4 files changed, 79 insertions(+) create mode 100644 pkg/api/admin/mimo.go create mode 100644 pkg/api/admin/mimo_convert.go diff --git a/pkg/api/admin/mimo.go b/pkg/api/admin/mimo.go new file mode 100644 index 00000000000..6aaf6953e0e --- /dev/null +++ b/pkg/api/admin/mimo.go @@ -0,0 +1,39 @@ +package admin + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +type MaintenanceManifestState string + +const ( + MaintenanceManifestStatePending MaintenanceManifestState = "Pending" + MaintenanceManifestStateInProgress MaintenanceManifestState = "InProgress" + MaintenanceManifestStateCompleted MaintenanceManifestState = "Completed" + MaintenanceManifestStateFailed MaintenanceManifestState = "Failed" + MaintenanceManifestStateTimedOut MaintenanceManifestState = "TimedOut" +) + +type MaintenanceManifest struct { + // The ID for the resource. + ID string `json:"id,omitempty"` + + State MaintenanceManifestState `json:"state,omitempty"` + StatusText string `json:"statusText,omitempty"` + + MaintenanceSetID string `json:"maintenanceSetID,omitempty"` + Priority int `json:"priority,omitempty"` + + // RunAfter defines the earliest that this manifest should start running + RunAfter int `json:"runAfter,omitempty"` + // RunBefore defines the latest that this manifest should start running + RunBefore int `json:"runBefore,omitempty"` +} + +// MaintenanceManifestList represents a list of MaintenanceManifests. +type MaintenanceManifestList struct { + // The list of MaintenanceManifests. + MaintenanceManifests []*MaintenanceManifest `json:"value"` + + // The link used to get the next page of operations. + NextLink string `json:"nextLink,omitempty"` +} diff --git a/pkg/api/admin/mimo_convert.go b/pkg/api/admin/mimo_convert.go new file mode 100644 index 00000000000..930e1d29d2a --- /dev/null +++ b/pkg/api/admin/mimo_convert.go @@ -0,0 +1,33 @@ +package admin + +import "github.com/Azure/ARO-RP/pkg/api" + +type maintenanceManifestConverter struct{} + +func (m maintenanceManifestConverter) ToExternal(d *api.MaintenanceManifestDocument) interface{} { + return &MaintenanceManifest{ + ID: d.ID, + + State: MaintenanceManifestState(d.MaintenanceManifest.State), + StatusText: d.MaintenanceManifest.StatusText, + + MaintenanceSetID: d.MaintenanceManifest.MaintenanceSetID, + Priority: d.MaintenanceManifest.Priority, + + RunAfter: d.MaintenanceManifest.RunAfter, + RunBefore: d.MaintenanceManifest.RunBefore, + } +} + +func (m maintenanceManifestConverter) ToExternalList(docs []*api.MaintenanceManifestDocument, nextLink string) interface{} { + l := &MaintenanceManifestList{ + MaintenanceManifests: make([]*MaintenanceManifest, 0, len(docs)), + NextLink: nextLink, + } + + for _, doc := range docs { + l.MaintenanceManifests = append(l.MaintenanceManifests, m.ToExternal(doc).(*MaintenanceManifest)) + } + + return l +} diff --git a/pkg/api/admin/register.go b/pkg/api/admin/register.go index 4a0613c276e..e24558fffa3 100644 --- a/pkg/api/admin/register.go +++ b/pkg/api/admin/register.go @@ -18,5 +18,6 @@ func init() { OpenShiftVersionStaticValidator: openShiftVersionStaticValidator{}, PlatformWorkloadIdentityRoleSetConverter: platformWorkloadIdentityRoleSetConverter{}, PlatformWorkloadIdentityRoleSetStaticValidator: platformWorkloadIdentityRoleSetStaticValidator{}, + MaintenanceManifestConverter: maintenanceManifestConverter{}, } } diff --git a/pkg/api/register.go b/pkg/api/register.go index 5e91693909b..9b50b26f8bd 100644 --- a/pkg/api/register.go +++ b/pkg/api/register.go @@ -71,6 +71,11 @@ type SecretConverter interface { ToInternal(interface{}, *Secret) } +type MaintenanceManifestConverter interface { + ToExternal(*MaintenanceManifestDocument) interface{} + ToExternalList([]*MaintenanceManifestDocument, string) interface{} +} + // Version is a set of endpoints implemented by each API version type Version struct { OpenShiftClusterConverter OpenShiftClusterConverter @@ -87,6 +92,7 @@ type Version struct { SyncIdentityProviderConverter SyncIdentityProviderConverter SecretConverter SecretConverter ClusterManagerStaticValidator ClusterManagerStaticValidator + MaintenanceManifestConverter MaintenanceManifestConverter } // APIs is the map of registered API versions From 727ece2f8c9888ea8507feece673c56e6cc8b122 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Thu, 18 Jul 2024 13:18:51 +1000 Subject: [PATCH 06/67] query + tests for maintenancemanifests --- pkg/api/admin/mimo_convert.go | 3 + pkg/frontend/admin_mimo_list.go | 67 +++++++-- pkg/frontend/admin_mimo_list_test.go | 214 +++++++++++++++++++++++++++ pkg/frontend/frontend.go | 4 + 4 files changed, 275 insertions(+), 13 deletions(-) create mode 100644 pkg/frontend/admin_mimo_list_test.go diff --git a/pkg/api/admin/mimo_convert.go b/pkg/api/admin/mimo_convert.go index 930e1d29d2a..a357bfc69de 100644 --- a/pkg/api/admin/mimo_convert.go +++ b/pkg/api/admin/mimo_convert.go @@ -1,5 +1,8 @@ package admin +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + import "github.com/Azure/ARO-RP/pkg/api" type maintenanceManifestConverter struct{} diff --git a/pkg/frontend/admin_mimo_list.go b/pkg/frontend/admin_mimo_list.go index 4a8308150bf..ba1d039e131 100644 --- a/pkg/frontend/admin_mimo_list.go +++ b/pkg/frontend/admin_mimo_list.go @@ -5,14 +5,18 @@ package frontend import ( "context" + "encoding/json" + "fmt" + "math" "net/http" "path/filepath" + "strconv" "strings" "github.com/sirupsen/logrus" - "github.com/ugorji/go/codec" "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/api/admin" "github.com/Azure/ARO-RP/pkg/frontend/middleware" ) @@ -20,7 +24,7 @@ func (f *frontend) getAdminMaintManifests(w http.ResponseWriter, r *http.Request ctx := r.Context() log := ctx.Value(middleware.ContextKeyLog).(*logrus.Entry) resourceID := strings.TrimPrefix(filepath.Dir(r.URL.Path), "/admin") - b, err := f._getAdminMaintManifests(ctx, resourceID) + b, err := f._getAdminMaintManifests(ctx, r, resourceID) if cloudErr, ok := err.(*api.CloudError); ok { api.WriteCloudError(w, cloudErr) @@ -30,28 +34,65 @@ func (f *frontend) getAdminMaintManifests(w http.ResponseWriter, r *http.Request adminReply(log, w, nil, b, err) } -func (f *frontend) _getAdminMaintManifests(ctx context.Context, resourceID string) ([]byte, error) { - doc, err := f.dbOpenShiftClusters.Get(ctx, resourceID) +func (f *frontend) _getAdminMaintManifests(ctx context.Context, r *http.Request, resourceID string) ([]byte, error) { + limitstr := r.URL.Query().Get("limit") + limit, err := strconv.Atoi(limitstr) + if err != nil { + limit = 100 + } + + converter := f.apis[admin.APIVersion].MaintenanceManifestConverter + + dbOpenShiftClusters, err := f.dbGroup.OpenShiftClusters() + if err != nil { + return nil, api.NewCloudError(http.StatusInternalServerError, api.CloudErrorCodeInternalServerError, "", err.Error()) + } + + dbMaintenanceManifests, err := f.dbGroup.MaintenanceManifests() + if err != nil { + return nil, api.NewCloudError(http.StatusInternalServerError, api.CloudErrorCodeInternalServerError, "", err.Error()) + } + + doc, err := dbOpenShiftClusters.Get(ctx, resourceID) if err != nil { return nil, api.NewCloudError(http.StatusNotFound, api.CloudErrorCodeNotFound, "", "cluster not found") } - f.dbOpenShiftVersions + if doc.OpenShiftCluster.Properties.ProvisioningState == api.ProvisioningStateDeleting { + return nil, api.NewCloudError(http.StatusNotFound, api.CloudErrorCodeNotFound, "", "cluster being deleted") + } - if doc.OpenShiftCluster.Properties.HiveProfile.Namespace == "" { - return nil, api.NewCloudError(http.StatusNoContent, api.CloudErrorCodeResourceNotFound, "", "cluster is not managed by hive") + skipToken, err := f.parseSkipToken(r.URL.String()) + if err != nil { + return nil, err } - cd, err := f.hiveClusterManager.GetClusterDeployment(ctx, doc) + i, err := dbMaintenanceManifests.GetByClusterResourceID(ctx, resourceID, skipToken) if err != nil { - return nil, api.NewCloudError(http.StatusNotFound, api.CloudErrorCodeNotFound, "", "cluster deployment not found") + return nil, api.NewCloudError(http.StatusInternalServerError, api.CloudErrorCodeInternalServerError, "", err.Error()) + } + + docList := make([]*api.MaintenanceManifestDocument, 0) + for { + docs, err := i.Next(ctx, int(math.Min(float64(limit), 10))) + if err != nil { + return nil, api.NewCloudError(http.StatusInternalServerError, api.CloudErrorCodeInternalServerError, "", fmt.Errorf("failed reading next manifest document: %w", err).Error()) + } + if docs == nil { + break + } + + docList = append(docList, docs.MaintenanceManifestDocuments...) + + if len(docList) >= limit { + break + } } - var b []byte - err = codec.NewEncoderBytes(&b, &codec.JsonHandle{}).Encode(cd) + nextLink, err := f.buildNextLink(r.Header.Get("Referer"), i.Continuation()) if err != nil { - return nil, api.NewCloudError(http.StatusInternalServerError, api.CloudErrorCodeInternalServerError, "", "unable to marshal response") + return nil, err } - return b, nil + return json.MarshalIndent(converter.ToExternalList(docList, nextLink), "", " ") } diff --git a/pkg/frontend/admin_mimo_list_test.go b/pkg/frontend/admin_mimo_list_test.go new file mode 100644 index 00000000000..0da37580329 --- /dev/null +++ b/pkg/frontend/admin_mimo_list_test.go @@ -0,0 +1,214 @@ +package frontend + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "encoding/base64" + "fmt" + "net/http" + "net/url" + "strings" + "testing" + "time" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/api/admin" + "github.com/Azure/ARO-RP/pkg/metrics/noop" + testdatabase "github.com/Azure/ARO-RP/test/database" +) + +func TestMIMOList(t *testing.T) { + mockSubID := "00000000-0000-0000-0000-000000000000" + mockTenantID := "00000000-0000-0000-0000-000000000000" + resourceID := fmt.Sprintf("/subscriptions/%s/resourcegroups/resourceGroup/providers/Microsoft.RedHatOpenShift/openShiftClusters/resourceName", mockSubID) + ctx := context.Background() + + type test struct { + name string + fixtures func(f *testdatabase.Fixture) + limit int + wantStatusCode int + wantResponse *admin.MaintenanceManifestList + wantError string + } + + for _, tt := range []*test{ + { + name: "no entries", + fixtures: func(f *testdatabase.Fixture) { + f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{ + Key: strings.ToLower(resourceID), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: resourceID, + Name: "resourceName", + Type: "Microsoft.RedHatOpenShift/openshiftClusters", + }, + }) + }, + wantResponse: &admin.MaintenanceManifestList{ + MaintenanceManifests: []*admin.MaintenanceManifest{}, + }, + wantStatusCode: http.StatusOK, + }, + { + name: "single entry", + fixtures: func(f *testdatabase.Fixture) { + f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{ + Key: strings.ToLower(resourceID), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: resourceID, + Name: "resourceName", + Type: "Microsoft.RedHatOpenShift/openshiftClusters", + }, + }) + f.AddMaintenanceManifestDocuments(&api.MaintenanceManifestDocument{ + ClusterResourceID: strings.ToLower(resourceID), + MaintenanceManifest: &api.MaintenanceManifest{ + MaintenanceSetID: "exampleset", + State: api.MaintenanceManifestStatePending, + RunAfter: 1, + RunBefore: 1, + }, + }) + }, + wantResponse: &admin.MaintenanceManifestList{ + MaintenanceManifests: []*admin.MaintenanceManifest{ + { + ID: "07070707-0707-0707-0707-070707070001", + MaintenanceSetID: "exampleset", + State: admin.MaintenanceManifestStatePending, + Priority: 0, + RunAfter: 1, + RunBefore: 1, + }, + }, + }, + wantStatusCode: http.StatusOK, + }, + { + name: "limit over", + limit: 1, + fixtures: func(f *testdatabase.Fixture) { + f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{ + Key: strings.ToLower(resourceID), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: resourceID, + Name: "resourceName", + Type: "Microsoft.RedHatOpenShift/openshiftClusters", + }, + }) + f.AddMaintenanceManifestDocuments(&api.MaintenanceManifestDocument{ + ClusterResourceID: strings.ToLower(resourceID), + MaintenanceManifest: &api.MaintenanceManifest{ + MaintenanceSetID: "exampleset", + State: api.MaintenanceManifestStatePending, + RunAfter: 1, + RunBefore: 1, + }, + }) + f.AddMaintenanceManifestDocuments(&api.MaintenanceManifestDocument{ + ClusterResourceID: strings.ToLower(resourceID), + MaintenanceManifest: &api.MaintenanceManifest{ + MaintenanceSetID: "exampleset2", + State: api.MaintenanceManifestStatePending, + RunAfter: 1, + RunBefore: 1, + }, + }) + }, + wantResponse: &admin.MaintenanceManifestList{ + NextLink: "https://mockrefererhost/?%24skipToken=" + url.QueryEscape(base64.StdEncoding.EncodeToString([]byte("FAKE1"))), + MaintenanceManifests: []*admin.MaintenanceManifest{ + { + ID: "07070707-0707-0707-0707-070707070001", + MaintenanceSetID: "exampleset", + State: admin.MaintenanceManifestStatePending, + Priority: 0, + RunAfter: 1, + RunBefore: 1, + }, + }, + }, + wantStatusCode: http.StatusOK, + }, + { + name: "cluster being deleted", + fixtures: func(f *testdatabase.Fixture) { + f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{ + Key: strings.ToLower(resourceID), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: resourceID, + Name: "resourceName", + Type: "Microsoft.RedHatOpenShift/openshiftClusters", + Properties: api.OpenShiftClusterProperties{ + ProvisioningState: api.ProvisioningStateDeleting, + }, + }, + }) + }, + wantError: "404: NotFound: : cluster being deleted", + wantStatusCode: http.StatusNotFound, + }, + { + name: "missing cluster", + wantError: "404: NotFound: : cluster not found", + wantStatusCode: http.StatusNotFound, + }, + } { + t.Run(tt.name, func(t *testing.T) { + now := func() time.Time { return time.Unix(1000, 0) } + + ti := newTestInfra(t).WithOpenShiftClusters().WithSubscriptions().WithMaintenanceManifests(now) + defer ti.done() + + ti.fixture.AddSubscriptionDocuments(&api.SubscriptionDocument{ + ID: mockSubID, + Subscription: &api.Subscription{ + State: api.SubscriptionStateRegistered, + Properties: &api.SubscriptionProperties{ + TenantID: mockTenantID, + }, + }, + }) + + if tt.fixtures != nil { + tt.fixtures(ti.fixture) + } + + err := ti.buildFixtures(nil) + if err != nil { + t.Fatal(err) + } + + f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.dbGroup, api.APIs, &noop.Noop{}, &noop.Noop{}, testdatabase.NewFakeAEAD(), nil, nil, nil, nil, nil) + + if err != nil { + t.Fatal(err) + } + + go f.Run(ctx, nil, nil) + + if tt.limit == 0 { + tt.limit = 100 + } + + fmt.Printf("limit: %d", tt.limit) + + resp, b, err := ti.request(http.MethodGet, + fmt.Sprintf("https://server/admin%s/maintenancemanifests?limit=%d", resourceID, tt.limit), + http.Header{ + "Referer": []string{"https://mockrefererhost/"}, + }, nil) + if err != nil { + t.Fatal(err) + } + + err = validateResponse(resp, b, tt.wantStatusCode, tt.wantError, tt.wantResponse) + if err != nil { + t.Error(err) + } + }) + } +} diff --git a/pkg/frontend/frontend.go b/pkg/frontend/frontend.go index 83e9d97b660..b34daa4406d 100644 --- a/pkg/frontend/frontend.go +++ b/pkg/frontend/frontend.go @@ -48,6 +48,7 @@ type frontendDBs interface { database.DatabaseGroupWithAsyncOperations database.DatabaseGroupWithSubscriptions database.DatabaseGroupWithPlatformWorkloadIdentityRoleSets + database.DatabaseGroupWithMaintenanceManifests } type kubeActionsFactory func(*logrus.Entry, env.Interface, *api.OpenShiftCluster) (adminactions.KubeActions, error) @@ -326,6 +327,9 @@ func (f *frontend) chiAuthenticatedRoutes(router chi.Router) { r.With(f.maintenanceMiddleware.UnplannedMaintenanceSignal).Post("/etcdcertificaterenew", f.postAdminOpenShiftClusterEtcdCertificateRenew) r.With(f.maintenanceMiddleware.UnplannedMaintenanceSignal).Post("/deletemanagedresource", f.postAdminOpenShiftDeleteManagedResource) + + // MIMO + r.Get("/maintenancemanifests", f.getAdminMaintManifests) }) }) From ca2c62f5091af09e9b992090eb319969b38e2dc3 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Thu, 18 Jul 2024 17:28:01 +1000 Subject: [PATCH 07/67] update with get --- pkg/frontend/admin_mimo_get.go | 65 ++++++++++++ pkg/frontend/admin_mimo_get_test.go | 152 ++++++++++++++++++++++++++++ pkg/frontend/frontend.go | 3 + 3 files changed, 220 insertions(+) create mode 100644 pkg/frontend/admin_mimo_get.go create mode 100644 pkg/frontend/admin_mimo_get_test.go diff --git a/pkg/frontend/admin_mimo_get.go b/pkg/frontend/admin_mimo_get.go new file mode 100644 index 00000000000..f5fd1764930 --- /dev/null +++ b/pkg/frontend/admin_mimo_get.go @@ -0,0 +1,65 @@ +package frontend + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "encoding/json" + "net/http" + "path/filepath" + "strings" + + "github.com/go-chi/chi/v5" + "github.com/sirupsen/logrus" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/api/admin" + "github.com/Azure/ARO-RP/pkg/frontend/middleware" +) + +func (f *frontend) getSingleAdminMaintManifest(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + log := ctx.Value(middleware.ContextKeyLog).(*logrus.Entry) + resourceID := strings.TrimPrefix(filepath.Dir(r.URL.Path), "/admin") + b, err := f._getSingleAdminMaintManifest(ctx, r, resourceID) + + if cloudErr, ok := err.(*api.CloudError); ok { + api.WriteCloudError(w, cloudErr) + return + } + + adminReply(log, w, nil, b, err) +} + +func (f *frontend) _getSingleAdminMaintManifest(ctx context.Context, r *http.Request, resourceID string) ([]byte, error) { + manifestId := chi.URLParam(r, "manifestId") + + converter := f.apis[admin.APIVersion].MaintenanceManifestConverter + + dbOpenShiftClusters, err := f.dbGroup.OpenShiftClusters() + if err != nil { + return nil, api.NewCloudError(http.StatusInternalServerError, api.CloudErrorCodeInternalServerError, "", err.Error()) + } + + dbMaintenanceManifests, err := f.dbGroup.MaintenanceManifests() + if err != nil { + return nil, api.NewCloudError(http.StatusInternalServerError, api.CloudErrorCodeInternalServerError, "", err.Error()) + } + + doc, err := dbOpenShiftClusters.Get(ctx, resourceID) + if err != nil { + return nil, api.NewCloudError(http.StatusNotFound, api.CloudErrorCodeNotFound, "", "cluster not found") + } + + if doc.OpenShiftCluster.Properties.ProvisioningState == api.ProvisioningStateDeleting { + return nil, api.NewCloudError(http.StatusNotFound, api.CloudErrorCodeNotFound, "", "cluster being deleted") + } + + manifest, err := dbMaintenanceManifests.Get(ctx, resourceID, manifestId) + if err != nil { + return nil, api.NewCloudError(http.StatusNotFound, api.CloudErrorCodeInternalServerError, "", err.Error()) + } + + return json.MarshalIndent(converter.ToExternal(manifest), "", " ") +} diff --git a/pkg/frontend/admin_mimo_get_test.go b/pkg/frontend/admin_mimo_get_test.go new file mode 100644 index 00000000000..5960ffd8c52 --- /dev/null +++ b/pkg/frontend/admin_mimo_get_test.go @@ -0,0 +1,152 @@ +package frontend + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "fmt" + "net/http" + "strings" + "testing" + "time" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/api/admin" + "github.com/Azure/ARO-RP/pkg/metrics/noop" + testdatabase "github.com/Azure/ARO-RP/test/database" +) + +func TestMIMOGet(t *testing.T) { + mockSubID := "00000000-0000-0000-0000-000000000000" + mockTenantID := "00000000-0000-0000-0000-000000000000" + resourceID := fmt.Sprintf("/subscriptions/%s/resourcegroups/resourceGroup/providers/Microsoft.RedHatOpenShift/openShiftClusters/resourceName", mockSubID) + ctx := context.Background() + + type test struct { + name string + fixtures func(f *testdatabase.Fixture) + wantStatusCode int + wantResponse *admin.MaintenanceManifest + wantError string + } + + for _, tt := range []*test{ + // { + // name: "no cluster", + // wantError: "404: NotFound: : cluster not found", + // fixtures: func(f *testdatabase.Fixture) {}, + // wantStatusCode: http.StatusNotFound, + // }, + { + name: "cluster being deleted", + fixtures: func(f *testdatabase.Fixture) { + f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{ + Key: strings.ToLower(resourceID), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: resourceID, + Name: "resourceName", + Type: "Microsoft.RedHatOpenShift/openshiftClusters", + Properties: api.OpenShiftClusterProperties{ + ProvisioningState: api.ProvisioningStateDeleting, + }, + }, + }) + }, + wantError: "404: NotFound: : cluster being deleted", + wantStatusCode: http.StatusNotFound, + }, + { + name: "no item", + fixtures: func(f *testdatabase.Fixture) { + f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{ + Key: strings.ToLower(resourceID), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: resourceID, + Name: "resourceName", + Type: "Microsoft.RedHatOpenShift/openshiftClusters", + }, + }) + }, + wantError: "404: NotFound: : cluster not found", + wantStatusCode: http.StatusNotFound, + }, + { + name: "get entry", + fixtures: func(f *testdatabase.Fixture) { + f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{ + Key: strings.ToLower(resourceID), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: resourceID, + Name: "resourceName", + Type: "Microsoft.RedHatOpenShift/openshiftClusters", + }, + }) + f.AddMaintenanceManifestDocuments(&api.MaintenanceManifestDocument{ + ClusterResourceID: strings.ToLower(resourceID), + MaintenanceManifest: &api.MaintenanceManifest{ + MaintenanceSetID: "exampleset", + State: api.MaintenanceManifestStatePending, + RunAfter: 1, + RunBefore: 1, + }, + }) + }, + wantResponse: &admin.MaintenanceManifest{ + ID: "07070707-0707-0707-0707-070707070001", + MaintenanceSetID: "exampleset", + State: admin.MaintenanceManifestStatePending, + Priority: 0, + RunAfter: 1, + RunBefore: 1, + }, + wantStatusCode: http.StatusOK, + }, + } { + t.Run(tt.name, func(t *testing.T) { + now := func() time.Time { return time.Unix(1000, 0) } + + ti := newTestInfra(t).WithOpenShiftClusters().WithSubscriptions().WithMaintenanceManifests(now) + defer ti.done() + + ti.fixture.AddSubscriptionDocuments(&api.SubscriptionDocument{ + ID: mockSubID, + Subscription: &api.Subscription{ + State: api.SubscriptionStateRegistered, + Properties: &api.SubscriptionProperties{ + TenantID: mockTenantID, + }, + }, + }) + + if tt.fixtures != nil { + tt.fixtures(ti.fixture) + } + + err := ti.buildFixtures(nil) + if err != nil { + t.Fatal(err) + } + + f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.dbGroup, api.APIs, &noop.Noop{}, &noop.Noop{}, testdatabase.NewFakeAEAD(), nil, nil, nil, nil, nil) + + if err != nil { + t.Fatal(err) + } + + go f.Run(ctx, nil, nil) + + resp, b, err := ti.request(http.MethodGet, + fmt.Sprintf("https://server/admin%s/maintenancemanifests/07070707-0707-0707-0707-070707070001", resourceID), + nil, nil) + if err != nil { + t.Fatal(err) + } + + err = validateResponse(resp, b, tt.wantStatusCode, tt.wantError, tt.wantResponse) + if err != nil { + t.Error(err) + } + }) + } +} diff --git a/pkg/frontend/frontend.go b/pkg/frontend/frontend.go index b34daa4406d..3871bb5b988 100644 --- a/pkg/frontend/frontend.go +++ b/pkg/frontend/frontend.go @@ -330,6 +330,9 @@ func (f *frontend) chiAuthenticatedRoutes(router chi.Router) { // MIMO r.Get("/maintenancemanifests", f.getAdminMaintManifests) + r.Route("/maintenancemanifests/{manifestId}", func(r chi.Router) { + r.Get("/", f.getSingleAdminMaintManifest) + }) }) }) From 95e95d8804ff3b9e7eb13d627e46c153469bc41e Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Fri, 19 Jul 2024 16:42:06 +1000 Subject: [PATCH 08/67] test fixes --- pkg/frontend/admin_mimo_get.go | 9 ++++----- pkg/frontend/admin_mimo_get_test.go | 23 +++++++++++++---------- pkg/frontend/admin_mimo_list.go | 6 ++---- pkg/frontend/admin_mimo_list_test.go | 8 ++------ pkg/frontend/frontend.go | 21 +++++++++++++++++++++ test/database/maintenancemanifests.go | 6 ++++++ 6 files changed, 48 insertions(+), 25 deletions(-) diff --git a/pkg/frontend/admin_mimo_get.go b/pkg/frontend/admin_mimo_get.go index f5fd1764930..d8de769cc7e 100644 --- a/pkg/frontend/admin_mimo_get.go +++ b/pkg/frontend/admin_mimo_get.go @@ -6,9 +6,8 @@ package frontend import ( "context" "encoding/json" + "fmt" "net/http" - "path/filepath" - "strings" "github.com/go-chi/chi/v5" "github.com/sirupsen/logrus" @@ -21,7 +20,7 @@ import ( func (f *frontend) getSingleAdminMaintManifest(w http.ResponseWriter, r *http.Request) { ctx := r.Context() log := ctx.Value(middleware.ContextKeyLog).(*logrus.Entry) - resourceID := strings.TrimPrefix(filepath.Dir(r.URL.Path), "/admin") + resourceID := resourceIdFromURLParams(r) b, err := f._getSingleAdminMaintManifest(ctx, r, resourceID) if cloudErr, ok := err.(*api.CloudError); ok { @@ -49,7 +48,7 @@ func (f *frontend) _getSingleAdminMaintManifest(ctx context.Context, r *http.Req doc, err := dbOpenShiftClusters.Get(ctx, resourceID) if err != nil { - return nil, api.NewCloudError(http.StatusNotFound, api.CloudErrorCodeNotFound, "", "cluster not found") + return nil, api.NewCloudError(http.StatusNotFound, api.CloudErrorCodeNotFound, "", fmt.Sprintf("cluster not found: %s", err.Error())) } if doc.OpenShiftCluster.Properties.ProvisioningState == api.ProvisioningStateDeleting { @@ -58,7 +57,7 @@ func (f *frontend) _getSingleAdminMaintManifest(ctx context.Context, r *http.Req manifest, err := dbMaintenanceManifests.Get(ctx, resourceID, manifestId) if err != nil { - return nil, api.NewCloudError(http.StatusNotFound, api.CloudErrorCodeInternalServerError, "", err.Error()) + return nil, api.NewCloudError(http.StatusNotFound, api.CloudErrorCodeNotFound, "", fmt.Sprintf("manifest not found: %s", err.Error())) } return json.MarshalIndent(converter.ToExternal(manifest), "", " ") diff --git a/pkg/frontend/admin_mimo_get_test.go b/pkg/frontend/admin_mimo_get_test.go index 5960ffd8c52..9e3ed80e368 100644 --- a/pkg/frontend/admin_mimo_get_test.go +++ b/pkg/frontend/admin_mimo_get_test.go @@ -32,12 +32,12 @@ func TestMIMOGet(t *testing.T) { } for _, tt := range []*test{ - // { - // name: "no cluster", - // wantError: "404: NotFound: : cluster not found", - // fixtures: func(f *testdatabase.Fixture) {}, - // wantStatusCode: http.StatusNotFound, - // }, + { + name: "no cluster", + wantError: "404: NotFound: : cluster not found: 404 : ", + fixtures: func(f *testdatabase.Fixture) {}, + wantStatusCode: http.StatusNotFound, + }, { name: "cluster being deleted", fixtures: func(f *testdatabase.Fixture) { @@ -68,7 +68,7 @@ func TestMIMOGet(t *testing.T) { }, }) }, - wantError: "404: NotFound: : cluster not found", + wantError: "404: NotFound: : manifest not found: 404 : ", wantStatusCode: http.StatusNotFound, }, { @@ -119,15 +119,18 @@ func TestMIMOGet(t *testing.T) { }, }) - if tt.fixtures != nil { - tt.fixtures(ti.fixture) + err := ti.buildFixtures(tt.fixtures) + if err != nil { + t.Fatal(err) } - err := ti.buildFixtures(nil) + a, err := ti.openShiftClustersClient.ListAll(ctx, nil) if err != nil { t.Fatal(err) } + fmt.Print(a) + f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.dbGroup, api.APIs, &noop.Noop{}, &noop.Noop{}, testdatabase.NewFakeAEAD(), nil, nil, nil, nil, nil) if err != nil { diff --git a/pkg/frontend/admin_mimo_list.go b/pkg/frontend/admin_mimo_list.go index ba1d039e131..48c222e8534 100644 --- a/pkg/frontend/admin_mimo_list.go +++ b/pkg/frontend/admin_mimo_list.go @@ -9,9 +9,7 @@ import ( "fmt" "math" "net/http" - "path/filepath" "strconv" - "strings" "github.com/sirupsen/logrus" @@ -23,7 +21,7 @@ import ( func (f *frontend) getAdminMaintManifests(w http.ResponseWriter, r *http.Request) { ctx := r.Context() log := ctx.Value(middleware.ContextKeyLog).(*logrus.Entry) - resourceID := strings.TrimPrefix(filepath.Dir(r.URL.Path), "/admin") + resourceID := resourceIdFromURLParams(r) b, err := f._getAdminMaintManifests(ctx, r, resourceID) if cloudErr, ok := err.(*api.CloudError); ok { @@ -55,7 +53,7 @@ func (f *frontend) _getAdminMaintManifests(ctx context.Context, r *http.Request, doc, err := dbOpenShiftClusters.Get(ctx, resourceID) if err != nil { - return nil, api.NewCloudError(http.StatusNotFound, api.CloudErrorCodeNotFound, "", "cluster not found") + return nil, api.NewCloudError(http.StatusNotFound, api.CloudErrorCodeNotFound, "", fmt.Sprintf("cluster not found: %s", err.Error())) } if doc.OpenShiftCluster.Properties.ProvisioningState == api.ProvisioningStateDeleting { diff --git a/pkg/frontend/admin_mimo_list_test.go b/pkg/frontend/admin_mimo_list_test.go index 0da37580329..4a557008333 100644 --- a/pkg/frontend/admin_mimo_list_test.go +++ b/pkg/frontend/admin_mimo_list_test.go @@ -153,7 +153,7 @@ func TestMIMOList(t *testing.T) { }, { name: "missing cluster", - wantError: "404: NotFound: : cluster not found", + wantError: "404: NotFound: : cluster not found: 404 : ", wantStatusCode: http.StatusNotFound, }, } { @@ -173,11 +173,7 @@ func TestMIMOList(t *testing.T) { }, }) - if tt.fixtures != nil { - tt.fixtures(ti.fixture) - } - - err := ti.buildFixtures(nil) + err := ti.buildFixtures(tt.fixtures) if err != nil { t.Fatal(err) } diff --git a/pkg/frontend/frontend.go b/pkg/frontend/frontend.go index 3871bb5b988..438d27b8ae9 100644 --- a/pkg/frontend/frontend.go +++ b/pkg/frontend/frontend.go @@ -10,6 +10,7 @@ import ( "log" "net" "net/http" + "strings" "sync" "sync/atomic" "time" @@ -33,6 +34,7 @@ import ( "github.com/Azure/ARO-RP/pkg/util/heartbeat" utillog "github.com/Azure/ARO-RP/pkg/util/log" "github.com/Azure/ARO-RP/pkg/util/recover" + "github.com/Azure/go-autorest/autorest/azure" ) type statusCodeError int @@ -332,6 +334,7 @@ func (f *frontend) chiAuthenticatedRoutes(router chi.Router) { r.Get("/maintenancemanifests", f.getAdminMaintManifests) r.Route("/maintenancemanifests/{manifestId}", func(r chi.Router) { r.Get("/", f.getSingleAdminMaintManifest) + r.Post("/cancel", f.postAdminMaintManifestCancel) }) }) }) @@ -500,3 +503,21 @@ func frontendOperationResultLog(log *logrus.Entry, method string, err error) { log = log.WithField("errorDetails", err.Error()) log.Info("front end operation failed") } + +// resourceIdFromURLParams returns an Azure Resource ID built out of the +// individual parameters of the URL. +func resourceIdFromURLParams(r *http.Request) string { + subID, resType, resProvider, resName, resGroupName := chi.URLParam(r, "subscriptionId"), + chi.URLParam(r, "resourceType"), + chi.URLParam(r, "resourceProviderNamespace"), + chi.URLParam(r, "resourceName"), + chi.URLParam(r, "resourceGroupName") + + return strings.ToLower(azure.Resource{ + SubscriptionID: subID, + ResourceGroup: resGroupName, + ResourceType: resType, + ResourceName: resName, + Provider: resProvider, + }.String()) +} diff --git a/test/database/maintenancemanifests.go b/test/database/maintenancemanifests.go index e6e511214ab..e20387253d9 100644 --- a/test/database/maintenancemanifests.go +++ b/test/database/maintenancemanifests.go @@ -4,7 +4,9 @@ package database // Licensed under the Apache License 2.0. import ( + "cmp" "context" + "slices" "strconv" "time" @@ -72,6 +74,10 @@ func fakeMaintenanceManifestsForCluster(client cosmosdb.MaintenanceManifestDocum } } + slices.SortFunc(results, func(a, b *api.MaintenanceManifestDocument) int { + return cmp.Compare(a.ID, b.ID) + }) + return cosmosdb.NewFakeMaintenanceManifestDocumentIterator(results, startingIndex) } From a55e649124fa6b5fe49b3d54bb058dcf67883370 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Fri, 19 Jul 2024 16:42:13 +1000 Subject: [PATCH 09/67] cancellation tests and impl --- pkg/api/admin/mimo.go | 1 + pkg/api/mimo.go | 1 + pkg/frontend/admin_mimo_cancel.go | 79 ++++++++++ pkg/frontend/admin_mimo_cancel_test.go | 206 +++++++++++++++++++++++++ 4 files changed, 287 insertions(+) create mode 100644 pkg/frontend/admin_mimo_cancel.go create mode 100644 pkg/frontend/admin_mimo_cancel_test.go diff --git a/pkg/api/admin/mimo.go b/pkg/api/admin/mimo.go index 6aaf6953e0e..18fbae98358 100644 --- a/pkg/api/admin/mimo.go +++ b/pkg/api/admin/mimo.go @@ -11,6 +11,7 @@ const ( MaintenanceManifestStateCompleted MaintenanceManifestState = "Completed" MaintenanceManifestStateFailed MaintenanceManifestState = "Failed" MaintenanceManifestStateTimedOut MaintenanceManifestState = "TimedOut" + MaintenanceManifestStateCancelled MaintenanceManifestState = "Cancelled" ) type MaintenanceManifest struct { diff --git a/pkg/api/mimo.go b/pkg/api/mimo.go index a3ffddf0804..d2616407ad2 100644 --- a/pkg/api/mimo.go +++ b/pkg/api/mimo.go @@ -11,6 +11,7 @@ const ( MaintenanceManifestStateCompleted MaintenanceManifestState = "Completed" MaintenanceManifestStateFailed MaintenanceManifestState = "Failed" MaintenanceManifestStateTimedOut MaintenanceManifestState = "TimedOut" + MaintenanceManifestStateCancelled MaintenanceManifestState = "Cancelled" ) type MaintenanceSet struct { diff --git a/pkg/frontend/admin_mimo_cancel.go b/pkg/frontend/admin_mimo_cancel.go new file mode 100644 index 00000000000..a453a4824cc --- /dev/null +++ b/pkg/frontend/admin_mimo_cancel.go @@ -0,0 +1,79 @@ +package frontend + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + + "github.com/go-chi/chi/v5" + "github.com/sirupsen/logrus" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/api/admin" + "github.com/Azure/ARO-RP/pkg/database/cosmosdb" + "github.com/Azure/ARO-RP/pkg/frontend/middleware" +) + +func (f *frontend) postAdminMaintManifestCancel(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + log := ctx.Value(middleware.ContextKeyLog).(*logrus.Entry) + resourceID := resourceIdFromURLParams(r) + b, err := f._postAdminMaintManifestCancel(ctx, r, resourceID) + + if cloudErr, ok := err.(*api.CloudError); ok { + api.WriteCloudError(w, cloudErr) + return + } + + adminReply(log, w, nil, b, err) +} + +func (f *frontend) _postAdminMaintManifestCancel(ctx context.Context, r *http.Request, resourceID string) ([]byte, error) { + manifestId := chi.URLParam(r, "manifestId") + + converter := f.apis[admin.APIVersion].MaintenanceManifestConverter + + dbOpenShiftClusters, err := f.dbGroup.OpenShiftClusters() + if err != nil { + return nil, api.NewCloudError(http.StatusInternalServerError, api.CloudErrorCodeInternalServerError, "", err.Error()) + } + + dbMaintenanceManifests, err := f.dbGroup.MaintenanceManifests() + if err != nil { + return nil, api.NewCloudError(http.StatusInternalServerError, api.CloudErrorCodeInternalServerError, "", err.Error()) + } + + doc, err := dbOpenShiftClusters.Get(ctx, resourceID) + if err != nil { + return nil, api.NewCloudError(http.StatusNotFound, api.CloudErrorCodeNotFound, "", fmt.Sprintf("cluster not found: %s", err.Error())) + } + + if doc.OpenShiftCluster.Properties.ProvisioningState == api.ProvisioningStateDeleting { + return nil, api.NewCloudError(http.StatusNotFound, api.CloudErrorCodeNotFound, "", "cluster being deleted") + } + + modifiedDoc, err := dbMaintenanceManifests.Patch(ctx, resourceID, manifestId, func(mmd *api.MaintenanceManifestDocument) error { + if mmd.MaintenanceManifest.State != api.MaintenanceManifestStatePending { + return api.NewCloudError(http.StatusNotAcceptable, api.CloudErrorCodePropertyChangeNotAllowed, "", fmt.Sprintf("cannot cancel task in state %s", mmd.MaintenanceManifest.State)) + } + + mmd.MaintenanceManifest.State = api.MaintenanceManifestStateCancelled + return nil + }) + if err != nil { + cloudErr, ok := err.(*api.CloudError) + if ok { + return nil, cloudErr + } else if cosmosdb.IsErrorStatusCode(err, http.StatusNotFound) { + return nil, api.NewCloudError(http.StatusNotFound, api.CloudErrorCodeNotFound, "", fmt.Sprintf("manifest not found: %s", err.Error())) + } else { + return nil, api.NewCloudError(http.StatusInternalServerError, api.CloudErrorCodeInternalServerError, "", err.Error()) + } + } + + return json.MarshalIndent(converter.ToExternal(modifiedDoc), "", " ") +} diff --git a/pkg/frontend/admin_mimo_cancel_test.go b/pkg/frontend/admin_mimo_cancel_test.go new file mode 100644 index 00000000000..fc7327f1c52 --- /dev/null +++ b/pkg/frontend/admin_mimo_cancel_test.go @@ -0,0 +1,206 @@ +package frontend + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "fmt" + "net/http" + "strings" + "testing" + "time" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/api/admin" + "github.com/Azure/ARO-RP/pkg/metrics/noop" + testdatabase "github.com/Azure/ARO-RP/test/database" +) + +func TestMIMOCancelTask(t *testing.T) { + mockSubID := "00000000-0000-0000-0000-000000000000" + mockTenantID := "00000000-0000-0000-0000-000000000000" + resourceID := fmt.Sprintf("/subscriptions/%s/resourcegroups/resourceGroup/providers/Microsoft.RedHatOpenShift/openShiftClusters/resourceName", mockSubID) + ctx := context.Background() + + type test struct { + name string + fixtures func(f *testdatabase.Fixture) + wantStatusCode int + wantResponse *admin.MaintenanceManifest + wantResult func(f *testdatabase.Checker) + wantError string + } + + for _, tt := range []*test{ + { + name: "no cluster", + wantError: "404: NotFound: : cluster not found: 404 : ", + fixtures: func(f *testdatabase.Fixture) {}, + wantStatusCode: http.StatusNotFound, + }, + { + name: "cluster being deleted", + fixtures: func(f *testdatabase.Fixture) { + f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{ + Key: strings.ToLower(resourceID), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: resourceID, + Name: "resourceName", + Type: "Microsoft.RedHatOpenShift/openshiftClusters", + Properties: api.OpenShiftClusterProperties{ + ProvisioningState: api.ProvisioningStateDeleting, + }, + }, + }) + }, + wantError: "404: NotFound: : cluster being deleted", + wantStatusCode: http.StatusNotFound, + }, + { + name: "no item", + fixtures: func(f *testdatabase.Fixture) { + f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{ + Key: strings.ToLower(resourceID), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: resourceID, + Name: "resourceName", + Type: "Microsoft.RedHatOpenShift/openshiftClusters", + }, + }) + }, + wantError: "404: NotFound: : manifest not found: 404 : ", + wantStatusCode: http.StatusNotFound, + }, + { + name: "cancel pending", + fixtures: func(f *testdatabase.Fixture) { + f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{ + Key: strings.ToLower(resourceID), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: resourceID, + Name: "resourceName", + Type: "Microsoft.RedHatOpenShift/openshiftClusters", + }, + }) + f.AddMaintenanceManifestDocuments(&api.MaintenanceManifestDocument{ + ClusterResourceID: strings.ToLower(resourceID), + MaintenanceManifest: &api.MaintenanceManifest{ + MaintenanceSetID: "exampleset", + State: api.MaintenanceManifestStatePending, + RunAfter: 1, + RunBefore: 1, + }, + }) + }, + wantResult: func(c *testdatabase.Checker) { + c.AddMaintenanceManifestDocuments(&api.MaintenanceManifestDocument{ + ID: "07070707-0707-0707-0707-070707070001", + ClusterResourceID: strings.ToLower(resourceID), + MaintenanceManifest: &api.MaintenanceManifest{ + MaintenanceSetID: "exampleset", + State: api.MaintenanceManifestStateCancelled, + RunAfter: 1, + RunBefore: 1, + }, + }) + }, + wantResponse: &admin.MaintenanceManifest{ + ID: "07070707-0707-0707-0707-070707070001", + MaintenanceSetID: "exampleset", + State: admin.MaintenanceManifestStateCancelled, + Priority: 0, + RunAfter: 1, + RunBefore: 1, + }, + wantStatusCode: http.StatusOK, + }, + { + name: "cannot cancel failed", + fixtures: func(f *testdatabase.Fixture) { + f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{ + Key: strings.ToLower(resourceID), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: resourceID, + Name: "resourceName", + Type: "Microsoft.RedHatOpenShift/openshiftClusters", + }, + }) + f.AddMaintenanceManifestDocuments(&api.MaintenanceManifestDocument{ + ClusterResourceID: strings.ToLower(resourceID), + MaintenanceManifest: &api.MaintenanceManifest{ + MaintenanceSetID: "exampleset", + State: api.MaintenanceManifestStateFailed, + RunAfter: 1, + RunBefore: 1, + }, + }) + }, + wantResult: func(c *testdatabase.Checker) { + c.AddMaintenanceManifestDocuments(&api.MaintenanceManifestDocument{ + ID: "07070707-0707-0707-0707-070707070001", + ClusterResourceID: strings.ToLower(resourceID), + MaintenanceManifest: &api.MaintenanceManifest{ + MaintenanceSetID: "exampleset", + State: api.MaintenanceManifestStateFailed, + RunAfter: 1, + RunBefore: 1, + }, + }) + }, + wantError: "406: PropertyChangeNotAllowed: : cannot cancel task in state Failed", + wantStatusCode: http.StatusNotAcceptable, + }, + } { + t.Run(tt.name, func(t *testing.T) { + now := func() time.Time { return time.Unix(1000, 0) } + + ti := newTestInfra(t).WithOpenShiftClusters().WithSubscriptions().WithMaintenanceManifests(now) + defer ti.done() + + ti.fixture.AddSubscriptionDocuments(&api.SubscriptionDocument{ + ID: mockSubID, + Subscription: &api.Subscription{ + State: api.SubscriptionStateRegistered, + Properties: &api.SubscriptionProperties{ + TenantID: mockTenantID, + }, + }, + }) + + err := ti.buildFixtures(tt.fixtures) + if err != nil { + t.Fatal(err) + } + + if tt.wantResult != nil { + tt.wantResult(ti.checker) + } + + f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.dbGroup, api.APIs, &noop.Noop{}, &noop.Noop{}, testdatabase.NewFakeAEAD(), nil, nil, nil, nil, nil) + + if err != nil { + t.Fatal(err) + } + + go f.Run(ctx, nil, nil) + + resp, b, err := ti.request(http.MethodPost, + fmt.Sprintf("https://server/admin%s/maintenancemanifests/07070707-0707-0707-0707-070707070001/cancel", resourceID), + nil, nil) + if err != nil { + t.Fatal(err) + } + + err = validateResponse(resp, b, tt.wantStatusCode, tt.wantError, tt.wantResponse) + if err != nil { + t.Error(err) + } + + for _, err := range ti.checker.CheckMaintenanceManifests(ti.maintenanceManifestsClient) { + t.Error(err) + } + + }) + } +} From a75fcd5a548779c8fa3015607f37310f666b39e0 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Mon, 22 Jul 2024 14:16:18 +1000 Subject: [PATCH 10/67] renaming and tweaking --- ...admin_mimo_cancel.go => admin_mimo_manifest_cancel.go} | 0 ..._cancel_test.go => admin_mimo_manifest_cancel_test.go} | 8 ++++---- .../{admin_mimo_get.go => admin_mimo_manifest_get.go} | 0 ...n_mimo_get_test.go => admin_mimo_manifest_get_test.go} | 2 +- .../{admin_mimo_list.go => admin_mimo_manifest_list.go} | 0 ...mimo_list_test.go => admin_mimo_manifest_list_test.go} | 6 +++--- 6 files changed, 8 insertions(+), 8 deletions(-) rename pkg/frontend/{admin_mimo_cancel.go => admin_mimo_manifest_cancel.go} (100%) rename pkg/frontend/{admin_mimo_cancel_test.go => admin_mimo_manifest_cancel_test.go} (96%) rename pkg/frontend/{admin_mimo_get.go => admin_mimo_manifest_get.go} (100%) rename pkg/frontend/{admin_mimo_get_test.go => admin_mimo_manifest_get_test.go} (98%) rename pkg/frontend/{admin_mimo_list.go => admin_mimo_manifest_list.go} (100%) rename pkg/frontend/{admin_mimo_list_test.go => admin_mimo_manifest_list_test.go} (97%) diff --git a/pkg/frontend/admin_mimo_cancel.go b/pkg/frontend/admin_mimo_manifest_cancel.go similarity index 100% rename from pkg/frontend/admin_mimo_cancel.go rename to pkg/frontend/admin_mimo_manifest_cancel.go diff --git a/pkg/frontend/admin_mimo_cancel_test.go b/pkg/frontend/admin_mimo_manifest_cancel_test.go similarity index 96% rename from pkg/frontend/admin_mimo_cancel_test.go rename to pkg/frontend/admin_mimo_manifest_cancel_test.go index fc7327f1c52..44d6a591fe7 100644 --- a/pkg/frontend/admin_mimo_cancel_test.go +++ b/pkg/frontend/admin_mimo_manifest_cancel_test.go @@ -85,7 +85,7 @@ func TestMIMOCancelTask(t *testing.T) { }) f.AddMaintenanceManifestDocuments(&api.MaintenanceManifestDocument{ ClusterResourceID: strings.ToLower(resourceID), - MaintenanceManifest: &api.MaintenanceManifest{ + MaintenanceManifest: api.MaintenanceManifest{ MaintenanceSetID: "exampleset", State: api.MaintenanceManifestStatePending, RunAfter: 1, @@ -97,7 +97,7 @@ func TestMIMOCancelTask(t *testing.T) { c.AddMaintenanceManifestDocuments(&api.MaintenanceManifestDocument{ ID: "07070707-0707-0707-0707-070707070001", ClusterResourceID: strings.ToLower(resourceID), - MaintenanceManifest: &api.MaintenanceManifest{ + MaintenanceManifest: api.MaintenanceManifest{ MaintenanceSetID: "exampleset", State: api.MaintenanceManifestStateCancelled, RunAfter: 1, @@ -128,7 +128,7 @@ func TestMIMOCancelTask(t *testing.T) { }) f.AddMaintenanceManifestDocuments(&api.MaintenanceManifestDocument{ ClusterResourceID: strings.ToLower(resourceID), - MaintenanceManifest: &api.MaintenanceManifest{ + MaintenanceManifest: api.MaintenanceManifest{ MaintenanceSetID: "exampleset", State: api.MaintenanceManifestStateFailed, RunAfter: 1, @@ -140,7 +140,7 @@ func TestMIMOCancelTask(t *testing.T) { c.AddMaintenanceManifestDocuments(&api.MaintenanceManifestDocument{ ID: "07070707-0707-0707-0707-070707070001", ClusterResourceID: strings.ToLower(resourceID), - MaintenanceManifest: &api.MaintenanceManifest{ + MaintenanceManifest: api.MaintenanceManifest{ MaintenanceSetID: "exampleset", State: api.MaintenanceManifestStateFailed, RunAfter: 1, diff --git a/pkg/frontend/admin_mimo_get.go b/pkg/frontend/admin_mimo_manifest_get.go similarity index 100% rename from pkg/frontend/admin_mimo_get.go rename to pkg/frontend/admin_mimo_manifest_get.go diff --git a/pkg/frontend/admin_mimo_get_test.go b/pkg/frontend/admin_mimo_manifest_get_test.go similarity index 98% rename from pkg/frontend/admin_mimo_get_test.go rename to pkg/frontend/admin_mimo_manifest_get_test.go index 9e3ed80e368..9b7e6a84265 100644 --- a/pkg/frontend/admin_mimo_get_test.go +++ b/pkg/frontend/admin_mimo_manifest_get_test.go @@ -84,7 +84,7 @@ func TestMIMOGet(t *testing.T) { }) f.AddMaintenanceManifestDocuments(&api.MaintenanceManifestDocument{ ClusterResourceID: strings.ToLower(resourceID), - MaintenanceManifest: &api.MaintenanceManifest{ + MaintenanceManifest: api.MaintenanceManifest{ MaintenanceSetID: "exampleset", State: api.MaintenanceManifestStatePending, RunAfter: 1, diff --git a/pkg/frontend/admin_mimo_list.go b/pkg/frontend/admin_mimo_manifest_list.go similarity index 100% rename from pkg/frontend/admin_mimo_list.go rename to pkg/frontend/admin_mimo_manifest_list.go diff --git a/pkg/frontend/admin_mimo_list_test.go b/pkg/frontend/admin_mimo_manifest_list_test.go similarity index 97% rename from pkg/frontend/admin_mimo_list_test.go rename to pkg/frontend/admin_mimo_manifest_list_test.go index 4a557008333..46379ae66c1 100644 --- a/pkg/frontend/admin_mimo_list_test.go +++ b/pkg/frontend/admin_mimo_manifest_list_test.go @@ -65,7 +65,7 @@ func TestMIMOList(t *testing.T) { }) f.AddMaintenanceManifestDocuments(&api.MaintenanceManifestDocument{ ClusterResourceID: strings.ToLower(resourceID), - MaintenanceManifest: &api.MaintenanceManifest{ + MaintenanceManifest: api.MaintenanceManifest{ MaintenanceSetID: "exampleset", State: api.MaintenanceManifestStatePending, RunAfter: 1, @@ -101,7 +101,7 @@ func TestMIMOList(t *testing.T) { }) f.AddMaintenanceManifestDocuments(&api.MaintenanceManifestDocument{ ClusterResourceID: strings.ToLower(resourceID), - MaintenanceManifest: &api.MaintenanceManifest{ + MaintenanceManifest: api.MaintenanceManifest{ MaintenanceSetID: "exampleset", State: api.MaintenanceManifestStatePending, RunAfter: 1, @@ -110,7 +110,7 @@ func TestMIMOList(t *testing.T) { }) f.AddMaintenanceManifestDocuments(&api.MaintenanceManifestDocument{ ClusterResourceID: strings.ToLower(resourceID), - MaintenanceManifest: &api.MaintenanceManifest{ + MaintenanceManifest: api.MaintenanceManifest{ MaintenanceSetID: "exampleset2", State: api.MaintenanceManifestStatePending, RunAfter: 1, From c1743a994b4070c6c13945457ecaaf74039faf62 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Mon, 22 Jul 2024 14:16:31 +1000 Subject: [PATCH 11/67] static validation --- pkg/api/admin/mimo_convert.go | 17 +++++++- pkg/api/admin/mimo_validatestatic.go | 59 ++++++++++++++++++++++++++++ pkg/api/admin/register.go | 1 + pkg/api/mimodocument.go | 4 +- pkg/api/register.go | 6 +++ pkg/mimo/actuator/actuator_test.go | 20 +++++----- pkg/mimo/actuator/service_test.go | 8 ++-- pkg/mimo/tasks/example/task_test.go | 2 +- 8 files changed, 99 insertions(+), 18 deletions(-) create mode 100644 pkg/api/admin/mimo_validatestatic.go diff --git a/pkg/api/admin/mimo_convert.go b/pkg/api/admin/mimo_convert.go index a357bfc69de..6046a81ccc7 100644 --- a/pkg/api/admin/mimo_convert.go +++ b/pkg/api/admin/mimo_convert.go @@ -3,7 +3,9 @@ package admin // Copyright (c) Microsoft Corporation. // Licensed under the Apache License 2.0. -import "github.com/Azure/ARO-RP/pkg/api" +import ( + "github.com/Azure/ARO-RP/pkg/api" +) type maintenanceManifestConverter struct{} @@ -34,3 +36,16 @@ func (m maintenanceManifestConverter) ToExternalList(docs []*api.MaintenanceMani return l } + +func (m maintenanceManifestConverter) ToInternal(_i interface{}, out *api.MaintenanceManifestDocument) { + + i := _i.(*MaintenanceManifest) + + out.ID = i.ID + out.MaintenanceManifest.MaintenanceSetID = i.MaintenanceSetID + out.MaintenanceManifest.Priority = i.Priority + out.MaintenanceManifest.RunAfter = i.RunAfter + out.MaintenanceManifest.RunBefore = i.RunBefore + out.MaintenanceManifest.State = api.MaintenanceManifestState(i.State) + out.MaintenanceManifest.StatusText = i.StatusText +} diff --git a/pkg/api/admin/mimo_validatestatic.go b/pkg/api/admin/mimo_validatestatic.go new file mode 100644 index 00000000000..0193a4404b1 --- /dev/null +++ b/pkg/api/admin/mimo_validatestatic.go @@ -0,0 +1,59 @@ +package admin + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "net/http" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/api/util/immutable" +) + +type maintenanceManifestStaticValidator struct{} + +// Validate validates an OpenShift cluster +func (sv maintenanceManifestStaticValidator) Static(_new interface{}, _current *api.MaintenanceManifestDocument) error { + new := _new.(*MaintenanceManifest) + + var current *MaintenanceManifest + if _current != nil { + current = (&maintenanceManifestConverter{}).ToExternal(_current).(*MaintenanceManifest) + } + + err := sv.validate(new) + if err != nil { + return err + } + + if current == nil { + return nil + } + + return sv.validateDelta(new, current) +} + +func (sv maintenanceManifestStaticValidator) validate(new *MaintenanceManifest) error { + if new.MaintenanceSetID == "" { + return api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidParameter, "maintenanceSetID", "Must be provided") + } + + if new.RunAfter == 0 { + return api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidParameter, "runAfter", "Must be provided") + } + + if new.RunBefore == 0 { + return api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidParameter, "runBefore", "Must be provided") + } + + return nil +} + +func (sv maintenanceManifestStaticValidator) validateDelta(new, current *MaintenanceManifest) error { + err := immutable.Validate("", new, current) + if err != nil { + err := err.(*immutable.ValidationError) + return api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodePropertyChangeNotAllowed, err.Target, err.Message) + } + return nil +} diff --git a/pkg/api/admin/register.go b/pkg/api/admin/register.go index e24558fffa3..028f7dd2bcb 100644 --- a/pkg/api/admin/register.go +++ b/pkg/api/admin/register.go @@ -19,5 +19,6 @@ func init() { PlatformWorkloadIdentityRoleSetConverter: platformWorkloadIdentityRoleSetConverter{}, PlatformWorkloadIdentityRoleSetStaticValidator: platformWorkloadIdentityRoleSetStaticValidator{}, MaintenanceManifestConverter: maintenanceManifestConverter{}, + MaintenanceManifestStaticValidator: maintenanceManifestStaticValidator{}, } } diff --git a/pkg/api/mimodocument.go b/pkg/api/mimodocument.go index 099d14e2a30..141b75f412c 100644 --- a/pkg/api/mimodocument.go +++ b/pkg/api/mimodocument.go @@ -26,8 +26,8 @@ type MaintenanceManifestDocument struct { LSN int `json:"_lsn,omitempty"` Metadata map[string]interface{} `json:"_metadata,omitempty"` - ClusterResourceID string `json:"clusterResourceID,omitempty"` - MaintenanceManifest *MaintenanceManifest `json:"maintenanceManifest,omitempty"` + ClusterResourceID string `json:"clusterResourceID,omitempty"` + MaintenanceManifest MaintenanceManifest `json:"maintenanceManifest,omitempty"` LeaseOwner string `json:"leaseOwner,omitempty" deep:"-"` LeaseExpires int `json:"leaseExpires,omitempty" deep:"-"` diff --git a/pkg/api/register.go b/pkg/api/register.go index 9b50b26f8bd..59fa1ebcda8 100644 --- a/pkg/api/register.go +++ b/pkg/api/register.go @@ -74,6 +74,11 @@ type SecretConverter interface { type MaintenanceManifestConverter interface { ToExternal(*MaintenanceManifestDocument) interface{} ToExternalList([]*MaintenanceManifestDocument, string) interface{} + ToInternal(interface{}, *MaintenanceManifestDocument) +} + +type MaintenanceManifestStaticValidator interface { + Static(interface{}, *MaintenanceManifestDocument) error } // Version is a set of endpoints implemented by each API version @@ -93,6 +98,7 @@ type Version struct { SecretConverter SecretConverter ClusterManagerStaticValidator ClusterManagerStaticValidator MaintenanceManifestConverter MaintenanceManifestConverter + MaintenanceManifestStaticValidator MaintenanceManifestStaticValidator } // APIs is the map of registered API versions diff --git a/pkg/mimo/actuator/actuator_test.go b/pkg/mimo/actuator/actuator_test.go index a1dd3dcf680..f4afe08ab3c 100644 --- a/pkg/mimo/actuator/actuator_test.go +++ b/pkg/mimo/actuator/actuator_test.go @@ -110,7 +110,7 @@ var _ = Describe("MIMO Actuator", Ordered, func() { fixtures.AddMaintenanceManifestDocuments(&api.MaintenanceManifestDocument{ ID: manifestID, ClusterResourceID: strings.ToLower(clusterResourceID), - MaintenanceManifest: &api.MaintenanceManifest{ + MaintenanceManifest: api.MaintenanceManifest{ State: api.MaintenanceManifestStatePending, RunBefore: 60, RunAfter: 0, @@ -121,7 +121,7 @@ var _ = Describe("MIMO Actuator", Ordered, func() { checker.AddMaintenanceManifestDocuments(&api.MaintenanceManifestDocument{ ID: manifestID, ClusterResourceID: strings.ToLower(clusterResourceID), - MaintenanceManifest: &api.MaintenanceManifest{ + MaintenanceManifest: api.MaintenanceManifest{ State: api.MaintenanceManifestStateTimedOut, StatusText: "timed out at 1970-01-01 00:02:00 +0000 UTC", RunBefore: 60, @@ -156,7 +156,7 @@ var _ = Describe("MIMO Actuator", Ordered, func() { fixtures.AddMaintenanceManifestDocuments(&api.MaintenanceManifestDocument{ ID: manifestID, ClusterResourceID: strings.ToLower(clusterResourceID), - MaintenanceManifest: &api.MaintenanceManifest{ + MaintenanceManifest: api.MaintenanceManifest{ State: api.MaintenanceManifestStatePending, MaintenanceSetID: "0", RunBefore: 600, @@ -168,7 +168,7 @@ var _ = Describe("MIMO Actuator", Ordered, func() { checker.AddMaintenanceManifestDocuments(&api.MaintenanceManifestDocument{ ID: manifestID, ClusterResourceID: strings.ToLower(clusterResourceID), - MaintenanceManifest: &api.MaintenanceManifest{ + MaintenanceManifest: api.MaintenanceManifest{ State: api.MaintenanceManifestStateCompleted, MaintenanceSetID: "0", StatusText: "done", @@ -208,7 +208,7 @@ var _ = Describe("MIMO Actuator", Ordered, func() { fixtures.AddMaintenanceManifestDocuments(&api.MaintenanceManifestDocument{ ID: manifestIDs[0], ClusterResourceID: strings.ToLower(clusterResourceID), - MaintenanceManifest: &api.MaintenanceManifest{ + MaintenanceManifest: api.MaintenanceManifest{ State: api.MaintenanceManifestStatePending, MaintenanceSetID: "0", RunBefore: 600, @@ -219,7 +219,7 @@ var _ = Describe("MIMO Actuator", Ordered, func() { &api.MaintenanceManifestDocument{ ID: manifestIDs[1], ClusterResourceID: strings.ToLower(clusterResourceID), - MaintenanceManifest: &api.MaintenanceManifest{ + MaintenanceManifest: api.MaintenanceManifest{ State: api.MaintenanceManifestStatePending, MaintenanceSetID: "1", RunBefore: 600, @@ -230,7 +230,7 @@ var _ = Describe("MIMO Actuator", Ordered, func() { &api.MaintenanceManifestDocument{ ID: manifestIDs[2], ClusterResourceID: strings.ToLower(clusterResourceID), - MaintenanceManifest: &api.MaintenanceManifest{ + MaintenanceManifest: api.MaintenanceManifest{ State: api.MaintenanceManifestStatePending, MaintenanceSetID: "2", RunBefore: 600, @@ -243,7 +243,7 @@ var _ = Describe("MIMO Actuator", Ordered, func() { checker.AddMaintenanceManifestDocuments(&api.MaintenanceManifestDocument{ ID: manifestIDs[0], ClusterResourceID: strings.ToLower(clusterResourceID), - MaintenanceManifest: &api.MaintenanceManifest{ + MaintenanceManifest: api.MaintenanceManifest{ State: api.MaintenanceManifestStateCompleted, MaintenanceSetID: "0", StatusText: "done", @@ -255,7 +255,7 @@ var _ = Describe("MIMO Actuator", Ordered, func() { &api.MaintenanceManifestDocument{ ID: manifestIDs[1], ClusterResourceID: strings.ToLower(clusterResourceID), - MaintenanceManifest: &api.MaintenanceManifest{ + MaintenanceManifest: api.MaintenanceManifest{ State: api.MaintenanceManifestStateCompleted, MaintenanceSetID: "1", StatusText: "done", @@ -267,7 +267,7 @@ var _ = Describe("MIMO Actuator", Ordered, func() { &api.MaintenanceManifestDocument{ ID: manifestIDs[2], ClusterResourceID: strings.ToLower(clusterResourceID), - MaintenanceManifest: &api.MaintenanceManifest{ + MaintenanceManifest: api.MaintenanceManifest{ State: api.MaintenanceManifestStateCompleted, MaintenanceSetID: "2", StatusText: "done", diff --git a/pkg/mimo/actuator/service_test.go b/pkg/mimo/actuator/service_test.go index 9613bb0c402..320715fdafa 100644 --- a/pkg/mimo/actuator/service_test.go +++ b/pkg/mimo/actuator/service_test.go @@ -164,7 +164,7 @@ var _ = Describe("MIMO Actuator Service", Ordered, func() { &api.MaintenanceManifestDocument{ ID: manifestID, ClusterResourceID: strings.ToLower(clusterResourceID), - MaintenanceManifest: &api.MaintenanceManifest{ + MaintenanceManifest: api.MaintenanceManifest{ State: api.MaintenanceManifestStatePending, RunBefore: 60, RunAfter: 0, @@ -173,7 +173,7 @@ var _ = Describe("MIMO Actuator Service", Ordered, func() { &api.MaintenanceManifestDocument{ ID: manifestID2, ClusterResourceID: strings.ToLower(clusterResourceID), - MaintenanceManifest: &api.MaintenanceManifest{ + MaintenanceManifest: api.MaintenanceManifest{ State: api.MaintenanceManifestStatePending, RunBefore: 300, RunAfter: 0, @@ -186,7 +186,7 @@ var _ = Describe("MIMO Actuator Service", Ordered, func() { &api.MaintenanceManifestDocument{ ID: manifestID, ClusterResourceID: strings.ToLower(clusterResourceID), - MaintenanceManifest: &api.MaintenanceManifest{ + MaintenanceManifest: api.MaintenanceManifest{ State: api.MaintenanceManifestStateTimedOut, StatusText: "timed out at 1970-01-01 00:02:00 +0000 UTC", RunBefore: 60, @@ -196,7 +196,7 @@ var _ = Describe("MIMO Actuator Service", Ordered, func() { &api.MaintenanceManifestDocument{ ID: manifestID2, ClusterResourceID: strings.ToLower(clusterResourceID), - MaintenanceManifest: &api.MaintenanceManifest{ + MaintenanceManifest: api.MaintenanceManifest{ State: api.MaintenanceManifestStateCompleted, StatusText: "ok", RunBefore: 300, diff --git a/pkg/mimo/tasks/example/task_test.go b/pkg/mimo/tasks/example/task_test.go index 055d3ec9a8f..ab5f3a8f95f 100644 --- a/pkg/mimo/tasks/example/task_test.go +++ b/pkg/mimo/tasks/example/task_test.go @@ -48,7 +48,7 @@ func TestTask(t *testing.T) { manifest := &api.MaintenanceManifestDocument{ ClusterResourceID: strings.ToLower(clusterResourceID), - MaintenanceManifest: &api.MaintenanceManifest{ + MaintenanceManifest: api.MaintenanceManifest{ State: api.MaintenanceManifestStatePending, RunBefore: 60, RunAfter: 0, From 505a210a1a500adaa8f5d4ae0e033f0bdc8e4b4c Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Mon, 22 Jul 2024 14:16:39 +1000 Subject: [PATCH 12/67] update frontend to have create --- pkg/frontend/admin_mimo_manifest_create.go | 99 ++++++++++ .../admin_mimo_manifest_create_test.go | 171 ++++++++++++++++++ pkg/frontend/frontend.go | 11 +- 3 files changed, 277 insertions(+), 4 deletions(-) create mode 100644 pkg/frontend/admin_mimo_manifest_create.go create mode 100644 pkg/frontend/admin_mimo_manifest_create_test.go diff --git a/pkg/frontend/admin_mimo_manifest_create.go b/pkg/frontend/admin_mimo_manifest_create.go new file mode 100644 index 00000000000..cc238d896da --- /dev/null +++ b/pkg/frontend/admin_mimo_manifest_create.go @@ -0,0 +1,99 @@ +package frontend + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "time" + + "github.com/sirupsen/logrus" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/api/admin" + "github.com/Azure/ARO-RP/pkg/frontend/middleware" +) + +func (f *frontend) putAdminMaintManifestCreate(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + log := ctx.Value(middleware.ContextKeyLog).(*logrus.Entry) + resourceID := resourceIdFromURLParams(r) + b, err := f._putAdminMaintManifestCreate(ctx, r, resourceID) + + if cloudErr, ok := err.(*api.CloudError); ok { + api.WriteCloudError(w, cloudErr) + return + } + + err = statusCodeError(http.StatusCreated) + adminReply(log, w, nil, b, err) +} + +func (f *frontend) _putAdminMaintManifestCreate(ctx context.Context, r *http.Request, resourceID string) ([]byte, error) { + converter := f.apis[admin.APIVersion].MaintenanceManifestConverter + validator := f.apis[admin.APIVersion].MaintenanceManifestStaticValidator + + body := r.Context().Value(middleware.ContextKeyBody).([]byte) + if len(body) == 0 || !json.Valid(body) { + return nil, api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidRequestContent, "", "The request content was invalid and could not be deserialized.") + } + + dbOpenShiftClusters, err := f.dbGroup.OpenShiftClusters() + if err != nil { + return nil, api.NewCloudError(http.StatusInternalServerError, api.CloudErrorCodeInternalServerError, "", err.Error()) + } + + dbMaintenanceManifests, err := f.dbGroup.MaintenanceManifests() + if err != nil { + return nil, api.NewCloudError(http.StatusInternalServerError, api.CloudErrorCodeInternalServerError, "", err.Error()) + } + + doc, err := dbOpenShiftClusters.Get(ctx, resourceID) + if err != nil { + return nil, api.NewCloudError(http.StatusNotFound, api.CloudErrorCodeNotFound, "", fmt.Sprintf("cluster not found: %s", err.Error())) + } + + if doc.OpenShiftCluster.Properties.ProvisioningState == api.ProvisioningStateDeleting { + return nil, api.NewCloudError(http.StatusNotFound, api.CloudErrorCodeNotFound, "", "cluster being deleted") + } + + var ext *admin.MaintenanceManifest + err = json.Unmarshal(body, &ext) + if err != nil { + return nil, api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidRequestContent, "", "The request content could not be deserialized: "+err.Error()) + } + + // fill in some sefaults + ext.ID = dbMaintenanceManifests.NewUUID() + ext.State = admin.MaintenanceManifestStatePending + + if ext.RunAfter == 0 { + ext.RunAfter = int(f.now().Unix()) + } + + // add a 7d timeout by default + if ext.RunBefore == 0 { + ext.RunBefore = int(f.now().Add(time.Hour * 7 * 24).Unix()) + } + + err = validator.Static(ext, nil) + if err != nil { + return nil, err + } + + manifestDoc := &api.MaintenanceManifestDocument{ + ClusterResourceID: resourceID, + } + converter.ToInternal(ext, manifestDoc) + + savedDoc, err := dbMaintenanceManifests.Create(ctx, manifestDoc) + if err != nil { + return nil, api.NewCloudError(http.StatusInternalServerError, api.CloudErrorCodeInternalServerError, "", err.Error()) + + } + + return json.MarshalIndent(converter.ToExternal(savedDoc), "", " ") +} diff --git a/pkg/frontend/admin_mimo_manifest_create_test.go b/pkg/frontend/admin_mimo_manifest_create_test.go new file mode 100644 index 00000000000..ac739adc698 --- /dev/null +++ b/pkg/frontend/admin_mimo_manifest_create_test.go @@ -0,0 +1,171 @@ +package frontend + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "fmt" + "net/http" + "strings" + "testing" + "time" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/api/admin" + "github.com/Azure/ARO-RP/pkg/metrics/noop" + testdatabase "github.com/Azure/ARO-RP/test/database" +) + +func TestMIMOCreateManifest(t *testing.T) { + mockSubID := "00000000-0000-0000-0000-000000000000" + mockTenantID := "00000000-0000-0000-0000-000000000000" + resourceID := fmt.Sprintf("/subscriptions/%s/resourcegroups/resourceGroup/providers/Microsoft.RedHatOpenShift/openShiftClusters/resourceName", mockSubID) + ctx := context.Background() + + type test struct { + name string + fixtures func(f *testdatabase.Fixture) + body *admin.MaintenanceManifest + wantStatusCode int + wantResponse *admin.MaintenanceManifest + wantResult func(f *testdatabase.Checker) + wantError string + } + + for _, tt := range []*test{ + { + name: "no cluster", + wantError: "404: NotFound: : cluster not found: 404 : ", + fixtures: func(f *testdatabase.Fixture) {}, + wantStatusCode: http.StatusNotFound, + }, + { + name: "cluster being deleted", + fixtures: func(f *testdatabase.Fixture) { + f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{ + Key: strings.ToLower(resourceID), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: resourceID, + Name: "resourceName", + Type: "Microsoft.RedHatOpenShift/openshiftClusters", + Properties: api.OpenShiftClusterProperties{ + ProvisioningState: api.ProvisioningStateDeleting, + }, + }, + }) + }, + wantError: "404: NotFound: : cluster being deleted", + wantStatusCode: http.StatusNotFound, + }, + { + name: "invalid", + fixtures: func(f *testdatabase.Fixture) { + f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{ + Key: strings.ToLower(resourceID), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: resourceID, + Name: "resourceName", + Type: "Microsoft.RedHatOpenShift/openshiftClusters", + }, + }) + }, + body: &admin.MaintenanceManifest{}, + wantError: "400: InvalidParameter: maintenanceSetID: Must be provided", + wantStatusCode: http.StatusBadRequest, + }, + + { + name: "good", + fixtures: func(f *testdatabase.Fixture) { + f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{ + Key: strings.ToLower(resourceID), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: resourceID, + Name: "resourceName", + Type: "Microsoft.RedHatOpenShift/openshiftClusters", + }, + }) + }, + body: &admin.MaintenanceManifest{ + MaintenanceSetID: "exampleset", + State: admin.MaintenanceManifestStatePending, + RunAfter: 1, + RunBefore: 1, + }, + wantResult: func(c *testdatabase.Checker) { + c.AddMaintenanceManifestDocuments(&api.MaintenanceManifestDocument{ + ID: "07070707-0707-0707-0707-070707070001", + ClusterResourceID: strings.ToLower(resourceID), + MaintenanceManifest: api.MaintenanceManifest{ + MaintenanceSetID: "exampleset", + State: api.MaintenanceManifestStatePending, + RunAfter: 1, + RunBefore: 1, + }, + }) + }, + wantResponse: &admin.MaintenanceManifest{ + ID: "07070707-0707-0707-0707-070707070001", + MaintenanceSetID: "exampleset", + State: admin.MaintenanceManifestStatePending, + RunAfter: 1, + RunBefore: 1, + }, + wantStatusCode: http.StatusCreated, + }, + } { + t.Run(tt.name, func(t *testing.T) { + now := func() time.Time { return time.Unix(1000, 0) } + + ti := newTestInfra(t).WithOpenShiftClusters().WithSubscriptions().WithMaintenanceManifests(now) + defer ti.done() + + ti.fixture.AddSubscriptionDocuments(&api.SubscriptionDocument{ + ID: mockSubID, + Subscription: &api.Subscription{ + State: api.SubscriptionStateRegistered, + Properties: &api.SubscriptionProperties{ + TenantID: mockTenantID, + }, + }, + }) + + err := ti.buildFixtures(tt.fixtures) + if err != nil { + t.Fatal(err) + } + + if tt.wantResult != nil { + tt.wantResult(ti.checker) + } + + f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.dbGroup, api.APIs, &noop.Noop{}, &noop.Noop{}, testdatabase.NewFakeAEAD(), nil, nil, nil, nil, nil) + + if err != nil { + t.Fatal(err) + } + + go f.Run(ctx, nil, nil) + + resp, b, err := ti.request(http.MethodPut, + fmt.Sprintf("https://server/admin%s/maintenancemanifests", resourceID), + http.Header{ + "Content-Type": []string{"application/json"}, + }, tt.body) + if err != nil { + t.Fatal(err) + } + + err = validateResponse(resp, b, tt.wantStatusCode, tt.wantError, tt.wantResponse) + if err != nil { + t.Error(err) + } + + for _, err := range ti.checker.CheckMaintenanceManifests(ti.maintenanceManifestsClient) { + t.Error(err) + } + + }) + } +} diff --git a/pkg/frontend/frontend.go b/pkg/frontend/frontend.go index 438d27b8ae9..d4773d182bd 100644 --- a/pkg/frontend/frontend.go +++ b/pkg/frontend/frontend.go @@ -331,10 +331,13 @@ func (f *frontend) chiAuthenticatedRoutes(router chi.Router) { r.With(f.maintenanceMiddleware.UnplannedMaintenanceSignal).Post("/deletemanagedresource", f.postAdminOpenShiftDeleteManagedResource) // MIMO - r.Get("/maintenancemanifests", f.getAdminMaintManifests) - r.Route("/maintenancemanifests/{manifestId}", func(r chi.Router) { - r.Get("/", f.getSingleAdminMaintManifest) - r.Post("/cancel", f.postAdminMaintManifestCancel) + r.Route("/maintenancemanifests", func(r chi.Router) { + r.Get("/", f.getAdminMaintManifests) + r.Put("/", f.putAdminMaintManifestCreate) + r.Route("/{manifestId}", func(r chi.Router) { + r.Get("/", f.getSingleAdminMaintManifest) + r.Post("/cancel", f.postAdminMaintManifestCancel) + }) }) }) }) From ea27dd1a6a8d3e7e3d609af0f2d60bf750f5c6d7 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Mon, 22 Jul 2024 17:39:33 +1000 Subject: [PATCH 13/67] code for deleting --- pkg/database/mimo.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pkg/database/mimo.go b/pkg/database/mimo.go index 582965c98cb..117346ff3a7 100644 --- a/pkg/database/mimo.go +++ b/pkg/database/mimo.go @@ -38,6 +38,7 @@ type MaintenanceManifests interface { Lease(ctx context.Context, clusterResourceID string, id string) (*api.MaintenanceManifestDocument, error) EndLease(context.Context, string, string, api.MaintenanceManifestState, *string) (*api.MaintenanceManifestDocument, error) Get(context.Context, string, string) (*api.MaintenanceManifestDocument, error) + Delete(context.Context, string, string) error NewUUID() string } @@ -229,3 +230,11 @@ func (c *maintenanceManifests) Lease(ctx context.Context, clusterResourceID stri return nil }, &cosmosdb.Options{PreTriggers: []string{"renewLease"}}) } + +func (c *maintenanceManifests) Delete(ctx context.Context, clusterResourceID string, id string) error { + if clusterResourceID != strings.ToLower(clusterResourceID) { + return fmt.Errorf("clusterID %q is not lower case", clusterResourceID) + } + + return c.c.Delete(ctx, clusterResourceID, &api.MaintenanceManifestDocument{ID: id}, nil) +} From 488a7c7687f7baa38a90f80ba00a9d3ac778920a Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Mon, 22 Jul 2024 17:39:44 +1000 Subject: [PATCH 14/67] add deleting endpoint --- .../admin_mimo_manifest_cancel_test.go | 2 +- pkg/frontend/admin_mimo_manifest_delete.go | 69 ++++++++ .../admin_mimo_manifest_delete_test.go | 152 ++++++++++++++++++ pkg/frontend/admin_mimo_manifest_list_test.go | 2 +- pkg/frontend/frontend.go | 1 + 5 files changed, 224 insertions(+), 2 deletions(-) create mode 100644 pkg/frontend/admin_mimo_manifest_delete.go create mode 100644 pkg/frontend/admin_mimo_manifest_delete_test.go diff --git a/pkg/frontend/admin_mimo_manifest_cancel_test.go b/pkg/frontend/admin_mimo_manifest_cancel_test.go index 44d6a591fe7..d65c5401a2e 100644 --- a/pkg/frontend/admin_mimo_manifest_cancel_test.go +++ b/pkg/frontend/admin_mimo_manifest_cancel_test.go @@ -17,7 +17,7 @@ import ( testdatabase "github.com/Azure/ARO-RP/test/database" ) -func TestMIMOCancelTask(t *testing.T) { +func TestMIMOCancelManifest(t *testing.T) { mockSubID := "00000000-0000-0000-0000-000000000000" mockTenantID := "00000000-0000-0000-0000-000000000000" resourceID := fmt.Sprintf("/subscriptions/%s/resourcegroups/resourceGroup/providers/Microsoft.RedHatOpenShift/openShiftClusters/resourceName", mockSubID) diff --git a/pkg/frontend/admin_mimo_manifest_delete.go b/pkg/frontend/admin_mimo_manifest_delete.go new file mode 100644 index 00000000000..73e62a46b11 --- /dev/null +++ b/pkg/frontend/admin_mimo_manifest_delete.go @@ -0,0 +1,69 @@ +package frontend + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + + "github.com/go-chi/chi/v5" + "github.com/sirupsen/logrus" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/database/cosmosdb" + "github.com/Azure/ARO-RP/pkg/frontend/middleware" +) + +func (f *frontend) deleteAdminMaintManifest(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + log := ctx.Value(middleware.ContextKeyLog).(*logrus.Entry) + resourceID := resourceIdFromURLParams(r) + b, err := f._deleteAdminMaintManifest(ctx, r, resourceID) + + if cloudErr, ok := err.(*api.CloudError); ok { + api.WriteCloudError(w, cloudErr) + return + } + + adminReply(log, w, nil, b, err) +} + +func (f *frontend) _deleteAdminMaintManifest(ctx context.Context, r *http.Request, resourceID string) ([]byte, error) { + manifestId := chi.URLParam(r, "manifestId") + + dbOpenShiftClusters, err := f.dbGroup.OpenShiftClusters() + if err != nil { + return nil, api.NewCloudError(http.StatusInternalServerError, api.CloudErrorCodeInternalServerError, "", err.Error()) + } + + dbMaintenanceManifests, err := f.dbGroup.MaintenanceManifests() + if err != nil { + return nil, api.NewCloudError(http.StatusInternalServerError, api.CloudErrorCodeInternalServerError, "", err.Error()) + } + + doc, err := dbOpenShiftClusters.Get(ctx, resourceID) + if err != nil { + return nil, api.NewCloudError(http.StatusNotFound, api.CloudErrorCodeNotFound, "", fmt.Sprintf("cluster not found: %s", err.Error())) + } + + if doc.OpenShiftCluster.Properties.ProvisioningState == api.ProvisioningStateDeleting { + return nil, api.NewCloudError(http.StatusNotFound, api.CloudErrorCodeNotFound, "", "cluster being deleted") + } + + err = dbMaintenanceManifests.Delete(ctx, resourceID, manifestId) + if err != nil { + cloudErr, ok := err.(*api.CloudError) + if ok { + return nil, cloudErr + } else if cosmosdb.IsErrorStatusCode(err, http.StatusNotFound) { + return nil, api.NewCloudError(http.StatusNotFound, api.CloudErrorCodeNotFound, "", fmt.Sprintf("manifest not found: %s", err.Error())) + } else { + return nil, api.NewCloudError(http.StatusInternalServerError, api.CloudErrorCodeInternalServerError, "", err.Error()) + } + } + + return json.MarshalIndent(map[string]string{}, "", " ") +} diff --git a/pkg/frontend/admin_mimo_manifest_delete_test.go b/pkg/frontend/admin_mimo_manifest_delete_test.go new file mode 100644 index 00000000000..53d427846fa --- /dev/null +++ b/pkg/frontend/admin_mimo_manifest_delete_test.go @@ -0,0 +1,152 @@ +package frontend + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "fmt" + "net/http" + "strings" + "testing" + "time" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/metrics/noop" + testdatabase "github.com/Azure/ARO-RP/test/database" +) + +func TestMIMODeleteManifest(t *testing.T) { + mockSubID := "00000000-0000-0000-0000-000000000000" + mockTenantID := "00000000-0000-0000-0000-000000000000" + resourceID := fmt.Sprintf("/subscriptions/%s/resourcegroups/resourceGroup/providers/Microsoft.RedHatOpenShift/openShiftClusters/resourceName", mockSubID) + ctx := context.Background() + + type test struct { + name string + fixtures func(f *testdatabase.Fixture) + wantStatusCode int + wantResponse []byte + wantResult func(f *testdatabase.Checker) + wantError string + } + + for _, tt := range []*test{ + { + name: "no cluster", + wantError: "404: NotFound: : cluster not found: 404 : ", + fixtures: func(f *testdatabase.Fixture) {}, + wantStatusCode: http.StatusNotFound, + }, + { + name: "cluster being deleted", + fixtures: func(f *testdatabase.Fixture) { + f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{ + Key: strings.ToLower(resourceID), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: resourceID, + Name: "resourceName", + Type: "Microsoft.RedHatOpenShift/openshiftClusters", + Properties: api.OpenShiftClusterProperties{ + ProvisioningState: api.ProvisioningStateDeleting, + }, + }, + }) + }, + wantError: "404: NotFound: : cluster being deleted", + wantStatusCode: http.StatusNotFound, + }, + { + name: "no item", + fixtures: func(f *testdatabase.Fixture) { + f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{ + Key: strings.ToLower(resourceID), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: resourceID, + Name: "resourceName", + Type: "Microsoft.RedHatOpenShift/openshiftClusters", + }, + }) + }, + wantError: "404: NotFound: : manifest not found: 404 : ", + wantStatusCode: http.StatusNotFound, + }, + { + name: "delete successfully", + fixtures: func(f *testdatabase.Fixture) { + f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{ + Key: strings.ToLower(resourceID), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: resourceID, + Name: "resourceName", + Type: "Microsoft.RedHatOpenShift/openshiftClusters", + }, + }) + f.AddMaintenanceManifestDocuments(&api.MaintenanceManifestDocument{ + ClusterResourceID: strings.ToLower(resourceID), + MaintenanceManifest: api.MaintenanceManifest{ + MaintenanceSetID: "exampleset", + State: api.MaintenanceManifestStatePending, + RunAfter: 1, + RunBefore: 1, + }, + }) + }, + wantResult: func(c *testdatabase.Checker) { + }, + wantResponse: []byte("{}\n"), + wantStatusCode: http.StatusOK, + }, + } { + t.Run(tt.name, func(t *testing.T) { + now := func() time.Time { return time.Unix(1000, 0) } + + ti := newTestInfra(t).WithOpenShiftClusters().WithSubscriptions().WithMaintenanceManifests(now) + defer ti.done() + + ti.fixture.AddSubscriptionDocuments(&api.SubscriptionDocument{ + ID: mockSubID, + Subscription: &api.Subscription{ + State: api.SubscriptionStateRegistered, + Properties: &api.SubscriptionProperties{ + TenantID: mockTenantID, + }, + }, + }) + + err := ti.buildFixtures(tt.fixtures) + if err != nil { + t.Fatal(err) + } + + if tt.wantResult != nil { + tt.wantResult(ti.checker) + } + + f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.dbGroup, api.APIs, &noop.Noop{}, &noop.Noop{}, testdatabase.NewFakeAEAD(), nil, nil, nil, nil, nil) + + if err != nil { + t.Fatal(err) + } + + go f.Run(ctx, nil, nil) + + resp, b, err := ti.request(http.MethodDelete, + fmt.Sprintf("https://server/admin%s/maintenancemanifests/07070707-0707-0707-0707-070707070001", resourceID), + nil, nil) + if err != nil { + t.Fatal(err) + } + + err = validateResponse(resp, b, tt.wantStatusCode, tt.wantError, tt.wantResponse) + if err != nil { + t.Error(err) + } + + for _, err := range ti.checker.CheckMaintenanceManifests(ti.maintenanceManifestsClient) { + t.Error(err) + } + + }) + } +} diff --git a/pkg/frontend/admin_mimo_manifest_list_test.go b/pkg/frontend/admin_mimo_manifest_list_test.go index 46379ae66c1..c82ebd51102 100644 --- a/pkg/frontend/admin_mimo_manifest_list_test.go +++ b/pkg/frontend/admin_mimo_manifest_list_test.go @@ -19,7 +19,7 @@ import ( testdatabase "github.com/Azure/ARO-RP/test/database" ) -func TestMIMOList(t *testing.T) { +func TestMIMOListManifests(t *testing.T) { mockSubID := "00000000-0000-0000-0000-000000000000" mockTenantID := "00000000-0000-0000-0000-000000000000" resourceID := fmt.Sprintf("/subscriptions/%s/resourcegroups/resourceGroup/providers/Microsoft.RedHatOpenShift/openShiftClusters/resourceName", mockSubID) diff --git a/pkg/frontend/frontend.go b/pkg/frontend/frontend.go index d4773d182bd..f5cdfd857d5 100644 --- a/pkg/frontend/frontend.go +++ b/pkg/frontend/frontend.go @@ -336,6 +336,7 @@ func (f *frontend) chiAuthenticatedRoutes(router chi.Router) { r.Put("/", f.putAdminMaintManifestCreate) r.Route("/{manifestId}", func(r chi.Router) { r.Get("/", f.getSingleAdminMaintManifest) + r.Delete("/", f.deleteAdminMaintManifest) r.Post("/cancel", f.postAdminMaintManifestCancel) }) }) From 41d72f78c7c575620beadbd8d11bdc20ffae815b Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Fri, 26 Jul 2024 13:10:41 +1000 Subject: [PATCH 15/67] move clusteroperator check code for reuse --- pkg/cluster/condition.go | 16 ++--- pkg/cluster/condition_test.go | 53 ---------------- pkg/util/clusteroperators/isavailable.go | 13 ++++ pkg/util/clusteroperators/isavailable_test.go | 61 +++++++++++++++++++ 4 files changed, 79 insertions(+), 64 deletions(-) create mode 100644 pkg/util/clusteroperators/isavailable.go create mode 100644 pkg/util/clusteroperators/isavailable_test.go diff --git a/pkg/cluster/condition.go b/pkg/cluster/condition.go index a5f1fbd9140..6a1744822db 100644 --- a/pkg/cluster/condition.go +++ b/pkg/cluster/condition.go @@ -12,6 +12,8 @@ import ( corev1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/Azure/ARO-RP/pkg/util/clusteroperators" ) const minimumWorkerNodes = 2 @@ -24,7 +26,7 @@ func (m *manager) apiServersReady(ctx context.Context) (bool, error) { if err != nil { return false, nil } - return isOperatorAvailable(apiserver), nil + return clusteroperators.IsOperatorAvailable(apiserver), nil } func (m *manager) minimumWorkerNodesReady(ctx context.Context) (bool, error) { @@ -57,7 +59,7 @@ func (m *manager) operatorConsoleReady(ctx context.Context) (bool, error) { if err != nil { return false, nil } - return isOperatorAvailable(consoleOperator), nil + return clusteroperators.IsOperatorAvailable(consoleOperator), nil } func (m *manager) clusterVersionReady(ctx context.Context) (bool, error) { @@ -77,15 +79,7 @@ func (m *manager) ingressControllerReady(ctx context.Context) (bool, error) { if err != nil { return false, nil } - return isOperatorAvailable(ingressOperator), nil -} - -func isOperatorAvailable(operator *configv1.ClusterOperator) bool { - m := make(map[configv1.ClusterStatusConditionType]configv1.ConditionStatus, len(operator.Status.Conditions)) - for _, cond := range operator.Status.Conditions { - m[cond.Type] = cond.Status - } - return m[configv1.OperatorAvailable] == configv1.ConditionTrue && m[configv1.OperatorProgressing] == configv1.ConditionFalse + return clusteroperators.IsOperatorAvailable(ingressOperator), nil } // aroCredentialsRequestReconciled evaluates whether the openshift-azure-operator CredentialsRequest has recently been reconciled and returns true diff --git a/pkg/cluster/condition_test.go b/pkg/cluster/condition_test.go index a0515659204..c035bce48ca 100644 --- a/pkg/cluster/condition_test.go +++ b/pkg/cluster/condition_test.go @@ -63,59 +63,6 @@ func TestOperatorConsoleExists(t *testing.T) { } } -func TestIsOperatorAvailable(t *testing.T) { - for _, tt := range []struct { - name string - availableCondition configv1.ConditionStatus - progressingCondition configv1.ConditionStatus - want bool - }{ - { - name: "Available && Progressing; not available", - availableCondition: configv1.ConditionTrue, - progressingCondition: configv1.ConditionTrue, - }, - { - name: "Available && !Progressing; available", - availableCondition: configv1.ConditionTrue, - progressingCondition: configv1.ConditionFalse, - want: true, - }, - { - name: "!Available && Progressing; not available", - availableCondition: configv1.ConditionFalse, - progressingCondition: configv1.ConditionTrue, - }, - { - name: "!Available && !Progressing; not available", - availableCondition: configv1.ConditionFalse, - progressingCondition: configv1.ConditionFalse, - }, - } { - operator := &configv1.ClusterOperator{ - ObjectMeta: metav1.ObjectMeta{ - Name: "name", - }, - Status: configv1.ClusterOperatorStatus{ - Conditions: []configv1.ClusterOperatorStatusCondition{ - { - Type: configv1.OperatorAvailable, - Status: tt.availableCondition, - }, - { - Type: configv1.OperatorProgressing, - Status: tt.progressingCondition, - }, - }, - }, - } - available := isOperatorAvailable(operator) - if available != tt.want { - t.Error(available) - } - } -} - func TestMinimumWorkerNodesReady(t *testing.T) { ctx := context.Background() diff --git a/pkg/util/clusteroperators/isavailable.go b/pkg/util/clusteroperators/isavailable.go new file mode 100644 index 00000000000..1af3bf1ba8e --- /dev/null +++ b/pkg/util/clusteroperators/isavailable.go @@ -0,0 +1,13 @@ +package clusteroperators + +import ( + configv1 "github.com/openshift/api/config/v1" +) + +func IsOperatorAvailable(operator *configv1.ClusterOperator) bool { + m := make(map[configv1.ClusterStatusConditionType]configv1.ConditionStatus, len(operator.Status.Conditions)) + for _, cond := range operator.Status.Conditions { + m[cond.Type] = cond.Status + } + return m[configv1.OperatorAvailable] == configv1.ConditionTrue && m[configv1.OperatorProgressing] == configv1.ConditionFalse +} diff --git a/pkg/util/clusteroperators/isavailable_test.go b/pkg/util/clusteroperators/isavailable_test.go new file mode 100644 index 00000000000..4b9189641a7 --- /dev/null +++ b/pkg/util/clusteroperators/isavailable_test.go @@ -0,0 +1,61 @@ +package clusteroperators + +import ( + "testing" + + configv1 "github.com/openshift/api/config/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestIsOperatorAvailable(t *testing.T) { + for _, tt := range []struct { + name string + availableCondition configv1.ConditionStatus + progressingCondition configv1.ConditionStatus + want bool + }{ + { + name: "Available && Progressing; not available", + availableCondition: configv1.ConditionTrue, + progressingCondition: configv1.ConditionTrue, + }, + { + name: "Available && !Progressing; available", + availableCondition: configv1.ConditionTrue, + progressingCondition: configv1.ConditionFalse, + want: true, + }, + { + name: "!Available && Progressing; not available", + availableCondition: configv1.ConditionFalse, + progressingCondition: configv1.ConditionTrue, + }, + { + name: "!Available && !Progressing; not available", + availableCondition: configv1.ConditionFalse, + progressingCondition: configv1.ConditionFalse, + }, + } { + operator := &configv1.ClusterOperator{ + ObjectMeta: metav1.ObjectMeta{ + Name: "name", + }, + Status: configv1.ClusterOperatorStatus{ + Conditions: []configv1.ClusterOperatorStatusCondition{ + { + Type: configv1.OperatorAvailable, + Status: tt.availableCondition, + }, + { + Type: configv1.OperatorProgressing, + Status: tt.progressingCondition, + }, + }, + }, + } + available := IsOperatorAvailable(operator) + if available != tt.want { + t.Error(available) + } + } +} From eaa0f41f31d2643c7b1a318f999767e9ac53f80b Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Fri, 26 Jul 2024 13:11:04 +1000 Subject: [PATCH 16/67] MIMO task/set cleanups --- pkg/mimo/actuator/actuator_test.go | 40 +++++++++--------- pkg/mimo/actuator/manager.go | 37 +++++++++-------- pkg/mimo/actuator/service.go | 10 ++--- pkg/mimo/actuator/service_test.go | 11 ++--- pkg/mimo/actuator/task.go | 64 ++++++++++++++++++++++++++--- pkg/mimo/cmd/cli.go | 6 +-- pkg/mimo/sets/interfaces.go | 11 +++++ pkg/mimo/sets/sets.go | 8 ++++ pkg/mimo/tasks/example/task.go | 18 +++++--- pkg/mimo/tasks/example/task_test.go | 39 ++---------------- pkg/mimo/tasks/interfaces.go | 22 ---------- pkg/util/mimo/errors.go | 37 +++++++++++++++++ pkg/util/mimo/taskcontext.go | 32 +++++++++++++++ test/mimo/tasks/testcontext.go | 31 +++++++++++--- 14 files changed, 242 insertions(+), 124 deletions(-) create mode 100644 pkg/mimo/sets/interfaces.go create mode 100644 pkg/mimo/sets/sets.go delete mode 100644 pkg/mimo/tasks/interfaces.go create mode 100644 pkg/util/mimo/errors.go create mode 100644 pkg/util/mimo/taskcontext.go diff --git a/pkg/mimo/actuator/actuator_test.go b/pkg/mimo/actuator/actuator_test.go index f4afe08ab3c..3da92fdeeb1 100644 --- a/pkg/mimo/actuator/actuator_test.go +++ b/pkg/mimo/actuator/actuator_test.go @@ -20,7 +20,8 @@ import ( "github.com/Azure/ARO-RP/pkg/database" "github.com/Azure/ARO-RP/pkg/database/cosmosdb" "github.com/Azure/ARO-RP/pkg/env" - "github.com/Azure/ARO-RP/pkg/mimo/tasks" + "github.com/Azure/ARO-RP/pkg/mimo/sets" + "github.com/Azure/ARO-RP/pkg/util/mimo" mock_env "github.com/Azure/ARO-RP/pkg/util/mocks/env" testdatabase "github.com/Azure/ARO-RP/test/database" testlog "github.com/Azure/ARO-RP/test/util/log" @@ -84,8 +85,8 @@ var _ = Describe("MIMO Actuator", Ordered, func() { mmf: manifests, oc: clusters, - tasks: map[string]tasks.TaskFunc{}, - now: now, + sets: map[string]sets.MaintenanceSet{}, + now: now, } }) @@ -179,8 +180,10 @@ var _ = Describe("MIMO Actuator", Ordered, func() { }) It("runs them", func() { - a.AddTask("0", func(ctx context.Context, th tasks.TaskContext, mmd *api.MaintenanceManifestDocument, oscd *api.OpenShiftClusterDocument) (api.MaintenanceManifestState, string) { - return api.MaintenanceManifestStateCompleted, "done" + a.AddMaintenanceSets(map[string]sets.MaintenanceSet{ + "0": func(th mimo.TaskContext, mmd *api.MaintenanceManifestDocument, oscd *api.OpenShiftClusterDocument) (api.MaintenanceManifestState, string) { + return api.MaintenanceManifestStateCompleted, "done" + }, }) didWork, err := a.Process(ctx) @@ -279,22 +282,21 @@ var _ = Describe("MIMO Actuator", Ordered, func() { }) It("runs them", func() { - ordering := []string{} - a.AddTask("0", func(ctx context.Context, th tasks.TaskContext, mmd *api.MaintenanceManifestDocument, oscd *api.OpenShiftClusterDocument) (api.MaintenanceManifestState, string) { - ordering = append(ordering, "0") - return api.MaintenanceManifestStateCompleted, "done" - }) - - a.AddTask("1", func(ctx context.Context, th tasks.TaskContext, mmd *api.MaintenanceManifestDocument, oscd *api.OpenShiftClusterDocument) (api.MaintenanceManifestState, string) { - ordering = append(ordering, "1") - return api.MaintenanceManifestStateCompleted, "done" - }) - - a.AddTask("2", func(ctx context.Context, th tasks.TaskContext, mmd *api.MaintenanceManifestDocument, oscd *api.OpenShiftClusterDocument) (api.MaintenanceManifestState, string) { - ordering = append(ordering, "2") - return api.MaintenanceManifestStateCompleted, "done" + a.AddMaintenanceSets(map[string]sets.MaintenanceSet{ + "0": func(th mimo.TaskContext, mmd *api.MaintenanceManifestDocument, oscd *api.OpenShiftClusterDocument) (api.MaintenanceManifestState, string) { + ordering = append(ordering, "0") + return api.MaintenanceManifestStateCompleted, "done" + }, + "1": func(th mimo.TaskContext, mmd *api.MaintenanceManifestDocument, oscd *api.OpenShiftClusterDocument) (api.MaintenanceManifestState, string) { + ordering = append(ordering, "1") + return api.MaintenanceManifestStateCompleted, "done" + }, + "2": func(th mimo.TaskContext, mmd *api.MaintenanceManifestDocument, oscd *api.OpenShiftClusterDocument) (api.MaintenanceManifestState, string) { + ordering = append(ordering, "2") + return api.MaintenanceManifestStateCompleted, "done" + }, }) didWork, err := a.Process(ctx) diff --git a/pkg/mimo/actuator/manager.go b/pkg/mimo/actuator/manager.go index 16531b96d9f..231cdb857d7 100644 --- a/pkg/mimo/actuator/manager.go +++ b/pkg/mimo/actuator/manager.go @@ -17,15 +17,14 @@ import ( "github.com/Azure/ARO-RP/pkg/api" "github.com/Azure/ARO-RP/pkg/database" "github.com/Azure/ARO-RP/pkg/env" - "github.com/Azure/ARO-RP/pkg/mimo/tasks" + "github.com/Azure/ARO-RP/pkg/mimo/sets" ) const maxDequeueCount = 5 type Actuator interface { Process(context.Context) (bool, error) - AddTask(string, tasks.TaskFunc) - AddTasks(map[string]tasks.TaskFunc) + AddMaintenanceSets(map[string]sets.MaintenanceSet) } type actuator struct { @@ -38,7 +37,7 @@ type actuator struct { oc database.OpenShiftClusters mmf database.MaintenanceManifests - tasks map[string]tasks.TaskFunc + sets map[string]sets.MaintenanceSet } func NewActuator( @@ -55,7 +54,7 @@ func NewActuator( clusterResourceID: strings.ToLower(clusterResourceID), oc: oc, mmf: mmf, - tasks: make(map[string]tasks.TaskFunc), + sets: make(map[string]sets.MaintenanceSet), now: now, } @@ -63,12 +62,8 @@ func NewActuator( return a, nil } -func (a *actuator) AddTask(u string, t tasks.TaskFunc) { - a.tasks[u] = t -} - -func (a *actuator) AddTasks(tasks map[string]tasks.TaskFunc) { - maps.Copy(a.tasks, tasks) +func (a *actuator) AddMaintenanceSets(sets map[string]sets.MaintenanceSet) { + maps.Copy(a.sets, sets) } func (a *actuator) Process(ctx context.Context) (bool, error) { @@ -141,12 +136,12 @@ func (a *actuator) Process(ctx context.Context) (bool, error) { return false, err // This will include StatusPreconditionFaileds } - taskContext := newTaskContext(a.env, a.log, oc) + taskContext := newTaskContext(ctx, a.env, a.log, oc) // Execute on the manifests we want to action for _, doc := range manifestsToAction { // here - f, ok := a.tasks[doc.MaintenanceManifest.MaintenanceSetID] + f, ok := a.sets[doc.MaintenanceManifest.MaintenanceSetID] if !ok { a.log.Infof("not found %v", doc.MaintenanceManifest.MaintenanceSetID) continue @@ -155,7 +150,7 @@ func (a *actuator) Process(ctx context.Context) (bool, error) { // Attempt a dequeue doc, err = a.mmf.Lease(ctx, a.clusterResourceID, doc.ID) if err != nil { - // log and continue if it doesn't work + // log and continue to the next task if it doesn't work a.log.Error(err) continue } @@ -170,8 +165,18 @@ func (a *actuator) Process(ctx context.Context) (bool, error) { continue } - // Perform the task - state, msg := f(ctx, taskContext, doc, oc) + var state api.MaintenanceManifestState + var msg string + + // Perform the task with a timeout + err = taskContext.RunInTimeout(time.Minute*60, func() error { + state, msg = f(taskContext, doc, oc) + return taskContext.Err() + }) + if err != nil { + a.log.Error(err) + } + _, err = a.mmf.EndLease(ctx, doc.ClusterResourceID, doc.ID, state, &msg) if err != nil { a.log.Error(err) diff --git a/pkg/mimo/actuator/service.go b/pkg/mimo/actuator/service.go index d588817cd8b..635e55a95f2 100644 --- a/pkg/mimo/actuator/service.go +++ b/pkg/mimo/actuator/service.go @@ -17,7 +17,7 @@ import ( "github.com/Azure/ARO-RP/pkg/database" "github.com/Azure/ARO-RP/pkg/env" "github.com/Azure/ARO-RP/pkg/metrics" - "github.com/Azure/ARO-RP/pkg/mimo/tasks" + "github.com/Azure/ARO-RP/pkg/mimo/sets" "github.com/Azure/ARO-RP/pkg/proxy" "github.com/Azure/ARO-RP/pkg/util/buckets" "github.com/Azure/ARO-RP/pkg/util/heartbeat" @@ -51,7 +51,7 @@ type service struct { pollTime time.Duration now func() time.Time - tasks map[string]tasks.TaskFunc + sets map[string]sets.MaintenanceSet } func NewService(env env.Interface, log *logrus.Entry, dialer proxy.Dialer, dbOpenShiftClusters database.OpenShiftClusters, dbMaintenanceManifests database.MaintenanceManifests, m metrics.Emitter) *service { @@ -77,8 +77,8 @@ func NewService(env env.Interface, log *logrus.Entry, dialer proxy.Dialer, dbOpe return s } -func (s *service) SetTasks(tasks map[string]tasks.TaskFunc) { - s.tasks = tasks +func (s *service) SetMaintenanceSets(sets map[string]sets.MaintenanceSet) { + s.sets = sets } func (s *service) Run(ctx context.Context, stop <-chan struct{}, done chan<- struct{}) error { @@ -207,7 +207,7 @@ func (s *service) worker(stop <-chan struct{}, delay time.Duration, id string) { } // load in the tasks for the Actuator from the controller - a.AddTasks(s.tasks) + a.AddMaintenanceSets(s.sets) t := time.NewTicker(s.pollTime) defer t.Stop() diff --git a/pkg/mimo/actuator/service_test.go b/pkg/mimo/actuator/service_test.go index 320715fdafa..e78400298c4 100644 --- a/pkg/mimo/actuator/service_test.go +++ b/pkg/mimo/actuator/service_test.go @@ -20,7 +20,8 @@ import ( "github.com/Azure/ARO-RP/pkg/database/cosmosdb" "github.com/Azure/ARO-RP/pkg/env" "github.com/Azure/ARO-RP/pkg/metrics" - "github.com/Azure/ARO-RP/pkg/mimo/tasks" + "github.com/Azure/ARO-RP/pkg/mimo/sets" + "github.com/Azure/ARO-RP/pkg/util/mimo" mock_env "github.com/Azure/ARO-RP/pkg/util/mocks/env" testdatabase "github.com/Azure/ARO-RP/test/database" testlog "github.com/Azure/ARO-RP/test/util/log" @@ -212,8 +213,8 @@ var _ = Describe("MIMO Actuator Service", Ordered, func() { done := make(chan struct{}) svc.pollTime = time.Second - svc.SetTasks(map[string]tasks.TaskFunc{ - "0000-0000-0001": func(ctx context.Context, th tasks.TaskContext, mmd *api.MaintenanceManifestDocument, oscd *api.OpenShiftClusterDocument) (api.MaintenanceManifestState, string) { + svc.SetMaintenanceSets(map[string]sets.MaintenanceSet{ + "0000-0000-0001": func(th mimo.TaskContext, mmd *api.MaintenanceManifestDocument, oscd *api.OpenShiftClusterDocument) (api.MaintenanceManifestState, string) { svc.stopping.Store(true) return api.MaintenanceManifestStateCompleted, "ok" }, @@ -230,8 +231,8 @@ var _ = Describe("MIMO Actuator Service", Ordered, func() { done := make(chan struct{}) svc.pollTime = time.Second - svc.SetTasks(map[string]tasks.TaskFunc{ - "0000-0000-0001": func(ctx context.Context, th tasks.TaskContext, mmd *api.MaintenanceManifestDocument, oscd *api.OpenShiftClusterDocument) (api.MaintenanceManifestState, string) { + svc.SetMaintenanceSets(map[string]sets.MaintenanceSet{ + "0000-0000-0001": func(th mimo.TaskContext, mmd *api.MaintenanceManifestDocument, oscd *api.OpenShiftClusterDocument) (api.MaintenanceManifestState, string) { // ProvisioningState is in the full document, not just the // ClusterResourceID only as in the bucket worker Expect(oscd.OpenShiftCluster.Properties.ProvisioningState).To(Equal(api.ProvisioningStateSucceeded)) diff --git a/pkg/mimo/actuator/task.go b/pkg/mimo/actuator/task.go index 6c5b71a4222..d81fbf03d5d 100644 --- a/pkg/mimo/actuator/task.go +++ b/pkg/mimo/actuator/task.go @@ -4,33 +4,73 @@ package actuator // Licensed under the Apache License 2.0. import ( + "context" + "time" + "github.com/sirupsen/logrus" "github.com/Azure/ARO-RP/pkg/api" "github.com/Azure/ARO-RP/pkg/env" - "github.com/Azure/ARO-RP/pkg/mimo/tasks" "github.com/Azure/ARO-RP/pkg/util/clienthelper" + "github.com/Azure/ARO-RP/pkg/util/mimo" "github.com/Azure/ARO-RP/pkg/util/restconfig" ) type th struct { + originalCtx context.Context + ctx context.Context + env env.Interface log *logrus.Entry + resultMessage string + oc *api.OpenShiftClusterDocument _ch clienthelper.Interface } -func newTaskContext(env env.Interface, log *logrus.Entry, oc *api.OpenShiftClusterDocument) tasks.TaskContext { +// force interface checking +var _ mimo.TaskContext = &th{} + +func newTaskContext(ctx context.Context, env env.Interface, log *logrus.Entry, oc *api.OpenShiftClusterDocument) *th { return &th{ - env: env, - log: log, - oc: oc, - _ch: nil, + originalCtx: ctx, + ctx: ctx, + env: env, + log: log, + oc: oc, + _ch: nil, } } +func (t *th) RunInTimeout(timeout time.Duration, f func() error) error { + newctx, cancel := context.WithTimeout(t.originalCtx, timeout) + t.ctx = newctx + defer func() { + cancel() + t.ctx = t.originalCtx + }() + return f() +} + +// context stuff +func (t *th) Deadline() (time.Time, bool) { + return t.ctx.Deadline() +} + +func (t *th) Done() <-chan struct{} { + return t.ctx.Done() +} + +func (t *th) Err() error { + return t.ctx.Err() +} + +func (t *th) Value(key any) any { + return t.ctx.Value(key) +} + func (t *th) Environment() env.Interface { return t.env } @@ -57,3 +97,15 @@ func (t *th) ClientHelper() (clienthelper.Interface, error) { func (t *th) Log() *logrus.Entry { return t.log } + +func (t *th) Now() time.Time { + return time.Now() +} + +func (t *th) SetResultMessage(msg string) { + t.resultMessage = msg +} + +func (t *th) GetResultMessage() string { + return t.resultMessage +} diff --git a/pkg/mimo/cmd/cli.go b/pkg/mimo/cmd/cli.go index feffce5a9be..6857953d224 100644 --- a/pkg/mimo/cmd/cli.go +++ b/pkg/mimo/cmd/cli.go @@ -16,8 +16,6 @@ import ( "github.com/Azure/ARO-RP/pkg/metrics/statsd" "github.com/Azure/ARO-RP/pkg/metrics/statsd/golang" "github.com/Azure/ARO-RP/pkg/mimo/actuator" - "github.com/Azure/ARO-RP/pkg/mimo/tasks" - "github.com/Azure/ARO-RP/pkg/mimo/tasks/example" "github.com/Azure/ARO-RP/pkg/proxy" utillog "github.com/Azure/ARO-RP/pkg/util/log" "github.com/Azure/ARO-RP/pkg/util/service" @@ -159,9 +157,7 @@ func main() { a := actuator.NewService(_env, _env.Logger(), dialer, clusters, manifests, m) - a.SetTasks(map[string]tasks.TaskFunc{ - "64d23e34-78c3-4d60-831c-6efb22f08508": example.ExampleTask, - }) + a.SetMaintenanceSets(sets.DEFAULT_MAINTENANCE_SETS) sigterm := make(chan os.Signal, 1) done := make(chan struct{}) diff --git a/pkg/mimo/sets/interfaces.go b/pkg/mimo/sets/interfaces.go new file mode 100644 index 00000000000..154f09a2bcf --- /dev/null +++ b/pkg/mimo/sets/interfaces.go @@ -0,0 +1,11 @@ +package sets + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/util/mimo" +) + +type MaintenanceSet func(mimo.TaskContext, *api.MaintenanceManifestDocument, *api.OpenShiftClusterDocument) (api.MaintenanceManifestState, string) diff --git a/pkg/mimo/sets/sets.go b/pkg/mimo/sets/sets.go new file mode 100644 index 00000000000..0e003bba2e8 --- /dev/null +++ b/pkg/mimo/sets/sets.go @@ -0,0 +1,8 @@ +package sets + +import "time" + +const DEFAULT_POLL_TIME = time.Second * 10 +const DEFAULT_TIMEOUT_DURATION = time.Minute * 20 + +var DEFAULT_MAINTENANCE_SETS = map[string]MaintenanceSet{} diff --git a/pkg/mimo/tasks/example/task.go b/pkg/mimo/tasks/example/task.go index b8186f2167d..3dc832029fd 100644 --- a/pkg/mimo/tasks/example/task.go +++ b/pkg/mimo/tasks/example/task.go @@ -10,22 +10,28 @@ import ( configv1 "github.com/openshift/api/config/v1" "k8s.io/apimachinery/pkg/types" - "github.com/Azure/ARO-RP/pkg/api" - "github.com/Azure/ARO-RP/pkg/mimo/tasks" + "github.com/Azure/ARO-RP/pkg/util/mimo" ) -func ExampleTask(ctx context.Context, th tasks.TaskContext, manifest *api.MaintenanceManifestDocument, oc *api.OpenShiftClusterDocument) (api.MaintenanceManifestState, string) { +func ReportClusterVersion(ctx context.Context) error { + th, err := mimo.GetTaskContext(ctx) + if err != nil { + return err + } + ch, err := th.ClientHelper() if err != nil { - return api.MaintenanceManifestStateFailed, err.Error() + return err } cv := &configv1.ClusterVersion{} err = ch.GetOne(ctx, types.NamespacedName{Name: "version"}, cv) if err != nil { - return api.MaintenanceManifestStateFailed, fmt.Errorf("unable to get ClusterVersion: %w", err).Error() + return fmt.Errorf("unable to get ClusterVersion: %w", err) } - return api.MaintenanceManifestStateCompleted, fmt.Sprintf("cluster version is: %s", cv.Status.History[0].Version) + th.SetResultMessage(fmt.Sprintf("cluster version is: %s", cv.Status.History[0].Version)) + + return nil } diff --git a/pkg/mimo/tasks/example/task_test.go b/pkg/mimo/tasks/example/task_test.go index ab5f3a8f95f..87fe75f1b61 100644 --- a/pkg/mimo/tasks/example/task_test.go +++ b/pkg/mimo/tasks/example/task_test.go @@ -5,9 +5,8 @@ package example import ( "context" - "fmt" - "strings" "testing" + "time" . "github.com/onsi/gomega" @@ -16,10 +15,8 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client/fake" - "github.com/Azure/ARO-RP/pkg/api" "github.com/Azure/ARO-RP/pkg/util/clienthelper" mock_env "github.com/Azure/ARO-RP/pkg/util/mocks/env" - testdatabase "github.com/Azure/ARO-RP/test/database" testtasks "github.com/Azure/ARO-RP/test/mimo/tasks" testclienthelper "github.com/Azure/ARO-RP/test/util/clienthelper" testlog "github.com/Azure/ARO-RP/test/util/log" @@ -29,35 +26,10 @@ func TestTask(t *testing.T) { RegisterTestingT(t) ctx := context.Background() - mockSubID := "00000000-0000-0000-0000-000000000000" - clusterResourceID := fmt.Sprintf("/subscriptions/%s/resourcegroups/resourceGroup/providers/Microsoft.RedHatOpenShift/openShiftClusters/resourceName", mockSubID) - controller := gomock.NewController(t) _env := mock_env.NewMockInterface(controller) _, log := testlog.New() - fixtures := testdatabase.NewFixture() - clusters, _ := testdatabase.NewFakeOpenShiftClusters() - - fixtures.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{ - Key: strings.ToLower(clusterResourceID), - OpenShiftCluster: &api.OpenShiftCluster{ - ID: clusterResourceID, - }, - }) - - manifest := &api.MaintenanceManifestDocument{ - ClusterResourceID: strings.ToLower(clusterResourceID), - MaintenanceManifest: api.MaintenanceManifest{ - State: api.MaintenanceManifestStatePending, - RunBefore: 60, - RunAfter: 0, - }, - } - - err := fixtures.WithOpenShiftClusters(clusters).Create() - Expect(err).ToNot(HaveOccurred()) - builder := fake.NewClientBuilder().WithRuntimeObjects( &configv1.ClusterVersion{ ObjectMeta: metav1.ObjectMeta{ @@ -74,12 +46,9 @@ func TestTask(t *testing.T) { }, ) ch := clienthelper.NewWithClient(log, testclienthelper.NewHookingClient(builder.Build())) - tc := testtasks.NewFakeTestContext(_env, log, ch) + tc := testtasks.NewFakeTestContext(ctx, _env, log, func() time.Time { return time.Unix(100, 0) }, ch) - oc, err := clusters.Get(ctx, strings.ToLower(clusterResourceID)) + err := ReportClusterVersion(tc) Expect(err).ToNot(HaveOccurred()) - r, text := ExampleTask(ctx, tc, manifest, oc) - - Expect(r).To(Equal(api.MaintenanceManifestStateCompleted)) - Expect(text).To(Equal("cluster version is: 4.99.123")) + Expect(tc.GetResultMessage()).To(Equal("cluster version is: 4.99.123")) } diff --git a/pkg/mimo/tasks/interfaces.go b/pkg/mimo/tasks/interfaces.go deleted file mode 100644 index f52974ff258..00000000000 --- a/pkg/mimo/tasks/interfaces.go +++ /dev/null @@ -1,22 +0,0 @@ -package tasks - -// Copyright (c) Microsoft Corporation. -// Licensed under the Apache License 2.0. - -import ( - "context" - - "github.com/sirupsen/logrus" - - "github.com/Azure/ARO-RP/pkg/api" - "github.com/Azure/ARO-RP/pkg/env" - "github.com/Azure/ARO-RP/pkg/util/clienthelper" -) - -type TaskContext interface { - Environment() env.Interface - ClientHelper() (clienthelper.Interface, error) - Log() *logrus.Entry -} - -type TaskFunc func(context.Context, TaskContext, *api.MaintenanceManifestDocument, *api.OpenShiftClusterDocument) (api.MaintenanceManifestState, string) diff --git a/pkg/util/mimo/errors.go b/pkg/util/mimo/errors.go new file mode 100644 index 00000000000..3a38afb3ce7 --- /dev/null +++ b/pkg/util/mimo/errors.go @@ -0,0 +1,37 @@ +package mimo + +type MIMOErrorVariety int + +const ( + MIMOErrorTypeTransientError MIMOErrorVariety = iota + MIMOErrorTypeNonRetryableError +) + +type MIMOError interface { + error + MIMOErrorVariety() MIMOErrorVariety +} + +type wrappedMIMOError struct { + error + variety MIMOErrorVariety +} + +func (f *wrappedMIMOError) MIMOErrorVariety() MIMOErrorVariety { + return f.variety +} + +func NewMIMOError(err error, variety MIMOErrorVariety) MIMOError { + return &wrappedMIMOError{ + error: err, + variety: variety, + } +} + +func UnretryableError(err error) MIMOError { + return NewMIMOError(err, MIMOErrorTypeNonRetryableError) +} + +func TransientError(err error) MIMOError { + return NewMIMOError(err, MIMOErrorTypeTransientError) +} diff --git a/pkg/util/mimo/taskcontext.go b/pkg/util/mimo/taskcontext.go new file mode 100644 index 00000000000..ecb96663ae6 --- /dev/null +++ b/pkg/util/mimo/taskcontext.go @@ -0,0 +1,32 @@ +package mimo + +import ( + "context" + "fmt" + "time" + + "github.com/sirupsen/logrus" + + "github.com/Azure/ARO-RP/pkg/env" + "github.com/Azure/ARO-RP/pkg/util/clienthelper" +) + +type TaskContext interface { + context.Context + Now() time.Time + Environment() env.Interface + ClientHelper() (clienthelper.Interface, error) + Log() *logrus.Entry + + SetResultMessage(string) + GetResultMessage() string +} + +func GetTaskContext(c context.Context) (TaskContext, error) { + r, ok := c.(TaskContext) + if !ok { + return nil, fmt.Errorf("cannot convert %v", r) + } + + return r, nil +} diff --git a/test/mimo/tasks/testcontext.go b/test/mimo/tasks/testcontext.go index 2d7f6aea095..55791ccb44e 100644 --- a/test/mimo/tasks/testcontext.go +++ b/test/mimo/tasks/testcontext.go @@ -4,25 +4,34 @@ package tasks // Licensed under the Apache License 2.0. import ( + "context" + "time" + "github.com/sirupsen/logrus" "github.com/Azure/ARO-RP/pkg/env" - "github.com/Azure/ARO-RP/pkg/mimo/tasks" "github.com/Azure/ARO-RP/pkg/util/clienthelper" + "github.com/Azure/ARO-RP/pkg/util/mimo" ) -func NewFakeTestContext(env env.Interface, log *logrus.Entry, ch clienthelper.Interface) tasks.TaskContext { +func NewFakeTestContext(ctx context.Context, env env.Interface, log *logrus.Entry, now func() time.Time, ch clienthelper.Interface) mimo.TaskContext { return &fakeTestContext{ - env: env, - log: log, - ch: ch, + Context: ctx, + env: env, + log: log, + ch: ch, + now: now, } } type fakeTestContext struct { + context.Context + now func() time.Time env env.Interface ch clienthelper.Interface log *logrus.Entry + + resultMessage string } func (t *fakeTestContext) Environment() env.Interface { @@ -36,3 +45,15 @@ func (t *fakeTestContext) ClientHelper() (clienthelper.Interface, error) { func (t *fakeTestContext) Log() *logrus.Entry { return t.log } + +func (t *fakeTestContext) Now() time.Time { + return t.now() +} + +func (t *fakeTestContext) SetResultMessage(s string) { + t.resultMessage = s +} + +func (t *fakeTestContext) GetResultMessage() string { + return t.resultMessage +} From f1916f8239184c72a0ef8f3072b9ce03fabed02c Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Fri, 26 Jul 2024 13:55:55 +1000 Subject: [PATCH 17/67] mimo error code --- pkg/mimo/tasks/cluster/apiserver_is_up.go | 43 +++++++ .../tasks/cluster/apiserver_is_up_test.go | 106 ++++++++++++++++++ pkg/util/clusteroperators/isavailable.go | 12 ++ pkg/util/clusteroperators/isavailable_test.go | 56 +++++++++ pkg/util/mimo/errors.go | 31 +++-- 5 files changed, 241 insertions(+), 7 deletions(-) create mode 100644 pkg/mimo/tasks/cluster/apiserver_is_up.go create mode 100644 pkg/mimo/tasks/cluster/apiserver_is_up_test.go diff --git a/pkg/mimo/tasks/cluster/apiserver_is_up.go b/pkg/mimo/tasks/cluster/apiserver_is_up.go new file mode 100644 index 00000000000..a4a1985bf70 --- /dev/null +++ b/pkg/mimo/tasks/cluster/apiserver_is_up.go @@ -0,0 +1,43 @@ +package cluster + +import ( + "context" + "errors" + + configv1 "github.com/openshift/api/config/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + + "github.com/Azure/ARO-RP/pkg/util/clusteroperators" + "github.com/Azure/ARO-RP/pkg/util/mimo" +) + +func EnsureAPIServerIsUp(ctx context.Context) error { + th, err := mimo.GetTaskContext(ctx) + if err != nil { + return err + } + + ch, err := th.ClientHelper() + if err != nil { + return mimo.TerminalError(err) + } + + co := &configv1.ClusterOperator{} + + err = ch.GetOne(ctx, types.NamespacedName{Name: "kube-apiserver"}, co) + if err != nil { + // 404 on kube-apiserver is likely terminal + if kerrors.IsNotFound(err) { + return mimo.TerminalError(err) + } + + return mimo.TransientError(err) + } + + available := clusteroperators.IsOperatorAvailable(co) + if !available { + return mimo.TransientError(errors.New(clusteroperators.OperatorStatusText(co))) + } + return nil +} diff --git a/pkg/mimo/tasks/cluster/apiserver_is_up_test.go b/pkg/mimo/tasks/cluster/apiserver_is_up_test.go new file mode 100644 index 00000000000..8eaee72d665 --- /dev/null +++ b/pkg/mimo/tasks/cluster/apiserver_is_up_test.go @@ -0,0 +1,106 @@ +package cluster + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "testing" + "time" + + . "github.com/onsi/gomega" + + "github.com/golang/mock/gomock" + configv1 "github.com/openshift/api/config/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/Azure/ARO-RP/pkg/util/clienthelper" + mock_env "github.com/Azure/ARO-RP/pkg/util/mocks/env" + testtasks "github.com/Azure/ARO-RP/test/mimo/tasks" + testclienthelper "github.com/Azure/ARO-RP/test/util/clienthelper" + testlog "github.com/Azure/ARO-RP/test/util/log" +) + +func TestAPIServerIsUp(t *testing.T) { + ctx := context.Background() + + for _, tt := range []struct { + name string + objects []runtime.Object + wantErr string + }{ + { + name: "not found", + objects: []runtime.Object{}, + wantErr: `NonRetryableError: clusteroperators.config.openshift.io "kube-apiserver" not found`, + }, + { + name: "not ready", + objects: []runtime.Object{ + &configv1.ClusterOperator{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kube-apiserver", + }, + Status: configv1.ClusterOperatorStatus{ + Conditions: []configv1.ClusterOperatorStatusCondition{ + { + Type: configv1.OperatorAvailable, + Status: configv1.ConditionFalse, + }, + { + Type: configv1.OperatorProgressing, + Status: configv1.ConditionTrue, + }, + }, + }, + }, + }, + wantErr: `TransientError: kube-apiserver Available=False, Progressing=True`, + }, + { + name: "ready", + objects: []runtime.Object{ + &configv1.ClusterOperator{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kube-apiserver", + }, + Status: configv1.ClusterOperatorStatus{ + Conditions: []configv1.ClusterOperatorStatusCondition{ + { + Type: configv1.OperatorAvailable, + Status: configv1.ConditionTrue, + }, + { + Type: configv1.OperatorProgressing, + Status: configv1.ConditionFalse, + }, + }, + }, + }, + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + controller := gomock.NewController(t) + _env := mock_env.NewMockInterface(controller) + _, log := testlog.New() + + builder := fake.NewClientBuilder().WithRuntimeObjects(tt.objects...) + ch := clienthelper.NewWithClient(log, testclienthelper.NewHookingClient(builder.Build())) + tc := testtasks.NewFakeTestContext(ctx, _env, log, func() time.Time { return time.Unix(100, 0) }, ch) + + err := EnsureAPIServerIsUp(tc) + if tt.wantErr != "" && err != nil { + g.Expect(err).To(MatchError(tt.wantErr)) + } else if tt.wantErr != "" && err == nil { + t.Errorf("wanted error %s", tt.wantErr) + } else if tt.wantErr == "" { + g.Expect(err).ToNot(HaveOccurred()) + } + }) + + } +} diff --git a/pkg/util/clusteroperators/isavailable.go b/pkg/util/clusteroperators/isavailable.go index 1af3bf1ba8e..8945fc14a41 100644 --- a/pkg/util/clusteroperators/isavailable.go +++ b/pkg/util/clusteroperators/isavailable.go @@ -1,6 +1,8 @@ package clusteroperators import ( + "fmt" + configv1 "github.com/openshift/api/config/v1" ) @@ -11,3 +13,13 @@ func IsOperatorAvailable(operator *configv1.ClusterOperator) bool { } return m[configv1.OperatorAvailable] == configv1.ConditionTrue && m[configv1.OperatorProgressing] == configv1.ConditionFalse } + +func OperatorStatusText(operator *configv1.ClusterOperator) string { + m := make(map[configv1.ClusterStatusConditionType]configv1.ConditionStatus, len(operator.Status.Conditions)) + for _, cond := range operator.Status.Conditions { + m[cond.Type] = cond.Status + } + return fmt.Sprintf("%s %s=%s, %s=%s", operator.Name, + configv1.OperatorAvailable, m[configv1.OperatorAvailable], configv1.OperatorProgressing, m[configv1.OperatorProgressing], + ) +} diff --git a/pkg/util/clusteroperators/isavailable_test.go b/pkg/util/clusteroperators/isavailable_test.go index 4b9189641a7..b216745801e 100644 --- a/pkg/util/clusteroperators/isavailable_test.go +++ b/pkg/util/clusteroperators/isavailable_test.go @@ -59,3 +59,59 @@ func TestIsOperatorAvailable(t *testing.T) { } } } + +func TestOperatorStatusText(t *testing.T) { + for _, tt := range []struct { + name string + availableCondition configv1.ConditionStatus + progressingCondition configv1.ConditionStatus + want string + }{ + { + name: "Available && Progressing; not available", + availableCondition: configv1.ConditionTrue, + progressingCondition: configv1.ConditionTrue, + want: "server Available=True, Progressing=True", + }, + { + name: "Available && !Progressing; available", + availableCondition: configv1.ConditionTrue, + progressingCondition: configv1.ConditionFalse, + want: "server Available=True, Progressing=False", + }, + { + name: "!Available && Progressing; not available", + availableCondition: configv1.ConditionFalse, + progressingCondition: configv1.ConditionTrue, + want: "server Available=False, Progressing=True", + }, + { + name: "!Available && !Progressing; not available", + availableCondition: configv1.ConditionFalse, + progressingCondition: configv1.ConditionFalse, + want: "server Available=False, Progressing=False", + }, + } { + operator := &configv1.ClusterOperator{ + ObjectMeta: metav1.ObjectMeta{ + Name: "server", + }, + Status: configv1.ClusterOperatorStatus{ + Conditions: []configv1.ClusterOperatorStatusCondition{ + { + Type: configv1.OperatorAvailable, + Status: tt.availableCondition, + }, + { + Type: configv1.OperatorProgressing, + Status: tt.progressingCondition, + }, + }, + }, + } + available := OperatorStatusText(operator) + if available != tt.want { + t.Error(available) + } + } +} diff --git a/pkg/util/mimo/errors.go b/pkg/util/mimo/errors.go index 3a38afb3ce7..b1ac174db5c 100644 --- a/pkg/util/mimo/errors.go +++ b/pkg/util/mimo/errors.go @@ -1,10 +1,12 @@ package mimo -type MIMOErrorVariety int +import "fmt" + +type MIMOErrorVariety string const ( - MIMOErrorTypeTransientError MIMOErrorVariety = iota - MIMOErrorTypeNonRetryableError + MIMOErrorTypeTransientError MIMOErrorVariety = "TransientError" + MIMOErrorTypeTerminalError MIMOErrorVariety = "TerminalError" ) type MIMOError interface { @@ -17,21 +19,36 @@ type wrappedMIMOError struct { variety MIMOErrorVariety } -func (f *wrappedMIMOError) MIMOErrorVariety() MIMOErrorVariety { +func (f wrappedMIMOError) MIMOErrorVariety() MIMOErrorVariety { return f.variety } +func (f wrappedMIMOError) Error() string { + return fmt.Sprintf("%s: %s", f.variety, f.error.Error()) +} + func NewMIMOError(err error, variety MIMOErrorVariety) MIMOError { - return &wrappedMIMOError{ + return wrappedMIMOError{ error: err, variety: variety, } } -func UnretryableError(err error) MIMOError { - return NewMIMOError(err, MIMOErrorTypeNonRetryableError) +func TerminalError(err error) MIMOError { + return NewMIMOError(err, MIMOErrorTypeTerminalError) } func TransientError(err error) MIMOError { return NewMIMOError(err, MIMOErrorTypeTransientError) } + +func IsRetryableError(err error) bool { + e, ok := err.(wrappedMIMOError) + if !ok { + return false + } + if e.MIMOErrorVariety() == MIMOErrorTypeTransientError { + return true + } + return false +} From 48129d61de8b9474ed57ff52d61690c0eebc4625 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Fri, 26 Jul 2024 13:56:01 +1000 Subject: [PATCH 18/67] more work on sets --- pkg/mimo/cmd/cli.go | 1 + pkg/mimo/sets/sets.go | 24 ++++++++++++++++++++++-- pkg/mimo/sets/tls_cert_rotate.go | 16 ++++++++++++++++ 3 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 pkg/mimo/sets/tls_cert_rotate.go diff --git a/pkg/mimo/cmd/cli.go b/pkg/mimo/cmd/cli.go index 6857953d224..ab22142be85 100644 --- a/pkg/mimo/cmd/cli.go +++ b/pkg/mimo/cmd/cli.go @@ -16,6 +16,7 @@ import ( "github.com/Azure/ARO-RP/pkg/metrics/statsd" "github.com/Azure/ARO-RP/pkg/metrics/statsd/golang" "github.com/Azure/ARO-RP/pkg/mimo/actuator" + "github.com/Azure/ARO-RP/pkg/mimo/sets" "github.com/Azure/ARO-RP/pkg/proxy" utillog "github.com/Azure/ARO-RP/pkg/util/log" "github.com/Azure/ARO-RP/pkg/util/service" diff --git a/pkg/mimo/sets/sets.go b/pkg/mimo/sets/sets.go index 0e003bba2e8..5b54581ea25 100644 --- a/pkg/mimo/sets/sets.go +++ b/pkg/mimo/sets/sets.go @@ -1,8 +1,28 @@ package sets -import "time" +import ( + "time" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/util/mimo" + "github.com/Azure/ARO-RP/pkg/util/steps" +) const DEFAULT_POLL_TIME = time.Second * 10 const DEFAULT_TIMEOUT_DURATION = time.Minute * 20 -var DEFAULT_MAINTENANCE_SETS = map[string]MaintenanceSet{} +var DEFAULT_MAINTENANCE_SETS = map[string]MaintenanceSet{ + "9b741734-6505-447f-8510-85eb0ae561a2": TLSCertRotation, +} + +func run(t mimo.TaskContext, s []steps.Step) (api.MaintenanceManifestState, string) { + _, err := steps.Run(t, t.Log(), DEFAULT_POLL_TIME, s, t.Now) + + if err != nil { + if mimo.IsRetryableError(err) { + return api.MaintenanceManifestStatePending, err.Error() + } + return api.MaintenanceManifestStateFailed, err.Error() + } + return api.MaintenanceManifestStateCompleted, t.GetResultMessage() +} diff --git a/pkg/mimo/sets/tls_cert_rotate.go b/pkg/mimo/sets/tls_cert_rotate.go new file mode 100644 index 00000000000..a687da00622 --- /dev/null +++ b/pkg/mimo/sets/tls_cert_rotate.go @@ -0,0 +1,16 @@ +package sets + +import ( + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/mimo/tasks/cluster" + "github.com/Azure/ARO-RP/pkg/util/mimo" + "github.com/Azure/ARO-RP/pkg/util/steps" +) + +func TLSCertRotation(t mimo.TaskContext, doc *api.MaintenanceManifestDocument, oc *api.OpenShiftClusterDocument) (api.MaintenanceManifestState, string) { + s := []steps.Step{ + steps.Action(cluster.EnsureAPIServerIsUp), + } + + return run(t, s) +} From 37c2f7eba900bb6a90a4aab34a1e233d7f0a5216 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Fri, 26 Jul 2024 15:06:08 +1000 Subject: [PATCH 19/67] tls tasks work --- pkg/api/admin/mimo_convert.go | 1 - .../admin_mimo_manifest_cancel_test.go | 1 - pkg/frontend/admin_mimo_manifest_create.go | 1 - .../admin_mimo_manifest_create_test.go | 1 - .../admin_mimo_manifest_delete_test.go | 1 - pkg/frontend/frontend.go | 2 +- pkg/mimo/actuator/task.go | 8 ++ pkg/mimo/sets/sets.go | 3 + pkg/mimo/sets/tls_cert_rotate.go | 6 + pkg/mimo/tasks/cluster/apiserver_is_up.go | 3 + .../tasks/cluster/apiserver_is_up_test.go | 8 +- pkg/mimo/tasks/cluster/tls.go | 104 +++++++++++++++++ pkg/mimo/tasks/cluster/tls_test.go | 106 ++++++++++++++++++ pkg/mimo/tasks/example/task_test.go | 6 +- pkg/util/clusteroperators/isavailable.go | 3 + pkg/util/clusteroperators/isavailable_test.go | 3 + pkg/util/mimo/errors.go | 3 + pkg/util/mimo/taskcontext.go | 8 ++ test/mimo/tasks/testcontext.go | 73 ++++++++++-- 19 files changed, 319 insertions(+), 22 deletions(-) create mode 100644 pkg/mimo/tasks/cluster/tls.go create mode 100644 pkg/mimo/tasks/cluster/tls_test.go diff --git a/pkg/api/admin/mimo_convert.go b/pkg/api/admin/mimo_convert.go index 6046a81ccc7..6cd9a518577 100644 --- a/pkg/api/admin/mimo_convert.go +++ b/pkg/api/admin/mimo_convert.go @@ -38,7 +38,6 @@ func (m maintenanceManifestConverter) ToExternalList(docs []*api.MaintenanceMani } func (m maintenanceManifestConverter) ToInternal(_i interface{}, out *api.MaintenanceManifestDocument) { - i := _i.(*MaintenanceManifest) out.ID = i.ID diff --git a/pkg/frontend/admin_mimo_manifest_cancel_test.go b/pkg/frontend/admin_mimo_manifest_cancel_test.go index d65c5401a2e..97669c6c356 100644 --- a/pkg/frontend/admin_mimo_manifest_cancel_test.go +++ b/pkg/frontend/admin_mimo_manifest_cancel_test.go @@ -200,7 +200,6 @@ func TestMIMOCancelManifest(t *testing.T) { for _, err := range ti.checker.CheckMaintenanceManifests(ti.maintenanceManifestsClient) { t.Error(err) } - }) } } diff --git a/pkg/frontend/admin_mimo_manifest_create.go b/pkg/frontend/admin_mimo_manifest_create.go index cc238d896da..c42c22f86d4 100644 --- a/pkg/frontend/admin_mimo_manifest_create.go +++ b/pkg/frontend/admin_mimo_manifest_create.go @@ -92,7 +92,6 @@ func (f *frontend) _putAdminMaintManifestCreate(ctx context.Context, r *http.Req savedDoc, err := dbMaintenanceManifests.Create(ctx, manifestDoc) if err != nil { return nil, api.NewCloudError(http.StatusInternalServerError, api.CloudErrorCodeInternalServerError, "", err.Error()) - } return json.MarshalIndent(converter.ToExternal(savedDoc), "", " ") diff --git a/pkg/frontend/admin_mimo_manifest_create_test.go b/pkg/frontend/admin_mimo_manifest_create_test.go index ac739adc698..777122ea49c 100644 --- a/pkg/frontend/admin_mimo_manifest_create_test.go +++ b/pkg/frontend/admin_mimo_manifest_create_test.go @@ -165,7 +165,6 @@ func TestMIMOCreateManifest(t *testing.T) { for _, err := range ti.checker.CheckMaintenanceManifests(ti.maintenanceManifestsClient) { t.Error(err) } - }) } } diff --git a/pkg/frontend/admin_mimo_manifest_delete_test.go b/pkg/frontend/admin_mimo_manifest_delete_test.go index 53d427846fa..781948b2a42 100644 --- a/pkg/frontend/admin_mimo_manifest_delete_test.go +++ b/pkg/frontend/admin_mimo_manifest_delete_test.go @@ -146,7 +146,6 @@ func TestMIMODeleteManifest(t *testing.T) { for _, err := range ti.checker.CheckMaintenanceManifests(ti.maintenanceManifestsClient) { t.Error(err) } - }) } } diff --git a/pkg/frontend/frontend.go b/pkg/frontend/frontend.go index f5cdfd857d5..d24938a3efa 100644 --- a/pkg/frontend/frontend.go +++ b/pkg/frontend/frontend.go @@ -15,6 +15,7 @@ import ( "sync/atomic" "time" + "github.com/Azure/go-autorest/autorest/azure" "github.com/go-chi/chi/v5" chiMiddlewares "github.com/go-chi/chi/v5/middleware" "github.com/sirupsen/logrus" @@ -34,7 +35,6 @@ import ( "github.com/Azure/ARO-RP/pkg/util/heartbeat" utillog "github.com/Azure/ARO-RP/pkg/util/log" "github.com/Azure/ARO-RP/pkg/util/recover" - "github.com/Azure/go-autorest/autorest/azure" ) type statusCodeError int diff --git a/pkg/mimo/actuator/task.go b/pkg/mimo/actuator/task.go index d81fbf03d5d..21d002e4410 100644 --- a/pkg/mimo/actuator/task.go +++ b/pkg/mimo/actuator/task.go @@ -109,3 +109,11 @@ func (t *th) SetResultMessage(msg string) { func (t *th) GetResultMessage() string { return t.resultMessage } + +func (t *th) GetClusterUUID() string { + return t.oc.ID +} + +func (t *th) GetOpenShiftClusterProperties() api.OpenShiftClusterProperties { + return t.oc.OpenShiftCluster.Properties +} diff --git a/pkg/mimo/sets/sets.go b/pkg/mimo/sets/sets.go index 5b54581ea25..c19dd54ae6c 100644 --- a/pkg/mimo/sets/sets.go +++ b/pkg/mimo/sets/sets.go @@ -1,5 +1,8 @@ package sets +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + import ( "time" diff --git a/pkg/mimo/sets/tls_cert_rotate.go b/pkg/mimo/sets/tls_cert_rotate.go index a687da00622..41e24d9de5e 100644 --- a/pkg/mimo/sets/tls_cert_rotate.go +++ b/pkg/mimo/sets/tls_cert_rotate.go @@ -1,5 +1,8 @@ package sets +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + import ( "github.com/Azure/ARO-RP/pkg/api" "github.com/Azure/ARO-RP/pkg/mimo/tasks/cluster" @@ -10,6 +13,9 @@ import ( func TLSCertRotation(t mimo.TaskContext, doc *api.MaintenanceManifestDocument, oc *api.OpenShiftClusterDocument) (api.MaintenanceManifestState, string) { s := []steps.Step{ steps.Action(cluster.EnsureAPIServerIsUp), + + steps.Action(cluster.RotateAPIServerCertificate), + steps.Action(cluster.EnsureAPIServerServingCertificateConfiguration), } return run(t, s) diff --git a/pkg/mimo/tasks/cluster/apiserver_is_up.go b/pkg/mimo/tasks/cluster/apiserver_is_up.go index a4a1985bf70..eecefa7e434 100644 --- a/pkg/mimo/tasks/cluster/apiserver_is_up.go +++ b/pkg/mimo/tasks/cluster/apiserver_is_up.go @@ -1,5 +1,8 @@ package cluster +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + import ( "context" "errors" diff --git a/pkg/mimo/tasks/cluster/apiserver_is_up_test.go b/pkg/mimo/tasks/cluster/apiserver_is_up_test.go index 8eaee72d665..6acffbb4994 100644 --- a/pkg/mimo/tasks/cluster/apiserver_is_up_test.go +++ b/pkg/mimo/tasks/cluster/apiserver_is_up_test.go @@ -34,7 +34,7 @@ func TestAPIServerIsUp(t *testing.T) { { name: "not found", objects: []runtime.Object{}, - wantErr: `NonRetryableError: clusteroperators.config.openshift.io "kube-apiserver" not found`, + wantErr: `TerminalError: clusteroperators.config.openshift.io "kube-apiserver" not found`, }, { name: "not ready", @@ -90,7 +90,10 @@ func TestAPIServerIsUp(t *testing.T) { builder := fake.NewClientBuilder().WithRuntimeObjects(tt.objects...) ch := clienthelper.NewWithClient(log, testclienthelper.NewHookingClient(builder.Build())) - tc := testtasks.NewFakeTestContext(ctx, _env, log, func() time.Time { return time.Unix(100, 0) }, ch) + tc := testtasks.NewFakeTestContext( + ctx, _env, log, func() time.Time { return time.Unix(100, 0) }, + testtasks.WithClientHelper(ch), + ) err := EnsureAPIServerIsUp(tc) if tt.wantErr != "" && err != nil { @@ -101,6 +104,5 @@ func TestAPIServerIsUp(t *testing.T) { g.Expect(err).ToNot(HaveOccurred()) } }) - } } diff --git a/pkg/mimo/tasks/cluster/tls.go b/pkg/mimo/tasks/cluster/tls.go new file mode 100644 index 00000000000..d291c76fcc8 --- /dev/null +++ b/pkg/mimo/tasks/cluster/tls.go @@ -0,0 +1,104 @@ +package cluster + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + + configv1 "github.com/openshift/api/config/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" + + "github.com/Azure/ARO-RP/pkg/cluster" + "github.com/Azure/ARO-RP/pkg/util/dns" + "github.com/Azure/ARO-RP/pkg/util/mimo" +) + +func RotateAPIServerCertificate(ctx context.Context) error { + th, err := mimo.GetTaskContext(ctx) + if err != nil { + return mimo.TerminalError(err) + } + + ch, err := th.ClientHelper() + if err != nil { + return mimo.TerminalError(err) + } + + env := th.Environment() + + secretName := th.GetClusterUUID() + "-apiserver" + + for _, namespace := range []string{"openshift-config", "openshift-azure-operator"} { + err = cluster.EnsureTLSSecretFromKeyvault( + ctx, env, ch, types.NamespacedName{Namespace: namespace, Name: secretName}, secretName, + ) + if err != nil { + return err + } + } + + return nil +} + +func EnsureAPIServerServingCertificateConfiguration(ctx context.Context) error { + th, err := mimo.GetTaskContext(ctx) + if err != nil { + return mimo.TerminalError(err) + } + + ch, err := th.ClientHelper() + if err != nil { + return mimo.TerminalError(err) + } + + env := th.Environment() + clusterProperties := th.GetOpenShiftClusterProperties() + + managedDomain, err := dns.ManagedDomain(env, clusterProperties.ClusterProfile.Domain) + if err != nil { + // if it fails the belt&braces check then not much we can do + return mimo.TerminalError(err) + } + + if managedDomain == "" { + th.SetResultMessage("apiserver certificate is not managed") + return nil + } + + return retry.RetryOnConflict(retry.DefaultRetry, func() error { + apiserver := &configv1.APIServer{} + + err := ch.GetOne(ctx, types.NamespacedName{Name: "cluster"}, apiserver) + if err != nil { + if kerrors.IsNotFound(err) { + // apiserver not being found is probably unrecoverable + return mimo.TerminalError(err) + } + return mimo.TransientError(err) + } + + apiserver.Spec.ServingCerts.NamedCertificates = []configv1.APIServerNamedServingCert{ + { + Names: []string{ + "api." + managedDomain, + }, + ServingCertificate: configv1.SecretNameReference{ + Name: th.GetClusterUUID() + "-apiserver", + }, + }, + } + + err = ch.Client().Update(ctx, apiserver) + if err != nil { + if kerrors.IsConflict(err) { + return err + } else { + return mimo.TransientError(err) + } + } + return nil + }) +} diff --git a/pkg/mimo/tasks/cluster/tls_test.go b/pkg/mimo/tasks/cluster/tls_test.go new file mode 100644 index 00000000000..ad850ed27ee --- /dev/null +++ b/pkg/mimo/tasks/cluster/tls_test.go @@ -0,0 +1,106 @@ +package cluster + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "testing" + "time" + + . "github.com/onsi/gomega" + + "github.com/golang/mock/gomock" + configv1 "github.com/openshift/api/config/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/util/clienthelper" + mock_env "github.com/Azure/ARO-RP/pkg/util/mocks/env" + testtasks "github.com/Azure/ARO-RP/test/mimo/tasks" + testlog "github.com/Azure/ARO-RP/test/util/log" +) + +func TestConfigureAPIServerCertificates(t *testing.T) { + ctx := context.Background() + clusterUUID := "512a50c8-2a43-4c2a-8fd9-a5539475df2a" + + for _, tt := range []struct { + name string + objects []runtime.Object + check func(clienthelper.Interface, Gomega) error + wantErr string + }{ + { + name: "not found", + objects: []runtime.Object{}, + wantErr: `TerminalError: apiservers.config.openshift.io "cluster" not found`, + }, + { + name: "secrets referenced", + objects: []runtime.Object{ + &configv1.APIServer{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster", + }, + Spec: configv1.APIServerSpec{}, + }, + }, + check: func(i clienthelper.Interface, g Gomega) error { + apiserver := &configv1.APIServer{} + err := i.GetOne(ctx, types.NamespacedName{Name: "cluster"}, apiserver) + if err != nil { + return err + } + + g.Expect(apiserver.Spec.ServingCerts.NamedCertificates).To(Equal([]configv1.APIServerNamedServingCert{ + { + Names: []string{"api.something.example.com"}, + ServingCertificate: configv1.SecretNameReference{ + Name: "512a50c8-2a43-4c2a-8fd9-a5539475df2a-apiserver", + }, + }, + })) + + return nil + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + controller := gomock.NewController(t) + _env := mock_env.NewMockInterface(controller) + _env.EXPECT().Domain().AnyTimes().Return("example.com") + + _, log := testlog.New() + + builder := fake.NewClientBuilder().WithRuntimeObjects(tt.objects...) + ch := clienthelper.NewWithClient(log, builder.Build()) + tc := testtasks.NewFakeTestContext( + ctx, _env, log, func() time.Time { return time.Unix(100, 0) }, + testtasks.WithClientHelper(ch), + testtasks.WithOpenShiftClusterProperties(clusterUUID, api.OpenShiftClusterProperties{ + ClusterProfile: api.ClusterProfile{ + Domain: "something", + }, + }), + ) + + err := EnsureAPIServerServingCertificateConfiguration(tc) + if tt.wantErr != "" && err != nil { + g.Expect(err).To(MatchError(tt.wantErr)) + } else if tt.wantErr != "" && err == nil { + t.Errorf("wanted error %s", tt.wantErr) + } else if tt.wantErr == "" { + g.Expect(err).ToNot(HaveOccurred()) + } + + if tt.check != nil { + g.Expect(tt.check(ch, g)).ToNot(HaveOccurred()) + } + }) + } +} diff --git a/pkg/mimo/tasks/example/task_test.go b/pkg/mimo/tasks/example/task_test.go index 87fe75f1b61..a912181cfc2 100644 --- a/pkg/mimo/tasks/example/task_test.go +++ b/pkg/mimo/tasks/example/task_test.go @@ -46,8 +46,10 @@ func TestTask(t *testing.T) { }, ) ch := clienthelper.NewWithClient(log, testclienthelper.NewHookingClient(builder.Build())) - tc := testtasks.NewFakeTestContext(ctx, _env, log, func() time.Time { return time.Unix(100, 0) }, ch) - + tc := testtasks.NewFakeTestContext( + ctx, _env, log, func() time.Time { return time.Unix(100, 0) }, + testtasks.WithClientHelper(ch), + ) err := ReportClusterVersion(tc) Expect(err).ToNot(HaveOccurred()) Expect(tc.GetResultMessage()).To(Equal("cluster version is: 4.99.123")) diff --git a/pkg/util/clusteroperators/isavailable.go b/pkg/util/clusteroperators/isavailable.go index 8945fc14a41..57a08dc55b8 100644 --- a/pkg/util/clusteroperators/isavailable.go +++ b/pkg/util/clusteroperators/isavailable.go @@ -1,5 +1,8 @@ package clusteroperators +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + import ( "fmt" diff --git a/pkg/util/clusteroperators/isavailable_test.go b/pkg/util/clusteroperators/isavailable_test.go index b216745801e..71672f799eb 100644 --- a/pkg/util/clusteroperators/isavailable_test.go +++ b/pkg/util/clusteroperators/isavailable_test.go @@ -1,5 +1,8 @@ package clusteroperators +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + import ( "testing" diff --git a/pkg/util/mimo/errors.go b/pkg/util/mimo/errors.go index b1ac174db5c..7ee6c2cf0a4 100644 --- a/pkg/util/mimo/errors.go +++ b/pkg/util/mimo/errors.go @@ -1,5 +1,8 @@ package mimo +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + import "fmt" type MIMOErrorVariety string diff --git a/pkg/util/mimo/taskcontext.go b/pkg/util/mimo/taskcontext.go index ecb96663ae6..ac32776ea35 100644 --- a/pkg/util/mimo/taskcontext.go +++ b/pkg/util/mimo/taskcontext.go @@ -1,5 +1,8 @@ package mimo +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + import ( "context" "fmt" @@ -7,6 +10,7 @@ import ( "github.com/sirupsen/logrus" + "github.com/Azure/ARO-RP/pkg/api" "github.com/Azure/ARO-RP/pkg/env" "github.com/Azure/ARO-RP/pkg/util/clienthelper" ) @@ -18,6 +22,10 @@ type TaskContext interface { ClientHelper() (clienthelper.Interface, error) Log() *logrus.Entry + // OpenShiftCluster + GetClusterUUID() string + GetOpenShiftClusterProperties() api.OpenShiftClusterProperties + SetResultMessage(string) GetResultMessage() string } diff --git a/test/mimo/tasks/testcontext.go b/test/mimo/tasks/testcontext.go index 55791ccb44e..bda019561fc 100644 --- a/test/mimo/tasks/testcontext.go +++ b/test/mimo/tasks/testcontext.go @@ -5,25 +5,16 @@ package tasks import ( "context" + "fmt" "time" "github.com/sirupsen/logrus" + "github.com/Azure/ARO-RP/pkg/api" "github.com/Azure/ARO-RP/pkg/env" "github.com/Azure/ARO-RP/pkg/util/clienthelper" - "github.com/Azure/ARO-RP/pkg/util/mimo" ) -func NewFakeTestContext(ctx context.Context, env env.Interface, log *logrus.Entry, now func() time.Time, ch clienthelper.Interface) mimo.TaskContext { - return &fakeTestContext{ - Context: ctx, - env: env, - log: log, - ch: ch, - now: now, - } -} - type fakeTestContext struct { context.Context now func() time.Time @@ -31,14 +22,59 @@ type fakeTestContext struct { ch clienthelper.Interface log *logrus.Entry + clusterUUID string + clusterResourceID string + properties api.OpenShiftClusterProperties + resultMessage string } +type Option func(*fakeTestContext) + +func WithClientHelper(ch clienthelper.Interface) Option { + return func(ftc *fakeTestContext) { + ftc.ch = ch + } +} + +func WithOpenShiftClusterDocument(oc *api.OpenShiftClusterDocument) Option { + return func(ftc *fakeTestContext) { + ftc.clusterUUID = oc.ID + ftc.clusterResourceID = oc.OpenShiftCluster.ID + ftc.properties = oc.OpenShiftCluster.Properties + } +} + +func WithOpenShiftClusterProperties(uuid string, oc api.OpenShiftClusterProperties) Option { + return func(ftc *fakeTestContext) { + ftc.clusterUUID = uuid + ftc.properties = oc + } +} + +func NewFakeTestContext(ctx context.Context, env env.Interface, log *logrus.Entry, now func() time.Time, o ...Option) *fakeTestContext { + ftc := &fakeTestContext{ + Context: ctx, + env: env, + log: log, + now: now, + } + for _, i := range o { + i(ftc) + } + return ftc +} + +// handle + func (t *fakeTestContext) Environment() env.Interface { return t.env } func (t *fakeTestContext) ClientHelper() (clienthelper.Interface, error) { + if t.ch == nil { + return nil, fmt.Errorf("missing clienthelper") + } return t.ch, nil } @@ -50,6 +86,21 @@ func (t *fakeTestContext) Now() time.Time { return t.now() } +// OpenShiftCluster +func (t *fakeTestContext) GetClusterUUID() string { + if t.clusterUUID == "" { + panic("didn't set up openshiftcluster in test") + } + return t.clusterUUID +} + +func (t *fakeTestContext) GetOpenShiftClusterProperties() api.OpenShiftClusterProperties { + if t.clusterUUID == "" { + panic("didn't set up openshiftcluster in test") + } + return t.properties +} + func (t *fakeTestContext) SetResultMessage(s string) { t.resultMessage = s } From fb2301e837006a498f46b164333e8104948df6ed Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Mon, 5 Aug 2024 14:59:45 +1000 Subject: [PATCH 20/67] update for cleanups --- pkg/mimo/actuator/task.go | 13 +++++++++++-- pkg/mimo/tasks/cluster/tls.go | 5 ++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/pkg/mimo/actuator/task.go b/pkg/mimo/actuator/task.go index 21d002e4410..38e8ff6cc6c 100644 --- a/pkg/mimo/actuator/task.go +++ b/pkg/mimo/actuator/task.go @@ -8,6 +8,8 @@ import ( "time" "github.com/sirupsen/logrus" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" "github.com/Azure/ARO-RP/pkg/api" "github.com/Azure/ARO-RP/pkg/env" @@ -85,12 +87,19 @@ func (t *th) ClientHelper() (clienthelper.Interface, error) { return nil, err } - ch, err := clienthelper.New(t.log, restConfig) + mapper, err := apiutil.NewDynamicRESTMapper(restConfig, apiutil.WithLazyDiscovery) if err != nil { return nil, err } - t._ch = ch + client, err := client.New(restConfig, client.Options{ + Mapper: mapper, + }) + if err != nil { + return nil, err + } + + t._ch = clienthelper.NewWithClient(t.log, client) return t._ch, nil } diff --git a/pkg/mimo/tasks/cluster/tls.go b/pkg/mimo/tasks/cluster/tls.go index d291c76fcc8..5501077ed5b 100644 --- a/pkg/mimo/tasks/cluster/tls.go +++ b/pkg/mimo/tasks/cluster/tls.go @@ -28,12 +28,11 @@ func RotateAPIServerCertificate(ctx context.Context) error { } env := th.Environment() - secretName := th.GetClusterUUID() + "-apiserver" for _, namespace := range []string{"openshift-config", "openshift-azure-operator"} { err = cluster.EnsureTLSSecretFromKeyvault( - ctx, env, ch, types.NamespacedName{Namespace: namespace, Name: secretName}, secretName, + ctx, env.ClusterKeyvault(), ch, types.NamespacedName{Namespace: namespace, Name: secretName}, secretName, ) if err != nil { return err @@ -91,7 +90,7 @@ func EnsureAPIServerServingCertificateConfiguration(ctx context.Context) error { }, } - err = ch.Client().Update(ctx, apiserver) + err = ch.Update(ctx, apiserver) if err != nil { if kerrors.IsConflict(err) { return err From c05b87146aee717225ee5476c243a7f252f6707a Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Fri, 16 Aug 2024 14:43:45 +1000 Subject: [PATCH 21/67] move into the main CLI endpoint --- cmd/aro/main.go | 4 + cmd/aro/mimoactuator.go | 98 ++++++++++++++++ pkg/mimo/actuator/service.go | 34 ++++-- pkg/mimo/actuator/service_test.go | 3 +- pkg/mimo/cmd/cli.go | 183 ------------------------------ 5 files changed, 131 insertions(+), 191 deletions(-) create mode 100644 cmd/aro/mimoactuator.go delete mode 100644 pkg/mimo/cmd/cli.go diff --git a/cmd/aro/main.go b/cmd/aro/main.go index e629e8202ab..eea63e22b06 100644 --- a/cmd/aro/main.go +++ b/cmd/aro/main.go @@ -28,6 +28,7 @@ func usage() { fmt.Fprintf(flag.CommandLine.Output(), " %s operator {master,worker}\n", os.Args[0]) fmt.Fprintf(flag.CommandLine.Output(), " %s update-versions\n", os.Args[0]) fmt.Fprintf(flag.CommandLine.Output(), " %s update-role-sets\n", os.Args[0]) + fmt.Fprintf(flag.CommandLine.Output(), " %s mimo-actuator\n", os.Args[0]) flag.PrintDefaults() } @@ -74,6 +75,9 @@ func main() { case "update-role-sets": checkArgs(1) err = updatePlatformWorkloadIdentityRoleSets(ctx, log) + case "mimo-actuator": + checkArgs(1) + err = mimoActuator(ctx, log) default: usage() os.Exit(2) diff --git a/cmd/aro/mimoactuator.go b/cmd/aro/mimoactuator.go new file mode 100644 index 00000000000..c534495e80e --- /dev/null +++ b/cmd/aro/mimoactuator.go @@ -0,0 +1,98 @@ +package main + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "os" + "os/signal" + "syscall" + + "github.com/sirupsen/logrus" + + "github.com/Azure/ARO-RP/pkg/database" + "github.com/Azure/ARO-RP/pkg/env" + "github.com/Azure/ARO-RP/pkg/metrics/statsd" + "github.com/Azure/ARO-RP/pkg/metrics/statsd/golang" + "github.com/Azure/ARO-RP/pkg/mimo/actuator" + "github.com/Azure/ARO-RP/pkg/mimo/sets" + "github.com/Azure/ARO-RP/pkg/proxy" + "github.com/Azure/ARO-RP/pkg/util/service" +) + +func mimoActuator(ctx context.Context, log *logrus.Entry) error { + stop := make(chan struct{}) + + _env, err := env.NewEnv(ctx, log, env.COMPONENT_MIMO_ACTUATOR) + if err != nil { + return err + } + + var keys []string + if _env.IsLocalDevelopmentMode() { + keys = []string{} + } else { + keys = []string{ + "MDM_ACCOUNT", + "MDM_NAMESPACE", + } + } + + if err = env.ValidateVars(keys...); err != nil { + return err + } + + m := statsd.New(ctx, log.WithField("component", "actuator"), _env, os.Getenv("MDM_ACCOUNT"), os.Getenv("MDM_NAMESPACE"), os.Getenv("MDM_STATSD_SOCKET")) + + g, err := golang.NewMetrics(_env.Logger(), m) + if err != nil { + return err + } + go g.Run() + + dbc, err := service.NewDatabase(ctx, _env, log, m, false) + if err != nil { + return err + } + + dbName, err := service.DBName(_env.IsLocalDevelopmentMode()) + if err != nil { + return err + } + + clusters, err := database.NewOpenShiftClusters(ctx, dbc, dbName) + if err != nil { + return err + } + + manifests, err := database.NewMaintenanceManifests(ctx, dbc, dbName) + if err != nil { + return err + } + + dbg := database.NewDBGroup(). + WithOpenShiftClusters(clusters). + WithMaintenanceManifests(manifests) + + dialer, err := proxy.NewDialer(_env.IsLocalDevelopmentMode()) + if err != nil { + return err + } + + a := actuator.NewService(_env, _env.Logger(), dialer, dbg, m) + a.SetMaintenanceSets(sets.DEFAULT_MAINTENANCE_SETS) + + sigterm := make(chan os.Signal, 1) + done := make(chan struct{}) + signal.Notify(sigterm, syscall.SIGTERM) + + go a.Run(ctx, stop, done) + + <-sigterm + log.Print("received SIGTERM") + close(stop) + <-done + + return nil +} diff --git a/pkg/mimo/actuator/service.go b/pkg/mimo/actuator/service.go index 635e55a95f2..c7b36d20ac3 100644 --- a/pkg/mimo/actuator/service.go +++ b/pkg/mimo/actuator/service.go @@ -34,8 +34,7 @@ type service struct { baseLog *logrus.Entry env env.Interface - dbOpenShiftClusters database.OpenShiftClusters - dbMaintenanceManifests database.MaintenanceManifests + dbGroup actuatorDBs m metrics.Emitter mu sync.RWMutex @@ -54,14 +53,18 @@ type service struct { sets map[string]sets.MaintenanceSet } -func NewService(env env.Interface, log *logrus.Entry, dialer proxy.Dialer, dbOpenShiftClusters database.OpenShiftClusters, dbMaintenanceManifests database.MaintenanceManifests, m metrics.Emitter) *service { +type actuatorDBs interface { + database.DatabaseGroupWithOpenShiftClusters + database.DatabaseGroupWithMaintenanceManifests +} + +func NewService(env env.Interface, log *logrus.Entry, dialer proxy.Dialer, dbg actuatorDBs, m metrics.Emitter) *service { s := &service{ env: env, baseLog: log, dialer: dialer, - dbOpenShiftClusters: dbOpenShiftClusters, - dbMaintenanceManifests: dbMaintenanceManifests, + dbGroup: dbg, m: m, stopping: &atomic.Bool{}, @@ -127,8 +130,13 @@ func (s *service) Run(ctx context.Context, stop <-chan struct{}, done chan<- str // Temporary method of updating without the changefeed -- the reason why is // complicated func (s *service) poll(ctx context.Context, oldDocs map[string]*api.OpenShiftClusterDocument) (map[string]*api.OpenShiftClusterDocument, error) { + dbOpenShiftClusters, err := s.dbGroup.OpenShiftClusters() + if err != nil { + return nil, err + } + // Fetch all of the cluster UUIDs - i, err := s.dbOpenShiftClusters.GetAllResourceIDs(ctx, "") + i, err := dbOpenShiftClusters.GetAllResourceIDs(ctx, "") if err != nil { return nil, err } @@ -200,7 +208,19 @@ func (s *service) worker(stop <-chan struct{}, delay time.Duration, id string) { log := utillog.EnrichWithResourceID(s.baseLog, id) - a, err := NewActuator(context.Background(), s.env, log, id, s.dbOpenShiftClusters, s.dbMaintenanceManifests, s.now) + dbOpenShiftClusters, err := s.dbGroup.OpenShiftClusters() + if err != nil { + log.Error(err) + return + } + + dbMaintenanceManifests, err := s.dbGroup.MaintenanceManifests() + if err != nil { + log.Error(err) + return + } + + a, err := NewActuator(context.Background(), s.env, log, id, dbOpenShiftClusters, dbMaintenanceManifests, s.now) if err != nil { log.Error(err) return diff --git a/pkg/mimo/actuator/service_test.go b/pkg/mimo/actuator/service_test.go index e78400298c4..ac893cbea29 100644 --- a/pkg/mimo/actuator/service_test.go +++ b/pkg/mimo/actuator/service_test.go @@ -96,8 +96,9 @@ var _ = Describe("MIMO Actuator Service", Ordered, func() { now := func() time.Time { return time.Unix(120, 0) } manifests, manifestsClient = testdatabase.NewFakeMaintenanceManifests(now) clusters, _ = testdatabase.NewFakeOpenShiftClusters() + dbg := database.NewDBGroup().WithMaintenanceManifests(manifests).WithOpenShiftClusters(clusters) - svc = NewService(_env, log, nil, clusters, manifests, m) + svc = NewService(_env, log, nil, dbg, m) svc.now = now }) diff --git a/pkg/mimo/cmd/cli.go b/pkg/mimo/cmd/cli.go deleted file mode 100644 index ab22142be85..00000000000 --- a/pkg/mimo/cmd/cli.go +++ /dev/null @@ -1,183 +0,0 @@ -package main - -// Copyright (c) Microsoft Corporation. -// Licensed under the Apache License 2.0. - -import ( - "os" - "os/signal" - "syscall" - - "github.com/sirupsen/logrus" - "github.com/urfave/cli/v2" - - "github.com/Azure/ARO-RP/pkg/database" - "github.com/Azure/ARO-RP/pkg/env" - "github.com/Azure/ARO-RP/pkg/metrics/statsd" - "github.com/Azure/ARO-RP/pkg/metrics/statsd/golang" - "github.com/Azure/ARO-RP/pkg/mimo/actuator" - "github.com/Azure/ARO-RP/pkg/mimo/sets" - "github.com/Azure/ARO-RP/pkg/proxy" - utillog "github.com/Azure/ARO-RP/pkg/util/log" - "github.com/Azure/ARO-RP/pkg/util/service" -) - -func main() { - log := utillog.GetLogger() - developmentMode := env.IsLocalDevelopmentMode() - - app := &cli.App{ - Name: "MIMO", - Usage: "Managed Infrastructure Maintenance Operator", - Commands: []*cli.Command{ - { - Name: "scheduler", - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "MDM_ACCOUNT", - EnvVars: []string{"MDM_ACCOUNT"}, - Required: !developmentMode, - }, - &cli.StringFlag{ - Name: "MDM_NAMESPACE", - EnvVars: []string{"MDM_NAMESPACE"}, - Required: !developmentMode, - }, - &cli.StringFlag{ - Name: "MDM_STATSD_SOCKET", - EnvVars: []string{"MDM_STATSD_SOCKET"}, - Required: false, - }, - }, - Action: func(ctx *cli.Context) error { - _env, err := env.NewEnv(ctx.Context, log, env.COMPONENT_MIMO_SCHEDULER) - if err != nil { - return err - } - - m := statsd.New(ctx.Context, log.WithField("component", "scheduler"), _env, os.Getenv("MDM_ACCOUNT"), os.Getenv("MDM_NAMESPACE"), os.Getenv("MDM_STATSD_SOCKET")) - - g, err := golang.NewMetrics(_env.Logger(), m) - if err != nil { - return err - } - go g.Run() - - // dbc, err := service.NewDatabase(ctx.Context, _env, log, m, service.DB_DBTOKEN_PROD_MASTERKEY_DEV, false) - // if err != nil { - // return err - // } - - // dbName, err := service.DBName(_env.IsLocalDevelopmentMode()) - // if err != nil { - // return err - // } - - // clusters, err := database.NewOpenShiftClusters(ctx.Context, dbc, dbName) - // if err != nil { - // return err - // } - - return nil - }, - }, - { - Name: "actuator", - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "MDM_ACCOUNT", - EnvVars: []string{"MDM_ACCOUNT"}, - Required: !developmentMode, - }, - &cli.StringFlag{ - Name: "MDM_NAMESPACE", - EnvVars: []string{"MDM_NAMESPACE"}, - Required: !developmentMode, - }, - &cli.StringFlag{ - Name: "MDM_STATSD_SOCKET", - EnvVars: []string{"MDM_STATSD_SOCKET"}, - Required: false, - }, - &cli.StringFlag{ - Name: "loglevel", - EnvVars: []string{"ARO_LOGLEVEL"}, - DefaultText: "info", - Required: false, - }, - }, - Before: func(ctx *cli.Context) error { - log.Print("MIMO actuator initialising") - return nil - }, - Action: func(ctx *cli.Context) error { - stop := make(chan struct{}) - - l, err := logrus.ParseLevel(ctx.String("loglevel")) - if err != nil { - log.Logger.SetLevel(l) - } - - _env, err := env.NewEnv(ctx.Context, log, env.COMPONENT_MIMO_ACTUATOR) - if err != nil { - return err - } - - m := statsd.New(ctx.Context, log.WithField("component", "actuator"), _env, os.Getenv("MDM_ACCOUNT"), os.Getenv("MDM_NAMESPACE"), os.Getenv("MDM_STATSD_SOCKET")) - - g, err := golang.NewMetrics(_env.Logger(), m) - if err != nil { - return err - } - go g.Run() - - dbc, err := service.NewDatabase(ctx.Context, _env, log, m, false) - if err != nil { - return err - } - - dbName, err := service.DBName(_env.IsLocalDevelopmentMode()) - if err != nil { - return err - } - - clusters, err := database.NewOpenShiftClusters(ctx.Context, dbc, dbName) - if err != nil { - return err - } - - manifests, err := database.NewMaintenanceManifests(ctx.Context, dbc, dbName) - if err != nil { - return err - } - - dialer, err := proxy.NewDialer(_env.IsLocalDevelopmentMode()) - if err != nil { - return err - } - - a := actuator.NewService(_env, _env.Logger(), dialer, clusters, manifests, m) - - a.SetMaintenanceSets(sets.DEFAULT_MAINTENANCE_SETS) - - sigterm := make(chan os.Signal, 1) - done := make(chan struct{}) - signal.Notify(sigterm, syscall.SIGTERM) - - go a.Run(ctx.Context, stop, done) - - <-sigterm - log.Print("received SIGTERM") - close(stop) - //<-done - - return nil - }, - }, - }, - } - - if err := app.Run(os.Args); err != nil { - log.Fatal(err) - } -} From 559caead0748d55b56f15e072b28a029825e867d Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Tue, 17 Sep 2024 15:56:14 +1000 Subject: [PATCH 22/67] add a task for updating the operator flags, for testing --- pkg/mimo/tasks/cluster/operatorflags.go | 55 ++++++++ pkg/mimo/tasks/cluster/operatorflags_test.go | 127 +++++++++++++++++++ 2 files changed, 182 insertions(+) create mode 100644 pkg/mimo/tasks/cluster/operatorflags.go create mode 100644 pkg/mimo/tasks/cluster/operatorflags_test.go diff --git a/pkg/mimo/tasks/cluster/operatorflags.go b/pkg/mimo/tasks/cluster/operatorflags.go new file mode 100644 index 00000000000..652f4392882 --- /dev/null +++ b/pkg/mimo/tasks/cluster/operatorflags.go @@ -0,0 +1,55 @@ +package cluster + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + + arov1alpha1 "github.com/Azure/ARO-RP/pkg/operator/apis/aro.openshift.io/v1alpha1" + "github.com/Azure/ARO-RP/pkg/util/mimo" + kerrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" +) + +// UpdateClusterOperatorFlags updates the OperatorFlags object in the ARO +// Cluster document. +func UpdateClusterOperatorFlags(ctx context.Context) error { + th, err := mimo.GetTaskContext(ctx) + if err != nil { + return mimo.TerminalError(err) + } + + props := th.GetOpenShiftClusterProperties() + + ch, err := th.ClientHelper() + if err != nil { + return mimo.TerminalError(err) + } + + return retry.RetryOnConflict(retry.DefaultRetry, func() error { + clusterObj := &arov1alpha1.Cluster{} + + err = ch.GetOne(ctx, types.NamespacedName{Name: arov1alpha1.SingletonClusterName}, clusterObj) + if err != nil { + if kerrors.IsNotFound(err) { + // cluster doc being gone is unrecoverable + return mimo.TerminalError(err) + } + return mimo.TransientError(err) + } + + clusterObj.Spec.OperatorFlags = arov1alpha1.OperatorFlags(props.OperatorFlags) + + err = ch.Update(ctx, clusterObj) + if err != nil { + if kerrors.IsConflict(err) { + return err + } else { + return mimo.TransientError(err) + } + } + return nil + }) +} diff --git a/pkg/mimo/tasks/cluster/operatorflags_test.go b/pkg/mimo/tasks/cluster/operatorflags_test.go new file mode 100644 index 00000000000..a9f355601f1 --- /dev/null +++ b/pkg/mimo/tasks/cluster/operatorflags_test.go @@ -0,0 +1,127 @@ +package cluster + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "testing" + "time" + + "github.com/go-test/deep" + . "github.com/onsi/gomega" + + "github.com/golang/mock/gomock" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/Azure/ARO-RP/pkg/api" + arov1alpha1 "github.com/Azure/ARO-RP/pkg/operator/apis/aro.openshift.io/v1alpha1" + "github.com/Azure/ARO-RP/pkg/util/clienthelper" + mock_env "github.com/Azure/ARO-RP/pkg/util/mocks/env" + testtasks "github.com/Azure/ARO-RP/test/mimo/tasks" + testclienthelper "github.com/Azure/ARO-RP/test/util/clienthelper" + testlog "github.com/Azure/ARO-RP/test/util/log" +) + +func TestOperatorFlags(t *testing.T) { + ctx := context.Background() + + for _, tt := range []struct { + name string + objects []runtime.Object + wantObjects []runtime.Object + wantErr string + }{ + { + name: "not found", + objects: []runtime.Object{}, + wantErr: `TerminalError: clusters.aro.openshift.io "cluster" not found`, + }, + { + name: "not ready", + objects: []runtime.Object{ + &arov1alpha1.Cluster{ + ObjectMeta: v1.ObjectMeta{ + Name: arov1alpha1.SingletonClusterName, + ResourceVersion: "1000", + }, + Spec: arov1alpha1.ClusterSpec{ + OperatorFlags: arov1alpha1.OperatorFlags{ + "foo": "bar", + }, + }, + }, + }, + wantObjects: []runtime.Object{ + &arov1alpha1.Cluster{ + ObjectMeta: v1.ObjectMeta{ + Name: arov1alpha1.SingletonClusterName, + ResourceVersion: "1001", + }, + TypeMeta: v1.TypeMeta{ + Kind: "Cluster", + APIVersion: arov1alpha1.SchemeGroupVersion.String(), + }, + Spec: arov1alpha1.ClusterSpec{ + OperatorFlags: arov1alpha1.OperatorFlags{ + "foo": "baz", + "gaz": "data", + }, + }, + }, + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + controller := gomock.NewController(t) + _env := mock_env.NewMockInterface(controller) + _, log := testlog.New() + + ocDoc := &api.OpenShiftClusterDocument{ + ID: "0000", + OpenShiftCluster: &api.OpenShiftCluster{ + Properties: api.OpenShiftClusterProperties{ + OperatorFlags: api.OperatorFlags{ + "foo": "baz", + "gaz": "data", + }, + }, + }, + } + + builder := fake.NewClientBuilder().WithRuntimeObjects(tt.objects...) + ch := clienthelper.NewWithClient(log, testclienthelper.NewHookingClient(builder.Build())) + tc := testtasks.NewFakeTestContext( + ctx, _env, log, func() time.Time { return time.Unix(100, 0) }, + testtasks.WithClientHelper(ch), testtasks.WithOpenShiftClusterDocument(ocDoc), + ) + + err := UpdateClusterOperatorFlags(tc) + if tt.wantErr != "" && err != nil { + g.Expect(err).To(MatchError(tt.wantErr)) + } else if tt.wantErr != "" && err == nil { + t.Errorf("wanted error %s", tt.wantErr) + } else if tt.wantErr == "" { + g.Expect(err).ToNot(HaveOccurred()) + } + + if len(tt.wantObjects) > 0 { + for _, i := range tt.wantObjects { + o, err := scheme.Scheme.New(i.GetObjectKind().GroupVersionKind()) + g.Expect(err).ToNot(HaveOccurred()) + + err = ch.GetOne(ctx, client.ObjectKeyFromObject(i.(client.Object)), o) + g.Expect(err).ToNot(HaveOccurred()) + + r := deep.Equal(i, o) + g.Expect(r).To(HaveLen(0)) + } + } + }) + } +} From 2e81eace894688f69d351ce3f21bcc07976cc22f Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 18 Sep 2024 11:08:38 +1000 Subject: [PATCH 23/67] add healthz endpoints for MIMO actuator --- pkg/env/dev.go | 3 ++ pkg/mimo/actuator/service.go | 55 +++++++++++++++++++++++++++++-- pkg/mimo/actuator/service_test.go | 3 +- 3 files changed, 57 insertions(+), 4 deletions(-) diff --git a/pkg/env/dev.go b/pkg/env/dev.go index 1dee4514c63..d4f0cf13752 100644 --- a/pkg/env/dev.go +++ b/pkg/env/dev.go @@ -77,6 +77,9 @@ func (d *dev) AROOperatorImage() string { } func (d *dev) Listen() (net.Listener, error) { + if d.Component() == string(COMPONENT_MIMO_ACTUATOR) { + return net.Listen("tcp", ":8445") + } return net.Listen("tcp", ":8443") } diff --git a/pkg/mimo/actuator/service.go b/pkg/mimo/actuator/service.go index c7b36d20ac3..6c103141386 100644 --- a/pkg/mimo/actuator/service.go +++ b/pkg/mimo/actuator/service.go @@ -5,6 +5,10 @@ package actuator import ( "context" + "errors" + "log" + "net" + "net/http" "strings" "sync" "sync/atomic" @@ -12,6 +16,7 @@ import ( "github.com/sirupsen/logrus" "golang.org/x/exp/maps" + "sigs.k8s.io/controller-runtime/pkg/healthz" "github.com/Azure/ARO-RP/pkg/api" "github.com/Azure/ARO-RP/pkg/database" @@ -51,6 +56,8 @@ type service struct { now func() time.Time sets map[string]sets.MaintenanceSet + + serveHealthz bool } type actuatorDBs interface { @@ -73,6 +80,8 @@ func NewService(env env.Interface, log *logrus.Entry, dialer proxy.Dialer, dbg a startTime: time.Now(), now: time.Now, pollTime: time.Minute, + + serveHealthz: true, } s.b = buckets.NewBucketWorker(log, s.worker, &s.mu) @@ -87,6 +96,43 @@ func (s *service) SetMaintenanceSets(sets map[string]sets.MaintenanceSet) { func (s *service) Run(ctx context.Context, stop <-chan struct{}, done chan<- struct{}) error { defer recover.Panic(s.baseLog) + // Only enable the healthz endpoint if configured (disabled in unit tests) + if s.serveHealthz { + c := &healthz.Handler{ + Checks: map[string]healthz.Checker{ + "ready": func(h *http.Request) error { + if !s.checkReady() { + return errors.New("not ready") + } + return nil + }, + }, + } + + m := http.NewServeMux() + m.Handle("/healthz", http.StripPrefix("/healthz", c)) + // Handle healthz subpaths + m.Handle("/healthz/", http.StripPrefix("/healthz", c)) + + h := &http.Server{ + Handler: m, + ErrorLog: log.New(s.baseLog.Writer(), "", 0), + BaseContext: func(net.Listener) context.Context { return ctx }, + } + + listener, err := s.env.Listen() + if err != nil { + return err + } + + go func() { + err := h.Serve(listener) + if err != http.ErrServerClosed { + s.baseLog.Error(err) + } + }() + } + t := time.NewTicker(10 * time.Second) defer t.Stop() @@ -196,8 +242,13 @@ func (s *service) checkReady() bool { if !ok { return false } - return (time.Since(lastChangefeedTime) < time.Minute) && // did we update our list of clusters recently? - (time.Since(s.startTime) > 2*time.Minute) // are we running for at least 2 minutes? + + if s.env.IsLocalDevelopmentMode() { + return (time.Since(lastChangefeedTime) < time.Minute) // did we update our list of clusters recently? + } else { + return (time.Since(lastChangefeedTime) < time.Minute) && // did we update our list of clusters recently? + (time.Since(s.startTime) > 2*time.Minute) // are we running for at least 2 minutes? + } } func (s *service) worker(stop <-chan struct{}, delay time.Duration, id string) { diff --git a/pkg/mimo/actuator/service_test.go b/pkg/mimo/actuator/service_test.go index ac893cbea29..d4bf43a153c 100644 --- a/pkg/mimo/actuator/service_test.go +++ b/pkg/mimo/actuator/service_test.go @@ -100,7 +100,7 @@ var _ = Describe("MIMO Actuator Service", Ordered, func() { svc = NewService(_env, log, nil, dbg, m) svc.now = now - + svc.serveHealthz = false }) JustBeforeEach(func() { @@ -249,5 +249,4 @@ var _ = Describe("MIMO Actuator Service", Ordered, func() { Expect(errs).To(BeNil(), fmt.Sprintf("%v", errs)) }) }) - }) From ae325ad07eb31ff4b1169c3ff630e490552add68 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 18 Sep 2024 11:08:51 +1000 Subject: [PATCH 24/67] makefile target for running actuator locally --- Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Makefile b/Makefile index 36c95bff746..931001ff2d0 100644 --- a/Makefile +++ b/Makefile @@ -198,6 +198,10 @@ proxy: runlocal-portal: go run -ldflags "-X github.com/Azure/ARO-RP/pkg/util/version.GitCommit=$(VERSION)" ./cmd/aro portal +.PHONY: runlocal-actuator +runlocal-actuator: + go run -ldflags "-X github.com/Azure/ARO-RP/pkg/util/version.GitCommit=$(VERSION)" ./cmd/aro mimo-actuator + .PHONY: build-portal build-portal: cd portal/v2 && npm install && npm run build From 9075618827c2237c5b687d62fde147887195708c Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 18 Sep 2024 11:10:47 +1000 Subject: [PATCH 25/67] add mimo actuator steps in e2e helper --- hack/e2e/run-rp-and-e2e.sh | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/hack/e2e/run-rp-and-e2e.sh b/hack/e2e/run-rp-and-e2e.sh index c20f67c4c6f..a802637db25 100755 --- a/hack/e2e/run-rp-and-e2e.sh +++ b/hack/e2e/run-rp-and-e2e.sh @@ -91,6 +91,43 @@ kill_portal() { wait $rppid } +run_mimo_actuator() { + echo "########## 🚀 Run MIMO Actuator in background ##########" + export AZURE_ENVIRONMENT=AzurePublicCloud + ./aro mimo-actuator & +} + +kill_mimo_actuator() { + echo "########## Kill the MIMO Actuator running in background ##########" + rppid=$(lsof -t -i :8445) + kill $rppid + wait $rppid +} + +validate_mimo_actuator_running() { + echo "########## ？Checking MIMO Actuator Status ##########" + ELAPSED=0 + while true; do + sleep 5 + http_code=$(curl -k -s -o /dev/null -w '%{http_code}' http://localhost:8445/healthz/ready) + case $http_code in + "200") + echo "########## ✅ ARO MIMO Actuator Running ##########" + break + ;; + *) + echo "Attempt $ELAPSED - local MIMO Actuator is NOT up. Code : $http_code, waiting" + sleep 2 + # after 40 secs return exit 1 to not block ci + ELAPSED=$((ELAPSED + 1)) + if [ $ELAPSED -eq 20 ]; then + exit 1 + fi + ;; + esac + done +} + run_vpn() { echo "########## 🚀 Run OpenVPN in background ##########" echo "Using Secret secrets/$VPN" From 90832a05f60909e7c6c2a6796b5f87cce5fb5667 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 18 Sep 2024 11:14:32 +1000 Subject: [PATCH 26/67] start mimo in e2e --- .pipelines/e2e.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.pipelines/e2e.yml b/.pipelines/e2e.yml index cc7cc47feab..db29add2a91 100644 --- a/.pipelines/e2e.yml +++ b/.pipelines/e2e.yml @@ -70,6 +70,8 @@ jobs: - script: | export CI=true + # Tell the E2E binary to run the MIMO tests + export CI_E2E_MIMO=true . secrets/env . ./hack/e2e/run-rp-and-e2e.sh @@ -84,6 +86,9 @@ jobs: run_selenium validate_selenium_running + run_mimo_actuator + validate_mimo_actuator_running + run_rp validate_rp_running @@ -128,6 +133,7 @@ jobs: delete_e2e_cluster kill_rp + kill_mimo_actuator kill_selenium kill_podman kill_vpn From 38c6f11029da836a016a19c33544d164d2ed74c4 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 18 Sep 2024 12:25:34 +1000 Subject: [PATCH 27/67] fix build --- pkg/deploy/generator/resources_rp.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/deploy/generator/resources_rp.go b/pkg/deploy/generator/resources_rp.go index 1cb2941f143..2fb56b9af25 100644 --- a/pkg/deploy/generator/resources_rp.go +++ b/pkg/deploy/generator/resources_rp.go @@ -1295,8 +1295,6 @@ func (g *generator) database(databaseName string, addDependsOn bool) []*arm.Reso g.rpCosmosDBTriggers(databaseName, "OpenShiftClusters", "renewLease", renewLeaseTriggerFunction, sdkcosmos.TriggerTypePre, sdkcosmos.TriggerOperationAll), // Monitors g.rpCosmosDBTriggers(databaseName, "Monitors", "renewLease", renewLeaseTriggerFunction, sdkcosmos.TriggerTypePre, sdkcosmos.TriggerOperationAll), - // MIMO - g.rpCosmosDBTriggers(databaseName, "MaintenanceManifests", "renewLease", renewLeaseTriggerFunction, sdkcosmos.TriggerTypePre, sdkcosmos.TriggerOperationAll), ) // Don't deploy the MIMO databases in production yet From 4655b11250ad19ca91d2716fd4e60d45c37e2717 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 18 Sep 2024 12:29:17 +1000 Subject: [PATCH 28/67] go generate --- ...z_generated_maintenancemanifestdocument.go | 313 +++++++++++++++ ...erated_maintenancemanifestdocument_fake.go | 361 ++++++++++++++++++ pkg/deploy/assets/databases-development.json | 40 ++ 3 files changed, 714 insertions(+) create mode 100644 pkg/database/cosmosdb/zz_generated_maintenancemanifestdocument.go create mode 100644 pkg/database/cosmosdb/zz_generated_maintenancemanifestdocument_fake.go diff --git a/pkg/database/cosmosdb/zz_generated_maintenancemanifestdocument.go b/pkg/database/cosmosdb/zz_generated_maintenancemanifestdocument.go new file mode 100644 index 00000000000..1aa237d5d06 --- /dev/null +++ b/pkg/database/cosmosdb/zz_generated_maintenancemanifestdocument.go @@ -0,0 +1,313 @@ +// Code generated by github.com/jewzaam/go-cosmosdb, DO NOT EDIT. + +package cosmosdb + +import ( + "context" + "net/http" + "strconv" + "strings" + + pkg "github.com/Azure/ARO-RP/pkg/api" +) + +type maintenanceManifestDocumentClient struct { + *databaseClient + path string +} + +// MaintenanceManifestDocumentClient is a maintenanceManifestDocument client +type MaintenanceManifestDocumentClient interface { + Create(context.Context, string, *pkg.MaintenanceManifestDocument, *Options) (*pkg.MaintenanceManifestDocument, error) + List(*Options) MaintenanceManifestDocumentIterator + ListAll(context.Context, *Options) (*pkg.MaintenanceManifestDocuments, error) + Get(context.Context, string, string, *Options) (*pkg.MaintenanceManifestDocument, error) + Replace(context.Context, string, *pkg.MaintenanceManifestDocument, *Options) (*pkg.MaintenanceManifestDocument, error) + Delete(context.Context, string, *pkg.MaintenanceManifestDocument, *Options) error + Query(string, *Query, *Options) MaintenanceManifestDocumentRawIterator + QueryAll(context.Context, string, *Query, *Options) (*pkg.MaintenanceManifestDocuments, error) + ChangeFeed(*Options) MaintenanceManifestDocumentIterator +} + +type maintenanceManifestDocumentChangeFeedIterator struct { + *maintenanceManifestDocumentClient + continuation string + options *Options +} + +type maintenanceManifestDocumentListIterator struct { + *maintenanceManifestDocumentClient + continuation string + done bool + options *Options +} + +type maintenanceManifestDocumentQueryIterator struct { + *maintenanceManifestDocumentClient + partitionkey string + query *Query + continuation string + done bool + options *Options +} + +// MaintenanceManifestDocumentIterator is a maintenanceManifestDocument iterator +type MaintenanceManifestDocumentIterator interface { + Next(context.Context, int) (*pkg.MaintenanceManifestDocuments, error) + Continuation() string +} + +// MaintenanceManifestDocumentRawIterator is a maintenanceManifestDocument raw iterator +type MaintenanceManifestDocumentRawIterator interface { + MaintenanceManifestDocumentIterator + NextRaw(context.Context, int, interface{}) error +} + +// NewMaintenanceManifestDocumentClient returns a new maintenanceManifestDocument client +func NewMaintenanceManifestDocumentClient(collc CollectionClient, collid string) MaintenanceManifestDocumentClient { + return &maintenanceManifestDocumentClient{ + databaseClient: collc.(*collectionClient).databaseClient, + path: collc.(*collectionClient).path + "/colls/" + collid, + } +} + +func (c *maintenanceManifestDocumentClient) all(ctx context.Context, i MaintenanceManifestDocumentIterator) (*pkg.MaintenanceManifestDocuments, error) { + allmaintenanceManifestDocuments := &pkg.MaintenanceManifestDocuments{} + + for { + maintenanceManifestDocuments, err := i.Next(ctx, -1) + if err != nil { + return nil, err + } + if maintenanceManifestDocuments == nil { + break + } + + allmaintenanceManifestDocuments.Count += maintenanceManifestDocuments.Count + allmaintenanceManifestDocuments.ResourceID = maintenanceManifestDocuments.ResourceID + allmaintenanceManifestDocuments.MaintenanceManifestDocuments = append(allmaintenanceManifestDocuments.MaintenanceManifestDocuments, maintenanceManifestDocuments.MaintenanceManifestDocuments...) + } + + return allmaintenanceManifestDocuments, nil +} + +func (c *maintenanceManifestDocumentClient) Create(ctx context.Context, partitionkey string, newmaintenanceManifestDocument *pkg.MaintenanceManifestDocument, options *Options) (maintenanceManifestDocument *pkg.MaintenanceManifestDocument, err error) { + headers := http.Header{} + headers.Set("X-Ms-Documentdb-Partitionkey", `["`+partitionkey+`"]`) + + if options == nil { + options = &Options{} + } + options.NoETag = true + + err = c.setOptions(options, newmaintenanceManifestDocument, headers) + if err != nil { + return + } + + err = c.do(ctx, http.MethodPost, c.path+"/docs", "docs", c.path, http.StatusCreated, &newmaintenanceManifestDocument, &maintenanceManifestDocument, headers) + return +} + +func (c *maintenanceManifestDocumentClient) List(options *Options) MaintenanceManifestDocumentIterator { + continuation := "" + if options != nil { + continuation = options.Continuation + } + + return &maintenanceManifestDocumentListIterator{maintenanceManifestDocumentClient: c, options: options, continuation: continuation} +} + +func (c *maintenanceManifestDocumentClient) ListAll(ctx context.Context, options *Options) (*pkg.MaintenanceManifestDocuments, error) { + return c.all(ctx, c.List(options)) +} + +func (c *maintenanceManifestDocumentClient) Get(ctx context.Context, partitionkey, maintenanceManifestDocumentid string, options *Options) (maintenanceManifestDocument *pkg.MaintenanceManifestDocument, err error) { + headers := http.Header{} + headers.Set("X-Ms-Documentdb-Partitionkey", `["`+partitionkey+`"]`) + + err = c.setOptions(options, nil, headers) + if err != nil { + return + } + + err = c.do(ctx, http.MethodGet, c.path+"/docs/"+maintenanceManifestDocumentid, "docs", c.path+"/docs/"+maintenanceManifestDocumentid, http.StatusOK, nil, &maintenanceManifestDocument, headers) + return +} + +func (c *maintenanceManifestDocumentClient) Replace(ctx context.Context, partitionkey string, newmaintenanceManifestDocument *pkg.MaintenanceManifestDocument, options *Options) (maintenanceManifestDocument *pkg.MaintenanceManifestDocument, err error) { + headers := http.Header{} + headers.Set("X-Ms-Documentdb-Partitionkey", `["`+partitionkey+`"]`) + + err = c.setOptions(options, newmaintenanceManifestDocument, headers) + if err != nil { + return + } + + err = c.do(ctx, http.MethodPut, c.path+"/docs/"+newmaintenanceManifestDocument.ID, "docs", c.path+"/docs/"+newmaintenanceManifestDocument.ID, http.StatusOK, &newmaintenanceManifestDocument, &maintenanceManifestDocument, headers) + return +} + +func (c *maintenanceManifestDocumentClient) Delete(ctx context.Context, partitionkey string, maintenanceManifestDocument *pkg.MaintenanceManifestDocument, options *Options) (err error) { + headers := http.Header{} + headers.Set("X-Ms-Documentdb-Partitionkey", `["`+partitionkey+`"]`) + + err = c.setOptions(options, maintenanceManifestDocument, headers) + if err != nil { + return + } + + err = c.do(ctx, http.MethodDelete, c.path+"/docs/"+maintenanceManifestDocument.ID, "docs", c.path+"/docs/"+maintenanceManifestDocument.ID, http.StatusNoContent, nil, nil, headers) + return +} + +func (c *maintenanceManifestDocumentClient) Query(partitionkey string, query *Query, options *Options) MaintenanceManifestDocumentRawIterator { + continuation := "" + if options != nil { + continuation = options.Continuation + } + + return &maintenanceManifestDocumentQueryIterator{maintenanceManifestDocumentClient: c, partitionkey: partitionkey, query: query, options: options, continuation: continuation} +} + +func (c *maintenanceManifestDocumentClient) QueryAll(ctx context.Context, partitionkey string, query *Query, options *Options) (*pkg.MaintenanceManifestDocuments, error) { + return c.all(ctx, c.Query(partitionkey, query, options)) +} + +func (c *maintenanceManifestDocumentClient) ChangeFeed(options *Options) MaintenanceManifestDocumentIterator { + continuation := "" + if options != nil { + continuation = options.Continuation + } + + return &maintenanceManifestDocumentChangeFeedIterator{maintenanceManifestDocumentClient: c, options: options, continuation: continuation} +} + +func (c *maintenanceManifestDocumentClient) setOptions(options *Options, maintenanceManifestDocument *pkg.MaintenanceManifestDocument, headers http.Header) error { + if options == nil { + return nil + } + + if maintenanceManifestDocument != nil && !options.NoETag { + if maintenanceManifestDocument.ETag == "" { + return ErrETagRequired + } + headers.Set("If-Match", maintenanceManifestDocument.ETag) + } + if len(options.PreTriggers) > 0 { + headers.Set("X-Ms-Documentdb-Pre-Trigger-Include", strings.Join(options.PreTriggers, ",")) + } + if len(options.PostTriggers) > 0 { + headers.Set("X-Ms-Documentdb-Post-Trigger-Include", strings.Join(options.PostTriggers, ",")) + } + if len(options.PartitionKeyRangeID) > 0 { + headers.Set("X-Ms-Documentdb-PartitionKeyRangeID", options.PartitionKeyRangeID) + } + + return nil +} + +func (i *maintenanceManifestDocumentChangeFeedIterator) Next(ctx context.Context, maxItemCount int) (maintenanceManifestDocuments *pkg.MaintenanceManifestDocuments, err error) { + headers := http.Header{} + headers.Set("A-IM", "Incremental feed") + + headers.Set("X-Ms-Max-Item-Count", strconv.Itoa(maxItemCount)) + if i.continuation != "" { + headers.Set("If-None-Match", i.continuation) + } + + err = i.setOptions(i.options, nil, headers) + if err != nil { + return + } + + err = i.do(ctx, http.MethodGet, i.path+"/docs", "docs", i.path, http.StatusOK, nil, &maintenanceManifestDocuments, headers) + if IsErrorStatusCode(err, http.StatusNotModified) { + err = nil + } + if err != nil { + return + } + + i.continuation = headers.Get("Etag") + + return +} + +func (i *maintenanceManifestDocumentChangeFeedIterator) Continuation() string { + return i.continuation +} + +func (i *maintenanceManifestDocumentListIterator) Next(ctx context.Context, maxItemCount int) (maintenanceManifestDocuments *pkg.MaintenanceManifestDocuments, err error) { + if i.done { + return + } + + headers := http.Header{} + headers.Set("X-Ms-Max-Item-Count", strconv.Itoa(maxItemCount)) + if i.continuation != "" { + headers.Set("X-Ms-Continuation", i.continuation) + } + + err = i.setOptions(i.options, nil, headers) + if err != nil { + return + } + + err = i.do(ctx, http.MethodGet, i.path+"/docs", "docs", i.path, http.StatusOK, nil, &maintenanceManifestDocuments, headers) + if err != nil { + return + } + + i.continuation = headers.Get("X-Ms-Continuation") + i.done = i.continuation == "" + + return +} + +func (i *maintenanceManifestDocumentListIterator) Continuation() string { + return i.continuation +} + +func (i *maintenanceManifestDocumentQueryIterator) Next(ctx context.Context, maxItemCount int) (maintenanceManifestDocuments *pkg.MaintenanceManifestDocuments, err error) { + err = i.NextRaw(ctx, maxItemCount, &maintenanceManifestDocuments) + return +} + +func (i *maintenanceManifestDocumentQueryIterator) NextRaw(ctx context.Context, maxItemCount int, raw interface{}) (err error) { + if i.done { + return + } + + headers := http.Header{} + headers.Set("X-Ms-Max-Item-Count", strconv.Itoa(maxItemCount)) + headers.Set("X-Ms-Documentdb-Isquery", "True") + headers.Set("Content-Type", "application/query+json") + if i.partitionkey != "" { + headers.Set("X-Ms-Documentdb-Partitionkey", `["`+i.partitionkey+`"]`) + } else { + headers.Set("X-Ms-Documentdb-Query-Enablecrosspartition", "True") + } + if i.continuation != "" { + headers.Set("X-Ms-Continuation", i.continuation) + } + + err = i.setOptions(i.options, nil, headers) + if err != nil { + return + } + + err = i.do(ctx, http.MethodPost, i.path+"/docs", "docs", i.path, http.StatusOK, &i.query, &raw, headers) + if err != nil { + return + } + + i.continuation = headers.Get("X-Ms-Continuation") + i.done = i.continuation == "" + + return +} + +func (i *maintenanceManifestDocumentQueryIterator) Continuation() string { + return i.continuation +} diff --git a/pkg/database/cosmosdb/zz_generated_maintenancemanifestdocument_fake.go b/pkg/database/cosmosdb/zz_generated_maintenancemanifestdocument_fake.go new file mode 100644 index 00000000000..b6a6a20413f --- /dev/null +++ b/pkg/database/cosmosdb/zz_generated_maintenancemanifestdocument_fake.go @@ -0,0 +1,361 @@ +// Code generated by github.com/jewzaam/go-cosmosdb, DO NOT EDIT. + +package cosmosdb + +import ( + "context" + "fmt" + "net/http" + "sync" + + "github.com/ugorji/go/codec" + + pkg "github.com/Azure/ARO-RP/pkg/api" +) + +type fakeMaintenanceManifestDocumentTriggerHandler func(context.Context, *pkg.MaintenanceManifestDocument) error +type fakeMaintenanceManifestDocumentQueryHandler func(MaintenanceManifestDocumentClient, *Query, *Options) MaintenanceManifestDocumentRawIterator + +var _ MaintenanceManifestDocumentClient = &FakeMaintenanceManifestDocumentClient{} + +// NewFakeMaintenanceManifestDocumentClient returns a FakeMaintenanceManifestDocumentClient +func NewFakeMaintenanceManifestDocumentClient(h *codec.JsonHandle) *FakeMaintenanceManifestDocumentClient { + return &FakeMaintenanceManifestDocumentClient{ + jsonHandle: h, + maintenanceManifestDocuments: make(map[string]*pkg.MaintenanceManifestDocument), + triggerHandlers: make(map[string]fakeMaintenanceManifestDocumentTriggerHandler), + queryHandlers: make(map[string]fakeMaintenanceManifestDocumentQueryHandler), + } +} + +// FakeMaintenanceManifestDocumentClient is a FakeMaintenanceManifestDocumentClient +type FakeMaintenanceManifestDocumentClient struct { + lock sync.RWMutex + jsonHandle *codec.JsonHandle + maintenanceManifestDocuments map[string]*pkg.MaintenanceManifestDocument + triggerHandlers map[string]fakeMaintenanceManifestDocumentTriggerHandler + queryHandlers map[string]fakeMaintenanceManifestDocumentQueryHandler + sorter func([]*pkg.MaintenanceManifestDocument) + etag int + + // returns true if documents conflict + conflictChecker func(*pkg.MaintenanceManifestDocument, *pkg.MaintenanceManifestDocument) bool + + // err, if not nil, is an error to return when attempting to communicate + // with this Client + err error +} + +// SetError sets or unsets an error that will be returned on any +// FakeMaintenanceManifestDocumentClient method invocation +func (c *FakeMaintenanceManifestDocumentClient) SetError(err error) { + c.lock.Lock() + defer c.lock.Unlock() + + c.err = err +} + +// SetSorter sets or unsets a sorter function which will be used to sort values +// returned by List() for test stability +func (c *FakeMaintenanceManifestDocumentClient) SetSorter(sorter func([]*pkg.MaintenanceManifestDocument)) { + c.lock.Lock() + defer c.lock.Unlock() + + c.sorter = sorter +} + +// SetConflictChecker sets or unsets a function which can be used to validate +// additional unique keys in a MaintenanceManifestDocument +func (c *FakeMaintenanceManifestDocumentClient) SetConflictChecker(conflictChecker func(*pkg.MaintenanceManifestDocument, *pkg.MaintenanceManifestDocument) bool) { + c.lock.Lock() + defer c.lock.Unlock() + + c.conflictChecker = conflictChecker +} + +// SetTriggerHandler sets or unsets a trigger handler +func (c *FakeMaintenanceManifestDocumentClient) SetTriggerHandler(triggerName string, trigger fakeMaintenanceManifestDocumentTriggerHandler) { + c.lock.Lock() + defer c.lock.Unlock() + + c.triggerHandlers[triggerName] = trigger +} + +// SetQueryHandler sets or unsets a query handler +func (c *FakeMaintenanceManifestDocumentClient) SetQueryHandler(queryName string, query fakeMaintenanceManifestDocumentQueryHandler) { + c.lock.Lock() + defer c.lock.Unlock() + + c.queryHandlers[queryName] = query +} + +func (c *FakeMaintenanceManifestDocumentClient) deepCopy(maintenanceManifestDocument *pkg.MaintenanceManifestDocument) (*pkg.MaintenanceManifestDocument, error) { + var b []byte + err := codec.NewEncoderBytes(&b, c.jsonHandle).Encode(maintenanceManifestDocument) + if err != nil { + return nil, err + } + + maintenanceManifestDocument = nil + err = codec.NewDecoderBytes(b, c.jsonHandle).Decode(&maintenanceManifestDocument) + if err != nil { + return nil, err + } + + return maintenanceManifestDocument, nil +} + +func (c *FakeMaintenanceManifestDocumentClient) apply(ctx context.Context, partitionkey string, maintenanceManifestDocument *pkg.MaintenanceManifestDocument, options *Options, isCreate bool) (*pkg.MaintenanceManifestDocument, error) { + c.lock.Lock() + defer c.lock.Unlock() + + if c.err != nil { + return nil, c.err + } + + maintenanceManifestDocument, err := c.deepCopy(maintenanceManifestDocument) // copy now because pretriggers can mutate maintenanceManifestDocument + if err != nil { + return nil, err + } + + if options != nil { + err := c.processPreTriggers(ctx, maintenanceManifestDocument, options) + if err != nil { + return nil, err + } + } + + existingMaintenanceManifestDocument, exists := c.maintenanceManifestDocuments[maintenanceManifestDocument.ID] + if isCreate && exists { + return nil, &Error{ + StatusCode: http.StatusConflict, + Message: "Entity with the specified id already exists in the system", + } + } + if !isCreate { + if !exists { + return nil, &Error{StatusCode: http.StatusNotFound} + } + + if maintenanceManifestDocument.ETag != existingMaintenanceManifestDocument.ETag { + return nil, &Error{StatusCode: http.StatusPreconditionFailed} + } + } + + if c.conflictChecker != nil { + for _, maintenanceManifestDocumentToCheck := range c.maintenanceManifestDocuments { + if c.conflictChecker(maintenanceManifestDocumentToCheck, maintenanceManifestDocument) { + return nil, &Error{ + StatusCode: http.StatusConflict, + Message: "Entity with the specified id already exists in the system", + } + } + } + } + + maintenanceManifestDocument.ETag = fmt.Sprint(c.etag) + c.etag++ + + c.maintenanceManifestDocuments[maintenanceManifestDocument.ID] = maintenanceManifestDocument + + return c.deepCopy(maintenanceManifestDocument) +} + +// Create creates a MaintenanceManifestDocument in the database +func (c *FakeMaintenanceManifestDocumentClient) Create(ctx context.Context, partitionkey string, maintenanceManifestDocument *pkg.MaintenanceManifestDocument, options *Options) (*pkg.MaintenanceManifestDocument, error) { + return c.apply(ctx, partitionkey, maintenanceManifestDocument, options, true) +} + +// Replace replaces a MaintenanceManifestDocument in the database +func (c *FakeMaintenanceManifestDocumentClient) Replace(ctx context.Context, partitionkey string, maintenanceManifestDocument *pkg.MaintenanceManifestDocument, options *Options) (*pkg.MaintenanceManifestDocument, error) { + return c.apply(ctx, partitionkey, maintenanceManifestDocument, options, false) +} + +// List returns a MaintenanceManifestDocumentIterator to list all MaintenanceManifestDocuments in the database +func (c *FakeMaintenanceManifestDocumentClient) List(*Options) MaintenanceManifestDocumentIterator { + c.lock.RLock() + defer c.lock.RUnlock() + + if c.err != nil { + return NewFakeMaintenanceManifestDocumentErroringRawIterator(c.err) + } + + maintenanceManifestDocuments := make([]*pkg.MaintenanceManifestDocument, 0, len(c.maintenanceManifestDocuments)) + for _, maintenanceManifestDocument := range c.maintenanceManifestDocuments { + maintenanceManifestDocument, err := c.deepCopy(maintenanceManifestDocument) + if err != nil { + return NewFakeMaintenanceManifestDocumentErroringRawIterator(err) + } + maintenanceManifestDocuments = append(maintenanceManifestDocuments, maintenanceManifestDocument) + } + + if c.sorter != nil { + c.sorter(maintenanceManifestDocuments) + } + + return NewFakeMaintenanceManifestDocumentIterator(maintenanceManifestDocuments, 0) +} + +// ListAll lists all MaintenanceManifestDocuments in the database +func (c *FakeMaintenanceManifestDocumentClient) ListAll(ctx context.Context, options *Options) (*pkg.MaintenanceManifestDocuments, error) { + iter := c.List(options) + return iter.Next(ctx, -1) +} + +// Get gets a MaintenanceManifestDocument from the database +func (c *FakeMaintenanceManifestDocumentClient) Get(ctx context.Context, partitionkey string, id string, options *Options) (*pkg.MaintenanceManifestDocument, error) { + c.lock.RLock() + defer c.lock.RUnlock() + + if c.err != nil { + return nil, c.err + } + + maintenanceManifestDocument, exists := c.maintenanceManifestDocuments[id] + if !exists { + return nil, &Error{StatusCode: http.StatusNotFound} + } + + return c.deepCopy(maintenanceManifestDocument) +} + +// Delete deletes a MaintenanceManifestDocument from the database +func (c *FakeMaintenanceManifestDocumentClient) Delete(ctx context.Context, partitionKey string, maintenanceManifestDocument *pkg.MaintenanceManifestDocument, options *Options) error { + c.lock.Lock() + defer c.lock.Unlock() + + if c.err != nil { + return c.err + } + + _, exists := c.maintenanceManifestDocuments[maintenanceManifestDocument.ID] + if !exists { + return &Error{StatusCode: http.StatusNotFound} + } + + delete(c.maintenanceManifestDocuments, maintenanceManifestDocument.ID) + return nil +} + +// ChangeFeed is unimplemented +func (c *FakeMaintenanceManifestDocumentClient) ChangeFeed(*Options) MaintenanceManifestDocumentIterator { + c.lock.RLock() + defer c.lock.RUnlock() + + if c.err != nil { + return NewFakeMaintenanceManifestDocumentErroringRawIterator(c.err) + } + + return NewFakeMaintenanceManifestDocumentErroringRawIterator(ErrNotImplemented) +} + +func (c *FakeMaintenanceManifestDocumentClient) processPreTriggers(ctx context.Context, maintenanceManifestDocument *pkg.MaintenanceManifestDocument, options *Options) error { + for _, triggerName := range options.PreTriggers { + if triggerHandler := c.triggerHandlers[triggerName]; triggerHandler != nil { + c.lock.Unlock() + err := triggerHandler(ctx, maintenanceManifestDocument) + c.lock.Lock() + if err != nil { + return err + } + } else { + return ErrNotImplemented + } + } + + return nil +} + +// Query calls a query handler to implement database querying +func (c *FakeMaintenanceManifestDocumentClient) Query(name string, query *Query, options *Options) MaintenanceManifestDocumentRawIterator { + c.lock.RLock() + defer c.lock.RUnlock() + + if c.err != nil { + return NewFakeMaintenanceManifestDocumentErroringRawIterator(c.err) + } + + if queryHandler := c.queryHandlers[query.Query]; queryHandler != nil { + c.lock.RUnlock() + i := queryHandler(c, query, options) + c.lock.RLock() + return i + } + + return NewFakeMaintenanceManifestDocumentErroringRawIterator(ErrNotImplemented) +} + +// QueryAll calls a query handler to implement database querying +func (c *FakeMaintenanceManifestDocumentClient) QueryAll(ctx context.Context, partitionkey string, query *Query, options *Options) (*pkg.MaintenanceManifestDocuments, error) { + iter := c.Query("", query, options) + return iter.Next(ctx, -1) +} + +func NewFakeMaintenanceManifestDocumentIterator(maintenanceManifestDocuments []*pkg.MaintenanceManifestDocument, continuation int) MaintenanceManifestDocumentRawIterator { + return &fakeMaintenanceManifestDocumentIterator{maintenanceManifestDocuments: maintenanceManifestDocuments, continuation: continuation} +} + +type fakeMaintenanceManifestDocumentIterator struct { + maintenanceManifestDocuments []*pkg.MaintenanceManifestDocument + continuation int + done bool +} + +func (i *fakeMaintenanceManifestDocumentIterator) NextRaw(ctx context.Context, maxItemCount int, out interface{}) error { + return ErrNotImplemented +} + +func (i *fakeMaintenanceManifestDocumentIterator) Next(ctx context.Context, maxItemCount int) (*pkg.MaintenanceManifestDocuments, error) { + if i.done { + return nil, nil + } + + var maintenanceManifestDocuments []*pkg.MaintenanceManifestDocument + if maxItemCount == -1 { + maintenanceManifestDocuments = i.maintenanceManifestDocuments[i.continuation:] + i.continuation = len(i.maintenanceManifestDocuments) + i.done = true + } else { + max := i.continuation + maxItemCount + if max > len(i.maintenanceManifestDocuments) { + max = len(i.maintenanceManifestDocuments) + } + maintenanceManifestDocuments = i.maintenanceManifestDocuments[i.continuation:max] + i.continuation += max + i.done = i.Continuation() == "" + } + + return &pkg.MaintenanceManifestDocuments{ + MaintenanceManifestDocuments: maintenanceManifestDocuments, + Count: len(maintenanceManifestDocuments), + }, nil +} + +func (i *fakeMaintenanceManifestDocumentIterator) Continuation() string { + if i.continuation >= len(i.maintenanceManifestDocuments) { + return "" + } + return fmt.Sprintf("%d", i.continuation) +} + +// NewFakeMaintenanceManifestDocumentErroringRawIterator returns a MaintenanceManifestDocumentRawIterator which +// whose methods return the given error +func NewFakeMaintenanceManifestDocumentErroringRawIterator(err error) MaintenanceManifestDocumentRawIterator { + return &fakeMaintenanceManifestDocumentErroringRawIterator{err: err} +} + +type fakeMaintenanceManifestDocumentErroringRawIterator struct { + err error +} + +func (i *fakeMaintenanceManifestDocumentErroringRawIterator) Next(ctx context.Context, maxItemCount int) (*pkg.MaintenanceManifestDocuments, error) { + return nil, i.err +} + +func (i *fakeMaintenanceManifestDocumentErroringRawIterator) NextRaw(context.Context, int, interface{}) error { + return i.err +} + +func (i *fakeMaintenanceManifestDocumentErroringRawIterator) Continuation() string { + return "" +} diff --git a/pkg/deploy/assets/databases-development.json b/pkg/deploy/assets/databases-development.json index 191670b7f6f..b3ac7d93da4 100644 --- a/pkg/deploy/assets/databases-development.json +++ b/pkg/deploy/assets/databases-development.json @@ -368,6 +368,46 @@ } }, "type": "Microsoft.DocumentDB/databaseAccounts/sqlDatabases/containers/triggers" + }, + { + "apiVersion": "2023-04-15", + "dependsOn": [ + "[resourceId('Microsoft.DocumentDB/databaseAccounts/sqlDatabases', parameters('databaseAccountName'), parameters('databaseName'))]" + ], + "location": "[resourceGroup().location]", + "name": "[concat(parameters('databaseAccountName'), '/', parameters('databaseName'), '/MaintenanceManifests')]", + "properties": { + "options": {}, + "resource": { + "defaultTtl": -1, + "id": "MaintenanceManifests", + "partitionKey": { + "kind": "Hash", + "paths": [ + "/id" + ] + } + } + }, + "type": "Microsoft.DocumentDB/databaseAccounts/sqlDatabases/containers" + }, + { + "apiVersion": "2023-04-15", + "dependsOn": [ + "[resourceId('Microsoft.DocumentDB/databaseAccounts/sqlDatabases', parameters('databaseAccountName'), parameters('databaseName'))]", + "[resourceId('Microsoft.DocumentDB/databaseAccounts/sqlDatabases/containers', parameters('databaseAccountName'), parameters('databaseName'), 'MaintenanceManifests')]" + ], + "location": "[resourceGroup().location]", + "name": "[concat(parameters('databaseAccountName'), '/', parameters('databaseName'), '/MaintenanceManifests/renewLease')]", + "properties": { + "resource": { + "body": "function trigger() {\n\t\t\t\tvar request = getContext().getRequest();\n\t\t\t\tvar body = request.getBody();\n\t\t\t\tvar date = new Date();\n\t\t\t\tbody[\"leaseExpires\"] = Math.floor(date.getTime() / 1000) + 60;\n\t\t\t\trequest.setBody(body);\n\t\t\t}", + "id": "renewLease", + "triggerOperation": "All", + "triggerType": "Pre" + } + }, + "type": "Microsoft.DocumentDB/databaseAccounts/sqlDatabases/containers/triggers" } ] } From 721d8f36bf991244f143a3530b09df0c406a8b11 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 18 Sep 2024 12:36:11 +1000 Subject: [PATCH 29/67] lint --- pkg/mimo/tasks/cluster/operatorflags.go | 5 +++-- pkg/mimo/tasks/cluster/operatorflags_test.go | 12 ++++++------ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/pkg/mimo/tasks/cluster/operatorflags.go b/pkg/mimo/tasks/cluster/operatorflags.go index 652f4392882..5c9b01eb9fb 100644 --- a/pkg/mimo/tasks/cluster/operatorflags.go +++ b/pkg/mimo/tasks/cluster/operatorflags.go @@ -6,11 +6,12 @@ package cluster import ( "context" - arov1alpha1 "github.com/Azure/ARO-RP/pkg/operator/apis/aro.openshift.io/v1alpha1" - "github.com/Azure/ARO-RP/pkg/util/mimo" kerrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/util/retry" + + arov1alpha1 "github.com/Azure/ARO-RP/pkg/operator/apis/aro.openshift.io/v1alpha1" + "github.com/Azure/ARO-RP/pkg/util/mimo" ) // UpdateClusterOperatorFlags updates the OperatorFlags object in the ARO diff --git a/pkg/mimo/tasks/cluster/operatorflags_test.go b/pkg/mimo/tasks/cluster/operatorflags_test.go index a9f355601f1..c4d8b1ee418 100644 --- a/pkg/mimo/tasks/cluster/operatorflags_test.go +++ b/pkg/mimo/tasks/cluster/operatorflags_test.go @@ -8,11 +8,11 @@ import ( "testing" "time" - "github.com/go-test/deep" . "github.com/onsi/gomega" + "github.com/go-test/deep" "github.com/golang/mock/gomock" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes/scheme" "sigs.k8s.io/controller-runtime/pkg/client" @@ -45,7 +45,7 @@ func TestOperatorFlags(t *testing.T) { name: "not ready", objects: []runtime.Object{ &arov1alpha1.Cluster{ - ObjectMeta: v1.ObjectMeta{ + ObjectMeta: metav1.ObjectMeta{ Name: arov1alpha1.SingletonClusterName, ResourceVersion: "1000", }, @@ -58,11 +58,11 @@ func TestOperatorFlags(t *testing.T) { }, wantObjects: []runtime.Object{ &arov1alpha1.Cluster{ - ObjectMeta: v1.ObjectMeta{ + ObjectMeta: metav1.ObjectMeta{ Name: arov1alpha1.SingletonClusterName, ResourceVersion: "1001", }, - TypeMeta: v1.TypeMeta{ + TypeMeta: metav1.TypeMeta{ Kind: "Cluster", APIVersion: arov1alpha1.SchemeGroupVersion.String(), }, @@ -119,7 +119,7 @@ func TestOperatorFlags(t *testing.T) { g.Expect(err).ToNot(HaveOccurred()) r := deep.Equal(i, o) - g.Expect(r).To(HaveLen(0)) + g.Expect(r).To(BeEmpty()) } } }) From 34f583916ab98b0c194a69b37f5b44905e5035ed Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Thu, 19 Sep 2024 14:26:39 +1000 Subject: [PATCH 30/67] updates for basic mimo e2e --- .pipelines/e2e.yml | 2 +- pkg/mimo/const.go | 9 +++ pkg/mimo/sets/operatorflags.go | 21 +++++++ pkg/mimo/sets/sets.go | 10 ++-- test/e2e/mimo_actuator.go | 104 +++++++++++++++++++++++++++++++++ test/e2e/setup.go | 6 ++ 6 files changed, 147 insertions(+), 5 deletions(-) create mode 100644 pkg/mimo/const.go create mode 100644 pkg/mimo/sets/operatorflags.go create mode 100644 test/e2e/mimo_actuator.go diff --git a/.pipelines/e2e.yml b/.pipelines/e2e.yml index db29add2a91..1f58b07f8bc 100644 --- a/.pipelines/e2e.yml +++ b/.pipelines/e2e.yml @@ -71,7 +71,7 @@ jobs: - script: | export CI=true # Tell the E2E binary to run the MIMO tests - export CI_E2E_MIMO=true + export ARO_E2E_MIMO=true . secrets/env . ./hack/e2e/run-rp-and-e2e.sh diff --git a/pkg/mimo/const.go b/pkg/mimo/const.go new file mode 100644 index 00000000000..b66bc025adb --- /dev/null +++ b/pkg/mimo/const.go @@ -0,0 +1,9 @@ +package mimo + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +const ( + TLS_CERT_ROTATION_ID = "9b741734-6505-447f-8510-85eb0ae561a2" + OPERATOR_FLAGS_UPDATE_ID = "b41749fc-af26-4ab7-b5a1-e03f3ee4cba6" +) diff --git a/pkg/mimo/sets/operatorflags.go b/pkg/mimo/sets/operatorflags.go new file mode 100644 index 00000000000..bb2167acef2 --- /dev/null +++ b/pkg/mimo/sets/operatorflags.go @@ -0,0 +1,21 @@ +package sets + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/mimo/tasks/cluster" + "github.com/Azure/ARO-RP/pkg/util/mimo" + "github.com/Azure/ARO-RP/pkg/util/steps" +) + +func UpdateOperatorFlags(t mimo.TaskContext, doc *api.MaintenanceManifestDocument, oc *api.OpenShiftClusterDocument) (api.MaintenanceManifestState, string) { + s := []steps.Step{ + steps.Action(cluster.EnsureAPIServerIsUp), + + steps.Action(cluster.UpdateClusterOperatorFlags), + } + + return run(t, s) +} diff --git a/pkg/mimo/sets/sets.go b/pkg/mimo/sets/sets.go index c19dd54ae6c..903891bc73c 100644 --- a/pkg/mimo/sets/sets.go +++ b/pkg/mimo/sets/sets.go @@ -7,7 +7,8 @@ import ( "time" "github.com/Azure/ARO-RP/pkg/api" - "github.com/Azure/ARO-RP/pkg/util/mimo" + "github.com/Azure/ARO-RP/pkg/mimo" + utilmimo "github.com/Azure/ARO-RP/pkg/util/mimo" "github.com/Azure/ARO-RP/pkg/util/steps" ) @@ -15,14 +16,15 @@ const DEFAULT_POLL_TIME = time.Second * 10 const DEFAULT_TIMEOUT_DURATION = time.Minute * 20 var DEFAULT_MAINTENANCE_SETS = map[string]MaintenanceSet{ - "9b741734-6505-447f-8510-85eb0ae561a2": TLSCertRotation, + mimo.TLS_CERT_ROTATION_ID: TLSCertRotation, + mimo.OPERATOR_FLAGS_UPDATE_ID: UpdateOperatorFlags, } -func run(t mimo.TaskContext, s []steps.Step) (api.MaintenanceManifestState, string) { +func run(t utilmimo.TaskContext, s []steps.Step) (api.MaintenanceManifestState, string) { _, err := steps.Run(t, t.Log(), DEFAULT_POLL_TIME, s, t.Now) if err != nil { - if mimo.IsRetryableError(err) { + if utilmimo.IsRetryableError(err) { return api.MaintenanceManifestStatePending, err.Error() } return api.MaintenanceManifestStateFailed, err.Error() diff --git a/test/e2e/mimo_actuator.go b/test/e2e/mimo_actuator.go new file mode 100644 index 00000000000..91928d3edcb --- /dev/null +++ b/test/e2e/mimo_actuator.go @@ -0,0 +1,104 @@ +package e2e + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "encoding/json" + "net/http" + "net/url" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/util/retry" + + "github.com/Azure/ARO-RP/pkg/api/admin" + "github.com/Azure/ARO-RP/pkg/mimo" + "github.com/Azure/ARO-RP/pkg/operator" + "github.com/Azure/ARO-RP/pkg/util/uuid" +) + +var _ = Describe("MIMO Actuator E2E Testing", func() { + BeforeEach(func() { + skipIfNotInDevelopmentEnv() + skipIfMIMOActuatorNotEnabled() + }) + + It("Should be able to schedule and run a maintenance set via the admin API", func(ctx context.Context) { + var oc = &admin.OpenShiftCluster{} + testflag := "aro.e2e.testflag." + uuid.DefaultGenerator.Generate() + + By("set a bogus flag on the cluster") + resp, err := adminRequest(ctx, + http.MethodPatch, clusterResourceID, nil, true, + json.RawMessage("{\"properties\": {\"operatorFlags\": {\""+testflag+"\": \"true\"}}}"), oc) + Expect(err).NotTo(HaveOccurred()) + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + + By("waiting for the update to complete") + Eventually(func(g Gomega, ctx context.Context) { + oc = adminGetCluster(g, ctx, clusterResourceID) + g.Expect(oc.Properties.ProvisioningState).To(Equal(admin.ProvisioningStateSucceeded)) + }).WithContext(ctx).WithTimeout(DefaultEventuallyTimeout).Should(Succeed()) + + By("check the flag is set in the cluster") + co, err := clients.AROClusters.AroV1alpha1().Clusters().Get(ctx, "cluster", metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + flag, ok := co.Spec.OperatorFlags[testflag] + Expect(ok).To(BeTrue()) + Expect(flag).To(Equal("true")) + + By("change the flag in-cluster to a wrong value") + // get the flag we want to check for + err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + co, err := clients.AROClusters.AroV1alpha1().Clusters().Get(ctx, "cluster", metav1.GetOptions{}) + if err != nil { + return err + } + + co.Spec.OperatorFlags[testflag] = operator.FlagFalse + _, err = clients.AROClusters.AroV1alpha1().Clusters().Update(ctx, co, metav1.UpdateOptions{}) + return err + }) + Expect(err).NotTo(HaveOccurred()) + + By("creating the flag update manifest via the API") + out := &admin.MaintenanceManifest{} + resp, err = adminRequest(ctx, + http.MethodPut, "/admin"+clusterResourceID+"/maintenancemanifests", + url.Values{}, true, &admin.MaintenanceManifest{ + MaintenanceSetID: mimo.OPERATOR_FLAGS_UPDATE_ID, + RunAfter: 1, + RunBefore: int(time.Now().Add(time.Hour).Unix()), + }, &out) + Expect(err).NotTo(HaveOccurred()) + Expect(resp.StatusCode).To(Equal(http.StatusCreated)) + + manifestID := out.ID + + By("waiting for the manifest run to complete") + Eventually(func(g Gomega, ctx context.Context) { + fetchedManifest := &admin.MaintenanceManifest{} + resp, err = adminRequest(ctx, + http.MethodPut, "/admin"+clusterResourceID+"/maintenancemanifests/"+manifestID, + url.Values{}, true, nil, &fetchedManifest) + + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(resp.StatusCode).To(Equal(http.StatusOK)) + g.Expect(fetchedManifest.State).To(Equal(admin.MaintenanceManifestStateCompleted)) + }).WithContext(ctx).WithTimeout(DefaultEventuallyTimeout).Should(Succeed()) + + By("checking the flag has been set back in the cluster") + co, err = clients.AROClusters.AroV1alpha1().Clusters().Get(ctx, "cluster", metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + flag, ok = co.Spec.OperatorFlags[testflag] + Expect(ok).To(BeTrue()) + Expect(flag).To(Equal("true"), "MIMO manifest has not run") + }) +}) diff --git a/test/e2e/setup.go b/test/e2e/setup.go index a337cb83b1d..cbd92ed265a 100644 --- a/test/e2e/setup.go +++ b/test/e2e/setup.go @@ -131,6 +131,12 @@ func skipIfSeleniumNotEnabled() { } } +func skipIfMIMOActuatorNotEnabled() { + if os.Getenv("ARO_E2E_MIMO") == "" { + Skip("ARO_E2E_MIMO not set, skipping MIMO e2e") + } +} + func skipIfNotHiveManagedCluster(adminAPICluster *admin.OpenShiftCluster) { if adminAPICluster.Properties.HiveProfile == (admin.HiveProfile{}) { Skip("skipping tests because this ARO cluster has not been created/adopted by Hive") From a3b1356b511b16f14097cccc42dc082bc341085d Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Thu, 19 Sep 2024 23:44:32 +1000 Subject: [PATCH 31/67] e2e testing --- Makefile | 2 +- test/e2e/mimo_actuator.go | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 931001ff2d0..4e491355a94 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ SHELL = /bin/bash TAG ?= $(shell git describe --exact-match 2>/dev/null) COMMIT = $(shell git rev-parse --short=7 HEAD)$(shell [[ $$(git status --porcelain) = "" ]] || echo -dirty) ARO_IMAGE_BASE = ${RP_IMAGE_ACR}.azurecr.io/aro -E2E_FLAGS ?= -test.v --ginkgo.v --ginkgo.timeout 180m --ginkgo.flake-attempts=2 --ginkgo.junit-report=e2e-report.xml +E2E_FLAGS ?= -test.v --ginkgo.vv --ginkgo.timeout 180m --ginkgo.flake-attempts=2 --ginkgo.junit-report=e2e-report.xml E2E_LABEL ?= !smoke&&!regressiontest GO_FLAGS ?= -tags=containers_image_openpgp,exclude_graphdriver_btrfs,exclude_graphdriver_devicemapper diff --git a/test/e2e/mimo_actuator.go b/test/e2e/mimo_actuator.go index 91928d3edcb..bd33d293da9 100644 --- a/test/e2e/mimo_actuator.go +++ b/test/e2e/mimo_actuator.go @@ -8,7 +8,6 @@ import ( "encoding/json" "net/http" "net/url" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -73,8 +72,6 @@ var _ = Describe("MIMO Actuator E2E Testing", func() { http.MethodPut, "/admin"+clusterResourceID+"/maintenancemanifests", url.Values{}, true, &admin.MaintenanceManifest{ MaintenanceSetID: mimo.OPERATOR_FLAGS_UPDATE_ID, - RunAfter: 1, - RunBefore: int(time.Now().Add(time.Hour).Unix()), }, &out) Expect(err).NotTo(HaveOccurred()) Expect(resp.StatusCode).To(Equal(http.StatusCreated)) From 72baacfda497434e8f14c844fdb75c3a7b8f0ebc Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Mon, 23 Sep 2024 13:32:51 +1000 Subject: [PATCH 32/67] try and see what e2e is breaking with --- test/e2e/adminapi.go | 50 +++++++++++++++++++++++++++++++++++++-- test/e2e/mimo_actuator.go | 4 ++-- 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/test/e2e/adminapi.go b/test/e2e/adminapi.go index 9b3d2375217..fea6a36178a 100644 --- a/test/e2e/adminapi.go +++ b/test/e2e/adminapi.go @@ -15,11 +15,40 @@ import ( . "github.com/onsi/gomega" + "github.com/sirupsen/logrus" + "github.com/Azure/ARO-RP/pkg/api/admin" "github.com/Azure/ARO-RP/pkg/env" ) -func adminRequest(ctx context.Context, method, path string, params url.Values, strict bool, in, out interface{}) (*http.Response, error) { +type adminReqOpts struct { + Option string + Value interface{} +} + +func logOnError(log *logrus.Entry) []adminReqOpts { + return []adminReqOpts{ + { + Option: "log", + Value: log, + }, + { + Option: "logOnError", + Value: true, + }, + } +} + +func _getAdminReqOpt(key string, opts []adminReqOpts) (interface{}, bool) { + for _, i := range opts { + if i.Option == key { + return i.Value, true + } + } + return nil, false +} + +func adminRequest(ctx context.Context, method, path string, params url.Values, strict bool, in, out interface{}, opts ...adminReqOpts) (*http.Response, error) { if !env.IsLocalDevelopmentMode() { return nil, errors.New("only development RP mode is supported") } @@ -69,7 +98,24 @@ func adminRequest(ctx context.Context, method, path string, params url.Values, s }() if out != nil && resp.Header.Get("Content-Type") == "application/json" { - decoder := json.NewDecoder(resp.Body) + body, err := io.ReadAll(resp.Body) + if err != nil { + return resp, err + } + + if resp.StatusCode == http.StatusInternalServerError { + _, ok := _getAdminReqOpt("logOnError", opts) + if ok { + log, ok := _getAdminReqOpt("log", opts) + if ok { + logger := log.(*logrus.Entry) + + logger.Errorf("Failed request, content: %s", string(body)) + } + } + } + + decoder := json.NewDecoder(bytes.NewBuffer(body)) // If strict is set, enable DisallowUnknownFields. This is used to // verify that the response doesn't contain any fields that are not // defined, namely systemData. diff --git a/test/e2e/mimo_actuator.go b/test/e2e/mimo_actuator.go index bd33d293da9..c9c760d23b0 100644 --- a/test/e2e/mimo_actuator.go +++ b/test/e2e/mimo_actuator.go @@ -34,7 +34,7 @@ var _ = Describe("MIMO Actuator E2E Testing", func() { By("set a bogus flag on the cluster") resp, err := adminRequest(ctx, http.MethodPatch, clusterResourceID, nil, true, - json.RawMessage("{\"properties\": {\"operatorFlags\": {\""+testflag+"\": \"true\"}}}"), oc) + json.RawMessage("{\"properties\": {\"maintenanceTask\": \"SyncClusterObject\", \"operatorFlags\": {\""+testflag+"\": \"true\"}}}"), oc) Expect(err).NotTo(HaveOccurred()) Expect(resp.StatusCode).To(Equal(http.StatusOK)) @@ -72,7 +72,7 @@ var _ = Describe("MIMO Actuator E2E Testing", func() { http.MethodPut, "/admin"+clusterResourceID+"/maintenancemanifests", url.Values{}, true, &admin.MaintenanceManifest{ MaintenanceSetID: mimo.OPERATOR_FLAGS_UPDATE_ID, - }, &out) + }, &out, logOnError(log)...) Expect(err).NotTo(HaveOccurred()) Expect(resp.StatusCode).To(Equal(http.StatusCreated)) From 1045abfc3fd1416da2738157819dacb310beb00f Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Tue, 24 Sep 2024 17:35:14 +1000 Subject: [PATCH 33/67] initial doc frame --- docs/mimo.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/mimo.md diff --git a/docs/mimo.md b/docs/mimo.md new file mode 100644 index 00000000000..bc4f21edd3e --- /dev/null +++ b/docs/mimo.md @@ -0,0 +1 @@ +# Managed Infrastructure Maintenance Operator From 4cd13e3b29a9c301818d512cc164117354256ba9 Mon Sep 17 00:00:00 2001 From: Edison Cardenas Date: Mon, 19 Aug 2024 18:44:00 +1200 Subject: [PATCH 34/67] ARO-9263: Add ACR Token expiry --- pkg/api/admin/openshiftcluster.go | 7 ++++-- .../openshiftcluster_validatestatic_test.go | 23 +++++++++++++++++-- pkg/api/openshiftcluster.go | 3 ++- pkg/util/acrtoken/acrtoken.go | 11 +++++++++ pkg/util/acrtoken/acrtoken_test.go | 3 ++- 5 files changed, 41 insertions(+), 6 deletions(-) diff --git a/pkg/api/admin/openshiftcluster.go b/pkg/api/admin/openshiftcluster.go index 2a209eb4bb5..279c40a2f8b 100644 --- a/pkg/api/admin/openshiftcluster.go +++ b/pkg/api/admin/openshiftcluster.go @@ -5,6 +5,8 @@ package admin import ( "time" + + "github.com/Azure/go-autorest/autorest/date" ) // OpenShiftClusterList represents a list of OpenShift clusters. @@ -466,8 +468,9 @@ const ( // RegistryProfile represents a registry profile type RegistryProfile struct { - Name string `json:"name,omitempty"` - Username string `json:"username,omitempty"` + Name string `json:"name,omitempty"` + Username string `json:"username,omitempty"` + Expiry *date.Time `json:"expiry,omitempty"` } // ArchitectureVersion represents an architecture version diff --git a/pkg/api/admin/openshiftcluster_validatestatic_test.go b/pkg/api/admin/openshiftcluster_validatestatic_test.go index 3088f319a4a..761268edc00 100644 --- a/pkg/api/admin/openshiftcluster_validatestatic_test.go +++ b/pkg/api/admin/openshiftcluster_validatestatic_test.go @@ -13,6 +13,7 @@ import ( "github.com/Azure/ARO-RP/pkg/api" "github.com/Azure/ARO-RP/pkg/util/uuid" "github.com/Azure/ARO-RP/test/validate" + "github.com/Azure/go-autorest/autorest/date" ) func TestOpenShiftClusterStaticValidateDelta(t *testing.T) { @@ -624,11 +625,11 @@ func TestOpenShiftClusterStaticValidateDelta(t *testing.T) { wantErr: "400: PropertyChangeNotAllowed: properties.provisionedBy: Changing property 'properties.provisionedBy' is not allowed.", }, { - name: "registryProfiles change is not allowed", + name: "registryProfiles username change is not allowed", oc: func() *OpenShiftCluster { return &OpenShiftCluster{ Properties: OpenShiftClusterProperties{ - RegistryProfiles: []RegistryProfile{{Name: "test", Username: "testuser"}}, + RegistryProfiles: []RegistryProfile{{Name: "test", Username: "testuser", Expiry: toDate(time.Now())}}, }, } }, @@ -637,6 +638,20 @@ func TestOpenShiftClusterStaticValidateDelta(t *testing.T) { }, wantErr: "400: PropertyChangeNotAllowed: properties.registryProfiles: Changing property 'properties.registryProfiles' is not allowed.", }, + { + name: "registryProfiles expiry change is not allowed", + oc: func() *OpenShiftCluster { + return &OpenShiftCluster{ + Properties: OpenShiftClusterProperties{ + RegistryProfiles: []RegistryProfile{{Name: "test", Username: "testuser", Expiry: toDate(time.Now())}}, + }, + } + }, + modify: func(oc *OpenShiftCluster) { + oc.Properties.RegistryProfiles[0].Expiry = toDate(time.Now().UTC().Add(time.Hour * 24 * 30)) + }, + wantErr: "400: PropertyChangeNotAllowed: properties.registryProfiles: Changing property 'properties.registryProfiles' is not allowed.", + }, { name: "maintenanceTask change to Everything is allowed", oc: func() *OpenShiftCluster { @@ -766,3 +781,7 @@ func TestOpenShiftClusterStaticValidateDelta(t *testing.T) { }) } } + +func toDate(t time.Time) *date.Time { + return &date.Time{Time: t} +} diff --git a/pkg/api/openshiftcluster.go b/pkg/api/openshiftcluster.go index cd5aa33a1b6..df3f780e778 100644 --- a/pkg/api/openshiftcluster.go +++ b/pkg/api/openshiftcluster.go @@ -9,6 +9,7 @@ import ( "time" "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" + "github.com/Azure/go-autorest/autorest/date" ) // OpenShiftCluster represents an OpenShift cluster @@ -211,7 +212,6 @@ const ( ProvisioningStateUpdating ProvisioningState = "Updating" ProvisioningStateAdminUpdating ProvisioningState = "AdminUpdating" ProvisioningStateCanceled ProvisioningState = "Canceled" - ProvisioningStateMaintenance ProvisioningState = "Maintenance" ProvisioningStateDeleting ProvisioningState = "Deleting" ProvisioningStateSucceeded ProvisioningState = "Succeeded" ProvisioningStateFailed ProvisioningState = "Failed" @@ -772,6 +772,7 @@ type RegistryProfile struct { Name string `json:"name,omitempty"` Username string `json:"username,omitempty"` Password SecureString `json:"password,omitempty"` + Expiry *date.Time `json:"expiry,omitempty"` } // Install represents an install process diff --git a/pkg/util/acrtoken/acrtoken.go b/pkg/util/acrtoken/acrtoken.go index b1c87b3ba0d..2812e46005c 100644 --- a/pkg/util/acrtoken/acrtoken.go +++ b/pkg/util/acrtoken/acrtoken.go @@ -7,10 +7,12 @@ import ( "context" "fmt" "net/http" + "time" mgmtcontainerregistry "github.com/Azure/azure-sdk-for-go/services/preview/containerregistry/mgmt/2020-11-01-preview/containerregistry" "github.com/Azure/go-autorest/autorest" "github.com/Azure/go-autorest/autorest/azure" + "github.com/Azure/go-autorest/autorest/date" "github.com/Azure/go-autorest/autorest/to" "github.com/Azure/ARO-RP/pkg/api" @@ -36,6 +38,11 @@ type manager struct { registries containerregistry.RegistriesClient } +const ( + HoursInADay = 24 + ACRTokenLifeInDays = 90 +) + func NewManager(env env.Interface, localFPAuthorizer autorest.Authorizer) (Manager, error) { r, err := azure.ParseResourceID(env.ACRResourceID()) if err != nil { @@ -64,9 +71,12 @@ func (m *manager) GetRegistryProfile(oc *api.OpenShiftCluster) *api.RegistryProf } func (m *manager) NewRegistryProfile(oc *api.OpenShiftCluster) *api.RegistryProfile { + var tokenExpiration = time.Now().UTC().Add(time.Hour * HoursInADay * ACRTokenLifeInDays) + return &api.RegistryProfile{ Name: fmt.Sprintf("%s.%s", m.r.ResourceName, m.env.Environment().ContainerRegistryDNSSuffix), Username: "token-" + uuid.DefaultGenerator.Generate(), + Expiry: &date.Time{Time: tokenExpiration}, } } @@ -148,6 +158,7 @@ func (m *manager) generateTokenPassword(ctx context.Context, passwordName mgmtco creds, err := m.registries.GenerateCredentials(ctx, m.r.ResourceGroup, m.r.ResourceName, mgmtcontainerregistry.GenerateCredentialsParameters{ TokenID: to.StringPtr(m.env.ACRResourceID() + "/tokens/" + rp.Username), Name: passwordName, + Expiry: rp.Expiry, }) if err != nil { return "", err diff --git a/pkg/util/acrtoken/acrtoken_test.go b/pkg/util/acrtoken/acrtoken_test.go index 05639752a68..b0244d158e8 100644 --- a/pkg/util/acrtoken/acrtoken_test.go +++ b/pkg/util/acrtoken/acrtoken_test.go @@ -26,6 +26,7 @@ const ( func TestEnsureTokenAndPassword(t *testing.T) { ctx := context.Background() + var tokenExpiration = time.Now().UTC().Add(time.Hour * 24 * 90) controller := gomock.NewController(t) defer controller.Finish() @@ -67,7 +68,7 @@ func TestEnsureTokenAndPassword(t *testing.T) { tokens: tokens, } - password, err := m.EnsureTokenAndPassword(ctx, &api.RegistryProfile{Username: tokenName}) + password, err := m.EnsureTokenAndPassword(ctx, &api.RegistryProfile{Username: tokenName, Expiry: &date.Time{Time: tokenExpiration}}) if err != nil { t.Fatal(err) } From 58ed3db65810cf729116a1f089dd6a0c4f297681 Mon Sep 17 00:00:00 2001 From: Edison Cardenas Date: Tue, 20 Aug 2024 20:20:32 +1200 Subject: [PATCH 35/67] ARO-9263: Add ACR Token Expiry Checker --- pkg/mimo/actuator/task.go | 15 ++++++++ pkg/mimo/sets/acrtoken.go | 19 ++++++++++ pkg/mimo/sets/sets.go | 6 ++-- pkg/mimo/tasks/cluster/acrtoken_checker.go | 41 ++++++++++++++++++++++ pkg/util/mimo/taskcontext.go | 3 ++ 5 files changed, 82 insertions(+), 2 deletions(-) create mode 100644 pkg/mimo/sets/acrtoken.go create mode 100644 pkg/mimo/tasks/cluster/acrtoken_checker.go diff --git a/pkg/mimo/actuator/task.go b/pkg/mimo/actuator/task.go index 38e8ff6cc6c..86af8b7c658 100644 --- a/pkg/mimo/actuator/task.go +++ b/pkg/mimo/actuator/task.go @@ -16,6 +16,7 @@ import ( "github.com/Azure/ARO-RP/pkg/util/clienthelper" "github.com/Azure/ARO-RP/pkg/util/mimo" "github.com/Azure/ARO-RP/pkg/util/restconfig" + "github.com/Azure/go-autorest/autorest" ) type th struct { @@ -126,3 +127,17 @@ func (t *th) GetClusterUUID() string { func (t *th) GetOpenShiftClusterProperties() api.OpenShiftClusterProperties { return t.oc.OpenShiftCluster.Properties } + +// localFpAuthorizer implements mimo.TaskContext. +func (t *th) LocalFpAuthorizer() (autorest.Authorizer, error) { + localFPAuthorizer, err := t.env.FPAuthorizer(t.env.TenantID(), t.env.Environment().ResourceManagerScope) + if err != nil { + return nil, err + } + return localFPAuthorizer, nil +} + +// GetOpenshiftClusterDocument implements mimo.TaskContext. +func (t *th) GetOpenshiftClusterDocument() *api.OpenShiftClusterDocument { + return t.oc +} diff --git a/pkg/mimo/sets/acrtoken.go b/pkg/mimo/sets/acrtoken.go new file mode 100644 index 00000000000..0dab7c33c8f --- /dev/null +++ b/pkg/mimo/sets/acrtoken.go @@ -0,0 +1,19 @@ +package sets + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/mimo/tasks/cluster" + "github.com/Azure/ARO-RP/pkg/util/mimo" + "github.com/Azure/ARO-RP/pkg/util/steps" +) + +func ACRTokenChecker(t mimo.TaskContext, doc *api.MaintenanceManifestDocument, oc *api.OpenShiftClusterDocument) (api.MaintenanceManifestState, string) { + s := []steps.Step{ + steps.Action(cluster.EnsureACRToken), + } + + return run(t, s) +} diff --git a/pkg/mimo/sets/sets.go b/pkg/mimo/sets/sets.go index 903891bc73c..0d7ccc4f707 100644 --- a/pkg/mimo/sets/sets.go +++ b/pkg/mimo/sets/sets.go @@ -16,8 +16,10 @@ const DEFAULT_POLL_TIME = time.Second * 10 const DEFAULT_TIMEOUT_DURATION = time.Minute * 20 var DEFAULT_MAINTENANCE_SETS = map[string]MaintenanceSet{ - mimo.TLS_CERT_ROTATION_ID: TLSCertRotation, - mimo.OPERATOR_FLAGS_UPDATE_ID: UpdateOperatorFlags, + mimo.TLS_CERT_ROTATION_ID: TLSCertRotation, + //TODO Add string + "xxxx-xxx-xxx-xxxx-xxxxxxxxxx": ACRTokenChecker, + mimo.OPERATOR_FLAGS_UPDATE_ID: UpdateOperatorFlags, } func run(t utilmimo.TaskContext, s []steps.Step) (api.MaintenanceManifestState, string) { diff --git a/pkg/mimo/tasks/cluster/acrtoken_checker.go b/pkg/mimo/tasks/cluster/acrtoken_checker.go new file mode 100644 index 00000000000..a62b36aa139 --- /dev/null +++ b/pkg/mimo/tasks/cluster/acrtoken_checker.go @@ -0,0 +1,41 @@ +package cluster + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "errors" + "time" + + "github.com/Azure/ARO-RP/pkg/util/acrtoken" + "github.com/Azure/ARO-RP/pkg/util/mimo" +) + +func EnsureACRToken(ctx context.Context) error { + th, err := mimo.GetTaskContext(ctx) + if err != nil { + return mimo.TerminalError(err) + } + + env := th.Environment() + localFpAuthorizer, err := th.LocalFpAuthorizer() + if err != nil { + return mimo.TerminalError(err) + } + + token, err := acrtoken.NewManager(env, localFpAuthorizer) + if err != nil { + return err + } + + rp := token.GetRegistryProfile(th.GetOpenshiftClusterDocument().OpenShiftCluster) + var now = time.Now().UTC() + expiry := rp.Expiry.Time + + if expiry.After(now) { + return mimo.TerminalError(errors.New("ACR Token has expired")) + } + + return nil +} diff --git a/pkg/util/mimo/taskcontext.go b/pkg/util/mimo/taskcontext.go index ac32776ea35..56c6b56b976 100644 --- a/pkg/util/mimo/taskcontext.go +++ b/pkg/util/mimo/taskcontext.go @@ -13,6 +13,7 @@ import ( "github.com/Azure/ARO-RP/pkg/api" "github.com/Azure/ARO-RP/pkg/env" "github.com/Azure/ARO-RP/pkg/util/clienthelper" + "github.com/Azure/go-autorest/autorest" ) type TaskContext interface { @@ -21,10 +22,12 @@ type TaskContext interface { Environment() env.Interface ClientHelper() (clienthelper.Interface, error) Log() *logrus.Entry + LocalFpAuthorizer() (autorest.Authorizer, error) // OpenShiftCluster GetClusterUUID() string GetOpenShiftClusterProperties() api.OpenShiftClusterProperties + GetOpenshiftClusterDocument() api.OpenShiftClusterDocument SetResultMessage(string) GetResultMessage() string From ac096bc5ba47a65676ac05c98b094b67e14cca30 Mon Sep 17 00:00:00 2001 From: Edison Cardenas Date: Wed, 21 Aug 2024 21:00:38 +1200 Subject: [PATCH 36/67] ARO-9263: Add unit test for checker --- pkg/mimo/tasks/cluster/acrtoken_checker.go | 19 ++++- .../tasks/cluster/acrtoken_checker_test.go | 81 +++++++++++++++++++ pkg/util/mimo/taskcontext.go | 2 +- 3 files changed, 99 insertions(+), 3 deletions(-) create mode 100644 pkg/mimo/tasks/cluster/acrtoken_checker_test.go diff --git a/pkg/mimo/tasks/cluster/acrtoken_checker.go b/pkg/mimo/tasks/cluster/acrtoken_checker.go index a62b36aa139..45171356adb 100644 --- a/pkg/mimo/tasks/cluster/acrtoken_checker.go +++ b/pkg/mimo/tasks/cluster/acrtoken_checker.go @@ -6,18 +6,26 @@ package cluster import ( "context" "errors" + "fmt" "time" + arov1alpha1 "github.com/Azure/ARO-RP/pkg/operator/apis/aro.openshift.io/v1alpha1" "github.com/Azure/ARO-RP/pkg/util/acrtoken" "github.com/Azure/ARO-RP/pkg/util/mimo" + "k8s.io/apimachinery/pkg/types" ) -func EnsureACRToken(ctx context.Context) error { +func EnsureACRTokenIsValid(ctx context.Context) error { th, err := mimo.GetTaskContext(ctx) if err != nil { return mimo.TerminalError(err) } + ch, err := th.ClientHelper() + if err != nil { + return err + } + env := th.Environment() localFpAuthorizer, err := th.LocalFpAuthorizer() if err != nil { @@ -29,13 +37,20 @@ func EnsureACRToken(ctx context.Context) error { return err } + cluster := &arov1alpha1.Cluster{} + err = ch.GetOne(ctx, types.NamespacedName{Name: arov1alpha1.SingletonClusterName}, cluster) + if err != nil { + return mimo.NewMIMOError(err, mimo.MIMOErrorTypeTerminalError) + } + rp := token.GetRegistryProfile(th.GetOpenshiftClusterDocument().OpenShiftCluster) var now = time.Now().UTC() expiry := rp.Expiry.Time if expiry.After(now) { - return mimo.TerminalError(errors.New("ACR Token has expired")) + return mimo.TerminalError(errors.New("ACR token has expired")) } + th.SetResultMessage(fmt.Sprintf("ACR token is valid")) return nil } diff --git a/pkg/mimo/tasks/cluster/acrtoken_checker_test.go b/pkg/mimo/tasks/cluster/acrtoken_checker_test.go new file mode 100644 index 00000000000..1d0bc4af8d4 --- /dev/null +++ b/pkg/mimo/tasks/cluster/acrtoken_checker_test.go @@ -0,0 +1,81 @@ +package cluster + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "testing" + "time" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/util/clienthelper" + mock_env "github.com/Azure/ARO-RP/pkg/util/mocks/env" + testtasks "github.com/Azure/ARO-RP/test/mimo/tasks" + testclienthelper "github.com/Azure/ARO-RP/test/util/clienthelper" + testlog "github.com/Azure/ARO-RP/test/util/log" + "github.com/Azure/go-autorest/autorest/date" + "github.com/golang/mock/gomock" + . "github.com/onsi/gomega" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestEnsureACRToken(t *testing.T) { + ctx := context.Background() + clusterUUID := "512a50c8-2a43-4c2a-8fd9-a5539475df2a" + + for _, tt := range []struct { + name string + oc func() *api.OpenShiftCluster + wantErr string + }{ + { + name: "not found", + oc: func() *api.OpenShiftCluster { + return &api.OpenShiftCluster{} + }, + wantErr: `No object found`, + }, + { + name: "expired", + oc: func() *api.OpenShiftCluster { + return &api.OpenShiftCluster{ + Properties: api.OpenShiftClusterProperties{ + RegistryProfiles: []*api.RegistryProfile{ + { + Name: "test", + Username: "testuser", + Expiry: &date.Time{Time: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)}, + }, + }, + }, + } + }, + wantErr: `TerminalError: ACR token has expired`, + }, + } { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + controller := gomock.NewController(t) + _env := mock_env.NewMockInterface(controller) + _, log := testlog.New() + + builder := fake.NewClientBuilder() + ch := clienthelper.NewWithClient(log, testclienthelper.NewHookingClient(builder.Build())) + tc := testtasks.NewFakeTestContext( + ctx, _env, log, func() time.Time { return time.Unix(100, 0) }, + testtasks.WithClientHelper(ch), + testtasks.WithOpenShiftClusterProperties(clusterUUID, tt.oc().Properties), + ) + + err := EnsureACRTokenIsValid(tc) + if tt.wantErr != "" && err != nil { + g.Expect(err).To(MatchError(tt.wantErr)) + } else if tt.wantErr != "" && err == nil { + t.Errorf("wanted error %s", tt.wantErr) + } else if tt.wantErr == "" { + g.Expect(err).ToNot(HaveOccurred()) + } + }) + } +} diff --git a/pkg/util/mimo/taskcontext.go b/pkg/util/mimo/taskcontext.go index 56c6b56b976..ce391e6b951 100644 --- a/pkg/util/mimo/taskcontext.go +++ b/pkg/util/mimo/taskcontext.go @@ -27,7 +27,7 @@ type TaskContext interface { // OpenShiftCluster GetClusterUUID() string GetOpenShiftClusterProperties() api.OpenShiftClusterProperties - GetOpenshiftClusterDocument() api.OpenShiftClusterDocument + GetOpenshiftClusterDocument() *api.OpenShiftClusterDocument SetResultMessage(string) GetResultMessage() string From d7cc1bb0b9f382fa8b7d8663870c1cfc99900118 Mon Sep 17 00:00:00 2001 From: Edison Cardenas Date: Thu, 22 Aug 2024 14:46:32 +1200 Subject: [PATCH 37/67] ARE-9263: Rename function --- pkg/mimo/sets/acrtoken.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/mimo/sets/acrtoken.go b/pkg/mimo/sets/acrtoken.go index 0dab7c33c8f..26eb2ad0037 100644 --- a/pkg/mimo/sets/acrtoken.go +++ b/pkg/mimo/sets/acrtoken.go @@ -12,7 +12,7 @@ import ( func ACRTokenChecker(t mimo.TaskContext, doc *api.MaintenanceManifestDocument, oc *api.OpenShiftClusterDocument) (api.MaintenanceManifestState, string) { s := []steps.Step{ - steps.Action(cluster.EnsureACRToken), + steps.Action(cluster.EnsureACRTokenIsValid), } return run(t, s) From 407b1bae0b29bd112dc15d38c212c3004f3fedd9 Mon Sep 17 00:00:00 2001 From: Edison Cardenas Date: Thu, 22 Aug 2024 17:04:08 +1200 Subject: [PATCH 38/67] ARO-9263: Restore missing "ProvisioningStateMaintenance" --- pkg/api/openshiftcluster.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/api/openshiftcluster.go b/pkg/api/openshiftcluster.go index df3f780e778..83d6e2423f0 100644 --- a/pkg/api/openshiftcluster.go +++ b/pkg/api/openshiftcluster.go @@ -212,6 +212,7 @@ const ( ProvisioningStateUpdating ProvisioningState = "Updating" ProvisioningStateAdminUpdating ProvisioningState = "AdminUpdating" ProvisioningStateCanceled ProvisioningState = "Canceled" + ProvisioningStateMaintenance ProvisioningState = "Maintenance" ProvisioningStateDeleting ProvisioningState = "Deleting" ProvisioningStateSucceeded ProvisioningState = "Succeeded" ProvisioningStateFailed ProvisioningState = "Failed" From 083d4f55d321b053d911612816186222903c1758 Mon Sep 17 00:00:00 2001 From: Edison Cardenas Date: Mon, 26 Aug 2024 15:08:22 +1200 Subject: [PATCH 39/67] ARO-9263: Fix import CI check failures --- pkg/api/admin/openshiftcluster_validatestatic_test.go | 3 ++- pkg/mimo/actuator/task.go | 2 +- pkg/mimo/tasks/cluster/acrtoken_checker.go | 6 +++--- pkg/mimo/tasks/cluster/acrtoken_checker_test.go | 10 ++++++---- pkg/util/mimo/taskcontext.go | 2 +- 5 files changed, 13 insertions(+), 10 deletions(-) diff --git a/pkg/api/admin/openshiftcluster_validatestatic_test.go b/pkg/api/admin/openshiftcluster_validatestatic_test.go index 761268edc00..a1ec2739f31 100644 --- a/pkg/api/admin/openshiftcluster_validatestatic_test.go +++ b/pkg/api/admin/openshiftcluster_validatestatic_test.go @@ -10,10 +10,11 @@ import ( "testing" "time" + "github.com/Azure/go-autorest/autorest/date" + "github.com/Azure/ARO-RP/pkg/api" "github.com/Azure/ARO-RP/pkg/util/uuid" "github.com/Azure/ARO-RP/test/validate" - "github.com/Azure/go-autorest/autorest/date" ) func TestOpenShiftClusterStaticValidateDelta(t *testing.T) { diff --git a/pkg/mimo/actuator/task.go b/pkg/mimo/actuator/task.go index 86af8b7c658..6392acf7767 100644 --- a/pkg/mimo/actuator/task.go +++ b/pkg/mimo/actuator/task.go @@ -7,6 +7,7 @@ import ( "context" "time" + "github.com/Azure/go-autorest/autorest" "github.com/sirupsen/logrus" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" @@ -16,7 +17,6 @@ import ( "github.com/Azure/ARO-RP/pkg/util/clienthelper" "github.com/Azure/ARO-RP/pkg/util/mimo" "github.com/Azure/ARO-RP/pkg/util/restconfig" - "github.com/Azure/go-autorest/autorest" ) type th struct { diff --git a/pkg/mimo/tasks/cluster/acrtoken_checker.go b/pkg/mimo/tasks/cluster/acrtoken_checker.go index 45171356adb..41119e1191c 100644 --- a/pkg/mimo/tasks/cluster/acrtoken_checker.go +++ b/pkg/mimo/tasks/cluster/acrtoken_checker.go @@ -6,13 +6,13 @@ package cluster import ( "context" "errors" - "fmt" "time" + "k8s.io/apimachinery/pkg/types" + arov1alpha1 "github.com/Azure/ARO-RP/pkg/operator/apis/aro.openshift.io/v1alpha1" "github.com/Azure/ARO-RP/pkg/util/acrtoken" "github.com/Azure/ARO-RP/pkg/util/mimo" - "k8s.io/apimachinery/pkg/types" ) func EnsureACRTokenIsValid(ctx context.Context) error { @@ -51,6 +51,6 @@ func EnsureACRTokenIsValid(ctx context.Context) error { return mimo.TerminalError(errors.New("ACR token has expired")) } - th.SetResultMessage(fmt.Sprintf("ACR token is valid")) + th.SetResultMessage("ACR token is valid") return nil } diff --git a/pkg/mimo/tasks/cluster/acrtoken_checker_test.go b/pkg/mimo/tasks/cluster/acrtoken_checker_test.go index 1d0bc4af8d4..4d788488a88 100644 --- a/pkg/mimo/tasks/cluster/acrtoken_checker_test.go +++ b/pkg/mimo/tasks/cluster/acrtoken_checker_test.go @@ -8,16 +8,18 @@ import ( "testing" "time" + . "github.com/onsi/gomega" + + "github.com/Azure/go-autorest/autorest/date" + "github.com/golang/mock/gomock" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "github.com/Azure/ARO-RP/pkg/api" "github.com/Azure/ARO-RP/pkg/util/clienthelper" mock_env "github.com/Azure/ARO-RP/pkg/util/mocks/env" testtasks "github.com/Azure/ARO-RP/test/mimo/tasks" testclienthelper "github.com/Azure/ARO-RP/test/util/clienthelper" testlog "github.com/Azure/ARO-RP/test/util/log" - "github.com/Azure/go-autorest/autorest/date" - "github.com/golang/mock/gomock" - . "github.com/onsi/gomega" - "sigs.k8s.io/controller-runtime/pkg/client/fake" ) func TestEnsureACRToken(t *testing.T) { diff --git a/pkg/util/mimo/taskcontext.go b/pkg/util/mimo/taskcontext.go index ce391e6b951..0a3ed2a8614 100644 --- a/pkg/util/mimo/taskcontext.go +++ b/pkg/util/mimo/taskcontext.go @@ -8,12 +8,12 @@ import ( "fmt" "time" + "github.com/Azure/go-autorest/autorest" "github.com/sirupsen/logrus" "github.com/Azure/ARO-RP/pkg/api" "github.com/Azure/ARO-RP/pkg/env" "github.com/Azure/ARO-RP/pkg/util/clienthelper" - "github.com/Azure/go-autorest/autorest" ) type TaskContext interface { From 3bf3b4d059dbd3254ddfba5c06bd993e6782913f Mon Sep 17 00:00:00 2001 From: Edison Cardenas Date: Thu, 12 Sep 2024 12:28:30 +1200 Subject: [PATCH 40/67] ARO-9263: Refactoring tests and revising logic to check expiry date. --- pkg/mimo/sets/sets.go | 7 ++- pkg/mimo/tasks/cluster/acrtoken_checker.go | 35 ++++++--------- .../tasks/cluster/acrtoken_checker_test.go | 44 ++++++++++++++----- pkg/util/acrtoken/acrtoken.go | 11 +++++ test/mimo/tasks/testcontext.go | 10 +++++ 5 files changed, 70 insertions(+), 37 deletions(-) diff --git a/pkg/mimo/sets/sets.go b/pkg/mimo/sets/sets.go index 0d7ccc4f707..403758073c1 100644 --- a/pkg/mimo/sets/sets.go +++ b/pkg/mimo/sets/sets.go @@ -16,10 +16,9 @@ const DEFAULT_POLL_TIME = time.Second * 10 const DEFAULT_TIMEOUT_DURATION = time.Minute * 20 var DEFAULT_MAINTENANCE_SETS = map[string]MaintenanceSet{ - mimo.TLS_CERT_ROTATION_ID: TLSCertRotation, - //TODO Add string - "xxxx-xxx-xxx-xxxx-xxxxxxxxxx": ACRTokenChecker, - mimo.OPERATOR_FLAGS_UPDATE_ID: UpdateOperatorFlags, + mimo.TLS_CERT_ROTATION_ID: TLSCertRotation, + "082978ce-3700-4972-835f-53d48658d291": ACRTokenChecker, + mimo.OPERATOR_FLAGS_UPDATE_ID: UpdateOperatorFlags, } func run(t utilmimo.TaskContext, s []steps.Step) (api.MaintenanceManifestState, string) { diff --git a/pkg/mimo/tasks/cluster/acrtoken_checker.go b/pkg/mimo/tasks/cluster/acrtoken_checker.go index 41119e1191c..76f52844cd6 100644 --- a/pkg/mimo/tasks/cluster/acrtoken_checker.go +++ b/pkg/mimo/tasks/cluster/acrtoken_checker.go @@ -8,47 +8,40 @@ import ( "errors" "time" - "k8s.io/apimachinery/pkg/types" - - arov1alpha1 "github.com/Azure/ARO-RP/pkg/operator/apis/aro.openshift.io/v1alpha1" "github.com/Azure/ARO-RP/pkg/util/acrtoken" "github.com/Azure/ARO-RP/pkg/util/mimo" ) +// EnsureACRTokenIsValid checks the expiry date of the Azure Container Registry (ACR) Token from the RegistryProfile. +// It returns an error if the expiry date is past the date now or if there is no registry profile found. func EnsureACRTokenIsValid(ctx context.Context) error { th, err := mimo.GetTaskContext(ctx) if err != nil { return mimo.TerminalError(err) } - ch, err := th.ClientHelper() - if err != nil { - return err - } - env := th.Environment() localFpAuthorizer, err := th.LocalFpAuthorizer() if err != nil { return mimo.TerminalError(err) } - token, err := acrtoken.NewManager(env, localFpAuthorizer) + manager, err := acrtoken.NewManager(env, localFpAuthorizer) if err != nil { return err } - cluster := &arov1alpha1.Cluster{} - err = ch.GetOne(ctx, types.NamespacedName{Name: arov1alpha1.SingletonClusterName}, cluster) - if err != nil { - return mimo.NewMIMOError(err, mimo.MIMOErrorTypeTerminalError) - } - - rp := token.GetRegistryProfile(th.GetOpenshiftClusterDocument().OpenShiftCluster) - var now = time.Now().UTC() - expiry := rp.Expiry.Time - - if expiry.After(now) { - return mimo.TerminalError(errors.New("ACR token has expired")) + registryProfiles := th.GetOpenShiftClusterProperties().RegistryProfiles + rp := manager.GetRegistryProfileFromSlice(registryProfiles) + if rp != nil { + var now = time.Now().UTC() + expiry := registryProfiles[0].Expiry.Time + + if expiry.Before(now) { + return mimo.TerminalError(errors.New("ACR token has expired")) + } + } else { + return mimo.TerminalError(errors.New("No registry profile detected.")) } th.SetResultMessage("ACR token is valid") diff --git a/pkg/mimo/tasks/cluster/acrtoken_checker_test.go b/pkg/mimo/tasks/cluster/acrtoken_checker_test.go index 4d788488a88..c0bcad97624 100644 --- a/pkg/mimo/tasks/cluster/acrtoken_checker_test.go +++ b/pkg/mimo/tasks/cluster/acrtoken_checker_test.go @@ -10,56 +10,76 @@ import ( . "github.com/onsi/gomega" + utilerror "github.com/Azure/ARO-RP/test/util/error" "github.com/Azure/go-autorest/autorest/date" "github.com/golang/mock/gomock" "sigs.k8s.io/controller-runtime/pkg/client/fake" "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/util/azureclient" "github.com/Azure/ARO-RP/pkg/util/clienthelper" + + // mock_acrtoken "github.com/Azure/ARO-RP/pkg/util/mocks/acrtoken" mock_env "github.com/Azure/ARO-RP/pkg/util/mocks/env" testtasks "github.com/Azure/ARO-RP/test/mimo/tasks" testclienthelper "github.com/Azure/ARO-RP/test/util/clienthelper" testlog "github.com/Azure/ARO-RP/test/util/log" ) +const ( + tokenName = "token-12345" + registryResourceID = "/subscriptions/93aeba23-2f76-4307-be82-02921df010cf/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arointsvc" + clusterUUID = "512a50c8-2a43-4c2a-8fd9-a5539475df2a" +) + func TestEnsureACRToken(t *testing.T) { ctx := context.Background() - clusterUUID := "512a50c8-2a43-4c2a-8fd9-a5539475df2a" - for _, tt := range []struct { - name string - oc func() *api.OpenShiftCluster - wantErr string + name string + azureEnv azureclient.AROEnvironment + oc func() *api.OpenShiftCluster + wantErr string }{ { - name: "not found", + name: "not found", + azureEnv: azureclient.PublicCloud, oc: func() *api.OpenShiftCluster { - return &api.OpenShiftCluster{} + return &api.OpenShiftCluster{ + Properties: api.OpenShiftClusterProperties{}, + } }, - wantErr: `No object found`, + wantErr: "TerminalError: No registry profile detected.", }, { - name: "expired", + name: "expired", + azureEnv: azureclient.PublicCloud, oc: func() *api.OpenShiftCluster { return &api.OpenShiftCluster{ Properties: api.OpenShiftClusterProperties{ RegistryProfiles: []*api.RegistryProfile{ { - Name: "test", + Name: "arosvc.azurecr.io", Username: "testuser", Expiry: &date.Time{Time: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)}, }, + { + Name: "arointsvc.azurecr.io", + Username: "testuser", + Expiry: &date.Time{Time: time.Date(2024, 1, 9, 0, 0, 0, 0, time.UTC)}, + }, }, }, } }, - wantErr: `TerminalError: ACR token has expired`, + wantErr: "TerminalError: ACR token has expired", }, } { t.Run(tt.name, func(t *testing.T) { g := NewWithT(t) controller := gomock.NewController(t) _env := mock_env.NewMockInterface(controller) + _env.EXPECT().ACRResourceID().AnyTimes().Return(registryResourceID) + _env.EXPECT().Environment().AnyTimes().Return(&tt.azureEnv) _, log := testlog.New() builder := fake.NewClientBuilder() @@ -72,7 +92,7 @@ func TestEnsureACRToken(t *testing.T) { err := EnsureACRTokenIsValid(tc) if tt.wantErr != "" && err != nil { - g.Expect(err).To(MatchError(tt.wantErr)) + utilerror.AssertErrorMessage(t, err, tt.wantErr) } else if tt.wantErr != "" && err == nil { t.Errorf("wanted error %s", tt.wantErr) } else if tt.wantErr == "" { diff --git a/pkg/util/acrtoken/acrtoken.go b/pkg/util/acrtoken/acrtoken.go index 2812e46005c..70aeafc1c01 100644 --- a/pkg/util/acrtoken/acrtoken.go +++ b/pkg/util/acrtoken/acrtoken.go @@ -23,6 +23,7 @@ import ( type Manager interface { GetRegistryProfile(oc *api.OpenShiftCluster) *api.RegistryProfile + GetRegistryProfileFromSlice(oc []*api.RegistryProfile) *api.RegistryProfile NewRegistryProfile(oc *api.OpenShiftCluster) *api.RegistryProfile PutRegistryProfile(oc *api.OpenShiftCluster, rp *api.RegistryProfile) EnsureTokenAndPassword(ctx context.Context, rp *api.RegistryProfile) (string, error) @@ -70,6 +71,16 @@ func (m *manager) GetRegistryProfile(oc *api.OpenShiftCluster) *api.RegistryProf return nil } +func (m *manager) GetRegistryProfileFromSlice(registryProfiles []*api.RegistryProfile) *api.RegistryProfile { + for _, rp := range registryProfiles { + if rp.Name == fmt.Sprintf("%s.%s", m.r.ResourceName, m.env.Environment().ContainerRegistryDNSSuffix) { + return rp + } + } + + return nil +} + func (m *manager) NewRegistryProfile(oc *api.OpenShiftCluster) *api.RegistryProfile { var tokenExpiration = time.Now().UTC().Add(time.Hour * HoursInADay * ACRTokenLifeInDays) diff --git a/test/mimo/tasks/testcontext.go b/test/mimo/tasks/testcontext.go index bda019561fc..0f137d5004f 100644 --- a/test/mimo/tasks/testcontext.go +++ b/test/mimo/tasks/testcontext.go @@ -13,6 +13,7 @@ import ( "github.com/Azure/ARO-RP/pkg/api" "github.com/Azure/ARO-RP/pkg/env" "github.com/Azure/ARO-RP/pkg/util/clienthelper" + "github.com/Azure/go-autorest/autorest" ) type fakeTestContext struct { @@ -65,6 +66,15 @@ func NewFakeTestContext(ctx context.Context, env env.Interface, log *logrus.Entr return ftc } +func (t *fakeTestContext) LocalFpAuthorizer() (autorest.Authorizer, error) { + myAuthorizer := autorest.NullAuthorizer{} + return myAuthorizer, nil +} +func (t *fakeTestContext) GetOpenshiftClusterDocument() *api.OpenShiftClusterDocument { + myCD := &api.OpenShiftClusterDocument{} + return myCD +} + // handle func (t *fakeTestContext) Environment() env.Interface { From 95ccde9b2fc68dc2129423019bfcfcbf1c48a34c Mon Sep 17 00:00:00 2001 From: Edison Cardenas Date: Thu, 12 Sep 2024 15:52:30 +1200 Subject: [PATCH 41/67] ARO-9263: Add another condition to check if expiry date is nil --- pkg/mimo/tasks/cluster/acrtoken_checker.go | 10 ++++++--- .../tasks/cluster/acrtoken_checker_test.go | 22 +++++++++++++++++-- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/pkg/mimo/tasks/cluster/acrtoken_checker.go b/pkg/mimo/tasks/cluster/acrtoken_checker.go index 76f52844cd6..36783940c8b 100644 --- a/pkg/mimo/tasks/cluster/acrtoken_checker.go +++ b/pkg/mimo/tasks/cluster/acrtoken_checker.go @@ -35,15 +35,19 @@ func EnsureACRTokenIsValid(ctx context.Context) error { rp := manager.GetRegistryProfileFromSlice(registryProfiles) if rp != nil { var now = time.Now().UTC() - expiry := registryProfiles[0].Expiry.Time + expiry := registryProfiles[0].Expiry - if expiry.Before(now) { + switch { + case expiry == nil: + return mimo.TerminalError(errors.New("No expiry date detected.")) + case expiry.Time.Before(now): return mimo.TerminalError(errors.New("ACR token has expired")) + default: + th.SetResultMessage("ACR token is valid") } } else { return mimo.TerminalError(errors.New("No registry profile detected.")) } - th.SetResultMessage("ACR token is valid") return nil } diff --git a/pkg/mimo/tasks/cluster/acrtoken_checker_test.go b/pkg/mimo/tasks/cluster/acrtoken_checker_test.go index c0bcad97624..453d7f822cf 100644 --- a/pkg/mimo/tasks/cluster/acrtoken_checker_test.go +++ b/pkg/mimo/tasks/cluster/acrtoken_checker_test.go @@ -27,8 +27,8 @@ import ( ) const ( - tokenName = "token-12345" - registryResourceID = "/subscriptions/93aeba23-2f76-4307-be82-02921df010cf/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arointsvc" + registryName = "arointsvc" + registryResourceID = "/subscriptions/93aeba23-2f76-4307-be82-02921df010cf/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/" + registryName clusterUUID = "512a50c8-2a43-4c2a-8fd9-a5539475df2a" ) @@ -50,6 +50,24 @@ func TestEnsureACRToken(t *testing.T) { }, wantErr: "TerminalError: No registry profile detected.", }, + { + name: "No expiry date", + azureEnv: azureclient.PublicCloud, + oc: func() *api.OpenShiftCluster { + return &api.OpenShiftCluster{ + Properties: api.OpenShiftClusterProperties{ + RegistryProfiles: []*api.RegistryProfile{ + { + Name: registryName + ".azurecr.io", + Username: "testuser", + Expiry: nil, + }, + }, + }, + } + }, + wantErr: "TerminalError: No expiry date detected.", + }, { name: "expired", azureEnv: azureclient.PublicCloud, From a9e02de70edfdae813af379b1fd19abd9b84cd9d Mon Sep 17 00:00:00 2001 From: Edison Cardenas Date: Thu, 12 Sep 2024 18:07:24 +1200 Subject: [PATCH 42/67] refactor: update package groupings and error messages to resolve issues detected by automated checks --- pkg/mimo/tasks/cluster/acrtoken_checker.go | 8 ++++---- pkg/mimo/tasks/cluster/acrtoken_checker_test.go | 10 ++++------ test/mimo/tasks/testcontext.go | 2 +- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/pkg/mimo/tasks/cluster/acrtoken_checker.go b/pkg/mimo/tasks/cluster/acrtoken_checker.go index 36783940c8b..5170bf088d0 100644 --- a/pkg/mimo/tasks/cluster/acrtoken_checker.go +++ b/pkg/mimo/tasks/cluster/acrtoken_checker.go @@ -39,14 +39,14 @@ func EnsureACRTokenIsValid(ctx context.Context) error { switch { case expiry == nil: - return mimo.TerminalError(errors.New("No expiry date detected.")) + return mimo.TerminalError(errors.New("no expiry date detected")) case expiry.Time.Before(now): - return mimo.TerminalError(errors.New("ACR token has expired")) + return mimo.TerminalError(errors.New("azure container registry (acr) token has expired")) default: - th.SetResultMessage("ACR token is valid") + th.SetResultMessage("azure container registry (acr) token is valid") } } else { - return mimo.TerminalError(errors.New("No registry profile detected.")) + return mimo.TerminalError(errors.New("no registry profile detected")) } return nil diff --git a/pkg/mimo/tasks/cluster/acrtoken_checker_test.go b/pkg/mimo/tasks/cluster/acrtoken_checker_test.go index 453d7f822cf..2fdaa9abb7b 100644 --- a/pkg/mimo/tasks/cluster/acrtoken_checker_test.go +++ b/pkg/mimo/tasks/cluster/acrtoken_checker_test.go @@ -10,7 +10,6 @@ import ( . "github.com/onsi/gomega" - utilerror "github.com/Azure/ARO-RP/test/util/error" "github.com/Azure/go-autorest/autorest/date" "github.com/golang/mock/gomock" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -18,11 +17,10 @@ import ( "github.com/Azure/ARO-RP/pkg/api" "github.com/Azure/ARO-RP/pkg/util/azureclient" "github.com/Azure/ARO-RP/pkg/util/clienthelper" - - // mock_acrtoken "github.com/Azure/ARO-RP/pkg/util/mocks/acrtoken" mock_env "github.com/Azure/ARO-RP/pkg/util/mocks/env" testtasks "github.com/Azure/ARO-RP/test/mimo/tasks" testclienthelper "github.com/Azure/ARO-RP/test/util/clienthelper" + utilerror "github.com/Azure/ARO-RP/test/util/error" testlog "github.com/Azure/ARO-RP/test/util/log" ) @@ -48,7 +46,7 @@ func TestEnsureACRToken(t *testing.T) { Properties: api.OpenShiftClusterProperties{}, } }, - wantErr: "TerminalError: No registry profile detected.", + wantErr: "TerminalError: no registry profile detected", }, { name: "No expiry date", @@ -66,7 +64,7 @@ func TestEnsureACRToken(t *testing.T) { }, } }, - wantErr: "TerminalError: No expiry date detected.", + wantErr: "TerminalError: no expiry date detected", }, { name: "expired", @@ -89,7 +87,7 @@ func TestEnsureACRToken(t *testing.T) { }, } }, - wantErr: "TerminalError: ACR token has expired", + wantErr: "TerminalError: azure container registry (acr) token has expired", }, } { t.Run(tt.name, func(t *testing.T) { diff --git a/test/mimo/tasks/testcontext.go b/test/mimo/tasks/testcontext.go index 0f137d5004f..791f26f2ef1 100644 --- a/test/mimo/tasks/testcontext.go +++ b/test/mimo/tasks/testcontext.go @@ -8,12 +8,12 @@ import ( "fmt" "time" + "github.com/Azure/go-autorest/autorest" "github.com/sirupsen/logrus" "github.com/Azure/ARO-RP/pkg/api" "github.com/Azure/ARO-RP/pkg/env" "github.com/Azure/ARO-RP/pkg/util/clienthelper" - "github.com/Azure/go-autorest/autorest" ) type fakeTestContext struct { From 4abdcc020c4fb87c54d12a01e95c1a322c7407bb Mon Sep 17 00:00:00 2001 From: Edison Cardenas Date: Mon, 16 Sep 2024 17:34:36 +1200 Subject: [PATCH 43/67] ARO-9263: Change expiry to the date the token was issued. --- pkg/api/admin/openshiftcluster.go | 2 +- .../openshiftcluster_validatestatic_test.go | 6 +++--- pkg/api/openshiftcluster.go | 2 +- pkg/cluster/acrtoken.go | 2 ++ pkg/mimo/tasks/cluster/acrtoken_checker.go | 2 +- .../tasks/cluster/acrtoken_checker_test.go | 18 +++++++++--------- pkg/util/acrtoken/acrtoken.go | 3 +-- pkg/util/acrtoken/acrtoken_test.go | 2 +- 8 files changed, 19 insertions(+), 18 deletions(-) diff --git a/pkg/api/admin/openshiftcluster.go b/pkg/api/admin/openshiftcluster.go index 279c40a2f8b..bc9beaf9df1 100644 --- a/pkg/api/admin/openshiftcluster.go +++ b/pkg/api/admin/openshiftcluster.go @@ -470,7 +470,7 @@ const ( type RegistryProfile struct { Name string `json:"name,omitempty"` Username string `json:"username,omitempty"` - Expiry *date.Time `json:"expiry,omitempty"` + IssueDate *date.Time `json:"issueDate,omitempty"` } // ArchitectureVersion represents an architecture version diff --git a/pkg/api/admin/openshiftcluster_validatestatic_test.go b/pkg/api/admin/openshiftcluster_validatestatic_test.go index a1ec2739f31..08ccb0eb9f1 100644 --- a/pkg/api/admin/openshiftcluster_validatestatic_test.go +++ b/pkg/api/admin/openshiftcluster_validatestatic_test.go @@ -630,7 +630,7 @@ func TestOpenShiftClusterStaticValidateDelta(t *testing.T) { oc: func() *OpenShiftCluster { return &OpenShiftCluster{ Properties: OpenShiftClusterProperties{ - RegistryProfiles: []RegistryProfile{{Name: "test", Username: "testuser", Expiry: toDate(time.Now())}}, + RegistryProfiles: []RegistryProfile{{Name: "test", Username: "testuser", IssueDate: toDate(time.Now())}}, }, } }, @@ -644,12 +644,12 @@ func TestOpenShiftClusterStaticValidateDelta(t *testing.T) { oc: func() *OpenShiftCluster { return &OpenShiftCluster{ Properties: OpenShiftClusterProperties{ - RegistryProfiles: []RegistryProfile{{Name: "test", Username: "testuser", Expiry: toDate(time.Now())}}, + RegistryProfiles: []RegistryProfile{{Name: "test", Username: "testuser", IssueDate: toDate(time.Now())}}, }, } }, modify: func(oc *OpenShiftCluster) { - oc.Properties.RegistryProfiles[0].Expiry = toDate(time.Now().UTC().Add(time.Hour * 24 * 30)) + oc.Properties.RegistryProfiles[0].IssueDate = toDate(time.Now().UTC().Add(time.Hour * 24 * 30)) }, wantErr: "400: PropertyChangeNotAllowed: properties.registryProfiles: Changing property 'properties.registryProfiles' is not allowed.", }, diff --git a/pkg/api/openshiftcluster.go b/pkg/api/openshiftcluster.go index 83d6e2423f0..2ff913b9441 100644 --- a/pkg/api/openshiftcluster.go +++ b/pkg/api/openshiftcluster.go @@ -773,7 +773,7 @@ type RegistryProfile struct { Name string `json:"name,omitempty"` Username string `json:"username,omitempty"` Password SecureString `json:"password,omitempty"` - Expiry *date.Time `json:"expiry,omitempty"` + IssueDate *date.Time `json:"issueDate,omitempty"` } // Install represents an install process diff --git a/pkg/cluster/acrtoken.go b/pkg/cluster/acrtoken.go index 9ffc25e65ed..ba7c4e8d9b7 100644 --- a/pkg/cluster/acrtoken.go +++ b/pkg/cluster/acrtoken.go @@ -7,6 +7,7 @@ import ( "context" "time" + "github.com/Azure/go-autorest/autorest/date" corev1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -57,6 +58,7 @@ func (m *manager) ensureACRToken(ctx context.Context) error { } rp.Password = api.SecureString(password) + rp.IssueDate = &date.Time{Time: time.Now().UTC()} m.doc, err = m.db.PatchWithLease(ctx, m.doc.Key, func(doc *api.OpenShiftClusterDocument) error { token.PutRegistryProfile(doc.OpenShiftCluster, rp) diff --git a/pkg/mimo/tasks/cluster/acrtoken_checker.go b/pkg/mimo/tasks/cluster/acrtoken_checker.go index 5170bf088d0..d9fce5f966b 100644 --- a/pkg/mimo/tasks/cluster/acrtoken_checker.go +++ b/pkg/mimo/tasks/cluster/acrtoken_checker.go @@ -35,7 +35,7 @@ func EnsureACRTokenIsValid(ctx context.Context) error { rp := manager.GetRegistryProfileFromSlice(registryProfiles) if rp != nil { var now = time.Now().UTC() - expiry := registryProfiles[0].Expiry + expiry := registryProfiles[0].IssueDate switch { case expiry == nil: diff --git a/pkg/mimo/tasks/cluster/acrtoken_checker_test.go b/pkg/mimo/tasks/cluster/acrtoken_checker_test.go index 2fdaa9abb7b..2e0d5d8229c 100644 --- a/pkg/mimo/tasks/cluster/acrtoken_checker_test.go +++ b/pkg/mimo/tasks/cluster/acrtoken_checker_test.go @@ -56,9 +56,9 @@ func TestEnsureACRToken(t *testing.T) { Properties: api.OpenShiftClusterProperties{ RegistryProfiles: []*api.RegistryProfile{ { - Name: registryName + ".azurecr.io", - Username: "testuser", - Expiry: nil, + Name: registryName + ".azurecr.io", + Username: "testuser", + IssueDate: nil, }, }, }, @@ -74,14 +74,14 @@ func TestEnsureACRToken(t *testing.T) { Properties: api.OpenShiftClusterProperties{ RegistryProfiles: []*api.RegistryProfile{ { - Name: "arosvc.azurecr.io", - Username: "testuser", - Expiry: &date.Time{Time: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)}, + Name: "arosvc.azurecr.io", + Username: "testuser", + IssueDate: &date.Time{Time: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)}, }, { - Name: "arointsvc.azurecr.io", - Username: "testuser", - Expiry: &date.Time{Time: time.Date(2024, 1, 9, 0, 0, 0, 0, time.UTC)}, + Name: "arointsvc.azurecr.io", + Username: "testuser", + IssueDate: &date.Time{Time: time.Date(2024, 1, 9, 0, 0, 0, 0, time.UTC)}, }, }, }, diff --git a/pkg/util/acrtoken/acrtoken.go b/pkg/util/acrtoken/acrtoken.go index 70aeafc1c01..49f3137db31 100644 --- a/pkg/util/acrtoken/acrtoken.go +++ b/pkg/util/acrtoken/acrtoken.go @@ -87,7 +87,7 @@ func (m *manager) NewRegistryProfile(oc *api.OpenShiftCluster) *api.RegistryProf return &api.RegistryProfile{ Name: fmt.Sprintf("%s.%s", m.r.ResourceName, m.env.Environment().ContainerRegistryDNSSuffix), Username: "token-" + uuid.DefaultGenerator.Generate(), - Expiry: &date.Time{Time: tokenExpiration}, + IssueDate: &date.Time{Time: tokenExpiration}, } } @@ -169,7 +169,6 @@ func (m *manager) generateTokenPassword(ctx context.Context, passwordName mgmtco creds, err := m.registries.GenerateCredentials(ctx, m.r.ResourceGroup, m.r.ResourceName, mgmtcontainerregistry.GenerateCredentialsParameters{ TokenID: to.StringPtr(m.env.ACRResourceID() + "/tokens/" + rp.Username), Name: passwordName, - Expiry: rp.Expiry, }) if err != nil { return "", err diff --git a/pkg/util/acrtoken/acrtoken_test.go b/pkg/util/acrtoken/acrtoken_test.go index b0244d158e8..5ea066951c2 100644 --- a/pkg/util/acrtoken/acrtoken_test.go +++ b/pkg/util/acrtoken/acrtoken_test.go @@ -68,7 +68,7 @@ func TestEnsureTokenAndPassword(t *testing.T) { tokens: tokens, } - password, err := m.EnsureTokenAndPassword(ctx, &api.RegistryProfile{Username: tokenName, Expiry: &date.Time{Time: tokenExpiration}}) + password, err := m.EnsureTokenAndPassword(ctx, &api.RegistryProfile{Username: tokenName, IssueDate: &date.Time{Time: tokenExpiration}}) if err != nil { t.Fatal(err) } From 449fc414aab58755d662b99d22e62d139b79bcc4 Mon Sep 17 00:00:00 2001 From: Edison Cardenas Date: Tue, 17 Sep 2024 15:59:38 +1200 Subject: [PATCH 44/67] ARO-9263: Revise logic to check issue date instead of expiry --- pkg/api/admin/openshiftcluster.go | 4 +-- pkg/mimo/tasks/cluster/acrtoken_checker.go | 22 ++++++++++---- .../tasks/cluster/acrtoken_checker_test.go | 29 +++++++++++++++++-- 3 files changed, 45 insertions(+), 10 deletions(-) diff --git a/pkg/api/admin/openshiftcluster.go b/pkg/api/admin/openshiftcluster.go index bc9beaf9df1..7a4c2ba9d88 100644 --- a/pkg/api/admin/openshiftcluster.go +++ b/pkg/api/admin/openshiftcluster.go @@ -468,8 +468,8 @@ const ( // RegistryProfile represents a registry profile type RegistryProfile struct { - Name string `json:"name,omitempty"` - Username string `json:"username,omitempty"` + Name string `json:"name,omitempty"` + Username string `json:"username,omitempty"` IssueDate *date.Time `json:"issueDate,omitempty"` } diff --git a/pkg/mimo/tasks/cluster/acrtoken_checker.go b/pkg/mimo/tasks/cluster/acrtoken_checker.go index d9fce5f966b..292d9cc954a 100644 --- a/pkg/mimo/tasks/cluster/acrtoken_checker.go +++ b/pkg/mimo/tasks/cluster/acrtoken_checker.go @@ -6,12 +6,18 @@ package cluster import ( "context" "errors" + "fmt" "time" "github.com/Azure/ARO-RP/pkg/util/acrtoken" "github.com/Azure/ARO-RP/pkg/util/mimo" ) +const ( + daysValid = 90 + daysShouldRotate = 45 +) + // EnsureACRTokenIsValid checks the expiry date of the Azure Container Registry (ACR) Token from the RegistryProfile. // It returns an error if the expiry date is past the date now or if there is no registry profile found. func EnsureACRTokenIsValid(ctx context.Context) error { @@ -35,13 +41,19 @@ func EnsureACRTokenIsValid(ctx context.Context) error { rp := manager.GetRegistryProfileFromSlice(registryProfiles) if rp != nil { var now = time.Now().UTC() - expiry := registryProfiles[0].IssueDate + issueDate := rp.IssueDate - switch { - case expiry == nil: + if issueDate == nil { return mimo.TerminalError(errors.New("no expiry date detected")) - case expiry.Time.Before(now): - return mimo.TerminalError(errors.New("azure container registry (acr) token has expired")) + } + + daysInterval := int32(now.Sub(issueDate.Time).Hours() / 24) + + switch { + case daysInterval > daysValid: + return mimo.TerminalError(fmt.Errorf("azure container registry (acr) token has expired, %d days have passed", daysInterval)) + case daysInterval >= daysShouldRotate: + return mimo.TerminalError(fmt.Errorf("%d days have passed since azure container registry (acr) token was issued, please rotate the token now", daysInterval)) default: th.SetResultMessage("azure container registry (acr) token is valid") } diff --git a/pkg/mimo/tasks/cluster/acrtoken_checker_test.go b/pkg/mimo/tasks/cluster/acrtoken_checker_test.go index 2e0d5d8229c..34b7b66f300 100644 --- a/pkg/mimo/tasks/cluster/acrtoken_checker_test.go +++ b/pkg/mimo/tasks/cluster/acrtoken_checker_test.go @@ -49,7 +49,7 @@ func TestEnsureACRToken(t *testing.T) { wantErr: "TerminalError: no registry profile detected", }, { - name: "No expiry date", + name: "No issue date", azureEnv: azureclient.PublicCloud, oc: func() *api.OpenShiftCluster { return &api.OpenShiftCluster{ @@ -67,7 +67,7 @@ func TestEnsureACRToken(t *testing.T) { wantErr: "TerminalError: no expiry date detected", }, { - name: "expired", + name: "Expired", azureEnv: azureclient.PublicCloud, oc: func() *api.OpenShiftCluster { return &api.OpenShiftCluster{ @@ -87,7 +87,30 @@ func TestEnsureACRToken(t *testing.T) { }, } }, - wantErr: "TerminalError: azure container registry (acr) token has expired", + wantErr: "TerminalError: azure container registry (acr) token has expired, 252 days have passed", + }, + { + name: "Should rotate token", + azureEnv: azureclient.PublicCloud, + oc: func() *api.OpenShiftCluster { + return &api.OpenShiftCluster{ + Properties: api.OpenShiftClusterProperties{ + RegistryProfiles: []*api.RegistryProfile{ + { + Name: "arosvc.azurecr.io", + Username: "testuser", + IssueDate: &date.Time{Time: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)}, + }, + { + Name: "arointsvc.azurecr.io", + Username: "testuser", + IssueDate: &date.Time{Time: time.Now().AddDate(0, 0, -50)}, + }, + }, + }, + } + }, + wantErr: "TerminalError: 50 days have passed since azure container registry (acr) token was issued, please rotate the token now", }, } { t.Run(tt.name, func(t *testing.T) { From 1a6810ea2155829eea924ce0f20bd24ddffed562 Mon Sep 17 00:00:00 2001 From: Edison Cardenas Date: Tue, 17 Sep 2024 16:42:21 +1200 Subject: [PATCH 45/67] ARO-9263: Add constants to reduce redunant values --- .../tasks/cluster/acrtoken_checker_test.go | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/pkg/mimo/tasks/cluster/acrtoken_checker_test.go b/pkg/mimo/tasks/cluster/acrtoken_checker_test.go index 34b7b66f300..a1ba3ef784d 100644 --- a/pkg/mimo/tasks/cluster/acrtoken_checker_test.go +++ b/pkg/mimo/tasks/cluster/acrtoken_checker_test.go @@ -25,13 +25,16 @@ import ( ) const ( - registryName = "arointsvc" - registryResourceID = "/subscriptions/93aeba23-2f76-4307-be82-02921df010cf/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/" + registryName + registryResourceID = "/subscriptions/93aeba23-2f76-4307-be82-02921df010cf/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arointsvc" clusterUUID = "512a50c8-2a43-4c2a-8fd9-a5539475df2a" + publicACR = "arosvc.azurecr.io" + intACR = "arointsvc.azurecr.io" + user = "testuser" ) func TestEnsureACRToken(t *testing.T) { ctx := context.Background() + for _, tt := range []struct { name string azureEnv azureclient.AROEnvironment @@ -56,8 +59,8 @@ func TestEnsureACRToken(t *testing.T) { Properties: api.OpenShiftClusterProperties{ RegistryProfiles: []*api.RegistryProfile{ { - Name: registryName + ".azurecr.io", - Username: "testuser", + Name: intACR, + Username: user, IssueDate: nil, }, }, @@ -74,13 +77,13 @@ func TestEnsureACRToken(t *testing.T) { Properties: api.OpenShiftClusterProperties{ RegistryProfiles: []*api.RegistryProfile{ { - Name: "arosvc.azurecr.io", - Username: "testuser", + Name: publicACR, + Username: user, IssueDate: &date.Time{Time: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)}, }, { - Name: "arointsvc.azurecr.io", - Username: "testuser", + Name: intACR, + Username: user, IssueDate: &date.Time{Time: time.Date(2024, 1, 9, 0, 0, 0, 0, time.UTC)}, }, }, @@ -97,13 +100,13 @@ func TestEnsureACRToken(t *testing.T) { Properties: api.OpenShiftClusterProperties{ RegistryProfiles: []*api.RegistryProfile{ { - Name: "arosvc.azurecr.io", - Username: "testuser", + Name: publicACR, + Username: user, IssueDate: &date.Time{Time: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)}, }, { - Name: "arointsvc.azurecr.io", - Username: "testuser", + Name: intACR, + Username: user, IssueDate: &date.Time{Time: time.Now().AddDate(0, 0, -50)}, }, }, From ebf3c4e7cd76347b8ea283481f068b09f58d8ae6 Mon Sep 17 00:00:00 2001 From: Edison Cardenas Date: Wed, 18 Sep 2024 19:01:13 +1200 Subject: [PATCH 46/67] ARO-9263: Update test to check issue date in constant time to avoid failure --- pkg/mimo/tasks/cluster/acrtoken_checker_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/mimo/tasks/cluster/acrtoken_checker_test.go b/pkg/mimo/tasks/cluster/acrtoken_checker_test.go index a1ba3ef784d..57f49cb3f44 100644 --- a/pkg/mimo/tasks/cluster/acrtoken_checker_test.go +++ b/pkg/mimo/tasks/cluster/acrtoken_checker_test.go @@ -84,13 +84,13 @@ func TestEnsureACRToken(t *testing.T) { { Name: intACR, Username: user, - IssueDate: &date.Time{Time: time.Date(2024, 1, 9, 0, 0, 0, 0, time.UTC)}, + IssueDate: &date.Time{Time: time.Now().AddDate(0, 0, -100)}, }, }, }, } }, - wantErr: "TerminalError: azure container registry (acr) token has expired, 252 days have passed", + wantErr: "TerminalError: azure container registry (acr) token has expired, 100 days have passed", }, { name: "Should rotate token", From 1973281331353e59adafd00bbc3ab5c511ba2d94 Mon Sep 17 00:00:00 2001 From: Edison Cardenas Date: Thu, 19 Sep 2024 18:04:32 +1200 Subject: [PATCH 47/67] ARO-9263: Change or remove any references about expiry to issue date. --- pkg/mimo/const.go | 1 + pkg/mimo/sets/sets.go | 6 +++--- pkg/mimo/tasks/cluster/acrtoken_checker.go | 4 ++-- pkg/mimo/tasks/cluster/acrtoken_checker_test.go | 4 ++-- pkg/util/acrtoken/acrtoken.go | 3 +-- pkg/util/acrtoken/acrtoken_test.go | 3 +-- 6 files changed, 10 insertions(+), 11 deletions(-) diff --git a/pkg/mimo/const.go b/pkg/mimo/const.go index b66bc025adb..7dea33e1c8e 100644 --- a/pkg/mimo/const.go +++ b/pkg/mimo/const.go @@ -6,4 +6,5 @@ package mimo const ( TLS_CERT_ROTATION_ID = "9b741734-6505-447f-8510-85eb0ae561a2" OPERATOR_FLAGS_UPDATE_ID = "b41749fc-af26-4ab7-b5a1-e03f3ee4cba6" + ACR_TOKEN_CHECKER_ID = "082978ce-3700-4972-835f-53d48658d291" ) diff --git a/pkg/mimo/sets/sets.go b/pkg/mimo/sets/sets.go index 403758073c1..a6d1e56f664 100644 --- a/pkg/mimo/sets/sets.go +++ b/pkg/mimo/sets/sets.go @@ -16,9 +16,9 @@ const DEFAULT_POLL_TIME = time.Second * 10 const DEFAULT_TIMEOUT_DURATION = time.Minute * 20 var DEFAULT_MAINTENANCE_SETS = map[string]MaintenanceSet{ - mimo.TLS_CERT_ROTATION_ID: TLSCertRotation, - "082978ce-3700-4972-835f-53d48658d291": ACRTokenChecker, - mimo.OPERATOR_FLAGS_UPDATE_ID: UpdateOperatorFlags, + mimo.TLS_CERT_ROTATION_ID: TLSCertRotation, + mimo.ACR_TOKEN_CHECKER_ID: ACRTokenChecker, + mimo.OPERATOR_FLAGS_UPDATE_ID: UpdateOperatorFlags, } func run(t utilmimo.TaskContext, s []steps.Step) (api.MaintenanceManifestState, string) { diff --git a/pkg/mimo/tasks/cluster/acrtoken_checker.go b/pkg/mimo/tasks/cluster/acrtoken_checker.go index 292d9cc954a..4a642766e17 100644 --- a/pkg/mimo/tasks/cluster/acrtoken_checker.go +++ b/pkg/mimo/tasks/cluster/acrtoken_checker.go @@ -44,14 +44,14 @@ func EnsureACRTokenIsValid(ctx context.Context) error { issueDate := rp.IssueDate if issueDate == nil { - return mimo.TerminalError(errors.New("no expiry date detected")) + return mimo.TerminalError(errors.New("no issue date detected")) } daysInterval := int32(now.Sub(issueDate.Time).Hours() / 24) switch { case daysInterval > daysValid: - return mimo.TerminalError(fmt.Errorf("azure container registry (acr) token has expired, %d days have passed", daysInterval)) + return mimo.TerminalError(fmt.Errorf("azure container registry (acr) token is not valid, %d days have passed", daysInterval)) case daysInterval >= daysShouldRotate: return mimo.TerminalError(fmt.Errorf("%d days have passed since azure container registry (acr) token was issued, please rotate the token now", daysInterval)) default: diff --git a/pkg/mimo/tasks/cluster/acrtoken_checker_test.go b/pkg/mimo/tasks/cluster/acrtoken_checker_test.go index 57f49cb3f44..b4e5086957f 100644 --- a/pkg/mimo/tasks/cluster/acrtoken_checker_test.go +++ b/pkg/mimo/tasks/cluster/acrtoken_checker_test.go @@ -67,7 +67,7 @@ func TestEnsureACRToken(t *testing.T) { }, } }, - wantErr: "TerminalError: no expiry date detected", + wantErr: "TerminalError: no issue date detected", }, { name: "Expired", @@ -90,7 +90,7 @@ func TestEnsureACRToken(t *testing.T) { }, } }, - wantErr: "TerminalError: azure container registry (acr) token has expired, 100 days have passed", + wantErr: "TerminalError: azure container registry (acr) token is not valid, 100 days have passed", }, { name: "Should rotate token", diff --git a/pkg/util/acrtoken/acrtoken.go b/pkg/util/acrtoken/acrtoken.go index 49f3137db31..41a75080a59 100644 --- a/pkg/util/acrtoken/acrtoken.go +++ b/pkg/util/acrtoken/acrtoken.go @@ -82,12 +82,11 @@ func (m *manager) GetRegistryProfileFromSlice(registryProfiles []*api.RegistryPr } func (m *manager) NewRegistryProfile(oc *api.OpenShiftCluster) *api.RegistryProfile { - var tokenExpiration = time.Now().UTC().Add(time.Hour * HoursInADay * ACRTokenLifeInDays) return &api.RegistryProfile{ Name: fmt.Sprintf("%s.%s", m.r.ResourceName, m.env.Environment().ContainerRegistryDNSSuffix), Username: "token-" + uuid.DefaultGenerator.Generate(), - IssueDate: &date.Time{Time: tokenExpiration}, + IssueDate: &date.Time{Time: time.Now().UTC()}, } } diff --git a/pkg/util/acrtoken/acrtoken_test.go b/pkg/util/acrtoken/acrtoken_test.go index 5ea066951c2..1864ce3b8f0 100644 --- a/pkg/util/acrtoken/acrtoken_test.go +++ b/pkg/util/acrtoken/acrtoken_test.go @@ -26,7 +26,6 @@ const ( func TestEnsureTokenAndPassword(t *testing.T) { ctx := context.Background() - var tokenExpiration = time.Now().UTC().Add(time.Hour * 24 * 90) controller := gomock.NewController(t) defer controller.Finish() @@ -68,7 +67,7 @@ func TestEnsureTokenAndPassword(t *testing.T) { tokens: tokens, } - password, err := m.EnsureTokenAndPassword(ctx, &api.RegistryProfile{Username: tokenName, IssueDate: &date.Time{Time: tokenExpiration}}) + password, err := m.EnsureTokenAndPassword(ctx, &api.RegistryProfile{Username: tokenName, IssueDate: &date.Time{Time: time.Now().AddDate(0, 0, -50)}}) if err != nil { t.Fatal(err) } From f472e787da68055c49d9f801064d5baf2ed14a87 Mon Sep 17 00:00:00 2001 From: Edison Cardenas Date: Fri, 20 Sep 2024 12:56:05 +1200 Subject: [PATCH 48/67] ARO-9263: Fix lint issues --- pkg/api/openshiftcluster.go | 6 +++--- pkg/util/acrtoken/acrtoken.go | 5 ++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/pkg/api/openshiftcluster.go b/pkg/api/openshiftcluster.go index 2ff913b9441..d52b938ab93 100644 --- a/pkg/api/openshiftcluster.go +++ b/pkg/api/openshiftcluster.go @@ -770,9 +770,9 @@ type IngressProfile struct { type RegistryProfile struct { MissingFields - Name string `json:"name,omitempty"` - Username string `json:"username,omitempty"` - Password SecureString `json:"password,omitempty"` + Name string `json:"name,omitempty"` + Username string `json:"username,omitempty"` + Password SecureString `json:"password,omitempty"` IssueDate *date.Time `json:"issueDate,omitempty"` } diff --git a/pkg/util/acrtoken/acrtoken.go b/pkg/util/acrtoken/acrtoken.go index 41a75080a59..762d97a8693 100644 --- a/pkg/util/acrtoken/acrtoken.go +++ b/pkg/util/acrtoken/acrtoken.go @@ -82,10 +82,9 @@ func (m *manager) GetRegistryProfileFromSlice(registryProfiles []*api.RegistryPr } func (m *manager) NewRegistryProfile(oc *api.OpenShiftCluster) *api.RegistryProfile { - return &api.RegistryProfile{ - Name: fmt.Sprintf("%s.%s", m.r.ResourceName, m.env.Environment().ContainerRegistryDNSSuffix), - Username: "token-" + uuid.DefaultGenerator.Generate(), + Name: fmt.Sprintf("%s.%s", m.r.ResourceName, m.env.Environment().ContainerRegistryDNSSuffix), + Username: "token-" + uuid.DefaultGenerator.Generate(), IssueDate: &date.Time{Time: time.Now().UTC()}, } } From 101db0182a06806388b4ead06fbca75201334b5f Mon Sep 17 00:00:00 2001 From: Edison Cardenas Date: Tue, 24 Sep 2024 13:03:40 +1200 Subject: [PATCH 49/67] ARO-9263: Revise error message and reorder return statement --- pkg/mimo/tasks/cluster/acrtoken_checker.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pkg/mimo/tasks/cluster/acrtoken_checker.go b/pkg/mimo/tasks/cluster/acrtoken_checker.go index 4a642766e17..9f17733058d 100644 --- a/pkg/mimo/tasks/cluster/acrtoken_checker.go +++ b/pkg/mimo/tasks/cluster/acrtoken_checker.go @@ -44,7 +44,7 @@ func EnsureACRTokenIsValid(ctx context.Context) error { issueDate := rp.IssueDate if issueDate == nil { - return mimo.TerminalError(errors.New("no issue date detected")) + return mimo.TerminalError(errors.New("no issue date detected, please rotate token")) } daysInterval := int32(now.Sub(issueDate.Time).Hours() / 24) @@ -57,9 +57,7 @@ func EnsureACRTokenIsValid(ctx context.Context) error { default: th.SetResultMessage("azure container registry (acr) token is valid") } - } else { - return mimo.TerminalError(errors.New("no registry profile detected")) } - return nil + return mimo.TerminalError(errors.New("no registry profile detected")) } From 409d8a93d18193d5f5bcd85e85a337f0c167ca72 Mon Sep 17 00:00:00 2001 From: Edison Cardenas Date: Tue, 24 Sep 2024 16:46:41 +1200 Subject: [PATCH 50/67] ARO-9263: Fix unit test --- pkg/mimo/tasks/cluster/acrtoken_checker_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/mimo/tasks/cluster/acrtoken_checker_test.go b/pkg/mimo/tasks/cluster/acrtoken_checker_test.go index b4e5086957f..3775454c4e1 100644 --- a/pkg/mimo/tasks/cluster/acrtoken_checker_test.go +++ b/pkg/mimo/tasks/cluster/acrtoken_checker_test.go @@ -67,7 +67,7 @@ func TestEnsureACRToken(t *testing.T) { }, } }, - wantErr: "TerminalError: no issue date detected", + wantErr: "TerminalError: no issue date detected, please rotate token", }, { name: "Expired", From 40f1d9e12d89ba67ddb773b7c0b6c046fc1ba384 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Thu, 26 Sep 2024 13:36:55 +1000 Subject: [PATCH 51/67] fix e2e, hopefully --- .../admin/openshiftcluster_validatestatic.go | 1 + pkg/api/openshiftcluster.go | 1 + pkg/frontend/admin_mimo_manifest_create.go | 2 +- .../admin_mimo_manifest_create_test.go | 38 +++++++++++++++++++ test/e2e/mimo_actuator.go | 13 +++++-- 5 files changed, 51 insertions(+), 4 deletions(-) diff --git a/pkg/api/admin/openshiftcluster_validatestatic.go b/pkg/api/admin/openshiftcluster_validatestatic.go index fe0d7d8d420..ea8fbf8aff9 100644 --- a/pkg/api/admin/openshiftcluster_validatestatic.go +++ b/pkg/api/admin/openshiftcluster_validatestatic.go @@ -39,6 +39,7 @@ func validateMaintenanceTask(task MaintenanceTask) error { task == MaintenanceTaskRenewCerts || task == MaintenanceTaskPending || task == MaintenanceTaskNone || + task == MaintenanceTaskSyncClusterObject || task == MaintenanceTaskCustomerActionNeeded) { return api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidParameter, "properties.maintenanceTask", "Invalid enum parameter.") } diff --git a/pkg/api/openshiftcluster.go b/pkg/api/openshiftcluster.go index d52b938ab93..b0ee2ad472b 100644 --- a/pkg/api/openshiftcluster.go +++ b/pkg/api/openshiftcluster.go @@ -262,6 +262,7 @@ func (t MaintenanceTask) IsMaintenanceOngoingTask() bool { result := (t == MaintenanceTaskEverything) || (t == MaintenanceTaskOperator) || (t == MaintenanceTaskRenewCerts) || + (t == MaintenanceTaskSyncClusterObject) || (t == "") return result } diff --git a/pkg/frontend/admin_mimo_manifest_create.go b/pkg/frontend/admin_mimo_manifest_create.go index c42c22f86d4..cf055d0062e 100644 --- a/pkg/frontend/admin_mimo_manifest_create.go +++ b/pkg/frontend/admin_mimo_manifest_create.go @@ -66,7 +66,7 @@ func (f *frontend) _putAdminMaintManifestCreate(ctx context.Context, r *http.Req return nil, api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidRequestContent, "", "The request content could not be deserialized: "+err.Error()) } - // fill in some sefaults + // fill in some defaults ext.ID = dbMaintenanceManifests.NewUUID() ext.State = admin.MaintenanceManifestStatePending diff --git a/pkg/frontend/admin_mimo_manifest_create_test.go b/pkg/frontend/admin_mimo_manifest_create_test.go index 777122ea49c..e23f1ccf376 100644 --- a/pkg/frontend/admin_mimo_manifest_create_test.go +++ b/pkg/frontend/admin_mimo_manifest_create_test.go @@ -114,6 +114,44 @@ func TestMIMOCreateManifest(t *testing.T) { }, wantStatusCode: http.StatusCreated, }, + { + name: "default set to pending", + fixtures: func(f *testdatabase.Fixture) { + f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{ + Key: strings.ToLower(resourceID), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: resourceID, + Name: "resourceName", + Type: "Microsoft.RedHatOpenShift/openshiftClusters", + }, + }) + }, + body: &admin.MaintenanceManifest{ + MaintenanceSetID: "exampleset", + RunAfter: 1, + RunBefore: 1, + }, + wantResult: func(c *testdatabase.Checker) { + c.AddMaintenanceManifestDocuments(&api.MaintenanceManifestDocument{ + ID: "07070707-0707-0707-0707-070707070001", + ClusterResourceID: strings.ToLower(resourceID), + MaintenanceManifest: api.MaintenanceManifest{ + MaintenanceSetID: "exampleset", + State: api.MaintenanceManifestStatePending, + RunAfter: 1, + RunBefore: 1, + }, + }) + }, + wantResponse: &admin.MaintenanceManifest{ + ID: "07070707-0707-0707-0707-070707070001", + MaintenanceSetID: "exampleset", + State: admin.MaintenanceManifestStatePending, + RunAfter: 1, + RunBefore: 1, + }, + wantStatusCode: http.StatusCreated, + }, } { t.Run(tt.name, func(t *testing.T) { now := func() time.Time { return time.Unix(1000, 0) } diff --git a/test/e2e/mimo_actuator.go b/test/e2e/mimo_actuator.go index c9c760d23b0..89ecdbf831e 100644 --- a/test/e2e/mimo_actuator.go +++ b/test/e2e/mimo_actuator.go @@ -5,7 +5,6 @@ package e2e import ( "context" - "encoding/json" "net/http" "net/url" @@ -31,10 +30,18 @@ var _ = Describe("MIMO Actuator E2E Testing", func() { var oc = &admin.OpenShiftCluster{} testflag := "aro.e2e.testflag." + uuid.DefaultGenerator.Generate() + ocIn := &admin.OpenShiftCluster{ + Properties: admin.OpenShiftClusterProperties{ + MaintenanceTask: admin.MaintenanceTaskSyncClusterObject, + OperatorFlags: admin.OperatorFlags{ + testflag: "true", + }, + }, + } + By("set a bogus flag on the cluster") resp, err := adminRequest(ctx, - http.MethodPatch, clusterResourceID, nil, true, - json.RawMessage("{\"properties\": {\"maintenanceTask\": \"SyncClusterObject\", \"operatorFlags\": {\""+testflag+"\": \"true\"}}}"), oc) + http.MethodPatch, clusterResourceID, nil, true, ocIn, oc) Expect(err).NotTo(HaveOccurred()) Expect(resp.StatusCode).To(Equal(http.StatusOK)) From 79a39dbf648f52a79632dc858a32b28821e3646b Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Thu, 26 Sep 2024 17:54:13 +1000 Subject: [PATCH 52/67] pls --- test/e2e/mimo_actuator.go | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/test/e2e/mimo_actuator.go b/test/e2e/mimo_actuator.go index 89ecdbf831e..c9c760d23b0 100644 --- a/test/e2e/mimo_actuator.go +++ b/test/e2e/mimo_actuator.go @@ -5,6 +5,7 @@ package e2e import ( "context" + "encoding/json" "net/http" "net/url" @@ -30,18 +31,10 @@ var _ = Describe("MIMO Actuator E2E Testing", func() { var oc = &admin.OpenShiftCluster{} testflag := "aro.e2e.testflag." + uuid.DefaultGenerator.Generate() - ocIn := &admin.OpenShiftCluster{ - Properties: admin.OpenShiftClusterProperties{ - MaintenanceTask: admin.MaintenanceTaskSyncClusterObject, - OperatorFlags: admin.OperatorFlags{ - testflag: "true", - }, - }, - } - By("set a bogus flag on the cluster") resp, err := adminRequest(ctx, - http.MethodPatch, clusterResourceID, nil, true, ocIn, oc) + http.MethodPatch, clusterResourceID, nil, true, + json.RawMessage("{\"properties\": {\"maintenanceTask\": \"SyncClusterObject\", \"operatorFlags\": {\""+testflag+"\": \"true\"}}}"), oc) Expect(err).NotTo(HaveOccurred()) Expect(resp.StatusCode).To(Equal(http.StatusOK)) From 2155b80811b9512f5556badee1c8b6e37e2a1b8e Mon Sep 17 00:00:00 2001 From: Srinivas Atmakuri Date: Thu, 2 Mar 2023 02:27:18 +0530 Subject: [PATCH 53/67] API OperatorFlagsMergeStrategy JIRA: https://issues.redhat.com/browse/ARO-1885 --- pkg/api/admin/operatorflags.go | 68 +++++++++++++++++++++ pkg/api/admin/operatorflags_test.go | 60 ++++++++++++++++++ pkg/frontend/openshiftcluster_putorpatch.go | 4 ++ 3 files changed, 132 insertions(+) create mode 100644 pkg/api/admin/operatorflags.go create mode 100644 pkg/api/admin/operatorflags_test.go diff --git a/pkg/api/admin/operatorflags.go b/pkg/api/admin/operatorflags.go new file mode 100644 index 00000000000..80561b0ae28 --- /dev/null +++ b/pkg/api/admin/operatorflags.go @@ -0,0 +1,68 @@ +package admin + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "encoding/json" + "net/http" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/operator" +) + +type operatorFlagsMergeStrategyStruct struct { + OperatorFlagsMergeStrategy string + Cluster *api.OpenShiftCluster +} + +const ( + operatorFlagsMergeStrategyDefault string = "merge" + operatorFlagsMergeStrategyMerge string = "merge" + operatorFlagsMergeStrategyReset string = "reset" +) + +// When a cluster is edited via the PATCH Cluster Geneva Action (aka an Admin Update) +// the flags given are treated according to the provided Update Strategy, +// provided in operatorFlagsMergeStrategy + +// merge (default): The provided cluster flags are laid on top of the cluster’s existing flags. +// reset: The provided cluster flags are laid on top of the default cluster flags, +// essentially ‘resetting’ the flags if no new flags are provided. +func OperatorFlagsMergeStrategy(oc *api.OpenShiftCluster, body []byte) error { + payload := operatorFlagsMergeStrategyStruct{ + OperatorFlagsMergeStrategy: operatorFlagsMergeStrategyDefault, + Cluster: &api.OpenShiftCluster{}, + } + + err := json.Unmarshal(body, &payload) + if err != nil { + return api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidRequestContent, "", "The request content was invalid and could not be deserialized: %q.", err) + } + + // return error if OperatorFlagsMergeStrategy is not merge or reset, default is merge + if payload.OperatorFlagsMergeStrategy != operatorFlagsMergeStrategyMerge && + payload.OperatorFlagsMergeStrategy != operatorFlagsMergeStrategyReset { + return api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidParameter, "", "invalid operatorFlagsMergeStrategy '%s', can only be 'merge' or 'reset'", payload.OperatorFlagsMergeStrategy) + } + // return if payload is empty + if payload.Cluster == nil { + return nil + } + properties := &payload.Cluster.Properties + if properties == nil || properties.OperatorFlags == nil { + return nil + } + // return nil, if OperatorFlagsMergeStrategy is merge and payload has not operatorFlags + // return operatorFlags of payload, if OperatorFlagsMergeStrategy is merge and payload has operatorFlags + // return defaultOperatorFlags, if OperatorFlagsMergeStrategy is reset and payload has not operatorFlags + // return defaultOperatorFlags + operatorFlags of payload, if OperatorFlagsMergeStrategy is reset and payload has operatorFlags + if payload.OperatorFlagsMergeStrategy == operatorFlagsMergeStrategyReset { + oc.Properties.OperatorFlags = operator.DefaultOperatorFlags() + for operatorflag, value := range payload.Cluster.Properties.OperatorFlags { + oc.Properties.OperatorFlags[operatorflag] = value + } + } + + return nil +} diff --git a/pkg/api/admin/operatorflags_test.go b/pkg/api/admin/operatorflags_test.go new file mode 100644 index 00000000000..54947f9fea4 --- /dev/null +++ b/pkg/api/admin/operatorflags_test.go @@ -0,0 +1,60 @@ +package admin + +import ( + "testing" + + "github.com/Azure/ARO-RP/pkg/api" + utilerror "github.com/Azure/ARO-RP/test/util/error" +) + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +func TestOperatorFlagsMergeStrategy(t *testing.T) { + tests := []struct { + name string + oc *api.OpenShiftCluster + wantOc *api.OpenShiftCluster + body []byte + wantErr string + }{ + { + name: "invalid_json", + oc: nil, + body: []byte(`{{}`), + wantErr: `400: InvalidRequestContent: : The request content was invalid and could not be deserialized: "invalid character '{' looking for beginning of object key string".`, + }, + { + name: "OperatorFlagsMergeStrategy_is_not_merge_or_reset", + oc: nil, + body: []byte(`{"operatorFlagsMergeStrategy": "xyz"}`), + wantErr: `400: InvalidParameter: : invalid operatorFlagsMergeStrategy 'xyz', can only be 'merge' or 'reset'`, + }, + { + name: "OperatorFlagsMergeStrategy_payload_is_empty", + oc: &api.OpenShiftCluster{ + Properties: api.OpenShiftClusterProperties{ + OperatorFlags: api.OperatorFlags{"aro.feature1.enabled": "false"}, + }, + }, + body: []byte(`{"operatorflagsmergestrategy":"merge"}`), + wantErr: "", + }, + { + name: "OperatorFlagsMergeStrategy_reset", + oc: &api.OpenShiftCluster{ + Properties: api.OpenShiftClusterProperties{ + OperatorFlags: api.OperatorFlags{"aro.feature1.enabled": "false"}, + }, + }, + body: []byte(`{"operatorflagsmergestrategy":"reset"}`), + wantErr: "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := OperatorFlagsMergeStrategy(tt.oc, tt.body) + utilerror.AssertErrorMessage(t, err, tt.wantErr) + }) + } +} diff --git a/pkg/frontend/openshiftcluster_putorpatch.go b/pkg/frontend/openshiftcluster_putorpatch.go index e802b5f3518..43798b27ad6 100644 --- a/pkg/frontend/openshiftcluster_putorpatch.go +++ b/pkg/frontend/openshiftcluster_putorpatch.go @@ -198,6 +198,10 @@ func (f *frontend) _putOrPatchOpenShiftCluster(ctx context.Context, log *logrus. // provide single field json to be updated in the database. // Patch should be used for updating individual fields of the document. case http.MethodPatch: + err = admin.OperatorFlagsMergeStrategy(doc.OpenShiftCluster, putOrPatchClusterParameters.body) + if err != nil { + return nil, err + } ext = putOrPatchClusterParameters.converter.ToExternal(doc.OpenShiftCluster) } From c82f3569d44b6818d143c8356227896eec35c8e1 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 2 Oct 2024 14:21:31 +1000 Subject: [PATCH 54/67] operator flags patches + tests --- pkg/api/admin/openshiftcluster.go | 15 +- pkg/api/admin/operatorflags.go | 38 +- pkg/frontend/openshiftcluster_putorpatch.go | 13 +- .../openshiftcluster_putorpatch_test.go | 862 ++++++++++-------- 4 files changed, 535 insertions(+), 393 deletions(-) diff --git a/pkg/api/admin/openshiftcluster.go b/pkg/api/admin/openshiftcluster.go index 7a4c2ba9d88..7bbd02e62cd 100644 --- a/pkg/api/admin/openshiftcluster.go +++ b/pkg/api/admin/openshiftcluster.go @@ -20,13 +20,14 @@ type OpenShiftClusterList struct { // OpenShiftCluster represents an Azure Red Hat OpenShift cluster. type OpenShiftCluster struct { - ID string `json:"id,omitempty" mutable:"case"` - Name string `json:"name,omitempty" mutable:"case"` - Type string `json:"type,omitempty" mutable:"case"` - Location string `json:"location,omitempty"` - Tags map[string]string `json:"tags,omitempty"` - Properties OpenShiftClusterProperties `json:"properties,omitempty"` - Identity *Identity `json:"identity,omitempty"` + ID string `json:"id,omitempty" mutable:"case"` + Name string `json:"name,omitempty" mutable:"case"` + Type string `json:"type,omitempty" mutable:"case"` + Location string `json:"location,omitempty"` + Tags map[string]string `json:"tags,omitempty"` + Properties OpenShiftClusterProperties `json:"properties,omitempty"` + Identity *Identity `json:"identity,omitempty"` + OperatorFlagsMergeStrategy string `json:"operatorFlagsMergeStrategy,omitempty" mutable:"true"` } // OpenShiftClusterProperties represents an OpenShift cluster's properties. diff --git a/pkg/api/admin/operatorflags.go b/pkg/api/admin/operatorflags.go index 80561b0ae28..b1caefe23e6 100644 --- a/pkg/api/admin/operatorflags.go +++ b/pkg/api/admin/operatorflags.go @@ -12,14 +12,13 @@ import ( ) type operatorFlagsMergeStrategyStruct struct { - OperatorFlagsMergeStrategy string - Cluster *api.OpenShiftCluster + OperatorFlagsMergeStrategy string `json:"operatorFlagsMergeStrategy,omitempty"` + Cluster *api.OpenShiftCluster `json:"cluster,omitempty"` } const ( - operatorFlagsMergeStrategyDefault string = "merge" - operatorFlagsMergeStrategyMerge string = "merge" - operatorFlagsMergeStrategyReset string = "reset" + OperatorFlagsMergeStrategyMerge string = "merge" + OperatorFlagsMergeStrategyReset string = "reset" ) // When a cluster is edited via the PATCH Cluster Geneva Action (aka an Admin Update) @@ -30,38 +29,31 @@ const ( // reset: The provided cluster flags are laid on top of the default cluster flags, // essentially ‘resetting’ the flags if no new flags are provided. func OperatorFlagsMergeStrategy(oc *api.OpenShiftCluster, body []byte) error { - payload := operatorFlagsMergeStrategyStruct{ - OperatorFlagsMergeStrategy: operatorFlagsMergeStrategyDefault, - Cluster: &api.OpenShiftCluster{}, - } + payload := &OpenShiftCluster{} err := json.Unmarshal(body, &payload) if err != nil { return api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidRequestContent, "", "The request content was invalid and could not be deserialized: %q.", err) } - // return error if OperatorFlagsMergeStrategy is not merge or reset, default is merge - if payload.OperatorFlagsMergeStrategy != operatorFlagsMergeStrategyMerge && - payload.OperatorFlagsMergeStrategy != operatorFlagsMergeStrategyReset { - return api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidParameter, "", "invalid operatorFlagsMergeStrategy '%s', can only be 'merge' or 'reset'", payload.OperatorFlagsMergeStrategy) - } - // return if payload is empty - if payload.Cluster == nil { + // if it's empty, use the default of merge, which is performed by + // deserialising the body JSON later + if payload.OperatorFlagsMergeStrategy == "" { return nil } - properties := &payload.Cluster.Properties - if properties == nil || properties.OperatorFlags == nil { - return nil + + // return error if OperatorFlagsMergeStrategy is not merge or reset, default is merge + if payload.OperatorFlagsMergeStrategy != OperatorFlagsMergeStrategyMerge && + payload.OperatorFlagsMergeStrategy != OperatorFlagsMergeStrategyReset { + return api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidParameter, "", "invalid operatorFlagsMergeStrategy '%s', can only be 'merge' or 'reset'", payload.OperatorFlagsMergeStrategy) } + // return nil, if OperatorFlagsMergeStrategy is merge and payload has not operatorFlags // return operatorFlags of payload, if OperatorFlagsMergeStrategy is merge and payload has operatorFlags // return defaultOperatorFlags, if OperatorFlagsMergeStrategy is reset and payload has not operatorFlags // return defaultOperatorFlags + operatorFlags of payload, if OperatorFlagsMergeStrategy is reset and payload has operatorFlags - if payload.OperatorFlagsMergeStrategy == operatorFlagsMergeStrategyReset { + if payload.OperatorFlagsMergeStrategy == OperatorFlagsMergeStrategyReset { oc.Properties.OperatorFlags = operator.DefaultOperatorFlags() - for operatorflag, value := range payload.Cluster.Properties.OperatorFlags { - oc.Properties.OperatorFlags[operatorflag] = value - } } return nil diff --git a/pkg/frontend/openshiftcluster_putorpatch.go b/pkg/frontend/openshiftcluster_putorpatch.go index 43798b27ad6..a4d1e193006 100644 --- a/pkg/frontend/openshiftcluster_putorpatch.go +++ b/pkg/frontend/openshiftcluster_putorpatch.go @@ -198,9 +198,15 @@ func (f *frontend) _putOrPatchOpenShiftCluster(ctx context.Context, log *logrus. // provide single field json to be updated in the database. // Patch should be used for updating individual fields of the document. case http.MethodPatch: - err = admin.OperatorFlagsMergeStrategy(doc.OpenShiftCluster, putOrPatchClusterParameters.body) - if err != nil { - return nil, err + if putOrPatchClusterParameters.apiVersion == admin.APIVersion { + // OperatorFlagsMergeStrategy==reset will place the default flags in + // the external object and then merge in the body's flags when the + // request is unmarshaled below. + err = admin.OperatorFlagsMergeStrategy(doc.OpenShiftCluster, putOrPatchClusterParameters.body) + if err != nil { + // OperatorFlagsMergeStrategy returns CloudErrors + return nil, err + } } ext = putOrPatchClusterParameters.converter.ToExternal(doc.OpenShiftCluster) } @@ -219,6 +225,7 @@ func (f *frontend) _putOrPatchOpenShiftCluster(ctx context.Context, log *logrus. return nil, err } } else { + err = putOrPatchClusterParameters.staticValidator.Static(ext, doc.OpenShiftCluster, f.env.Location(), f.env.Domain(), f.env.FeatureIsSet(env.FeatureRequireD2sV3Workers), putOrPatchClusterParameters.path) if err != nil { return nil, err diff --git a/pkg/frontend/openshiftcluster_putorpatch_test.go b/pkg/frontend/openshiftcluster_putorpatch_test.go index d7e32ae7d9a..5dd0d09caf0 100644 --- a/pkg/frontend/openshiftcluster_putorpatch_test.go +++ b/pkg/frontend/openshiftcluster_putorpatch_test.go @@ -58,7 +58,7 @@ func TestPutOrPatchOpenShiftClusterAdminAPI(t *testing.T) { wantStatusCode int wantEnriched []string wantDocuments func(*testdatabase.Checker) - wantResponse *admin.OpenShiftCluster + wantResponse func() *admin.OpenShiftCluster wantAsync bool wantError string wantSystemDataEnriched bool @@ -133,33 +133,34 @@ func TestPutOrPatchOpenShiftClusterAdminAPI(t *testing.T) { }, wantAsync: true, wantStatusCode: http.StatusOK, - wantResponse: &admin.OpenShiftCluster{ - ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), - Type: "Microsoft.RedHatOpenShift/openShiftClusters", - Tags: map[string]string{"tag": "will-be-kept"}, - Properties: admin.OpenShiftClusterProperties{ - ProvisioningState: admin.ProvisioningStateAdminUpdating, - LastProvisioningState: admin.ProvisioningStateSucceeded, - ClusterProfile: admin.ClusterProfile{ - FipsValidatedModules: admin.FipsValidatedModulesDisabled, - }, - MaintenanceTask: admin.MaintenanceTaskEverything, - NetworkProfile: admin.NetworkProfile{ - OutboundType: admin.OutboundTypeLoadbalancer, - LoadBalancerProfile: &admin.LoadBalancerProfile{ - ManagedOutboundIPs: &admin.ManagedOutboundIPs{ - Count: 1, + wantResponse: func() *admin.OpenShiftCluster { + return &admin.OpenShiftCluster{ + ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), + Type: "Microsoft.RedHatOpenShift/openShiftClusters", + Tags: map[string]string{"tag": "will-be-kept"}, + Properties: admin.OpenShiftClusterProperties{ + ProvisioningState: admin.ProvisioningStateAdminUpdating, + LastProvisioningState: admin.ProvisioningStateSucceeded, + ClusterProfile: admin.ClusterProfile{ + FipsValidatedModules: admin.FipsValidatedModulesDisabled, + }, + MaintenanceTask: admin.MaintenanceTaskEverything, + NetworkProfile: admin.NetworkProfile{ + OutboundType: admin.OutboundTypeLoadbalancer, + LoadBalancerProfile: &admin.LoadBalancerProfile{ + ManagedOutboundIPs: &admin.ManagedOutboundIPs{ + Count: 1, + }, }, }, + MasterProfile: admin.MasterProfile{ + EncryptionAtHost: admin.EncryptionAtHostDisabled, + }, + OperatorFlags: admin.OperatorFlags{"testFlag": "true"}, + MaintenanceState: admin.MaintenanceStateUnplanned, }, - MasterProfile: admin.MasterProfile{ - EncryptionAtHost: admin.EncryptionAtHostDisabled, - }, - OperatorFlags: admin.OperatorFlags{"testFlag": "true"}, - MaintenanceState: admin.MaintenanceStateUnplanned, - }, - }, - }, + } + }}, { name: "patch with flags merges the flags together", request: func(oc *admin.OpenShiftCluster) { @@ -234,33 +235,34 @@ func TestPutOrPatchOpenShiftClusterAdminAPI(t *testing.T) { }, wantAsync: true, wantStatusCode: http.StatusOK, - wantResponse: &admin.OpenShiftCluster{ - ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), - Type: "Microsoft.RedHatOpenShift/openShiftClusters", - Tags: map[string]string{"tag": "will-be-kept"}, - Properties: admin.OpenShiftClusterProperties{ - ProvisioningState: admin.ProvisioningStateAdminUpdating, - LastProvisioningState: admin.ProvisioningStateSucceeded, - MaintenanceTask: admin.MaintenanceTaskOperator, - NetworkProfile: admin.NetworkProfile{ - OutboundType: admin.OutboundTypeLoadbalancer, - LoadBalancerProfile: &admin.LoadBalancerProfile{ - ManagedOutboundIPs: &admin.ManagedOutboundIPs{ - Count: 1, + wantResponse: func() *admin.OpenShiftCluster { + return &admin.OpenShiftCluster{ + ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), + Type: "Microsoft.RedHatOpenShift/openShiftClusters", + Tags: map[string]string{"tag": "will-be-kept"}, + Properties: admin.OpenShiftClusterProperties{ + ProvisioningState: admin.ProvisioningStateAdminUpdating, + LastProvisioningState: admin.ProvisioningStateSucceeded, + MaintenanceTask: admin.MaintenanceTaskOperator, + NetworkProfile: admin.NetworkProfile{ + OutboundType: admin.OutboundTypeLoadbalancer, + LoadBalancerProfile: &admin.LoadBalancerProfile{ + ManagedOutboundIPs: &admin.ManagedOutboundIPs{ + Count: 1, + }, }, }, + ClusterProfile: admin.ClusterProfile{ + FipsValidatedModules: admin.FipsValidatedModulesDisabled, + }, + MasterProfile: admin.MasterProfile{ + EncryptionAtHost: admin.EncryptionAtHostDisabled, + }, + OperatorFlags: admin.OperatorFlags{"exploding-flag": "true", "overwrittenFlag": "true", "testFlag": "true"}, + MaintenanceState: admin.MaintenanceStateUnplanned, }, - ClusterProfile: admin.ClusterProfile{ - FipsValidatedModules: admin.FipsValidatedModulesDisabled, - }, - MasterProfile: admin.MasterProfile{ - EncryptionAtHost: admin.EncryptionAtHostDisabled, - }, - OperatorFlags: admin.OperatorFlags{"exploding-flag": "true", "overwrittenFlag": "true", "testFlag": "true"}, - MaintenanceState: admin.MaintenanceStateUnplanned, - }, - }, - }, + } + }}, { name: "patch an existing cluster with no flags in db will use defaults", request: func(oc *admin.OpenShiftCluster) { @@ -331,33 +333,34 @@ func TestPutOrPatchOpenShiftClusterAdminAPI(t *testing.T) { }, wantAsync: true, wantStatusCode: http.StatusOK, - wantResponse: &admin.OpenShiftCluster{ - ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), - Type: "Microsoft.RedHatOpenShift/openShiftClusters", - Tags: map[string]string{"tag": "will-be-kept"}, - Properties: admin.OpenShiftClusterProperties{ - ProvisioningState: admin.ProvisioningStateAdminUpdating, - LastProvisioningState: admin.ProvisioningStateSucceeded, - MaintenanceTask: admin.MaintenanceTaskEverything, - NetworkProfile: admin.NetworkProfile{ - OutboundType: admin.OutboundTypeLoadbalancer, - LoadBalancerProfile: &admin.LoadBalancerProfile{ - ManagedOutboundIPs: &admin.ManagedOutboundIPs{ - Count: 1, + wantResponse: func() *admin.OpenShiftCluster { + return &admin.OpenShiftCluster{ + ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), + Type: "Microsoft.RedHatOpenShift/openShiftClusters", + Tags: map[string]string{"tag": "will-be-kept"}, + Properties: admin.OpenShiftClusterProperties{ + ProvisioningState: admin.ProvisioningStateAdminUpdating, + LastProvisioningState: admin.ProvisioningStateSucceeded, + MaintenanceTask: admin.MaintenanceTaskEverything, + NetworkProfile: admin.NetworkProfile{ + OutboundType: admin.OutboundTypeLoadbalancer, + LoadBalancerProfile: &admin.LoadBalancerProfile{ + ManagedOutboundIPs: &admin.ManagedOutboundIPs{ + Count: 1, + }, }, }, + MasterProfile: admin.MasterProfile{ + EncryptionAtHost: admin.EncryptionAtHostDisabled, + }, + ClusterProfile: admin.ClusterProfile{ + FipsValidatedModules: admin.FipsValidatedModulesDisabled, + }, + OperatorFlags: admin.OperatorFlags(operator.DefaultOperatorFlags()), + MaintenanceState: admin.MaintenanceStateUnplanned, }, - MasterProfile: admin.MasterProfile{ - EncryptionAtHost: admin.EncryptionAtHostDisabled, - }, - ClusterProfile: admin.ClusterProfile{ - FipsValidatedModules: admin.FipsValidatedModulesDisabled, - }, - OperatorFlags: admin.OperatorFlags(operator.DefaultOperatorFlags()), - MaintenanceState: admin.MaintenanceStateUnplanned, - }, - }, - }, + } + }}, { name: "patch with operator update request", request: func(oc *admin.OpenShiftCluster) { @@ -426,31 +429,145 @@ func TestPutOrPatchOpenShiftClusterAdminAPI(t *testing.T) { }, wantAsync: true, wantStatusCode: http.StatusOK, - wantResponse: &admin.OpenShiftCluster{ - ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), - Type: "Microsoft.RedHatOpenShift/openShiftClusters", - Tags: map[string]string{"tag": "will-be-kept"}, - Properties: admin.OpenShiftClusterProperties{ - ProvisioningState: admin.ProvisioningStateAdminUpdating, - LastProvisioningState: admin.ProvisioningStateSucceeded, - ClusterProfile: admin.ClusterProfile{ - FipsValidatedModules: admin.FipsValidatedModulesDisabled, - }, - MaintenanceTask: admin.MaintenanceTaskOperator, - NetworkProfile: admin.NetworkProfile{ - OutboundType: admin.OutboundTypeLoadbalancer, - LoadBalancerProfile: &admin.LoadBalancerProfile{ - ManagedOutboundIPs: &admin.ManagedOutboundIPs{ - Count: 1, + wantResponse: func() *admin.OpenShiftCluster { + return &admin.OpenShiftCluster{ + ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), + Type: "Microsoft.RedHatOpenShift/openShiftClusters", + Tags: map[string]string{"tag": "will-be-kept"}, + Properties: admin.OpenShiftClusterProperties{ + ProvisioningState: admin.ProvisioningStateAdminUpdating, + LastProvisioningState: admin.ProvisioningStateSucceeded, + ClusterProfile: admin.ClusterProfile{ + FipsValidatedModules: admin.FipsValidatedModulesDisabled, + }, + MaintenanceTask: admin.MaintenanceTaskOperator, + NetworkProfile: admin.NetworkProfile{ + OutboundType: admin.OutboundTypeLoadbalancer, + LoadBalancerProfile: &admin.LoadBalancerProfile{ + ManagedOutboundIPs: &admin.ManagedOutboundIPs{ + Count: 1, + }, }, }, + MasterProfile: admin.MasterProfile{ + EncryptionAtHost: admin.EncryptionAtHostDisabled, + }, + OperatorFlags: admin.OperatorFlags(operator.DefaultOperatorFlags()), + MaintenanceState: admin.MaintenanceStateUnplanned, }, - MasterProfile: admin.MasterProfile{ - EncryptionAtHost: admin.EncryptionAtHostDisabled, + } + }, + }, + { + name: "patch with OperatorFlagsMergeStrategy=reset will reset flags to defaults and merge in request flags", + request: func(oc *admin.OpenShiftCluster) { + oc.OperatorFlagsMergeStrategy = admin.OperatorFlagsMergeStrategyReset + oc.Properties.MaintenanceTask = admin.MaintenanceTaskOperator + oc.Properties.OperatorFlags = admin.OperatorFlags{"exploding-flag": "true", "overwrittenFlag": "true"} + }, + isPatch: true, + fixture: func(f *testdatabase.Fixture) { + f.AddSubscriptionDocuments(&api.SubscriptionDocument{ + ID: mockSubID, + Subscription: &api.Subscription{ + State: api.SubscriptionStateRegistered, }, - OperatorFlags: admin.OperatorFlags(operator.DefaultOperatorFlags()), - MaintenanceState: admin.MaintenanceStateUnplanned, - }, + }) + f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{ + Key: strings.ToLower(testdatabase.GetResourcePath(mockSubID, "resourceName")), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), + Type: "Microsoft.RedHatOpenShift/openShiftClusters", + + Tags: map[string]string{"tag": "will-be-kept"}, + Properties: api.OpenShiftClusterProperties{ + ClusterProfile: api.ClusterProfile{ + FipsValidatedModules: api.FipsValidatedModulesDisabled, + }, + ProvisioningState: api.ProvisioningStateSucceeded, + OperatorFlags: api.OperatorFlags{"testFlag": "true", "overwrittenFlag": "false"}, + }, + }, + }) + }, + wantSystemDataEnriched: true, + wantEnriched: []string{testdatabase.GetResourcePath(mockSubID, "resourceName")}, + wantDocuments: func(c *testdatabase.Checker) { + expectedFlags := operator.DefaultOperatorFlags() + expectedFlags["exploding-flag"] = "true" + expectedFlags["overwrittenFlag"] = "true" + + c.AddAsyncOperationDocuments(&api.AsyncOperationDocument{ + OpenShiftClusterKey: strings.ToLower(testdatabase.GetResourcePath(mockSubID, "resourceName")), + AsyncOperation: &api.AsyncOperation{ + InitialProvisioningState: api.ProvisioningStateAdminUpdating, + ProvisioningState: api.ProvisioningStateAdminUpdating, + }, + }) + c.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{ + Key: strings.ToLower(testdatabase.GetResourcePath(mockSubID, "resourceName")), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), + Type: "Microsoft.RedHatOpenShift/openShiftClusters", + Tags: map[string]string{"tag": "will-be-kept"}, + Properties: api.OpenShiftClusterProperties{ + ProvisioningState: api.ProvisioningStateAdminUpdating, + LastProvisioningState: api.ProvisioningStateSucceeded, + MaintenanceTask: api.MaintenanceTaskOperator, + ClusterProfile: api.ClusterProfile{ + FipsValidatedModules: api.FipsValidatedModulesDisabled, + }, + NetworkProfile: api.NetworkProfile{ + OutboundType: api.OutboundTypeLoadbalancer, + PreconfiguredNSG: api.PreconfiguredNSGDisabled, + LoadBalancerProfile: &api.LoadBalancerProfile{ + ManagedOutboundIPs: &api.ManagedOutboundIPs{ + Count: 1, + }, + }, + }, + MasterProfile: api.MasterProfile{ + EncryptionAtHost: api.EncryptionAtHostDisabled, + }, + OperatorFlags: api.OperatorFlags(expectedFlags), + MaintenanceState: api.MaintenanceStateUnplanned, + }, + }, + }) + }, + wantAsync: true, + wantStatusCode: http.StatusOK, + wantResponse: func() *admin.OpenShiftCluster { + expectedFlags := operator.DefaultOperatorFlags() + expectedFlags["exploding-flag"] = "true" + expectedFlags["overwrittenFlag"] = "true" + + return &admin.OpenShiftCluster{ + ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), + Type: "Microsoft.RedHatOpenShift/openShiftClusters", + Tags: map[string]string{"tag": "will-be-kept"}, + Properties: admin.OpenShiftClusterProperties{ + ProvisioningState: admin.ProvisioningStateAdminUpdating, + LastProvisioningState: admin.ProvisioningStateSucceeded, + MaintenanceTask: admin.MaintenanceTaskOperator, + NetworkProfile: admin.NetworkProfile{ + OutboundType: admin.OutboundTypeLoadbalancer, + LoadBalancerProfile: &admin.LoadBalancerProfile{ + ManagedOutboundIPs: &admin.ManagedOutboundIPs{ + Count: 1, + }, + }, + }, + ClusterProfile: admin.ClusterProfile{ + FipsValidatedModules: admin.FipsValidatedModulesDisabled, + }, + MasterProfile: admin.MasterProfile{ + EncryptionAtHost: admin.EncryptionAtHostDisabled, + }, + OperatorFlags: admin.OperatorFlags(expectedFlags), + MaintenanceState: admin.MaintenanceStateUnplanned, + }, + } }, }, { @@ -524,31 +641,33 @@ func TestPutOrPatchOpenShiftClusterAdminAPI(t *testing.T) { }, wantAsync: true, wantStatusCode: http.StatusOK, - wantResponse: &admin.OpenShiftCluster{ - ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), - Type: "Microsoft.RedHatOpenShift/openShiftClusters", - Tags: map[string]string{"tag": "will-be-kept"}, - Properties: admin.OpenShiftClusterProperties{ - ProvisioningState: admin.ProvisioningStateAdminUpdating, - LastProvisioningState: admin.ProvisioningStateSucceeded, - ClusterProfile: admin.ClusterProfile{ - FipsValidatedModules: admin.FipsValidatedModulesDisabled, - }, - MaintenanceTask: admin.MaintenanceTaskOperator, - NetworkProfile: admin.NetworkProfile{ - OutboundType: admin.OutboundTypeLoadbalancer, - LoadBalancerProfile: &admin.LoadBalancerProfile{ - ManagedOutboundIPs: &admin.ManagedOutboundIPs{ - Count: 1, + wantResponse: func() *admin.OpenShiftCluster { + return &admin.OpenShiftCluster{ + ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), + Type: "Microsoft.RedHatOpenShift/openShiftClusters", + Tags: map[string]string{"tag": "will-be-kept"}, + Properties: admin.OpenShiftClusterProperties{ + ProvisioningState: admin.ProvisioningStateAdminUpdating, + LastProvisioningState: admin.ProvisioningStateSucceeded, + ClusterProfile: admin.ClusterProfile{ + FipsValidatedModules: admin.FipsValidatedModulesDisabled, + }, + MaintenanceTask: admin.MaintenanceTaskOperator, + NetworkProfile: admin.NetworkProfile{ + OutboundType: admin.OutboundTypeLoadbalancer, + LoadBalancerProfile: &admin.LoadBalancerProfile{ + ManagedOutboundIPs: &admin.ManagedOutboundIPs{ + Count: 1, + }, }, }, + MasterProfile: admin.MasterProfile{ + EncryptionAtHost: admin.EncryptionAtHostDisabled, + }, + OperatorFlags: admin.OperatorFlags{"testFlag": "true"}, + MaintenanceState: admin.MaintenanceStateUnplanned, }, - MasterProfile: admin.MasterProfile{ - EncryptionAtHost: admin.EncryptionAtHostDisabled, - }, - OperatorFlags: admin.OperatorFlags{"testFlag": "true"}, - MaintenanceState: admin.MaintenanceStateUnplanned, - }, + } }, }, { @@ -631,6 +750,7 @@ func TestPutOrPatchOpenShiftClusterAdminAPI(t *testing.T) { wantAsync: false, wantStatusCode: http.StatusBadRequest, wantError: `400: PropertyChangeNotAllowed: properties.registryProfiles: Changing property 'properties.registryProfiles' is not allowed.`, + wantResponse: func() *admin.OpenShiftCluster { return nil }, }, { name: "patch an empty maintenance state cluster with maintenance pending request", @@ -702,31 +822,33 @@ func TestPutOrPatchOpenShiftClusterAdminAPI(t *testing.T) { }, wantAsync: true, wantStatusCode: http.StatusOK, - wantResponse: &admin.OpenShiftCluster{ - ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), - Type: "Microsoft.RedHatOpenShift/openShiftClusters", - Tags: map[string]string{"tag": "will-be-kept"}, - Properties: admin.OpenShiftClusterProperties{ - ProvisioningState: admin.ProvisioningStateSucceeded, - LastProvisioningState: admin.ProvisioningStateSucceeded, - ClusterProfile: admin.ClusterProfile{ - FipsValidatedModules: admin.FipsValidatedModulesDisabled, - }, - MaintenanceTask: "", - NetworkProfile: admin.NetworkProfile{ - OutboundType: admin.OutboundTypeLoadbalancer, - LoadBalancerProfile: &admin.LoadBalancerProfile{ - ManagedOutboundIPs: &admin.ManagedOutboundIPs{ - Count: 1, + wantResponse: func() *admin.OpenShiftCluster { + return &admin.OpenShiftCluster{ + ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), + Type: "Microsoft.RedHatOpenShift/openShiftClusters", + Tags: map[string]string{"tag": "will-be-kept"}, + Properties: admin.OpenShiftClusterProperties{ + ProvisioningState: admin.ProvisioningStateSucceeded, + LastProvisioningState: admin.ProvisioningStateSucceeded, + ClusterProfile: admin.ClusterProfile{ + FipsValidatedModules: admin.FipsValidatedModulesDisabled, + }, + MaintenanceTask: "", + NetworkProfile: admin.NetworkProfile{ + OutboundType: admin.OutboundTypeLoadbalancer, + LoadBalancerProfile: &admin.LoadBalancerProfile{ + ManagedOutboundIPs: &admin.ManagedOutboundIPs{ + Count: 1, + }, }, }, + MaintenanceState: admin.MaintenanceStatePending, + MasterProfile: admin.MasterProfile{ + EncryptionAtHost: admin.EncryptionAtHostDisabled, + }, + OperatorFlags: admin.OperatorFlags(operator.DefaultOperatorFlags()), }, - MaintenanceState: admin.MaintenanceStatePending, - MasterProfile: admin.MasterProfile{ - EncryptionAtHost: admin.EncryptionAtHostDisabled, - }, - OperatorFlags: admin.OperatorFlags(operator.DefaultOperatorFlags()), - }, + } }, }, { @@ -800,31 +922,33 @@ func TestPutOrPatchOpenShiftClusterAdminAPI(t *testing.T) { }, wantAsync: true, wantStatusCode: http.StatusOK, - wantResponse: &admin.OpenShiftCluster{ - ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), - Type: "Microsoft.RedHatOpenShift/openShiftClusters", - Tags: map[string]string{"tag": "will-be-kept"}, - Properties: admin.OpenShiftClusterProperties{ - ProvisioningState: admin.ProvisioningStateSucceeded, - LastProvisioningState: admin.ProvisioningStateSucceeded, - ClusterProfile: admin.ClusterProfile{ - FipsValidatedModules: admin.FipsValidatedModulesDisabled, - }, - MaintenanceTask: "", - NetworkProfile: admin.NetworkProfile{ - OutboundType: admin.OutboundTypeLoadbalancer, - LoadBalancerProfile: &admin.LoadBalancerProfile{ - ManagedOutboundIPs: &admin.ManagedOutboundIPs{ - Count: 1, + wantResponse: func() *admin.OpenShiftCluster { + return &admin.OpenShiftCluster{ + ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), + Type: "Microsoft.RedHatOpenShift/openShiftClusters", + Tags: map[string]string{"tag": "will-be-kept"}, + Properties: admin.OpenShiftClusterProperties{ + ProvisioningState: admin.ProvisioningStateSucceeded, + LastProvisioningState: admin.ProvisioningStateSucceeded, + ClusterProfile: admin.ClusterProfile{ + FipsValidatedModules: admin.FipsValidatedModulesDisabled, + }, + MaintenanceTask: "", + NetworkProfile: admin.NetworkProfile{ + OutboundType: admin.OutboundTypeLoadbalancer, + LoadBalancerProfile: &admin.LoadBalancerProfile{ + ManagedOutboundIPs: &admin.ManagedOutboundIPs{ + Count: 1, + }, }, }, + MaintenanceState: admin.MaintenanceStatePending, + MasterProfile: admin.MasterProfile{ + EncryptionAtHost: admin.EncryptionAtHostDisabled, + }, + OperatorFlags: admin.OperatorFlags(operator.DefaultOperatorFlags()), }, - MaintenanceState: admin.MaintenanceStatePending, - MasterProfile: admin.MasterProfile{ - EncryptionAtHost: admin.EncryptionAtHostDisabled, - }, - OperatorFlags: admin.OperatorFlags(operator.DefaultOperatorFlags()), - }, + } }, }, { @@ -897,31 +1021,33 @@ func TestPutOrPatchOpenShiftClusterAdminAPI(t *testing.T) { }, wantAsync: true, wantStatusCode: http.StatusOK, - wantResponse: &admin.OpenShiftCluster{ - ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), - Type: "Microsoft.RedHatOpenShift/openShiftClusters", - Tags: map[string]string{"tag": "will-be-kept"}, - Properties: admin.OpenShiftClusterProperties{ - ProvisioningState: admin.ProvisioningStateAdminUpdating, - LastProvisioningState: admin.ProvisioningStateSucceeded, - ClusterProfile: admin.ClusterProfile{ - FipsValidatedModules: admin.FipsValidatedModulesDisabled, - }, - MaintenanceTask: admin.MaintenanceTaskEverything, - NetworkProfile: admin.NetworkProfile{ - OutboundType: admin.OutboundTypeLoadbalancer, - LoadBalancerProfile: &admin.LoadBalancerProfile{ - ManagedOutboundIPs: &admin.ManagedOutboundIPs{ - Count: 1, + wantResponse: func() *admin.OpenShiftCluster { + return &admin.OpenShiftCluster{ + ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), + Type: "Microsoft.RedHatOpenShift/openShiftClusters", + Tags: map[string]string{"tag": "will-be-kept"}, + Properties: admin.OpenShiftClusterProperties{ + ProvisioningState: admin.ProvisioningStateAdminUpdating, + LastProvisioningState: admin.ProvisioningStateSucceeded, + ClusterProfile: admin.ClusterProfile{ + FipsValidatedModules: admin.FipsValidatedModulesDisabled, + }, + MaintenanceTask: admin.MaintenanceTaskEverything, + NetworkProfile: admin.NetworkProfile{ + OutboundType: admin.OutboundTypeLoadbalancer, + LoadBalancerProfile: &admin.LoadBalancerProfile{ + ManagedOutboundIPs: &admin.ManagedOutboundIPs{ + Count: 1, + }, }, }, + MasterProfile: admin.MasterProfile{ + EncryptionAtHost: admin.EncryptionAtHostDisabled, + }, + OperatorFlags: admin.OperatorFlags(operator.DefaultOperatorFlags()), + MaintenanceState: admin.MaintenanceStatePlanned, }, - MasterProfile: admin.MasterProfile{ - EncryptionAtHost: admin.EncryptionAtHostDisabled, - }, - OperatorFlags: admin.OperatorFlags(operator.DefaultOperatorFlags()), - MaintenanceState: admin.MaintenanceStatePlanned, - }, + } }, }, { @@ -995,31 +1121,33 @@ func TestPutOrPatchOpenShiftClusterAdminAPI(t *testing.T) { }, wantAsync: true, wantStatusCode: http.StatusOK, - wantResponse: &admin.OpenShiftCluster{ - ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), - Type: "Microsoft.RedHatOpenShift/openShiftClusters", - Tags: map[string]string{"tag": "will-be-kept"}, - Properties: admin.OpenShiftClusterProperties{ - ProvisioningState: admin.ProvisioningStateSucceeded, - LastProvisioningState: admin.ProvisioningStateSucceeded, - ClusterProfile: admin.ClusterProfile{ - FipsValidatedModules: admin.FipsValidatedModulesDisabled, - }, - MaintenanceTask: "", - NetworkProfile: admin.NetworkProfile{ - OutboundType: admin.OutboundTypeLoadbalancer, - LoadBalancerProfile: &admin.LoadBalancerProfile{ - ManagedOutboundIPs: &admin.ManagedOutboundIPs{ - Count: 1, + wantResponse: func() *admin.OpenShiftCluster { + return &admin.OpenShiftCluster{ + ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), + Type: "Microsoft.RedHatOpenShift/openShiftClusters", + Tags: map[string]string{"tag": "will-be-kept"}, + Properties: admin.OpenShiftClusterProperties{ + ProvisioningState: admin.ProvisioningStateSucceeded, + LastProvisioningState: admin.ProvisioningStateSucceeded, + ClusterProfile: admin.ClusterProfile{ + FipsValidatedModules: admin.FipsValidatedModulesDisabled, + }, + MaintenanceTask: "", + NetworkProfile: admin.NetworkProfile{ + OutboundType: admin.OutboundTypeLoadbalancer, + LoadBalancerProfile: &admin.LoadBalancerProfile{ + ManagedOutboundIPs: &admin.ManagedOutboundIPs{ + Count: 1, + }, }, }, + MaintenanceState: admin.MaintenanceStateNone, + MasterProfile: admin.MasterProfile{ + EncryptionAtHost: admin.EncryptionAtHostDisabled, + }, + OperatorFlags: admin.OperatorFlags(operator.DefaultOperatorFlags()), }, - MaintenanceState: admin.MaintenanceStateNone, - MasterProfile: admin.MasterProfile{ - EncryptionAtHost: admin.EncryptionAtHostDisabled, - }, - OperatorFlags: admin.OperatorFlags(operator.DefaultOperatorFlags()), - }, + } }, }, { @@ -1093,31 +1221,33 @@ func TestPutOrPatchOpenShiftClusterAdminAPI(t *testing.T) { }, wantAsync: true, wantStatusCode: http.StatusOK, - wantResponse: &admin.OpenShiftCluster{ - ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), - Type: "Microsoft.RedHatOpenShift/openShiftClusters", - Tags: map[string]string{"tag": "will-be-kept"}, - Properties: admin.OpenShiftClusterProperties{ - ProvisioningState: admin.ProvisioningStateSucceeded, - LastProvisioningState: admin.ProvisioningStateSucceeded, - ClusterProfile: admin.ClusterProfile{ - FipsValidatedModules: admin.FipsValidatedModulesDisabled, - }, - MaintenanceTask: "", - NetworkProfile: admin.NetworkProfile{ - OutboundType: admin.OutboundTypeLoadbalancer, - LoadBalancerProfile: &admin.LoadBalancerProfile{ - ManagedOutboundIPs: &admin.ManagedOutboundIPs{ - Count: 1, + wantResponse: func() *admin.OpenShiftCluster { + return &admin.OpenShiftCluster{ + ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), + Type: "Microsoft.RedHatOpenShift/openShiftClusters", + Tags: map[string]string{"tag": "will-be-kept"}, + Properties: admin.OpenShiftClusterProperties{ + ProvisioningState: admin.ProvisioningStateSucceeded, + LastProvisioningState: admin.ProvisioningStateSucceeded, + ClusterProfile: admin.ClusterProfile{ + FipsValidatedModules: admin.FipsValidatedModulesDisabled, + }, + MaintenanceTask: "", + NetworkProfile: admin.NetworkProfile{ + OutboundType: admin.OutboundTypeLoadbalancer, + LoadBalancerProfile: &admin.LoadBalancerProfile{ + ManagedOutboundIPs: &admin.ManagedOutboundIPs{ + Count: 1, + }, }, }, + MaintenanceState: admin.MaintenanceStateNone, + MasterProfile: admin.MasterProfile{ + EncryptionAtHost: admin.EncryptionAtHostDisabled, + }, + OperatorFlags: admin.OperatorFlags(operator.DefaultOperatorFlags()), }, - MaintenanceState: admin.MaintenanceStateNone, - MasterProfile: admin.MasterProfile{ - EncryptionAtHost: admin.EncryptionAtHostDisabled, - }, - OperatorFlags: admin.OperatorFlags(operator.DefaultOperatorFlags()), - }, + } }, }, { @@ -1189,31 +1319,33 @@ func TestPutOrPatchOpenShiftClusterAdminAPI(t *testing.T) { }, wantAsync: true, wantStatusCode: http.StatusOK, - wantResponse: &admin.OpenShiftCluster{ - ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), - Type: "Microsoft.RedHatOpenShift/openShiftClusters", - Tags: map[string]string{"tag": "will-be-kept"}, - Properties: admin.OpenShiftClusterProperties{ - ProvisioningState: admin.ProvisioningStateAdminUpdating, - LastProvisioningState: admin.ProvisioningStateSucceeded, - ClusterProfile: admin.ClusterProfile{ - FipsValidatedModules: admin.FipsValidatedModulesDisabled, - }, - MaintenanceTask: admin.MaintenanceTaskEverything, - NetworkProfile: admin.NetworkProfile{ - OutboundType: admin.OutboundTypeLoadbalancer, - LoadBalancerProfile: &admin.LoadBalancerProfile{ - ManagedOutboundIPs: &admin.ManagedOutboundIPs{ - Count: 1, + wantResponse: func() *admin.OpenShiftCluster { + return &admin.OpenShiftCluster{ + ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), + Type: "Microsoft.RedHatOpenShift/openShiftClusters", + Tags: map[string]string{"tag": "will-be-kept"}, + Properties: admin.OpenShiftClusterProperties{ + ProvisioningState: admin.ProvisioningStateAdminUpdating, + LastProvisioningState: admin.ProvisioningStateSucceeded, + ClusterProfile: admin.ClusterProfile{ + FipsValidatedModules: admin.FipsValidatedModulesDisabled, + }, + MaintenanceTask: admin.MaintenanceTaskEverything, + NetworkProfile: admin.NetworkProfile{ + OutboundType: admin.OutboundTypeLoadbalancer, + LoadBalancerProfile: &admin.LoadBalancerProfile{ + ManagedOutboundIPs: &admin.ManagedOutboundIPs{ + Count: 1, + }, }, }, + MasterProfile: admin.MasterProfile{ + EncryptionAtHost: admin.EncryptionAtHostDisabled, + }, + OperatorFlags: admin.OperatorFlags{"testFlag": "true"}, + MaintenanceState: admin.MaintenanceStateUnplanned, }, - MasterProfile: admin.MasterProfile{ - EncryptionAtHost: admin.EncryptionAtHostDisabled, - }, - OperatorFlags: admin.OperatorFlags{"testFlag": "true"}, - MaintenanceState: admin.MaintenanceStateUnplanned, - }, + } }, }, { @@ -1289,31 +1421,33 @@ func TestPutOrPatchOpenShiftClusterAdminAPI(t *testing.T) { }, wantAsync: true, wantStatusCode: http.StatusOK, - wantResponse: &admin.OpenShiftCluster{ - ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), - Type: "Microsoft.RedHatOpenShift/openShiftClusters", - Tags: map[string]string{"tag": "will-be-kept"}, - Properties: admin.OpenShiftClusterProperties{ - ProvisioningState: admin.ProvisioningStateSucceeded, - LastProvisioningState: admin.ProvisioningStateSucceeded, - ClusterProfile: admin.ClusterProfile{ - FipsValidatedModules: admin.FipsValidatedModulesDisabled, - }, - NetworkProfile: admin.NetworkProfile{ - OutboundType: admin.OutboundTypeLoadbalancer, - LoadBalancerProfile: &admin.LoadBalancerProfile{ - ManagedOutboundIPs: &admin.ManagedOutboundIPs{ - Count: 1, + wantResponse: func() *admin.OpenShiftCluster { + return &admin.OpenShiftCluster{ + ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), + Type: "Microsoft.RedHatOpenShift/openShiftClusters", + Tags: map[string]string{"tag": "will-be-kept"}, + Properties: admin.OpenShiftClusterProperties{ + ProvisioningState: admin.ProvisioningStateSucceeded, + LastProvisioningState: admin.ProvisioningStateSucceeded, + ClusterProfile: admin.ClusterProfile{ + FipsValidatedModules: admin.FipsValidatedModulesDisabled, + }, + NetworkProfile: admin.NetworkProfile{ + OutboundType: admin.OutboundTypeLoadbalancer, + LoadBalancerProfile: &admin.LoadBalancerProfile{ + ManagedOutboundIPs: &admin.ManagedOutboundIPs{ + Count: 1, + }, }, }, + MasterProfile: admin.MasterProfile{ + EncryptionAtHost: admin.EncryptionAtHostDisabled, + }, + OperatorFlags: admin.OperatorFlags{"testFlag": "true"}, + MaintenanceState: admin.MaintenanceStateCustomerActionNeeded, + LastAdminUpdateError: "error", }, - MasterProfile: admin.MasterProfile{ - EncryptionAtHost: admin.EncryptionAtHostDisabled, - }, - OperatorFlags: admin.OperatorFlags{"testFlag": "true"}, - MaintenanceState: admin.MaintenanceStateCustomerActionNeeded, - LastAdminUpdateError: "error", - }, + } }, }, { @@ -1389,31 +1523,33 @@ func TestPutOrPatchOpenShiftClusterAdminAPI(t *testing.T) { }, wantAsync: true, wantStatusCode: http.StatusOK, - wantResponse: &admin.OpenShiftCluster{ - ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), - Type: "Microsoft.RedHatOpenShift/openShiftClusters", - Tags: map[string]string{"tag": "will-be-kept"}, - Properties: admin.OpenShiftClusterProperties{ - ProvisioningState: admin.ProvisioningStateSucceeded, - FailedProvisioningState: admin.ProvisioningStateUpdating, - ClusterProfile: admin.ClusterProfile{ - FipsValidatedModules: admin.FipsValidatedModulesDisabled, - }, - NetworkProfile: admin.NetworkProfile{ - OutboundType: admin.OutboundTypeLoadbalancer, - LoadBalancerProfile: &admin.LoadBalancerProfile{ - ManagedOutboundIPs: &admin.ManagedOutboundIPs{ - Count: 1, + wantResponse: func() *admin.OpenShiftCluster { + return &admin.OpenShiftCluster{ + ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), + Type: "Microsoft.RedHatOpenShift/openShiftClusters", + Tags: map[string]string{"tag": "will-be-kept"}, + Properties: admin.OpenShiftClusterProperties{ + ProvisioningState: admin.ProvisioningStateSucceeded, + FailedProvisioningState: admin.ProvisioningStateUpdating, + ClusterProfile: admin.ClusterProfile{ + FipsValidatedModules: admin.FipsValidatedModulesDisabled, + }, + NetworkProfile: admin.NetworkProfile{ + OutboundType: admin.OutboundTypeLoadbalancer, + LoadBalancerProfile: &admin.LoadBalancerProfile{ + ManagedOutboundIPs: &admin.ManagedOutboundIPs{ + Count: 1, + }, }, }, + MasterProfile: admin.MasterProfile{ + EncryptionAtHost: admin.EncryptionAtHostDisabled, + }, + OperatorFlags: admin.OperatorFlags{"testFlag": "true"}, + MaintenanceState: admin.MaintenanceStateCustomerActionNeeded, + LastAdminUpdateError: "error", }, - MasterProfile: admin.MasterProfile{ - EncryptionAtHost: admin.EncryptionAtHostDisabled, - }, - OperatorFlags: admin.OperatorFlags{"testFlag": "true"}, - MaintenanceState: admin.MaintenanceStateCustomerActionNeeded, - LastAdminUpdateError: "error", - }, + } }, }, { @@ -1489,31 +1625,33 @@ func TestPutOrPatchOpenShiftClusterAdminAPI(t *testing.T) { }, wantAsync: true, wantStatusCode: http.StatusOK, - wantResponse: &admin.OpenShiftCluster{ - ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), - Type: "Microsoft.RedHatOpenShift/openShiftClusters", - Tags: map[string]string{"tag": "will-be-kept"}, - Properties: admin.OpenShiftClusterProperties{ - ProvisioningState: admin.ProvisioningStateSucceeded, - FailedProvisioningState: admin.ProvisioningStateUpdating, - ClusterProfile: admin.ClusterProfile{ - FipsValidatedModules: admin.FipsValidatedModulesDisabled, - }, - NetworkProfile: admin.NetworkProfile{ - OutboundType: admin.OutboundTypeLoadbalancer, - LoadBalancerProfile: &admin.LoadBalancerProfile{ - ManagedOutboundIPs: &admin.ManagedOutboundIPs{ - Count: 1, + wantResponse: func() *admin.OpenShiftCluster { + return &admin.OpenShiftCluster{ + ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), + Type: "Microsoft.RedHatOpenShift/openShiftClusters", + Tags: map[string]string{"tag": "will-be-kept"}, + Properties: admin.OpenShiftClusterProperties{ + ProvisioningState: admin.ProvisioningStateSucceeded, + FailedProvisioningState: admin.ProvisioningStateUpdating, + ClusterProfile: admin.ClusterProfile{ + FipsValidatedModules: admin.FipsValidatedModulesDisabled, + }, + NetworkProfile: admin.NetworkProfile{ + OutboundType: admin.OutboundTypeLoadbalancer, + LoadBalancerProfile: &admin.LoadBalancerProfile{ + ManagedOutboundIPs: &admin.ManagedOutboundIPs{ + Count: 1, + }, }, }, + MasterProfile: admin.MasterProfile{ + EncryptionAtHost: admin.EncryptionAtHostDisabled, + }, + OperatorFlags: admin.OperatorFlags{"testFlag": "true"}, + MaintenanceState: admin.MaintenanceStateCustomerActionNeeded, + LastAdminUpdateError: "error", }, - MasterProfile: admin.MasterProfile{ - EncryptionAtHost: admin.EncryptionAtHostDisabled, - }, - OperatorFlags: admin.OperatorFlags{"testFlag": "true"}, - MaintenanceState: admin.MaintenanceStateCustomerActionNeeded, - LastAdminUpdateError: "error", - }, + } }, }, { @@ -1589,31 +1727,33 @@ func TestPutOrPatchOpenShiftClusterAdminAPI(t *testing.T) { }, wantAsync: true, wantStatusCode: http.StatusOK, - wantResponse: &admin.OpenShiftCluster{ - ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), - Type: "Microsoft.RedHatOpenShift/openShiftClusters", - Tags: map[string]string{"tag": "will-be-kept"}, - Properties: admin.OpenShiftClusterProperties{ - ProvisioningState: admin.ProvisioningStateSucceeded, - FailedProvisioningState: admin.ProvisioningStateUpdating, - ClusterProfile: admin.ClusterProfile{ - FipsValidatedModules: admin.FipsValidatedModulesDisabled, - }, - NetworkProfile: admin.NetworkProfile{ - OutboundType: admin.OutboundTypeLoadbalancer, - LoadBalancerProfile: &admin.LoadBalancerProfile{ - ManagedOutboundIPs: &admin.ManagedOutboundIPs{ - Count: 1, + wantResponse: func() *admin.OpenShiftCluster { + return &admin.OpenShiftCluster{ + ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), + Type: "Microsoft.RedHatOpenShift/openShiftClusters", + Tags: map[string]string{"tag": "will-be-kept"}, + Properties: admin.OpenShiftClusterProperties{ + ProvisioningState: admin.ProvisioningStateSucceeded, + FailedProvisioningState: admin.ProvisioningStateUpdating, + ClusterProfile: admin.ClusterProfile{ + FipsValidatedModules: admin.FipsValidatedModulesDisabled, + }, + NetworkProfile: admin.NetworkProfile{ + OutboundType: admin.OutboundTypeLoadbalancer, + LoadBalancerProfile: &admin.LoadBalancerProfile{ + ManagedOutboundIPs: &admin.ManagedOutboundIPs{ + Count: 1, + }, }, }, + MasterProfile: admin.MasterProfile{ + EncryptionAtHost: admin.EncryptionAtHostDisabled, + }, + OperatorFlags: admin.OperatorFlags{"testFlag": "true"}, + MaintenanceState: admin.MaintenanceStateNone, + LastAdminUpdateError: "error", }, - MasterProfile: admin.MasterProfile{ - EncryptionAtHost: admin.EncryptionAtHostDisabled, - }, - OperatorFlags: admin.OperatorFlags{"testFlag": "true"}, - MaintenanceState: admin.MaintenanceStateNone, - LastAdminUpdateError: "error", - }, + } }, }, { @@ -1689,32 +1829,34 @@ func TestPutOrPatchOpenShiftClusterAdminAPI(t *testing.T) { }, wantAsync: true, wantStatusCode: http.StatusOK, - wantResponse: &admin.OpenShiftCluster{ - ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), - Type: "Microsoft.RedHatOpenShift/openShiftClusters", - Tags: map[string]string{"tag": "will-be-kept"}, - Properties: admin.OpenShiftClusterProperties{ - ProvisioningState: admin.ProvisioningStateAdminUpdating, - LastProvisioningState: admin.ProvisioningStateSucceeded, - FailedProvisioningState: admin.ProvisioningStateUpdating, - ClusterProfile: admin.ClusterProfile{ - FipsValidatedModules: admin.FipsValidatedModulesDisabled, - }, - NetworkProfile: admin.NetworkProfile{ - OutboundType: admin.OutboundTypeLoadbalancer, - LoadBalancerProfile: &admin.LoadBalancerProfile{ - ManagedOutboundIPs: &admin.ManagedOutboundIPs{ - Count: 1, + wantResponse: func() *admin.OpenShiftCluster { + return &admin.OpenShiftCluster{ + ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), + Type: "Microsoft.RedHatOpenShift/openShiftClusters", + Tags: map[string]string{"tag": "will-be-kept"}, + Properties: admin.OpenShiftClusterProperties{ + ProvisioningState: admin.ProvisioningStateAdminUpdating, + LastProvisioningState: admin.ProvisioningStateSucceeded, + FailedProvisioningState: admin.ProvisioningStateUpdating, + ClusterProfile: admin.ClusterProfile{ + FipsValidatedModules: admin.FipsValidatedModulesDisabled, + }, + NetworkProfile: admin.NetworkProfile{ + OutboundType: admin.OutboundTypeLoadbalancer, + LoadBalancerProfile: &admin.LoadBalancerProfile{ + ManagedOutboundIPs: &admin.ManagedOutboundIPs{ + Count: 1, + }, }, }, + MasterProfile: admin.MasterProfile{ + EncryptionAtHost: admin.EncryptionAtHostDisabled, + }, + OperatorFlags: admin.OperatorFlags{"testFlag": "true"}, + MaintenanceState: admin.MaintenanceStateUnplanned, + MaintenanceTask: admin.MaintenanceTaskEverything, }, - MasterProfile: admin.MasterProfile{ - EncryptionAtHost: admin.EncryptionAtHostDisabled, - }, - OperatorFlags: admin.OperatorFlags{"testFlag": "true"}, - MaintenanceState: admin.MaintenanceStateUnplanned, - MaintenanceTask: admin.MaintenanceTaskEverything, - }, + } }, }, } { @@ -1772,7 +1914,7 @@ func TestPutOrPatchOpenShiftClusterAdminAPI(t *testing.T) { t.Error(azureAsyncOperation) } } - err = validateResponse(resp, b, tt.wantStatusCode, tt.wantError, tt.wantResponse) + err = validateResponse(resp, b, tt.wantStatusCode, tt.wantError, tt.wantResponse()) if err != nil { t.Error(err) } From 4ae6d67a49c80b7915052ef02e5aa662a7611159 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 2 Oct 2024 14:25:40 +1000 Subject: [PATCH 55/67] add the maintmanifests client to the RP frontend/backend in dev --- cmd/aro/rp.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cmd/aro/rp.go b/cmd/aro/rp.go index 1dd9f6e3335..5e8227a68ff 100644 --- a/cmd/aro/rp.go +++ b/cmd/aro/rp.go @@ -172,6 +172,15 @@ func rp(ctx context.Context, log, audit *logrus.Entry) error { WithPlatformWorkloadIdentityRoleSets(dbPlatformWorkloadIdentityRoleSets). WithSubscriptions(dbSubscriptions) + // MIMO only activated in development for now + if _env.IsLocalDevelopmentMode() { + dbMaintenanceManifests, err := database.NewMaintenanceManifests(ctx, dbc, dbName) + if err != nil { + return err + } + dbg.WithMaintenanceManifests(dbMaintenanceManifests) + } + f, err := frontend.NewFrontend(ctx, audit, log.WithField("component", "frontend"), _env, dbg, api.APIs, metrics, clusterm, feAead, hiveClusterManager, adminactions.NewKubeActions, adminactions.NewAzureActions, adminactions.NewAppLensActions, clusterdata.NewParallelEnricher(metrics, _env)) if err != nil { return err From a985949278ae9143426d43d67868915dbfafb576 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 2 Oct 2024 14:32:29 +1000 Subject: [PATCH 56/67] reset the cluster flags to stop other tests failing --- test/e2e/mimo_actuator.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/e2e/mimo_actuator.go b/test/e2e/mimo_actuator.go index c9c760d23b0..eebd84e0cb8 100644 --- a/test/e2e/mimo_actuator.go +++ b/test/e2e/mimo_actuator.go @@ -25,6 +25,16 @@ var _ = Describe("MIMO Actuator E2E Testing", func() { BeforeEach(func() { skipIfNotInDevelopmentEnv() skipIfMIMOActuatorNotEnabled() + + DeferCleanup(func(ctx context.Context) { + // reset feature flags to their default values + var oc = &admin.OpenShiftCluster{} + resp, err := adminRequest(ctx, + http.MethodPatch, clusterResourceID, nil, true, + json.RawMessage("{\"operatorFlagsMergeStrategy\": \"reset\", \"properties\": {\"maintenanceTask\": \"SyncClusterObject\"}}"), oc) + Expect(err).NotTo(HaveOccurred()) + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + }) }) It("Should be able to schedule and run a maintenance set via the admin API", func(ctx context.Context) { From 637bda9111a6fbb9620924482c8db9e83e406d16 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 2 Oct 2024 15:15:11 +1000 Subject: [PATCH 57/67] Bump test file --- pkg/mimo/actuator/actuator_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/mimo/actuator/actuator_test.go b/pkg/mimo/actuator/actuator_test.go index 3da92fdeeb1..9ce44a6a17c 100644 --- a/pkg/mimo/actuator/actuator_test.go +++ b/pkg/mimo/actuator/actuator_test.go @@ -13,8 +13,8 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/golang/mock/gomock" "github.com/sirupsen/logrus" + "github.com/uber-go/mock/gomock" "github.com/Azure/ARO-RP/pkg/api" "github.com/Azure/ARO-RP/pkg/database" From 46f13be9a30b88bb917cb246ebbd7292d5b9db74 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 2 Oct 2024 15:17:01 +1000 Subject: [PATCH 58/67] Update actuator_test.go --- pkg/mimo/actuator/actuator_test.go | 2 +- pkg/mimo/actuator/service_test.go | 2 +- pkg/mimo/tasks/cluster/acrtoken_checker_test.go | 2 +- pkg/mimo/tasks/cluster/apiserver_is_up_test.go | 2 +- pkg/mimo/tasks/cluster/operatorflags_test.go | 2 +- pkg/mimo/tasks/cluster/tls_test.go | 2 +- pkg/mimo/tasks/example/task_test.go | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/mimo/actuator/actuator_test.go b/pkg/mimo/actuator/actuator_test.go index 9ce44a6a17c..06d71f62020 100644 --- a/pkg/mimo/actuator/actuator_test.go +++ b/pkg/mimo/actuator/actuator_test.go @@ -14,7 +14,7 @@ import ( . "github.com/onsi/gomega" "github.com/sirupsen/logrus" - "github.com/uber-go/mock/gomock" + "go.uber.org/mock/gomock" "github.com/Azure/ARO-RP/pkg/api" "github.com/Azure/ARO-RP/pkg/database" diff --git a/pkg/mimo/actuator/service_test.go b/pkg/mimo/actuator/service_test.go index d4bf43a153c..16811aa43fb 100644 --- a/pkg/mimo/actuator/service_test.go +++ b/pkg/mimo/actuator/service_test.go @@ -12,8 +12,8 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/golang/mock/gomock" "github.com/sirupsen/logrus" + "go.uber.org/mock/gomock" "github.com/Azure/ARO-RP/pkg/api" "github.com/Azure/ARO-RP/pkg/database" diff --git a/pkg/mimo/tasks/cluster/acrtoken_checker_test.go b/pkg/mimo/tasks/cluster/acrtoken_checker_test.go index 3775454c4e1..76b455e8ce1 100644 --- a/pkg/mimo/tasks/cluster/acrtoken_checker_test.go +++ b/pkg/mimo/tasks/cluster/acrtoken_checker_test.go @@ -11,7 +11,7 @@ import ( . "github.com/onsi/gomega" "github.com/Azure/go-autorest/autorest/date" - "github.com/golang/mock/gomock" + "go.uber.org/mock/gomock" "sigs.k8s.io/controller-runtime/pkg/client/fake" "github.com/Azure/ARO-RP/pkg/api" diff --git a/pkg/mimo/tasks/cluster/apiserver_is_up_test.go b/pkg/mimo/tasks/cluster/apiserver_is_up_test.go index 6acffbb4994..c89544766a1 100644 --- a/pkg/mimo/tasks/cluster/apiserver_is_up_test.go +++ b/pkg/mimo/tasks/cluster/apiserver_is_up_test.go @@ -10,8 +10,8 @@ import ( . "github.com/onsi/gomega" - "github.com/golang/mock/gomock" configv1 "github.com/openshift/api/config/v1" + "go.uber.org/mock/gomock" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client/fake" diff --git a/pkg/mimo/tasks/cluster/operatorflags_test.go b/pkg/mimo/tasks/cluster/operatorflags_test.go index c4d8b1ee418..f9b29f5fa3f 100644 --- a/pkg/mimo/tasks/cluster/operatorflags_test.go +++ b/pkg/mimo/tasks/cluster/operatorflags_test.go @@ -11,7 +11,7 @@ import ( . "github.com/onsi/gomega" "github.com/go-test/deep" - "github.com/golang/mock/gomock" + "go.uber.org/mock/gomock" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes/scheme" diff --git a/pkg/mimo/tasks/cluster/tls_test.go b/pkg/mimo/tasks/cluster/tls_test.go index ad850ed27ee..64b26995264 100644 --- a/pkg/mimo/tasks/cluster/tls_test.go +++ b/pkg/mimo/tasks/cluster/tls_test.go @@ -10,8 +10,8 @@ import ( . "github.com/onsi/gomega" - "github.com/golang/mock/gomock" configv1 "github.com/openshift/api/config/v1" + "go.uber.org/mock/gomock" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" diff --git a/pkg/mimo/tasks/example/task_test.go b/pkg/mimo/tasks/example/task_test.go index a912181cfc2..af1e07fa8e2 100644 --- a/pkg/mimo/tasks/example/task_test.go +++ b/pkg/mimo/tasks/example/task_test.go @@ -10,8 +10,8 @@ import ( . "github.com/onsi/gomega" - "github.com/golang/mock/gomock" configv1 "github.com/openshift/api/config/v1" + "go.uber.org/mock/gomock" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client/fake" From 2e8020619fdf1f5ba52f45727b65840d0a3f5f08 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Thu, 3 Oct 2024 13:05:46 +1000 Subject: [PATCH 59/67] fix the ARM resource deploying the partition key --- pkg/deploy/generator/resources_rp.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/deploy/generator/resources_rp.go b/pkg/deploy/generator/resources_rp.go index 2fb56b9af25..e630e4b31d5 100644 --- a/pkg/deploy/generator/resources_rp.go +++ b/pkg/deploy/generator/resources_rp.go @@ -1030,7 +1030,7 @@ func (g *generator) database(databaseName string, addDependsOn bool) []*arm.Reso ID: to.StringPtr("MaintenanceManifests"), PartitionKey: &sdkcosmos.ContainerPartitionKey{ Paths: []*string{ - to.StringPtr("/id"), + to.StringPtr("/clusterResourceID"), }, Kind: &hashPartitionKey, }, From dc7692fcb61319fe6e59285f6ba763cc57c3bb4e Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Thu, 3 Oct 2024 13:05:50 +1000 Subject: [PATCH 60/67] regen --- pkg/deploy/assets/databases-development.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/deploy/assets/databases-development.json b/pkg/deploy/assets/databases-development.json index b3ac7d93da4..dd4daa9375c 100644 --- a/pkg/deploy/assets/databases-development.json +++ b/pkg/deploy/assets/databases-development.json @@ -384,7 +384,7 @@ "partitionKey": { "kind": "Hash", "paths": [ - "/id" + "/clusterResourceID" ] } } From b1c0829f8fbd29d09f6217b8464be883b9dd825a Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Thu, 3 Oct 2024 13:45:31 +1000 Subject: [PATCH 61/67] lint fix --- pkg/api/admin/operatorflags.go | 5 ----- pkg/frontend/openshiftcluster_putorpatch.go | 1 - 2 files changed, 6 deletions(-) diff --git a/pkg/api/admin/operatorflags.go b/pkg/api/admin/operatorflags.go index b1caefe23e6..88d5e0873a6 100644 --- a/pkg/api/admin/operatorflags.go +++ b/pkg/api/admin/operatorflags.go @@ -11,11 +11,6 @@ import ( "github.com/Azure/ARO-RP/pkg/operator" ) -type operatorFlagsMergeStrategyStruct struct { - OperatorFlagsMergeStrategy string `json:"operatorFlagsMergeStrategy,omitempty"` - Cluster *api.OpenShiftCluster `json:"cluster,omitempty"` -} - const ( OperatorFlagsMergeStrategyMerge string = "merge" OperatorFlagsMergeStrategyReset string = "reset" diff --git a/pkg/frontend/openshiftcluster_putorpatch.go b/pkg/frontend/openshiftcluster_putorpatch.go index a4d1e193006..f5e9fdee896 100644 --- a/pkg/frontend/openshiftcluster_putorpatch.go +++ b/pkg/frontend/openshiftcluster_putorpatch.go @@ -225,7 +225,6 @@ func (f *frontend) _putOrPatchOpenShiftCluster(ctx context.Context, log *logrus. return nil, err } } else { - err = putOrPatchClusterParameters.staticValidator.Static(ext, doc.OpenShiftCluster, f.env.Location(), f.env.Domain(), f.env.FeatureIsSet(env.FeatureRequireD2sV3Workers), putOrPatchClusterParameters.path) if err != nil { return nil, err From af70386641a77c131c807c31c31a953e72382425 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Thu, 3 Oct 2024 23:53:58 +1000 Subject: [PATCH 62/67] fixes for e2e --- cmd/aro/mimoactuator.go | 2 +- pkg/database/mimo.go | 6 +++--- pkg/database/openshiftclusters.go | 2 +- pkg/mimo/actuator/manager.go | 9 ++++++--- pkg/mimo/actuator/service.go | 5 +++-- pkg/util/buckets/cache.go | 10 ++++++++-- 6 files changed, 22 insertions(+), 12 deletions(-) diff --git a/cmd/aro/mimoactuator.go b/cmd/aro/mimoactuator.go index c534495e80e..85421d17935 100644 --- a/cmd/aro/mimoactuator.go +++ b/cmd/aro/mimoactuator.go @@ -51,7 +51,7 @@ func mimoActuator(ctx context.Context, log *logrus.Entry) error { } go g.Run() - dbc, err := service.NewDatabase(ctx, _env, log, m, false) + dbc, err := service.NewDatabase(ctx, _env, log, m, true) if err != nil { return err } diff --git a/pkg/database/mimo.go b/pkg/database/mimo.go index 117346ff3a7..387342f05c1 100644 --- a/pkg/database/mimo.go +++ b/pkg/database/mimo.go @@ -15,8 +15,8 @@ import ( ) const ( - MaintenanceManifestDequeueQueryForCluster = `SELECT * FROM MaintenanceManifests doc WHERE doc.maintenanceManifest.state IN ("Pending") AND doc.clusterResourceID = @clusterID` - MaintenanceManifestQueryForCluster = `SELECT * FROM MaintenanceManifests doc WHERE doc.clusterResourceID = @clusterID` + MaintenanceManifestDequeueQueryForCluster = `SELECT * FROM MaintenanceManifests doc WHERE doc.maintenanceManifest.state IN ("Pending") AND doc.clusterResourceID = @clusterResourceID` + MaintenanceManifestQueryForCluster = `SELECT * FROM MaintenanceManifests doc WHERE doc.clusterResourceID = @clusterResourceID` MaintenanceManifestQueueLengthQuery = `SELECT VALUE COUNT(1) FROM MaintenanceManifests doc WHERE doc.maintenanceManifest.state IN ("Pending") AND (doc.leaseExpires ?? 0) < GetCurrentTimestamp() / 1000` ) @@ -224,7 +224,7 @@ func (c *maintenanceManifests) Lease(ctx context.Context, clusterResourceID stri return nil, fmt.Errorf("clusterID %q is not lower case", clusterResourceID) } - return c.patchWithLease(ctx, clusterResourceID, id, func(doc *api.MaintenanceManifestDocument) error { + return c.patch(ctx, clusterResourceID, id, func(doc *api.MaintenanceManifestDocument) error { doc.LeaseOwner = c.uuid doc.Dequeues++ return nil diff --git a/pkg/database/openshiftclusters.go b/pkg/database/openshiftclusters.go index 6c667f25d9b..409e4c4f0a0 100644 --- a/pkg/database/openshiftclusters.go +++ b/pkg/database/openshiftclusters.go @@ -23,7 +23,7 @@ const ( OpenshiftClustersPrefixQuery = `SELECT * FROM OpenShiftClusters doc WHERE STARTSWITH(doc.key, @prefix)` OpenshiftClustersClientIdQuery = `SELECT * FROM OpenShiftClusters doc WHERE doc.clientIdKey = @clientID` OpenshiftClustersResourceGroupQuery = `SELECT * FROM OpenShiftClusters doc WHERE doc.clusterResourceGroupIdKey = @resourceGroupID` - OpenshiftClustersClusterResourceIDOnlyQuery = `SELECT {key: doc.key} FROM OpenShiftClusters doc WHERE doc.openShiftCluster.properties.provisioningState IN ("Creating", "Deleting", "Updating", "AdminUpdating")` + OpenshiftClustersClusterResourceIDOnlyQuery = `SELECT doc.id, doc.key FROM OpenShiftClusters doc WHERE doc.openShiftCluster.properties.provisioningState NOT IN ("Creating", "Deleting")` ) type OpenShiftClusterDocumentMutator func(*api.OpenShiftClusterDocument) error diff --git a/pkg/mimo/actuator/manager.go b/pkg/mimo/actuator/manager.go index 231cdb857d7..0f016df2a8b 100644 --- a/pkg/mimo/actuator/manager.go +++ b/pkg/mimo/actuator/manager.go @@ -128,12 +128,12 @@ func (a *actuator) Process(ctx context.Context) (bool, error) { // Dequeue the document oc, err := a.oc.Get(ctx, a.clusterResourceID) if err != nil { - return false, err + return false, fmt.Errorf("failed getting cluster document: %w", err) } oc, err = a.oc.DoDequeue(ctx, oc) if err != nil { - return false, err // This will include StatusPreconditionFaileds + return false, fmt.Errorf("failed dequeuing cluster document: %w", err) // This will include StatusPreconditionFaileds } taskContext := newTaskContext(ctx, a.env, a.log, oc) @@ -185,5 +185,8 @@ func (a *actuator) Process(ctx context.Context) (bool, error) { // release the OpenShiftCluster _, err = a.oc.EndLease(ctx, a.clusterResourceID, oc.OpenShiftCluster.Properties.ProvisioningState, api.ProvisioningStateMaintenance, nil) - return true, err + if err != nil { + return false, fmt.Errorf("failed ending lease on cluster document: %w", err) + } + return true, nil } diff --git a/pkg/mimo/actuator/service.go b/pkg/mimo/actuator/service.go index 6c103141386..051c6338f13 100644 --- a/pkg/mimo/actuator/service.go +++ b/pkg/mimo/actuator/service.go @@ -254,11 +254,12 @@ func (s *service) checkReady() bool { func (s *service) worker(stop <-chan struct{}, delay time.Duration, id string) { defer recover.Panic(s.baseLog) + log := utillog.EnrichWithResourceID(s.baseLog, id) + log.Debugf("starting worker for %s in %s...", id, delay.String()) + // Wait for a randomised delay before starting time.Sleep(delay) - log := utillog.EnrichWithResourceID(s.baseLog, id) - dbOpenShiftClusters, err := s.dbGroup.OpenShiftClusters() if err != nil { log.Error(err) diff --git a/pkg/util/buckets/cache.go b/pkg/util/buckets/cache.go index 12f8d09d539..603bcf4c62d 100644 --- a/pkg/util/buckets/cache.go +++ b/pkg/util/buckets/cache.go @@ -52,9 +52,15 @@ func (mon *monitor) UpsertDoc(doc *api.OpenShiftClusterDocument) { func (mon *monitor) FixDoc(doc *api.OpenShiftClusterDocument) { id := strings.ToLower(doc.ID) v := mon.docs[id] - _, ours := mon.buckets[v.doc.Bucket] + + mon.baseLog.Debugf("fixing doc %s (%s)", doc.ID, doc.Key) + + // TODO: bucketing logic + //_, ours := mon.buckets[v.doc.Bucket] + ours := true if !ours && v.stop != nil { + mon.baseLog.Debugf("stopping channel for %s", doc.ID) close(v.stop) v.stop = nil } else if ours && v.stop == nil { @@ -63,7 +69,7 @@ func (mon *monitor) FixDoc(doc *api.OpenShiftClusterDocument) { delay := time.Duration(rand.Intn(60)) * time.Second - go mon.worker(ch, delay, id) + go mon.worker(ch, delay, doc.Key) } } From 470aa41c82b6c6dfde634b75ee142baad5af1b67 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Fri, 4 Oct 2024 12:19:01 +1000 Subject: [PATCH 63/67] add the ability to add a debug flag --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 4e491355a94..219a36c0abb 100644 --- a/Makefile +++ b/Makefile @@ -67,7 +67,7 @@ aro: check-release generate .PHONY: runlocal-rp runlocal-rp: - go run -ldflags "-X github.com/Azure/ARO-RP/pkg/util/version.GitCommit=$(VERSION)" ./cmd/aro rp + go run -ldflags "-X github.com/Azure/ARO-RP/pkg/util/version.GitCommit=$(VERSION)" ./cmd/aro ${ARO_CMD_ARGS} rp .PHONY: az az: pyenv @@ -196,11 +196,11 @@ proxy: .PHONY: runlocal-portal runlocal-portal: - go run -ldflags "-X github.com/Azure/ARO-RP/pkg/util/version.GitCommit=$(VERSION)" ./cmd/aro portal + go run -ldflags "-X github.com/Azure/ARO-RP/pkg/util/version.GitCommit=$(VERSION)" ./cmd/aro ${ARO_CMD_ARGS} portal .PHONY: runlocal-actuator runlocal-actuator: - go run -ldflags "-X github.com/Azure/ARO-RP/pkg/util/version.GitCommit=$(VERSION)" ./cmd/aro mimo-actuator + go run -ldflags "-X github.com/Azure/ARO-RP/pkg/util/version.GitCommit=$(VERSION)" ./cmd/aro ${ARO_CMD_ARGS} mimo-actuator .PHONY: build-portal build-portal: From 6dcb1e51458575ba6990cf50602d51b9e5a9e9a7 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Fri, 4 Oct 2024 15:34:02 +1000 Subject: [PATCH 64/67] e2e fix --- test/e2e/mimo_actuator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/mimo_actuator.go b/test/e2e/mimo_actuator.go index eebd84e0cb8..9cff1d54241 100644 --- a/test/e2e/mimo_actuator.go +++ b/test/e2e/mimo_actuator.go @@ -92,7 +92,7 @@ var _ = Describe("MIMO Actuator E2E Testing", func() { Eventually(func(g Gomega, ctx context.Context) { fetchedManifest := &admin.MaintenanceManifest{} resp, err = adminRequest(ctx, - http.MethodPut, "/admin"+clusterResourceID+"/maintenancemanifests/"+manifestID, + http.MethodGet, "/admin"+clusterResourceID+"/maintenancemanifests/"+manifestID, url.Values{}, true, nil, &fetchedManifest) g.Expect(err).NotTo(HaveOccurred()) From 1380fd2c4bdda1db0fae52559f09a4a087d6b8a2 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 9 Oct 2024 12:50:15 +1100 Subject: [PATCH 65/67] renames and fixes --- cmd/aro/mimoactuator.go | 4 +- pkg/api/admin/mimo.go | 4 +- pkg/api/admin/mimo_convert.go | 6 +- pkg/api/admin/mimo_validatestatic.go | 4 +- pkg/api/mimo.go | 12 +-- .../admin_mimo_manifest_cancel_test.go | 44 ++++----- .../admin_mimo_manifest_create_test.go | 52 +++++----- .../admin_mimo_manifest_delete_test.go | 8 +- pkg/frontend/admin_mimo_manifest_get_test.go | 20 ++-- pkg/frontend/admin_mimo_manifest_list_test.go | 48 +++++----- pkg/mimo/actuator/actuator_test.go | 94 +++++++++---------- pkg/mimo/actuator/manager.go | 16 ++-- pkg/mimo/actuator/service.go | 10 +- pkg/mimo/actuator/service_test.go | 24 ++--- .../cluster/acrtoken_checker.go | 0 .../cluster/acrtoken_checker_test.go | 4 +- .../cluster/apiserver_is_up.go | 0 .../cluster/apiserver_is_up_test.go | 0 .../{tasks => steps}/cluster/operatorflags.go | 0 .../cluster/operatorflags_test.go | 0 pkg/mimo/{tasks => steps}/cluster/tls.go | 0 pkg/mimo/{tasks => steps}/cluster/tls_test.go | 0 pkg/mimo/{tasks => steps}/example/task.go | 0 .../{tasks => steps}/example/task_test.go | 0 pkg/mimo/{sets => tasks}/acrtoken.go | 4 +- pkg/mimo/{sets => tasks}/interfaces.go | 4 +- pkg/mimo/{sets => tasks}/operatorflags.go | 4 +- pkg/mimo/{sets => tasks}/sets.go | 4 +- pkg/mimo/{sets => tasks}/tls_cert_rotate.go | 4 +- test/e2e/mimo_actuator.go | 2 +- 30 files changed, 183 insertions(+), 189 deletions(-) rename pkg/mimo/{tasks => steps}/cluster/acrtoken_checker.go (100%) rename pkg/mimo/{tasks => steps}/cluster/acrtoken_checker_test.go (96%) rename pkg/mimo/{tasks => steps}/cluster/apiserver_is_up.go (100%) rename pkg/mimo/{tasks => steps}/cluster/apiserver_is_up_test.go (100%) rename pkg/mimo/{tasks => steps}/cluster/operatorflags.go (100%) rename pkg/mimo/{tasks => steps}/cluster/operatorflags_test.go (100%) rename pkg/mimo/{tasks => steps}/cluster/tls.go (100%) rename pkg/mimo/{tasks => steps}/cluster/tls_test.go (100%) rename pkg/mimo/{tasks => steps}/example/task.go (100%) rename pkg/mimo/{tasks => steps}/example/task_test.go (100%) rename pkg/mimo/{sets => tasks}/acrtoken.go (87%) rename pkg/mimo/{sets => tasks}/interfaces.go (51%) rename pkg/mimo/{sets => tasks}/operatorflags.go (88%) rename pkg/mimo/{sets => tasks}/sets.go (92%) rename pkg/mimo/{sets => tasks}/tls_cert_rotate.go (90%) diff --git a/cmd/aro/mimoactuator.go b/cmd/aro/mimoactuator.go index 85421d17935..1136f3fb572 100644 --- a/cmd/aro/mimoactuator.go +++ b/cmd/aro/mimoactuator.go @@ -16,7 +16,7 @@ import ( "github.com/Azure/ARO-RP/pkg/metrics/statsd" "github.com/Azure/ARO-RP/pkg/metrics/statsd/golang" "github.com/Azure/ARO-RP/pkg/mimo/actuator" - "github.com/Azure/ARO-RP/pkg/mimo/sets" + "github.com/Azure/ARO-RP/pkg/mimo/tasks" "github.com/Azure/ARO-RP/pkg/proxy" "github.com/Azure/ARO-RP/pkg/util/service" ) @@ -81,7 +81,7 @@ func mimoActuator(ctx context.Context, log *logrus.Entry) error { } a := actuator.NewService(_env, _env.Logger(), dialer, dbg, m) - a.SetMaintenanceSets(sets.DEFAULT_MAINTENANCE_SETS) + a.SetMaintenanceTasks(tasks.DEFAULT_MAINTENANCE_SETS) sigterm := make(chan os.Signal, 1) done := make(chan struct{}) diff --git a/pkg/api/admin/mimo.go b/pkg/api/admin/mimo.go index 18fbae98358..b57c5664766 100644 --- a/pkg/api/admin/mimo.go +++ b/pkg/api/admin/mimo.go @@ -21,8 +21,8 @@ type MaintenanceManifest struct { State MaintenanceManifestState `json:"state,omitempty"` StatusText string `json:"statusText,omitempty"` - MaintenanceSetID string `json:"maintenanceSetID,omitempty"` - Priority int `json:"priority,omitempty"` + MaintenanceTaskID string `json:"maintenanceTaskID,omitempty"` + Priority int `json:"priority,omitempty"` // RunAfter defines the earliest that this manifest should start running RunAfter int `json:"runAfter,omitempty"` diff --git a/pkg/api/admin/mimo_convert.go b/pkg/api/admin/mimo_convert.go index 6cd9a518577..3bc1b468554 100644 --- a/pkg/api/admin/mimo_convert.go +++ b/pkg/api/admin/mimo_convert.go @@ -16,8 +16,8 @@ func (m maintenanceManifestConverter) ToExternal(d *api.MaintenanceManifestDocum State: MaintenanceManifestState(d.MaintenanceManifest.State), StatusText: d.MaintenanceManifest.StatusText, - MaintenanceSetID: d.MaintenanceManifest.MaintenanceSetID, - Priority: d.MaintenanceManifest.Priority, + MaintenanceTaskID: d.MaintenanceManifest.MaintenanceTaskID, + Priority: d.MaintenanceManifest.Priority, RunAfter: d.MaintenanceManifest.RunAfter, RunBefore: d.MaintenanceManifest.RunBefore, @@ -41,7 +41,7 @@ func (m maintenanceManifestConverter) ToInternal(_i interface{}, out *api.Mainte i := _i.(*MaintenanceManifest) out.ID = i.ID - out.MaintenanceManifest.MaintenanceSetID = i.MaintenanceSetID + out.MaintenanceManifest.MaintenanceTaskID = i.MaintenanceTaskID out.MaintenanceManifest.Priority = i.Priority out.MaintenanceManifest.RunAfter = i.RunAfter out.MaintenanceManifest.RunBefore = i.RunBefore diff --git a/pkg/api/admin/mimo_validatestatic.go b/pkg/api/admin/mimo_validatestatic.go index 0193a4404b1..ca052458631 100644 --- a/pkg/api/admin/mimo_validatestatic.go +++ b/pkg/api/admin/mimo_validatestatic.go @@ -34,8 +34,8 @@ func (sv maintenanceManifestStaticValidator) Static(_new interface{}, _current * } func (sv maintenanceManifestStaticValidator) validate(new *MaintenanceManifest) error { - if new.MaintenanceSetID == "" { - return api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidParameter, "maintenanceSetID", "Must be provided") + if new.MaintenanceTaskID == "" { + return api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidParameter, "maintenanceTaskID", "Must be provided") } if new.RunAfter == 0 { diff --git a/pkg/api/mimo.go b/pkg/api/mimo.go index d2616407ad2..b8e983b2992 100644 --- a/pkg/api/mimo.go +++ b/pkg/api/mimo.go @@ -14,13 +14,7 @@ const ( MaintenanceManifestStateCancelled MaintenanceManifestState = "Cancelled" ) -type MaintenanceSet struct { - MissingFields - - Name string `json:"name,omitempty"` -} - -// MaintenanceManifest represents an instance of a MaintenanceSet running on a +// MaintenanceManifest represents an instance of a MaintenanceTask running on a // given cluster. type MaintenanceManifest struct { MissingFields @@ -28,8 +22,8 @@ type MaintenanceManifest struct { State MaintenanceManifestState `json:"state,omitempty"` StatusText string `json:"statusText,omitempty"` - MaintenanceSetID string `json:"maintenanceSetID,omitempty"` - Priority int `json:"priority,omitempty"` + MaintenanceTaskID string `json:"maintenanceTaskID,omitempty"` + Priority int `json:"priority,omitempty"` // RunAfter defines the earliest that this manifest should start running RunAfter int `json:"runAfter,omitempty"` diff --git a/pkg/frontend/admin_mimo_manifest_cancel_test.go b/pkg/frontend/admin_mimo_manifest_cancel_test.go index 97669c6c356..ae8536a4fc6 100644 --- a/pkg/frontend/admin_mimo_manifest_cancel_test.go +++ b/pkg/frontend/admin_mimo_manifest_cancel_test.go @@ -86,10 +86,10 @@ func TestMIMOCancelManifest(t *testing.T) { f.AddMaintenanceManifestDocuments(&api.MaintenanceManifestDocument{ ClusterResourceID: strings.ToLower(resourceID), MaintenanceManifest: api.MaintenanceManifest{ - MaintenanceSetID: "exampleset", - State: api.MaintenanceManifestStatePending, - RunAfter: 1, - RunBefore: 1, + MaintenanceTaskID: "exampletask", + State: api.MaintenanceManifestStatePending, + RunAfter: 1, + RunBefore: 1, }, }) }, @@ -98,20 +98,20 @@ func TestMIMOCancelManifest(t *testing.T) { ID: "07070707-0707-0707-0707-070707070001", ClusterResourceID: strings.ToLower(resourceID), MaintenanceManifest: api.MaintenanceManifest{ - MaintenanceSetID: "exampleset", - State: api.MaintenanceManifestStateCancelled, - RunAfter: 1, - RunBefore: 1, + MaintenanceTaskID: "exampletask", + State: api.MaintenanceManifestStateCancelled, + RunAfter: 1, + RunBefore: 1, }, }) }, wantResponse: &admin.MaintenanceManifest{ - ID: "07070707-0707-0707-0707-070707070001", - MaintenanceSetID: "exampleset", - State: admin.MaintenanceManifestStateCancelled, - Priority: 0, - RunAfter: 1, - RunBefore: 1, + ID: "07070707-0707-0707-0707-070707070001", + MaintenanceTaskID: "exampletask", + State: admin.MaintenanceManifestStateCancelled, + Priority: 0, + RunAfter: 1, + RunBefore: 1, }, wantStatusCode: http.StatusOK, }, @@ -129,10 +129,10 @@ func TestMIMOCancelManifest(t *testing.T) { f.AddMaintenanceManifestDocuments(&api.MaintenanceManifestDocument{ ClusterResourceID: strings.ToLower(resourceID), MaintenanceManifest: api.MaintenanceManifest{ - MaintenanceSetID: "exampleset", - State: api.MaintenanceManifestStateFailed, - RunAfter: 1, - RunBefore: 1, + MaintenanceTaskID: "exampletask", + State: api.MaintenanceManifestStateFailed, + RunAfter: 1, + RunBefore: 1, }, }) }, @@ -141,10 +141,10 @@ func TestMIMOCancelManifest(t *testing.T) { ID: "07070707-0707-0707-0707-070707070001", ClusterResourceID: strings.ToLower(resourceID), MaintenanceManifest: api.MaintenanceManifest{ - MaintenanceSetID: "exampleset", - State: api.MaintenanceManifestStateFailed, - RunAfter: 1, - RunBefore: 1, + MaintenanceTaskID: "exampletask", + State: api.MaintenanceManifestStateFailed, + RunAfter: 1, + RunBefore: 1, }, }) }, diff --git a/pkg/frontend/admin_mimo_manifest_create_test.go b/pkg/frontend/admin_mimo_manifest_create_test.go index e23f1ccf376..d309e3d8c71 100644 --- a/pkg/frontend/admin_mimo_manifest_create_test.go +++ b/pkg/frontend/admin_mimo_manifest_create_test.go @@ -71,7 +71,7 @@ func TestMIMOCreateManifest(t *testing.T) { }) }, body: &admin.MaintenanceManifest{}, - wantError: "400: InvalidParameter: maintenanceSetID: Must be provided", + wantError: "400: InvalidParameter: maintenanceTaskID: Must be provided", wantStatusCode: http.StatusBadRequest, }, @@ -88,29 +88,29 @@ func TestMIMOCreateManifest(t *testing.T) { }) }, body: &admin.MaintenanceManifest{ - MaintenanceSetID: "exampleset", - State: admin.MaintenanceManifestStatePending, - RunAfter: 1, - RunBefore: 1, + MaintenanceTaskID: "exampletask", + State: admin.MaintenanceManifestStatePending, + RunAfter: 1, + RunBefore: 1, }, wantResult: func(c *testdatabase.Checker) { c.AddMaintenanceManifestDocuments(&api.MaintenanceManifestDocument{ ID: "07070707-0707-0707-0707-070707070001", ClusterResourceID: strings.ToLower(resourceID), MaintenanceManifest: api.MaintenanceManifest{ - MaintenanceSetID: "exampleset", - State: api.MaintenanceManifestStatePending, - RunAfter: 1, - RunBefore: 1, + MaintenanceTaskID: "exampletask", + State: api.MaintenanceManifestStatePending, + RunAfter: 1, + RunBefore: 1, }, }) }, wantResponse: &admin.MaintenanceManifest{ - ID: "07070707-0707-0707-0707-070707070001", - MaintenanceSetID: "exampleset", - State: admin.MaintenanceManifestStatePending, - RunAfter: 1, - RunBefore: 1, + ID: "07070707-0707-0707-0707-070707070001", + MaintenanceTaskID: "exampletask", + State: admin.MaintenanceManifestStatePending, + RunAfter: 1, + RunBefore: 1, }, wantStatusCode: http.StatusCreated, }, @@ -127,28 +127,28 @@ func TestMIMOCreateManifest(t *testing.T) { }) }, body: &admin.MaintenanceManifest{ - MaintenanceSetID: "exampleset", - RunAfter: 1, - RunBefore: 1, + MaintenanceTaskID: "exampletask", + RunAfter: 1, + RunBefore: 1, }, wantResult: func(c *testdatabase.Checker) { c.AddMaintenanceManifestDocuments(&api.MaintenanceManifestDocument{ ID: "07070707-0707-0707-0707-070707070001", ClusterResourceID: strings.ToLower(resourceID), MaintenanceManifest: api.MaintenanceManifest{ - MaintenanceSetID: "exampleset", - State: api.MaintenanceManifestStatePending, - RunAfter: 1, - RunBefore: 1, + MaintenanceTaskID: "exampletask", + State: api.MaintenanceManifestStatePending, + RunAfter: 1, + RunBefore: 1, }, }) }, wantResponse: &admin.MaintenanceManifest{ - ID: "07070707-0707-0707-0707-070707070001", - MaintenanceSetID: "exampleset", - State: admin.MaintenanceManifestStatePending, - RunAfter: 1, - RunBefore: 1, + ID: "07070707-0707-0707-0707-070707070001", + MaintenanceTaskID: "exampletask", + State: admin.MaintenanceManifestStatePending, + RunAfter: 1, + RunBefore: 1, }, wantStatusCode: http.StatusCreated, }, diff --git a/pkg/frontend/admin_mimo_manifest_delete_test.go b/pkg/frontend/admin_mimo_manifest_delete_test.go index 781948b2a42..221d02cabca 100644 --- a/pkg/frontend/admin_mimo_manifest_delete_test.go +++ b/pkg/frontend/admin_mimo_manifest_delete_test.go @@ -85,10 +85,10 @@ func TestMIMODeleteManifest(t *testing.T) { f.AddMaintenanceManifestDocuments(&api.MaintenanceManifestDocument{ ClusterResourceID: strings.ToLower(resourceID), MaintenanceManifest: api.MaintenanceManifest{ - MaintenanceSetID: "exampleset", - State: api.MaintenanceManifestStatePending, - RunAfter: 1, - RunBefore: 1, + MaintenanceTaskID: "exampletask", + State: api.MaintenanceManifestStatePending, + RunAfter: 1, + RunBefore: 1, }, }) }, diff --git a/pkg/frontend/admin_mimo_manifest_get_test.go b/pkg/frontend/admin_mimo_manifest_get_test.go index 9b7e6a84265..ddad3d36f9f 100644 --- a/pkg/frontend/admin_mimo_manifest_get_test.go +++ b/pkg/frontend/admin_mimo_manifest_get_test.go @@ -85,20 +85,20 @@ func TestMIMOGet(t *testing.T) { f.AddMaintenanceManifestDocuments(&api.MaintenanceManifestDocument{ ClusterResourceID: strings.ToLower(resourceID), MaintenanceManifest: api.MaintenanceManifest{ - MaintenanceSetID: "exampleset", - State: api.MaintenanceManifestStatePending, - RunAfter: 1, - RunBefore: 1, + MaintenanceTaskID: "exampletask", + State: api.MaintenanceManifestStatePending, + RunAfter: 1, + RunBefore: 1, }, }) }, wantResponse: &admin.MaintenanceManifest{ - ID: "07070707-0707-0707-0707-070707070001", - MaintenanceSetID: "exampleset", - State: admin.MaintenanceManifestStatePending, - Priority: 0, - RunAfter: 1, - RunBefore: 1, + ID: "07070707-0707-0707-0707-070707070001", + MaintenanceTaskID: "exampletask", + State: admin.MaintenanceManifestStatePending, + Priority: 0, + RunAfter: 1, + RunBefore: 1, }, wantStatusCode: http.StatusOK, }, diff --git a/pkg/frontend/admin_mimo_manifest_list_test.go b/pkg/frontend/admin_mimo_manifest_list_test.go index c82ebd51102..4c7664e5f4d 100644 --- a/pkg/frontend/admin_mimo_manifest_list_test.go +++ b/pkg/frontend/admin_mimo_manifest_list_test.go @@ -66,22 +66,22 @@ func TestMIMOListManifests(t *testing.T) { f.AddMaintenanceManifestDocuments(&api.MaintenanceManifestDocument{ ClusterResourceID: strings.ToLower(resourceID), MaintenanceManifest: api.MaintenanceManifest{ - MaintenanceSetID: "exampleset", - State: api.MaintenanceManifestStatePending, - RunAfter: 1, - RunBefore: 1, + MaintenanceTaskID: "exampletask", + State: api.MaintenanceManifestStatePending, + RunAfter: 1, + RunBefore: 1, }, }) }, wantResponse: &admin.MaintenanceManifestList{ MaintenanceManifests: []*admin.MaintenanceManifest{ { - ID: "07070707-0707-0707-0707-070707070001", - MaintenanceSetID: "exampleset", - State: admin.MaintenanceManifestStatePending, - Priority: 0, - RunAfter: 1, - RunBefore: 1, + ID: "07070707-0707-0707-0707-070707070001", + MaintenanceTaskID: "exampletask", + State: admin.MaintenanceManifestStatePending, + Priority: 0, + RunAfter: 1, + RunBefore: 1, }, }, }, @@ -102,19 +102,19 @@ func TestMIMOListManifests(t *testing.T) { f.AddMaintenanceManifestDocuments(&api.MaintenanceManifestDocument{ ClusterResourceID: strings.ToLower(resourceID), MaintenanceManifest: api.MaintenanceManifest{ - MaintenanceSetID: "exampleset", - State: api.MaintenanceManifestStatePending, - RunAfter: 1, - RunBefore: 1, + MaintenanceTaskID: "exampletask", + State: api.MaintenanceManifestStatePending, + RunAfter: 1, + RunBefore: 1, }, }) f.AddMaintenanceManifestDocuments(&api.MaintenanceManifestDocument{ ClusterResourceID: strings.ToLower(resourceID), MaintenanceManifest: api.MaintenanceManifest{ - MaintenanceSetID: "exampleset2", - State: api.MaintenanceManifestStatePending, - RunAfter: 1, - RunBefore: 1, + MaintenanceTaskID: "exampleset2", + State: api.MaintenanceManifestStatePending, + RunAfter: 1, + RunBefore: 1, }, }) }, @@ -122,12 +122,12 @@ func TestMIMOListManifests(t *testing.T) { NextLink: "https://mockrefererhost/?%24skipToken=" + url.QueryEscape(base64.StdEncoding.EncodeToString([]byte("FAKE1"))), MaintenanceManifests: []*admin.MaintenanceManifest{ { - ID: "07070707-0707-0707-0707-070707070001", - MaintenanceSetID: "exampleset", - State: admin.MaintenanceManifestStatePending, - Priority: 0, - RunAfter: 1, - RunBefore: 1, + ID: "07070707-0707-0707-0707-070707070001", + MaintenanceTaskID: "exampletask", + State: admin.MaintenanceManifestStatePending, + Priority: 0, + RunAfter: 1, + RunBefore: 1, }, }, }, diff --git a/pkg/mimo/actuator/actuator_test.go b/pkg/mimo/actuator/actuator_test.go index 06d71f62020..1cfc82a9844 100644 --- a/pkg/mimo/actuator/actuator_test.go +++ b/pkg/mimo/actuator/actuator_test.go @@ -20,7 +20,7 @@ import ( "github.com/Azure/ARO-RP/pkg/database" "github.com/Azure/ARO-RP/pkg/database/cosmosdb" "github.com/Azure/ARO-RP/pkg/env" - "github.com/Azure/ARO-RP/pkg/mimo/sets" + "github.com/Azure/ARO-RP/pkg/mimo/tasks" "github.com/Azure/ARO-RP/pkg/util/mimo" mock_env "github.com/Azure/ARO-RP/pkg/util/mocks/env" testdatabase "github.com/Azure/ARO-RP/test/database" @@ -85,8 +85,8 @@ var _ = Describe("MIMO Actuator", Ordered, func() { mmf: manifests, oc: clusters, - sets: map[string]sets.MaintenanceSet{}, - now: now, + tasks: map[string]tasks.MaintenanceTask{}, + now: now, } }) @@ -158,10 +158,10 @@ var _ = Describe("MIMO Actuator", Ordered, func() { ID: manifestID, ClusterResourceID: strings.ToLower(clusterResourceID), MaintenanceManifest: api.MaintenanceManifest{ - State: api.MaintenanceManifestStatePending, - MaintenanceSetID: "0", - RunBefore: 600, - RunAfter: 0, + State: api.MaintenanceManifestStatePending, + MaintenanceTaskID: "0", + RunBefore: 600, + RunAfter: 0, }, }) @@ -170,17 +170,17 @@ var _ = Describe("MIMO Actuator", Ordered, func() { ID: manifestID, ClusterResourceID: strings.ToLower(clusterResourceID), MaintenanceManifest: api.MaintenanceManifest{ - State: api.MaintenanceManifestStateCompleted, - MaintenanceSetID: "0", - StatusText: "done", - RunBefore: 600, - RunAfter: 0, + State: api.MaintenanceManifestStateCompleted, + MaintenanceTaskID: "0", + StatusText: "done", + RunBefore: 600, + RunAfter: 0, }, }) }) It("runs them", func() { - a.AddMaintenanceSets(map[string]sets.MaintenanceSet{ + a.AddMaintenanceTasks(map[string]tasks.MaintenanceTask{ "0": func(th mimo.TaskContext, mmd *api.MaintenanceManifestDocument, oscd *api.OpenShiftClusterDocument) (api.MaintenanceManifestState, string) { return api.MaintenanceManifestStateCompleted, "done" }, @@ -212,33 +212,33 @@ var _ = Describe("MIMO Actuator", Ordered, func() { ID: manifestIDs[0], ClusterResourceID: strings.ToLower(clusterResourceID), MaintenanceManifest: api.MaintenanceManifest{ - State: api.MaintenanceManifestStatePending, - MaintenanceSetID: "0", - RunBefore: 600, - RunAfter: 0, - Priority: 2, + State: api.MaintenanceManifestStatePending, + MaintenanceTaskID: "0", + RunBefore: 600, + RunAfter: 0, + Priority: 2, }, }, &api.MaintenanceManifestDocument{ ID: manifestIDs[1], ClusterResourceID: strings.ToLower(clusterResourceID), MaintenanceManifest: api.MaintenanceManifest{ - State: api.MaintenanceManifestStatePending, - MaintenanceSetID: "1", - RunBefore: 600, - RunAfter: 0, - Priority: 1, + State: api.MaintenanceManifestStatePending, + MaintenanceTaskID: "1", + RunBefore: 600, + RunAfter: 0, + Priority: 1, }, }, &api.MaintenanceManifestDocument{ ID: manifestIDs[2], ClusterResourceID: strings.ToLower(clusterResourceID), MaintenanceManifest: api.MaintenanceManifest{ - State: api.MaintenanceManifestStatePending, - MaintenanceSetID: "2", - RunBefore: 600, - RunAfter: 1, - Priority: 0, + State: api.MaintenanceManifestStatePending, + MaintenanceTaskID: "2", + RunBefore: 600, + RunAfter: 1, + Priority: 0, }, }) @@ -247,36 +247,36 @@ var _ = Describe("MIMO Actuator", Ordered, func() { ID: manifestIDs[0], ClusterResourceID: strings.ToLower(clusterResourceID), MaintenanceManifest: api.MaintenanceManifest{ - State: api.MaintenanceManifestStateCompleted, - MaintenanceSetID: "0", - StatusText: "done", - RunBefore: 600, - RunAfter: 0, - Priority: 2, + State: api.MaintenanceManifestStateCompleted, + MaintenanceTaskID: "0", + StatusText: "done", + RunBefore: 600, + RunAfter: 0, + Priority: 2, }, }, &api.MaintenanceManifestDocument{ ID: manifestIDs[1], ClusterResourceID: strings.ToLower(clusterResourceID), MaintenanceManifest: api.MaintenanceManifest{ - State: api.MaintenanceManifestStateCompleted, - MaintenanceSetID: "1", - StatusText: "done", - RunBefore: 600, - RunAfter: 0, - Priority: 1, + State: api.MaintenanceManifestStateCompleted, + MaintenanceTaskID: "1", + StatusText: "done", + RunBefore: 600, + RunAfter: 0, + Priority: 1, }, }, &api.MaintenanceManifestDocument{ ID: manifestIDs[2], ClusterResourceID: strings.ToLower(clusterResourceID), MaintenanceManifest: api.MaintenanceManifest{ - State: api.MaintenanceManifestStateCompleted, - MaintenanceSetID: "2", - StatusText: "done", - RunBefore: 600, - RunAfter: 1, - Priority: 0, + State: api.MaintenanceManifestStateCompleted, + MaintenanceTaskID: "2", + StatusText: "done", + RunBefore: 600, + RunAfter: 1, + Priority: 0, }, }) }) @@ -284,7 +284,7 @@ var _ = Describe("MIMO Actuator", Ordered, func() { It("runs them", func() { ordering := []string{} - a.AddMaintenanceSets(map[string]sets.MaintenanceSet{ + a.AddMaintenanceTasks(map[string]tasks.MaintenanceTask{ "0": func(th mimo.TaskContext, mmd *api.MaintenanceManifestDocument, oscd *api.OpenShiftClusterDocument) (api.MaintenanceManifestState, string) { ordering = append(ordering, "0") return api.MaintenanceManifestStateCompleted, "done" diff --git a/pkg/mimo/actuator/manager.go b/pkg/mimo/actuator/manager.go index 0f016df2a8b..3b8aae922e8 100644 --- a/pkg/mimo/actuator/manager.go +++ b/pkg/mimo/actuator/manager.go @@ -17,14 +17,14 @@ import ( "github.com/Azure/ARO-RP/pkg/api" "github.com/Azure/ARO-RP/pkg/database" "github.com/Azure/ARO-RP/pkg/env" - "github.com/Azure/ARO-RP/pkg/mimo/sets" + "github.com/Azure/ARO-RP/pkg/mimo/tasks" ) const maxDequeueCount = 5 type Actuator interface { Process(context.Context) (bool, error) - AddMaintenanceSets(map[string]sets.MaintenanceSet) + AddMaintenanceTasks(map[string]tasks.MaintenanceTask) } type actuator struct { @@ -37,7 +37,7 @@ type actuator struct { oc database.OpenShiftClusters mmf database.MaintenanceManifests - sets map[string]sets.MaintenanceSet + tasks map[string]tasks.MaintenanceTask } func NewActuator( @@ -54,7 +54,7 @@ func NewActuator( clusterResourceID: strings.ToLower(clusterResourceID), oc: oc, mmf: mmf, - sets: make(map[string]sets.MaintenanceSet), + tasks: make(map[string]tasks.MaintenanceTask), now: now, } @@ -62,8 +62,8 @@ func NewActuator( return a, nil } -func (a *actuator) AddMaintenanceSets(sets map[string]sets.MaintenanceSet) { - maps.Copy(a.sets, sets) +func (a *actuator) AddMaintenanceTasks(tasks map[string]tasks.MaintenanceTask) { + maps.Copy(a.tasks, tasks) } func (a *actuator) Process(ctx context.Context) (bool, error) { @@ -141,9 +141,9 @@ func (a *actuator) Process(ctx context.Context) (bool, error) { // Execute on the manifests we want to action for _, doc := range manifestsToAction { // here - f, ok := a.sets[doc.MaintenanceManifest.MaintenanceSetID] + f, ok := a.tasks[doc.MaintenanceManifest.MaintenanceTaskID] if !ok { - a.log.Infof("not found %v", doc.MaintenanceManifest.MaintenanceSetID) + a.log.Infof("not found %v", doc.MaintenanceManifest.MaintenanceTaskID) continue } diff --git a/pkg/mimo/actuator/service.go b/pkg/mimo/actuator/service.go index 051c6338f13..62a1133fcd8 100644 --- a/pkg/mimo/actuator/service.go +++ b/pkg/mimo/actuator/service.go @@ -22,7 +22,7 @@ import ( "github.com/Azure/ARO-RP/pkg/database" "github.com/Azure/ARO-RP/pkg/env" "github.com/Azure/ARO-RP/pkg/metrics" - "github.com/Azure/ARO-RP/pkg/mimo/sets" + "github.com/Azure/ARO-RP/pkg/mimo/tasks" "github.com/Azure/ARO-RP/pkg/proxy" "github.com/Azure/ARO-RP/pkg/util/buckets" "github.com/Azure/ARO-RP/pkg/util/heartbeat" @@ -55,7 +55,7 @@ type service struct { pollTime time.Duration now func() time.Time - sets map[string]sets.MaintenanceSet + tasks map[string]tasks.MaintenanceTask serveHealthz bool } @@ -89,8 +89,8 @@ func NewService(env env.Interface, log *logrus.Entry, dialer proxy.Dialer, dbg a return s } -func (s *service) SetMaintenanceSets(sets map[string]sets.MaintenanceSet) { - s.sets = sets +func (s *service) SetMaintenanceTasks(tasks map[string]tasks.MaintenanceTask) { + s.tasks = tasks } func (s *service) Run(ctx context.Context, stop <-chan struct{}, done chan<- struct{}) error { @@ -279,7 +279,7 @@ func (s *service) worker(stop <-chan struct{}, delay time.Duration, id string) { } // load in the tasks for the Actuator from the controller - a.AddMaintenanceSets(s.sets) + a.AddMaintenanceTasks(s.tasks) t := time.NewTicker(s.pollTime) defer t.Stop() diff --git a/pkg/mimo/actuator/service_test.go b/pkg/mimo/actuator/service_test.go index 16811aa43fb..0d686271f02 100644 --- a/pkg/mimo/actuator/service_test.go +++ b/pkg/mimo/actuator/service_test.go @@ -20,7 +20,7 @@ import ( "github.com/Azure/ARO-RP/pkg/database/cosmosdb" "github.com/Azure/ARO-RP/pkg/env" "github.com/Azure/ARO-RP/pkg/metrics" - "github.com/Azure/ARO-RP/pkg/mimo/sets" + "github.com/Azure/ARO-RP/pkg/mimo/tasks" "github.com/Azure/ARO-RP/pkg/util/mimo" mock_env "github.com/Azure/ARO-RP/pkg/util/mocks/env" testdatabase "github.com/Azure/ARO-RP/test/database" @@ -176,10 +176,10 @@ var _ = Describe("MIMO Actuator Service", Ordered, func() { ID: manifestID2, ClusterResourceID: strings.ToLower(clusterResourceID), MaintenanceManifest: api.MaintenanceManifest{ - State: api.MaintenanceManifestStatePending, - RunBefore: 300, - RunAfter: 0, - MaintenanceSetID: "0000-0000-0001", + State: api.MaintenanceManifestStatePending, + RunBefore: 300, + RunAfter: 0, + MaintenanceTaskID: "0000-0000-0001", }, }) @@ -199,11 +199,11 @@ var _ = Describe("MIMO Actuator Service", Ordered, func() { ID: manifestID2, ClusterResourceID: strings.ToLower(clusterResourceID), MaintenanceManifest: api.MaintenanceManifest{ - State: api.MaintenanceManifestStateCompleted, - StatusText: "ok", - RunBefore: 300, - RunAfter: 0, - MaintenanceSetID: "0000-0000-0001", + State: api.MaintenanceManifestStateCompleted, + StatusText: "ok", + RunBefore: 300, + RunAfter: 0, + MaintenanceTaskID: "0000-0000-0001", }, }, ) @@ -214,7 +214,7 @@ var _ = Describe("MIMO Actuator Service", Ordered, func() { done := make(chan struct{}) svc.pollTime = time.Second - svc.SetMaintenanceSets(map[string]sets.MaintenanceSet{ + svc.SetMaintenanceTasks(map[string]tasks.MaintenanceTask{ "0000-0000-0001": func(th mimo.TaskContext, mmd *api.MaintenanceManifestDocument, oscd *api.OpenShiftClusterDocument) (api.MaintenanceManifestState, string) { svc.stopping.Store(true) return api.MaintenanceManifestStateCompleted, "ok" @@ -232,7 +232,7 @@ var _ = Describe("MIMO Actuator Service", Ordered, func() { done := make(chan struct{}) svc.pollTime = time.Second - svc.SetMaintenanceSets(map[string]sets.MaintenanceSet{ + svc.SetMaintenanceTasks(map[string]tasks.MaintenanceTask{ "0000-0000-0001": func(th mimo.TaskContext, mmd *api.MaintenanceManifestDocument, oscd *api.OpenShiftClusterDocument) (api.MaintenanceManifestState, string) { // ProvisioningState is in the full document, not just the // ClusterResourceID only as in the bucket worker diff --git a/pkg/mimo/tasks/cluster/acrtoken_checker.go b/pkg/mimo/steps/cluster/acrtoken_checker.go similarity index 100% rename from pkg/mimo/tasks/cluster/acrtoken_checker.go rename to pkg/mimo/steps/cluster/acrtoken_checker.go diff --git a/pkg/mimo/tasks/cluster/acrtoken_checker_test.go b/pkg/mimo/steps/cluster/acrtoken_checker_test.go similarity index 96% rename from pkg/mimo/tasks/cluster/acrtoken_checker_test.go rename to pkg/mimo/steps/cluster/acrtoken_checker_test.go index 76b455e8ce1..8a389150d6e 100644 --- a/pkg/mimo/tasks/cluster/acrtoken_checker_test.go +++ b/pkg/mimo/steps/cluster/acrtoken_checker_test.go @@ -84,7 +84,7 @@ func TestEnsureACRToken(t *testing.T) { { Name: intACR, Username: user, - IssueDate: &date.Time{Time: time.Now().AddDate(0, 0, -100)}, + IssueDate: &date.Time{Time: time.Now().UTC().AddDate(0, 0, -100)}, }, }, }, @@ -107,7 +107,7 @@ func TestEnsureACRToken(t *testing.T) { { Name: intACR, Username: user, - IssueDate: &date.Time{Time: time.Now().AddDate(0, 0, -50)}, + IssueDate: &date.Time{Time: time.Now().UTC().AddDate(0, 0, -50)}, }, }, }, diff --git a/pkg/mimo/tasks/cluster/apiserver_is_up.go b/pkg/mimo/steps/cluster/apiserver_is_up.go similarity index 100% rename from pkg/mimo/tasks/cluster/apiserver_is_up.go rename to pkg/mimo/steps/cluster/apiserver_is_up.go diff --git a/pkg/mimo/tasks/cluster/apiserver_is_up_test.go b/pkg/mimo/steps/cluster/apiserver_is_up_test.go similarity index 100% rename from pkg/mimo/tasks/cluster/apiserver_is_up_test.go rename to pkg/mimo/steps/cluster/apiserver_is_up_test.go diff --git a/pkg/mimo/tasks/cluster/operatorflags.go b/pkg/mimo/steps/cluster/operatorflags.go similarity index 100% rename from pkg/mimo/tasks/cluster/operatorflags.go rename to pkg/mimo/steps/cluster/operatorflags.go diff --git a/pkg/mimo/tasks/cluster/operatorflags_test.go b/pkg/mimo/steps/cluster/operatorflags_test.go similarity index 100% rename from pkg/mimo/tasks/cluster/operatorflags_test.go rename to pkg/mimo/steps/cluster/operatorflags_test.go diff --git a/pkg/mimo/tasks/cluster/tls.go b/pkg/mimo/steps/cluster/tls.go similarity index 100% rename from pkg/mimo/tasks/cluster/tls.go rename to pkg/mimo/steps/cluster/tls.go diff --git a/pkg/mimo/tasks/cluster/tls_test.go b/pkg/mimo/steps/cluster/tls_test.go similarity index 100% rename from pkg/mimo/tasks/cluster/tls_test.go rename to pkg/mimo/steps/cluster/tls_test.go diff --git a/pkg/mimo/tasks/example/task.go b/pkg/mimo/steps/example/task.go similarity index 100% rename from pkg/mimo/tasks/example/task.go rename to pkg/mimo/steps/example/task.go diff --git a/pkg/mimo/tasks/example/task_test.go b/pkg/mimo/steps/example/task_test.go similarity index 100% rename from pkg/mimo/tasks/example/task_test.go rename to pkg/mimo/steps/example/task_test.go diff --git a/pkg/mimo/sets/acrtoken.go b/pkg/mimo/tasks/acrtoken.go similarity index 87% rename from pkg/mimo/sets/acrtoken.go rename to pkg/mimo/tasks/acrtoken.go index 26eb2ad0037..1f30a9f9e35 100644 --- a/pkg/mimo/sets/acrtoken.go +++ b/pkg/mimo/tasks/acrtoken.go @@ -1,11 +1,11 @@ -package sets +package tasks // Copyright (c) Microsoft Corporation. // Licensed under the Apache License 2.0. import ( "github.com/Azure/ARO-RP/pkg/api" - "github.com/Azure/ARO-RP/pkg/mimo/tasks/cluster" + "github.com/Azure/ARO-RP/pkg/mimo/steps/cluster" "github.com/Azure/ARO-RP/pkg/util/mimo" "github.com/Azure/ARO-RP/pkg/util/steps" ) diff --git a/pkg/mimo/sets/interfaces.go b/pkg/mimo/tasks/interfaces.go similarity index 51% rename from pkg/mimo/sets/interfaces.go rename to pkg/mimo/tasks/interfaces.go index 154f09a2bcf..13b71d1f19d 100644 --- a/pkg/mimo/sets/interfaces.go +++ b/pkg/mimo/tasks/interfaces.go @@ -1,4 +1,4 @@ -package sets +package tasks // Copyright (c) Microsoft Corporation. // Licensed under the Apache License 2.0. @@ -8,4 +8,4 @@ import ( "github.com/Azure/ARO-RP/pkg/util/mimo" ) -type MaintenanceSet func(mimo.TaskContext, *api.MaintenanceManifestDocument, *api.OpenShiftClusterDocument) (api.MaintenanceManifestState, string) +type MaintenanceTask func(mimo.TaskContext, *api.MaintenanceManifestDocument, *api.OpenShiftClusterDocument) (api.MaintenanceManifestState, string) diff --git a/pkg/mimo/sets/operatorflags.go b/pkg/mimo/tasks/operatorflags.go similarity index 88% rename from pkg/mimo/sets/operatorflags.go rename to pkg/mimo/tasks/operatorflags.go index bb2167acef2..fc256a5d0fa 100644 --- a/pkg/mimo/sets/operatorflags.go +++ b/pkg/mimo/tasks/operatorflags.go @@ -1,11 +1,11 @@ -package sets +package tasks // Copyright (c) Microsoft Corporation. // Licensed under the Apache License 2.0. import ( "github.com/Azure/ARO-RP/pkg/api" - "github.com/Azure/ARO-RP/pkg/mimo/tasks/cluster" + "github.com/Azure/ARO-RP/pkg/mimo/steps/cluster" "github.com/Azure/ARO-RP/pkg/util/mimo" "github.com/Azure/ARO-RP/pkg/util/steps" ) diff --git a/pkg/mimo/sets/sets.go b/pkg/mimo/tasks/sets.go similarity index 92% rename from pkg/mimo/sets/sets.go rename to pkg/mimo/tasks/sets.go index a6d1e56f664..b38fca9e218 100644 --- a/pkg/mimo/sets/sets.go +++ b/pkg/mimo/tasks/sets.go @@ -1,4 +1,4 @@ -package sets +package tasks // Copyright (c) Microsoft Corporation. // Licensed under the Apache License 2.0. @@ -15,7 +15,7 @@ import ( const DEFAULT_POLL_TIME = time.Second * 10 const DEFAULT_TIMEOUT_DURATION = time.Minute * 20 -var DEFAULT_MAINTENANCE_SETS = map[string]MaintenanceSet{ +var DEFAULT_MAINTENANCE_SETS = map[string]MaintenanceTask{ mimo.TLS_CERT_ROTATION_ID: TLSCertRotation, mimo.ACR_TOKEN_CHECKER_ID: ACRTokenChecker, mimo.OPERATOR_FLAGS_UPDATE_ID: UpdateOperatorFlags, diff --git a/pkg/mimo/sets/tls_cert_rotate.go b/pkg/mimo/tasks/tls_cert_rotate.go similarity index 90% rename from pkg/mimo/sets/tls_cert_rotate.go rename to pkg/mimo/tasks/tls_cert_rotate.go index 41e24d9de5e..61e536dbacd 100644 --- a/pkg/mimo/sets/tls_cert_rotate.go +++ b/pkg/mimo/tasks/tls_cert_rotate.go @@ -1,11 +1,11 @@ -package sets +package tasks // Copyright (c) Microsoft Corporation. // Licensed under the Apache License 2.0. import ( "github.com/Azure/ARO-RP/pkg/api" - "github.com/Azure/ARO-RP/pkg/mimo/tasks/cluster" + "github.com/Azure/ARO-RP/pkg/mimo/steps/cluster" "github.com/Azure/ARO-RP/pkg/util/mimo" "github.com/Azure/ARO-RP/pkg/util/steps" ) diff --git a/test/e2e/mimo_actuator.go b/test/e2e/mimo_actuator.go index 9cff1d54241..053c9beae32 100644 --- a/test/e2e/mimo_actuator.go +++ b/test/e2e/mimo_actuator.go @@ -81,7 +81,7 @@ var _ = Describe("MIMO Actuator E2E Testing", func() { resp, err = adminRequest(ctx, http.MethodPut, "/admin"+clusterResourceID+"/maintenancemanifests", url.Values{}, true, &admin.MaintenanceManifest{ - MaintenanceSetID: mimo.OPERATOR_FLAGS_UPDATE_ID, + MaintenanceTaskID: mimo.OPERATOR_FLAGS_UPDATE_ID, }, &out, logOnError(log)...) Expect(err).NotTo(HaveOccurred()) Expect(resp.StatusCode).To(Equal(http.StatusCreated)) From efc24b42118f6900b603dc4f48a1f0886f4794d2 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Thu, 10 Oct 2024 13:31:17 +1100 Subject: [PATCH 66/67] go mod tidy --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 3221751d4c2..2d967e4cf63 100644 --- a/go.mod +++ b/go.mod @@ -77,6 +77,7 @@ require ( github.com/vincent-petithory/dataurl v1.0.0 go.uber.org/mock v0.4.0 golang.org/x/crypto v0.28.0 + golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 golang.org/x/net v0.30.0 golang.org/x/oauth2 v0.18.0 golang.org/x/sync v0.8.0 @@ -260,7 +261,6 @@ require ( go.opentelemetry.io/otel/metric v1.22.0 // indirect go.opentelemetry.io/otel/trace v1.22.0 // indirect go.starlark.net v0.0.0-20220328144851-d1966c6b9fcd // indirect - golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect golang.org/x/mod v0.17.0 // indirect golang.org/x/sys v0.26.0 // indirect golang.org/x/term v0.25.0 // indirect From 3176e49ef580c0c37b6153420174ab4e7f499a0b Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Fri, 11 Oct 2024 15:51:52 +1100 Subject: [PATCH 67/67] add some documentation --- docs/mimo.md | 1 - docs/mimo/README.md | 22 ++++++++++++++++++++++ docs/mimo/actuator.md | 30 ++++++++++++++++++++++++++++++ docs/mimo/admin-api.md | 0 docs/mimo/scheduler.md | 3 +++ docs/mimo/writing-tasks.md | 1 + 6 files changed, 56 insertions(+), 1 deletion(-) delete mode 100644 docs/mimo.md create mode 100644 docs/mimo/README.md create mode 100644 docs/mimo/actuator.md create mode 100644 docs/mimo/admin-api.md create mode 100644 docs/mimo/scheduler.md create mode 100644 docs/mimo/writing-tasks.md diff --git a/docs/mimo.md b/docs/mimo.md deleted file mode 100644 index bc4f21edd3e..00000000000 --- a/docs/mimo.md +++ /dev/null @@ -1 +0,0 @@ -# Managed Infrastructure Maintenance Operator diff --git a/docs/mimo/README.md b/docs/mimo/README.md new file mode 100644 index 00000000000..8d03ab961c4 --- /dev/null +++ b/docs/mimo/README.md @@ -0,0 +1,22 @@ +# MIMO Documentation + +The Managed Infrastructure Maintenance Operator, or MIMO, is a component of the Azure Red Hat OpenShift Resource Provider (ARO-RP) which is responsible for automated maintenance of clusters provisioned by the platform. +MIMO specifically focuses on "managed infrastructure", the parts of ARO that are deployed and maintained by the RP and ARO Operator instead of by OCP (in-cluster) or Hive (out-of-cluster). + +MIMO consists of two main components, the [Actuator](./actuator.md) and the [Scheduler](./scheduler.md). It is primarily interfaced with via the [Admin API](./admin-api.md). + +## A Primer On MIMO + +The smallest thing that you can tell MIMO to run is a **Task** (see [`pkg/mimo/tasks/`](../../pkg/mimo/tasks/)). +A Task is composed of reusable **Steps** (see [`pkg/mimo/steps/`](../../pkg/mimo/steps/)), reusing the framework utilised by AdminUpdate/Update/Install methods in `pkg/cluster/`. +A Task only runs in the scope of a singular cluster. +These steps are run in sequence and can return either **Terminal** errors (causing the ran Task to fail and not be retried) or **Transient** errors (which indicates that the Task can be retried later). + +Tasks are executed by the **Actuator** by way of creation of a **Maintenance Manifest**. +This Manifest is created with the cluster ID (which is elided from the cluster-scoped Admin APIs), the Task ID (which is currently a UUID), and optional priority, "start after", and "start before" times which are filled in with defaults if not provided. +The Actuator will treat these Maintenance Manifests as a work queue, taking ones which are past their "start after" time and executing them in order of earliest start-after and priority. +After running each, a state will be written into the Manifest (with optional free-form status text) with the result of the ran Task. +Manifests past their start-before times are marked as having a "timed out" state and not ran. + +Currently, Manifests are created by the Admin API. +In the future, the Scheduler will create some these Manifests depending on cluster state/version and wall-clock time, providing the ability to perform tasks like rotations of secrets autonomously. diff --git a/docs/mimo/actuator.md b/docs/mimo/actuator.md new file mode 100644 index 00000000000..5950662890e --- /dev/null +++ b/docs/mimo/actuator.md @@ -0,0 +1,30 @@ +# Managed Infrastructure Maintenance Operator: Actuator + +The Actuator is the MIMO component that performs execution of tasks. +The process of running tasks looks like this: + +```mermaid +graph TD; + START((Start))-->QUERY; + QUERY[Fetch all State = Pending] -->SORT; + SORT[Sort tasks by RUNAFTER and PRIORITY]-->ITERATE[Iterate over tasks]; + ITERATE-- Per Task -->ISEXPIRED; + subgraph PerTask[ ] + ISEXPIRED{{Is RUNBEFORE > now?}}-- Yes --> STATETIMEDOUT([State = TimedOut]) --> CONTINUE[Continue]; + ISEXPIRED-- No --> DEQUEUECLUSTER; + DEQUEUECLUSTER[Claim lease on OpenShiftClusterDocument] --> DEQUEUE; + DEQUEUE[Actuator dequeues task]--> ISRETRYLIMIT; + ISRETRYLIMIT{{Have we retried the task too many times?}} -- Yes --> STATETIMEDOUT; + ISRETRYLIMIT -- No -->STATEINPROGRESS; + STATEINPROGRESS([State = InProgress]) -->RUN[[Task is run]]; + RUN -- Success --> SUCCESS + RUN-- Terminal Error-->TERMINALERROR; + RUN-- Transient Error-->TRANSIENTERROR; + SUCCESS([State = Completed])-->DELEASECLUSTER + TERMINALERROR([State = Failed])-->DELEASECLUSTER; + TRANSIENTERROR([State = Pending])-->DELEASECLUSTER; + DELEASECLUSTER[Release Lease on OpenShiftClusterDocument] -->CONTINUE; + end + CONTINUE-->ITERATE; + ITERATE-- Finished -->END; +``` diff --git a/docs/mimo/admin-api.md b/docs/mimo/admin-api.md new file mode 100644 index 00000000000..e69de29bb2d diff --git a/docs/mimo/scheduler.md b/docs/mimo/scheduler.md new file mode 100644 index 00000000000..8a457798a25 --- /dev/null +++ b/docs/mimo/scheduler.md @@ -0,0 +1,3 @@ +# MIMO Scheduler + +The MIMO Scheduler is a planned component, but is not yet implemented. diff --git a/docs/mimo/writing-tasks.md b/docs/mimo/writing-tasks.md new file mode 100644 index 00000000000..91861535fd2 --- /dev/null +++ b/docs/mimo/writing-tasks.md @@ -0,0 +1 @@ +# Writing MIMO Tasks