From cd34ad1572803a1b0898b7c3051d0fb787780f88 Mon Sep 17 00:00:00 2001 From: cadenmarchese Date: Fri, 31 May 2024 09:09:53 -0400 Subject: [PATCH] new cmd to populate role sets in prod --- cmd/aro/const.go | 13 +-- cmd/aro/main.go | 4 + cmd/aro/update_role_sets.go | 190 ++++++++++++++++++++++++++++++++++++ pkg/util/cluster/cluster.go | 61 +++++++----- 4 files changed, 236 insertions(+), 32 deletions(-) create mode 100644 cmd/aro/update_role_sets.go diff --git a/cmd/aro/const.go b/cmd/aro/const.go index 65d924c316d..499d14e677e 100644 --- a/cmd/aro/const.go +++ b/cmd/aro/const.go @@ -4,10 +4,11 @@ package main // Licensed under the Apache License 2.0. const ( - envDatabaseName = "DATABASE_NAME" - envDatabaseAccountName = "DATABASE_ACCOUNT_NAME" - envKeyVaultPrefix = "KEYVAULT_PREFIX" - envDBTokenUrl = "DBTOKEN_URL" - envOpenShiftVersions = "OPENSHIFT_VERSIONS" - envInstallerImageDigests = "INSTALLER_IMAGE_DIGESTS" + envDatabaseName = "DATABASE_NAME" + envDatabaseAccountName = "DATABASE_ACCOUNT_NAME" + envKeyVaultPrefix = "KEYVAULT_PREFIX" + envDBTokenUrl = "DBTOKEN_URL" + envOpenShiftVersions = "OPENSHIFT_VERSIONS" + envInstallerImageDigests = "INSTALLER_IMAGE_DIGESTS" + envPlatformWorkloadIdentityRoleSets = "PLATFORM_WORKLOAD_IDENTITY_ROLE_SETS" ) diff --git a/cmd/aro/main.go b/cmd/aro/main.go index 1bdd4b32c4d..a332ee46e15 100644 --- a/cmd/aro/main.go +++ b/cmd/aro/main.go @@ -29,6 +29,7 @@ func usage() { fmt.Fprintf(flag.CommandLine.Output(), " %s rp\n", os.Args[0]) fmt.Fprintf(flag.CommandLine.Output(), " %s operator {master,worker}\n", os.Args[0]) fmt.Fprintf(flag.CommandLine.Output(), " %s update-versions\n", os.Args[0]) + fmt.Fprintf(flag.CommandLine.Output(), " %s update-role-sets\n", os.Args[0]) flag.PrintDefaults() } @@ -75,6 +76,9 @@ func main() { case "update-versions": checkArgs(1) err = updateOCPVersions(ctx, log) + case "update-role-sets": + checkArgs(1) + err = updatePlatformWorkloadIdentityRoleSets(ctx, log) default: usage() os.Exit(2) diff --git a/cmd/aro/update_role_sets.go b/cmd/aro/update_role_sets.go new file mode 100644 index 00000000000..7add0366b4c --- /dev/null +++ b/cmd/aro/update_role_sets.go @@ -0,0 +1,190 @@ +package main + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "fmt" + "os" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/database" + "github.com/Azure/ARO-RP/pkg/env" + "github.com/Azure/ARO-RP/pkg/metrics/statsd" + "github.com/Azure/ARO-RP/pkg/util/encryption" + "github.com/Azure/ARO-RP/pkg/util/keyvault" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm/policy" + "github.com/sirupsen/logrus" +) + +// 1 - Get env data from agent VMs (with getEnvironemntData) and write to types created in step 1 +func getPlatformWorkloadIdentityRoleSets() ([]api.PlatformWorkloadIdentityRoleSet, error) { + const envKey = envPlatformWorkloadIdentityRoleSets + var PlatformWorkloadIdentityRoleSet []api.PlatformWorkloadIdentityRoleSet + + // marshall env data into type []api.PlatformWorkloadIdentityRoleSet + if err := getEnvironmentData(envKey, PlatformWorkloadIdentityRoleSet); err != nil { + return nil, err + } + + return PlatformWorkloadIdentityRoleSet, nil +} + +func getRoleSetFromEnv() ([]api.PlatformWorkloadIdentityRoleSet, error) { + roleSet, err := getPlatformWorkloadIdentityRoleSets() + if err != nil { + return nil, err + } + + finalRoleSet := []api.PlatformWorkloadIdentityRoleSet{} + finalRoleSet = append(finalRoleSet, roleSet...) + + return finalRoleSet, nil +} + +// 2 - Get the existing role set documents, if existing +// Mostly copied from update_ocp_versions.go +func getPlatformWorkloadIdentityRoleSetDatabase(ctx context.Context, log *logrus.Entry) (database.PlatformWorkloadIdentityRoleSets, error) { + _env, err := env.NewCore(ctx, log, env.COMPONENT_UPDATE_OCP_VERSIONS) + if err != nil { + return nil, err + } + + msiToken, err := _env.NewMSITokenCredential() + if err != nil { + return nil, fmt.Errorf("MSI Authorizer failed with: %s", err.Error()) + } + + msiKVAuthorizer, err := _env.NewMSIAuthorizer(_env.Environment().KeyVaultScope) + if err != nil { + return nil, fmt.Errorf("MSI KeyVault Authorizer failed with: %s", err.Error()) + } + + m := statsd.New(ctx, log.WithField("component", "update-role-sets"), _env, os.Getenv("MDM_ACCOUNT"), os.Getenv("MDM_NAMESPACE"), os.Getenv("MDM_STATSD_SOCKET")) + + keyVaultPrefix := os.Getenv(envKeyVaultPrefix) + serviceKeyvaultURI := keyvault.URI(_env, env.ServiceKeyvaultSuffix, keyVaultPrefix) + serviceKeyvault := keyvault.NewManager(msiKVAuthorizer, serviceKeyvaultURI) + + aead, err := encryption.NewMulti(ctx, serviceKeyvault, env.EncryptionSecretV2Name, env.EncryptionSecretName) + if err != nil { + return nil, err + } + + if err := env.ValidateVars(envDatabaseAccountName); err != nil { + return nil, err + } + dbAccountName := os.Getenv(envDatabaseAccountName) + clientOptions := &policy.ClientOptions{ + ClientOptions: _env.Environment().ManagedIdentityCredentialOptions().ClientOptions, + } + + logrusEntry := log.WithField("component", "database") + dbAuthorizer, err := database.NewMasterKeyAuthorizer(ctx, logrusEntry, msiToken, clientOptions, _env.SubscriptionID(), _env.ResourceGroup(), dbAccountName) + if err != nil { + return nil, err + } + + dbc, err := database.NewDatabaseClient(log.WithField("component", "database"), _env, dbAuthorizer, m, aead, dbAccountName) + if err != nil { + return nil, err + } + + dbName, err := DBName(_env.IsLocalDevelopmentMode()) + if err != nil { + return nil, err + } + dbPlatformWorkloadIdentityRoleSetsDocument, err := database.NewPlatformWorkloadIdentityRoleSets(ctx, dbc, dbName) + if err != nil { + return nil, err + } + + return dbPlatformWorkloadIdentityRoleSetsDocument, nil +} + +// 3 - Put/patch the new role sets to the doc, overwriting whatever is there for that version, or adding if new +// Mostly copied from update_ocp_versions.go +func updatePlatformWorkloadIdentityRoleSetsInCosmosDB(ctx context.Context, dbPlatformWorkloadIdentityRoleSets database.PlatformWorkloadIdentityRoleSets, log *logrus.Entry) error { + dbPlatformWorkloadIdentityRoleSet, err := dbPlatformWorkloadIdentityRoleSets.ListAll(ctx) + if err != nil { + return nil + } + + incomingRoleSet, err := getRoleSetFromEnv() + if err != nil { + return err + } + + newRoleSets := make(map[string]api.PlatformWorkloadIdentityRoleSet) + for _, doc := range incomingRoleSet { + newRoleSets[doc.Properties.OpenShiftVersion] = doc + } + + for _, doc := range dbPlatformWorkloadIdentityRoleSet.PlatformWorkloadIdentityRoleSetDocuments { + existing, found := newRoleSets[doc.PlatformWorkloadIdentityRoleSet.Properties.OpenShiftVersion] + if found { + log.Printf("Found Version %q, patching", existing.Properties.OpenShiftVersion) + _, err := dbPlatformWorkloadIdentityRoleSets.Patch(ctx, doc.ID, func(inFlightDoc *api.PlatformWorkloadIdentityRoleSetDocument) error { + inFlightDoc.PlatformWorkloadIdentityRoleSet = &existing + return nil + }) + if err != nil { + return err + } + log.Printf("Version %q found", existing.Properties.OpenShiftVersion) + delete(newRoleSets, existing.Properties.OpenShiftVersion) + continue + } + + log.Printf("Version %q not found, deleting", doc.PlatformWorkloadIdentityRoleSet.Properties.OpenShiftVersion) + // Delete via changefeed + _, err := dbPlatformWorkloadIdentityRoleSets.Patch(ctx, doc.ID, + func(d *api.PlatformWorkloadIdentityRoleSetDocument) error { + d.PlatformWorkloadIdentityRoleSet.Deleting = true + d.TTL = 60 + return nil + }) + if err != nil { + return err + } + } + + for _, doc := range newRoleSets { + log.Printf("Version %q not found in database, creating", doc.Properties.OpenShiftVersion) + newDoc := api.PlatformWorkloadIdentityRoleSetDocument{ + ID: dbPlatformWorkloadIdentityRoleSets.NewUUID(), + PlatformWorkloadIdentityRoleSet: &doc, + } + _, err := dbPlatformWorkloadIdentityRoleSets.Create(ctx, &newDoc) + if err != nil { + return err + } + } + + return nil +} + +func updatePlatformWorkloadIdentityRoleSets(ctx context.Context, log *logrus.Entry) error { + if err := env.ValidateVars("PLATFORM_WORKLOAD_IDENTITY_ROLES", "OPENSHIFT_VERSION", "OPERATOR_NAME", "ROLE_DEFINITION_NAME", "ROLE_DEFINITION_ID", "SERVICE_ACCOUNTS"); err != nil { + return err + } + + if !env.IsLocalDevelopmentMode() { + if err := env.ValidateVars("MDM_ACCOUNT", "MDM_NAMESPACE"); err != nil { + return err + } + } + + dbRoleSets, err := getPlatformWorkloadIdentityRoleSetDatabase(ctx, log) + if err != nil { + return err + } + + err = updatePlatformWorkloadIdentityRoleSetsInCosmosDB(ctx, dbRoleSets, log) + if err != nil { + return err + } + + return nil +} diff --git a/pkg/util/cluster/cluster.go b/pkg/util/cluster/cluster.go index 828c0ee0121..bb67cb5a746 100644 --- a/pkg/util/cluster/cluster.go +++ b/pkg/util/cluster/cluster.go @@ -41,6 +41,9 @@ import ( "github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/authorization" "github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/features" "github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/network" + redhatopenshift20200430 "github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/redhatopenshift/2020-04-30/redhatopenshift" + redhatopenshift20210901preview "github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/redhatopenshift/2021-09-01-preview/redhatopenshift" + redhatopenshift20220401 "github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/redhatopenshift/2022-04-01/redhatopenshift" redhatopenshift20230904 "github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/redhatopenshift/2023-09-04/redhatopenshift" utilgraph "github.com/Azure/ARO-RP/pkg/util/graph" "github.com/Azure/ARO-RP/pkg/util/rbac" @@ -55,17 +58,20 @@ type Cluster struct { ci bool ciParentVnet string - spGraphClient *utilgraph.GraphServiceClient - deployments features.DeploymentsClient - groups features.ResourceGroupsClient - openshiftclusters redhatopenshift20230904.OpenShiftClustersClient - securitygroups network.SecurityGroupsClient - subnets network.SubnetsClient - routetables network.RouteTablesClient - roleassignments authorization.RoleAssignmentsClient - peerings network.VirtualNetworkPeeringsClient - ciParentVnetPeerings network.VirtualNetworkPeeringsClient - vaultsClient armkeyvault.VaultsClient + spGraphClient *utilgraph.GraphServiceClient + deployments features.DeploymentsClient + groups features.ResourceGroupsClient + openshiftclustersv20200430 redhatopenshift20200430.OpenShiftClustersClient + openshiftclustersv20210901preview redhatopenshift20210901preview.OpenShiftClustersClient + openshiftclustersv20220401 redhatopenshift20220401.OpenShiftClustersClient + openshiftclustersv20230904 redhatopenshift20230904.OpenShiftClustersClient + securitygroups network.SecurityGroupsClient + subnets network.SubnetsClient + routetables network.RouteTablesClient + roleassignments authorization.RoleAssignmentsClient + peerings network.VirtualNetworkPeeringsClient + ciParentVnetPeerings network.VirtualNetworkPeeringsClient + vaultsClient armkeyvault.VaultsClient } func New(log *logrus.Entry, environment env.Core, ci bool) (*Cluster, error) { @@ -106,16 +112,19 @@ func New(log *logrus.Entry, environment env.Core, ci bool) (*Cluster, error) { env: environment, ci: ci, - spGraphClient: spGraphClient, - deployments: features.NewDeploymentsClient(environment.Environment(), environment.SubscriptionID(), authorizer), - groups: features.NewResourceGroupsClient(environment.Environment(), environment.SubscriptionID(), authorizer), - openshiftclusters: redhatopenshift20230904.NewOpenShiftClustersClient(environment.Environment(), environment.SubscriptionID(), authorizer), - securitygroups: network.NewSecurityGroupsClient(environment.Environment(), environment.SubscriptionID(), authorizer), - subnets: network.NewSubnetsClient(environment.Environment(), environment.SubscriptionID(), authorizer), - routetables: network.NewRouteTablesClient(environment.Environment(), environment.SubscriptionID(), authorizer), - roleassignments: authorization.NewRoleAssignmentsClient(environment.Environment(), environment.SubscriptionID(), authorizer), - peerings: network.NewVirtualNetworkPeeringsClient(environment.Environment(), environment.SubscriptionID(), authorizer), - vaultsClient: vaultClient, + spGraphClient: spGraphClient, + deployments: features.NewDeploymentsClient(environment.Environment(), environment.SubscriptionID(), authorizer), + groups: features.NewResourceGroupsClient(environment.Environment(), environment.SubscriptionID(), authorizer), + openshiftclustersv20200430: redhatopenshift20200430.NewOpenShiftClustersClient(environment.Environment(), environment.SubscriptionID(), authorizer), + openshiftclustersv20210901preview: redhatopenshift20210901preview.NewOpenShiftClustersClient(environment.Environment(), environment.SubscriptionID(), authorizer), + openshiftclustersv20220401: redhatopenshift20220401.NewOpenShiftClustersClient(environment.Environment(), environment.SubscriptionID(), authorizer), + openshiftclustersv20230904: redhatopenshift20230904.NewOpenShiftClustersClient(environment.Environment(), environment.SubscriptionID(), authorizer), + securitygroups: network.NewSecurityGroupsClient(environment.Environment(), environment.SubscriptionID(), authorizer), + subnets: network.NewSubnetsClient(environment.Environment(), environment.SubscriptionID(), authorizer), + routetables: network.NewRouteTablesClient(environment.Environment(), environment.SubscriptionID(), authorizer), + roleassignments: authorization.NewRoleAssignmentsClient(environment.Environment(), environment.SubscriptionID(), authorizer), + peerings: network.NewVirtualNetworkPeeringsClient(environment.Environment(), environment.SubscriptionID(), authorizer), + vaultsClient: vaultClient, } if ci && env.IsLocalDevelopmentMode() { @@ -161,7 +170,7 @@ func (c *Cluster) DeleteApp(ctx context.Context) error { } func (c *Cluster) Create(ctx context.Context, vnetResourceGroup, clusterName string, osClusterVersion string) error { - clusterGet, err := c.openshiftclusters.Get(ctx, vnetResourceGroup, clusterName) + clusterGet, err := c.openshiftclustersv20230904.Get(ctx, vnetResourceGroup, clusterName) if err == nil { if clusterGet.ProvisioningState == mgmtredhatopenshift20230904.Failed { return fmt.Errorf("cluster exists and is in failed provisioning state, please delete and retry") @@ -487,7 +496,7 @@ func (c *Cluster) createCluster(ctx context.Context, vnetResourceGroup, clusterN return err } - return c.openshiftclusters.CreateOrUpdateAndWait(ctx, vnetResourceGroup, clusterName, ocExt) + return c.openshiftclustersv20230904.CreateOrUpdateAndWait(ctx, vnetResourceGroup, clusterName, ocExt) } func (c *Cluster) registerSubscription(ctx context.Context) error { @@ -578,7 +587,7 @@ func (c *Cluster) insertPlatformWorkloadIdentityRoleSetsIntoCosmosdb() error { return err } - req, err := http.NewRequest(http.MethodPut, "https://localhost:8443/platformworkloadidentityrolesets/", bytes.NewReader(b)) + req, err := http.NewRequest(http.MethodPut, "https://localhost:8443/admin/platformworkloadidentityrolesets/", bytes.NewReader(b)) if err != nil { return err } @@ -637,7 +646,7 @@ func (c *Cluster) fixupNSGs(ctx context.Context, vnetResourceGroup, clusterName func (c *Cluster) deleteRoleAssignments(ctx context.Context, vnetResourceGroup, clusterName string) error { c.log.Print("deleting role assignments") - oc, err := c.openshiftclusters.Get(ctx, vnetResourceGroup, clusterName) + oc, err := c.openshiftclustersv20200430.Get(ctx, vnetResourceGroup, clusterName) if err != nil { return fmt.Errorf("error getting cluster document: %w", err) } @@ -673,7 +682,7 @@ func (c *Cluster) deleteRoleAssignments(ctx context.Context, vnetResourceGroup, func (c *Cluster) deleteCluster(ctx context.Context, resourceGroup, clusterName string) error { c.log.Printf("deleting cluster %s", clusterName) - if err := c.openshiftclusters.DeleteAndWait(ctx, resourceGroup, clusterName); err != nil { + if err := c.openshiftclustersv20200430.DeleteAndWait(ctx, resourceGroup, clusterName); err != nil { return fmt.Errorf("error deleting cluster %s: %w", clusterName, err) } return nil