[oadp-1.2] OADP-1057: AWS STS Creds support for internal image backup (…

…#202) * add support for aws sts creds for registry Signed-off-by: Tiger Kaovilai <[email protected]> * remove image-spec replace breaking build * OADP-1057: STS creds support for internal image backup Signed-off-by: Tiger Kaovilai <[email protected]> * Remove non-working code for backing up completed build pods No longer needed after by vmware-tanzu/velero#6189 or vmware-tanzu/velero#6188 which fix vmware-tanzu/velero#6182 Signed-off-by: Tiger Kaovilai <[email protected]> remove pod backup plugin from main.go * Resolve Dockerfile perm issues running go build by removing specified UID when copying Resolves ``` > [builder 6/6] RUN go build -installsuffix "static" -tags "containers_image_ostree_stub exclude_graphdriver_devicemapper exclude_graphdriver_btrfs containers_image_openpgp exclude_graphdriver_overlay include_gcs include_oss" -o _output/velero-plugins ./velero-plugins: #0 4.584 error obtaining VCS status: exit status 128 #0 4.584 Use -buildvcs=false to disable VCS stamping. ``` Signed-off-by: Tiger Kaovilai <[email protected]> * update go.mod to openshift/velero #276 * add sts cred secret handling Signed-off-by: Tiger Kaovilai <[email protected]> * Pass secret path instead of web_identity_token_file Signed-off-by: Tiger Kaovilai <[email protected]> * Update go.mod to openshift/oadp-1.2 --------- Signed-off-by: Tiger Kaovilai <[email protected]> Co-authored-by: Tiger Kaovilai <[email protected]>
openshift · Aug 22, 2023 · 4a17e00 · 4a17e00
1 parent de9e2cd
commit 4a17e00
Show file tree

Hide file tree

Showing 11 changed files with 843 additions and 1,239 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -16,9 +16,9 @@ ENV GOPATH=$APP_ROOT
 ENV BUILDTAGS containers_image_ostree_stub exclude_graphdriver_devicemapper exclude_graphdriver_btrfs containers_image_openpgp exclude_graphdriver_overlay include_gcs include_oss
 ENV BIN velero-plugins
 WORKDIR $APP_ROOT/src/github.com/konveyor/openshift-velero-plugin
-COPY --chown=1001 go.mod go.sum $APP_ROOT/src/github.com/konveyor/openshift-velero-plugin/
+COPY go.mod go.sum $APP_ROOT/src/github.com/konveyor/openshift-velero-plugin/
 RUN go mod download
-COPY --chown=1001 . $APP_ROOT/src/github.com/konveyor/openshift-velero-plugin
+COPY . $APP_ROOT/src/github.com/konveyor/openshift-velero-plugin
 RUN go build -installsuffix "static" -tags "$BUILDTAGS" -o _output/$BIN ./$BIN
 
 FROM registry.access.redhat.com/ubi8-minimal

diff --git a/go.mod b/go.mod
diff --git a/go.sum b/go.sum
diff --git a/velero-plugins/clients/clients.go b/velero-plugins/clients/clients.go
@@ -34,6 +34,10 @@ var buildClientError error
 
 var inClusterConfig *rest.Config
 
+func SetInClusterConfig(config *rest.Config) {
+	inClusterConfig = config
+}
+
 func GetInClusterConfig() (*rest.Config, error) {
 	if inClusterConfig != nil {
 		return inClusterConfig, nil

diff --git a/velero-plugins/imagestream/registry.go b/velero-plugins/imagestream/registry.go
@@ -2,6 +2,7 @@ package imagestream
 
 import (
 	"errors"
+	"fmt"
 
 	"github.com/openshift/oadp-operator/pkg/credentials"
 
@@ -13,14 +14,15 @@ import (
 // Registry Env var keys
 const (
 	// AWS registry env vars
-	RegistryStorageEnvVarKey                 = "REGISTRY_STORAGE"
-	RegistryStorageS3AccesskeyEnvVarKey      = "REGISTRY_STORAGE_S3_ACCESSKEY"
-	RegistryStorageS3BucketEnvVarKey         = "REGISTRY_STORAGE_S3_BUCKET"
-	RegistryStorageS3RegionEnvVarKey         = "REGISTRY_STORAGE_S3_REGION"
-	RegistryStorageS3SecretkeyEnvVarKey      = "REGISTRY_STORAGE_S3_SECRETKEY"
-	RegistryStorageS3RegionendpointEnvVarKey = "REGISTRY_STORAGE_S3_REGIONENDPOINT"
-	RegistryStorageS3RootdirectoryEnvVarKey  = "REGISTRY_STORAGE_S3_ROOTDIRECTORY"
-	RegistryStorageS3SkipverifyEnvVarKey     = "REGISTRY_STORAGE_S3_SKIPVERIFY"
+	RegistryStorageEnvVarKey                        = "REGISTRY_STORAGE"
+	RegistryStorageS3AccesskeyEnvVarKey             = "REGISTRY_STORAGE_S3_ACCESSKEY"
+	RegistryStorageS3BucketEnvVarKey                = "REGISTRY_STORAGE_S3_BUCKET"
+	RegistryStorageS3RegionEnvVarKey                = "REGISTRY_STORAGE_S3_REGION"
+	RegistryStorageS3SecretkeyEnvVarKey             = "REGISTRY_STORAGE_S3_SECRETKEY"
+	RegistryStorageS3CredentialsConfigPathEnvVarKey = "REGISTRY_STORAGE_S3_CREDENTIALSCONFIGPATH"
+	RegistryStorageS3RegionendpointEnvVarKey        = "REGISTRY_STORAGE_S3_REGIONENDPOINT"
+	RegistryStorageS3RootdirectoryEnvVarKey         = "REGISTRY_STORAGE_S3_ROOTDIRECTORY"
+	RegistryStorageS3SkipverifyEnvVarKey            = "REGISTRY_STORAGE_S3_SKIPVERIFY"
 	// Azure registry env vars
 	RegistryStorageAzureContainerEnvVarKey       = "REGISTRY_STORAGE_AZURE_CONTAINER"
 	RegistryStorageAzureAccountnameEnvVarKey     = "REGISTRY_STORAGE_AZURE_ACCOUNTNAME"
@@ -35,7 +37,7 @@ const (
 	RegistryStorageGCSRootdirectory = "REGISTRY_STORAGE_GCS_ROOTDIRECTORY"
 )
 
-// provider specific object storage
+// provider specific object storage config
 const (
 	S3                    = "s3"
 	Azure                 = "azure"
@@ -50,40 +52,23 @@ const (
 	InsecureSkipTLSVerify = "insecureSkipTLSVerify"
 	StorageAccount        = "storageAccount"
 	ResourceGroup         = "resourceGroup"
+	enableSharedConfig    = "enableSharedConfig"
 )
 
+// secret data keys
+const (
+	webIdentityTokenFile = "web_identity_token_file"
+)
+
+// velero constants
+const (
+	// https://github.com/vmware-tanzu/velero/blob/5afe837f76aea4dd59b1bf2792e7802d4966f0a7/pkg/cmd/server/server.go#L110
+	defaultCredentialsDirectory = "/tmp/credentials"
+)
+
+// TODO: remove this map and just define them in each function
 // creating skeleton for provider based env var map
 var cloudProviderEnvVarMap = map[string][]corev1.EnvVar{
-	"aws": {
-		{
-			Name:  RegistryStorageEnvVarKey,
-			Value: S3,
-		},
-		{
-			Name:  RegistryStorageS3AccesskeyEnvVarKey,
-			Value: "",
-		},
-		{
-			Name:  RegistryStorageS3BucketEnvVarKey,
-			Value: "",
-		},
-		{
-			Name:  RegistryStorageS3RegionEnvVarKey,
-			Value: "",
-		},
-		{
-			Name:  RegistryStorageS3SecretkeyEnvVarKey,
-			Value: "",
-		},
-		{
-			Name:  RegistryStorageS3RegionendpointEnvVarKey,
-			Value: "",
-		},
-		{
-			Name:  RegistryStorageS3SkipverifyEnvVarKey,
-			Value: "",
-		},
-	},
 	"azure": {
 		{
 			Name:  RegistryStorageEnvVarKey,
@@ -149,72 +134,93 @@ type azureCredentials struct {
 }
 
 func getRegistryEnvVars(bsl *velerov1.BackupStorageLocation) ([]corev1.EnvVar, error) {
-	envVar := []corev1.EnvVar{}
+	var envVars []corev1.EnvVar
 	provider := bsl.Spec.Provider
 	var err error
 	switch provider {
 	case AWSProvider:
-		envVar, err = getAWSRegistryEnvVars(bsl, cloudProviderEnvVarMap[AWSProvider])
+		envVars, err = getAWSRegistryEnvVars(bsl)
 
 	case AzureProvider:
-		envVar, err = getAzureRegistryEnvVars(bsl, cloudProviderEnvVarMap[AzureProvider])
+		envVars, err = getAzureRegistryEnvVars(bsl, cloudProviderEnvVarMap[AzureProvider])
 
 	case GCPProvider:
-		envVar, err = getGCPRegistryEnvVars(bsl, cloudProviderEnvVarMap[GCPProvider])
+		envVars, err = getGCPRegistryEnvVars(bsl, cloudProviderEnvVarMap[GCPProvider])
 	default:
 		return nil, errors.New("unsupported provider")
 	}
 	if err != nil {
 		return nil, err
 	}
-	return envVar, nil
+	return envVars, nil
 }
 
-func getAWSRegistryEnvVars(bsl *velerov1.BackupStorageLocation, awsEnvVars []corev1.EnvVar) ([]corev1.EnvVar, error) {
-
+func getAWSRegistryEnvVars(bsl *velerov1.BackupStorageLocation) ([]corev1.EnvVar, error) {
+	// validation
+	bslSpecRegion, regionInConfig := bsl.Spec.Config[Region]
+	if !regionInConfig {
+		return nil, errors.New("region not found in backupstoragelocation spec")
+	}
 	// create secret data and fill up the values and return from here
-	for i := range awsEnvVars {
-		if awsEnvVars[i].Name == RegistryStorageS3AccesskeyEnvVarKey {
-			awsEnvVars[i].ValueFrom = &corev1.EnvVarSource{
-				SecretKeyRef: &corev1.SecretKeySelector{
-					LocalObjectReference: corev1.LocalObjectReference{Name: "oadp-" + bsl.Name + "-" + bsl.Spec.Provider + "-registry-secret"},
-					Key:                  "access_key",
+	awsEnvs := []corev1.EnvVar{
+		{
+			Name:  RegistryStorageEnvVarKey,
+			Value: S3,
+		},
+		{
+			Name:  RegistryStorageS3BucketEnvVarKey,
+			Value: bsl.Spec.StorageType.ObjectStorage.Bucket,
+		},
+		{
+			Name:  RegistryStorageS3RegionEnvVarKey,
+			Value: bslSpecRegion,
+		},
+		{
+			Name:  RegistryStorageS3RegionendpointEnvVarKey,
+			Value: bsl.Spec.Config[S3URL],
+		},
+		{
+			Name:  RegistryStorageS3SkipverifyEnvVarKey,
+			Value: bsl.Spec.Config[InsecureSkipTLSVerify],
+		},
+	}
+	// if credential is sts, then add sts specific env vars
+	if bsl.Spec.Config[enableSharedConfig] == "true" {
+		awsEnvs = append(awsEnvs, corev1.EnvVar{
+			Name:  RegistryStorageS3CredentialsConfigPathEnvVarKey,
+			Value: getBslSecretPath(bsl),
+		})
+	} else {
+		awsEnvs = append(awsEnvs,
+			corev1.EnvVar{
+				Name: RegistryStorageS3AccesskeyEnvVarKey,
+				ValueFrom: &corev1.EnvVarSource{
+					SecretKeyRef: &corev1.SecretKeySelector{
+						LocalObjectReference: corev1.LocalObjectReference{Name: "oadp-" + bsl.Name + "-" + bsl.Spec.Provider + "-registry-secret"},
+						Key:                  "access_key",
+					},
 				},
-			}
-		}
-
-		if awsEnvVars[i].Name == RegistryStorageS3BucketEnvVarKey {
-			awsEnvVars[i].Value = bsl.Spec.StorageType.ObjectStorage.Bucket
-		}
-
-		if awsEnvVars[i].Name == RegistryStorageS3RegionEnvVarKey {
-			bslSpecRegion, regionInConfig := bsl.Spec.Config[Region]
-			if regionInConfig {
-				awsEnvVars[i].Value = bslSpecRegion
-			} else {
-				return nil, errors.New("region not found in backupstoragelocation spec")
-			}
-		}
-
-		if awsEnvVars[i].Name == RegistryStorageS3SecretkeyEnvVarKey {
-			awsEnvVars[i].ValueFrom = &corev1.EnvVarSource{
-				SecretKeyRef: &corev1.SecretKeySelector{
-					LocalObjectReference: corev1.LocalObjectReference{Name: "oadp-" + bsl.Name + "-" + bsl.Spec.Provider + "-registry-secret"},
-					Key:                  "secret_key",
+			},
+			corev1.EnvVar{
+				Name: RegistryStorageS3SecretkeyEnvVarKey,
+				ValueFrom: &corev1.EnvVarSource{
+					SecretKeyRef: &corev1.SecretKeySelector{
+						LocalObjectReference: corev1.LocalObjectReference{Name: "oadp-" + bsl.Name + "-" + bsl.Spec.Provider + "-registry-secret"},
+						Key:                  "secret_key",
+					},
 				},
-			}
-		}
-
-		if awsEnvVars[i].Name == RegistryStorageS3RegionendpointEnvVarKey {
-			awsEnvVars[i].Value = bsl.Spec.Config[S3URL]
-		}
-
-		if awsEnvVars[i].Name == RegistryStorageS3SkipverifyEnvVarKey {
-			awsEnvVars[i].Value = bsl.Spec.Config[InsecureSkipTLSVerify]
-		}
-
+			})
 	}
-	return awsEnvVars, nil
+	return awsEnvs, nil
+}
+
+// return path to cloud credentials secret file as defined by
+// https://github.com/vmware-tanzu/velero/blob/5afe837f76aea4dd59b1bf2792e7802d4966f0a7/pkg/cmd/server/server.go#L334
+// https://github.com/vmware-tanzu/velero/blob/5afe837f76aea4dd59b1bf2792e7802d4966f0a7/internal/credentials/file_store.go#L50
+// https://github.com/vmware-tanzu/velero/blob/5afe837f76aea4dd59b1bf2792e7802d4966f0a7/internal/credentials/file_store.go#L72
+// This file is written by velero server on startup
+func getBslSecretPath(bsl *velerov1.BackupStorageLocation) string {
+	return fmt.Sprintf("%s/%s/%s-%s", defaultCredentialsDirectory, bsl.Namespace, bsl.Spec.Credential.LocalObjectReference.Name, bsl.Spec.Credential.Key)
 }
 
 func getAzureRegistryEnvVars(bsl *velerov1.BackupStorageLocation, azureEnvVars []corev1.EnvVar) ([]corev1.EnvVar, error) {