From 9388c2b08ae543e5d24ce02862a9d2a9fa9debff Mon Sep 17 00:00:00 2001 From: Joel Diaz Date: Wed, 11 Nov 2020 10:31:57 -0500 Subject: [PATCH 1/2] UPSTREAM: docker/distribution: 3296: allow pointing to an AWS config file as a parameter for the s3 driver Recognize a new parameter when setting up the AWS client so that a generic AWS config file can be used instead of having to specify AWS access and secret keys. This should allow someone to use different authentication methods beyond just access key, secret key (and optionally session token). Using the current supported auth methods a valid file would look like: ``` [default] aws_access_key_id = AKMYAWSACCCESSKEYID aws_secret_access_key = myawssecretaccesskey ``` But you can also specify alternative auth methods: ``` [default] role_arn = arn:aws:iam:ACCOUNT_NUM:role/ROLE_NAME web_identity_token_file = /path/to/token ``` Signed-off-by: Tiger Kaovilai --- registry/storage/driver/s3-aws/s3.go | 28 +++++++++++++--- registry/storage/driver/s3-aws/s3_test.go | 39 +++++++++++++---------- 2 files changed, 46 insertions(+), 21 deletions(-) diff --git a/registry/storage/driver/s3-aws/s3.go b/registry/storage/driver/s3-aws/s3.go index 7e0c48650d2..528cb8fd341 100644 --- a/registry/storage/driver/s3-aws/s3.go +++ b/registry/storage/driver/s3-aws/s3.go @@ -118,6 +118,7 @@ type DriverParameters struct { SessionToken string UseDualStack bool Accelerate bool + CredentialsConfigPath string } func init() { @@ -197,6 +198,11 @@ func FromParameters(parameters map[string]interface{}) (*Driver, error) { secretKey = "" } + credentialsConfigPath := parameters["credentialsconfigpath"] + if credentialsConfigPath == nil { + credentialsConfigPath = "" + } + regionEndpoint := parameters["regionendpoint"] if regionEndpoint == nil { regionEndpoint = "" @@ -460,6 +466,7 @@ func FromParameters(parameters map[string]interface{}) (*Driver, error) { fmt.Sprint(sessionToken), useDualStackBool, accelerateBool, + fmt.Sprint(credentialsConfigPath), } return New(params) @@ -503,6 +510,12 @@ func New(params DriverParameters) (*Driver, error) { return nil, fmt.Errorf("on Amazon S3 this storage driver can only be used with v4 authentication") } + // Makes no sense to provide access/secret key and the location of a + // config file with credentials. + if (params.AccessKey != "" || params.SecretKey != "") && params.CredentialsConfigPath != "" { + return nil, fmt.Errorf("cannot set both access/secret key and credentials file path") + } + awsConfig := aws.NewConfig() if params.AccessKey != "" && params.SecretKey != "" { @@ -522,9 +535,7 @@ func New(params DriverParameters) (*Driver, error) { awsConfig.WithS3UseAccelerate(params.Accelerate) awsConfig.WithRegion(params.Region) awsConfig.WithDisableSSL(!params.Secure) - if params.UseDualStack { - awsConfig.UseDualStackEndpoint = endpoints.DualStackEndpointStateEnabled - } + awsConfig.WithUseDualStack(params.UseDualStack) if params.UserAgent != "" || params.SkipVerify { httpTransport := http.DefaultTransport @@ -544,7 +555,16 @@ func New(params DriverParameters) (*Driver, error) { } } - sess, err := session.NewSession(awsConfig) + sessionOptions := session.Options{ + Config: *awsConfig, + } + if params.CredentialsConfigPath != "" { + sessionOptions.SharedConfigState = session.SharedConfigEnable + sessionOptions.SharedConfigFiles = []string{ + params.CredentialsConfigPath, + } + } + sess, err := session.NewSessionWithOptions(sessionOptions) if err != nil { return nil, fmt.Errorf("failed to create new session with aws config: %v", err) } diff --git a/registry/storage/driver/s3-aws/s3_test.go b/registry/storage/driver/s3-aws/s3_test.go index 74a3226aab6..c41685b03d1 100644 --- a/registry/storage/driver/s3-aws/s3_test.go +++ b/registry/storage/driver/s3-aws/s3_test.go @@ -31,23 +31,27 @@ var s3DriverConstructor func(rootDirectory, storageClass string) (*Driver, error var skipS3 func() string func init() { - accessKey := os.Getenv("AWS_ACCESS_KEY") - secretKey := os.Getenv("AWS_SECRET_KEY") - bucket := os.Getenv("S3_BUCKET") - encrypt := os.Getenv("S3_ENCRYPT") - keyID := os.Getenv("S3_KEY_ID") - secure := os.Getenv("S3_SECURE") - skipVerify := os.Getenv("S3_SKIP_VERIFY") - v4Auth := os.Getenv("S3_V4_AUTH") - region := os.Getenv("AWS_REGION") - objectACL := os.Getenv("S3_OBJECT_ACL") - root, err := ioutil.TempDir("", "driver-") - regionEndpoint := os.Getenv("REGION_ENDPOINT") - forcePathStyle := os.Getenv("AWS_S3_FORCE_PATH_STYLE") - sessionToken := os.Getenv("AWS_SESSION_TOKEN") - useDualStack := os.Getenv("S3_USE_DUALSTACK") - combineSmallPart := os.Getenv("MULTIPART_COMBINE_SMALL_PART") - accelerate := os.Getenv("S3_ACCELERATE") + var ( + accessKey = os.Getenv("AWS_ACCESS_KEY") + secretKey = os.Getenv("AWS_SECRET_KEY") + bucket = os.Getenv("S3_BUCKET") + encrypt = os.Getenv("S3_ENCRYPT") + keyID = os.Getenv("S3_KEY_ID") + secure = os.Getenv("S3_SECURE") + skipVerify = os.Getenv("S3_SKIP_VERIFY") + v4Auth = os.Getenv("S3_V4_AUTH") + region = os.Getenv("AWS_REGION") + objectACL = os.Getenv("S3_OBJECT_ACL") + regionEndpoint = os.Getenv("REGION_ENDPOINT") + forcePathStyle = os.Getenv("AWS_S3_FORCE_PATH_STYLE") + sessionToken = os.Getenv("AWS_SESSION_TOKEN") + useDualStack = os.Getenv("S3_USE_DUALSTACK") + combineSmallPart = os.Getenv("MULTIPART_COMBINE_SMALL_PART") + accelerate = os.Getenv("S3_ACCELERATE") + credentialsConfigPath = os.Getenv("AWS_SHARED_CREDENTIALS_FILE") + ) + + root, err := os.MkdirTemp("", "driver-") if err != nil { panic(err) } @@ -138,6 +142,7 @@ func init() { sessionToken, useDualStackBool, accelerateBool, + credentialsConfigPath, } return New(parameters) From 4b50a948f1c004069e1adf73a4fdb32d6992941c Mon Sep 17 00:00:00 2001 From: Tiger Kaovilai Date: Tue, 11 Jul 2023 13:55:14 -0400 Subject: [PATCH 2/2] `make vendor` Signed-off-by: Tiger Kaovilai --- go.mod | 4 +-- go.sum | 9 +++--- .../v3/registry/storage/driver/s3-aws/s3.go | 28 ++++++++++++++++--- vendor/modules.txt | 8 +++--- 4 files changed, 35 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index 3964b052ada..797e25a964d 100644 --- a/go.mod +++ b/go.mod @@ -45,11 +45,11 @@ require ( github.com/Azure/go-autorest/logger v0.2.1 // indirect github.com/Azure/go-autorest/tracing v0.6.0 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/bitly/go-simplejson v0.5.0 // indirect + github.com/bitly/go-simplejson v0.5.1 // indirect github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b // indirect github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect - github.com/dnaeon/go-vcr v1.0.1 // indirect + github.com/dnaeon/go-vcr v1.2.0 // indirect github.com/felixge/httpsnoop v1.0.1 // indirect github.com/gofrs/uuid v4.0.0+incompatible // indirect github.com/golang-jwt/jwt/v4 v4.2.0 // indirect diff --git a/go.sum b/go.sum index fa2d82b68c0..c67e7e683ce 100644 --- a/go.sum +++ b/go.sum @@ -95,8 +95,8 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bitly/go-simplejson v0.5.0 h1:6IH+V8/tVMab511d5bn4M7EwGXZf9Hj6i2xSwkNEM+Y= -github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= +github.com/bitly/go-simplejson v0.5.1 h1:xgwPbetQScXt1gh9BmoJ6j9JMr3TElvuIyjR8pgdoow= +github.com/bitly/go-simplejson v0.5.1/go.mod h1:YOPVLzCfwK14b4Sff3oP1AmGhI9T9Vsg84etUnlyp+Q= github.com/bshuster-repo/logrus-logstash-hook v1.0.0 h1:e+C0SB5R1pu//O4MQ3f9cFuPGoOVeF2fE4Og9otCc70= github.com/bshuster-repo/logrus-logstash-hook v1.0.0/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk= github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd h1:rFt+Y/IK1aEZkEHchZRSq9OQbsSzIT/OrI8YFFmRIng= @@ -136,8 +136,8 @@ github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba h1:p6poVbjHDkK github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/dnaeon/go-vcr v1.0.1 h1:r8L/HqC0Hje5AXMu1ooW8oyQyOFv4GxqpL0nRP7SLLY= -github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= +github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= +github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c h1:+pKlWGMw7gf6bQ+oDZB4KHQFypsfjYlq/C4rfL7D3g8= github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQV8= @@ -319,6 +319,7 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/ncw/swift v1.0.47 h1:4DQRPj35Y41WogBxyhOXlrI37nzGlyEcsforeudyYPQ= diff --git a/vendor/github.com/distribution/distribution/v3/registry/storage/driver/s3-aws/s3.go b/vendor/github.com/distribution/distribution/v3/registry/storage/driver/s3-aws/s3.go index 7e0c48650d2..528cb8fd341 100644 --- a/vendor/github.com/distribution/distribution/v3/registry/storage/driver/s3-aws/s3.go +++ b/vendor/github.com/distribution/distribution/v3/registry/storage/driver/s3-aws/s3.go @@ -118,6 +118,7 @@ type DriverParameters struct { SessionToken string UseDualStack bool Accelerate bool + CredentialsConfigPath string } func init() { @@ -197,6 +198,11 @@ func FromParameters(parameters map[string]interface{}) (*Driver, error) { secretKey = "" } + credentialsConfigPath := parameters["credentialsconfigpath"] + if credentialsConfigPath == nil { + credentialsConfigPath = "" + } + regionEndpoint := parameters["regionendpoint"] if regionEndpoint == nil { regionEndpoint = "" @@ -460,6 +466,7 @@ func FromParameters(parameters map[string]interface{}) (*Driver, error) { fmt.Sprint(sessionToken), useDualStackBool, accelerateBool, + fmt.Sprint(credentialsConfigPath), } return New(params) @@ -503,6 +510,12 @@ func New(params DriverParameters) (*Driver, error) { return nil, fmt.Errorf("on Amazon S3 this storage driver can only be used with v4 authentication") } + // Makes no sense to provide access/secret key and the location of a + // config file with credentials. + if (params.AccessKey != "" || params.SecretKey != "") && params.CredentialsConfigPath != "" { + return nil, fmt.Errorf("cannot set both access/secret key and credentials file path") + } + awsConfig := aws.NewConfig() if params.AccessKey != "" && params.SecretKey != "" { @@ -522,9 +535,7 @@ func New(params DriverParameters) (*Driver, error) { awsConfig.WithS3UseAccelerate(params.Accelerate) awsConfig.WithRegion(params.Region) awsConfig.WithDisableSSL(!params.Secure) - if params.UseDualStack { - awsConfig.UseDualStackEndpoint = endpoints.DualStackEndpointStateEnabled - } + awsConfig.WithUseDualStack(params.UseDualStack) if params.UserAgent != "" || params.SkipVerify { httpTransport := http.DefaultTransport @@ -544,7 +555,16 @@ func New(params DriverParameters) (*Driver, error) { } } - sess, err := session.NewSession(awsConfig) + sessionOptions := session.Options{ + Config: *awsConfig, + } + if params.CredentialsConfigPath != "" { + sessionOptions.SharedConfigState = session.SharedConfigEnable + sessionOptions.SharedConfigFiles = []string{ + params.CredentialsConfigPath, + } + } + sess, err := session.NewSessionWithOptions(sessionOptions) if err != nil { return nil, fmt.Errorf("failed to create new session with aws config: %v", err) } diff --git a/vendor/modules.txt b/vendor/modules.txt index dd8391d8c10..b5ab48da2a9 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -99,8 +99,8 @@ github.com/aws/aws-sdk-go/service/sts/stsiface # github.com/beorn7/perks v1.0.1 ## explicit; go 1.11 github.com/beorn7/perks/quantile -# github.com/bitly/go-simplejson v0.5.0 -## explicit +# github.com/bitly/go-simplejson v0.5.1 +## explicit; go 1.17 # github.com/bshuster-repo/logrus-logstash-hook v1.0.0 ## explicit github.com/bshuster-repo/logrus-logstash-hook @@ -181,8 +181,8 @@ github.com/distribution/distribution/v3/registry/storage/driver/testsuites github.com/distribution/distribution/v3/testutil github.com/distribution/distribution/v3/uuid github.com/distribution/distribution/v3/version -# github.com/dnaeon/go-vcr v1.0.1 -## explicit +# github.com/dnaeon/go-vcr v1.2.0 +## explicit; go 1.15 # github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c ## explicit github.com/docker/go-events