From 1c363295851b106bd57305d71e9fec3abe7457a8 Mon Sep 17 00:00:00 2001 From: Xuyang Tao Date: Mon, 10 Aug 2020 09:01:23 -0700 Subject: [PATCH] b/148306474,b/156410254: move generating access token by service account credential to configmanager (#251) --- api/envoy/v7/http/common/base.proto | 19 +-- .../v7/http/service_control/config.proto | 10 +- .../grpc_dynamic_routing/envoy_config.json | 2 +- examples/service_control/envoy_config.json | 2 +- go.mod | 1 + go.sum | 1 + prow/gcpproxy-e2e.sh | 11 +- scripts/all-utilities.sh | 5 +- .../http/backend_auth/config_parser_impl.cc | 5 - .../backend_auth/config_parser_impl_test.cc | 20 ---- .../http/service_control/client_cache.cc | 14 ++- .../service_control_call_impl.cc | 39 ------- .../service_control_call_impl.h | 3 - src/go/bootstrap/ads/flags/flags.go | 3 +- src/go/commonflags/flags.go | 2 +- src/go/configgenerator/cluster_generator.go | 41 +++++-- .../configgenerator/cluster_generator_test.go | 25 ++++ src/go/configgenerator/listener_generator.go | 23 +--- .../listener_generator_test.go | 95 +++++++++++++++ src/go/configinfo/service_info.go | 18 +-- src/go/configinfo/service_info_test.go | 57 +++++++++ src/go/configmanager/config_manager.go | 3 +- src/go/configmanager/config_manager_test.go | 2 +- src/go/configmanager/flags/flags.go | 2 + src/go/configmanager/main/server.go | 19 ++- .../testdata/test_fetch_listeners.go | 4 +- src/go/metadata/metadata_fetcher.go | 18 +-- src/go/metadata/metadata_fetcher_test.go | 18 +-- src/go/options/adsbootstrapper.go | 5 +- src/go/options/common.go | 2 +- src/go/options/configgenerator.go | 5 +- .../token_generator.go | 38 +++++- src/go/tokengenerator/token_generator_test.go | 110 ++++++++++++++++++ src/go/tracing/tracing_test.go | 2 +- src/go/util/httptest_util.go | 20 ++++ src/go/util/token_generator_test.go | 46 -------- src/go/util/url_util.go | 4 +- src/go/util/util.go | 28 +++-- tests/e2e/scripts/cloud-run/deploy.sh | 2 +- tests/e2e/scripts/gke/deploy.sh | 27 ++++- tests/e2e/scripts/prow-utilities.sh | 9 +- .../gke/http-bookstore.yaml.template | 9 ++ tests/env/components/mock_metadata.go | 12 +- tests/env/components/mock_metadata_test.go | 2 +- tests/env/components/ports.go | 3 +- tests/env/platform/files.go | 7 +- .../backend_auth_disable_auth_test.go | 4 +- .../backend_auth_using_iam_test.go | 8 +- .../backend_auth_using_imds_test.go | 10 +- .../jwt_auth_integration_test.go | 2 +- .../report_gcp_attributes_test.go | 14 +-- .../service_control_access_token_test.go | 69 ++++++++++- .../service_control_check_fail_test.go | 2 +- tests/utils/http.go | 5 +- 54 files changed, 641 insertions(+), 266 deletions(-) rename src/go/{util => tokengenerator}/token_generator.go (64%) create mode 100644 src/go/tokengenerator/token_generator_test.go delete mode 100644 src/go/util/token_generator_test.go diff --git a/api/envoy/v7/http/common/base.proto b/api/envoy/v7/http/common/base.proto index 97afc5c4b..88ccdc1fe 100644 --- a/api/envoy/v7/http/common/base.proto +++ b/api/envoy/v7/http/common/base.proto @@ -67,21 +67,9 @@ message HttpUri { }]; } -// Duplicate DataSource from envoy for self containment. -// https://aip.dev/213 -message DataSource { - oneof specifier { - option (validate.required) = true; - - // Local filesystem data source. - string filename = 1 [(validate.rules).string.min_bytes = 1]; - - // String inlined in the configuration. - string inline_string = 2 [(validate.rules).string.min_bytes = 1]; - } -} - message AccessToken { + reserved 2; + oneof token_type { option (validate.required) = true; @@ -91,9 +79,6 @@ message AccessToken { // Query parameters are added by the filter // - Token cluster address to fetch JWT token. HttpUri remote_token = 1; - - // The local path or inline content of the service account json file. - DataSource service_account_secret = 2; } } diff --git a/api/envoy/v7/http/service_control/config.proto b/api/envoy/v7/http/service_control/config.proto index b69e0ada8..542104426 100644 --- a/api/envoy/v7/http/service_control/config.proto +++ b/api/envoy/v7/http/service_control/config.proto @@ -98,6 +98,8 @@ message GcpAttributes { } message FilterConfig { + reserved 5; + // A list of services supported on this Envoy server. repeated Service services = 1; // ref:multi-service @@ -110,14 +112,10 @@ message FilterConfig { oneof access_token { option (validate.required) = true; - // The Instance Metadata Server uri used to fetch access token from Instance - // Metadata Server. + // Uri used to fetch access token from Instance Metadata Server or the local + // token agent server. espv2.api.envoy.v7.http.common.HttpUri imds_token = 4; - // The local path or inline content of the service account json file used to - // generate access token. - espv2.api.envoy.v7.http.common.DataSource service_account_secret = 5; - // Information used to fetch access token from Google Cloud IAM. espv2.api.envoy.v7.http.common.IamTokenInfo iam_token = 6; } diff --git a/examples/grpc_dynamic_routing/envoy_config.json b/examples/grpc_dynamic_routing/envoy_config.json index f53df3e72..d9927946c 100644 --- a/examples/grpc_dynamic_routing/envoy_config.json +++ b/examples/grpc_dynamic_routing/envoy_config.json @@ -508,7 +508,7 @@ "serviceControlUri": { "cluster": "service-control-cluster", "timeout": "30s", - "uri": "https://servicecontrol.googleapis.com/v1/services/" + "uri": "https://servicecontrol.googleapis.com/v1/services" }, "services": [ { diff --git a/examples/service_control/envoy_config.json b/examples/service_control/envoy_config.json index dfaf863c5..5efeb6584 100644 --- a/examples/service_control/envoy_config.json +++ b/examples/service_control/envoy_config.json @@ -200,7 +200,7 @@ "serviceControlUri": { "cluster": "service-control-cluster", "timeout": "30s", - "uri": "https://servicecontrol.googleapis.com/v1/services/" + "uri": "https://servicecontrol.googleapis.com/v1/services" }, "services": [ { diff --git a/go.mod b/go.mod index c473ba3dc..fe322d4a0 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/miekg/dns v1.1.29 golang.org/x/net v0.0.0-20190923162816-aa69164e4478 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 + golang.org/x/tools v0.0.0-20191216052735-49a3e744a425 google.golang.org/api v0.7.0 google.golang.org/genproto v0.0.0-20200617032506-f1bdc9086088 google.golang.org/grpc v1.27.0 diff --git a/go.sum b/go.sum index 85a9d2ead..5a1a21b44 100644 --- a/go.sum +++ b/go.sum @@ -138,6 +138,7 @@ golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0 h1:Dh6fw+p6FyRl5x/FvNswO1ji0lIGzm3KP8Y9VkS9PTE= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20191216052735-49a3e744a425 h1:VvQyQJN0tSuecqgcIxMWnnfG5kSmgy9KZR9sW3W5QeA= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= diff --git a/prow/gcpproxy-e2e.sh b/prow/gcpproxy-e2e.sh index 50a88e1a2..444a7fac9 100755 --- a/prow/gcpproxy-e2e.sh +++ b/prow/gcpproxy-e2e.sh @@ -28,7 +28,7 @@ exit 1; } function runE2E() { local OPTIND OPTARG arg - while getopts :f:p:c:g:m:R:t: arg; do + while getopts :f:p:c:g:m:R:St: arg; do case ${arg} in f) local backend_platform="${OPTARG}" ;; p) local platform="${OPTARG}" ;; @@ -36,11 +36,12 @@ function runE2E() { g) local backend="${OPTARG}" ;; m) local apiproxy_image="${OPTARG}" ;; R) local rollout_strategy="${OPTARG}" ;; + S) local using_sa_cred='true';; t) local test_type="$(echo ${OPTARG} | tr '[A-Z]' '[a-z]')" ;; esac done - local apiproxy_service=$(get_apiproxy_service ${backend}) + local apiproxy_service=$(get_apiproxy_service "${backend}" "${using_sa_cred}") local unique_id=$(get_unique_id "gke-${test_type}-${backend}") if [ "${platform}" == "anthos-cloud-run" ] then @@ -61,7 +62,8 @@ function runE2E() { -i "${unique_id}" \ -B "${BUCKET}" \ -l "${DURATION_IN_HOUR}" \ - -f "${backend_platform}" + -f "${backend_platform}" \ + -S "${using_sa_cred}" } if [ ! -d "$GOPATH/bin" ]; then @@ -87,6 +89,9 @@ case ${TEST_CASE} in "tight-http-bookstore-managed") runE2E -p "gke" -c "tight" -t "http" -g "bookstore" -R "managed" -m "$(get_proxy_image_name_with_sha)" ;; + "tight-http-bookstore-managed-using-sa-cred") + runE2E -p "gke" -c "tight" -t "http" -g "bookstore" -R "managed" -S -m "$(get_proxy_image_name_with_sha)" + ;; "tight-grpc-echo-managed") runE2E -p "gke" -c "tight" -t "grpc" -g "echo" -R "managed" -m "$(get_proxy_image_name_with_sha)" ;; diff --git a/scripts/all-utilities.sh b/scripts/all-utilities.sh index 6ce8f9fbe..7710f9b8c 100755 --- a/scripts/all-utilities.sh +++ b/scripts/all-utilities.sh @@ -142,8 +142,7 @@ function set_api_keys() { function get_test_client_key() { local remote_file_name=$1 local key_path=$2 - [[ -e $key_path ]] || $GSUTIL \ - cp "gs://apiproxy-testing-client-secret-files/$remote_file_name" "$key_path" + $GSUTIL cp "gs://apiproxy-testing-client-secret-files/$remote_file_name" "$key_path" echo -n "$key_path" return 0 } @@ -326,7 +325,7 @@ function get_gcsrunner_image_release_name() { function get_tag_name() { local tag_format="%H" tag_name="$(git show -q HEAD --pretty=format:"${tag_format}")" - echo -n ${tag_name} + echo -n "${tag_name}" } function get_envoy_image_name_with_sha() { diff --git a/src/envoy/http/backend_auth/config_parser_impl.cc b/src/envoy/http/backend_auth/config_parser_impl.cc index 269bd699d..bd82253e4 100644 --- a/src/envoy/http/backend_auth/config_parser_impl.cc +++ b/src/envoy/http/backend_auth/config_parser_impl.cc @@ -107,11 +107,6 @@ FilterConfigParserImpl::FilterConfigParserImpl( }); break; } - case AccessToken::TokenTypeCase::kServiceAccountSecret: - // TODO(taoxuy): support getting access token from service account file. - ENVOY_LOG(error, - "Not support getting access token by service account file"); - return; default: NOT_REACHED_GCOVR_EXCL_LINE; } diff --git a/src/envoy/http/backend_auth/config_parser_impl_test.cc b/src/envoy/http/backend_auth/config_parser_impl_test.cc index e5e473d3a..487417529 100644 --- a/src/envoy/http/backend_auth/config_parser_impl_test.cc +++ b/src/envoy/http/backend_auth/config_parser_impl_test.cc @@ -64,26 +64,6 @@ rules { "Duplicated operation"); } -TEST_F(ConfigParserImplTest, IamIdTokenWithServiceAccountAsAccessToken) { - const char filter_config[] = R"( -iam_token { - access_token { - service_account_secret{} - } -} -rules { - operation: "append-with-audience" - jwt_audience: "this-is-audience" -} -)"; - - EXPECT_CALL(mock_token_subscriber_factory_, createImdsTokenSubscriber) - .Times(0); - EXPECT_CALL(mock_token_subscriber_factory_, createIamTokenSubscriber) - .Times(0); - setUp(filter_config); -} - TEST_F(ConfigParserImplTest, GetIdTokenByImds) { const char filter_config[] = R"( imds_token { diff --git a/src/envoy/http/service_control/client_cache.cc b/src/envoy/http/service_control/client_cache.cc index 5a6475f32..f5cbff1ed 100644 --- a/src/envoy/http/service_control/client_cache.cc +++ b/src/envoy/http/service_control/client_cache.cc @@ -211,17 +211,19 @@ ClientCache::ClientCache( initHttpRequestSetting(filter_config); check_call_factory_ = std::make_unique( cm, dispatcher, filter_config.service_control_uri(), - config_.service_name() + ":check", sc_token_fn, check_timeout_ms_, - check_retries_, time_source, "Service Control remote call: Check"); + absl::StrCat("/", config_.service_name(), ":check"), sc_token_fn, + check_timeout_ms_, check_retries_, time_source, + "Service Control remote call: Check"); quota_call_factory_ = std::make_unique( cm, dispatcher, filter_config.service_control_uri(), - config_.service_name() + ":allocateQuota", quota_token_fn, - quota_timeout_ms_, quota_retries_, time_source, + absl::StrCat("/", config_.service_name(), ":allocateQuota"), + quota_token_fn, quota_timeout_ms_, quota_retries_, time_source, "Service Control remote call: Allocate Quota"); report_call_factory_ = std::make_unique( cm, dispatcher, filter_config.service_control_uri(), - config_.service_name() + ":report", sc_token_fn, report_timeout_ms_, - report_retries_, time_source, "Service Control remote call: Report"); + absl::StrCat("/", config_.service_name(), ":report"), sc_token_fn, + report_timeout_ms_, report_retries_, time_source, + "Service Control remote call: Report"); // Note: Check transport is also defined per request. // But this must be defined, it will be called on each flush of the cache diff --git a/src/envoy/http/service_control/service_control_call_impl.cc b/src/envoy/http/service_control/service_control_call_impl.cc index f9e7b0710..bbb312c65 100644 --- a/src/envoy/http/service_control/service_control_call_impl.cc +++ b/src/envoy/http/service_control/service_control_call_impl.cc @@ -34,17 +34,6 @@ using token::ServiceAccountTokenGenerator; using token::TokenSubscriber; using token::TokenType; -namespace { -// The service_control service name. used for as audience to generate JWT token. -constexpr char kServiceControlService[] = - "/google.api.servicecontrol.v1.ServiceController"; - -// The quota_control service name. used for as audience to generate JWT token. -constexpr char kQuotaControlService[] = - "/google.api.servicecontrol.v1.QuotaController"; - -} // namespace - void ServiceControlCallImpl::createImdsTokenSub() { const std::string& token_cluster = filter_config_.imds_token().cluster(); const std::string& token_uri = filter_config_.imds_token().uri(); @@ -61,31 +50,6 @@ void ServiceControlCallImpl::createImdsTokenSub() { }); } -void ServiceControlCallImpl::createTokenGen() { - const std::string service_control_audience = - filter_config_.service_control_uri().uri() + kServiceControlService; - sc_token_gen_ = token_subscriber_factory_.createServiceAccountTokenGenerator( - filter_config_.service_account_secret().inline_string(), - service_control_audience, [this](const std::string& token) { - TokenSharedPtr new_token = std::make_shared(token); - tls_->runOnAllThreads([this, new_token]() { - tls_->getTyped().set_sc_token(new_token); - }); - }); - - const std::string quota_audience = - filter_config_.service_control_uri().uri() + kQuotaControlService; - quota_token_gen_ = - token_subscriber_factory_.createServiceAccountTokenGenerator( - filter_config_.service_account_secret().inline_string(), - quota_audience, [this](const std::string& token) { - TokenSharedPtr new_token = std::make_shared(token); - tls_->runOnAllThreads([this, new_token]() { - tls_->getTyped().set_quota_token(new_token); - }); - }); -} - void ServiceControlCallImpl::createIamTokenSub() { switch (filter_config_.iam_token().access_token().token_type_case()) { case AccessToken::kRemoteToken: { @@ -151,9 +115,6 @@ ServiceControlCallImpl::ServiceControlCallImpl( case FilterConfig::kImdsToken: createImdsTokenSub(); break; - case FilterConfig::kServiceAccountSecret: - createTokenGen(); - break; case FilterConfig::kIamToken: createIamTokenSub(); break; diff --git a/src/envoy/http/service_control/service_control_call_impl.h b/src/envoy/http/service_control/service_control_call_impl.h index 21c569509..c05c902a3 100644 --- a/src/envoy/http/service_control/service_control_call_impl.h +++ b/src/envoy/http/service_control/service_control_call_impl.h @@ -103,7 +103,6 @@ class ServiceControlCallImpl } void createImdsTokenSub(); - void createTokenGen(); void createIamTokenSub(); const ::espv2::api::envoy::v7::http::service_control::FilterConfig& @@ -123,8 +122,6 @@ class ServiceControlCallImpl // Token subscriber used to fetch access token from iam for service control token::TokenSubscriberPtr iam_token_sub_; - token::ServiceAccountTokenPtr sc_token_gen_; - token::ServiceAccountTokenPtr quota_token_gen_; Envoy::ThreadLocal::SlotPtr tls_; }; // namespace ServiceControl diff --git a/src/go/bootstrap/ads/flags/flags.go b/src/go/bootstrap/ads/flags/flags.go index e312d8a49..d326c2aa2 100644 --- a/src/go/bootstrap/ads/flags/flags.go +++ b/src/go/bootstrap/ads/flags/flags.go @@ -21,6 +21,7 @@ import ( "github.com/GoogleCloudPlatform/esp-v2/src/go/commonflags" "github.com/GoogleCloudPlatform/esp-v2/src/go/options" + "github.com/GoogleCloudPlatform/esp-v2/src/go/util" "github.com/golang/glog" ) @@ -33,7 +34,7 @@ func DefaultBootstrapperOptionsFromFlags() options.AdsBootstrapperOptions { opts := options.AdsBootstrapperOptions{ CommonOptions: common_option, AdsConnectTimeout: *AdsConnectTimeout, - DiscoveryAddress: fmt.Sprintf("127.0.0.1:%d", common_option.DiscoveryPort), + DiscoveryAddress: fmt.Sprintf("%s:%d", util.LoopbackIPv4Addr, common_option.DiscoveryPort), } glog.Infof("ADS Bootstrapper options: %+v", opts) diff --git a/src/go/commonflags/flags.go b/src/go/commonflags/flags.go index 7a26a8fa8..d81561f9d 100644 --- a/src/go/commonflags/flags.go +++ b/src/go/commonflags/flags.go @@ -52,7 +52,7 @@ var ( //listener is marked as ready but the whole Envoy server is not marked as ready //(worker did not start) somehow. To work around this problem, use IP for //metadata server to fetch access token. - MetadataURL = flag.String("metadata_url", "http://169.254.169.254/computeMetadata", "url of metadata server") + MetadataURL = flag.String("metadata_url", "http://169.254.169.254", "url of metadata server") IamURL = flag.String("iam_url", "https://iamcredentials.googleapis.com", "url of iam server") ServiceControlIamServiceAccount = flag.String("service_control_iam_service_account", "", "The service account used to fetch access token for the Service Control from Google Cloud IAM") diff --git a/src/go/configgenerator/cluster_generator.go b/src/go/configgenerator/cluster_generator.go index c22dd4228..b99c97507 100644 --- a/src/go/configgenerator/cluster_generator.go +++ b/src/go/configgenerator/cluster_generator.go @@ -40,12 +40,27 @@ func MakeClusters(serviceInfo *sc.ServiceInfo) ([]*clusterpb.Cluster, error) { clusters = append(clusters, backendCluster) } - metadataCluster, err := makeMetadataCluster(serviceInfo) - if err != nil { - return nil, err - } - if metadataCluster != nil { - clusters = append(clusters, metadataCluster) + // When ServiceAccountKey is undefined, envoy will access Instance + // Metadata to get access token and identity token. It can be either sidecar + // mode and gateway mode. + // + // When ServiceAccountKey is defined, configmanager will setup a local + // server to provide access token generated by the service account credential. + // It can only be sidecar mode. + if serviceInfo.Options.ServiceAccountKey == "" { + metadataCluster, err := makeMetadataCluster(serviceInfo) + if err != nil { + return nil, err + } + if metadataCluster != nil { + clusters = append(clusters, metadataCluster) + } + + } else { + tokenAgentCluster := makeTokenAgentCluster(serviceInfo) + + clusters = append(clusters, tokenAgentCluster) + } iamCluster, err := makeIamCluster(serviceInfo) @@ -136,6 +151,18 @@ func makeMetadataCluster(serviceInfo *sc.ServiceInfo) (*clusterpb.Cluster, error return c, nil } +func makeTokenAgentCluster(serviceInfo *sc.ServiceInfo) *clusterpb.Cluster { + return &clusterpb.Cluster{ + Name: util.TokenAgentClusterName, + LbPolicy: clusterpb.Cluster_ROUND_ROBIN, + ConnectTimeout: ptypes.DurationProto(serviceInfo.Options.ClusterConnectTimeout), + ClusterDiscoveryType: &clusterpb.Cluster_Type{ + Type: clusterpb.Cluster_STATIC, + }, + LoadAssignment: util.CreateLoadAssignment(util.LoopbackIPv4Addr, uint32(serviceInfo.Options.TokenAgentPort)), + } +} + func makeIamCluster(serviceInfo *sc.ServiceInfo) (*clusterpb.Cluster, error) { if serviceInfo.Options.ServiceControlCredentials == nil && serviceInfo.Options.BackendAuthCredentials == nil { return nil, nil @@ -286,7 +313,7 @@ func makeServiceControlCluster(serviceInfo *sc.ServiceInfo) (*clusterpb.Cluster, } connectTimeoutProto := ptypes.DurationProto(5 * time.Second) - serviceInfo.ServiceControlURI = scheme + "://" + hostname + "/v1/services/" + serviceInfo.ServiceControlURI = scheme + "://" + hostname + "/v1/services" c := &clusterpb.Cluster{ Name: util.ServiceControlClusterName, LbPolicy: clusterpb.Cluster_ROUND_ROBIN, diff --git a/src/go/configgenerator/cluster_generator_test.go b/src/go/configgenerator/cluster_generator_test.go index a61b67bd0..5c7be69d4 100644 --- a/src/go/configgenerator/cluster_generator_test.go +++ b/src/go/configgenerator/cluster_generator_test.go @@ -783,3 +783,28 @@ func TestMakeIamCluster(t *testing.T) { } } } + +func TestMakeTokenAgentCluster(t *testing.T) { + fakeServiceInfo, _ := configinfo.NewServiceInfoFromServiceConfig(&confpb.Service{ + Apis: []*apipb.Api{ + { + Name: testApiName, + }, + }, + }, testConfigID, options.DefaultConfigGeneratorOptions()) + + cluster := makeTokenAgentCluster(fakeServiceInfo) + wantCluster := &clusterpb.Cluster{ + Name: util.TokenAgentClusterName, + LbPolicy: clusterpb.Cluster_ROUND_ROBIN, + ConnectTimeout: ptypes.DurationProto(fakeServiceInfo.Options.ClusterConnectTimeout), + ClusterDiscoveryType: &clusterpb.Cluster_Type{ + Type: clusterpb.Cluster_STATIC, + }, + LoadAssignment: util.CreateLoadAssignment("127.0.0.1", uint32(fakeServiceInfo.Options.TokenAgentPort)), + } + + if !proto.Equal(cluster, wantCluster) { + t.Errorf("Test makeTokenAgentClusters, \ngot: %v,\nwant: %v", cluster, wantCluster) + } +} diff --git a/src/go/configgenerator/listener_generator.go b/src/go/configgenerator/listener_generator.go index 6364e1fea..f80a467ca 100644 --- a/src/go/configgenerator/listener_generator.go +++ b/src/go/configgenerator/listener_generator.go @@ -620,7 +620,7 @@ func makeServiceControlFilter(serviceInfo *sc.ServiceInfo) *hcmpb.HttpFilter { filterConfig.AccessToken = &scpb.FilterConfig_IamToken{ IamToken: &commonpb.IamTokenInfo{ IamUri: &commonpb.HttpUri{ - Uri: fmt.Sprintf("%s%s", serviceInfo.Options.IamURL, util.IamAccessTokenSuffix(serviceInfo.Options.ServiceControlCredentials.ServiceAccountEmail)), + Uri: fmt.Sprintf("%s%s", serviceInfo.Options.IamURL, util.IamAccessTokenPath(serviceInfo.Options.ServiceControlCredentials.ServiceAccountEmail)), Cluster: util.IamServerClusterName, Timeout: ptypes.DurationProto(serviceInfo.Options.HttpRequestTimeout), }, @@ -630,21 +630,10 @@ func makeServiceControlFilter(serviceInfo *sc.ServiceInfo) *hcmpb.HttpFilter { }, } } else { - // Use access token from fetched the Instance Metadata Server to talk to Service Controller - switch serviceInfo.AccessToken.TokenType.(type) { - case *commonpb.AccessToken_RemoteToken: - filterConfig.AccessToken = &scpb.FilterConfig_ImdsToken{ - ImdsToken: serviceInfo.AccessToken.GetRemoteToken(), - } - break - case *commonpb.AccessToken_ServiceAccountSecret: - filterConfig.AccessToken = &scpb.FilterConfig_ServiceAccountSecret{ - ServiceAccountSecret: serviceInfo.AccessToken.GetServiceAccountSecret(), - } - break - default: - break + filterConfig.AccessToken = &scpb.FilterConfig_ImdsToken{ + ImdsToken: serviceInfo.AccessToken.GetRemoteToken(), } + } if serviceInfo.GcpAttributes != nil { @@ -783,7 +772,7 @@ func makeBackendAuthFilter(serviceInfo *sc.ServiceInfo) *hcmpb.HttpFilter { backendAuthConfig.IdTokenInfo = &bapb.FilterConfig_IamToken{ IamToken: &commonpb.IamTokenInfo{ IamUri: &commonpb.HttpUri{ - Uri: fmt.Sprintf("%s%s", serviceInfo.Options.IamURL, util.IamIdentityTokenSuffix(serviceInfo.Options.BackendAuthCredentials.ServiceAccountEmail)), + Uri: fmt.Sprintf("%s%s", serviceInfo.Options.IamURL, util.IamIdentityTokenPath(serviceInfo.Options.BackendAuthCredentials.ServiceAccountEmail)), Cluster: util.IamServerClusterName, Timeout: ptypes.DurationProto(serviceInfo.Options.HttpRequestTimeout), }, @@ -796,7 +785,7 @@ func makeBackendAuthFilter(serviceInfo *sc.ServiceInfo) *hcmpb.HttpFilter { } else { backendAuthConfig.IdTokenInfo = &bapb.FilterConfig_ImdsToken{ ImdsToken: &commonpb.HttpUri{ - Uri: fmt.Sprintf("%s%s", serviceInfo.Options.MetadataURL, util.IdentityTokenSuffix), + Uri: fmt.Sprintf("%s%s", serviceInfo.Options.MetadataURL, util.IdentityTokenPath), Cluster: util.MetadataServerClusterName, Timeout: ptypes.DurationProto(serviceInfo.Options.HttpRequestTimeout), }, diff --git a/src/go/configgenerator/listener_generator_test.go b/src/go/configgenerator/listener_generator_test.go index 4e2162de9..016566965 100644 --- a/src/go/configgenerator/listener_generator_test.go +++ b/src/go/configgenerator/listener_generator_test.go @@ -1472,6 +1472,101 @@ func TestPathMatcherFilter(t *testing.T) { } } +func TestServiceControl(t *testing.T) { + fakeServiceConfig := &confpb.Service{ + Name: testProjectName, + Apis: []*apipb.Api{ + { + Name: testApiName, + Methods: []*apipb.Method{ + { + Name: "ListShelves", + }, + }, + }, + }, + Control: &confpb.Control{ + Environment: statPrefix, + }, + } + testData := []struct { + desc string + serviceControlCredentials *options.IAMCredentialsOptions + serviceAccountKey string + wantPartialServiceControlFilter string + }{ + { + desc: "get access token from imds", + wantPartialServiceControlFilter: ` + "imdsToken": { + "cluster": "metadata-cluster", + "timeout": "30s", + "uri": "http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/token" + },`, + }, + { + desc: "get access token from iam", + serviceControlCredentials: &options.IAMCredentialsOptions{ + ServiceAccountEmail: "ServiceControl@iam.com", + Delegates: []string{"delegate_foo", "delegate_bar"}, + }, + wantPartialServiceControlFilter: ` + "iamToken": { + "accessToken": { + "remoteToken": { + "cluster": "metadata-cluster", + "timeout": "30s", + "uri": "http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/token" + } + }, + "delegates": [ + "delegate_foo", + "delegate_bar" + ], + "iamUri": { + "cluster": "iam-cluster", + "timeout": "30s", + "uri": "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/ServiceControl@iam.com:generateAccessToken" + }, + "serviceAccountEmail": "ServiceControl@iam.com" + },`, + }, + { + desc: "get access token from the token agent server", + serviceAccountKey: "this-is-sa-cred", + wantPartialServiceControlFilter: ` + "imdsToken": { + "cluster": "token-agent-cluster", + "timeout": "30s", + "uri": "http://127.0.0.1:8791/local/access_token" + },`, + }, + } + for i, tc := range testData { + opts := options.DefaultConfigGeneratorOptions() + opts.ServiceControlCredentials = tc.serviceControlCredentials + opts.ServiceAccountKey = tc.serviceAccountKey + + fakeServiceInfo, err := configinfo.NewServiceInfoFromServiceConfig(fakeServiceConfig, testConfigID, opts) + if err != nil { + t.Error(err) + } + + marshaler := &jsonpb.Marshaler{} + filter := makeServiceControlFilter(fakeServiceInfo) + + gotFilter, err := marshaler.MarshalToString(filter) + if err != nil { + t.Fatal(err) + } + + if err := util.JsonContains(gotFilter, tc.wantPartialServiceControlFilter); err != nil { + t.Errorf("Test Desc(%d): %s, makeServiceControlFilter failed,\n%v", i, tc.desc, err) + } + + } +} + func TestHealthCheckFilter(t *testing.T) { testdata := []struct { desc string diff --git a/src/go/configinfo/service_info.go b/src/go/configinfo/service_info.go index 0b8531f18..237cc31c2 100644 --- a/src/go/configinfo/service_info.go +++ b/src/go/configinfo/service_info.go @@ -16,7 +16,6 @@ package configinfo import ( "fmt" - "io/ioutil" "math" "net" "sort" @@ -243,27 +242,30 @@ func (s *ServiceInfo) addGrpcHttpRules() { func (s *ServiceInfo) processAccessToken() { if s.Options.ServiceAccountKey != "" { - data, _ := ioutil.ReadFile(s.Options.ServiceAccountKey) s.AccessToken = &commonpb.AccessToken{ - TokenType: &commonpb.AccessToken_ServiceAccountSecret{ - ServiceAccountSecret: &commonpb.DataSource{ - Specifier: &commonpb.DataSource_InlineString{ - InlineString: string(data), - }, + TokenType: &commonpb.AccessToken_RemoteToken{ + RemoteToken: &commonpb.HttpUri{ + // Use http://127.0.0.1:8791/local/access_token by default. + Uri: fmt.Sprintf("http://%s:%v%s", util.LoopbackIPv4Addr, s.Options.TokenAgentPort, util.TokenAgentAccessTokenPath), + Cluster: util.TokenAgentClusterName, + Timeout: ptypes.DurationProto(s.Options.HttpRequestTimeout), }, }, } + return } + s.AccessToken = &commonpb.AccessToken{ TokenType: &commonpb.AccessToken_RemoteToken{ RemoteToken: &commonpb.HttpUri{ - Uri: fmt.Sprintf("%s%s", s.Options.MetadataURL, util.AccessTokenSuffix), + Uri: fmt.Sprintf("%s%s", s.Options.MetadataURL, util.AccessTokenPath), Cluster: util.MetadataServerClusterName, Timeout: ptypes.DurationProto(s.Options.HttpRequestTimeout), }, }, } + } func (s *ServiceInfo) processQuota() { diff --git a/src/go/configinfo/service_info_test.go b/src/go/configinfo/service_info_test.go index e92559fc0..a82249049 100644 --- a/src/go/configinfo/service_info_test.go +++ b/src/go/configinfo/service_info_test.go @@ -28,6 +28,7 @@ import ( "github.com/GoogleCloudPlatform/esp-v2/src/go/options" "github.com/GoogleCloudPlatform/esp-v2/src/go/util" "github.com/golang/protobuf/proto" + "github.com/golang/protobuf/ptypes" "github.com/google/go-cmp/cmp" "github.com/gorilla/mux" @@ -2334,3 +2335,59 @@ func TestProcessTypes(t *testing.T) { } } } + +func TestProcessAccessToken(t *testing.T) { + fakeServiceConfig := &confpb.Service{ + Apis: []*apipb.Api{ + { + Name: testApiName, + }, + }, + } + testCases := []struct { + desc string + serviceAccountKey string + wantAccessToken *commonpb.AccessToken + }{ + { + desc: "get access token from imds", + wantAccessToken: &commonpb.AccessToken{ + TokenType: &commonpb.AccessToken_RemoteToken{ + RemoteToken: &commonpb.HttpUri{ + Uri: "http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/token", + Cluster: "metadata-cluster", + Timeout: ptypes.DurationProto(30 * time.Second), + }, + }, + }, + }, + { + desc: "get access token from lmds", + serviceAccountKey: "this-is-service-account-key", + wantAccessToken: &commonpb.AccessToken{ + TokenType: &commonpb.AccessToken_RemoteToken{ + RemoteToken: &commonpb.HttpUri{ + Uri: "http://127.0.0.1:8791/local/access_token", + Cluster: "token-agent-cluster", + Timeout: ptypes.DurationProto(30 * time.Second), + }, + }, + }, + }, + } + + for _, tc := range testCases { + opts := options.DefaultConfigGeneratorOptions() + opts.ServiceAccountKey = tc.serviceAccountKey + serviceInfo, err := NewServiceInfoFromServiceConfig(fakeServiceConfig, "ConfigID", opts) + if err != nil { + t.Fatal(err) + } + + serviceInfo.processAccessToken() + if !reflect.DeepEqual(serviceInfo.AccessToken, tc.wantAccessToken) { + t.Errorf("fail(%s): expect accessToken: %v, get accessToken: %v", tc.desc, tc.wantAccessToken, serviceInfo.AccessToken) + } + } + +} diff --git a/src/go/configmanager/config_manager.go b/src/go/configmanager/config_manager.go index 8d59c37bf..17a4da26e 100644 --- a/src/go/configmanager/config_manager.go +++ b/src/go/configmanager/config_manager.go @@ -27,6 +27,7 @@ import ( "github.com/GoogleCloudPlatform/esp-v2/src/go/configinfo" "github.com/GoogleCloudPlatform/esp-v2/src/go/metadata" "github.com/GoogleCloudPlatform/esp-v2/src/go/options" + "github.com/GoogleCloudPlatform/esp-v2/src/go/tokengenerator" "github.com/GoogleCloudPlatform/esp-v2/src/go/util" "github.com/envoyproxy/go-control-plane/pkg/cache/types" "github.com/envoyproxy/go-control-plane/pkg/cache/v3" @@ -132,7 +133,7 @@ func NewConfigManager(mf *metadata.MetadataFetcher, opts options.ConfigGenerator accessToken := func() (string, time.Duration, error) { if opts.ServiceAccountKey != "" { - return util.GenerateAccessTokenFromFile(opts.ServiceAccountKey) + return tokengenerator.GenerateAccessTokenFromFile(opts.ServiceAccountKey) } return mf.FetchAccessToken() } diff --git a/src/go/configmanager/config_manager_test.go b/src/go/configmanager/config_manager_test.go index 5126848e3..95192f01b 100644 --- a/src/go/configmanager/config_manager_test.go +++ b/src/go/configmanager/config_manager_test.go @@ -464,7 +464,7 @@ func runTest(t *testing.T, fakeScReport, fakeRollouts, fakeConfig *safeData, opt } mockMetadataServer := util.InitMockServerFromPathResp(map[string]string{ - util.AccessTokenSuffix: fakeToken, + util.AccessTokenPath: fakeToken, }) defer mockMetadataServer.Close() diff --git a/src/go/configmanager/flags/flags.go b/src/go/configmanager/flags/flags.go index 661039b9a..d1223d74b 100644 --- a/src/go/configmanager/flags/flags.go +++ b/src/go/configmanager/flags/flags.go @@ -66,6 +66,7 @@ var ( ServiceAccountKey = flag.String("service_account_key", "", `Use the service account key JSON file to access the service control and the service management. You can also set {creds_key} environment variable to the location of the service account credentials JSON file. If the option is omitted, the proxy contacts the metadata service to fetch an access token`) + TokenAgentPort = flag.Uint("token_agent_port", 8791, "Port that configmanager use to setup server to provide envoy with access token using service account credential, for accessing servicecontrol.") // Envoy configurations. AccessLog = flag.String("access_log", "", "Path to a local file to which the access log entries will be written") @@ -145,6 +146,7 @@ func EnvoyConfigOptionsFromFlags() options.ConfigGeneratorOptions { EnableHSTS: *EnableHSTS, DnsResolverAddresses: *DnsResolverAddresses, ServiceAccountKey: *ServiceAccountKey, + TokenAgentPort: *TokenAgentPort, SkipJwtAuthnFilter: *SkipJwtAuthnFilter, SkipServiceControlFilter: *SkipServiceControlFilter, EnvoyUseRemoteAddress: *EnvoyUseRemoteAddress, diff --git a/src/go/configmanager/main/server.go b/src/go/configmanager/main/server.go index 96ceb10cc..5b6a467a2 100644 --- a/src/go/configmanager/main/server.go +++ b/src/go/configmanager/main/server.go @@ -19,6 +19,7 @@ import ( "flag" "fmt" "net" + "net/http" "os" "os/signal" "syscall" @@ -26,6 +27,8 @@ import ( "github.com/GoogleCloudPlatform/esp-v2/src/go/configmanager" "github.com/GoogleCloudPlatform/esp-v2/src/go/configmanager/flags" "github.com/GoogleCloudPlatform/esp-v2/src/go/metadata" + "github.com/GoogleCloudPlatform/esp-v2/src/go/tokengenerator" + "github.com/GoogleCloudPlatform/esp-v2/src/go/util" "github.com/golang/glog" "google.golang.org/grpc" @@ -53,7 +56,7 @@ func main() { } server := xds.NewServer(ctx, m.Cache(), nil) grpcServer := grpc.NewServer() - lis, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", opts.DiscoveryPort)) + lis, err := net.Listen("tcp", fmt.Sprintf("%s:%d", util.LoopbackIPv4Addr, opts.DiscoveryPort)) if err != nil { glog.Exitf("Server failed to listen: %v", err) } @@ -74,6 +77,20 @@ func main() { grpcServer.Stop() }() + if opts.ServiceAccountKey != "" { + // Setup token agent server + r := tokengenerator.MakeTokenAgentHandler(opts.ServiceAccountKey) + go func() { + err := http.ListenAndServe(fmt.Sprintf(":%v", opts.TokenAgentPort), r) + + if err != nil { + glog.Errorf("token agent fail to serve: %v", err) + } + + }() + + } + if err := grpcServer.Serve(lis); err != nil { glog.Exitf("Server fail to serve: %v", err) } diff --git a/src/go/configmanager/testdata/test_fetch_listeners.go b/src/go/configmanager/testdata/test_fetch_listeners.go index 79a656c7c..70115561d 100644 --- a/src/go/configmanager/testdata/test_fetch_listeners.go +++ b/src/go/configmanager/testdata/test_fetch_listeners.go @@ -952,7 +952,7 @@ var ( "serviceControlUri":{ "cluster":"service-control-cluster", "timeout":"30s", - "uri":"https://servicecontrol.googleapis.com/v1/services/" + "uri":"https://servicecontrol.googleapis.com/v1/services" }, "services":[ { @@ -1329,7 +1329,7 @@ var ( "serviceControlUri":{ "cluster":"service-control-cluster", "timeout":"30s", - "uri":"https://servicecontrol.googleapis.com/v1/services/" + "uri":"https://servicecontrol.googleapis.com/v1/services" }, "services":[ { diff --git a/src/go/metadata/metadata_fetcher.go b/src/go/metadata/metadata_fetcher.go index e2a2ab3d7..a6e0d0199 100644 --- a/src/go/metadata/metadata_fetcher.go +++ b/src/go/metadata/metadata_fetcher.go @@ -97,7 +97,7 @@ func (mf *MetadataFetcher) FetchAccessToken() (string, time.Duration, error) { return mf.tokenInfo.accessToken, mf.tokenInfo.tokenTimeout.Sub(now), nil } - tokenBody, err := mf.getMetadata(mf.createUrl(util.AccessTokenSuffix)) + tokenBody, err := mf.getMetadata(mf.createUrl(util.AccessTokenPath)) if err != nil { return "", 0, err } @@ -124,15 +124,15 @@ func (mf *MetadataFetcher) fetchMetadata(key string) (string, error) { } func (mf *MetadataFetcher) FetchServiceName() (string, error) { - return mf.fetchMetadata(util.ServiceNameSuffix) + return mf.fetchMetadata(util.ServiceNamePath) } func (mf *MetadataFetcher) FetchConfigId() (string, error) { - return mf.fetchMetadata(util.ConfigIDSuffix) + return mf.fetchMetadata(util.ConfigIDPath) } func (mf *MetadataFetcher) FetchRolloutStrategy() (string, error) { - return mf.fetchMetadata(util.RolloutStrategySuffix) + return mf.fetchMetadata(util.RolloutStrategyPath) } func (mf *MetadataFetcher) FetchIdentityJWTToken(audience string) (string, time.Duration, error) { @@ -146,7 +146,7 @@ func (mf *MetadataFetcher) FetchIdentityJWTToken(audience string) (string, time. } } - identityTokenURI := util.IdentityTokenSuffix + "?audience=" + audience + "&format=standard" + identityTokenURI := util.IdentityTokenPath + "?audience=" + audience + "&format=standard" token, err := mf.fetchMetadata(identityTokenURI) if err != nil { return "", 0, err @@ -181,12 +181,12 @@ func (mf *MetadataFetcher) FetchGCPAttributes() (*scpb.GcpAttributes, error) { } func (mf *MetadataFetcher) FetchProjectId() (string, error) { - return mf.fetchMetadata(util.ProjectIDSuffix) + return mf.fetchMetadata(util.ProjectIDPath) } // Do not directly use this function. Use fetchGCPAttributes instead. func (mf *MetadataFetcher) fetchZone() (string, error) { - zonePath, err := mf.fetchMetadata(util.ZoneSuffix) + zonePath, err := mf.fetchMetadata(util.ZonePath) if err != nil { return "", err } @@ -203,11 +203,11 @@ func (mf *MetadataFetcher) fetchZone() (string, error) { // Do not directly use this function. Use fetchGCPAttributes instead. func (mf *MetadataFetcher) fetchPlatform() string { - if _, err := mf.fetchMetadata(util.GAEServerSoftwareSuffix); err == nil { + if _, err := mf.fetchMetadata(util.GAEServerSoftwarePath); err == nil { return util.GAEFlex } - if _, err := mf.fetchMetadata(util.KubeEnvSuffix); err == nil { + if _, err := mf.fetchMetadata(util.KubeEnvPath); err == nil { return util.GKE } diff --git a/src/go/metadata/metadata_fetcher_test.go b/src/go/metadata/metadata_fetcher_test.go index 1c884e193..183bb5022 100644 --- a/src/go/metadata/metadata_fetcher_test.go +++ b/src/go/metadata/metadata_fetcher_test.go @@ -204,7 +204,7 @@ func TestFetchGCPAttributes(t *testing.T) { { desc: "ProjectID", mockedResp: map[string]string{ - util.ProjectIDSuffix: fakeProjectID, + util.ProjectIDPath: fakeProjectID, }, expectedGCPAttributes: &scpb.GcpAttributes{ ProjectId: fakeProjectID, @@ -214,7 +214,7 @@ func TestFetchGCPAttributes(t *testing.T) { { desc: "Valid Zone", mockedResp: map[string]string{ - util.ZoneSuffix: fakeZonePath, + util.ZonePath: fakeZonePath, }, expectedGCPAttributes: &scpb.GcpAttributes{ Zone: fakeZone, @@ -224,7 +224,7 @@ func TestFetchGCPAttributes(t *testing.T) { { desc: "Invalid Zone - without '/'", mockedResp: map[string]string{ - util.ZoneSuffix: "noslash", + util.ZonePath: "noslash", }, expectedGCPAttributes: &scpb.GcpAttributes{ Platform: util.GCE, @@ -233,7 +233,7 @@ func TestFetchGCPAttributes(t *testing.T) { { desc: "Invalid Zone - ends with '/'", mockedResp: map[string]string{ - util.ZoneSuffix: "project/123123/", + util.ZonePath: "project/123123/", }, expectedGCPAttributes: &scpb.GcpAttributes{ Platform: util.GCE, @@ -242,7 +242,7 @@ func TestFetchGCPAttributes(t *testing.T) { { desc: "Platform - GAE_FLEX", mockedResp: map[string]string{ - util.GAEServerSoftwareSuffix: "foo", + util.GAEServerSoftwarePath: "foo", }, expectedGCPAttributes: &scpb.GcpAttributes{ Platform: util.GAEFlex, @@ -251,7 +251,7 @@ func TestFetchGCPAttributes(t *testing.T) { { desc: "Platform - GKE", mockedResp: map[string]string{ - util.KubeEnvSuffix: "foo", + util.KubeEnvPath: "foo", }, expectedGCPAttributes: &scpb.GcpAttributes{ Platform: util.GKE, @@ -267,9 +267,9 @@ func TestFetchGCPAttributes(t *testing.T) { { desc: "Combined - ProjectID, Zone, and Platform", mockedResp: map[string]string{ - util.ProjectIDSuffix: fakeProjectID, - util.ZoneSuffix: fakeZonePath, - util.GAEServerSoftwareSuffix: "foo", + util.ProjectIDPath: fakeProjectID, + util.ZonePath: fakeZonePath, + util.GAEServerSoftwarePath: "foo", }, expectedGCPAttributes: &scpb.GcpAttributes{ ProjectId: fakeProjectID, diff --git a/src/go/options/adsbootstrapper.go b/src/go/options/adsbootstrapper.go index d02ec0756..34562dcb1 100644 --- a/src/go/options/adsbootstrapper.go +++ b/src/go/options/adsbootstrapper.go @@ -15,7 +15,10 @@ package options import ( + "fmt" "time" + + "github.com/GoogleCloudPlatform/esp-v2/src/go/util" ) // AdsBootstrapperOptions describes the possible overrides used by the ADS bootstrapper to create the envoy bootstrap config. @@ -34,6 +37,6 @@ func DefaultAdsBootstrapperOptions() AdsBootstrapperOptions { return AdsBootstrapperOptions{ CommonOptions: DefaultCommonOptions(), AdsConnectTimeout: 10 * time.Second, - DiscoveryAddress: "127.0.0.1:8790", + DiscoveryAddress: fmt.Sprintf("%s:8790", util.LoopbackIPv4Addr), } } diff --git a/src/go/options/common.go b/src/go/options/common.go index 0a1b8adfa..70986ce1e 100644 --- a/src/go/options/common.go +++ b/src/go/options/common.go @@ -90,7 +90,7 @@ func DefaultCommonOptions() CommonOptions { TracingMaxNumAnnotations: 32, TracingMaxNumMessageEvents: 128, TracingMaxNumLinks: 128, - MetadataURL: "http://169.254.169.254/computeMetadata", + MetadataURL: "http://169.254.169.254", IamURL: "https://iamcredentials.googleapis.com", GeneratedHeaderPrefix: "X-Endpoint-", } diff --git a/src/go/options/configgenerator.go b/src/go/options/configgenerator.go index 38b538f51..96705acc4 100644 --- a/src/go/options/configgenerator.go +++ b/src/go/options/configgenerator.go @@ -15,6 +15,7 @@ package options import ( + "fmt" "time" "github.com/GoogleCloudPlatform/esp-v2/src/go/util" @@ -59,6 +60,7 @@ type ConfigGeneratorOptions struct { // Flags for non_gcp deployment. ServiceAccountKey string + TokenAgentPort uint // Flags for testing purpose. SkipJwtAuthnFilter bool @@ -107,12 +109,13 @@ func DefaultConfigGeneratorOptions() ConfigGeneratorOptions { return ConfigGeneratorOptions{ CommonOptions: DefaultCommonOptions(), BackendDnsLookupFamily: "auto", - BackendAddress: "http://127.0.0.1:8082", + BackendAddress: fmt.Sprintf("http://%s:8082", util.LoopbackIPv4Addr), ClusterConnectTimeout: 20 * time.Second, EnvoyXffNumTrustedHops: 2, JwksCacheDurationInS: 300, ListenerAddress: "0.0.0.0", ListenerPort: 8080, + TokenAgentPort: 8791, RootCertsPath: util.DefaultRootCAPaths, SuppressEnvoyHeaders: true, ServiceControlNetworkFailOpen: true, diff --git a/src/go/util/token_generator.go b/src/go/tokengenerator/token_generator.go similarity index 64% rename from src/go/util/token_generator.go rename to src/go/tokengenerator/token_generator.go index 8ab6db523..edaac6391 100644 --- a/src/go/util/token_generator.go +++ b/src/go/tokengenerator/token_generator.go @@ -12,26 +12,33 @@ // See the License for the specific language governing permissions and // limitations under the License. -package util +package tokengenerator import ( + "fmt" "io/ioutil" + "net/http" "sync" "time" + "github.com/GoogleCloudPlatform/esp-v2/src/go/util" + "github.com/gorilla/mux" "golang.org/x/oauth2" "golang.org/x/oauth2/google" ) var ( _GOOGLE_API_SCOPE = []string{ + // Call servicemanagement to fetch service config. "https://www.googleapis.com/auth/service.management.readonly", + // Call servicecontrol to get latest rollout id. + "https://www.googleapis.com/auth/servicecontrol", } tokenCache = &oauth2.Token{} tokenMux = sync.Mutex{} ) -func GenerateAccessTokenFromFile(saFilePath string) (string, time.Duration, error) { +var GenerateAccessTokenFromFile = func(saFilePath string) (string, time.Duration, error) { if token, duration := activeAccessToken(); token != "" { return token, duration, nil } @@ -85,3 +92,30 @@ func generateAccessToken(keyData []byte) (string, time.Duration, error) { tokenCache = token return token.AccessToken, token.Expiry.Sub(time.Now()), nil } + +// Create the token agent handler to provide envoy with access +// token generated by the service account credential. +// +// It follows the following scheme: +// Request: GET /local/access_token. +// Response: access token response is a JSON payload in the format: +// { +// "access_token": "string", +// "expires_in": uint +// } +func MakeTokenAgentHandler(serviceAccountKey string) http.Handler { + r := mux.NewRouter() + + r.PathPrefix(util.TokenAgentAccessTokenPath).Methods("GET").HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + token, expire, err := GenerateAccessTokenFromFile(serviceAccountKey) + + if err != nil { + http.Error(w, err.Error(), 500) + return + } + + _, _ = w.Write([]byte(fmt.Sprintf(`{"access_token": "%s", "expires_in": %v}`, token, int(expire.Seconds())))) + }) + + return r +} diff --git a/src/go/tokengenerator/token_generator_test.go b/src/go/tokengenerator/token_generator_test.go new file mode 100644 index 000000000..17ba725e3 --- /dev/null +++ b/src/go/tokengenerator/token_generator_test.go @@ -0,0 +1,110 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tokengenerator + +import ( + "fmt" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/GoogleCloudPlatform/esp-v2/src/go/util" + "github.com/GoogleCloudPlatform/esp-v2/src/go/util/testdata" + "github.com/GoogleCloudPlatform/esp-v2/tests/env/platform" + "github.com/GoogleCloudPlatform/esp-v2/tests/utils" +) + +func TestGenerateAccessToken(t *testing.T) { + + fakeToken := `{"access_token": "ya29.new", "expires_in":3599, "token_type":"Bearer"}` + mockTokenServer := util.InitMockServer(fakeToken) + defer mockTokenServer.Close() + + fakeKey := strings.Replace(testdata.FakeServiceAccountKeyData, "FAKE-TOKEN-URI", mockTokenServer.GetURL(), 1) + fakeKeyData := []byte(fakeKey) + + token, duration, err := generateAccessTokenFromData(fakeKeyData) + if token != "ya29.new" || duration.Seconds() < 3598 || err != nil { + t.Errorf("Test : Fail to make access token, got token: %s, duration: %v, err: %v", token, duration, err) + } + + latestFakeToken := `{"access_token": "ya29.latest", "expires_in":3599, "token_type":"Bearer"}` + mockTokenServer.SetResp(latestFakeToken) + + // The token is cached so the old token gets returned. + token, duration, err = generateAccessTokenFromData([]byte("Invalid data, not a service account")) + if token != "ya29.new" || err != nil { + t.Errorf("Test : Fail to make access token, got token: %s, duration: %v, err: %v", token, duration, err) + } +} + +func TestMakeTokenAgentHandler(t *testing.T) { + + s := httptest.NewServer(MakeTokenAgentHandler(platform.GetFilePath(platform.FakeServiceAccountFile))) + + testCases := []struct { + desc string + path string + genAccessTokenFromFile func(saFilePath string) (string, time.Duration, error) + method string + wantResp string + wantError string + }{ + { + desc: "success, get access token", + genAccessTokenFromFile: func(saFilePath string) (string, time.Duration, error) { + return "ya29.new", time.Duration(time.Second * 100), nil + }, + path: "/local/access_token", + method: "GET", + wantResp: `{"access_token": "ya29.new", "expires_in": 100}`, + }, + { + desc: "fail, error in generating access token", + genAccessTokenFromFile: func(saFilePath string) (string, time.Duration, error) { + return "", 0, fmt.Errorf("gen-access-token-error") + }, + path: "/local/access_token", + method: "GET", + wantError: "500 Internal Server Error, gen-access-token-error", + }, + { + desc: "fail, wrong path", + genAccessTokenFromFile: func(saFilePath string) (string, time.Duration, error) { + return "ya29.new", time.Duration(time.Second * 100), nil + }, + path: "/wrong-path", + method: "GET", + wantError: "404 Not Found", + }, + } + + for _, tc := range testCases { + GenerateAccessTokenFromFile = tc.genAccessTokenFromFile + _, resp, err := utils.DoWithHeaders(s.URL+tc.path, "GET", "", nil) + if tc.wantError != "" { + if err == nil || !strings.Contains(err.Error(), tc.wantError) { + t.Errorf("test(%s): get error: %v, want error: %s", tc.desc, err, tc.wantError) + } + } + + if tc.wantResp != "" && tc.wantResp != string(resp) { + t.Errorf("test(%s): get resp: %s, want resp %s", tc.desc, string(resp), tc.wantResp) + + } + + } +} diff --git a/src/go/tracing/tracing_test.go b/src/go/tracing/tracing_test.go index 26abecf3c..4053812cb 100644 --- a/src/go/tracing/tracing_test.go +++ b/src/go/tracing/tracing_test.go @@ -450,7 +450,7 @@ func runTest(_ *testing.T, shouldRunServer bool, f func()) { if shouldRunServer { // Run a mock server and point injected client to mock server mockMetadataServer := util.InitMockServerFromPathResp(map[string]string{ - util.ProjectIDSuffix: fakeMetadataProjectId, + util.ProjectIDPath: fakeMetadataProjectId, }) defer mockMetadataServer.Close() metadata.SetMockMetadataFetcher(mockMetadataServer.URL, time.Now()) diff --git a/src/go/util/httptest_util.go b/src/go/util/httptest_util.go index 64afe452a..87d911342 100644 --- a/src/go/util/httptest_util.go +++ b/src/go/util/httptest_util.go @@ -19,6 +19,7 @@ import ( "fmt" "net/http" "net/http/httptest" + "regexp" "strings" "time" ) @@ -91,6 +92,25 @@ func JsonEqual(want, got string) error { return nil } +// JsonContains should be used for test only. It can check whether partial +// target json string belong to the source json string. +// JsonContains will remove regex(`(\t|\n|\s)`) inside target so it shouldn't +// contains these chars in its content. +func JsonContains(source, target string) error { + normalizedSource, err := normalizeJson(source) + if err != nil { + return err + } + + var re = regexp.MustCompile(`(\t|\n|\s)`) + normalizedTarget := re.ReplaceAllString(target, "") + + if !strings.Contains(normalizedSource, normalizedTarget) { + return fmt.Errorf("source doesn't contain the target,\nsource: %s,\ntarget: %s\n", normalizedSource, normalizedTarget) + } + return nil +} + // normalizeJson returns normalized JSON string. func normalizeJson(input string) (string, error) { var jsonObject map[string]interface{} diff --git a/src/go/util/token_generator_test.go b/src/go/util/token_generator_test.go deleted file mode 100644 index b6d0604ad..000000000 --- a/src/go/util/token_generator_test.go +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2019 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package util - -import ( - "strings" - "testing" - - "github.com/GoogleCloudPlatform/esp-v2/src/go/util/testdata" -) - -func TestGenerateAccessToken(t *testing.T) { - - fakeToken := `{"access_token": "ya29.new", "expires_in":3599, "token_type":"Bearer"}` - mockTokenServer := InitMockServer(fakeToken) - defer mockTokenServer.Close() - - fakeKey := strings.Replace(testdata.FakeServiceAccountKeyData, "FAKE-TOKEN-URI", mockTokenServer.GetURL(), 1) - fakeKeyData := []byte(fakeKey) - - token, duration, err := generateAccessTokenFromData(fakeKeyData) - if token != "ya29.new" || duration.Seconds() < 3598 || err != nil { - t.Errorf("Test : Fail to make access token, got token: %s, duration: %v, err: %v", token, duration, err) - } - - latestFakeToken := `{"access_token": "ya29.latest", "expires_in":3599, "token_type":"Bearer"}` - mockTokenServer.SetResp(latestFakeToken) - - // The token is cached so the old token gets returned. - token, duration, err = generateAccessTokenFromData([]byte("Invalid data, not a service account")) - if token != "ya29.new" || err != nil { - t.Errorf("Test : Fail to make access token, got token: %s, duration: %v, err: %v", token, duration, err) - } -} diff --git a/src/go/util/url_util.go b/src/go/util/url_util.go index bc823ffcd..66a004efc 100644 --- a/src/go/util/url_util.go +++ b/src/go/util/url_util.go @@ -161,11 +161,11 @@ func ResolveJwksUriUsingOpenID(uri string) (string, error) { return jwksURI, nil } -func IamIdentityTokenSuffix(IamServiceAccount string) string { +func IamIdentityTokenPath(IamServiceAccount string) string { return fmt.Sprintf("/v1/projects/-/serviceAccounts/%s:generateIdToken", IamServiceAccount) } -func IamAccessTokenSuffix(IamServiceAccount string) string { +func IamAccessTokenPath(IamServiceAccount string) string { return fmt.Sprintf("/v1/projects/-/serviceAccounts/%s:generateAccessToken", IamServiceAccount) } diff --git a/src/go/util/util.go b/src/go/util/util.go index 87cc38488..82b36e645 100644 --- a/src/go/util/util.go +++ b/src/go/util/util.go @@ -77,16 +77,19 @@ const ( // Metadata suffix - ConfigIDSuffix = "/v1/instance/attributes/endpoints-service-version" - GAEServerSoftwareSuffix = "/v1/instance/attributes/gae_server_software" - KubeEnvSuffix = "/v1/instance/attributes/kube-env" - RolloutStrategySuffix = "/v1/instance/attributes/endpoints-rollout-strategy" - ServiceNameSuffix = "/v1/instance/attributes/endpoints-service-name" + ConfigIDPath = "/computeMetadata/v1/instance/attributes/endpoints-service-version" + GAEServerSoftwarePath = "/computeMetadata/v1/instance/attributes/gae_server_software" + KubeEnvPath = "/computeMetadata/v1/instance/attributes/kube-env" + RolloutStrategyPath = "/computeMetadata/v1/instance/attributes/endpoints-rollout-strategy" + ServiceNamePath = "/computeMetadata/v1/instance/attributes/endpoints-service-name" - AccessTokenSuffix = "/v1/instance/service-accounts/default/token" - IdentityTokenSuffix = "/v1/instance/service-accounts/default/identity" - ProjectIDSuffix = "/v1/project/project-id" - ZoneSuffix = "/v1/instance/zone" + AccessTokenPath = "/computeMetadata/v1/instance/service-accounts/default/token" + IdentityTokenPath = "/computeMetadata/v1/instance/service-accounts/default/identity" + ProjectIDPath = "/computeMetadata/v1/project/project-id" + ZonePath = "/computeMetadata/v1/instance/zone" + + // The path of getting access token from token agent server + TokenAgentAccessTokenPath = "/local/access_token" // b/147591854: This string must NOT have a trailing slash OpenIDDiscoveryCfgURLSuffix = "/.well-known/openid-configuration" @@ -94,6 +97,9 @@ const ( // The metadata server cluster name. MetadataServerClusterName = "metadata-cluster" + // The token agent server cluster name. + TokenAgentClusterName = "token-agent-cluster" + // The iam server cluster name. IamServerClusterName = "iam-cluster" @@ -104,7 +110,6 @@ const ( LoopbackListenerName = "loopback_listener" // Platforms - GAEFlex = "GAE_FLEX(ESPv2)" GKE = "GKE(ESPv2)" GCE = "GCE(ESPv2)" @@ -139,6 +144,9 @@ const ( // Standard type url prefix. TypeUrlPrefix = "type.googleapis.com/" + + // Loopback Address + LoopbackIPv4Addr = "127.0.0.1" ) type BackendProtocol int32 diff --git a/tests/e2e/scripts/cloud-run/deploy.sh b/tests/e2e/scripts/cloud-run/deploy.sh index 15f567783..f46cfb7af 100755 --- a/tests/e2e/scripts/cloud-run/deploy.sh +++ b/tests/e2e/scripts/cloud-run/deploy.sh @@ -36,7 +36,7 @@ PROJECT_ID="cloudesf-testing" TEST_ID="cloud-run-${BACKEND}" PROXY_RUNTIME_SERVICE_ACCOUNT="e2e-cloud-run-proxy-rt@${PROJECT_ID}.iam.gserviceaccount.com" BACKEND_RUNTIME_SERVICE_ACCOUNT="e2e-${BACKEND_PLATFORM}-backend-rt@${PROJECT_ID}.iam.gserviceaccount.com" -JOB_KEY_PATH="${ROOT}/tests/e2e/client/gob-prow-jobs-secret.json" +JOB_KEY_PATH="$(mktemp /tmp/servie_account_cred.XXXX)" LOG_DIR="$(mktemp -d /tmp/log.XXXX)" # Determine names of all resources diff --git a/tests/e2e/scripts/gke/deploy.sh b/tests/e2e/scripts/gke/deploy.sh index cc62016eb..a2f633b5c 100755 --- a/tests/e2e/scripts/gke/deploy.sh +++ b/tests/e2e/scripts/gke/deploy.sh @@ -60,9 +60,15 @@ case "${BACKEND}" in *) echo "Invalid backend ${BACKEND}" return 1 ;; - esac +if [ ${BACKEND} == "bookstore" ]; then + SA_CRED_PATH="$(mktemp /tmp/servie_account_cred.XXXX)" + + # This file mount path is set in tests/e2e/testdata/bookstore/gke/http-bookstore.yaml + [[ -n ${USING_SA_CRED} ]] && ARGS="$ARGS, \"--service_account_key=/etc/creds/$(basename "${SA_CRED_PATH}")\"" +fi + sed "s|APIPROXY_IMAGE|${APIPROXY_IMAGE}|g" ${YAML_TEMPLATE} \ | sed "s|ARGS|${ARGS}|g" | tee ${YAML_FILE} @@ -72,7 +78,14 @@ sed "s|APIPROXY_IMAGE|${APIPROXY_IMAGE}|g" ${YAML_TEMPLATE} \ # case "${BACKEND}" in 'bookstore') - SERVICE_IDL="${ROOT}/tests/endpoints/bookstore/bookstore_swagger_template.json" + SERVICE_IDL_TMPL="${ROOT}/tests/endpoints/bookstore/bookstore_swagger_template.json" + SERVICE_IDL="${ROOT}/tests/endpoints/bookstore/bookstore_swagger.json" + + cat "${SERVICE_IDL_TMPL}" \ + | jq ".host = \"${APIPROXY_SERVICE}\" \ + | .securityDefinitions.auth0_jwk.\"x-google-audiences\" = \"${APIPROXY_SERVICE}\"" \ + > "${SERVICE_IDL}" + CREATE_SERVICE_ARGS="${SERVICE_IDL}" ;; 'echo') @@ -87,9 +100,11 @@ case "${BACKEND}" in ARGS="$ARGS -g" ;; *) echo "Invalid backend ${BACKEND}" - return 1 ;; + exit 1;; esac + + LOG_DIR="$(mktemp -d /tmp/log.XXXX)" create_service ${CREATE_SERVICE_ARGS} @@ -97,6 +112,12 @@ create_service ${CREATE_SERVICE_ARGS} # Creates service on GKE cluster. NAMESPACE="${UNIQUE_ID}" run kubectl create namespace "${NAMESPACE}" || error_exit "Namespace already exists" + +if [ "${BACKEND}" == 'bookstore' ]; then + get_test_client_key "e2e-non-gcp-instance-proxy-rt-sa.json" "${SA_CRED_PATH}" + run kubectl create secret generic service-account-cred --from-file="${SA_CRED_PATH}" --namespace "${NAMESPACE}" +fi + run kubectl create -f ${YAML_FILE} --namespace "${NAMESPACE}" HOST=$(get_cluster_host "${NAMESPACE}") diff --git a/tests/e2e/scripts/prow-utilities.sh b/tests/e2e/scripts/prow-utilities.sh index 38a8cd7a9..1dc3bd777 100755 --- a/tests/e2e/scripts/prow-utilities.sh +++ b/tests/e2e/scripts/prow-utilities.sh @@ -25,7 +25,7 @@ set -eo pipefail # End to End tests common options function e2e_options() { local OPTIND OPTARG arg - while getopts :a:b:B:m:g:i:k:l:r:p:R:s:t:v:V:f: arg; do + while getopts :a:b:B:m:g:i:k:l:r:p:R:s:S:t:v:V:f: arg; do case ${arg} in a) APIPROXY_SERVICE="${OPTARG}" ;; b) BOOKSTORE_IMAGE="${OPTARG}" ;; @@ -39,6 +39,7 @@ function e2e_options() { p) PROXY_PLATFORM="${OPTARG}" ;; R) ROLLOUT_STRATEGY="${OPTARG}" ;; s) SKIP_CLEANUP='true' ;; + S) USING_SA_CRED="${OPTARG}" ;; t) TEST_TYPE="$(echo ${OPTARG} | tr '[A-Z]' '[a-z]')" ;; v) VM_IMAGE="${OPTARG}" ;; V) ENDPOINTS_RUNTIME_VERSION="${OPTARG}" ;; @@ -294,7 +295,11 @@ function download_client_binaries() { function get_apiproxy_service() { if [[ "${1}" == "bookstore" ]]; then - echo "bookstore.endpoints.cloudesf-testing.cloud.goog" + if [ -n "${2}" ]; then + echo "bookstore-using-sa-cred.endpoints.cloudesf-testing.cloud.goog" + else + echo "bookstore.endpoints.cloudesf-testing.cloud.goog" + fi elif [[ "${1}" == "echo" ]]; then echo "echo.endpoints.cloudesf-testing.cloud.goog" elif [[ "${1}" == "interop" ]]; then diff --git a/tests/e2e/testdata/bookstore/gke/http-bookstore.yaml.template b/tests/e2e/testdata/bookstore/gke/http-bookstore.yaml.template index d1ade46c8..2b4b7b2ad 100644 --- a/tests/e2e/testdata/bookstore/gke/http-bookstore.yaml.template +++ b/tests/e2e/testdata/bookstore/gke/http-bookstore.yaml.template @@ -42,12 +42,21 @@ spec: labels: app: app spec: + volumes: + - name: service-account-cred + secret: + secretName: service-account-cred containers: - name: apiproxy image: APIPROXY_IMAGE args: [ARGS] ports: - containerPort: 8080 + volumeMounts: + - mountPath: /etc/creds + name: service-account-cred + readOnly: true + imagePullPolicy: Always - name: bookstore image: gcr.io/cloudesf-testing/http-bookstore:2 ports: diff --git a/tests/env/components/mock_metadata.go b/tests/env/components/mock_metadata.go index 046b3ef5a..5e6956f8f 100644 --- a/tests/env/components/mock_metadata.go +++ b/tests/env/components/mock_metadata.go @@ -34,12 +34,12 @@ const ( ) var defaultResp = map[string]string{ - util.ConfigIDSuffix: fakeConfigID, - util.ServiceNameSuffix: fakeServiceName, - util.AccessTokenSuffix: fakeToken, - util.IdentityTokenSuffix: fakeIdentityToken, - util.ProjectIDSuffix: FakeProjectID, - util.ZoneSuffix: fakeZonePath, + util.ConfigIDPath: fakeConfigID, + util.ServiceNamePath: fakeServiceName, + util.AccessTokenPath: fakeToken, + util.IdentityTokenPath: fakeIdentityToken, + util.ProjectIDPath: FakeProjectID, + util.ZonePath: fakeZonePath, } // MockMetadataServer mocks the Metadata server. diff --git a/tests/env/components/mock_metadata_test.go b/tests/env/components/mock_metadata_test.go index a461b3f68..4f22569e0 100644 --- a/tests/env/components/mock_metadata_test.go +++ b/tests/env/components/mock_metadata_test.go @@ -72,7 +72,7 @@ func TestMockMetadata(t *testing.T) { }, { desc: "Success, query hard-code metadata", - url: util.ConfigIDSuffix, + url: util.ConfigIDPath, wantedResp: fakeConfigID, }, { diff --git a/tests/env/components/ports.go b/tests/env/components/ports.go index 740d303ee..650ee9c24 100644 --- a/tests/env/components/ports.go +++ b/tests/env/components/ports.go @@ -85,7 +85,8 @@ const ( TestPreflightCorsWithBasicPreset TestPreflightRequestWithAllowCors TestReportGCPAttributes - TestServiceControlAccessToken + TestServiceControlAccessTokenFromIam + TestServiceControlAccessTokenFromTokenAgent TestServiceControlAllHTTPMethod TestServiceControlAllHTTPPath TestServiceControlAPIKeyCustomLocation diff --git a/tests/env/platform/files.go b/tests/env/platform/files.go index be322a98b..05f68b436 100644 --- a/tests/env/platform/files.go +++ b/tests/env/platform/files.go @@ -45,6 +45,7 @@ const ( LogMetrics Version AccessLog + ServiceAccountFile TestRootCaCerts // Configurations from examples directory @@ -61,6 +62,7 @@ const ( // Other configurations for testing FixedDrServiceConfig + FakeServiceAccountFile ) // go tests are not executed from the root fo the repository. @@ -105,8 +107,9 @@ var fileMap = map[RuntimeFile]string{ GrpcEchoEnvoyConfig: "../../../../examples/grpc_dynamic_routing/envoy_config.json", // Used by other unit tests. - TestRootCaCerts: "../../../tests/env/testdata/roots.pem", - FixedDrServiceConfig: "../../../tests/env/testdata/service_config_for_fixed_dynamic_routing.json", + TestRootCaCerts: "../../../tests/env/testdata/roots.pem", + FixedDrServiceConfig: "../../../tests/env/testdata/service_config_for_fixed_dynamic_routing.json", + FakeServiceAccountFile: "./service_account.json", } // Get the runtime file path for the specified file. diff --git a/tests/integration_test/backend_auth_disable_auth_test.go b/tests/integration_test/backend_auth_disable_auth_test.go index 2bbc79923..b151887e2 100644 --- a/tests/integration_test/backend_auth_disable_auth_test.go +++ b/tests/integration_test/backend_auth_disable_auth_test.go @@ -33,8 +33,8 @@ func TestBackendAuthDisableAuth(t *testing.T) { s := env.NewTestEnv(comp.TestBackendAuthDisableAuth, platform.EchoRemote) s.OverrideMockMetadata( map[string]string{ - util.IdentityTokenSuffix + "?format=standard&audience=https://localhost/bearertoken/constant": "ya29.JwtAudienceSet", - util.IdentityTokenSuffix + "?format=standard&audience=https://localhost": "ya29.DefaultAuth", + util.IdentityTokenPath + "?format=standard&audience=https://localhost/bearertoken/constant": "ya29.JwtAudienceSet", + util.IdentityTokenPath + "?format=standard&audience=https://localhost": "ya29.DefaultAuth", }, 0) defer s.TearDown(t) diff --git a/tests/integration_test/backend_auth_using_iam_test.go b/tests/integration_test/backend_auth_using_iam_test.go index d91dba68d..a0393aada 100644 --- a/tests/integration_test/backend_auth_using_iam_test.go +++ b/tests/integration_test/backend_auth_using_iam_test.go @@ -39,8 +39,8 @@ func TestBackendAuthWithIamIdToken(t *testing.T) { s.SetBackendAuthIamServiceAccount(serviceAccount) s.SetIamResps( map[string]string{ - fmt.Sprintf("%s?audience=https://localhost/bearertoken/constant", util.IamIdentityTokenSuffix(serviceAccount)): `{"token": "id-token-for-constant"}`, - fmt.Sprintf("%s?audience=https://localhost/bearertoken/append", util.IamIdentityTokenSuffix(serviceAccount)): `{"token": "id-token-for-append"}`, + fmt.Sprintf("%s?audience=https://localhost/bearertoken/constant", util.IamIdentityTokenPath(serviceAccount)): `{"token": "id-token-for-constant"}`, + fmt.Sprintf("%s?audience=https://localhost/bearertoken/append", util.IamIdentityTokenPath(serviceAccount)): `{"token": "id-token-for-append"}`, }, 0, 0) defer s.TearDown(t) @@ -117,7 +117,7 @@ func TestBackendAuthWithIamIdTokenRetries(t *testing.T) { func() { s.SetIamResps( map[string]string{ - fmt.Sprintf("%s?audience=https://localhost/bearertoken/constant", util.IamIdentityTokenSuffix(serviceAccount)): `{"token": "id-token-for-constant"}`, + fmt.Sprintf("%s?audience=https://localhost/bearertoken/constant", util.IamIdentityTokenPath(serviceAccount)): `{"token": "id-token-for-constant"}`, }, tc.wantNumFails, 0) defer s.TearDown(t) @@ -205,7 +205,7 @@ func TestBackendAuthWithIamIdTokenTimeouts(t *testing.T) { // Setup IAM with the iam response time. s.SetIamResps( map[string]string{ - fmt.Sprintf("%s?audience=https://localhost/bearertoken/constant", util.IamIdentityTokenSuffix(serviceAccount)): `{"token": "id-token-for-constant"}`, + fmt.Sprintf("%s?audience=https://localhost/bearertoken/constant", util.IamIdentityTokenPath(serviceAccount)): `{"token": "id-token-for-constant"}`, }, 0, tc.iamResponseTime) // Setup ESPv2 with the http request timeout (used for making calls to IAM). diff --git a/tests/integration_test/backend_auth_using_imds_test.go b/tests/integration_test/backend_auth_using_imds_test.go index 94f886868..c33493319 100644 --- a/tests/integration_test/backend_auth_using_imds_test.go +++ b/tests/integration_test/backend_auth_using_imds_test.go @@ -35,8 +35,8 @@ func TestBackendAuthWithImdsIdToken(t *testing.T) { s := env.NewTestEnv(comp.TestBackendAuthWithImdsIdToken, platform.EchoRemote) s.OverrideMockMetadata( map[string]string{ - util.IdentityTokenSuffix + "?format=standard&audience=https://localhost/bearertoken/constant": "ya29.constant", - util.IdentityTokenSuffix + "?format=standard&audience=https://localhost/bearertoken/append": "ya29.append", + util.IdentityTokenPath + "?format=standard&audience=https://localhost/bearertoken/constant": "ya29.constant", + util.IdentityTokenPath + "?format=standard&audience=https://localhost/bearertoken/append": "ya29.append", }, 0) defer s.TearDown(t) @@ -118,7 +118,7 @@ func TestBackendAuthWithImdsIdTokenRetries(t *testing.T) { func() { s.OverrideMockMetadata( map[string]string{ - util.IdentityTokenSuffix + "?format=standard&audience=https://localhost/bearertoken/constant": "ya29.constant", + util.IdentityTokenPath + "?format=standard&audience=https://localhost/bearertoken/constant": "ya29.constant", }, tc.wantNumFails) defer s.TearDown(t) @@ -164,8 +164,8 @@ func TestBackendAuthWithImdsIdTokenWhileAllowCors(t *testing.T) { s := env.NewTestEnv(comp.TestBackendAuthWithImdsIdTokenWhileAllowCors, platform.EchoRemote) s.OverrideMockMetadata( map[string]string{ - util.IdentityTokenSuffix + "?format=standard&audience=https://localhost/bearertoken/constant": "ya29.constant", - util.IdentityTokenSuffix + "?format=standard&audience=https://localhost/bearertoken/append": "ya29.append", + util.IdentityTokenPath + "?format=standard&audience=https://localhost/bearertoken/constant": "ya29.constant", + util.IdentityTokenPath + "?format=standard&audience=https://localhost/bearertoken/append": "ya29.append", }, 0) s.SetAllowCors() diff --git a/tests/integration_test/jwt_auth_integration_test.go b/tests/integration_test/jwt_auth_integration_test.go index e22aa7f8d..fe4a0640e 100644 --- a/tests/integration_test/jwt_auth_integration_test.go +++ b/tests/integration_test/jwt_auth_integration_test.go @@ -276,7 +276,7 @@ func TestFrontendAndBackendAuthHeaders(t *testing.T) { }) s.OverrideMockMetadata( map[string]string{ - util.IdentityTokenSuffix + "?format=standard&audience=https://localhost/bearertoken/constant": "ya29.BackendAuthToken", + util.IdentityTokenPath + "?format=standard&audience=https://localhost/bearertoken/constant": "ya29.BackendAuthToken", }, 0) defer s.TearDown(t) diff --git a/tests/integration_test/report_gcp_attributes_test.go b/tests/integration_test/report_gcp_attributes_test.go index d31d285ab..ca559cdfb 100644 --- a/tests/integration_test/report_gcp_attributes_test.go +++ b/tests/integration_test/report_gcp_attributes_test.go @@ -45,7 +45,7 @@ func TestReportGCPAttributes(t *testing.T) { { desc: "Valid Zone", mockMetadataOverride: map[string]string{ - util.ZoneSuffix: "projects/4242424242/zones/us-west-1b", + util.ZonePath: "projects/4242424242/zones/us-west-1b", }, wantLocation: "us-west-1b", wantPlatform: "GCE(ESPv2)", @@ -53,7 +53,7 @@ func TestReportGCPAttributes(t *testing.T) { { desc: "Invalid Zone - without '/'", mockMetadataOverride: map[string]string{ - util.ZoneSuffix: "some-invalid-zone", + util.ZonePath: "some-invalid-zone", }, wantLocation: "", wantPlatform: "GCE(ESPv2)", @@ -61,7 +61,7 @@ func TestReportGCPAttributes(t *testing.T) { { desc: "Invalid Zone - ends with '/'", mockMetadataOverride: map[string]string{ - util.ZoneSuffix: "project/123123/", + util.ZonePath: "project/123123/", }, wantLocation: "", wantPlatform: "GCE(ESPv2)", @@ -69,7 +69,7 @@ func TestReportGCPAttributes(t *testing.T) { { desc: "Platform - GAE FLEX", mockMetadataOverride: map[string]string{ - util.GAEServerSoftwareSuffix: "gae", + util.GAEServerSoftwarePath: "gae", }, wantLocation: "test-zone", wantPlatform: "GAE_FLEX(ESPv2)", @@ -77,7 +77,7 @@ func TestReportGCPAttributes(t *testing.T) { { desc: "Platform - GKE", mockMetadataOverride: map[string]string{ - util.KubeEnvSuffix: "kube-env", + util.KubeEnvPath: "kube-env", }, wantLocation: "test-zone", wantPlatform: "GKE(ESPv2)", @@ -92,8 +92,8 @@ func TestReportGCPAttributes(t *testing.T) { { desc: "Platform and Zone", mockMetadataOverride: map[string]string{ - util.ZoneSuffix: "projects/4242424242/zones/us-west-1b", - util.GAEServerSoftwareSuffix: "gae", + util.ZonePath: "projects/4242424242/zones/us-west-1b", + util.GAEServerSoftwarePath: "gae", }, wantLocation: "us-west-1b", wantPlatform: "GAE_FLEX(ESPv2)", diff --git a/tests/integration_test/service_control_access_token_test.go b/tests/integration_test/service_control_access_token_test.go index 53e9ca660..c436ca942 100644 --- a/tests/integration_test/service_control_access_token_test.go +++ b/tests/integration_test/service_control_access_token_test.go @@ -16,16 +16,20 @@ package integration_test import ( "fmt" + "io/ioutil" + "os" + "strings" "testing" + "github.com/GoogleCloudPlatform/esp-v2/src/go/util" + "github.com/GoogleCloudPlatform/esp-v2/src/go/util/testdata" "github.com/GoogleCloudPlatform/esp-v2/tests/endpoints/echo/client" "github.com/GoogleCloudPlatform/esp-v2/tests/env" - "github.com/GoogleCloudPlatform/esp-v2/tests/env/platform" - comp "github.com/GoogleCloudPlatform/esp-v2/tests/env/components" + "github.com/GoogleCloudPlatform/esp-v2/tests/env/platform" ) -func TestServiceControlAccessToken(t *testing.T) { +func TestServiceControlAccessTokenFromIam(t *testing.T) { t.Parallel() configId := "test-config-id" @@ -33,7 +37,7 @@ func TestServiceControlAccessToken(t *testing.T) { args := []string{"--service_config_id=" + configId, "--rollout_strategy=fixed", "--suppress_envoy_headers"} - s := env.NewTestEnv(comp.TestServiceControlAccessToken, platform.EchoSidecar) + s := env.NewTestEnv(comp.TestServiceControlAccessTokenFromIam, platform.EchoSidecar) serviceAccount := "ServiceAccount@google.com" s.SetServiceControlIamServiceAccount(serviceAccount) s.SetServiceControlIamDelegates("delegate_foo,delegate_bar,delegate_baz") @@ -98,3 +102,60 @@ func TestServiceControlAccessToken(t *testing.T) { } } } + +func TestServiceControlAccessTokenFromTokenAgent(t *testing.T) { + t.Parallel() + + // Setup token server which will be queried by configmanager. + fakeToken := `{"access_token": "this-is-sa_gen_token", "expires_in":3599, "token_type":"Bearer"}` + mockTokenServer := util.InitMockServer(fakeToken) + defer mockTokenServer.Close() + + fakeKey := strings.Replace(testdata.FakeServiceAccountKeyData, "FAKE-TOKEN-URI", mockTokenServer.GetURL(), 1) + serviceAccountFile, err := ioutil.TempFile(os.TempDir(), "sa-cred-") + if err != nil { + t.Fatal("fail to create a temp service account file") + } + _ = ioutil.WriteFile(serviceAccountFile.Name(), []byte(fakeKey), 0644) + + args := []string{"--service_config_id=test-config-id", + "--rollout_strategy=fixed", "--suppress_envoy_headers", "--service_account_key=" + serviceAccountFile.Name()} + + s := env.NewTestEnv(comp.TestServiceControlAccessTokenFromTokenAgent, platform.EchoSidecar) + + defer s.TearDown(t) + if err := s.Setup(args); err != nil { + t.Fatalf("fail to setup test env, %v", err) + } + testData := []struct { + desc string + url string + method string + wantScRequestAccessToken string + }{ + { + desc: "succeed, fetching access token from local server", + url: fmt.Sprintf("http://localhost:%v%v%v", s.Ports().ListenerPort, "/echo", "?key=api-key"), + method: "POST", + wantScRequestAccessToken: "Bearer this-is-sa_gen_token", + }, + } + for _, tc := range testData { + _, err := client.DoWithHeaders(tc.url, tc.method, "message", nil) + if err != nil { + t.Fatalf("Test (%s): failed, %v", tc.desc, err) + } + + // The check call and the report call will be sent. + scRequests, err := s.ServiceControlServer.GetRequests(2) + if err != nil { + t.Fatalf("Test (%s): failed, GetRequests returns error: %v", tc.desc, err) + } + + for _, scRequest := range scRequests { + if gotAccessToken := scRequest.ReqHeader.Get("Authorization"); gotAccessToken != tc.wantScRequestAccessToken { + t.Errorf("Test (%s): failed, different access token received by service controller, expected: %v, but got: %v", tc.desc, tc.wantScRequestAccessToken, gotAccessToken) + } + } + } +} diff --git a/tests/integration_test/service_control_check_fail_test.go b/tests/integration_test/service_control_check_fail_test.go index 5c4597d1b..f999ace89 100644 --- a/tests/integration_test/service_control_check_fail_test.go +++ b/tests/integration_test/service_control_check_fail_test.go @@ -170,7 +170,7 @@ func TestServiceControlCheckError(t *testing.T) { }, }, // Note: first request is from Config Manager, second is from ESPv2 - wantRequestsToMetaServer: &expectedRequestCount{"/v1/instance/service-accounts/default/token", 2}, + wantRequestsToMetaServer: &expectedRequestCount{util.AccessTokenPath, 2}, wantRequestsToProvider: &expectedRequestCount{provider, 1}, wantError: `400 Bad Request, {"code":400,"message":"INVALID_ARGUMENT:Client project not valid. Please pass a valid project."}`, wantScRequests: []interface{}{ diff --git a/tests/utils/http.go b/tests/utils/http.go index fb7835cee..e6149d1ca 100644 --- a/tests/utils/http.go +++ b/tests/utils/http.go @@ -88,7 +88,10 @@ func DoWithHeaders(url, method, message string, headers map[string]string) (http } if resp.StatusCode != http.StatusOK { - return nil, nil, fmt.Errorf("http response status is not 200 OK: %s, %s", resp.Status, RpcStatusDeterministicJsonFormat(bodyBytes)) + if resp.Header.Get("Content-Type") == "application/json" { + return nil, nil, fmt.Errorf("http response status is not 200 OK: %s, %s", resp.Status, RpcStatusDeterministicJsonFormat(bodyBytes)) + } + return nil, nil, fmt.Errorf("http response status is not 200 OK: %s, %s", resp.Status, bodyBytes) } return resp.Header, bodyBytes, err }