b/148306474,b/156410254: move generating access token by service acco…

…unt credential to configmanager (#251)

api/envoy/v7/http/common/base.proto

@@ -67,21 +67,9 @@ message HttpUri {
   }];
 }
 
-// Duplicate DataSource from envoy for self containment.
-// https://aip.dev/213
-message DataSource {
-  oneof specifier {
-    option (validate.required) = true;
-
-    // Local filesystem data source.
-    string filename = 1 [(validate.rules).string.min_bytes = 1];
-
-    // String inlined in the configuration.
-    string inline_string = 2 [(validate.rules).string.min_bytes = 1];
-  }
-}
-
 message AccessToken {
+  reserved 2;
+
   oneof token_type {
     option (validate.required) = true;
 
@@ -91,9 +79,6 @@ message AccessToken {
     //   Query parameters are added by the filter
     // - Token cluster address to fetch JWT token.
     HttpUri remote_token = 1;
-
-    // The local path or inline content of the service account json file.
-    DataSource service_account_secret = 2;
   }
 }
 

api/envoy/v7/http/service_control/config.proto

@@ -98,6 +98,8 @@ message GcpAttributes {
 }
 
 message FilterConfig {
+  reserved 5;
+
   // A list of services supported on this Envoy server.
   repeated Service services = 1;  // ref:multi-service
 
@@ -110,14 +112,10 @@ message FilterConfig {
   oneof access_token {
     option (validate.required) = true;
 
-    // The Instance Metadata Server uri used to fetch access token from Instance
-    // Metadata Server.
+    // Uri used to fetch access token from Instance Metadata Server or the local
+    // token agent server.
     espv2.api.envoy.v7.http.common.HttpUri imds_token = 4;
 
-    // The local path or inline content of the service account json file used to
-    // generate access token.
-    espv2.api.envoy.v7.http.common.DataSource service_account_secret = 5;
-
     // Information used to fetch access token from Google Cloud IAM.
     espv2.api.envoy.v7.http.common.IamTokenInfo iam_token = 6;
   }

examples/grpc_dynamic_routing/envoy_config.json

@@ -508,7 +508,7 @@
                         "serviceControlUri": {
                           "cluster": "service-control-cluster",
                           "timeout": "30s",
-                          "uri": "https://servicecontrol.googleapis.com/v1/services/"
+                          "uri": "https://servicecontrol.googleapis.com/v1/services"
                         },
                         "services": [
                           {

examples/service_control/envoy_config.json

@@ -200,7 +200,7 @@
                         "serviceControlUri": {
                           "cluster": "service-control-cluster",
                           "timeout": "30s",
-                          "uri": "https://servicecontrol.googleapis.com/v1/services/"
+                          "uri": "https://servicecontrol.googleapis.com/v1/services"
                         },
                         "services": [
                           {

go.mod

@@ -16,6 +16,7 @@ require (
 	github.com/miekg/dns v1.1.29
 	golang.org/x/net v0.0.0-20190923162816-aa69164e4478
 	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
+	golang.org/x/tools v0.0.0-20191216052735-49a3e744a425
 	google.golang.org/api v0.7.0
 	google.golang.org/genproto v0.0.0-20200617032506-f1bdc9086088
 	google.golang.org/grpc v1.27.0

go.sum

@@ -138,6 +138,7 @@ golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBn
 golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0 h1:Dh6fw+p6FyRl5x/FvNswO1ji0lIGzm3KP8Y9VkS9PTE=
 golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20191216052735-49a3e744a425 h1:VvQyQJN0tSuecqgcIxMWnnfG5kSmgy9KZR9sW3W5QeA=
 golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=

prow/gcpproxy-e2e.sh

@@ -28,19 +28,20 @@ exit 1; }
 
 function runE2E() {
   local OPTIND OPTARG arg
-  while getopts :f:p:c:g:m:R:t: arg; do
+  while getopts :f:p:c:g:m:R:St: arg; do
     case ${arg} in
       f) local backend_platform="${OPTARG}" ;;
       p) local platform="${OPTARG}" ;;
       c) local coupling_option="$(echo ${OPTARG} | tr '[A-Z]' '[a-z]')" ;;
       g) local backend="${OPTARG}" ;;
       m) local apiproxy_image="${OPTARG}" ;;
       R) local rollout_strategy="${OPTARG}" ;;
+      S) local using_sa_cred='true';;
       t) local test_type="$(echo ${OPTARG} | tr '[A-Z]' '[a-z]')" ;;
     esac
   done
 
-  local apiproxy_service=$(get_apiproxy_service ${backend})
+  local apiproxy_service=$(get_apiproxy_service "${backend}" "${using_sa_cred}")
   local unique_id=$(get_unique_id "gke-${test_type}-${backend}")
   if [ "${platform}" == "anthos-cloud-run" ]
   then
@@ -61,7 +62,8 @@ function runE2E() {
     -i "${unique_id}"  \
     -B "${BUCKET}"  \
     -l "${DURATION_IN_HOUR}" \
-    -f "${backend_platform}"
+    -f "${backend_platform}" \
+    -S "${using_sa_cred}"
 }
 
 if [ ! -d "$GOPATH/bin" ]; then
@@ -87,6 +89,9 @@ case ${TEST_CASE} in
   "tight-http-bookstore-managed")
     runE2E -p "gke" -c "tight" -t "http" -g "bookstore" -R "managed" -m "$(get_proxy_image_name_with_sha)"
     ;;
+    "tight-http-bookstore-managed-using-sa-cred")
+    runE2E -p "gke" -c "tight" -t "http" -g "bookstore" -R "managed" -S -m "$(get_proxy_image_name_with_sha)"
+    ;;
   "tight-grpc-echo-managed")
     runE2E -p "gke" -c "tight" -t "grpc" -g "echo" -R "managed" -m "$(get_proxy_image_name_with_sha)"
     ;;

scripts/all-utilities.sh

@@ -142,8 +142,7 @@ function set_api_keys() {
 function get_test_client_key() {
   local remote_file_name=$1
   local key_path=$2
-  [[ -e $key_path ]] || $GSUTIL  \
-    cp "gs://apiproxy-testing-client-secret-files/$remote_file_name" "$key_path"
+  $GSUTIL  cp "gs://apiproxy-testing-client-secret-files/$remote_file_name" "$key_path"
   echo -n "$key_path"
   return 0
 }
@@ -326,7 +325,7 @@ function get_gcsrunner_image_release_name() {
 function get_tag_name() {
   local tag_format="%H"
   tag_name="$(git show -q HEAD --pretty=format:"${tag_format}")"
-  echo -n ${tag_name}
+  echo -n "${tag_name}"
 }
 
 function get_envoy_image_name_with_sha() {

src/envoy/http/backend_auth/config_parser_impl.cc

@@ -107,11 +107,6 @@ FilterConfigParserImpl::FilterConfigParserImpl(
                 });
         break;
       }
-      case AccessToken::TokenTypeCase::kServiceAccountSecret:
-        // TODO(taoxuy): support getting access token from service account file.
-        ENVOY_LOG(error,
-                  "Not support getting access token by service account file");
-        return;
       default:
         NOT_REACHED_GCOVR_EXCL_LINE;
     }

src/envoy/http/backend_auth/config_parser_impl_test.cc

@@ -64,26 +64,6 @@ rules {
                           "Duplicated operation");
 }
 
-TEST_F(ConfigParserImplTest, IamIdTokenWithServiceAccountAsAccessToken) {
-  const char filter_config[] = R"(
-iam_token {
-  access_token {
-    service_account_secret{}
-  }
-}
-rules {
-  operation: "append-with-audience"
-  jwt_audience: "this-is-audience"
-}
-)";
-
-  EXPECT_CALL(mock_token_subscriber_factory_, createImdsTokenSubscriber)
-      .Times(0);
-  EXPECT_CALL(mock_token_subscriber_factory_, createIamTokenSubscriber)
-      .Times(0);
-  setUp(filter_config);
-}
-
 TEST_F(ConfigParserImplTest, GetIdTokenByImds) {
   const char filter_config[] = R"(
 imds_token {

src/envoy/http/service_control/client_cache.cc

@@ -211,17 +211,19 @@ ClientCache::ClientCache(
   initHttpRequestSetting(filter_config);
   check_call_factory_ = std::make_unique<HttpCallFactoryImpl>(
       cm, dispatcher, filter_config.service_control_uri(),
-      config_.service_name() + ":check", sc_token_fn, check_timeout_ms_,
-      check_retries_, time_source, "Service Control remote call: Check");
+      absl::StrCat("/", config_.service_name(), ":check"), sc_token_fn,
+      check_timeout_ms_, check_retries_, time_source,
+      "Service Control remote call: Check");
   quota_call_factory_ = std::make_unique<HttpCallFactoryImpl>(
       cm, dispatcher, filter_config.service_control_uri(),
-      config_.service_name() + ":allocateQuota", quota_token_fn,
-      quota_timeout_ms_, quota_retries_, time_source,
+      absl::StrCat("/", config_.service_name(), ":allocateQuota"),
+      quota_token_fn, quota_timeout_ms_, quota_retries_, time_source,
       "Service Control remote call: Allocate Quota");
   report_call_factory_ = std::make_unique<HttpCallFactoryImpl>(
       cm, dispatcher, filter_config.service_control_uri(),
-      config_.service_name() + ":report", sc_token_fn, report_timeout_ms_,
-      report_retries_, time_source, "Service Control remote call: Report");
+      absl::StrCat("/", config_.service_name(), ":report"), sc_token_fn,
+      report_timeout_ms_, report_retries_, time_source,
+      "Service Control remote call: Report");
 
   // Note: Check transport is also defined per request.
   // But this must be defined, it will be called on each flush of the cache

src/envoy/http/service_control/service_control_call_impl.cc

@@ -34,17 +34,6 @@ using token::ServiceAccountTokenGenerator;
 using token::TokenSubscriber;
 using token::TokenType;
 
-namespace {
-// The service_control service name. used for as audience to generate JWT token.
-constexpr char kServiceControlService[] =
-    "/google.api.servicecontrol.v1.ServiceController";
-
-// The quota_control service name. used for as audience to generate JWT token.
-constexpr char kQuotaControlService[] =
-    "/google.api.servicecontrol.v1.QuotaController";
-
-}  // namespace
-
 void ServiceControlCallImpl::createImdsTokenSub() {
   const std::string& token_cluster = filter_config_.imds_token().cluster();
   const std::string& token_uri = filter_config_.imds_token().uri();
@@ -61,31 +50,6 @@ void ServiceControlCallImpl::createImdsTokenSub() {
       });
 }
 
-void ServiceControlCallImpl::createTokenGen() {
-  const std::string service_control_audience =
-      filter_config_.service_control_uri().uri() + kServiceControlService;
-  sc_token_gen_ = token_subscriber_factory_.createServiceAccountTokenGenerator(
-      filter_config_.service_account_secret().inline_string(),
-      service_control_audience, [this](const std::string& token) {
-        TokenSharedPtr new_token = std::make_shared<std::string>(token);
-        tls_->runOnAllThreads([this, new_token]() {
-          tls_->getTyped<ThreadLocalCache>().set_sc_token(new_token);
-        });
-      });
-
-  const std::string quota_audience =
-      filter_config_.service_control_uri().uri() + kQuotaControlService;
-  quota_token_gen_ =
-      token_subscriber_factory_.createServiceAccountTokenGenerator(
-          filter_config_.service_account_secret().inline_string(),
-          quota_audience, [this](const std::string& token) {
-            TokenSharedPtr new_token = std::make_shared<std::string>(token);
-            tls_->runOnAllThreads([this, new_token]() {
-              tls_->getTyped<ThreadLocalCache>().set_quota_token(new_token);
-            });
-          });
-}
-
 void ServiceControlCallImpl::createIamTokenSub() {
   switch (filter_config_.iam_token().access_token().token_type_case()) {
     case AccessToken::kRemoteToken: {
@@ -151,9 +115,6 @@ ServiceControlCallImpl::ServiceControlCallImpl(
     case FilterConfig::kImdsToken:
       createImdsTokenSub();
       break;
-    case FilterConfig::kServiceAccountSecret:
-      createTokenGen();
-      break;
     case FilterConfig::kIamToken:
       createIamTokenSub();
       break;

src/envoy/http/service_control/service_control_call_impl.h

@@ -103,7 +103,6 @@ class ServiceControlCallImpl
   }
 
   void createImdsTokenSub();
-  void createTokenGen();
   void createIamTokenSub();
 
   const ::espv2::api::envoy::v7::http::service_control::FilterConfig&
@@ -123,8 +122,6 @@ class ServiceControlCallImpl
   // Token subscriber used to fetch access token from iam for service control
   token::TokenSubscriberPtr iam_token_sub_;
 
-  token::ServiceAccountTokenPtr sc_token_gen_;
-  token::ServiceAccountTokenPtr quota_token_gen_;
   Envoy::ThreadLocal::SlotPtr tls_;
 };  // namespace ServiceControl
 

src/go/bootstrap/ads/flags/flags.go

@@ -21,6 +21,7 @@ import (
 
 	"github.com/GoogleCloudPlatform/esp-v2/src/go/commonflags"
 	"github.com/GoogleCloudPlatform/esp-v2/src/go/options"
+	"github.com/GoogleCloudPlatform/esp-v2/src/go/util"
 	"github.com/golang/glog"
 )
 
@@ -33,7 +34,7 @@ func DefaultBootstrapperOptionsFromFlags() options.AdsBootstrapperOptions {
 	opts := options.AdsBootstrapperOptions{
 		CommonOptions:     common_option,
 		AdsConnectTimeout: *AdsConnectTimeout,
-		DiscoveryAddress:  fmt.Sprintf("127.0.0.1:%d", common_option.DiscoveryPort),
+		DiscoveryAddress:  fmt.Sprintf("%s:%d", util.LoopbackIPv4Addr, common_option.DiscoveryPort),
 	}
 
 	glog.Infof("ADS Bootstrapper options: %+v", opts)

src/go/commonflags/flags.go

@@ -52,7 +52,7 @@ var (
 	//listener is marked as ready but the whole Envoy server is not marked as ready
 	//(worker did not start) somehow. To work around this problem, use IP for
 	//metadata server to fetch access token.
-	MetadataURL = flag.String("metadata_url", "http://169.254.169.254/computeMetadata", "url of metadata server")
+	MetadataURL = flag.String("metadata_url", "http://169.254.169.254", "url of metadata server")
 	IamURL      = flag.String("iam_url", "https://iamcredentials.googleapis.com", "url of iam server")
 
 	ServiceControlIamServiceAccount = flag.String("service_control_iam_service_account", "", "The service account used to fetch access token for the Service Control from Google Cloud IAM")

src/go/configgenerator/cluster_generator.go

@@ -40,12 +40,27 @@ func MakeClusters(serviceInfo *sc.ServiceInfo) ([]*clusterpb.Cluster, error) {
 		clusters = append(clusters, backendCluster)
 	}
 
-	metadataCluster, err := makeMetadataCluster(serviceInfo)
-	if err != nil {
-		return nil, err
-	}
-	if metadataCluster != nil {
-		clusters = append(clusters, metadataCluster)
+	// When ServiceAccountKey is undefined, envoy will access Instance
+	// Metadata to get access token and identity token. It can be either sidecar
+	// mode and gateway mode.
+	//
+	// When ServiceAccountKey is defined, configmanager will setup a local
+	// server to provide access token generated by the service account credential.
+	// It can only be sidecar mode.
+	if serviceInfo.Options.ServiceAccountKey == "" {
+		metadataCluster, err := makeMetadataCluster(serviceInfo)
+		if err != nil {
+			return nil, err
+		}
+		if metadataCluster != nil {
+			clusters = append(clusters, metadataCluster)
+		}
+
+	} else {
+		tokenAgentCluster := makeTokenAgentCluster(serviceInfo)
+
+		clusters = append(clusters, tokenAgentCluster)
+
 	}
 
 	iamCluster, err := makeIamCluster(serviceInfo)
@@ -136,6 +151,18 @@ func makeMetadataCluster(serviceInfo *sc.ServiceInfo) (*clusterpb.Cluster, error
 	return c, nil
 }
 
+func makeTokenAgentCluster(serviceInfo *sc.ServiceInfo) *clusterpb.Cluster {
+	return &clusterpb.Cluster{
+		Name:           util.TokenAgentClusterName,
+		LbPolicy:       clusterpb.Cluster_ROUND_ROBIN,
+		ConnectTimeout: ptypes.DurationProto(serviceInfo.Options.ClusterConnectTimeout),
+		ClusterDiscoveryType: &clusterpb.Cluster_Type{
+			Type: clusterpb.Cluster_STATIC,
+		},
+		LoadAssignment: util.CreateLoadAssignment(util.LoopbackIPv4Addr, uint32(serviceInfo.Options.TokenAgentPort)),
+	}
+}
+
 func makeIamCluster(serviceInfo *sc.ServiceInfo) (*clusterpb.Cluster, error) {
 	if serviceInfo.Options.ServiceControlCredentials == nil && serviceInfo.Options.BackendAuthCredentials == nil {
 		return nil, nil
@@ -286,7 +313,7 @@ func makeServiceControlCluster(serviceInfo *sc.ServiceInfo) (*clusterpb.Cluster,
 	}
 
 	connectTimeoutProto := ptypes.DurationProto(5 * time.Second)
-	serviceInfo.ServiceControlURI = scheme + "://" + hostname + "/v1/services/"
+	serviceInfo.ServiceControlURI = scheme + "://" + hostname + "/v1/services"
 	c := &clusterpb.Cluster{
 		Name:                 util.ServiceControlClusterName,
 		LbPolicy:             clusterpb.Cluster_ROUND_ROBIN,