Skip to content

Commit 2d99282

Browse files
authored
Aws sts conn pool (#78)
* moved sts cred provider into TLS slot * got rid of thread local cache * stash * Merge branch 'master' of github.com:solo-io/envoy-gloo into aws-sts-conn-pool * should not be committed * fetcher compiles again * building with slight class refactor * formatting * static construtor * working on cred provider * credential provider lib working * it builds * it's working * using local dispatcher * new file watcher compiling * formatting * working * Delete envoy_env.yaml * fetcher tests passing again * basic connection pool test * tests for connection pool passing * added connection pool factory to mock fetcher * formatting * in the middle of getting tests to compile * credential provider tests passing * fixed filter test * compiles, need to debug * passing * Merge branch 'master' into aws-sts-conn-pool * add pprof * oops * maybe strict * no more memory issues * using function for onFailure * building with new names * made private constructor * added tests for file watcher * formatting * only call when STS is used * fixed lifetime of config * formatting * changelog
1 parent 2602267 commit 2d99282

26 files changed

+1174
-694
lines changed

.bazelrc

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,7 @@ build:libc++ --define force_libcpp=enabled
113113
build:sizeopt -c opt --copt -Os
114114

115115
# Test options
116-
build --test_env=HEAPCHECK=normal --test_env=PPROF_PATH
116+
build --test_env=HEAPCHECK=strict --test_env=PPROF_PATH
117117

118118
# Coverage options
119119
coverage --config=coverage
Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
changelog:
2+
- type: NEW_FEATURE
3+
description: Add connection pooling to AWS STS connections for better performance
4+
issueLink: https://github.com/solo-io/envoy-gloo/issues/80
5+

ci/do_ci.sh

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,5 +30,8 @@ export NUM_CPUS=16
3030
# google cloud build doesn't like ipv6
3131
export BAZEL_EXTRA_TEST_OPTIONS="--test_env=ENVOY_IP_TEST_VERSIONS=v4only --test_output=errors --jobs=${NUM_CPUS}"
3232

33+
# sudo apt-get install google-perftools -y
34+
# export PPROF_PATH=$(which google-pprof)
35+
3336
echo Building
3437
bash -x $UPSTREAM_ENVOY_SRCDIR/ci/do_ci.sh "$@"

source/extensions/filters/http/aws_lambda/BUILD

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,9 +88,27 @@ envoy_cc_library(
8888
srcs = ["sts_credentials_provider.cc"],
8989
hdrs = ["sts_credentials_provider.h"],
9090
repository = "@envoy",
91+
deps = [
92+
":sts_connection_pool_lib",
93+
"@envoy//source/common/common:minimal_logger_lib",
94+
"@envoy//source/common/common:linked_object",
95+
"@envoy//source/common/config:datasource_lib",
96+
"@envoy//source/common/protobuf:utility_lib",
97+
"@envoy//source/common/filesystem:watcher_lib",
98+
"@envoy//source/extensions/common/aws:credentials_provider_interface",
99+
"//api/envoy/config/filter/http/aws_lambda/v2:pkg_cc_proto",
100+
],
101+
)
102+
103+
envoy_cc_library(
104+
name = "sts_connection_pool_lib",
105+
srcs = ["sts_connection_pool.cc"],
106+
hdrs = ["sts_connection_pool.h"],
107+
repository = "@envoy",
91108
deps = [
92109
":sts_fetcher_lib",
93110
"@envoy//source/common/common:minimal_logger_lib",
111+
"@envoy//source/common/common:linked_object",
94112
"@envoy//source/common/config:datasource_lib",
95113
"@envoy//source/common/protobuf:utility_lib",
96114
"@envoy//source/common/filesystem:watcher_lib",

source/extensions/filters/http/aws_lambda/aws_lambda_filter.cc

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,7 @@ void AWSLambdaFilter::onSuccess(
124124
std::shared_ptr<const Envoy::Extensions::Common::Aws::Credentials>
125125
credentials) {
126126
credentials_ = credentials;
127-
// Do not null context here; all hell will break loose.
127+
context_ = nullptr;
128128
state_ = State::Complete;
129129

130130
const std::string *access_key{};
@@ -171,6 +171,8 @@ void AWSLambdaFilter::onSuccess(
171171

172172
// TODO: Use the failure status in the local reply
173173
void AWSLambdaFilter::onFailure(CredentialsFailureStatus) {
174+
// cancel mustn't be called
175+
context_ = nullptr;
174176
state_ = State::Responded;
175177
decoder_callbacks_->sendLocalReply(
176178
Http::Code::InternalServerError, RcDetails::get().CredentialsNotFoundBody,

source/extensions/filters/http/aws_lambda/aws_lambda_filter.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ namespace AwsLambda {
2222
* it expects retrieveFunction to be called before decodeHeaders.
2323
*/
2424
class AWSLambdaFilter : public Http::StreamDecoderFilter,
25-
StsCredentialsProvider::Callbacks,
25+
StsConnectionPool::Context::Callbacks,
2626
Logger::Loggable<Logger::Id::filter> {
2727
public:
2828
AWSLambdaFilter(Upstream::ClusterManager &cluster_manager, Api::Api &api,
@@ -75,7 +75,7 @@ class AWSLambdaFilter : public Http::StreamDecoderFilter,
7575

7676
CredentialsConstSharedPtr credentials_;
7777

78-
ContextSharedPtr context_;
78+
StsConnectionPool::Context *context_;
7979
// The state of the request
8080
enum State { Init, Calling, Responded, Complete };
8181
// The state of the get credentials request.

source/extensions/filters/http/aws_lambda/aws_lambda_filter_config_factory.cc

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -18,15 +18,14 @@ AWSLambdaFilterConfigFactory::createFilterFactoryFromProtoTyped(
1818
const std::string &stats_prefix,
1919
Server::Configuration::FactoryContext &context) {
2020

21-
auto sts_factory = StsCredentialsProviderFactoryImpl(
22-
context.api(), context.threadLocal(), context.dispatcher());
23-
auto config = std::make_shared<AWSLambdaConfigImpl>(
21+
auto config = AWSLambdaConfigImpl::create(
2422
std::make_unique<
2523
Extensions::Common::Aws::DefaultCredentialsProviderChain>(
2624
context.api(), Extensions::Common::Aws::Utility::metadataFetcher),
27-
context.clusterManager(), sts_factory, context.dispatcher(),
28-
context.threadLocal(), stats_prefix, context.scope(), context.api(),
29-
proto_config);
25+
StsCredentialsProviderFactory::create(context.api(),
26+
context.clusterManager()),
27+
context.dispatcher(), context.api(), context.threadLocal(), stats_prefix,
28+
context.scope(), proto_config);
3029

3130
return
3231
[&context, config](Http::FilterChainFactoryCallbacks &callbacks) -> void {

source/extensions/filters/http/aws_lambda/config.cc

Lines changed: 109 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -34,21 +34,24 @@ constexpr std::chrono::milliseconds REFRESH_AWS_CREDS =
3434
struct ThreadLocalCredentials : public Envoy::ThreadLocal::ThreadLocalObject {
3535
ThreadLocalCredentials(CredentialsConstSharedPtr credentials)
3636
: credentials_(credentials) {}
37+
ThreadLocalCredentials(StsCredentialsProviderPtr credentials)
38+
: sts_credentials_(std::move(credentials)) {}
3739
CredentialsConstSharedPtr credentials_;
40+
StsCredentialsProviderPtr sts_credentials_;
3841
};
3942

4043
} // namespace
4144

4245
AWSLambdaConfigImpl::AWSLambdaConfigImpl(
4346
std::unique_ptr<Extensions::Common::Aws::CredentialsProvider> &&provider,
44-
Upstream::ClusterManager &cluster_manager,
45-
StsCredentialsProviderFactory &sts_factory, Event::Dispatcher &dispatcher,
47+
std::unique_ptr<StsCredentialsProviderFactory> &&sts_factory,
48+
Event::Dispatcher &dispatcher, Api::Api &api,
4649
Envoy::ThreadLocal::SlotAllocator &tls, const std::string &stats_prefix,
47-
Stats::Scope &scope, Api::Api &api,
50+
Stats::Scope &scope,
4851
const envoy::config::filter::http::aws_lambda::v2::AWSLambdaConfig
4952
&protoconfig)
50-
: context_factory_(cluster_manager, api),
51-
stats_(generateStats(stats_prefix, scope)) {
53+
: stats_(generateStats(stats_prefix, scope)), api_(api),
54+
file_watcher_(dispatcher.createFilesystemWatcher()) {
5255

5356
// Initialize Credential fetcher, if none exists do nothing. Filter will
5457
// implicitly use protocol options data
@@ -73,9 +76,23 @@ AWSLambdaConfigImpl::AWSLambdaConfigImpl(
7376
case envoy::config::filter::http::aws_lambda::v2::AWSLambdaConfig::
7477
CredentialsFetcherCase::kServiceAccountCredentials: {
7578
ENVOY_LOG(debug, "{}: Using STS credentials source", __func__);
79+
80+
// Load all of the env data for STS credentials
81+
loadSTSData();
7682
// use service account credentials provider
83+
tls_slot_ = tls.allocateSlot();
84+
// transfer ptr ownership to sts_factor isn't cleaned up before we get into
85+
// tls set
86+
sts_factory_ = std::move(sts_factory);
7787
auto service_account_creds = protoconfig.service_account_credentials();
78-
sts_credentials_provider_ = sts_factory.create(service_account_creds);
88+
tls_slot_->set([this, web_token = web_token_, role_arn = role_arn_,
89+
service_account_creds](Event::Dispatcher &dispatcher) {
90+
StsCredentialsProviderPtr sts_cred_provider = sts_factory_->build(
91+
service_account_creds, dispatcher, web_token, role_arn);
92+
return std::make_shared<ThreadLocalCredentials>(
93+
std::move(sts_cred_provider));
94+
});
95+
sts_enabled_ = true;
7996
break;
8097
}
8198
case envoy::config::filter::http::aws_lambda::v2::AWSLambdaConfig::
@@ -85,15 +102,75 @@ AWSLambdaConfigImpl::AWSLambdaConfigImpl(
85102
}
86103
}
87104

105+
void AWSLambdaConfigImpl::loadSTSData() {
106+
// AWS_WEB_IDENTITY_TOKEN_FILE and AWS_ROLE_ARN must be set for STS
107+
// credentials to be enabled
108+
token_file_ =
109+
absl::NullSafeStringView(std::getenv(AWS_WEB_IDENTITY_TOKEN_FILE));
110+
if (token_file_ == "") {
111+
throw EnvoyException(fmt::format("Env var {} must be present, and set",
112+
AWS_WEB_IDENTITY_TOKEN_FILE));
113+
}
114+
role_arn_ = absl::NullSafeStringView(std::getenv(AWS_ROLE_ARN));
115+
if (role_arn_ == "") {
116+
throw EnvoyException(
117+
fmt::format("Env var {} must be present, and set", AWS_ROLE_ARN));
118+
}
119+
// File must exist on system
120+
if (!api_.fileSystem().fileExists(token_file_)) {
121+
throw EnvoyException(
122+
fmt::format("Web token file {} does not exist", token_file_));
123+
}
124+
125+
web_token_ = api_.fileSystem().fileReadToEnd(token_file_);
126+
// File should not be empty
127+
if (web_token_ == "") {
128+
throw EnvoyException(
129+
fmt::format("Web token file {} exists but is empty", token_file_));
130+
}
131+
}
132+
133+
void AWSLambdaConfigImpl::init() {
134+
if (sts_enabled_) {
135+
// Add file watcher for token file
136+
auto shared_this = shared_from_this();
137+
file_watcher_->addWatch(
138+
token_file_, Filesystem::Watcher::Events::Modified,
139+
[shared_this](uint32_t) {
140+
try {
141+
const auto web_token = shared_this->api_.fileSystem().fileReadToEnd(
142+
shared_this->token_file_);
143+
// Set the web token on all sts credentials providers
144+
shared_this->tls_slot_->runOnAllThreads(
145+
[web_token](ThreadLocal::ThreadLocalObjectSharedPtr previous)
146+
-> ThreadLocal::ThreadLocalObjectSharedPtr {
147+
auto prev_config =
148+
std::dynamic_pointer_cast<ThreadLocalCredentials>(
149+
previous);
150+
prev_config->sts_credentials_->setWebToken(web_token);
151+
return previous;
152+
});
153+
// TODO: check if web_token is valid
154+
// TODO: stats here
155+
} catch (const EnvoyException &e) {
156+
ENVOY_LOG_TO_LOGGER(
157+
Envoy::Logger::Registry::getLog(Logger::Id::aws), warn,
158+
"{}: Exception while reading file during watch ({}): {}",
159+
__func__, shared_this->token_file_, e.what());
160+
}
161+
});
162+
}
163+
}
164+
88165
/*
89166
* Three options, in order of precedence
90167
* 1. Protocol Options
91168
* 2. Default Provider
92169
* 3. STS
93170
*/
94-
ContextSharedPtr AWSLambdaConfigImpl::getCredentials(
171+
StsConnectionPool::Context *AWSLambdaConfigImpl::getCredentials(
95172
SharedAWSLambdaProtocolExtensionConfig ext_cfg,
96-
StsCredentialsProvider::Callbacks *callbacks) const {
173+
StsConnectionPool::Context::Callbacks *callbacks) const {
97174
// Always check extension config first, as it overrides
98175
if (ext_cfg->accessKey().has_value() && ext_cfg->secretKey().has_value()) {
99176
ENVOY_LOG(trace, "{}: Credentials found from protocol options", __func__);
@@ -111,22 +188,20 @@ ContextSharedPtr AWSLambdaConfigImpl::getCredentials(
111188
return nullptr;
112189
}
113190

191+
auto &thread_local_credentials =
192+
tls_slot_->getTyped<ThreadLocalCredentials>();
114193
if (provider_) {
115194
ENVOY_LOG(trace, "{}: Credentials found from default source", __func__);
116-
callbacks->onSuccess(
117-
tls_slot_->getTyped<ThreadLocalCredentials>().credentials_);
195+
callbacks->onSuccess(thread_local_credentials.credentials_);
118196
// no context necessary as these credentials are available immediately
119197
return nullptr;
120198
}
121199

122-
if (sts_credentials_provider_) {
200+
if (sts_enabled_) {
123201
ENVOY_LOG(trace, "{}: Credentials being retrieved from STS provider",
124202
__func__);
125-
// return the context directly to the filter, as no direct credentials can
126-
// be sent
127-
auto context = context_factory_.create(callbacks);
128-
sts_credentials_provider_->find(ext_cfg->roleArn(), context);
129-
return context;
203+
return thread_local_credentials.sts_credentials_->find(ext_cfg->roleArn(),
204+
callbacks);
130205
}
131206

132207
ENVOY_LOG(debug, "{}: No valid credentials source found", __func__);
@@ -164,6 +239,24 @@ void AWSLambdaConfigImpl::timerCallback() {
164239
}
165240
}
166241

242+
std::shared_ptr<AWSLambdaConfigImpl> AWSLambdaConfigImpl::create(
243+
std::unique_ptr<Envoy::Extensions::Common::Aws::CredentialsProvider>
244+
&&provider,
245+
std::unique_ptr<StsCredentialsProviderFactory> &&sts_factory,
246+
Event::Dispatcher &dispatcher, Api::Api &api,
247+
Envoy::ThreadLocal::SlotAllocator &tls, const std::string &stats_prefix,
248+
Stats::Scope &scope,
249+
const envoy::config::filter::http::aws_lambda::v2::AWSLambdaConfig
250+
&protoconfig) {
251+
// We can't use make_shared here because the constructor of this class is
252+
// private.
253+
std::shared_ptr<AWSLambdaConfigImpl> ptr(new AWSLambdaConfigImpl(
254+
std::move(provider), std::move(sts_factory), dispatcher, api, tls,
255+
stats_prefix, scope, protoconfig));
256+
ptr->init();
257+
return ptr;
258+
}
259+
167260
AwsLambdaFilterStats
168261
AWSLambdaConfigImpl::generateStats(const std::string &prefix,
169262
Stats::Scope &scope) {

source/extensions/filters/http/aws_lambda/config.h

Lines changed: 40 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -70,50 +70,74 @@ using SharedAWSLambdaProtocolExtensionConfig =
7070

7171
class AWSLambdaConfig {
7272
public:
73-
virtual ContextSharedPtr
73+
virtual StsConnectionPool::Context *
7474
getCredentials(SharedAWSLambdaProtocolExtensionConfig ext_cfg,
75-
StsCredentialsProvider::Callbacks *callbacks) const PURE;
75+
StsConnectionPool::Context::Callbacks *callbacks) const PURE;
7676
virtual ~AWSLambdaConfig() = default;
7777
};
7878

7979
class AWSLambdaConfigImpl
8080
: public AWSLambdaConfig,
81-
public Envoy::Logger::Loggable<Envoy::Logger::Id::filter> {
81+
public Envoy::Logger::Loggable<Envoy::Logger::Id::filter>,
82+
public std::enable_shared_from_this<AWSLambdaConfigImpl> {
8283
public:
84+
~AWSLambdaConfigImpl() = default;
85+
86+
static std::shared_ptr<AWSLambdaConfigImpl>
87+
create(std::unique_ptr<Envoy::Extensions::Common::Aws::CredentialsProvider>
88+
&&provider,
89+
std::unique_ptr<StsCredentialsProviderFactory> &&sts_factory,
90+
Event::Dispatcher &dispatcher, Api::Api &api,
91+
Envoy::ThreadLocal::SlotAllocator &tls,
92+
const std::string &stats_prefix, Stats::Scope &scope,
93+
const envoy::config::filter::http::aws_lambda::v2::AWSLambdaConfig
94+
&protoconfig);
95+
96+
StsConnectionPool::Context *getCredentials(
97+
SharedAWSLambdaProtocolExtensionConfig ext_cfg,
98+
StsConnectionPool::Context::Callbacks *callbacks) const override;
99+
100+
private:
83101
AWSLambdaConfigImpl(
84102
std::unique_ptr<Envoy::Extensions::Common::Aws::CredentialsProvider>
85103
&&provider,
86-
Upstream::ClusterManager &cluster_manager,
87-
StsCredentialsProviderFactory &sts_factory, Event::Dispatcher &dispatcher,
104+
std::unique_ptr<StsCredentialsProviderFactory> &&sts_factory,
105+
Event::Dispatcher &dispatcher, Api::Api &api,
88106
Envoy::ThreadLocal::SlotAllocator &tls, const std::string &stats_prefix,
89-
Stats::Scope &scope, Api::Api &api,
107+
Stats::Scope &scope,
90108
const envoy::config::filter::http::aws_lambda::v2::AWSLambdaConfig
91109
&protoconfig);
92-
~AWSLambdaConfigImpl() = default;
93-
94-
ContextSharedPtr
95-
getCredentials(SharedAWSLambdaProtocolExtensionConfig ext_cfg,
96-
StsCredentialsProvider::Callbacks *callbacks) const override;
97110

98-
private:
99111
CredentialsConstSharedPtr getProviderCredentials() const;
112+
100113
static AwsLambdaFilterStats generateStats(const std::string &prefix,
101114
Stats::Scope &scope);
102115

103116
void timerCallback();
104117

105-
ContextFactory context_factory_;
118+
void init();
119+
120+
void loadSTSData();
121+
122+
AwsLambdaFilterStats stats_;
123+
124+
Api::Api &api_;
125+
126+
Envoy::Filesystem::WatcherPtr file_watcher_;
106127

107128
std::unique_ptr<Envoy::Extensions::Common::Aws::CredentialsProvider>
108129
provider_;
109130

110-
ThreadLocal::SlotPtr tls_slot_;
131+
bool sts_enabled_ = false;
132+
std::string token_file_;
133+
std::string web_token_;
134+
std::string role_arn_;
111135

112-
StsCredentialsProviderPtr sts_credentials_provider_;
136+
ThreadLocal::SlotPtr tls_slot_;
113137

114138
Event::TimerPtr timer_;
115139

116-
AwsLambdaFilterStats stats_;
140+
std::unique_ptr<StsCredentialsProviderFactory> sts_factory_;
117141
};
118142

119143
typedef std::shared_ptr<const AWSLambdaConfig> AWSLambdaConfigConstSharedPtr;

0 commit comments

Comments
 (0)