Skip to content

Commit 0dbf335

Browse files
kalroyJay Mundrawala
kalroy
and
Jay Mundrawala
authored
Add support for AWS Elasticsearch Service (#4664)
* Initial changes to pass role_arn to backup Signed-off-by: Kallol Roy <[email protected]> * WIP * Add test and fix bug for esgateway config Signed-off-by: Jay Mundrawala <[email protected]> * Use optional parameters Signed-off-by: Jay Mundrawala <[email protected]> * Fix unit tests Signed-off-by: Jay Mundrawala <[email protected]> * fixup go.mod Signed-off-by: Jay Mundrawala <[email protected]> Co-authored-by: Jay Mundrawala <[email protected]>
1 parent 68d5586 commit 0dbf335

File tree

15 files changed

+985
-626
lines changed

15 files changed

+985
-626
lines changed

‎api/config/es_sidecar/config_request.go

+14-2
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package es_sidecar
33
import (
44
ac "github.com/chef/automate/api/config/shared"
55
w "github.com/chef/automate/api/config/shared/wrappers"
6+
"google.golang.org/protobuf/types/known/wrapperspb"
67
)
78

89
// NewConfigRequest returns a new instance of ConfigRequest with zero values.
@@ -91,7 +92,8 @@ func (c *ConfigRequest) SetGlobalConfig(g *ac.GlobalConfig) {
9192
c.applyGlobalBackupConfig(backupsCfg, g.GetV1().GetBackups())
9293

9394
if externalCfg.GetEnable().GetValue() {
94-
c.applyExternalConfig(backupsCfg, g.GetV1().GetBackups(), externalCfg.GetBackup())
95+
c.applyExternalConfig(backupsCfg, g.GetV1().GetBackups(), externalCfg.GetBackup(),
96+
externalCfg.GetAuth(), externalCfg.GetNodes())
9597
// Early return as we don't need to migrate to the backup-gateway with
9698
// external es
9799
return
@@ -107,7 +109,9 @@ func (c *ConfigRequest) SetGlobalConfig(g *ac.GlobalConfig) {
107109
func (c *ConfigRequest) applyExternalConfig(
108110
cfg *ConfigRequest_V1_System_Backups,
109111
global *ac.Backups,
110-
override *ac.External_Elasticsearch_Backup) {
112+
override *ac.External_Elasticsearch_Backup,
113+
auth *ac.External_Elasticsearch_Authentication,
114+
nodes []*wrapperspb.StringValue) {
111115

112116
// If backups are disabled we don't need to worry about applying any other
113117
// config.
@@ -148,6 +152,14 @@ func (c *ConfigRequest) applyExternalConfig(
148152
if s := extS3cfg.GetSettings(); s != nil {
149153
cfg.S3.Es = s
150154
}
155+
156+
if auth.GetScheme().GetValue() == "aws_es" {
157+
cfg.S3.EnableAwsAuth = w.Bool(true)
158+
cfg.S3.AwsAuth = auth.GetAwsEs()
159+
if len(nodes) > 0 {
160+
cfg.S3.EsUrl = nodes[0]
161+
}
162+
}
151163
case "gcs":
152164
cfg.Backend = w.String(backend)
153165
if cfg.Gcs == nil {

‎api/config/es_sidecar/config_request.pb.go

+110-67
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎api/config/es_sidecar/config_request.proto

+3
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,9 @@ message ConfigRequest {
5151
google.protobuf.StringValue client = 2;
5252
google.protobuf.StringValue base_path = 3;
5353
chef.automate.infra.config.Backups.S3.Elasticsearch es = 4;
54+
google.protobuf.BoolValue enable_aws_auth = 5;
55+
chef.automate.infra.config.External.Elasticsearch.Authentication.AwsElasticsearchAuth aws_auth = 6;
56+
google.protobuf.StringValue es_url = 7;
5457
}
5558
message GCSSettings {
5659
google.protobuf.StringValue bucket = 1;

‎api/config/esgateway/config_request.go

+17-2
Original file line numberDiff line numberDiff line change
@@ -122,10 +122,10 @@ func (c *ConfigRequest) SetGlobalConfig(g *ac.GlobalConfig) {
122122
c.V1.Sys.External = &ConfigRequest_V1_System_External{
123123
Enable: w.Bool(true),
124124
}
125+
endpoints := make([]*ConfigRequest_V1_System_Endpoint, 0, len(nodes))
125126
if len(nodes) > 0 {
126127
isSSL := false
127128

128-
endpoints := make([]*ConfigRequest_V1_System_Endpoint, 0, len(nodes))
129129
for _, n := range nodes {
130130
endpoint, ssl := uriToEndpoint(n.GetValue())
131131
endpoints = append(endpoints, endpoint)
@@ -144,14 +144,29 @@ func (c *ConfigRequest) SetGlobalConfig(g *ac.GlobalConfig) {
144144
}
145145
}
146146

147-
if auth := g.GetV1().GetExternal().GetElasticsearch().GetAuth(); auth.GetScheme().GetValue() == "basic_auth" {
147+
switch auth := g.GetV1().GetExternal().GetElasticsearch().GetAuth(); auth.GetScheme().GetValue() {
148+
case "basic_auth":
148149
c.V1.Sys.External.BasicAuthCredentials = w.String(base64.StdEncoding.EncodeToString([]byte(
149150
fmt.Sprintf(
150151
"%s:%s",
151152
auth.GetBasicAuth().GetUsername().GetValue(),
152153
auth.GetBasicAuth().GetPassword().GetValue(),
153154
),
154155
)))
156+
case "aws_es":
157+
// If we only have 1 AWS Elasticsearch Service endpoint specified, we can assume that
158+
// the host header should be the name of that endpoint.
159+
if c.V1.Sys.Ngx.Http.ProxySetHeaderHost.Value == "$http_host" && len(endpoints) == 1 {
160+
c.V1.Sys.Ngx.Http.ProxySetHeaderHost = endpoints[0].Address
161+
}
162+
c.V1.Sys.External.BasicAuthCredentials = w.String(base64.StdEncoding.EncodeToString([]byte(
163+
fmt.Sprintf(
164+
"%s:%s",
165+
auth.GetAwsEs().GetUsername().GetValue(),
166+
auth.GetAwsEs().GetPassword().GetValue(),
167+
),
168+
)))
169+
default:
155170
}
156171

157172
c.V1.Sys.External.RootCert = g.GetV1().GetExternal().GetElasticsearch().GetSsl().GetRootCert()

‎api/config/esgateway/config_request_test.go

+32-1
Original file line numberDiff line numberDiff line change
@@ -154,7 +154,6 @@ func TestExternalElasticsearch(t *testing.T) {
154154

155155
assert.Equal(t, "server1", c.GetV1().GetSys().GetExternal().GetParsedEndpoints()[0].GetAddress().GetValue())
156156
assert.Equal(t, "443", c.GetV1().GetSys().GetExternal().GetParsedEndpoints()[0].GetPort().GetValue())
157-
158157
})
159158

160159
t.Run("single http endpoint with IP", func(t *testing.T) {
@@ -276,4 +275,36 @@ func TestExternalElasticsearch(t *testing.T) {
276275

277276
require.Equal(t, c.V1.Sys.Ngx.Main.Resolvers.NameserversString.GetValue(), "111.11.11.11:50", "does not match with the nameserver passed")
278277
})
278+
279+
t.Run("aws elasticsearch", func(t *testing.T) {
280+
c := DefaultConfigRequest()
281+
c.SetGlobalConfig(&ac.GlobalConfig{
282+
V1: &ac.V1{
283+
External: &ac.External{
284+
Elasticsearch: &ac.External_Elasticsearch{
285+
Enable: w.Bool(true),
286+
Nodes: []*wrappers.StringValue{
287+
w.String("https://server1"),
288+
},
289+
Auth: &ac.External_Elasticsearch_Authentication{
290+
Scheme: w.String("aws_es"),
291+
AwsEs: &ac.External_Elasticsearch_Authentication_AwsElasticsearchAuth{
292+
Username: w.String("testuser"),
293+
Password: w.String("testpassword"),
294+
},
295+
},
296+
},
297+
},
298+
},
299+
})
300+
301+
require.True(t,
302+
c.GetV1().GetSys().GetExternal().GetEnable().GetValue(),
303+
"expected external ES to be enabled")
304+
305+
require.Equal(t, "server1", c.GetV1().GetSys().GetExternal().GetParsedEndpoints()[0].GetAddress().GetValue())
306+
require.Equal(t, "443", c.GetV1().GetSys().GetExternal().GetParsedEndpoints()[0].GetPort().GetValue())
307+
require.Equal(t, "server1", c.GetV1().GetSys().GetNgx().GetHttp().GetProxySetHeaderHost().GetValue())
308+
309+
})
279310
}

‎api/config/shared/global.go

+21-15
Original file line numberDiff line numberDiff line change
@@ -215,23 +215,29 @@ func (c *GlobalConfig) Validate() error { // nolint gocyclo
215215

216216
auth := c.GetV1().GetExternal().GetElasticsearch().GetAuth()
217217
scheme := auth.GetScheme().GetValue()
218-
if scheme != "" {
219-
// External ES uses a supported auth scheme
220-
if scheme != "basic_auth" {
221-
cfgErr.AddInvalidValue("global.v1.external.elasticsearch.auth.scheme", "Scheme should be 'basic_auth'.")
218+
switch scheme {
219+
case "basic_auth":
220+
u := auth.GetBasicAuth().GetUsername().GetValue()
221+
p := auth.GetBasicAuth().GetPassword().GetValue()
222+
if u == "" {
223+
cfgErr.AddMissingKey("global.v1.external.elasticsearch.auth.basic_auth.username")
222224
}
223-
224-
// Username and password specified in config if using basic auth
225-
if scheme == "basic_auth" {
226-
u := auth.GetBasicAuth().GetUsername().GetValue()
227-
p := auth.GetBasicAuth().GetPassword().GetValue()
228-
if u == "" {
229-
cfgErr.AddMissingKey("global.v1.external.elasticsearch.basic_auth.username")
230-
}
231-
if p == "" {
232-
cfgErr.AddMissingKey("global.v1.external.elasticsearch.basic_auth.password")
233-
}
225+
if p == "" {
226+
cfgErr.AddMissingKey("global.v1.external.elasticsearch.auth.basic_auth.password")
227+
}
228+
case "aws_es":
229+
u := auth.GetAwsEs().GetUsername().GetValue()
230+
p := auth.GetAwsEs().GetPassword().GetValue()
231+
if u == "" {
232+
cfgErr.AddMissingKey("global.v1.external.elasticsearch.auth.aws_es.username")
233+
}
234+
if p == "" {
235+
cfgErr.AddMissingKey("global.v1.external.elasticsearch.auth.aws_es.password")
234236
}
237+
case "":
238+
default:
239+
cfgErr.AddInvalidValue("global.v1.external.elasticsearch.auth.scheme",
240+
"Scheme should be one of 'basic_auth', 'aws_es'.")
235241
}
236242
}
237243

0 commit comments

Comments
 (0)