Elasticsearch TLS support (cadence-workflow#4154)

Author: spmistry

CONTRIBUTING.md

@@ -118,7 +118,8 @@ This will run all the tests excluding end-to-end integration test in host/ packa
 ```bash
 make test
 ```
-> Note that you will see some test failures because of errors connecting to MySQL/Postgres if only Cassandra is up. This is okay if you don't write any code related to persistence layer. 
+:warning: Note:
+> You will see some test failures because of errors connecting to MySQL/Postgres if only Cassandra is up. This is okay if you don't write any code related to persistence layer. 
  
 To run all end-to-end integration tests in **host/** package: 
 ```bash

common/config/elasticsearch.go

@@ -48,6 +48,8 @@ type (
 		// optional to use AWS signing client
 		// See more info https://github.com/olivere/elastic/wiki/Using-with-AWS-Elasticsearch-Service
 		AWSSigning AWSSigning `yaml:"awsSigning"`
+		// optional to use Signed Certificates over https
+		TLS TLS `yaml:"tls"`
 	}
 
 	// AWSSigning contains config to enable signing,

common/elasticsearch/client_v6.go

@@ -120,6 +120,16 @@ func NewV6Client(
 		}
 		clientOptFuncs = append(clientOptFuncs, elastic.SetHttpClient(signingClient))
 	}
+	if connectConfig.TLS.Enabled {
+		var tlsClient *http.Client
+		var err error
+		tlsClient, err = buildTLSHTTPClient(connectConfig.TLS)
+		if err != nil {
+			return nil, err
+		}
+		clientOptFuncs = append(clientOptFuncs, elastic.SetHttpClient(tlsClient))
+	}
+
 	client, err := elastic.NewClient(clientOptFuncs...)
 	if err != nil {
 		return nil, err

common/elasticsearch/client_v7.go

@@ -113,6 +113,15 @@ func NewV7Client(
 		}
 		clientOptFuncs = append(clientOptFuncs, elastic.SetHttpClient(signingClient))
 	}
+	if connectConfig.TLS.Enabled {
+		var tlsClient *http.Client
+		var err error
+		tlsClient, err = buildTLSHTTPClient(connectConfig.TLS)
+		if err != nil {
+			return nil, err
+		}
+		clientOptFuncs = append(clientOptFuncs, elastic.SetHttpClient(tlsClient))
+	}
 	client, err := elastic.NewClient(clientOptFuncs...)
 	if err != nil {
 		return nil, err

common/elasticsearch/common.go

@@ -21,10 +21,56 @@
 package elasticsearch
 
 import (
+	"crypto/tls"
+	"crypto/x509"
+	"io/ioutil"
+	"net/http"
 	"time"
+
+	"github.com/uber/cadence/common/config"
 )
 
 const unknownStatusCode = -1
 
 // TODO https://github.com/uber/cadence/issues/3686
 const oneMicroSecondInNano = int64(time.Microsecond / time.Nanosecond)
+
+// Build Http Client with TLS
+func buildTLSHTTPClient(config config.TLS) (*http.Client, error) {
+	// Setup base TLS config
+	// EnableHostVerification is a secure flag vs insecureSkipVerify is insecure so inverse the valu
+	tlsConfig := &tls.Config{
+		InsecureSkipVerify: !config.EnableHostVerification,
+	}
+
+	// Setup server name
+	if config.ServerName != "" {
+		tlsConfig.ServerName = config.ServerName
+	}
+
+	// Load client cert
+	if config.CertFile != "" && config.KeyFile != "" {
+		cert, err := tls.LoadX509KeyPair(config.CertFile, config.KeyFile)
+		if err != nil {
+			return nil, err
+		}
+		tlsConfig.Certificates = []tls.Certificate{cert}
+	}
+
+	// Load CA cert
+	if config.CaFile != "" {
+		caCert, err := ioutil.ReadFile(config.CaFile)
+		if err != nil {
+			return nil, err
+		}
+		caCertPool := x509.NewCertPool()
+		caCertPool.AppendCertsFromPEM(caCert)
+		tlsConfig.RootCAs = caCertPool
+	}
+
+	// Setup HTTPS client
+	transport := &http.Transport{TLSClientConfig: tlsConfig}
+	tlsClient := &http.Client{Transport: transport}
+
+	return tlsClient, nil
+}

docs/visibility-on-elasticsearch.md

@@ -75,6 +75,25 @@ This part is used to config advanced visibility store to ElasticSearch.
  - `url` is for Cadence to discover ES 
  - `indices/visibility` is ElasticSearch index name for the deployment.  
 
+Optional TLS Support can be enabled by setting the TLS config as follows:
+```yaml
+elasticsearch:
+  url:
+    scheme: "https"
+    host: "127.0.0.1:9200"
+  indices:
+    visibility: cadence-visibility-dev
+  tls:
+    enabled: true
+    caFile: /secrets/cadence/elasticsearch_cert.pem
+    enableHostVerification: true
+    serverName: myServerName
+    certFile: /secrets/cadence/certfile.crt
+    keyFile: /secrets/cadence/keyfile.key
+    sslmode: false
+```
+
+Also need to add a kafka topic to visibility, as shown below.  
 ```
 kafka:
   ...
@@ -84,7 +103,6 @@ kafka:
       dlq-topic: cadence-visibility-dev-dlq
   ...
 ``` 
-Also need to add a kafka topic to visibility, see above for example.  
 
 There are dynamic configs to control ElasticSearch visibility features:
 - `system.advancedVisibilityWritingMode` is an int property to control how to write visibility to data store.