Added app level encryption feature. (#599)

* Added app level encryption feature.

* Add validation of config parameters.

* test

* better name

* naming

* nil encrypter for kek

* docs.

* integration fix

cmd/ssiservice/main.go

@@ -6,6 +6,7 @@ import (
 	"io"
 	"os"
 	"os/signal"
+	"path"
 	"strconv"
 	"syscall"
 	"time"
@@ -52,7 +53,9 @@ func run() error {
 		logrus.Infof("loading config from env var path: %s", envConfigPath)
 		configPath = envConfigPath
 	}
-	cfg, err := config.LoadConfig(configPath)
+
+	dir, file := path.Split(configPath)
+	cfg, err := config.LoadConfig(file, os.DirFS(dir))
 	if err != nil {
 		logrus.Fatalf("could not instantiate config: %s", err.Error())
 	}

config/config.go

@@ -2,6 +2,7 @@ package config
 
 import (
 	"fmt"
+	"io/fs"
 	"os"
 	"path/filepath"
 	"reflect"
@@ -72,6 +73,10 @@ type ServicesConfig struct {
 	StorageOptions  []storage.Option `toml:"storage_option"`
 	ServiceEndpoint string           `toml:"service_endpoint"`
 
+	// Application level encryption configuration. Defines how values are encrypted before they are stored in the
+	// configured KV store.
+	AppLevelEncryptionConfiguration EncryptionConfig `toml:"storage_encryption,omitempty"`
+
 	// Embed all service-specific configs here. The order matters: from which should be instantiated first, to last
 	KeyStoreConfig        KeyStoreServiceConfig     `toml:"keystore,omitempty"`
 	DIDConfig             DIDServiceConfig          `toml:"did,omitempty"`
@@ -94,21 +99,53 @@ type BaseServiceConfig struct {
 type KeyStoreServiceConfig struct {
 	*BaseServiceConfig
 
-	// The URI for the master key. We use tink for envelope encryption as described in https://github.com/google/tink/blob/9bc2667963e20eb42611b7581e570f0dddf65a2b/docs/KEY-MANAGEMENT.md#key-management-with-tink
-	// When left empty, then a random key is generated and used.
+	// Configuration describing the encryption of the private keys that are under ssi-service's custody.
+	EncryptionConfig
+}
+
+type EncryptionConfig struct {
+	DisableEncryption bool `toml:"disable_encryption"`
+
+	// The URI for a master key. We use tink for envelope encryption as described in https://github.com/google/tink/blob/9bc2667963e20eb42611b7581e570f0dddf65a2b/docs/KEY-MANAGEMENT.md#key-management-with-tink
+	// When left empty and DisableEncryption is off, then a random key is generated and used. This random key is persisted unencrypted in the
+	// configured storage. Production deployments should never leave this field empty.
 	MasterKeyURI string `toml:"master_key_uri"`
 
-	// Path for credentials. Required when using an external KMS. More info at https://github.com/google/tink/blob/9bc2667963e20eb42611b7581e570f0dddf65a2b/docs/KEY-MANAGEMENT.md#credentials
+	// Path for credentials. Required when MasterKeyURI is set. More info at https://github.com/google/tink/blob/9bc2667963e20eb42611b7581e570f0dddf65a2b/docs/KEY-MANAGEMENT.md#credentials
 	KMSCredentialsPath string `toml:"kms_credentials_path"`
 }
 
+func (e EncryptionConfig) GetMasterKeyURI() string {
+	return e.MasterKeyURI
+}
+
+func (e EncryptionConfig) GetKMSCredentialsPath() string {
+	return e.KMSCredentialsPath
+}
+
+func (e EncryptionConfig) EncryptionEnabled() bool {
+	return !e.DisableEncryption
+}
+
 func (k *KeyStoreServiceConfig) IsEmpty() bool {
 	if k == nil {
 		return true
 	}
 	return reflect.DeepEqual(k, &KeyStoreServiceConfig{})
 }
 
+func (k *KeyStoreServiceConfig) GetMasterKeyURI() string {
+	return k.MasterKeyURI
+}
+
+func (k *KeyStoreServiceConfig) GetKMSCredentialsPath() string {
+	return k.KMSCredentialsPath
+}
+
+func (k *KeyStoreServiceConfig) EncryptionEnabled() bool {
+	return !k.DisableEncryption
+}
+
 type DIDServiceConfig struct {
 	*BaseServiceConfig
 	Methods                  []string `toml:"methods"`
@@ -211,7 +248,10 @@ func (p *WebhookServiceConfig) IsEmpty() bool {
 
 // LoadConfig attempts to load a TOML config file from the given path, and coerce it into our object model.
 // Before loading, defaults are applied on certain properties, which are overwritten if specified in the TOML file.
-func LoadConfig(path string) (*SSIServiceConfig, error) {
+func LoadConfig(path string, fs fs.FS) (*SSIServiceConfig, error) {
+	if fs == nil {
+		fs = os.DirFS(".")
+	}
 	loadDefaultConfig, err := checkValidConfigPath(path)
 	if err != nil {
 		return nil, errors.Wrap(err, "validate config path")
@@ -226,17 +266,33 @@ func LoadConfig(path string) (*SSIServiceConfig, error) {
 	if loadDefaultConfig {
 		defaultServicesConfig := getDefaultServicesConfig()
 		config.Services = defaultServicesConfig
-	} else if err = loadTOMLConfig(path, &config); err != nil {
+	} else if err = loadTOMLConfig(path, &config, fs); err != nil {
 		return nil, errors.Wrap(err, "load toml config")
 	}
 
 	if err = applyEnvVariables(&config); err != nil {
 		return nil, errors.Wrap(err, "apply env variables")
 	}
 
+	if err = validateConfig(&config); err != nil {
+		return nil, errors.Wrap(err, "validating config values")
+	}
+
 	return &config, nil
 }
 
+func validateConfig(s *SSIServiceConfig) error {
+	if s.Server.Environment == EnvironmentProd {
+		if s.Services.KeyStoreConfig.DisableEncryption {
+			return errors.New("prod environment cannot disable key encryption")
+		}
+		if s.Services.AppLevelEncryptionConfiguration.DisableEncryption {
+			logrus.Warn("prod environment detected without app level encryption. This is strongly discouraged.")
+		}
+	}
+	return nil
+}
+
 func checkValidConfigPath(path string) (bool, error) {
 	// no path, load default config
 	defaultConfig := false
@@ -314,9 +370,13 @@ func getDefaultServicesConfig() ServicesConfig {
 	}
 }
 
-func loadTOMLConfig(path string, config *SSIServiceConfig) error {
+func loadTOMLConfig(path string, config *SSIServiceConfig, fs fs.FS) error {
 	// load from TOML file
-	if _, err := toml.DecodeFile(path, &config); err != nil {
+	file, err := fs.Open(path)
+	if err != nil {
+		return errors.Wrapf(err, "opening path %s", path)
+	}
+	if _, err := toml.NewDecoder(file).Decode(&config); err != nil {
 		return errors.Wrapf(err, "could not load config: %s", path)
 	}
 

config/config_test.go

@@ -1,20 +1,32 @@
 package config
 
 import (
+	"embed"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 )
 
-func TestConfig(t *testing.T) {
-	config, err := LoadConfig(Filename)
-	assert.NoError(t, err)
-	assert.NotEmpty(t, config)
+//go:embed testdata
+var testdata embed.FS
 
-	assert.False(t, config.Server.ReadTimeout.String() == "")
-	assert.False(t, config.Server.WriteTimeout.String() == "")
-	assert.False(t, config.Server.ShutdownTimeout.String() == "")
-	assert.False(t, config.Server.APIHost == "")
+func TestLoadConfig(t *testing.T) {
+	t.Run("returns no errors when passed in file", func(t *testing.T) {
+		config, err := LoadConfig(Filename, nil)
+		assert.NoError(t, err)
+		assert.NotEmpty(t, config)
 
-	assert.NotEmpty(t, config.Services.StorageProvider)
+		assert.False(t, config.Server.ReadTimeout.String() == "")
+		assert.False(t, config.Server.WriteTimeout.String() == "")
+		assert.False(t, config.Server.ShutdownTimeout.String() == "")
+		assert.False(t, config.Server.APIHost == "")
+
+		assert.NotEmpty(t, config.Services.StorageProvider)
+	})
+
+	t.Run("returns errors when prod disables encryption", func(t *testing.T) {
+		_, err := LoadConfig("testdata/test1.toml", testdata)
+		assert.Error(t, err)
+		assert.ErrorContains(t, err, "prod environment cannot disable key encryption")
+	})
 }

config/dev.toml

@@ -29,6 +29,10 @@ service_endpoint = "http://localhost:3000"
 # example bolt config with filepath option
 storage = "bolt"
 
+[services.storage_encryption]
+# encryption
+disable_encryption = true
+
 [[services.storage_option]]
 id = "boltdb-filepath-option"
 option = "bolt.db"

config/prod.toml

@@ -20,7 +20,12 @@ log_location = "log"
 log_level = "info"
 
 enable_schema_caching = true
-enable_allow_all_cors = true
+enable_allow_all_cors = false
+
+[services.storage_encryption]
+# master_key_uri = "gcp-kms://projects/*/locations/*/keyRings/*/cryptoKeys/*"
+# kms_credentials_path = "credentials.json"
+disable_encryption = false
 
 # Storage Configuration
 [services]
@@ -38,6 +43,7 @@ option = "password"
 # per-service configuration
 [services.keystore]
 name = "keystore"
+disable_encryption = false
 # master_key_uri = "gcp-kms://projects/*/locations/*/keyRings/*/cryptoKeys/*"
 # kms_credentials_path = "credentials.json"
 

config/test.toml

@@ -25,6 +25,11 @@ log_level = "warn"
 enable_schema_caching = true
 enable_allow_all_cors = true
 
+[services.storage_encryption]
+# master_key_uri = "gcp-kms://projects/*/locations/*/keyRings/*/cryptoKeys/*"
+# kms_credentials_path = "credentials.json"
+disable_encryption = false
+
 # Storage Configuration
 [services]
 service_endpoint = "http://localhost:8080"
@@ -43,6 +48,7 @@ option = "password"
 name = "keystore"
 # master_key_uri = "gcp-kms://projects/*/locations/*/keyRings/*/cryptoKeys/*"
 # kms_credentials_path = "credentials.json"
+disable_encryption = false
 
 [services.did]
 name = "did"

config/testdata/test1.toml

@@ -0,0 +1,6 @@
+[server]
+env = "prod" # either 'dev', 'test', or 'prod'
+
+[services.keystore]
+name = "keystore"
+disable_encryption = true

doc/STORAGE.md

@@ -70,4 +70,55 @@ For a working example, see this [dev.toml file](https://github.com/TBD54566975/s
 
 You need to implement the [ServiceStorage interface](../pkg/storage/storage.go), similar to how [Redis](../pkg/storage/redis.go)
 is implemented. For an example, see [this PR](https://github.com/TBD54566975/ssi-service/pull/590/files#diff-606358579107e7ad1221525001aed8c776a141d4cc5aab9ef7a3ddbcec10d9f9)
-which introduces the SQL based implementation.
+which introduces the SQL based implementation.
+
+## Encryption
+
+SSI Service supports application level encryption of values before sending them to the configured KV store. Please note
+that keys (i.e. the key of the KV store) are not currently encrypted. See the [Privacy Considerations](#privacy-considerations) for more information.
+A MasterKey is used (a.k.a. a Data Encryption Key or DEK) to encrypt all data before it's sent to the configured storage.
+The MasterKey can be stored in the configured storage system or in an external Key Management System (KMS) like GCP KMS or AWS KMS.
+When storing locally, the key will be automatically generated if it doesn't exist already.
+
+**For production deployments, it is strongly recommended to store the MasterKey in an external KMS.**
+
+To use an external KMS:
+1. Create a symmetric encryption key in your KMS. You MUST select the algorithm that uses AES-256 block cipher in Galois/Counter Mode (GCM). At the time of writing, this is the only algorithm supported by AWS and GCP.
+2. Set the `master_key_uri` field of the `[services.storage_encryption]` section using the format described in [tink](https://github.com/google/tink/blob/9bc2667963e20eb42611b7581e570f0dddf65a2b/docs/KEY-MANAGEMENT.md#key-management-systems)
+   (we use the tink library under the hood).
+3. Set the `kms_credentials_path` field of the `[services.storage_encryption]` section to point to your credentials file, according to [this section](https://github.com/google/tink/blob/9bc2667963e20eb42611b7581e570f0dddf65a2b/docs/KEY-MANAGEMENT.md#credentials).
+4. Win!
+
+Below, there is an example snippet of what the TOML configuration should look like.
+```toml
+[services.storage_encryption]
+# Make sure the following values are valid.
+master_key_uri = "gcp-kms://projects/*/locations/*/keyRings/*/cryptoKeys/*"
+kms_credentials_path = "credentials.json"
+disable_encryption = false
+```
+
+Storing the MasterKey in the configured storage system is done with the following options in your TOML configuration.
+
+```toml
+[services.storage_encryption]
+# ensure that master_key_uri is NOT set.
+disable_encryption = false
+```
+
+Disabling app level encryption is also possible using the following options in your TOML configuration:
+
+```toml
+[services.storage_encryption]
+# encryption
+disable_encryption = true
+```
+
+### Privacy Considerations
+
+From the perspective of SSI-Service, all keys are stored in plaintext (this doesn't preclude configuring encryption at rest
+in your deployment of the storage configuration). Making all keys readable by any actor may have an impact in your organization's
+use cases around privacy. You should consider whether this is acceptable. Notably, a DID that was created by SSI Service
+is stored as a key. This can fit some definition of PII, as it could be correlated to identify and individual.
+
+Encrypting keys is being considered in https://github.com/TBD54566975/ssi-service/issues/603.