Encryption and decryption of secrets at rest

conf/st2.dev.conf

@@ -40,6 +40,7 @@ logging = st2reactor/conf/logging.timersengine.conf
 [actionrunner]
 logging = st2actions/conf/logging.conf
 stream_output = True
+encryption_key_path = conf/st2_kvstore_demo.crypto.key.json
 
 [rbac]
 enable = False

st2api/tests/unit/controllers/v1/test_action_alias.py

@@ -13,6 +13,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from oslo_config import cfg
+
+opts = [
+    cfg.StrOpt(
+        "encryption_key_path",
+        default="conf/st2_kvstore_demo.crypto.key.json",
+        help="Location of the symmetric encryption key for encrypting values in kvstore. "
+        "This key should be in JSON and should've been generated using "
+        "st2-generate-symmetric-crypto-key tool.",
+    ),
+]
+cfg.CONF.register_opts(opts, group="actionrunner")
 from st2common.models.api.action import ActionAliasAPI
 from st2api.controllers.v1.actionalias import ActionAliasController
 

st2api/tests/unit/controllers/v1/test_action_views.py

@@ -21,6 +21,19 @@
 import mock
 
 from st2common.content import utils as content_utils
+from oslo_config import cfg
+
+opts = [
+    cfg.StrOpt(
+        "encryption_key_path",
+        default="conf/st2_kvstore_demo.crypto.key.json",
+        help="Location of the symmetric encryption key for encrypting values in kvstore. "
+        "This key should be in JSON and should've been generated using "
+        "st2-generate-symmetric-crypto-key tool.",
+    ),
+]
+cfg.CONF.register_opts(opts, group="actionrunner")
+
 import st2common.validators.api.action as action_validator
 from st2common.util.compat import mock_open_name
 from st2api.controllers.v1.action_views import OverviewController

st2api/tests/unit/controllers/v1/test_actions.py

@@ -28,6 +28,18 @@
 import mock
 import unittest2
 from six.moves import http_client
+from oslo_config import cfg
+
+opts = [
+    cfg.StrOpt(
+        "encryption_key_path",
+        default="conf/st2_kvstore_demo.crypto.key.json",
+        help="Location of the symmetric encryption key for encrypting values in kvstore. "
+        "This key should be in JSON and should've been generated using "
+        "st2-generate-symmetric-crypto-key tool.",
+    ),
+]
+cfg.CONF.register_opts(opts, group="actionrunner")
 
 from st2common.persistence.action import Action
 import st2common.validators.api.action as action_validator

st2api/tests/unit/controllers/v1/test_alias_execution.py

@@ -18,6 +18,18 @@
 import mock
 
 from st2common.constants.action import LIVEACTION_STATUS_SUCCEEDED
+from oslo_config import cfg
+
+opts = [
+    cfg.StrOpt(
+        "encryption_key_path",
+        default="conf/st2_kvstore_demo.crypto.key.json",
+        help="Location of the symmetric encryption key for encrypting values in kvstore. "
+        "This key should be in JSON and should've been generated using "
+        "st2-generate-symmetric-crypto-key tool.",
+    ),
+]
+cfg.CONF.register_opts(opts, group="actionrunner")
 from st2common.models.db.execution import ActionExecutionDB
 from st2common.services import action as action_service
 from st2tests.api import SUPER_SECRET_PARAMETER

st2api/tests/unit/controllers/v1/test_executions.py

@@ -30,6 +30,18 @@
 from st2common.content import utils as content_utils
 from st2common.models.api.keyvalue import KeyValuePairAPI
 from st2common.models.db.auth import UserDB
+from oslo_config import cfg
+
+opts = [
+    cfg.StrOpt(
+        "encryption_key_path",
+        default="conf/st2_kvstore_demo.crypto.key.json",
+        help="Location of the symmetric encryption key for encrypting values in kvstore. "
+        "This key should be in JSON and should've been generated using "
+        "st2-generate-symmetric-crypto-key tool.",
+    ),
+]
+cfg.CONF.register_opts(opts, group="actionrunner")
 from st2common.models.db.execution import ActionExecutionDB
 from st2common.models.db.execution import ActionExecutionOutputDB
 from st2common.models.db.keyvalue import KeyValuePairDB
@@ -1728,7 +1740,6 @@ def test_get_include_attributes_and_secret_parameters(self):
             ("/v1/actionexecutions?include_attributes=parameters,runner"),
             ("/v1/actionexecutions?include_attributes=parameters,action,runner"),
         ]
-
         for url in urls:
             resp = self.app.get(url + "&limit=1&show_secrets=True")
 

st2common/st2common/config.py

@@ -494,6 +494,13 @@ def register_opts(ignore_errors=False):
                 "that size"
             ),
         ),
+        cfg.StrOpt(
+            "encryption_key_path",
+            default="conf/st2_kvstore_demo.crypto.key.json",
+            help="Location of the symmetric encryption key for encrypting values in kvstore. "
+            "This key should be in JSON and should've been generated using "
+            "st2-generate-symmetric-crypto-key tool.",
+        ),
     ]
 
     do_register_opts(

st2common/st2common/models/db/execution.py

@@ -19,23 +19,24 @@
 
 import mongoengine as me
 
-from st2common import log as logging
 from st2common.models.db import stormbase
 from st2common.fields import JSONDictEscapedFieldCompatibilityField
 from st2common.fields import ComplexDateTimeField
 from st2common.util import date as date_utils
 from st2common.util import output_schema
+from oslo_config import cfg
+
 from st2common.util.secrets import get_secret_parameters
 from st2common.util.secrets import mask_inquiry_response
 from st2common.util.secrets import mask_secret_parameters
+from st2common.util.secrets import encrypt_secret_parameters
+from st2common.util.crypto import read_crypto_key
+
 from st2common.constants.types import ResourceType
 
 __all__ = ["ActionExecutionDB", "ActionExecutionOutputDB"]
 
 
-LOG = logging.getLogger(__name__)
-
-
 class ActionExecutionDB(stormbase.StormFoundationDB):
     RESOURCE_TYPE = ResourceType.EXECUTION
     UID_FIELDS = ["id"]
@@ -99,6 +100,7 @@ class ActionExecutionDB(stormbase.StormFoundationDB):
             {"fields": ["task_execution"]},
         ]
     }
+    encryption_key = read_crypto_key(cfg.CONF.actionrunner.encryption_key_path)
 
     def get_uid(self):
         # TODO Construct id from non id field:
@@ -178,6 +180,89 @@ def get_masked_parameters(self):
         serializable_dict = self.to_serializable_dict(mask_secrets=True)
         return serializable_dict["parameters"]
 
+    def save(self, *args, **kwargs):
+        original_parameters = copy.deepcopy(self.parameters)
+        parameters = {}
+        parameters.update(dict(self.action).get("parameters", {}))
+        parameters.update(dict(self.runner).get("runner_parameters", {}))
+        secret_parameters = get_secret_parameters(parameters=parameters)
+        encrpyted_parameters = encrypt_secret_parameters(
+            self.parameters, secret_parameters, self.encryption_key
+        )
+        self.parameters = encrpyted_parameters
+        liveaction_dict = dict(self.liveaction)
+        if "parameters" in liveaction_dict:
+            # We need to also encrypt the parameters inside liveaction
+            original_liveaction_parameters = liveaction_dict.get("parameters", {})
+            encrpyted_parameters = encrypt_secret_parameters(
+                original_liveaction_parameters, secret_parameters, self.encryption_key
+            )
+            liveaction_dict["parameters"] = encrpyted_parameters
+            self.liveaction = liveaction_dict
+            # We also mask response found inside parameters under liveaction.
+            # As mentioned above in mask_secrets function but I don't know what should be
+            # the expected behaviour as there we are making all the values because
+            # the schema is unknown
+        original_output_value = None
+        if self.result:
+            original_output_value = self.result
+            schema = dict(self.action).get("output_schema")
+            if schema is not None:
+                self.result = output_schema.encrypt_secret_output(
+                    self.encryption_key, self.result, schema
+                )
+                # # Need output key
+                # schema = self.action.get("output_schema")
+                # for key, spec in schema.items():
+                #     if key in self.result and spec.get("secret", False):
+                #         self.result[key] = str(symmetric_encrypt(self.encryption_key, self.result[key]))
+
+        self = super(ActionExecutionDB, self).save(*args, **kwargs)
+        # Resetting to the original values
+        setattr(self, "parameters", original_parameters)
+        if hasattr(self, "liveaction"):
+            liveaction_dict = dict(self.liveaction)
+            if "parameters" in liveaction_dict:
+                liveaction_dict["parameters"] = original_liveaction_parameters
+            self.liveaction = liveaction_dict
+
+        if hasattr(self, "result") and original_output_value is not None:
+            setattr(self, "result", original_output_value)
+        return self
+
+    def update(self, **kwargs):
+        parameters = {}
+        parameters.update(dict(self.action).get("parameters", {}))
+        parameters.update(dict(self.runner).get("runner_parameters", {}))
+        secret_parameters = get_secret_parameters(parameters=parameters)
+        encrpyted_parameters = encrypt_secret_parameters(
+            self.parameters, secret_parameters, self.encryption_key
+        )
+        self.parameters = encrpyted_parameters
+        if "set__liveaction" in kwargs and "parameters" in kwargs["set__liveaction"]:
+            encrpyted_parameters = encrypt_secret_parameters(
+                kwargs["set__liveaction"]["parameters"],
+                secret_parameters,
+                self.encryption_key,
+            )
+            kwargs["set__liveaction"]["parameters"] = encrpyted_parameters
+        if "set__result" in kwargs and "result" in kwargs["set__result"]:
+            output_value = kwargs["set__result"]["result"]
+            # Need output key
+            schema = dict(self.action).get("output_schema")
+            kwargs["set__result"]["result"] = output_schema.encrypt_secret_output(
+                self.encryption_key, output_value, schema
+            )
+
+        if "parameters" in self.liveaction:
+            original_liveaction_parameters = self.liveaction.get("parameters", {})
+            encrpyted_parameters = encrypt_secret_parameters(
+                original_liveaction_parameters, secret_parameters, self.encryption_key
+            )
+            self.liveaction.parameters = encrpyted_parameters
+
+        return super(ActionExecutionDB, self).update(**kwargs)
+
 
 class ActionExecutionOutputDB(stormbase.StormFoundationDB):
     """

st2common/st2common/models/db/liveaction.py

@@ -25,8 +25,11 @@
 from st2common.fields import ComplexDateTimeField
 from st2common.fields import JSONDictEscapedFieldCompatibilityField
 from st2common.util import date as date_utils
+from oslo_config import cfg
 from st2common.util.secrets import get_secret_parameters
 from st2common.util.secrets import mask_secret_parameters
+from st2common.util.secrets import encrypt_secret_parameters
+from st2common.util.crypto import read_crypto_key
 
 __all__ = [
     "LiveActionDB",
@@ -94,6 +97,7 @@ class LiveActionDB(stormbase.StormFoundationDB):
             {"fields": ["task_execution"]},
         ],
     }
+    encryption_key = read_crypto_key(cfg.CONF.actionrunner.encryption_key_path)
 
     def mask_secrets(self, value):
         from st2common.util import action_db
@@ -125,6 +129,29 @@ def get_masked_parameters(self):
         serializable_dict = self.to_serializable_dict(mask_secrets=True)
         return serializable_dict["parameters"]
 
+    def save(self, *args, **kwargs):
+        from st2common.util import action_db
+
+        original_parameters = copy.deepcopy(self.parameters)
+        action_parameters = action_db.get_action_parameters_specs(
+            action_ref=self.action
+        )
+        secret_parameters = get_secret_parameters(parameters=action_parameters)
+        encrpyted_parameters = encrypt_secret_parameters(
+            self.parameters, secret_parameters, self.encryption_key
+        )
+        self.parameters = encrpyted_parameters
+        self = super(LiveActionDB, self).save(*args, **kwargs)
+        # Reset parameters to original value after saving them to mongo
+        if "parameters" in self:
+            self.parameters = original_parameters
+        return self
+
+    def update(self, **kwargs):
+        # TODO : As of now update is not being used for LiveAction and that is why we did not
+        # add the encryption logic here but we should add it here soon
+        return super(LiveActionDB, self).update(**kwargs)
+
 
 # specialized access objects
 liveaction_access = MongoDBAccess(LiveActionDB)

st2common/st2common/persistence/execution.py

@@ -19,6 +19,10 @@
 from st2common.models.db.execution import ActionExecutionDB
 from st2common.models.db.execution import ActionExecutionOutputDB
 from st2common.persistence.base import Access
+from oslo_config import cfg
+from st2common.util.crypto import read_crypto_key
+from st2common.util.secrets import get_secret_parameters
+from st2common.util.secrets import decrypt_secret_parameters
 
 __all__ = [
     "ActionExecution",
@@ -29,6 +33,7 @@
 class ActionExecution(Access):
     impl = MongoDBAccess(ActionExecutionDB)
     publisher = None
+    encryption_key = read_crypto_key(cfg.CONF.actionrunner.encryption_key_path)
 
     @classmethod
     def _get_impl(cls):
@@ -44,6 +49,66 @@ def _get_publisher(cls):
     def delete_by_query(cls, *args, **query):
         return cls._get_impl().delete_by_query(*args, **query)
 
+    @classmethod
+    def get_by_name(cls, value):
+        result = cls.get(name=value, raise_exception=True)
+        return result
+
+    @classmethod
+    def get_by_id(cls, value):
+        instance = super(ActionExecution, cls).get_by_id(value)
+        instance = cls._decrypt_secrets(instance)
+        return instance
+
+    @classmethod
+    def get_by_uid(cls, value):
+        result = cls.get(uid=value, raise_exception=True)
+        return result
+
+    @classmethod
+    def get_by_ref(cls, value):
+        result = cls.get(ref=value, raise_exception=True)
+        return result
+
+    @classmethod
+    def get_by_pack(cls, value):
+        result = cls.get(pack=value, raise_exception=True)
+        return result
+
+    @classmethod
+    def get(cls, *args, **kwargs):
+        instance = super(ActionExecution, cls).get(*args, **kwargs)
+        if instance is None:
+            return instance
+        # Decrypt secrets if any
+        instance = cls._decrypt_secrets(instance)
+        return instance
+
+    @classmethod
+    def _decrypt_secrets(cls, instance):
+        if instance is None:
+            return instance
+        action = getattr(instance, "action", {})
+        runner = getattr(instance, "runner", {})
+        parameters = {}
+        parameters.update(action.get("parameters", {}))
+        parameters.update(runner.get("runner_parameters", {}))
+        secret_parameters = get_secret_parameters(parameters=parameters)
+
+        decrypt_parameters = decrypt_secret_parameters(
+            getattr(instance, "parameters", {}), secret_parameters, cls.encryption_key
+        )
+        setattr(instance, "parameters", decrypt_parameters)
+
+        liveaction_parameter = getattr(instance, "liveaction", {}).get("parameters", {})
+        if liveaction_parameter:
+            decrypt_liveaction_parameters = decrypt_secret_parameters(
+                liveaction_parameter, secret_parameters, cls.encryption_key
+            )
+            instance.liveaction["parameters"] = decrypt_liveaction_parameters
+
+        return instance
+
 
 class ActionExecutionOutput(Access):
     impl = MongoDBAccess(ActionExecutionOutputDB)