From 4b527904947b13680570fbdb91e76a23252ac0ad Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 6 Aug 2021 15:37:47 -0500 Subject: [PATCH 01/19] Process defaults and datastore values in more pack config locations --- st2common/st2common/util/config_loader.py | 121 ++++++++++++++++++---- 1 file changed, 102 insertions(+), 19 deletions(-) diff --git a/st2common/st2common/util/config_loader.py b/st2common/st2common/util/config_loader.py index aec424d75e..ce1cd2c18a 100644 --- a/st2common/st2common/util/config_loader.py +++ b/st2common/st2common/util/config_loader.py @@ -15,6 +15,7 @@ from __future__ import absolute_import import copy +import re import six @@ -101,7 +102,8 @@ def _get_values_for_config(self, config_schema_db, config_db): @staticmethod def _get_object_property_schema(object_schema, additional_properties_keys=None): """ - Create a schema for an object property using both additionalProperties and properties. + Create a schema for an object property using all of: properties, + patternProperties, and additionalProperties. :rtype: ``dict`` """ @@ -112,9 +114,52 @@ def _get_object_property_schema(object_schema, additional_properties_keys=None): # ensure that these keys are present in the object for key in additional_properties_keys: property_schema[key] = additional_properties - property_schema.update(object_schema.get("properties", {})) + + properties_schema = object_schema.get("properties", {}) + property_schema.update(properties_schema) + + potential_patterned_keys = set(additional_properties_keys) - set( + properties_schema.keys() + ) + + pattern_properties = object_schema.get("patternProperties", {}) + # patternProperties can be a boolean or a dict + if pattern_properties and isinstance(pattern_properties, dict): + # update any matching key + for raw_pattern, pattern_schema in pattern_properties.items(): + if not potential_patterned_keys: + # nothing to check. Don't compile any more patterns + break + pattern = re.compile(raw_pattern) + for key in list(potential_patterned_keys): + if pattern.search(key): + property_schema[key] = pattern_schema + potential_patterned_keys.remove(key) return property_schema + @staticmethod + def _get_array_items_schema(object_schema, items_count=0): + """ + Create a schema for array items using both additionalItems and items. + + :rtype: ``list`` + """ + items_schema = object_schema.get("items", []) + if isinstance(items_schema, dict): + items_schema = [items_schema] * items_count + items_schema_count = len(items_schema) + if items_schema_count >= items_count: + # no additional items to account for. + return items_schema + + additional_items = object_schema.get("additionalItems", {}) + # additionalItems can be a boolean or a dict + if additional_items and isinstance(additional_items, dict): + # ensure that these keys are present in the object + items_schema.extend([additional_items] * (items_count - items_schema_count)) + + return items_schema + def _assign_dynamic_config_values(self, schema, config, parent_keys=None): """ Assign dynamic config value for a particular config item if the ite utilizes a Jinja @@ -137,7 +182,13 @@ def _assign_dynamic_config_values(self, schema, config, parent_keys=None): if config_is_dict: # different schema for each key/value pair schema_item = schema.get(config_item_key, {}) - if config_is_list: + if config_is_list and isinstance(schema, list): + # positional schema for list items + try: + schema_item = schema[config_item_key] + except IndexError: + schema_item = {} + elif config_is_list: # same schema is shared between every item in the list schema_item = schema @@ -160,8 +211,12 @@ def _assign_dynamic_config_values(self, schema, config, parent_keys=None): ) # Inspect nested list items elif is_list: + items_schema = self._get_array_items_schema( + schema_item, + items_count=len(config[config_item_key]), + ) self._assign_dynamic_config_values( - schema=schema_item.get("items", {}), + schema=items_schema, config=config[config_item_key], parent_keys=current_keys, ) @@ -195,32 +250,60 @@ def _assign_default_values(self, schema, config): :rtype: ``dict`` """ - for schema_item_key, schema_item in six.iteritems(schema): + schema_is_dict = isinstance(schema, dict) + iterator = six.iteritems(schema) if schema_is_dict else enumerate(schema) + + for schema_item_key, schema_item in iterator: has_default_value = "default" in schema_item has_config_value = schema_item_key in config default_value = schema_item.get("default", None) - is_object = schema_item.get("type", None) == "object" - has_properties = schema_item.get("properties", None) - has_additional_properties = schema_item.get("additionalProperties", None) - if has_default_value and not has_config_value: # Config value is not provided, but default value is, use a default value config[schema_item_key] = default_value - # Inspect nested object properties - if is_object and (has_properties or has_additional_properties): - if not config.get(schema_item_key, None): - config[schema_item_key] = {} + schema_item_type = schema_item.get("type", None) - property_schema = self._get_object_property_schema( - schema_item, - additional_properties_keys=config[schema_item_key].keys(), + if schema_item_type == "object": + has_properties = schema_item.get("properties", None) + has_pattern_properties = schema_item.get("patternProperties", None) + has_additional_properties = schema_item.get( + "additionalProperties", None ) - self._assign_default_values( - schema=property_schema, config=config[schema_item_key] - ) + # Inspect nested object properties + if ( + has_properties + or has_pattern_properties + or has_additional_properties + ): + if not config.get(schema_item_key, None): + config[schema_item_key] = {} + + property_schema = self._get_object_property_schema( + schema_item, + additional_properties_keys=config[schema_item_key].keys(), + ) + + self._assign_default_values( + schema=property_schema, config=config[schema_item_key] + ) + elif schema_item_type == "array": + has_items = schema_item.get("items", None) + has_additional_items = schema_item.get("additionalItems", None) + + # Inspect nested array items + if has_items or has_additional_items: + if not config.get(schema_item_key, None): + config[schema_item_key] = [] + + items_schema = self._get_array_items_schema( + schema_item, + items_count=len(config[schema_item_key]), + ) + self._assign_default_values( + schema=items_schema, config=config[schema_item_key] + ) return config From 4c637ec899068e8fa50bb702b24b6e286b1a02a0 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 6 Aug 2021 17:08:05 -0500 Subject: [PATCH 02/19] do not edit object_schema --- st2common/st2common/util/config_loader.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/st2common/st2common/util/config_loader.py b/st2common/st2common/util/config_loader.py index ce1cd2c18a..7b26b7909a 100644 --- a/st2common/st2common/util/config_loader.py +++ b/st2common/st2common/util/config_loader.py @@ -144,9 +144,13 @@ def _get_array_items_schema(object_schema, items_count=0): :rtype: ``list`` """ - items_schema = object_schema.get("items", []) - if isinstance(items_schema, dict): - items_schema = [items_schema] * items_count + items_schema = [] + object_items_schema = object_schema.get("items", []) + if isinstance(object_items_schema, dict): + items_schema.extend([object_items_schema] * items_count) + else: + items_schema.extend(object_items_schema) + items_schema_count = len(items_schema) if items_schema_count >= items_count: # no additional items to account for. From 6ed8e5d297561f257daed38bd5cd6a0b6afa4132 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 6 Aug 2021 17:11:04 -0500 Subject: [PATCH 03/19] do not use six on new lines of code --- st2common/st2common/util/config_loader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/st2common/st2common/util/config_loader.py b/st2common/st2common/util/config_loader.py index 7b26b7909a..1fe15114ab 100644 --- a/st2common/st2common/util/config_loader.py +++ b/st2common/st2common/util/config_loader.py @@ -255,7 +255,7 @@ def _assign_default_values(self, schema, config): :rtype: ``dict`` """ schema_is_dict = isinstance(schema, dict) - iterator = six.iteritems(schema) if schema_is_dict else enumerate(schema) + iterator = schema.items() if schema_is_dict else enumerate(schema) for schema_item_key, schema_item in iterator: has_default_value = "default" in schema_item From e6794870bd81725d86e9bc31e283c0c03a29490a Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 31 Mar 2022 00:43:16 -0500 Subject: [PATCH 04/19] Add tests for pack config patternProperties and additionalItems --- st2common/tests/unit/test_config_loader.py | 110 ++++++++++++++++++ .../config.schema.yaml | 24 ++++ .../pack.yaml | 6 + .../config.schema.yaml | 25 ++++ .../pack.yaml | 6 + 5 files changed, 171 insertions(+) create mode 100644 st2tests/st2tests/fixtures/packs/dummy_pack_schema_with_additional_items_1/config.schema.yaml create mode 100644 st2tests/st2tests/fixtures/packs/dummy_pack_schema_with_additional_items_1/pack.yaml create mode 100644 st2tests/st2tests/fixtures/packs/dummy_pack_schema_with_pattern_properties_1/config.schema.yaml create mode 100644 st2tests/st2tests/fixtures/packs/dummy_pack_schema_with_pattern_properties_1/pack.yaml diff --git a/st2common/tests/unit/test_config_loader.py b/st2common/tests/unit/test_config_loader.py index d8616a3b44..5e5421214c 100644 --- a/st2common/tests/unit/test_config_loader.py +++ b/st2common/tests/unit/test_config_loader.py @@ -575,6 +575,116 @@ def test_get_config_dynamic_config_item_under_additional_properties(self): config_db.delete() + def test_get_config_dynamic_config_item_under_pattern_properties(self): + pack_name = "dummy_pack_schema_with_pattern_properties_1" + loader = ContentPackConfigLoader(pack_name=pack_name) + + encrypted_value = crypto.symmetric_encrypt( + KeyValuePairAPI.crypto_key, "v1_encrypted" + ) + KeyValuePair.add_or_update( + KeyValuePairDB(name="k1_encrypted", value=encrypted_value, secret=True) + ) + + #################### + # values in objects under an object with additionalProperties + values = { + "profiles": { + "dev": { + # no host or port to test default value + "token": "hard-coded-secret", + }, + "prod": { + "host": "127.1.2.7", + "port": 8282, + # encrypted in datastore + "token": "{{st2kv.system.k1_encrypted}}", + # schema declares `secret: true` which triggers auto-decryption. + # If this were not encrypted, it would try to decrypt it and fail. + }, + } + } + config_db = ConfigDB(pack=pack_name, values=values) + config_db = Config.add_or_update(config_db) + + config_rendered = loader.get_config() + + self.assertEqual( + config_rendered, + { + "region": "us-east-1", + "profiles": { + "dev": { + "host": "127.0.0.3", + "port": 8080, + "token": "hard-coded-secret", + }, + "prod": { + "host": "127.1.2.7", + "port": 8282, + "token": "v1_encrypted", + }, + }, + }, + ) + + config_db.delete() + + def test_get_config_dynamic_config_item_under_additional_items(self): + pack_name = "dummy_pack_schema_with_additional_items_1" + loader = ContentPackConfigLoader(pack_name=pack_name) + + encrypted_value = crypto.symmetric_encrypt( + KeyValuePairAPI.crypto_key, "v1_encrypted" + ) + KeyValuePair.add_or_update( + KeyValuePairDB(name="k1_encrypted", value=encrypted_value, secret=True) + ) + + #################### + # values in objects under an object with additionalProperties + values = { + "profiles": [ + { + # no host or port to test default value + "token": "hard-coded-secret", + }, + { + "host": "127.1.2.7", + "port": 8282, + # encrypted in datastore + "token": "{{st2kv.system.k1_encrypted}}", + # schema declares `secret: true` which triggers auto-decryption. + # If this were not encrypted, it would try to decrypt it and fail. + }, + ] + } + config_db = ConfigDB(pack=pack_name, values=values) + config_db = Config.add_or_update(config_db) + + config_rendered = loader.get_config() + + self.assertEqual( + config_rendered, + { + "region": "us-east-1", + "profiles": [ + { + "host": "127.0.0.3", + "port": 8080, + "token": "hard-coded-secret", + }, + { + "host": "127.1.2.7", + "port": 8282, + "token": "v1_encrypted", + }, + ], + }, + ) + + config_db.delete() + def test_empty_config_object_in_the_database(self): pack_name = "dummy_pack_empty_config" diff --git a/st2tests/st2tests/fixtures/packs/dummy_pack_schema_with_additional_items_1/config.schema.yaml b/st2tests/st2tests/fixtures/packs/dummy_pack_schema_with_additional_items_1/config.schema.yaml new file mode 100644 index 0000000000..b0bcb2091b --- /dev/null +++ b/st2tests/st2tests/fixtures/packs/dummy_pack_schema_with_additional_items_1/config.schema.yaml @@ -0,0 +1,24 @@ +--- + region: + type: "string" + required: false + default: "us-east-1" + profiles: + type: "array" + required: false + additionalItems: + type: object + additionalProperties: false + properties: + host: + type: "string" + required: false + default: "127.0.0.3" + port: + type: "integer" + required: false + default: 8080 + token: + type: "string" + required: true + secret: true diff --git a/st2tests/st2tests/fixtures/packs/dummy_pack_schema_with_additional_items_1/pack.yaml b/st2tests/st2tests/fixtures/packs/dummy_pack_schema_with_additional_items_1/pack.yaml new file mode 100644 index 0000000000..172b029d27 --- /dev/null +++ b/st2tests/st2tests/fixtures/packs/dummy_pack_schema_with_additional_items_1/pack.yaml @@ -0,0 +1,6 @@ +--- +name : dummy_pack_schema_with_additional_items_1 +description : dummy pack with nested objects under additionalItems +version : 0.1.0 +author : st2-dev +email : info@stackstorm.com diff --git a/st2tests/st2tests/fixtures/packs/dummy_pack_schema_with_pattern_properties_1/config.schema.yaml b/st2tests/st2tests/fixtures/packs/dummy_pack_schema_with_pattern_properties_1/config.schema.yaml new file mode 100644 index 0000000000..237507215f --- /dev/null +++ b/st2tests/st2tests/fixtures/packs/dummy_pack_schema_with_pattern_properties_1/config.schema.yaml @@ -0,0 +1,25 @@ +--- + region: + type: "string" + required: false + default: "us-east-1" + profiles: + type: "object" + required: false + patternProperties: + "^\\w+$": + type: object + additionalProperties: false + properties: + host: + type: "string" + required: false + default: "127.0.0.3" + port: + type: "integer" + required: false + default: 8080 + token: + type: "string" + required: true + secret: true diff --git a/st2tests/st2tests/fixtures/packs/dummy_pack_schema_with_pattern_properties_1/pack.yaml b/st2tests/st2tests/fixtures/packs/dummy_pack_schema_with_pattern_properties_1/pack.yaml new file mode 100644 index 0000000000..b0816cf159 --- /dev/null +++ b/st2tests/st2tests/fixtures/packs/dummy_pack_schema_with_pattern_properties_1/pack.yaml @@ -0,0 +1,6 @@ +--- +name : dummy_pack_schema_with_pattern_properties_1 +description : dummy pack with nested objects under patternProperties +version : 0.1.0 +author : st2-dev +email : info@stackstorm.com From 81053603c86c5b8161968020442cb5f22dc9ab11 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 31 Mar 2022 11:44:06 -0500 Subject: [PATCH 05/19] Fix list access in config_loader._assign_default_values It was trying to access lists as if they were dictionaries. So, adjust to allow both dicts and lists. --- st2common/st2common/util/config_loader.py | 29 +++++++++++++++-------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/st2common/st2common/util/config_loader.py b/st2common/st2common/util/config_loader.py index 1fe15114ab..fb7d268f06 100644 --- a/st2common/st2common/util/config_loader.py +++ b/st2common/st2common/util/config_loader.py @@ -252,20 +252,29 @@ def _assign_default_values(self, schema, config): Note: This method mutates config argument in place. - :rtype: ``dict`` + :rtype: ``dict|list`` """ schema_is_dict = isinstance(schema, dict) iterator = schema.items() if schema_is_dict else enumerate(schema) + # _get_*_schema ensures that schema_item is always a dict for schema_item_key, schema_item in iterator: has_default_value = "default" in schema_item - has_config_value = schema_item_key in config + if isinstance(config, dict): + has_config_value = schema_item_key in config + else: + has_config_value = schema_item_key < len(config) default_value = schema_item.get("default", None) if has_default_value and not has_config_value: # Config value is not provided, but default value is, use a default value config[schema_item_key] = default_value + try: + config_value = config[schema_item_key] + except (KeyError, IndexError): + config_value = None + schema_item_type = schema_item.get("type", None) if schema_item_type == "object": @@ -281,16 +290,16 @@ def _assign_default_values(self, schema, config): or has_pattern_properties or has_additional_properties ): - if not config.get(schema_item_key, None): - config[schema_item_key] = {} + if not config_value: + config_value = config[schema_item_key] = {} property_schema = self._get_object_property_schema( schema_item, - additional_properties_keys=config[schema_item_key].keys(), + additional_properties_keys=config_value.keys(), ) self._assign_default_values( - schema=property_schema, config=config[schema_item_key] + schema=property_schema, config=config_value ) elif schema_item_type == "array": has_items = schema_item.get("items", None) @@ -298,15 +307,15 @@ def _assign_default_values(self, schema, config): # Inspect nested array items if has_items or has_additional_items: - if not config.get(schema_item_key, None): - config[schema_item_key] = [] + if not config_value: + config_value = config[schema_item_key] = [] items_schema = self._get_array_items_schema( schema_item, - items_count=len(config[schema_item_key]), + items_count=len(config_value), ) self._assign_default_values( - schema=items_schema, config=config[schema_item_key] + schema=items_schema, config=config_value ) return config From 9aead0f9dbaee5eb72d139a35e466b2910fa3b84 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 31 Mar 2022 14:36:02 -0500 Subject: [PATCH 06/19] Add changelog entry --- CHANGELOG.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6770482b3a..3d34f4f956 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -185,6 +185,12 @@ Added Contributed by @nzlosh +* Fix a bug in the pack config loader so that objects covered by an ``patternProperties`` schema + or arrays using ``additionalItems`` schema(s) can use encrypted datastore keys and have their + default values applied correctly. #5321 + + Contributed by @cognifloyd. + Changed ~~~~~~~ From ba06db4bba5b1d2d3db833ce0d29911b940da297 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 28 Jun 2022 15:01:32 -0500 Subject: [PATCH 07/19] Move changelog entry --- CHANGELOG.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 93c01533f8..13e0f03702 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -9,6 +9,12 @@ Fixed * Fix redis SSL problems with sentinel #5660 +* Fix a bug in the pack config loader so that objects covered by an ``patternProperties`` schema + or arrays using ``additionalItems`` schema(s) can use encrypted datastore keys and have their + default values applied correctly. #5321 + + Contributed by @cognifloyd. + 3.7.0 - May 05, 2022 -------------------- @@ -227,12 +233,6 @@ Added Contributed by @nzlosh -* Fix a bug in the pack config loader so that objects covered by an ``patternProperties`` schema - or arrays using ``additionalItems`` schema(s) can use encrypted datastore keys and have their - default values applied correctly. #5321 - - Contributed by @cognifloyd. - Changed ~~~~~~~ From 8bffa2f04eef1f87409e1da742cefd439c0b7c39 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 29 Jun 2022 01:16:09 -0500 Subject: [PATCH 08/19] Refactor var/method names for clarity This documents the properties schema flattening algorithm with code comments. That highlighted some vars and a method that could be renamed to clarify the code even more. --- st2common/st2common/util/config_loader.py | 57 ++++++++++++++++------- 1 file changed, 41 insertions(+), 16 deletions(-) diff --git a/st2common/st2common/util/config_loader.py b/st2common/st2common/util/config_loader.py index 2288a54a35..69ebb1f2ec 100644 --- a/st2common/st2common/util/config_loader.py +++ b/st2common/st2common/util/config_loader.py @@ -100,27 +100,52 @@ def _get_values_for_config(self, config_schema_db, config_db): return config @staticmethod - def _get_object_property_schema(object_schema, additional_properties_keys=None): + def _get_object_properties_schema(object_schema, objecy_keys=None): """ Create a schema for an object property using all of: properties, patternProperties, and additionalProperties. + This 'flattens' properties, patternProperties, and additionalProperties + so that we can handle patternProperties and additionalProperties + as if they were defined in properties. + So, every key in objecy_keys will be assigned a schema + from properties, patternProperties, or additionalProperties. + + NOTE: order of precedence: properties, patternProperties, additionalProperties + So, the additionalProperties schema is only used for keys that are not in + properties and that do not match any of the patterns in patternProperties. + And, patternProperties schemas only apply to keys missing from properties. + :rtype: ``dict`` """ - property_schema = {} + flattened_properties_schema = {} + + # First, eagerly add the additionalProperties schema for all object_keys to + # avoid tracking which keys are covered by patternProperties and properties. + # This schema will subsequently be replaced by the more-specific key matches + # in patternProperties and properties. + additional_properties = object_schema.get("additionalProperties", {}) # additionalProperties can be a boolean or a dict if additional_properties and isinstance(additional_properties, dict): # ensure that these keys are present in the object - for key in additional_properties_keys: - property_schema[key] = additional_properties + for key in objecy_keys: + flattened_properties_schema[key] = additional_properties + + # Second, replace the additionalProperties schemas with any + # explicit property schemas in propertiea. properties_schema = object_schema.get("properties", {}) - property_schema.update(properties_schema) + flattened_properties_schema.update(properties_schema) - potential_patterned_keys = set(additional_properties_keys) - set( - properties_schema.keys() - ) + # Third, calculate which keys are in object_keys but not in properties. + # These are the only keys that can be matched with patternnProperties. + + potential_patterned_keys = set(objecy_keys) - set(properties_schema.keys()) + + # Fourth, match the remaining keys with patternProperties, + # and replace the additionalProperties schema with the patternProperties schema + # because patternProperties is more specific than additionalProperties. pattern_properties = object_schema.get("patternProperties", {}) # patternProperties can be a boolean or a dict @@ -133,9 +158,9 @@ def _get_object_property_schema(object_schema, additional_properties_keys=None): pattern = re.compile(raw_pattern) for key in list(potential_patterned_keys): if pattern.search(key): - property_schema[key] = pattern_schema + flattened_properties_schema[key] = pattern_schema potential_patterned_keys.remove(key) - return property_schema + return flattened_properties_schema @staticmethod def _get_array_items_schema(object_schema, items_count=0): @@ -204,12 +229,12 @@ def _assign_dynamic_config_values(self, schema, config, parent_keys=None): # Inspect nested object properties if is_dictionary: - property_schema = self._get_object_property_schema( + properties_schema = self._get_object_properties_schema( schema_item, - additional_properties_keys=config_item_value.keys(), + objecy_keys=config_item_value.keys(), ) self._assign_dynamic_config_values( - schema=property_schema, + schema=properties_schema, config=config[config_item_key], parent_keys=current_keys, ) @@ -293,13 +318,13 @@ def _assign_default_values(self, schema, config): if not config_value: config_value = config[schema_item_key] = {} - property_schema = self._get_object_property_schema( + properties_schema = self._get_object_properties_schema( schema_item, - additional_properties_keys=config_value.keys(), + objecy_keys=config_value.keys(), ) self._assign_default_values( - schema=property_schema, config=config_value + schema=properties_schema, config=config_value ) elif schema_item_type == "array": has_items = schema_item.get("items", None) From bbfb5108bb88d63010ae43dfd83d80c106e3e778 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 29 Jun 2022 20:51:45 -0500 Subject: [PATCH 09/19] extend docstring for pack config array schema handling --- st2common/st2common/util/config_loader.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/st2common/st2common/util/config_loader.py b/st2common/st2common/util/config_loader.py index 69ebb1f2ec..7a90b49d0b 100644 --- a/st2common/st2common/util/config_loader.py +++ b/st2common/st2common/util/config_loader.py @@ -167,6 +167,13 @@ def _get_array_items_schema(object_schema, items_count=0): """ Create a schema for array items using both additionalItems and items. + This 'flattens' items and additionalItems so that we can handle additionalItems + as if each additional item was defined in items. + + The additionalItems schema will only be used if the items schema is shorter + than items_count. So, when additionalItems is defined, the items schema will be + extended to be at least as long as items_count. + :rtype: ``list`` """ items_schema = [] @@ -184,7 +191,7 @@ def _get_array_items_schema(object_schema, items_count=0): additional_items = object_schema.get("additionalItems", {}) # additionalItems can be a boolean or a dict if additional_items and isinstance(additional_items, dict): - # ensure that these keys are present in the object + # ensure that these indexes are present in the array items_schema.extend([additional_items] * (items_count - items_schema_count)) return items_schema From 3d8b2f63209cc69468ce3dc5931d0d29a5d5507e Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 29 Jun 2022 21:12:03 -0500 Subject: [PATCH 10/19] refactor var naming in pack config items schema handling for clarity --- st2common/st2common/util/config_loader.py | 28 +++++++++++++---------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/st2common/st2common/util/config_loader.py b/st2common/st2common/util/config_loader.py index 7a90b49d0b..ee966effca 100644 --- a/st2common/st2common/util/config_loader.py +++ b/st2common/st2common/util/config_loader.py @@ -163,7 +163,7 @@ def _get_object_properties_schema(object_schema, objecy_keys=None): return flattened_properties_schema @staticmethod - def _get_array_items_schema(object_schema, items_count=0): + def _get_array_items_schema(array_schema, items_count=0): """ Create a schema for array items using both additionalItems and items. @@ -176,25 +176,29 @@ def _get_array_items_schema(object_schema, items_count=0): :rtype: ``list`` """ - items_schema = [] - object_items_schema = object_schema.get("items", []) - if isinstance(object_items_schema, dict): - items_schema.extend([object_items_schema] * items_count) + flattened_items_schema = [] + items_schema = array_schema.get("items", []) + if isinstance(items_schema, dict): + # with only one schema for all items, additionalItems will be ignored. + flattened_items_schema.extend([items_schema] * items_count) else: - items_schema.extend(object_items_schema) + # items is a positional array of schemas + flattened_items_schema.extend(items_schema) - items_schema_count = len(items_schema) - if items_schema_count >= items_count: + flattened_items_schema_count = len(flattened_items_schema) + if flattened_items_schema_count >= items_count: # no additional items to account for. - return items_schema + return flattened_items_schema - additional_items = object_schema.get("additionalItems", {}) + additional_items = array_schema.get("additionalItems", {}) # additionalItems can be a boolean or a dict if additional_items and isinstance(additional_items, dict): # ensure that these indexes are present in the array - items_schema.extend([additional_items] * (items_count - items_schema_count)) + flattened_items_schema.extend( + [additional_items] * (items_count - flattened_items_schema_count) + ) - return items_schema + return flattened_items_schema def _assign_dynamic_config_values(self, schema, config, parent_keys=None): """ From 2c9ebf156832fec9f15ac1e226bb27ca2fbebbff Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 30 Jun 2022 11:51:59 -0500 Subject: [PATCH 11/19] typo --- st2common/st2common/util/config_loader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/st2common/st2common/util/config_loader.py b/st2common/st2common/util/config_loader.py index ee966effca..d5345f59a6 100644 --- a/st2common/st2common/util/config_loader.py +++ b/st2common/st2common/util/config_loader.py @@ -129,7 +129,7 @@ def _get_object_properties_schema(object_schema, objecy_keys=None): # additionalProperties can be a boolean or a dict if additional_properties and isinstance(additional_properties, dict): # ensure that these keys are present in the object - for key in objecy_keys: + for key in object_keys: flattened_properties_schema[key] = additional_properties # Second, replace the additionalProperties schemas with any From b62016a54e46418212202f3620314a97d79b3f75 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 30 Jun 2022 16:09:23 -0500 Subject: [PATCH 12/19] refactor pack config properties schema flattening for clarity --- st2common/st2common/util/config_loader.py | 58 +++++++++++------------ 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/st2common/st2common/util/config_loader.py b/st2common/st2common/util/config_loader.py index d5345f59a6..356714a5cc 100644 --- a/st2common/st2common/util/config_loader.py +++ b/st2common/st2common/util/config_loader.py @@ -118,48 +118,48 @@ def _get_object_properties_schema(object_schema, objecy_keys=None): :rtype: ``dict`` """ - flattened_properties_schema = {} - - # First, eagerly add the additionalProperties schema for all object_keys to - # avoid tracking which keys are covered by patternProperties and properties. - # This schema will subsequently be replaced by the more-specific key matches - # in patternProperties and properties. - - additional_properties = object_schema.get("additionalProperties", {}) - # additionalProperties can be a boolean or a dict - if additional_properties and isinstance(additional_properties, dict): - # ensure that these keys are present in the object - for key in object_keys: - flattened_properties_schema[key] = additional_properties - - # Second, replace the additionalProperties schemas with any - # explicit property schemas in propertiea. + # preserve the order in object_keys + flattened_properties_schema = {key: {} for key in object_keys} + # properties takes precedence over patternProperties and additionalProperties properties_schema = object_schema.get("properties", {}) flattened_properties_schema.update(properties_schema) - # Third, calculate which keys are in object_keys but not in properties. - # These are the only keys that can be matched with patternnProperties. + # extra_keys has keys that may use patternProperties or additionalProperties + # we remove keys when they have been assigned a schema + extra_keys = set(objecy_keys) - set(properties_schema.keys()) - potential_patterned_keys = set(objecy_keys) - set(properties_schema.keys()) - - # Fourth, match the remaining keys with patternProperties, - # and replace the additionalProperties schema with the patternProperties schema - # because patternProperties is more specific than additionalProperties. + if not extra_keys: + # nothing to check. Don't look at patternProperties or additionalProperties. + return flattened_properties_schema + # match each key against patternPropetties pattern_properties = object_schema.get("patternProperties", {}) # patternProperties can be a boolean or a dict if pattern_properties and isinstance(pattern_properties, dict): - # update any matching key for raw_pattern, pattern_schema in pattern_properties.items(): - if not potential_patterned_keys: - # nothing to check. Don't compile any more patterns - break pattern = re.compile(raw_pattern) - for key in list(potential_patterned_keys): + for key in list(extra_keys): if pattern.search(key): + # update matched key flattened_properties_schema[key] = pattern_schema - potential_patterned_keys.remove(key) + # don't check matched key against any more patterns + # and don't overwrite with additionalProperties + extra_keys.remove(key) + + if not extra_keys: + # nothing to check. Don't compile any more patterns + # and don't look at additionalProperties. + return flattened_properties_schema + + # fill in any remaining keys with additionalProperties + additional_properties = object_schema.get("additionalProperties", {}) + # additionalProperties can be a boolean or a dict + if additional_properties and isinstance(additional_properties, dict): + # ensure that these keys are present in the object + for key in extra_keys: + flattened_properties_schema[key] = additional_properties + return flattened_properties_schema @staticmethod From 815b41530a82dbac9e18343b8bb4fef58aeb1673 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 30 Jun 2022 16:10:49 -0500 Subject: [PATCH 13/19] typo --- st2common/st2common/util/config_loader.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/st2common/st2common/util/config_loader.py b/st2common/st2common/util/config_loader.py index 356714a5cc..6940bbc1df 100644 --- a/st2common/st2common/util/config_loader.py +++ b/st2common/st2common/util/config_loader.py @@ -100,7 +100,7 @@ def _get_values_for_config(self, config_schema_db, config_db): return config @staticmethod - def _get_object_properties_schema(object_schema, objecy_keys=None): + def _get_object_properties_schema(object_schema, object_keys=None): """ Create a schema for an object property using all of: properties, patternProperties, and additionalProperties. @@ -108,7 +108,7 @@ def _get_object_properties_schema(object_schema, objecy_keys=None): This 'flattens' properties, patternProperties, and additionalProperties so that we can handle patternProperties and additionalProperties as if they were defined in properties. - So, every key in objecy_keys will be assigned a schema + So, every key in object_keys will be assigned a schema from properties, patternProperties, or additionalProperties. NOTE: order of precedence: properties, patternProperties, additionalProperties @@ -127,7 +127,7 @@ def _get_object_properties_schema(object_schema, objecy_keys=None): # extra_keys has keys that may use patternProperties or additionalProperties # we remove keys when they have been assigned a schema - extra_keys = set(objecy_keys) - set(properties_schema.keys()) + extra_keys = set(object_keys) - set(properties_schema.keys()) if not extra_keys: # nothing to check. Don't look at patternProperties or additionalProperties. @@ -242,7 +242,7 @@ def _assign_dynamic_config_values(self, schema, config, parent_keys=None): if is_dictionary: properties_schema = self._get_object_properties_schema( schema_item, - objecy_keys=config_item_value.keys(), + object_keys=config_item_value.keys(), ) self._assign_dynamic_config_values( schema=properties_schema, @@ -331,7 +331,7 @@ def _assign_default_values(self, schema, config): properties_schema = self._get_object_properties_schema( schema_item, - objecy_keys=config_value.keys(), + object_keys=config_value.keys(), ) self._assign_default_values( From 398539eb5b0310790fefec4429d053dc10180772 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 4 Jul 2022 18:23:14 -0500 Subject: [PATCH 14/19] add order of precedence test for config_loader --- st2common/tests/unit/test_config_loader.py | 101 +++++++++++++++++- .../config.schema.yaml | 61 +++++++++++ .../pack.yaml | 6 ++ 3 files changed, 167 insertions(+), 1 deletion(-) create mode 100644 st2tests/st2tests/fixtures/packs/dummy_pack_schema_with_pattern_and_additional_properties_1/config.schema.yaml create mode 100644 st2tests/st2tests/fixtures/packs/dummy_pack_schema_with_pattern_and_additional_properties_1/pack.yaml diff --git a/st2common/tests/unit/test_config_loader.py b/st2common/tests/unit/test_config_loader.py index 5e5421214c..7a68d05de6 100644 --- a/st2common/tests/unit/test_config_loader.py +++ b/st2common/tests/unit/test_config_loader.py @@ -587,7 +587,7 @@ def test_get_config_dynamic_config_item_under_pattern_properties(self): ) #################### - # values in objects under an object with additionalProperties + # values in objects under an object with patternProperties values = { "profiles": { "dev": { @@ -630,6 +630,105 @@ def test_get_config_dynamic_config_item_under_pattern_properties(self): config_db.delete() + def test_get_config_dynamic_config_item_properties_order_of_precedence(self): + pack_name = "dummy_pack_schema_with_properties_1" + loader = ContentPackConfigLoader(pack_name=pack_name) + + encrypted_value = crypto.symmetric_encrypt( + KeyValuePairAPI.crypto_key, "v1_encrypted" + ) + KeyValuePair.add_or_update( + KeyValuePairDB(name="k1_encrypted", value=encrypted_value, secret=True) + ) + KeyValuePair.add_or_update( + KeyValuePairDB(name="k2_encrypted", value=encrypted_value, secret=True) + ) + KeyValuePair.add_or_update( + KeyValuePairDB(name="k3_encrypted", value=encrypted_value, secret=True) + ) + + #################### + # values in objects under an object with additionalProperties + values = { + "profiles": { + # properties + "foo": { + "domain": "foo.example.com", + "token": "hard-coded-secret", + }, + "bar": { + "domain": "bar.example.com", + "token": "{{st2kv.system.k1_encrypted}}", + }, + # patternProperties start with env- + "env-dev": { + "host": "127.0.0.127", + "token": "hard-coded-secret", + }, + "env-prod": { + "host": "127.1.2.7", + "port": 8282, + # encrypted in datastore + "token": "{{st2kv.system.k2_encrypted}}", + # schema declares `secret: true` which triggers auto-decryption. + # If this were not encrypted, it would try to decrypt it and fail. + }, + # additionalProperties + "dev": { + "url": "https://example.com", + "token": "hard-coded-secret", + }, + "prod": { + "url": "https://other.example.com", + "port": 2345, + "token": "{{st2kv.system.k3_encrypted}}", + }, + } + } + config_db = ConfigDB(pack=pack_name, values=values) + config_db = Config.add_or_update(config_db) + + config_rendered = loader.get_config() + + self.assertEqual( + config_rendered, + { + "region": "us-east-1", + "profiles": { + "foo": { + "domain": "foo.example.com", + "token": "hard-coded-secret", + }, + "bar": { + "domain": "bar.example.com", + "token": "v1_encrypted", + }, + "env-dev": { + "host": "127.0.0.127", + "port": 8080, + "token": "hard-coded-secret", + }, + "env-prod": { + "host": "127.1.2.7", + "port": 8282, + "token": "v1_encrypted", + }, + "dev": { + "url": "https://example.com", + "port": 1234, + "token": "hard-coded-secret", + }, + "prod": { + "url": "https://other.example.com", + "port": 2345, + "token": "v1_encrypted", + }, + }, + }, + ) + + config_db.delete() + def test_get_config_dynamic_config_item_under_additional_items(self): pack_name = "dummy_pack_schema_with_additional_items_1" loader = ContentPackConfigLoader(pack_name=pack_name) diff --git a/st2tests/st2tests/fixtures/packs/dummy_pack_schema_with_pattern_and_additional_properties_1/config.schema.yaml b/st2tests/st2tests/fixtures/packs/dummy_pack_schema_with_pattern_and_additional_properties_1/config.schema.yaml new file mode 100644 index 0000000000..eb60d04cb0 --- /dev/null +++ b/st2tests/st2tests/fixtures/packs/dummy_pack_schema_with_pattern_and_additional_properties_1/config.schema.yaml @@ -0,0 +1,61 @@ +--- + region: + type: "string" + required: false + default: "us-east-1" + profiles: + type: "object" + required: false + # order of precedence: properties, patternProperties, additionalProperties + properties: + foo: + type: object + properties: + domain: + type: "string" + required: true + token: + type: "string" + required: true + secret: true + bar: + type: object + properties: + domain: + type: "string" + required: true + token: + type: "string" + required: true + secret: true + patternProperties: + "^env-\\w+$": + type: object + additionalProperties: false + properties: + host: + type: "string" + required: true + port: + type: "integer" + required: false + default: 8080 + token: + type: "string" + required: true + secret: true + additionalProperties: + type: object + additionalProperties: false + properties: + url: + type: "string" + required: true + port: + type: "integer" + required: false + default: 1234 + token: + type: "string" + required: true + secret: true diff --git a/st2tests/st2tests/fixtures/packs/dummy_pack_schema_with_pattern_and_additional_properties_1/pack.yaml b/st2tests/st2tests/fixtures/packs/dummy_pack_schema_with_pattern_and_additional_properties_1/pack.yaml new file mode 100644 index 0000000000..5861470ddd --- /dev/null +++ b/st2tests/st2tests/fixtures/packs/dummy_pack_schema_with_pattern_and_additional_properties_1/pack.yaml @@ -0,0 +1,6 @@ +--- +name : dummy_pack_schema_with_pattern_and_additional_properties_1 +description : dummy pack with nested objects under patternProperties and additionalProperties +version : 0.1.0 +author : st2-dev +email : info@stackstorm.com From 5931e6d266cbfcb9eba5719102773d47986cbea0 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 4 Jul 2022 19:41:59 -0500 Subject: [PATCH 15/19] patternProperties matches all patterns against all keys In draft 4, patternProperties creates an all-of set of schemas for each property. Each property gets compared against all patterns. And it must match all of the schemas for the patterns it matches. So, we cannot avoid compiling some of the patterns. https://datatracker.ietf.org/doc/html/draft-fge-json-schema-validation-00#section-8.3 --- st2common/st2common/util/config_loader.py | 34 ++++++++++++++--------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/st2common/st2common/util/config_loader.py b/st2common/st2common/util/config_loader.py index 6940bbc1df..5b59c87de5 100644 --- a/st2common/st2common/util/config_loader.py +++ b/st2common/st2common/util/config_loader.py @@ -137,20 +137,28 @@ def _get_object_properties_schema(object_schema, object_keys=None): pattern_properties = object_schema.get("patternProperties", {}) # patternProperties can be a boolean or a dict if pattern_properties and isinstance(pattern_properties, dict): - for raw_pattern, pattern_schema in pattern_properties.items(): - pattern = re.compile(raw_pattern) - for key in list(extra_keys): + # we need to match all extra_keys against all patterns + # and then compose the per-property schema from all + # the matched patterns' properties. + pattern_properties = { + re.compile(raw_pattern): pattern_schema + for raw_pattern, pattern_schema in pattern_properties.items() + } + for key in list(extra_keys): + key_schemas = [] + for pattern, pattern_schema in pattern_properties.items(): if pattern.search(key): - # update matched key - flattened_properties_schema[key] = pattern_schema - # don't check matched key against any more patterns - # and don't overwrite with additionalProperties - extra_keys.remove(key) - - if not extra_keys: - # nothing to check. Don't compile any more patterns - # and don't look at additionalProperties. - return flattened_properties_schema + key_schemas.append(pattern_schema) + if key_schemas: + composed_schema = {**schema for schema in key_schemas} + # update matched key + flattened_properties_schema[key] = composed_schema + # don't overwrite matched key's schema with additionalProperties + extra_keys.remove(key) + + if not extra_keys: + # nothing else to check. Don't look at additionalProperties. + return flattened_properties_schema # fill in any remaining keys with additionalProperties additional_properties = object_schema.get("additionalProperties", {}) From 62761ca93735250536bac848ce5a639dfd7dff95 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 4 Jul 2022 20:06:01 -0500 Subject: [PATCH 16/19] improve code comments --- st2common/st2common/util/config_loader.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/st2common/st2common/util/config_loader.py b/st2common/st2common/util/config_loader.py index 5b59c87de5..cd1d8af6a4 100644 --- a/st2common/st2common/util/config_loader.py +++ b/st2common/st2common/util/config_loader.py @@ -135,7 +135,7 @@ def _get_object_properties_schema(object_schema, object_keys=None): # match each key against patternPropetties pattern_properties = object_schema.get("patternProperties", {}) - # patternProperties can be a boolean or a dict + # patternProperties should be a dict if defined if pattern_properties and isinstance(pattern_properties, dict): # we need to match all extra_keys against all patterns # and then compose the per-property schema from all @@ -150,6 +150,9 @@ def _get_object_properties_schema(object_schema, object_keys=None): if pattern.search(key): key_schemas.append(pattern_schema) if key_schemas: + # This naive schema composition approximates allOf. + # We can improve this later if someone provides examples that need + # a better allOf schema implementation for patternProperties. composed_schema = {**schema for schema in key_schemas} # update matched key flattened_properties_schema[key] = composed_schema From 7110c75cd362c1fcc3ba9dd8174ecb655d54b662 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 4 Jul 2022 20:13:31 -0500 Subject: [PATCH 17/19] fix syntax in schema merging --- st2common/st2common/util/config_loader.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/st2common/st2common/util/config_loader.py b/st2common/st2common/util/config_loader.py index cd1d8af6a4..42f750e1fc 100644 --- a/st2common/st2common/util/config_loader.py +++ b/st2common/st2common/util/config_loader.py @@ -153,7 +153,9 @@ def _get_object_properties_schema(object_schema, object_keys=None): # This naive schema composition approximates allOf. # We can improve this later if someone provides examples that need # a better allOf schema implementation for patternProperties. - composed_schema = {**schema for schema in key_schemas} + composed_schema = {} + for schema in key_schemas: + composed_schema.update(schema) # update matched key flattened_properties_schema[key] = composed_schema # don't overwrite matched key's schema with additionalProperties From 1463f4dc4c945abf7951ec78aafedc372c1050fc Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 4 Jul 2022 20:31:03 -0500 Subject: [PATCH 18/19] fix test pack fixture usage --- st2common/tests/unit/test_config_loader.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/st2common/tests/unit/test_config_loader.py b/st2common/tests/unit/test_config_loader.py index 7a68d05de6..d4a4e77f61 100644 --- a/st2common/tests/unit/test_config_loader.py +++ b/st2common/tests/unit/test_config_loader.py @@ -631,20 +631,26 @@ def test_get_config_dynamic_config_item_under_pattern_properties(self): config_db.delete() def test_get_config_dynamic_config_item_properties_order_of_precedence(self): - pack_name = "dummy_pack_schema_with_properties_1" + pack_name = "dummy_pack_schema_with_pattern_and_additional_properties_1" loader = ContentPackConfigLoader(pack_name=pack_name) - encrypted_value = crypto.symmetric_encrypt( + encrypted_value_1 = crypto.symmetric_encrypt( KeyValuePairAPI.crypto_key, "v1_encrypted" ) KeyValuePair.add_or_update( - KeyValuePairDB(name="k1_encrypted", value=encrypted_value, secret=True) + KeyValuePairDB(name="k1_encrypted", value=encrypted_value_1, secret=True) + ) + encrypted_value_2 = crypto.symmetric_encrypt( + KeyValuePairAPI.crypto_key, "v2_encrypted" ) KeyValuePair.add_or_update( - KeyValuePairDB(name="k2_encrypted", value=encrypted_value, secret=True) + KeyValuePairDB(name="k2_encrypted", value=encrypted_value_2, secret=True) + ) + encrypted_value_3 = crypto.symmetric_encrypt( + KeyValuePairAPI.crypto_key, "v3_encrypted" ) KeyValuePair.add_or_update( - KeyValuePairDB(name="k3_encrypted", value=encrypted_value, secret=True) + KeyValuePairDB(name="k3_encrypted", value=encrypted_value_3, secret=True) ) #################### @@ -711,7 +717,7 @@ def test_get_config_dynamic_config_item_properties_order_of_precedence(self): "env-prod": { "host": "127.1.2.7", "port": 8282, - "token": "v1_encrypted", + "token": "v2_encrypted", }, "dev": { "url": "https://example.com", @@ -721,7 +727,7 @@ def test_get_config_dynamic_config_item_properties_order_of_precedence(self): "prod": { "url": "https://other.example.com", "port": 2345, - "token": "v1_encrypted", + "token": "v3_encrypted", }, }, }, From a5084e762fffcc1402724b5c88ac842fa84e33f8 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 5 Jul 2022 12:16:38 -0500 Subject: [PATCH 19/19] add a pack config test for additionalItems: true --- st2common/tests/unit/test_config_loader.py | 26 ++++++++++++++++++- .../config.schema.yaml | 4 +++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/st2common/tests/unit/test_config_loader.py b/st2common/tests/unit/test_config_loader.py index d4a4e77f61..3f8e23d5be 100644 --- a/st2common/tests/unit/test_config_loader.py +++ b/st2common/tests/unit/test_config_loader.py @@ -762,7 +762,21 @@ def test_get_config_dynamic_config_item_under_additional_items(self): # schema declares `secret: true` which triggers auto-decryption. # If this were not encrypted, it would try to decrypt it and fail. }, - ] + ], + # foobar has additionalItems: true + "foobar": [ + # there are no types to validate here + 5, + "a string", + { + # there are no defaults to interpolate here + "token": "hard-coded-secret", + }, + { + # nothing is marked `secret: true` so no auto-decryption occurs. + "token": "{{st2kv.system.k1_encrypted|decrypt_kv}}", + }, + ], } config_db = ConfigDB(pack=pack_name, values=values) config_db = Config.add_or_update(config_db) @@ -785,6 +799,16 @@ def test_get_config_dynamic_config_item_under_additional_items(self): "token": "v1_encrypted", }, ], + "foobar": [ + 5, + "a string", + { + "token": "hard-coded-secret", + }, + { + "token": "v1_encrypted", + }, + ], }, ) diff --git a/st2tests/st2tests/fixtures/packs/dummy_pack_schema_with_additional_items_1/config.schema.yaml b/st2tests/st2tests/fixtures/packs/dummy_pack_schema_with_additional_items_1/config.schema.yaml index b0bcb2091b..88c8b1c5d7 100644 --- a/st2tests/st2tests/fixtures/packs/dummy_pack_schema_with_additional_items_1/config.schema.yaml +++ b/st2tests/st2tests/fixtures/packs/dummy_pack_schema_with_additional_items_1/config.schema.yaml @@ -22,3 +22,7 @@ type: "string" required: true secret: true + foobar: + type: "array" + required: false + additionalItems: true