From 1ca3cdb1b1bbf1821e502a92cfbb52996bc7cc91 Mon Sep 17 00:00:00 2001 From: Martin Jackson Date: Thu, 26 Sep 2024 13:55:32 -0500 Subject: [PATCH 01/11] Simplify sanity test excludes by using more general config.yml --- tests/config.yml | 3 +++ tests/sanity/ignore-2.17.txt | 18 ------------------ 2 files changed, 3 insertions(+), 18 deletions(-) create mode 100644 tests/config.yml diff --git a/tests/config.yml b/tests/config.yml new file mode 100644 index 0000000..056fd1c --- /dev/null +++ b/tests/config.yml @@ -0,0 +1,3 @@ +--- +modules: + python_requires: '!= 2.6, != 2.7' diff --git a/tests/sanity/ignore-2.17.txt b/tests/sanity/ignore-2.17.txt index 8f7ac9c..2182076 100644 --- a/tests/sanity/ignore-2.17.txt +++ b/tests/sanity/ignore-2.17.txt @@ -1,21 +1,3 @@ -plugins/module_utils/__init__.py compile-2.7!skip -plugins/module_utils/load_secrets_common.py compile-2.7!skip -plugins/module_utils/load_secrets_v1.py compile-2.7!skip -plugins/module_utils/load_secrets_v2.py compile-2.7!skip -plugins/module_utils/parse_secrets_v2.py compile-2.7!skip -plugins/modules/__init__.py compile-2.7!skip -plugins/modules/parse_secrets_info.py compile-2.7!skip -plugins/modules/vault_load_parsed_secrets.py compile-2.7!skip -plugins/modules/vault_load_secrets.py compile-2.7!skip -plugins/module_utils/__init__.py import-2.7!skip -plugins/module_utils/load_secrets_common.py import-2.7!skip -plugins/module_utils/load_secrets_v1.py import-2.7!skip -plugins/module_utils/load_secrets_v2.py import-2.7!skip -plugins/module_utils/parse_secrets_v2.py import-2.7!skip -plugins/modules/__init__.py import-2.7!skip -plugins/modules/parse_secrets_info.py import-2.7!skip -plugins/modules/vault_load_parsed_secrets.py import-2.7!skip -plugins/modules/vault_load_secrets.py import-2.7!skip plugins/modules/parse_secrets_info.py validate-modules:missing-gplv3-license plugins/modules/vault_load_secrets.py validate-modules:missing-gplv3-license plugins/modules/vault_load_parsed_secrets.py validate-modules:missing-gplv3-license From 83d836fa377fff6ef42e8187c44ab414a3de02f5 Mon Sep 17 00:00:00 2001 From: Martin Jackson Date: Thu, 26 Sep 2024 13:58:50 -0500 Subject: [PATCH 02/11] Only skip 2.7 as 2.6 was dropped long ago --- tests/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/config.yml b/tests/config.yml index 056fd1c..a9f8718 100644 --- a/tests/config.yml +++ b/tests/config.yml @@ -1,3 +1,3 @@ --- modules: - python_requires: '!= 2.6, != 2.7' + python_requires: '!= 2.7' From 2f82f42e5ed23370286adaf727f54060dc28c20f Mon Sep 17 00:00:00 2001 From: Michele Baldessari Date: Wed, 25 Sep 2024 18:47:53 +0200 Subject: [PATCH 03/11] Add jsonschema CI job We test a few values secret files against the json schema --- .github/workflows/jsonschema.yaml | 31 +++++++++++++++++++++++++++++++ Makefile | 5 +++++ 2 files changed, 36 insertions(+) create mode 100644 .github/workflows/jsonschema.yaml diff --git a/.github/workflows/jsonschema.yaml b/.github/workflows/jsonschema.yaml new file mode 100644 index 0000000..0b0db8a --- /dev/null +++ b/.github/workflows/jsonschema.yaml @@ -0,0 +1,31 @@ +--- +name: Verify json schema + +on: [push, pull_request] + +jobs: + jsonschema_tests: + name: Json Schema tests + strategy: + matrix: + python-version: [3.11.3] + runs-on: ubuntu-latest + + steps: + - name: Checkout Code + uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install check-jsonschema + + - name: Verify secrets json schema + run: | + set -e + for i in values-secret-v2-base values-secret-v2-generic-onlygenerate values-secret-v2-block-yamlstring; do echo "$i"; check-jsonschema --schemafile ./roles/vault_utils/values-secrets.v2.schema.json "tests/unit/v2/$i.yaml"; done diff --git a/Makefile b/Makefile index 893811d..8c8f76d 100644 --- a/Makefile +++ b/Makefile @@ -43,3 +43,8 @@ ansible-unittest: ## run ansible unit tests .PHONY: test test: ansible-sanitytest ansible-unittest + +.PHONY: check-jsonschema +check-jsonschema: ## Runs check-jsonschema against all unit test files except known broken ones + set -e; \ + for i in values-secret-v2-base values-secret-v2-generic-onlygenerate values-secret-v2-block-yamlstring; do echo "$$i"; check-jsonschema --schemafile ./roles/vault_utils/values-secrets.v2.schema.json "tests/unit/v2/$$i.yaml"; done From a282712dfa38fc4be6f44ea0f945d9040eb60eb9 Mon Sep 17 00:00:00 2001 From: Michele Baldessari Date: Fri, 27 Sep 2024 10:51:51 +0200 Subject: [PATCH 04/11] Fix typo in README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 19dee83..68af1ea 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Ansible Collection - rhvp.cluser_utils +# Ansible Collection - rhvp.cluster_utils This collection represents the collected Ansible code from the Validated Patterns framework common repository. From 383495128479bb550d6ae51aeb83e03652f0d387 Mon Sep 17 00:00:00 2001 From: Michele Baldessari Date: Tue, 1 Oct 2024 10:22:31 +0200 Subject: [PATCH 05/11] Boolean defaults cannot be strings --- roles/vault_utils/values-secrets.v2.schema.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/vault_utils/values-secrets.v2.schema.json b/roles/vault_utils/values-secrets.v2.schema.json index c8b5c02..2d29589 100644 --- a/roles/vault_utils/values-secrets.v2.schema.json +++ b/roles/vault_utils/values-secrets.v2.schema.json @@ -276,12 +276,12 @@ "base64": { "type": "boolean", "description": "Before uploading the secret the content is base-64 encoded. It is recommended to set this to true when dealing with files", - "default": "false" + "default": false }, "override": { "type": "boolean", "description": "When onMissingValue is set to 'generate' and the secret already exists in the vault update it", - "default": "false" + "default": false } }, "dependentRequired": { From 274745a1662b4d746b1fe50fc3bb34034ab0eb67 Mon Sep 17 00:00:00 2001 From: Michele Baldessari Date: Tue, 1 Oct 2024 10:23:23 +0200 Subject: [PATCH 06/11] Use --fill-defaults when running json schema check Reason for this is https://python-jsonschema.readthedocs.io/en/stable/faq/#why-doesn-t-my-schema-s-default-property-set-the-default-on-my-instance --- .github/workflows/jsonschema.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/jsonschema.yaml b/.github/workflows/jsonschema.yaml index 0b0db8a..ea127e2 100644 --- a/.github/workflows/jsonschema.yaml +++ b/.github/workflows/jsonschema.yaml @@ -28,4 +28,4 @@ jobs: - name: Verify secrets json schema run: | set -e - for i in values-secret-v2-base values-secret-v2-generic-onlygenerate values-secret-v2-block-yamlstring; do echo "$i"; check-jsonschema --schemafile ./roles/vault_utils/values-secrets.v2.schema.json "tests/unit/v2/$i.yaml"; done + for i in values-secret-v2-base values-secret-v2-generic-onlygenerate values-secret-v2-block-yamlstring; do echo "$i"; check-jsonschema --fill-defaults --schemafile ./roles/vault_utils/values-secrets.v2.schema.json "tests/unit/v2/$i.yaml"; done From c09634c0b5df9a643ca1e448de3912b8e43b0eb2 Mon Sep 17 00:00:00 2001 From: Akos Eros Date: Tue, 8 Oct 2024 11:07:29 +0200 Subject: [PATCH 07/11] Fix secret parsing, when there is no secret in secret yaml file This will fix the edge case when there is an empty list ([]) or no secret in the secret values file for a pattern, the validation and the parsing would fail the module. Now it only throws a warning for both validation and parsing. --- plugins/module_utils/parse_secrets_v2.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/plugins/module_utils/parse_secrets_v2.py b/plugins/module_utils/parse_secrets_v2.py index 00115fb..83bc6a4 100644 --- a/plugins/module_utils/parse_secrets_v2.py +++ b/plugins/module_utils/parse_secrets_v2.py @@ -194,6 +194,11 @@ def parse(self): secrets = self._get_secrets() total_secrets = 0 # Counter for all the secrets uploaded + + if secrets == "None" or len(secrets) == 0: + self.module.warn("No secrets were parsed") + return total_secrets + for s in secrets: total_secrets += 1 counter = 0 # This counter is to use kv put on first secret and kv patch on latter @@ -322,8 +327,9 @@ def _validate_field(self, f): def _validate_secrets(self): backing_store = self._get_backingstore() secrets = self._get_secrets() - if len(secrets) == 0: - self.module.fail_json("No secrets found") + if secrets == "None" or len(secrets) == 0: + self.module.warn("No secrets found") + return (True, "") names = [] for s in secrets: From f3978829570a71fd9a036ca3389d66d758400d89 Mon Sep 17 00:00:00 2001 From: Akos Eros Date: Tue, 8 Oct 2024 17:56:07 +0200 Subject: [PATCH 08/11] Move null check to _get_secrets The check for "None" is better placed inside the get_secrets function, with this, get_secrets will always return a list, either empty list, or the list with the secrets from the yaml. Changed checks accordingly. Changed default get to be an empty list if the "secret" key is not present in the secret values file. --- plugins/module_utils/parse_secrets_v2.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/plugins/module_utils/parse_secrets_v2.py b/plugins/module_utils/parse_secrets_v2.py index 83bc6a4..76ea97f 100644 --- a/plugins/module_utils/parse_secrets_v2.py +++ b/plugins/module_utils/parse_secrets_v2.py @@ -96,7 +96,9 @@ def _get_vault_policies(self, enable_default_vp_policies=True): return policies def _get_secrets(self): - return self.syaml.get("secrets", {}) + secrets = self.syaml.get("secrets", []) + # We check for "None" here because the yaml file is filtered thru' from_yaml + return [] if secrets == "None" else secrets def _get_field_on_missing_value(self, f): # By default if 'onMissingValue' is missing we assume we need to @@ -195,7 +197,7 @@ def parse(self): total_secrets = 0 # Counter for all the secrets uploaded - if secrets == "None" or len(secrets) == 0: + if len(secrets) == 0: self.module.warn("No secrets were parsed") return total_secrets @@ -327,7 +329,7 @@ def _validate_field(self, f): def _validate_secrets(self): backing_store = self._get_backingstore() secrets = self._get_secrets() - if secrets == "None" or len(secrets) == 0: + if len(secrets) == 0: self.module.warn("No secrets found") return (True, "") From 754cc624b1a49fc303d9f6b41a784712c7c70dcc Mon Sep 17 00:00:00 2001 From: Akos Eros Date: Wed, 9 Oct 2024 15:26:52 +0200 Subject: [PATCH 09/11] Add unit tests for empty secret logic Added check for None type as well, in case the module is used witout from_yaml filter (like in unit tests) --- plugins/module_utils/parse_secrets_v2.py | 6 ++- tests/unit/test_parse_secrets.py | 42 +++++++++++++++++++ .../v2/values-secret-v2-empty-secret.yaml | 2 + .../unit/v2/values-secret-v2-null-secret.yaml | 2 + 4 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 tests/unit/v2/values-secret-v2-empty-secret.yaml create mode 100644 tests/unit/v2/values-secret-v2-null-secret.yaml diff --git a/plugins/module_utils/parse_secrets_v2.py b/plugins/module_utils/parse_secrets_v2.py index 76ea97f..4d27511 100644 --- a/plugins/module_utils/parse_secrets_v2.py +++ b/plugins/module_utils/parse_secrets_v2.py @@ -97,8 +97,10 @@ def _get_vault_policies(self, enable_default_vp_policies=True): def _get_secrets(self): secrets = self.syaml.get("secrets", []) - # We check for "None" here because the yaml file is filtered thru' from_yaml - return [] if secrets == "None" else secrets + # We check for "None" here because the yaml file is currently + # filtered thru' from_yaml in module + # We also check for None here to cover when there is no jinja filter is used (unit tests) + return [] if secrets == "None" or secrets is None else secrets def _get_field_on_missing_value(self, f): # By default if 'onMissingValue' is missing we assume we need to diff --git a/tests/unit/test_parse_secrets.py b/tests/unit/test_parse_secrets.py index 1b93d11..2d10455 100644 --- a/tests/unit/test_parse_secrets.py +++ b/tests/unit/test_parse_secrets.py @@ -930,6 +930,48 @@ def test_ensure_generate_errors_on_none_generate(self, getpass): == "You cannot have onMissingValue set to 'generate' unless using vault backingstore for secret config-demo field secret" # noqa: E501 ) + def test_ensure_success_empty_secrets(self, getpass): + testfile_output = self.get_file_as_stdout( + os.path.join(self.testdir_v2, "values-secret-v2-empty-secret.yaml") + ) + with self.assertRaises(AnsibleExitJson) as ansible_err: + set_module_args( + { + "values_secrets_plaintext": testfile_output, + "secrets_backing_store": "vault", + } + ) + parse_secrets_info.main() + + ret = ansible_err.exception.args[0] + self.assertTrue( + (ret["failed"] is False) + and (ret["changed"] is False) + and (len(ret["parsed_secrets"])) == 0 + and (len(ret["kubernetes_secret_objects"]) == 0) + ) + + def test_ensure_success_null_secrets(self, getpass): + testfile_output = self.get_file_as_stdout( + os.path.join(self.testdir_v2, "values-secret-v2-null-secret.yaml") + ) + with self.assertRaises(AnsibleExitJson) as ansible_err: + set_module_args( + { + "values_secrets_plaintext": testfile_output, + "secrets_backing_store": "vault", + } + ) + parse_secrets_info.main() + + ret = ansible_err.exception.args[0] + self.assertTrue( + (ret["failed"] is False) + and (ret["changed"] is False) + and (len(ret["parsed_secrets"])) == 0 + and (len(ret["kubernetes_secret_objects"]) == 0) + ) + if __name__ == "__main__": unittest.main() diff --git a/tests/unit/v2/values-secret-v2-empty-secret.yaml b/tests/unit/v2/values-secret-v2-empty-secret.yaml new file mode 100644 index 0000000..1c21da4 --- /dev/null +++ b/tests/unit/v2/values-secret-v2-empty-secret.yaml @@ -0,0 +1,2 @@ +version: "2.0" +secrets: [] diff --git a/tests/unit/v2/values-secret-v2-null-secret.yaml b/tests/unit/v2/values-secret-v2-null-secret.yaml new file mode 100644 index 0000000..dfd340a --- /dev/null +++ b/tests/unit/v2/values-secret-v2-null-secret.yaml @@ -0,0 +1,2 @@ +version: "2.0" +secrets: From ad1e74ddaaec6c08b9025fea5e5ca11cdefe8c05 Mon Sep 17 00:00:00 2001 From: Akos Eros Date: Wed, 9 Oct 2024 15:56:00 +0200 Subject: [PATCH 10/11] Bump version number --- galaxy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy.yml b/galaxy.yml index cea2f07..dcb0de9 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -8,7 +8,7 @@ namespace: rhvp name: cluster_utils # The version of the collection. Must be compatible with semantic versioning -version: 1.0.1 +version: 1.0.2 # The path to the Markdown (.md) readme file. This path is relative to the root of the collection readme: README.md From 5aa8600caf4e672cc676956e382b075a37b8f74e Mon Sep 17 00:00:00 2001 From: shellclear <12013211+shellclear@users.noreply.github.com> Date: Mon, 11 Nov 2024 14:53:50 +0100 Subject: [PATCH 11/11] Rremove links with - in the name because the usage of dash is not supported to run playbook names in collections --- playbooks/auto-approve-installplans.yml | 1 - playbooks/hello-world.yml | 1 - playbooks/iib-ci.yml | 1 - playbooks/write-token-kubeconfig.yml | 1 - 4 files changed, 4 deletions(-) delete mode 120000 playbooks/auto-approve-installplans.yml delete mode 120000 playbooks/hello-world.yml delete mode 120000 playbooks/iib-ci.yml delete mode 120000 playbooks/write-token-kubeconfig.yml diff --git a/playbooks/auto-approve-installplans.yml b/playbooks/auto-approve-installplans.yml deleted file mode 120000 index 8910a47..0000000 --- a/playbooks/auto-approve-installplans.yml +++ /dev/null @@ -1 +0,0 @@ -auto_approve_installplans.yml \ No newline at end of file diff --git a/playbooks/hello-world.yml b/playbooks/hello-world.yml deleted file mode 120000 index 6e3a7c4..0000000 --- a/playbooks/hello-world.yml +++ /dev/null @@ -1 +0,0 @@ -hello_world.yml \ No newline at end of file diff --git a/playbooks/iib-ci.yml b/playbooks/iib-ci.yml deleted file mode 120000 index 29697fe..0000000 --- a/playbooks/iib-ci.yml +++ /dev/null @@ -1 +0,0 @@ -iib_ci.yml \ No newline at end of file diff --git a/playbooks/write-token-kubeconfig.yml b/playbooks/write-token-kubeconfig.yml deleted file mode 120000 index 841f0b3..0000000 --- a/playbooks/write-token-kubeconfig.yml +++ /dev/null @@ -1 +0,0 @@ -write_token_kubeconfig.yml \ No newline at end of file