diff --git a/CHANGELOG.md b/CHANGELOG.md index 6670740b..fa93003f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,9 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project closely adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## 0.145.0 +- Fix - reporting when file names have secrets + ## 0.144.0 - Fix - preserve parent security CUCU_SECRETS setting - Change - ignore objects in hide_secrets diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 74aaf873..bc623684 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -55,7 +55,7 @@ By making a contribution to this project, I certify that: # Design `cucu` is a "batteries-included" approach to testing -1. Automatically stand up or connect to a selenium container +1. Connect to local browser or docker container 2. Does fuzzy matching on DOM elements 3. Implements a large set of standard steps 4. Enables customization of steps & linter rules diff --git a/features/cli/run_outputs.feature b/features/cli/run_outputs.feature index 9afcfe65..cee65402 100644 --- a/features/cli/run_outputs.feature +++ b/features/cli/run_outputs.feature @@ -164,7 +164,7 @@ Feature: Run outputs """ Scenario: User gets exact expected output from various console outputs - Given I run the command "cucu run data/features/echo.feature --env SHELL=/foo/bar/zsh --env USER=that_guy --env PWD=/some/place/nice --results {CUCU_RESULTS_DIR}/validate_junit_xml_results" and save stdout to "STDOUT", stderr to "STDERR" and expect exit code "0" + Given I run the command "cucu run data/features/echo.feature --env SHELL=/foo/bar/zsh --env USER=that_guy --env PWD=/some/place/nice --results {CUCU_RESULTS_DIR}/validate_junit_xml_results --no-color-output" and save stdout to "STDOUT", stderr to "STDERR" and expect exit code "0" # {SHELL} and {PWD} contain slashes which we don't have a good way of # escaping in the tests yet so we'll just .* to match them and for the # crazy looking 4 backslashes its because the original test has 2 @@ -189,11 +189,10 @@ Feature: Run outputs current working directory is '/some/place/nice' And I echo "current working directory is '\{PWD\}'" .* - # PWD="/some/place/nice*" + # PWD="/some/place/nice" \{ "user": "that_guy" \} - And I echo the following .* \"\"\" \\\\{ diff --git a/features/cli/secrets.feature b/features/cli/secrets.feature index 7039fd3c..3de20eae 100644 --- a/features/cli/secrets.feature +++ b/features/cli/secrets.feature @@ -86,6 +86,32 @@ Feature: Secrets When I run the command "cucu run {CUCU_RESULTS_DIR}/features_with_secrets --secrets MY_SECRET --results={CUCU_RESULTS_DIR}/features_with_secrets_results" and save stdout to "STDOUT", stderr to "STDERR" and expect exit code "0" Then I should see "{STDOUT}" does not contain "super secret" + Scenario: User can run a test that has the secret value in its scenario name + Given I create a file at "{CUCU_RESULTS_DIR}/features_with_secrets/environment.py" with the following: + """ + from cucu.environment import * + """ + And I create a file at "{CUCU_RESULTS_DIR}/features_with_secrets/steps/__init__.py" with the following: + """ + from cucu.steps import * + """ + And I create a file at "{CUCU_RESULTS_DIR}/features_with_secrets/cucurc.yml" with the following: + """ + CUCU_SECRETS: MY_SECRET + MY_SECRET: 'secret-value' + """ + And I create a file at "{CUCU_RESULTS_DIR}/features_with_secrets/features/feature_that_spills_the_beans.feature" with the following: + """ + Feature: Feature that spills the beans + + Scenario: This scenario has the secret-value in its name + Given I echo "the current user is \{USER\}" + And I echo "\{MY_SECRET\}" + And I echo "your home directory is at \{HOME\}" + """ + When I run the command "cucu run {CUCU_RESULTS_DIR}/features_with_secrets --results={CUCU_RESULTS_DIR}/features_with_secrets_results -g" and save stdout to "STDOUT", stderr to "STDERR" and expect exit code "0" + Then I should see "{STDOUT}" does not contain "secret-value" + Scenario: User gets expected behavior when not using variable passthru Given I create a file at "{CUCU_RESULTS_DIR}/substeps_without_variable_passthru/environment.py" with the following: """ diff --git a/pyproject.toml b/pyproject.toml index 31d825f5..4a1357fe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "cucu" -version = "0.144.0" +version = "0.145.0" license = "MIT" description = "Easy BDD web testing" authors = ["Domino Data Lab "] diff --git a/src/cucu/ansi_parser.py b/src/cucu/ansi_parser.py index 63e732bf..eec71e36 100644 --- a/src/cucu/ansi_parser.py +++ b/src/cucu/ansi_parser.py @@ -3,8 +3,6 @@ from behave.formatter.ansi_escapes import colors, escapes -from cucu import logger - ESC_SEQ = r"\x1b[" TRANSLATION = {v: f'' for k, v in colors.items()} | { escapes["reset"]: "", @@ -22,7 +20,9 @@ } RE_TO_HTML = re.compile("|".join(map(re.escape, TRANSLATION))) -RE_TO_REMOVE = re.compile(r"\x1b\[0m|\x1b\[0;\d\dm") +RE_TO_REMOVE = re.compile( + r"\x1b\[(0;)?[0-9A-F]{1,2}m" +) # detect hex values, not just decimal digits def remove_ansi(input: str) -> str: @@ -39,6 +39,9 @@ def parse_log_to_html(input: str) -> str: result = f"{body_start}
\n{RE_TO_HTML.sub(lambda match: TRANSLATION[match.group(0)], html.escape(input, quote=False))}\n
{body_end}" if ESC_SEQ in result: lines = "\n".join([x for x in result.split("\n") if ESC_SEQ in x]) - logger.info(f"Detected unmapped ansi escape code!:\n{lines}") + + print( + f"Detected unmapped ansi escape code!:\n{lines}" + ) # use print instead of logger to avoid circular import return result diff --git a/src/cucu/config.py b/src/cucu/config.py index 5c92e062..2fe2245c 100644 --- a/src/cucu/config.py +++ b/src/cucu/config.py @@ -1,3 +1,4 @@ +import json import os import re import socket @@ -155,22 +156,60 @@ def expand(self, string): return references - def hide_secrets(self, line): + def hide_secrets(self, text): secret_keys = [x for x in self.get("CUCU_SECRETS", "").split(",") if x] secret_values = [self.get(x) for x in secret_keys if self.get(x)] secret_values = [x for x in secret_values if isinstance(x, str)] - # here's where we can hide secrets - for value in secret_values: - replacement = "*" * len(value) + is_bytes = isinstance(text, bytes) + if is_bytes: + text = text.decode() - if isinstance(line, bytes): - value = bytes(value, "utf8") - replacement = bytes(replacement, "utf8") + result = None + if text.startswith("{"): + try: + result = self._hide_secrets_json(secret_values, text) + except Exception as e: + print( + f"Couldn't parse json, falling back to text processing: {e}" + ) - line = line.replace(value, replacement) + if result is None: + result = self._hide_secrets_text(secret_values, text) - return line + if is_bytes: + result = result.encode() + + return result + + def _hide_secrets_json(self, secret_values, text): + data = json.loads(text) + + def hide_node(value, parent, key): + if not isinstance(value, str): + return value + + if ( + key == "name" + and isinstance(parent, dict) + and parent.get("keyword", "") in ["Feature", "Scenario"] + ): + return value + + return self._hide_secrets_text(secret_values, value) + + leaf_map(data, hide_node) + return json.dumps(data, indent=2, sort_keys=True) + + def _hide_secrets_text(self, secret_values, text): + lines = text.split("\n") + + for x in range(len(lines)): + # here's where we can hide secrets + for value in secret_values: + lines[x] = lines[x].replace(value, "*" * len(value)) + + return "\n".join(lines) def resolve(self, string): """ @@ -350,3 +389,25 @@ def _get_local_address(): "when set to 'true' results in stacktraces showing in the JUnit XML failure output", default="false", ) + + +# define re_map here instead of in utils.py to avoid circular import +def leaf_map(data, value_func, parent=None, key=None): + """ + Utility to apply a map function recursively to a dict or list. + + Args: + data: The dict or list or value to use + value_func: Callable function that accepts data and parent + parent: The parent object (or None) + """ + if isinstance(data, dict): + for key, value in data.items(): + data[key] = leaf_map(value, value_func, data, key) + return data + elif isinstance(data, list): + for x, value in enumerate(data): + data[x] = leaf_map(value, value_func, data, key) + return data + else: + return value_func(data, parent, key) diff --git a/tests/test_config.py b/tests/test_config.py index 3d9bde33..02edd754 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -2,7 +2,7 @@ import cucu import pytest -from cucu.config import CONFIG +from cucu.config import CONFIG, leaf_map def test_config_lookup_for_inexistent_is_none(): @@ -110,3 +110,20 @@ def test_config_expand_with_custom_variable_handling(): # if the custom resolution takes precedence then we'll never see the # "wassup" value assert CONFIG.resolve("{CUSTOM_BAR}") == "boom" + + +def test_leaf_map(): + data = { + "a": 1, + "b": ["x", "k", "c", "d", {"one": "bee", "two": "straws"}], + } + + def something(value, parent, key): + if key in ["a", 3, "two"]: + return "z" + + return value + + expected = {"a": "z", "b": ["x", "k", "c", "d", {"one": "bee", "two": "z"}]} + actual = leaf_map(data, something) + assert actual == expected