QE-12272 fix path names (#346)

- Fix - reporting when file names have secrets

CHANGELOG.md

@@ -5,6 +5,9 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project closely adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## 0.145.0
+- Fix - reporting when file names have secrets
+
 ## 0.144.0
 - Fix - preserve parent security CUCU_SECRETS setting
 - Change - ignore objects in hide_secrets

CONTRIBUTING.md

@@ -55,7 +55,7 @@ By making a contribution to this project, I certify that:
 
 # Design
 `cucu` is a "batteries-included" approach to testing
-1. Automatically stand up or connect to a selenium container
+1. Connect to local browser or docker container
 2. Does fuzzy matching on DOM elements
 3. Implements a large set of standard steps
 4. Enables customization of steps & linter rules

features/cli/run_outputs.feature

@@ -164,7 +164,7 @@ Feature: Run outputs
       """
 
   Scenario: User gets exact expected output from various console outputs
-    Given I run the command "cucu run data/features/echo.feature --env SHELL=/foo/bar/zsh --env USER=that_guy --env PWD=/some/place/nice --results {CUCU_RESULTS_DIR}/validate_junit_xml_results" and save stdout to "STDOUT", stderr to "STDERR" and expect exit code "0"
+    Given I run the command "cucu run data/features/echo.feature --env SHELL=/foo/bar/zsh --env USER=that_guy --env PWD=/some/place/nice --results {CUCU_RESULTS_DIR}/validate_junit_xml_results  --no-color-output" and save stdout to "STDOUT", stderr to "STDERR" and expect exit code "0"
       # {SHELL} and {PWD} contain slashes which we don't have a good way of
       # escaping in the tests yet so we'll just .* to match them and for the
       # crazy looking 4 backslashes its because the original test has 2
@@ -189,11 +189,10 @@ Feature: Run outputs
       current working directory is '/some/place/nice'
 
             And I echo "current working directory is '\{PWD\}'" .*
-            # PWD="/some/place/nice*"
+            # PWD="/some/place/nice"
       \{
         "user": "that_guy"
       \}
-
             And I echo the following .*
               \"\"\"
               \\\\{

features/cli/secrets.feature

@@ -86,6 +86,32 @@ Feature: Secrets
      When I run the command "cucu run {CUCU_RESULTS_DIR}/features_with_secrets --secrets MY_SECRET --results={CUCU_RESULTS_DIR}/features_with_secrets_results" and save stdout to "STDOUT", stderr to "STDERR" and expect exit code "0"
      Then I should see "{STDOUT}" does not contain "super secret"
 
+  Scenario: User can run a test that has the secret value in its scenario name
+    Given I create a file at "{CUCU_RESULTS_DIR}/features_with_secrets/environment.py" with the following:
+      """
+      from cucu.environment import *
+      """
+      And I create a file at "{CUCU_RESULTS_DIR}/features_with_secrets/steps/__init__.py" with the following:
+      """
+      from cucu.steps import *
+      """
+      And I create a file at "{CUCU_RESULTS_DIR}/features_with_secrets/cucurc.yml" with the following:
+      """
+      CUCU_SECRETS: MY_SECRET
+      MY_SECRET: 'secret-value'
+      """
+      And I create a file at "{CUCU_RESULTS_DIR}/features_with_secrets/features/feature_that_spills_the_beans.feature" with the following:
+      """
+      Feature: Feature that spills the beans
+
+        Scenario: This scenario has the secret-value in its name
+         Given I echo "the current user is \{USER\}"
+           And I echo "\{MY_SECRET\}"
+           And I echo "your home directory is at \{HOME\}"
+      """
+     When I run the command "cucu run {CUCU_RESULTS_DIR}/features_with_secrets --results={CUCU_RESULTS_DIR}/features_with_secrets_results -g" and save stdout to "STDOUT", stderr to "STDERR" and expect exit code "0"
+     Then I should see "{STDOUT}" does not contain "secret-value"
+
   Scenario: User gets expected behavior when not using variable passthru
     Given I create a file at "{CUCU_RESULTS_DIR}/substeps_without_variable_passthru/environment.py" with the following:
       """

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "cucu"
-version = "0.144.0"
+version = "0.145.0"
 license = "MIT"
 description = "Easy BDD web testing"
 authors = ["Domino Data Lab <[email protected]>"]

src/cucu/ansi_parser.py

@@ -3,8 +3,6 @@
 
 from behave.formatter.ansi_escapes import colors, escapes
 
-from cucu import logger
-
 ESC_SEQ = r"\x1b["
 TRANSLATION = {v: f'<span style="color: {k};">' for k, v in colors.items()} | {
     escapes["reset"]: "</span>",
@@ -22,7 +20,9 @@
 }
 RE_TO_HTML = re.compile("|".join(map(re.escape, TRANSLATION)))
 
-RE_TO_REMOVE = re.compile(r"\x1b\[0m|\x1b\[0;\d\dm")
+RE_TO_REMOVE = re.compile(
+    r"\x1b\[(0;)?[0-9A-F]{1,2}m"
+)  # detect hex values, not just decimal digits
 
 
 def remove_ansi(input: str) -> str:
@@ -39,6 +39,9 @@ def parse_log_to_html(input: str) -> str:
     result = f"{body_start}<pre>\n{RE_TO_HTML.sub(lambda match: TRANSLATION[match.group(0)], html.escape(input, quote=False))}\n</pre>{body_end}"
     if ESC_SEQ in result:
         lines = "\n".join([x for x in result.split("\n") if ESC_SEQ in x])
-        logger.info(f"Detected unmapped ansi escape code!:\n{lines}")
+
+        print(
+            f"Detected unmapped ansi escape code!:\n{lines}"
+        )  # use print instead of logger to avoid circular import
 
     return result

src/cucu/config.py

@@ -1,3 +1,4 @@
+import json
 import os
 import re
 import socket
@@ -155,22 +156,60 @@ def expand(self, string):
 
         return references
 
-    def hide_secrets(self, line):
+    def hide_secrets(self, text):
         secret_keys = [x for x in self.get("CUCU_SECRETS", "").split(",") if x]
         secret_values = [self.get(x) for x in secret_keys if self.get(x)]
         secret_values = [x for x in secret_values if isinstance(x, str)]
 
-        # here's where we can hide secrets
-        for value in secret_values:
-            replacement = "*" * len(value)
+        is_bytes = isinstance(text, bytes)
+        if is_bytes:
+            text = text.decode()
 
-            if isinstance(line, bytes):
-                value = bytes(value, "utf8")
-                replacement = bytes(replacement, "utf8")
+        result = None
+        if text.startswith("{"):
+            try:
+                result = self._hide_secrets_json(secret_values, text)
+            except Exception as e:
+                print(
+                    f"Couldn't parse json, falling back to text processing: {e}"
+                )
 
-            line = line.replace(value, replacement)
+        if result is None:
+            result = self._hide_secrets_text(secret_values, text)
 
-        return line
+        if is_bytes:
+            result = result.encode()
+
+        return result
+
+    def _hide_secrets_json(self, secret_values, text):
+        data = json.loads(text)
+
+        def hide_node(value, parent, key):
+            if not isinstance(value, str):
+                return value
+
+            if (
+                key == "name"
+                and isinstance(parent, dict)
+                and parent.get("keyword", "") in ["Feature", "Scenario"]
+            ):
+                return value
+
+            return self._hide_secrets_text(secret_values, value)
+
+        leaf_map(data, hide_node)
+        return json.dumps(data, indent=2, sort_keys=True)
+
+    def _hide_secrets_text(self, secret_values, text):
+        lines = text.split("\n")
+
+        for x in range(len(lines)):
+            # here's where we can hide secrets
+            for value in secret_values:
+                lines[x] = lines[x].replace(value, "*" * len(value))
+
+        return "\n".join(lines)
 
     def resolve(self, string):
         """
@@ -350,3 +389,25 @@ def _get_local_address():
     "when set to 'true' results in stacktraces showing in the JUnit XML failure output",
     default="false",
 )
+
+
+# define re_map here instead of in utils.py to avoid circular import
+def leaf_map(data, value_func, parent=None, key=None):
+    """
+    Utility to apply a map function recursively to a dict or list.
+
+    Args:
+        data: The dict or list or value to use
+        value_func: Callable function that accepts data and parent
+        parent: The parent object (or None)
+    """
+    if isinstance(data, dict):
+        for key, value in data.items():
+            data[key] = leaf_map(value, value_func, data, key)
+        return data
+    elif isinstance(data, list):
+        for x, value in enumerate(data):
+            data[x] = leaf_map(value, value_func, data, key)
+        return data
+    else:
+        return value_func(data, parent, key)

tests/test_config.py

@@ -2,7 +2,7 @@
 
 import cucu
 import pytest
-from cucu.config import CONFIG
+from cucu.config import CONFIG, leaf_map
 
 
 def test_config_lookup_for_inexistent_is_none():
@@ -110,3 +110,20 @@ def test_config_expand_with_custom_variable_handling():
     # if the custom resolution takes precedence then we'll never see the
     # "wassup" value
     assert CONFIG.resolve("{CUSTOM_BAR}") == "boom"
+
+
+def test_leaf_map():
+    data = {
+        "a": 1,
+        "b": ["x", "k", "c", "d", {"one": "bee", "two": "straws"}],
+    }
+
+    def something(value, parent, key):
+        if key in ["a", 3, "two"]:
+            return "z"
+
+        return value
+
+    expected = {"a": "z", "b": ["x", "k", "c", "d", {"one": "bee", "two": "z"}]}
+    actual = leaf_map(data, something)
+    assert actual == expected