feat: Adding support for redacted environment variable values through openjd_redacted_env

Signed-off-by: Brian Axelson <86568017+baxeaz@users.noreply.github.com>

Author: baxeaz

requirements-testing.txt

@@ -1,7 +1,7 @@
 coverage[toml] == 7.*
 pytest == 8.3.*
 pytest-cov == 6.1.*
-pytest-timeout == 2.3.*
+pytest-timeout == 2.4.*
 pytest-xdist == 3.6.*
 types-PyYAML ~= 6.0
 black == 25.*

src/openjd/model/v2023_09/_model.py

@@ -89,6 +89,8 @@ class ExtensionName(str, Enum):
 
     # # https://github.com/OpenJobDescription/openjd-specifications/blob/mainline/rfcs/0001-task-chunking.md
     TASK_CHUNKING = "TASK_CHUNKING"
+    # Extension that enables the use of openjd_redacted_env for setting environment variables with redacted values in logs
+    REDACTED_ENV_VARS = "REDACTED_ENV_VARS"
 
 
 ExtensionNameList = Annotated[list[str], Field(min_length=1)]
@@ -512,24 +514,6 @@ class RangeString(FormatString):
 TaskRangeList = list[Union[TaskParameterStringValueAsJob, int, float, Decimal]]
 
 
-# Target model for task parameters when instantiating a job.
-class RangeListTaskParameterDefinition(OpenJDModel_v2023_09):
-    # element type of items in the range
-    type: TaskParameterType
-    # NOTE: Pydantic V1 was allowing non-string values in this range, V2 is enforcing that type.
-    range: TaskRangeList
-    # has a value when type is CHUNK[INT], which is only possible from the TASK_CHUNKING extension
-    chunks: Optional[TaskChunksDefinition] = None
-
-
-class RangeExpressionTaskParameterDefinition(OpenJDModel_v2023_09):
-    # element type of items in the range
-    type: TaskParameterType
-    range: IntRangeExpr
-    # has a value when type is CHUNK[INT], which is only possible from the TASK_CHUNKING extension
-    chunks: Optional[TaskChunksDefinition] = None
-
-
 class TaskChunksRangeConstraint(str, Enum):
     CONTIGUOUS = "CONTIGUOUS"
     NONCONTIGUOUS = "NONCONTIGUOUS"
@@ -559,6 +543,24 @@ def _validate_target_runtime_seconds(cls, value: Any, info: ValidationInfo) -> A
         return validate_int_fmtstring_field(value, ge=0, context=context)
 
 
+# Target model for task parameters when instantiating a job.
+class RangeListTaskParameterDefinition(OpenJDModel_v2023_09):
+    # element type of items in the range
+    type: TaskParameterType
+    # NOTE: Pydantic V1 was allowing non-string values in this range, V2 is enforcing that type.
+    range: TaskRangeList
+    # has a value when type is CHUNK[INT], which is only possible from the TASK_CHUNKING extension
+    chunks: Optional[TaskChunksDefinition] = None
+
+
+class RangeExpressionTaskParameterDefinition(OpenJDModel_v2023_09):
+    # element type of items in the range
+    type: TaskParameterType
+    range: IntRangeExpr
+    # has a value when type is CHUNK[INT], which is only possible from the TASK_CHUNKING extension
+    chunks: Optional[TaskChunksDefinition] = None
+
+
 class IntTaskParameterDefinition(OpenJDModel_v2023_09):
     """Definition of an integer-typed Task Parameter and its value range.
 

test/openjd/model/v2023_09/test_definitions.py

@@ -2,17 +2,37 @@
 
 import pytest
 from pydantic import BaseModel
-from typing import Type
+from typing import Type, ForwardRef
 import openjd.model.v2023_09 as mod
 from inspect import getmembers, getmodule, isclass
 
 
+from .test_module import ClassWithForwardRef, ClassWithoutForwardRef
+
+
 ALL_MODELS = sorted(
     [obj for name, obj in getmembers(mod) if isclass(obj) and issubclass(obj, BaseModel)],
     key=lambda o: o.__name__,
 )
 
 
+def test_forward_ref_detection():
+    """Test that our ForwardRef detection works by checking classes that use ForwardRefs vs direct references."""
+    # When referencing a class that's already defined, no ForwardRef is created
+    field = ClassWithoutForwardRef.model_fields["ref"]
+    assert not isinstance(field.annotation, ForwardRef), (
+        "Expected ClassWithoutForwardRef.ref to NOT be a ForwardRef since ReferencedClass "
+        "is defined before it's used"
+    )
+
+    # When referencing a class that's defined later, a ForwardRef is created
+    field = ClassWithForwardRef.model_fields["ref"]
+    assert isinstance(field.annotation, ForwardRef), (
+        "Expected ClassWithForwardRef.ref to be a ForwardRef since SecondReferencedClass "
+        "is defined after it's used"
+    )
+
+
 @pytest.mark.parametrize("model", ALL_MODELS)
 def test_models_in_same_module(model: Type[BaseModel]) -> None:
     # For our error reporting of discriminated union fields to be correctly reported
@@ -21,3 +41,18 @@ def test_models_in_same_module(model: Type[BaseModel]) -> None:
     # This is to identify when a name in an error location is actually a class name from
     # a typed union.
     assert getmodule(mod.JobTemplate) == getmodule(model)
+
+
+@pytest.mark.parametrize("model", ALL_MODELS)
+def test_no_forward_refs_in_models(model: Type[BaseModel]) -> None:
+    """Test that no models in _model.py use ForwardRefs in their field annotations.
+
+    ForwardRefs indicate that a type is being referenced before it's defined, which can lead
+    to issues in pydantic validation. This test ensures all types are properly defined before
+    they're used.
+    """
+    for field_name, field in model.model_fields.items():
+        assert not isinstance(field.annotation, ForwardRef), (
+            f"Field '{field_name}' in model '{model.__name__}' uses a ForwardRef. "
+            "The referenced type should be defined before it's used."
+        )

test/openjd/model/v2023_09/test_module.py

@@ -0,0 +1,27 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+
+from __future__ import annotations
+
+from typing import Optional
+from openjd.model.v2023_09._model import OpenJDModel_v2023_09
+
+
+# First define a class we'll reference properly
+class ReferencedClass(OpenJDModel_v2023_09):
+    value: str
+
+
+class ClassWithoutForwardRef(OpenJDModel_v2023_09):
+    # This won't create a ForwardRef since ReferencedClass is already defined
+    ref: ReferencedClass
+
+
+# Now try to reference a class before it's defined, like we did in _model.py
+class ClassWithForwardRef(OpenJDModel_v2023_09):
+    # This should create a ForwardRef since SecondReferencedClass isn't defined yet
+    ref: Optional[SecondReferencedClass] = None
+
+
+# Define the class after it's referenced
+class SecondReferencedClass(OpenJDModel_v2023_09):
+    value: str

test/openjd/model/v2023_09/test_redacted_env_vars.py

@@ -0,0 +1,65 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+
+import pytest
+from pydantic import ValidationError
+
+from openjd.model._parse import _parse_model
+from openjd.model.v2023_09 import (
+    JobTemplate,
+    ModelParsingContext,
+)
+
+
+def test_redacted_env_vars_extension_supported() -> None:
+    """Test that the REDACTED_ENV_VARS extension can be used in a job template."""
+    data = {
+        "specificationVersion": "jobtemplate-2023-09",
+        "extensions": ["REDACTED_ENV_VARS"],
+        "name": "Test Job",
+        "steps": [
+            {
+                "name": "step1",
+                "script": {
+                    "actions": {"onRun": {"command": "python", "args": ["{{Task.File.Run}}"]}},
+                    "embeddedFiles": [
+                        {
+                            "name": "Run",
+                            "type": "TEXT",
+                            "data": 'print("openjd_redacted_env: SECRETVAR=SECRETVAL")',
+                        }
+                    ],
+                },
+            }
+        ],
+    }
+
+    # It parses successfully when the REDACTED_ENV_VARS extension is requested
+    _parse_model(
+        model=JobTemplate,
+        obj=data,
+        context=ModelParsingContext(supported_extensions=["REDACTED_ENV_VARS"]),
+    )
+
+
+def test_redacted_env_vars_extension_not_supported() -> None:
+    """Test that using REDACTED_ENV_VARS extension fails when not supported."""
+    data = {
+        "specificationVersion": "jobtemplate-2023-09",
+        "extensions": ["REDACTED_ENV_VARS"],
+        "name": "Test Job",
+        "steps": [
+            {
+                "name": "step1",
+                "script": {"actions": {"onRun": {"command": "echo", "args": ["test"]}}},
+            }
+        ],
+    }
+
+    # It fails to parse when REDACTED_ENV_VARS extension is not supported
+    with pytest.raises(ValidationError) as excinfo:
+        _parse_model(
+            model=JobTemplate,
+            obj=data,
+            context=ModelParsingContext(),
+        )
+    assert "Unsupported extension names: REDACTED_ENV_VARS" in str(excinfo.value)