From 65a10cf7f49bcff516e2cc88670efa8623e153c3 Mon Sep 17 00:00:00 2001 From: Daryl Lim <5508348+daryllimyt@users.noreply.github.com> Date: Fri, 8 Nov 2024 19:04:17 -0800 Subject: [PATCH 1/4] app: Rename static inputs to variables --- ...cf72d_rename_static_inputs_to_variables.py | 79 ++++++++++++++++++ tests/unit/test_error_handling.py | 6 +- tests/unit/test_expressions.py | 58 +++++++------- tests/unit/test_workflows.py | 80 +++++++++---------- tracecat/db/schemas.py | 4 +- tracecat/dsl/_converter.py | 30 ------- tracecat/dsl/common.py | 22 ++--- tracecat/dsl/models.py | 4 +- tracecat/dsl/workflow.py | 2 +- tracecat/expressions/parser/evaluator.py | 4 +- tracecat/expressions/parser/grammar.py | 4 +- tracecat/expressions/parser/validator.py | 14 ++-- tracecat/expressions/shared.py | 6 +- tracecat/validation/service.py | 2 +- tracecat/workflow/management/management.py | 4 +- tracecat/workflow/management/models.py | 4 +- tracecat/workflow/management/router.py | 2 +- 17 files changed, 187 insertions(+), 138 deletions(-) create mode 100644 alembic/versions/01c0ce9cf72d_rename_static_inputs_to_variables.py diff --git a/alembic/versions/01c0ce9cf72d_rename_static_inputs_to_variables.py b/alembic/versions/01c0ce9cf72d_rename_static_inputs_to_variables.py new file mode 100644 index 000000000..a70abdb36 --- /dev/null +++ b/alembic/versions/01c0ce9cf72d_rename_static_inputs_to_variables.py @@ -0,0 +1,79 @@ +"""Rename static_inputs to variables + +Revision ID: 01c0ce9cf72d +Revises: 046d417c113f +Create Date: 2024-11-08 18:18:54.464298 + +""" +from collections.abc import Sequence + +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "01c0ce9cf72d" +down_revision: str | None = "046d417c113f" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + # Add new column + op.add_column( + "workflow", + sa.Column("variables", postgresql.JSONB(astext_type=sa.Text()), nullable=True), + ) + + # Transfer existing data from static_inputs to variables + op.execute("UPDATE workflow SET variables = static_inputs") + + # Update WorkflowDefinition content field + op.execute( + """ + UPDATE workflowdefinition + SET content = jsonb_set( + content, + '{variables}', + content->'inputs' + ) - 'inputs' + WHERE content ? 'inputs' + """ + ) + + # Drop old column + op.drop_column("workflow", "static_inputs") + + +def downgrade() -> None: + # Add old column back + op.add_column( + "workflow", + sa.Column( + "static_inputs", + postgresql.JSONB(astext_type=sa.Text()), + autoincrement=False, + nullable=True, + ), + ) + + # Transfer data back from variables to static_inputs + op.execute("UPDATE workflow SET static_inputs = variables") + + # Revert WorkflowDefinition content field changes + op.execute( + """ + UPDATE workflowdefinition + SET content = jsonb_set( + content, + '{inputs}', + content->'variables' + ) - 'variables' + WHERE content ? 'variables' + """ + ) + + # Drop new column + op.drop_column("workflow", "variables") + # ### end Alembic commands ### diff --git a/tests/unit/test_error_handling.py b/tests/unit/test_error_handling.py index d487c3d80..43c111b06 100644 --- a/tests/unit/test_error_handling.py +++ b/tests/unit/test_error_handling.py @@ -60,8 +60,8 @@ async def test_execution_fails_fatal(temporal_cluster, test_role): # ("${{ 'string' + 5 }}", "unsupported operand type(s) for +: 'str' and 'int'"), # ("${{ {'key': 'value'}['nonexistent'] }}", "KeyError"), # ( - # "${{ INPUTS.undefined_input }}", - # "Couldn't resolve expression 'INPUTS.undefined_input'", + # "${{ VARS.undefined_input }}", + # "Couldn't resolve expression 'VARS.undefined_input'", # ), # ("${{ True and 'not boolean' }}", "TypeError"), ("${{ 1 if True else }}", "SyntaxError"), @@ -98,7 +98,7 @@ async def test_execution_fails_invalid_expressions( "description": "", }, ], - inputs={}, + variables={}, returns=None, tests=[], triggers=[], diff --git a/tests/unit/test_expressions.py b/tests/unit/test_expressions.py index 930344802..dd040adf3 100644 --- a/tests/unit/test_expressions.py +++ b/tests/unit/test_expressions.py @@ -66,20 +66,20 @@ def test_eval_jsonpath(): [ ("${{ACTIONS.webhook.result}}", 1), ("${{ ACTIONS.webhook.result -> int }}", 1), - ("${{ INPUTS.arg1 -> int }}", 1), - ("${{ INPUTS.arg1 }}", 1), # Doesn't cast - ("${{ INPUTS.arg2 -> str }}", "2"), + ("${{ VARS.arg1 -> int }}", 1), + ("${{ VARS.arg1 }}", 1), # Doesn't cast + ("${{ VARS.arg2 -> str }}", "2"), ("${{ ACTIONS.webhook.result -> str }}", "1"), ("${{ ACTIONS.path_A_first.result.path.nested.value -> int }}", 9999), ( - "${{ FN.add(INPUTS.arg1, ACTIONS.path_A_first.result.path.nested.value) }}", + "${{ FN.add(VARS.arg1, ACTIONS.path_A_first.result.path.nested.value) }}", 10000, ), ], ) def test_templated_expression_result(expression, expected_result): exec_vars = { - ExprContext.INPUTS: { + ExprContext.VARS: { "arg1": 1, "arg2": 2, }, @@ -109,18 +109,18 @@ def test_templated_expression_result(expression, expected_result): 1234.5, ), ( - "${{ FN.less_than(INPUTS.arg1, INPUTS.arg2) -> bool }}", + "${{ FN.less_than(VARS.arg1, VARS.arg2) -> bool }}", True, ), ( - "${{ FN.is_equal(INPUTS.arg1, ACTIONS.webhook.result) -> bool }}", + "${{ FN.is_equal(VARS.arg1, ACTIONS.webhook.result) -> bool }}", True, ), ], ) def test_templated_expression_function(expression, expected_result): exec_vars = { - ExprContext.INPUTS: { + ExprContext.VARS: { "arg1": 1, "arg2": 2, }, @@ -379,7 +379,7 @@ def test_eval_templated_object_inline_fails_if_not_str(): ("FN.concat(ENV.item, '5')", "ITEM5"), ("FN.add(5, 2)", 7), (" FN.is_null(None) ", True), - ("FN.contains('a', INPUTS.my.module.items)", True), + ("FN.contains('a', VARS.my.module.items)", True), ("FN.length([1, 2, 3])", 3), ("FN.join(['A', 'B', 'C'], ',')", "A,B,C"), ("FN.join(['A', 'B', 'C'], '@')", "A@B@C"), @@ -390,7 +390,7 @@ def test_eval_templated_object_inline_fails_if_not_str(): ["Hey Alice!", "Hey Bob!", "Hey Charlie!"], ), ( - "FN.format.map('Hello, {}! You are {}.', ['Alice', 'Bob', 'Charlie'], INPUTS.adjectives)", + "FN.format.map('Hello, {}! You are {}.', ['Alice', 'Bob', 'Charlie'], VARS.adjectives)", [ "Hello, Alice! You are cool.", "Hello, Bob! You are awesome.", @@ -403,15 +403,15 @@ def test_eval_templated_object_inline_fails_if_not_str(): ), # Ternary expressions ( - "'It contains 1' if FN.contains(1, INPUTS.list) else 'it does not contain 1'", + "'It contains 1' if FN.contains(1, VARS.list) else 'it does not contain 1'", "It contains 1", ), - ("True if FN.contains('key1', INPUTS.dict) else False", True), - ("True if FN.contains('key2', INPUTS.dict) else False", False), - ("True if FN.does_not_contain('key2', INPUTS.dict) else False", True), - ("True if FN.does_not_contain('key1', INPUTS.dict) else False", False), + ("True if FN.contains('key1', VARS.dict) else False", True), + ("True if FN.contains('key2', VARS.dict) else False", False), + ("True if FN.does_not_contain('key2', VARS.dict) else False", True), + ("True if FN.does_not_contain('key1', VARS.dict) else False", False), ( - "None if FN.does_not_contain('key1', INPUTS.dict) else INPUTS.dict.key1", + "None if FN.does_not_contain('key1', VARS.dict) else VARS.dict.key1", 1, ), ( @@ -425,7 +425,7 @@ def test_eval_templated_object_inline_fails_if_not_str(): ("True if TRIGGER.hits2._source.host.ip else False", False), # Truthy expressions ("TRIGGER.hits2._source.host.ip", None), - ("INPUTS.people[4].name", None), + ("VARS.people[4].name", None), # Typecast expressions ("int(5)", 5), ("float(5.0)", 5.0), @@ -466,12 +466,12 @@ def test_eval_templated_object_inline_fails_if_not_str(): ("var.y", "100"), ("var.y -> int", 100), # Test jsonpath - ("INPUTS.people[1].name", "Bob"), - ("INPUTS.people[2].age -> str", "50"), - ("INPUTS.people[*].age", [30, 40, 50]), - ("INPUTS.people[*].name", ["Alice", "Bob", "Charlie"]), - ("INPUTS.people[*].gender", ["female", "male"]), - # ('INPUTS.["user@tracecat.com"].name', "Bob"), TODO: Add support for object key indexing + ("VARS.people[1].name", "Bob"), + ("VARS.people[2].age -> str", "50"), + ("VARS.people[*].age", [30, 40, 50]), + ("VARS.people[*].name", ["Alice", "Bob", "Charlie"]), + ("VARS.people[*].gender", ["female", "male"]), + # ('VARS.["user@tracecat.com"].name', "Bob"), TODO: Add support for object key indexing # Combination ("'a' if FN.is_equal(var.y, '100') else 'b'", "a"), ("'a' if var.y == '100' else 'b'", "a"), @@ -503,7 +503,7 @@ def test_expression_parser(expr, expected): "KEY": "SECRET", }, }, - ExprContext.INPUTS: { + ExprContext.VARS: { "list": [1, 2, 3], "dict": { "key1": 1, @@ -740,7 +740,7 @@ def assert_validation_result( "url": "${{ int(100) }}", }, "test2": "fail 1 ${{ ACTIONS.my_action.invalid }} ", - "test3": "fail 2 ${{ int(INPUTS.my_action.invalid_inner) }} ", + "test3": "fail 2 ${{ int(VARS.my_action.invalid_inner) }} ", }, [ { @@ -754,7 +754,7 @@ def assert_validation_result( "contains_msg": "invalid", }, { - "type": ExprType.INPUT, + "type": ExprType.VARS, "status": "error", "contains_msg": "invalid_inner", }, @@ -763,12 +763,12 @@ def assert_validation_result( ( { "test": { - "data": "INLINE: ${{ INPUTS.invalid }}", + "data": "INLINE: ${{ VARS.invalid }}", }, }, [ { - "type": ExprType.INPUT, + "type": ExprType.VARS, "status": "error", "contains_msg": "invalid", }, @@ -795,7 +795,7 @@ async def test_extract_expressions_errors(expr, expected, test_role, env_sandbox # The only defined action reference is "my_action" validation_context = ExprValidationContext( action_refs={"my_action"}, - inputs_context={"arg": 2}, + variables_context={"arg": 2}, ) validators = get_validators() diff --git a/tests/unit/test_workflows.py b/tests/unit/test_workflows.py index 6b7f48392..29821ca2d 100644 --- a/tests/unit/test_workflows.py +++ b/tests/unit/test_workflows.py @@ -367,7 +367,7 @@ async def set_environment(value: str) -> dict[str, Any]: }, ], "description": "Test that a UDF with declared secrets will pull the corresponding secrets given the runtime environment.", - "inputs": {}, + "variables": {}, "returns": "${{ ACTIONS.a.result }}", "tests": [], "title": f"{test_name}", @@ -501,7 +501,7 @@ async def set_environment(value: str) -> dict[str, Any]: }, ], "description": "Stress testing", - "inputs": {}, + "variables": {}, "returns": "${{ ACTIONS.a.result }}", "tests": [], "title": f"{test_name}", @@ -601,7 +601,7 @@ async def test_stress_workflow_correctness( }, ], "description": "Stress testing", - "inputs": {}, + "variables": {}, "returns": "${{ ACTIONS.d.result }}", "tests": [], "title": f"{test_name}", @@ -672,7 +672,7 @@ async def test_workflow_set_environment_correct( } ], "description": test_description, - "inputs": {}, + "variables": {}, "returns": "${{ ACTIONS.a.result }}", "tests": [], "title": f"{test_name}", @@ -732,7 +732,7 @@ async def test_workflow_override_environment_correct( } ], "description": test_description, - "inputs": {}, + "variables": {}, "returns": "${{ ACTIONS.a.result }}", "tests": [], "title": f"{test_name}", @@ -793,7 +793,7 @@ async def test_workflow_default_environment_correct( } ], "description": test_description, - "inputs": {}, + "variables": {}, "returns": "${{ ACTIONS.a.result }}", "tests": [], "title": f"{test_name}", @@ -889,7 +889,7 @@ async def test_child_workflow_success(temporal_cluster, test_role, temporal_clie } ], "description": "Test child workflow success", - "inputs": {}, + "variables": {}, "returns": "${{ ACTIONS.a.result }}", "tests": [], "title": "Test child workflow success", @@ -911,7 +911,7 @@ async def test_child_workflow_success(temporal_cluster, test_role, temporal_clie { "ref": "parent", "action": "core.workflow.execute", - "for_each": "${{ for var.x in INPUTS.data }}", + "for_each": "${{ for var.x in VARS.data }}", "args": { "workflow_id": child_workflow.id, "trigger_inputs": { @@ -921,7 +921,7 @@ async def test_child_workflow_success(temporal_cluster, test_role, temporal_clie "depends_on": [], }, ], - "inputs": { + "variables": { "data": [1, 2, 3, 4, 5, 6, 7], }, "returns": None, @@ -944,7 +944,7 @@ async def test_child_workflow_success(temporal_cluster, test_role, temporal_clie "result_typename": "list", } }, - "INPUTS": {"data": [1, 2, 3, 4, 5, 6, 7]}, + "VARS": {"data": [1, 2, 3, 4, 5, 6, 7]}, "TRIGGER": {}, } assert result == expected @@ -976,7 +976,7 @@ async def test_child_workflow_context_passing( } ], "description": "Testing child workflow", - "inputs": {}, + "variables": {}, "returns": None, "tests": [], "title": "Aug 16, 2024, 13:44:37", @@ -1020,7 +1020,7 @@ async def test_child_workflow_context_passing( "description": "", }, ], - "inputs": {}, + "variables": {}, "returns": None, "tests": [], "triggers": [], @@ -1055,7 +1055,7 @@ async def test_child_workflow_context_passing( "result_typename": "dict", } }, - "INPUTS": {}, + "VARS": {}, "TRIGGER": { "data_from_parent": "Parent sent child __EXPECTED_DATA__" }, @@ -1063,7 +1063,7 @@ async def test_child_workflow_context_passing( "result_typename": "dict", }, }, - "INPUTS": {}, + "VARS": {}, "TRIGGER": {"data": "__EXPECTED_DATA__"}, } assert result == expected @@ -1093,7 +1093,7 @@ async def test_single_child_workflow_override_environment_correct( } ], "description": test_description, - "inputs": {}, + "variables": {}, "returns": "${{ ACTIONS.a.result }}", "tests": [], "title": f"{test_name}:child", @@ -1121,7 +1121,7 @@ async def test_single_child_workflow_override_environment_correct( "depends_on": [], }, ], - "inputs": {}, + "variables": {}, "returns": None, "tests": [], "triggers": [], @@ -1142,7 +1142,7 @@ async def test_single_child_workflow_override_environment_correct( "result_typename": "str", } }, - "INPUTS": {}, + "VARS": {}, "TRIGGER": {}, } assert result == expected @@ -1171,7 +1171,7 @@ async def test_multiple_child_workflow_override_environment_correct( "description": "", } ], - "inputs": {}, + "variables": {}, "returns": "${{ ACTIONS.a.result }}", "tests": [], "title": f"{test_name}:child", @@ -1200,7 +1200,7 @@ async def test_multiple_child_workflow_override_environment_correct( "depends_on": [], }, ], - "inputs": {}, + "variables": {}, "returns": None, "tests": [], "triggers": [], @@ -1221,7 +1221,7 @@ async def test_multiple_child_workflow_override_environment_correct( "result_typename": "list", } }, - "INPUTS": {}, + "VARS": {}, "TRIGGER": {}, } assert result == expected @@ -1251,7 +1251,7 @@ async def test_single_child_workflow_environment_has_correct_default( } ], "description": test_description, - "inputs": {}, + "variables": {}, "returns": "${{ ACTIONS.a.result }}", "config": {"environment": "__TESTING_DEFAULT__"}, "tests": [], @@ -1279,7 +1279,7 @@ async def test_single_child_workflow_environment_has_correct_default( "depends_on": [], }, ], - "inputs": {}, + "variables": {}, "returns": None, "tests": [], "triggers": [], @@ -1300,7 +1300,7 @@ async def test_single_child_workflow_environment_has_correct_default( "result_typename": "str", } }, - "INPUTS": {}, + "VARS": {}, "TRIGGER": {}, } assert result == expected @@ -1330,7 +1330,7 @@ async def test_multiple_child_workflow_environments_have_correct_defaults( } ], "description": test_description, - "inputs": {}, + "variables": {}, "returns": "${{ ACTIONS.a.result }}", "config": { "environment": "__TESTING_DEFAULT__" @@ -1364,7 +1364,7 @@ async def test_multiple_child_workflow_environments_have_correct_defaults( "depends_on": [], }, ], - "inputs": {}, + "variables": {}, "returns": None, "tests": [], "triggers": [], @@ -1389,7 +1389,7 @@ async def test_multiple_child_workflow_environments_have_correct_defaults( "result_typename": "list", } }, - "INPUTS": {}, + "VARS": {}, "TRIGGER": {}, } assert result == expected @@ -1437,7 +1437,7 @@ async def test_single_child_workflow_get_correct_secret_environment( } ], "description": test_description, - "inputs": {}, + "variables": {}, "returns": "${{ ACTIONS.a.result }}", "config": { "environment": "__TESTING_DEFAULT__" @@ -1468,7 +1468,7 @@ async def test_single_child_workflow_get_correct_secret_environment( "depends_on": [], }, ], - "inputs": {}, + "variables": {}, "returns": None, "tests": [], "triggers": [], @@ -1492,7 +1492,7 @@ async def test_single_child_workflow_get_correct_secret_environment( "result_typename": "list", } }, - "INPUTS": {}, + "VARS": {}, "TRIGGER": {}, } assert result == expected @@ -1527,7 +1527,7 @@ async def test_pull_based_workflow_fetches_latest_version(temporal_client, test_ } ], "description": "Test that a pull-based workflow fetches the latest version", - "inputs": {}, + "variables": {}, "returns": "${{ ACTIONS.a.result }}", "tests": [], "title": f"{test_name}:first", @@ -1576,7 +1576,7 @@ async def test_pull_based_workflow_fetches_latest_version(temporal_client, test_ } ], "description": "Test that a pull-based workflow fetches the latest version", - "inputs": {}, + "variables": {}, "returns": "${{ ACTIONS.a.result }}", "tests": [], "title": f"{test_name}:second", @@ -1661,7 +1661,7 @@ def _get_test_id(test_case): "depends_on": ["start.error"], # RUNS }, ], - "inputs": {}, + "variables": {}, "returns": { "start": "${{ ACTIONS.start.result }}", "start.error": "${{ ACTIONS.start.error }}", @@ -1710,7 +1710,7 @@ def _get_test_id(test_case): "depends_on": ["c"], }, ], - "inputs": {}, + "variables": {}, "returns": { "a": "${{ ACTIONS.a.result }}", "b": "${{ ACTIONS.b.result }}", @@ -1751,7 +1751,7 @@ def _get_test_id(test_case): "depends_on": ["a.error"], # RUNS }, ], - "inputs": {}, + "variables": {}, "returns": { "success": "${{ ACTIONS.success_path.result }}", "error": "${{ ACTIONS.error_path.result }}", @@ -1788,7 +1788,7 @@ def _get_test_id(test_case): "depends_on": ["a.error"], # This action should NOT run }, ], - "inputs": {}, + "variables": {}, "returns": { "success": "${{ ACTIONS.success_path.result }}", "error": "${{ ACTIONS.error_path.result }}", @@ -1831,7 +1831,7 @@ def _get_test_id(test_case): "depends_on": ["error_path_1"], # This action should NOT run }, ], - "inputs": {}, + "variables": {}, "returns": { "success_path": "${{ ACTIONS.success_path.result }}", "error_path_1": "${{ ACTIONS.error_path_1.result }}", @@ -1865,7 +1865,7 @@ def _get_test_id(test_case): "run_if": "${{ False }}", }, ], - "inputs": {}, + "variables": {}, "returns": { "a": "${{ ACTIONS.a.result }}", "b": "${{ ACTIONS.b.result }}", @@ -1897,7 +1897,7 @@ def _get_test_id(test_case): "join_strategy": "any", }, ], - "inputs": {}, + "variables": {}, "returns": { "start": "${{ ACTIONS.start.result }}", "join": "${{ ACTIONS.join.result }}", @@ -1945,7 +1945,7 @@ def _get_test_id(test_case): "join_strategy": "any", }, ], - "inputs": {}, + "variables": {}, "returns": { "start": "${{ ACTIONS.start.result }}", "left": "${{ ACTIONS.left.result }}", @@ -1997,7 +1997,7 @@ def _get_test_id(test_case): "join_strategy": "any", }, ], - "inputs": {}, + "variables": {}, "returns": { "start": "${{ ACTIONS.start.result }}", "left": "${{ ACTIONS.left.result }}", diff --git a/tracecat/db/schemas.py b/tracecat/db/schemas.py index 3d9da7782..48e377dd1 100644 --- a/tracecat/db/schemas.py +++ b/tracecat/db/schemas.py @@ -243,10 +243,10 @@ class Workflow(Resource, table=True): None, description="ID of the node directly connected to the trigger.", ) - static_inputs: dict[str, Any] = Field( + variables: dict[str, Any] = Field( default_factory=dict, sa_column=Column(JSONB), - description="Static inputs for the workflow", + description="Variables for the workflow", ) expects: dict[str, Any] = Field( default_factory=dict, diff --git a/tracecat/dsl/_converter.py b/tracecat/dsl/_converter.py index 2001b2067..8a1b94fa3 100644 --- a/tracecat/dsl/_converter.py +++ b/tracecat/dsl/_converter.py @@ -59,33 +59,3 @@ def __init__(self) -> None: payload_converter_class=PydanticPayloadConverter ) """Data converter using Pydantic JSON conversion.""" - - -# test = { -# "role": { -# "type": "service", -# "workspace_id": UUID("c0963ef7-8577-4da6-9860-6ea7b4db900b"), -# "user_id": UUID("00000000-0000-4444-aaaa-000000000000"), -# "service_id": "tracecat-runner", -# }, -# "dsl": { -# "title": "test_workflow_override_environment_correct", -# "description": "Test that we can set the runtime environment for a workflow. The workflow should use the environment set in the DSL config.", -# "entrypoint": {"ref": "a", "expects": {}}, -# "actions": [ -# { -# "ref": "a", -# "description": "", -# "action": "core.transform.reshape", -# "args": {"value": "${{ ENV.environment }}"}, -# "depends_on": [], -# } -# ], -# "config": {"environment": "__TEST_ENVIRONMENT__"}, -# "triggers": [], -# "inputs": {}, -# "tests": [], -# "returns": "${{ ACTIONS.a.result }}", -# }, -# "wf_id": "wf-00000000000000000000000000000000", -# } diff --git a/tracecat/dsl/common.py b/tracecat/dsl/common.py index 457cf5f8b..51159dd3f 100644 --- a/tracecat/dsl/common.py +++ b/tracecat/dsl/common.py @@ -66,8 +66,8 @@ class DSLInput(BaseModel): actions: list[ActionStatement] config: DSLConfig = Field(default_factory=DSLConfig) triggers: list[Trigger] = Field(default_factory=list) - inputs: dict[str, Any] = Field( - default_factory=dict, description="Static input parameters" + variables: dict[str, Any] = Field( + default_factory=dict, description="Variables for the workflow" ) returns: Any | None = Field(None, description="The action ref or value to return.") @@ -114,25 +114,25 @@ def validate_structure(self) -> Self: ) return self - @field_validator("inputs") + @field_validator("variables") @classmethod - def inputs_cannot_have_expressions(cls, inputs: Any) -> dict[str, Any]: + def variables_cannot_have_expressions(cls, variables: Any) -> dict[str, Any]: try: exceptions = [] - for loc, value in traverse_leaves(inputs): + for loc, value in traverse_leaves(variables): if not isinstance(value, str): continue for match in patterns.TEMPLATE_STRING.finditer(value): template = match.group("template") exceptions.append( TracecatDSLError( - "Static `INPUTS` context cannot contain expressions," - f" but found {template!r} in INPUTS.{loc}" + "`VARS` context cannot contain expressions," + f" but found {template!r} in VARS.{loc}" ) ) if exceptions: - raise ExceptionGroup("Static `INPUTS` validation failed", exceptions) - return inputs + raise ExceptionGroup("`VARS` validation failed", exceptions) + return variables except* TracecatDSLError as eg: raise eg @@ -303,13 +303,13 @@ def build_action_statements( def create_default_dsl_context( - INPUTS: dict[str, Any] | None = None, + VARS: dict[str, Any] | None = None, ACTIONS: dict[str, Any] | None = None, TRIGGER: dict[str, Any] | None = None, ENV: DSLEnvironment | None = None, ) -> DSLContext: return DSLContext( - INPUTS=INPUTS or {}, + VARS=VARS or {}, ACTIONS=ACTIONS or {}, TRIGGER=TRIGGER or {}, ENV=cast(DSLEnvironment, ENV or {}), diff --git a/tracecat/dsl/models.py b/tracecat/dsl/models.py index 26e27c2c2..cd0f7fd15 100644 --- a/tracecat/dsl/models.py +++ b/tracecat/dsl/models.py @@ -163,8 +163,8 @@ class DSLEnvironment(TypedDict, total=False): class DSLContext(TypedDict, total=False): - INPUTS: dict[str, Any] - """DSL Static Inputs context""" + VARS: dict[str, Any] + """DSL Variables context""" ACTIONS: dict[str, Any] """DSL Actions context""" diff --git a/tracecat/dsl/workflow.py b/tracecat/dsl/workflow.py index 9f9af2831..ef941e6df 100644 --- a/tracecat/dsl/workflow.py +++ b/tracecat/dsl/workflow.py @@ -214,7 +214,7 @@ async def run(self, args: DSLRunArgs) -> Any: # Prepare user facing context self.context = DSLContext( ACTIONS={}, - INPUTS=self.dsl.inputs, + VARS=self.dsl.variables, TRIGGER=trigger_inputs, ENV=DSLEnvironment( workflow={ diff --git a/tracecat/expressions/parser/evaluator.py b/tracecat/expressions/parser/evaluator.py index 14231b160..f2bd83f43 100644 --- a/tracecat/expressions/parser/evaluator.py +++ b/tracecat/expressions/parser/evaluator.py @@ -110,10 +110,10 @@ def secrets(self, jsonpath: str): ) @v_args(inline=True) - def inputs(self, jsonpath: str): + def variables(self, jsonpath: str): logger.trace("Visiting inputs:", args=jsonpath) return functions.eval_jsonpath( - ExprContext.INPUTS + jsonpath, self._context, strict=self._strict + ExprContext.VARS + jsonpath, self._context, strict=self._strict ) @v_args(inline=True) diff --git a/tracecat/expressions/parser/grammar.py b/tracecat/expressions/parser/grammar.py index 854f56a44..be392b81a 100644 --- a/tracecat/expressions/parser/grammar.py +++ b/tracecat/expressions/parser/grammar.py @@ -20,7 +20,7 @@ ?context: actions | secrets - | inputs + | variables | env | local_vars | trigger @@ -33,7 +33,7 @@ actions: "ACTIONS" jsonpath_expression secrets: "SECRETS" ATTRIBUTE_PATH -inputs: "INPUTS" jsonpath_expression +variables: "VARS" jsonpath_expression env: "ENV" jsonpath_expression local_vars: "var" jsonpath_expression trigger: "TRIGGER" [jsonpath_expression] diff --git a/tracecat/expressions/parser/validator.py b/tracecat/expressions/parser/validator.py index 581f54f65..5b1db058a 100644 --- a/tracecat/expressions/parser/validator.py +++ b/tracecat/expressions/parser/validator.py @@ -25,7 +25,7 @@ class ExprValidationContext(BaseModel): """Container for the validation context of an expression tree.""" action_refs: set[str] - inputs_context: Any = Field(default_factory=dict) + variables_context: Any = Field(default_factory=dict) trigger_context: Any = Field(default_factory=dict) @@ -164,19 +164,19 @@ def secrets(self, node: Tree): ) self._task_group.create_task(coro) - def inputs(self, node: Tree): - self.logger.trace("Visit input expression", node=node) + def variables(self, node: Tree): + self.logger.trace("Visit variables expression", node=node) jsonpath = get_jsonpath_body_from_context(node).lstrip(".") try: functions.eval_jsonpath( jsonpath, - self._context.inputs_context, - context_type=ExprContext.INPUTS, + self._context.variables_context, + context_type=ExprContext.VARS, strict=self._strict, ) - self.add(status="success", type=ExprType.INPUT) + self.add(status="success", type=ExprType.VARS) except TracecatExpressionError as e: - return self.add(status="error", msg=str(e), type=ExprType.INPUT) + return self.add(status="error", msg=str(e), type=ExprType.VARS) def trigger(self, node: Tree): self.logger.trace("Visit trigger expression", node=node) diff --git a/tracecat/expressions/shared.py b/tracecat/expressions/shared.py index 2164bc2cd..7e5068a71 100644 --- a/tracecat/expressions/shared.py +++ b/tracecat/expressions/shared.py @@ -15,7 +15,7 @@ class ExprContext(TracecatEnum): ACTIONS = "ACTIONS" SECRETS = "SECRETS" FN = "FN" - INPUTS = "INPUTS" + VARS = "VARS" ENV = "ENV" TRIGGER = "TRIGGER" @@ -30,7 +30,7 @@ class ExprType(TracecatEnum): ACTION = auto() SECRET = auto() FUNCTION = auto() - INPUT = auto() + VARS = auto() ENV = auto() LOCAL_VARS = auto() LITERAL = auto() @@ -45,7 +45,7 @@ class ExprType(TracecatEnum): "actions": ExprType.ACTION, "secrets": ExprType.SECRET, "function": ExprType.FUNCTION, - "inputs": ExprType.INPUT, + "variables": ExprType.VARS, "env": ExprType.ENV, "local_vars": ExprType.LOCAL_VARS, "literal": ExprType.LITERAL, diff --git a/tracecat/validation/service.py b/tracecat/validation/service.py index 87748736e..97da7e5d7 100644 --- a/tracecat/validation/service.py +++ b/tracecat/validation/service.py @@ -289,7 +289,7 @@ async def validate_dsl_expressions( ) -> list[ExprValidationResult]: """Validate the DSL expressions at commit time.""" validation_context = ExprValidationContext( - action_refs={a.ref for a in dsl.actions}, inputs_context=dsl.inputs + action_refs={a.ref for a in dsl.actions}, variables_context=dsl.variables ) validators = {ExprType.SECRET: secret_validator} diff --git a/tracecat/workflow/management/management.py b/tracecat/workflow/management/management.py index cb2c32200..abff538b4 100644 --- a/tracecat/workflow/management/management.py +++ b/tracecat/workflow/management/management.py @@ -217,7 +217,7 @@ async def build_dsl_from_workflow(self, workflow: Workflow) -> DSLInput: ref=graph.logical_entrypoint.ref, expects=workflow.expects ), actions=action_statements, - inputs=workflow.static_inputs, + variables=workflow.variables, config=DSLConfig(**workflow.config), returns=workflow.returns, # triggers=workflow.triggers, @@ -264,7 +264,7 @@ async def _create_db_workflow_from_dsl( "title": dsl.title, "description": dsl.description, "owner_id": self.role.workspace_id, - "static_inputs": dsl.inputs, + "variables": dsl.variables, "returns": dsl.returns, "config": dsl.config.model_dump(), "expects": entrypoint.get("expects"), diff --git a/tracecat/workflow/management/models.py b/tracecat/workflow/management/models.py index 04c8718b8..f94a72369 100644 --- a/tracecat/workflow/management/models.py +++ b/tracecat/workflow/management/models.py @@ -34,7 +34,7 @@ class WorkflowResponse(BaseModel): webhook: WebhookResponse schedules: list[Schedule] entrypoint: str | None - static_inputs: dict[str, Any] + variables: dict[str, Any] expects: dict[str, ExpectedField] | None = None returns: Any config: DSLConfig | None @@ -48,7 +48,7 @@ class UpdateWorkflowParams(BaseModel): version: int | None = None entrypoint: str | None = None icon_url: str | None = None - static_inputs: dict[str, Any] | None = None + variables: dict[str, Any] | None = None expects: dict[str, ExpectedField] | None = None returns: Any | None = None config: DSLConfig | None = None diff --git a/tracecat/workflow/management/router.py b/tracecat/workflow/management/router.py index 596b9f62a..d397bf40a 100644 --- a/tracecat/workflow/management/router.py +++ b/tracecat/workflow/management/router.py @@ -192,7 +192,7 @@ async def get_workflow( returns=workflow.returns, entrypoint=workflow.entrypoint, object=workflow.object, - static_inputs=workflow.static_inputs, + variables=workflow.variables, config=DSLConfig(**workflow.config), actions=actions_responses, webhook=WebhookResponse(**workflow.webhook.model_dump()), From 853441fe42a0ddaedf644ea3478f146ab1e884d5 Mon Sep 17 00:00:00 2001 From: Daryl Lim <5508348+daryllimyt@users.noreply.github.com> Date: Fri, 8 Nov 2024 19:04:32 -0800 Subject: [PATCH 2/4] ui: Update frontend --- frontend/src/client/schemas.gen.ts | 20 ++++++++--------- frontend/src/client/types.gen.ts | 10 ++++----- .../workbench/panel/workflow-panel.tsx | 22 +++++++++---------- 3 files changed, 26 insertions(+), 26 deletions(-) diff --git a/frontend/src/client/schemas.gen.ts b/frontend/src/client/schemas.gen.ts index c9b30e2af..e490471a7 100644 --- a/frontend/src/client/schemas.gen.ts +++ b/frontend/src/client/schemas.gen.ts @@ -733,9 +733,9 @@ Activities don't need access to this.` export const $DSLContext = { properties: { - INPUTS: { + VARS: { type: 'object', - title: 'Inputs' + title: 'Vars' }, ACTIONS: { type: 'object', @@ -834,10 +834,10 @@ export const $DSLInput = { type: 'array', title: 'Triggers' }, - inputs: { + variables: { type: 'object', - title: 'Inputs', - description: 'Static input parameters' + title: 'Variables', + description: 'Variables for the workflow' }, returns: { anyOf: [ @@ -2898,7 +2898,7 @@ export const $UpdateWorkflowParams = { ], title: 'Icon Url' }, - static_inputs: { + variables: { anyOf: [ { type: 'object' @@ -2907,7 +2907,7 @@ export const $UpdateWorkflowParams = { type: 'null' } ], - title: 'Static Inputs' + title: 'Variables' }, expects: { anyOf: [ @@ -3626,9 +3626,9 @@ export const $WorkflowResponse = { ], title: 'Entrypoint' }, - static_inputs: { + variables: { type: 'object', - title: 'Static Inputs' + title: 'Variables' }, expects: { anyOf: [ @@ -3659,7 +3659,7 @@ export const $WorkflowResponse = { } }, type: 'object', - required: ['id', 'title', 'description', 'status', 'actions', 'object', 'owner_id', 'webhook', 'schedules', 'entrypoint', 'static_inputs', 'returns', 'config'], + required: ['id', 'title', 'description', 'status', 'actions', 'object', 'owner_id', 'webhook', 'schedules', 'entrypoint', 'variables', 'returns', 'config'], title: 'WorkflowResponse' } as const; diff --git a/frontend/src/client/types.gen.ts b/frontend/src/client/types.gen.ts index 33b44a8c5..4a7fa9504 100644 --- a/frontend/src/client/types.gen.ts +++ b/frontend/src/client/types.gen.ts @@ -261,7 +261,7 @@ export type DSLConfig_Output = { }; export type DSLContext = { - INPUTS?: { + VARS?: { [key: string]: unknown; }; ACTIONS?: { @@ -316,9 +316,9 @@ export type DSLInput = { config?: DSLConfig_Output; triggers?: Array; /** - * Static input parameters + * Variables for the workflow */ - inputs?: { + variables?: { [key: string]: unknown; }; /** @@ -973,7 +973,7 @@ export type UpdateWorkflowParams = { version?: number | null; entrypoint?: string | null; icon_url?: string | null; - static_inputs?: { + variables?: { [key: string]: unknown; } | null; expects?: { @@ -1155,7 +1155,7 @@ export type WorkflowResponse = { webhook: WebhookResponse; schedules: Array; entrypoint: string | null; - static_inputs: { + variables: { [key: string]: unknown; }; expects?: { diff --git a/frontend/src/components/workbench/panel/workflow-panel.tsx b/frontend/src/components/workbench/panel/workflow-panel.tsx index 1c6d3de00..8c08a40a1 100644 --- a/frontend/src/components/workbench/panel/workflow-panel.tsx +++ b/frontend/src/components/workbench/panel/workflow-panel.tsx @@ -64,7 +64,7 @@ const workflowConfigFormSchema = z.object({ return z.NEVER } }), - static_inputs: z.string().transform((val, ctx) => { + variables: z.string().transform((val, ctx) => { try { return YAML.parse(val) || {} } catch (error) { @@ -123,9 +123,9 @@ export function WorkflowPanel({ environment: null, }) : YAML.stringify(workflow.config), - static_inputs: isEmptyObjectOrNullish(workflow.static_inputs) + variables: isEmptyObjectOrNullish(workflow.variables) ? "" - : YAML.stringify(workflow.static_inputs), + : YAML.stringify(workflow.variables), expects: isEmptyObjectOrNullish(workflow.expects) ? "" : YAML.stringify(workflow.expects), @@ -195,7 +195,7 @@ export function WorkflowPanel({ value="workflow-static-inputs" > - Static Inputs + Variables @@ -459,7 +459,7 @@ export function WorkflowPanel({
- Static Inputs + Variables
@@ -477,15 +477,15 @@ export function WorkflowPanel({ side="left" sideOffset={20} > - + - Define optional static inputs for the workflow. + Define optional variables for the workflow. (
- Static Inputs + Variables (optional)
@@ -525,7 +525,7 @@ function StaticInputTooltip() { 
-          {"${{ INPUTS.my_static_key }}"}
+          {"${{ VARS.my_static_key }}"}
From 19eed82f294ad41bddb9c169000b7e0e9f9a6c56 Mon Sep 17 00:00:00 2001 From: Daryl Lim <5508348+daryllimyt@users.noreply.github.com> Date: Fri, 8 Nov 2024 19:07:05 -0800 Subject: [PATCH 3/4] chore: Update tests and playbooks --- playbooks/detect/webhook_alerts/elastic.yml | 6 +- playbooks/respond/notify_users/slack.yml | 4 +- .../tutorials/execute_caldera_operations.yml | 854 +++++++++--------- playbooks/tutorials/limacharlie/list_tags.yml | 14 +- .../limacharlie/run_investigation.yml | 22 +- playbooks/tutorials/limacharlie/run_tests.yml | 22 +- playbooks/tutorials/virustotal_quickstart.yml | 2 +- .../workflows/integration_core_ai_action.yml | 4 +- .../workflows/integration_io_linear_http.yml | 6 +- tests/data/workflows/shared_adder_tree.yml | 2 +- tests/data/workflows/shared_kite.yml | 2 +- tests/data/workflows/shared_tree.yml | 2 +- tests/data/workflows/stress_adder_tree.yml | 4 +- .../workflows/unit_child_workflow_parent.yml | 4 +- .../unit_child_workflow_parent_expected.yml | 2 +- ...iamond_skip_with_join_strong_dep_fails.yml | 2 +- ..._adder_diamond_skip_with_join_weak_dep.yml | 2 +- ...amond_skip_with_join_weak_dep_expected.yml | 2 +- .../unit_conditional_adder_tree_continues.yml | 2 +- ...ditional_adder_tree_continues_expected.yml | 2 +- ...conditional_adder_tree_skip_propagates.yml | 2 +- ...al_adder_tree_skip_propagates_expected.yml | 2 +- .../unit_conditional_adder_tree_skips.yml | 2 +- ..._conditional_adder_tree_skips_expected.yml | 2 +- tests/data/workflows/unit_error_fatal.yml | 6 +- tests/data/workflows/unit_secrets.yml | 2 +- .../workflows/unit_transform_filter_dict.yml | 4 +- .../unit_transform_filter_dict_expected.yml | 2 +- .../unit_transform_filter_function.yml | 4 +- ...nit_transform_filter_function_expected.yml | 2 +- .../unit_transform_reshape_arrange.yml | 20 +- ...nit_transform_reshape_arrange_expected.yml | 2 +- .../unit_transform_reshape_arrange_loop.yml | 4 +- ...ransform_reshape_arrange_loop_expected.yml | 2 +- .../workflows/unit_transform_reshape_loop.yml | 4 +- .../unit_transform_reshape_loop_chained.yml | 8 +- ...ransform_reshape_loop_chained_expected.yml | 2 +- .../unit_transform_reshape_loop_expected.yml | 2 +- .../unit_transform_reshape_map_loop.yml | 4 +- ...it_transform_reshape_map_loop_expected.yml | 2 +- .../workflows/unit_transform_reshape_zip.yml | 6 +- .../unit_transform_reshape_zip_expected.yml | 2 +- 42 files changed, 523 insertions(+), 523 deletions(-) diff --git a/playbooks/detect/webhook_alerts/elastic.yml b/playbooks/detect/webhook_alerts/elastic.yml index 5ab003c8e..83c7e0796 100644 --- a/playbooks/detect/webhook_alerts/elastic.yml +++ b/playbooks/detect/webhook_alerts/elastic.yml @@ -17,7 +17,7 @@ definition: - type: webhook ref: receive_elastic_alerts entrypoint: deserialize_elastic_alerts - inputs: + variables: child_workflow_id: REPLACE_WITH_CHILD_WORKFLOW_ID limit: 10 batch_size: 10 @@ -35,10 +35,10 @@ definition: - deserialize_elastic_alerts for_each: ${{ for var.alert in ACTIONS.deserialize_elastic_alerts.result }} args: - workflow_id: ${{ INPUTS.child_workflow_id }} + workflow_id: ${{ VARS.child_workflow_id }} loop_strategy: parallel fail_strategy: isolated - batch_size: ${{ INPUTS.batch_size }} + batch_size: ${{ VARS.batch_size }} trigger_inputs: title: ${{ var.alert.message }} integration: Elastic Security diff --git a/playbooks/respond/notify_users/slack.yml b/playbooks/respond/notify_users/slack.yml index 3e2a42e0f..219291d23 100644 --- a/playbooks/respond/notify_users/slack.yml +++ b/playbooks/respond/notify_users/slack.yml @@ -23,7 +23,7 @@ definition: - type: webhook ref: receive_alert entrypoint: send_to_slack - inputs: + variables: # ID to uniquely identify the actions in this playbook slack_actions_id: alert-from-tracecat @@ -63,7 +63,7 @@ definition: text: "Respond to alert?" emoji: true - type: actions - block_id: ${{ INPUTS.slack_actions_id }} + block_id: ${{ VARS.slack_actions_id }} elements: - type: button text: diff --git a/playbooks/tutorials/execute_caldera_operations.yml b/playbooks/tutorials/execute_caldera_operations.yml index 9efa579c6..27baca639 100644 --- a/playbooks/tutorials/execute_caldera_operations.yml +++ b/playbooks/tutorials/execute_caldera_operations.yml @@ -8,439 +8,439 @@ definition: entrypoint: expects: {} ref: generate_uuid - inputs: + variables: caldera_api_url: /api/v2 returns: null triggers: [] actions: - - action: core.transform.reshape - args: - value: ${{ FN.slice(FN.uuid4(), 0, 6) }} - depends_on: [] - description: '' - for_each: null - ref: generate_uuid - retry_policy: - max_attempts: 1 - timeout: 300 - run_if: null - start_delay: 0.0 - # Receive operations from Caldera - - action: core.http_request - args: - headers: - KEY: ${{ SECRETS.caldera.API_KEY }} - method: GET - url: ${{ INPUTS.caldera_api_url }}/operations - verify_ssl: false - depends_on: - - generate_uuid - description: '' - for_each: null - ref: receive_operations - retry_policy: - max_attempts: 1 - timeout: 300 - run_if: null - start_delay: 0.0 - # Execute the Discovery operation - - action: core.http_request - args: - headers: - KEY: ${{ SECRETS.caldera.API_KEY }} - method: POST - payload: - adversary: - adversary_id: 0f4c3c67-845e-49a0-927e-90ed33c044e0 - atomic_ordering: - - c0da588f-79f0-4263-8998-7496b1a40596 - - c1cd6388-3ced-48c7-a511-0434c6ba8f48 - - feaced8f-f43f-452a-9500-a5219488abb8 - - b6f545ef-f802-4537-b59d-2cb19831c8ed - - 3b5db901-2cb8-4df7-8043-c4628a6a5d5a - - 530e47c6-8592-42bf-91df-c59ffbd8541b - - 26c8b8b5-7b5b-4de1-a128-7d37fb14f517 - - 2dece965-37a0-4f70-a391-0f30e3331aba - - 5c4dd985-89e3-4590-9b57-71fed66ff4e2 - - 8c06ebf8-bacf-486b-bd77-21ba8c5a5777 - - ce485320-41a4-42e8-a510-f5a8fe96a644 - - b007fc38-9eb7-4320-92b3-9a3ad3e6ec25 - description: A discovery adversary - name: Discovery - objective: 495a9828-cab1-44dd-a0ca-66e58177d8cc - plugin: stockpile - tags: [] - auto_close: false - autonomous: 1 - group: '' - id: ${{ FN.uuid4() }} - jitter: 2/8 - name: Operation Discovery - ${{ ACTIONS.generate_uuid.result }} - obfuscator: plain-text - objective: - description: string - goals: - - count: 0 - operator: string - target: string - value: string - id: string - name: string - planner: - allow_repeatable_abilities: true - description: string - id: aaa7c857-37a0-4c4a-85f7-4e9f7f30e31a - ignore_enforcement_modules: - - string - module: app.planners.atomic - name: atomic - params: {} - plugin: null - stopping_conditions: [] - source: - adjustments: - - ability_id: string - offset: 0 - trait: string - value: string - facts: - - collected_by: - - string - limit_count: 0 - links: - - string - origin_type: string - relationships: - - string - score: 0 - source: string - technique_id: string - trait: string - value: string - id: ed32b9c3-9593-4c33-b0db-e2007315096b - name: basic - plugin: null - relationships: - - edge: string - origin: string - score: 0 - source: - collected_by: - - string - limit_count: 0 - links: - - string - origin_type: string - relationships: - - string - score: 0 - source: string - technique_id: string - trait: string - value: string - target: - collected_by: + - action: core.transform.reshape + args: + value: ${{ FN.slice(FN.uuid4(), 0, 6) }} + depends_on: [] + description: "" + for_each: null + ref: generate_uuid + retry_policy: + max_attempts: 1 + timeout: 300 + run_if: null + start_delay: 0.0 + # Receive operations from Caldera + - action: core.http_request + args: + headers: + KEY: ${{ SECRETS.caldera.API_KEY }} + method: GET + url: ${{ VARS.caldera_api_url }}/operations + verify_ssl: false + depends_on: + - generate_uuid + description: "" + for_each: null + ref: receive_operations + retry_policy: + max_attempts: 1 + timeout: 300 + run_if: null + start_delay: 0.0 + # Execute the Discovery operation + - action: core.http_request + args: + headers: + KEY: ${{ SECRETS.caldera.API_KEY }} + method: POST + payload: + adversary: + adversary_id: 0f4c3c67-845e-49a0-927e-90ed33c044e0 + atomic_ordering: + - c0da588f-79f0-4263-8998-7496b1a40596 + - c1cd6388-3ced-48c7-a511-0434c6ba8f48 + - feaced8f-f43f-452a-9500-a5219488abb8 + - b6f545ef-f802-4537-b59d-2cb19831c8ed + - 3b5db901-2cb8-4df7-8043-c4628a6a5d5a + - 530e47c6-8592-42bf-91df-c59ffbd8541b + - 26c8b8b5-7b5b-4de1-a128-7d37fb14f517 + - 2dece965-37a0-4f70-a391-0f30e3331aba + - 5c4dd985-89e3-4590-9b57-71fed66ff4e2 + - 8c06ebf8-bacf-486b-bd77-21ba8c5a5777 + - ce485320-41a4-42e8-a510-f5a8fe96a644 + - b007fc38-9eb7-4320-92b3-9a3ad3e6ec25 + description: A discovery adversary + name: Discovery + objective: 495a9828-cab1-44dd-a0ca-66e58177d8cc + plugin: stockpile + tags: [] + auto_close: false + autonomous: 1 + group: "" + id: ${{ FN.uuid4() }} + jitter: 2/8 + name: Operation Discovery - ${{ ACTIONS.generate_uuid.result }} + obfuscator: plain-text + objective: + description: string + goals: + - count: 0 + operator: string + target: string + value: string + id: string + name: string + planner: + allow_repeatable_abilities: true + description: string + id: aaa7c857-37a0-4c4a-85f7-4e9f7f30e31a + ignore_enforcement_modules: - string - limit_count: 0 - links: - - string - origin_type: string - relationships: - - string - score: 0 - source: string - technique_id: string - trait: string - value: string - rules: - - action: DENY - match: .* - trait: file.sensitive.extension - - action: ALLOW - match: png - trait: file.sensitive.extension - - action: ALLOW - match: yml - trait: file.sensitive.extension - - action: ALLOW - match: wav - trait: file.sensitive.extension - state: running - use_learning_parsers: true - visibility: 51 - url: ${{ INPUTS.caldera_api_url }}/operations - verify_ssl: false - depends_on: - - receive_operations - description: '' - for_each: null - ref: operation_discovery - retry_policy: - max_attempts: 1 - timeout: 300 - run_if: null - start_delay: 10.0 - # Execute the Thief operation - - action: core.http_request - args: - headers: - KEY: ${{ SECRETS.caldera.API_KEY }} - method: POST - payload: - adversary: - adversary_id: 1a98b8e6-18ce-4617-8cc5-e65a1a9d490e - atomic_ordering: - - 6469befa-748a-4b9c-a96d-f191fde47d89 - - 90c2efaa-8205-480d-8bb6-61d90dbaf81b - - 4e97e699-93d7-4040-b5a3-2e906a58199e - - 300157e5-f4ad-4569-b533-9d1fa0e74d74 - - ea713bc4-63f0-491c-9a6f-0b01d560b87e - description: An adversary to steal sensitive files - name: Thief - objective: 495a9828-cab1-44dd-a0ca-66e58177d8cc - plugin: stockpile - tags: [] - auto_close: false - autonomous: 1 - group: '' - id: ${{ FN.uuid4() }} - jitter: 2/8 - name: Operation Thief - ${{ ACTIONS.generate_uuid.result }} - obfuscator: plain-text - objective: - description: string - goals: - - count: 0 - operator: string - target: string - value: string - id: string - name: string - planner: - allow_repeatable_abilities: true - description: string - id: aaa7c857-37a0-4c4a-85f7-4e9f7f30e31a - ignore_enforcement_modules: - - string - module: app.planners.atomic - name: atomic - params: {} - plugin: null - stopping_conditions: [] - source: - adjustments: - - ability_id: string - offset: 0 - trait: string - value: string - facts: - - collected_by: - - string - limit_count: 0 - links: - - string - origin_type: string + module: app.planners.atomic + name: atomic + params: {} + plugin: null + stopping_conditions: [] + source: + adjustments: + - ability_id: string + offset: 0 + trait: string + value: string + facts: + - collected_by: + - string + limit_count: 0 + links: + - string + origin_type: string + relationships: + - string + score: 0 + source: string + technique_id: string + trait: string + value: string + id: ed32b9c3-9593-4c33-b0db-e2007315096b + name: basic + plugin: null relationships: - - string - score: 0 - source: string - technique_id: string - trait: string - value: string - id: ed32b9c3-9593-4c33-b0db-e2007315096b - name: basic - plugin: null - relationships: - - edge: string - origin: string - score: 0 - source: - collected_by: - - string - limit_count: 0 - links: - - string - origin_type: string - relationships: + - edge: string + origin: string + score: 0 + source: + collected_by: + - string + limit_count: 0 + links: + - string + origin_type: string + relationships: + - string + score: 0 + source: string + technique_id: string + trait: string + value: string + target: + collected_by: + - string + limit_count: 0 + links: + - string + origin_type: string + relationships: + - string + score: 0 + source: string + technique_id: string + trait: string + value: string + rules: + - action: DENY + match: .* + trait: file.sensitive.extension + - action: ALLOW + match: png + trait: file.sensitive.extension + - action: ALLOW + match: yml + trait: file.sensitive.extension + - action: ALLOW + match: wav + trait: file.sensitive.extension + state: running + use_learning_parsers: true + visibility: 51 + url: ${{ VARS.caldera_api_url }}/operations + verify_ssl: false + depends_on: + - receive_operations + description: "" + for_each: null + ref: operation_discovery + retry_policy: + max_attempts: 1 + timeout: 300 + run_if: null + start_delay: 10.0 + # Execute the Thief operation + - action: core.http_request + args: + headers: + KEY: ${{ SECRETS.caldera.API_KEY }} + method: POST + payload: + adversary: + adversary_id: 1a98b8e6-18ce-4617-8cc5-e65a1a9d490e + atomic_ordering: + - 6469befa-748a-4b9c-a96d-f191fde47d89 + - 90c2efaa-8205-480d-8bb6-61d90dbaf81b + - 4e97e699-93d7-4040-b5a3-2e906a58199e + - 300157e5-f4ad-4569-b533-9d1fa0e74d74 + - ea713bc4-63f0-491c-9a6f-0b01d560b87e + description: An adversary to steal sensitive files + name: Thief + objective: 495a9828-cab1-44dd-a0ca-66e58177d8cc + plugin: stockpile + tags: [] + auto_close: false + autonomous: 1 + group: "" + id: ${{ FN.uuid4() }} + jitter: 2/8 + name: Operation Thief - ${{ ACTIONS.generate_uuid.result }} + obfuscator: plain-text + objective: + description: string + goals: + - count: 0 + operator: string + target: string + value: string + id: string + name: string + planner: + allow_repeatable_abilities: true + description: string + id: aaa7c857-37a0-4c4a-85f7-4e9f7f30e31a + ignore_enforcement_modules: - string - score: 0 - source: string - technique_id: string - trait: string - value: string - target: - collected_by: - - string - limit_count: 0 - links: - - string - origin_type: string - relationships: - - string - score: 0 - source: string - technique_id: string - trait: string - value: string - rules: - - action: DENY - match: .* - trait: file.sensitive.extension - - action: ALLOW - match: png - trait: file.sensitive.extension - - action: ALLOW - match: yml - trait: file.sensitive.extension - - action: ALLOW - match: wav - trait: file.sensitive.extension - state: running - use_learning_parsers: true - visibility: 51 - url: ${{ INPUTS.caldera_api_url }}/operations - verify_ssl: false - depends_on: - - operation_discovery - description: '' - for_each: null - ref: operation_thief - retry_policy: - max_attempts: 1 - timeout: 300 - run_if: null - start_delay: 10.0 - # Execute the Alice 2.0 operation - - action: core.http_request - args: - headers: - KEY: ${{ SECRETS.caldera.API_KEY }} - method: POST - payload: - adversary: - adversary_id: 50855e29-3b4e-4562-aa55-b3d7f93c26b8 - atomic_ordering: - - 13379ae1-d20e-4162-91f8-320d78a35e7f - - 7049e3ec-b822-4fdf-a4ac-18190f9b66d1 - - 14a21534-350f-4d83-9dd7-3c56b93a0c17 - - 6d90e6fa-9324-4eb5-93be-9f737245bd7z - - 2afae782-6d0a-4fbd-a6b6-d1ce90090eac - - 921055f4-5970-4707-909e-62f594234d91 - - aa6ec4dd-db09-4925-b9b9-43adeb154686 - - 65048ec1-f7ca-49d3-9410-10813e472b30 - - ece5dde3-d370-4c20-b213-a1f424aa8d03 - description: Adversary used for demoing restricted lateral movement - name: Alice 2.0 - objective: 495a9828-cab1-44dd-a0ca-66e58177d8cc - plugin: stockpile - tags: [] - auto_close: false - autonomous: 1 - group: '' - id: ${{ FN.uuid4() }} - jitter: 2/8 - name: Operation Alice 2.0 - ${{ ACTIONS.generate_uuid.result }} - obfuscator: plain-text - objective: - description: string - goals: - - count: 0 - operator: string - target: string - value: string - id: string - name: string - planner: - allow_repeatable_abilities: true - description: string - id: aaa7c857-37a0-4c4a-85f7-4e9f7f30e31a - ignore_enforcement_modules: - - string - module: app.planners.atomic - name: atomic - params: {} - plugin: null - stopping_conditions: [] - source: - adjustments: - - ability_id: string - offset: 0 - trait: string - value: string - facts: - - collected_by: - - string - limit_count: 0 - links: - - string - origin_type: string + module: app.planners.atomic + name: atomic + params: {} + plugin: null + stopping_conditions: [] + source: + adjustments: + - ability_id: string + offset: 0 + trait: string + value: string + facts: + - collected_by: + - string + limit_count: 0 + links: + - string + origin_type: string + relationships: + - string + score: 0 + source: string + technique_id: string + trait: string + value: string + id: ed32b9c3-9593-4c33-b0db-e2007315096b + name: basic + plugin: null relationships: - - string - score: 0 - source: string - technique_id: string - trait: string - value: string - id: ed32b9c3-9593-4c33-b0db-e2007315096b - name: basic - plugin: null - relationships: - - edge: string - origin: string - score: 0 - source: - collected_by: - - string - limit_count: 0 - links: - - string - origin_type: string - relationships: + - edge: string + origin: string + score: 0 + source: + collected_by: + - string + limit_count: 0 + links: + - string + origin_type: string + relationships: + - string + score: 0 + source: string + technique_id: string + trait: string + value: string + target: + collected_by: + - string + limit_count: 0 + links: + - string + origin_type: string + relationships: + - string + score: 0 + source: string + technique_id: string + trait: string + value: string + rules: + - action: DENY + match: .* + trait: file.sensitive.extension + - action: ALLOW + match: png + trait: file.sensitive.extension + - action: ALLOW + match: yml + trait: file.sensitive.extension + - action: ALLOW + match: wav + trait: file.sensitive.extension + state: running + use_learning_parsers: true + visibility: 51 + url: ${{ VARS.caldera_api_url }}/operations + verify_ssl: false + depends_on: + - operation_discovery + description: "" + for_each: null + ref: operation_thief + retry_policy: + max_attempts: 1 + timeout: 300 + run_if: null + start_delay: 10.0 + # Execute the Alice 2.0 operation + - action: core.http_request + args: + headers: + KEY: ${{ SECRETS.caldera.API_KEY }} + method: POST + payload: + adversary: + adversary_id: 50855e29-3b4e-4562-aa55-b3d7f93c26b8 + atomic_ordering: + - 13379ae1-d20e-4162-91f8-320d78a35e7f + - 7049e3ec-b822-4fdf-a4ac-18190f9b66d1 + - 14a21534-350f-4d83-9dd7-3c56b93a0c17 + - 6d90e6fa-9324-4eb5-93be-9f737245bd7z + - 2afae782-6d0a-4fbd-a6b6-d1ce90090eac + - 921055f4-5970-4707-909e-62f594234d91 + - aa6ec4dd-db09-4925-b9b9-43adeb154686 + - 65048ec1-f7ca-49d3-9410-10813e472b30 + - ece5dde3-d370-4c20-b213-a1f424aa8d03 + description: Adversary used for demoing restricted lateral movement + name: Alice 2.0 + objective: 495a9828-cab1-44dd-a0ca-66e58177d8cc + plugin: stockpile + tags: [] + auto_close: false + autonomous: 1 + group: "" + id: ${{ FN.uuid4() }} + jitter: 2/8 + name: Operation Alice 2.0 - ${{ ACTIONS.generate_uuid.result }} + obfuscator: plain-text + objective: + description: string + goals: + - count: 0 + operator: string + target: string + value: string + id: string + name: string + planner: + allow_repeatable_abilities: true + description: string + id: aaa7c857-37a0-4c4a-85f7-4e9f7f30e31a + ignore_enforcement_modules: - string - score: 0 - source: string - technique_id: string - trait: string - value: string - target: - collected_by: - - string - limit_count: 0 - links: - - string - origin_type: string - relationships: - - string - score: 0 - source: string - technique_id: string - trait: string - value: string - rules: - - action: DENY - match: .* - trait: file.sensitive.extension - - action: ALLOW - match: png - trait: file.sensitive.extension - - action: ALLOW - match: yml - trait: file.sensitive.extension - - action: ALLOW - match: wav - trait: file.sensitive.extension - state: running - use_learning_parsers: true - visibility: 51 - url: ${{ INPUTS.caldera_api_url }}/operations - verify_ssl: false - depends_on: - - operation_thief - description: '' - for_each: null - ref: operation_alice_2_0 - retry_policy: - max_attempts: 1 - timeout: 300 - run_if: null - start_delay: 10.0 + module: app.planners.atomic + name: atomic + params: {} + plugin: null + stopping_conditions: [] + source: + adjustments: + - ability_id: string + offset: 0 + trait: string + value: string + facts: + - collected_by: + - string + limit_count: 0 + links: + - string + origin_type: string + relationships: + - string + score: 0 + source: string + technique_id: string + trait: string + value: string + id: ed32b9c3-9593-4c33-b0db-e2007315096b + name: basic + plugin: null + relationships: + - edge: string + origin: string + score: 0 + source: + collected_by: + - string + limit_count: 0 + links: + - string + origin_type: string + relationships: + - string + score: 0 + source: string + technique_id: string + trait: string + value: string + target: + collected_by: + - string + limit_count: 0 + links: + - string + origin_type: string + relationships: + - string + score: 0 + source: string + technique_id: string + trait: string + value: string + rules: + - action: DENY + match: .* + trait: file.sensitive.extension + - action: ALLOW + match: png + trait: file.sensitive.extension + - action: ALLOW + match: yml + trait: file.sensitive.extension + - action: ALLOW + match: wav + trait: file.sensitive.extension + state: running + use_learning_parsers: true + visibility: 51 + url: ${{ VARS.caldera_api_url }}/operations + verify_ssl: false + depends_on: + - operation_thief + description: "" + for_each: null + ref: operation_alice_2_0 + retry_policy: + max_attempts: 1 + timeout: 300 + run_if: null + start_delay: 10.0 diff --git a/playbooks/tutorials/limacharlie/list_tags.yml b/playbooks/tutorials/limacharlie/list_tags.yml index befe39557..5d5ad1516 100644 --- a/playbooks/tutorials/limacharlie/list_tags.yml +++ b/playbooks/tutorials/limacharlie/list_tags.yml @@ -5,7 +5,7 @@ definition: Requires secret named `limacharlie` with keys `LIMACHARLIE_API_SECRET` and `LIMACHARLIE_OID` entrypoint: ref: list_tags - inputs: + variables: api_url: https://api.limacharlie.io/v1 jwt_url: https://jwt.limacharlie.io token_response_key: token @@ -17,9 +17,9 @@ definition: - action: core.http_request ref: list_tags args: - url: "${{ INPUTS.api_url }}/tags/${{ SECRETS.limacharlie.LIMACHARLIE_OID }}" - token_response_key: ${{ INPUTS.token_response_key }} - jwt_url: "${{ INPUTS.jwt_url }}?oid=${{ SECRETS.limacharlie.LIMACHARLIE_OID }}&secret=${{ SECRETS.limacharlie.LIMACHARLIE_API_SECRET }}" + url: "${{ VARS.api_url }}/tags/${{ SECRETS.limacharlie.LIMACHARLIE_OID }}" + token_response_key: ${{ VARS.token_response_key }} + jwt_url: "${{ VARS.jwt_url }}?oid=${{ SECRETS.limacharlie.LIMACHARLIE_OID }}&secret=${{ SECRETS.limacharlie.LIMACHARLIE_API_SECRET }}" method: GET - action: core.http_request @@ -27,7 +27,7 @@ definition: depends_on: - list_tags args: - url: "${{ INPUTS.api_url }}/sensors/${{ SECRETS.limacharlie.LIMACHARLIE_OID }}" - token_response_key: ${{ INPUTS.token_response_key }} - jwt_url: "${{ INPUTS.jwt_url }}?oid=${{ SECRETS.limacharlie.LIMACHARLIE_OID }}&secret=${{ SECRETS.limacharlie.LIMACHARLIE_API_SECRET }}" + url: "${{ VARS.api_url }}/sensors/${{ SECRETS.limacharlie.LIMACHARLIE_OID }}" + token_response_key: ${{ VARS.token_response_key }} + jwt_url: "${{ VARS.jwt_url }}?oid=${{ SECRETS.limacharlie.LIMACHARLIE_OID }}&secret=${{ SECRETS.limacharlie.LIMACHARLIE_API_SECRET }}" method: GET diff --git a/playbooks/tutorials/limacharlie/run_investigation.yml b/playbooks/tutorials/limacharlie/run_investigation.yml index da1897395..9e69fa901 100644 --- a/playbooks/tutorials/limacharlie/run_investigation.yml +++ b/playbooks/tutorials/limacharlie/run_investigation.yml @@ -13,7 +13,7 @@ definition: type: list[str] description: List of Atomic Red Team test IDs to run. ref: list_sensors_by_tags - inputs: + variables: api_url: https://api.limacharlie.io/v1 jwt_url: https://jwt.limacharlie.io token_response_key: token @@ -25,10 +25,10 @@ definition: - action: core.http_request ref: list_sensors_by_tags args: - url: "{{ INPUTS.api_url }}/tags/{{ SECRETS.limacharlie.LIMACHARLIE_OID }}/{ TRIGGER.tags }" + url: "{{ VARS.api_url }}/tags/{{ SECRETS.limacharlie.LIMACHARLIE_OID }}/{ TRIGGER.tags }" method: GET - token_response_key: ${{ INPUTS.token_response_key }} - jwt_url: "{{ INPUTS.jwt_url }}?oid={{ SECRETS.limacharlie.LIMACHARLIE_OID }}&secret={{ SECRETS.limacharlie.LIMACHARLIE_API_SECRET }}" + token_response_key: ${{ VARS.token_response_key }} + jwt_url: "{{ VARS.jwt_url }}?oid={{ SECRETS.limacharlie.LIMACHARLIE_OID }}&secret={{ SECRETS.limacharlie.LIMACHARLIE_API_SECRET }}" - action: core.transform.reshape ref: extract_sensor_ids @@ -44,13 +44,13 @@ definition: - extract_sensor_ids for_each: ${{ for var.sensor_id in ACTIONS.extract_sensor_ids.result.sensor_ids }} args: - url: "${{ INPUTS.api_url }}/${{ var.sensor_id }}/" + url: "${{ VARS.api_url }}/${{ var.sensor_id }}/" method: GET - token_response_key: ${{ INPUTS.token_response_key }} - jwt_url: "${{ INPUTS.jwt_url }}?oid=${{ SECRETS.limacharlie.LIMACHARLIE_OID }}&secret=${{ SECRETS.limacharlie.LIMACHARLIE_API_SECRET }}" + token_response_key: ${{ VARS.token_response_key }} + jwt_url: "${{ VARS.jwt_url }}?oid=${{ SECRETS.limacharlie.LIMACHARLIE_OID }}&secret=${{ SECRETS.limacharlie.LIMACHARLIE_API_SECRET }}" payload: tasks: - - "run --shell-command 'del ${{ INPUTS.payload_dir_path }}\\*.exe'" + - "run --shell-command 'del ${{ VARS.payload_dir_path }}\\*.exe'" headers: Content-Type: application/x-www-form-urlencoded @@ -60,12 +60,12 @@ definition: - erase_past_executables for_each: ${{ for var.sensor_id in ACTIONS.extract_sensor_ids.result.sensor_ids }} args: - workflow_id: ${{ INPUTS.child_workflow_id }} + workflow_id: ${{ VARS.child_workflow_id }} loop_strategy: parallel fail_strategy: isolated - batch_size: ${{ INPUTS.batch_size }} + batch_size: ${{ VARS.batch_size }} trigger_inputs: sensor_id: ${{ var.sensor_id }} investigation_id: ${{ FN.uuid4() }} test_ids: ${{ TRIGGER.test_ids }} - payload_dir_path: ${{ INPUTS.payload_dir_path }} + payload_dir_path: ${{ VARS.payload_dir_path }} diff --git a/playbooks/tutorials/limacharlie/run_tests.yml b/playbooks/tutorials/limacharlie/run_tests.yml index c9e98ac41..b38703af1 100644 --- a/playbooks/tutorials/limacharlie/run_tests.yml +++ b/playbooks/tutorials/limacharlie/run_tests.yml @@ -18,7 +18,7 @@ definition: type: str description: The directory path where the payloads are stored. ref: upload_emulation_tests - inputs: + variables: api_url: https://api.limacharlie.io/v1 jwt_url: https://jwt.limacharlie.io token_response_key: token @@ -30,10 +30,10 @@ definition: ref: upload_emulation_tests for_each: ${{ for var.test_id in TRIGGER.test_ids }} args: - url: "${{ INPUTS.api_url }}/${{ TRIGGER.sensor_id }}/" + url: "${{ VARS.api_url }}/${{ TRIGGER.sensor_id }}/" method: GET - token_response_key: ${{ INPUTS.token_response_key }} - jwt_url: "${{ INPUTS.jwt_url }}?oid=${{ SECRETS.limacharlie.LIMACHARLIE_OID }}&secret=${{ SECRETS.limacharlie.LIMACHARLIE_API_SECRET }}" + token_response_key: ${{ VARS.token_response_key }} + jwt_url: "${{ VARS.jwt_url }}?oid=${{ SECRETS.limacharlie.LIMACHARLIE_OID }}&secret=${{ SECRETS.limacharlie.LIMACHARLIE_API_SECRET }}" payload: tasks: - "put --payload-name ${{ var.test_id }} --payload-path ${{ TRIGGER.payload_dir_path }}\\${{ var.test_id }}.exe" @@ -47,10 +47,10 @@ definition: - upload_emulation_tests for_each: ${{ for var.test_id in TRIGGER.test_ids }} args: - url: "${{ INPUTS.api_url }}/${{ TRIGGER.sensor_id }}/" + url: "${{ VARS.api_url }}/${{ TRIGGER.sensor_id }}/" method: GET - token_response_key: ${{ INPUTS.token_response_key }} - jwt_url: "${{ INPUTS.jwt_url }}?oid=${{ SECRETS.limacharlie.LIMACHARLIE_OID }}&secret=${{ SECRETS.limacharlie.LIMACHARLIE_API_SECRET }}" + token_response_key: ${{ VARS.token_response_key }} + jwt_url: "${{ VARS.jwt_url }}?oid=${{ SECRETS.limacharlie.LIMACHARLIE_OID }}&secret=${{ SECRETS.limacharlie.LIMACHARLIE_API_SECRET }}" payload: tasks: - "run --shell-command '${{ TRIGGER.payload_dir_path }}\\${{ var.test_id }}.exe'" @@ -64,8 +64,8 @@ definition: - run_emulation_tests args: value: - start: ${{ FN.to_datestring(ENV.workflow.start_time, INPUTS.datetime_format) }} - end: ${{ FN.to_datestring(FN.now(), INPUTS.datetime_format) }} + start: ${{ FN.to_datestring(ENV.workflow.start_time, VARS.datetime_format) }} + end: ${{ FN.to_datestring(FN.now(), VARS.datetime_format) }} - action: core.http_request ref: list_emulation_results @@ -74,11 +74,11 @@ definition: # Filter for results between start of workflow and now args: url: | - ${{ INPUTS.api_url }}/v1/insight/${{ ACTIONS.emulation_times.result.start }}/ + ${{ VARS.api_url }}/v1/insight/${{ ACTIONS.emulation_times.result.start }}/ ${{ TRIGGER.sensor_id }}? start=${{ ACTIONS.emulation_times.result.start }}& end=${{ ACTIONS.emulation_times.result.end }}& - event_type=${{ INPUTS.event_type }} + event_type=${{ VARS.event_type }} method: GET # TODO: Upload results to records diff --git a/playbooks/tutorials/virustotal_quickstart.yml b/playbooks/tutorials/virustotal_quickstart.yml index ec6e9f990..43589bb2d 100644 --- a/playbooks/tutorials/virustotal_quickstart.yml +++ b/playbooks/tutorials/virustotal_quickstart.yml @@ -14,7 +14,7 @@ definition: entrypoint: expects: {} ref: search_url - inputs: {} + variables: {} returns: null tests: [] title: Test virustotal diff --git a/tests/data/workflows/integration_core_ai_action.yml b/tests/data/workflows/integration_core_ai_action.yml index d00cbb45e..89f1caa33 100644 --- a/tests/data/workflows/integration_core_ai_action.yml +++ b/tests/data/workflows/integration_core_ai_action.yml @@ -4,11 +4,11 @@ description: | Requires `openai.OPENAI_API_KEY` secret. entrypoint: ref: tell_me_something_intereesting -inputs: +variables: prompt: "Please tell me something interesting" actions: - ref: tell_me_something_intereesting action: core.ai_action args: - prompt: ${{ INPUTS.prompt }} + prompt: ${{ VARS.prompt }} diff --git a/tests/data/workflows/integration_io_linear_http.yml b/tests/data/workflows/integration_io_linear_http.yml index fc56b6905..19546d249 100644 --- a/tests/data/workflows/integration_io_linear_http.yml +++ b/tests/data/workflows/integration_io_linear_http.yml @@ -4,7 +4,7 @@ config: scheduler: dynamic entrypoint: ref: pull_aws_guardduty_findings -inputs: +variables: uim_url: http://host.docker.internal:8005 num_days: 1 @@ -21,7 +21,7 @@ actions: action: core.http_request version: "0.1.0" args: - url: ${{ INPUTS.uim_url }}/cdr/alerts + url: ${{ VARS.uim_url }}/cdr/alerts method: GET params: start_date: "2024-05-26T01:33:58.799180Z" @@ -34,7 +34,7 @@ actions: depends_on: - pull_aws_guardduty_findings args: - url: ${{ INPUTS.uim_url }}/notifications + url: ${{ VARS.uim_url }}/notifications headers: Content-Type: application/json method: POST diff --git a/tests/data/workflows/shared_adder_tree.yml b/tests/data/workflows/shared_adder_tree.yml index 4ab85e028..72a33a221 100644 --- a/tests/data/workflows/shared_adder_tree.yml +++ b/tests/data/workflows/shared_adder_tree.yml @@ -9,7 +9,7 @@ config: scheduler: dynamic entrypoint: ref: a -inputs: +variables: another_url: "http://api:8000" value: 1 diff --git a/tests/data/workflows/shared_kite.yml b/tests/data/workflows/shared_kite.yml index 10aa82018..28cda9a47 100644 --- a/tests/data/workflows/shared_kite.yml +++ b/tests/data/workflows/shared_kite.yml @@ -6,7 +6,7 @@ config: scheduler: dynamic entrypoint: ref: a -inputs: +variables: another_url: "http://api:8000" value: 1 diff --git a/tests/data/workflows/shared_tree.yml b/tests/data/workflows/shared_tree.yml index 3a3ff854c..3c7f3f502 100644 --- a/tests/data/workflows/shared_tree.yml +++ b/tests/data/workflows/shared_tree.yml @@ -9,7 +9,7 @@ config: scheduler: dynamic entrypoint: ref: a -inputs: +variables: another_url: "http://api:8000" value: 1 diff --git a/tests/data/workflows/stress_adder_tree.yml b/tests/data/workflows/stress_adder_tree.yml index a61dc2018..2f1ee6b3e 100644 --- a/tests/data/workflows/stress_adder_tree.yml +++ b/tests/data/workflows/stress_adder_tree.yml @@ -9,7 +9,7 @@ config: scheduler: dynamic entrypoint: ref: a -inputs: +variables: value: 1 triggers: @@ -21,7 +21,7 @@ actions: - ref: a action: core.transform.reshape args: - value: ${{ INPUTS.value }} + value: ${{ VARS.value }} - ref: b action: example.add diff --git a/tests/data/workflows/unit_child_workflow_parent.yml b/tests/data/workflows/unit_child_workflow_parent.yml index 2b02e4bea..13e1f4a3f 100644 --- a/tests/data/workflows/unit_child_workflow_parent.yml +++ b/tests/data/workflows/unit_child_workflow_parent.yml @@ -2,12 +2,12 @@ title: Parent workflow description: Test child workflow entrypoint: ref: parent -inputs: +variables: data: [1, 2, 3, 4, 5, 6, 7] actions: - ref: parent - for_each: ${{ for var.x in INPUTS.data }} + for_each: ${{ for var.x in VARS.data }} action: core.workflow.execute args: workflow_id: <> diff --git a/tests/data/workflows/unit_child_workflow_parent_expected.yml b/tests/data/workflows/unit_child_workflow_parent_expected.yml index ff5624765..5f58af499 100644 --- a/tests/data/workflows/unit_child_workflow_parent_expected.yml +++ b/tests/data/workflows/unit_child_workflow_parent_expected.yml @@ -3,6 +3,6 @@ ACTIONS: result: [1001, 1002, 1003, 1004, 1005, 1006, 1007] result_typename: "list" -INPUTS: +VARS: data: [1, 2, 3, 4, 5, 6, 7] TRIGGER: diff --git a/tests/data/workflows/unit_conditional_adder_diamond_skip_with_join_strong_dep_fails.yml b/tests/data/workflows/unit_conditional_adder_diamond_skip_with_join_strong_dep_fails.yml index 7aa209e25..424e3bf74 100644 --- a/tests/data/workflows/unit_conditional_adder_diamond_skip_with_join_strong_dep_fails.yml +++ b/tests/data/workflows/unit_conditional_adder_diamond_skip_with_join_strong_dep_fails.yml @@ -17,7 +17,7 @@ config: scheduler: dynamic entrypoint: ref: a -inputs: +variables: another_url: http://api:8000 value: 1 diff --git a/tests/data/workflows/unit_conditional_adder_diamond_skip_with_join_weak_dep.yml b/tests/data/workflows/unit_conditional_adder_diamond_skip_with_join_weak_dep.yml index f78222e0a..3c3732c47 100644 --- a/tests/data/workflows/unit_conditional_adder_diamond_skip_with_join_weak_dep.yml +++ b/tests/data/workflows/unit_conditional_adder_diamond_skip_with_join_weak_dep.yml @@ -17,7 +17,7 @@ config: scheduler: dynamic entrypoint: ref: a -inputs: +variables: another_url: http://api:8000 value: 1 diff --git a/tests/data/workflows/unit_conditional_adder_diamond_skip_with_join_weak_dep_expected.yml b/tests/data/workflows/unit_conditional_adder_diamond_skip_with_join_weak_dep_expected.yml index 5c5da6968..97b765ea6 100644 --- a/tests/data/workflows/unit_conditional_adder_diamond_skip_with_join_weak_dep_expected.yml +++ b/tests/data/workflows/unit_conditional_adder_diamond_skip_with_join_weak_dep_expected.yml @@ -8,7 +8,7 @@ ACTIONS: d: result: 6 result_typename: "int" -INPUTS: +VARS: another_url: "http://api:8000" value: 1 TRIGGER: diff --git a/tests/data/workflows/unit_conditional_adder_tree_continues.yml b/tests/data/workflows/unit_conditional_adder_tree_continues.yml index 8e72e27ac..6deab6619 100644 --- a/tests/data/workflows/unit_conditional_adder_tree_continues.yml +++ b/tests/data/workflows/unit_conditional_adder_tree_continues.yml @@ -7,7 +7,7 @@ config: scheduler: dynamic entrypoint: ref: a -inputs: +variables: another_url: http://api:8000 value: 1 diff --git a/tests/data/workflows/unit_conditional_adder_tree_continues_expected.yml b/tests/data/workflows/unit_conditional_adder_tree_continues_expected.yml index a0a314473..d0eb68b2e 100644 --- a/tests/data/workflows/unit_conditional_adder_tree_continues_expected.yml +++ b/tests/data/workflows/unit_conditional_adder_tree_continues_expected.yml @@ -9,7 +9,7 @@ ACTIONS: result: 4 result_typename: "int" -INPUTS: +VARS: another_url: http://api:8000 value: 1 TRIGGER: diff --git a/tests/data/workflows/unit_conditional_adder_tree_skip_propagates.yml b/tests/data/workflows/unit_conditional_adder_tree_skip_propagates.yml index a851151a0..274749502 100644 --- a/tests/data/workflows/unit_conditional_adder_tree_skip_propagates.yml +++ b/tests/data/workflows/unit_conditional_adder_tree_skip_propagates.yml @@ -9,7 +9,7 @@ config: scheduler: dynamic entrypoint: ref: a -inputs: +variables: another_url: http://api:8000 value: 1 diff --git a/tests/data/workflows/unit_conditional_adder_tree_skip_propagates_expected.yml b/tests/data/workflows/unit_conditional_adder_tree_skip_propagates_expected.yml index 4c0d8a630..a5eb72223 100644 --- a/tests/data/workflows/unit_conditional_adder_tree_skip_propagates_expected.yml +++ b/tests/data/workflows/unit_conditional_adder_tree_skip_propagates_expected.yml @@ -5,7 +5,7 @@ ACTIONS: b: result: 2 result_typename: "int" -INPUTS: +VARS: another_url: "http://api:8000" value: 1 TRIGGER: diff --git a/tests/data/workflows/unit_conditional_adder_tree_skips.yml b/tests/data/workflows/unit_conditional_adder_tree_skips.yml index d61558a46..e64b7d779 100644 --- a/tests/data/workflows/unit_conditional_adder_tree_skips.yml +++ b/tests/data/workflows/unit_conditional_adder_tree_skips.yml @@ -7,7 +7,7 @@ config: scheduler: dynamic entrypoint: ref: a -inputs: +variables: another_url: http://api:8000 value: 1 diff --git a/tests/data/workflows/unit_conditional_adder_tree_skips_expected.yml b/tests/data/workflows/unit_conditional_adder_tree_skips_expected.yml index 4c0d8a630..a5eb72223 100644 --- a/tests/data/workflows/unit_conditional_adder_tree_skips_expected.yml +++ b/tests/data/workflows/unit_conditional_adder_tree_skips_expected.yml @@ -5,7 +5,7 @@ ACTIONS: b: result: 2 result_typename: "int" -INPUTS: +VARS: another_url: "http://api:8000" value: 1 TRIGGER: diff --git a/tests/data/workflows/unit_error_fatal.yml b/tests/data/workflows/unit_error_fatal.yml index 95bf67c4a..49a5d9c96 100644 --- a/tests/data/workflows/unit_error_fatal.yml +++ b/tests/data/workflows/unit_error_fatal.yml @@ -8,7 +8,7 @@ config: entrypoint: ref: a -inputs: +variables: valid: 1 value: another_valid: 2 @@ -23,7 +23,7 @@ actions: action: core.transform.reshape args: value: - valid: ${{ INPUTS.valid }} + valid: ${{ VARS.valid }} - ref: b action: example.add @@ -32,4 +32,4 @@ actions: args: # This should fail lhs: ${{ ACTIONS.a.result.invalid }} - rhs: ${{ INPUTS.value.another_valid }} + rhs: ${{ VARS.value.another_valid }} diff --git a/tests/data/workflows/unit_secrets.yml b/tests/data/workflows/unit_secrets.yml index fad7d4cab..438550b62 100644 --- a/tests/data/workflows/unit_secrets.yml +++ b/tests/data/workflows/unit_secrets.yml @@ -6,7 +6,7 @@ config: scheduler: dynamic entrypoint: ref: a -inputs: +variables: another_url: "http://api:8000" value: 1 diff --git a/tests/data/workflows/unit_transform_filter_dict.yml b/tests/data/workflows/unit_transform_filter_dict.yml index fd1144053..8160cead8 100644 --- a/tests/data/workflows/unit_transform_filter_dict.yml +++ b/tests/data/workflows/unit_transform_filter_dict.yml @@ -2,7 +2,7 @@ title: Filter data description: Test to filter data entrypoint: ref: filter_empty -inputs: #2D list +variables: #2D list matrix: - signal: a data: [1, 2, 3] @@ -22,7 +22,7 @@ actions: action: core.transform.filter args: # List of objects to filter - items: ${{ INPUTS.matrix }} + items: ${{ VARS.matrix }} constraint: jsonpath: $.data function: not_empty diff --git a/tests/data/workflows/unit_transform_filter_dict_expected.yml b/tests/data/workflows/unit_transform_filter_dict_expected.yml index 7cd243dd7..8b061047c 100644 --- a/tests/data/workflows/unit_transform_filter_dict_expected.yml +++ b/tests/data/workflows/unit_transform_filter_dict_expected.yml @@ -11,7 +11,7 @@ ACTIONS: data: [10, 11, 12] result_typename: list -INPUTS: +VARS: matrix: - signal: a data: [1, 2, 3] diff --git a/tests/data/workflows/unit_transform_filter_function.yml b/tests/data/workflows/unit_transform_filter_function.yml index aa4275a8b..fe6b1cabc 100644 --- a/tests/data/workflows/unit_transform_filter_function.yml +++ b/tests/data/workflows/unit_transform_filter_function.yml @@ -2,7 +2,7 @@ title: Filter data inline description: Test to filter data inline entrypoint: ref: filter_empty -inputs: #2D list +variables: #2D list matrix: - [1, 2, 3] - [4, 5, 6] @@ -15,4 +15,4 @@ actions: - ref: filter_empty action: core.transform.reshape args: - value: ${{ FN.filter(INPUTS.matrix, 'len(x) > 0') }} + value: ${{ FN.filter(VARS.matrix, 'len(x) > 0') }} diff --git a/tests/data/workflows/unit_transform_filter_function_expected.yml b/tests/data/workflows/unit_transform_filter_function_expected.yml index 96d1da639..d9db1fbc7 100644 --- a/tests/data/workflows/unit_transform_filter_function_expected.yml +++ b/tests/data/workflows/unit_transform_filter_function_expected.yml @@ -3,7 +3,7 @@ ACTIONS: result: [[1, 2, 3], [4, 5, 6], [7, 8, 9], [10, 11, 12]] result_typename: list -INPUTS: +VARS: matrix: - [1, 2, 3] - [4, 5, 6] diff --git a/tests/data/workflows/unit_transform_reshape_arrange.yml b/tests/data/workflows/unit_transform_reshape_arrange.yml index 91bb9abde..427d43b61 100644 --- a/tests/data/workflows/unit_transform_reshape_arrange.yml +++ b/tests/data/workflows/unit_transform_reshape_arrange.yml @@ -2,7 +2,7 @@ title: Reshape data description: Test reshape data entrypoint: ref: flatten -inputs: +variables: details: name: John Doe age: 30 @@ -23,12 +23,12 @@ actions: # Flatten the mapping into a list of key-value pairs args: value: - name: ${{ INPUTS.details.name }} - age: ${{ INPUTS.details.age }} - job_title: ${{ INPUTS.details.job.title }} - job_company: ${{ INPUTS.details.job.company }} - job_city: ${{ INPUTS.details.job.city }} - residence_street_address: ${{ INPUTS.details.residence.street_address }} - residence_city: ${{ INPUTS.details.residence.city }} - contact_email: ${{ INPUTS.details.contact.email }} - contact_phone: ${{ INPUTS.details.contact.phone }} + name: ${{ VARS.details.name }} + age: ${{ VARS.details.age }} + job_title: ${{ VARS.details.job.title }} + job_company: ${{ VARS.details.job.company }} + job_city: ${{ VARS.details.job.city }} + residence_street_address: ${{ VARS.details.residence.street_address }} + residence_city: ${{ VARS.details.residence.city }} + contact_email: ${{ VARS.details.contact.email }} + contact_phone: ${{ VARS.details.contact.phone }} diff --git a/tests/data/workflows/unit_transform_reshape_arrange_expected.yml b/tests/data/workflows/unit_transform_reshape_arrange_expected.yml index 5aa0b0382..7909ce96f 100644 --- a/tests/data/workflows/unit_transform_reshape_arrange_expected.yml +++ b/tests/data/workflows/unit_transform_reshape_arrange_expected.yml @@ -12,7 +12,7 @@ ACTIONS: contact_phone: 123-456-7890 result_typename: "dict" -INPUTS: +VARS: details: name: John Doe age: 30 diff --git a/tests/data/workflows/unit_transform_reshape_arrange_loop.yml b/tests/data/workflows/unit_transform_reshape_arrange_loop.yml index a9b79acdc..fe62ec70a 100644 --- a/tests/data/workflows/unit_transform_reshape_arrange_loop.yml +++ b/tests/data/workflows/unit_transform_reshape_arrange_loop.yml @@ -2,7 +2,7 @@ title: Reshape data in a loop description: Test reshaping data from a list of mappings to a list of key-value pairs entrypoint: ref: flatten -inputs: +variables: contacts: - name: John Doe age: 30 @@ -44,7 +44,7 @@ inputs: actions: - ref: flatten action: core.transform.reshape - for_each: ${{ for var.person in INPUTS.contacts }} + for_each: ${{ for var.person in VARS.contacts }} # Flatten the mapping into a list of key-value pairs args: value: diff --git a/tests/data/workflows/unit_transform_reshape_arrange_loop_expected.yml b/tests/data/workflows/unit_transform_reshape_arrange_loop_expected.yml index 17e786fd7..a6792ea8e 100644 --- a/tests/data/workflows/unit_transform_reshape_arrange_loop_expected.yml +++ b/tests/data/workflows/unit_transform_reshape_arrange_loop_expected.yml @@ -30,7 +30,7 @@ ACTIONS: contact_phone: 456-789-0123 result_typename: "list" -INPUTS: +VARS: contacts: - name: John Doe age: 30 diff --git a/tests/data/workflows/unit_transform_reshape_loop.yml b/tests/data/workflows/unit_transform_reshape_loop.yml index fecb492f2..c127a36fd 100644 --- a/tests/data/workflows/unit_transform_reshape_loop.yml +++ b/tests/data/workflows/unit_transform_reshape_loop.yml @@ -2,7 +2,7 @@ title: Reshape loop description: Test that we can loop reshape actions entrypoint: ref: a -inputs: +variables: first: 1 second: 2 list: [1, 2, 3] @@ -10,6 +10,6 @@ inputs: actions: - ref: a action: core.transform.reshape - for_each: ${{ for var.x in INPUTS.list }} + for_each: ${{ for var.x in VARS.list }} args: value: I received ${{ var.x }} from you diff --git a/tests/data/workflows/unit_transform_reshape_loop_chained.yml b/tests/data/workflows/unit_transform_reshape_loop_chained.yml index fb73cf296..54930a544 100644 --- a/tests/data/workflows/unit_transform_reshape_loop_chained.yml +++ b/tests/data/workflows/unit_transform_reshape_loop_chained.yml @@ -2,7 +2,7 @@ title: Chained reshape loop description: Test that we can chain reshaping for loops back to back entrypoint: ref: a -inputs: +variables: first: 1 second: 2 list: [1, 2, 3] @@ -10,13 +10,13 @@ inputs: actions: - ref: a action: core.transform.reshape - for_each: ${{ for var.x in INPUTS.list }} + for_each: ${{ for var.x in VARS.list }} args: - value: ${{ FN.add(var.x, INPUTS.first) }} + value: ${{ FN.add(var.x, VARS.first) }} - ref: b action: core.transform.reshape depends_on: - a for_each: ${{ for var.x in ACTIONS.a.result }} args: - value: ${{ FN.add(INPUTS.second, var.x) }} + value: ${{ FN.add(VARS.second, var.x) }} diff --git a/tests/data/workflows/unit_transform_reshape_loop_chained_expected.yml b/tests/data/workflows/unit_transform_reshape_loop_chained_expected.yml index 0720aef0f..d81164441 100644 --- a/tests/data/workflows/unit_transform_reshape_loop_chained_expected.yml +++ b/tests/data/workflows/unit_transform_reshape_loop_chained_expected.yml @@ -12,7 +12,7 @@ ACTIONS: - 6 result_typename: "list" -INPUTS: +VARS: first: 1 second: 2 list: [1, 2, 3] diff --git a/tests/data/workflows/unit_transform_reshape_loop_expected.yml b/tests/data/workflows/unit_transform_reshape_loop_expected.yml index e6d434024..3e8851970 100644 --- a/tests/data/workflows/unit_transform_reshape_loop_expected.yml +++ b/tests/data/workflows/unit_transform_reshape_loop_expected.yml @@ -6,7 +6,7 @@ ACTIONS: - I received 3 from you result_typename: "list" -INPUTS: +VARS: first: 1 second: 2 list: [1, 2, 3] diff --git a/tests/data/workflows/unit_transform_reshape_map_loop.yml b/tests/data/workflows/unit_transform_reshape_map_loop.yml index 88ef6e804..26fef0df1 100644 --- a/tests/data/workflows/unit_transform_reshape_map_loop.yml +++ b/tests/data/workflows/unit_transform_reshape_map_loop.yml @@ -2,7 +2,7 @@ title: Reshape with map in loop description: Test that we can loop and map inside entrypoint: ref: a -inputs: +variables: nested: - ["user_1", "user_2", "user_3"] - ["user_4", "user_5", "user_6"] @@ -11,7 +11,7 @@ inputs: actions: - ref: a action: core.transform.reshape - for_each: ${{ for var.list in INPUTS.nested }} + for_each: ${{ for var.list in VARS.nested }} args: # We map the list into the template. the `map` function automatically # broadcasts the template to each element of the list. diff --git a/tests/data/workflows/unit_transform_reshape_map_loop_expected.yml b/tests/data/workflows/unit_transform_reshape_map_loop_expected.yml index d76fe41e6..7fcdf582b 100644 --- a/tests/data/workflows/unit_transform_reshape_map_loop_expected.yml +++ b/tests/data/workflows/unit_transform_reshape_map_loop_expected.yml @@ -6,7 +6,7 @@ ACTIONS: - ["Got user_7!", "Got user_8!", "Got user_9!"] result_typename: list -INPUTS: +VARS: nested: - ["user_1", "user_2", "user_3"] - ["user_4", "user_5", "user_6"] diff --git a/tests/data/workflows/unit_transform_reshape_zip.yml b/tests/data/workflows/unit_transform_reshape_zip.yml index 80fe6b832..db9b6294a 100644 --- a/tests/data/workflows/unit_transform_reshape_zip.yml +++ b/tests/data/workflows/unit_transform_reshape_zip.yml @@ -2,7 +2,7 @@ title: Reshape loop with a list of iterables (zip) description: Test that we can loop entrypoint: ref: a -inputs: +variables: first: 1 second: 2 list_one: [1, 2, 3] @@ -12,7 +12,7 @@ actions: - ref: a action: core.transform.reshape for_each: - - ${{ for var.x in INPUTS.list_one }} - - ${{ for var.y in INPUTS.list_two }} + - ${{ for var.x in VARS.list_one }} + - ${{ for var.y in VARS.list_two }} args: value: I received ${{ FN.concat(str(var.x), var.y) }} from you diff --git a/tests/data/workflows/unit_transform_reshape_zip_expected.yml b/tests/data/workflows/unit_transform_reshape_zip_expected.yml index a99cb6023..79fd67c12 100644 --- a/tests/data/workflows/unit_transform_reshape_zip_expected.yml +++ b/tests/data/workflows/unit_transform_reshape_zip_expected.yml @@ -6,7 +6,7 @@ ACTIONS: - I received 3three from you result_typename: "list" -INPUTS: +VARS: first: 1 second: 2 list_one: [1, 2, 3] From 3fe75270f7ac46da7c8abc7dadf29ca3942e80d1 Mon Sep 17 00:00:00 2001 From: Daryl Lim <5508348+daryllimyt@users.noreply.github.com> Date: Fri, 8 Nov 2024 19:15:29 -0800 Subject: [PATCH 4/4] ui: Update tooltip --- frontend/src/components/workbench/panel/workflow-panel.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/components/workbench/panel/workflow-panel.tsx b/frontend/src/components/workbench/panel/workflow-panel.tsx index 8c08a40a1..0526cc675 100644 --- a/frontend/src/components/workbench/panel/workflow-panel.tsx +++ b/frontend/src/components/workbench/panel/workflow-panel.tsx @@ -525,7 +525,7 @@ function VariablesTooltip() { 
-          {"${{ VARS.my_static_key }}"}
+          {"${{ VARS.my_variable }}"}