diff --git a/docs/examples/workflows/experimental/new_dag_decorator_artifacts.md b/docs/examples/workflows/experimental/new_dag_decorator_artifacts.md index d4c297b6e..de676f7b2 100644 --- a/docs/examples/workflows/experimental/new_dag_decorator_artifacts.md +++ b/docs/examples/workflows/experimental/new_dag_decorator_artifacts.md @@ -11,7 +11,7 @@ from typing_extensions import Annotated from hera.shared import global_config - from hera.workflows import Artifact, Input, Output, Workflow + from hera.workflows import Artifact, ArtifactLoader, Input, Output, Workflow global_config.experimental_features["decorator_syntax"] = True @@ -24,8 +24,8 @@ class ConcatInput(Input): - word_a: Annotated[str, Artifact(name="word_a")] - word_b: Annotated[str, Artifact(name="word_b")] + word_a: Annotated[str, Artifact(name="word_a", loader=ArtifactLoader.json)] + word_b: Annotated[str, Artifact(name="word_b", loader=ArtifactLoader.json)] @w.script() @@ -105,23 +105,21 @@ name: word_a - from: '{{inputs.artifacts.artifact_b}}' name: word_b - name: concat_1 + name: concat-1 template: concat - arguments: artifacts: - - from: '{{tasks.concat_1.outputs.artifacts.an-artifact}}' + - from: '{{tasks.concat-1.outputs.artifacts.an-artifact}}' name: word_a - - from: '{{tasks.concat_1.outputs.artifacts.an-artifact}}' + - from: '{{tasks.concat-1.outputs.artifacts.an-artifact}}' name: word_b - depends: concat_1 + depends: concat-1 name: concat-2-custom-name template: concat inputs: artifacts: - name: artifact_a - path: /tmp/hera-inputs/artifacts/artifact_a - name: artifact_b - path: /tmp/hera-inputs/artifacts/artifact_b name: worker outputs: artifacts: diff --git a/docs/examples/workflows/experimental/new_dag_decorator_inner_dag.md b/docs/examples/workflows/experimental/new_dag_decorator_inner_dag.md index f718b856f..8d392c694 100644 --- a/docs/examples/workflows/experimental/new_dag_decorator_inner_dag.md +++ b/docs/examples/workflows/experimental/new_dag_decorator_inner_dag.md @@ -123,34 +123,34 @@ source: '{{inputs.parameters}}' - dag: tasks: - - name: setup_task + - name: setup-task template: setup - arguments: parameters: - name: word_a value: '{{inputs.parameters.value_a}}' - name: word_b - value: '{{tasks.setup_task.outputs.parameters.environment_parameter}}' - depends: setup_task - name: task_a + value: '{{tasks.setup-task.outputs.parameters.environment_parameter}}' + depends: setup-task + name: task-a template: concat - arguments: parameters: - name: word_a value: '{{inputs.parameters.value_b}}' - name: word_b - value: '{{tasks.setup_task.outputs.result}}' - depends: setup_task - name: task_b + value: '{{tasks.setup-task.outputs.result}}' + depends: setup-task + name: task-b template: concat - arguments: parameters: - name: word_a - value: '{{tasks.task_a.outputs.result}}' + value: '{{tasks.task-a.outputs.result}}' - name: word_b - value: '{{tasks.task_b.outputs.result}}' - depends: task_a && task_b - name: final_task + value: '{{tasks.task-b.outputs.result}}' + depends: task-a && task-b + name: final-task template: concat inputs: parameters: @@ -161,7 +161,7 @@ parameters: - name: value valueFrom: - parameter: '{{tasks.final_task.outputs.result}}' + parameter: '{{tasks.final-task.outputs.result}}' - dag: tasks: - arguments: @@ -170,7 +170,7 @@ value: dag_a - name: value_b value: '{{inputs.parameters.value_a}}' - name: sub_dag_a + name: sub-dag-a template: worker - arguments: parameters: @@ -178,16 +178,16 @@ value: dag_b - name: value_b value: '{{inputs.parameters.value_b}}' - name: sub_dag_b + name: sub-dag-b template: worker - arguments: parameters: - name: value_a - value: '{{tasks.sub_dag_a.outputs.parameters.value}}' + value: '{{tasks.sub-dag-a.outputs.parameters.value}}' - name: value_b - value: '{{tasks.sub_dag_b.outputs.parameters.value}}' - depends: sub_dag_a && sub_dag_b - name: sub_dag_c + value: '{{tasks.sub-dag-b.outputs.parameters.value}}' + depends: sub-dag-a && sub-dag-b + name: sub-dag-c template: worker inputs: parameters: @@ -198,6 +198,6 @@ parameters: - name: value valueFrom: - parameter: '{{tasks.sub_dag_c.outputs.parameters.value}}' + parameter: '{{tasks.sub-dag-c.outputs.parameters.value}}' ``` diff --git a/docs/examples/workflows/experimental/new_dag_decorator_params.md b/docs/examples/workflows/experimental/new_dag_decorator_params.md index 87ab9b605..2efec733e 100644 --- a/docs/examples/workflows/experimental/new_dag_decorator_params.md +++ b/docs/examples/workflows/experimental/new_dag_decorator_params.md @@ -26,7 +26,7 @@ class SetupOutput(Output): environment_parameter: str - an_annotated_parameter: Annotated[int, Parameter(name="dummy-param")] # use an annotated non-str + an_annotated_parameter: Annotated[int, Parameter()] # use an annotated non-str, infer name from field setup_config: Annotated[SetupConfig, Parameter(name="setup-config")] # use a pydantic BaseModel @@ -71,7 +71,8 @@ class WorkerOutput(Output): - value: str + result_value: str + another_value: str @w.set_entrypoint @@ -87,7 +88,7 @@ task_b = concat(ConcatInput(word_a=worker_input.value_b, word_b=setup_task.result)) final_task = concat(ConcatInput(word_a=task_a.result, word_b=task_b.result)) - return WorkerOutput(value=final_task.result) + return WorkerOutput(result_value=final_task.result, another_value=setup_task.an_annotated_parameter) ``` === "YAML" @@ -106,9 +107,9 @@ - name: environment_parameter valueFrom: path: /tmp/hera-outputs/parameters/environment_parameter - - name: dummy-param + - name: an_annotated_parameter valueFrom: - path: /tmp/hera-outputs/parameters/dummy-param + path: /tmp/hera-outputs/parameters/an_annotated_parameter - name: setup-config valueFrom: path: /tmp/hera-outputs/parameters/setup-config @@ -156,40 +157,40 @@ source: '{{inputs.parameters}}' - dag: tasks: - - name: setup_task + - name: setup-task template: setup - arguments: parameters: - name: word_a value: '{{inputs.parameters.value_a}}' - name: word_b - value: '{{tasks.setup_task.outputs.parameters.environment_parameter}}{{tasks.setup_task.outputs.parameters.dummy-param}}' + value: '{{tasks.setup-task.outputs.parameters.environment_parameter}}{{tasks.setup-task.outputs.parameters.an_annotated_parameter}}' - name: concat_config value: '{"reverse": false}' - depends: setup_task - name: task_a + depends: setup-task + name: task-a template: concat - arguments: parameters: - name: word_a value: '{{inputs.parameters.value_b}}' - name: word_b - value: '{{tasks.setup_task.outputs.result}}' + value: '{{tasks.setup-task.outputs.result}}' - name: concat_config value: '{"reverse": false}' - depends: setup_task - name: task_b + depends: setup-task + name: task-b template: concat - arguments: parameters: - name: word_a - value: '{{tasks.task_a.outputs.result}}' + value: '{{tasks.task-a.outputs.result}}' - name: word_b - value: '{{tasks.task_b.outputs.result}}' + value: '{{tasks.task-b.outputs.result}}' - name: concat_config value: '{"reverse": false}' - depends: task_a && task_b - name: final_task + depends: task-a && task-b + name: final-task template: concat inputs: parameters: @@ -203,8 +204,11 @@ name: worker outputs: parameters: - - name: value + - name: result_value valueFrom: - parameter: '{{tasks.final_task.outputs.result}}' + parameter: '{{tasks.final-task.outputs.result}}' + - name: another_value + valueFrom: + parameter: '{{tasks.setup-task.outputs.parameters.an_annotated_parameter}}' ``` diff --git a/docs/examples/workflows/experimental/new_decorators_auto_template_refs.md b/docs/examples/workflows/experimental/new_decorators_auto_template_refs.md index a6ecf31b1..cb49bdf87 100644 --- a/docs/examples/workflows/experimental/new_decorators_auto_template_refs.md +++ b/docs/examples/workflows/experimental/new_decorators_auto_template_refs.md @@ -116,7 +116,7 @@ clusterScope: true name: my-cluster-workflow-template template: run-setup-dag - - name: setup_task + - name: setup-task templateRef: clusterScope: true name: my-cluster-workflow-template @@ -126,11 +126,11 @@ - name: word_a value: '{{inputs.parameters.value_a}}' - name: word_b - value: '{{tasks.setup_task.outputs.parameters.environment_parameter}}{{tasks.setup_task.outputs.parameters.dummy-param}}' + value: '{{tasks.setup-task.outputs.parameters.environment_parameter}}{{tasks.setup-task.outputs.parameters.dummy-param}}' - name: concat_config value: '{"reverse": false}' - depends: setup_task - name: task_a + depends: setup-task + name: task-a templateRef: name: my-workflow-template template: concat @@ -139,24 +139,24 @@ - name: word_a value: '{{inputs.parameters.value_b}}' - name: word_b - value: '{{tasks.setup_task.outputs.result}}' + value: '{{tasks.setup-task.outputs.result}}' - name: concat_config value: '{"reverse": false}' - depends: setup_task - name: task_b + depends: setup-task + name: task-b templateRef: name: my-workflow-template template: concat - arguments: parameters: - name: word_a - value: '{{tasks.task_a.outputs.result}}' + value: '{{tasks.task-a.outputs.result}}' - name: word_b - value: '{{tasks.task_b.outputs.result}}' + value: '{{tasks.task-b.outputs.result}}' - name: concat_config value: '{"reverse": false}' - depends: task_a && task_b - name: final_task + depends: task-a && task-b + name: final-task templateRef: name: my-workflow-template template: concat @@ -174,6 +174,6 @@ parameters: - name: value valueFrom: - parameter: '{{tasks.final_task.outputs.result}}' + parameter: '{{tasks.final-task.outputs.result}}' ``` diff --git a/docs/examples/workflows/experimental/new_steps_decorator_with_parallel_steps.md b/docs/examples/workflows/experimental/new_steps_decorator_with_parallel_steps.md index d68391714..c1ae3a4a1 100644 --- a/docs/examples/workflows/experimental/new_steps_decorator_with_parallel_steps.md +++ b/docs/examples/workflows/experimental/new_steps_decorator_with_parallel_steps.md @@ -138,33 +138,33 @@ parameters: - name: value valueFrom: - parameter: '{{steps.final_step.outputs.result}}' + parameter: '{{steps.final-step.outputs.result}}' steps: - - - name: setup_step + - - name: setup-step template: setup - - arguments: parameters: - name: word_a value: '{{inputs.parameters.value_a}}' - name: word_b - value: '{{steps.setup_step.outputs.parameters.environment_parameter}}{{steps.setup_step.outputs.parameters.dummy-param}}' - name: step_a + value: '{{steps.setup-step.outputs.parameters.environment_parameter}}{{steps.setup-step.outputs.parameters.dummy-param}}' + name: step-a template: concat - arguments: parameters: - name: word_a value: '{{inputs.parameters.value_b}}' - name: word_b - value: '{{steps.setup_step.outputs.result}}' - name: step_b + value: '{{steps.setup-step.outputs.result}}' + name: step-b template: concat - - arguments: parameters: - name: word_a - value: '{{steps.step_a.outputs.result}}' + value: '{{steps.step-a.outputs.result}}' - name: word_b - value: '{{steps.step_b.outputs.result}}' - name: final_step + value: '{{steps.step-b.outputs.result}}' + name: final-step template: concat ``` diff --git a/docs/examples/workflows/experimental/template_sets.md b/docs/examples/workflows/experimental/template_sets.md index 5d0fac151..04c450d93 100644 --- a/docs/examples/workflows/experimental/template_sets.md +++ b/docs/examples/workflows/experimental/template_sets.md @@ -22,6 +22,12 @@ return Output(result="Setting things up") + @templates.dag() + def my_dag(): + setup(name="task-a") + setup(name="task-b") + + w.add_template_set(templates) ``` @@ -52,5 +58,12 @@ value: '' image: python:3.9 source: '{{inputs.parameters}}' + - dag: + tasks: + - name: task-a + template: setup + - name: task-b + template: setup + name: my-dag ``` diff --git a/docs/user-guides/decorators.md b/docs/user-guides/decorators.md index 7f932aea1..1526261a0 100644 --- a/docs/user-guides/decorators.md +++ b/docs/user-guides/decorators.md @@ -99,8 +99,8 @@ def goodbye(my_input: MyInput) -> Output: @w.set_entrypoint @w.steps() def my_steps() -> None: - hello_world(my_input=MyInput(user="elliot")) - goodbye(my_input=MyInput(user="elliot")) + hello_world(MyInput(user="elliot")) + goodbye(MyInput(user="elliot")) ``` For the line-by-line explanation, let's start with @@ -162,8 +162,8 @@ Then, we set up a `steps` template, setting it as the entrypoint, as follows: @w.set_entrypoint @w.steps() def my_steps() -> None: - hello_world(my_input=MyInput(user="elliot")) - goodbye(my_input=MyInput(user="elliot")) + hello_world(MyInput(user="elliot")) + goodbye(MyInput(user="elliot")) ``` We can simply call the script templates, passing the input objects in. diff --git a/examples/workflows/experimental/new-dag-decorator-artifacts.yaml b/examples/workflows/experimental/new-dag-decorator-artifacts.yaml index 56ce9923e..527aa2428 100644 --- a/examples/workflows/experimental/new-dag-decorator-artifacts.yaml +++ b/examples/workflows/experimental/new-dag-decorator-artifacts.yaml @@ -41,23 +41,21 @@ spec: name: word_a - from: '{{inputs.artifacts.artifact_b}}' name: word_b - name: concat_1 + name: concat-1 template: concat - arguments: artifacts: - - from: '{{tasks.concat_1.outputs.artifacts.an-artifact}}' + - from: '{{tasks.concat-1.outputs.artifacts.an-artifact}}' name: word_a - - from: '{{tasks.concat_1.outputs.artifacts.an-artifact}}' + - from: '{{tasks.concat-1.outputs.artifacts.an-artifact}}' name: word_b - depends: concat_1 + depends: concat-1 name: concat-2-custom-name template: concat inputs: artifacts: - name: artifact_a - path: /tmp/hera-inputs/artifacts/artifact_a - name: artifact_b - path: /tmp/hera-inputs/artifacts/artifact_b name: worker outputs: artifacts: diff --git a/examples/workflows/experimental/new-dag-decorator-inner-dag.yaml b/examples/workflows/experimental/new-dag-decorator-inner-dag.yaml index 583987c88..126e0faed 100644 --- a/examples/workflows/experimental/new-dag-decorator-inner-dag.yaml +++ b/examples/workflows/experimental/new-dag-decorator-inner-dag.yaml @@ -52,34 +52,34 @@ spec: source: '{{inputs.parameters}}' - dag: tasks: - - name: setup_task + - name: setup-task template: setup - arguments: parameters: - name: word_a value: '{{inputs.parameters.value_a}}' - name: word_b - value: '{{tasks.setup_task.outputs.parameters.environment_parameter}}' - depends: setup_task - name: task_a + value: '{{tasks.setup-task.outputs.parameters.environment_parameter}}' + depends: setup-task + name: task-a template: concat - arguments: parameters: - name: word_a value: '{{inputs.parameters.value_b}}' - name: word_b - value: '{{tasks.setup_task.outputs.result}}' - depends: setup_task - name: task_b + value: '{{tasks.setup-task.outputs.result}}' + depends: setup-task + name: task-b template: concat - arguments: parameters: - name: word_a - value: '{{tasks.task_a.outputs.result}}' + value: '{{tasks.task-a.outputs.result}}' - name: word_b - value: '{{tasks.task_b.outputs.result}}' - depends: task_a && task_b - name: final_task + value: '{{tasks.task-b.outputs.result}}' + depends: task-a && task-b + name: final-task template: concat inputs: parameters: @@ -90,7 +90,7 @@ spec: parameters: - name: value valueFrom: - parameter: '{{tasks.final_task.outputs.result}}' + parameter: '{{tasks.final-task.outputs.result}}' - dag: tasks: - arguments: @@ -99,7 +99,7 @@ spec: value: dag_a - name: value_b value: '{{inputs.parameters.value_a}}' - name: sub_dag_a + name: sub-dag-a template: worker - arguments: parameters: @@ -107,16 +107,16 @@ spec: value: dag_b - name: value_b value: '{{inputs.parameters.value_b}}' - name: sub_dag_b + name: sub-dag-b template: worker - arguments: parameters: - name: value_a - value: '{{tasks.sub_dag_a.outputs.parameters.value}}' + value: '{{tasks.sub-dag-a.outputs.parameters.value}}' - name: value_b - value: '{{tasks.sub_dag_b.outputs.parameters.value}}' - depends: sub_dag_a && sub_dag_b - name: sub_dag_c + value: '{{tasks.sub-dag-b.outputs.parameters.value}}' + depends: sub-dag-a && sub-dag-b + name: sub-dag-c template: worker inputs: parameters: @@ -127,4 +127,4 @@ spec: parameters: - name: value valueFrom: - parameter: '{{tasks.sub_dag_c.outputs.parameters.value}}' + parameter: '{{tasks.sub-dag-c.outputs.parameters.value}}' diff --git a/examples/workflows/experimental/new-dag-decorator-params.yaml b/examples/workflows/experimental/new-dag-decorator-params.yaml index 23d24956b..d30d31e04 100644 --- a/examples/workflows/experimental/new-dag-decorator-params.yaml +++ b/examples/workflows/experimental/new-dag-decorator-params.yaml @@ -11,9 +11,9 @@ spec: - name: environment_parameter valueFrom: path: /tmp/hera-outputs/parameters/environment_parameter - - name: dummy-param + - name: an_annotated_parameter valueFrom: - path: /tmp/hera-outputs/parameters/dummy-param + path: /tmp/hera-outputs/parameters/an_annotated_parameter - name: setup-config valueFrom: path: /tmp/hera-outputs/parameters/setup-config @@ -61,40 +61,40 @@ spec: source: '{{inputs.parameters}}' - dag: tasks: - - name: setup_task + - name: setup-task template: setup - arguments: parameters: - name: word_a value: '{{inputs.parameters.value_a}}' - name: word_b - value: '{{tasks.setup_task.outputs.parameters.environment_parameter}}{{tasks.setup_task.outputs.parameters.dummy-param}}' + value: '{{tasks.setup-task.outputs.parameters.environment_parameter}}{{tasks.setup-task.outputs.parameters.an_annotated_parameter}}' - name: concat_config value: '{"reverse": false}' - depends: setup_task - name: task_a + depends: setup-task + name: task-a template: concat - arguments: parameters: - name: word_a value: '{{inputs.parameters.value_b}}' - name: word_b - value: '{{tasks.setup_task.outputs.result}}' + value: '{{tasks.setup-task.outputs.result}}' - name: concat_config value: '{"reverse": false}' - depends: setup_task - name: task_b + depends: setup-task + name: task-b template: concat - arguments: parameters: - name: word_a - value: '{{tasks.task_a.outputs.result}}' + value: '{{tasks.task-a.outputs.result}}' - name: word_b - value: '{{tasks.task_b.outputs.result}}' + value: '{{tasks.task-b.outputs.result}}' - name: concat_config value: '{"reverse": false}' - depends: task_a && task_b - name: final_task + depends: task-a && task-b + name: final-task template: concat inputs: parameters: @@ -108,6 +108,9 @@ spec: name: worker outputs: parameters: - - name: value + - name: result_value valueFrom: - parameter: '{{tasks.final_task.outputs.result}}' + parameter: '{{tasks.final-task.outputs.result}}' + - name: another_value + valueFrom: + parameter: '{{tasks.setup-task.outputs.parameters.an_annotated_parameter}}' diff --git a/examples/workflows/experimental/new-decorators-auto-template-refs.yaml b/examples/workflows/experimental/new-decorators-auto-template-refs.yaml index 82affcae7..3fedda938 100644 --- a/examples/workflows/experimental/new-decorators-auto-template-refs.yaml +++ b/examples/workflows/experimental/new-decorators-auto-template-refs.yaml @@ -12,7 +12,7 @@ spec: clusterScope: true name: my-cluster-workflow-template template: run-setup-dag - - name: setup_task + - name: setup-task templateRef: clusterScope: true name: my-cluster-workflow-template @@ -22,11 +22,11 @@ spec: - name: word_a value: '{{inputs.parameters.value_a}}' - name: word_b - value: '{{tasks.setup_task.outputs.parameters.environment_parameter}}{{tasks.setup_task.outputs.parameters.dummy-param}}' + value: '{{tasks.setup-task.outputs.parameters.environment_parameter}}{{tasks.setup-task.outputs.parameters.dummy-param}}' - name: concat_config value: '{"reverse": false}' - depends: setup_task - name: task_a + depends: setup-task + name: task-a templateRef: name: my-workflow-template template: concat @@ -35,24 +35,24 @@ spec: - name: word_a value: '{{inputs.parameters.value_b}}' - name: word_b - value: '{{tasks.setup_task.outputs.result}}' + value: '{{tasks.setup-task.outputs.result}}' - name: concat_config value: '{"reverse": false}' - depends: setup_task - name: task_b + depends: setup-task + name: task-b templateRef: name: my-workflow-template template: concat - arguments: parameters: - name: word_a - value: '{{tasks.task_a.outputs.result}}' + value: '{{tasks.task-a.outputs.result}}' - name: word_b - value: '{{tasks.task_b.outputs.result}}' + value: '{{tasks.task-b.outputs.result}}' - name: concat_config value: '{"reverse": false}' - depends: task_a && task_b - name: final_task + depends: task-a && task-b + name: final-task templateRef: name: my-workflow-template template: concat @@ -70,4 +70,4 @@ spec: parameters: - name: value valueFrom: - parameter: '{{tasks.final_task.outputs.result}}' + parameter: '{{tasks.final-task.outputs.result}}' diff --git a/examples/workflows/experimental/new-steps-decorator-with-parallel-steps.yaml b/examples/workflows/experimental/new-steps-decorator-with-parallel-steps.yaml index 308200c58..cc938f328 100644 --- a/examples/workflows/experimental/new-steps-decorator-with-parallel-steps.yaml +++ b/examples/workflows/experimental/new-steps-decorator-with-parallel-steps.yaml @@ -66,31 +66,31 @@ spec: parameters: - name: value valueFrom: - parameter: '{{steps.final_step.outputs.result}}' + parameter: '{{steps.final-step.outputs.result}}' steps: - - - name: setup_step + - - name: setup-step template: setup - - arguments: parameters: - name: word_a value: '{{inputs.parameters.value_a}}' - name: word_b - value: '{{steps.setup_step.outputs.parameters.environment_parameter}}{{steps.setup_step.outputs.parameters.dummy-param}}' - name: step_a + value: '{{steps.setup-step.outputs.parameters.environment_parameter}}{{steps.setup-step.outputs.parameters.dummy-param}}' + name: step-a template: concat - arguments: parameters: - name: word_a value: '{{inputs.parameters.value_b}}' - name: word_b - value: '{{steps.setup_step.outputs.result}}' - name: step_b + value: '{{steps.setup-step.outputs.result}}' + name: step-b template: concat - - arguments: parameters: - name: word_a - value: '{{steps.step_a.outputs.result}}' + value: '{{steps.step-a.outputs.result}}' - name: word_b - value: '{{steps.step_b.outputs.result}}' - name: final_step + value: '{{steps.step-b.outputs.result}}' + name: final-step template: concat diff --git a/examples/workflows/experimental/new_dag_decorator_artifacts.py b/examples/workflows/experimental/new_dag_decorator_artifacts.py index 6c66684a8..18bcee536 100644 --- a/examples/workflows/experimental/new_dag_decorator_artifacts.py +++ b/examples/workflows/experimental/new_dag_decorator_artifacts.py @@ -1,7 +1,7 @@ from typing_extensions import Annotated from hera.shared import global_config -from hera.workflows import Artifact, Input, Output, Workflow +from hera.workflows import Artifact, ArtifactLoader, Input, Output, Workflow global_config.experimental_features["decorator_syntax"] = True @@ -14,8 +14,8 @@ class ArtifactOutput(Output): class ConcatInput(Input): - word_a: Annotated[str, Artifact(name="word_a")] - word_b: Annotated[str, Artifact(name="word_b")] + word_a: Annotated[str, Artifact(name="word_a", loader=ArtifactLoader.json)] + word_b: Annotated[str, Artifact(name="word_b", loader=ArtifactLoader.json)] @w.script() diff --git a/examples/workflows/experimental/new_dag_decorator_params.py b/examples/workflows/experimental/new_dag_decorator_params.py index 2a7b719de..89d903472 100644 --- a/examples/workflows/experimental/new_dag_decorator_params.py +++ b/examples/workflows/experimental/new_dag_decorator_params.py @@ -16,7 +16,7 @@ class SetupConfig(BaseModel): class SetupOutput(Output): environment_parameter: str - an_annotated_parameter: Annotated[int, Parameter(name="dummy-param")] # use an annotated non-str + an_annotated_parameter: Annotated[int, Parameter()] # use an annotated non-str, infer name from field setup_config: Annotated[SetupConfig, Parameter(name="setup-config")] # use a pydantic BaseModel @@ -61,7 +61,8 @@ class WorkerInput(Input): class WorkerOutput(Output): - value: str + result_value: str + another_value: str @w.set_entrypoint @@ -77,4 +78,4 @@ def worker(worker_input: WorkerInput) -> WorkerOutput: task_b = concat(ConcatInput(word_a=worker_input.value_b, word_b=setup_task.result)) final_task = concat(ConcatInput(word_a=task_a.result, word_b=task_b.result)) - return WorkerOutput(value=final_task.result) + return WorkerOutput(result_value=final_task.result, another_value=setup_task.an_annotated_parameter) diff --git a/examples/workflows/experimental/template-sets.yaml b/examples/workflows/experimental/template-sets.yaml index afa2f9a4d..02ce3656c 100644 --- a/examples/workflows/experimental/template-sets.yaml +++ b/examples/workflows/experimental/template-sets.yaml @@ -22,3 +22,10 @@ spec: value: '' image: python:3.9 source: '{{inputs.parameters}}' + - dag: + tasks: + - name: task-a + template: setup + - name: task-b + template: setup + name: my-dag diff --git a/examples/workflows/experimental/template_sets.py b/examples/workflows/experimental/template_sets.py index 2a242c637..0e61d8bb6 100644 --- a/examples/workflows/experimental/template_sets.py +++ b/examples/workflows/experimental/template_sets.py @@ -12,4 +12,10 @@ def setup() -> Output: return Output(result="Setting things up") +@templates.dag() +def my_dag(): + setup(name="task-a") + setup(name="task-b") + + w.add_template_set(templates) diff --git a/src/hera/workflows/_context.py b/src/hera/workflows/_context.py index b7f4b07a9..bf9d75392 100644 --- a/src/hera/workflows/_context.py +++ b/src/hera/workflows/_context.py @@ -8,6 +8,7 @@ from typing import List, Optional, TypeVar, Union from hera.shared import BaseMixin +from hera.shared._global_config import _DECORATOR_SYNTAX_FLAG, _flag_enabled from hera.workflows.exceptions import InvalidType from hera.workflows.protocol import Subbable, TTemplate @@ -96,8 +97,14 @@ def add_sub_node(self, node: Union[SubNodeMixin, TTemplate]) -> None: ): from hera.workflows.workflow import Workflow - if not isinstance(pieces[0], Workflow): - raise SyntaxError("Not under a Workflow context") + if _flag_enabled(_DECORATOR_SYNTAX_FLAG): + from hera.workflows.template_set import TemplateSet + + if not isinstance(pieces[0], (TemplateSet, Workflow)): + raise SyntaxError("Not under a TemplateSet/Workflow context") + else: + if not isinstance(pieces[0], Workflow): + raise SyntaxError("Not under a Workflow context") found = False for t in pieces[0].templates: diff --git a/src/hera/workflows/_meta_mixins.py b/src/hera/workflows/_meta_mixins.py index 010265cf2..d32dd55b6 100644 --- a/src/hera/workflows/_meta_mixins.py +++ b/src/hera/workflows/_meta_mixins.py @@ -535,6 +535,11 @@ def _create_subnode( if len(args) == 1 and isinstance(args[0], (InputV1, InputV2)): subnode_args = args[0]._get_as_arguments() + if input_in_kwargs := [k for k, v in kwargs.items() if isinstance(v, (InputV1, InputV2))]: + raise SyntaxError( + f"Found Input argument(s) in kwargs: {input_in_kwargs}. Input must be passed as a positional-only argument." + ) + signature = inspect.signature(func) output_class = signature.return_annotation @@ -632,6 +637,16 @@ def script_decorator(func: Callable[FuncIns, FuncR]) -> Callable: if "constructor" not in script_kwargs and "constructor" not in global_config._get_class_defaults(Script): script_kwargs["constructor"] = RunnerScriptConstructor() + signature = inspect.signature(func) + func_inputs = signature.parameters + if len(func_inputs) > 1: + raise SyntaxError("script decorator must be used with a single `Input` arg, or no args.") + + if len(func_inputs) == 1: + func_input = list(func_inputs.values())[0].annotation + if not issubclass(func_input, (InputV1, InputV2)): + raise SyntaxError("script decorator must be used with a single `Input` arg, or no args.") + # Open (Workflow) context to add `Script` object automatically with self: script_template = Script(name=name, source=source, **script_kwargs) @@ -643,21 +658,24 @@ def script_decorator(func: Callable[FuncIns, FuncR]) -> Callable: def script_call_wrapper(*args, **kwargs) -> Union[FuncR, Step, Task, None]: """Invokes a CallableTemplateMixin's `__call__` method using the given SubNode (Step or Task) args/kwargs.""" if _context.declaring: - try: - # ignore decorator function assignment - subnode_name = varname() - except ImproperUseError: - # Template is being used without variable assignment (so use function name or provided name) - subnode_name = script_template.name # type: ignore - - subnode_name = kwargs.pop("name", subnode_name) - assert isinstance(subnode_name, str) + subnode_name = kwargs.pop("name", None) + if not subnode_name: + try: + # ignore decorator function assignment + subnode_name = varname() + except ImproperUseError: + # Template is being used without variable assignment (so use function name or provided name) + subnode_name = script_template.name # type: ignore + + assert isinstance(subnode_name, str) + subnode_name = subnode_name.replace("_", "-") return self._create_subnode(subnode_name, func, script_template, *args, **kwargs) if _context.pieces: return script_template.__call__(*args, **kwargs) - return func(*args, **kwargs) + + return func(*args) # Set the wrapped function to the original function so that we can use it later script_call_wrapper.wrapped_function = func # type: ignore @@ -682,7 +700,7 @@ def container_decorator(func: Callable[FuncIns, FuncR]) -> Callable: if len(func_inputs) >= 1: input_arg = list(func_inputs.values())[0].annotation if issubclass(input_arg, (InputV1, InputV2)): - inputs = input_arg._get_inputs() + inputs = input_arg._get_inputs(add_missing_path=True) func_return = signature.return_annotation outputs = [] @@ -702,21 +720,23 @@ def container_decorator(func: Callable[FuncIns, FuncR]) -> Callable: def container_call_wrapper(*args, **kwargs) -> Union[FuncR, Step, Task, None]: """Invokes a CallableTemplateMixin's `__call__` method using the given SubNode (Step or Task) args/kwargs.""" if _context.declaring: - try: - # ignore decorator function assignment - subnode_name = varname() - except ImproperUseError: - # Template is being used without variable assignment (so use function name or provided name) - subnode_name = container_template.name # type: ignore - - subnode_name = kwargs.pop("name", subnode_name) - assert isinstance(subnode_name, str) + subnode_name = kwargs.pop("name", None) + if not subnode_name: + try: + # ignore decorator function assignment + subnode_name = varname() + except ImproperUseError: + # Template is being used without variable assignment (so use function name or provided name) + subnode_name = container_template.name # type: ignore + + assert isinstance(subnode_name, str) + subnode_name = subnode_name.replace("_", "-") return self._create_subnode(subnode_name, func, container_template, *args, **kwargs) if _context.pieces: return container_template.__call__(*args, **kwargs) - return func(*args, **kwargs) + return func(*args) # Set the template name to the inferred name container_call_wrapper.template_name = name # type: ignore @@ -763,15 +783,28 @@ def decorator(func: Callable[FuncIns, FuncR]) -> Callable: signature = inspect.signature(func) func_inputs = signature.parameters inputs = [] + if len(func_inputs) > 1: + raise SyntaxError( + f"{invocator_type.__name__.lower()} decorator must be used with a single `Input` arg, or no args." + ) + if len(func_inputs) == 1: - arg_class = list(func_inputs.values())[0].annotation - if issubclass(arg_class, (InputV1, InputV2)): - inputs = arg_class._get_inputs() + input_arg = list(func_inputs.values())[0].annotation + if not issubclass(input_arg, (InputV1, InputV2)): + raise SyntaxError( + f"{invocator_type.__name__.lower()} decorator must be used with a single `Input` arg, or no args." + ) + inputs = input_arg._get_inputs() - func_return = signature.return_annotation + func_return_annotation = signature.return_annotation outputs = [] - if func_return and issubclass(func_return, (OutputV1, OutputV2)): - outputs = func_return._get_outputs() + if func_return_annotation and issubclass(func_return_annotation, (OutputV1, OutputV2)): + outputs = func_return_annotation._get_outputs() + elif func_return_annotation is not inspect.Signature.empty and func_return_annotation is not None: + raise SyntaxError( + f"{invocator_type.__name__.lower()} decorator must be used with a single " + "`Output` return annotation, or a `None`/empty return annotation." + ) # Add dag/steps to workflow with self: @@ -785,35 +818,65 @@ def decorator(func: Callable[FuncIns, FuncR]) -> Callable: @functools.wraps(func) def call_wrapper(*args, **kwargs): if _context.declaring: - # A sub-dag as a Step or Task is being created - try: - subnode_name = varname() - except ImproperUseError: - # Template is being used without variable assignment (so use function name or provided name) - subnode_name = name - - subnode_name = kwargs.pop("name", subnode_name) - assert isinstance(subnode_name, str) + subnode_name = kwargs.pop("name", None) + if not subnode_name: + try: + # ignore decorator function assignment + subnode_name = varname() + except ImproperUseError: + # Template is being used without variable assignment (so use function name or provided name) + subnode_name = name # type: ignore + + assert isinstance(subnode_name, str) + subnode_name = subnode_name.replace("_", "-") return self._create_subnode(subnode_name, func, template, *args, **kwargs) - return func(*args, **kwargs) + return func(*args) call_wrapper.template_name = name # type: ignore # Open workflow context to cross-check task template names with self, template: + input_objs = [] if len(func_inputs) == 1: - arg_class = list(func_inputs.values())[0].annotation - if issubclass(arg_class, (InputV1, InputV2)): - input_obj = arg_class._get_as_templated_arguments() - # "run" the dag/steps function to collect the tasks/steps - _context.declaring = True - func_return = func(input_obj) - _context.declaring = False - - if func_return and isinstance(func_return, (OutputV1, OutputV2)): - template.outputs = func_return._get_as_invocator_output() + input_arg = list(func_inputs.values())[0].annotation + if issubclass(input_arg, (InputV1, InputV2)): + input_objs.append(input_arg._get_as_templated_arguments()) + + # "run" the dag/steps function to collect the tasks/steps + _context.declaring = True + func_return = func(*input_objs) + _context.declaring = False + + if func_return is not None: + if func_return_annotation is inspect.Signature.empty or func_return_annotation is None: + raise SyntaxError( + f"Function returned {func_return.__class__}, expected None " + "(the function may be missing a return annotation)." + ) + + from hera.workflows.steps import Step + from hera.workflows.task import Task + + if isinstance(func_return, (Step, Task)): + # User tried to return an `Output` from another step/task directly + raise SyntaxError("Function return must be a new Output object.") + + if issubclass(func_return_annotation, (OutputV1, OutputV2)): + if type(func_return) is not func_return_annotation: + raise SyntaxError( + "Function return does not match annotation, " + f"expected: {func_return_annotation}; got: {func_return.__class__}." + ) + assert isinstance(func_return, func_return_annotation) # for type-checking + + if func_return.result or func_return.exit_code: + raise SyntaxError( + "Cannot set `result` or `exit_code` on Output when used in a dag/steps function." + ) + + template.outputs = func_return._get_as_invocator_output() return call_wrapper diff --git a/src/hera/workflows/_mixins.py b/src/hera/workflows/_mixins.py index 341466b17..9acf61062 100644 --- a/src/hera/workflows/_mixins.py +++ b/src/hera/workflows/_mixins.py @@ -739,14 +739,11 @@ def __getattribute__(self, name: str) -> Any: return result_templated_str if param_or_artifact := get_workflow_annotation(annotations[name]): + output_name = param_or_artifact.name or name if isinstance(param_or_artifact, Parameter): - return ( - "{{" + f"{subnode_type}.{subnode_name}.outputs.parameters.{param_or_artifact.name}" + "}}" - ) + return "{{" + f"{subnode_type}.{subnode_name}.outputs.parameters.{output_name}" + "}}" else: - return ( - "{{" + f"{subnode_type}.{subnode_name}.outputs.artifacts.{param_or_artifact.name}" + "}}" - ) + return "{{" + f"{subnode_type}.{subnode_name}.outputs.artifacts.{output_name}" + "}}" return "{{" + f"{subnode_type}.{subnode_name}.outputs.parameters.{name}" + "}}" diff --git a/src/hera/workflows/io/_io_mixins.py b/src/hera/workflows/io/_io_mixins.py index 6d7f2d767..3ef106122 100644 --- a/src/hera/workflows/io/_io_mixins.py +++ b/src/hera/workflows/io/_io_mixins.py @@ -106,19 +106,19 @@ def _get_parameters(cls, object_override: Optional[Self] = None) -> List[Paramet return parameters @classmethod - def _get_artifacts(cls) -> List[Artifact]: + def _get_artifacts(cls, add_missing_path: bool = False) -> List[Artifact]: artifacts = [] for _, _, artifact in _construct_io_from_fields(cls): if isinstance(artifact, Artifact): - if artifact.path is None: + if add_missing_path and artifact.path is None: artifact.path = artifact._get_default_inputs_path() artifacts.append(artifact) return artifacts @classmethod - def _get_inputs(cls) -> List[Union[Artifact, Parameter]]: - return cls._get_artifacts() + cls._get_parameters() + def _get_inputs(cls, add_missing_path: bool = False) -> List[Union[Artifact, Parameter]]: + return cls._get_artifacts(add_missing_path) + cls._get_parameters() @classmethod def _get_as_templated_arguments(cls) -> Self: @@ -226,8 +226,10 @@ def _get_as_invocator_output(self) -> List[Union[Artifact, Parameter]]: templated_value = self_dict[field] # a string such as `"{{tasks.task_a.outputs.parameter.my_param}}"` if isinstance(annotation, Parameter): - outputs.append(Parameter(name=annotation.name, value_from=ValueFrom(parameter=templated_value))) + annotation.value_from = ValueFrom(parameter=templated_value) else: - outputs.append(Artifact(name=annotation.name, from_=templated_value)) + annotation.from_ = templated_value + + outputs.append(annotation) return outputs diff --git a/src/hera/workflows/script.py b/src/hera/workflows/script.py index 058f5a1b9..15cf5a8e9 100644 --- a/src/hera/workflows/script.py +++ b/src/hera/workflows/script.py @@ -494,7 +494,7 @@ class will be used as inputs, rather than the class itself. else: parameters.extend(input_class._get_parameters()) - artifacts.extend(input_class._get_artifacts()) + artifacts.extend(input_class._get_artifacts(add_missing_path=True)) elif param_or_artifact := get_workflow_annotation(func_param.annotation): if param_or_artifact.output: diff --git a/tests/conftest.py b/tests/conftest.py index 2a4e5182e..d91eb9cf1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,7 @@ import pytest from hera.shared import global_config +from hera.workflows._context import _context @pytest.fixture @@ -8,3 +9,8 @@ def global_config_fixture(): global_config.reset() yield global_config global_config.reset() + + +@pytest.fixture(autouse=True) +def clear_context(): + _context.declaring = False diff --git a/tests/test_unit/test_context.py b/tests/test_unit/test_context.py index 710146aba..6b7aa4c90 100644 --- a/tests/test_unit/test_context.py +++ b/tests/test_unit/test_context.py @@ -131,3 +131,15 @@ def hello(): with pytest.raises(SyntaxError, match="Not under a Workflow context"): with Steps(name="test"): hello() + + +def test_error_outside_of_workflow_context_decorator_flag(global_config_fixture): + global_config_fixture.experimental_features["decorator_syntax"] = True + + @script() + def hello(): + print("hello") + + with pytest.raises(SyntaxError, match="Not under a TemplateSet/Workflow context"): + with Steps(name="test"): + hello() diff --git a/tests/test_decorators.py b/tests/test_unit/test_decorators.py similarity index 50% rename from tests/test_decorators.py rename to tests/test_unit/test_decorators.py index cca81ea31..4729a1835 100644 --- a/tests/test_decorators.py +++ b/tests/test_unit/test_decorators.py @@ -1,8 +1,13 @@ import importlib import logging +import re from typing import cast +import pytest + +import hera.workflows.models as m from hera.workflows import DAG, Workflow +from hera.workflows.io.v1 import Input, Output from hera.workflows.models import ( Artifact as ModelArtifact, Parameter as ModelParameter, @@ -31,26 +36,24 @@ def test_dag_io_declaration(): assert len(model_workflow.spec.templates) == 1 - template = model_workflow.spec.templates[0] + dag_template = model_workflow.spec.templates[0] - assert template.inputs - assert len(template.inputs.parameters) == 2 - assert template.inputs.parameters == [ + assert dag_template.inputs + assert len(dag_template.inputs.parameters) == 2 + assert dag_template.inputs.parameters == [ ModelParameter(name="basic_input_parameter"), ModelParameter(name="my-input-param"), ] - assert len(template.inputs.artifacts) == 1 - assert template.inputs.artifacts == [ - ModelArtifact(name="my-input-artifact", path="/tmp/hera-inputs/artifacts/my-input-artifact"), - ] + assert len(dag_template.inputs.artifacts) == 1 + assert dag_template.inputs.artifacts == [ModelArtifact(name="my-input-artifact")] - assert template.outputs - assert len(template.outputs.parameters) == 2 - assert template.outputs.parameters == [ + assert dag_template.outputs + assert len(dag_template.outputs.parameters) == 2 + assert dag_template.outputs.parameters == [ ModelParameter(name="basic_output_parameter"), ModelParameter(name="my-output-param"), ] - assert template.outputs.artifacts == [ + assert dag_template.outputs.artifacts == [ ModelArtifact(name="my-output-artifact"), ] @@ -94,7 +97,7 @@ def test_dag_tasks_with_masked_attributes_in_arguments(): assert dag_template.dag is not None assert len(dag_template.dag.tasks) == 2 - task_a = next(filter(lambda t: t.name == "task_a", dag_template.dag.tasks)) + task_a = next(filter(lambda t: t.name == "task-a", dag_template.dag.tasks)) assert task_a.arguments and task_a.arguments.parameters assert task_a.arguments.parameters == [ @@ -106,23 +109,23 @@ def test_dag_tasks_with_masked_attributes_in_arguments(): ModelParameter(name="target", value="{{inputs.parameters.target}}"), ] - task_b = next(filter(lambda t: t.name == "task_b", dag_template.dag.tasks)) + task_b = next(filter(lambda t: t.name == "task-b", dag_template.dag.tasks)) assert task_b.arguments and task_b.arguments.parameters assert task_b.arguments.parameters == [ - ModelParameter(name="name", value="{{tasks.task_a.outputs.parameters.name}}"), - ModelParameter(name="hooks", value="{{tasks.task_a.outputs.parameters.hooks}}"), + ModelParameter(name="name", value="{{tasks.task-a.outputs.parameters.name}}"), + ModelParameter(name="hooks", value="{{tasks.task-a.outputs.parameters.hooks}}"), ModelParameter(name="target", value="{{inputs.parameters.target}}"), ] assert dag_template.outputs.parameters == [ ModelParameter( name="name", - value_from={"parameter": "{{tasks.task_a.outputs.parameters.name}}"}, + value_from={"parameter": "{{tasks.task-a.outputs.parameters.name}}"}, ), ModelParameter( name="hooks", - value_from={"parameter": "{{tasks.task_b.outputs.parameters.hooks}}"}, + value_from={"parameter": "{{tasks.task-b.outputs.parameters.hooks}}"}, ), ModelParameter( name="target", @@ -144,7 +147,7 @@ def test_dag_task_io_hoisting(): assert dag_template.outputs and dag_template.outputs.parameters and dag_template.outputs.artifacts assert dag_template.outputs.parameters assert dag_template.outputs.artifacts == [ - ModelArtifact(name="my-output-artifact", from_="{{tasks.a_task.outputs.artifacts.my-output-artifact}}"), + ModelArtifact(name="my-output-artifact", from_="{{tasks.a-task.outputs.artifacts.my-output-artifact}}"), ] @@ -158,17 +161,17 @@ def test_dag_task_auto_depends(): assert len(dag_template.tasks) == 4 - setup_task = next(iter([t for t in dag_template.tasks if t.name == "setup_task"]), None) + setup_task = next(iter([t for t in dag_template.tasks if t.name == "setup-task"]), None) assert setup_task.depends is None - task_a = next(iter([t for t in dag_template.tasks if t.name == "task_a"]), None) - assert task_a.depends == "setup_task" + task_a = next(iter([t for t in dag_template.tasks if t.name == "task-a"]), None) + assert task_a.depends == "setup-task" - task_b = next(iter([t for t in dag_template.tasks if t.name == "task_b"]), None) - assert task_b.depends == "setup_task" + task_b = next(iter([t for t in dag_template.tasks if t.name == "task-b"]), None) + assert task_b.depends == "setup-task" - final_task = next(iter([t for t in dag_template.tasks if t.name == "final_task"]), None) - assert final_task.depends == "task_a && task_b" + final_task = next(iter([t for t in dag_template.tasks if t.name == "final-task-name"]), None) + assert final_task.depends == "task-a && task-b" def test_dag_with_inner_dag(): @@ -180,7 +183,7 @@ def test_dag_with_inner_dag(): assert len(outer_dag_template.tasks) == 3 - dag_a = next(iter([t for t in outer_dag_template.tasks if t.name == "sub_dag_a"]), None) + dag_a = next(iter([t for t in outer_dag_template.tasks if t.name == "sub-dag-a"]), None) assert dag_a assert dag_a.arguments and dag_a.arguments.parameters == [ ModelParameter( @@ -193,7 +196,7 @@ def test_dag_with_inner_dag(): ), ] - dag_b = next(iter([t for t in outer_dag_template.tasks if t.name == "sub_dag_b"]), None) + dag_b = next(iter([t for t in outer_dag_template.tasks if t.name == "sub-dag-b"]), None) assert dag_b assert dag_b.arguments and dag_b.arguments.parameters == [ ModelParameter( @@ -206,16 +209,16 @@ def test_dag_with_inner_dag(): ), ] - dag_c = next(iter([t for t in outer_dag_template.tasks if t.name == "sub_dag_c"]), None) + dag_c = next(iter([t for t in outer_dag_template.tasks if t.name == "sub-dag-c"]), None) assert dag_c assert dag_c.arguments and dag_c.arguments.parameters == [ ModelParameter( name="value_a", - value="{{tasks.sub_dag_a.outputs.parameters.value}}", + value="{{tasks.sub-dag-a.outputs.parameters.value}}", ), ModelParameter( name="value_b", - value="{{tasks.sub_dag_b.outputs.parameters.value}}", + value="{{tasks.sub-dag-b.outputs.parameters.value}}", ), ] @@ -237,3 +240,176 @@ def test_steps_with_parallel_steps_is_runnable(global_config_fixture): assert worker(WorkerInput(value_a="hello", value_b="world")) == WorkerOutput( value="hello linux42 world Setting things up" ) + + +def test_dag_func_no_inputs(global_config_fixture): + # GIVEN + global_config_fixture.experimental_features["decorator_syntax"] = True + w = Workflow() + + @w.dag() + def no_arg_dag() -> None: + pass + + # WHEN + model_workflow = cast(m.Workflow, w.build()) + + # THEN + assert model_workflow.spec.templates and model_workflow.spec.templates[0].inputs is None + + +def test_dag_func_one_input(global_config_fixture): + # GIVEN + global_config_fixture.experimental_features["decorator_syntax"] = True + w = Workflow() + + class DagInput(Input): + my_int: int + + @w.dag() + def one_arg_dag_func(_: DagInput) -> None: + pass + + # WHEN + model_workflow = cast(m.Workflow, w.build()) + + # THEN + assert model_workflow.spec.templates and model_workflow.spec.templates[0].inputs == m.Inputs( + parameters=[m.Parameter(name="my_int")] + ) + + +def test_dag_func_two_inputs_errors(global_config_fixture): + # GIVEN + global_config_fixture.experimental_features["decorator_syntax"] = True + w = Workflow() + + # WHEN/THEN + with pytest.raises( + SyntaxError, match=re.escape("dag decorator must be used with a single `Input` arg, or no args.") + ): + + @w.dag() + def two_args_dag_func(_1: int, _2: str) -> None: + pass + + +def test_dag_return_must_be_new_output(global_config_fixture): + global_config_fixture.experimental_features["decorator_syntax"] = True + + # WHEN/THEN + with pytest.raises(SyntaxError, match=re.escape("Function return must be a new Output object.")): + importlib.import_module("tests.workflow_decorators.dag_return_task_error").w + + +def test_dag_return_without_annotation_errors_if_output(global_config_fixture): + global_config_fixture.experimental_features["decorator_syntax"] = True + + # WHEN/THEN + with pytest.raises( + SyntaxError, + match=re.escape( + "Function returned , " + "expected None (the function may be missing a return annotation)." + ), + ): + importlib.import_module("tests.workflow_decorators.dag_return_unannotated_error").w + + +def test_dag_return_none_errors_if_output(global_config_fixture): + global_config_fixture.experimental_features["decorator_syntax"] = True + + # WHEN/THEN + with pytest.raises( + SyntaxError, + match=re.escape( + "Function returned , " + "expected None (the function may be missing a return annotation)." + ), + ): + importlib.import_module("tests.workflow_decorators.dag_return_none_error").w + + +def test_passing_input_as_kwarg_errors(global_config_fixture): + with pytest.raises( + SyntaxError, + match=re.escape( + "Found Input argument(s) in kwargs: ['concat_input']. Input must be passed as a positional-only argument." + ), + ): + importlib.import_module("tests.workflow_decorators.dag_kwarg_error").w + + +def test_script_single_non_input_arg_errors(global_config_fixture): + global_config_fixture.experimental_features["decorator_syntax"] = True + w = Workflow() + + # WHEN/THEN + with pytest.raises( + SyntaxError, match=re.escape("script decorator must be used with a single `Input` arg, or no args.") + ): + + @w.script() + def two_args_script_func(_1: int) -> None: + pass + + +def test_script_regular_args_errors(global_config_fixture): + global_config_fixture.experimental_features["decorator_syntax"] = True + w = Workflow() + + # WHEN/THEN + with pytest.raises( + SyntaxError, match=re.escape("script decorator must be used with a single `Input` arg, or no args.") + ): + + @w.script() + def two_args_script_func(_1: int, _2: str) -> None: + pass + + +def test_dag_set_output_result_errors(global_config_fixture): + global_config_fixture.experimental_features["decorator_syntax"] = True + w = Workflow() + + # WHEN/THEN + with pytest.raises( + SyntaxError, match="Cannot set `result` or `exit_code` on Output when used in a dag/steps function." + ): + + @w.dag() + def set_output_result() -> Output: + return Output(result="foo") + + +def test_dag_set_output_exit_code_errors(global_config_fixture): + global_config_fixture.experimental_features["decorator_syntax"] = True + w = Workflow() + + # WHEN/THEN + with pytest.raises( + SyntaxError, match="Cannot set `result` or `exit_code` on Output when used in a dag/steps function." + ): + + @w.dag() + def set_output_result() -> Output: + return Output(exit_code=1) + + +def test_dag_set_outputs_subclass_errors(global_config_fixture): + global_config_fixture.experimental_features["decorator_syntax"] = True + w = Workflow() + + # WHEN/THEN + with pytest.raises( + SyntaxError, + match="Function return does not match annotation, " + "expected: ; got: .SubOfOutput'>.", + ): + + class SubOfOutput(Output): + pass + + @w.dag() + def return_subclass() -> Output: + return SubOfOutput() diff --git a/tests/test_unit/test_io_mixins.py b/tests/test_unit/test_io_mixins.py index f4f3e1161..012fc8007 100644 --- a/tests/test_unit/test_io_mixins.py +++ b/tests/test_unit/test_io_mixins.py @@ -79,7 +79,7 @@ class Foo(Input): foo: int bar: str = "a default" - assert Foo._get_artifacts() == [] + assert Foo._get_artifacts(add_missing_path=True) == [] def test_get_artifacts_with_pydantic_annotations(): @@ -87,7 +87,7 @@ class Foo(Input): foo: Annotated[int, Field(gt=0)] bar: Annotated[str, Field(max_length=10)] = "a default" - assert Foo._get_artifacts() == [] + assert Foo._get_artifacts(add_missing_path=True) == [] def test_get_artifacts_annotated_with_name(): @@ -96,7 +96,7 @@ class Foo(Input): bar: Annotated[str, Parameter(name="b_ar")] = "a default" baz: Annotated[str, Artifact(name="b_az")] - assert Foo._get_artifacts() == [Artifact(name="b_az", path="/tmp/hera-inputs/artifacts/b_az")] + assert Foo._get_artifacts(add_missing_path=True) == [Artifact(name="b_az", path="/tmp/hera-inputs/artifacts/b_az")] def test_get_artifacts_annotated_with_description(): @@ -105,7 +105,7 @@ class Foo(Input): bar: Annotated[str, Parameter(description="param bar")] = "a default" baz: Annotated[str, Artifact(description="artifact baz")] - assert Foo._get_artifacts() == [ + assert Foo._get_artifacts(add_missing_path=True) == [ Artifact(name="baz", path="/tmp/hera-inputs/artifacts/baz", description="artifact baz") ] @@ -114,7 +114,16 @@ def test_get_artifacts_annotated_with_path(): class Foo(Input): baz: Annotated[str, Artifact(path="/tmp/hera-inputs/artifacts/bishbosh")] - assert Foo._get_artifacts() == [Artifact(name="baz", path="/tmp/hera-inputs/artifacts/bishbosh")] + assert Foo._get_artifacts(add_missing_path=True) == [ + Artifact(name="baz", path="/tmp/hera-inputs/artifacts/bishbosh") + ] + + +def test_get_artifacts_annotated_do_not_add_path(): + class Foo(Input): + baz: Annotated[str, Artifact()] + + assert Foo._get_artifacts(add_missing_path=False) == [Artifact(name="baz")] def test_get_artifacts_with_multiple_annotations(): @@ -123,7 +132,7 @@ class Foo(Input): bar: Annotated[str, Field(max_length=10), Parameter(description="param bar")] = "a default" baz: Annotated[str, Field(max_length=15), Artifact()] - assert Foo._get_artifacts() == [Artifact(name="baz", path="/tmp/hera-inputs/artifacts/baz")] + assert Foo._get_artifacts(add_missing_path=True) == [Artifact(name="baz", path="/tmp/hera-inputs/artifacts/baz")] def test_get_as_arguments_unannotated(): @@ -554,9 +563,9 @@ class Foo(Output): parameters = foo._get_as_invocator_output() assert parameters == [ - Parameter(name="foo", value_from=ValueFrom(parameter="{{...foo}}")), - Parameter(name="bar", value_from=ValueFrom(parameter="{{...bar}}")), - Artifact(name="baz", from_="{{...baz}}"), + Parameter(name="foo", description="param foo", value_from=ValueFrom(parameter="{{...foo}}")), + Parameter(name="bar", description="param bar", value_from=ValueFrom(parameter="{{...bar}}")), + Artifact(name="baz", description="artifact baz", from_="{{...baz}}"), ] @@ -571,6 +580,6 @@ class Foo(Output): assert parameters == [ Parameter(name="f_oo", value_from=ValueFrom(parameter="{{...foo}}")), - Parameter(name="bar", value_from=ValueFrom(parameter="{{...bar}}")), + Parameter(name="bar", description="param bar", value_from=ValueFrom(parameter="{{...bar}}")), Artifact(name="baz", from_="{{...baz}}"), ] diff --git a/tests/workflow_decorators/dag.py b/tests/workflow_decorators/dag.py index 0e395ba83..feba26b45 100644 --- a/tests/workflow_decorators/dag.py +++ b/tests/workflow_decorators/dag.py @@ -43,6 +43,6 @@ def worker(worker_input: WorkerInput) -> WorkerOutput: setup_task = setup() task_a = concat(ConcatInput(word_a=worker_input.value_a, word_b=setup_task.environment_parameter)) task_b = concat(ConcatInput(word_a=worker_input.value_b, word_b=setup_task.result)) - final_task = concat(ConcatInput(word_a=task_a.result, word_b=task_b.result)) + final_task = concat(ConcatInput(word_a=task_a.result, word_b=task_b.result), name="final-task-name") return WorkerOutput(value=final_task.result) diff --git a/tests/workflow_decorators/dag_kwarg_error.py b/tests/workflow_decorators/dag_kwarg_error.py new file mode 100644 index 000000000..34acc2bd9 --- /dev/null +++ b/tests/workflow_decorators/dag_kwarg_error.py @@ -0,0 +1,23 @@ +from hera.shared import global_config +from hera.workflows import Input, Output, Workflow + +global_config.experimental_features["decorator_syntax"] = True + + +w = Workflow(generate_name="my-workflow-") + + +class ConcatInput(Input): + word_a: str + word_b: str + + +@w.script() +def concat(concat_input: ConcatInput) -> Output: + return Output(result=f"{concat_input.word_a} {concat_input.word_b}") + + +@w.set_entrypoint +@w.dag() +def worker() -> None: + concat(concat_input=ConcatInput(word_a="hello", word_b="world")) diff --git a/tests/workflow_decorators/dag_return_none_error.py b/tests/workflow_decorators/dag_return_none_error.py new file mode 100644 index 000000000..b25f0e1e6 --- /dev/null +++ b/tests/workflow_decorators/dag_return_none_error.py @@ -0,0 +1,22 @@ +from hera.shared import global_config +from hera.workflows import Input, Output, Workflow + +global_config.experimental_features["decorator_syntax"] = True + + +class ExampleOutput(Output): + field: str + + +w = Workflow(generate_name="my-workflow-") + + +@w.script() +def my_script(_: Input) -> ExampleOutput: + return ExampleOutput(field="Hello world!") + + +@w.dag() +def dag(_: Input) -> None: + task = my_script(Input()) + return ExampleOutput(field=task.field) diff --git a/tests/workflow_decorators/dag_return_task_error.py b/tests/workflow_decorators/dag_return_task_error.py new file mode 100644 index 000000000..e89b8063e --- /dev/null +++ b/tests/workflow_decorators/dag_return_task_error.py @@ -0,0 +1,23 @@ +from hera.shared import global_config +from hera.workflows import Input, Output, Workflow + +global_config.experimental_features["decorator_syntax"] = True + + +class ExampleOutput(Output): + field: str + + +w = Workflow(generate_name="my-workflow-") + + +@w.script() +def my_script(_: Input) -> ExampleOutput: + return ExampleOutput(field="Hello world!") + + +@w.dag() +def dag(_: Input) -> ExampleOutput: + # example_output is actually a Task when running the DAG decorator logic + example_output = my_script(Input()) + return example_output diff --git a/tests/workflow_decorators/dag_return_unannotated_error.py b/tests/workflow_decorators/dag_return_unannotated_error.py new file mode 100644 index 000000000..3a8692e50 --- /dev/null +++ b/tests/workflow_decorators/dag_return_unannotated_error.py @@ -0,0 +1,22 @@ +from hera.shared import global_config +from hera.workflows import Input, Output, Workflow + +global_config.experimental_features["decorator_syntax"] = True + + +class ExampleOutput(Output): + field: str + + +w = Workflow(generate_name="my-workflow-") + + +@w.script() +def my_script(_: Input) -> ExampleOutput: + return ExampleOutput(field="Hello world!") + + +@w.dag() +def dag(_: Input): + task = my_script(Input()) + return ExampleOutput(field=task.field)