Add ability to get task output parameters, more syntactic sugar around complex dependencies, and easy way to define env variables (#517)

Signed-off-by: Flaviu Vadan <[email protected]>

Author: flaviuvadan

docs/examples/workflows/complex_deps.md

@@ -0,0 +1,83 @@
+# Complex Deps
+
+
+
+
+
+
+=== "Hera"
+
+    ```python linenums="1"
+    from hera.workflows import DAG, Workflow, script
+
+
+    @script()
+    def foo(p):
+        if p < 0.5:
+            raise Exception(p)
+        print(42)
+
+
+    with Workflow(generate_name="complex-deps-", entrypoint="d") as w:
+        with DAG(name="d"):
+            A = foo(name="a", arguments={"p": 0.6})
+            B = foo(name="b", arguments={"p": 0.3})
+            C = foo(name="c", arguments={"p": 0.7})
+            D = foo(name="d", arguments={"p": 0.9})
+            # here, D depends on A, B, and C. If A succeeds and one of B or C fails, D still runs
+            A >> [B, C], [A, (B | C)] >> D
+    ```
+
+=== "YAML"
+
+    ```yaml linenums="1"
+    apiVersion: argoproj.io/v1alpha1
+    kind: Workflow
+    metadata:
+      generateName: complex-deps-
+    spec:
+      entrypoint: d
+      templates:
+      - dag:
+          tasks:
+          - arguments:
+              parameters:
+              - name: p
+                value: '0.6'
+            name: a
+            template: foo
+          - arguments:
+              parameters:
+              - name: p
+                value: '0.3'
+            depends: a
+            name: b
+            template: foo
+          - arguments:
+              parameters:
+              - name: p
+                value: '0.7'
+            depends: a
+            name: c
+            template: foo
+          - arguments:
+              parameters:
+              - name: p
+                value: '0.9'
+            depends: a && (b || c)
+            name: d
+            template: foo
+        name: d
+      - inputs:
+          parameters:
+          - name: p
+        name: foo
+        script:
+          command:
+          - python
+          image: python:3.7
+          source: "import os\nimport sys\nsys.path.append(os.getcwd())\nimport json\n\
+            try: p = json.loads(r'''{{inputs.parameters.p}}''')\nexcept: p = r'''{{inputs.parameters.p}}'''\n\
+            \nif p < 0.5:\n    raise Exception(p)\nprint(42)\n"
+    ```
+

docs/examples/workflows/dag_with_param_passing.md

@@ -0,0 +1,75 @@
+# Dag With Param Passing
+
+
+
+
+
+
+=== "Hera"
+
+    ```python linenums="1"
+    from hera.workflows import DAG, Container, Parameter, Task, Workflow
+
+    with Workflow(generate_name="param-passing-", entrypoint="d") as w:
+        out = Container(
+            name="out",
+            image="docker/whalesay:latest",
+            command=["cowsay"],
+            outputs=Parameter(name="x", value=42),
+        )
+        in_ = Container(
+            name="in",
+            image="docker/whalesay:latest",
+            command=["cowsay"],
+            args=["{{inputs.parameters.a}}"],
+            inputs=Parameter(name="a"),
+        )
+        with DAG(name="d"):
+            t1 = Task(name="a", template=out)
+            t2 = Task(name="b", template=in_, arguments=t1.get_parameter("x").with_name("a"))
+            t1 >> t2
+    ```
+
+=== "YAML"
+
+    ```yaml linenums="1"
+    apiVersion: argoproj.io/v1alpha1
+    kind: Workflow
+    metadata:
+      generateName: param-passing-
+    spec:
+      entrypoint: d
+      templates:
+      - container:
+          command:
+          - cowsay
+          image: docker/whalesay:latest
+        name: out
+        outputs:
+          parameters:
+          - name: x
+            value: '42'
+      - container:
+          args:
+          - '{{inputs.parameters.a}}'
+          command:
+          - cowsay
+          image: docker/whalesay:latest
+        inputs:
+          parameters:
+          - name: a
+        name: in
+      - dag:
+          tasks:
+          - name: a
+            template: out
+          - arguments:
+              parameters:
+              - name: a
+                value: '{{tasks.a.outputs.parameters.x}}'
+            depends: a
+            name: b
+            template: in
+        name: d
+    ```
+

docs/examples/workflows/dag_with_script_param_passing.md

@@ -0,0 +1,92 @@
+# Dag With Script Param Passing
+
+
+
+
+
+
+=== "Hera"
+
+    ```python linenums="1"
+    from hera.workflows import DAG, Parameter, Task, Workflow, script
+
+
+    @script()
+    def out():
+        print(42)
+
+
+    @script()
+    def in_(a):
+        print(a)
+
+
+    with Workflow(generate_name="script-param-passing-", entrypoint="d") as w:
+        with DAG(name="d"):
+            t1: Task = out()
+            t2 = in_(arguments=Parameter(name="a", value=t1.result))
+            t1 >> t2
+    ```
+
+=== "YAML"
+
+    ```yaml linenums="1"
+    apiVersion: argoproj.io/v1alpha1
+    kind: Workflow
+    metadata:
+      generateName: script-param-passing-
+    spec:
+      entrypoint: d
+      templates:
+      - dag:
+          tasks:
+          - name: out
+            template: out
+          - arguments:
+              parameters:
+              - name: a
+                value: '{{tasks.out.outputs.result}}'
+            depends: out
+            name: in-
+            template: in-
+        name: d
+      - name: out
+        script:
+          command:
+          - python
+          image: python:3.7
+          source: 'import os
+
+            import sys
+
+            sys.path.append(os.getcwd())
+
+            print(42)
+
+            '
+      - inputs:
+          parameters:
+          - name: a
+        name: in-
+        script:
+          command:
+          - python
+          image: python:3.7
+          source: 'import os
+
+            import sys
+
+            sys.path.append(os.getcwd())
+
+            import json
+
+            try: a = json.loads(r''''''{{inputs.parameters.a}}'''''')
+
+            except: a = r''''''{{inputs.parameters.a}}''''''
+
+
+            print(a)
+
+            '
+    ```
+

docs/examples/workflows/multi_env.md

@@ -0,0 +1,75 @@
+# Multi Env
+
+
+
+
+
+
+=== "Hera"
+
+    ```python linenums="1"
+    from hera.workflows import DAG, Workflow, script
+
+
+    @script(env={"a": 1, "b": 2, "c": 3})
+    def env():
+        import os
+
+        # note that env params come in as strings
+        assert os.environ["a"] == "1", os.environ["a"]
+        assert os.environ["b"] == "2", os.environ["b"]
+        assert os.environ["c"] == "3", os.environ["c"]
+
+
+    with Workflow(generate_name="multi-env-", entrypoint="d") as w:
+        with DAG(name="d"):
+            env()
+    ```
+
+=== "YAML"
+
+    ```yaml linenums="1"
+    apiVersion: argoproj.io/v1alpha1
+    kind: Workflow
+    metadata:
+      generateName: multi-env-
+    spec:
+      entrypoint: d
+      templates:
+      - dag:
+          tasks:
+          - name: env
+            template: env
+        name: d
+      - name: env
+        script:
+          command:
+          - python
+          env:
+          - name: a
+            value: '1'
+          - name: b
+            value: '2'
+          - name: c
+            value: '3'
+          image: python:3.7
+          source: 'import os
+
+            import sys
+
+            sys.path.append(os.getcwd())
+
+            import os
+
+
+            # note that env params come in as strings
+
+            assert os.environ["a"] == "1", os.environ["a"]
+
+            assert os.environ["b"] == "2", os.environ["b"]
+
+            assert os.environ["c"] == "3", os.environ["c"]
+
+            '
+    ```
+