feat: Long waited annotated pattern

* docs: updating examples

* refactor: nodes as modules

* refactor: symbol changes

* refactor: symbol changes

* fix: remove interference with logging within the application

* chore: minor changes to the logging structure

* fix: log messages being part of the catalog

* test: removing assertion to have parallel step root to be part of logs

* test: removing assertion to have parallel step root to be part of logs

* chore: formatting

* refactor: getting annotated pattern

* refactor: pipeline in good shape except for notebook

* feat: refactored to a much simpler code base

* fix: notebooks work now

* fix: adding better tests

* test: writting more unit tests

* test: writting more unit tests

* refactor: working pipeline execution

* refactor: working pipeline execution

* fix: removing unwanted examples

* feat: Jobs working

* feat: Jobs working

* feat: Revamped codebase with emulator test pattern

* feat: Revamped codebase with emulator test pattern

* feat: Revamped codebase with emulator test pattern

* feat: Revamped codebase with emulator test pattern

Author: vijayvammi

.github/workflows/pr.yaml

@@ -39,7 +39,7 @@ jobs:
         run: uv python install
 
       - name: Install the project
-        run: uv sync --all-extras --dev --frozen
+        run: uv sync --dev --frozen --extra examples --extra notebook
 
       - name: Run lint
         # For example, using `flake8`

.pre-commit-config.yaml

@@ -17,9 +17,9 @@ repos:
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]
-        exclude: ^tests
+        exclude: ^(tests/|examples)
       - id: ruff-format
-        exclude: ^tests
+        exclude: ^(tests/|examples)
   - repo: https://github.com/jorisroovers/gitlint
     rev: v0.19.1
     hooks:

Dockerfile

@@ -22,6 +22,7 @@ WORKDIR /app
 ENV http_proxy=
 ENV https_proxy=
 
-RUN uv sync --index https://artifactory.astrazeneca.net/api/pypi/pypi-virtual/simple/ --frozen
+RUN uv sync --verbose  --frozen --extra examples --extra notebook --extra k8s
+
 
 ENV PATH="/app/.venv/bin:$PATH"

examples/01-tasks/notebook.py

@@ -3,10 +3,12 @@
 
     python examples/01-tasks/notebook.py
 
-The notebook is executed in the same environment so any installed packages are available for the
+The notebook is executed in the same environment
+so any installed packages are available for the
 notebook.
 
-Upon successful execution, the output notebook with cell outputs is stored in the catalog.
+Upon successful execution, the output notebook with
+cell outputs is stored in the catalog.
 
 """
 
@@ -20,7 +22,6 @@ def main():
     hello_task = NotebookTask(
         name="hello",
         notebook="examples/common/simple_notebook.ipynb",
-        terminate_with_success=True,
     )
 
     # The pipeline has only one step.

examples/01-tasks/python_task_as_pipeline.py

@@ -12,13 +12,11 @@
 
 
 def main():
-    # Create a tasks which calls the function "hello"
-    # If this step executes successfully,
-    # the pipeline will terminate with success
+    # A single step pipeline can also be created using the
+    # task.as_pipeline() method.
     pipeline = PythonTask(
         name="hello",
         function=hello,
-        terminate_with_success=True,
     ).as_pipeline()
 
     pipeline.execute()

examples/01-tasks/python_tasks.py

@@ -18,7 +18,6 @@ def main():
     hello_task = PythonTask(
         name="hello",
         function=hello,
-        terminate_with_success=True,
     )
 
     # The pipeline has only one step.

examples/01-tasks/scripts.py

@@ -3,8 +3,10 @@
 
     python examples/01-tasks/scripts.py
 
-The command can be anything that can be executed in a shell.
-The stdout/stderr of the execution is captured as execution log and stored in the catalog.
+The command can be anything that can be
+executed in a shell.
+The stdout/stderr of the execution is
+captured as execution log and stored in the catalog.
 
 """
 
@@ -16,7 +18,6 @@ def main():
     hello_task = ShellTask(
         name="hello",
         command="echo 'Hello World!'",
-        terminate_with_success=True,
     )
 
     # The pipeline has only one step.
@@ -29,3 +30,4 @@ def main():
 
 if __name__ == "__main__":
     main()
+    main()

examples/01-tasks/scripts.yaml

@@ -11,5 +11,5 @@ dag:
     shell:
       type: task
       command_type: shell
-      command: echo "hello world!!" # The path is relative to the root of the project.
+      command: echo "hello world!!"
       next: success

examples/01-tasks/stub.py

@@ -3,7 +3,7 @@
 
     step 1 >> step 2 >> step 3 >> success
 
-    All the steps are mocked and they will just pass through.
+    All the steps are stubbed and they will just pass through.
     Use this pattern to define the skeleton of your pipeline
     and flesh out the steps later.
 
@@ -27,7 +27,7 @@ def main():
     # mature pipelines
     step2 = Stub(name="step2", what="is this thing")
 
-    step3 = Stub(name="step3", terminate_with_success=True)
+    step3 = Stub(name="step3")
 
     pipeline = Pipeline(steps=[step1, step2, step3])
 

examples/01-tasks/stub.yaml

@@ -4,7 +4,7 @@ dag:
 
     step 1 >> step 2 >> step 3 >> success
 
-    All the steps are mocked and they will just pass through.
+    All the steps are stubbed and they will just pass through.
     Use this pattern to define the skeleton of your pipeline
     and flesh out the steps later.