Migrate Dragon backend to 0.14.1

README.md

@@ -23,11 +23,11 @@ A unified runtime for executing **AI and HPC workloads** on supercomputing infra
 ```python
 import asyncio
 from rhapsody.api import Session, ComputeTask, AITask
-from rhapsody.backends import DragonExecutionBackendV3, DragonVllmInferenceBackend
+from rhapsody.backends import DragonExecutionBackend, DragonVllmInferenceBackend
 
 async def main():
     # Initialize backends
-    hpc_backend = await DragonExecutionBackendV3(name="hpc")
+    hpc_backend = await DragonExecutionBackend(name="hpc")
     ai_backend = await DragonVllmInferenceBackend(name="vllm", model="Qwen2.5-7B")
 
     # Create session with multiple backends

docs/getting-started/advanced-usage.md

@@ -216,8 +216,11 @@ named after the task UID (`task.000001.stdout`, `task.000001.stderr`).
 
 !!! note "Supported backends"
     `capture_stdio` is supported by `ConcurrentExecutionBackend`,
-    `DaskExecutionBackend`, and `DragonExecutionBackendV3`. Function tasks
-    (`function=`) are unaffected regardless of the flag value.
+    `DaskExecutionBackend`, and `DragonExecutionBackend`. On
+    `ConcurrentExecutionBackend` and `DaskExecutionBackend`, function tasks
+    (`function=`) are unaffected by this flag. On `DragonExecutionBackend`,
+    `capture_stdio` applies to **all** task types including function tasks — see
+    [Dragon: Task Output and File Paths](#dragon-task-output-and-file-paths).
 
 !!! note "Session work_dir"
     Files are placed in `{session.work_dir}/{session.uid}/`. Pass `work_dir=`
@@ -240,7 +243,7 @@ import asyncio
 import rhapsody
 from rhapsody.api import Session
 from rhapsody.api import ComputeTask
-from rhapsody.backends import ConcurrentExecutionBackend, DragonExecutionBackendV3
+from rhapsody.backends import ConcurrentExecutionBackend, DragonExecutionBackend
 
 
 async def compute_function():
@@ -250,7 +253,7 @@ async def compute_function():
 async def multi_backend_demo():
     # Initialize two different backends
     be_local = ConcurrentExecutionBackend(name="local_cpu")
-    be_remote = DragonExecutionBackendV3(name="hpc_gpu")
+    be_remote = DragonExecutionBackend(name="hpc_gpu")
     await asyncio.gather(be_local, be_remote)
 
 
@@ -302,12 +305,12 @@ For large-scale HPC deployments, the Dragon backend provides native integration
 import asyncio
 import rhapsody
 from rhapsody.api import Session, ComputeTask
-from rhapsody.backends import DragonExecutionBackendV3
+from rhapsody.backends import DragonExecutionBackend
 
 async def dragon_demo():
     # Initialize the Dragon backend (optimized for HPC)
     # V3 uses native Dragon Batch for high-performance wait/callbacks
-    dragon_be = await DragonExecutionBackendV3(
+    dragon_be = await DragonExecutionBackend(
         name="dragon_hpc"
     )
 
@@ -336,7 +339,7 @@ if __name__ == "__main__":
 
 ## Dragon Process Templates
 
-When using `DragonExecutionBackendV3`, you can pass Dragon-native process configuration to `ComputeTask` via the `task_backend_specific_kwargs` parameter. This exposes the `process_template` and `process_templates` options, which map directly to Dragon's [ProcessTemplate](https://dragonhpc.github.io/dragon/doc/_build/html/ref/native/dragon.native.process.html#dragon.native.process.ProcessTemplate).
+When using `DragonExecutionBackend`, you can pass Dragon-native process configuration to `ComputeTask` via the `task_backend_specific_kwargs` parameter. This exposes the `process_template` and `process_templates` options, which map directly to Dragon's [ProcessTemplate](https://dragonhpc.github.io/dragon/doc/_build/html/ref/native/dragon.native.process.html#dragon.native.process.ProcessTemplate).
 
 The `ProcessTemplate` accepts the following parameters:
 
@@ -351,7 +354,7 @@ The `ProcessTemplate` accepts the following parameters:
 | `options` | `ProcessOptions`, optional | Process options, such as allowing the process to connect to the infrastructure |
 
 !!! warning "Excluded Parameters"
-    `DragonExecutionBackendV3` manages `target` (the binary or Python callable), `args`, and `kwargs` internally through the `ComputeTask` interface. These three parameters are **not** available in `process_template` / `process_templates` — use `ComputeTask.executable`, `ComputeTask.arguments`, and `ComputeTask.function` instead.
+    `DragonExecutionBackend` manages `target` (the binary or Python callable), `args`, and `kwargs` internally through the `ComputeTask` interface. These three parameters are **not** available in `process_template` / `process_templates` — use `ComputeTask.executable`, `ComputeTask.arguments`, and `ComputeTask.function` instead.
 
 ### Single-Process Template
 
@@ -409,7 +412,7 @@ from dragon.infrastructure.policy import Policy
 
 import rhapsody
 from rhapsody.api import ComputeTask, Session
-from rhapsody.backends import DragonExecutionBackendV3
+from rhapsody.backends import DragonExecutionBackend
 
 
 from dragon.native.machine import System, Node
@@ -446,7 +449,7 @@ def make_policies(all_gpus, nprocs=32):
 
 
 async def main():
-    backend = await DragonExecutionBackendV3()
+    backend = await DragonExecutionBackend()
     session = Session(backends=[backend])
 
     all_gpus = find_gpus()  # e.g. [("node-0", 0), ("node-0", 1), ("node-1", 0), ...]
@@ -497,11 +500,11 @@ The following example demonstrates mixing native functions, single-process tasks
 import asyncio
 import rhapsody
 from rhapsody.api import ComputeTask, Session
-from rhapsody.backends import DragonExecutionBackendV3
+from rhapsody.backends import DragonExecutionBackend
 
 
 async def main():
-    backend = await DragonExecutionBackendV3()
+    backend = await DragonExecutionBackend()
     session = Session(backends=[backend])
 
     async def single_function():
@@ -573,6 +576,25 @@ if __name__ == "__main__":
     dragon my_heterogeneous_workload.py
     ```
 
+## Dragon: Task Output and File Paths
+
+After a task completes, `task.stdout` and `task.stderr` work as follows:
+
+- **`capture_stdio=False` (default)** — content is an inline string (whatever the task printed). Read it directly: `task.stdout`.
+- **`capture_stdio=True`** — content is a file path under the Rhapsody session directory (`rhapsody.session.{uid}/task.000001.stdout`). Open it to read: `open(task.stdout).read()`.
+
+!!! note "Performance at scale"
+    At 100K+ tasks on shared filesystems, the default file-read overhead (~100–400 s on Lustre) can be avoided by passing `batch_kwargs={"task_logs": False}` to `DragonExecutionBackend`. With this option, `task.stdout` and `task.stderr` will be **empty** for all `capture_stdio=False` tasks — output goes to the console only. A warning is logged when this option is active.
+
+!!! warning "Functions launched via process_template or process_templates"
+    If you use `function=` together with `process_template` or `process_templates`,
+    Dragon runs the callable as a subprocess — **not** in Dragon's native function pool.
+    In this mode `task.return_value` is the **exit code** (`0` on success), not the
+    Python return value of your function. Use a bare `ComputeTask(function=my_fn)` (no
+    templates) when you need the actual return value.
+
+---
+
 ## Mixed HPC and AI Workloads
 
 One of RHAPSODY's most powerful features is the ability to orchestrate traditional binaries alongside AI inference services (like vLLM).

docs/getting-started/configuration.md

@@ -62,9 +62,9 @@ ComputeTask(
 High-performance execution using the Dragon runtime.
 
 ```python
-from rhapsody.backends import DragonExecutionBackendV3
+from rhapsody.backends import DragonExecutionBackend
 
-backend = DragonExecutionBackendV3(
+backend = DragonExecutionBackend(
     name="dragon",
     num_nodes=4,            # Total nodes for the worker pool (optional)
     pool_nodes=2,           # Nodes per worker pool (optional)
@@ -80,12 +80,12 @@ backend = DragonExecutionBackendV3(
 | `disable_telemetry` | `bool` | `False` | Disable Dragon internal telemetry |
 
 !!! note "Streaming pipeline"
-    `DragonExecutionBackendV3` uses Dragon's streaming batch pipeline — tasks submitted via
+    `DragonExecutionBackend` uses Dragon's streaming batch pipeline — tasks submitted via
     `session.submit_tasks()` are dispatched individually by a continuously running background
     thread. There is no compile or start step; tasks begin executing as soon as they are submitted.
 
 !!! note "Dragon Versions"
-    While `DragonExecutionBackendV3` is recommended for most users and will be always maintained, V1 and V2 are also available for legacy compatibility.
+    `DragonExecutionBackend` is the only supported Dragon backend as of RHAPSODY 0.5.0.
 
 ### RADICAL-Pilot Backend
 Large-scale HPC execution using RADICAL-Pilot.

docs/getting-started/resource-specification.md

@@ -13,7 +13,7 @@ This design keeps the task API thin and makes resource semantics explicit per ba
 |---------|-------------|-----|-----|-------|
 | `ConcurrentExecutionBackend` | not supported | `task_backend_specific_kwargs={"cwd": "..."}` | `task_backend_specific_kwargs={"env": {...}}` | `task_backend_specific_kwargs={"shell": True}` |
 | `DaskExecutionBackend` | `task_backend_specific_kwargs={"resources": {"GPU": 1}}` | `task_backend_specific_kwargs={"cwd": "..."}` | `task_backend_specific_kwargs={"env": {...}}` | `task_backend_specific_kwargs={"shell": True}` |
-| `DragonExecutionBackendV3` | `task_backend_specific_kwargs={"process_template": {"policy": ...}}` | `task_backend_specific_kwargs={"process_template": {"cwd": "..."}}` | `task_backend_specific_kwargs={"process_template": {"env": {...}}}` | not applicable |
+| `DragonExecutionBackend` | `task_backend_specific_kwargs={"process_template": {"policy": ...}}` | `task_backend_specific_kwargs={"process_template": {"cwd": "..."}}` | `task_backend_specific_kwargs={"process_template": {"env": {...}}}` | not applicable |
 
 ---
 
@@ -140,7 +140,7 @@ backend = await DaskExecutionBackend(cluster=cluster)
 
 ## Dragon Backend (V3)
 
-`DragonExecutionBackendV3` submits tasks to the Dragon runtime. All per-task resource
+`DragonExecutionBackend` submits tasks to the Dragon runtime. All per-task resource
 and placement settings flow through Dragon's [ProcessTemplate](https://dragonhpc.github.io/dragon/doc/_build/html/ref/native/dragon.native.process.html#dragon.native.process.ProcessTemplate).
 
 There is **no backend-level `working_directory`** in V3. Every process
@@ -152,7 +152,7 @@ Pass a `batch_kwargs` dict when constructing the backend to tune the underlying
 `dragon.workflows.batch.Batch` instance:
 
 ```python
-backend = await DragonExecutionBackendV3(batch_kwargs={
+backend = await DragonExecutionBackend(batch_kwargs={
     "num_nodes": 8,
     "results_ddict_mem": 4 * 1024**3,  # 4 GiB results DDict
     "scheduler_workers": 16,

docs/hpc-machines/anvil.md

@@ -63,7 +63,7 @@ dragon <script.py>
 
 ## Example
 
-This example runs 32 MPI jobs concurrently using RHAPSODY's `Session` API with `DragonExecutionBackendV3`. Each job gets a random number of ranks (between 2 and 32), all scheduled across the 2 allocated nodes. Worker count is determined automatically by Dragon from the allocation.
+This example runs 32 MPI jobs concurrently using RHAPSODY's `Session` API with `DragonExecutionBackend`. Each job gets a random number of ranks (between 2 and 32), all scheduled across the 2 allocated nodes. Worker count is determined automatically by Dragon from the allocation.
 
 ```python title="rhapsody-mpi.py"
 import asyncio
@@ -73,7 +73,7 @@ import time
 
 import rhapsody
 from rhapsody.api import ComputeTask, Session
-from rhapsody.backends import DragonExecutionBackendV3
+from rhapsody.backends import DragonExecutionBackend
 
 rhapsody.enable_logging(level=logging.DEBUG)
 
@@ -100,7 +100,7 @@ async def main():
     sleepsecs = 2
     maxranks = 32
 
-    backend = await DragonExecutionBackendV3()
+    backend = await DragonExecutionBackend()
     session = Session(backends=[backend])
 
     print("--- Submitting Tasks ---")

docs/hpc-machines/bridges2.md

@@ -84,15 +84,15 @@ dragon -w ssh --network-config slurm.yaml -t tcp 01-workload-async-gather.py
 ## Example
 
 This example submits a heterogeneous workload mixing function tasks and
-executable tasks using RHAPSODY's `Session` API with `DragonExecutionBackendV3`.
+executable tasks using RHAPSODY's `Session` API with `DragonExecutionBackend`.
 
 ```python title="01-workload-async-gather.py"
 import asyncio
 import logging
 
 import rhapsody
 from rhapsody.api import ComputeTask, Session
-from rhapsody.backends import DragonExecutionBackendV3
+from rhapsody.backends import DragonExecutionBackend
 
 rhapsody.enable_logging(level=logging.DEBUG)
 
@@ -102,7 +102,7 @@ def compute(x: int) -> int:
 
 
 async def main():
-    backend = await DragonExecutionBackendV3()
+    backend = await DragonExecutionBackend()
     session = Session(backends=[backend])
 
     tasks = [ComputeTask(function=compute, args=(i,)) for i in range(16)]

docs/hpc-machines/index.md

@@ -6,7 +6,7 @@ This section provides machine-specific setup guides and verified examples for ru
 
 **`ConcurrentExecutionBackend (designed for debug and tests single node only)`** works out of the box on every machine listed here — no machine-specific configuration is required. Install RHAPSODY and run.
 
-**`DragonExecutionBackendV3 (designed for large scale multi-node)`** requires machine-specific setup: each HPC system has its own software stack, MPI library, interconnect, and job launcher. The exact steps — module loading, virtual environment creation, transport configuration, and launch command — differ per machine and are documented in each section below.
+**`DragonExecutionBackend (designed for large scale multi-node)`** requires machine-specific setup: each HPC system has its own software stack, MPI library, interconnect, and job launcher. The exact steps — module loading, virtual environment creation, transport configuration, and launch command — differ per machine and are documented in each section below.
 
 Each guide covers environment setup, module loading, job allocation, and working examples.
 

docs/hpc-machines/ncsa-delta.md

@@ -60,7 +60,7 @@ dragon <script.py>
 
 ## Example
 
-This example runs 32 MPI jobs concurrently using RHAPSODY's `Session` API with the `DragonExecutionBackendV3`. Each job gets a random number of ranks (between 2 and 32), all scheduled across the 2 allocated nodes. Worker count is determined automatically by Dragon from the allocation.
+This example runs 32 MPI jobs concurrently using RHAPSODY's `Session` API with the `DragonExecutionBackend`. Each job gets a random number of ranks (between 2 and 32), all scheduled across the 2 allocated nodes. Worker count is determined automatically by Dragon from the allocation.
 
 ```python title="mpi-rhapsody.py"
 import asyncio
@@ -70,7 +70,7 @@ import time
 
 import rhapsody
 from rhapsody.api import ComputeTask, Session
-from rhapsody.backends import DragonExecutionBackendV3
+from rhapsody.backends import DragonExecutionBackend
 
 rhapsody.enable_logging(level=logging.DEBUG)
 
@@ -97,7 +97,7 @@ async def main():
     sleepsecs = 2
     maxranks = 32
 
-    backend = await DragonExecutionBackendV3()
+    backend = await DragonExecutionBackend()
     session = Session(backends=[backend])
 
     print("--- Submitting Tasks ---")
@@ -140,7 +140,7 @@ dragon mpi-rhapsody.py
     2026-03-25 23:07:36,938 | DEBUG    | [api_setup] | got handshake
     2026-03-25 23:07:36,938 | INFO     | [api_setup] | debug entry hooked
     2026-03-25 23:07:36,942 | DEBUG    | [dragon.native.queue] | Created queue {self!r}
-    2026-03-25 23:07:37,648 | INFO     | [rhapsody.backends.execution.dragon] | DragonExecutionBackendV3: 128 workers, 2 managers
+    2026-03-25 23:07:37,648 | INFO     | [rhapsody.backends.execution.dragon] | DragonExecutionBackend: 128 workers, 2 managers
     2026-03-25 23:07:37,648 | DEBUG    | [rhapsody.backends.execution.dragon] | Starting Dragon backend V3 async initialization...
     2026-03-25 23:07:37,648 | DEBUG    | [rhapsody.backends.execution.dragon] | Registering backend states...
     2026-03-25 23:07:37,649 | DEBUG    | [rhapsody.backends.execution.dragon] | Registering task states...

docs/hpc-machines/perlmutter.md

@@ -79,7 +79,7 @@ dragon <script.py>
 This example demonstrates a two-phase HPC-AI workflow on Perlmutter:
 
 1. **Simulation phase** — run a batch of scientific simulations using LLM inference (`DragonVllmInferenceBackend`). Each `AITask` sends a domain-specific prompt and collects the generated output as synthetic simulation data.
-2. **Training phase** — fine-tune a small model on the collected simulation outputs using a `ComputeTask` dispatched through `DragonExecutionBackendV3`.
+2. **Training phase** — fine-tune a small model on the collected simulation outputs using a `ComputeTask` dispatched through `DragonExecutionBackend`.
 
 Both phases run within the same RHAPSODY `Session`, sharing the same Dragon runtime across GPU and CPU resources.
 
@@ -90,7 +90,7 @@ import multiprocessing as mp
 
 import rhapsody
 from rhapsody.api import AITask, ComputeTask, Session
-from rhapsody.backends import DragonExecutionBackendV3, DragonVllmInferenceBackend
+from rhapsody.backends import DragonExecutionBackend, DragonVllmInferenceBackend
 
 rhapsody.enable_logging(level=logging.INFO)
 
@@ -160,7 +160,7 @@ def fine_tune(simulation_outputs: list):
 async def main():
     mp.set_start_method("dragon")
 
-    execution_backend = await DragonExecutionBackendV3()
+    execution_backend = await DragonExecutionBackend()
 
     inference_backend = await DragonVllmInferenceBackend(
         config_file="config.yaml",

docs/index.md

@@ -19,11 +19,11 @@ RHAPSODY is a high-performance runtime system designed for orchestrating complex
 ```python
 import asyncio
 from rhapsody import Session, ComputeTask
-from rhapsody.backends import DragonExecutionBackendV3
+from rhapsody.backends import DragonExecutionBackend
 
 async def main():
     # 1. Initialize session with a backend
-    backend = await DragonExecutionBackendV3()
+    backend = await DragonExecutionBackend()
     async with Session(backends=[backend]) as session:
 
         # 2. Define a task

docs/integrations.md

@@ -43,14 +43,14 @@ import multiprocessing as mp
 
 from rhapsody import Session
 from rhapsody.api import AITask, ComputeTask
-from rhapsody.backends import DragonExecutionBackendV3, DragonVllmInferenceBackend
+from rhapsody.backends import DragonExecutionBackend, DragonVllmInferenceBackend
 
 async def main():
     # Set Dragon as the multiprocessing start method
     mp.set_start_method("dragon")
 
     # Initialize execution backend for compute tasks
-    execution_backend = await DragonExecutionBackendV3()
+    execution_backend = await DragonExecutionBackend()
 
     # Initialize inference backend for AI tasks
     inference_backend = DragonVllmInferenceBackend(
@@ -231,14 +231,14 @@ import asyncio
 import multiprocessing as mp
 
 from radical.asyncflow import WorkflowEngine
-from rhapsody.backends import DragonExecutionBackendV3
+from rhapsody.backends import DragonExecutionBackend
 
 async def main():
     # Set Dragon as the multiprocessing start method
     mp.set_start_method("dragon")
 
     # Initialize RHAPSODY backend
-    backend = await DragonExecutionBackendV3()
+    backend = await DragonExecutionBackend()
 
     # Create AsyncFlow workflow engine with RHAPSODY backend
     flow = await WorkflowEngine.create(backend=backend)
@@ -347,8 +347,8 @@ from rhapsody.backends import DaskExecutionBackend
 backend = await DaskExecutionBackend()
 
 # Dragon HPC
-from rhapsody.backends import DragonExecutionBackendV3
-backend = await DragonExecutionBackendV3()
+from rhapsody.backends import DragonExecutionBackend
+backend = await DragonExecutionBackend()
 
 # Create workflow with chosen backend
 flow = await WorkflowEngine.create(backend=backend)