packaged-workflow gold path works end-to-end: compiler fixes, sandbox excision, canonical example, harness + CI coverage

.angreal/demos/features/features.py

@@ -1,5 +1,6 @@
 """demos features — run feature-focused Cloacina examples."""
 
+import json
 import shutil
 import subprocess
 
@@ -40,6 +41,32 @@ def _cmd():
     for name, path in get_rust_feature_directories()
 }
 
+# Bespoke feature demos defined below (excluded from auto-registration because
+# `cargo run` is the wrong verb for them). Keep this list next to their
+# definitions — `demos matrix` includes it so CI runs them too.
+_BESPOKE_FEATURES = ["python-workflow", "simple-packaged"]
+
+
+@demos()
+@angreal.command(
+    name="matrix",
+    about="emit every runnable feature demo as a JSON list (the CI examples matrix source)",
+    when_to_use=[
+        "generating the CI rust-examples matrix (examples-docs.yml discover job)",
+        "checking which examples the harness will execute",
+    ],
+    when_not_to_use=["running the demos themselves (use demos features <name>)"],
+)
+def matrix():
+    """CI executes ALL runnable examples. This is the single source of truth:
+    the same discovery that registers `demos features <name>` commands feeds
+    the CI matrix, so a new example directory automatically joins CI — no
+    hand-maintained workflow list to drift."""
+    names = sorted(
+        [name.replace("_", "-") for name in _rust_feature_commands] + _BESPOKE_FEATURES
+    )
+    print(json.dumps(names))
+
 
 # --- bespoke Python-workflow feature demo -----------------------------------
 
@@ -97,3 +124,149 @@ def python_workflow():
     finally:
         if venv_path.exists():
             shutil.rmtree(venv_path)
+
+
+# --- canonical packaged-workflow demo (the gold path) ------------------------
+#
+# `simple-packaged` is the CANONICAL example (CLOACI-I-0138): it runs through
+# the PRIMARY interface — pack → upload → (server compiles & reconciles) →
+# workflow run → execution Completed — never an in-process runner. It's in the
+# auto-registration exclude list because `cargo run` is the wrong verb for it;
+# this bespoke command drives the real lifecycle instead, reusing the
+# service-lifecycle helpers from the compiler e2e harness.
+
+@demos()
+@features()
+@angreal.command(
+    name="simple-packaged",
+    about="run the canonical packaged example through the primary interface (pack → upload → compile → run)",
+    long_about=(
+        "Drives examples/features/workflows/simple-packaged the way the README "
+        "says a user does: postgres (dev stack) + cloacina-server + "
+        "cloacina-compiler (with --dev-workspace so the example's crates.io "
+        "version deps resolve against this checkout), then cloacinactl "
+        "pack → upload → poll the build to success → workflow run "
+        "data_processing → poll the execution to Completed. First run "
+        "cold-compiles the package deps (~5-10 min); warm cache finishes in "
+        "under a minute."
+    ),
+    when_to_use=[
+        "verifying the packaged/server gold path end to end",
+        "validating the canonical example after compiler/server changes",
+    ],
+    when_not_to_use=[
+        "compiler-pipeline regression assertions (use `test e2e compiler`)",
+        "running without docker",
+    ],
+)
+def simple_packaged():
+    import tempfile
+    from pathlib import Path
+
+    from test.e2e.compiler import (
+        _assert_ports_free,
+        _build_binaries,
+        _cloacinactl,
+        _kill,
+        _poll_build_status,
+        _poll_execution_status,
+        _poll_run_workflow,
+        _start_postgres,
+        _upload,
+        _wait_http,
+    )
+
+    print("=== simple-packaged: packaged-workflow gold path ===")
+    _build_binaries()
+    _start_postgres()
+    # Distinct ports from the other harnesses (compiler e2e 18083/19003,
+    # ui e2e 18085/19001) so lanes can't collide on a shared machine.
+    _assert_ports_free(18087, 19005)
+
+    db_url = "postgres://cloacina:cloacina@localhost:15432/cloacina"
+    bootstrap_key = "demo-simple-packaged-key"
+    server_bind = "127.0.0.1:18087"
+    compiler_bind = "127.0.0.1:19005"
+
+    example_dir = PROJECT_ROOT / "examples" / "features" / "workflows" / "simple-packaged"
+    home = Path(tempfile.mkdtemp(prefix="simple-packaged-demo-"))
+    print(f"demo home (service logs): {home}")
+
+    server_proc = None
+    compiler_proc = None
+    try:
+        server_log = open(home / "server.log", "w")
+        server_proc = subprocess.Popen(
+            [
+                "target/debug/cloacina-server",
+                "--home", str(home),
+                "--database-url", db_url,
+                "--bind", server_bind,
+                "--bootstrap-key", bootstrap_key,
+                "--verbose",
+            ],
+            cwd=PROJECT_ROOT,
+            stdout=server_log,
+            stderr=subprocess.STDOUT,
+        )
+        _wait_http(f"http://{server_bind}/health", "server", proc=server_proc)
+        print("  ok: server up")
+
+        # Shared target cache with the compiler e2e so the cloacina deps only
+        # cold-compile once across all harness lanes.
+        shared_target = PROJECT_ROOT / "target" / "compiler-e2e-cache"
+        shared_target.mkdir(parents=True, exist_ok=True)
+        compiler_log = open(home / "compiler.log", "w")
+        compiler_proc = subprocess.Popen(
+            [
+                "target/debug/cloacina-compiler",
+                "--home", str(home),
+                "--database-url", db_url,
+                "--bind", compiler_bind,
+                "--poll-interval-ms", "500",
+                "--cargo-target-dir", str(shared_target),
+                "--cargo-flag=build",
+                "--cargo-flag=--lib",
+                # DEV ESCAPE HATCH (CLOACI-T-0887): the example ships crates.io
+                # version deps (the form users ship); resolve them against THIS
+                # checkout's unpublished crates.
+                "--dev-workspace", str(PROJECT_ROOT),
+                "--verbose",
+            ],
+            cwd=PROJECT_ROOT,
+            stdout=compiler_log,
+            stderr=subprocess.STDOUT,
+        )
+        _wait_http(
+            f"http://{compiler_bind}/health", "compiler",
+            timeout_s=60.0, proc=compiler_proc,
+        )
+        print("  ok: compiler up")
+
+        _cloacinactl(
+            home,
+            "config", "profile", "set", "local", f"http://{server_bind}",
+            "--api-key", bootstrap_key,
+            "--default",
+        )
+
+        print("  pack + upload (server compiles the source archive; "
+              "first run: ~5-10 min cold build)")
+        pkg_id = _upload(home, example_dir)
+        _poll_build_status(home, pkg_id, {"success"}, timeout_s=900.0)
+        print("  ok: build_status = success")
+
+        exec_id = _poll_run_workflow(home, "data_processing", timeout_s=180.0)
+        print(f"  ok: workflow run accepted (execution {exec_id})")
+
+        _poll_execution_status(home, exec_id, {"Completed"}, timeout_s=300.0)
+        print("  ok: execution Completed")
+
+        print(
+            "\nSUCCESS: gold path verified — "
+            "pack → upload → compile → reconcile → execute → Completed"
+        )
+        return 0
+    finally:
+        _kill(compiler_proc)
+        _kill(server_proc)

.angreal/demos/tutorials/python.py

@@ -1,5 +1,6 @@
 """demos tutorials python — run individual Python tutorial examples."""
 
+import os
 import shutil
 import subprocess
 import sys
@@ -68,6 +69,12 @@ def _run_python_tutorial(tutorial_num, tutorial_rel_path, backend="sqlite"):
             print(f"[diagnostic] smart_postgres_reset returned {reset_ok}", flush=True)
 
         print(f"Executing tutorial {tutorial_num}...", flush=True)
+        # The harness owns the DB wiring: the dev stack publishes postgres on
+        # host 15432 (not 5432 — that's the user's own DB). Tutorials honor
+        # DATABASE_URL and keep a user-facing 5432 fallback.
+        env = os.environ.copy()
+        if backend == "postgres":
+            env["DATABASE_URL"] = "postgres://cloacina:cloacina@localhost:15432/cloacina"
         # `python -u` forces unbuffered stdio in the child so CI sees
         # progress + tracebacks even if the tutorial crashes mid-stream.
         result = subprocess.run(
@@ -76,6 +83,7 @@ def _run_python_tutorial(tutorial_num, tutorial_rel_path, backend="sqlite"):
             capture_output=True,
             text=True,
             timeout=300,
+            env=env,
         )
 
         if result.returncode == 0:

.angreal/docker-compose.yaml

@@ -1,4 +1,10 @@
 ---
+# Pinned project name so this backing-services stack is unambiguously Cloacina's
+# and never collides with the default `angreal` project used by other angreal
+# repos on the same machine. `angreal ui`'s full demo/e2e stack is a SEPARATE
+# project (`cloacina-demo`, docker/docker-compose.demo.yml); `angreal purge`
+# tears down both.
+name: cloacina-dev
 services:
   postgres:
     image: postgres:16
@@ -8,7 +14,7 @@ services:
       POSTGRES_PASSWORD: cloacina
       POSTGRES_DB: cloacina
     ports:
-      - "5432:5432"
+      - "15432:5432"
     volumes:
       - postgres_data:/var/lib/postgresql/data
     command: postgres -c max_connections=500

.angreal/task_purge.py

@@ -109,18 +109,30 @@ def _cargo_cache_dirs() -> List[Path]:
 
 
 def _docker_compose_down(cwd: Path) -> int:
-    """Stop docker services and remove volumes. Returns docker's exit code,
-    or 0 if compose isn't on PATH."""
-    compose_file = cwd / ".angreal" / "docker-compose.yaml"
-    if not compose_file.exists():
-        return 0
+    """Stop EVERY Cloacina docker stack and remove volumes. Returns the worst
+    exit code, or 0 if compose isn't on PATH.
+
+    Purge must clean ALL stacks a command can bring up — otherwise a stale one
+    (notably the `angreal ui` demo stack) survives a purge and its DB pool
+    later exhausts. Historically this only tore down the backing-services stack
+    and left `cloacina-demo` running for days."""
     if shutil.which("docker") is None:
         return 0
-    return subprocess.run(
-        ["docker", "compose", "-f", str(compose_file), "down", "-v", "--remove-orphans"],
-        cwd=str(cwd),
-        check=False,
-    ).returncode
+    compose_files = [
+        cwd / ".angreal" / "docker-compose.yaml",    # backing services (angreal services / test integration)
+        cwd / "docker" / "docker-compose.demo.yml",  # demo / e2e stack (angreal ui)
+    ]
+    rc = 0
+    for compose_file in compose_files:
+        if not compose_file.exists():
+            continue
+        code = subprocess.run(
+            ["docker", "compose", "-f", str(compose_file), "down", "-v", "--remove-orphans"],
+            cwd=str(cwd),
+            check=False,
+        ).returncode
+        rc = rc or code
+    return rc
 
 
 # ---------------------------------------------------------------------------

.angreal/test/auth.py

@@ -116,7 +116,7 @@ def auth_integration_test():
     start_postgres()
 
     base_url = "http://127.0.0.1:18082"
-    db_url = "postgres://cloacina:cloacina@localhost:5432/cloacina"
+    db_url = "postgres://cloacina:cloacina@localhost:15432/cloacina"
     test_home = Path("target/auth-integration-test")
     if test_home.exists():
         import shutil

.angreal/test/e2e/cli.py

@@ -109,7 +109,7 @@ def cli():
     _build_binaries()
     _start_postgres()
 
-    db_url = "postgres://cloacina:cloacina@localhost:5432/cloacina"
+    db_url = "postgres://cloacina:cloacina@localhost:15432/cloacina"
     bootstrap_key = "test-bootstrap-e2e"
     bind = "127.0.0.1:18082"
     base_url = f"http://{bind}"
@@ -659,7 +659,7 @@ def default_executor():
     _build_binaries()
     _start_postgres()
 
-    db_url = "postgres://cloacina:cloacina@localhost:5432/cloacina"
+    db_url = "postgres://cloacina:cloacina@localhost:15432/cloacina"
 
     # --- negative: unknown executor key aborts startup ----------------------
     with tempfile.TemporaryDirectory() as home_s: