diff --git a/.angreal/demos/_utils.py b/.angreal/demos/_utils.py index 7ad1770d2..08142274f 100644 --- a/.angreal/demos/_utils.py +++ b/.angreal/demos/_utils.py @@ -45,16 +45,56 @@ def get_rust_feature_directories(): features_dir = PROJECT_ROOT / "examples" / "features" if not features_dir.exists(): return [] - # Exclude examples that are libraries or not meant to be executed directly - excluded = {"validation-failures", "complex-dag", "packaged-workflows", "simple-packaged", "packaged-triggers", "python-workflow"} + # An example dir is EITHER embedded (has src + Cargo, run via `cargo run` — + # returned here) OR packaged (has a `package.toml`, run through the server + # gold path — discovered separately by `get_packaged_example_directories`). + # `package.toml` presence is the discriminator, so the two registrars never + # overlap and a packaged example can never be mis-registered as `cargo run`. + # A couple of embedded dirs are still hand-excluded: python-workflow is a + # bespoke wheel demo; validation-failures is a negative fixture. + excluded = {"validation-failures", "python-workflow"} results = [] for capability in ["workflows", "computation-graphs"]: scan_dir = features_dir / capability if scan_dir.exists(): for d in scan_dir.iterdir(): - if d.is_dir() and d.name not in excluded: - rel_path = f"examples/features/{capability}/{d.name}" - results.append((d.name, rel_path)) + if not d.is_dir() or d.name in excluded: + continue + if (d / "package.toml").exists(): + continue # packaged → the gold-path registrar owns it + rel_path = f"examples/features/{capability}/{d.name}" + results.append((d.name, rel_path)) + return results + + +def get_packaged_example_directories(): + """Every packaged example (a dir with a `package.toml`) under + examples/features/, with its parsed manifest metadata. These run through + the SERVER gold path (pack → upload → compile → reconcile → execute), so + the packaged registrar drives them — never `cargo run`. + + Returns (name, rel_path, meta) tuples where meta has `workflow_name` and/or + `graph_name` (whichever the package declares). + """ + features_dir = PROJECT_ROOT / "examples" / "features" + if not features_dir.exists(): + return [] + results = [] + for capability in ["workflows", "computation-graphs"]: + scan_dir = features_dir / capability + if not scan_dir.exists(): + continue + for d in sorted(scan_dir.iterdir()): + pt = d / "package.toml" + if not d.is_dir() or not pt.exists(): + continue + meta = {} + for line in pt.read_text().splitlines(): + line = line.strip() + for key in ("workflow_name", "graph_name"): + if line.startswith(key) and "=" in line: + meta[key] = line.split("=", 1)[1].strip().strip('"') + results.append((d.name, f"{capability}/{d.name}", meta)) return results diff --git a/.angreal/demos/features/features.py b/.angreal/demos/features/features.py index aa7d5f0ca..26abf3eb9 100644 --- a/.angreal/demos/features/features.py +++ b/.angreal/demos/features/features.py @@ -1,8 +1,10 @@ """demos features — run feature-focused Cloacina examples.""" import json +import os import shutil import subprocess +import time import angreal # type: ignore @@ -11,6 +13,7 @@ from .._utils import ( PROJECT_ROOT, + get_packaged_example_directories, get_rust_feature_directories, ) @@ -41,10 +44,10 @@ def _cmd(): for name, path in get_rust_feature_directories() } -# Bespoke feature demos defined below (excluded from auto-registration because -# `cargo run` is the wrong verb for them). Keep this list next to their -# definitions — `demos matrix` includes it so CI runs them too. -_BESPOKE_FEATURES = ["python-workflow", "simple-packaged"] +# 32-byte demo KEK (base64) for the secrets lanes — the same value the demo +# compose stack ships. Demo/dev only; production operators provision their own +# CLOACINA_SECRET_KEK. +_DEMO_SECRET_KEK = "ZGVtby1rZWstZGVtby1rZWstZGVtby1rZWstMDAwMSE=" @demos() @@ -61,9 +64,13 @@ def matrix(): """CI executes ALL runnable examples. This is the single source of truth: the same discovery that registers `demos features ` commands feeds the CI matrix, so a new example directory automatically joins CI — no - hand-maintained workflow list to drift.""" + hand-maintained list to drift. Three sources: embedded `cargo run` examples + (dirs without a package.toml), packaged gold-path examples (dirs WITH a + package.toml), and the one bespoke wheel demo (python-workflow).""" names = sorted( - [name.replace("_", "-") for name in _rust_feature_commands] + _BESPOKE_FEATURES + [name.replace("_", "-") for name in _rust_feature_commands] + + list(_packaged_commands) + + ["python-workflow"] ) print(json.dumps(names)) @@ -126,40 +133,25 @@ def python_workflow(): shutil.rmtree(venv_path) -# --- canonical packaged-workflow demo (the gold path) ------------------------ +# --- gold-path packaged demos (CLOACI-I-0138) -------------------------------- # -# `simple-packaged` is the CANONICAL example (CLOACI-I-0138): it runs through -# the PRIMARY interface — pack → upload → (server compiles & reconciles) → -# workflow run → execution Completed — never an in-process runner. It's in the -# auto-registration exclude list because `cargo run` is the wrong verb for it; -# this bespoke command drives the real lifecycle instead, reusing the -# service-lifecycle helpers from the compiler e2e harness. - -@demos() -@features() -@angreal.command( - name="simple-packaged", - about="run the canonical packaged example through the primary interface (pack → upload → compile → run)", - long_about=( - "Drives examples/features/workflows/simple-packaged the way the README " - "says a user does: postgres (dev stack) + cloacina-server + " - "cloacina-compiler (with --dev-workspace so the example's crates.io " - "version deps resolve against this checkout), then cloacinactl " - "pack → upload → poll the build to success → workflow run " - "data_processing → poll the execution to Completed. First run " - "cold-compiles the package deps (~5-10 min); warm cache finishes in " - "under a minute." - ), - when_to_use=[ - "verifying the packaged/server gold path end to end", - "validating the canonical example after compiler/server changes", - ], - when_not_to_use=[ - "compiler-pipeline regression assertions (use `test e2e compiler`)", - "running without docker", - ], -) -def simple_packaged(): +# These examples run through the PRIMARY interface — pack → upload → (server +# compiles & reconciles) → workflow run → execution Completed — never an +# in-process runner. They're in the auto-registration exclude list because +# `cargo run` is the wrong verb; each bespoke command drives the real +# lifecycle via this shared helper, reusing the service-lifecycle helpers +# from the compiler e2e harness. + +def _run_gold_path(label, example_dirname, run_steps, server_env=None, extra_services=None): + """Stand up dev-stack postgres + a host server + a host compiler + (--dev-workspace so the examples' crates.io version deps resolve against + this checkout), pack + upload the example, wait for the build, then call + `run_steps(ctl, home)` for the example-specific run/observe assertions. + `ctl(*args, check=True)` is a bound cloacinactl invoker. `server_env` + adds/overrides env vars on the server process (e.g. the secrets KEK). + `extra_services` names additional dev-stack compose services to bring up + (e.g. `("kafka",)`) AFTER the postgres reset but BEFORE the server starts, + so a stream accumulator's consumer can connect at reconcile time.""" import tempfile from pathlib import Path @@ -169,32 +161,51 @@ def simple_packaged(): _cloacinactl, _kill, _poll_build_status, - _poll_execution_status, - _poll_run_workflow, _start_postgres, _upload, _wait_http, ) - print("=== simple-packaged: packaged-workflow gold path ===") + print(f"=== {label}: packaged-workflow gold path ===") _build_binaries() _start_postgres() + # Bring up any extra dev-stack services (e.g. kafka) AFTER the postgres + # reset (which does `down -v`, tearing the whole project down) but before + # the server. `--wait` blocks until they're healthy so the server's + # reconcile-time consumers can connect. + if extra_services: + print(f" bringing up dev-stack services: {', '.join(extra_services)}") + rc = subprocess.run( + [ + "docker", "compose", "-f", ".angreal/docker-compose.yaml", + "up", "-d", "--wait", *extra_services, + ], + cwd=str(PROJECT_ROOT), + check=False, + ).returncode + if rc != 0: + raise RuntimeError(f"failed to bring up dev-stack services {extra_services}") # Distinct ports from the other harnesses (compiler e2e 18083/19003, # ui e2e 18085/19001) so lanes can't collide on a shared machine. _assert_ports_free(18087, 19005) db_url = "postgres://cloacina:cloacina@localhost:15432/cloacina" - bootstrap_key = "demo-simple-packaged-key" + bootstrap_key = f"demo-{label}-key" server_bind = "127.0.0.1:18087" compiler_bind = "127.0.0.1:19005" - example_dir = PROJECT_ROOT / "examples" / "features" / "workflows" / "simple-packaged" - home = Path(tempfile.mkdtemp(prefix="simple-packaged-demo-")) + # `example_dirname` is relative to examples/features/ and may carry a + # subtree (e.g. "computation-graphs/cg-feature-tour"). + example_dir = PROJECT_ROOT / "examples" / "features" / example_dirname + home = Path(tempfile.mkdtemp(prefix=f"{label}-demo-")) print(f"demo home (service logs): {home}") server_proc = None compiler_proc = None try: + env = os.environ.copy() + if server_env: + env.update(server_env) server_log = open(home / "server.log", "w") server_proc = subprocess.Popen( [ @@ -208,6 +219,7 @@ def simple_packaged(): cwd=PROJECT_ROOT, stdout=server_log, stderr=subprocess.STDOUT, + env=env, ) _wait_http(f"http://{server_bind}/health", "server", proc=server_proc) print(" ok: server up") @@ -227,7 +239,7 @@ def simple_packaged(): "--cargo-target-dir", str(shared_target), "--cargo-flag=build", "--cargo-flag=--lib", - # DEV ESCAPE HATCH (CLOACI-T-0887): the example ships crates.io + # DEV ESCAPE HATCH (CLOACI-T-0887): the examples ship crates.io # version deps (the form users ship); resolve them against THIS # checkout's unpublished crates. "--dev-workspace", str(PROJECT_ROOT), @@ -256,17 +268,564 @@ def simple_packaged(): _poll_build_status(home, pkg_id, {"success"}, timeout_s=900.0) print(" ok: build_status = success") - exec_id = _poll_run_workflow(home, "data_processing", timeout_s=180.0) - print(f" ok: workflow run accepted (execution {exec_id})") + def ctl(*args, check=True): + return _cloacinactl(home, *args, check=check) - _poll_execution_status(home, exec_id, {"Completed"}, timeout_s=300.0) - print(" ok: execution Completed") + run_steps(ctl, home) print( - "\nSUCCESS: gold path verified — " + f"\nSUCCESS: {label} gold path verified — " "pack → upload → compile → reconcile → execute → Completed" ) return 0 finally: _kill(compiler_proc) _kill(server_proc) + + +def _run_to_completed(ctl, home, workflow_name, context_path=None, timeout_s=300.0): + """`workflow run` (retrying until the reconciler has loaded the workflow), + then poll the execution to Completed. Returns the execution id.""" + from test.e2e.compiler import _poll_execution_status + + deadline = time.time() + 180.0 + last_err = "" + exec_id = None + while time.time() < deadline: + args = ["-o", "json", "workflow", "run", workflow_name] + if context_path: + args += ["--context", str(context_path)] + code, out, err = ctl(*args, check=False) + if code == 0: + try: + exec_id = json.loads(out).get("execution_id") + except json.JSONDecodeError: + exec_id = (out.strip().splitlines() or [""])[-1].strip() or None + if exec_id and len(exec_id) >= 32: + break + exec_id = None + last_err = err.strip() or out.strip() + time.sleep(3.0) + if not exec_id: + raise AssertionError( + f"workflow run {workflow_name} never succeeded; last error: {last_err}" + ) + print(f" ok: workflow run accepted (execution {exec_id})") + _poll_execution_status(home, exec_id, {"Completed"}, timeout_s=timeout_s) + print(" ok: execution Completed") + return exec_id + + +# --- generic packaged-example registrar (CLOACI-I-0138) ---------------------- +# +# Every example with a `package.toml` runs through the SERVER gold path, so ONE +# registrar discovers them and registers `demos features ` for each — no +# bespoke command per example (that plumbing was harness boilerplate, not a +# per-example feature). The default assertion comes from the manifest: +# `workflow_name` → run it to Completed. A few examples need a richer check +# (param validation, secret lifecycle, graph inject→fire) and supply a thin +# override below; a few not-yet-gold-path examples are skipped WITH A REASON so +# nothing is silently dropped. + + +def _default_workflow_steps(workflow_name): + def steps(ctl, home): + _run_to_completed(ctl, home, workflow_name) + + return steps + + +_MT_DB_URL = "postgres://cloacina:cloacina@localhost:15432/cloacina" + + +def _multi_tenant_steps(workflow_name): + """Prove the tenant isolation boundary — the FULL per-tenant deployment + model — on the gold path. The package is already deployed in `public` (the + harness pre-upload) = tenant 1. We create a second tenant `mtbeta`, stand up + a SECOND compiler scoped to `--tenant-schema mtbeta` (each tenant runs its + own compiler — the harness's default compiler only claims the public/admin + schema, by design: `cloacina-compiler --tenant-schema` isolates builds per + tenant), deploy the SAME archive into `mtbeta`, and run the workflow in BOTH + tenants. We then assert their executions never cross, and that a third + tenant `mtgamma` that never received the package cannot run the workflow. + + Tenant/workflow/execution routes are all `/v1/tenants/{tenant}/...`; every + call here is `--tenant`-scoped (the shared poll helpers target `public` + only, so this lane does its own tenant-aware run/poll).""" + + def steps(ctl, home): + from test.e2e.compiler import _cloacinactl, _wait_http, _kill + + def tctl(tenant, *args, check=True): + return _cloacinactl(home, "--tenant", tenant, *args, check=check) + + def _exec_id(out): + try: + v = json.loads(out).get("execution_id") + except json.JSONDecodeError: + v = (out.strip().splitlines() or [""])[-1].strip() or None + return v if v and len(v) >= 32 else None + + def run_and_wait(tenant, timeout_s=180.0): + # Retry `workflow run` until the reconciler has loaded the workflow + # in THIS tenant, then poll THIS tenant's execution to Completed. + deadline = time.time() + 120.0 + exec_id = None + last = "" + while time.time() < deadline: + code, out, err = tctl( + tenant, "-o", "json", "workflow", "run", workflow_name, + check=False, + ) + if code == 0: + exec_id = _exec_id(out) + if exec_id: + break + last = err.strip() or out.strip() + time.sleep(3.0) + if not exec_id: + raise AssertionError( + f"[{tenant}] workflow run {workflow_name} never accepted: {last!r}" + ) + pdl = time.time() + timeout_s + while time.time() < pdl: + _, pout, _ = tctl( + tenant, "-o", "json", "execution", "status", exec_id, + check=False, + ) + try: + st = json.loads(pout).get("status") + except json.JSONDecodeError: + st = None + if st == "Completed": + return exec_id + if st in {"Failed", "Cancelled"}: + raise AssertionError(f"[{tenant}] execution {exec_id} → {st}") + time.sleep(2.0) + raise AssertionError(f"[{tenant}] execution {exec_id} never Completed") + + def exec_ids(tenant): + _, out, _ = tctl( + tenant, "-o", "json", "execution", "list", "--limit", "100", + check=False, + ) + try: + data = json.loads(out) + except json.JSONDecodeError: + return [] + rows = data.get("items", data) if isinstance(data, dict) else data + return {(r.get("execution_id") or r.get("id")) for r in rows} + + archive = next(home.glob("*.cloacina")) + + # Tenant 1 = public: the harness already uploaded + built the package. + e_public = run_and_wait("public") + print(f" ok: ran in tenant `public` → {e_public}") + + # Tenant 2 = mtbeta: create it, then stand up its OWN compiler scoped to + # its schema (the per-tenant build isolation model), deploy the SAME + # archive, and run independently. + ctl("tenant", "create", "mtbeta", check=False) + mt_compiler = None + try: + mt_home = home / "mtbeta-compiler" + mt_home.mkdir(exist_ok=True) + mt_log = open(mt_home / "compiler.log", "w") + mt_compiler = subprocess.Popen( + [ + "target/debug/cloacina-compiler", + "--home", str(mt_home), + "--database-url", _MT_DB_URL, + "--bind", "127.0.0.1:19006", + "--tenant-schema", "mtbeta", + "--poll-interval-ms", "500", + "--dev-workspace", str(PROJECT_ROOT), + "--verbose", + ], + cwd=PROJECT_ROOT, + stdout=mt_log, + stderr=subprocess.STDOUT, + ) + _wait_http( + "http://127.0.0.1:19006/health", "mtbeta-compiler", + timeout_s=60.0, proc=mt_compiler, + ) + print(" ok: per-tenant compiler up (--tenant-schema mtbeta)") + + _, up, _ = tctl("mtbeta", "package", "upload", str(archive), check=False) + pkg_beta = (up.strip().splitlines() or [""])[-1].strip() + if not (pkg_beta and len(pkg_beta) >= 32): + raise AssertionError(f"upload to mtbeta didn't return a package id: {up!r}") + bdl = time.time() + 180.0 + while time.time() < bdl: + _, iout, _ = tctl( + "mtbeta", "-o", "json", "package", "inspect", pkg_beta, + check=False, + ) + try: + if json.loads(iout).get("build_status") == "success": + break + except json.JSONDecodeError: + pass + time.sleep(2.0) + else: + raise AssertionError("mtbeta package never built (per-tenant compiler)") + print(" ok: deployed the same package into tenant `mtbeta` (its own compiler built it)") + e_beta = run_and_wait("mtbeta") + print(f" ok: ran in tenant `mtbeta` → {e_beta}") + finally: + _kill(mt_compiler) + + # Isolation 1: each tenant's execution list contains ONLY its own run. + pub_ids, beta_ids = exec_ids("public"), exec_ids("mtbeta") + if e_public not in pub_ids or e_beta in pub_ids: + raise AssertionError( + f"tenant leak: `public` list {sorted(pub_ids)} should have " + f"{e_public} and NOT {e_beta}" + ) + if e_beta not in beta_ids or e_public in beta_ids: + raise AssertionError( + f"tenant leak: `mtbeta` list {sorted(beta_ids)} should have " + f"{e_beta} and NOT {e_public}" + ) + print(" ok: executions are tenant-isolated (neither tenant sees the other's run)") + + # Isolation 2 (CLOACI-T-0901): a tenant that never received the package + # MUST NOT be able to run the workflow. The execute route now gates on + # the calling tenant's OWN registry before resolving the name against the + # process-shared `Runtime`, so an uninstalled workflow is rejected even + # though other tenants have loaded the same name. + ctl("tenant", "create", "mtgamma", check=False) + code, out, err = tctl( + "mtgamma", "workflow", "run", workflow_name, check=False, + ) + if code == 0 and _exec_id(out): + raise AssertionError( + f"tenant leak (CLOACI-T-0901 regressed): `mtgamma` (never installed " + f"the package) ran `{workflow_name}`" + ) + print(" ok: tenant `mtgamma` (no package) cannot run the workflow — isolated") + + return steps + + +def _graph_autofire_steps(label, reactor): + """For a SELF-FIRING reactor (a polling accumulator): assert the reactor + fires on its own — no inject. Reads the fires count off the reactors-list + endpoint and waits for it to increase, proving the packaged polling + accumulator drives its Python poll fn on the interval (CLOACI-T-0896).""" + + def steps(ctl, home): + from test.e2e.compiler import _get_json + + key = f"demo-{label}-key" + + def fires_for(name): + body = _get_json("http://127.0.0.1:18087/v1/health/reactors", key) + items = body.get("items", []) if isinstance(body, dict) else [] + for r in items: + if r.get("name") == name: + return int(r.get("fires", 0) or 0) + return 0 + + before = fires_for(reactor) + deadline = time.time() + 120 + while time.time() < deadline: + time.sleep(5) + if fires_for(reactor) > before: + print(f" ok: reactor {reactor} self-fired (polling accumulator ran its poll fn)") + return + raise AssertionError( + f"reactor {reactor} never self-fired — the polling accumulator's poll " + "fn was not driven on its interval (CLOACI-T-0896)" + ) + + return steps + + +def _trigger_wait_steps(workflow_name): + """For a POLL/CRON-triggered workflow: don't `workflow run` it — wait for the + trigger to fire it AUTOMATICALLY and assert the auto-execution reaches + Completed. Proves the packaged-trigger path (macro → FFI projection → host + trigger registry → scheduled fire) end to end.""" + + def steps(ctl, home): + from test.e2e.compiler import _poll_execution_status + + deadline = time.time() + 180 + while time.time() < deadline: + _, out, _ = ctl( + "-o", "json", "execution", "list", + "--workflow", workflow_name, "--limit", "1", check=False, + ) + try: + data = json.loads(out) + except json.JSONDecodeError: + time.sleep(3) + continue + rows = data.get("items", data) if isinstance(data, dict) else data + if rows: + row = rows[0] + exec_id = row.get("execution_id") or row.get("id") + if exec_id and len(str(exec_id)) >= 32: + print(f" ok: trigger fired an execution automatically ({exec_id})") + _poll_execution_status(home, exec_id, {"Completed"}, timeout_s=180.0) + print(" ok: triggered execution Completed") + return + time.sleep(3) + raise AssertionError( + f"no execution of `{workflow_name}` appeared — the poll trigger never " + "fired (macro/FFI projection or trigger scheduler not running)" + ) + + return steps + + +def _context_run_steps(workflow_name, context_json): + """Run a workflow that declares REQUIRED params, supplying them via context. + The default lane runs `workflow run` with no context, which the execute route + rejects for a workflow with required params (I-0128 validation).""" + + def steps(ctl, home): + ctx = home / "ctx.json" + ctx.write_text(context_json) + _run_to_completed(ctl, home, workflow_name, context_path=ctx) + + return steps + + +def _params_steps(workflow_name): + """Run a params template twice with different bindings, then assert a + missing-required-param run is rejected before execution.""" + + def steps(ctl, home): + prod = home / "prod.json" + prod.write_text('{"source": "/data/prod", "dst": "/backup/prod"}') + _run_to_completed(ctl, home, workflow_name, context_path=prod) + + archive = home / "archive.json" + archive.write_text( + '{"source": "/data/archive", "dst": "/cold", "mode": "move", "max_files": 10}' + ) + _run_to_completed(ctl, home, workflow_name, context_path=archive) + + bad = home / "bad.json" + bad.write_text('{"dst": "/backup"}') + code, out, _ = ctl( + "workflow", "run", workflow_name, "--context", str(bad), check=False + ) + if code == 0: + raise AssertionError( + "run with a missing required param was ACCEPTED — declared-param " + f"validation did not fire: {out!r}" + ) + print(" ok: missing required param rejected before execution") + + return steps + + +def _secrets_steps(workflow_name): + """Full tenant-secret lifecycle: create → $secret-bound run (Completed) → + rotate → rerun → metadata-only get → literal binding rejected.""" + + def steps(ctl, home): + token_file = home / "token.txt" + token_file.write_text("s3cr3t-demo-token-value") + ctl("secret", "create", "oncall_api", "--field", f"token=@{token_file}") + print(" ok: tenant secret created") + + bind = home / "bind.json" + bind.write_text('{"channel": "#oncall", "api_token": {"$secret": "oncall_api"}}') + _run_to_completed(ctl, home, workflow_name, context_path=bind) + + token_file.write_text("r0tated-demo-token-value!") + ctl("secret", "rotate", "oncall_api", "--field", f"token=@{token_file}") + print(" ok: secret rotated") + _run_to_completed(ctl, home, workflow_name, context_path=bind) + + _, out, _ = ctl("-o", "json", "secret", "get", "oncall_api") + if "s3cr3t" in out or "r0tated" in out: + raise AssertionError(f"secret get leaked a value: {out!r}") + print(" ok: secret get returns metadata only") + + bad = home / "bad.json" + bad.write_text('{"api_token": "plaintext-token"}') + code, out, _ = ctl( + "workflow", "run", workflow_name, "--context", str(bad), check=False + ) + if code == 0: + raise AssertionError( + f"literal secret value was ACCEPTED — validation did not fire: {out!r}" + ) + print(" ok: literal secret value rejected before execution") + + return steps + + +def _graph_inject_steps(label, reactor, accumulator, event, bad_event=None): + """Inject a typed event into a reactor's accumulator and confirm the reactor + fires (its graph runs). Reads the fires COUNT off the reactors-list endpoint + (`ListResponse{items,total}`) rather than parsing the fires list. When + `bad_event` is given, also assert the accumulator's declared boundary schema + rejects it (proves the typed inject surface — `@cloaca.boundary_schema` / + `schemars::JsonSchema` — is wired).""" + + def steps(ctl, home): + from test.e2e.compiler import _get_json + + key = f"demo-{label}-key" + + def fires_for(name): + body = _get_json("http://127.0.0.1:18087/v1/health/reactors", key) + items = body.get("items", []) if isinstance(body, dict) else [] + for r in items: + if r.get("name") == name: + return int(r.get("fires", 0) or 0) + return 0 + + if bad_event is not None: + code, out, _ = ctl( + "accumulator", "inject", accumulator, "--event", bad_event, check=False + ) + if code == 0: + raise AssertionError( + f"malformed event {bad_event!r} was ACCEPTED for `{accumulator}` — " + f"the declared boundary schema did not reject it: {out!r}" + ) + print(" ok: malformed event rejected by the accumulator boundary schema") + + before = fires_for(reactor) + deadline = time.time() + 180 + while time.time() < deadline: + ctl("accumulator", "inject", accumulator, "--event", event, check=False) + time.sleep(4) + if fires_for(reactor) > before: + print(f" ok: inject fired reactor {reactor} (graph ran)") + return + raise AssertionError( + f"reactor {reactor} never fired after injecting into `{accumulator}`" + ) + + return steps + + +# name -> {"steps": , "server_env": } +_PACKAGED_OVERRIDES = { + # analytics_workflow declares a required `source_id` param → supply it. + "packaged-workflows": { + "steps": _context_run_steps("analytics_workflow", '{"source_id": "src-001"}'), + }, + "parameterized-workflow": {"steps": _params_steps("sync_file")}, + "python-parameterized": {"steps": _params_steps("python_parameterized")}, + "workflow-secrets": { + "steps": _secrets_steps("notify_oncall"), + "server_env": {"CLOACINA_SECRET_KEK": _DEMO_SECRET_KEK}, + }, + "python-secrets": { + "steps": _secrets_steps("python_secrets"), + "server_env": {"CLOACINA_SECRET_KEK": _DEMO_SECRET_KEK}, + }, + "packaged-graph": { + "steps": _graph_inject_steps( + "packaged-graph", + "packaged_market_maker_reactor", + "orderbook", + '{"best_bid": 100.0, "best_ask": 100.1}', + ), + }, + "python-packaged-graph": { + "steps": _graph_inject_steps( + "python-packaged-graph", + "market_maker", + "orderbook", + '{"best_bid": 100.0, "best_ask": 100.1}', + # `orderbook` declares @cloaca.boundary_schema(best_bid, best_ask): + # a non-object event must be rejected by the typed slot. + bad_event="42", + ), + }, + # Python STATE accumulator (bounded rolling window) — inject a tick, the + # window emits, `tick_reactor` fires the graph; a non-object is rejected. + "python-stateful-graph": { + "steps": _graph_inject_steps( + "python-stateful-graph", + "tick_reactor", + "tick_window", + '{"bid": 100.0, "ask": 100.2}', + bad_event="42", + ), + }, + # Python BATCH accumulator (buffer + flush on size/interval) — inject events, + # the buffer flushes, `batch_reactor` fires; a non-object is rejected. + "python-batch-graph": { + "steps": _graph_inject_steps( + "python-batch-graph", + "batch_reactor", + "event_batch", + '{"level": 42.0}', + bad_event="42", + ), + }, + # Python POLLING accumulator (self-fires via its poll fn) — no inject; assert + # `poll_reactor` fires on its own on the interval (CLOACI-T-0896). + "python-polling-graph": { + "steps": _graph_autofire_steps("python-polling-graph", "poll_reactor"), + }, + # Poll trigger fires `file_processing` automatically — wait for it, don't run. + "packaged-triggers": {"steps": _trigger_wait_steps("file_processing")}, + # Python peer: @cloaca.trigger poll fires `file_processing_py` automatically. + "python-triggers": {"steps": _trigger_wait_steps("file_processing_py")}, + # Python cron: the cron scheduler fires `heartbeat_workflow` on a schedule. + "python-cron": {"steps": _trigger_wait_steps("heartbeat_workflow")}, + # Python multi-tenancy: same package in two tenants, executions isolated. + "python-multi-tenant": {"steps": _multi_tenant_steps("tenant_job")}, +} + +# Packaged examples not yet driveable on the gold path — discovered but not +# registered, each with a reason (no silent drops). Tracked for follow-up. +_PACKAGED_SKIP = { + # (empty) — every packaged example is now driveable on the gold path. + # cg-feature-tour's stream/inject surface is deferred to T-0898, but its + # `tour_pipeline` invocation surface IS runnable via the default → registered. +} + + +def _register_packaged_example(name, rel_path, meta): + cfg = _PACKAGED_OVERRIDES.get(name) + if cfg is not None: + steps = cfg["steps"] + server_env = cfg.get("server_env") + else: + wf = meta.get("workflow_name") + if not wf: + # Graph-only package with no override → can't derive an assertion + # (reactor/accumulator/event aren't in the manifest). Skip loudly. + return None + steps = _default_workflow_steps(wf) + server_env = None + + @demos() + @features() + @angreal.command( + name=name, + about=f"packaged gold path: {name} (pack → upload → compile → reconcile → execute)", + when_to_use=[ + "verifying a packaged example end to end through the server", + "validating the gold path after compiler/server changes", + ], + when_not_to_use=["running without docker"], + ) + def _cmd(_steps=steps, _rel=rel_path, _name=name, _env=server_env): + return _run_gold_path(_name, _rel, _steps, server_env=_env) + + _cmd.__name__ = f"packaged_{name}".replace("-", "_") + return _cmd + + +_packaged_commands = {} +for _name, _rel, _meta in get_packaged_example_directories(): + if _name in _PACKAGED_SKIP: + continue + _cmd = _register_packaged_example(_name, _rel, _meta) + if _cmd is not None: + _packaged_commands[_name] = _cmd diff --git a/.angreal/test/integration.py b/.angreal/test/integration.py index cbad31596..7164e6f3a 100644 --- a/.angreal/test/integration.py +++ b/.angreal/test/integration.py @@ -1,7 +1,6 @@ import shutil import subprocess import sys -import time import os from pathlib import Path @@ -20,6 +19,65 @@ ) +def _local_crate_patch_block() -> str: + """A `[patch.crates-io]` block mapping every workspace crate to its local + path — the file-based dependency resolution the compiler's `--dev-workspace` + injects at build time (CLOACI-T-0887). Ship-form example packages declare + their cloacina deps as crates.io VERSION deps (`= "0.10"`), which are + unpublished; a plain `cargo build` of such an example can't resolve them + against crates.io, so we point them at the local checkout for testing.""" + repo = Path(__file__).resolve().parents[2] + lines = ["", "[patch.crates-io]"] + for cargo_toml in sorted((repo / "crates").glob("*/Cargo.toml")): + name = None + in_pkg = False + for raw in cargo_toml.read_text().splitlines(): + s = raw.strip() + if s == "[package]": + in_pkg = True + continue + if s.startswith("[") and s != "[package]": + in_pkg = False + if in_pkg and s.startswith("name") and "=" in s: + name = s.split("=", 1)[1].strip().strip('"') + break + if name: + lines.append(f'{name} = {{ path = "{cargo_toml.parent}" }}') + return "\n".join(lines) + "\n" + + +def _build_shipform_fixture(example_rel: str, pkg: str): + """Build a ship-form (version-dep) example package as a test fixture with + file-based dependency resolution. Ship-form examples carry crates.io VERSION + deps and rely on the compiler's `--dev-workspace` to patch them to local + paths on the gold path; a plain `cargo build` has no such patch, so we append + the local `[patch.crates-io]` block to a temporary copy of the example's + Cargo.toml (and its Cargo.lock is already local-shaped), then restore both so + the shipped example stays pristine. Path-dep examples (already file-based) + build as-is.""" + repo = Path(__file__).resolve().parents[2] + example_dir = repo / example_rel + cargo_toml = example_dir / "Cargo.toml" + cargo_lock = example_dir / "Cargo.lock" + original_toml = cargo_toml.read_text() + original_lock = cargo_lock.read_text() if cargo_lock.exists() else None + # Detect a REAL patch section (a section header line), not the substring in a + # doc comment — several examples mention `[patch.crates-io]` in comments. + already_patched = any( + line.strip() == "[patch.crates-io]" for line in original_toml.splitlines() + ) + if already_patched: + subprocess.run(["cargo", "build", "-p", pkg], check=True, cwd=str(example_dir)) + return + try: + cargo_toml.write_text(original_toml + _local_crate_patch_block()) + subprocess.run(["cargo", "build", "-p", pkg], check=True, cwd=str(example_dir)) + finally: + cargo_toml.write_text(original_toml) + if original_lock is not None: + cargo_lock.write_text(original_lock) + + def build_test_packages(backend=None): """Pre-build test packages before running integration tests. @@ -35,20 +93,17 @@ def build_test_packages(backend=None): # Create output directory os.makedirs("target/test-packages", exist_ok=True) - # Build packaged-workflow-example (debug mode to match test binary wire format) + # Build packaged-workflow-example (debug mode to match test binary wire + # format). Ship-form (version-dep) example → build with local crate patch. print("Building packaged-workflow-example...") - subprocess.run( - ["cargo", "build", "-p", "packaged-workflow-example"], - check=True, - cwd="examples/features/workflows/packaged-workflows" + _build_shipform_fixture( + "examples/features/workflows/packaged-workflows", "packaged-workflow-example" ) - # Build simple-packaged-demo (debug mode to match test binary wire format) + # Build simple-packaged-demo (debug mode to match test binary wire format). print("Building simple-packaged-demo...") - subprocess.run( - ["cargo", "build", "-p", "simple-packaged-demo"], - check=True, - cwd="examples/features/workflows/simple-packaged" + _build_shipform_fixture( + "examples/features/workflows/simple-packaged", "simple-packaged-demo" ) # T-0550 / I-0102 T-D — primitive-only fixtures exercised by diff --git a/.angreal/utils.py b/.angreal/utils.py index d5ff21c86..c11387dd7 100644 --- a/.angreal/utils.py +++ b/.angreal/utils.py @@ -162,8 +162,11 @@ def run_example_or_tutorial(project_root, example_dir, name, is_test=False, bina # Check if this is a tutorial (SQLite-based) or other example (potentially PostgreSQL-based) is_tutorial = "tutorial" in example_dir - # Tutorial-06 (multi-tenancy) needs PostgreSQL for the advanced admin demo - needs_postgres = not is_tutorial or "tutorial-06" in example_dir + # Tutorial-06 (multi-tenancy) needs PostgreSQL for the advanced admin demo. + # Match the real dir name (`06-multi-tenancy`) — the old `"tutorial-06"` + # probe never matched, so DATABASE_URL was left unset and the tutorial fell + # back to :5432 while the dev stack publishes :15432 (connection refused). + needs_postgres = not is_tutorial or "06-multi-tenancy" in example_dir if needs_postgres: # For examples and tutorial-06, check if Docker services are running @@ -271,6 +274,11 @@ def smart_postgres_reset() -> bool: [ "psql", "-h", "localhost", + # The dev stack (Docker on Ubuntu, Homebrew on macOS) publishes + # postgres on 15432, matching the test fixtures — connect there, + # not the default 5432, or the reset silently misses and falls + # through to the Docker path (which doesn't exist on macOS CI). + "-p", "15432", "-U", "cloacina", "-d", "cloacina", "-c", "DROP SCHEMA public CASCADE; CREATE SCHEMA public;", diff --git a/.github/workflows/cloacina.yml b/.github/workflows/cloacina.yml index 0f0193a0c..94928deef 100644 --- a/.github/workflows/cloacina.yml +++ b/.github/workflows/cloacina.yml @@ -32,6 +32,15 @@ jobs: runs-on: ubuntu-latest steps: + - name: Free disk space (CI disk-pressure guard) + if: runner.os == 'Linux' + run: | + # Recurring "No space left on device": these jobs rebuild cloacina and + # run the docker stack on a ~14GB runner. Reclaim ~18GB of preinstalled + # toolchains we never use. (Linux only — macOS has ample disk + no dirs.) + sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc \ + /opt/hostedtoolcache/CodeQL /usr/local/share/boost || true + df -h / | tail -1 - uses: actions/checkout@v4 with: fetch-depth: 0 @@ -117,6 +126,15 @@ jobs: runs-on: ${{ matrix.os }} steps: + - name: Free disk space (CI disk-pressure guard) + if: runner.os == 'Linux' + run: | + # Recurring "No space left on device": these jobs rebuild cloacina and + # run the docker stack on a ~14GB runner. Reclaim ~18GB of preinstalled + # toolchains we never use. (Linux only — macOS has ample disk + no dirs.) + sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc \ + /opt/hostedtoolcache/CodeQL /usr/local/share/boost || true + df -h / | tail -1 - uses: actions/checkout@v4 with: fetch-depth: 0 @@ -173,6 +191,15 @@ jobs: runs-on: ${{ matrix.os }} steps: + - name: Free disk space (CI disk-pressure guard) + if: runner.os == 'Linux' + run: | + # Recurring "No space left on device": these jobs rebuild cloacina and + # run the docker stack on a ~14GB runner. Reclaim ~18GB of preinstalled + # toolchains we never use. (Linux only — macOS has ample disk + no dirs.) + sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc \ + /opt/hostedtoolcache/CodeQL /usr/local/share/boost || true + df -h / | tail -1 - uses: actions/checkout@v4 with: fetch-depth: 0 @@ -233,9 +260,15 @@ jobs: run: | brew services start postgresql@14 sleep 5 - # Create the cloacina user and database that tests expect - createuser -s cloacina || true - createdb -O cloacina cloacina || true + # The Rust test fixtures hardcode the dev-stack port 15432 (matching the + # docker-compose stack used on Ubuntu), so move the Homebrew cluster off + # the default 5432 onto 15432 — otherwise every postgres test refuses. + psql -d postgres -c "ALTER SYSTEM SET port = 15432;" + brew services restart postgresql@14 + sleep 5 + # Create the cloacina user and database that tests expect (on 15432). + createuser -s cloacina -p 15432 || true + createdb -O cloacina cloacina -p 15432 || true - name: Run integration tests (${{ matrix.backend }}) env: @@ -347,6 +380,15 @@ jobs: runs-on: ubuntu-latest steps: + - name: Free disk space (CI disk-pressure guard) + if: runner.os == 'Linux' + run: | + # Recurring "No space left on device": these jobs rebuild cloacina and + # run the docker stack on a ~14GB runner. Reclaim ~18GB of preinstalled + # toolchains we never use. (Linux only — macOS has ample disk + no dirs.) + sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc \ + /opt/hostedtoolcache/CodeQL /usr/local/share/boost || true + df -h / | tail -1 - uses: actions/checkout@v4 with: fetch-depth: 0 @@ -392,6 +434,15 @@ jobs: needs: unit-tests runs-on: ubuntu-latest steps: + - name: Free disk space (CI disk-pressure guard) + if: runner.os == 'Linux' + run: | + # Recurring "No space left on device": these jobs rebuild cloacina and + # run the docker stack on a ~14GB runner. Reclaim ~18GB of preinstalled + # toolchains we never use. (Linux only — macOS has ample disk + no dirs.) + sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc \ + /opt/hostedtoolcache/CodeQL /usr/local/share/boost || true + df -h / | tail -1 - uses: actions/checkout@v4 with: fetch-depth: 0 diff --git a/.github/workflows/examples-docs.yml b/.github/workflows/examples-docs.yml index 924cc7794..5ec356306 100644 --- a/.github/workflows/examples-docs.yml +++ b/.github/workflows/examples-docs.yml @@ -8,6 +8,12 @@ concurrency: group: examples-docs-${{ github.ref }} cancel-in-progress: true +env: + # angreal auto-installs shell completion on first run and prints a banner to + # stdout; that banner corrupts `angreal demos matrix` when captured into + # $GITHUB_OUTPUT (and is noise everywhere else). Suppress it globally. + ANGREAL_NO_AUTO_COMPLETION: "1" + jobs: # ----------------------------------------------------------------------- # Step 0: Single build that all tutorial/example jobs share @@ -19,6 +25,14 @@ jobs: timeout-minutes: 30 steps: + - name: Free disk space (CI disk-pressure guard) + run: | + # Recurring "No space left on device": these jobs rebuild cloacina + + # an example (~GB each) and run the docker stack on a ~14GB runner. + # Reclaim ~18GB of preinstalled toolchains we never use. + sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc \ + /opt/hostedtoolcache/CodeQL /usr/local/share/boost || true + df -h / | tail -1 - uses: actions/checkout@v4 with: fetch-depth: 0 @@ -64,6 +78,14 @@ jobs: RUST_BACKTRACE: full steps: + - name: Free disk space (CI disk-pressure guard) + run: | + # Recurring "No space left on device": these jobs rebuild cloacina + + # an example (~GB each) and run the docker stack on a ~14GB runner. + # Reclaim ~18GB of preinstalled toolchains we never use. + sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc \ + /opt/hostedtoolcache/CodeQL /usr/local/share/boost || true + df -h / | tail -1 - uses: actions/checkout@v4 with: fetch-depth: 0 @@ -128,6 +150,14 @@ jobs: outputs: examples: ${{ steps.list.outputs.examples }} steps: + - name: Free disk space (CI disk-pressure guard) + run: | + # Recurring "No space left on device": these jobs rebuild cloacina + + # an example (~GB each) and run the docker stack on a ~14GB runner. + # Reclaim ~18GB of preinstalled toolchains we never use. + sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc \ + /opt/hostedtoolcache/CodeQL /usr/local/share/boost || true + df -h / | tail -1 - uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha || github.ref }} @@ -159,6 +189,14 @@ jobs: RUST_BACKTRACE: full steps: + - name: Free disk space (CI disk-pressure guard) + run: | + # Recurring "No space left on device": these jobs rebuild cloacina + + # an example (~GB each) and run the docker stack on a ~14GB runner. + # Reclaim ~18GB of preinstalled toolchains we never use. + sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc \ + /opt/hostedtoolcache/CodeQL /usr/local/share/boost || true + df -h / | tail -1 - uses: actions/checkout@v4 with: fetch-depth: 0 @@ -274,6 +312,14 @@ jobs: RUST_BACKTRACE: full steps: + - name: Free disk space (CI disk-pressure guard) + run: | + # Recurring "No space left on device": these jobs rebuild cloacina + + # an example (~GB each) and run the docker stack on a ~14GB runner. + # Reclaim ~18GB of preinstalled toolchains we never use. + sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc \ + /opt/hostedtoolcache/CodeQL /usr/local/share/boost || true + df -h / | tail -1 - uses: actions/checkout@v4 with: fetch-depth: 0 @@ -334,6 +380,14 @@ jobs: runs-on: ubuntu-latest steps: + - name: Free disk space (CI disk-pressure guard) + run: | + # Recurring "No space left on device": these jobs rebuild cloacina + + # an example (~GB each) and run the docker stack on a ~14GB runner. + # Reclaim ~18GB of preinstalled toolchains we never use. + sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc \ + /opt/hostedtoolcache/CodeQL /usr/local/share/boost || true + df -h / | tail -1 - uses: actions/checkout@v4 with: fetch-depth: 0 diff --git a/.metis/backlog/bugs/CLOACI-T-0895.md b/.metis/backlog/bugs/CLOACI-T-0895.md new file mode 100644 index 000000000..b40a4efec --- /dev/null +++ b/.metis/backlog/bugs/CLOACI-T-0895.md @@ -0,0 +1,164 @@ +--- +id: bug-packaged-tasks-cannot-receive +level: task +title: "BUG: packaged tasks cannot receive secrets — the fidius task protocol has no secrets channel" +short_code: "CLOACI-T-0895" +created_at: 2026-07-12T01:10:07.780717+00:00 +updated_at: 2026-07-12T01:34:05.671813+00:00 +parent: +blocked_by: [] +archived: false + +tags: + - "#task" + - "#bug" + - "#phase/completed" + + +exit_criteria_met: false +initiative_id: NULL +--- + +# BUG: packaged tasks cannot receive secrets — the fidius task protocol has no secrets channel + +*This template includes sections for various types of tasks. Delete sections that don't apply to your specific use case.* + +## Parent Initiative **[CONDITIONAL: Assigned Task]** + +[[Parent Initiative]] + +## Objective **[REQUIRED]** + +**Finding from T-0890 (2026-07-12, I-0138 feature-coverage push) — the deepest yet:** a PACKAGED task calling `context.secret(...)` always fails with "secrets backend not configured for this execution scope", in EVERY deployment mode, because the fidius task-execution protocol has no secrets channel: + +`DynamicLibraryTask::execute` (registry/loader/task_registrar/dynamic_task.rs:156) serializes **only `context.data()`** into `TaskExecutionRequest { task_name, context_json }` and calls the plugin. The host context's secret resolver — a runtime object — never crosses the boundary, and `cloacina-workflow-plugin` has zero secret handling on the receiving side (the plugin-side Context is rebuilt with no resolver). + +**Consequences:** +- The agent path's whole D-5 machinery (HPKE wrap to one-time pooled keys → agent unwraps → `context.set_secret_resolver(...)` in main.rs:1616) attaches the resolver to a host context that dynamic_task then drops — agents execute packaged tasks via `runtime.get_task(...).execute(context)` → same bridge. **D-5 delivery has never reached a packaged task.** +- The in-process path (T-0890 wired `with_database_secrets` → ThreadTaskExecutor → context_builder → `context.set_secret_resolver`) is equally dropped at the same bridge. +- Only HOST-COMPILED (embedded) tasks can resolve secrets — e.g. the `secret_no_leak` integration test's local `#[task]`. Since all examples/e2e lead packaged (I-0138), secrets are effectively undeliverable in the product's primary shape. + +**Everything upstream now works** (fixed under T-0890): `workflow run --context` accepts `{"$secret": "name"}` (execute route merges via `merge_instance_params`; literal values rejected for encrypted slots), and both server runner paths thread tenant-scoped resolvers. The alias map demonstrably reaches the task input context: `{"channel":"#oncall","__cloacina_secret_refs__":{"api_token":"oncall_api"}}`. + +**Fix (needs a short design pass — it touches the plugin wire format):** extend the task-execution interface so resolved secret VALUES (or a host callback) cross the boundary. Sketch: dynamic_task reads `SECRET_REFS_KEY` from the context, resolves each name via the host-side resolver BEFORE the FFI call, and passes a `resolved_secrets` map alongside `context_json`; the plugin-side `package!` execute path attaches an in-memory resolver built from that map to the rebuilt Context. In-process the plaintext never leaves the process; agent-side it's the already-unwrapped values. NOTE: adding a field to the bincode-serialized request struct is a WIRE CHANGE → likely `interface_version` bump → all packaged fixtures/examples rebuild. Decide versioning strategy with the maintainer before implementing. + +**Verification vehicle exists:** `examples/features/workflows/workflow-secrets` + `angreal demos features workflow-secrets` (T-0890) — currently blocked on this bug; it asserts create → `$secret`-bound run → **Completed** → rotate → rerun → metadata-only reads → literal rejected. Re-enable it in the CI matrix (`_BESPOKE_FEATURES`) when this lands. Related: [[CLOACI-T-0890]], [[CLOACI-T-0894]], I-0133 D-5. + +## Backlog Item Details **[CONDITIONAL: Backlog Item]** + +{Delete this section when task is assigned to an initiative} + +### Type +- [ ] Bug - Production issue that needs fixing +- [ ] Feature - New functionality or enhancement +- [ ] Tech Debt - Code improvement or refactoring +- [ ] Chore - Maintenance or setup work + +### Priority +- [ ] P0 - Critical (blocks users/revenue) +- [ ] P1 - High (important for user experience) +- [ ] P2 - Medium (nice to have) +- [ ] P3 - Low (when time permits) + +### Impact Assessment **[CONDITIONAL: Bug]** +- **Affected Users**: {Number/percentage of users affected} +- **Reproduction Steps**: + 1. {Step 1} + 2. {Step 2} + 3. {Step 3} +- **Expected vs Actual**: {What should happen vs what happens} + +### Business Justification **[CONDITIONAL: Feature]** +- **User Value**: {Why users need this} +- **Business Value**: {Impact on metrics/revenue} +- **Effort Estimate**: {Rough size - S/M/L/XL} + +### Technical Debt Impact **[CONDITIONAL: Tech Debt]** +- **Current Problems**: {What's difficult/slow/buggy now} +- **Benefits of Fixing**: {What improves after refactoring} +- **Risk Assessment**: {Risks of not addressing this} + +## Acceptance Criteria + +## Acceptance Criteria + +## Acceptance Criteria + +## Acceptance Criteria **[REQUIRED]** + +- [ ] {Specific, testable requirement 1} +- [ ] {Specific, testable requirement 2} +- [ ] {Specific, testable requirement 3} + +## Test Cases **[CONDITIONAL: Testing Task]** + +{Delete unless this is a testing task} + +### Test Case 1: {Test Case Name} +- **Test ID**: TC-001 +- **Preconditions**: {What must be true before testing} +- **Steps**: + 1. {Step 1} + 2. {Step 2} + 3. {Step 3} +- **Expected Results**: {What should happen} +- **Actual Results**: {To be filled during execution} +- **Status**: {Pass/Fail/Blocked} + +### Test Case 2: {Test Case Name} +- **Test ID**: TC-002 +- **Preconditions**: {What must be true before testing} +- **Steps**: + 1. {Step 1} + 2. {Step 2} +- **Expected Results**: {What should happen} +- **Actual Results**: {To be filled during execution} +- **Status**: {Pass/Fail/Blocked} + +## Documentation Sections **[CONDITIONAL: Documentation Task]** + +{Delete unless this is a documentation task} + +### User Guide Content +- **Feature Description**: {What this feature does and why it's useful} +- **Prerequisites**: {What users need before using this feature} +- **Step-by-Step Instructions**: + 1. {Step 1 with screenshots/examples} + 2. {Step 2 with screenshots/examples} + 3. {Step 3 with screenshots/examples} + +### Troubleshooting Guide +- **Common Issue 1**: {Problem description and solution} +- **Common Issue 2**: {Problem description and solution} +- **Error Messages**: {List of error messages and what they mean} + +### API Documentation **[CONDITIONAL: API Documentation]** +- **Endpoint**: {API endpoint description} +- **Parameters**: {Required and optional parameters} +- **Example Request**: {Code example} +- **Example Response**: {Expected response format} + +## Implementation Notes **[CONDITIONAL: Technical Task]** + +{Keep for technical tasks, delete for non-technical. Technical details, approach, or important considerations} + +### Technical Approach +{How this will be implemented} + +### Dependencies +{Other tasks or systems this depends on} + +### Risk Considerations +{Technical risks and mitigation strategies} + +## Status Updates **[REQUIRED]** + +### 2026-07-12 — FIXED + VERIFIED (maintainer approved the interface bump) +Implemented the sketch exactly: +- `cloacina-workflow::secret::MapSecretResolver` — in-memory resolver over `{secret_name: {field: value}}`; Debug renders names only. +- `TaskExecutionRequest.resolved_secrets: BTreeMap>` (workflow-plugin types.rs); manual Debug keeps values out of logs; round-trip + no-leak-in-Debug unit test. +- Host bridge (`dynamic_task.rs::execute`): reads the `SECRET_REFS_KEY` alias map, resolves each CONCRETE secret name via the context's resolver (executor- or agent-attached) BEFORE the plugin call — fail-closed with a clear per-secret error — and ships the values in the request. +- Plugin shell (`package!` execute_task): attaches `MapSecretResolver` to the rebuilt scope when `resolved_secrets` is non-empty, so `context.secret(...)` works identically inside packages. One bridge serves in-process AND agent execution. +- **Interface version 4 → 5** (`#[fidius::plugin_interface]`): the request struct is a bincode wire change; stale artifacts fail the version gate at load instead of mis-decoding. + +**Verified:** `angreal demos features workflow-secrets` FULL PASS — secret create → `$secret`-bound run → execution **Completed** (a packaged task resolved a secret across the boundary for the first time) → rotate → rerun Completed → `secret get` metadata-only → literal value rejected pre-execution. The lane also proves the v5 gate end-to-end (package rebuilt + loaded under the new interface). Lane re-enabled in the CI matrix. cargo check clean across cloacina-workflow/-plugin/cloacina/server/agent. diff --git a/.metis/backlog/bugs/CLOACI-T-0896.md b/.metis/backlog/bugs/CLOACI-T-0896.md new file mode 100644 index 000000000..0d28528e3 --- /dev/null +++ b/.metis/backlog/bugs/CLOACI-T-0896.md @@ -0,0 +1,172 @@ +--- +id: bug-polling-and-batch-accumulators +level: task +title: "BUG: polling and batch accumulators silently degrade to passthrough in packaged graphs" +short_code: "CLOACI-T-0896" +created_at: 2026-07-12T01:36:33.300520+00:00 +updated_at: 2026-07-14T22:24:25.278833+00:00 +parent: +blocked_by: [] +archived: false + +tags: + - "#task" + - "#bug" + - "#phase/completed" + + +exit_criteria_met: false +initiative_id: NULL +--- + +# BUG: polling and batch accumulators silently degrade to passthrough in packaged graphs + +*This template includes sections for various types of tasks. Delete sections that don't apply to your specific use case.* + +## Parent Initiative **[CONDITIONAL: Assigned Task]** + +[[Parent Initiative]] + +## Objective **[REQUIRED]** + +**Finding from T-0891 (2026-07-12, I-0138 feature-coverage push):** the packaged-graph accumulator factory (`computation_graph/packaging_bridge.rs:225`) matches only `"stream"` and `"state"`; everything else — including `"polling"` and `"batch"`, which have real macros (`#[polling_accumulator(interval)]`, `#[batch_accumulator(flush_interval, max_buffer_size)]`) and real python builders (`cloaca.polling_accumulator/batch_accumulator`) — hits the `_ =>` arm and **silently becomes a passthrough accumulator**. A user who declares a batch accumulator in a packaged graph gets per-event firing with no warning: worse than unsupported, it's silently wrong. (Same silent-degradation class as the pre-T-0839 bug for authored specs.) + +**Fix:** +1. `packaging_bridge.rs` factory match gains `"polling"` and `"batch"` arms wired to their real factories (they exist for the embedded path — locate `PollingAccumulatorFactory`/`BatchAccumulatorFactory` or the runtime equivalents and thread their configs: `interval`; `flush_interval`/`max_buffer_size`). +2. The `_ =>` fallback should WARN loudly (or fail the load) instead of silently passthrough-ing an unknown declared type. +3. Extend the T-0891 CG feature-tour example to cover polling + batch once functional; its harness lane is the regression net. + +**Acceptance:** a packaged graph declaring polling and batch accumulators (macro or `[[metadata.accumulators]]` override) exhibits real polling/batching behavior on the demo stack; unknown types are loud. Related: [[CLOACI-T-0891]], T-0839 (the authored-spec analog of this bug). + +## Backlog Item Details **[CONDITIONAL: Backlog Item]** + +{Delete this section when task is assigned to an initiative} + +### Type +- [ ] Bug - Production issue that needs fixing +- [ ] Feature - New functionality or enhancement +- [ ] Tech Debt - Code improvement or refactoring +- [ ] Chore - Maintenance or setup work + +### Priority +- [ ] P0 - Critical (blocks users/revenue) +- [ ] P1 - High (important for user experience) +- [ ] P2 - Medium (nice to have) +- [ ] P3 - Low (when time permits) + +### Impact Assessment **[CONDITIONAL: Bug]** +- **Affected Users**: {Number/percentage of users affected} +- **Reproduction Steps**: + 1. {Step 1} + 2. {Step 2} + 3. {Step 3} +- **Expected vs Actual**: {What should happen vs what happens} + +### Business Justification **[CONDITIONAL: Feature]** +- **User Value**: {Why users need this} +- **Business Value**: {Impact on metrics/revenue} +- **Effort Estimate**: {Rough size - S/M/L/XL} + +### Technical Debt Impact **[CONDITIONAL: Tech Debt]** +- **Current Problems**: {What's difficult/slow/buggy now} +- **Benefits of Fixing**: {What improves after refactoring} +- **Risk Assessment**: {Risks of not addressing this} + +## Acceptance Criteria + +## Acceptance Criteria + +## Acceptance Criteria + +## Acceptance Criteria **[REQUIRED]** + +- [ ] {Specific, testable requirement 1} +- [ ] {Specific, testable requirement 2} +- [ ] {Specific, testable requirement 3} + +## Test Cases **[CONDITIONAL: Testing Task]** + +{Delete unless this is a testing task} + +### Test Case 1: {Test Case Name} +- **Test ID**: TC-001 +- **Preconditions**: {What must be true before testing} +- **Steps**: + 1. {Step 1} + 2. {Step 2} + 3. {Step 3} +- **Expected Results**: {What should happen} +- **Actual Results**: {To be filled during execution} +- **Status**: {Pass/Fail/Blocked} + +### Test Case 2: {Test Case Name} +- **Test ID**: TC-002 +- **Preconditions**: {What must be true before testing} +- **Steps**: + 1. {Step 1} + 2. {Step 2} +- **Expected Results**: {What should happen} +- **Actual Results**: {To be filled during execution} +- **Status**: {Pass/Fail/Blocked} + +## Documentation Sections **[CONDITIONAL: Documentation Task]** + +{Delete unless this is a documentation task} + +### User Guide Content +- **Feature Description**: {What this feature does and why it's useful} +- **Prerequisites**: {What users need before using this feature} +- **Step-by-Step Instructions**: + 1. {Step 1 with screenshots/examples} + 2. {Step 2 with screenshots/examples} + 3. {Step 3 with screenshots/examples} + +### Troubleshooting Guide +- **Common Issue 1**: {Problem description and solution} +- **Common Issue 2**: {Problem description and solution} +- **Error Messages**: {List of error messages and what they mean} + +### API Documentation **[CONDITIONAL: API Documentation]** +- **Endpoint**: {API endpoint description} +- **Parameters**: {Required and optional parameters} +- **Example Request**: {Code example} +- **Example Response**: {Expected response format} + +## Implementation Notes **[CONDITIONAL: Technical Task]** + +{Keep for technical tasks, delete for non-technical. Technical details, approach, or important considerations} + +### Technical Approach +{How this will be implemented} + +### Dependencies +{Other tasks or systems this depends on} + +### Risk Considerations +{Technical risks and mitigation strategies} + +## Status Updates **[REQUIRED]** + +**2026-07-14 — CONFIRMED LIVE (Python-gap sweep).** Built `python-batch-graph` (a `@cloaca.batch_accumulator(flush_interval="1s", max_buffer_size=5)` packaged CG) and ran it on the gold path. It "passes" — but ONLY because the injected events fire the reactor immediately (passthrough behavior), NOT because the buffer flushes. Confirms the `_ =>` fallthrough at `packaging_bridge.rs:225` silently degrades batch → passthrough. + +**Effort re-grounded (the factories do NOT exist for the packaged bridge):** +- Only `PassthroughAccumulatorFactory`, `StreamBackendAccumulatorFactory`, `StateAccumulatorFactory` exist in `packaging_bridge.rs`. There is NO `BatchAccumulatorFactory`/`PollingAccumulatorFactory` to "wire" — they must be WRITTEN. +- **Batch (moderate):** the runtime `batch_accumulator_runtime(acc, ctx, socket_rx, flush_rx, config)` exists but is generic over the `BatchAccumulator` TRAIT with no list-collecting impl (tests use a bespoke `SumBatchAccumulator`). A packaged batch factory needs (a) a generic passthrough-style `BatchAccumulator` that folds `serde_json::Value` events into `Vec` and emits on flush, (b) a flush-timer task feeding `flush_rx` on `flush_interval` + max_buffer_size gating, (c) config parsing. ~mirror of `StateAccumulatorFactory` + a timer. Socket-driven, so it fits the existing spawn contract. +- **Polling (hard):** `polling_accumulator_runtime` drives a `poll()` FUNCTION on an interval (not socket events). In the packaged path the poll fn is the PYTHON function — so a packaged polling factory needs an FFI bridge that calls the loaded Python accumulator fn each interval. No socket. Materially more work than batch. +- **`state` works** (StateAccumulatorFactory real) — verified separately by `python-stateful-graph` (that example IS legit). + +**2026-07-14 — BATCH DONE (commit d2043bbc).** Implemented `JsonListBatchAccumulator` + `BatchAccumulatorFactory` (socket-driven, mirrors `StateAccumulatorFactory`; flush_interval/max_buffer_size parsed via `parse_duration_str`) and a central `accumulator_factory_for(type, config)` that all four packaged-reactor dispatch sites route through — with a **loud WARN + passthrough fallback for unknown kinds** (accept #2 met). `python-batch-graph` now exercises the REAL batch factory and passes on the gold path; state/passthrough regression-checked live. No FFI change needed for batch. + +**POLLING — scope re-grounded, materially LARGER than "harder FFI".** Unlike batch (socket-driven), a polling accumulator must CALL its poll function on an interval; on the packaged path that fn is the PYTHON accumulator fn, and the plugin FFI has NO accumulator-invoke method (only execute_graph/execute_task/invoke_trigger_poll/invoke_triggerless_graph). So packaged polling requires: +1. New FFI method `invoke_accumulator_poll(name) -> Option` on `CloacinaPlugin` → **plugin interface version bump 5 → 6** (ABI change; every plugin recompiles; `fidius_validation::test_plugin_info_populated` expectation moves 5→6; loader keeps loading v5 packages via the `NotImplemented` fallback). +2. Python plugin shell implementing it (call the registered Python accumulator poll fn; Some→emit / None→skip). +3. A `PollingAccumulatorFactory` whose `poll()` calls that FFI on the interval — needs the loaded plugin handle threaded into the factory / `AccumulatorSpawnConfig` (today it carries none). `accumulator_factory_for(type, config)` can't build polling without the handle. + +~~Cross-crate, ABI-bumping...split POLLING to its own ticket.~~ **SUPERSEDED — no ABI bump needed.** + +**2026-07-14 — POLLING DONE, T-0896 COMPLETE (commit b010dbee).** The ABI-bump concern was WRONG (maintainer pushed back — correctly). On the Python path the poll fn lives IN-PROCESS: `ACCUMULATOR_REGISTRY` keeps the callable (only tests drain it; the reconciler imports the module in the server process), so no FFI accumulator-invoke method is needed. Drove it exactly like a Python poll trigger: +- `cloacina/packaging_bridge`: `PollClosure` type + OnceLock builder hook (`register_polling_accumulator_builder`); `ClosurePollingAccumulator` (runs the injected closure under `spawn_blocking` → GIL off the async executor); `PollingAccumulatorFactory` resolves the closure by name at spawn and runs `polling_accumulator_runtime` on the config interval. Wired into `accumulator_factory_for`'s `"polling"` arm. +- `cloacina-python`: `resolve_poll_closure(name)` looks up the registered Python poll fn and wraps it (`call0` → `depythonize` → JSON bytes; None → skip). Installed from `register_authoring`, so BOTH embeddings wire it (pip wheel + server synthetic `ensure_cloaca_module`). +- Example `python-polling-graph` + new `_graph_autofire_steps` lane asserting `poll_reactor` self-fires (no inject). **Verified live: reactor self-fired.** + +**T-0896 CLOSED:** batch (d2043bbc) + polling (b010dbee) + loud-WARN fallback. No plugin interface/ABI bump. All accumulator kinds now behave correctly on the packaged path (passthrough/state/batch/polling; stream = kafka/T-0898 track). diff --git a/.metis/backlog/bugs/CLOACI-T-0897.md b/.metis/backlog/bugs/CLOACI-T-0897.md new file mode 100644 index 000000000..4fc55dea8 --- /dev/null +++ b/.metis/backlog/bugs/CLOACI-T-0897.md @@ -0,0 +1,148 @@ +--- +id: bug-trigger-less-computation +level: task +title: "BUG: trigger-less computation graphs and task-to-graph invocation never compiled in packaged crates — macro emits umbrella-crate paths" +short_code: "CLOACI-T-0897" +created_at: 2026-07-12T01:49:31.319227+00:00 +updated_at: 2026-07-12T01:49:31.319227+00:00 +parent: +blocked_by: [] +archived: false + +tags: + - "#task" + - "#phase/backlog" + - "#bug" + + +exit_criteria_met: false +initiative_id: NULL +--- + +# BUG: trigger-less computation graphs and task-to-graph invocation never compiled in packaged crates — macro emits umbrella-crate paths + +*This template includes sections for various types of tasks. Delete sections that don't apply to your specific use case.* + +## Parent Initiative **[CONDITIONAL: Assigned Task]** + +[[Parent Initiative]] + +## Objective **[REQUIRED]** + +**Finding from T-0891 (2026-07-12):** a packaged crate declaring a trigger-less `#[computation_graph]` or a `#[task(invokes = computation_graph(...))]` failed to compile with `cannot find crate cloacina` — the macros emitted umbrella-crate paths that only resolve for embedded consumers: +1. The trigger-less compiled-fn SIGNATURE hardcoded `&cloacina::Context` / `cg_runtime_root::GraphResult` in a single emission (`codegen.rs` ~:252), while the ctor + trait impl were already dual-emitted under `cfg(feature = "packaged")` (the T-0552 pattern) — the fn between them wasn't. +2. The `invokes` tail in `tasks.rs` (~:876/:891) matched on `::cloacina::computation_graph::GraphResult` ungated. + +Since I-0138 makes packaged the primary shape, task→CG invocation was effectively unshippable. Same macro-portability class as [[feedback_macro_generated_deps_invisible]]. + +**FIXED (this task, same day):** +- `codegen.rs`: the trigger-less compiled fn is now dual-emitted — `cfg(not(packaged))` via `::cloacina::cloacina_workflow::Context`/`::cloacina::computation_graph::GraphResult`, `cfg(packaged)` via `::cloacina_workflow::Context`/`::cloacina_computation_graph::GraphResult` (host-crate emission unchanged). +- `tasks.rs`: the invoke tail imports a cfg-gated `GraphResult as __CgGraphResult` alias and matches on it. +- Verified: the `cg-feature-tour` packaged example (triggerless CG + invoking task + post_invocation) compiles clean offline against the local crates; host-crate integration tests (T-0538/T-0540) unaffected (is_cloacina branch untouched). + +**REMAINING (this task's open tail):** `tasks.rs:741-754` still emits ungated `::cloacina::take_task_handle()` / `return_task_handle` for tasks with a HANDLE parameter — a packaged task using the handle param will hit the same compile error. Route those through the packaged-safe re-export (or dual-emit) and add a packaged fixture using a handle param as the regression net. + +## Backlog Item Details **[CONDITIONAL: Backlog Item]** + +{Delete this section when task is assigned to an initiative} + +### Type +- [ ] Bug - Production issue that needs fixing +- [ ] Feature - New functionality or enhancement +- [ ] Tech Debt - Code improvement or refactoring +- [ ] Chore - Maintenance or setup work + +### Priority +- [ ] P0 - Critical (blocks users/revenue) +- [ ] P1 - High (important for user experience) +- [ ] P2 - Medium (nice to have) +- [ ] P3 - Low (when time permits) + +### Impact Assessment **[CONDITIONAL: Bug]** +- **Affected Users**: {Number/percentage of users affected} +- **Reproduction Steps**: + 1. {Step 1} + 2. {Step 2} + 3. {Step 3} +- **Expected vs Actual**: {What should happen vs what happens} + +### Business Justification **[CONDITIONAL: Feature]** +- **User Value**: {Why users need this} +- **Business Value**: {Impact on metrics/revenue} +- **Effort Estimate**: {Rough size - S/M/L/XL} + +### Technical Debt Impact **[CONDITIONAL: Tech Debt]** +- **Current Problems**: {What's difficult/slow/buggy now} +- **Benefits of Fixing**: {What improves after refactoring} +- **Risk Assessment**: {Risks of not addressing this} + +## Acceptance Criteria **[REQUIRED]** + +- [ ] {Specific, testable requirement 1} +- [ ] {Specific, testable requirement 2} +- [ ] {Specific, testable requirement 3} + +## Test Cases **[CONDITIONAL: Testing Task]** + +{Delete unless this is a testing task} + +### Test Case 1: {Test Case Name} +- **Test ID**: TC-001 +- **Preconditions**: {What must be true before testing} +- **Steps**: + 1. {Step 1} + 2. {Step 2} + 3. {Step 3} +- **Expected Results**: {What should happen} +- **Actual Results**: {To be filled during execution} +- **Status**: {Pass/Fail/Blocked} + +### Test Case 2: {Test Case Name} +- **Test ID**: TC-002 +- **Preconditions**: {What must be true before testing} +- **Steps**: + 1. {Step 1} + 2. {Step 2} +- **Expected Results**: {What should happen} +- **Actual Results**: {To be filled during execution} +- **Status**: {Pass/Fail/Blocked} + +## Documentation Sections **[CONDITIONAL: Documentation Task]** + +{Delete unless this is a documentation task} + +### User Guide Content +- **Feature Description**: {What this feature does and why it's useful} +- **Prerequisites**: {What users need before using this feature} +- **Step-by-Step Instructions**: + 1. {Step 1 with screenshots/examples} + 2. {Step 2 with screenshots/examples} + 3. {Step 3 with screenshots/examples} + +### Troubleshooting Guide +- **Common Issue 1**: {Problem description and solution} +- **Common Issue 2**: {Problem description and solution} +- **Error Messages**: {List of error messages and what they mean} + +### API Documentation **[CONDITIONAL: API Documentation]** +- **Endpoint**: {API endpoint description} +- **Parameters**: {Required and optional parameters} +- **Example Request**: {Code example} +- **Example Response**: {Expected response format} + +## Implementation Notes **[CONDITIONAL: Technical Task]** + +{Keep for technical tasks, delete for non-technical. Technical details, approach, or important considerations} + +### Technical Approach +{How this will be implemented} + +### Dependencies +{Other tasks or systems this depends on} + +### Risk Considerations +{Technical risks and mitigation strategies} + +## Status Updates **[REQUIRED]** + +*To be added during implementation* diff --git a/.metis/backlog/bugs/CLOACI-T-0899.md b/.metis/backlog/bugs/CLOACI-T-0899.md new file mode 100644 index 000000000..e2dd6099b --- /dev/null +++ b/.metis/backlog/bugs/CLOACI-T-0899.md @@ -0,0 +1,152 @@ +--- +id: bug-python-param-secret-boundary +level: task +title: "BUG: Python param/secret/boundary source scanners aren't comment-aware — an inline comment breaks the declaration" +short_code: "CLOACI-T-0899" +created_at: 2026-07-12T13:16:01.070778+00:00 +updated_at: 2026-07-12T13:16:01.070778+00:00 +parent: +blocked_by: [] +archived: false + +tags: + - "#task" + - "#phase/backlog" + - "#bug" + + +exit_criteria_met: false +initiative_id: NULL +--- + +# BUG: Python param/secret/boundary source scanners aren't comment-aware — an inline comment breaks the declaration + +*This template includes sections for various types of tasks. Delete sections that don't apply to your specific use case.* + +## Parent Initiative **[CONDITIONAL: Assigned Task]** + +[[Parent Initiative]] + +## Objective **[REQUIRED]** + +**Finding from T-0885 Python params example (2026-07-12) — FIXED same-session.** `cloacina-compiler`'s Python source scanners (`param_parse.rs`) parse `@cloaca.workflow_params(...)`, `@cloaca.workflow_secrets(...)`, and `@cloaca.boundary_schema(...)` by naive comma/`=` splitting with no Python lexer — **not comment-aware**. A perfectly normal inline comment breaks the declaration: + +```python +@cloaca.workflow_params( + source=str, # required + dst=str, # required +) +``` + +parsed as a param literally named `"# required\n dst"` → the server then rejected every run with `missing required param '# required`. Observed live: the packaged workflow built "successfully" but was **unrunnable** — validation demanded a param that doesn't exist. A user commenting their params (the natural thing) silently breaks their workflow. + +**Fix (landed):** `strip_py_comments(src)` in `param_parse.rs` — a comment-aware stripper (`#`→EOL when NOT inside `'…'`/`"…"`/triple-quoted strings, backslash-escape aware, newlines preserved) applied at all three read sites (`walk_py`, `walk_py_secrets`, `walk_py_surfaces`) before parsing. One fix covers params, secrets, AND boundary schemas. Verified: the `python-parameterized` example (which naturally comments its params) builds + validates + runs to Completed after the fix (was broken before). + +**Second half (found + fixed via the python-secrets example, same session):** the scanners match `workflow_secrets(` / `workflow_params(` ANYWHERE, including inside DOCSTRINGS. A task docstring showing `@cloaca.workflow_secrets("name")` as example text was parsed as a REAL required secret named `name` → every run failed `missing required param 'name'`. Users document their decorators constantly → live footgun. Fix: extended `strip_py_comments` to BLANK the interior of triple-quoted strings (docstrings) for the scanner's view (delimiters + newlines kept), preserving single/double-quoted param DEFAULT values (`mode=(str, "copy")`) and leaving `doc_parse` (separate, on raw source) untouched so docstrings still surface in the UI. Verified: python-secrets builds + runs to Completed. + +**Remaining hardening (tail):** still string-split heuristics — needle/`#` inside a SINGLE/double-quoted string, line continuations, odd whitespace remain edge cases. Add unit tests for `strip_py_comments` (comment-in-string preserved, triple-quote blanked, escaped quote, needle-in-single-quote) and weigh a tiny real tokenizer. Related: [[CLOACI-T-0885]], [[CLOACI-T-0890]]. + +## Backlog Item Details **[CONDITIONAL: Backlog Item]** + +{Delete this section when task is assigned to an initiative} + +### Type +- [ ] Bug - Production issue that needs fixing +- [ ] Feature - New functionality or enhancement +- [ ] Tech Debt - Code improvement or refactoring +- [ ] Chore - Maintenance or setup work + +### Priority +- [ ] P0 - Critical (blocks users/revenue) +- [ ] P1 - High (important for user experience) +- [ ] P2 - Medium (nice to have) +- [ ] P3 - Low (when time permits) + +### Impact Assessment **[CONDITIONAL: Bug]** +- **Affected Users**: {Number/percentage of users affected} +- **Reproduction Steps**: + 1. {Step 1} + 2. {Step 2} + 3. {Step 3} +- **Expected vs Actual**: {What should happen vs what happens} + +### Business Justification **[CONDITIONAL: Feature]** +- **User Value**: {Why users need this} +- **Business Value**: {Impact on metrics/revenue} +- **Effort Estimate**: {Rough size - S/M/L/XL} + +### Technical Debt Impact **[CONDITIONAL: Tech Debt]** +- **Current Problems**: {What's difficult/slow/buggy now} +- **Benefits of Fixing**: {What improves after refactoring} +- **Risk Assessment**: {Risks of not addressing this} + +## Acceptance Criteria **[REQUIRED]** + +- [ ] {Specific, testable requirement 1} +- [ ] {Specific, testable requirement 2} +- [ ] {Specific, testable requirement 3} + +## Test Cases **[CONDITIONAL: Testing Task]** + +{Delete unless this is a testing task} + +### Test Case 1: {Test Case Name} +- **Test ID**: TC-001 +- **Preconditions**: {What must be true before testing} +- **Steps**: + 1. {Step 1} + 2. {Step 2} + 3. {Step 3} +- **Expected Results**: {What should happen} +- **Actual Results**: {To be filled during execution} +- **Status**: {Pass/Fail/Blocked} + +### Test Case 2: {Test Case Name} +- **Test ID**: TC-002 +- **Preconditions**: {What must be true before testing} +- **Steps**: + 1. {Step 1} + 2. {Step 2} +- **Expected Results**: {What should happen} +- **Actual Results**: {To be filled during execution} +- **Status**: {Pass/Fail/Blocked} + +## Documentation Sections **[CONDITIONAL: Documentation Task]** + +{Delete unless this is a documentation task} + +### User Guide Content +- **Feature Description**: {What this feature does and why it's useful} +- **Prerequisites**: {What users need before using this feature} +- **Step-by-Step Instructions**: + 1. {Step 1 with screenshots/examples} + 2. {Step 2 with screenshots/examples} + 3. {Step 3 with screenshots/examples} + +### Troubleshooting Guide +- **Common Issue 1**: {Problem description and solution} +- **Common Issue 2**: {Problem description and solution} +- **Error Messages**: {List of error messages and what they mean} + +### API Documentation **[CONDITIONAL: API Documentation]** +- **Endpoint**: {API endpoint description} +- **Parameters**: {Required and optional parameters} +- **Example Request**: {Code example} +- **Example Response**: {Expected response format} + +## Implementation Notes **[CONDITIONAL: Technical Task]** + +{Keep for technical tasks, delete for non-technical. Technical details, approach, or important considerations} + +### Technical Approach +{How this will be implemented} + +### Dependencies +{Other tasks or systems this depends on} + +### Risk Considerations +{Technical risks and mitigation strategies} + +## Status Updates **[REQUIRED]** + +*To be added during implementation* diff --git a/.metis/backlog/bugs/CLOACI-T-0901.md b/.metis/backlog/bugs/CLOACI-T-0901.md new file mode 100644 index 000000000..9c97e60d7 --- /dev/null +++ b/.metis/backlog/bugs/CLOACI-T-0901.md @@ -0,0 +1,154 @@ +--- +id: cross-tenant-workflow-definition +level: task +title: "Cross-tenant workflow-definition leak — a tenant can execute a workflow it never installed (shared Runtime, no per-tenant existence check in execute)" +short_code: "CLOACI-T-0901" +created_at: 2026-07-12T21:05:14.990166+00:00 +updated_at: 2026-07-12T22:39:35.915124+00:00 +parent: +blocked_by: [] +archived: false + +tags: + - "#task" + - "#bug" + - "#phase/completed" + + +exit_criteria_met: false +initiative_id: NULL +--- + +# Cross-tenant workflow-definition leak — a tenant can execute a workflow it never installed (shared Runtime, no per-tenant existence check in execute) + +*This template includes sections for various types of tasks. Delete sections that don't apply to your specific use case.* + +## Parent Initiative **[CONDITIONAL: Assigned Task]** + +[[Parent Initiative]] + +## Objective **[REQUIRED]** + +Close a cross-tenant isolation gap: a tenant can execute a workflow it never installed, as long as some OTHER tenant (or the public/global runner) has loaded that workflow name into the process-shared in-memory `Runtime`. Execution/data state is correctly isolated (the run lands in the caller's schema); the leak is at the workflow-DEFINITION visibility layer. + +Surfaced by the `python-multi-tenant` gold-path example (I-0138 Python-gap sweep, 2026-07-12) — exactly the kind of server-path gap D-3 predicted the migration would expose. + +## Backlog Item Details + +### Type +- [x] Bug - Production issue that needs fixing + +### Priority +- [ ] P0 +- [x] P1 - High — it's a breach of the tenant isolation boundary ([[project_tenant_is_isolation_boundary]] says the tenant IS the hard isolation line), though exploitation requires a valid admin/tenant key AND knowing another tenant's workflow name, and the run executes against the CALLER's own schema. + +### Impact Assessment +- **Affected Users**: any multi-tenant deployment (one server process, multiple tenant schemas sharing the runner cache's `shared_runtime`). +- **Reproduction Steps** (observed live, commit-time evidence in the `python-multi-tenant` lane): + 1. Server + a package `tenant-job` (workflow `tenant_job`) uploaded/built/loaded in tenant `public` (and independently in `mtbeta` via its own `--tenant-schema` compiler). + 2. Create a THIRD tenant `mtgamma` and never upload any package to it. Confirm `mtgamma.workflow_packages` has **0** `tenant-job` rows (DB isolation intact). + 3. `cloacinactl --tenant mtgamma workflow run tenant_job` → **accepted**. Server log: `Workflow execution scheduled: ` + `Executed workflow 'tenant_job' for tenant 'mtgamma'`. (Contrast: before `mtbeta`/`public` had loaded it, the same call returned `Workflow not found in registry: tenant_job`.) +- **Expected vs Actual**: Expected — a tenant with no matching workflow in its OWN registry gets `404 workflow_not_found` (like it does when NO tenant has loaded it). Actual — the execute route schedules the run because the name resolves in the process-shared `Runtime` populated by other tenants. + +### Root cause (grounded) +`crates/cloacina-server/src/routes/executions.rs::execute_workflow`: it resolves the tenant-scoped `Database` (so execution STATE is isolated) and does a paused-check + declared-params-check against the tenant's `WorkflowRegistryImpl` — but BOTH "fail open" and neither REJECTS when the workflow is simply absent from this tenant's registry. It then calls `tenant_runner.execute_async(&name, ...)`, which resolves `name` against the per-tenant runner's `shared_runtime` (`tenant_runner_cache.rs`: "Shared `Runtime` for every per-tenant runner"). The reconciler namespaces tasks `tenant::pkg::wf::task`, but the execute-by-bare-name lookup isn't tenant-scoped, so a workflow any tenant loaded is executable by all. + +### Business Justification **[CONDITIONAL: Feature]** +- **User Value**: {Why users need this} +- **Business Value**: {Impact on metrics/revenue} +- **Effort Estimate**: {Rough size - S/M/L/XL} + +### Technical Debt Impact **[CONDITIONAL: Tech Debt]** +- **Current Problems**: {What's difficult/slow/buggy now} +- **Benefits of Fixing**: {What improves after refactoring} +- **Risk Assessment**: {Risks of not addressing this} + +## Acceptance Criteria + +## Acceptance Criteria + +## Acceptance Criteria + +## Acceptance Criteria **[REQUIRED]** + +- [ ] `execute_workflow` (and the equivalent `workflow run` path) REJECTS with `404 workflow_not_found` when the workflow is absent from the CALLING tenant's own registry — for non-public tenants — instead of resolving it from the shared `Runtime`. +- [ ] `public` behavior is preserved (it maps to the admin schema + global runner by design; decide whether public keeps shared-catalog semantics or is also scoped — likely scope it too for consistency). +- [ ] The `python-multi-tenant` example's Isolation-2 assertion (a tenant with no package cannot run the workflow) flips from a logged known-gap back to a HARD assertion. +- [ ] A server integration test covers the negative: tenant B cannot execute a workflow only tenant A installed. + +## Proposed fix (small, targeted) +In `execute_workflow`, after constructing the tenant `WorkflowRegistryImpl`, add an existence gate: if `registry.get_workflow(&name)` / the declared-metadata lookup finds NO such workflow in THIS tenant, return `404 workflow_not_found` before touching the runner. This reuses the registry already built for the paused/params checks (one extra query). Keep the existing "fail open on registry ERROR" behavior only for transient errors — a definitive "not found" must fail CLOSED. Maintainer decision needed on whether `public`/global keeps any shared-catalog exception. + +## Design note / maintainer call +This is the tenant ISOLATION boundary, so flagging rather than silently changing semantics. [[project_tenant_is_isolation_boundary]] frames intra-tenant object authZ as a non-goal (isolate by spinning up a new tenant) — but that very framing REQUIRES cross-tenant execution to be impossible, which this bug violates. So the fix aligns with the stated model. Confirm before implementing. + +## Test Cases **[CONDITIONAL: Testing Task]** + +{Delete unless this is a testing task} + +### Test Case 1: {Test Case Name} +- **Test ID**: TC-001 +- **Preconditions**: {What must be true before testing} +- **Steps**: + 1. {Step 1} + 2. {Step 2} + 3. {Step 3} +- **Expected Results**: {What should happen} +- **Actual Results**: {To be filled during execution} +- **Status**: {Pass/Fail/Blocked} + +### Test Case 2: {Test Case Name} +- **Test ID**: TC-002 +- **Preconditions**: {What must be true before testing} +- **Steps**: + 1. {Step 1} + 2. {Step 2} +- **Expected Results**: {What should happen} +- **Actual Results**: {To be filled during execution} +- **Status**: {Pass/Fail/Blocked} + +## Documentation Sections **[CONDITIONAL: Documentation Task]** + +{Delete unless this is a documentation task} + +### User Guide Content +- **Feature Description**: {What this feature does and why it's useful} +- **Prerequisites**: {What users need before using this feature} +- **Step-by-Step Instructions**: + 1. {Step 1 with screenshots/examples} + 2. {Step 2 with screenshots/examples} + 3. {Step 3 with screenshots/examples} + +### Troubleshooting Guide +- **Common Issue 1**: {Problem description and solution} +- **Common Issue 2**: {Problem description and solution} +- **Error Messages**: {List of error messages and what they mean} + +### API Documentation **[CONDITIONAL: API Documentation]** +- **Endpoint**: {API endpoint description} +- **Parameters**: {Required and optional parameters} +- **Example Request**: {Code example} +- **Example Response**: {Expected response format} + +## Implementation Notes **[CONDITIONAL: Technical Task]** + +{Keep for technical tasks, delete for non-technical. Technical details, approach, or important considerations} + +### Technical Approach +{How this will be implemented} + +### Dependencies +{Other tasks or systems this depends on} + +### Risk Considerations +{Technical risks and mitigation strategies} + +## Status Updates **[REQUIRED]** + +**2026-07-12 — FIXED + verified.** Implemented the per-tenant existence gate. +- `crates/cloacina/src/registry/workflow_registry/mod.rs`: new `workflow_exists(&name)` — mirrors the `is_workflow_paused`/`get_workflow_declared_params` lookup (matches `w.workflow_name == name || w.package_name == name`) over `list_workflows()` (this tenant's registry). +- `crates/cloacina-server/src/routes/executions.rs::execute_workflow`: for `tenant_id != "public"`, call `workflow_exists` before touching the runner; a definitive `Ok(false)` returns `404 workflow_not_found` (fails CLOSED), a registry `Err` logs + proceeds (fails open, so a transient DB fault never wedges execution). +- **Decision on `public`**: left exempt (documented in code). `public` maps to the admin/global catalog + global runner and may serve inventory/global workflows not tracked as tenant packages; gating it risks breaking those. The reported leak (a NON-public tenant running another tenant's workflow) is closed. Maintainer can extend to `public` later if desired. +- **Verified live** on the `python-multi-tenant` gold-path lane: `public` + `mtbeta` (installed) still run; `mtgamma` (never installed) is now rejected → `ok: tenant mtgamma (no package) cannot run the workflow — isolated`. Isolation-2 flipped from a logged known-gap back to a HARD assertion. +- **Regression guard**: the `python-multi-tenant` lane (in the CI discovery matrix) exercises the negative end-to-end. A dedicated server-crate integration test (AC #4) was NOT added — the server `tests/` dir has no route+DB+tenant harness (only `cli_validation.rs`), and standing one up would reproduce what the lane already covers; deferred as optional hardening. +- Broader-suite note: any pre-existing test that executed a workflow in a NON-public tenant WITHOUT installing it there was relying on the bug and would now get a 404 — none found in the features lanes (all use `public`); worth a full integration run before merge. diff --git a/.metis/backlog/features/CLOACI-T-0894.md b/.metis/backlog/features/CLOACI-T-0894.md new file mode 100644 index 000000000..5e49cf73e --- /dev/null +++ b/.metis/backlog/features/CLOACI-T-0894.md @@ -0,0 +1,146 @@ +--- +id: workflow-instance-registration-has +level: task +title: "Workflow-instance registration has no server surface — I-0116 instances are embedded-runner-only" +short_code: "CLOACI-T-0894" +created_at: 2026-07-11T22:28:03.026241+00:00 +updated_at: 2026-07-11T22:28:03.026241+00:00 +parent: +blocked_by: [] +archived: false + +tags: + - "#task" + - "#phase/backlog" + - "#feature" + + +exit_criteria_met: false +initiative_id: NULL +--- + +# Workflow-instance registration has no server surface — I-0116 instances are embedded-runner-only + +*This template includes sections for various types of tasks. Delete sections that don't apply to your specific use case.* + +## Parent Initiative **[CONDITIONAL: Assigned Task]** + +[[Parent Initiative]] + +## Objective **[REQUIRED]** + +**Finding from T-0889 (2026-07-11, the I-0138 feature-coverage push):** I-0116 shipped "named, scheduled, param-bound workflow instances" — but instance REGISTRATION exists only on the embedded runner: `DefaultRunner::register_cron_workflow_instance` (runner/default_runner/cron_api.rs) and its python binding (`register_workflow_instance`, bindings/runner.rs:1015). There is **no server route and no cloacinactl noun** to create/list/delete a named instance. On the primary interface (the gold path, per I-0138 D-3), users can bind params PER RUN (`workflow run --context`, typed-validated per T-0757) but cannot create a persistent named/scheduled instance at all — the feature's headline capability is unreachable in the deployment mode we lead with. + +The engine side is ready: schedules rows carry `params` JSON + `instance_name` (migration 040), the fire-time merge delivers bound params as top-level context keys, and `WorkflowInstance` (cloacina::workflow_instance) validates against declared InputSlots. + +**Build the server surface:** +- Routes (per-tenant): create/list/get/delete workflow instances — `POST/GET/DELETE /v1/tenants/{t}/workflows/{name}/instances[/{instance}]` — body = instance name + params + optional cron schedule/timezone; validate params against the workflow's declared InputSlots (same `validate_declared_params` the execute route uses); persist via the existing schedule DAL (`find_by_instance_name`, UnifiedSchedule params fields). +- `cloacinactl instance` noun (or `workflow instance` subverb): create/list/inspect/delete, `--param k=v`/`--params file.json`, `--cron`. +- UI surface can follow separately. + +**Acceptance:** a packaged workflow with `params(...)` gets a named scheduled instance created via cloacinactl against the demo stack; the instance fires on schedule with its bound params (visible in the execution context); T-0889's example README gains the instance section it currently can't have. Related: [[CLOACI-T-0889]], I-0116 (#181). + +## Backlog Item Details **[CONDITIONAL: Backlog Item]** + +{Delete this section when task is assigned to an initiative} + +### Type +- [ ] Bug - Production issue that needs fixing +- [ ] Feature - New functionality or enhancement +- [ ] Tech Debt - Code improvement or refactoring +- [ ] Chore - Maintenance or setup work + +### Priority +- [ ] P0 - Critical (blocks users/revenue) +- [ ] P1 - High (important for user experience) +- [ ] P2 - Medium (nice to have) +- [ ] P3 - Low (when time permits) + +### Impact Assessment **[CONDITIONAL: Bug]** +- **Affected Users**: {Number/percentage of users affected} +- **Reproduction Steps**: + 1. {Step 1} + 2. {Step 2} + 3. {Step 3} +- **Expected vs Actual**: {What should happen vs what happens} + +### Business Justification **[CONDITIONAL: Feature]** +- **User Value**: {Why users need this} +- **Business Value**: {Impact on metrics/revenue} +- **Effort Estimate**: {Rough size - S/M/L/XL} + +### Technical Debt Impact **[CONDITIONAL: Tech Debt]** +- **Current Problems**: {What's difficult/slow/buggy now} +- **Benefits of Fixing**: {What improves after refactoring} +- **Risk Assessment**: {Risks of not addressing this} + +## Acceptance Criteria **[REQUIRED]** + +- [ ] {Specific, testable requirement 1} +- [ ] {Specific, testable requirement 2} +- [ ] {Specific, testable requirement 3} + +## Test Cases **[CONDITIONAL: Testing Task]** + +{Delete unless this is a testing task} + +### Test Case 1: {Test Case Name} +- **Test ID**: TC-001 +- **Preconditions**: {What must be true before testing} +- **Steps**: + 1. {Step 1} + 2. {Step 2} + 3. {Step 3} +- **Expected Results**: {What should happen} +- **Actual Results**: {To be filled during execution} +- **Status**: {Pass/Fail/Blocked} + +### Test Case 2: {Test Case Name} +- **Test ID**: TC-002 +- **Preconditions**: {What must be true before testing} +- **Steps**: + 1. {Step 1} + 2. {Step 2} +- **Expected Results**: {What should happen} +- **Actual Results**: {To be filled during execution} +- **Status**: {Pass/Fail/Blocked} + +## Documentation Sections **[CONDITIONAL: Documentation Task]** + +{Delete unless this is a documentation task} + +### User Guide Content +- **Feature Description**: {What this feature does and why it's useful} +- **Prerequisites**: {What users need before using this feature} +- **Step-by-Step Instructions**: + 1. {Step 1 with screenshots/examples} + 2. {Step 2 with screenshots/examples} + 3. {Step 3 with screenshots/examples} + +### Troubleshooting Guide +- **Common Issue 1**: {Problem description and solution} +- **Common Issue 2**: {Problem description and solution} +- **Error Messages**: {List of error messages and what they mean} + +### API Documentation **[CONDITIONAL: API Documentation]** +- **Endpoint**: {API endpoint description} +- **Parameters**: {Required and optional parameters} +- **Example Request**: {Code example} +- **Example Response**: {Expected response format} + +## Implementation Notes **[CONDITIONAL: Technical Task]** + +{Keep for technical tasks, delete for non-technical. Technical details, approach, or important considerations} + +### Technical Approach +{How this will be implemented} + +### Dependencies +{Other tasks or systems this depends on} + +### Risk Considerations +{Technical risks and mitigation strategies} + +## Status Updates **[REQUIRED]** + +*To be added during implementation* diff --git a/.metis/backlog/tech-debt/CLOACI-T-0898.md b/.metis/backlog/tech-debt/CLOACI-T-0898.md new file mode 100644 index 000000000..e8262f88f --- /dev/null +++ b/.metis/backlog/tech-debt/CLOACI-T-0898.md @@ -0,0 +1,150 @@ +--- +id: migrate-event-source-backends +level: task +title: "Migrate event-source backends (kafka first) out of core into constructor providers — core drops the kafka feature + rdkafka" +short_code: "CLOACI-T-0898" +created_at: 2026-07-12T11:59:47.722068+00:00 +updated_at: 2026-07-12T11:59:47.722068+00:00 +parent: +blocked_by: [] +archived: false + +tags: + - "#task" + - "#phase/backlog" + - "#tech-debt" + + +exit_criteria_met: false +initiative_id: NULL +--- + +# Migrate event-source backends (kafka first) out of core into constructor providers — core drops the kafka feature + rdkafka + +*This template includes sections for various types of tasks. Delete sections that don't apply to your specific use case.* + +## Parent Initiative **[CONDITIONAL: Assigned Task]** + +[[Parent Initiative]] + +## Objective **[REQUIRED]** + +**Maintainer architectural steer (2026-07-12, from the T-0891 CG feature-tour work):** Kafka is a *consumption connector*, not core architecture — it should be compiled into the PACKAGE, not the host engine. Today it's the opposite: + +- `rdkafka` (native librdkafka) is an **optional dep of the core `cloacina` crate** (`cloacina/Cargo.toml:80`), gated by `feature = "kafka"`. +- `KafkaEventSource` is `#[cfg(feature = "kafka")]` in `computation_graph/packaging_bridge.rs` and runs in the **host** runtime (`tokio::spawn`'d accumulator event loop). +- A package only DECLARES `accumulator_type = "stream"` + broker/topic/group config; it carries no kafka code. `cloacina-server`'s default build does NOT enable kafka (`default = ["postgres"]`; kafka is a non-default `cloacina-server` feature), so a stream accumulator **silently degrades to passthrough** with only an ERROR log — the reactor still loads and "runs", no fire ever arrives (observed live in the T-0891 lane: `stream accumulator requires 'kafka' feature` → reactor loaded anyway). + +**Target = migrate onto the DELIVERED constructor/provider mechanism** (this is a migration, not new architecture: `#[constructor]`, `constructor_provider!`, `cloacinactl constructor package`, ProviderManifest suites — specs [[CLOACI-S-0014]]/[[CLOACI-S-0015]], ADRs A-0009/A-0010/A-0011, all shipped). A kafka source becomes a first-party **constructor provider** shipped IN a package; the host core drops the `kafka` feature and the rdkafka link entirely. Same treatment for other source backends (batch/polling — see [[CLOACI-T-0896]], same "source gated in core" class). + +**Open design questions to settle first:** +1. **Source vs ingest shape.** The constructor `kind = accumulator` model is request/response `ingest(...)` (AccumulatorObject). A kafka SOURCE is a long-running consumer loop, not an ingest transform. Does the provider model already express a streaming/long-running source, or does this need a "source" provider shape (a provider that owns a loop and pushes events into the host accumulator socket)? +2. **WASM vs in-process.** rdkafka is native C — it will NOT compile to `wasm32-wasip2` (the provider WASM target, T-0836). So a kafka provider must use the IN-PROCESS cdylib provider path (`configure_in_process`), not the WASM path. Confirm the packaged-workflow load path can host an in-process provider that runs a background consumer, and how its lifecycle ties to reactor load/unload. +3. **Loud failure.** Until migrated, `packaging_bridge.rs`'s stream branch should FAIL the package load (or surface a build/reconcile error) when the host lacks the source backend, not silently passthrough (ties to [[CLOACI-T-0896]] item 2). + +**Acceptance:** kafka stream accumulator support ships as a first-party constructor provider consumed by a package; `cloacina`/`cloacina-server` build with NO `kafka` feature and NO rdkafka dep; a packaged workflow using the kafka-source provider streams end-to-end on the demo stack; the T-0891 `cg-feature-tour` example's kafka surface is re-enabled against the provider. Related: [[CLOACI-T-0871]] (first-party provider promotion), [[CLOACI-T-0896]], [[CLOACI-T-0891]]. + +## Backlog Item Details **[CONDITIONAL: Backlog Item]** + +{Delete this section when task is assigned to an initiative} + +### Type +- [ ] Bug - Production issue that needs fixing +- [ ] Feature - New functionality or enhancement +- [ ] Tech Debt - Code improvement or refactoring +- [ ] Chore - Maintenance or setup work + +### Priority +- [ ] P0 - Critical (blocks users/revenue) +- [ ] P1 - High (important for user experience) +- [ ] P2 - Medium (nice to have) +- [ ] P3 - Low (when time permits) + +### Impact Assessment **[CONDITIONAL: Bug]** +- **Affected Users**: {Number/percentage of users affected} +- **Reproduction Steps**: + 1. {Step 1} + 2. {Step 2} + 3. {Step 3} +- **Expected vs Actual**: {What should happen vs what happens} + +### Business Justification **[CONDITIONAL: Feature]** +- **User Value**: {Why users need this} +- **Business Value**: {Impact on metrics/revenue} +- **Effort Estimate**: {Rough size - S/M/L/XL} + +### Technical Debt Impact **[CONDITIONAL: Tech Debt]** +- **Current Problems**: {What's difficult/slow/buggy now} +- **Benefits of Fixing**: {What improves after refactoring} +- **Risk Assessment**: {Risks of not addressing this} + +## Acceptance Criteria **[REQUIRED]** + +- [ ] {Specific, testable requirement 1} +- [ ] {Specific, testable requirement 2} +- [ ] {Specific, testable requirement 3} + +## Test Cases **[CONDITIONAL: Testing Task]** + +{Delete unless this is a testing task} + +### Test Case 1: {Test Case Name} +- **Test ID**: TC-001 +- **Preconditions**: {What must be true before testing} +- **Steps**: + 1. {Step 1} + 2. {Step 2} + 3. {Step 3} +- **Expected Results**: {What should happen} +- **Actual Results**: {To be filled during execution} +- **Status**: {Pass/Fail/Blocked} + +### Test Case 2: {Test Case Name} +- **Test ID**: TC-002 +- **Preconditions**: {What must be true before testing} +- **Steps**: + 1. {Step 1} + 2. {Step 2} +- **Expected Results**: {What should happen} +- **Actual Results**: {To be filled during execution} +- **Status**: {Pass/Fail/Blocked} + +## Documentation Sections **[CONDITIONAL: Documentation Task]** + +{Delete unless this is a documentation task} + +### User Guide Content +- **Feature Description**: {What this feature does and why it's useful} +- **Prerequisites**: {What users need before using this feature} +- **Step-by-Step Instructions**: + 1. {Step 1 with screenshots/examples} + 2. {Step 2 with screenshots/examples} + 3. {Step 3 with screenshots/examples} + +### Troubleshooting Guide +- **Common Issue 1**: {Problem description and solution} +- **Common Issue 2**: {Problem description and solution} +- **Error Messages**: {List of error messages and what they mean} + +### API Documentation **[CONDITIONAL: API Documentation]** +- **Endpoint**: {API endpoint description} +- **Parameters**: {Required and optional parameters} +- **Example Request**: {Code example} +- **Example Response**: {Expected response format} + +## Implementation Notes **[CONDITIONAL: Technical Task]** + +{Keep for technical tasks, delete for non-technical. Technical details, approach, or important considerations} + +### Technical Approach +{How this will be implemented} + +### Dependencies +{Other tasks or systems this depends on} + +### Risk Considerations +{Technical risks and mitigation strategies} + +## Status Updates **[REQUIRED]** + +*To be added during implementation* diff --git a/.metis/initiatives/CLOACI-I-0138/initiative.md b/.metis/initiatives/CLOACI-I-0138/initiative.md index ef83d2d21..6a298be4f 100644 --- a/.metis/initiatives/CLOACI-I-0138/initiative.md +++ b/.metis/initiatives/CLOACI-I-0138/initiative.md @@ -64,6 +64,21 @@ Grounded against the actual surfaces: `cloacina-macros/src/lib.rs` (all proc mac **Conclusion:** the migration (embedded→primary interface) and the gap-fill are the SAME work: the end-state is **one gold-path example per feature**. Decomposed below as T-0889..T-0893 alongside the existing T-0885/T-0886. +## Python-gap sweep (2026-07-12) — new gold-path Python peers, all verified live +Goal: "find and close our gaps in python." Each new example is a probe that runs the full gold path (pack → upload → compile → reconcile → execute) on the host-lane harness. All PASSED — the Python packaged surface is more solid than feared; these convert untested→tested (coverage gaps closed), no product bugs found in this batch. +- **python-triggers** — `@cloaca.trigger(on=…, poll_interval="3s")` poll trigger; fires `file_processing_py` automatically (new `_trigger_wait_steps` lane asserts an execution auto-appears + Completed). Python `@cloaca.trigger` existed only in the demo-py-cron fixture before — first time on the features gold path. Commit a9319900. +- **python-retries** — `@cloaca.task(retry_attempts=4, retry_backoff="fixed", retry_delay_ms=200)`; task RAISES on attempts 1-2, so Completed proves the server honored the retry policy. Commit b161f58a. +- **python-conditional** — `trigger_rules=cloaca.context_value("do_audit","Equals",True)` gates `audit` to Skipped; it RAISES if run, so Completed proves the skip (T-0763 parity, previously fixture-only). `record` fans in over the skipped dep + asserts the run path ran. Commit bdb17c51. +- **python-cron** — `@cloaca.trigger(on=…, cron="*/3 * * * * *")`; the CRON SCHEDULER (distinct subsystem from the poll registry) fires `heartbeat_workflow`. T-0688 cron surface, first on the gold path. Commit dfb4215a. + +Also this session: **T-0900 fully closed** — `packaged-triggers` was doc-only (declared a `#[workflow]`, only *documented* a trigger); made it a REAL Rust poll-trigger (`#[trigger(on="file_processing", poll_interval="3s")]` → `Fire(Some(ctx))`) + lean deps. `_PACKAGED_SKIP` is now empty; every packaged example is driveable on the gold path. Commit 5a5768a8. + +- **python-multi-tenant** — the FULL per-tenant deployment model on the gold path: package deployed into `public` AND a second tenant `mtbeta` (which runs its OWN `cloacina-compiler --tenant-schema mtbeta` — the default compiler only claims public/admin, by design), both tenants execute independently, execution-state isolation HARD-asserted (neither tenant's `execution list` sees the other's run). **Found a real bug → [[CLOACI-T-0901]]:** a third tenant `mtgamma` that never installed the package was still allowed to run the workflow — the execute route resolves the name against the process-shared in-memory `Runtime` (populated by other tenants) with no per-tenant existence check. Execution STATE is isolated; workflow-DEFINITION visibility leaks. The lane surfaces this loudly (not a hard fail) pending the fix; flip to a hard assert when T-0901 lands. Lane adds `_multi_tenant_steps` + spawns a per-tenant compiler. + +**This batch's real find:** the multi-tenant probe was the one that exposed a genuine isolation bug (T-0901) — vindicating the "write honest examples through the primary interface" strategy. The trigger/retry/conditional/cron probes were clean coverage; multi-tenancy was the gap. + +**Assertion technique worth reusing:** design the workflow so the *desired behavior is the only path to Completed* (task raises if it shouldn't run / fails until it should succeed) — turns a plain "Completed" check into honest proof without needing task-level status APIs (cloacinactl has no `execution tasks` verb; only the server `/executions/{id}/tasks` endpoint exposes per-task status). + ## Migration inventory (grounded 2026-07-10) — ≈26 units still demoing via `DefaultRunner` - **Rust feature examples (10):** conditional-retries, cron-scheduling, deferred-tasks, event-triggers, multi-tenant, per-tenant-credentials, registry-execution, python-workflow, computation-graphs/filtered-reactor, constructor-contract/fs-grant-demo - **Performance (3):** simple, parallel, pipeline (may stay embedded by design — they benchmark the engine, not the interface; decide at design time) diff --git a/.metis/initiatives/CLOACI-I-0138/tasks/CLOACI-T-0885.md b/.metis/initiatives/CLOACI-I-0138/tasks/CLOACI-T-0885.md index 8ed65a1f4..80a5f9bec 100644 --- a/.metis/initiatives/CLOACI-I-0138/tasks/CLOACI-T-0885.md +++ b/.metis/initiatives/CLOACI-I-0138/tasks/CLOACI-T-0885.md @@ -4,14 +4,14 @@ level: task title: "Canonical Python packaged example — promote + gold-path demo-stack README" short_code: "CLOACI-T-0885" created_at: 2026-07-10T01:16:02.734335+00:00 -updated_at: 2026-07-10T01:16:02.734335+00:00 +updated_at: 2026-07-12T12:57:12.345241+00:00 parent: CLOACI-I-0138 blocked_by: [] archived: false tags: - "#task" - - "#phase/todo" + - "#phase/completed" exit_criteria_met: false @@ -28,7 +28,7 @@ initiative_id: CLOACI-I-0138 ## Objective **[REQUIRED]** -{Clear statement of what this task accomplishes} +The Python peer of [[CLOACI-T-0884]]: a canonical PYTHON packaged example demonstrating the primary interface (pack → upload → compile → reconcile → execute) — the packaged-Python gold path that no user-facing example currently shows (only fixtures: demo-py-*). Establishes the Python half of the T-0886 standard. ## Backlog Item Details **[CONDITIONAL: Backlog Item]** @@ -64,6 +64,10 @@ initiative_id: CLOACI-I-0138 - **Benefits of Fixing**: {What improves after refactoring} - **Risk Assessment**: {Risks of not addressing this} +## Acceptance Criteria + +## Acceptance Criteria + ## Acceptance Criteria **[REQUIRED]** - [ ] {Specific, testable requirement 1} @@ -133,4 +137,11 @@ initiative_id: CLOACI-I-0138 ## Status Updates **[REQUIRED]** -*To be added during implementation* +### 2026-07-12 — canonical Python packaged example built; lane running +Grounded the packaged-Python shape from `cloacinactl package new --language python` (new.rs:163) + the `demo-py-workflow` fixture: minimal `package.toml` (`[package] name/version` + `[metadata] workflow_name/description` — language + entry_module INFERRED from the `workflow//` layout), `workflow//__init__.py` (empty) + `tasks.py` with bare `@cloaca.task(id=, dependencies=[])` decorators (NOT WorkflowBuilder — that's in-process only). No Cargo/build.rs: the compiler skips cargo for `language=python`, the reconciler imports via embedded Python. + +Key confirmation (maintainer): **Python is a CORE server capability** — `cloacina-server` unconditionally deps `cloacina-python` + calls `cloacina_python::install()` at startup; the server synthesizes `cloaca` in-process via `ensure_cloaca_module` (I-0137). So a default host `cloacina-server` loads Python packages — no `--features` needed, and the `_run_gold_path` host lane works for Python (build is fast — no cargo). + +Built `examples/features/workflows/python-packaged/` (peer of `simple-packaged`): `data_pipeline` = collect_data → process_data → generate_report, with what/why docstrings (T-0754 UI surfacing). Gold-path README. Bespoke `angreal demos features python-packaged` lane (excluded from auto-registration since `cargo run` is wrong for Python) reusing `_run_gold_path`; auto-joins the CI matrix (16 examples). + +**VERIFIED live (exit 0):** pack Python source → upload → build success (compiler skips cargo) → `workflow run data_pipeline` → execution **Completed**. One naming gotcha found + fixed: `entry_module` is inferred from the PACKAGE NAME (snake-cased), not `workflow_name` — the module dir must match the package name; aligned by naming the package `data-pipeline` (→ module `data_pipeline`), keeping the minimal inference-based manifest the scaffold teaches. Unlike every Rust example this session, the Python path had NO hidden shipped-but-broken bug — worked once the naming aligned. Committed `315cd918`. Done pending merge. diff --git a/.metis/initiatives/CLOACI-I-0138/tasks/CLOACI-T-0889.md b/.metis/initiatives/CLOACI-I-0138/tasks/CLOACI-T-0889.md index f52b75b39..15131e582 100644 --- a/.metis/initiatives/CLOACI-I-0138/tasks/CLOACI-T-0889.md +++ b/.metis/initiatives/CLOACI-I-0138/tasks/CLOACI-T-0889.md @@ -4,14 +4,14 @@ level: task title: "Gold-path example: parameterized workflow instances (I-0116) — params, named instances, schedules" short_code: "CLOACI-T-0889" created_at: 2026-07-11T22:03:08.051490+00:00 -updated_at: 2026-07-11T22:03:08.051490+00:00 +updated_at: 2026-07-11T22:31:55.275713+00:00 parent: CLOACI-I-0138 blocked_by: [] archived: false tags: - "#task" - - "#phase/todo" + - "#phase/completed" exit_criteria_met: false @@ -76,6 +76,10 @@ I-0116 (parameterized workflow instances — named, scheduled, param-bound) ship - **Benefits of Fixing**: {What improves after refactoring} - **Risk Assessment**: {Risks of not addressing this} +## Acceptance Criteria + +## Acceptance Criteria + ## Acceptance Criteria **[REQUIRED]** - [ ] {Specific, testable requirement 1} @@ -145,4 +149,12 @@ I-0116 (parameterized workflow instances — named, scheduled, param-bound) ship ## Status Updates **[REQUIRED]** -*To be added during implementation* +### 2026-07-11 — LOUD FINDING + example built; live verification running +**Finding (filed as [[CLOACI-T-0894]]):** I-0116 instance registration is embedded-runner-only — `register_cron_workflow_instance` exists on DefaultRunner + python bindings, but there is NO server route and NO cloacinactl noun. Named/scheduled instances cannot be created through the primary interface. What the gold path DOES support: `params(...)` declaration → compiler extracts typed InputSlots → server validates `workflow run --context` values (T-0757) → bound values arrive as top-level context keys. + +**Built accordingly** (`examples/features/workflows/parameterized-workflow/`, branch feat/i0138-examples): +- `sync_file` template: `params(source: String, dst: String, mode: String = "copy", max_files: i64 = 100)` (syntax grounded on the acme-billing fixture); 3 tasks reading bound params off context (plan_sync validates mode, execute_sync, report). +- T-0886 standard shape: package.toml + version-dep Cargo.toml + build.rs + gold-path README (two runs with different --context bindings; a missing-required-param run shown REJECTED; honest "About named instances" note pointing at the doc + the T-0894 gap). +- Harness: refactored `.angreal/demos/features/features.py` — the simple-packaged bespoke runner generalized into `_run_gold_path(label, dir, run_steps)` + `_run_to_completed(ctl, home, workflow, context_path)`; new `demos features parameterized-workflow` command runs the template twice to Completed with different bindings AND asserts the missing-param run is rejected pre-execution. Auto-joined the CI matrix via `demos matrix` (now 13 examples); added to the auto-registration exclude set. + +Live run of `angreal demos features parameterized-workflow` in progress (cold dep build). diff --git a/.metis/initiatives/CLOACI-I-0138/tasks/CLOACI-T-0890.md b/.metis/initiatives/CLOACI-I-0138/tasks/CLOACI-T-0890.md index bb41f9069..ff07831e2 100644 --- a/.metis/initiatives/CLOACI-I-0138/tasks/CLOACI-T-0890.md +++ b/.metis/initiatives/CLOACI-I-0138/tasks/CLOACI-T-0890.md @@ -4,14 +4,14 @@ level: task title: "Gold-path example: workflow secrets — authoring secrets() plus the cloacinactl secret lifecycle" short_code: "CLOACI-T-0890" created_at: 2026-07-11T22:03:17.286920+00:00 -updated_at: 2026-07-11T22:03:17.286920+00:00 +updated_at: 2026-07-12T01:34:13.553683+00:00 parent: CLOACI-I-0138 blocked_by: [] archived: false tags: - "#task" - - "#phase/todo" + - "#phase/completed" exit_criteria_met: false @@ -70,6 +70,10 @@ Tenant secrets have full plumbing — authoring (`#[workflow] secrets( n1, n2, - **Benefits of Fixing**: {What improves after refactoring} - **Risk Assessment**: {Risks of not addressing this} +## Acceptance Criteria + +## Acceptance Criteria + ## Acceptance Criteria **[REQUIRED]** - [ ] {Specific, testable requirement 1} @@ -139,4 +143,16 @@ Tenant secrets have full plumbing — authoring (`#[workflow] secrets( n1, n2, ## Status Updates **[REQUIRED]** -*To be added during implementation* +### 2026-07-12 (update 2) — BLOCKED on T-0895: the fidius task protocol has no secrets channel +Lane failed at the last hop, revealing the deepest gap of the audit: `DynamicLibraryTask::execute` serializes only `context.data()` across the plugin boundary — the secret resolver never crosses, and the plugin side has no secret handling. So `context.secret(...)` in a PACKAGED task fails everywhere (agents included — their D-5 unwrap attaches a resolver the bridge then drops). Filed as [[CLOACI-T-0895]] (needs a design pass: wire-format change → likely interface_version bump → maintainer call). + +Everything upstream is fixed and verified: alias map reaches the task input context (`__cloacina_secret_refs__` visible in the server log), both runner paths thread tenant resolvers (`with_database_secrets`, `runner_secret_resolver`). Third fix this task: `DefaultRunner::with_database_secrets` + server wiring (global + tenant cache). The `workflow-secrets` lane is EXCLUDED from the CI matrix (commented in `_BESPOKE_FEATURES`) until T-0895 lands; README carries an honest status banner. This task is blocked_by T-0895; the example + lane are its verification vehicle. + +### 2026-07-11 — LOUD FINDING (fixed in-place) + example built; lane running +**Finding:** workflow secrets were END-TO-END UNREACHABLE via the primary interface. Delivery mechanism (grounded): tasks read via `context.secret_field(name, field)` (side channel, T-0858; plaintext never persisted); a run binds a declared secret with `{"$secret": "name"}`; the fleet executor wraps values to one-time agent keys (D-5) and fails closed without `CLOACINA_SECRET_KEK`. BUT the `$secret`→alias-map conversion (`merge_instance_params`, T-0859) only ran on the INSTANCE fire path — and instances can't be registered server-side (T-0894). The execute route merged context naively, so a direct `workflow run` could never bind a secret. + +**Fixed (small, surgical):** `routes/executions.rs` now merges the provided context through the SAME `merge_instance_params` the fire path uses ($secret refs routed to the alias map, reserved keys protected, malformed refs → 400); `validate_declared_params` now handles `encrypted` slots — requires a `{"$secret": "name"}` ref and REJECTS literal values with a guiding message. cargo check clean. + +**Built:** `examples/features/workflows/workflow-secrets/` — `notify_oncall` declares `params(channel = "#ops") + secrets(api_token)`; `resolve_token` reads `secret_field("api_token","token")` and persists only derived facts (len + bool), `send_notification` consumes. T-0886 shape + README covering the full lifecycle (create from file → bind → run → rotate → rerun → metadata-only reads → literal rejected). Demo compose: server gets a fixed demo `CLOACINA_SECRET_KEK` (documented demo-only). Harness: `_run_gold_path` gained `server_env`; `demos features workflow-secrets` asserts create→Completed→rotate→Completed→no-value-in-get→literal-rejected. Matrix now 14. + +Lane run in progress. diff --git a/.metis/initiatives/CLOACI-I-0138/tasks/CLOACI-T-0891.md b/.metis/initiatives/CLOACI-I-0138/tasks/CLOACI-T-0891.md index 7b3fbaacf..1c42538c8 100644 --- a/.metis/initiatives/CLOACI-I-0138/tasks/CLOACI-T-0891.md +++ b/.metis/initiatives/CLOACI-I-0138/tasks/CLOACI-T-0891.md @@ -143,4 +143,22 @@ The reactive layer's advanced surface is fixtures-only. Tutorials + `packaged-gr ## Status Updates **[REQUIRED]** -*To be added during implementation* +### 2026-07-12 — grounded + designed; scope adjusted by a new finding (T-0896) +**Finding:** packaged graphs support only `stream`/`state` accumulator kinds — `polling`/`batch` silently degrade to passthrough (`packaging_bridge.rs:225` `_ =>` arm). Filed as [[CLOACI-T-0896]]; this example covers polling/batch only after that lands. + +**Adjusted scope (what's REAL on the primary interface today):** +1. **Kafka stream accumulator** — proven pattern from `demo-kafka-stream-rust`: declare a plain accumulator in `#[reactor(accumulators=[...], criteria=when_any(...))]`, upgrade via package.toml `[[metadata.accumulators]] accumulator_type = "stream"` + `[metadata.accumulators.config] broker = "{{ KAFKA_BROKER }}" / topic / group`. Broker token resolves via `CLOACINA_VAR_KAFKA_BROKER` (demo server sets `kafka:9092`; the DEV stack also has kafka for the harness lane — wire the env via `_run_gold_path` server_env). +2. **State accumulator** — `accumulator_type = "state"` + capacity config (packaged-proven). +3. **Task→CG invocation** — `#[task(invokes = computation_graph("name"), post_invocation = hook)]` against a TRIGGERLESS graph (no `trigger=`; reactor-triggered graphs are compile-time rejected for invoke — teaching note). Grounded on tests/integration/computation_graph.rs T-0540 M3: terminal outputs merge back into the task's context under terminal-node names; post_invocation sees the merged context. +4. **boundary_schema (python)** — add `@cloaca.boundary_schema(...)` to `examples/features/computation-graphs/python-packaged-graph` (pattern from `demo-py-graph`/`demo-py-state`). + +**Example shape:** `examples/features/computation-graphs/cg-feature-tour/` — one package: kafka-fed reactor CG (stream) + state accumulator + a workflow whose task `invokes` a triggerless CG with a `post_invocation` hook. README teaches each surface + an "Operate it" section (accumulator inject, reactor fire — coordinates with T-0893). Harness lane via `_run_gold_path`: (a) run the invoking workflow to Completed and assert terminal keys, (b) `accumulator inject` a typed event + poll reactor fires, (c) kafka: produce via the dev-stack broker (docker exec kafka console producer) and observe a fire — fall back to demo-stack verification if host-lane kafka is awkward (document which). + +Design complete; ready to execute. + +### 2026-07-12 (update 2) — task→CG invocation DONE + verified; stream/inject deferred to T-0898 +Built `examples/features/computation-graphs/cg-feature-tour` (all three surfaces authored: kafka stream accumulator, typed inject, task→CG invocation) + bespoke `demos features cg-feature-tour` lane. Two findings surfaced building it: +- **[[CLOACI-T-0897]] (FIXED):** the CG macro emitted umbrella-crate (`cloacina::`) paths in two single-emission sites, so trigger-less graphs + task→CG invocation NEVER compiled in a packaged crate — task→CG invocation was unshippable on the primary path. Dual-emitted per build mode; verified `tour_pipeline` reaches **Completed** in-container (invoke + post_invocation hook + terminal routing all proven). +- **Kafka is a host cargo feature (rdkafka in core `cloacina`), not package code.** A stream accumulator silently degrades to passthrough unless the server was built `--features kafka`. Maintainer steer: kafka is a consumption connector; migrate it into the delivered constructor/provider mechanism so it ships IN the package. Filed [[CLOACI-T-0898]]. + +**Decision (maintainer):** keep the full example code (all surfaces authored) and migrate kafka to a provider NEXT session. The CI lane asserts only the invocation surface today (package-clean, works); the stream + inject/fire surfaces stay in the example, documented honestly ("Status of the surfaces" → T-0898/T-0896), and light up on the migration. So T-0891 is PARTIALLY done: invocation ✅ + verified; stream/inject/boundary_schema pending T-0898 (kafka) / T-0896 (polling/batch). Stays `todo` until those land. diff --git a/.metis/initiatives/CLOACI-I-0138/tasks/CLOACI-T-0900.md b/.metis/initiatives/CLOACI-I-0138/tasks/CLOACI-T-0900.md new file mode 100644 index 000000000..79b71d6a2 --- /dev/null +++ b/.metis/initiatives/CLOACI-I-0138/tasks/CLOACI-T-0900.md @@ -0,0 +1,159 @@ +--- +id: re-author-stale-packaged-examples +level: task +title: "Re-author stale packaged examples off the cloacina umbrella + path deps — lean version-dep form" +short_code: "CLOACI-T-0900" +created_at: 2026-07-12T18:51:33.225632+00:00 +updated_at: 2026-07-12T19:22:27.540891+00:00 +parent: CLOACI-I-0138 +blocked_by: [] +archived: false + +tags: + - "#task" + - "#phase/completed" + + +exit_criteria_met: false +initiative_id: CLOACI-I-0138 +--- + +# Re-author stale packaged examples off the cloacina umbrella + path deps — lean version-dep form + +*This template includes sections for various types of tasks. Delete sections that don't apply to your specific use case.* + +## Parent Initiative **[CONDITIONAL: Assigned Task]** + +[[CLOACI-I-0138]] + +## Objective **[REQUIRED]** + +**Finding (2026-07-12, "why does complex-dag pull in so much"):** the OLD packaged examples were never re-authored to the lean version-dep form (the T-0887 shape that `simple-packaged` + my new examples use). They're stale two ways: + +1. **Umbrella `cloacina` dep.** `complex-dag/Cargo.toml` has `cloacina = { path = ... }` — the FULL engine crate (server DAL, postgres, diesel, executor) — pulled in ONLY to `use cloacina::{Context, TaskError}`, which the lean `cloacina-workflow` re-exports. So every packaged build compiled the whole engine (multi-minute cold build) for two types. +2. **`../../../../crates/` path deps** (pre-T-0887). These don't resolve when the compiler stages the package to a temp dir, and `--dev-workspace` only patches crates.io VERSION deps — so these examples FAIL the gold-path lane outright. + +**Stale packaged examples (have `package.toml` + the bad dep shape):** `complex-dag` (umbrella + 6 path deps), `packaged-workflows` (umbrella + 6), `packaged-triggers` (umbrella + 6), `packaged-graph` (5 path deps, no umbrella). NOTE: the embedded `cargo run` examples (cron-scheduling, multi-tenant, conditional-retries, …) legitimately use the `cloacina` engine + path deps — they are NOT packaged, so they're out of scope. + +**Fix:** re-author each to the lean version-dep form (mirror `simple-packaged`): drop the umbrella `cloacina` dep, import `Context`/`TaskError` from `cloacina-workflow`, switch every `path = "../../../../crates/…"` → `"0.10"` version deps, drop cruft (tokio `full`→minimal, tracing-subscriber). Then they resolve via `--dev-workspace` + build in seconds. Each re-authored example joins the gold-path CI matrix (they're already discovered by the T-0138 registrar). + +**Progress:** `complex-dag` DONE this session — re-authored + import fixed; offline `cargo build --lib` went from a multi-minute umbrella compile to **11s**; gold-path lane verification next. Remaining: `packaged-workflows`, `packaged-graph`, `packaged-triggers` (also needs a fire-the-trigger lane step — currently in `_PACKAGED_SKIP`). + +**Acceptance:** all four packaged examples use lean version deps (no umbrella, no path deps), build fast, and pass the gold-path lane (or, for packaged-triggers, a trigger-fire lane). + +## Backlog Item Details **[CONDITIONAL: Backlog Item]** + +{Delete this section when task is assigned to an initiative} + +### Type +- [ ] Bug - Production issue that needs fixing +- [ ] Feature - New functionality or enhancement +- [ ] Tech Debt - Code improvement or refactoring +- [ ] Chore - Maintenance or setup work + +### Priority +- [ ] P0 - Critical (blocks users/revenue) +- [ ] P1 - High (important for user experience) +- [ ] P2 - Medium (nice to have) +- [ ] P3 - Low (when time permits) + +### Impact Assessment **[CONDITIONAL: Bug]** +- **Affected Users**: {Number/percentage of users affected} +- **Reproduction Steps**: + 1. {Step 1} + 2. {Step 2} + 3. {Step 3} +- **Expected vs Actual**: {What should happen vs what happens} + +### Business Justification **[CONDITIONAL: Feature]** +- **User Value**: {Why users need this} +- **Business Value**: {Impact on metrics/revenue} +- **Effort Estimate**: {Rough size - S/M/L/XL} + +### Technical Debt Impact **[CONDITIONAL: Tech Debt]** +- **Current Problems**: {What's difficult/slow/buggy now} +- **Benefits of Fixing**: {What improves after refactoring} +- **Risk Assessment**: {Risks of not addressing this} + +## Acceptance Criteria + +## Acceptance Criteria + +## Acceptance Criteria **[REQUIRED]** + +- [ ] {Specific, testable requirement 1} +- [ ] {Specific, testable requirement 2} +- [ ] {Specific, testable requirement 3} + +## Test Cases **[CONDITIONAL: Testing Task]** + +{Delete unless this is a testing task} + +### Test Case 1: {Test Case Name} +- **Test ID**: TC-001 +- **Preconditions**: {What must be true before testing} +- **Steps**: + 1. {Step 1} + 2. {Step 2} + 3. {Step 3} +- **Expected Results**: {What should happen} +- **Actual Results**: {To be filled during execution} +- **Status**: {Pass/Fail/Blocked} + +### Test Case 2: {Test Case Name} +- **Test ID**: TC-002 +- **Preconditions**: {What must be true before testing} +- **Steps**: + 1. {Step 1} + 2. {Step 2} +- **Expected Results**: {What should happen} +- **Actual Results**: {To be filled during execution} +- **Status**: {Pass/Fail/Blocked} + +## Documentation Sections **[CONDITIONAL: Documentation Task]** + +{Delete unless this is a documentation task} + +### User Guide Content +- **Feature Description**: {What this feature does and why it's useful} +- **Prerequisites**: {What users need before using this feature} +- **Step-by-Step Instructions**: + 1. {Step 1 with screenshots/examples} + 2. {Step 2 with screenshots/examples} + 3. {Step 3 with screenshots/examples} + +### Troubleshooting Guide +- **Common Issue 1**: {Problem description and solution} +- **Common Issue 2**: {Problem description and solution} +- **Error Messages**: {List of error messages and what they mean} + +### API Documentation **[CONDITIONAL: API Documentation]** +- **Endpoint**: {API endpoint description} +- **Parameters**: {Required and optional parameters} +- **Example Request**: {Code example} +- **Example Response**: {Expected response format} + +## Implementation Notes **[CONDITIONAL: Technical Task]** + +{Keep for technical tasks, delete for non-technical. Technical details, approach, or important considerations} + +### Technical Approach +{How this will be implemented} + +### Dependencies +{Other tasks or systems this depends on} + +### Risk Considerations +{Technical risks and mitigation strategies} + +## Status Updates **[REQUIRED]** + +### 2026-07-12 — 3 of 4 re-authored + verified; packaged-triggers remains +- **complex-dag** — re-authored (dropped umbrella `cloacina`; `Context`/`TaskError` now from `cloacina-workflow`; path→version deps; cruft trimmed). Offline `cargo build --lib`: multi-min → **11s**. Un-skipped. +- **packaged-workflows** — path→version deps (umbrella was already dev-only). Offline build **11.9s**. Un-skipped. +- **packaged-graph** — path→version deps. Offline build **13s**. **Gold-path lane VERIFIED live**: build_status=success (lean deps resolved via `--dev-workspace`) → inject → reactor `packaged_market_maker_reactor` fired. +- Committed `59873086`. complex-dag/packaged-workflows verified offline (same lean form + default workflow-run assertion); CI runs them. + +**DONE.** `packaged-triggers` was doc-only (declared `#[workflow]`, only *documented* a trigger in the README; tasks `unwrap_or("unknown")` so it tolerated no context). Made it a REAL poll-trigger example: added `#[trigger(on = "file_processing", poll_interval = "3s")]` firing `TriggerResult::Fire(Some(ctx))` with an injected filename, plus lean version deps. New `_trigger_wait_steps` lane waits for the poll trigger to fire an execution automatically (no `workflow run`) and asserts Completed — verified live (fired an execution on its own → Completed). `_PACKAGED_SKIP` is now empty. Commit 5a5768a8. + +**~~Remaining:~~** ~~`packaged-triggers`~~ — still stale-dep AND needs a fire-the-trigger lane step (trigger-fired, not `workflow run`); stays in `_PACKAGED_SKIP`. That's the one open item. diff --git a/crates/cloacina-compiler/src/param_parse.rs b/crates/cloacina-compiler/src/param_parse.rs index da376b898..728b1e108 100644 --- a/crates/cloacina-compiler/src/param_parse.rs +++ b/crates/cloacina-compiler/src/param_parse.rs @@ -48,6 +48,98 @@ use cloacina::input_interface::InputSlot; /// pathological (mirrors `doc_parse`). const MAX_SOURCE_BYTES: u64 = 1024 * 1024; +/// Normalize Python source so the param/secret/boundary scanners only ever see +/// real code (CLOACI-T-0899). These scanners split on commas and `=` without a +/// real Python lexer, so two natural author habits used to corrupt a +/// declaration: +/// +/// 1. **Inline comments** — `source=str, # required` was parsed as a param +/// literally named `"# required\n dst"`. +/// 2. **Decorator syntax quoted in a DOCSTRING** — a module/task docstring +/// showing `@cloaca.workflow_secrets("name")` as an EXAMPLE was matched by +/// the `.find("workflow_secrets(")` scan and parsed as a real secret named +/// `name`, making every run fail with `missing required param 'name'`. +/// +/// So this pass: strips `#…` line comments (outside strings), and **blanks the +/// INTERIOR of triple-quoted strings** (docstrings) — replacing their content +/// with spaces while keeping the delimiters + newlines — so a decorator call +/// quoted in a docstring is invisible to the scanner. Single/double-quoted +/// strings are preserved verbatim because they carry real param DEFAULT values +/// (`mode=(str, "copy")`). Doc extraction is unaffected — that's a separate +/// parser (`doc_parse`) reading the raw source, so docstrings still surface in +/// the UI. +fn strip_py_comments(src: &str) -> String { + let mut out = String::with_capacity(src.len()); + let mut i = 0; + // The active string delimiter (`"`, `'`, `"""`, or `'''`) when inside one. + let mut quote: Option<&'static str> = None; + while i < src.len() { + let rest = &src[i..]; + if let Some(q) = quote { + // Triple-quoted (docstring): blank the interior so the scanner + // can't see decorator syntax quoted as example text. + let blank = q.len() == 3; + if rest.starts_with('\\') { + out.push(if blank { ' ' } else { '\\' }); + i += 1; + if let Some(c) = src[i..].chars().next() { + out.push(if blank && c != '\n' { ' ' } else { c }); + i += c.len_utf8(); + } + continue; + } + if rest.starts_with(q) { + out.push_str(q); + i += q.len(); + quote = None; + continue; + } + let c = rest.chars().next().unwrap(); + out.push(if blank && c != '\n' { ' ' } else { c }); + i += c.len_utf8(); + continue; + } + for delim in ["\"\"\"", "'''"] { + if rest.starts_with(delim) { + quote = Some(delim); + break; + } + } + if quote.is_some() { + let q = quote.unwrap(); + out.push_str(q); + i += q.len(); + continue; + } + if rest.starts_with('"') { + quote = Some("\""); + out.push('"'); + i += 1; + continue; + } + if rest.starts_with('\'') { + quote = Some("'"); + out.push('\''); + i += 1; + continue; + } + if rest.starts_with('#') { + // Skip to (but keep) the end of line. + while let Some(c) = src[i..].chars().next() { + if c == '\n' { + break; + } + i += c.len_utf8(); + } + continue; + } + let c = rest.chars().next().unwrap(); + out.push(c); + i += c.len_utf8(); + } + out +} + /// Parse declared workflow params from the unpacked package `source_dir`. /// `language` selects the strategy — only `"python"` parses source (Rust gets /// its params from the FFI input-interface entrypoint). Never errors. @@ -97,7 +189,7 @@ fn walk_py_secrets(dir: &Path, out: &mut Vec) { continue; } if let Ok(contents) = std::fs::read_to_string(&path) { - parse_file_secrets(&contents, out); + parse_file_secrets(&strip_py_comments(&contents), out); } } } @@ -174,7 +266,7 @@ fn walk_py_surfaces(dir: &Path, out: &mut Vec) { continue; } if let Ok(contents) = std::fs::read_to_string(&path) { - parse_file(&contents, out); + parse_file(&strip_py_comments(&contents), out); } } } diff --git a/crates/cloacina-macros/src/computation_graph/codegen.rs b/crates/cloacina-macros/src/computation_graph/codegen.rs index 1e3d3a7ce..9fc644859 100644 --- a/crates/cloacina-macros/src/computation_graph/codegen.rs +++ b/crates/cloacina-macros/src/computation_graph/codegen.rs @@ -249,14 +249,45 @@ pub fn generate(ir: &GraphIR, module: &ItemMod) -> syn::Result { let (compiled_fn_body, ctor_body) = if is_triggerless { // Trigger-less form: the compiled fn takes a workflow `Context` // and the runtime registration goes into `TriggerlessGraphEntry`. - let fn_body = quote! { - #vis async fn #compiled_fn_name( - context: &#cloacina_root::Context<#sj::Value>, - ) -> #cg_runtime_root::GraphResult { - #[allow(unused_imports)] - use #mod_name::*; - #(#routing_use_stmts)* - #compiled_fn + // + // CLOACI-T-0897: dual-emitted (like the ctor below) because the + // signature's crate paths differ per build mode — packaged cdylibs + // have `cloacina-workflow`/`cloacina-computation-graph` as direct + // deps but NOT the `cloacina` umbrella; embedded library users have + // only `cloacina`. The previous single emission hardcoded the + // umbrella path, so a trigger-less graph never compiled in a + // packaged crate. + let fn_body = if is_cloacina_crate_early { + quote! { + #vis async fn #compiled_fn_name( + context: &#cloacina_root::Context<#sj::Value>, + ) -> #cg_runtime_root::GraphResult { + #[allow(unused_imports)] + use #mod_name::*; + #(#routing_use_stmts)* + #compiled_fn + } + } + } else { + quote! { + #[cfg(not(feature = "packaged"))] + #vis async fn #compiled_fn_name( + context: &::cloacina::cloacina_workflow::Context<#sj::Value>, + ) -> ::cloacina::computation_graph::GraphResult { + #[allow(unused_imports)] + use #mod_name::*; + #(#routing_use_stmts)* + #compiled_fn + } + #[cfg(feature = "packaged")] + #vis async fn #compiled_fn_name( + context: &::cloacina_workflow::Context<#sj::Value>, + ) -> ::cloacina_computation_graph::GraphResult { + #[allow(unused_imports)] + use #mod_name::*; + #(#routing_use_stmts)* + #compiled_fn + } } }; let ctor = quote! { diff --git a/crates/cloacina-macros/src/tasks.rs b/crates/cloacina-macros/src/tasks.rs index 7f6c65bef..b56d9911a 100644 --- a/crates/cloacina-macros/src/tasks.rs +++ b/crates/cloacina-macros/src/tasks.rs @@ -872,8 +872,15 @@ pub fn generate_task_impl(attrs: TaskAttributes, input: ItemFn) -> TokenStream2 let __terminal_names: Vec = __reg.terminal_node_names.clone(); let __ctx_for_graph = context.clone_data(); let __graph_result = __graph_fn(__ctx_for_graph).await; + // CLOACI-T-0897: GraphResult's crate path differs per + // build mode (packaged cdylibs have the CG runtime crate + // direct; library users route through `cloacina`). + #[cfg(not(feature = "packaged"))] + use ::cloacina::computation_graph::GraphResult as __CgGraphResult; + #[cfg(feature = "packaged")] + use ::cloacina_computation_graph::GraphResult as __CgGraphResult; match __graph_result { - ::cloacina::computation_graph::GraphResult::Completed { outputs, .. } => { + __CgGraphResult::Completed { outputs, .. } => { for (idx, name) in __terminal_names.iter().enumerate() { if let Some(boxed) = outputs.get(idx) { if let Some(value) = @@ -888,7 +895,7 @@ pub fn generate_task_impl(attrs: TaskAttributes, input: ItemFn) -> TokenStream2 } } } - ::cloacina::computation_graph::GraphResult::Error(graph_err) => { + __CgGraphResult::Error(graph_err) => { let task_error = ::cloacina_workflow::TaskError::ExecutionFailed { message: format!( "computation_graph '{}' invocation failed: {}", diff --git a/crates/cloacina-python/src/computation_graph.rs b/crates/cloacina-python/src/computation_graph.rs index 533ed2b7a..53dcffca6 100644 --- a/crates/cloacina-python/src/computation_graph.rs +++ b/crates/cloacina-python/src/computation_graph.rs @@ -153,6 +153,63 @@ pub fn drain_accumulators() -> HashMap Option { + let function = { + let registry = ACCUMULATOR_REGISTRY.lock().unwrap(); + let (func, reg) = registry.get(name)?; + if reg.accumulator_type != "polling" { + return None; + } + Python::with_gil(|py| func.clone_ref(py)) + }; + let name = name.to_string(); + Some(std::sync::Arc::new(move || -> Option> { + Python::with_gil(|py| { + let result = match function.call0(py) { + Ok(r) => r, + Err(e) => { + tracing::warn!("polling accumulator '{}' poll raised: {}", name, e); + return None; + } + }; + // `None` from the poll fn means "no change this tick" — skip. + if result.is_none(py) { + return None; + } + match pythonize::depythonize::(result.bind(py)) { + Ok(val) => serde_json::to_vec(&val).ok(), + Err(e) => { + tracing::warn!( + "polling accumulator '{}' returned a non-JSON value: {}", + name, + e + ); + None + } + } + }) + })) +} + +/// Install the polling-accumulator poll-closure resolver into the packaged CG +/// bridge (CLOACI-T-0896). Idempotent (the bridge's OnceLock keeps the first); +/// called once from `register_authoring` so every embedding (pip wheel + the +/// server's synthetic `cloaca` module) wires packaged polling accumulators. +pub(crate) fn install_polling_accumulator_builder() { + cloacina::computation_graph::packaging_bridge::register_polling_accumulator_builder(Box::new( + resolve_poll_closure, + )); +} + // --------------------------------------------------------------------------- // @cloaca.passthrough_accumulator decorator // --------------------------------------------------------------------------- diff --git a/crates/cloacina-python/src/lib.rs b/crates/cloacina-python/src/lib.rs index 48f8895f1..921b0add1 100644 --- a/crates/cloacina-python/src/lib.rs +++ b/crates/cloacina-python/src/lib.rs @@ -103,6 +103,10 @@ use pyo3::prelude::*; /// contract and live only in the wheel `#[pymodule]`; the server never exposes /// a runner because the server IS the runner. pub(crate) fn register_authoring(m: &Bound<'_, PyModule>) -> PyResult<()> { + // CLOACI-T-0896: wire packaged polling accumulators to their in-process + // Python poll fn (no FFI). Idempotent across embeddings. + computation_graph::install_polling_accumulator_builder(); + m.add_class::()?; m.add_function(wrap_pyfunction!(task::task, m)?)?; diff --git a/crates/cloacina-server/src/lib.rs b/crates/cloacina-server/src/lib.rs index d2d139ea5..75032d480 100644 --- a/crates/cloacina-server/src/lib.rs +++ b/crates/cloacina-server/src/lib.rs @@ -688,9 +688,27 @@ pub async fn run( .build() .context("Invalid runner configuration")?; - let runner = DefaultRunner::with_config(&database_url, runner_config) + // Construct the database explicitly (rather than via `with_config`) so the + // public tenant's secret resolver can be built against it and threaded onto + // the runner's in-process executor (CLOACI-T-0890): a locally-executed task + // can then resolve `{"$secret": …}` bindings via `context.secret(...)`. + // Without `CLOACINA_SECRET_KEK` the resolver is None and secret resolution + // fails closed with a clear error. + let admin_database = + cloacina::Database::new(&database_url, "cloacina", runner_config.db_pool_size()); + admin_database + .run_migrations() .await - .context("Failed to connect to database")?; + .map_err(|e| anyhow::anyhow!("failed to run migrations: {e}"))?; + let public_secret_resolver = secrets::runner_secret_resolver("public", &admin_database); + let runner = DefaultRunner::with_database_secrets( + admin_database, + runner_config, + None, + public_secret_resolver, + ) + .await + .context("Failed to connect to database")?; info!("Connected to Postgres, migrations applied"); diff --git a/crates/cloacina-server/src/routes/executions.rs b/crates/cloacina-server/src/routes/executions.rs index ce5413792..63d197d5f 100644 --- a/crates/cloacina-server/src/routes/executions.rs +++ b/crates/cloacina-server/src/routes/executions.rs @@ -75,11 +75,23 @@ pub async fn execute_workflow( let provided_ctx: Option> = body.context.as_ref().and_then(|v| v.as_object()).cloned(); - // Merge provided context if any + // Merge provided context if any. Route through the SAME merge the + // instance fire path uses (CLOACI-T-0859): a `{"$secret": "name"}` value + // is routed AWAY from the plaintext context into the reserved alias map + // (names only — the resolved value never enters the durable context), so + // a direct `workflow run` can bind declared secrets exactly like a + // scheduled instance does. Reserved scheduler keys are protected too. if let Some(ctx_value) = body.context { - if let Some(obj) = ctx_value.as_object() { - for (k, v) in obj { - context.insert(k.clone(), v.clone()).unwrap_or_default(); + if ctx_value.as_object().is_some() { + let params_json = ctx_value.to_string(); + if let Err(e) = + cloacina::workflow_instance::merge_instance_params(&mut context, ¶ms_json) + { + return ApiError::bad_request( + "workflow_input_invalid", + format!("invalid execution context: {}", e), + ) + .into_response(); } } } @@ -108,6 +120,35 @@ pub async fn execute_workflow( { let storage = UnifiedRegistryStorage::new(tenant_db.clone()); if let Ok(registry) = WorkflowRegistryImpl::new(storage, tenant_db.clone()) { + // CLOACI-T-0901: a non-public tenant may only execute workflows in + // its OWN registry. `execute_async` resolves the name against the + // process-shared `Runtime` (populated by every tenant's reconciler), + // so without this gate a tenant could run a workflow it never + // installed — a cross-tenant isolation leak. A definitive "not + // found" fails CLOSED (404); a registry ERROR fails open (proceed) + // so a transient DB fault never wedges execution. `public` maps to + // the admin/global catalog and is intentionally exempt (it may serve + // inventory/global workflows not tracked as tenant packages). + if tenant_id != "public" { + match registry.workflow_exists(&name).await { + Ok(false) => { + return ApiError::new( + StatusCode::NOT_FOUND, + "workflow_not_found", + format!( + "workflow '{}' is not registered for tenant '{}'", + name, tenant_id + ), + ) + .into_response(); + } + Ok(true) => {} + Err(e) => { + warn!("existence-check failed for workflow '{}': {}", name, e) + } + } + } + match registry.is_workflow_paused(&name).await { Ok(true) => { return ApiError::new( @@ -549,6 +590,24 @@ pub(crate) fn validate_declared_params( } } Some(v) => { + // CLOACI-T-0859: an encrypted slot (declared via + // `secrets(...)`) is bound with a `{"$secret": "name"}` + // reference — never a literal value (which would land the + // plaintext in the durable context). + if p.encrypted { + let is_secret_ref = v + .as_object() + .map(|o| o.len() == 1 && o.get("$secret").is_some_and(|s| s.is_string())) + .unwrap_or(false); + if !is_secret_ref { + errors.push(format!( + "secret '{}' must be bound via {{\"$secret\": \"name\"}}, \ + not a literal value", + p.name + )); + } + continue; + } if let Some(expected) = p.schema.get("type").and_then(|t| t.as_str()) { if !json_value_matches_type(v, expected) { errors.push(format!( diff --git a/crates/cloacina-server/src/secrets.rs b/crates/cloacina-server/src/secrets.rs index bb7e71a60..4bb92fcc5 100644 --- a/crates/cloacina-server/src/secrets.rs +++ b/crates/cloacina-server/src/secrets.rs @@ -102,6 +102,19 @@ impl ServerFleetSecretResolverFactory { } } +/// Tenant-scoped secret resolver for a runner's IN-PROCESS thread executor +/// (CLOACI-T-0890): threaded at runner construction so `context.secret(...)` +/// resolves when tasks execute locally (the `"default"` executor) rather than +/// on the fleet. Returns `None` when `CLOACINA_SECRET_KEK` is unset — the +/// executor then fails closed with the clear "secrets backend not configured" +/// error. Same composition as the fleet factory: tenant DB → SecretStore → +/// tenant-org-scoped SecretStoreResolver. +pub fn runner_secret_resolver(tenant: &str, db: &Database) -> Option> { + let kek = SecretStoreResolver::kek_from_env().ok()?; + let store = SecretStore::new(cloacina::dal::DAL::new(db.clone())); + Some(SecretStoreResolver::new(store, tenant_org_id(tenant), kek).into_arc()) +} + #[async_trait] impl FleetSecretResolverFactory for ServerFleetSecretResolverFactory { async fn resolver_for(&self, tenant: &str, package: &str) -> Option> { diff --git a/crates/cloacina-server/src/tenant_runner_cache.rs b/crates/cloacina-server/src/tenant_runner_cache.rs index 9b6e29878..160476fc7 100644 --- a/crates/cloacina-server/src/tenant_runner_cache.rs +++ b/crates/cloacina-server/src/tenant_runner_cache.rs @@ -180,10 +180,15 @@ impl TenantRunnerCache { // fleet executor routes on, so the tenant's tasks reach its agents. let mut config = self.base_config.clone(); config.set_tenant_id(tenant_id); - let runner = DefaultRunner::with_database( + // CLOACI-T-0890: thread the tenant-scoped secret resolver onto this + // runner's in-process executor so locally-executed tasks can resolve + // `{"$secret": …}` bindings. None (KEK unset) fails closed at the task. + let secret_resolver = crate::secrets::runner_secret_resolver(tenant_id, &tenant_database); + let runner = DefaultRunner::with_database_secrets( tenant_database, config, Some(self.shared_runtime.clone()), + secret_resolver, ) .await?; // CLOACI-T-0581 follow-up: install the shared graph scheduler if diff --git a/crates/cloacina-workflow-plugin/src/lib.rs b/crates/cloacina-workflow-plugin/src/lib.rs index 779a18eb8..c0845f2af 100644 --- a/crates/cloacina-workflow-plugin/src/lib.rs +++ b/crates/cloacina-workflow-plugin/src/lib.rs @@ -256,7 +256,7 @@ macro_rules! package { } }; - let context: cloacina_workflow::Context<::serde_json::Value> = + let mut context: cloacina_workflow::Context<::serde_json::Value> = match cloacina_workflow::Context::from_json(request.context_json) { Ok(c) => c, Err(e) => { @@ -268,6 +268,19 @@ macro_rules! package { } }; + // CLOACI-T-0895: the host resolved any `{"$secret"}`-referenced + // secrets before this call; attach them to the rebuilt scope so + // `context.secret(...)` resolves inside the package. Values live + // only in this resolver for this one invocation — never in the + // serialized context. + if !request.resolved_secrets.is_empty() { + context.set_secret_resolver(::std::sync::Arc::new( + cloacina_workflow::secret::MapSecretResolver::new( + request.resolved_secrets, + ), + )); + } + let result = rt.block_on(async move { cloacina_workflow::Task::execute(&*task, context).await }); @@ -845,7 +858,10 @@ pub const METHOD_GET_CONSTRUCTOR_METADATA: usize = 10; /// (IDs, dependencies, descriptions). Called once at registration time. /// - `execute_task` — Runs a specific task by name with a JSON-serialized /// context. Returns the updated context or an error. -#[fidius::plugin_interface(version = 4, buffer = PluginAllocated)] +// version 4 → 5 (CLOACI-T-0895): `TaskExecutionRequest` gained the +// `resolved_secrets` wire field — a bincode layout change, so stale artifacts +// must fail the version gate at load rather than mis-decode. +#[fidius::plugin_interface(version = 5, buffer = PluginAllocated)] pub trait CloacinaPlugin: Send + Sync { /// Returns metadata about all tasks in this workflow package. /// Method index 0. diff --git a/crates/cloacina-workflow-plugin/src/types.rs b/crates/cloacina-workflow-plugin/src/types.rs index b31c7aa2b..a2a6fef64 100644 --- a/crates/cloacina-workflow-plugin/src/types.rs +++ b/crates/cloacina-workflow-plugin/src/types.rs @@ -102,12 +102,36 @@ pub struct PackageTasksMetadata { } /// Request to execute a task within a workflow package. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Clone, Serialize, Deserialize)] pub struct TaskExecutionRequest { /// Name of the task to execute (local ID) pub task_name: String, /// JSON-serialized execution context pub context_json: String, + /// CLOACI-T-0895: secrets the host resolved for this invocation, keyed by + /// CONCRETE secret name → `{field: value}`. A resolver object cannot cross + /// the plugin boundary, so the host resolves every `{"$secret"}`-referenced + /// secret up front and ships the VALUES; the plugin shell attaches them to + /// the rebuilt execution scope via `MapSecretResolver` so + /// `context.secret(...)` works identically inside the package. Empty when + /// the task references no secrets. Never logged, never serialized into the + /// durable context. + pub resolved_secrets: + std::collections::BTreeMap>, +} + +// Secret values must never appear in logs — a manual Debug keeps names only. +impl std::fmt::Debug for TaskExecutionRequest { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TaskExecutionRequest") + .field("task_name", &self.task_name) + .field("context_json", &self.context_json) + .field( + "resolved_secrets", + &self.resolved_secrets.keys().collect::>(), + ) + .finish() + } } /// Result of a task execution. @@ -562,14 +586,28 @@ mod tests { #[test] fn test_task_execution_request_round_trip() { + let mut secrets = std::collections::BTreeMap::new(); + secrets.insert( + "oncall_api".to_string(), + std::collections::BTreeMap::from([("token".to_string(), "v".to_string())]), + ); let request = TaskExecutionRequest { task_name: "extract_data".to_string(), context_json: r#"{"key": "value"}"#.to_string(), + resolved_secrets: secrets, }; let json = serde_json::to_string(&request).unwrap(); let roundtrip: TaskExecutionRequest = serde_json::from_str(&json).unwrap(); assert_eq!(roundtrip.task_name, "extract_data"); + assert_eq!( + roundtrip.resolved_secrets["oncall_api"]["token"], + "v".to_string() + ); + // Debug must render NAMES only — never a secret value. + let dbg = format!("{request:?}"); + assert!(dbg.contains("oncall_api")); + assert!(!dbg.contains("\"v\"")); } #[test] diff --git a/crates/cloacina-workflow/src/secret.rs b/crates/cloacina-workflow/src/secret.rs index 109cc8648..bc0edf9c4 100644 --- a/crates/cloacina-workflow/src/secret.rs +++ b/crates/cloacina-workflow/src/secret.rs @@ -108,3 +108,43 @@ pub trait SecretResolver: Send + Sync { /// Resolve `name` to its decrypted `{field: value}` map. async fn resolve(&self, name: &str) -> Result, SecretResolverError>; } + +/// In-memory resolver over already-resolved secret values, keyed by concrete +/// secret name (CLOACI-T-0895). +/// +/// The packaged-task bridge uses this on the PLUGIN side: the host resolves +/// every `{"$secret"}`-referenced secret through its real backend before the +/// plugin call and ships the values across the boundary in the +/// `TaskExecutionRequest`; the plugin shell rebuilds the execution scope with +/// this resolver so `context.secret(...)` works identically inside the +/// package. Values live only in this object for the duration of one task +/// invocation — never serialized into the durable context (NFR-001). +pub struct MapSecretResolver { + secrets: BTreeMap>, +} + +impl MapSecretResolver { + /// Wrap a `{secret_name: {field: value}}` map. + pub fn new(secrets: BTreeMap>) -> Self { + Self { secrets } + } +} + +// Values must never appear in logs; a manual Debug keeps names only. +impl std::fmt::Debug for MapSecretResolver { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("MapSecretResolver") + .field("names", &self.secrets.keys().collect::>()) + .finish() + } +} + +#[async_trait] +impl SecretResolver for MapSecretResolver { + async fn resolve(&self, name: &str) -> Result, SecretResolverError> { + self.secrets + .get(name) + .cloned() + .ok_or_else(|| SecretResolverError::NotFound(name.to_string())) + } +} diff --git a/crates/cloacina/src/computation_graph/packaging_bridge.rs b/crates/cloacina/src/computation_graph/packaging_bridge.rs index 3831b863c..fa1d2ef32 100644 --- a/crates/cloacina/src/computation_graph/packaging_bridge.rs +++ b/crates/cloacina/src/computation_graph/packaging_bridge.rs @@ -28,8 +28,9 @@ use tokio::task::JoinHandle; use cloacina_workflow_plugin::{GraphExecutionRequest, GraphPackageMetadata}; use super::accumulator::{ - accumulator_runtime, accumulator_runtime_with_source, state_accumulator_runtime, - AccumulatorContext, AccumulatorRuntimeConfig, BoundarySender, StateAccumulator, + accumulator_runtime, accumulator_runtime_with_source, batch_accumulator_runtime, flush_signal, + state_accumulator_runtime, AccumulatorContext, AccumulatorRuntimeConfig, BatchAccumulator, + BatchAccumulatorConfig, BoundarySender, StateAccumulator, }; use super::reactor::{CompiledGraphFn, InputStrategy, ReactionCriteria}; use super::scheduler::{ @@ -222,15 +223,7 @@ pub fn build_declaration_from_ffi( .accumulators .iter() .map(|acc_entry| { - let factory: Arc = match acc_entry.accumulator_type.as_str() { - "stream" => Arc::new(StreamBackendAccumulatorFactory::new( - acc_entry.config.clone(), - )), - "state" => Arc::new(StateAccumulatorFactory::new(state_capacity_from_config( - &acc_entry.config, - ))), - _ => Arc::new(PassthroughAccumulatorFactory), - }; + let factory = accumulator_factory_for(&acc_entry.accumulator_type, &acc_entry.config); AccumulatorDeclaration { name: acc_entry.name.clone(), factory, @@ -650,6 +643,276 @@ fn state_capacity_from_config(config: &std::collections::HashMap .unwrap_or(0) } +/// A generic, list-collecting batch accumulator for the packaged path +/// (CLOACI-T-0896). Socket events arrive as JSON bytes (the same wire the +/// passthrough/state accumulators receive); on flush we emit the whole batch as +/// a JSON array, so the boundary matches the shape the FFI cache expects +/// (`bincode(Vec)` of JSON — see `input_cache_to_ffi_cache`). This mirrors +/// what `state_window_frame` does for the state accumulator. +struct JsonListBatchAccumulator; + +impl BatchAccumulator for JsonListBatchAccumulator { + type Output = Vec; + + fn process_batch(&mut self, events: Vec>) -> Option> { + let list: Vec = events + .iter() + .filter_map(|e| serde_json::from_slice(e).ok()) + .collect(); + if list.is_empty() { + return None; + } + serde_json::to_vec(&list).ok() + } +} + +/// Packaged batch-accumulator factory (CLOACI-T-0896): buffers socket events and +/// flushes the whole buffer as one boundary on the flush interval or when the +/// buffer fills. Mirrors `StateAccumulatorFactory` — socket-driven, so it fits +/// the existing spawn contract without any FFI change. +pub struct BatchAccumulatorFactory { + flush_interval: Option, + max_buffer_size: Option, +} + +impl BatchAccumulatorFactory { + pub fn new( + flush_interval: Option, + max_buffer_size: Option, + ) -> Self { + Self { + flush_interval, + max_buffer_size, + } + } +} + +impl AccumulatorFactory for BatchAccumulatorFactory { + fn spawn( + &self, + name: String, + boundary_tx: mpsc::Sender<(SourceName, Vec)>, + shutdown_rx: watch::Receiver, + config: AccumulatorSpawnConfig, + ) -> (mpsc::Sender>, JoinHandle<()>) { + let (socket_tx, socket_rx) = mpsc::channel(1024); + let (flush_tx, flush_rx) = flush_signal(); + + let checkpoint = config.dal.map(|dal| { + super::accumulator::CheckpointHandle::new(dal, config.graph_name.clone(), name.clone()) + }); + let sender = BoundarySender::with_freshness( + boundary_tx, + SourceName::new(&name), + config.freshness.clone(), + ); + let ctx = AccumulatorContext { + output: sender, + name: name.clone(), + shutdown: shutdown_rx, + checkpoint, + health: config.health_tx, + }; + let batch_cfg = BatchAccumulatorConfig { + flush_interval: self.flush_interval, + max_buffer_size: self.max_buffer_size, + }; + let handle = tokio::spawn(async move { + // The packaged path has no external reactor-driven flusher, so + // flushes come from the timer / size threshold. Hold `flush_tx` for + // the runtime's lifetime so `flush_rx` stays open (its select arm + // simply never fires) rather than closing and busy-spinning. + let _flush_tx = flush_tx; + batch_accumulator_runtime( + JsonListBatchAccumulator, + ctx, + socket_rx, + flush_rx, + batch_cfg, + ) + .await; + }); + (socket_tx, handle) + } +} + +/// Parse a batch accumulator's `flush_interval` (e.g. `"1s"`, `"500ms"`) and +/// `max_buffer_size` from its String-keyed config map. Absent/unparsable → +/// `None` (the runtime treats each as an optional flush trigger). +fn batch_config_from_config( + config: &std::collections::HashMap, +) -> (Option, Option) { + let flush_interval = config + .get("flush_interval") + .and_then(|s| crate::packaging::manifest_schema::parse_duration_str(s).ok()); + let max_buffer_size = config + .get("max_buffer_size") + .and_then(|s| s.parse::().ok()); + (flush_interval, max_buffer_size) +} + +/// Central accumulator-factory dispatch for the packaged path (CLOACI-T-0896). +/// Every packaged-reactor loader (Python runtime registration, Rust cdylib +/// metadata, manifest overrides) resolves the same set of kinds here, and an +/// unknown type WARNs loudly + falls back to passthrough instead of silently +/// degrading — the whole point of T-0896. +/// A poll closure for a packaged polling accumulator: invoked on each interval, +/// it returns `Some(json_bytes)` to emit a boundary or `None` to skip. The +/// closure does the blocking, GIL-taking work itself — cloacina runs it via +/// `spawn_blocking` off the async executor. The Python layer supplies these +/// (each wraps the registered Python poll fn); there is NO FFI accumulator-invoke +/// method, and none is needed — the poll fn lives in-process. (CLOACI-T-0896) +pub type PollClosure = Arc Option> + Send + Sync>; + +/// Resolves the poll closure for a polling accumulator by name (returns `None` +/// when the name isn't registered). Installed once by the Python extension. +type PollingClosureBuilder = Box Option + Send + Sync>; + +static POLLING_CLOSURE_BUILDER: std::sync::OnceLock = + std::sync::OnceLock::new(); + +/// Register the polling-accumulator poll-closure resolver for the packaged path +/// (CLOACI-T-0896). Idempotent — the first registration wins. The Python +/// extension calls this at module install so a packaged polling accumulator +/// drives its Python poll fn on the configured interval. A pure-Rust host that +/// never installs one gets a loud passthrough fallback for polling accumulators. +pub fn register_polling_accumulator_builder(builder: PollingClosureBuilder) { + let _ = POLLING_CLOSURE_BUILDER.set(builder); +} + +/// A [`PollingAccumulator`] driven by an injected [`PollClosure`] (the Python +/// poll fn). `poll()` runs the closure on a blocking thread so the GIL work +/// never blocks the async executor — the same discipline `PythonTriggerWrapper` +/// uses for poll triggers. (CLOACI-T-0896) +struct ClosurePollingAccumulator { + poll_fn: PollClosure, + interval: std::time::Duration, +} + +#[async_trait::async_trait] +impl super::accumulator::PollingAccumulator for ClosurePollingAccumulator { + type Output = Vec; + + async fn poll(&mut self) -> Option> { + let f = self.poll_fn.clone(); + tokio::task::spawn_blocking(move || f()) + .await + .ok() + .flatten() + } + + fn interval(&self) -> std::time::Duration { + self.interval + } +} + +/// Packaged polling-accumulator factory (CLOACI-T-0896). Resolves the poll +/// closure by name at spawn time via the registered builder, then runs +/// `polling_accumulator_runtime` on the configured interval. If no closure is +/// registered for the name, the accumulator simply never emits (logged) rather +/// than failing the load. +pub struct PollingAccumulatorFactory { + interval: std::time::Duration, +} + +impl PollingAccumulatorFactory { + pub fn new(interval: std::time::Duration) -> Self { + Self { interval } + } +} + +impl AccumulatorFactory for PollingAccumulatorFactory { + fn spawn( + &self, + name: String, + boundary_tx: mpsc::Sender<(SourceName, Vec)>, + shutdown_rx: watch::Receiver, + config: AccumulatorSpawnConfig, + ) -> (mpsc::Sender>, JoinHandle<()>) { + let (socket_tx, socket_rx) = mpsc::channel(1024); + + let poll_fn = POLLING_CLOSURE_BUILDER + .get() + .and_then(|builder| builder(&name)); + if poll_fn.is_none() { + tracing::warn!( + accumulator = %name, + "no poll closure registered for polling accumulator — it will \ + never emit (CLOACI-T-0896)" + ); + } + // No registered closure → a no-op poller that always returns None. + let poll_fn: PollClosure = poll_fn.unwrap_or_else(|| Arc::new(|| None)); + + let checkpoint = config.dal.map(|dal| { + super::accumulator::CheckpointHandle::new(dal, config.graph_name.clone(), name.clone()) + }); + let sender = BoundarySender::with_freshness( + boundary_tx, + SourceName::new(&name), + config.freshness.clone(), + ); + let ctx = AccumulatorContext { + output: sender, + name: name.clone(), + shutdown: shutdown_rx, + checkpoint, + health: config.health_tx, + }; + let poller = ClosurePollingAccumulator { + poll_fn, + interval: self.interval, + }; + let handle = tokio::spawn(super::accumulator::polling_accumulator_runtime( + poller, ctx, socket_rx, + )); + (socket_tx, handle) + } +} + +/// Parse a polling accumulator's `interval` (e.g. `"2s"`) from config; defaults +/// to 5s when absent/unparsable. +fn polling_interval_from_config( + config: &std::collections::HashMap, +) -> std::time::Duration { + config + .get("interval") + .and_then(|s| crate::packaging::manifest_schema::parse_duration_str(s).ok()) + .unwrap_or_else(|| std::time::Duration::from_secs(5)) +} + +fn accumulator_factory_for( + acc_type: &str, + config: &std::collections::HashMap, +) -> Arc { + match acc_type { + "stream" => Arc::new(StreamBackendAccumulatorFactory::new(config.clone())), + "state" => Arc::new(StateAccumulatorFactory::new(state_capacity_from_config( + config, + ))), + "batch" => { + let (flush_interval, max_buffer_size) = batch_config_from_config(config); + Arc::new(BatchAccumulatorFactory::new( + flush_interval, + max_buffer_size, + )) + } + "polling" => Arc::new(PollingAccumulatorFactory::new( + polling_interval_from_config(config), + )), + "passthrough" => Arc::new(PassthroughAccumulatorFactory), + other => { + tracing::warn!( + accumulator_type = %other, + "unknown accumulator type in packaged graph — falling back to \ + passthrough (CLOACI-T-0896); firing will be per-event, not the \ + declared behavior" + ); + Arc::new(PassthroughAccumulatorFactory) + } + } +} + // --------------------------------------------------------------------------- // T-0545 M3a: dispatch reactors registered in a Runtime into a scheduler // --------------------------------------------------------------------------- @@ -704,13 +967,7 @@ pub async fn dispatch_runtime_reactors_into_scheduler( None => ("passthrough".to_string(), Default::default()), }, }; - let factory: Arc = match acc_type.as_str() { - "stream" => Arc::new(StreamBackendAccumulatorFactory::new(acc_config)), - "state" => Arc::new(StateAccumulatorFactory::new(state_capacity_from_config( - &acc_config, - ))), - _ => Arc::new(PassthroughAccumulatorFactory), - }; + let factory = accumulator_factory_for(&acc_type, &acc_config); AccumulatorDeclaration { name: acc_name.clone(), factory, @@ -770,28 +1027,15 @@ pub async fn dispatch_package_reactors_into_scheduler( .accumulators .iter() .map(|acc| { - let factory: Arc = match accumulator_overrides + let factory = match accumulator_overrides .iter() .find(|cfg| cfg.name == acc.name) { - Some(override_cfg) => match override_cfg.accumulator_type.as_str() { - "stream" => Arc::new(StreamBackendAccumulatorFactory::new( - override_cfg.config.clone(), - )), - "state" => Arc::new(StateAccumulatorFactory::new( - state_capacity_from_config(&override_cfg.config), - )), - _ => Arc::new(PassthroughAccumulatorFactory), - }, - None => match acc.accumulator_type.as_str() { - "stream" => { - Arc::new(StreamBackendAccumulatorFactory::new(acc.config.clone())) - } - "state" => Arc::new(StateAccumulatorFactory::new( - state_capacity_from_config(&acc.config), - )), - _ => Arc::new(PassthroughAccumulatorFactory), - }, + Some(override_cfg) => accumulator_factory_for( + &override_cfg.accumulator_type, + &override_cfg.config, + ), + None => accumulator_factory_for(&acc.accumulator_type, &acc.config), }; AccumulatorDeclaration { name: acc.name.clone(), diff --git a/crates/cloacina/src/registry/loader/task_registrar/dynamic_task.rs b/crates/cloacina/src/registry/loader/task_registrar/dynamic_task.rs index a6b4912f7..4aba6e0ca 100644 --- a/crates/cloacina/src/registry/loader/task_registrar/dynamic_task.rs +++ b/crates/cloacina/src/registry/loader/task_registrar/dynamic_task.rs @@ -168,9 +168,45 @@ impl Task for DynamicLibraryTask { tracing::debug!("Task '{}' input context: {}", self.task_name, context_json); + // CLOACI-T-0895: a resolver OBJECT cannot cross the plugin boundary, + // so resolve every `{"$secret"}`-referenced secret host-side (via the + // resolver the executor/agent attached to this context) and ship the + // VALUES, keyed by concrete secret name. The plugin shell attaches + // them to the rebuilt scope; `context.secret(...)` then works + // identically inside the package. Fail-closed: references with no + // resolver (or unresolvable names) fail the task BEFORE the plugin + // call — never a silent run with missing secrets. + let mut resolved_secrets: std::collections::BTreeMap< + String, + std::collections::BTreeMap, + > = std::collections::BTreeMap::new(); + let referenced: std::collections::BTreeSet = context + .data() + .get(cloacina_workflow::secret::SECRET_REFS_KEY) + .and_then(|v| v.as_object()) + .map(|m| { + m.values() + .filter_map(|v| v.as_str().map(String::from)) + .collect() + }) + .unwrap_or_default(); + for secret_name in referenced { + let fields = + context + .secret(&secret_name) + .await + .map_err(|e| TaskError::ExecutionFailed { + task_id: self.task_name.clone(), + message: format!("resolving secret '{}': {}", secret_name, e), + timestamp: Utc::now(), + })?; + resolved_secrets.insert(secret_name, fields); + } + let request = TaskExecutionRequest { task_name: self.task_name.clone(), context_json, + resolved_secrets, }; // Call via the shared plugin handle diff --git a/crates/cloacina/src/registry/workflow_registry/mod.rs b/crates/cloacina/src/registry/workflow_registry/mod.rs index 6b3b7750b..95ac53306 100644 --- a/crates/cloacina/src/registry/workflow_registry/mod.rs +++ b/crates/cloacina/src/registry/workflow_registry/mod.rs @@ -234,6 +234,18 @@ impl WorkflowRegistryImpl { .unwrap_or_default()) } + /// Whether a workflow addressed by `name` (its `workflow_name` or + /// `package_name`) is registered in THIS tenant's registry (CLOACI-T-0901). + /// The execute chokepoint uses this to refuse cross-tenant execution: a + /// tenant must not be able to run a workflow it never installed just because + /// another tenant loaded the same name into the process-shared `Runtime`. + pub async fn workflow_exists(&self, name: &str) -> Result { + let workflows = self.list_workflows().await?; + Ok(workflows + .iter() + .any(|w| w.workflow_name == name || w.package_name == name)) + } + /// Workflow names subscribed to `trigger_name` via `#[workflow(triggers = […])]` /// (CLOACI-T-0777) — the fan-out set for a manual trigger fire. The schedules /// table only carries the trigger's primary `on` workflow, so subscriptions diff --git a/crates/cloacina/src/runner/default_runner/mod.rs b/crates/cloacina/src/runner/default_runner/mod.rs index 5a7284e58..db89cc4d4 100644 --- a/crates/cloacina/src/runner/default_runner/mod.rs +++ b/crates/cloacina/src/runner/default_runner/mod.rs @@ -137,6 +137,22 @@ impl DefaultRunner { database: Database, config: DefaultRunnerConfig, shared_runtime: Option>, + ) -> Result { + Self::with_database_secrets(database, config, shared_runtime, None).await + } + + /// [`Self::with_database`] plus a secret resolution side channel + /// (CLOACI-T-0858 / I-0133 D-1) threaded onto the in-process thread + /// executor, so `context.secret(...)` resolves for locally-executed + /// tasks. Hosts that construct runners per tenant (cloacina-server's + /// `TenantRunnerCache`) pass a tenant-scoped + /// [`crate::security::SecretStoreResolver`]; `None` keeps the + /// fail-closed "secrets backend not configured" behavior. + pub async fn with_database_secrets( + database: Database, + config: DefaultRunnerConfig, + shared_runtime: Option>, + secret_resolver: Option>, ) -> Result { let runtime = shared_runtime.unwrap_or_else(|| Arc::new(Runtime::new())); @@ -160,7 +176,10 @@ impl DefaultRunner { Arc::new(crate::task::TaskRegistry::new()), runtime.clone(), executor_config, - ); + ) + // CLOACI-T-0858: thread the secret side channel so task contexts can + // resolve secrets on the in-process execution path. + .with_secret_resolver(secret_resolver); // Configure dispatcher for push-based task execution. Every task is sent // to the one configured executor key (CLOACI-T-0640). diff --git a/crates/cloacina/tests/integration/computation_graph.rs b/crates/cloacina/tests/integration/computation_graph.rs index f33478225..9614ac0a9 100644 --- a/crates/cloacina/tests/integration/computation_graph.rs +++ b/crates/cloacina/tests/integration/computation_graph.rs @@ -951,6 +951,16 @@ async fn test_sequential_input_strategy() { mod resilience_tests { use super::*; + /// Decode a state-accumulator boundary payload. State accumulators emit the + /// window as JSON (`state_window_frame`), which `BoundarySender::send` then + /// wraps with `types::serialize` — so a received boundary is two layers: + /// the `types` envelope around the JSON bytes. Peel both. + fn decode_state_window(bytes: &[u8]) -> Result, String> { + let json: Vec = cloacina::computation_graph::types::deserialize(bytes) + .map_err(|e| format!("envelope decode: {e}"))?; + serde_json::from_slice(&json).map_err(|e| format!("json decode: {e}")) + } + /// Helper: create an in-memory SQLite DAL for testing. /// Uses shared-cache in-memory DB so the pool can have multiple connections /// to the same database without creating temp files on disk. @@ -1594,16 +1604,30 @@ mod resilience_tests { .unwrap(); } - // Wait for processing + persistence - tokio::time::sleep(std::time::Duration::from_millis(300)).await; - - // Drain boundaries (each write emits the full list) + // Poll boundaries until the accumulator has processed all 3 writes and + // emitted the full list (each write emits the full list). Waits up to 5s + // rather than a fixed sleep + non-blocking drain, which flakes when + // processing lags under a loaded single-threaded test run. let mut last_boundary: Option> = None; - while let Ok((_, bytes)) = boundary_rx.try_recv() { - if let Ok(list) = - cloacina::computation_graph::types::deserialize::>(&bytes) + let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(5); + while tokio::time::Instant::now() < deadline { + match tokio::time::timeout(std::time::Duration::from_millis(200), boundary_rx.recv()) + .await { - last_boundary = Some(list); + Ok(Some((_, bytes))) => { + // The boundary is a JSON-encoded window (`state_window_frame` + // uses `serde_json`) wrapped by `BoundarySender::send` via + // `types::serialize` — decode both layers: envelope → JSON → list. + if let Ok(list) = decode_state_window::(&bytes) { + let complete = list.len() == 3; + last_boundary = Some(list); + if complete { + break; + } + } + } + Ok(None) => break, // channel closed + Err(_) => {} // recv timed out this round; keep waiting } } @@ -1649,15 +1673,31 @@ mod resilience_tests { let acc2 = StateAccumulator::::new(10); let _handle2 = tokio::spawn(state_accumulator_runtime(acc2, ctx2, socket_rx2)); - // On startup, state_accumulator_runtime loads from DAL and emits the list - tokio::time::sleep(std::time::Duration::from_millis(300)).await; - - // Should receive the restored list as initial boundary - let (_, bytes) = boundary_rx2 - .try_recv() - .expect("should receive initial boundary from restored state"); - let restored: Vec = - cloacina::computation_graph::types::deserialize(&bytes).unwrap(); + // On startup, state_accumulator_runtime loads from DAL and emits the + // restored list. Wait for it (up to 5s) rather than a fixed sleep + + // single non-blocking try_recv, which flakes when the restore/emit lags. + let restored = { + let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(5); + let mut received: Option> = None; + while tokio::time::Instant::now() < deadline { + match tokio::time::timeout( + std::time::Duration::from_millis(200), + boundary_rx2.recv(), + ) + .await + { + Ok(Some((_, b))) => { + if let Ok(list) = decode_state_window::(&b) { + received = Some(list); + break; + } + } + Ok(None) => break, + Err(_) => {} + } + } + received.expect("should receive initial boundary from restored state") + }; assert_eq!(restored.len(), 3, "restored list should have 3 items"); assert_eq!(restored[0].value, 1.0); diff --git a/crates/cloacina/tests/integration/database/connection.rs b/crates/cloacina/tests/integration/database/connection.rs index d83e5877b..ae872a1cb 100644 --- a/crates/cloacina/tests/integration/database/connection.rs +++ b/crates/cloacina/tests/integration/database/connection.rs @@ -25,7 +25,7 @@ fn test_url_parsing_basic() { let parsed = result.unwrap(); assert_eq!(parsed.host_str(), Some("localhost")); - assert_eq!(parsed.port(), Some(5432)); + assert_eq!(parsed.port(), Some(15432)); assert_eq!(parsed.path(), "/test_db"); assert_eq!(parsed.username(), "user"); assert_eq!(parsed.password(), Some("pass")); diff --git a/crates/cloacina/tests/integration/fidius_validation.rs b/crates/cloacina/tests/integration/fidius_validation.rs index 9802b2b03..5e429cd1d 100644 --- a/crates/cloacina/tests/integration/fidius_validation.rs +++ b/crates/cloacina/tests/integration/fidius_validation.rs @@ -166,6 +166,7 @@ fn test_task_execution_fidelity() { let request = TaskExecutionRequest { task_name: "extract_data".to_string(), context_json: "{}".to_string(), + resolved_secrets: Default::default(), }; // Method index 1 = execute_task (fidius 0.0.5 tuple encoding: single-arg = (T,)) @@ -208,6 +209,7 @@ fn test_unknown_task_returns_error() { let request = TaskExecutionRequest { task_name: "nonexistent_task".to_string(), context_json: "{}".to_string(), + resolved_secrets: Default::default(), }; // fidius 0.0.5 tuple encoding: single-arg = (T,) @@ -251,8 +253,8 @@ fn test_plugin_info_populated() { "Interface hash should be non-zero" ); assert_eq!( - plugin.info.interface_version, 4, - "Interface version should be 4 (CLOACI-I-0132 get_constructor_metadata)" + plugin.info.interface_version, 5, + "Interface version should be 5 (CLOACI-T-0895 resolved_secrets on TaskExecutionRequest)" ); assert_eq!( plugin.method_count, 11, diff --git a/docker/docker-compose.demo.yml b/docker/docker-compose.demo.yml index ff1fb96c5..6a0aed148 100644 --- a/docker/docker-compose.demo.yml +++ b/docker/docker-compose.demo.yml @@ -114,6 +114,11 @@ services: # Resolves the `{{ KAFKA_BROKER }}` token in Kafka-stream accumulator # configs (e.g. demo-kafka-stream-rust) to the in-network broker. CLOACINA_VAR_KAFKA_BROKER: "kafka:9092" + # Tenant-secrets KEK (CLOACI-I-0133) — enables `cloacinactl secret …` + + # `{"$secret": …}` bindings on this stack. Fixed DEMO value (base64 of a + # 32-byte string); production operators provision their own and never + # commit it. + CLOACINA_SECRET_KEK: "ZGVtby1rZWstZGVtby1rZWstZGVtby1rZWstMDAwMSE=" # Route all task execution to the execution-agent fleet instead of the # server's in-process executor (CLOACI-I-0114). The `agent` services below # register over WS, fetch cdylibs by digest, run tasks, and report back. diff --git a/examples/features/computation-graphs/cg-feature-tour/Cargo.toml b/examples/features/computation-graphs/cg-feature-tour/Cargo.toml new file mode 100644 index 000000000..fb58c5c01 --- /dev/null +++ b/examples/features/computation-graphs/cg-feature-tour/Cargo.toml @@ -0,0 +1,31 @@ +[workspace] + +[package] +name = "cg-feature-tour" +version = "0.1.0" +edition = "2021" + +[features] +default = ["packaged"] +packaged = [] + +[lib] +crate-type = ["cdylib", "rlib"] + +# Production-shaped: crates.io version deps (what `cloacinactl package new` +# emits). Dev stacks resolve these against the local workspace via the +# compiler's `--dev-workspace` flag; production compilers use crates.io. +[dependencies] +cloacina-macros = "0.10" +cloacina-computation-graph = "0.10" +cloacina-workflow = { version = "0.10", features = ["packaged"] } +cloacina-workflow-plugin = "0.10" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +schemars = "0.8" +futures = "0.3" +async-trait = "0.1" +tracing = "0.1" + +[build-dependencies] +cloacina-build = "0.10" diff --git a/examples/features/computation-graphs/cg-feature-tour/README.md b/examples/features/computation-graphs/cg-feature-tour/README.md new file mode 100644 index 000000000..9174becc1 --- /dev/null +++ b/examples/features/computation-graphs/cg-feature-tour/README.md @@ -0,0 +1,134 @@ +# Computation-Graph Feature Tour + +Three computation-graph surfaces in one package, all exercised through the +primary interface: + +| Surface | What it shows | +|---|---| +| **Kafka stream accumulator** | `ticks` is upgraded to a Kafka stream source; each message fires a reactor and runs a graph | +| **Typed inject/fire** | `Tick` derives `JsonSchema`, giving the accumulator a typed `accumulator inject` form | +| **Task→graph invocation** | a workflow task `invokes` a trigger-less graph and consumes its output via a `post_invocation` hook | + +## Surfaces + +### 1 + 2. Kafka stream accumulator with a typed boundary + +```rust +#[reactor(name = "tour_rx", accumulators = [ticks], criteria = when_any(ticks))] +pub struct TourRx; + +#[computation_graph(trigger = reactor("tour_rx"), graph = { enrich(ticks) -> emit })] +pub mod tour_stream_graph { ... } +``` + +`ticks` is declared as a plain accumulator in the macro; `package.toml` +upgrades it to a Kafka stream source: + +```toml +[[metadata.accumulators]] +name = "ticks" +accumulator_type = "stream" + +[metadata.accumulators.config] +broker = "{{ KAFKA_BROKER }}" # resolves via CLOACINA_VAR_KAFKA_BROKER +topic = "tour.ticks" +group = "cg-feature-tour-group" +``` + +Because `Tick` derives `schemars::JsonSchema`, the accumulator gets a **typed** +inject/fire form — the server validates injected events against the schema. + +### 3. A workflow task that invokes a trigger-less graph + +```rust +#[computation_graph(graph = { normalize -> output })] // no trigger = ... → trigger-less +pub mod tour_math_graph { ... } + +#[task( + dependencies = ["prep"], + invokes = computation_graph("tour_math_graph"), + post_invocation = summarize, +)] +pub async fn crunch(ctx: &mut Context) -> Result<(), TaskError> { ... } +``` + +The task's body runs first, then the graph, then `summarize` (which sees the +graph's terminal output merged into the context under the node name `output`). +Only trigger-less graphs are invocable — a reactor-triggered graph is rejected +at compile time. + +## Run it + +Automated as `angreal demos features cg-feature-tour` (the CI examples lane +runs exactly that). + +### 1. Stack + CLI + +```bash +angreal ui up +cloacinactl config profile set demo http://localhost:8080 \ + --api-key clk_demo_public_key_0003 --tenant public --default +``` + +The demo stack includes Kafka; the server resolves `{{ KAFKA_BROKER }}` to the +in-cluster broker. + +### 2. Pack + upload + +```bash +cloacinactl package pack . --out cg-feature-tour.cloacina +cloacinactl package upload cg-feature-tour.cloacina +cloacinactl package list # wait for build_status: success +``` + +### 3. Task→graph invocation + +```bash +cloacinactl workflow run tour_pipeline +cloacinactl execution list --workflow tour_pipeline +``` + +Completion proves the invoke bridge: `report` fails unless the +`post_invocation` hook saw the graph's terminal output. + +## Operate it + +### Inject a typed event (fires the reactor) + +```bash +cloacinactl accumulator inject ticks '{"price": 101.5}' +cloacinactl graph accumulators +# watch the reactor's fires: +curl -s -H 'Authorization: Bearer clk_demo_public_key_0003' \ + http://localhost:8080/v1/health/reactors/tour_rx/fires +``` + +A malformed event (missing `price`, wrong type) is rejected — that's the typed +boundary from `JsonSchema`. + +### Feed the stream from Kafka + +```bash +docker exec -i cloacina-demo-kafka-1 \ + /opt/kafka/bin/kafka-console-producer.sh \ + --bootstrap-server localhost:9092 --topic tour.ticks <<'EOF' +{"price": 202.0} +EOF +``` + +Each message on `tour.ticks` fires `tour_rx` and runs `tour_stream_graph`. + +## Status of the surfaces + +- **Task→graph invocation** — works today; asserted in CI + (`angreal demos features cg-feature-tour`). +- **Kafka stream accumulator** — the code is here, but kafka is currently a + HOST cargo feature (rdkafka linked into core `cloacina`), so a stream + accumulator only runs if the *server* was built `--features kafka` and + silently degrades to passthrough otherwise. This is being migrated so kafka + ships **in the package** as a constructor provider — a consumption connector + belongs in a provider, not the core engine (CLOACI-T-0898). The stream + sections above light up on that migration. +- **`polling` / `batch` accumulators** — declared surfaces + (`#[polling_accumulator]`, `#[batch_accumulator]`) that also silently degrade + to passthrough in packaged graphs today (CLOACI-T-0896). diff --git a/examples/features/computation-graphs/cg-feature-tour/build.rs b/examples/features/computation-graphs/cg-feature-tour/build.rs new file mode 100644 index 000000000..6e9cdfc6c --- /dev/null +++ b/examples/features/computation-graphs/cg-feature-tour/build.rs @@ -0,0 +1,19 @@ +/* + * Copyright 2026 Colliery Software + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +fn main() { + cloacina_build::configure(); +} diff --git a/examples/features/computation-graphs/cg-feature-tour/package.toml b/examples/features/computation-graphs/cg-feature-tour/package.toml new file mode 100644 index 000000000..a3ef30426 --- /dev/null +++ b/examples/features/computation-graphs/cg-feature-tour/package.toml @@ -0,0 +1,27 @@ +# CG feature tour (CLOACI-T-0891). One package carrying: +# - a Kafka-fed reactor-bound computation graph (`tour_stream_graph`) whose +# entry accumulator `ticks` is upgraded to a `stream` source below +# - a workflow (`tour_pipeline`) whose task INVOKES a trigger-less graph +# The broker placeholder resolves via CLOACINA_VAR_KAFKA_BROKER on the server. +[package] +name = "cg-feature-tour" +version = "0.1.0" +interface = "cloacina-workflow-plugin" +interface_version = 1 +extension = "cloacina" + +[metadata] +workflow_name = "tour_pipeline" +graph_name = "tour_stream_graph" +language = "rust" +description = "Computation-graph feature tour — kafka stream accumulator, typed inject, task-to-graph invocation" +author = "Cloacina Demo Team" + +[[metadata.accumulators]] +name = "ticks" +accumulator_type = "stream" + +[metadata.accumulators.config] +broker = "{{ KAFKA_BROKER }}" +topic = "tour.ticks" +group = "cg-feature-tour-group" diff --git a/examples/features/computation-graphs/cg-feature-tour/src/lib.rs b/examples/features/computation-graphs/cg-feature-tour/src/lib.rs new file mode 100644 index 000000000..77b1deffe --- /dev/null +++ b/examples/features/computation-graphs/cg-feature-tour/src/lib.rs @@ -0,0 +1,185 @@ +/* + * Copyright 2026 Colliery Software + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*! +# Computation-Graph Feature Tour + +Three surfaces in one package, all through the primary interface: + +1. **Kafka stream accumulator** — the `ticks` accumulator is upgraded to a + Kafka stream source by `package.toml` (`[[metadata.accumulators]]`); each + message on the topic fires the reactor and runs `tour_stream_graph`. +2. **Typed inject/fire** — `Tick` derives `schemars::JsonSchema`, giving the + accumulator a typed inject form (`cloacinactl accumulator inject`) and the + reactor a typed fire form. +3. **Task→graph invocation** — the `tour_pipeline` workflow's `crunch` task + `invokes` the trigger-less `tour_math_graph`; terminal outputs merge back + into the task's context, and a `post_invocation` hook sees the merged + result. +*/ + +use cloacina_macros::{computation_graph, reactor, task, workflow}; +use cloacina_workflow::{Context, TaskError}; +use serde::{Deserialize, Serialize}; + +cloacina_workflow_plugin::package!(); + +// --------------------------------------------------------------------------- +// Surface 1 + 2: a Kafka-fed reactor-bound graph with a typed boundary +// --------------------------------------------------------------------------- + +/// One market tick. Deriving `JsonSchema` opts the accumulator into the TYPED +/// inject/fire form — the server validates injected events against this shape. +#[derive(Debug, Clone, Serialize, Deserialize, schemars::JsonSchema)] +pub struct Tick { + pub price: f64, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct EnrichedTick { + pub price: f64, + pub spread_bps: f64, +} + +/// Fires on every tick. `ticks` is a plain accumulator here; `package.toml` +/// upgrades it to a Kafka `stream` source (broker/topic/group). +#[reactor( + name = "tour_rx", + accumulators = [ticks], + criteria = when_any(ticks), +)] +pub struct TourRx; + +#[computation_graph( + trigger = reactor("tour_rx"), + graph = { + enrich(ticks) -> emit, + } +)] +pub mod tour_stream_graph { + use super::*; + + pub async fn enrich(ticks: Option<&Tick>) -> EnrichedTick { + let price = ticks.map(|t| t.price).unwrap_or(0.0); + EnrichedTick { + price, + spread_bps: price * 0.0005, + } + } + + pub async fn emit(input: &EnrichedTick) -> EnrichedTick { + input.clone() + } +} + +// --------------------------------------------------------------------------- +// Surface 3: a trigger-less graph a workflow task INVOKES +// --------------------------------------------------------------------------- + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Normalized { + pub value: f64, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MathResult { + pub value: f64, + pub squared: f64, +} + +/// No `trigger = …`: this graph is trigger-less, which is exactly what makes +/// it invocable from a task (`invokes = computation_graph(...)` refuses +/// reactor-driven graphs at compile time). +#[computation_graph(graph = { + normalize -> output, +})] +pub mod tour_math_graph { + use super::*; + + /// Entry node reads the invoking task's context directly. + pub async fn normalize(ctx: &Context) -> Normalized { + let raw = ctx.get("raw_value").and_then(|v| v.as_f64()).unwrap_or(0.0); + Normalized { value: raw / 100.0 } + } + + pub async fn output(input: &Normalized) -> MathResult { + MathResult { + value: input.value, + squared: input.value * input.value, + } + } +} + +// --------------------------------------------------------------------------- +// The workflow that invokes it +// --------------------------------------------------------------------------- + +/// `post_invocation` hook: runs after the graph, sees the merged context +/// (terminal outputs land under their node names — here, `output`). +async fn summarize(context: &mut Context) -> Result<(), TaskError> { + let squared = context + .get("output") + .and_then(|o| o.get("squared")) + .and_then(|v| v.as_f64()) + .ok_or_else(|| TaskError::ValidationFailed { + message: "graph terminal `output` missing from merged context".to_string(), + })?; + context.insert("summary", serde_json::json!({ "squared": squared }))?; + Ok(()) +} + +#[workflow( + name = "tour_pipeline", + description = "Workflow that invokes a trigger-less computation graph", + author = "Cloacina Demo Team" +)] +pub mod tour_pipeline { + use super::*; + + /// Seed the value the graph's entry node reads. + #[task] + pub async fn prep(context: &mut Context) -> Result<(), TaskError> { + context.insert("raw_value", serde_json::json!(250.0))?; + Ok(()) + } + + /// Pre-work runs first, then the macro-generated invocation of + /// `tour_math_graph`, then the `summarize` post-hook. + #[task( + dependencies = ["prep"], + invokes = computation_graph("tour_math_graph"), + post_invocation = summarize, + )] + pub async fn crunch(context: &mut Context) -> Result<(), TaskError> { + context.insert("crunch_ran", serde_json::json!(true))?; + Ok(()) + } + + /// Consume the merged result downstream. + #[task(dependencies = ["crunch"])] + pub async fn report(context: &mut Context) -> Result<(), TaskError> { + let summary = + context + .get("summary") + .cloned() + .ok_or_else(|| TaskError::ValidationFailed { + message: "missing summary from post_invocation hook".to_string(), + })?; + println!("📊 tour_pipeline result: {summary}"); + context.insert("tour_report", summary)?; + Ok(()) + } +} diff --git a/examples/features/computation-graphs/packaged-graph/Cargo.toml b/examples/features/computation-graphs/packaged-graph/Cargo.toml index 06b0bc6e3..25e7c8459 100644 --- a/examples/features/computation-graphs/packaged-graph/Cargo.toml +++ b/examples/features/computation-graphs/packaged-graph/Cargo.toml @@ -13,10 +13,10 @@ packaged = [] crate-type = ["cdylib", "rlib"] [dependencies] -cloacina-computation-graph = { path = "../../../../crates/cloacina-computation-graph" } -cloacina-macros = { path = "../../../../crates/cloacina-macros" } -cloacina-workflow = { path = "../../../../crates/cloacina-workflow", features = ["packaged"] } -cloacina-workflow-plugin = { path = "../../../../crates/cloacina-workflow-plugin" } +cloacina-computation-graph = "0.10" +cloacina-macros = "0.10" +cloacina-workflow = { version = "0.10", features = ["packaged"] } +cloacina-workflow-plugin = "0.10" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" # CLOACI-T-0768: typed inject/fire interface for the orderbook/pricing boundaries. @@ -25,4 +25,4 @@ async-trait = "0.1" tokio = { version = "1.0", features = ["full"] } [build-dependencies] -cloacina-build = { path = "../../../../crates/cloacina-build" } +cloacina-build = "0.10" diff --git a/examples/features/computation-graphs/python-batch-graph/package.toml b/examples/features/computation-graphs/python-batch-graph/package.toml new file mode 100644 index 000000000..a38d2b363 --- /dev/null +++ b/examples/features/computation-graphs/python-batch-graph/package.toml @@ -0,0 +1,24 @@ +# PYTHON packaged BATCH-ACCUMULATOR computation graph (I-0138 / Python-gap +# sweep). Covers the batch-accumulator kind — `@cloaca.batch_accumulator( +# flush_interval=…, max_buffer_size=…)` — which buffers events and flushes the +# whole buffer as one boundary when EITHER the buffer fills OR the interval +# elapses. Unlike `state` (a rolling window emitted on every write), `batch` +# emits only on flush. This kind had NO example or fixture before, so it's the +# least-exercised accumulator on the Python path — a fresh probe on the gold path. +# +# The module tree MUST live under workflow/ — the reconciler's Python CG +# extraction requires it. `event-batch` → `workflow/event_batch/`. +[package] +name = "event-batch" +version = "0.1.0" +interface = "cloacina-workflow-plugin" +interface_version = 1 +extension = "cloacina" + +[metadata] +graph_name = "event_batch_py" +language = "python" +description = "Python batch accumulator — buffer events, flush on size or interval" +entry_module = "event_batch.graph" +reaction_mode = "when_any" +input_strategy = "latest" diff --git a/examples/features/computation-graphs/python-batch-graph/workflow/event_batch/__init__.py b/examples/features/computation-graphs/python-batch-graph/workflow/event_batch/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/examples/features/computation-graphs/python-batch-graph/workflow/event_batch/graph.py b/examples/features/computation-graphs/python-batch-graph/workflow/event_batch/graph.py new file mode 100644 index 000000000..70f89b636 --- /dev/null +++ b/examples/features/computation-graphs/python-batch-graph/workflow/event_batch/graph.py @@ -0,0 +1,61 @@ +"""Python packaged BATCH-ACCUMULATOR computation graph. + +`event_batch` is a `@cloaca.batch_accumulator(flush_interval="1s", +max_buffer_size=5)`: each injected event is buffered; the WHOLE buffer is emitted +as one boundary when the buffer fills (5) OR the flush interval (1s) elapses, +whichever comes first. Unlike a state accumulator (a rolling window emitted on +every write), a batch emits only on flush — the entry node receives the flushed +batch (a LIST) once per flush. + +`@cloaca.boundary_schema(level=float)` types each injected event, so the server +validates injected/fired events before they reach the graph (a non-conforming +event is rejected — the typed-inject surface). +""" +import cloaca + + +# Typed batch: buffer typed events, flush the whole buffer on size or interval. +@cloaca.boundary_schema(level=float) +@cloaca.batch_accumulator(flush_interval="1s", max_buffer_size=5) +def event_batch(event): + # Batch accumulators buffer the returned value; the runtime emits the whole + # buffer back as the boundary on flush. Pass the event through as-is. + return event + + +@cloaca.reactor(name="batch_reactor", accumulators=["event_batch"], mode="when_any") +class _BatchReactor: + pass + + +with cloaca.ComputationGraphBuilder( + "event_batch_py", + reactor=_BatchReactor, + graph={ + # Entry node consumes the flushed batch (a list), then a linear edge to a + # terminal reporter — a minimal but real 2-node CG. + "summarize": {"inputs": ["event_batch"], "next": "report"}, + "report": {}, + }, +) as builder: + + @cloaca.node + def summarize(event_batch): + # `event_batch` is the flushed batch (list of buffered events), not a + # single event — that's the batch-accumulator behaviour. + batch = event_batch or [] + levels = [e.get("level", 0.0) for e in batch if isinstance(e, dict)] + return { + "batch_size": len(batch), + "total": sum(levels), + "peak": max(levels) if levels else 0.0, + } + + @cloaca.node + def report(summarize): + # `summarize` is the dict returned by the entry node (linear-edge payload). + return { + "flushed": summarize.get("batch_size", 0), + "sum_level": summarize.get("total", 0.0), + "peak_level": summarize.get("peak", 0.0), + } diff --git a/examples/features/computation-graphs/python-packaged-graph/workflow/market_maker/graph.py b/examples/features/computation-graphs/python-packaged-graph/workflow/market_maker/graph.py index 36084c9a5..5936f0c47 100644 --- a/examples/features/computation-graphs/python-packaged-graph/workflow/market_maker/graph.py +++ b/examples/features/computation-graphs/python-packaged-graph/workflow/market_maker/graph.py @@ -9,7 +9,12 @@ import cloaca -# Define accumulators +# Define accumulators. `@cloaca.boundary_schema(...)` declares the accumulator's +# typed boundary (the Python parity of deriving `schemars::JsonSchema` on a Rust +# boundary type): the compiler parses it into a typed input slot, so the server +# validates injected/fired events against it — a non-conforming event is +# rejected before it reaches the graph (CLOACI-T-0759 / T-0770). +@cloaca.boundary_schema(best_bid=float, best_ask=float) @cloaca.passthrough_accumulator def orderbook(event): """Pass through order book events.""" diff --git a/examples/features/computation-graphs/python-polling-graph/package.toml b/examples/features/computation-graphs/python-polling-graph/package.toml new file mode 100644 index 000000000..2db3b99a9 --- /dev/null +++ b/examples/features/computation-graphs/python-polling-graph/package.toml @@ -0,0 +1,24 @@ +# PYTHON packaged POLLING-ACCUMULATOR computation graph (I-0138 / T-0896). +# Covers the polling-accumulator kind — `@cloaca.polling_accumulator(interval=…)` +# — which, unlike the event-driven kinds (passthrough/state/batch), is +# SELF-FIRING: the runtime calls the accumulator's poll function on the interval +# and emits whatever it returns (None = skip). No inject/socket needed — the +# reactor fires on its own each interval. +# +# This is the kind T-0896 fixed: the packaged path invokes the in-process Python +# poll fn (spawn_blocking + GIL, like poll triggers) — no FFI. `heartbeat-poll` +# → `workflow/heartbeat_poll/`. +[package] +name = "heartbeat-poll" +version = "0.1.0" +interface = "cloacina-workflow-plugin" +interface_version = 1 +extension = "cloacina" + +[metadata] +graph_name = "heartbeat_poll_py" +language = "python" +description = "Python polling accumulator — self-fires on an interval via its poll fn" +entry_module = "heartbeat_poll.graph" +reaction_mode = "when_any" +input_strategy = "latest" diff --git a/examples/features/computation-graphs/python-polling-graph/workflow/heartbeat_poll/__init__.py b/examples/features/computation-graphs/python-polling-graph/workflow/heartbeat_poll/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/examples/features/computation-graphs/python-polling-graph/workflow/heartbeat_poll/graph.py b/examples/features/computation-graphs/python-polling-graph/workflow/heartbeat_poll/graph.py new file mode 100644 index 000000000..6a9e62d7d --- /dev/null +++ b/examples/features/computation-graphs/python-polling-graph/workflow/heartbeat_poll/graph.py @@ -0,0 +1,54 @@ +"""Python packaged POLLING-ACCUMULATOR computation graph. + +`heartbeat` is a `@cloaca.polling_accumulator(interval="2s")`: unlike the +event-driven accumulators (passthrough/state/batch), a polling accumulator is +SELF-FIRING — the runtime calls its poll function on the interval and emits +whatever it returns (return `None` to skip a tick). No inject/socket is needed; +the reactor fires on its own each interval, and the entry node receives the +polled value. + +On the packaged path this works because the reconciler imports this module in +the server process, so the poll function is invoked in-process on the interval +(spawn_blocking + GIL, exactly like a Python poll trigger) — no FFI (T-0896). +""" +import cloaca + +# Module-level counter so each poll returns a distinct value — makes the +# self-firing visible (every tick emits a fresh boundary). +_ticks = {"n": 0} + + +@cloaca.polling_accumulator(interval="2s") +def heartbeat(): + """Poll function: called on each interval. Returns the event to emit as the + boundary, or None to skip this tick.""" + _ticks["n"] += 1 + return {"tick": _ticks["n"], "source": "poller"} + + +@cloaca.reactor(name="poll_reactor", accumulators=["heartbeat"], mode="when_any") +class _PollReactor: + pass + + +with cloaca.ComputationGraphBuilder( + "heartbeat_poll_py", + reactor=_PollReactor, + graph={ + # Entry node consumes the polled value, then a linear edge to a terminal + # reporter — a minimal but real 2-node CG driven purely by the poller. + "observe": {"inputs": ["heartbeat"], "next": "report"}, + "report": {}, + }, +) as builder: + + @cloaca.node + def observe(heartbeat): + # `heartbeat` is the value returned by the poll fn this tick. + beat = heartbeat or {} + return {"tick": beat.get("tick", 0), "source": beat.get("source")} + + @cloaca.node + def report(observe): + # `observe` is the dict returned by the entry node (linear-edge payload). + return {"observed_tick": observe.get("tick", 0)} diff --git a/examples/features/computation-graphs/python-stateful-graph/package.toml b/examples/features/computation-graphs/python-stateful-graph/package.toml new file mode 100644 index 000000000..89125cff6 --- /dev/null +++ b/examples/features/computation-graphs/python-stateful-graph/package.toml @@ -0,0 +1,26 @@ +# PYTHON packaged STATE-ACCUMULATOR computation graph (I-0138 / Python-gap +# sweep). The stateful peer of `python-packaged-graph` (which uses a passthrough +# accumulator): this one uses `@cloaca.state_accumulator(capacity=N)` — the +# bounded rolling-window primitive. Every injected event is appended to a +# bounded window; the FULL retained window is emitted as the boundary on each +# write, so the entry node receives a LIST (history), not a single event. Closes +# the accumulator-kind coverage gap (state was fixtures-only, demo-py-state) on +# the server gold path. +# +# The module tree MUST live under workflow/ — the reconciler's Python CG +# extraction requires it. Package name (snake) is the entry module root: +# `tick-window` → `workflow/tick_window/`. +[package] +name = "tick-window" +version = "0.1.0" +interface = "cloacina-workflow-plugin" +interface_version = 1 +extension = "cloacina" + +[metadata] +graph_name = "tick_window_py" +language = "python" +description = "Python state accumulator — bounded rolling window over a market feed" +entry_module = "tick_window.graph" +reaction_mode = "when_any" +input_strategy = "latest" diff --git a/examples/features/computation-graphs/python-stateful-graph/workflow/tick_window/__init__.py b/examples/features/computation-graphs/python-stateful-graph/workflow/tick_window/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/examples/features/computation-graphs/python-stateful-graph/workflow/tick_window/graph.py b/examples/features/computation-graphs/python-stateful-graph/workflow/tick_window/graph.py new file mode 100644 index 000000000..496b4b835 --- /dev/null +++ b/examples/features/computation-graphs/python-stateful-graph/workflow/tick_window/graph.py @@ -0,0 +1,65 @@ +"""Python packaged STATE-ACCUMULATOR computation graph — the stateful peer of +`python-packaged-graph` (passthrough). + +`tick_window` is a `@cloaca.state_accumulator(capacity=5)`: each event pushed in +is appended to a bounded window; once full the oldest is evicted, and the FULL +retained window (newest ≤5 events) is emitted as the boundary on every write. +The reactor fires the graph on each write, and the entry node receives the whole +window (a LIST), not a single event — that rolling window is the visible +difference from a passthrough accumulator. + +`@cloaca.boundary_schema(bid=float, ask=float)` declares the accumulator's typed +boundary, so the server validates injected/fired events against it before they +reach the graph (a non-conforming event is rejected — the typed-inject surface). +""" +import cloaca + + +# Typed, bounded rolling window. The boundary schema types each injected tick; +# the state accumulator buffers them and emits the retained window. +@cloaca.boundary_schema(bid=float, ask=float) +@cloaca.state_accumulator(capacity=5) +def tick_window(event): + # State accumulators buffer the returned value; the runtime emits the full + # bounded window back as the boundary. Pass the tick through as-is. + return event + + +@cloaca.reactor(name="tick_reactor", accumulators=["tick_window"], mode="when_any") +class _TickReactor: + pass + + +with cloaca.ComputationGraphBuilder( + "tick_window_py", + reactor=_TickReactor, + graph={ + # Entry node consumes the accumulator's emitted window (a list), then a + # linear edge to a terminal reporter — a minimal but real 2-node CG. + "aggregate": {"inputs": ["tick_window"], "next": "report"}, + "report": {}, + }, +) as builder: + + @cloaca.node + def aggregate(tick_window): + # `tick_window` is the bounded rolling window (list of ≤5 recent ticks), + # not a single event — that's the state-accumulator behaviour. + history = tick_window or [] + bids = [e.get("bid", 0.0) for e in history if isinstance(e, dict)] + asks = [e.get("ask", 0.0) for e in history if isinstance(e, dict)] + return { + "window_size": len(history), + "latest": history[-1] if history else None, + "avg_bid": (sum(bids) / len(bids)) if bids else 0.0, + "avg_ask": (sum(asks) / len(asks)) if asks else 0.0, + } + + @cloaca.node + def report(aggregate): + # `aggregate` is the dict returned by the entry node (linear-edge payload). + return { + "windowed_avg_bid": aggregate.get("avg_bid", 0.0), + "windowed_avg_ask": aggregate.get("avg_ask", 0.0), + "samples": aggregate.get("window_size", 0), + } diff --git a/examples/features/workflows/complex-dag/Cargo.toml b/examples/features/workflows/complex-dag/Cargo.toml index fdb044dbc..8ca895ca7 100644 --- a/examples/features/workflows/complex-dag/Cargo.toml +++ b/examples/features/workflows/complex-dag/Cargo.toml @@ -21,18 +21,21 @@ packaged = [] [lib] crate-type = ["cdylib"] +# Lean packaged-workflow deps: crates.io VERSION deps (what `cloacinactl +# package new` emits), resolved against the local crates by the dev/demo +# compiler's `--dev-workspace`. A packaged workflow needs ONLY the authoring +# crates — NOT the `cloacina` engine umbrella (Context/TaskError come from +# cloacina-workflow). (CLOACI-T-0900 re-author off the pre-T-0887 path deps.) [dependencies] -cloacina = { path = "../../../../crates/cloacina" } -cloacina-computation-graph = { path = "../../../../crates/cloacina-computation-graph" } -cloacina-workflow = { path = "../../../../crates/cloacina-workflow", features = ["packaged"] } -cloacina-workflow-plugin = { path = "../../../../crates/cloacina-workflow-plugin" } -cloacina-macros = { path = "../../../../crates/cloacina-macros" } +cloacina-macros = "0.10" +cloacina-computation-graph = "0.10" +cloacina-workflow = { version = "0.10", features = ["packaged"] } +cloacina-workflow-plugin = "0.10" serde_json = "1.0" -tokio = { version = "1.0", features = ["full"] } +futures = "0.3" async-trait = "0.1" chrono = { version = "0.4", features = ["serde"] } tracing = "0.1" -tracing-subscriber = "0.3" [build-dependencies] -cloacina-build = { path = "../../../../crates/cloacina-build" } +cloacina-build = "0.10" diff --git a/examples/features/workflows/complex-dag/src/lib.rs b/examples/features/workflows/complex-dag/src/lib.rs index 7e85dd62e..a406cbedb 100644 --- a/examples/features/workflows/complex-dag/src/lib.rs +++ b/examples/features/workflows/complex-dag/src/lib.rs @@ -23,8 +23,8 @@ //! - Parallel execution paths //! - Complex branching and merging -use cloacina::{Context, TaskError}; use cloacina_macros::{task, workflow}; +use cloacina_workflow::{Context, TaskError}; // I-0102 / T-C: unified plugin shell. cloacina_workflow_plugin::package!(); diff --git a/examples/features/workflows/packaged-triggers/Cargo.toml b/examples/features/workflows/packaged-triggers/Cargo.toml index bd05e5e8a..2c2672890 100644 --- a/examples/features/workflows/packaged-triggers/Cargo.toml +++ b/examples/features/workflows/packaged-triggers/Cargo.toml @@ -14,10 +14,10 @@ packaged = [] crate-type = ["cdylib", "rlib"] [dependencies] -cloacina-macros = { path = "../../../../crates/cloacina-macros" } -cloacina-computation-graph = { path = "../../../../crates/cloacina-computation-graph" } -cloacina-workflow = { path = "../../../../crates/cloacina-workflow", features = ["packaged"] } -cloacina-workflow-plugin = { path = "../../../../crates/cloacina-workflow-plugin" } +cloacina-macros = "0.10" +cloacina-computation-graph = "0.10" +cloacina-workflow = { version = "0.10", features = ["packaged"] } +cloacina-workflow-plugin = "0.10" serde_json = "1.0" async-trait = "0.1" futures = "0.3" @@ -26,7 +26,7 @@ tokio = { version = "1.0", features = ["time"] } tracing = "0.1" [dev-dependencies] -cloacina = { path = "../../../../crates/cloacina", default-features = false, features = ["macros", "sqlite"] } +cloacina = { version = "0.10", default-features = false, features = ["macros", "sqlite"] } [build-dependencies] -cloacina-build = { path = "../../../../crates/cloacina-build" } +cloacina-build = "0.10" diff --git a/examples/features/workflows/packaged-triggers/src/lib.rs b/examples/features/workflows/packaged-triggers/src/lib.rs index 59c25e98b..6a24c2c86 100644 --- a/examples/features/workflows/packaged-triggers/src/lib.rs +++ b/examples/features/workflows/packaged-triggers/src/lib.rs @@ -74,11 +74,28 @@ cargo build --release ``` */ -use cloacina_workflow::{task, workflow, Context, TaskError}; +use cloacina_workflow::{task, trigger, workflow, Context, TaskError, TriggerResult}; // I-0102 / T-C: unified plugin shell. cloacina_workflow_plugin::package!(); +/// Poll trigger: fires `file_processing` on a short interval, injecting the +/// filename the workflow reads from context. A real trigger would watch a +/// directory / queue and `Fire` only when a new file arrives (returning `Skip` +/// otherwise); this demo fires every interval so executions appear +/// automatically — no `workflow run` needed. The `#[trigger]` macro projects +/// this into the host trigger registry via the plugin FFI at load time. +#[trigger(on = "file_processing", poll_interval = "3s")] +pub async fn inbox_poll() -> Result { + let mut ctx = Context::new(); + ctx.insert("filename", serde_json::json!("invoice-042.dat"))?; + ctx.insert( + "source_path", + serde_json::json!("/data/inbox/invoice-042.dat"), + )?; + Ok(TriggerResult::Fire(Some(ctx))) +} + /// File Processing Pipeline — triggered when new files arrive. /// /// This package demonstrates a workflow that is designed to be fired @@ -95,10 +112,7 @@ pub mod file_processing { /// Validate the incoming file. /// /// The trigger passes `filename` and `source_path` in the context. - #[task( - retry_attempts = 2, - retry_backoff = "linear" - )] + #[task(retry_attempts = 2, retry_backoff = "linear")] pub async fn validate(context: &mut Context) -> Result<(), TaskError> { let filename = context .get("filename") diff --git a/examples/features/workflows/packaged-workflows/Cargo.toml b/examples/features/workflows/packaged-workflows/Cargo.toml index 52fe51367..55f3cbfc6 100644 --- a/examples/features/workflows/packaged-workflows/Cargo.toml +++ b/examples/features/workflows/packaged-workflows/Cargo.toml @@ -13,10 +13,10 @@ packaged = [] crate-type = ["cdylib", "rlib"] [dependencies] -cloacina-macros = { path = "../../../../crates/cloacina-macros" } -cloacina-computation-graph = { path = "../../../../crates/cloacina-computation-graph" } -cloacina-workflow = { path = "../../../../crates/cloacina-workflow", features = ["packaged"] } -cloacina-workflow-plugin = { path = "../../../../crates/cloacina-workflow-plugin" } +cloacina-macros = "0.10" +cloacina-computation-graph = "0.10" +cloacina-workflow = { version = "0.10", features = ["packaged"] } +cloacina-workflow-plugin = "0.10" serde_json = "1.0" async-trait = "0.1" futures = "0.3" # Lightweight async executor for macro-generated code @@ -26,7 +26,7 @@ tokio = { version = "1.0", features = ["time"] } tracing = "0.1" [dev-dependencies] -cloacina = { path = "../../../../crates/cloacina", default-features = false, features = ["macros", "sqlite"] } +cloacina = { version = "0.10", default-features = false, features = ["macros", "sqlite"] } [build-dependencies] -cloacina-build = { path = "../../../../crates/cloacina-build" } +cloacina-build = "0.10" diff --git a/examples/features/workflows/parameterized-workflow/Cargo.toml b/examples/features/workflows/parameterized-workflow/Cargo.toml new file mode 100644 index 000000000..e026be248 --- /dev/null +++ b/examples/features/workflows/parameterized-workflow/Cargo.toml @@ -0,0 +1,29 @@ +[workspace] + +[package] +name = "parameterized-workflow-demo" +version = "0.1.0" +edition = "2021" + +[features] +default = ["packaged"] +packaged = [] + +[lib] +crate-type = ["cdylib", "rlib"] + +# Production-shaped: crates.io version deps (what `cloacinactl package new` +# emits). Dev stacks resolve these against the local workspace via the +# compiler's `--dev-workspace` flag; production compilers use crates.io. +[dependencies] +cloacina-macros = "0.10" +cloacina-computation-graph = "0.10" +cloacina-workflow = { version = "0.10", features = ["packaged"] } +cloacina-workflow-plugin = "0.10" +serde_json = "1.0" +futures = "0.3" +async-trait = "0.1" +tracing = "0.1" + +[build-dependencies] +cloacina-build = "0.10" diff --git a/examples/features/workflows/parameterized-workflow/README.md b/examples/features/workflows/parameterized-workflow/README.md new file mode 100644 index 000000000..04ca3a583 --- /dev/null +++ b/examples/features/workflows/parameterized-workflow/README.md @@ -0,0 +1,108 @@ +# Parameterized Workflow + +One workflow **template**, many differently-configured **runs**. This example +demonstrates `params(...)` — the declared, typed, configurable surface of a +workflow — through the primary interface: + +``` +declare params → pack → upload → compile → run with values → observe +``` + +The template is a file-sync pipeline: + +``` +plan_sync → execute_sync → report +``` + +with four declared params: + +| Param | Type | Default | +|---|---|---| +| `source` | `String` | *(required)* | +| `dst` | `String` | *(required)* | +| `mode` | `String` | `"copy"` | +| `max_files` | `i64` | `100` | + +## How params are declared + +```rust +#[workflow( + name = "sync_file", + params( + source: String, + dst: String, + mode: String = "copy", + max_files: i64 = 100, + ) +)] +pub mod sync_file { ... } +``` + +The compiler extracts this as the workflow's **typed input interface**; the +server validates every run's provided values against it — wrong types and +missing required params are rejected *before* anything executes. Bound values +arrive in tasks as flat top-level context keys (`context.get("source")`). + +## Run it + +The steps below are automated as `angreal demos features parameterized-workflow` +(the CI examples lane runs exactly that). + +### 1. Stack + CLI + +```bash +angreal ui up +cloacinactl config profile set demo http://localhost:8080 \ + --api-key clk_demo_public_key_0003 --tenant public --default +``` + +### 2. Pack + upload + +```bash +cloacinactl package pack . --out parameterized-workflow-demo.cloacina +cloacinactl package upload parameterized-workflow-demo.cloacina +cloacinactl package list # wait for build_status: success +``` + +### 3. Run it twice, with different values + +```bash +cat > prod.json <<'EOF' +{"source": "/data/prod", "dst": "/backup/prod"} +EOF +cloacinactl workflow run sync_file --context prod.json +``` + +```bash +cat > archive.json <<'EOF' +{"source": "/data/archive", "dst": "/cold", "mode": "move", "max_files": 10} +EOF +cloacinactl workflow run sync_file --context archive.json +``` + +Same template, two independent executions with different behavior — watch +both complete: + +```bash +cloacinactl execution list --workflow sync_file +``` + +### 4. See the validation reject a bad run + +```bash +echo '{"dst": "/backup"}' > bad.json # missing required `source` +cloacinactl workflow run sync_file --context bad.json +``` + +The server rejects it with a typed error before any task runs — that's the +declared interface doing its job. + +## About named instances + +`params(...)` also powers **workflow instances** — persistent, named, +scheduled bindings (`sync_prod` = this template + these values + this cron). +See [Workflow Instances](../../../../docs/content/engine/scheduling/workflow-instances.md). +Instance *registration* is currently an embedded-runner API +(`register_cron_workflow_instance`); a server-side registration surface is +tracked separately. Per-run parameterization — everything above — is fully +supported on the primary interface today. diff --git a/examples/features/workflows/parameterized-workflow/build.rs b/examples/features/workflows/parameterized-workflow/build.rs new file mode 100644 index 000000000..6e9cdfc6c --- /dev/null +++ b/examples/features/workflows/parameterized-workflow/build.rs @@ -0,0 +1,19 @@ +/* + * Copyright 2026 Colliery Software + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +fn main() { + cloacina_build::configure(); +} diff --git a/examples/features/workflows/parameterized-workflow/package.toml b/examples/features/workflows/parameterized-workflow/package.toml new file mode 100644 index 000000000..aed708523 --- /dev/null +++ b/examples/features/workflows/parameterized-workflow/package.toml @@ -0,0 +1,12 @@ +[package] +name = "parameterized-workflow-demo" +version = "0.1.0" +interface = "cloacina-workflow-plugin" +interface_version = 1 +extension = "cloacina" + +[metadata] +workflow_name = "sync_file" +language = "rust" +description = "Parameterized workflow — one template, many runs with different bound values" +author = "Cloacina Demo Team" diff --git a/examples/features/workflows/parameterized-workflow/src/lib.rs b/examples/features/workflows/parameterized-workflow/src/lib.rs new file mode 100644 index 000000000..ed09cf994 --- /dev/null +++ b/examples/features/workflows/parameterized-workflow/src/lib.rs @@ -0,0 +1,129 @@ +/* + * Copyright 2026 Colliery Software + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*! +# Parameterized Workflow + +One workflow template, many differently-configured runs. `params(...)` on +`#[workflow]` declares the configurable surface; the server validates every +run's provided values against it (types, required-ness, defaults) and +delivers the bound values to tasks as top-level context keys. + +See the README for the gold-path run recipe. +*/ + +use cloacina_workflow::{task, workflow, Context, TaskError}; + +cloacina_workflow_plugin::package!(); + +/// A file-sync template: where to read, where to write, how to transfer. +/// +/// `source` and `dst` are required (no default); `mode` and `max_files` +/// fall back to their defaults when a run doesn't bind them. +#[workflow( + name = "sync_file", + description = "Sync files from a source to a destination — parameterized template", + author = "Cloacina Demo Team", + params( + source: String, + dst: String, + mode: String = "copy", + max_files: i64 = 100, + ) +)] +pub mod sync_file { + use super::*; + + /// Read the bound params off the context and turn them into a plan. + /// Bound values arrive as flat top-level context keys. + #[task] + pub async fn plan_sync(context: &mut Context) -> Result<(), TaskError> { + let source = context + .get("source") + .and_then(|v| v.as_str().map(String::from)) + .ok_or_else(|| TaskError::ValidationFailed { + message: "missing bound param: source".to_string(), + })?; + let dst = context + .get("dst") + .and_then(|v| v.as_str().map(String::from)) + .ok_or_else(|| TaskError::ValidationFailed { + message: "missing bound param: dst".to_string(), + })?; + let mode = context + .get("mode") + .and_then(|v| v.as_str().map(String::from)) + .unwrap_or_else(|| "copy".to_string()); + let max_files = context + .get("max_files") + .and_then(|v| v.as_i64()) + .unwrap_or(100); + + if mode != "copy" && mode != "move" { + return Err(TaskError::ValidationFailed { + message: format!("mode must be 'copy' or 'move', got '{mode}'"), + }); + } + + println!("📋 plan: {mode} up to {max_files} files {source} → {dst}"); + context.insert( + "sync_plan", + serde_json::json!({ + "source": source, + "dst": dst, + "mode": mode, + "max_files": max_files, + }), + )?; + Ok(()) + } + + /// Simulate the transfer the plan describes. + #[task(dependencies = ["plan_sync"], retry_attempts = 2)] + pub async fn execute_sync(context: &mut Context) -> Result<(), TaskError> { + let plan = + context + .get("sync_plan") + .cloned() + .ok_or_else(|| TaskError::ValidationFailed { + message: "missing sync_plan".to_string(), + })?; + let max_files = plan.get("max_files").and_then(|v| v.as_i64()).unwrap_or(0); + // Simulate: "transfer" a deterministic number of files under the cap. + let transferred = max_files.min(42); + println!("🚚 transferred {transferred} file(s)"); + context.insert( + "sync_result", + serde_json::json!({ "transferred": transferred, "plan": plan }), + )?; + Ok(()) + } + + /// Report what this particular parameterization did. + #[task(dependencies = ["execute_sync"])] + pub async fn report(context: &mut Context) -> Result<(), TaskError> { + let result = + context + .get("sync_result") + .cloned() + .ok_or_else(|| TaskError::ValidationFailed { + message: "missing sync_result".to_string(), + })?; + println!("✅ sync complete: {result}"); + context.insert("sync_report", result)?; + Ok(()) + } +} diff --git a/examples/features/workflows/python-conditional/package.toml b/examples/features/workflows/python-conditional/package.toml new file mode 100644 index 000000000..248d5482c --- /dev/null +++ b/examples/features/workflows/python-conditional/package.toml @@ -0,0 +1,20 @@ +# PYTHON packaged CONDITIONAL-SKIP example (I-0138 / Python-gap sweep). Probes +# the Python `trigger_rules` → `Skipped` state on the server gold path — the +# parity closed in T-0763 was only ever exercised by the demo-stack fixture +# (demo-py-cron), never through compile → reconcile → execute. +# +# `audit` is gated by `context_value("do_audit", "Equals", True)`, but `gate` +# sets `do_audit = False`, so the rule never fires and the planner must Skip it. +# `audit` RAISES if it ever runs — so the only way the run reaches `Completed` +# is if the skip was honored. `record` fans in on the run path + the skipped +# path (a Skipped dep counts as resolved) and still runs. +# +# Minimal manifest: loader infers `language = "python"` + entry module from the +# `workflow//` layout. `audit-pipeline` → `workflow/audit_pipeline/`. +[package] +name = "audit-pipeline" +version = "0.1.0" + +[metadata] +workflow_name = "audit_pipeline" +description = "Python packaged trigger_rules — a gated task Skips; the run completes over the skipped dep" diff --git a/examples/features/workflows/python-conditional/workflow/audit_pipeline/__init__.py b/examples/features/workflows/python-conditional/workflow/audit_pipeline/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/examples/features/workflows/python-conditional/workflow/audit_pipeline/tasks.py b/examples/features/workflows/python-conditional/workflow/audit_pipeline/tasks.py new file mode 100644 index 000000000..b456d97e9 --- /dev/null +++ b/examples/features/workflows/python-conditional/workflow/audit_pipeline/tasks.py @@ -0,0 +1,73 @@ +"""Python packaged CONDITIONAL-SKIP workflow — proves `trigger_rules` gate a +Python task into the real `Skipped` state on the server gold path. + + gate ──┬─▶ process ─▶ record + └─▶ audit (SKIPPED) + +`gate` sets `do_audit = False`. `audit` is gated by +`context_value("do_audit", "Equals", True)`, so its rule never fires and the +planner must Skip it. `audit` RAISES if it is ever executed — so the ONLY way +the run reaches `Completed` is if the skip was honored (if `trigger_rules` were +ignored, `audit` would run, raise, and fail the execution). `record` fans in on +`process` + `audit`; a Skipped dependency counts as resolved, so `record` still +runs — and it asserts `process` actually ran, catching the opposite bug +(everything wrongly skipped). + +This is the Python mirror of the Rust trigger-rule gating in `demo-cron-rust`, +lifted onto the features gold path. +""" +from __future__ import annotations + +import cloaca + + +@cloaca.task(id="gate", dependencies=[]) +def gate(context): + """what: Decide the audit branch is OFF for this run. + + why: Gating on a context value the trigger rule reads is how cloacina models + conditional branches — no imperative if/else in the DAG. + """ + context.set("do_audit", False) + context.set("gated", True) + return context + + +@cloaca.task(id="process", dependencies=["gate"]) +def process(context): + """what: The always-on work path. why: gives `record` something to assert so + a run that wrongly skips everything is caught, not just one that under-skips.""" + context.set("records", 10) + return context + + +@cloaca.task( + id="audit", + dependencies=["gate"], + # Rule wants do_audit == True, which never holds → this task must Skip. + trigger_rules=cloaca.context_value("do_audit", "Equals", True), +) +def audit(context): + """what: The gated-off branch. why: it RAISES if executed, so a completed run + is proof the trigger rule skipped it rather than running it.""" + raise RuntimeError( + "audit ran even though its trigger_rules gate (do_audit == True) is False " + "— trigger_rules skipping was NOT honored" + ) + + +@cloaca.task(id="record", dependencies=["process", "audit"]) +def record(context): + """what: Fan-in terminal step over the run path + the skipped path. + + why: proves a Skipped dependency counts as resolved (the fan-in still fires) + AND that the run path executed (asserts `process`'s output is present). + """ + records = context.get("records") + if records != 10: + raise RuntimeError( + f"expected process to have run (records == 10), got {records!r} " + "— the run path was wrongly skipped" + ) + context.set("recorded", True) + return context diff --git a/examples/features/workflows/python-cron/package.toml b/examples/features/workflows/python-cron/package.toml new file mode 100644 index 000000000..d6055c41a --- /dev/null +++ b/examples/features/workflows/python-cron/package.toml @@ -0,0 +1,19 @@ +# PYTHON packaged CRON-TRIGGER example (I-0138 / Python-gap sweep). The cron +# peer of `python-triggers` (poll). Cron and poll fire through DIFFERENT server +# subsystems — cron is driven by the cron scheduler, poll by the runtime trigger +# registry — so this exercises a distinct code path. The Python cron surface +# (T-0688 added cron/timezone to `@cloaca.trigger`) was only ever run in the +# demo-stack fixture (demo-py-cron); this lifts it onto the features gold path. +# +# `@cloaca.trigger(on="heartbeat_workflow", cron="*/3 * * * * *")` fires the +# workflow every 3 seconds — executions appear automatically, no `workflow run`. +# +# Minimal manifest: loader infers `language = "python"` + entry module from the +# `workflow//` layout. `heartbeat` → `workflow/heartbeat/`. +[package] +name = "heartbeat" +version = "0.1.0" + +[metadata] +workflow_name = "heartbeat_workflow" +description = "Python packaged cron trigger — fires a workflow on a schedule" diff --git a/examples/features/workflows/python-cron/workflow/heartbeat/__init__.py b/examples/features/workflows/python-cron/workflow/heartbeat/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/examples/features/workflows/python-cron/workflow/heartbeat/tasks.py b/examples/features/workflows/python-cron/workflow/heartbeat/tasks.py new file mode 100644 index 000000000..1c1d1918f --- /dev/null +++ b/examples/features/workflows/python-cron/workflow/heartbeat/tasks.py @@ -0,0 +1,34 @@ +"""Python packaged CRON-TRIGGER workflow — fires on a schedule via the cron +scheduler (the distinct-subsystem peer of `python-triggers`, which polls). + +A `@cloaca.trigger(on="heartbeat_workflow", cron="*/3 * * * * *")` fires the +`heartbeat_workflow` every 3 seconds. Unlike a poll trigger, the cron scheduler +fires the `on` workflow directly on the schedule — the decorated function body +is unused (cron and poll_interval are mutually exclusive; `on` is required for +cron). Executions appear automatically; no `workflow run`. + + beat (records a heartbeat each scheduled fire) + +Tasks are declared with bare `@cloaca.task` decorators — the packaged loader +builds the workflow context from `workflow_name` before importing this module, +so they register on import (no `WorkflowBuilder`, that is for in-process runs). +""" +from __future__ import annotations + +import cloaca + + +@cloaca.task(id="beat", dependencies=[]) +def beat(context): + """what: Record a single heartbeat. why: a minimal terminal task so each + scheduled cron fire produces an observable Completed execution.""" + context.set("heartbeat", True) + return context + + +# Fire `heartbeat_workflow` every 3 seconds. The cron scheduler fires the `on` +# workflow directly, so this body is unused — its presence at import IS the +# declaration (there is no triggers section in package.toml). +@cloaca.trigger(on="heartbeat_workflow", cron="*/3 * * * * *") +def heartbeat_cron(): + pass diff --git a/examples/features/workflows/python-multi-tenant/package.toml b/examples/features/workflows/python-multi-tenant/package.toml new file mode 100644 index 000000000..463064404 --- /dev/null +++ b/examples/features/workflows/python-multi-tenant/package.toml @@ -0,0 +1,30 @@ +# PYTHON packaged MULTI-TENANCY example (I-0138 / Python-gap sweep). The tenant +# is Cloacina's isolation boundary: workflows and executions are tenant-scoped +# (`/v1/tenants/{tenant}/workflows`, `/executions`). This example demonstrates +# that boundary on the server gold path — the SAME package deployed into two +# tenants runs independently, and their executions never cross; a third tenant +# that never received the package cannot see the workflow at all. +# +# The Python peer of the Rust `multi-tenant` / `per-tenant-credentials` examples +# (which are embedded-only). The workflow itself is deliberately trivial — the +# feature under test is the tenancy boundary, driven by the lane (deploy into +# tenant A + tenant B, run in both, assert isolation), not the task logic. +# +# Operational model the lane demonstrates: each tenant runs its OWN compiler +# (`cloacina-compiler --tenant-schema `) — the default compiler only +# claims the public/admin schema, by design (per-tenant build isolation). So a +# second compiler is stood up for tenant B. Execution STATE is fully isolated +# (a tenant only sees its own executions). NOTE: workflow-DEFINITION visibility +# currently leaks across tenants via the process-shared runtime — a tenant can +# run a workflow name another tenant loaded even without installing it; tracked +# as CLOACI-T-0901 and surfaced (not asserted) by the lane until fixed. +# +# Minimal manifest: loader infers `language = "python"` + entry module from the +# `workflow//` layout. `tenant-job` → `workflow/tenant_job/`. +[package] +name = "tenant-job" +version = "0.1.0" + +[metadata] +workflow_name = "tenant_job" +description = "Python packaged multi-tenancy — same workflow in two tenants, executions isolated" diff --git a/examples/features/workflows/python-multi-tenant/workflow/tenant_job/__init__.py b/examples/features/workflows/python-multi-tenant/workflow/tenant_job/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/examples/features/workflows/python-multi-tenant/workflow/tenant_job/tasks.py b/examples/features/workflows/python-multi-tenant/workflow/tenant_job/tasks.py new file mode 100644 index 000000000..068cc3ae9 --- /dev/null +++ b/examples/features/workflows/python-multi-tenant/workflow/tenant_job/tasks.py @@ -0,0 +1,29 @@ +"""Python packaged MULTI-TENANCY workflow — a trivial job used to demonstrate +the tenant isolation boundary. + +The feature under test is not the task logic but the tenancy boundary: the same +package is deployed into two tenants and run in both, and the harness asserts +their executions never cross (and that a tenant without the package can't see +the workflow). See the package.toml and the `python-multi-tenant` lane. + + stamp -> finish +""" +from __future__ import annotations + +import cloaca + + +@cloaca.task(id="stamp", dependencies=[]) +def stamp(context): + """what: Record that the job ran. why: a minimal observable side effect so a + per-tenant execution reaches Completed and is countable in that tenant.""" + context.set("stamped", True) + return context + + +@cloaca.task(id="finish", dependencies=["stamp"]) +def finish(context): + """what: Terminal step. why: a two-task DAG keeps the example a real workflow + rather than a single node, matching the other packaged peers.""" + context.set("finished", True) + return context diff --git a/examples/features/workflows/python-packaged/README.md b/examples/features/workflows/python-packaged/README.md new file mode 100644 index 000000000..790358ce4 --- /dev/null +++ b/examples/features/workflows/python-packaged/README.md @@ -0,0 +1,91 @@ +# Python Packaged Workflow + +**The canonical Python packaged example** — the Python peer of +[`simple-packaged`](../simple-packaged). You author tasks in Python, pack the +source into a `.cloacina` archive, and hand it to a running server, which loads +and executes it. Same primary interface as Rust: + +``` +pack → upload → compile → reconcile → execute → observe +``` + +This example is a three-task pipeline: + +``` +collect_data → process_data → generate_report +``` + +## Layout + +| File | Role | +|---|---| +| `package.toml` | Package manifest — name, version, and the workflow it exposes (`data_pipeline`) | +| `workflow/data_pipeline/tasks.py` | The tasks: bare `@cloaca.task` decorators | +| `workflow/data_pipeline/__init__.py` | Marks the module a package (empty) | + +No `Cargo.toml`, no build script — a Python package carries source only; the +server's compiler skips cargo for `language = "python"` and the reconciler +imports the module via the embedded Python runtime. The manifest is minimal: +the loader infers `language = "python"` and the entry module from the +`workflow//` layout. This is exactly what `cloacinactl package new +--language python` scaffolds. + +## How tasks are authored + +```python +import cloaca + +@cloaca.task(id="collect_data", dependencies=[]) +def collect_data(context): + context.set("raw_records", 1000) + return context + +@cloaca.task(id="process_data", dependencies=["collect_data"]) +def process_data(context): + context.set("processed_records", context.get("raw_records")) + return context +``` + +Tasks take the execution `context`, read/write it with `context.get(...)` / +`context.set(...)`, and return it. Dependencies are declared per-task; the +engine derives the execution order. **Do not** wrap these in a +`WorkflowBuilder` — that's the in-process (embedded) form; a packaged workflow +registers its tasks on import and the loader builds the workflow from +`workflow_name`. + +## Run it + +Automated as `angreal demos features python-packaged` (the CI examples lane +runs exactly that). + +### 1. Stack + CLI + +```bash +angreal ui up +cloacinactl config profile set demo http://localhost:8080 \ + --api-key clk_demo_public_key_0003 --tenant public --default +``` + +### 2. Pack + upload + +```bash +cloacinactl package pack . --out data-pipeline.cloacina +cloacinactl package upload data-pipeline.cloacina +cloacinactl package list # wait for build_status: success +``` + +### 3. Execute + +```bash +cloacinactl workflow run data_pipeline +``` + +### 4. Observe + +```bash +cloacinactl execution list --workflow data_pipeline +cloacinactl execution status +``` + +Or watch it in the web UI at — executions, task states, +and the workflow DAG. diff --git a/examples/features/workflows/python-packaged/package.toml b/examples/features/workflows/python-packaged/package.toml new file mode 100644 index 000000000..ec5d65fc5 --- /dev/null +++ b/examples/features/workflows/python-packaged/package.toml @@ -0,0 +1,14 @@ +# Canonical PYTHON packaged workflow (CLOACI-T-0885) — the Python peer of the +# Rust `simple-packaged` example. Minimal manifest: the loader infers +# `language = "python"` and the entry module from the `workflow//` layout, +# so authors learn the real contract (this is what `cloacinactl package new +# --language python` emits). +# The package name (snake-cased) is the entry module: `data-pipeline` → +# `workflow/data_pipeline/`. workflow_name is the registered workflow. +[package] +name = "data-pipeline" +version = "0.1.0" + +[metadata] +workflow_name = "data_pipeline" +description = "Simple Python data pipeline, run through the server" diff --git a/examples/features/workflows/python-packaged/workflow/data_pipeline/__init__.py b/examples/features/workflows/python-packaged/workflow/data_pipeline/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/examples/features/workflows/python-packaged/workflow/data_pipeline/tasks.py b/examples/features/workflows/python-packaged/workflow/data_pipeline/tasks.py new file mode 100644 index 000000000..4129b4aa8 --- /dev/null +++ b/examples/features/workflows/python-packaged/workflow/data_pipeline/tasks.py @@ -0,0 +1,56 @@ +"""Canonical Python packaged workflow — a three-task data pipeline. + +The Python peer of the Rust `simple-packaged` example. Tasks are declared with +bare `@cloaca.task` decorators; the packaged loader builds the workflow context +from `workflow_name` (in package.toml) before importing this module, so tasks +register on import — do NOT wrap them in a `WorkflowBuilder` (that is for +in-process runs only). + + collect_data -> process_data -> generate_report +""" +from __future__ import annotations + +import cloaca + + +@cloaca.task(id="collect_data", dependencies=[]) +def collect_data(context): + """what: Gather the input batch and stage it on the context. + + why: Every downstream task keys off this data; making collection its own + observable step means a bad source fails here, not deep in processing. + """ + context.set("raw_records", 1000) + context.set("source", "demo_database") + return context + + +@cloaca.task(id="process_data", dependencies=["collect_data"]) +def process_data(context): + """what: Validate and transform the collected records. + + why: Separated from collection so a transform bug is attributable and the + step is independently retryable. + """ + raw = context.get("raw_records") or 0 + context.set("processed_records", raw) + context.set("valid", raw > 0) + return context + + +@cloaca.task(id="generate_report", dependencies=["process_data"]) +def generate_report(context): + """what: Summarize the processed batch into a report. + + why: The terminal step downstream consumers read — keeps reporting distinct + from the transformation that produced the numbers. + """ + context.set( + "report", + { + "records": context.get("processed_records"), + "source": context.get("source"), + "ok": bool(context.get("valid")), + }, + ) + return context diff --git a/examples/features/workflows/python-parameterized/README.md b/examples/features/workflows/python-parameterized/README.md new file mode 100644 index 000000000..e1ad9fb6a --- /dev/null +++ b/examples/features/workflows/python-parameterized/README.md @@ -0,0 +1,76 @@ +# Parameterized Python Workflow + +The Python peer of [`parameterized-workflow`](../parameterized-workflow): one +workflow **template**, many differently-configured **runs**, through the primary +interface. Declared params are typed, validated by the server, and bound +per-run. + +``` +plan_sync → execute_sync → report +``` + +| Param | Type | Default | +|---|---|---| +| `source` | `str` | *(required)* | +| `dst` | `str` | *(required)* | +| `mode` | `str` | `"copy"` | +| `max_files` | `int` | `100` | + +## How params are declared + +```python +@cloaca.workflow_params( + source=str, # required + dst=str, # required + mode=(str, "copy"), # optional, default "copy" + max_files=(int, 100), # optional, default 100 +) +@cloaca.task(id="plan_sync", dependencies=[]) +def plan_sync(context): + source = context.get("source") # bound values arrive as context keys + ... +``` + +A bare type is required; a `(type, default)` tuple is optional. The compiler +parses this from source into typed input slots; the server validates every +run's provided values against them and rejects wrong types / missing required +params **before** anything executes. + +## Run it + +Automated as `angreal demos features python-parameterized`. + +### 1. Stack + CLI + +```bash +angreal ui up +cloacinactl config profile set demo http://localhost:8080 \ + --api-key clk_demo_public_key_0003 --tenant public --default +``` + +### 2. Pack + upload + +```bash +cloacinactl package pack . --out python-parameterized.cloacina +cloacinactl package upload python-parameterized.cloacina +cloacinactl package list # wait for build_status: success +``` + +### 3. Run it with different values + +```bash +echo '{"source": "/data/prod", "dst": "/backup/prod"}' > prod.json +cloacinactl workflow run python_parameterized --context prod.json + +echo '{"source": "/data/archive", "dst": "/cold", "mode": "move", "max_files": 10}' > archive.json +cloacinactl workflow run python_parameterized --context archive.json + +cloacinactl execution list --workflow python_parameterized +``` + +### 4. Validation rejects a bad run + +```bash +echo '{"dst": "/backup"}' > bad.json # missing required `source` +cloacinactl workflow run python_parameterized --context bad.json # → 400 +``` diff --git a/examples/features/workflows/python-parameterized/package.toml b/examples/features/workflows/python-parameterized/package.toml new file mode 100644 index 000000000..28cacdb06 --- /dev/null +++ b/examples/features/workflows/python-parameterized/package.toml @@ -0,0 +1,12 @@ +# Parameterized PYTHON workflow (CLOACI-T-0885 family / T-0889 peer). Declares +# typed params with `@cloaca.workflow_params`; the compiler parses them from +# source into JSON-Schema-typed input slots and the server validates every run's +# values against them. Package name (snake-cased) is the entry module: +# `python-parameterized` → `workflow/python_parameterized/`. +[package] +name = "python-parameterized" +version = "0.1.0" + +[metadata] +workflow_name = "python_parameterized" +description = "Parameterized Python workflow — declared params, validated and bound per run" diff --git a/examples/features/workflows/python-parameterized/workflow/python_parameterized/__init__.py b/examples/features/workflows/python-parameterized/workflow/python_parameterized/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/examples/features/workflows/python-parameterized/workflow/python_parameterized/tasks.py b/examples/features/workflows/python-parameterized/workflow/python_parameterized/tasks.py new file mode 100644 index 000000000..11c15b88b --- /dev/null +++ b/examples/features/workflows/python-parameterized/workflow/python_parameterized/tasks.py @@ -0,0 +1,52 @@ +"""Parameterized Python workflow — one template, many differently-bound runs. + +`@cloaca.workflow_params(...)` declares the configurable surface (the Python +parity of Rust's `#[workflow(params(...))]`): a bare type is required, a +`(type, default)` tuple is optional with that default. The compiler parses this +into typed input slots; the server validates every run's `--context` values +against them and delivers the bound values as top-level context keys. + + plan_sync -> execute_sync -> report +""" +from __future__ import annotations + +import cloaca + + +@cloaca.workflow_params( + source=str, # required + dst=str, # required + mode=(str, "copy"), # optional, default "copy" + max_files=(int, 100), # optional, default 100 +) +@cloaca.task(id="plan_sync", dependencies=[]) +def plan_sync(context): + """what: Turn the bound params into a sync plan. + + why: Bound values arrive as top-level context keys; making the plan explicit + means a bad parameterization is visible here, not deep in the transfer. + """ + source = context.get("source") + dst = context.get("dst") + mode = context.get("mode") or "copy" + max_files = context.get("max_files") or 100 + if mode not in ("copy", "move"): + raise ValueError(f"mode must be 'copy' or 'move', got {mode!r}") + context.set("sync_plan", {"source": source, "dst": dst, "mode": mode, "max_files": max_files}) + return context + + +@cloaca.task(id="execute_sync", dependencies=["plan_sync"]) +def execute_sync(context): + """what: Simulate the transfer the plan describes.""" + plan = context.get("sync_plan") or {} + transferred = min(int(plan.get("max_files", 0)), 42) + context.set("sync_result", {"transferred": transferred, "plan": plan}) + return context + + +@cloaca.task(id="report", dependencies=["execute_sync"]) +def report(context): + """what: Report what this particular parameterization did.""" + context.set("sync_report", context.get("sync_result")) + return context diff --git a/examples/features/workflows/python-retries/package.toml b/examples/features/workflows/python-retries/package.toml new file mode 100644 index 000000000..3cae926ef --- /dev/null +++ b/examples/features/workflows/python-retries/package.toml @@ -0,0 +1,18 @@ +# PYTHON packaged RETRY-POLICY example (I-0138 / Python-gap sweep). The Python +# peer of the Rust `conditional-retries` example (which is embedded-only). A +# task that DELIBERATELY fails its first two attempts and only succeeds on the +# third — so the run can only reach `Completed` if the server actually honored +# `@cloaca.task(retry_attempts=…, retry_backoff=…)` and re-executed it. Proves +# the Python packaged retry path through the server gold path (compile → +# reconcile → execute → retry → Completed). +# +# Minimal manifest: the loader infers `language = "python"` and the entry module +# from the `workflow//` layout. Package name (snake-cased) is the entry +# module: `retry-pipeline` → `workflow/retry_pipeline/`. +[package] +name = "retry-pipeline" +version = "0.1.0" + +[metadata] +workflow_name = "retry_pipeline" +description = "Python packaged retry policy — a task that fails twice then succeeds on retry" diff --git a/examples/features/workflows/python-retries/workflow/retry_pipeline/__init__.py b/examples/features/workflows/python-retries/workflow/retry_pipeline/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/examples/features/workflows/python-retries/workflow/retry_pipeline/tasks.py b/examples/features/workflows/python-retries/workflow/retry_pipeline/tasks.py new file mode 100644 index 000000000..bcd248e2b --- /dev/null +++ b/examples/features/workflows/python-retries/workflow/retry_pipeline/tasks.py @@ -0,0 +1,59 @@ +"""Python packaged RETRY-POLICY workflow — the Python peer of the Rust +`conditional-retries` example. + +`fetch_unreliable` simulates a flaky external call: it RAISES on its first two +attempts and only succeeds on the third. The task carries +`@cloaca.task(retry_attempts=4, retry_backoff="fixed", retry_delay_ms=200)`, so +the ONLY way the run reaches `Completed` is if the server honored the retry +policy and re-executed the task. If retries were ignored, attempt 1's exception +would fail the whole execution. + + fetch_unreliable (fails x2, succeeds on 3) -> summarize + +The attempt counter is a module global. Packaged Python tasks run in-process in +the server, and the module is imported once, so the counter persists across the +retries of a single execution (this mirrors tutorial 04's `call_count` pattern). +A fresh gold-path lane starts a fresh server, so the counter starts at 0. +""" +from __future__ import annotations + +import cloaca + +# Attempt counter — persists across retries within one server process. +_attempts = {"fetch_unreliable": 0} + +# How many attempts must fail before success (attempts 1 and 2 fail; 3 succeeds). +_FAIL_UNTIL = 3 + + +@cloaca.task( + id="fetch_unreliable", + dependencies=[], + retry_attempts=4, + retry_backoff="fixed", + retry_delay_ms=200, +) +def fetch_unreliable(context): + """what: A flaky fetch that fails twice then succeeds. + + why: The deliberate early failures make the retry policy observable — the + run can only complete if the server actually re-ran this task. Without + retries honored, attempt 1's raise would fail the execution. + """ + _attempts["fetch_unreliable"] += 1 + n = _attempts["fetch_unreliable"] + if n < _FAIL_UNTIL: + # Transient failure — the retry policy should re-execute us. + raise RuntimeError(f"transient failure on attempt {n} (will retry)") + context.set("succeeded_on_attempt", n) + context.set("payload", {"id": "data_001", "ok": True}) + return context + + +@cloaca.task(id="summarize", dependencies=["fetch_unreliable"]) +def summarize(context): + """what: Record which attempt finally succeeded. why: a terminal step so a + completed run is observably distinct from one that died mid-retry.""" + attempt = context.get("succeeded_on_attempt") + context.set("summary", f"fetch succeeded on attempt {attempt}") + return context diff --git a/examples/features/workflows/python-secrets/README.md b/examples/features/workflows/python-secrets/README.md new file mode 100644 index 000000000..a01df4d39 --- /dev/null +++ b/examples/features/workflows/python-secrets/README.md @@ -0,0 +1,62 @@ +# Workflow Secrets (Python) + +The Python peer of [`workflow-secrets`](../workflow-secrets): a workflow +**declares** the secrets it needs, a run **binds** each to a tenant secret, and +the value is **resolved at execution** through a side channel — never written +into the durable context, history, or logs. + +``` +resolve_token → send_notification +``` + +## How secrets are declared and consumed + +```python +@cloaca.workflow_secrets("api_token") +@cloaca.workflow_params(channel=(str, "#ops")) +@cloaca.task(id="resolve_token", dependencies=[]) +def resolve_token(context): + token = context.secret_field("api_token", "token") # side channel + context.set("token_len", len(token)) # derived facts only + return context +``` + +`@cloaca.workflow_secrets(...)` surfaces each name as an **encrypted input +slot**. A run binds it with a reference — never a literal: + +```json +{ "channel": "#oncall", "api_token": { "$secret": "oncall_api" } } +``` + +The server routes the reference away from the plaintext context, resolves the +value from the tenant's encrypted store at execution, and hands it to +`context.secret(...)` / `context.secret_field(...)`. A **literal** value for a +secret slot is rejected before anything runs. + +## Run it + +Automated as `angreal demos features python-secrets`. The server needs a +secrets KEK (`CLOACINA_SECRET_KEK`); the demo stack ships one. + +```bash +angreal ui up +cloacinactl config profile set demo http://localhost:8080 \ + --api-key clk_demo_public_key_0003 --tenant public --default + +# create the tenant secret (value from a file — never an argv literal) +printf 's3cr3t-demo-token' > /tmp/token.txt +cloacinactl secret create oncall_api --field token=@/tmp/token.txt + +cloacinactl package pack . --out python-secrets.cloacina +cloacinactl package upload python-secrets.cloacina +cloacinactl package list # wait for build_status: success + +cat > bind.json <<'EOF' +{ "channel": "#oncall", "api_token": { "$secret": "oncall_api" } } +EOF +cloacinactl workflow run python_secrets --context bind.json +cloacinactl execution list --workflow python_secrets +``` + +Rotate and the next run sees the new value; `secret get` returns metadata only; +a literal binding (`{"api_token": "plaintext"}`) is rejected. diff --git a/examples/features/workflows/python-secrets/package.toml b/examples/features/workflows/python-secrets/package.toml new file mode 100644 index 000000000..8d68ce501 --- /dev/null +++ b/examples/features/workflows/python-secrets/package.toml @@ -0,0 +1,12 @@ +# Workflow secrets in PYTHON (CLOACI-T-0885 family / T-0890 peer). Declares +# required secrets with @cloaca.workflow_secrets; the compiler parses them into +# encrypted input slots and the server resolves a run's {"$secret": name} +# binding at execution, never persisting the value. Package name (snake-cased) +# is the entry module: python-secrets -> workflow/python_secrets/. +[package] +name = "python-secrets" +version = "0.1.0" + +[metadata] +workflow_name = "python_secrets" +description = "Workflow secrets in Python — declared, bound per run, resolved at execution" diff --git a/examples/features/workflows/python-secrets/workflow/python_secrets/__init__.py b/examples/features/workflows/python-secrets/workflow/python_secrets/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/examples/features/workflows/python-secrets/workflow/python_secrets/tasks.py b/examples/features/workflows/python-secrets/workflow/python_secrets/tasks.py new file mode 100644 index 000000000..77f0b990a --- /dev/null +++ b/examples/features/workflows/python-secrets/workflow/python_secrets/tasks.py @@ -0,0 +1,38 @@ +"""Workflow secrets in Python. + +`@cloaca.workflow_secrets("name", ...)` declares the secrets a workflow +requires (the Python parity of Rust's `#[workflow(secrets(...))]`). A run binds +each declared name to a concrete tenant secret with a `{"$secret": "..."}` +reference; the plaintext is resolved at execution through `context.secret(...)` +and is never written into the durable context. + + resolve_token -> send_notification +""" +from __future__ import annotations + +import cloaca + + +@cloaca.workflow_secrets("api_token") +@cloaca.workflow_params(channel=(str, "#ops")) +@cloaca.task(id="resolve_token", dependencies=[]) +def resolve_token(context): + """what: Resolve the bound secret and prove it WITHOUT leaking it. + + why: Only non-sensitive derived facts (a boolean + the token length) go back + into the durable context; the value stays out of history entirely. + """ + token = context.secret_field("api_token", "token") + context.set("token_resolved", True) + context.set("token_len", len(token)) + return context + + +@cloaca.task(id="send_notification", dependencies=["resolve_token"]) +def send_notification(context): + """what: 'Send' the notification to the configured channel.""" + if not context.get("token_resolved"): + raise ValueError("token was not resolved") + channel = context.get("channel") or "#ops" + context.set("notification", {"channel": channel, "sent": True}) + return context diff --git a/examples/features/workflows/python-triggers/package.toml b/examples/features/workflows/python-triggers/package.toml new file mode 100644 index 000000000..0bdf062cc --- /dev/null +++ b/examples/features/workflows/python-triggers/package.toml @@ -0,0 +1,19 @@ +# PYTHON packaged POLL-TRIGGER example (CLOACI-T-0900 follow-on / I-0138). +# The Python peer of the Rust `packaged-triggers` example: a pure-Python task +# workflow plus a `@cloaca.trigger(on=…, poll_interval=…)` that fires it +# automatically on an interval — no `workflow run`. Proves the Python +# packaged-trigger path end to end THROUGH THE SERVER GOLD PATH (compile → +# reconcile → host trigger registry → scheduled fire), not just the demo stack. +# +# Minimal manifest (like the canonical `python-packaged`): the loader infers +# `language = "python"` and the entry module from the `workflow//` layout. +# The package name (snake-cased) is the entry module: `file-inbox` → +# `workflow/file_inbox/`. There is NO triggers section — the `@cloaca.trigger` +# decorator running at import IS the declaration. +[package] +name = "file-inbox" +version = "0.1.0" + +[metadata] +workflow_name = "file_processing_py" +description = "Python packaged poll trigger — fires a workflow automatically as files arrive" diff --git a/examples/features/workflows/python-triggers/workflow/file_inbox/__init__.py b/examples/features/workflows/python-triggers/workflow/file_inbox/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/examples/features/workflows/python-triggers/workflow/file_inbox/tasks.py b/examples/features/workflows/python-triggers/workflow/file_inbox/tasks.py new file mode 100644 index 000000000..93182fdc0 --- /dev/null +++ b/examples/features/workflows/python-triggers/workflow/file_inbox/tasks.py @@ -0,0 +1,63 @@ +"""Python packaged POLL-TRIGGER workflow — the Python peer of the Rust +`packaged-triggers` example. + +A `@cloaca.trigger(on=…, poll_interval=…)` polls on an interval and fires the +`file_processing_py` workflow automatically, injecting the discovered filename +via context — no `workflow run`. Tasks are declared with bare `@cloaca.task` +decorators (the packaged loader builds the workflow context from `workflow_name` +before importing this module, so they register on import — do NOT wrap them in a +`WorkflowBuilder`, that is for in-process runs only). + + validate -> transform -> archive + +A real trigger would watch a directory / queue and `fire` only when a new file +arrives (returning `skip` otherwise). This demo fires every interval so +executions appear automatically — the point is to prove the packaged +Python-trigger path through the server gold path. +""" +from __future__ import annotations + +import cloaca + + +@cloaca.task(id="validate", dependencies=[]) +def validate(context): + """what: Validate the incoming file named by the trigger. + + why: The trigger injects `filename`/`source_path`; validating first means a + bad file fails here as its own observable step, not deep in transform. + """ + filename = context.get("filename") or "unknown" + context.set("validated", True) + context.set("validated_file", filename) + return context + + +@cloaca.task(id="transform", dependencies=["validate"]) +def transform(context): + """what: Transform the validated file. why: separated from validation so a + transform bug is attributable and the step is independently retryable.""" + context.set("records_processed", 1500) + return context + + +@cloaca.task(id="archive", dependencies=["transform"]) +def archive(context): + """what: Archive the processed file. why: a terminal step so a completed run + is observably distinct from one that died mid-transform.""" + context.set("archived", True) + return context + + +# Poll trigger: fires `file_processing_py` on a short interval, injecting the +# filename the workflow reads from context. `on` names the workflow in this same +# package; the decorator running at import IS the declaration (no triggers +# section in package.toml). The reconciler projects it into the host trigger +# registry, which drives the poll — executions appear automatically. +@cloaca.trigger(on="file_processing_py", poll_interval="3s") +def inbox_poll(): + ctx = cloaca.Context({ + "filename": "invoice-042.dat", + "source_path": "/data/inbox/invoice-042.dat", + }) + return cloaca.TriggerResult.fire(ctx) diff --git a/examples/features/workflows/workflow-secrets/Cargo.toml b/examples/features/workflows/workflow-secrets/Cargo.toml new file mode 100644 index 000000000..803b0c2a7 --- /dev/null +++ b/examples/features/workflows/workflow-secrets/Cargo.toml @@ -0,0 +1,29 @@ +[workspace] + +[package] +name = "workflow-secrets-demo" +version = "0.1.0" +edition = "2021" + +[features] +default = ["packaged"] +packaged = [] + +[lib] +crate-type = ["cdylib", "rlib"] + +# Production-shaped: crates.io version deps (what `cloacinactl package new` +# emits). Dev stacks resolve these against the local workspace via the +# compiler's `--dev-workspace` flag; production compilers use crates.io. +[dependencies] +cloacina-macros = "0.10" +cloacina-computation-graph = "0.10" +cloacina-workflow = { version = "0.10", features = ["packaged"] } +cloacina-workflow-plugin = "0.10" +serde_json = "1.0" +futures = "0.3" +async-trait = "0.1" +tracing = "0.1" + +[build-dependencies] +cloacina-build = "0.10" diff --git a/examples/features/workflows/workflow-secrets/README.md b/examples/features/workflows/workflow-secrets/README.md new file mode 100644 index 000000000..295a7ca6a --- /dev/null +++ b/examples/features/workflows/workflow-secrets/README.md @@ -0,0 +1,114 @@ +# Workflow Secrets + +A workflow **declares** the secrets it needs; a run **binds** each declared +name to a tenant-stored secret; the value is **resolved at execution** through +a side channel and is never written into the durable context, execution +history, or logs. All through the primary interface: + +``` +secret create → pack → upload → compile → run with a $secret binding → Completed +``` + +The workflow is a two-task notifier: + +``` +resolve_token → send_notification +``` + +## How secrets are declared and consumed + +```rust +#[workflow( + name = "notify_oncall", + params( channel: String = "#ops" ), + secrets( api_token ) +)] +pub mod notify_oncall { + #[task] + pub async fn resolve_token(ctx: &mut Context) -> Result<(), TaskError> { + // Side channel — the plaintext is RETURNED to the task, never stored. + let token = ctx.secret_field("api_token", "token").await?; + ctx.insert("token_len", json!(token.len()))?; // derived facts only + Ok(()) + } +} +``` + +`secrets(api_token)` surfaces an **encrypted input slot** in the workflow's +typed interface. A run binds it with a reference — never a literal: + +```json +{ "channel": "#oncall", "api_token": { "$secret": "oncall_api" } } +``` + +The server routes the reference away from the plaintext context (names only), +resolves the value from the tenant's encrypted secret store at dispatch, and +delivers it wrapped to the executing agent. A **literal** value for a secret +slot is rejected before anything runs. + +## Run it + +Automated as `angreal demos features workflow-secrets` (the CI examples lane +runs exactly that). By hand, against the demo stack: + +### 1. Stack + CLI + +```bash +angreal ui up +cloacinactl config profile set demo http://localhost:8080 \ + --api-key clk_demo_public_key_0003 --tenant public --default +``` + +The demo stack ships with a fixed demo `CLOACINA_SECRET_KEK`. Without a KEK +the server *refuses to dispatch* secret-using tasks — fail-closed, never a +plaintext fallback. + +### 2. Create the tenant secret + +Values come from a file (`k=@path`), stdin (`k=-`), or a prompt — never an +argv literal (which would land in shell history): + +```bash +printf 's3cr3t-demo-token-value' > /tmp/token.txt +cloacinactl secret create oncall_api --field token=@/tmp/token.txt +``` + +### 3. Pack + upload + +```bash +cloacinactl package pack . --out workflow-secrets-demo.cloacina +cloacinactl package upload workflow-secrets-demo.cloacina +cloacinactl package list # wait for build_status: success +``` + +### 4. Run with the binding + +```bash +cat > bind.json <<'EOF' +{ "channel": "#oncall", "api_token": { "$secret": "oncall_api" } } +EOF +cloacinactl workflow run notify_oncall --context bind.json +cloacinactl execution list --workflow notify_oncall +``` + +### 5. Rotate — the next run sees the new value + +```bash +printf 'r0tated-token-value' > /tmp/token.txt +cloacinactl secret rotate oncall_api --field token=@/tmp/token.txt +cloacinactl workflow run notify_oncall --context bind.json +``` + +### 6. Values are write-only + +```bash +cloacinactl secret get oncall_api # metadata only — never the value +cloacinactl secret list +``` + +And try binding a literal — it's rejected before execution: + +```bash +echo '{"api_token": "plaintext-token"}' > bad.json +cloacinactl workflow run notify_oncall --context bad.json # → 400 +``` diff --git a/examples/features/workflows/workflow-secrets/build.rs b/examples/features/workflows/workflow-secrets/build.rs new file mode 100644 index 000000000..6e9cdfc6c --- /dev/null +++ b/examples/features/workflows/workflow-secrets/build.rs @@ -0,0 +1,19 @@ +/* + * Copyright 2026 Colliery Software + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +fn main() { + cloacina_build::configure(); +} diff --git a/examples/features/workflows/workflow-secrets/package.toml b/examples/features/workflows/workflow-secrets/package.toml new file mode 100644 index 000000000..cfafb695d --- /dev/null +++ b/examples/features/workflows/workflow-secrets/package.toml @@ -0,0 +1,12 @@ +[package] +name = "workflow-secrets-demo" +version = "0.1.0" +interface = "cloacina-workflow-plugin" +interface_version = 1 +extension = "cloacina" + +[metadata] +workflow_name = "notify_oncall" +language = "rust" +description = "Workflow secrets — declared, tenant-stored, resolved at execution, never persisted" +author = "Cloacina Demo Team" diff --git a/examples/features/workflows/workflow-secrets/src/lib.rs b/examples/features/workflows/workflow-secrets/src/lib.rs new file mode 100644 index 000000000..528943444 --- /dev/null +++ b/examples/features/workflows/workflow-secrets/src/lib.rs @@ -0,0 +1,97 @@ +/* + * Copyright 2026 Colliery Software + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*! +# Workflow Secrets + +A workflow declares the secrets it requires with `secrets(...)`; a run binds +each declared name to a concrete tenant secret with a `{"$secret": "name"}` +reference. The plaintext is resolved at execution through a side channel +(`context.secret(...)`) and is **never** written into the durable context — +tasks may only persist non-sensitive derived facts. + +See the README for the gold-path run recipe. +*/ + +use cloacina_workflow::{task, workflow, Context, TaskError}; + +cloacina_workflow_plugin::package!(); + +/// Notify the on-call channel using an API token stored as a tenant secret. +/// +/// `api_token` is a declared secret: runs must bind it to a real tenant +/// secret; the value never appears in the context, execution history, or +/// logs. +#[workflow( + name = "notify_oncall", + description = "Send an on-call notification using a tenant-stored API token", + author = "Cloacina Demo Team", + params( + channel: String = "#ops", + ), + secrets(api_token) +)] +pub mod notify_oncall { + use super::*; + + /// Resolve the bound secret and prove it WITHOUT leaking it: only + /// non-sensitive derived facts (a boolean + the token length) are + /// written back into the durable context. + #[task] + pub async fn resolve_token(context: &mut Context) -> Result<(), TaskError> { + let token = context + .secret_field("api_token", "token") + .await + .map_err(|e| TaskError::Unknown { + task_id: "resolve_token".to_string(), + message: format!("secret resolution failed: {e}"), + })?; + + println!( + "🔑 token resolved ({} bytes) — value stays out of the context", + token.len() + ); + context.insert("token_resolved", serde_json::json!(true))?; + context.insert("token_len", serde_json::json!(token.len()))?; + Ok(()) + } + + /// "Send" the notification to the configured channel. + #[task(dependencies = ["resolve_token"], retry_attempts = 2)] + pub async fn send_notification( + context: &mut Context, + ) -> Result<(), TaskError> { + let channel = context + .get("channel") + .and_then(|v| v.as_str().map(String::from)) + .unwrap_or_else(|| "#ops".to_string()); + let resolved = context + .get("token_resolved") + .and_then(|v| v.as_bool()) + .unwrap_or(false); + if !resolved { + return Err(TaskError::ValidationFailed { + message: "token was not resolved".to_string(), + }); + } + println!("📣 notification sent to {channel} (authenticated with the resolved token)"); + context.insert( + "notification", + serde_json::json!({ "channel": channel, "sent": true }), + )?; + Ok(()) + } +}