Skip to content
100 changes: 95 additions & 5 deletions cli/remote_provider.py
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,68 @@ def _reject_api_conflicts(args: argparse.Namespace) -> None:
)


class _TaskServing(NamedTuple):
"""What this join decided about task serving, and what the operator must be told (issue 58)."""

#: The operator opted in. False means they did not, and nothing here applies.
requested: bool
#: Why this provider cannot run a task, in the words of whichever check found out.
problem: str | None

@property
def allowed(self) -> bool:
return self.requested and self.problem is None


def _decide_task_serving() -> _TaskServing:
"""Ask, in the parent, whether the child about to be spawned could actually run a task.

Every one of these answers exists already — and until this ran, `run_task` was the only place
that asked, which is *after a member is waiting*. So `grid join` printed "serving",
`grid project status` said online, and the provider looked healthy right up until somebody
else's task died on it. The fail-closed work of issues 22 and 23 made these failures loud; they
were loud at the wrong moment and to the wrong person.

⚠️ **The refusal cannot be printed by the serve child.** Measured: `_spawn_remote_engine`
detaches it with BOTH stdout and stderr redirected into the engine log, and this process then
waits 3s only to tell "alive" from "died" — tailing that log only when it died. Task serving
must not kill the child, so anything it printed would land in a file nobody has a reason to
open. The parent is the only place the sentence reaches the person who typed the command.

Costs nothing when the opt-in is off, which is the default: no `claude --version`, no probe of a
root nobody configured.

`(Exception, SystemExit)` because these checks use both — `task_sandbox` raises `SystemExit` as
a clean-error idiom, and a *bug* in any of them must degrade to "task serving off, inference
fine" rather than taking down a provider that was only asked to serve inference.
"""
from remote import task_opt_in

if not task_opt_in.serving_enabled():
return _TaskServing(requested=False, problem=None)
from remote import task_agent

try:
task_agent.preflight_before_serving()
except (Exception, SystemExit) as exc: # noqa: BLE001 — inference must survive any of them
return _TaskServing(requested=True, problem=str(exc) or exc.__class__.__name__)
return _TaskServing(requested=True, problem=None)


def _task_serving_override(decision: _TaskServing) -> dict[str, str] | None:
"""What to change in the serve child's environment, or `None` to hand it over untouched.

Only ever an OFF. Turning the opt-in *on* for somebody is the one thing this feature may not do
— a task loop spends the operator's own agent subscription — so a join that decided task serving
is fine passes the operator's own environment through rather than asserting it.
"""
from remote import task_opt_in

if decision.requested and not decision.allowed:
return {task_opt_in.SERVING_ENV: "0"}
return None


def cmd_remote_join(args: argparse.Namespace) -> int:
from remote import credentials

Expand Down Expand Up @@ -203,6 +265,21 @@ def cmd_remote_join(args: argparse.Namespace) -> int:
engine_id = _REMOTE_IDENTITY
meta_name = getattr(args, "name", None) or socket.gethostname()

# ⚠️ **Before the lock, and before anything is stopped.** A provider that was serving inference
# a second ago must still be serving it after a task check that failed — so this may not run
# from inside the block that has already terminated the prior child on its way to finding out.
# It is also the last point where nothing has been written: a pure read, then the mutation.
task_serving = _decide_task_serving()
if task_serving.problem:
print(
f"Task serving is off for this join: {task_serving.problem}\n"
f"Inference is unaffected. Fix that and re-run `grid join --respawn` to claim tasks.",
file=sys.stderr,
)
# Withheld from the CHILD, never from this process: the opt-in travels in the environment the
# parent hands over, and the child reads it once at startup.
engine_env = _task_serving_override(task_serving)

# Remote has ONE identity per grid (the token pins the relay node_id), so `grid join` is additive:
# merge this join's engines into whatever is already serving, then respawn the single detached engine.
# The read-merge-write is serialized so two concurrent joins can't lost-update the union (ADR 0010).
Expand Down Expand Up @@ -302,7 +379,8 @@ def cmd_remote_join(args: argparse.Namespace) -> int:
if reloaded:
reloaded = _hot_reload_identity(network_id, record, live) # False if it fell back to a respawn
else:
_respawn_identity(network_id, record, live) # stops prior process(es) then respawns; aborts on failure
# stops prior process(es) then respawns; aborts on failure
_respawn_identity(network_id, record, live, env_overrides=engine_env)

appended = bool(live)
verb = "Appended to" if appended else "Joining"
Expand All @@ -315,6 +393,8 @@ def cmd_remote_join(args: argparse.Namespace) -> int:
print(f"models={','.join(record['models'])}")
if media: # the comfyui:* models are resolved from bundle gating at serve time, not here
print("media=on (serving comfyui:* workflows via the relay)")
if task_serving.allowed: # said only when it is true — the refusal has already gone to stderr
print("tasks=on (claiming tasks for this grid)")
print(f"log={paths.engines_dir(network_id) / f'{engine_id}.log'}")
if reloaded: # the live process re-advertised in place — nothing restarted, nothing dropped
print(f"(hot-reloaded — no in-flight requests dropped; stop with `grid leave {label}`)")
Expand Down Expand Up @@ -997,7 +1077,8 @@ def _hot_reload_identity(


def _respawn_identity(
network_id: str, record: dict[str, object], priors: list[dict[str, object]]
network_id: str, record: dict[str, object], priors: list[dict[str, object]],
*, env_overrides: dict[str, str] | None = None,
) -> None:
"""Stop the prior process(es), then write ``record`` and (re)spawn the one detached engine, setting
``record["pid"]``. Shared by join-append and leave-shrink (respawn is Slice 1's update mechanism).
Expand Down Expand Up @@ -1041,7 +1122,7 @@ def _respawn_identity(
service_truth.clear_service_truth(record)
record["started_at"] = runtime.utc_now()
run_records.write_record(network_id, engine_id, record)
proc = _spawn_remote_engine(network_id, engine_id)
proc = _spawn_remote_engine(network_id, engine_id, env_overrides=env_overrides)
# Stamp the identity, not just the pid: this is the ONLY stamp a `grid leave` racing this join
# can see, because the child's own self-stamp has not run yet (POSIX `flock` gives the lock to
# whoever asks, in no order). With it, that leave can verify the pid it is about to signal and —
Expand Down Expand Up @@ -1223,7 +1304,16 @@ def _build_record(
}


def _spawn_remote_engine(network_id: str, engine_id: str) -> subprocess.Popen:
def _spawn_remote_engine(
network_id: str, engine_id: str, *, env_overrides: dict[str, str] | None = None
) -> subprocess.Popen:
"""Start the detached serve child. ``env_overrides`` wins over this process's own environment.

The child reads its opt-ins from the environment it inherits, so an override is how the parent
withholds one it has just decided this provider cannot honour (issue 58). A new dict rather than
a mutation of `os.environ`: this process may still have work to do, and a knob turned off for a
child must not be turned off for its parent.
"""
from local import runtime

log_path = paths.engines_dir(network_id) / f"{engine_id}.log"
Expand All @@ -1237,7 +1327,7 @@ def _spawn_remote_engine(network_id: str, engine_id: str) -> subprocess.Popen:
stdout=log,
stderr=subprocess.STDOUT,
start_new_session=True,
env={**os.environ, "PYTHONUNBUFFERED": "1"},
env={**os.environ, "PYTHONUNBUFFERED": "1", **(env_overrides or {})},
)


Expand Down
57 changes: 57 additions & 0 deletions cli/remote_task.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import base64
import datetime
import json
import shlex
import sys
from pathlib import Path
from typing import Any
Expand Down Expand Up @@ -1844,8 +1845,64 @@ def _task_get(args: argparse.Namespace) -> int:
print(f"\nThe grid could not combine this work with a colleague's change. Your work is "
f"safe in this conversation — ask for it again with:\n"
f" grid task send {conversation or '<conversation-id>'} --prompt '<what to do>'")
for line in _changed_since_note(task):
print(line)
result = task.get("result_text")
if result:
print("\n--- result ---")
print(result.rstrip("\n"))
return code


# How many of the relay's sample this prints. The relay already caps what it sends
# (`turn_superseded.MAX_PATHS`); this is the second, smaller bound on what a person reads in a
# terminal, and the count beside it is what stays true either way.
_CHANGED_SINCE_NAMED = 5


def _changed_since_note(task: dict) -> list[str]:
"""Say when the project no longer holds what this turn changed (ND-16/F-3). Lines, or none.

⚠️ **The failure this closes is a turn that reads like a success everywhere.** Under ADR 0034
D-d the relay applies every finished turn itself, and D-g hands a collision to a turn in the
OTHER conversation — whose decision to drop somebody's work is recorded where that somebody
cannot read it, because `grid task list --project` returns only the asker's own turns. Measured
live 2026-08-20: `state=completed`, `grid task diff` showing their own change, and the project
holding somebody else's line.

⚠️ **BOTH keys, and both type-checked, or this says nothing.** They arrive together from a relay
that can answer at all, and absence is what every relay deployed before this feature sends — so a
missing or unreadable pair must read as *nothing to show*, never as a warning made up here. A
count of `0` is the relay's positive answer that the work still stands, and it is the ordinary
case: a note that appeared on every task would be furniture within a day.

⚠️ `bool` is refused explicitly. It is a subclass of `int`, so `True` would otherwise print
*1 of the files …* for a relay that sent a flag where a count belongs.
"""
count = task.get("changed_since_count")
paths = task.get("changed_since_paths")
if isinstance(count, bool) or not isinstance(count, int) or count <= 0:
return []
if not isinstance(paths, list):
return []
named = [path for path in paths[:_CHANGED_SINCE_NAMED] if isinstance(path, str) and path]
if not named:
return []
files = "file" if count == 1 else "files"
lines = ["",
f"{count} {files} this task changed {'has' if count == 1 else 'have'} been changed "
f"again since it ran:"]
lines += [f" {path}" for path in named]
if count > len(named):
lines.append(f" … and {count - len(named)} more")
# The project id comes off the task rather than from the caller's arguments: `grid task get`
# takes only a task id, and a person who has to go and find the project id first is a person who
# reads the warning and does nothing about it.
project = task.get("project_id")
lines.append("What the project holds now may not be what this result describes. Read what is "
"there with:")
# `shlex.quote` leaves an ordinary path exactly as it is and wraps one with a space in it. A
# command somebody is told to run has to be a command they can paste, and a repository that came
# in through `POST …/import` can hold any name its author gave it.
lines.append(f" grid project file {project or '<project-id>'} {shlex.quote(named[0])}")
return lines
Original file line number Diff line number Diff line change
Expand Up @@ -427,6 +427,36 @@ allowlist with no domain constraint, so this rule is **wider** than the change r
reaches every non-private project, including imported company repositories. That is not the supported
topology, and `visibility` is the only instrument for it. Named here rather than discovered.

⚠️ **CORRECTED — measured 2026-08-23 (ND-21). "Grid *is* domain, structurally" is true of the DOMAIN
and false of the ROSTER, and it is false on the supported topology.** grid-apis
`store.member_for_access` checks an allowlist row **before** anything else, on every network type,
and grid-src `grid_auth._requires_allowlist` names three ways onto a `private-domain` grid — the
right domain, an allowlist row, or owning the grid — carrying the sentence *"an invited account from
another domain keeps working after the switch"*. An invited outsider is therefore **authenticated**,
so D-k gives them every non-private project on the grid, imported company repositories included,
plus a `project_members` row minted on their first read that needs a key.

**The relay does not compare domains and cannot.** `grid_auth.GridAuthContext` carries `email`;
`relay.AuthContext`, the object a route is handed, does not. The premise is enforced entirely at the
control plane, and the rule there **admits** rather than restricts.

**Decided 2026-08-23: correct the claim, do not narrow the gate.** D-k's premise reads *authenticated
on this grid ⇒ a colleague **or somebody a colleague invited***, and the blast radius above is
accepted with it. Three reasons, in order of weight: the alternative asks the control plane for a new
value saying WHY a caller was admitted (domain vs allowlist vs owner), which is a lockstep value and
a rollout order for a boundary that is already live; narrowing it would revoke access from invited
accounts that work today, which grid-apis explicitly promises will keep working; and `visibility`
already exists as the instrument for a project that must not be grid-wide. What is **not** accepted
is the justification standing while being false — that is what this correction removes.

⚖️ In fairness to the shape being accepted: the minted row is **visible** afterwards in
`grid project member list`, so an owner can see who reached the project. Nobody is notified, and the
access is granted before anyone looks.

Pinned by grid-src `test_project_visibility.TestAnInvitedOutsiderIsAColleagueToThisRule` — a
characterization test, so a future narrowing announces itself here rather than in a support ticket.
Narrowing is tracked as follow-up work, not as a defect against this decision.

⚠️ **`TASK_SERVED_DOMAINS` is untouched.** It is 0033 issue 24's *a provider serves only its own
company's domain* gate — a different axis with a different failure mode. This is the third thing in
the codebase called "domain".
Expand Down
28 changes: 28 additions & 0 deletions docs/cli.md
Original file line number Diff line number Diff line change
Expand Up @@ -1626,6 +1626,25 @@ can `get`, `list` and `follow` any task in it. That is deliberate — a member c
project and read any task branch — and it is what makes reviewing a colleague's run possible without
one.

`get` also says when the project **no longer holds** what a task changed:

```
2 files this task changed have been changed again since it ran:
shared.txt
notes.md
What the project holds now may not be what this result describes. Read what is there with:
grid project file 4f0e… shared.txt
```

The grid applies every finished task to the project by itself, so a colleague's later work can land
over yours — and when two tasks collide, the step that combines them runs in **their** conversation,
where you never see it. Without this line your task reads `completed` on every surface you have,
including `grid task diff`, which shows what your task changed rather than what the project holds
now. The line appears only when something really has changed, and it counts a later task of your own
the same as anybody else's: what it means is *look before you rely on this*. `--json` carries the
same fact as `changed_since_count` and `changed_since_paths`; a grid whose relay predates this sends
neither key and the line simply does not appear.

### Acting on the outcome from a script

Both `get` and `follow` exit with the task's own outcome, so a shell can branch on it without
Expand Down Expand Up @@ -2167,6 +2186,15 @@ a 24.04 host that a task can then read its workspace, install a dependency from
read a file outside the workspace. **Do not** task that sysctl off to "fix" a task that fails to run
commands; check `bubblewrap` and `socat` first.

**`grid join` checks all of this before it serves.** With `GRID_TASKS` on, the join asks the same
questions a task would — is Claude Code installed and new enough, is the permission mode and
passthrough list valid, can the sandbox start, and can this account create a workspace under
`GRID_TASK_ROOT` — and it asks them *in the terminal you typed the command in*. A provider that
fails any of them still **serves inference**: the join lands, the answer says what to change, and
only task serving is withheld until you fix it and re-run `grid join --respawn`. Before this, every
one of those was checked only after a task had been claimed, so the first person to find out was a
member of your grid whose task died.

The environment variables that tune a provider, all optional:

| variable | default | what it does |
Expand Down
Loading
Loading