diff --git a/bin/lib/onboard-session.js b/bin/lib/onboard-session.js index 819790173..3327bd39f 100644 --- a/bin/lib/onboard-session.js +++ b/bin/lib/onboard-session.js @@ -34,6 +34,13 @@ function defaultSteps() { }; } +function buildSessionMetadata(meta = {}) { + return { + gatewayName: meta.gatewayName || "nemoclaw", + fromDockerfile: meta.fromDockerfile || null, + }; +} + function createSession(overrides = {}) { const now = new Date().toISOString(); return { @@ -55,9 +62,7 @@ function createSession(overrides = {}) { preferredInferenceApi: overrides.preferredInferenceApi || null, nimContainer: overrides.nimContainer || null, policyPresets: Array.isArray(overrides.policyPresets) ? overrides.policyPresets.filter((value) => typeof value === "string") : null, - metadata: { - gatewayName: overrides.metadata?.gatewayName || "nemoclaw", - }, + metadata: buildSessionMetadata(overrides.metadata), steps: { ...defaultSteps(), ...(overrides.steps || {}), @@ -368,6 +373,7 @@ function filterSafeUpdates(updates) { if (isObject(updates.metadata) && typeof updates.metadata.gatewayName === "string") { safe.metadata = { gatewayName: updates.metadata.gatewayName, + fromDockerfile: updates.metadata.fromDockerfile || null, }; } return safe; diff --git a/bin/lib/onboard.js b/bin/lib/onboard.js index 53069e339..37f1a5088 100644 --- a/bin/lib/onboard.js +++ b/bin/lib/onboard.js @@ -1324,6 +1324,16 @@ function getResumeConfigConflicts(session, opts = {}) { }); } + const requestedFrom = opts.fromDockerfile ? path.resolve(opts.fromDockerfile) : null; + const recordedFrom = session?.metadata?.fromDockerfile ? path.resolve(session.metadata.fromDockerfile) : null; + if (requestedFrom !== recordedFrom) { + conflicts.push({ + field: "fromDockerfile", + requested: requestedFrom, + recorded: recordedFrom, + }); + } + return conflicts; } @@ -1729,7 +1739,7 @@ async function promptValidatedSandboxName() { } // eslint-disable-next-line complexity -async function createSandbox(gpu, model, provider, preferredInferenceApi = null, sandboxNameOverride = null) { +async function createSandbox(gpu, model, provider, preferredInferenceApi = null, sandboxNameOverride = null, fromDockerfile = null) { step(5, 7, "Creating sandbox"); const sandboxName = sandboxNameOverride || (await promptValidatedSandboxName()); @@ -1763,10 +1773,27 @@ async function createSandbox(gpu, model, provider, preferredInferenceApi = null, // Stage build context const buildCtx = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-build-")); const stagedDockerfile = path.join(buildCtx, "Dockerfile"); - fs.copyFileSync(path.join(ROOT, "Dockerfile"), stagedDockerfile); - copyBuildContextDir(path.join(ROOT, "nemoclaw"), path.join(buildCtx, "nemoclaw")); - copyBuildContextDir(path.join(ROOT, "nemoclaw-blueprint"), path.join(buildCtx, "nemoclaw-blueprint")); - copyBuildContextDir(path.join(ROOT, "scripts"), path.join(buildCtx, "scripts")); + if (fromDockerfile) { + const fromResolved = path.resolve(fromDockerfile); + if (!fs.existsSync(fromResolved)) { + console.error(` Custom Dockerfile not found: ${fromResolved}`); + process.exit(1); + } + // Copy the entire parent directory as build context. copyBuildContextDir + // already filters out node_modules, .git, .venv, __pycache__, etc. + copyBuildContextDir(path.dirname(fromResolved), buildCtx); + // If the caller pointed at a file not named "Dockerfile", copy it to the + // location openshell expects (buildCtx/Dockerfile). + if (path.basename(fromResolved) !== "Dockerfile") { + fs.copyFileSync(fromResolved, stagedDockerfile); + } + console.log(` Using custom Dockerfile: ${fromResolved}`); + } else { + fs.copyFileSync(path.join(ROOT, "Dockerfile"), stagedDockerfile); + copyBuildContextDir(path.join(ROOT, "nemoclaw"), path.join(buildCtx, "nemoclaw")); + copyBuildContextDir(path.join(ROOT, "nemoclaw-blueprint"), path.join(buildCtx, "nemoclaw-blueprint")); + copyBuildContextDir(path.join(ROOT, "scripts"), path.join(buildCtx, "scripts")); + } // Create sandbox (use -- echo to avoid dropping into interactive shell) // Pass the base policy so sandbox starts in proxy mode (required for policy updates later) @@ -2777,8 +2804,13 @@ async function onboard(opts = {}) { NON_INTERACTIVE = opts.nonInteractive || process.env.NEMOCLAW_NON_INTERACTIVE === "1"; delete process.env.OPENSHELL_GATEWAY; const resume = opts.resume === true; + // In non-interactive mode also accept the env var so CI pipelines can set it. + // This is the explicitly requested value; on resume it may be absent and the + // session-recorded path is used instead (see below). + const requestedFromDockerfile = + opts.fromDockerfile || (isNonInteractive() ? (process.env.NEMOCLAW_FROM_DOCKERFILE || null) : null); const lockResult = onboardSession.acquireOnboardLock( - `nemoclaw onboard${resume ? " --resume" : ""}${isNonInteractive() ? " --non-interactive" : ""}` + `nemoclaw onboard${resume ? " --resume" : ""}${isNonInteractive() ? " --non-interactive" : ""}${requestedFromDockerfile ? ` --from ${requestedFromDockerfile}` : ""}` ); if (!lockResult.acquired) { console.error(" Another NemoClaw onboarding run is already in progress."); @@ -2803,6 +2835,10 @@ async function onboard(opts = {}) { try { let session; + // Merged, absolute fromDockerfile: explicit flag/env takes precedence; on + // resume falls back to what the original session recorded so the same image + // is used even when --from is omitted from the resume invocation. + let fromDockerfile; if (resume) { session = onboardSession.loadSession(); if (!session || session.resumable === false) { @@ -2810,13 +2846,25 @@ async function onboard(opts = {}) { console.error(" Run: nemoclaw onboard"); process.exit(1); } - const resumeConflicts = getResumeConfigConflicts(session, { nonInteractive: isNonInteractive() }); + const sessionFrom = session?.metadata?.fromDockerfile || null; + fromDockerfile = requestedFromDockerfile + ? path.resolve(requestedFromDockerfile) + : (sessionFrom ? path.resolve(sessionFrom) : null); + const resumeConflicts = getResumeConfigConflicts(session, { nonInteractive: isNonInteractive(), fromDockerfile: requestedFromDockerfile }); if (resumeConflicts.length > 0) { for (const conflict of resumeConflicts) { if (conflict.field === "sandbox") { console.error( ` Resumable state belongs to sandbox '${conflict.recorded}', not '${conflict.requested}'.` ); + } else if (conflict.field === "fromDockerfile") { + if (!conflict.recorded) { + console.error(` Session was started without --from; add --from '${conflict.requested}' to resume it.`); + } else if (!conflict.requested) { + console.error(` Session was started with --from '${conflict.recorded}'; rerun with that path to resume it.`); + } else { + console.error(` Session was started with --from '${conflict.recorded}', not '${conflict.requested}'.`); + } } else { console.error( ` Resumable state recorded ${conflict.field} '${conflict.recorded}', not '${conflict.requested}'.` @@ -2835,10 +2883,11 @@ async function onboard(opts = {}) { }); session = onboardSession.loadSession(); } else { + fromDockerfile = requestedFromDockerfile ? path.resolve(requestedFromDockerfile) : null; session = onboardSession.saveSession( onboardSession.createSession({ mode: isNonInteractive() ? "non-interactive" : "interactive", - metadata: { gatewayName: "nemoclaw" }, + metadata: { gatewayName: "nemoclaw", fromDockerfile: fromDockerfile || null }, }) ); } @@ -2973,7 +3022,7 @@ async function onboard(opts = {}) { } sandboxName = sandboxName || (await promptValidatedSandboxName()); startRecordedStep("sandbox", { sandboxName, provider, model }); - sandboxName = await createSandbox(gpu, model, provider, preferredInferenceApi, sandboxName); + sandboxName = await createSandbox(gpu, model, provider, preferredInferenceApi, sandboxName, fromDockerfile); onboardSession.markStepComplete("sandbox", { sandboxName, provider, model, nimContainer }); } diff --git a/bin/nemoclaw.js b/bin/nemoclaw.js index 76e9512f5..a6b365f2b 100755 --- a/bin/nemoclaw.js +++ b/bin/nemoclaw.js @@ -334,16 +334,31 @@ function exitWithSpawnResult(result) { async function onboard(args) { const { onboard: runOnboard } = require("./lib/onboard"); + + // Extract --from before the unknown-arg validator: it takes a value + // so the set-based check would reject the value token as an unknown flag. + let fromDockerfile = null; + const fromIdx = args.indexOf("--from"); + if (fromIdx !== -1) { + fromDockerfile = args[fromIdx + 1]; + if (!fromDockerfile || fromDockerfile.startsWith("--")) { + console.error(" --from requires a path to a Dockerfile"); + console.error(" Usage: nemoclaw onboard [--non-interactive] [--resume] [--from ]"); + process.exit(1); + } + args = [...args.slice(0, fromIdx), ...args.slice(fromIdx + 2)]; + } + const allowedArgs = new Set(["--non-interactive", "--resume"]); const unknownArgs = args.filter((arg) => !allowedArgs.has(arg)); if (unknownArgs.length > 0) { console.error(` Unknown onboard option(s): ${unknownArgs.join(", ")}`); - console.error(" Usage: nemoclaw onboard [--non-interactive] [--resume]"); + console.error(" Usage: nemoclaw onboard [--non-interactive] [--resume] [--from ]"); process.exit(1); } const nonInteractive = args.includes("--non-interactive"); const resume = args.includes("--resume"); - await runOnboard({ nonInteractive, resume }); + await runOnboard({ nonInteractive, resume, fromDockerfile }); } async function setup() { @@ -716,6 +731,7 @@ function help() { ${G}Getting Started:${R} ${B}nemoclaw onboard${R} Configure inference endpoint and credentials + nemoclaw onboard ${D}--from ${R} Use a custom Dockerfile for the sandbox image nemoclaw setup-spark Set up on DGX Spark ${D}(fixes cgroup v2 + Docker)${R} ${G}Sandbox Management:${R} diff --git a/docs/reference/commands.md b/docs/reference/commands.md index 82f92405f..c10fb451a 100644 --- a/docs/reference/commands.md +++ b/docs/reference/commands.md @@ -41,7 +41,7 @@ The wizard creates an OpenShell gateway, registers inference providers, builds t Use this command for new installs and for recreating a sandbox after changes to policy or configuration. ```console -$ nemoclaw onboard +$ nemoclaw onboard [--non-interactive] [--resume] [--from ] ``` The wizard prompts for a provider first, then collects the provider credential if needed. @@ -55,6 +55,26 @@ Uppercase letters are automatically lowercased. Before creating the gateway, the wizard runs preflight checks. On systems with cgroup v2 (Ubuntu 24.04, DGX Spark, WSL2), it verifies that Docker is configured with `"default-cgroupns-mode": "host"` and provides fix instructions if the setting is missing. +#### `--from ` + +Build the sandbox image from a custom Dockerfile instead of the stock NemoClaw image. +The entire parent directory of the specified file is used as the Docker build context, so any files your Dockerfile references (scripts, config, etc.) must live alongside it. + +```console +$ nemoclaw onboard --from path/to/Dockerfile +``` + +The file can have any name; if it is not already named `Dockerfile`, onboard copies it to `Dockerfile` inside the staged build context automatically. +All NemoClaw build arguments (`NEMOCLAW_MODEL`, `NEMOCLAW_PROVIDER_KEY`, `NEMOCLAW_INFERENCE_BASE_URL`, etc.) are injected as `ARG` overrides at build time, so declare them in your Dockerfile if you need to reference them. + +In non-interactive mode, the path can also be supplied via the `NEMOCLAW_FROM_DOCKERFILE` environment variable: + +```console +$ NEMOCLAW_NON_INTERACTIVE=1 NEMOCLAW_FROM_DOCKERFILE=path/to/Dockerfile nemoclaw onboard +``` + +If a `--resume` is attempted with a different `--from` path than the original session, onboarding exits with a conflict error rather than silently building from the wrong image. + ### `nemoclaw list` List all registered sandboxes with their model, provider, and policy presets. diff --git a/package-lock.json b/package-lock.json index 8b9e57e0b..5b2e34b04 100644 --- a/package-lock.json +++ b/package-lock.json @@ -28,7 +28,7 @@ "vitest": "^4.1.0" }, "engines": { - "node": ">=22.0.0" + "node": ">=22.16.0" } }, "node_modules/@agentclientprotocol/sdk": { @@ -947,6 +947,14 @@ "scripts/actions/documentation" ] }, + "node_modules/@buape/carbon/node_modules/opusscript": { + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/opusscript/-/opusscript-0.0.8.tgz", + "integrity": "sha512-VSTi1aWFuCkRCVq+tx/BQ5q9fMnQ9pVZ3JU4UHKqTkf0ED3fKEPdr+gKAAl3IA2hj9rrP6iyq3hlcJq3HELtNQ==", + "license": "MIT", + "optional": true, + "peer": true + }, "node_modules/@buape/carbon/node_modules/prism-media": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/prism-media/-/prism-media-1.3.5.tgz", @@ -1339,6 +1347,14 @@ "url": "https://github.com/discordjs/discord.js?sponsor" } }, + "node_modules/@discordjs/voice/node_modules/opusscript": { + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/opusscript/-/opusscript-0.0.8.tgz", + "integrity": "sha512-VSTi1aWFuCkRCVq+tx/BQ5q9fMnQ9pVZ3JU4UHKqTkf0ED3fKEPdr+gKAAl3IA2hj9rrP6iyq3hlcJq3HELtNQ==", + "license": "MIT", + "optional": true, + "peer": true + }, "node_modules/@discordjs/voice/node_modules/prism-media": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/prism-media/-/prism-media-1.3.5.tgz", diff --git a/test/onboard.test.js b/test/onboard.test.js index 16b7e5453..4e01cb190 100644 --- a/test/onboard.test.js +++ b/test/onboard.test.js @@ -345,6 +345,40 @@ describe("onboard helpers", () => { } }); + it("detects resume conflicts when a different --from Dockerfile is requested", () => { + const session = { metadata: { fromDockerfile: "/project/Dockerfile" } }; + + // Different paths → conflict + const conflicts = getResumeConfigConflicts(session, { + nonInteractive: false, + fromDockerfile: "/other/Dockerfile", + }); + expect(conflicts).toHaveLength(1); + expect(conflicts[0].field).toBe("fromDockerfile"); + + // Same resolved path → no conflict + const same = getResumeConfigConflicts(session, { + nonInteractive: false, + fromDockerfile: "/project/Dockerfile", + }); + expect(same.filter((c) => c.field === "fromDockerfile")).toHaveLength(0); + + // Session recorded a custom Dockerfile but resume omits --from → conflict (would switch to stock image) + const switchToStock = getResumeConfigConflicts(session, { nonInteractive: false }); + expect(switchToStock.filter((c) => c.field === "fromDockerfile")).toHaveLength(1); + + // Neither session nor resume specifies --from → no conflict (both stock) + const bothStock = getResumeConfigConflicts({ metadata: {} }, { nonInteractive: false }); + expect(bothStock.filter((c) => c.field === "fromDockerfile")).toHaveLength(0); + + // Session used stock image but resume specifies --from → conflict + const switchToCustom = getResumeConfigConflicts( + { metadata: {} }, + { nonInteractive: false, fromDockerfile: "/project/Dockerfile" } + ); + expect(switchToCustom.filter((c) => c.field === "fromDockerfile")).toHaveLength(1); + }); + it("returns provider and model hints only for non-interactive runs", () => { const previousProvider = process.env.NEMOCLAW_PROVIDER; const previousModel = process.env.NEMOCLAW_MODEL; @@ -1330,4 +1364,179 @@ const { setupInference } = require(${onboardPath}); assert.equal(commands.length, 3); }); + it("uses the custom Dockerfile parent directory as build context when --from is given", async () => { + const repoRoot = path.join(import.meta.dirname, ".."); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-from-dockerfile-")); + const fakeBin = path.join(tmpDir, "bin"); + const scriptPath = path.join(tmpDir, "create-sandbox-from.js"); + const onboardPath = JSON.stringify(path.join(repoRoot, "bin", "lib", "onboard.js")); + const runnerPath = JSON.stringify(path.join(repoRoot, "bin", "lib", "runner.js")); + const registryPath = JSON.stringify(path.join(repoRoot, "bin", "lib", "registry.js")); + const preflightPath = JSON.stringify(path.join(repoRoot, "bin", "lib", "preflight.js")); + const credentialsPath = JSON.stringify(path.join(repoRoot, "bin", "lib", "credentials.js")); + + // Create a minimal custom Dockerfile in a temporary directory + const customBuildDir = path.join(tmpDir, "custom-image"); + fs.mkdirSync(customBuildDir, { recursive: true }); + fs.writeFileSync( + path.join(customBuildDir, "Dockerfile"), + [ + "FROM ubuntu:22.04", + "ARG NEMOCLAW_MODEL=nvidia/nemotron-super-49b-v1", + "ARG NEMOCLAW_PROVIDER_KEY=nvidia", + "ARG NEMOCLAW_PRIMARY_MODEL_REF=nvidia/nemotron-super-49b-v1", + "ARG CHAT_UI_URL=http://127.0.0.1:18789", + "ARG NEMOCLAW_INFERENCE_BASE_URL=https://inference.local/v1", + "ARG NEMOCLAW_INFERENCE_API=openai-completions", + "ARG NEMOCLAW_INFERENCE_COMPAT_B64=e30=", + "ARG NEMOCLAW_BUILD_ID=default", + "RUN echo done", + ].join("\n") + ); + fs.writeFileSync(path.join(customBuildDir, "extra.txt"), "extra build context file"); + + fs.mkdirSync(fakeBin, { recursive: true }); + fs.writeFileSync(path.join(fakeBin, "openshell"), "#!/usr/bin/env bash\nexit 0\n", { mode: 0o755 }); + + const customDockerfilePath = JSON.stringify(path.join(customBuildDir, "Dockerfile")); + + const script = String.raw` +const runner = require(${runnerPath}); +const registry = require(${registryPath}); +const preflight = require(${preflightPath}); +const credentials = require(${credentialsPath}); +const childProcess = require("node:child_process"); +const { EventEmitter } = require("node:events"); +const fs = require("node:fs"); + +const commands = []; +runner.run = (command, opts = {}) => { + commands.push({ command, env: opts.env || null }); + return { status: 0 }; +}; +runner.runCapture = (command) => { + if (command.includes("'sandbox' 'get' 'my-assistant'")) return ""; + if (command.includes("'sandbox' 'list'")) return "my-assistant Ready"; + return ""; +}; +registry.registerSandbox = () => true; +registry.removeSandbox = () => true; +preflight.checkPortAvailable = async () => ({ ok: true }); +credentials.prompt = async () => ""; + +childProcess.spawn = (...args) => { + const child = new EventEmitter(); + child.stdout = new EventEmitter(); + child.stderr = new EventEmitter(); + commands.push({ command: args[1][1], env: args[2]?.env || null }); + process.nextTick(() => { + child.stdout.emit("data", Buffer.from("Created sandbox: my-assistant\n")); + child.emit("close", 0); + }); + return child; +}; + +const { createSandbox } = require(${onboardPath}); + +(async () => { + process.env.OPENSHELL_GATEWAY = "nemoclaw"; + const sandboxName = await createSandbox(null, "gpt-5.4", "openai-api", null, "my-assistant", ${customDockerfilePath}); + // Verify the staged build context contains the extra file from the custom dir + const createCmd = commands.find((e) => e.command.includes("'sandbox' 'create'")); + const fromMatch = createCmd && createCmd.command.match(/--from['\s]+'([^']+)'/); + let stagedDir = null; + let hasExtraFile = false; + if (fromMatch) { + const dockerfilePath = fromMatch[1]; + stagedDir = require("node:path").dirname(dockerfilePath); + hasExtraFile = fs.existsSync(require("node:path").join(stagedDir, "extra.txt")); + } + console.log(JSON.stringify({ sandboxName, hasExtraFile })); +})().catch((error) => { + console.error(error); + process.exit(1); +}); +`; + fs.writeFileSync(scriptPath, script); + + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...process.env, + HOME: tmpDir, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + NEMOCLAW_NON_INTERACTIVE: "1", + }, + }); + + assert.equal(result.status, 0, result.stderr); + const payloadLine = result.stdout + .trim() + .split("\n") + .slice() + .reverse() + .find((line) => line.startsWith("{") && line.endsWith("}")); + assert.ok(payloadLine, `expected JSON payload in stdout:\n${result.stdout}`); + const payload = JSON.parse(payloadLine); + assert.equal(payload.sandboxName, "my-assistant"); + assert.equal(payload.hasExtraFile, true, "extra.txt from custom build context should be staged"); + }); + + it("exits with an error when the --from Dockerfile path does not exist", async () => { + const repoRoot = path.join(import.meta.dirname, ".."); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-from-missing-")); + const fakeBin = path.join(tmpDir, "bin"); + const scriptPath = path.join(tmpDir, "create-sandbox-missing.js"); + const onboardPath = JSON.stringify(path.join(repoRoot, "bin", "lib", "onboard.js")); + const runnerPath = JSON.stringify(path.join(repoRoot, "bin", "lib", "runner.js")); + const registryPath = JSON.stringify(path.join(repoRoot, "bin", "lib", "registry.js")); + const preflightPath = JSON.stringify(path.join(repoRoot, "bin", "lib", "preflight.js")); + const credentialsPath = JSON.stringify(path.join(repoRoot, "bin", "lib", "credentials.js")); + + fs.mkdirSync(fakeBin, { recursive: true }); + fs.writeFileSync(path.join(fakeBin, "openshell"), "#!/usr/bin/env bash\nexit 0\n", { mode: 0o755 }); + + const missingPath = JSON.stringify(path.join(tmpDir, "does-not-exist", "Dockerfile")); + + const script = String.raw` +const runner = require(${runnerPath}); +const registry = require(${registryPath}); +const preflight = require(${preflightPath}); +const credentials = require(${credentialsPath}); + +runner.run = () => ({ status: 0 }); +runner.runCapture = () => ""; +registry.registerSandbox = () => true; +registry.removeSandbox = () => true; +preflight.checkPortAvailable = async () => ({ ok: true }); +credentials.prompt = async () => ""; + +const { createSandbox } = require(${onboardPath}); + +(async () => { + process.env.OPENSHELL_GATEWAY = "nemoclaw"; + await createSandbox(null, "gpt-5.4", "openai-api", null, "my-assistant", ${missingPath}); +})().catch((error) => { + console.error(error); + process.exit(1); +}); +`; + fs.writeFileSync(scriptPath, script); + + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...process.env, + HOME: tmpDir, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + NEMOCLAW_NON_INTERACTIVE: "1", + }, + }); + + assert.equal(result.status, 1, "should exit 1 when fromDockerfile path is missing"); + assert.match(result.stderr, /Custom Dockerfile not found/); + }); + });