feat: remote environment support

Author: devversion

examples/environments/remote_env/config.js

@@ -0,0 +1,12 @@
+import { getBuiltInRatings } from 'web-codegen-scorer';
+
+/** @type {import("web-codegen-scorer").EnvironmentConfig} */
+export default {
+  displayName: 'Remote Env (example)',
+  clientSideFramework: 'angular',
+  sourceDirectory: './project',
+  ratings: getBuiltInRatings(),
+  generationSystemPrompt: './system-instructions.md',
+  executablePrompts: ['../../prompts/**/*.md'],
+  packageManager: 'npm',
+};

report-app/src/app/pages/report-viewer/report-viewer.html

@@ -257,22 +257,22 @@ <h2>Generated applications</h2>
               {{ result.promptDef.name }}
 
               <div class="status-badge-group">
-                @let initialBuild = result.attemptDetails[0].buildResult;
-                @let repairBuild =
+                @let initialAttempt = result.attemptDetails[0];
+                @let repairAttempt =
                   result.attemptDetails.length > 1
-                    ? result.attemptDetails[1].buildResult
+                    ? result.attemptDetails[1]
                     : null;
-                @let finalBuild = repairBuild ?? initialBuild;
+                @let finalAttempt = result.attemptDetails.at(-1)!;
 
-                @if (finalBuild.runtimeErrors) {
+                @if (finalAttempt.serveTestingResult?.runtimeErrors) {
                   <span class="status-badge error">Runtime error</span>
                 }
 
-                @if (repairBuild?.status === 'error') {
+                @if (repairAttempt?.buildResult?.status === 'error') {
                   <span class="status-badge error">Build after repair</span>
                 }
 
-                @if (initialBuild.status === 'error') {
+                @if (initialAttempt?.buildResult?.status === 'error') {
                   <span class="status-badge error">Initial build failed</span>
                 }
               </div>
@@ -354,7 +354,8 @@ <h5>
                 <h4>Additional info</h4>
                 @for (attempt of result.attemptDetails; track attempt) {
                   @let isBuilt = attempt.buildResult.status === 'success';
-                  @let axeViolations = attempt.buildResult.axeViolations;
+                  @let axeViolations =
+                    attempt.serveTestingResult?.axeViolations;
                   @let hasAxeViolations =
                     axeViolations && axeViolations.length > 0;
 
@@ -501,12 +502,12 @@ <h5>Response</h5>
                 }
               </div>
 
-              @if (finalBuild.runtimeErrors) {
+              @let finalRuntimeErrors =
+                finalAttempt.serveTestingResult?.runtimeErrors;
+              @if (finalRuntimeErrors) {
                 <div class="app-details-section">
                   <h4>Runtime errors</h4>
-                  <pre class="callout warn code">{{
-                    finalBuild.runtimeErrors
-                  }}</pre>
+                  <pre class="callout warn code">{{ finalRuntimeErrors }}</pre>
                 </div>
               }
 

report-app/src/app/pages/report-viewer/report-viewer.ts

@@ -12,7 +12,7 @@ import {
   viewChild,
 } from '@angular/core';
 import { NgxJsonViewerModule } from 'ngx-json-viewer';
-import { BuildErrorType } from '../../../../../runner/builder/builder-types';
+import { BuildErrorType } from '../../../../../runner/workers/builder/builder-types';
 import {
   AssessmentResult,
   IndividualAssessment,
@@ -237,7 +237,7 @@ export class ReportViewer {
   });
 
   protected getScreenshotUrl(result: AssessmentResult): string | null {
-    return result.build.screenshotPngUrl ?? null;
+    return result.finalAttempt.serveTestingResult?.screenshotPngUrl ?? null;
   }
 
   protected isLoading = this.reportsFetcher.isLoadingSingleReport;

report-app/src/app/shared/debugging-zip.ts

@@ -1,4 +1,4 @@
-import { BuildResultStatus } from '../../../../runner/builder/builder-types';
+import { BuildResultStatus } from '../../../../runner/workers/builder/builder-types';
 import {
   AssessmentResult,
   RunInfo,
@@ -31,11 +31,11 @@ export async function createPromptDebuggingZip(
   zip.file('generated-files.md', generatedFiles);
 
   let errors = ``;
-  if (app.build.runtimeErrors) {
-    errors += `## Runtime errors\n${app.build.runtimeErrors}\n`;
+  if (app.finalAttempt.serveTestingResult?.runtimeErrors) {
+    errors += `## Runtime errors\n${app.finalAttempt.serveTestingResult?.runtimeErrors}\n`;
   }
-  if (app.build.status === BuildResultStatus.ERROR) {
-    errors += `## Build error\n  ${app.build.message}`;
+  if (app.finalAttempt.buildResult.status === BuildResultStatus.ERROR) {
+    errors += `## Build error\n  ${app.finalAttempt.buildResult.message}`;
   }
 
   zip.file('errors.md', errors);

runner/builder/builder-types.ts

@@ -18,6 +18,7 @@ import {
 import { DirectorySnapshot } from './directory-snapshot.js';
 import { LlmResponseFile } from '../../shared-interfaces.js';
 import { UserFacingError } from '../../utils/errors.js';
+import assert from 'assert';
 
 const SUPPORTED_MODELS = [
   'gemini-2.5-pro',
@@ -45,6 +46,19 @@ export class GeminiCliRunner implements LlmRunner {
     options: LlmGenerateFilesRequestOptions
   ): Promise<LlmGenerateFilesResponse> {
     const { context, model } = options;
+
+    // TODO: Consider removing these assertions when we have better types here.
+    // These fields are always set when running in a local environment, and this
+    // is a requirement for selecting the `gemini-cli` runner.
+    assert(
+      context.buildCommand,
+      'Expected a `buildCommand` to be set in the LLM generate request context'
+    );
+    assert(
+      context.packageManager,
+      'Expected a `packageManager` to be set in the LLM generate request context'
+    );
+
     const ignoreFilePath = join(context.directory, '.geminiignore');
     const instructionFilePath = join(context.directory, 'GEMINI.md');
     const settingsDir = join(context.directory, '.gemini');

runner/codegen/gemini-cli/gemini-cli-runner.ts

@@ -98,14 +98,24 @@ export interface LlmGenerateFilesContext {
   /**
    * Combined system instructions and prompt for the environments
    * where the two can't be provided separately.
+   *
+   * TODO(crisbeto): Can we explain the reason for this better?
    */
   combinedPrompt: string;
   /** Directory in which the generation will occur. */
   directory: string;
-  /** Command that the LLM can use to verify that the build works. */
-  buildCommand: string;
-  /** Package manager that the LLM can use. */
-  packageManager: string;
+  /**
+   * Command that the LLM can use to verify that the build works.
+   *
+   * Can be `null` for remote environments.
+   */
+  buildCommand: string | undefined;
+  /**
+   * Package manager that the LLM can use.
+   *
+   * Can be `null` for remote environments.
+   */
+  packageManager: string | undefined;
   /** All available package managers supported by the runner. */
   possiblePackageManagers: string[];
 }

runner/codegen/llm-runner.ts

@@ -0,0 +1,81 @@
+import z from 'zod';
+import { ratingSchema } from '../ratings/rating-types.js';
+import { MultiStepPrompt } from './multi-step-prompt.js';
+import { mcpServerOptionsSchema } from '../codegen/llm-runner.js';
+import { getPossiblePackageManagers } from './environment-config.js';
+
+export const baseEnvironmentConfigSchema = z.strictObject({
+  /** Display name for the environment. */
+  displayName: z.string(),
+  /**
+   * Optional unique ID for the environment.
+   * If one isn't provided, it will be computed from the `displayName`.
+   */
+  id: z.string().optional(),
+  /** ID of the client-side framework used within the environment. */
+  clientSideFramework: z.string(),
+  /** Ratings to run when evaluating the environment. */
+  ratings: z.array(ratingSchema),
+  /** Path to the prompt used by the LLM for generating files. */
+  generationSystemPrompt: z.string(),
+  /**
+   * Path to the prompt used by the LLM for repairing builds or failures.
+   *
+   * If unset or `null`, the eval tool will use its default repair instructions.
+   */
+  repairSystemPrompt: z.union([z.string(), z.null()]).optional(),
+  /**
+   * Path to the prompt used by the LLM for editing.
+   *
+   * Prompts running after the initial generation are considered as editing (e.g. multi step prompts).
+   * If `null`, the eval tool will use the generation prompt for edits.
+   */
+  editingSystemPrompt: z.union([z.string(), z.null()]).optional(),
+  /** Prompts that should be sent to the LLM and written into the output. */
+  executablePrompts: z.array(
+    z.union([
+      z.string(),
+      z.strictObject({
+        path: z.string(),
+        name: z.string().optional(),
+        ratings: z.array(ratingSchema).optional(),
+      }),
+      z.custom<MultiStepPrompt>((data) => data instanceof MultiStepPrompt),
+    ])
+  ),
+  /** MCP servers that can be started for this environment. */
+  mcpServers: z.array(mcpServerOptionsSchema).optional(),
+  /** Relative path to the environment's source code in which to generate new code. */
+  sourceDirectory: z.string().optional(),
+  /**
+   * Path to the template directory to use when creating
+   * the project which the LLM will run against.
+   */
+  projectTemplate: z.string().optional(),
+  /** Package manager to use for the eval. */
+  packageManager: z.enum(getPossiblePackageManagers()).optional(),
+  /**
+   * Command to run when building the generated code.
+   * Defaults to `<package manager> run build`.
+   */
+  buildCommand: z.string().optional(),
+  /**
+   * Command to run when starting a development server inside the app.
+   * Defaults to `<package manager> run start --port 0`.
+   */
+  serveCommand: z.string().optional(),
+  /**
+   * Whether to skip installing dependencies when running evals in the environment.
+   * Useful if you're managing dependencies yourself.
+   */
+  skipInstall: z.boolean().optional(),
+  /**
+   * ID of the fullstack framework used within the environment.
+   * If omitted, it will default to the `clientSideFramework`.
+   */
+  fullStackFramework: z.string().optional(),
+  /** Path to the prompt to use when rating code. */
+  codeRatingPrompt: z.string().optional(),
+  /** When enabled, the system prompts for this environment won't be included in the report. */
+  classifyPrompts: z.boolean().optional(),
+});