Update Vally packages to 0.20.0

.github/dependabot.yml

@@ -3,6 +3,9 @@ registries:
   dotnet-public:
     type: nuget-feed
     url: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/index.json
+  dotnet-npm:
+    type: npm-registry
+    url: https://packagefeedproxy.microsoft.io/npm/
 updates:
   - package-ecosystem: "nuget"
     directory: "/"
@@ -290,6 +293,8 @@ updates:
 
   - package-ecosystem: "npm"
     directory: "/eng/harness-evaluation"
+    registries:
+      - dotnet-npm
     open-pull-requests-limit: 5
     schedule:
       interval: "weekly"

eng/harness-evaluation/.npmrc

@@ -0,0 +1,7 @@
+registry=https://packagefeedproxy.microsoft.io/npm/
+audit=false
+fetch-retries=5
+fetch-retry-factor=2
+fetch-retry-maxtimeout=120000
+fetch-retry-mintimeout=10000
+fetch-timeout=600000

eng/harness-evaluation/package.json

@@ -12,8 +12,13 @@
     "eval": "node src/cli.mjs eval"
   },
   "devDependencies": {
-    "@microsoft/vally": "0.19.0",
-    "@microsoft/vally-cli": "0.19.0",
+    "@microsoft/vally": "0.20.0",
+    "@microsoft/vally-cli": "0.20.0",
     "yaml": "2.9.1"
+  },
+  "overrides": {
+    "@github/copilot-sdk": {
+      "zod": "^4.6.6"
+    }
   }
 }

eng/harness-evaluation/src/judge-timeout-loader.mjs

@@ -32,13 +32,7 @@ export async function resolve(specifier, context, nextResolve) {
 
 export async function load(url, context, nextLoad) {
   const result = await nextLoad(url, context);
-  const originalUrl = new URL(clientUrl);
-  originalUrl.searchParams.set('efcore-original', 'true');
   const replacements = {
-    [originalUrl.href]: [
-      'const timeoutMs = Math.min(options.timeoutMs ?? 120_000, 120_000);',
-      'const timeoutMs = options.timeoutMs ?? 120_000;',
-    ],
     [promptGraderUrl]: [
       'const COMPARISON_DEADLINE_MS = 400_000;',
       `const COMPARISON_DEADLINE_MS = ${limits.comparisonDeadlineMs};`,
@@ -52,8 +46,8 @@ export async function load(url, context, nextLoad) {
   if (!replacement) {
     return result;
   }
-  const [before, after] = replacement;
   const source = typeof result.source === 'string' ? result.source : Buffer.from(result.source).toString('utf8');
+  const [before, after] = replacement;
   if (source.split(before).length !== 2) {
     throw new Error(`Unsupported Vally timeout implementation in ${url}; review judge-timeout-loader.mjs before upgrading Vally.`);
   }

eng/harness-evaluation/src/vally.mjs

@@ -7,8 +7,11 @@ const require = createRequire(import.meta.url);
 const entryUrl = pathToFileURL(join(dirname(require.resolve('@microsoft/vally-cli/package.json')), 'dist/index.js'));
 let timeoutMs = defaultJudgeTimeoutMs;
 if (process.argv[2] === '--judge-timeout') {
-  timeoutMs = timeoutMilliseconds(process.argv[3], '--judge-timeout');
+  const judgeTimeout = process.argv[3];
+  timeoutMs = timeoutMilliseconds(judgeTimeout, '--judge-timeout');
   process.argv.splice(2, 2);
+  const commandIndex = process.argv[2] === 'experiment' && process.argv[3] === 'run' ? 4 : 3;
+  process.argv.splice(commandIndex, 0, '--judge-timeout', judgeTimeout);
 }
 judgeTimeoutLimits(timeoutMs);
 

eng/harness-evaluation/test/harness.test.mjs

@@ -110,7 +110,7 @@ variants:
     const loaderPath = join(root, 'judge-loader.mjs');
     const fakeClientSource = `export async function createCopilotLlmClient() {
       return { supportsWorkspaceDelivery: true, shutdown() {}, judge: async (options) => {
-        const timeoutMs = Math.min(options.timeoutMs ?? 120_000, 120_000);
+        const timeoutMs = options.timeoutMs ?? 600_000;
         if (timeoutMs !== 600000) throw new Error('The comparison did not receive the extended judge timeout.');
         return { latencyMs: 0, args: {
           rubric_results: [{ criterion: 'Inspect the anchor.', winner: 'B', magnitude: 'much-better', reasoning: 'Contract probe.' }],
@@ -130,9 +130,12 @@ variants:
     const comparisonPath = join(outputRoot, 'comparison.jsonl');
     const comparisonDirectory = await prepareComparison(experimentDirectory, join(outputRoot, 'comparison-input'));
     const comparisonResult = runVally(['compare', comparisonDirectory, '--output', comparisonPath], {
-      cwd: root, env: { NODE_OPTIONS: `--import=${pathToFileURL(preloadPath).href}` },
+      cwd: root,
+      judgeTimeout: '10m',
+      env: { NODE_OPTIONS: `--import=${pathToFileURL(preloadPath).href}` },
     });
-    assert.equal(comparisonResult.status, 0, comparisonResult.stderr);
+    assert.equal(comparisonResult.status, 0,
+      comparisonResult.stderr + comparisonResult.stdout + await readFile(comparisonPath, 'utf8'));
     const comparisons = (await readFile(comparisonPath, 'utf8')).split(/\r?\n/).filter(Boolean).map(JSON.parse);
     assert.equal(comparisons.length, 1);
     const comparison = comparisons[0];
@@ -458,7 +461,7 @@ test('native scoring timeout reaches graders and bounds the whole grading pass',
   }
 });
 
-test('judge timeout loader removes the SDK cap without changing the native package', async () => {
+test('judge timeout loader preserves explicit timeouts without changing the native package', async () => {
   const root = await mkdtemp(join(tmpdir(), 'efcore-native-judge-timeout-'));
   try {
     const clientUrl = new URL('./graders/llm/llm-client.js', import.meta.resolve('@microsoft/vally')).href;
@@ -469,9 +472,10 @@ test('judge timeout loader removes the SDK cap without changing the native packa
       export class CopilotClient {
         async start() {}
         async stop() { return []; }
+        async listModels() { return [{ id: 'mock' }]; }
         async createSession(options) {
           return {
-            on() {}, async abort() {}, async disconnect() {},
+            on() { return () => {}; }, async abort() {}, async disconnect() {},
             async sendAndWait(message, timeoutMs) {
               globalThis.sdkTimeoutMs = timeoutMs;
               await options.tools[0].handler({ passed: true });
@@ -504,7 +508,7 @@ test('judge timeout loader removes the SDK cap without changing the native packa
     const originalSource = await readFile(fileURLToPath(clientUrl), 'utf8');
     const adapterUrl = new URL('../src/judge-timeout.mjs', import.meta.url).href;
     const timeoutLoaderUrl = new URL('../src/judge-timeout-loader.mjs', import.meta.url).href;
-    for (const [adapted, remainingBudget, expected] of [[false, '10m', 120_000], [true, '15m', 600_000], [true, '30s', 30_000]]) {
+    for (const [adapted, remainingBudget, expected] of [[false, '10m', 600_000], [true, '15m', 600_000], [true, '30s', 30_000]]) {
       await writeFile(preloadPath, `import { register } from 'node:module';
         register(${JSON.stringify(pathToFileURL(loaderPath).href)}, import.meta.url);
         ${adapted ? `register(${JSON.stringify(timeoutLoaderUrl)}, import.meta.url, {
@@ -531,17 +535,20 @@ test('judge timeout loader removes the SDK cap without changing the native packa
     }`);
     await writeFile(preloadPath, `import { register } from 'node:module';
       register(${JSON.stringify(pathToFileURL(loaderPath).href)}, import.meta.url);`);
-    for (const command of ['experiment', 'compare']) {
+    for (const [command, subcommand, fixture] of [['experiment', 'run', 'fixture'], ['compare', undefined, 'fixture']]) {
+      const commandArguments = subcommand ? [command, subcommand, fixture] : [command, fixture];
       for (const [timeout, expected] of [[undefined, 600_000], ['15m', 900_000], ['30s', 30_000]]) {
         const result = spawnSync(process.execPath, ['--import', pathToFileURL(preloadPath).href,
           fileURLToPath(new URL('../src/vally.mjs', import.meta.url)),
-          ...(timeout ? ['--judge-timeout', timeout] : []), command, 'fixture'],
+          ...(timeout ? ['--judge-timeout', timeout] : []), ...commandArguments],
         { encoding: 'utf8', env: { ...process.env, EF_HARNESS_JUDGE_TIMEOUT_MS: '1' } });
         assert.equal(result.status, 0, result.stderr);
         const lines = result.stdout.trim().split(/\r?\n/).map(JSON.parse);
         assert.deepEqual(lines, [
           { sdkTimeoutMs: expected, verdict: { passed: true } },
-          [command, 'fixture'],
+          subcommand
+            ? [command, subcommand, ...(timeout ? ['--judge-timeout', timeout] : []), fixture]
+            : [command, ...(timeout ? ['--judge-timeout', timeout] : []), fixture],
         ]);
       }
     }
@@ -834,7 +841,7 @@ test('eval rejects invalid timeouts and unsupported execution overrides before d
     const invalidJudgeDefault = spawnSync(process.execPath, [cliPath, 'eval', 'example', '--repo-root', root,
       '--output', output], { encoding: 'utf8' });
     assert.notEqual(invalidJudgeDefault.status, 0);
-    assert.match(invalidJudgeDefault.stderr, /defaults.judge_timeout must be a positive duration/);
+    assert.match(invalidJudgeDefault.stderr, /defaults.judge_timeout must be a duration string with a unit suffix/);
     assert.equal(await readFile(sentinel, 'utf8'), 'keep');
     await writeFile(evalPath, evalYaml('example'));
     const overflow = spawnSync(process.execPath, [cliPath, 'eval', 'example', '--repo-root', root,