From cf6a808756ed353c3a6775a6929d4c1d9307d800 Mon Sep 17 00:00:00 2001 From: Himanshu Singh Date: Mon, 22 Sep 2025 13:24:28 +0200 Subject: [PATCH 1/2] chore: add missing expectations for accuracy tests --- tests/accuracy/collectionIndexes.test.ts | 2 +- tests/accuracy/explain.test.ts | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/accuracy/collectionIndexes.test.ts b/tests/accuracy/collectionIndexes.test.ts index 5db4de1e2..45ad2b7e0 100644 --- a/tests/accuracy/collectionIndexes.test.ts +++ b/tests/accuracy/collectionIndexes.test.ts @@ -26,7 +26,7 @@ describeAccuracyTests([ ], }, { - prompt: `Is the following query: ${JSON.stringify({ runtime: { $lt: 100 } })} on the namespace 'mflix.movies' indexed?`, + prompt: `Is there an index covering the following query: ${JSON.stringify({ runtime: { $lt: 100 } })} on the namespace 'mflix.movies'?`, expectedToolCalls: [ { toolName: "collection-indexes", diff --git a/tests/accuracy/explain.test.ts b/tests/accuracy/explain.test.ts index cb9ac0c15..0630a6ab7 100644 --- a/tests/accuracy/explain.test.ts +++ b/tests/accuracy/explain.test.ts @@ -1,4 +1,5 @@ import { describeAccuracyTests } from "./sdk/describeAccuracyTests.js"; +import { Matcher } from "./sdk/matcher.js"; /** * None of these tests score a parameter match on any of the models, likely @@ -22,6 +23,7 @@ describeAccuracyTests([ }, }, ], + verbosity: Matcher.string(), }, }, ], @@ -46,6 +48,7 @@ describeAccuracyTests([ }, }, ], + verbosity: Matcher.string(), }, }, ], @@ -66,6 +69,7 @@ describeAccuracyTests([ }, }, ], + verbosity: Matcher.string(), }, }, ], From a14f641e50142227076c16584a529e48d1f04025 Mon Sep 17 00:00:00 2001 From: Himanshu Singh Date: Mon, 22 Sep 2025 13:52:27 +0200 Subject: [PATCH 2/2] chore: disable parallel test runs for accuracy tests --- scripts/accuracy/runAccuracyTests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/accuracy/runAccuracyTests.sh b/scripts/accuracy/runAccuracyTests.sh index 312d08a19..180bd96fb 100644 --- a/scripts/accuracy/runAccuracyTests.sh +++ b/scripts/accuracy/runAccuracyTests.sh @@ -17,7 +17,7 @@ export MDB_ACCURACY_RUN_ID=$(npx uuid v4) # specified in the command line. Such as: # npm run test:accuracy -- tests/accuracy/some-test.test.ts echo "Running accuracy tests with MDB_ACCURACY_RUN_ID '$MDB_ACCURACY_RUN_ID'" -vitest --config vitest.config.ts --project=accuracy --coverage=false --run "$@" +vitest --config vitest.config.ts --project=accuracy --coverage=false --no-file-parallelism --run "$@" # Preserving the exit code from test run to correctly notify in the CI # environments when the tests fail.