Skip to content

Commit 336d29a

Browse files
feat(js): Add current repetition property to jestlike test wrapper (#1856)
### Problem When running tests with multiple repetitions using LangSmith's Jest/Vitest test wrapper, there's no way to get the current repetition number. ### Solution This PR adds an `repetition` property that gets passed to the test function, providing the current repetition index ### Use Case For multi-turn conversations, each repetition needs its own unique threadId that remains consistent throughout all tests in that conversation. ```javascripts const ConversationTestDataset = [{ inputs: { userId: "test-user-123", message: "What's the weather like today?", conversationId: "weather-chat", }, referenceOutputs: { responseType: "weather_info", requiresLocation: true, }, }, { inputs: { userId: "test-user-123", message: "How about tomorrow?", conversationId: "weather-chat", }, referenceOutputs: { responseType: "forecast", }, }, ]; ls.describe("Conversation Flow Tests", () => { ls.test.each(ConversationTestDataset, { repetitions: 5, // Run each test case 5 times })( "Multi-turn conversation", async ({ inputs, referenceOutputs, repetition }) => { // Now we can create unique thread IDs for each repetition const threadId = `${inputs.conversationId}-run-${repetition}`; const response = await conversationAgent({ ...inputs, threadId, }); // Validate response matches expected outputs expect(response.type).toBe(referenceOutputs.responseType); // Log outputs with repetition context ls.logOutputs({ response: response.content, threadId, repetitionIndex: repetition }); } ); }); ``` --------- Co-authored-by: jacoblee93 <[email protected]>
1 parent 87736aa commit 336d29a

File tree

2 files changed

+27
-9
lines changed

2 files changed

+27
-9
lines changed

js/src/utils/jestlike/index.ts

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ import type {
3434
LangSmithJestlikeWrapperParams,
3535
LangSmithJestlikeDescribeWrapper,
3636
LangSmithJestlikeDescribeWrapperConfig,
37+
LangSmithJestlikeTestFunction,
3738
} from "./types.js";
3839
import { getEnvironmentVariable, isJsDom } from "../env.js";
3940
import {
@@ -465,9 +466,7 @@ export function generateWrapperFromJestlikeMethods(
465466
>(
466467
name: string,
467468
lsParams: LangSmithJestlikeWrapperParams<I, O>,
468-
testFn: (
469-
data: { inputs: I; referenceOutputs?: O } & Record<string, any>
470-
) => unknown | Promise<unknown>,
469+
testFn: LangSmithJestlikeTestFunction<I, O>,
471470
timeout?: number
472471
) {
473472
// Due to https://github.com/jestjs/jest/issues/13653,
@@ -572,6 +571,13 @@ export function generateWrapperFromJestlikeMethods(
572571
...rest,
573572
inputs: testInput,
574573
referenceOutputs: testOutput,
574+
testMetadata: {
575+
exampleId,
576+
experimentId: project?.id,
577+
datasetId: dataset?.id,
578+
testTrackingEnabled: trackingEnabled(testContext),
579+
repetition: i,
580+
},
575581
}
576582
)
577583
);
@@ -767,12 +773,7 @@ export function generateWrapperFromJestlikeMethods(
767773
}
768774
return function (
769775
name: string,
770-
fn: (
771-
params: { id?: string; inputs: I; referenceOutputs?: O } & Record<
772-
string,
773-
any
774-
>
775-
) => unknown | Promise<unknown>,
776+
fn: LangSmithJestlikeTestFunction<I, O>,
776777
timeout?: number
777778
) {
778779
for (let i = 0; i < table.length; i += 1) {

js/src/utils/jestlike/types.ts

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,3 +42,20 @@ export type SimpleEvaluationResult = {
4242
score: NonNullable<EvaluationResult["score"]>;
4343
comment?: EvaluationResult["comment"];
4444
};
45+
46+
export type LangSmithJestlikeTestMetadata = {
47+
exampleId?: string;
48+
experimentId?: string;
49+
datasetId?: string;
50+
testTrackingEnabled: boolean;
51+
repetition: number;
52+
};
53+
54+
export type LangSmithJestlikeTestFunction<I, O> = (
55+
data: {
56+
inputs: I;
57+
referenceOutputs?: O;
58+
testMetadata: LangSmithJestlikeTestMetadata;
59+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
60+
} & Record<string, any>
61+
) => unknown | Promise<unknown>;

0 commit comments

Comments
 (0)