feat: support rubric_instructions for annotation queues in the SDK (#1705)

samnoyes · web-flow · commit b01be87e903f · 2025-04-30T08:44:09.000-07:00
Fixes LS-3628
diff --git a/js/package.json b/js/package.json
@@ -1,6 +1,6 @@
 {
   "name": "langsmith",
-  "version": "0.3.22",
+  "version": "0.3.23",
   "description": "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform.",
   "packageManager": "yarn@1.22.19",
   "files": [
diff --git a/js/src/client.ts b/js/src/client.ts
@@ -41,6 +41,7 @@ import {
   AttachmentInfo,
   AttachmentData,
   DatasetVersion,
+  AnnotationQueueWithDetails,
 } from "./schemas.js";
 import {
   convertLangChainMessageToExample,
@@ -3801,12 +3802,14 @@ export class Client implements LangSmithTracingClientInterface {
     name: string;
     description?: string;
     queueId?: string;
-  }): Promise<AnnotationQueue> {
-    const { name, description, queueId } = options;
+    rubricInstructions?: string;
+  }): Promise<AnnotationQueueWithDetails> {
+    const { name, description, queueId, rubricInstructions } = options;
     const body = {
       name,
       description,
       id: queueId || uuid.v4(),
+      rubric_instructions: rubricInstructions,
     };
 
     const response = await this.caller.call(
@@ -3832,17 +3835,24 @@ export class Client implements LangSmithTracingClientInterface {
   /**
    * Read an annotation queue with the specified queue ID.
    * @param queueId - The ID of the annotation queue to read
-   * @returns The AnnotationQueue object
+   * @returns The AnnotationQueueWithDetails object
    */
-  public async readAnnotationQueue(queueId: string): Promise<AnnotationQueue> {
-    // TODO: Replace when actual endpoint is added
-    const queueIteratorResult = await this.listAnnotationQueues({
-      queueIds: [queueId],
-    }).next();
-    if (queueIteratorResult.done) {
-      throw new Error(`Annotation queue with ID ${queueId} not found`);
-    }
-    return queueIteratorResult.value;
+  public async readAnnotationQueue(
+    queueId: string
+  ): Promise<AnnotationQueueWithDetails> {
+    const response = await this.caller.call(
+      _getFetchImplementation(this.debug),
+      `${this.apiUrl}/annotation-queues/${assertUuid(queueId, "queueId")}`,
+      {
+        method: "GET",
+        headers: this.headers,
+        signal: AbortSignal.timeout(this.timeout_ms),
+        ...this.fetchOptions,
+      }
+    );
+    await raiseForStatus(response, "read annotation queue");
+    const data = await response.json();
+    return data as AnnotationQueueWithDetails;
   }
 
   /**
@@ -3857,16 +3867,21 @@ export class Client implements LangSmithTracingClientInterface {
     options: {
       name: string;
       description?: string;
+      rubricInstructions?: string;
     }
   ): Promise<void> {
-    const { name, description } = options;
+    const { name, description, rubricInstructions } = options;
     const response = await this.caller.call(
       _getFetchImplementation(this.debug),
       `${this.apiUrl}/annotation-queues/${assertUuid(queueId, "queueId")}`,
       {
         method: "PATCH",
         headers: { ...this.headers, "Content-Type": "application/json" },
-        body: JSON.stringify({ name, description }),
+        body: JSON.stringify({
+          name,
+          description,
+          rubric_instructions: rubricInstructions,
+        }),
         signal: AbortSignal.timeout(this.timeout_ms),
         ...this.fetchOptions,
       }
diff --git a/js/src/index.ts b/js/src/index.ts
@@ -18,4 +18,4 @@ export { RunTree, type RunTreeConfig } from "./run_trees.js";
 export { overrideFetchImplementation } from "./singletons/fetch.js";
 
 // Update using yarn bump-version
-export const __version__ = "0.3.22";
+export const __version__ = "0.3.23";
diff --git a/js/src/schemas.ts b/js/src/schemas.ts
@@ -567,6 +567,11 @@ export interface AnnotationQueue {
   tenant_id: string;
 }
 
+export interface AnnotationQueueWithDetails extends AnnotationQueue {
+  /** The rubric instructions for the annotation queue. */
+  rubric_instructions?: string;
+}
+
 export interface RunWithAnnotationQueueInfo extends BaseRun {
   /** The last time this run was reviewed. */
   last_reviewed_time?: string;
diff --git a/js/src/tests/client.int.test.ts b/js/src/tests/client.int.test.ts
@@ -1257,6 +1257,92 @@ test("annotationqueue crud", async () => {
   }
 });
 
+test("annotationqueue crud with rubric instructions", async () => {
+  const client = new Client();
+  const queueName = `test-queue-${uuidv4().substring(0, 8)}`;
+  const projectName = `test-project-${uuidv4().substring(0, 8)}`;
+  const queueId = uuidv4();
+
+  try {
+    // 1. Create an annotation queue
+    const queue = await client.createAnnotationQueue({
+      name: queueName,
+      description: "Initial description",
+      queueId,
+      rubricInstructions: "This is a rubric instruction",
+    });
+    expect(queue).toBeDefined();
+    expect(queue.name).toBe(queueName);
+
+    // 1a. Get the annotation queue
+    const fetchedQueue = await client.readAnnotationQueue(queue.id);
+    expect(fetchedQueue).toBeDefined();
+    expect(fetchedQueue.name).toBe(queueName);
+    expect(fetchedQueue.rubric_instructions).toBe(
+      "This is a rubric instruction"
+    );
+
+    // 1b. Update the annotation queue rubric instructions
+    const newInstructions = "Updated rubric instructions";
+    await client.updateAnnotationQueue(queue.id, {
+      name: queueName,
+      rubricInstructions: newInstructions,
+    });
+    const updatedQueue = await client.readAnnotationQueue(queue.id);
+    expect(updatedQueue.rubric_instructions).toBe(newInstructions);
+  } finally {
+    // 6. Delete the annotation queue
+    await client.deleteAnnotationQueue(queueId);
+
+    // Clean up the project
+    if (await client.hasProject({ projectName })) {
+      await client.deleteProject({ projectName });
+    }
+  }
+});
+
+test("annotationqueue crud with rubric instructions 2", async () => {
+  const client = new Client();
+  const queueName = `test-queue-${uuidv4().substring(0, 8)}`;
+  const projectName = `test-project-${uuidv4().substring(0, 8)}`;
+  const queueId = uuidv4();
+
+  try {
+    // 1. Create an annotation queue
+    const queue = await client.createAnnotationQueue({
+      name: queueName,
+      description: "Initial description",
+      queueId,
+    });
+    expect(queue).toBeDefined();
+    expect(queue.name).toBe(queueName);
+    expect(queue.rubric_instructions).toBeUndefined();
+
+    // 1a. Get the annotation queue
+    const fetchedQueue = await client.readAnnotationQueue(queue.id);
+    expect(fetchedQueue).toBeDefined();
+    expect(fetchedQueue.name).toBe(queueName);
+    expect(fetchedQueue.rubric_instructions).toBeNull();
+
+    // 1b. Update the annotation queue rubric instructions
+    const newInstructions = "Updated rubric instructions";
+    await client.updateAnnotationQueue(queue.id, {
+      name: queueName,
+      rubricInstructions: newInstructions,
+    });
+    const updatedQueue = await client.readAnnotationQueue(queue.id);
+    expect(updatedQueue.rubric_instructions).toBe(newInstructions);
+  } finally {
+    // 6. Delete the annotation queue
+    await client.deleteAnnotationQueue(queueId);
+
+    // Clean up the project
+    if (await client.hasProject({ projectName })) {
+      await client.deleteProject({ projectName });
+    }
+  }
+});
+
 test("upload examples multipart", async () => {
   const client = new Client();
   const datasetName = `__test_upload_examples_multipart${uuidv4().slice(0, 4)}`;
diff --git a/python/langsmith/__init__.py b/python/langsmith/__init__.py
@@ -20,7 +20,7 @@
     from langsmith.utils import ContextThreadPoolExecutor
 
 # Avoid calling into importlib on every call to __version__
-__version__ = "0.3.38"
+__version__ = "0.3.39"
 version = __version__  # for backwards compatibility
 
 
diff --git a/python/langsmith/client.py b/python/langsmith/client.py
@@ -6259,7 +6259,8 @@ def create_annotation_queue(
         name: str,
         description: Optional[str] = None,
         queue_id: Optional[ID_TYPE] = None,
-    ) -> ls_schemas.AnnotationQueue:
+        rubric_instructions: Optional[str] = None,
+    ) -> ls_schemas.AnnotationQueueWithDetails:
         """Create an annotation queue on the LangSmith API.
 
         Args:
@@ -6269,6 +6270,8 @@ def create_annotation_queue(
                 The description of the annotation queue.
             queue_id (Optional[Union[UUID, str]]):
                 The ID of the annotation queue.
+            rubric_instructions (Optional[str]):
+                The rubric instructions for the annotation queue.
 
         Returns:
             AnnotationQueue: The created annotation queue object.
@@ -6277,14 +6280,15 @@ def create_annotation_queue(
             "name": name,
             "description": description,
             "id": str(queue_id) if queue_id is not None else str(uuid.uuid4()),
+            "rubric_instructions": rubric_instructions,
         }
         response = self.request_with_retries(
             "POST",
             "/annotation-queues",
             json={k: v for k, v in body.items() if v is not None},
         )
         ls_utils.raise_for_status_with_text(response)
-        return ls_schemas.AnnotationQueue(
+        return ls_schemas.AnnotationQueueWithDetails(
             **response.json(),
         )
 
@@ -6297,11 +6301,22 @@ def read_annotation_queue(self, queue_id: ID_TYPE) -> ls_schemas.AnnotationQueue
         Returns:
             AnnotationQueue: The annotation queue object.
         """
-        # TODO: Replace when actual endpoint is added
-        return next(self.list_annotation_queues(queue_ids=[queue_id]))
+        base_url = f"/annotation-queues/{_as_uuid(queue_id, 'queue_id')}"
+        response = self.request_with_retries(
+            "GET",
+            f"{base_url}",
+            headers=self._headers,
+        )
+        ls_utils.raise_for_status_with_text(response)
+        return ls_schemas.AnnotationQueueWithDetails(**response.json())
 
     def update_annotation_queue(
-        self, queue_id: ID_TYPE, *, name: str, description: Optional[str] = None
+        self,
+        queue_id: ID_TYPE,
+        *,
+        name: str,
+        description: Optional[str] = None,
+        rubric_instructions: Optional[str] = None,
     ) -> None:
         """Update an annotation queue with the specified queue_id.
 
@@ -6310,6 +6325,8 @@ def update_annotation_queue(
             name (str): The new name for the annotation queue.
             description (Optional[str]): The new description for the
                 annotation queue. Defaults to None.
+            rubric_instructions (Optional[str]): The new rubric instructions for the
+                annotation queue. Defaults to None.
 
         Returns:
             None
@@ -6320,6 +6337,7 @@ def update_annotation_queue(
             json={
                 "name": name,
                 "description": description,
+                "rubric_instructions": rubric_instructions,
             },
         )
         ls_utils.raise_for_status_with_text(response)
diff --git a/python/langsmith/schemas.py b/python/langsmith/schemas.py
@@ -790,6 +790,13 @@ class AnnotationQueue(BaseModel):
     """The ID of the tenant associated with the annotation queue."""
 
 
+class AnnotationQueueWithDetails(AnnotationQueue):
+    """Represents an annotation queue with details."""
+
+    rubric_instructions: Optional[str] = None
+    """The rubric instructions for the annotation queue."""
+
+
 class BatchIngestConfig(TypedDict, total=False):
     """Configuration for batch ingestion."""
 
diff --git a/python/pyproject.toml b/python/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "langsmith"
-version = "0.3.38"
+version = "0.3.39"
 description = "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform."
 authors = ["LangChain <support@langchain.dev>"]
 license = "MIT"
diff --git a/python/tests/integration_tests/test_client.py b/python/tests/integration_tests/test_client.py
@@ -3110,3 +3110,84 @@ def _get_run(run_id: ID_TYPE) -> bool:
 
     # Clean up
     langchain_client.delete_annotation_queue(queue.id)
+
+
+def test_annotation_queue_with_rubric_instructions(langchain_client: Client):
+    """Test CRUD operations on annotation queue with rubric instructions."""
+    queue_name = f"test-queue-{str(uuid.uuid4())[:8]}"
+    project_name = f"test-project-{str(uuid.uuid4())[:8]}"
+    queue_id = uuid.uuid4()
+
+    try:
+        # 1. Create an annotation queue
+        queue = langchain_client.create_annotation_queue(
+            name=queue_name,
+            description="Initial description",
+            queue_id=queue_id,
+            rubric_instructions="This is a rubric instruction",
+        )
+        assert queue is not None
+        assert queue.name == queue_name
+
+        # 1a. Get the annotation queue
+        fetched_queue = langchain_client.read_annotation_queue(queue.id)
+        assert fetched_queue is not None
+        assert fetched_queue.name == queue_name
+        assert fetched_queue.rubric_instructions == "This is a rubric instruction"
+
+        # 1b. Update the annotation queue rubric instructions
+        new_instructions = "Updated rubric instructions"
+        langchain_client.update_annotation_queue(
+            queue.id,
+            name=queue_name,
+            rubric_instructions=new_instructions,
+        )
+        updated_queue = langchain_client.read_annotation_queue(queue.id)
+        assert updated_queue.rubric_instructions == new_instructions
+    finally:
+        # 6. Delete the annotation queue
+        langchain_client.delete_annotation_queue(queue_id)
+
+        # Clean up the project
+        if langchain_client.has_project(project_name=project_name):
+            langchain_client.delete_project(project_name=project_name)
+
+
+def test_annotation_queue_with_rubric_instructions_2(langchain_client: Client):
+    """Test CRUD operations on annotation queue with rubric instructions."""
+    queue_name = f"test-queue-{str(uuid.uuid4())[:8]}"
+    project_name = f"test-project-{str(uuid.uuid4())[:8]}"
+    queue_id = uuid.uuid4()
+
+    try:
+        # 1. Create an annotation queue without rubric instructions
+        queue = langchain_client.create_annotation_queue(
+            name=queue_name,
+            description="Initial description",
+            queue_id=queue_id,
+        )
+        assert queue is not None
+        assert queue.name == queue_name
+
+        # 1a. Get the annotation queue
+        fetched_queue = langchain_client.read_annotation_queue(queue.id)
+        assert fetched_queue is not None
+        assert fetched_queue.name == queue_name
+        assert fetched_queue.rubric_instructions is None
+
+        # 1b. Update the annotation queue rubric instructions
+        new_instructions = "Updated rubric instructions"
+        langchain_client.update_annotation_queue(
+            queue.id,
+            name=queue_name,
+            rubric_instructions=new_instructions,
+        )
+        updated_queue = langchain_client.read_annotation_queue(queue.id)
+        assert updated_queue.rubric_instructions == new_instructions
+    finally:
+        # 6. Delete the annotation queue
+        langchain_client.delete_annotation_queue(queue_id)
+
+        # Clean up the project
+        if langchain_client.has_project(project_name=project_name):
+            langchain_client.delete_project(project_name=project_name)

Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`{`
`2`	`2`	`"name": "langsmith",`
`3`		`- "version": "0.3.22",`
	`3`	`+ "version": "0.3.23",`
`4`	`4`	`"description": "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform.",`
`5`	`5`	`"packageManager": "[email protected]",`
`6`	`6`	`"files": [`