diff --git a/.changeset/chilly-cows-taste.md b/.changeset/chilly-cows-taste.md new file mode 100644 index 000000000000..7c43cd5b70bf --- /dev/null +++ b/.changeset/chilly-cows-taste.md @@ -0,0 +1,5 @@ +--- +'@ai-sdk/google-vertex': patch +--- + +feat (provider/vertex): add schema support diff --git a/content/providers/01-ai-sdk-providers/11-google-vertex.mdx b/content/providers/01-ai-sdk-providers/11-google-vertex.mdx index 8177b6fcfb8c..5b83ce4a2212 100644 --- a/content/providers/01-ai-sdk-providers/11-google-vertex.mdx +++ b/content/providers/01-ai-sdk-providers/11-google-vertex.mdx @@ -111,6 +111,17 @@ const model = vertex('gemini-1.5-pro', { The following optional settings are available for Google Vertex models: +- **structuredOutputs** _boolean_ + + Optional. Enable structured output. Default is true. + + This is useful when the JSON Schema contains elements that are + not supported by the OpenAPI schema version that + Google Vertex uses. You can use this to disable + structured outputs if you need to. + + See [Troubleshooting: Schema Limitations](#troubleshooting-schema-limitations) for more details. + - **safetySettings** _Array\<\{ category: string; threshold: string \}\>_ Optional. Safety settings for the model. @@ -207,14 +218,46 @@ const { text } = await generateText({ }); ``` +### Troubleshooting: Schema Limitations + +The Google Vertex API uses a subset of the OpenAPI 3.0 schema, +which does not support features such as unions. +The errors that you get in this case look like this: + +`GenerateContentRequest.generation_config.response_schema.properties[occupation].type: must be specified` + +By default, structured outputs are enabled (and for tool calling they are required). +You can disable structured outputs for object generation as a workaround: + +```ts highlight="3,8" +const result = await generateObject({ + model: vertex('gemini-1.5-pro', { + structuredOutputs: false, + }), + schema: z.object({ + name: z.string(), + age: z.number(), + contact: z.union([ + z.object({ + type: z.literal('email'), + value: z.string(), + }), + z.object({ + type: z.literal('phone'), + value: z.string(), + }), + ]), + }), + prompt: 'Generate an example person for testing.', +}); +``` + ### Model Capabilities -| Model | Image Input | Object Generation | Tool Usage | Tool Streaming | -| ----------------------- | ------------------- | ------------------- | ------------------- | ------------------- | -| `gemini-1.5-flash` | | | | | -| `gemini-1.5-pro` | | | | | -| `gemini-1.0-pro-vision` | | | | | -| `gemini-1.0-pro` | | | | | +| Model | Image Input | Object Generation | Tool Usage | Tool Streaming | +| ------------------ | ------------------- | ------------------- | ------------------- | ------------------- | +| `gemini-1.5-flash` | | | | | +| `gemini-1.5-pro` | | | | | The table above lists popular models. You can also pass any available provider diff --git a/examples/ai-core/src/generate-object/google-vertex-tool.ts b/examples/ai-core/src/generate-object/google-vertex-tool.ts new file mode 100644 index 000000000000..f3676a99dc65 --- /dev/null +++ b/examples/ai-core/src/generate-object/google-vertex-tool.ts @@ -0,0 +1,31 @@ +import { vertex } from '@ai-sdk/google-vertex'; +import { generateObject } from 'ai'; +import 'dotenv/config'; +import { z } from 'zod'; + +async function main() { + const result = await generateObject({ + model: vertex('gemini-1.5-pro'), + mode: 'tool', + schema: z.object({ + recipe: z.object({ + name: z.string(), + ingredients: z.array( + z.object({ + name: z.string(), + amount: z.string(), + }), + ), + steps: z.array(z.string()), + }), + }), + prompt: 'Generate a lasagna recipe.', + }); + + console.log(JSON.stringify(result.object.recipe, null, 2)); + console.log(); + console.log('Token usage:', result.usage); + console.log('Finish reason:', result.finishReason); +} + +main().catch(console.error); diff --git a/packages/google-vertex/src/google-vertex-language-model.test.ts b/packages/google-vertex/src/google-vertex-language-model.test.ts index bb8780bae783..268a001810b5 100644 --- a/packages/google-vertex/src/google-vertex-language-model.test.ts +++ b/packages/google-vertex/src/google-vertex-language-model.test.ts @@ -153,6 +153,7 @@ describe('doGenerate', () => { frequencyPenalty: undefined, maxOutputTokens: undefined, responseMimeType: undefined, + responseSchema: undefined, temperature: undefined, topK: undefined, topP: undefined, @@ -234,6 +235,7 @@ describe('doGenerate', () => { generationConfig: { maxOutputTokens: 100, responseMimeType: undefined, + responseSchema: undefined, temperature: 0.5, topK: 0.1, topP: 0.9, @@ -267,6 +269,7 @@ describe('doGenerate', () => { frequencyPenalty: undefined, maxOutputTokens: undefined, responseMimeType: undefined, + responseSchema: undefined, stopSequences: undefined, temperature: undefined, topK: undefined, @@ -321,46 +324,102 @@ describe('doGenerate', () => { }); }); - it('should set name & description in object-json mode', async () => { + it('should pass specification in object-json mode with structuredOutputs = true (default)', async () => { const { model, mockVertexAI } = createModel({ - modelId: 'test-model', generateContent: prepareResponse({ - parts: [{ text: '{"value":"Spark"}' }], + text: '{"property1":"value1","property2":"value2"}', }), }); - const response = await model.doGenerate({ + const result = await model.doGenerate({ inputFormat: 'prompt', mode: { type: 'object-json', - name: 'test-name', - description: 'test description', schema: { type: 'object', - properties: { value: { type: 'string' } }, - required: ['value'], + properties: { + property1: { type: 'string' }, + property2: { type: 'number' }, + }, + required: ['property1', 'property2'], + additionalProperties: false, + }, + }, + prompt: TEST_PROMPT, + }); + + expect(mockVertexAI.lastModelParams).toStrictEqual({ + generationConfig: { + frequencyPenalty: undefined, + maxOutputTokens: undefined, + responseMimeType: 'application/json', + responseSchema: { + properties: { + property1: { type: 'string' }, + property2: { type: 'number' }, + }, + required: ['property1', 'property2'], + type: 'object', + }, + stopSequences: undefined, + temperature: undefined, + topK: undefined, + topP: undefined, + }, + model: 'gemini-1.0-pro-002', + safetySettings: undefined, + }); + + expect(result.text).toStrictEqual( + '{"property1":"value1","property2":"value2"}', + ); + }); + + it('should not pass specification in object-json mode with structuredOutputs = false', async () => { + const { model, mockVertexAI } = createModel({ + generateContent: prepareResponse({ + text: '{"property1":"value1","property2":"value2"}', + }), + settings: { + structuredOutputs: false, + }, + }); + + const result = await model.doGenerate({ + inputFormat: 'prompt', + mode: { + type: 'object-json', + schema: { + type: 'object', + properties: { + property1: { type: 'string' }, + property2: { type: 'number' }, + }, + required: ['property1', 'property2'], additionalProperties: false, - $schema: 'http://json-schema.org/draft-07/schema#', }, }, prompt: TEST_PROMPT, }); expect(mockVertexAI.lastModelParams).toStrictEqual({ - model: 'test-model', generationConfig: { frequencyPenalty: undefined, maxOutputTokens: undefined, responseMimeType: 'application/json', + responseSchema: undefined, stopSequences: undefined, temperature: undefined, topK: undefined, topP: undefined, }, + model: 'gemini-1.0-pro-002', safetySettings: undefined, }); - expect(response.text).toStrictEqual('{"value":"Spark"}'); + expect(result.text).toStrictEqual( + '{"property1":"value1","property2":"value2"}', + ); }); it('should support object-tool mode', async () => { @@ -403,6 +462,7 @@ describe('doGenerate', () => { frequencyPenalty: undefined, maxOutputTokens: undefined, responseMimeType: undefined, + responseSchema: undefined, temperature: undefined, topK: undefined, topP: undefined, diff --git a/packages/google-vertex/src/google-vertex-language-model.ts b/packages/google-vertex/src/google-vertex-language-model.ts index 2879d34e8b52..d9a267d0f5d6 100644 --- a/packages/google-vertex/src/google-vertex-language-model.ts +++ b/packages/google-vertex/src/google-vertex-language-model.ts @@ -13,6 +13,7 @@ import { GenerateContentResponse, GenerationConfig, Part, + ResponseSchema, SafetySetting, Tool, ToolConfig, @@ -33,9 +34,13 @@ type GoogleVertexAIConfig = { export class GoogleVertexLanguageModel implements LanguageModelV1 { readonly specificationVersion = 'v1'; readonly provider = 'google-vertex'; - readonly defaultObjectGenerationMode = 'tool'; + readonly defaultObjectGenerationMode = 'json'; readonly supportsImageUrls = false; + get supportsObjectGeneration() { + return this.settings.structuredOutputs !== false; + } + readonly modelId: GoogleVertexModelId; readonly settings: GoogleVertexSettings; @@ -98,8 +103,20 @@ export class GoogleVertexLanguageModel implements LanguageModelV1 { temperature, topP, stopSequences, + + // response format: responseMimeType: responseFormat?.type === 'json' ? 'application/json' : undefined, + responseSchema: + responseFormat?.type === 'json' && + responseFormat.schema != null && + // Google Vertex does not support all OpenAPI Schema features, + // so this is needed as an escape hatch: + this.supportsObjectGeneration + ? (convertJSONSchemaToOpenAPISchema( + responseFormat.schema, + ) as ResponseSchema) + : undefined, }; const type = mode.type; @@ -132,6 +149,15 @@ export class GoogleVertexLanguageModel implements LanguageModelV1 { generationConfig: { ...generationConfig, responseMimeType: 'application/json', + responseSchema: + mode.schema != null && + // Google Vertex does not support all OpenAPI Schema features, + // so this is needed as an escape hatch: + this.supportsObjectGeneration + ? (convertJSONSchemaToOpenAPISchema( + mode.schema, + ) as ResponseSchema) + : undefined, }, safetySettings: this.settings.safetySettings as | undefined diff --git a/packages/google-vertex/src/google-vertex-settings.ts b/packages/google-vertex/src/google-vertex-settings.ts index 06b0382cd1f5..6e76ca5eb3ae 100644 --- a/packages/google-vertex/src/google-vertex-settings.ts +++ b/packages/google-vertex/src/google-vertex-settings.ts @@ -18,6 +18,16 @@ Models running with nucleus sampling don't allow topK setting. */ topK?: number; + /** + * Optional. Enable structured output. Default is true. + * + * This is useful when the JSON Schema contains elements that are + * not supported by the OpenAPI schema version that + * Google Generative AI uses. You can use this to disable + * structured outputs if you need to. + */ + structuredOutputs?: boolean; + /** Optional. A list of unique safety settings for blocking unsafe content. */