diff --git a/.claude/skills/harness-test-writer/SKILL.md b/.claude/skills/harness-test-writer/SKILL.md index 3857c67db33..4075c92914d 100644 --- a/.claude/skills/harness-test-writer/SKILL.md +++ b/.claude/skills/harness-test-writer/SKILL.md @@ -104,9 +104,10 @@ Conventions (match the existing collection exactly): pins the same invariant independently (e.g. native `/v1/responses` alongside the converted `/v1/chat/completions` path). - **Test scripts** (Postman `event[].script.exec`, plain ES5 JavaScript): - - Start with an infra guard so auth/rate/server noise skips instead of false-failing: - `if ([401, 403, 429, 500, 502, 503, 504].indexOf(pm.response.code) !== -1) { return; }` - Do NOT guard on 400 when a 400 IS the regression signature - that must fail loudly. + - Never skip any error status code. Do NOT open the script with an early-return + guard like `if ([401, 403, 429, 500, 502, 503, 504].indexOf(pm.response.code) !== -1) { return; }` + - every unexpected status (auth failure, rate limit, 5xx) must fail the case + loudly instead of silently passing it. Assert the exact status the case expects. - Assert the specific failure signature is absent (error substring, param name) AND that the happy path succeeded (status below 400, expected fields present). - Include the response text in failure messages: diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml new file mode 100644 index 00000000000..a36ba277af8 --- /dev/null +++ b/.github/workflows/codspeed.yml @@ -0,0 +1,133 @@ +name: CodSpeed Benchmarks + +# Continuous performance measurement for the request hot path. +# +# Bifrost sits inline on every LLM call, so the CPU cost of the translation +# layers (payload parsing, provider request/response conversion, SSE framing, +# key selection) is paid once per request - and once per chunk on a stream. +# `go test -bench` already covers those paths; this workflow runs the same +# benchmarks under the CodSpeed walltime instrument on every PR so a regression +# shows up as a diff against the base commit instead of surfacing in a load +# test later. +# +# Walltime is the only instrument the Go integration supports, so the job runs +# on CodSpeed macro runners (bare metal, pinned CPU) - GitHub-hosted runners are +# too noisy for wall-clock comparisons. + +on: + push: + branches: + - dev + - main + pull_request: + # `workflow_dispatch` allows CodSpeed to trigger backtest performance + # analysis in order to generate initial data. + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: codspeed-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + benchmarks: + name: Benchmarks (${{ matrix.name }}) + # CodSpeed's managed bare-metal runner for walltime benchmarks. + # https://codspeed.io/docs/benchmarks/go + # The org's Default runner group must "Allow public repositories", or the + # jobs queue without ever starting. + runs-on: codspeed-macro + timeout-minutes: 45 + + env: + # The modules declare go 1.27.0; keep the toolchain resolution identical to + # the test workflows rather than depending on the runner image default. + GOTOOLCHAIN: auto + + permissions: + contents: read # required for actions/checkout + id-token: write # required for OIDC authentication with CodSpeed + + strategy: + # One module's benchmarks failing should not hide the others' results. + fail-fast: false + matrix: + include: + # Package lists are explicit rather than ./... : most packages carry no + # benchmarks, and the ones excluded here either need live provider + # credentials or benchmark a comparison baseline that is not worth + # tracking as a trend. + - name: core + module: core + packages: . ./schemas/ ./keyselectors/ ./providers/openai/ ./providers/anthropic/ ./providers/utils/ + - name: framework + module: framework + packages: ./modelcatalog/ ./tracing/ + - name: plugin-governance + module: plugins/governance + packages: . + - name: plugin-mocker + module: plugins/mocker + packages: . + + steps: + # CodSpeed macro runners are self-hosted: their image must have the + # StepSecurity agent installed for this policy to be enforced. The action + # configures an existing agent; it does not install one on self-hosted VMs. + - name: Harden Runner + uses: step-security/harden-runner@fe104658747b27e96e4f7e80cd0a94068e53901d # v2.16.1 + with: + egress-policy: block + # Infrastructure only: Go toolchain/modules, GitHub release assets, + # and CodSpeed installation/results. Benchmarks use local fixtures; + # provider APIs and live-test assets are intentionally not allowed. + allowed-endpoints: > + api.codspeed.io:443 + api.github.com:443 + app.codspeed.io:443 + codspeed.io:443 + dl.google.com:443 + github.com:443 + go.dev:443 + gql.codspeed.io:443 + objects.githubusercontent.com:443 + proxy.golang.org:443 + raw.githubusercontent.com:443 + release-assets.githubusercontent.com:443 + storage.googleapis.com:443 + sum.golang.org:443 + + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Set up Go + uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 + with: + go-version: "1.27.0" + cache-dependency-path: | + core/go.sum + framework/go.sum + plugins/*/go.sum + + # The modules are versioned independently and carry no replace directives, + # so without a workspace the framework and plugin benchmarks would measure + # the last PUBLISHED core rather than the commit under test. Same script + # the test workflows use, so the module list cannot drift. + - name: Set up Go workspace + run: source ./.github/workflows/scripts/setup-go-workspace.sh + shell: bash + + - name: Download module dependencies + working-directory: ${{ matrix.module }} + run: go mod download + + - name: Run benchmarks + uses: CodSpeedHQ/action@373d6868929f444bc08d901fd0eb0ad52a8875ea # v5.2.1 + with: + mode: walltime + working-directory: ${{ matrix.module }} + # CodSpeed's Go runner discovers benchmarks without running regular tests. + # Its Go integration currently supports only the -bench flag. + run: go test -bench=. ${{ matrix.packages }} diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml index e513412f8a9..4910e79303d 100644 --- a/.github/workflows/pr-tests.yml +++ b/.github/workflows/pr-tests.yml @@ -138,6 +138,7 @@ jobs: SGL_API_KEY: ${{ secrets.SGL_API_KEY }} CEREBRAS_API_KEY: ${{ secrets.CEREBRAS_API_KEY }} SARVAM_API_KEY: ${{ secrets.SARVAM_API_KEY }} + TYPESAFE_API_KEY: ${{ secrets.TYPESAFE_API_KEY }} DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} DATABRICKS_WORKSPACE_URL: ${{ secrets.DATABRICKS_WORKSPACE_URL }} GITHUB_COPILOT_APP_ID: ${{ secrets.GITHUB_COPILOT_APP_ID }} diff --git a/.github/workflows/release-pipeline.yml b/.github/workflows/release-pipeline.yml index 289958263bc..f229d261f98 100644 --- a/.github/workflows/release-pipeline.yml +++ b/.github/workflows/release-pipeline.yml @@ -447,6 +447,7 @@ jobs: CEREBRAS_API_KEY: ${{ secrets.CEREBRAS_API_KEY }} DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }} SARVAM_API_KEY: ${{ secrets.SARVAM_API_KEY }} + TYPESAFE_API_KEY: ${{ secrets.TYPESAFE_API_KEY }} DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} DATABRICKS_WORKSPACE_URL: ${{ secrets.DATABRICKS_WORKSPACE_URL }} GITHUB_COPILOT_APP_ID: ${{ secrets.GITHUB_COPILOT_APP_ID }} @@ -678,6 +679,7 @@ jobs: CEREBRAS_API_KEY: ${{ secrets.CEREBRAS_API_KEY }} DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }} SARVAM_API_KEY: ${{ secrets.SARVAM_API_KEY }} + TYPESAFE_API_KEY: ${{ secrets.TYPESAFE_API_KEY }} DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} DATABRICKS_WORKSPACE_URL: ${{ secrets.DATABRICKS_WORKSPACE_URL }} GITHUB_COPILOT_APP_ID: ${{ secrets.GITHUB_COPILOT_APP_ID }} @@ -798,6 +800,101 @@ jobs: id: approve run: echo "approved=true" >> $GITHUB_OUTPUT + + # Memory-regression gate. + # + # Two production memory bugs shipped in quick succession with opposite + # signatures: a raw-JSON rewrite that was 23.6% of all allocation and 0% of + # the live heap, and a leaked watcher goroutine that was ~0% of allocation and + # ~80% of the live heap. Unit tests passed throughout both. + # + # These assert allocation SHAPE (grows with input, not with input squared) and + # that every loop-bound raw-JSON mutation in core/providers has been + # classified, so a new provider cannot add the same bug silently. + test-memory: + needs: [check-skip, detect-changes] + if: needs.check-skip.outputs.should-skip != 'true' && needs.check-skip.outputs.skip-tests != 'true' && (needs.detect-changes.outputs.core-needs-release == 'true' || needs.detect-changes.outputs.bifrost-http-needs-release == 'true') + runs-on: ${{ vars.BIFROST_RUNNER_AMD64 || 'ubuntu-24.04' }} + permissions: + contents: read + steps: + - name: Harden Runner + uses: step-security/harden-runner@fe104658747b27e96e4f7e80cd0a94068e53901d # v2.16.1 + with: + egress-policy: block + allowed-endpoints: *test-core-egress + + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + # This job uploads JUnit reports written by checked-out test code, and + # actions/checkout persists its token in git config by default. Dropping it + # removes the only path by which that token could reach an artifact. + persist-credentials: false + + - name: Set up Go + uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 + with: + go-version: "1.27.0" + + - name: Run memory regression tests + run: | + # Transport tests need the core/framework/plugins from this checkout. + source ./.github/workflows/scripts/setup-go-workspace.sh + make test-memory + + - name: Upload memory test report + if: ${{ !cancelled() }} + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: memory-regression-report + path: test-reports/memory-*.xml + retention-days: 30 + + # Break-glass for the memory gate. + # + # Unlike a flaky provider call, these assertions are deterministic: a failure + # here is a real regression, not noise. This exists so a release is never + # permanently wedged by it, but approving means consciously shipping a known + # memory regression. Prefer fixing forward. + approve-memory-regression: + needs: [check-skip, detect-changes, test-memory] + if: | + always() && + needs.check-skip.outputs.should-skip != 'true' && + needs.check-skip.outputs.skip-tests != 'true' && + (needs.detect-changes.outputs.core-needs-release == 'true' || needs.detect-changes.outputs.bifrost-http-needs-release == 'true') && + needs.test-memory.result == 'failure' + runs-on: ${{ vars.BIFROST_RUNNER_AMD64 || 'ubuntu-24.04' }} + environment: + name: memory-regression-override + url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + outputs: + approved: ${{ steps.approve.outputs.approved }} + steps: + - name: Harden Runner + uses: step-security/harden-runner@fe104658747b27e96e4f7e80cd0a94068e53901d # v2.16.1 + with: + egress-policy: block + allowed-endpoints: >+ + + - name: Display failed test info + run: | + echo "::warning::test-memory failed. These assertions are deterministic - this is a real regression, not a flake." + echo "Download the memory-regression-report artifact to see which assertion failed." + echo "" + echo "An allocation-scaling failure means a function now allocates with the SQUARE of its input." + echo "The usual cause is an sjson Set/Delete inside a loop: each call reserialises the whole" + echo "document, so N edits cost N copies of it. Batch the edits into a single write instead." + echo "" + echo "A coverage failure means a loop-bound raw-JSON mutation was added without being classified." + echo "Add it to reviewedSites in core/internal/memtest/coverage_test.go." + echo "" + echo "Approving ships a known memory regression. Prefer fixing forward." + - name: Mark as approved + id: approve + run: echo "approved=true" >> $GITHUB_OUTPUT + # Resolves the three most recent published versions of each coding CLI, fresh # on every run - nothing is pinned in the repo. Its output fans out into # test-cli-harness's matrix below. @@ -1259,6 +1356,7 @@ jobs: SGL_API_KEY: ${{ secrets.SGL_API_KEY }} CEREBRAS_API_KEY: ${{ secrets.CEREBRAS_API_KEY }} SARVAM_API_KEY: ${{ secrets.SARVAM_API_KEY }} + TYPESAFE_API_KEY: ${{ secrets.TYPESAFE_API_KEY }} DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} DATABRICKS_WORKSPACE_URL: ${{ secrets.DATABRICKS_WORKSPACE_URL }} COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} @@ -1387,6 +1485,7 @@ jobs: SGL_API_KEY: ${{ secrets.SGL_API_KEY }} CEREBRAS_API_KEY: ${{ secrets.CEREBRAS_API_KEY }} SARVAM_API_KEY: ${{ secrets.SARVAM_API_KEY }} + TYPESAFE_API_KEY: ${{ secrets.TYPESAFE_API_KEY }} DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} DATABRICKS_WORKSPACE_URL: ${{ secrets.DATABRICKS_WORKSPACE_URL }} COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} @@ -1719,6 +1818,7 @@ jobs: CEREBRAS_API_KEY: ${{ secrets.CEREBRAS_API_KEY }} DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }} SARVAM_API_KEY: ${{ secrets.SARVAM_API_KEY }} + TYPESAFE_API_KEY: ${{ secrets.TYPESAFE_API_KEY }} DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} DATABRICKS_WORKSPACE_URL: ${{ secrets.DATABRICKS_WORKSPACE_URL }} GITHUB_COPILOT_APP_ID: ${{ secrets.GITHUB_COPILOT_APP_ID }} @@ -2069,6 +2169,7 @@ jobs: PERPLEXITY_API_KEY: ${{ secrets.PERPLEXITY_API_KEY }} CEREBRAS_API_KEY: ${{ secrets.CEREBRAS_API_KEY }} SARVAM_API_KEY: ${{ secrets.SARVAM_API_KEY }} + TYPESAFE_API_KEY: ${{ secrets.TYPESAFE_API_KEY }} DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} DATABRICKS_WORKSPACE_URL: ${{ secrets.DATABRICKS_WORKSPACE_URL }} GITHUB_COPILOT_APP_ID: ${{ secrets.GITHUB_COPILOT_APP_ID }} @@ -2219,6 +2320,7 @@ jobs: PERPLEXITY_API_KEY: ${{ secrets.PERPLEXITY_API_KEY }} CEREBRAS_API_KEY: ${{ secrets.CEREBRAS_API_KEY }} SARVAM_API_KEY: ${{ secrets.SARVAM_API_KEY }} + TYPESAFE_API_KEY: ${{ secrets.TYPESAFE_API_KEY }} DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} DATABRICKS_WORKSPACE_URL: ${{ secrets.DATABRICKS_WORKSPACE_URL }} GITHUB_COPILOT_APP_ID: ${{ secrets.GITHUB_COPILOT_APP_ID }} @@ -2263,6 +2365,8 @@ jobs: test-core-integrations, test-core-unit, approve-flaky-test-core, + test-memory, + approve-memory-regression, resolve-cli-versions, test-cli-harness, test-vllm, @@ -2275,7 +2379,7 @@ jobs: test-docker-image-amd64, test-docker-image-arm64, ] - if: "always() && needs.check-skip.outputs.should-skip != 'true' && needs.detect-changes.result == 'success' && needs.detect-changes.outputs.core-needs-release == 'true' && (needs.test-core.result == 'success' || (needs.test-core.result == 'skipped' && (needs.check-skip.outputs.skip-tests == 'true' || needs.check-skip.outputs.skip-core-test == 'true')) || (needs.test-core.result == 'failure' && needs.approve-flaky-test-core.result == 'success')) && (needs.test-core-integrations.result == 'success' || (needs.test-core-integrations.result == 'skipped' && (needs.check-skip.outputs.skip-tests == 'true' || needs.check-skip.outputs.skip-core-test == 'true'))) && (needs.test-core-unit.result == 'success' || (needs.test-core-unit.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.test-cli-harness.result == 'success' || (needs.test-cli-harness.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.test-vllm.result == 'success' || needs.test-vllm.result == 'skipped') && (needs.resolve-cli-versions.result == 'success' || (needs.resolve-cli-versions.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.test-framework.result == 'success' || (needs.test-framework.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.test-plugins.result == 'success' || (needs.test-plugins.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.test-bifrost-http.result == 'success' || needs.test-bifrost-http.result == 'skipped') && (needs.test-migrations.result == 'success' || needs.test-migrations.result == 'skipped') && (needs.test-api-integrations.result == 'success' || needs.test-api-integrations.result == 'skipped') && (needs.test-cost-accuracy.result == 'success' || needs.test-cost-accuracy.result == 'skipped') && (needs.test-docker-image-amd64.result == 'success' || needs.test-docker-image-amd64.result == 'skipped') && (needs.test-docker-image-arm64.result == 'success' || needs.test-docker-image-arm64.result == 'skipped')" + if: "always() && needs.check-skip.outputs.should-skip != 'true' && needs.detect-changes.result == 'success' && needs.detect-changes.outputs.core-needs-release == 'true' && (needs.test-core.result == 'success' || (needs.test-core.result == 'skipped' && (needs.check-skip.outputs.skip-tests == 'true' || needs.check-skip.outputs.skip-core-test == 'true')) || (needs.test-core.result == 'failure' && needs.approve-flaky-test-core.result == 'success')) && (needs.test-core-integrations.result == 'success' || (needs.test-core-integrations.result == 'skipped' && (needs.check-skip.outputs.skip-tests == 'true' || needs.check-skip.outputs.skip-core-test == 'true'))) && (needs.test-core-unit.result == 'success' || (needs.test-core-unit.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.test-cli-harness.result == 'success' || (needs.test-cli-harness.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.test-vllm.result == 'success' || needs.test-vllm.result == 'skipped') && (needs.resolve-cli-versions.result == 'success' || (needs.resolve-cli-versions.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.test-framework.result == 'success' || (needs.test-framework.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.test-plugins.result == 'success' || (needs.test-plugins.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.test-memory.result == 'success' || (needs.test-memory.result == 'skipped' && (needs.check-skip.outputs.skip-tests == 'true' || (needs.detect-changes.outputs.core-needs-release != 'true' && needs.detect-changes.outputs.bifrost-http-needs-release != 'true'))) || (needs.test-memory.result == 'failure' && needs.approve-memory-regression.result == 'success')) && (needs.test-bifrost-http.result == 'success' || needs.test-bifrost-http.result == 'skipped') && (needs.test-migrations.result == 'success' || needs.test-migrations.result == 'skipped') && (needs.test-api-integrations.result == 'success' || needs.test-api-integrations.result == 'skipped') && (needs.test-cost-accuracy.result == 'success' || needs.test-cost-accuracy.result == 'skipped') && (needs.test-docker-image-amd64.result == 'success' || needs.test-docker-image-amd64.result == 'skipped') && (needs.test-docker-image-arm64.result == 'success' || needs.test-docker-image-arm64.result == 'skipped')" runs-on: ${{ vars.BIFROST_RUNNER_AMD64 || 'ubuntu-24.04' }} permissions: contents: write @@ -2347,6 +2451,7 @@ jobs: SGL_API_KEY: ${{ secrets.SGL_API_KEY }} CEREBRAS_API_KEY: ${{ secrets.CEREBRAS_API_KEY }} SARVAM_API_KEY: ${{ secrets.SARVAM_API_KEY }} + TYPESAFE_API_KEY: ${{ secrets.TYPESAFE_API_KEY }} DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} DATABRICKS_WORKSPACE_URL: ${{ secrets.DATABRICKS_WORKSPACE_URL }} COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} @@ -2470,6 +2575,7 @@ jobs: SGL_API_KEY: ${{ secrets.SGL_API_KEY }} CEREBRAS_API_KEY: ${{ secrets.CEREBRAS_API_KEY }} SARVAM_API_KEY: ${{ secrets.SARVAM_API_KEY }} + TYPESAFE_API_KEY: ${{ secrets.TYPESAFE_API_KEY }} DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} DATABRICKS_WORKSPACE_URL: ${{ secrets.DATABRICKS_WORKSPACE_URL }} COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} @@ -2605,6 +2711,7 @@ jobs: SGL_API_KEY: ${{ secrets.SGL_API_KEY }} CEREBRAS_API_KEY: ${{ secrets.CEREBRAS_API_KEY }} SARVAM_API_KEY: ${{ secrets.SARVAM_API_KEY }} + TYPESAFE_API_KEY: ${{ secrets.TYPESAFE_API_KEY }} DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} DATABRICKS_WORKSPACE_URL: ${{ secrets.DATABRICKS_WORKSPACE_URL }} COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} @@ -2631,6 +2738,8 @@ jobs: test-core-integrations, test-core-unit, approve-flaky-test-core, + test-memory, + approve-memory-regression, resolve-cli-versions, test-cli-harness, test-framework, @@ -2645,7 +2754,7 @@ jobs: framework-release, plugins-release, ] - if: "always() && needs.check-skip.outputs.should-skip != 'true' && needs.detect-changes.result == 'success' && needs.detect-changes.outputs.bifrost-http-needs-release == 'true' && (needs.test-core.result == 'success' || (needs.test-core.result == 'skipped' && (needs.check-skip.outputs.skip-tests == 'true' || needs.check-skip.outputs.skip-core-test == 'true')) || (needs.test-core.result == 'failure' && needs.approve-flaky-test-core.result == 'success')) && (needs.test-core-integrations.result == 'success' || (needs.test-core-integrations.result == 'skipped' && (needs.check-skip.outputs.skip-tests == 'true' || needs.check-skip.outputs.skip-core-test == 'true'))) && (needs.test-core-unit.result == 'success' || (needs.test-core-unit.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.test-cli-harness.result == 'success' || (needs.test-cli-harness.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.resolve-cli-versions.result == 'success' || (needs.resolve-cli-versions.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.test-framework.result == 'success' || (needs.test-framework.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.test-plugins.result == 'success' || (needs.test-plugins.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.test-bifrost-http.result == 'success' || needs.test-bifrost-http.result == 'skipped') && (needs.test-migrations.result == 'success' || needs.test-migrations.result == 'skipped') && (needs.test-api-integrations.result == 'success' || needs.test-api-integrations.result == 'skipped') && (needs.test-cost-accuracy.result == 'success' || needs.test-cost-accuracy.result == 'skipped') && (needs.test-docker-image-amd64.result == 'success' || needs.test-docker-image-amd64.result == 'skipped') && (needs.test-docker-image-arm64.result == 'success' || needs.test-docker-image-arm64.result == 'skipped') && (needs.detect-changes.outputs.core-needs-release == 'false' || needs.core-release.result == 'success' || needs.core-release.result == 'skipped') && (needs.detect-changes.outputs.framework-needs-release == 'false' || needs.framework-release.result == 'success' || needs.framework-release.result == 'skipped') && (needs.detect-changes.outputs.plugins-need-release == 'false' || needs.plugins-release.result == 'success' || needs.plugins-release.result == 'skipped')" + if: "always() && needs.check-skip.outputs.should-skip != 'true' && needs.detect-changes.result == 'success' && needs.detect-changes.outputs.bifrost-http-needs-release == 'true' && (needs.test-core.result == 'success' || (needs.test-core.result == 'skipped' && (needs.check-skip.outputs.skip-tests == 'true' || needs.check-skip.outputs.skip-core-test == 'true')) || (needs.test-core.result == 'failure' && needs.approve-flaky-test-core.result == 'success')) && (needs.test-core-integrations.result == 'success' || (needs.test-core-integrations.result == 'skipped' && (needs.check-skip.outputs.skip-tests == 'true' || needs.check-skip.outputs.skip-core-test == 'true'))) && (needs.test-core-unit.result == 'success' || (needs.test-core-unit.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.test-cli-harness.result == 'success' || (needs.test-cli-harness.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.resolve-cli-versions.result == 'success' || (needs.resolve-cli-versions.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.test-framework.result == 'success' || (needs.test-framework.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.test-plugins.result == 'success' || (needs.test-plugins.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.test-memory.result == 'success' || (needs.test-memory.result == 'skipped' && (needs.check-skip.outputs.skip-tests == 'true' || (needs.detect-changes.outputs.core-needs-release != 'true' && needs.detect-changes.outputs.bifrost-http-needs-release != 'true'))) || (needs.test-memory.result == 'failure' && needs.approve-memory-regression.result == 'success')) && (needs.test-bifrost-http.result == 'success' || needs.test-bifrost-http.result == 'skipped') && (needs.test-migrations.result == 'success' || needs.test-migrations.result == 'skipped') && (needs.test-api-integrations.result == 'success' || needs.test-api-integrations.result == 'skipped') && (needs.test-cost-accuracy.result == 'success' || needs.test-cost-accuracy.result == 'skipped') && (needs.test-docker-image-amd64.result == 'success' || needs.test-docker-image-amd64.result == 'skipped') && (needs.test-docker-image-arm64.result == 'success' || needs.test-docker-image-arm64.result == 'skipped') && (needs.detect-changes.outputs.core-needs-release == 'false' || needs.core-release.result == 'success' || needs.core-release.result == 'skipped') && (needs.detect-changes.outputs.framework-needs-release == 'false' || needs.framework-release.result == 'success' || needs.framework-release.result == 'skipped') && (needs.detect-changes.outputs.plugins-need-release == 'false' || needs.plugins-release.result == 'success' || needs.plugins-release.result == 'skipped')" runs-on: ${{ vars.BIFROST_RUNNER_AMD64 || 'ubuntu-24.04' }} permissions: contents: read @@ -2922,6 +3031,8 @@ jobs: test-core-integrations, test-core-unit, approve-flaky-test-core, + test-memory, + approve-memory-regression, resolve-cli-versions, test-cli-harness, test-framework, @@ -2938,7 +3049,7 @@ jobs: bifrost-http-prep, bifrost-http-release, ] - if: "always() && needs.check-skip.outputs.should-skip != 'true' && needs.detect-changes.result == 'success' && needs.detect-changes.outputs.docker-needs-release == 'true' && (needs.test-core.result == 'success' || (needs.test-core.result == 'skipped' && (needs.check-skip.outputs.skip-tests == 'true' || needs.check-skip.outputs.skip-core-test == 'true')) || (needs.test-core.result == 'failure' && needs.approve-flaky-test-core.result == 'success')) && (needs.test-core-integrations.result == 'success' || (needs.test-core-integrations.result == 'skipped' && (needs.check-skip.outputs.skip-tests == 'true' || needs.check-skip.outputs.skip-core-test == 'true'))) && (needs.test-core-unit.result == 'success' || (needs.test-core-unit.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.test-cli-harness.result == 'success' || (needs.test-cli-harness.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.resolve-cli-versions.result == 'success' || (needs.resolve-cli-versions.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.test-framework.result == 'success' || (needs.test-framework.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.test-plugins.result == 'success' || (needs.test-plugins.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.test-bifrost-http.result == 'success' || needs.test-bifrost-http.result == 'skipped') && (needs.test-migrations.result == 'success' || needs.test-migrations.result == 'skipped') && (needs.test-api-integrations.result == 'success' || needs.test-api-integrations.result == 'skipped') && (needs.test-cost-accuracy.result == 'success' || needs.test-cost-accuracy.result == 'skipped') && (needs.test-docker-image-amd64.result == 'success' || needs.test-docker-image-amd64.result == 'skipped') && (needs.test-docker-image-arm64.result == 'success' || needs.test-docker-image-arm64.result == 'skipped') && (needs.detect-changes.outputs.core-needs-release == 'false' || needs.core-release.result == 'success' || needs.core-release.result == 'skipped') && (needs.detect-changes.outputs.framework-needs-release == 'false' || needs.framework-release.result == 'success' || needs.framework-release.result == 'skipped') && (needs.detect-changes.outputs.plugins-need-release == 'false' || needs.plugins-release.result == 'success' || needs.plugins-release.result == 'skipped') && (needs.detect-changes.outputs.bifrost-http-needs-release == 'false' || (needs.bifrost-http-prep.result == 'success' && needs.bifrost-http-release.result == 'success'))" + if: "always() && needs.check-skip.outputs.should-skip != 'true' && needs.detect-changes.result == 'success' && needs.detect-changes.outputs.docker-needs-release == 'true' && (needs.test-core.result == 'success' || (needs.test-core.result == 'skipped' && (needs.check-skip.outputs.skip-tests == 'true' || needs.check-skip.outputs.skip-core-test == 'true')) || (needs.test-core.result == 'failure' && needs.approve-flaky-test-core.result == 'success')) && (needs.test-core-integrations.result == 'success' || (needs.test-core-integrations.result == 'skipped' && (needs.check-skip.outputs.skip-tests == 'true' || needs.check-skip.outputs.skip-core-test == 'true'))) && (needs.test-core-unit.result == 'success' || (needs.test-core-unit.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.test-cli-harness.result == 'success' || (needs.test-cli-harness.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.resolve-cli-versions.result == 'success' || (needs.resolve-cli-versions.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.test-framework.result == 'success' || (needs.test-framework.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.test-plugins.result == 'success' || (needs.test-plugins.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.test-memory.result == 'success' || (needs.test-memory.result == 'skipped' && (needs.check-skip.outputs.skip-tests == 'true' || (needs.detect-changes.outputs.core-needs-release != 'true' && needs.detect-changes.outputs.bifrost-http-needs-release != 'true'))) || (needs.test-memory.result == 'failure' && needs.approve-memory-regression.result == 'success')) && (needs.test-bifrost-http.result == 'success' || needs.test-bifrost-http.result == 'skipped') && (needs.test-migrations.result == 'success' || needs.test-migrations.result == 'skipped') && (needs.test-api-integrations.result == 'success' || needs.test-api-integrations.result == 'skipped') && (needs.test-cost-accuracy.result == 'success' || needs.test-cost-accuracy.result == 'skipped') && (needs.test-docker-image-amd64.result == 'success' || needs.test-docker-image-amd64.result == 'skipped') && (needs.test-docker-image-arm64.result == 'success' || needs.test-docker-image-arm64.result == 'skipped') && (needs.detect-changes.outputs.core-needs-release == 'false' || needs.core-release.result == 'success' || needs.core-release.result == 'skipped') && (needs.detect-changes.outputs.framework-needs-release == 'false' || needs.framework-release.result == 'success' || needs.framework-release.result == 'skipped') && (needs.detect-changes.outputs.plugins-need-release == 'false' || needs.plugins-release.result == 'success' || needs.plugins-release.result == 'skipped') && (needs.detect-changes.outputs.bifrost-http-needs-release == 'false' || (needs.bifrost-http-prep.result == 'success' && needs.bifrost-http-release.result == 'success'))" runs-on: ${{ vars.BIFROST_RUNNER_AMD64 || 'ubuntu-24.04' }} permissions: contents: write @@ -3055,6 +3166,8 @@ jobs: test-core-integrations, test-core-unit, approve-flaky-test-core, + test-memory, + approve-memory-regression, resolve-cli-versions, test-cli-harness, test-framework, @@ -3071,7 +3184,7 @@ jobs: bifrost-http-prep, bifrost-http-release, ] - if: "always() && needs.check-skip.outputs.should-skip != 'true' && needs.detect-changes.result == 'success' && needs.detect-changes.outputs.docker-needs-release == 'true' && (needs.test-core.result == 'success' || (needs.test-core.result == 'skipped' && (needs.check-skip.outputs.skip-tests == 'true' || needs.check-skip.outputs.skip-core-test == 'true')) || (needs.test-core.result == 'failure' && needs.approve-flaky-test-core.result == 'success')) && (needs.test-core-integrations.result == 'success' || (needs.test-core-integrations.result == 'skipped' && (needs.check-skip.outputs.skip-tests == 'true' || needs.check-skip.outputs.skip-core-test == 'true'))) && (needs.test-core-unit.result == 'success' || (needs.test-core-unit.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.test-cli-harness.result == 'success' || (needs.test-cli-harness.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.resolve-cli-versions.result == 'success' || (needs.resolve-cli-versions.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.test-framework.result == 'success' || (needs.test-framework.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.test-plugins.result == 'success' || (needs.test-plugins.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.test-bifrost-http.result == 'success' || needs.test-bifrost-http.result == 'skipped') && (needs.test-migrations.result == 'success' || needs.test-migrations.result == 'skipped') && (needs.test-api-integrations.result == 'success' || needs.test-api-integrations.result == 'skipped') && (needs.test-cost-accuracy.result == 'success' || needs.test-cost-accuracy.result == 'skipped') && (needs.test-docker-image-amd64.result == 'success' || needs.test-docker-image-amd64.result == 'skipped') && (needs.test-docker-image-arm64.result == 'success' || needs.test-docker-image-arm64.result == 'skipped') && (needs.detect-changes.outputs.core-needs-release == 'false' || needs.core-release.result == 'success' || needs.core-release.result == 'skipped') && (needs.detect-changes.outputs.framework-needs-release == 'false' || needs.framework-release.result == 'success' || needs.framework-release.result == 'skipped') && (needs.detect-changes.outputs.plugins-need-release == 'false' || needs.plugins-release.result == 'success' || needs.plugins-release.result == 'skipped') && (needs.detect-changes.outputs.bifrost-http-needs-release == 'false' || (needs.bifrost-http-prep.result == 'success' && needs.bifrost-http-release.result == 'success'))" + if: "always() && needs.check-skip.outputs.should-skip != 'true' && needs.detect-changes.result == 'success' && needs.detect-changes.outputs.docker-needs-release == 'true' && (needs.test-core.result == 'success' || (needs.test-core.result == 'skipped' && (needs.check-skip.outputs.skip-tests == 'true' || needs.check-skip.outputs.skip-core-test == 'true')) || (needs.test-core.result == 'failure' && needs.approve-flaky-test-core.result == 'success')) && (needs.test-core-integrations.result == 'success' || (needs.test-core-integrations.result == 'skipped' && (needs.check-skip.outputs.skip-tests == 'true' || needs.check-skip.outputs.skip-core-test == 'true'))) && (needs.test-core-unit.result == 'success' || (needs.test-core-unit.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.test-cli-harness.result == 'success' || (needs.test-cli-harness.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.resolve-cli-versions.result == 'success' || (needs.resolve-cli-versions.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.test-framework.result == 'success' || (needs.test-framework.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.test-plugins.result == 'success' || (needs.test-plugins.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.test-memory.result == 'success' || (needs.test-memory.result == 'skipped' && (needs.check-skip.outputs.skip-tests == 'true' || (needs.detect-changes.outputs.core-needs-release != 'true' && needs.detect-changes.outputs.bifrost-http-needs-release != 'true'))) || (needs.test-memory.result == 'failure' && needs.approve-memory-regression.result == 'success')) && (needs.test-bifrost-http.result == 'success' || needs.test-bifrost-http.result == 'skipped') && (needs.test-migrations.result == 'success' || needs.test-migrations.result == 'skipped') && (needs.test-api-integrations.result == 'success' || needs.test-api-integrations.result == 'skipped') && (needs.test-cost-accuracy.result == 'success' || needs.test-cost-accuracy.result == 'skipped') && (needs.test-docker-image-amd64.result == 'success' || needs.test-docker-image-amd64.result == 'skipped') && (needs.test-docker-image-arm64.result == 'success' || needs.test-docker-image-arm64.result == 'skipped') && (needs.detect-changes.outputs.core-needs-release == 'false' || needs.core-release.result == 'success' || needs.core-release.result == 'skipped') && (needs.detect-changes.outputs.framework-needs-release == 'false' || needs.framework-release.result == 'success' || needs.framework-release.result == 'skipped') && (needs.detect-changes.outputs.plugins-need-release == 'false' || needs.plugins-release.result == 'success' || needs.plugins-release.result == 'skipped') && (needs.detect-changes.outputs.bifrost-http-needs-release == 'false' || (needs.bifrost-http-prep.result == 'success' && needs.bifrost-http-release.result == 'success'))" runs-on: ${{ vars.BIFROST_RUNNER_ARM64 || 'ubuntu-24.04-arm' }} permissions: contents: write @@ -3247,6 +3360,8 @@ jobs: test-core-integrations, test-core-unit, approve-flaky-test-core, + test-memory, + approve-memory-regression, resolve-cli-versions, test-cli-harness, test-framework, @@ -3263,7 +3378,7 @@ jobs: bifrost-http-prep, bifrost-http-release, ] - if: "always() && needs.check-skip.outputs.should-skip != 'true' && needs.detect-changes.result == 'success' && needs.detect-changes.outputs.docker-needs-release == 'true' && (needs.test-core.result == 'success' || (needs.test-core.result == 'skipped' && (needs.check-skip.outputs.skip-tests == 'true' || needs.check-skip.outputs.skip-core-test == 'true')) || (needs.test-core.result == 'failure' && needs.approve-flaky-test-core.result == 'success')) && (needs.test-core-integrations.result == 'success' || (needs.test-core-integrations.result == 'skipped' && (needs.check-skip.outputs.skip-tests == 'true' || needs.check-skip.outputs.skip-core-test == 'true'))) && (needs.test-core-unit.result == 'success' || (needs.test-core-unit.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.test-cli-harness.result == 'success' || (needs.test-cli-harness.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.resolve-cli-versions.result == 'success' || (needs.resolve-cli-versions.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.test-framework.result == 'success' || (needs.test-framework.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.test-plugins.result == 'success' || (needs.test-plugins.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.test-bifrost-http.result == 'success' || needs.test-bifrost-http.result == 'skipped') && (needs.test-migrations.result == 'success' || needs.test-migrations.result == 'skipped') && (needs.test-api-integrations.result == 'success' || needs.test-api-integrations.result == 'skipped') && (needs.test-cost-accuracy.result == 'success' || needs.test-cost-accuracy.result == 'skipped') && (needs.test-docker-image-amd64.result == 'success' || needs.test-docker-image-amd64.result == 'skipped') && (needs.test-docker-image-arm64.result == 'success' || needs.test-docker-image-arm64.result == 'skipped') && (needs.detect-changes.outputs.core-needs-release == 'false' || needs.core-release.result == 'success' || needs.core-release.result == 'skipped') && (needs.detect-changes.outputs.framework-needs-release == 'false' || needs.framework-release.result == 'success' || needs.framework-release.result == 'skipped') && (needs.detect-changes.outputs.plugins-need-release == 'false' || needs.plugins-release.result == 'success' || needs.plugins-release.result == 'skipped') && (needs.detect-changes.outputs.bifrost-http-needs-release == 'false' || (needs.bifrost-http-prep.result == 'success' && needs.bifrost-http-release.result == 'success'))" + if: "always() && needs.check-skip.outputs.should-skip != 'true' && needs.detect-changes.result == 'success' && needs.detect-changes.outputs.docker-needs-release == 'true' && (needs.test-core.result == 'success' || (needs.test-core.result == 'skipped' && (needs.check-skip.outputs.skip-tests == 'true' || needs.check-skip.outputs.skip-core-test == 'true')) || (needs.test-core.result == 'failure' && needs.approve-flaky-test-core.result == 'success')) && (needs.test-core-integrations.result == 'success' || (needs.test-core-integrations.result == 'skipped' && (needs.check-skip.outputs.skip-tests == 'true' || needs.check-skip.outputs.skip-core-test == 'true'))) && (needs.test-core-unit.result == 'success' || (needs.test-core-unit.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.test-cli-harness.result == 'success' || (needs.test-cli-harness.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.resolve-cli-versions.result == 'success' || (needs.resolve-cli-versions.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.test-framework.result == 'success' || (needs.test-framework.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.test-plugins.result == 'success' || (needs.test-plugins.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.test-memory.result == 'success' || (needs.test-memory.result == 'skipped' && (needs.check-skip.outputs.skip-tests == 'true' || (needs.detect-changes.outputs.core-needs-release != 'true' && needs.detect-changes.outputs.bifrost-http-needs-release != 'true'))) || (needs.test-memory.result == 'failure' && needs.approve-memory-regression.result == 'success')) && (needs.test-bifrost-http.result == 'success' || needs.test-bifrost-http.result == 'skipped') && (needs.test-migrations.result == 'success' || needs.test-migrations.result == 'skipped') && (needs.test-api-integrations.result == 'success' || needs.test-api-integrations.result == 'skipped') && (needs.test-cost-accuracy.result == 'success' || needs.test-cost-accuracy.result == 'skipped') && (needs.test-docker-image-amd64.result == 'success' || needs.test-docker-image-amd64.result == 'skipped') && (needs.test-docker-image-arm64.result == 'success' || needs.test-docker-image-arm64.result == 'skipped') && (needs.detect-changes.outputs.core-needs-release == 'false' || needs.core-release.result == 'success' || needs.core-release.result == 'skipped') && (needs.detect-changes.outputs.framework-needs-release == 'false' || needs.framework-release.result == 'success' || needs.framework-release.result == 'skipped') && (needs.detect-changes.outputs.plugins-need-release == 'false' || needs.plugins-release.result == 'success' || needs.plugins-release.result == 'skipped') && (needs.detect-changes.outputs.bifrost-http-needs-release == 'false' || (needs.bifrost-http-prep.result == 'success' && needs.bifrost-http-release.result == 'success'))" runs-on: ${{ vars.BIFROST_RUNNER_AMD64 || 'ubuntu-24.04' }} permissions: contents: write @@ -3384,6 +3499,8 @@ jobs: test-core-integrations, test-core-unit, approve-flaky-test-core, + test-memory, + approve-memory-regression, resolve-cli-versions, test-cli-harness, test-framework, @@ -3400,7 +3517,7 @@ jobs: bifrost-http-prep, bifrost-http-release, ] - if: "always() && needs.check-skip.outputs.should-skip != 'true' && needs.detect-changes.result == 'success' && needs.detect-changes.outputs.docker-needs-release == 'true' && (needs.test-core.result == 'success' || (needs.test-core.result == 'skipped' && (needs.check-skip.outputs.skip-tests == 'true' || needs.check-skip.outputs.skip-core-test == 'true')) || (needs.test-core.result == 'failure' && needs.approve-flaky-test-core.result == 'success')) && (needs.test-core-integrations.result == 'success' || (needs.test-core-integrations.result == 'skipped' && (needs.check-skip.outputs.skip-tests == 'true' || needs.check-skip.outputs.skip-core-test == 'true'))) && (needs.test-core-unit.result == 'success' || (needs.test-core-unit.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.test-cli-harness.result == 'success' || (needs.test-cli-harness.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.resolve-cli-versions.result == 'success' || (needs.resolve-cli-versions.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.test-framework.result == 'success' || (needs.test-framework.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.test-plugins.result == 'success' || (needs.test-plugins.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.test-bifrost-http.result == 'success' || needs.test-bifrost-http.result == 'skipped') && (needs.test-migrations.result == 'success' || needs.test-migrations.result == 'skipped') && (needs.test-api-integrations.result == 'success' || needs.test-api-integrations.result == 'skipped') && (needs.test-cost-accuracy.result == 'success' || needs.test-cost-accuracy.result == 'skipped') && (needs.test-docker-image-amd64.result == 'success' || needs.test-docker-image-amd64.result == 'skipped') && (needs.test-docker-image-arm64.result == 'success' || needs.test-docker-image-arm64.result == 'skipped') && (needs.detect-changes.outputs.core-needs-release == 'false' || needs.core-release.result == 'success' || needs.core-release.result == 'skipped') && (needs.detect-changes.outputs.framework-needs-release == 'false' || needs.framework-release.result == 'success' || needs.framework-release.result == 'skipped') && (needs.detect-changes.outputs.plugins-need-release == 'false' || needs.plugins-release.result == 'success' || needs.plugins-release.result == 'skipped') && (needs.detect-changes.outputs.bifrost-http-needs-release == 'false' || (needs.bifrost-http-prep.result == 'success' && needs.bifrost-http-release.result == 'success'))" + if: "always() && needs.check-skip.outputs.should-skip != 'true' && needs.detect-changes.result == 'success' && needs.detect-changes.outputs.docker-needs-release == 'true' && (needs.test-core.result == 'success' || (needs.test-core.result == 'skipped' && (needs.check-skip.outputs.skip-tests == 'true' || needs.check-skip.outputs.skip-core-test == 'true')) || (needs.test-core.result == 'failure' && needs.approve-flaky-test-core.result == 'success')) && (needs.test-core-integrations.result == 'success' || (needs.test-core-integrations.result == 'skipped' && (needs.check-skip.outputs.skip-tests == 'true' || needs.check-skip.outputs.skip-core-test == 'true'))) && (needs.test-core-unit.result == 'success' || (needs.test-core-unit.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.test-cli-harness.result == 'success' || (needs.test-cli-harness.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.resolve-cli-versions.result == 'success' || (needs.resolve-cli-versions.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.test-framework.result == 'success' || (needs.test-framework.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.test-plugins.result == 'success' || (needs.test-plugins.result == 'skipped' && needs.check-skip.outputs.skip-tests == 'true')) && (needs.test-memory.result == 'success' || (needs.test-memory.result == 'skipped' && (needs.check-skip.outputs.skip-tests == 'true' || (needs.detect-changes.outputs.core-needs-release != 'true' && needs.detect-changes.outputs.bifrost-http-needs-release != 'true'))) || (needs.test-memory.result == 'failure' && needs.approve-memory-regression.result == 'success')) && (needs.test-bifrost-http.result == 'success' || needs.test-bifrost-http.result == 'skipped') && (needs.test-migrations.result == 'success' || needs.test-migrations.result == 'skipped') && (needs.test-api-integrations.result == 'success' || needs.test-api-integrations.result == 'skipped') && (needs.test-cost-accuracy.result == 'success' || needs.test-cost-accuracy.result == 'skipped') && (needs.test-docker-image-amd64.result == 'success' || needs.test-docker-image-amd64.result == 'skipped') && (needs.test-docker-image-arm64.result == 'success' || needs.test-docker-image-arm64.result == 'skipped') && (needs.detect-changes.outputs.core-needs-release == 'false' || needs.core-release.result == 'success' || needs.core-release.result == 'skipped') && (needs.detect-changes.outputs.framework-needs-release == 'false' || needs.framework-release.result == 'success' || needs.framework-release.result == 'skipped') && (needs.detect-changes.outputs.plugins-need-release == 'false' || needs.plugins-release.result == 'success' || needs.plugins-release.result == 'skipped') && (needs.detect-changes.outputs.bifrost-http-needs-release == 'false' || (needs.bifrost-http-prep.result == 'success' && needs.bifrost-http-release.result == 'success'))" runs-on: ${{ vars.BIFROST_RUNNER_ARM64 || 'ubuntu-24.04-arm' }} permissions: contents: write diff --git a/.github/workflows/run-core-tests.yml b/.github/workflows/run-core-tests.yml index d7f68b93d6c..277d8a430b1 100644 --- a/.github/workflows/run-core-tests.yml +++ b/.github/workflows/run-core-tests.yml @@ -60,6 +60,7 @@ jobs: CEREBRAS_API_KEY: ${{ secrets.CEREBRAS_API_KEY }} DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }} SARVAM_API_KEY: ${{ secrets.SARVAM_API_KEY }} + TYPESAFE_API_KEY: ${{ secrets.TYPESAFE_API_KEY }} DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} DATABRICKS_WORKSPACE_URL: ${{ secrets.DATABRICKS_WORKSPACE_URL }} GITHUB_COPILOT_APP_ID: ${{ secrets.GITHUB_COPILOT_APP_ID }} diff --git a/.github/workflows/scripts/check-egress-allowlist.sh b/.github/workflows/scripts/check-egress-allowlist.sh index 1431418ce03..30b0aa9f050 100755 --- a/.github/workflows/scripts/check-egress-allowlist.sh +++ b/.github/workflows/scripts/check-egress-allowlist.sh @@ -152,6 +152,7 @@ HARNESS_MARKERS = ("test-core.sh", "test-provider-harness.sh") NOT_DIALLED_HOSTS = { # Fetched by the provider's own server-side tooling, never by Bifrost. "example.com": "MCP server_url and web_fetch targets - dereferenced by the provider", + "mcp.deepwiki.com": "Anthropic mcp_servers url (12.B17 beta-header row) - Anthropic's MCP connector dials it", "www.youtube.com": "Gemini video input - Google fetches the URL server-side", "en.wikipedia.org": "googleSearch excludeDomains filter value, not a URL anyone fetches", "placeholder.search.windows.net": "Azure AI Search placeholder in a request body; never resolved", diff --git a/.github/workflows/scripts/list-models-stub.js b/.github/workflows/scripts/list-models-stub.js index be83ac888ae..2e2ad746809 100644 --- a/.github/workflows/scripts/list-models-stub.js +++ b/.github/workflows/scripts/list-models-stub.js @@ -35,6 +35,15 @@ const MODELS = { ], }; +// Anthropic's list-models shape, which its provider client parses instead of +// OpenAI's. The api-management "VK Allow All Providers" folder (upstream #7375) +// needs a provider the VK grants by allow_all_providers alone, with no config of +// its own, to show up in the /v1/models fan-out; anthropic is that provider. +// Anthropic is stubbed for list-models only: it has no chat route here. +const ANTHROPIC_MODELS = [ + { id: "claude-haiku-4-5", type: "model", display_name: "Claude Haiku 4.5", created_at: "2025-10-01T00:00:00Z" }, +]; + const port = Number(process.argv[2]); if (!Number.isInteger(port) || port <= 0) { console.error("usage: node list-models-stub.js "); @@ -90,6 +99,11 @@ const server = http.createServer((req, res) => { return; } const match = /^\/([a-z0-9-]+)\/v1\/models\/?$/.exec(path); + if (match && req.method === "GET" && match[1] === "anthropic") { + const ids = ANTHROPIC_MODELS.map((m) => m.id); + res.end(JSON.stringify({ data: ANTHROPIC_MODELS, has_more: false, first_id: ids[0], last_id: ids[ids.length - 1] })); + return; + } const models = match && req.method === "GET" ? MODELS[match[1]] : undefined; if (!models) { notFound(res, req, path); @@ -99,5 +113,5 @@ const server = http.createServer((req, res) => { }); server.listen(port, "127.0.0.1", () => { - console.log(`list-models stub listening on 127.0.0.1:${port} (${Object.keys(MODELS).join(", ")})`); + console.log(`list-models stub listening on 127.0.0.1:${port} (${Object.keys(MODELS).concat("anthropic").join(", ")})`); }); diff --git a/.github/workflows/scripts/test-api-integrations.sh b/.github/workflows/scripts/test-api-integrations.sh index 3e3352791d0..4074ad3d20e 100755 --- a/.github/workflows/scripts/test-api-integrations.sh +++ b/.github/workflows/scripts/test-api-integrations.sh @@ -160,11 +160,12 @@ jq --arg host "$POSTGRES_HOST" --arg port "$POSTGRES_PORT" --arg user "$POSTGRES # VK-allowed provider (openai) that can list models and some disallowed ones # (mistral, groq) that can too, so its exclusion checks are not vacuous. The # governance suites send chat completions through openai, which the stub answers -# with fixed token counts (TXE-5026). A run with the real secret set keeps the -# real provider. +# with fixed token counts (TXE-5026). The VK Allow All Providers folder needs +# anthropic, which the VK grants without a config of its own, to list models too +# (upstream #7375). A run with the real secret set keeps the real provider. STUB_PORT="${LIST_MODELS_STUB_PORT:-18090}" STUBBED_PROVIDERS="" -for provider in openai mistral groq; do +for provider in openai mistral groq anthropic; do key_var="$(printf '%s' "$provider" | tr '[:lower:]' '[:upper:]')_API_KEY" if [ -z "${!key_var:-}" ]; then STUBBED_PROVIDERS="$STUBBED_PROVIDERS $provider" diff --git a/.github/workflows/scripts/test-release-gates.py b/.github/workflows/scripts/test-release-gates.py index be892127339..615fb394e94 100644 --- a/.github/workflows/scripts/test-release-gates.py +++ b/.github/workflows/scripts/test-release-gates.py @@ -223,6 +223,45 @@ def test_images_need_the_jobs_their_condition_reads(self): self.assertIn("bifrost-http-release", JOBS[publisher]["needs"]) +class MemoryGate(unittest.TestCase): + """test-memory runs for a core or bifrost-http release and gates every job that names + it in its condition. A failure passes only with the break-glass approval; a skip passes + only under skip-tests or when the gate had nothing to cover, as for an image-only + republish, which must not be stranded.""" + + # The break-glass job itself reads the result to run on a failure; it is not gated. + GATED = [job for job, spec in JOBS.items() if job != "approve-memory-regression" and "needs.test-memory.result" in str(spec.get("if", ""))] + + def test_the_gate_covers_every_image_publisher(self): + for publisher in DOCKER_PUBLISHERS: + with self.subTest(publisher=publisher): + self.assertIn(publisher, self.GATED) + + def test_an_image_only_release_is_not_blocked_by_the_skipped_gate(self): + context = transport_only_context({"test-memory": "skipped", "bifrost-http-prep": "skipped", "bifrost-http-release": "skipped"}, transport_release="false") + self.assertEqual(evaluate(str(JOBS["test-memory"]["if"]), context), "false") + for publisher in DOCKER_PUBLISHERS: + with self.subTest(publisher=publisher): + self.assertEqual(evaluate(str(JOBS[publisher]["if"]), context), "true") + + def test_a_transport_release_needs_the_gate_to_pass(self): + for result in ("skipped", "failure", "cancelled"): + context = transport_only_context({"test-memory": result, "approve-memory-regression": "skipped"}) + self.assertEqual(evaluate(str(JOBS["test-memory"]["if"]), context), "true") + for job in self.GATED: + with self.subTest(result=result, job=job): + self.assertEqual(evaluate(str(JOBS[job]["if"]), context), "false") + + def test_a_failed_gate_passes_only_with_approval(self): + for approval, expected in (("success", "true"), ("skipped", "false"), ("failure", "false")): + context = transport_only_context({"test-memory": "failure", "approve-memory-regression": approval}) + for publisher in PUBLISHERS: + if publisher not in self.GATED: + continue + with self.subTest(approval=approval, publisher=publisher): + self.assertEqual(evaluate(str(JOBS[publisher]["if"]), context), expected) + + class ReleaseJobsLeaveMainToReviewedPRs(unittest.TestCase): """main requires a reviewed codex-review check, so no release job writes it: generated changes leave as patch artifacts, and every artifact is built from the tested commit.""" diff --git a/AGENTS.md b/AGENTS.md index edb6b602f72..6d172b0b396 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -579,6 +579,8 @@ Any change under `core/`, `framework/`, `transports/bifrost-http/`, or `plugins/ These layers all sit on the request path, so any of them can change the bytes a client sees - and that end-to-end behaviour is what the harness exists to pin. A Go unit test proves the function does what you meant; only the harness proves the bytes a real client sends still come back correct through the whole stack. The gap between those two is where regressions live: a fail-soft that fires on one request shape and silently skips a sibling shape passes every unit test it has. +**Never skip any error status code in a harness test script.** Do not open a test with an early-return guard like `if ([401, 403, 429, 500, 502, 503, 504].indexOf(pm.response.code) !== -1) { return; }` — every unexpected status, including auth failures, rate limits, and 5xx, must fail the assertion loudly rather than silently passing the case. Assert the exact status (or bound) the case expects and include `pm.response.text()` in the failure message. + Write the case so it is **red before the change and green after**, and validate it structurally while developing — no live paid run needed: ```bash diff --git a/Makefile b/Makefile index 5acf4009aa3..fb230ead3a8 100644 --- a/Makefile +++ b/Makefile @@ -67,7 +67,7 @@ define EXPOSE_ENV fi endef -.PHONY: all help dev dev-pulse build-ui build build-cli run run-cli install-air install-pulse clean test test-cli install-ui setup-workspace work-init work-clean docs docker-image docker-run cleanup-enterprise mod-tidy test-integrations test-integrations-py test-integrations-ts install-playwright run-e2e run-e2e-ui run-e2e-headed run-e2e-api format ui install-newman run-provider-harness-test smoke-provider-harness-test run-cli-harness-test cli-harness-report test-harness-runner-lib run-video-costing-test list-video-costing-cases test-semantic-cache test-semantic-cache-complete _test-semantic-cache-complete-inner helm-index install-microsocks socks5-proxy install-tinyproxy http-proxy +.PHONY: test-memory all help dev dev-pulse build-ui build build-cli run run-cli install-air install-pulse clean test test-cli install-ui setup-workspace work-init work-clean docs docker-image docker-run cleanup-enterprise mod-tidy test-integrations test-integrations-py test-integrations-ts install-playwright run-e2e run-e2e-ui run-e2e-headed run-e2e-api format ui install-newman run-provider-harness-test smoke-provider-harness-test run-cli-harness-test cli-harness-report test-harness-runner-lib run-video-costing-test list-video-costing-cases test-semantic-cache test-semantic-cache-complete _test-semantic-cache-complete-inner helm-index install-microsocks socks5-proxy install-tinyproxy http-proxy all: help @@ -853,6 +853,29 @@ test-plugins: install-gotestsum ## Run plugin tests SUMMARY_STRIP="plugin-" \ SUMMARY_FILES="$(TEST_REPORTS_DIR)/plugin-*.xml" +test-memory: install-gotestsum ## Run memory-regression tests (allocation scaling + coverage gate) + @$(ECHO) "$(GREEN)Running memory regression tests...$(NC)" + @$(ECHO) "$(CYAN)Asserts allocation grows with input size, not its square, and that every$(NC)" + @$(ECHO) "$(CYAN)loop-bound raw-JSON mutation in core/providers has been classified.$(NC)" + @mkdir -p $(TEST_REPORTS_DIR) + @rc=0; \ + for mod in core transports; do \ + pkgs=$$(cd $$mod && grep -rl "_AllocationScaling\|_Retention" --include="*_test.go" . 2>/dev/null | xargs -n1 dirname | sort -u); \ + if [ "$$mod" = "core" ]; then pkgs="./internal/memtest $$(echo "$$pkgs" | grep -v '^./internal/memtest$$')"; fi; \ + [ -z "$$(echo $$pkgs)" ] && continue; \ + $(ECHO) "$(CYAN)$$mod:$(NC)"; \ + for p in $$pkgs; do $(ECHO) " $$p"; done; \ + ( cd $$mod && gotestsum \ + --format=$(GOTESTSUM_FORMAT) \ + --junitfile=$(CURDIR)/$(TEST_REPORTS_DIR)/memory-$$mod.xml \ + -- -timeout 15m -count=1 \ + -run '_AllocationScaling|_Retention|TestEveryLoopBoundJSONMutationIsReviewed|TestCoveredSitesHaveARealTest' \ + $$pkgs ) || rc=1; \ + done; \ + exit $$rc + @$(ECHO) "" + @$(ECHO) "$(CYAN)JUnit XML report saved to $(TEST_REPORTS_DIR)/memory-*.xml$(NC)" + test-framework: install-gotestsum ## Run framework tests @$(EXPOSE_ENV); \ $(ECHO) "$(GREEN)Running framework tests...$(NC)"; \ @@ -1446,24 +1469,24 @@ test-integrations-py: ## Run Python integration tests (Usage: make test-integrat if [ -n "$(TESTCASE)" ]; then \ $(ECHO) "$(CYAN)Running $(INTEGRATION) integration test: $(TESTCASE)...$(NC)"; \ if [[ "$(TESTCASE)" == *::* ]]; then \ - uv run pytest tests/test_$(INTEGRATION).py::$(TESTCASE) $(if $(VERBOSE),-v,-q) || TEST_FAILED=1; \ + uv run --frozen pytest tests/test_$(INTEGRATION).py::$(TESTCASE) $(if $(VERBOSE),-v,-q) || TEST_FAILED=1; \ else \ - uv run pytest tests/test_$(INTEGRATION).py -k "$(TESTCASE)" $(if $(VERBOSE),-v,-q) || TEST_FAILED=1; \ + uv run --frozen pytest tests/test_$(INTEGRATION).py -k "$(TESTCASE)" $(if $(VERBOSE),-v,-q) || TEST_FAILED=1; \ fi; \ elif [ -n "$(PATTERN)" ]; then \ $(ECHO) "$(CYAN)Running $(INTEGRATION) integration tests matching '$(PATTERN)'...$(NC)"; \ - uv run pytest tests/test_$(INTEGRATION).py -k "$(PATTERN)" $(if $(VERBOSE),-v,-q) || TEST_FAILED=1; \ + uv run --frozen pytest tests/test_$(INTEGRATION).py -k "$(PATTERN)" $(if $(VERBOSE),-v,-q) || TEST_FAILED=1; \ else \ $(ECHO) "$(CYAN)Running $(INTEGRATION) integration tests...$(NC)"; \ - uv run pytest tests/test_$(INTEGRATION).py $(if $(VERBOSE),-v,-q) || TEST_FAILED=1; \ + uv run --frozen pytest tests/test_$(INTEGRATION).py $(if $(VERBOSE),-v,-q) || TEST_FAILED=1; \ fi; \ else \ if [ -n "$(PATTERN)" ]; then \ $(ECHO) "$(CYAN)Running all integration tests matching '$(PATTERN)'...$(NC)"; \ - uv run pytest -k "$(PATTERN)" $(if $(VERBOSE),-v,-q) || TEST_FAILED=1; \ + uv run --frozen pytest -k "$(PATTERN)" $(if $(VERBOSE),-v,-q) || TEST_FAILED=1; \ else \ $(ECHO) "$(CYAN)Running all integration tests...$(NC)"; \ - uv run pytest $(if $(VERBOSE),-v,-q) || TEST_FAILED=1; \ + uv run --frozen pytest $(if $(VERBOSE),-v,-q) || TEST_FAILED=1; \ fi; \ fi; \ fi; \ diff --git a/README.md b/README.md index 70d49e40e8b..130091408aa 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ ![Docker Pulls](https://img.shields.io/docker/pulls/maximhq/bifrost) [Run In Postman](https://app.getpostman.com/run-collection/31642484-2ba0e658-4dcd-49f4-845a-0c7ed745b916?action=collection%2Ffork&source=rip_markdown&collection-url=entityId%3D31642484-2ba0e658-4dcd-49f4-845a-0c7ed745b916%26entityType%3Dcollection%26workspaceId%3D63e853c8-9aec-477f-909c-7f02f543150e) [![Artifact Hub](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/bifrost)](https://artifacthub.io/packages/search?repo=bifrost) +[![CodSpeed](https://img.shields.io/endpoint?url=https://codspeed.io/badge.json)](https://app.codspeed.io/maximhq/bifrost?utm_source=badge) [![License](https://img.shields.io/github/license/maximhq/bifrost)](LICENSE) ## The fastest way to build AI applications that never go down diff --git a/core/azurestreamretry_test.go b/core/azurestreamretry_test.go index 666958a7792..9dc782a8f7a 100644 --- a/core/azurestreamretry_test.go +++ b/core/azurestreamretry_test.go @@ -285,6 +285,165 @@ func TestAzureResponsesOutputPreamble(t *testing.T) { } } +// OpenAI models emit startup events (response.created, in_progress, an empty +// role delta) before an in-stream error on every host that serves them, not +// only Azure. A retryable error must retry; a terminal one (an overload, which +// OpenAI sends with no HTTP status) must return synchronously so the provider +// fallback runs, instead of being forwarded inside an already-committed stream. +func TestOpenAIModelsRetryAfterStartupEvents(t *testing.T) { + responsesPreamble := func() []*schemas.BifrostStreamChunk { + return []*schemas.BifrostStreamChunk{ + {BifrostResponsesStreamResponse: &schemas.BifrostResponsesStreamResponse{ + Type: schemas.ResponsesStreamResponseTypeCreated, + }}, + {BifrostResponsesStreamResponse: &schemas.BifrostResponsesStreamResponse{ + Type: schemas.ResponsesStreamResponseTypeInProgress, + }}, + } + } + chatPreamble := func() []*schemas.BifrostStreamChunk { + return []*schemas.BifrostStreamChunk{{ + BifrostChatResponse: &schemas.BifrostChatResponse{ + Choices: []schemas.BifrostResponseChoice{{ + ChatStreamResponseChoice: &schemas.ChatStreamResponseChoice{ + Delta: &schemas.ChatStreamResponseChoiceDelta{ + Role: schemas.Ptr("assistant"), + }, + }, + }}, + }, + }} + } + requests := []struct { + name string + requestType schemas.RequestType + preamble func() []*schemas.BifrostStreamChunk + }{ + {"responses", schemas.ResponsesStreamRequest, responsesPreamble}, + {"chat", schemas.ChatCompletionStreamRequest, chatPreamble}, + } + failures := []struct { + name string + err func() *schemas.BifrostError + retryable bool + }{ + {"rate_limit", func() *schemas.BifrostError { + return createBifrostError("rate limit exceeded", nil, nil, false) + }, true}, + // Shape of an in-stream OpenAI overload after conversion: no status code. + {"overloaded", func() *schemas.BifrostError { + return &schemas.BifrostError{ + IsBifrostError: false, + Error: &schemas.ErrorField{ + Message: "The server is overloaded. Please try again later.", + Type: schemas.Ptr("server_error"), + Code: schemas.Ptr("server_is_overloaded"), + }, + } + }, false}, + } + hosts := []struct { + provider schemas.ModelProvider + baseProvider schemas.ModelProvider // set on ctx as the worker does for custom providers + model string + checkStartup bool + }{ + {schemas.OpenAI, "", "gpt-4o", true}, + {schemas.Bedrock, "", "openai.gpt-oss-120b-1:0", true}, + {schemas.BedrockMantle, "", "gpt-oss-120b", true}, + {schemas.Vertex, "", "openai/gpt-oss-20b", true}, + {schemas.ModelProvider("astra-openai"), "", "gpt-5", true}, + // OpenAI upstreams whose model ids carry no OpenAI family marker. + {schemas.OpenAI, "", "preamble-error", true}, + {schemas.ModelProvider("astra"), schemas.OpenAI, "astra-large", true}, + // Non-OpenAI families outside Azure keep first-chunk behavior. + {schemas.Vertex, "", "gemini-2.5-pro", false}, + } + + for _, host := range hosts { + for _, req := range requests { + for _, failure := range failures { + name := string(host.provider) + "/" + host.model + "/" + req.name + "/" + failure.name + t.Run(name, func(t *testing.T) { + parent, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + ctx := schemas.NewBifrostContext(parent, schemas.NoDeadline) + ctx.SetValue(schemas.BifrostContextKeyTracer, &schemas.NoOpTracer{}) + if host.baseProvider != "" { + ctx.SetValue(schemas.BifrostContextKeyBaseProviderType, host.baseProvider) + } + config := createTestConfig(1, time.Millisecond, time.Millisecond) + logger := NewDefaultLogger(schemas.LogLevelError) + attempts := 0 + success := &schemas.BifrostStreamChunk{ + BifrostResponsesStreamResponse: &schemas.BifrostResponsesStreamResponse{ + Type: schemas.ResponsesStreamResponseTypeCompleted, + }, + } + + handler := func(_ schemas.Key) (chan *schemas.BifrostStreamChunk, *schemas.BifrostError) { + attempts++ + stream := make(chan *schemas.BifrostStreamChunk, 4) + if attempts == 1 { + for _, chunk := range req.preamble() { + stream <- chunk + } + stream <- &schemas.BifrostStreamChunk{BifrostError: failure.err()} + } else { + stream <- success + } + close(stream) + return stream, nil + } + + stream, err := executeRequestWithRetries( + ctx, config, handler, nil, req.requestType, + host.provider, host.model, nil, logger, + ) + + switch { + case !host.checkStartup: + // The first startup event commits the stream; the error stays inside it. + if err != nil || attempts != 1 { + t.Fatalf("expected first-chunk behavior (stream, 1 attempt), got err=%v attempts=%d", err, attempts) + } + for range stream { + } + + case failure.retryable: + if err != nil || attempts != 2 { + t.Fatalf("expected retry to succeed (2 attempts), got err=%v attempts=%d", err, attempts) + } + select { + case chunk := <-stream: + if chunk != success { + t.Fatal("expected successful attempt; failed preamble must not escape") + } + case <-parent.Done(): + t.Fatal("timed out waiting for successful retry") + } + + default: + // Terminal on this provider: must surface synchronously so the + // caller's fallback chain runs, not ride inside the stream. + if err == nil { + for range stream { + } + t.Fatal("overload after startup events was committed into the stream; fallback cannot run") + } + if attempts != 1 { + t.Fatalf("expected 1 attempt for a non-retryable error, got %d", attempts) + } + if err.AllowFallbacks != nil && !*err.AllowFallbacks { + t.Fatal("overload must stay fallback-eligible") + } + } + }) + } + } + } +} + func TestAzureChatRetriesAfterStartupEvents(t *testing.T) { parent, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() diff --git a/core/bifrost.go b/core/bifrost.go index 2b373c72c39..b844bd5aa5b 100644 --- a/core/bifrost.go +++ b/core/bifrost.go @@ -14,7 +14,6 @@ import ( "sync/atomic" "time" - "github.com/bytedance/sonic" "github.com/google/uuid" "github.com/capsohq/bifrost/core/keyselectors" @@ -52,6 +51,7 @@ import ( "github.com/capsohq/bifrost/core/providers/runway" "github.com/capsohq/bifrost/core/providers/sarvam" "github.com/capsohq/bifrost/core/providers/sgl" + "github.com/capsohq/bifrost/core/providers/typesafe" providerUtils "github.com/capsohq/bifrost/core/providers/utils" "github.com/capsohq/bifrost/core/providers/vertex" "github.com/capsohq/bifrost/core/providers/vllm" @@ -133,6 +133,7 @@ type Bifrost struct { keySelector schemas.KeySelector // Custom key selector function keyPoolFilter schemas.KeyPoolFilter // optional hook to veto keys before selection (nil = all eligible) kvStore schemas.KVStore // optional KV store for session stickiness (nil = disabled) + sessionAffinity schemas.SessionAffinity // decides which key a session stays on; never nil after Init } // ProviderQueue wraps a provider's request channel with lifecycle management @@ -303,6 +304,12 @@ func Init(ctx context.Context, config schemas.BifrostConfig) (*Bifrost, error) { if bifrost.keySelector == nil { bifrost.keySelector = keyselectors.WeightedRandom } + // Always present, so key selection never checks for it: without a KV store the shipped + // key affinity has nowhere to keep a binding and declines every request. + bifrost.sessionAffinity = config.SessionAffinity + if bifrost.sessionAffinity == nil { + bifrost.sessionAffinity = NewSessionAffinity(bifrost.kvStore, bifrost.logger) + } // Initialize object pools bifrost.channelMessagePool = sync.Pool{ @@ -1444,6 +1451,58 @@ func (bifrost *Bifrost) RerankRequest(ctx *schemas.BifrostContext, req *schemas. return response.RerankResponse, nil } +// DecisionRequest sends an decision request to the specified provider. +func (bifrost *Bifrost) DecisionRequest(ctx *schemas.BifrostContext, req *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + if req == nil { + return nil, &schemas.BifrostError{ + IsBifrostError: false, + Error: &schemas.ErrorField{ + Message: "decision request is nil", + }, + ExtraFields: schemas.BifrostErrorExtraFields{ + RequestType: schemas.DecisionRequest, + }, + } + } + if req.State == nil { + return nil, &schemas.BifrostError{ + IsBifrostError: false, + Error: &schemas.ErrorField{ + Message: "state not provided for decision request", + }, + ExtraFields: schemas.BifrostErrorExtraFields{ + RequestType: schemas.DecisionRequest, + Provider: req.Provider, + OriginalModelRequested: req.Model, + ResolvedModelUsed: req.Model, + }, + } + } + if len(req.Questions) == 0 { + return nil, &schemas.BifrostError{ + IsBifrostError: false, + Error: &schemas.ErrorField{ + Message: "questions not provided for decision request", + }, + ExtraFields: schemas.BifrostErrorExtraFields{ + RequestType: schemas.DecisionRequest, + Provider: req.Provider, + OriginalModelRequested: req.Model, + ResolvedModelUsed: req.Model, + }, + } + } + bifrostReq := bifrost.getBifrostRequest() + bifrostReq.RequestType = schemas.DecisionRequest + bifrostReq.DecisionRequest = req + + response, err := bifrost.handleRequest(ctx, bifrostReq) + if err != nil { + return nil, err + } + return response.DecisionResponse, nil +} + // OCRRequest sends an OCR request to the specified provider. func (bifrost *Bifrost) OCRRequest(ctx *schemas.BifrostContext, req *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { if req == nil { @@ -4598,6 +4657,8 @@ func (bifrost *Bifrost) createBaseProvider(providerKey schemas.ModelProvider, co return sarvam.NewSarvamProvider(config, bifrost.logger) case schemas.Databricks: return databricks.NewDatabricksProvider(config, bifrost.logger) + case schemas.Typesafe: + return typesafe.NewTypesafeProvider(config, bifrost.logger) default: return nil, fmt.Errorf("unsupported provider: %s", targetProviderKey) } @@ -5187,6 +5248,12 @@ func (bifrost *Bifrost) prepareFallbackRequest(req *schemas.BifrostRequest, fall tmp.Model = fallback.Model fallbackReq.RerankRequest = &tmp } + if req.DecisionRequest != nil { + tmp := *req.DecisionRequest + tmp.Provider = fallback.Provider + tmp.Model = fallback.Model + fallbackReq.DecisionRequest = &tmp + } if req.OCRRequest != nil { tmp := *req.OCRRequest tmp.Provider = fallback.Provider @@ -5294,6 +5361,13 @@ func (bifrost *Bifrost) handleRequest(ctx *schemas.BifrostContext, req *schemas. ctx = bifrost.ctx } + // What the caller asked for, before any hook rewrites it: the name under which a session + // remembers where these requests were served. Reported back with how the request ended. + requested := schemas.Route{Provider: provider, Model: model} + var served *schemas.Route + servedFallback := false + defer func() { bifrost.observeSessionOutcome(ctx, requested, served, servedFallback, bifrostErr) }() + // Reset first: bifrost.ctx is shared across every nil-ctx caller. ctx.ResetUpstreamLatency() // Whole-request start for top-down overhead (total minus upstream). On ctx so @@ -5327,6 +5401,8 @@ func (bifrost *Bifrost) handleRequest(ctx *schemas.BifrostContext, req *schemas. preReqPipeline := bifrost.getPluginPipeline() preReqPipeline.RunPreRequestHooks(ctx, req) bifrost.releasePluginPipeline(preReqPipeline) + // The session has the last word on the chain the hooks produced. + bifrost.resolveSessionRoute(ctx, requested, req) bifrost.endCoreSpan(setupSpan) // "miscellaneous" phase: pre-dispatch glue (field re-read + validation) on no other // span. Closed before tryRequest so pipeline-pre stays a separate bucket. @@ -5362,6 +5438,9 @@ func (bifrost *Bifrost) handleRequest(ctx *schemas.BifrostContext, req *schemas. // Check if we should proceed with fallbacks shouldTryFallbacks := bifrost.shouldTryFallbacks(req, primaryErr) if !shouldTryFallbacks { + if primaryErr == nil { + served = &schemas.Route{Provider: provider, Model: model} + } return primaryResult, primaryErr } @@ -5383,6 +5462,11 @@ func (bifrost *Bifrost) handleRequest(ctx *schemas.BifrostContext, req *schemas. ctx.AppendRoutingEngineLog(schemas.RoutingEngineCore, schemas.LogLevelInfo, fmt.Sprintf("Trying fallback %d/%d: %s/%s (previous attempt failed: %s)", i+1, len(fallbacks), fallback.Provider, fallback.Model, routingErrorSummary(lastErr))) ctx.SetValue(schemas.BifrostContextKeyFallbackRequestID, uuid.New().String()) clearCtxForFallback(ctx) + // Re-pin after the clear: RunPreRequestHooks, which commits a target's pin, runs once per request, not per fallback. + if keyID := strings.TrimSpace(fallback.KeyID); keyID != "" { + ctx.SetFallbackPinnedAPIKeyID(keyID) + ctx.AppendRoutingEngineLog(schemas.RoutingEngineCore, schemas.LogLevelInfo, fmt.Sprintf("Fallback %d/%d pinned to provider key %s", i+1, len(fallbacks), keyID)) + } // Start span for fallback attempt tracer := bifrost.getTracer() @@ -5413,6 +5497,8 @@ func (bifrost *Bifrost) handleRequest(ctx *schemas.BifrostContext, req *schemas. bifrost.logger.Debug("successfully used fallback provider %s with model %s", fallback.Provider, fallback.Model) ctx.AppendRoutingEngineLog(schemas.RoutingEngineCore, schemas.LogLevelInfo, fmt.Sprintf("Request served by fallback %s/%s (attempt %d/%d)", fallback.Provider, fallback.Model, i+1, len(fallbacks))) tracer.EndSpan(handle, schemas.SpanStatusOk, "") + served = &schemas.Route{Provider: fallback.Provider, Model: fallback.Model} + servedFallback = true return result, nil } @@ -5440,7 +5526,7 @@ func (bifrost *Bifrost) handleRequest(ctx *schemas.BifrostContext, req *schemas. // It handles plugin hooks, request validation, response processing, and fallback providers. // If the primary provider fails, it will try each fallback provider in order until one succeeds. // It is the wrapper for all streaming public API methods. -func (bifrost *Bifrost) handleStreamRequest(ctx *schemas.BifrostContext, req *schemas.BifrostRequest) (chan *schemas.BifrostStreamChunk, *schemas.BifrostError) { +func (bifrost *Bifrost) handleStreamRequest(ctx *schemas.BifrostContext, req *schemas.BifrostRequest) (stream chan *schemas.BifrostStreamChunk, bifrostErr *schemas.BifrostError) { defer bifrost.releaseBifrostRequest(req) provider, model, fallbacks := req.GetRequestFields() @@ -5449,6 +5535,13 @@ func (bifrost *Bifrost) handleStreamRequest(ctx *schemas.BifrostContext, req *sc ctx = bifrost.ctx } + // What the caller asked for, before any hook rewrites it: the name under which a session + // remembers where these requests were served. Reported back with how the request ended. + requested := schemas.Route{Provider: provider, Model: model} + var served *schemas.Route + servedFallback := false + defer func() { bifrost.observeSessionOutcome(ctx, requested, served, servedFallback, bifrostErr) }() + ctx.ResetUpstreamLatency() ctx.ResetStreamOverhead() // Whole-request start for overhead on the streaming short-circuit path; the @@ -5470,6 +5563,8 @@ func (bifrost *Bifrost) handleStreamRequest(ctx *schemas.BifrostContext, req *sc preReqPipeline := bifrost.getPluginPipeline() preReqPipeline.RunPreRequestHooks(ctx, req) bifrost.releasePluginPipeline(preReqPipeline) + // The session has the last word on the chain the hooks produced. + bifrost.resolveSessionRoute(ctx, requested, req) // "miscellaneous" phase: pre-dispatch glue (field re-read + validation) on no other // span. Mirrors handleRequest so streaming classifies this cost too. preDispatchSpan := bifrost.startCoreSpan(ctx, "miscellaneous") @@ -5504,6 +5599,9 @@ func (bifrost *Bifrost) handleStreamRequest(ctx *schemas.BifrostContext, req *sc // Check if we should proceed with fallbacks shouldTryFallbacks := bifrost.shouldTryFallbacks(req, primaryErr) if !shouldTryFallbacks { + if primaryErr == nil { + served = &schemas.Route{Provider: provider, Model: model} + } return primaryResult, primaryErr } @@ -5521,6 +5619,11 @@ func (bifrost *Bifrost) handleStreamRequest(ctx *schemas.BifrostContext, req *sc ctx.AppendRoutingEngineLog(schemas.RoutingEngineCore, schemas.LogLevelInfo, fmt.Sprintf("Trying fallback %d/%d: %s/%s (previous attempt failed: %s)", i+1, len(fallbacks), fallback.Provider, fallback.Model, routingErrorSummary(lastErr))) ctx.SetValue(schemas.BifrostContextKeyFallbackRequestID, uuid.New().String()) clearCtxForFallback(ctx) + // Re-pin after the clear: RunPreRequestHooks, which commits a target's pin, runs once per request, not per fallback. + if keyID := strings.TrimSpace(fallback.KeyID); keyID != "" { + ctx.SetFallbackPinnedAPIKeyID(keyID) + ctx.AppendRoutingEngineLog(schemas.RoutingEngineCore, schemas.LogLevelInfo, fmt.Sprintf("Fallback %d/%d pinned to provider key %s", i+1, len(fallbacks), keyID)) + } // Start span for fallback attempt tracer := bifrost.getTracer() @@ -5566,6 +5669,8 @@ func (bifrost *Bifrost) handleStreamRequest(ctx *schemas.BifrostContext, req *sc bifrost.logger.Debug("successfully used fallback provider %s with model %s", fallback.Provider, fallback.Model) ctx.AppendRoutingEngineLog(schemas.RoutingEngineCore, schemas.LogLevelInfo, fmt.Sprintf("Request served by fallback %s/%s (attempt %d/%d)", fallback.Provider, fallback.Model, i+1, len(fallbacks))) tracer.EndSpan(handle, schemas.SpanStatusOk, "") + served = &schemas.Route{Provider: fallback.Provider, Model: fallback.Model} + servedFallback = true return result, nil } @@ -6218,7 +6323,10 @@ func executeRequestWithRetries[T any]( logger schemas.Logger, ) (result T, bifrostError *schemas.BifrostError) { var attempts int - checkAzurePreamble := providerKey == schemas.Azure && IsStreamRequestType(requestType) + isStreamRequest := IsStreamRequestType(requestType) + // Whether the previous attempt buffered startup events; its stream-end + // markers must be cleared before the next attempt reads a fresh stream. + prevCheckedPreamble := false // Emit the terminal routing-engine entry on every return path — including // early returns from key-selection failures and tracer-missing — so the @@ -6553,7 +6661,7 @@ func executeRequestWithRetries[T any]( } // The previous failed stream has drained before reaching this retry. - if checkAzurePreamble && attempts > 0 { + if prevCheckedPreamble && attempts > 0 { ctx.ClearValue(schemas.BifrostContextKeyStreamEndIndicator) ctx.ClearValue(schemas.BifrostContextKeyStreamBodyExhausted) ctx.ClearValue(schemas.BifrostContextKeyStreamParkedAfterFinish) @@ -6563,7 +6671,17 @@ func executeRequestWithRetries[T any]( result, bifrostError = requestHandler(currentKey) // Detect errors carried inside HTTP 200 streams before returning success. - // Azure Chat and Responses may emit startup metadata before an error. + // Azure and OpenAI upstreams (including custom providers built on them, + // whatever their model ids) and OpenAI models on any other host (Bedrock, + // Bedrock Mantle, Vertex) may emit startup events (response.created, + // in_progress, an empty role delta) before an overload or throttle error. + // Checked after requestHandler so the key's resolved alias decides the + // model family. + baseProvider := schemas.ResolveBaseProvider(ctx, providerKey) + checkPreamble := isStreamRequest && + (baseProvider == schemas.Azure || baseProvider == schemas.OpenAI || + schemas.IsOpenAIModelFamily(ctx, model)) + prevCheckedPreamble = checkPreamble emptyStream := false if bifrostError == nil { if streamChan, ok := any(result).(chan *schemas.BifrostStreamChunk); ok { @@ -6572,7 +6690,7 @@ func executeRequestWithRetries[T any]( drainDone <-chan struct{} firstChunkErr *schemas.BifrostError ) - if checkAzurePreamble { + if checkPreamble { requestID, _ := ctx.Value(schemas.BifrostContextKeyRequestID).(string) checkedStream, drainDone, firstChunkErr = providerUtils.CheckStreamPreambleForError( ctx, requestID, streamChan, azure.IsStreamPreamble, @@ -7581,7 +7699,8 @@ func promptCacheResponsesRequest(ctx *schemas.BifrostContext, config *schemas.Pr } // prepareResponsesRequest returns the Responses request to dispatch for one attempt: -// prompt-cache breakpoints first, then embedded client tools promoted and namespace +// billing metadata restored for Anthropic models, prompt-cache breakpoints +// injected, then embedded client tools promoted and namespace // tools flattened when the target wire does not understand them. All steps are copy-on-write, so the shared // req.BifrostRequest keeps the caller's namespaces for a later fallback attempt against // a wire that does. @@ -7591,10 +7710,12 @@ func promptCacheResponsesRequest(ctx *schemas.BifrostContext, config *schemas.Pr // the per-provider default in providerUtils applies, keyed on the BASE provider so a // custom provider wrapping OpenAI is treated like OpenAI. func prepareResponsesRequest(ctx *schemas.BifrostContext, config *schemas.ProviderConfig, provider schemas.Provider, key schemas.Key, r *schemas.BifrostResponsesRequest) (*schemas.BifrostResponsesRequest, *schemas.BifrostError) { + r = restoreResponsesBillingHeader(ctx, provider.GetProviderKey(), r) r = promptCacheResponsesRequest(ctx, config, provider.GetProviderKey(), r) if r == nil { return nil, nil } + r = normalizeResponsesToolSchemas(ctx, r) var supported bool if capable, ok := provider.(schemas.ResponsesNamespaceToolProvider); ok { supported = capable.SupportsResponsesNamespaceTools(ctx, key, r.Model) @@ -7622,14 +7743,85 @@ func prepareResponsesRequest(ctx *schemas.BifrostContext, config *schemas.Provid return providerUtils.FlattenResponsesNamespaceTools(ctx, r) } +// normalizeResponsesToolSchemas applies toolSchemaPatternRewriter to the request's +// tool schemas, with the same copy-on-write guarantee as promptCacheResponsesRequest: +// a request whose model is outside the relaxed set, or whose tools need no +// rewrite, is returned unchanged and allocates nothing. +// +// This sits on the shared dispatch path rather than in any one provider because +// the affected models are reached through several: DeepSeek natively, +// moonshotai.kimi-k3 through Bedrock, and OpenAI-compatible gateways fronting +// either. +func normalizeResponsesToolSchemas(ctx *schemas.BifrostContext, r *schemas.BifrostResponsesRequest) *schemas.BifrostResponsesRequest { + if r == nil || r.Params == nil || len(r.Params.Tools) == 0 { + return r + } + rewrite := toolSchemaPatternRewriter(ctx, r.Model) + if rewrite == nil { + return r + } + tools, changed := providerUtils.RewriteResponsesToolSchemas(r.Params.Tools, rewrite) + if !changed { + return r + } + params := *r.Params + params.Tools = tools + cp := *r + cp.Params = ¶ms + return &cp +} + +// relaxedPatternRewriter is the tool-schema pattern rewrite for the two model +// families whose validators reject regex syntax that every other tested backend +// accepts: the lossless `\0` -> `\x00` normalization followed by the lossy +// lookaround strip. Built once so the per-request path allocates no closure. +var relaxedPatternRewriter = providerUtils.ComposePatternRewriters( + providerUtils.NormalizeRegexNULEscape, + providerUtils.StripRegexLookaround, +) + +// toolSchemaPatternRewriter returns the pattern rewrite for a request's model, or +// nil when the model's tool schemas must reach the provider untouched. +// +// This is a positive list on purpose. Verified live on 2026-09-22: OpenAI, +// Anthropic (directly and on Bedrock) and Gemini all accept both `\0` and +// lookaround; DeepSeek rejects `\0` with a 400; moonshotai.kimi-k3 on Bedrock +// rejects both with HTTP 200 and an empty event stream. Only the two families +// with a verified rejection are rewritten, so every other model keeps its exact +// bytes (and its prompt-cache key), and a new backend that chokes on a pattern +// fails loudly through the Bedrock empty-stream guard rather than being silently +// rewritten. The check is on the model, never the provider: kimi behind an +// OpenAI-compatible custom provider is still kimi, and Claude on Bedrock is still +// Claude. +func toolSchemaPatternRewriter(ctx *schemas.BifrostContext, model string) providerUtils.PatternRewriter { + canonical := schemas.ResolveCanonicalModel(ctx, model) + if schemas.IsMoonshotModel(canonical) || schemas.IsDeepSeekModel(canonical) { + return relaxedPatternRewriter + } + return nil +} + // promptCacheChatRequest is the Chat Completions parallel of // promptCacheResponsesRequest, with the same copy-on-write guarantee. +// It also strips Bedrock cachePoint markers when the attempt targets a wire or model +// that cannot act on them. func promptCacheChatRequest(ctx *schemas.BifrostContext, config *schemas.ProviderConfig, provider schemas.ModelProvider, r *schemas.BifrostChatRequest) *schemas.BifrostChatRequest { - if r == nil || config == nil { + if r == nil { + return r + } + baseProvider := schemas.ResolveBaseProvider(ctx, provider) + if !providerUtils.ChatCachePointsSupported(baseProvider, schemas.ResolveCanonicalModel(ctx, r.Model)) { + if input, stripped := providerUtils.StripChatCachePoints(r.Input); stripped { + cp := *r + cp.Input = input + r = &cp + } + } + if config == nil { return r } promptCache := providerUtils.ResolvePromptCacheConfig(ctx, config.PromptCache) - if !providerUtils.PromptCacheInjectionEnabled(promptCache, schemas.ResolveBaseProvider(ctx, provider), r.Model) { + if !providerUtils.PromptCacheInjectionEnabled(promptCache, baseProvider, r.Model) { return r } cp := *r @@ -7781,6 +7973,32 @@ func (bifrost *Bifrost) handleProviderRequest(provider schemas.Provider, config return nil, bifrostError } response.RerankResponse = rerankResponse + case schemas.DecisionRequest: + decisionResponse, bifrostError := provider.Decision(req.Context, key, req.BifrostRequest.DecisionRequest) + // A provider without native decision support returns unsupported_operation; + // emulate the judgment through that provider's model (tool-calling / + // structured output). Covers both an LLM named as the decision model and an + // LLM reached as a fallback - both flow through this one case. + if isUnsupportedOperation(bifrostError) { + // unsupported_operation also covers a policy denial (AllowedRequests without + // Decision). Emulate only when the operation is actually permitted, so a + // config that denies Decision cannot run it through the chat/responses path. + var customProviderConfig *schemas.CustomProviderConfig + if config != nil { + customProviderConfig = config.CustomProviderConfig + } + if denyErr := providerUtils.CheckOperationAllowed(provider.GetProviderKey(), customProviderConfig, schemas.DecisionRequest); denyErr != nil { + if req.BifrostRequest.DecisionRequest != nil { + denyErr.ExtraFields.OriginalModelRequested = req.BifrostRequest.DecisionRequest.Model + } + return nil, denyErr + } + decisionResponse, bifrostError = bifrost.emulateDecisionViaResponses(req.Context, provider, key, req.BifrostRequest.DecisionRequest) + } + if bifrostError != nil { + return nil, bifrostError + } + response.DecisionResponse = decisionResponse case schemas.OCRRequest: var customProviderConfig *schemas.CustomProviderConfig if config != nil { @@ -8974,6 +9192,7 @@ func resetBifrostRequest(req *schemas.BifrostRequest) { req.CompactionRequest = nil req.EmbeddingRequest = nil req.RerankRequest = nil + req.DecisionRequest = nil req.OCRRequest = nil req.SpeechRequest = nil req.TranscriptionRequest = nil @@ -9167,7 +9386,7 @@ func (bifrost *Bifrost) getKeysForBatchAndFileOps(ctx *schemas.BifrostContext, p // canRotate=false is returned for cases where the caller must always use the same key: // - SkipKeySelection (Claude Code OAuth passthrough to Anthropic; empty slice returned) // - Explicit BifrostContextKeyAPIKeyID / APIKeyName (user pinned a specific key) -// - Session stickiness (key persisted in KV store for the session lifetime) +// - Session affinity (the registered policy named a key for the request's session) // - Single-key pool (only one eligible key — rotation is a no-op, KV write skipped) // // canRotate=true is returned when there are two or more eligible keys and no pinning @@ -9291,105 +9510,25 @@ func (bifrost *Bifrost) selectKeyFromProviderForModelWithPool(ctx *schemas.Bifro return []schemas.Key{supportedKeys[0]}, false, nil } - // Session stickiness: on the first request for a session ID, the randomly selected key is - // persisted in the KV store. Subsequent requests reuse it for the session lifetime. The sticky - // key is intentionally kept fixed across all retry attempts — return it as a single-element - // pool with canRotate=false so rate-limit retries also stay on the same key. - sessionID := "" - if ctx != nil { - if id, ok := ctx.Value(schemas.BifrostContextKeySessionID).(string); ok && id != "" { - sessionID = id - } - } - fallbackIndex := 0 - if ctx != nil { - fallbackIndex, _ = ctx.Value(schemas.BifrostContextKeyFallbackIndex).(int) - } - stickinessActive := sessionID != "" && bifrost.kvStore != nil && fallbackIndex == 0 - - if stickinessActive { - kvKey := buildSessionKey(providerKey, sessionID, model) - ttl, _ := ctx.Value(schemas.BifrostContextKeySessionTTL).(time.Duration) - if ttl <= 0 { - ttl = schemas.DefaultSessionStickyTTL - } - - if cachedKey, found, stale := getCachedKeyFromStore(bifrost.kvStore, kvKey, supportedKeys); found { - if err := bifrost.kvStore.SetWithTTL(kvKey, cachedKey.ID, ttl); err != nil { - bifrost.logger.Warn("error setting session cache for provider=%s key_id=%s: %s", providerKey, cachedKey.ID, err.Error()) - } - return []schemas.Key{cachedKey}, false, nil - } else if stale { - if _, err := bifrost.kvStore.Delete(kvKey); err != nil { - bifrost.logger.Warn("error deleting stale session cache for provider=%s: %s", providerKey, err.Error()) - } - } - - selectedKey, err := bifrost.keySelector(ctx, supportedKeys, providerKey, model) - if err != nil { - return nil, false, err - } - - wasSet, err := bifrost.kvStore.SetNXWithTTL(kvKey, selectedKey.ID, ttl) - if err != nil { - bifrost.logger.Warn("error setting session cache for provider=%s key_id=%s: %s", providerKey, selectedKey.ID, err.Error()) - return []schemas.Key{selectedKey}, false, nil - } - if wasSet { - return []schemas.Key{selectedKey}, false, nil - } - - // Another concurrent request won the race — re-read the persisted key. - if currentKey, found, stale := getCachedKeyFromStore(bifrost.kvStore, kvKey, supportedKeys); found { - return []schemas.Key{currentKey}, false, nil - } else if stale { - if _, err := bifrost.kvStore.Delete(kvKey); err != nil { - bifrost.logger.Warn("error deleting stale session cache for provider=%s: %s", providerKey, err.Error()) + // Session affinity: a request that takes part and already has a key here stays on it. The + // policy decides what that means; core honors the key it names, for every attempt, as long + // as that key is one of the eligible ones. A policy naming anything else is ignored, so a + // disabled or model-incompatible key can never reach a request through it. + if schemas.IsSessionAffinityActive(ctx) { + if key, ok := bifrost.sessionAffinity.ResolveKey(ctx, providerKey, model, supportedKeys); ok { + for _, eligible := range supportedKeys { + if eligible.ID == key.ID { + return []schemas.Key{eligible}, false, nil + } } - return []schemas.Key{selectedKey}, false, nil + bifrost.logger.Warn("session affinity named key %s for provider %s, which is not in the eligible pool; selecting a key normally", key.ID, providerKey) } - - return []schemas.Key{selectedKey}, false, nil } // Normal case: return the full filtered pool with rotation enabled. return supportedKeys, true, nil } -// getCachedKeyFromStore retrieves a key ID from the KV store and looks it up in supportedKeys. -// Returns the matching Key, found (true if key exists in supportedKeys), and stale (true if -// KV contains an ID but it is not in supportedKeys—caller should delete before SetNXWithTTL). -func getCachedKeyFromStore(kvStore schemas.KVStore, kvKey string, supportedKeys []schemas.Key) (schemas.Key, bool, bool) { - raw, err := kvStore.Get(kvKey) - if err != nil { - return schemas.Key{}, false, false - } - - var cachedKeyID string - switch v := raw.(type) { - case string: - cachedKeyID = v - case []byte: - var s string - if err := sonic.Unmarshal(v, &s); err == nil { - cachedKeyID = s - } else { - cachedKeyID = string(v) - } - } - - if cachedKeyID != "" { - for _, k := range supportedKeys { - if k.ID == cachedKeyID { - return k, true, false - } - } - return schemas.Key{}, false, true - } - - return schemas.Key{}, false, false -} - // Shutdown gracefully stops all workers when triggered. // It closes all request channels and waits for workers to exit. func (bifrost *Bifrost) Shutdown() { diff --git a/core/bifrost_test.go b/core/bifrost_test.go index f6135b7b14f..68bb94cda9a 100644 --- a/core/bifrost_test.go +++ b/core/bifrost_test.go @@ -13,6 +13,7 @@ import ( "testing" "time" + "github.com/capsohq/bifrost/core/internal/memtest" mistralprovider "github.com/capsohq/bifrost/core/providers/mistral" schemas "github.com/capsohq/bifrost/core/schemas" "golang.org/x/text/cases" @@ -1214,12 +1215,13 @@ func (m *mockKVStore) Delete(key string) (bool, error) { return false, nil } -// Test selectKeyFromProviderForModelWithPool with session stickiness +// Test selectKeyFromProviderForModelWithPool with session stickiness: nothing is bound until a +// request is served, and a bound key then comes back alone with rotation off. func TestSelectKeyFromProviderForModel_SessionStickiness(t *testing.T) { kvStore := newMockKVStore() account := NewMockAccount() account.AddProvider(schemas.OpenAI, 5, 1000) - // Use 2 keys so we hit the keySelector path (single key returns early) + // Use 2 keys so the pool can rotate (single key returns early) account.SetKeysForProvider(schemas.OpenAI, []schemas.Key{ {ID: "key-a", Name: "Key A", Value: *schemas.NewSecretVar("sk-a"), Models: schemas.WhiteList{"*"}, Weight: 1}, {ID: "key-b", Name: "Key B", Value: *schemas.NewSecretVar("sk-b"), Models: schemas.WhiteList{"*"}, Weight: 1}, @@ -1245,40 +1247,44 @@ func TestSelectKeyFromProviderForModel_SessionStickiness(t *testing.T) { bfCtx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) bfCtx.SetValue(schemas.BifrostContextKeySessionID, "sess-123") - // First call: cache miss, keySelector runs, key stored; returns single-element pool (canRotate=false) + // First request: nothing bound, so the whole pool comes back with rotation allowed and the + // pool builder neither selects nor writes. keys1, canRotate1, err := bifrost.selectKeyFromProviderForModelWithPool(bfCtx, schemas.ChatCompletionRequest, schemas.OpenAI, "gpt-4", schemas.OpenAI) if err != nil { t.Fatalf("first selectKeyFromProviderForModelWithPool: %v", err) } - if canRotate1 { - t.Error("first call: canRotate should be false for session-sticky request") + if !canRotate1 || len(keys1) != 2 { + t.Fatalf("first call: got %d keys canRotate=%v, want the full pool with rotation", len(keys1), canRotate1) } - if len(keys1) != 1 || keys1[0].ID != "key-a" { - t.Errorf("first call: expected [key-a], got %v", keys1) + if keySelectorCalls != 0 { + t.Errorf("first call: the pool builder should not select, got %d keySelector calls", keySelectorCalls) } - if keySelectorCalls != 1 { - t.Errorf("first call: expected 1 keySelector call, got %d", keySelectorCalls) + kvKey := SessionStateKey(bfCtx, SessionStateKindKey, string(schemas.OpenAI), "gpt-4") + if _, err := kvStore.Get(kvKey); err == nil { + t.Error("session bound before anything served it") } - // Verify kvstore was written - kvKey := buildSessionKey(schemas.OpenAI, "sess-123", "gpt-4") + // The request is served by key-a, which binds the session to it. + bfCtx.SetValue(schemas.BifrostContextKeySelectedKeyID, "key-a") + route := schemas.Route{Provider: schemas.OpenAI, Model: "gpt-4"} + bifrost.observeSessionOutcome(bfCtx, route, &route, false, nil) if raw, err := kvStore.Get(kvKey); err != nil || raw != "key-a" { - t.Errorf("kvstore after first call: expected key-a, got %v (err=%v)", raw, err) + t.Errorf("kvstore after the request served: expected key-a, got %v (err=%v)", raw, err) } - // Second call: cache hit, same key returned, keySelector NOT called + // Second request: the bound key comes back alone, rotation off, selector not consulted. keys2, canRotate2, err := bifrost.selectKeyFromProviderForModelWithPool(bfCtx, schemas.ChatCompletionRequest, schemas.OpenAI, "gpt-4", schemas.OpenAI) if err != nil { t.Fatalf("second selectKeyFromProviderForModelWithPool: %v", err) } if canRotate2 { - t.Error("second call: canRotate should be false for session-sticky request") + t.Error("second call: canRotate should be false for a session-bound key") } if len(keys2) != 1 || keys2[0].ID != "key-a" { - t.Errorf("second call: expected [key-a] (sticky), got %v", keys2) + t.Errorf("second call: expected [key-a] (bound), got %v", keys2) } - if keySelectorCalls != 1 { - t.Errorf("second call: keySelector should not run (cache hit), got %d calls", keySelectorCalls) + if keySelectorCalls != 0 { + t.Errorf("second call: keySelector should not run, got %d calls", keySelectorCalls) } } @@ -1328,13 +1334,16 @@ func TestSelectKeyFromProviderForModel_NoStickinessWithoutSessionID(t *testing.T t.Errorf("expected 0 keySelector calls from pool building (no session id), got %d", keySelectorCalls) } // KVStore should not have a sticky entry for an empty session id - if _, err := kvStore.Get(buildSessionKey(schemas.OpenAI, "", "gpt-4")); err == nil { - t.Error("kvstore should not have a sticky entry for an empty session id") + kvStore.mu.RLock() + entries := len(kvStore.data) + kvStore.mu.RUnlock() + if entries != 0 { + t.Errorf("kvstore should not have a sticky entry for an empty session id, got %d entries", entries) } } -// TestSelectKeyFromProviderForModel_SessionStickinessNoRotation verifies that when a session ID -// is present, rate-limit retries reuse the sticky key rather than rotating to another key. +// TestSelectKeyFromProviderForModel_SessionStickinessNoRotation verifies that once a session +// is bound to a key, rate-limit retries reuse that key rather than rotating to another. func TestSelectKeyFromProviderForModel_SessionStickinessNoRotation(t *testing.T) { kvStore := newMockKVStore() account := NewMockAccount() @@ -1363,6 +1372,11 @@ func TestSelectKeyFromProviderForModel_SessionStickinessNoRotation(t *testing.T) bfCtx.SetValue(schemas.BifrostContextKeySessionID, "sess-sticky") bfCtx.SetValue(schemas.BifrostContextKeyTracer, &schemas.NoOpTracer{}) + // An earlier request served by key-a bound the session to it. + bfCtx.SetValue(schemas.BifrostContextKeySelectedKeyID, "key-a") + route := schemas.Route{Provider: schemas.OpenAI, Model: "gpt-4"} + bifrost.observeSessionOutcome(bfCtx, route, &route, false, nil) + config := createTestConfig(3, 0, 0) logger := NewDefaultLogger(schemas.LogLevelError) @@ -3798,3 +3812,341 @@ func TestPrepareFallbackRequestRetargetsEveryFallbackCapableType(t *testing.T) { t.Fatal("no fallback-capable sub-request types found on BifrostRequest; the reflection walk is broken") } } + +// TestShutdown_RetentionNoGoroutineLeak is the core-side retention assertion. +// +// Bifrost runs a worker pool per provider, and a production profile showed 5120 of those +// goroutines on a single pod. Each holds channel references and, while serving, the +// request-scoped state that flows through them. A Shutdown that returns while workers are +// still parked leaks the whole pool plus everything it can still reach, and nothing in the +// suite asserted otherwise: the existing tests all `defer client.Shutdown()` and never +// check it did anything. +// +// This is the same class as the client-disconnect watcher leak that pinned roughly 2.0 GB +// of a 2.66 GB heap while GC ran perfectly, because every byte of it was reachable. +func TestShutdown_RetentionNoGoroutineLeak(t *testing.T) { + memtest.AssertNoGoroutineLeak(t, func() { + account := NewMockAccount() + account.AddProviderWithBaseURL(schemas.ModelProvider("custom-openai-shutdown"), 1, 1, "http://127.0.0.1:1") + account.SetKeysForProvider(schemas.ModelProvider("custom-openai-shutdown"), []schemas.Key{ + { + ID: "test-key-shutdown", + Value: *schemas.NewSecretVar("sk-test-shutdown"), + Models: schemas.WhiteList{"*"}, + Weight: 100, + }, + }) + account.SetCustomProviderConfig(schemas.ModelProvider("custom-openai-shutdown"), &schemas.CustomProviderConfig{ + BaseProviderType: schemas.OpenAI, + }) + + ctx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) + client, err := Init(ctx, schemas.BifrostConfig{ + Account: account, + Logger: NewDefaultLogger(schemas.LogLevelError), + }) + if err != nil { + t.Fatalf("Error initializing Bifrost: %v", err) + } + client.Shutdown() + }) +} + +// TestShutdown_RetentionClientReleased complements TestShutdown_RetentionNoGoroutineLeak. +// +// A goroutine count returning to baseline proves nothing is still running; it does not +// prove nothing still holds a reference. A package-level registry, a pool entry or a +// closure captured somewhere can keep the whole client (and its provider queues) alive +// with zero goroutines running. That is precisely the shape that made a production heap +// unreclaimable: not busy, just reachable. +// +// The weak pointer asserts unreachability directly rather than inferring it from a number. +func TestShutdown_RetentionClientReleased(t *testing.T) { + memtest.AssertReleased(t, "the Bifrost client after Shutdown", func() *Bifrost { + account := NewMockAccount() + account.AddProviderWithBaseURL(schemas.ModelProvider("custom-openai-release"), 1, 1, "http://127.0.0.1:1") + account.SetKeysForProvider(schemas.ModelProvider("custom-openai-release"), []schemas.Key{ + { + ID: "test-key-release", + Value: *schemas.NewSecretVar("sk-test-release"), + Models: schemas.WhiteList{"*"}, + Weight: 100, + }, + }) + account.SetCustomProviderConfig(schemas.ModelProvider("custom-openai-release"), &schemas.CustomProviderConfig{ + BaseProviderType: schemas.OpenAI, + }) + + ctx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) + client, err := Init(ctx, schemas.BifrostConfig{ + Account: account, + Logger: NewDefaultLogger(schemas.LogLevelError), + }) + if err != nil { + t.Fatalf("Error initializing Bifrost: %v", err) + } + client.Shutdown() + return client + }) +} + +// patternOf reads properties.p.pattern from a tool schema without a JSON round trip. +func patternOf(t *testing.T, params *schemas.ToolFunctionParameters) string { + t.Helper() + v, ok := params.Properties.Get("p") + if !ok { + t.Fatal("property p missing") + } + pm, ok := v.(*schemas.OrderedMap) + if !ok { + t.Fatalf("property p is %T, want *OrderedMap", v) + } + pat, _ := pm.Get("pattern") + s, _ := pat.(string) + return s +} + +func lookaroundToolParams() *schemas.ToolFunctionParameters { + prop := schemas.NewOrderedMap() + prop.Set("type", "string") + prop.Set("pattern", `^(?!\.\.?$)[^\0]{1,200}$`) + props := schemas.NewOrderedMap() + props.Set("p", prop) + return &schemas.ToolFunctionParameters{Type: "object", Properties: props} +} + +// Only Moonshot and DeepSeek models get their tool-schema patterns rewritten. +// Every other model, on every provider, must reach the wire byte-identical, and +// the no-op path must hand back the caller's request itself so nothing is copied. +func TestToolSchemaPatternRewriteAppliesOnlyToMoonshotAndDeepSeek(t *testing.T) { + ctx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) + cases := []struct { + model string + rewritten bool + }{ + {"gpt-4o-mini", false}, + {"claude-haiku-4-5", false}, + {"global.anthropic.claude-sonnet-5", false}, + {"gemini-3.6-flash", false}, + {"mistral-large-latest", false}, + {"kimina-prover", false}, // contains "kimi" but is not a Moonshot model + {"global.moonshotai.kimi-k3", true}, + {"kimi-k3", true}, + {"moonshotai/kimi-k2-instruct", true}, + {"deepseek-chat", true}, + {"deepseek-v4.1-flash", true}, + {"us.deepseek.r1-v1:0", true}, + } + for _, tc := range cases { + t.Run(tc.model, func(t *testing.T) { + req := &schemas.BifrostResponsesRequest{ + Model: tc.model, + Params: &schemas.ResponsesParameters{Tools: []schemas.ResponsesTool{{ + Type: schemas.ResponsesToolTypeFunction, + Name: schemas.Ptr("probe"), + ResponsesToolFunction: &schemas.ResponsesToolFunction{Parameters: lookaroundToolParams()}, + }}}, + } + out := normalizeResponsesToolSchemas(ctx, req) + got := patternOf(t, out.Params.Tools[0].ResponsesToolFunction.Parameters) + if !tc.rewritten { + if out != req { + t.Fatal("a model outside the relaxed set must get its own request back, not a copy") + } + if !strings.Contains(got, `\0`) || !strings.Contains(got, `(?!`) { + t.Fatalf("pattern was rewritten for a model outside the relaxed set: %q", got) + } + return + } + if strings.Contains(got, `\0`) || strings.Contains(got, `(?!`) || !strings.Contains(got, `\x00`) { + t.Fatalf("pattern not fully rewritten for a relaxed-set model: %q", got) + } + if orig := patternOf(t, req.Params.Tools[0].ResponsesToolFunction.Parameters); !strings.Contains(orig, `\0`) { + t.Fatalf("caller's request was mutated in place: %q", orig) + } + }) + } +} + +// TestSetFallbackPinnedAPIKeyID_SurvivesBlockedWrites covers the streaming race, where a prior attempt's async post-hooks hold blockRestrictedWrites. +func TestSetFallbackPinnedAPIKeyID_SurvivesBlockedWrites(t *testing.T) { + ctx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) + ctx.BlockRestrictedWrites() + defer ctx.UnblockRestrictedWrites() + + ctx.SetValue(schemas.BifrostContextKeyAPIKeyID, "dropped") + if pin, _ := ctx.Value(schemas.BifrostContextKeyAPIKeyID).(string); pin != "" { + t.Fatalf("SetValue on a reserved key should have been dropped, got %q", pin) + } + + ctx.SetFallbackPinnedAPIKeyID("fallback-key") + if pin, _ := ctx.Value(schemas.BifrostContextKeyAPIKeyID).(string); pin != "fallback-key" { + t.Fatalf("fallback pin did not land: got %q", pin) + } +} + +// TestClearCtxForFallback_ClearsPreviousFallbackPin keeps one fallback's pinned key from leaking into the next attempt. +func TestClearCtxForFallback_ClearsPreviousFallbackPin(t *testing.T) { + ctx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) + ctx.SetFallbackPinnedAPIKeyID("fallback-1-key") + + clearCtxForFallback(ctx) + + if pin, ok := ctx.Value(schemas.BifrostContextKeyAPIKeyID).(string); ok && pin != "" { + t.Fatalf("fallback pin survived clearCtxForFallback: %q", pin) + } +} + +// TestStreamFallbackUsesPinnedKey proves the pin, not the weighted selector, picks which credential goes upstream (the unpinned key carries all the weight). +func TestStreamFallbackUsesPinnedKey(t *testing.T) { + primary := httptest.NewServer(sseHandler(`{"error":{"message":"rate limited","type":"rate_limit_error"}}`)) + defer primary.Close() + + var gotAPIKey atomic.Value + fallback := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotAPIKey.Store(r.Header.Get("x-api-key")) + anthropicMessagesHandler()(w, r) + })) + defer fallback.Close() + + account := NewMockAccount() + account.AddProviderWithBaseURL(schemas.OpenAI, 1, 1, primary.URL) + account.AddProviderWithBaseURL(schemas.Anthropic, 1, 1, fallback.URL) + account.configs[schemas.OpenAI].NetworkConfig.MaxRetries = 0 + account.configs[schemas.Anthropic].NetworkConfig.MaxRetries = 0 + account.SetKeysForProvider(schemas.OpenAI, []schemas.Key{ + {ID: "primary-key", Value: *schemas.NewSecretVar("sk-primary"), Models: schemas.WhiteList{"*"}, Weight: 100}, + }) + account.SetKeysForProvider(schemas.Anthropic, []schemas.Key{ + {ID: "unpinned-key", Value: *schemas.NewSecretVar("sk-unpinned"), Models: schemas.WhiteList{"*"}, Weight: 100}, + {ID: "pinned-key", Value: *schemas.NewSecretVar("sk-pinned"), Models: schemas.WhiteList{"*"}, Weight: 0}, + }) + client := newStreamTestClient(t, account) + + ctx := schemas.NewBifrostContext(context.Background(), time.Now().Add(30*time.Second)) + stream, bifrostErr := client.ChatCompletionStreamRequest(ctx, &schemas.BifrostChatRequest{ + Provider: schemas.OpenAI, + Model: "gpt-4o-mini", + Input: []schemas.ChatMessage{ + {Role: schemas.ChatMessageRoleUser, Content: &schemas.ChatMessageContent{ContentStr: schemas.Ptr("hi")}}, + }, + Fallbacks: []schemas.Fallback{{ + Provider: schemas.Anthropic, + Model: "claude-3-5-haiku-20241022", + KeyID: "pinned-key", + }}, + }) + if bifrostErr != nil { + t.Fatalf("pinned fallback stream failed: %s", bifrostErr.Error.Message) + } + if _, errs := drainChatStream(stream); len(errs) > 0 { + t.Fatalf("pinned fallback stream emitted error chunks: %v", errs) + } + + if got, _ := gotAPIKey.Load().(string); got != "sk-pinned" { + t.Fatalf("fallback used api key %q, want %q", got, "sk-pinned") + } +} + +// TestFallbackWithUnknownPinnedKeyIsSkipped covers a pin that names no key in the provider's pool: the attempt is skipped, not load-balanced. +func TestFallbackWithUnknownPinnedKeyIsSkipped(t *testing.T) { + primary := httptest.NewServer(sseHandler(`{"error":{"message":"rate limited","type":"rate_limit_error"}}`)) + defer primary.Close() + + var hits atomic.Int32 + fallback := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + anthropicMessagesHandler()(w, r) + })) + defer fallback.Close() + + account := NewMockAccount() + account.AddProviderWithBaseURL(schemas.OpenAI, 1, 1, primary.URL) + account.AddProviderWithBaseURL(schemas.Anthropic, 1, 1, fallback.URL) + account.configs[schemas.OpenAI].NetworkConfig.MaxRetries = 0 + account.configs[schemas.Anthropic].NetworkConfig.MaxRetries = 0 + account.SetKeysForProvider(schemas.OpenAI, []schemas.Key{ + {ID: "primary-key", Value: *schemas.NewSecretVar("sk-primary"), Models: schemas.WhiteList{"*"}, Weight: 100}, + }) + account.SetKeysForProvider(schemas.Anthropic, []schemas.Key{ + {ID: "anthropic-key", Value: *schemas.NewSecretVar("sk-anthropic"), Models: schemas.WhiteList{"*"}, Weight: 100}, + }) + client := newStreamTestClient(t, account) + + ctx := schemas.NewBifrostContext(context.Background(), time.Now().Add(30*time.Second)) + stream, bifrostErr := client.ChatCompletionStreamRequest(ctx, &schemas.BifrostChatRequest{ + Provider: schemas.OpenAI, + Model: "gpt-4o-mini", + Input: []schemas.ChatMessage{ + {Role: schemas.ChatMessageRoleUser, Content: &schemas.ChatMessageContent{ContentStr: schemas.Ptr("hi")}}, + }, + Fallbacks: []schemas.Fallback{ + {Provider: schemas.Anthropic, Model: "claude-3-5-haiku-20241022", KeyID: "not-in-this-pool"}, + {Provider: schemas.Anthropic, Model: "claude-3-5-haiku-20241022"}, + }, + }) + if bifrostErr != nil { + t.Fatalf("chain did not recover after the mis-pinned fallback: %s", bifrostErr.Error.Message) + } + if _, errs := drainChatStream(stream); len(errs) > 0 { + t.Fatalf("stream emitted error chunks: %v", errs) + } + if got := hits.Load(); got != 1 { + t.Fatalf("fallback server hits = %d, want 1 (the mis-pinned attempt must not reach upstream)", got) + } +} + +// TestFallbackUsesPinnedKey is the non-streaming twin of TestStreamFallbackUsesPinnedKey; the two orchestrator loops are independent copies. +func TestFallbackUsesPinnedKey(t *testing.T) { + primary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusTooManyRequests) + fmt.Fprint(w, `{"error":{"message":"rate limited","type":"rate_limit_error"}}`) + })) + defer primary.Close() + + var gotAPIKey atomic.Value + fallback := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotAPIKey.Store(r.Header.Get("x-api-key")) + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"id":"msg_1","type":"message","role":"assistant","model":"claude-3-5-haiku-20241022",`+ + `"content":[{"type":"text","text":"hello"}],"stop_reason":"end_turn",`+ + `"usage":{"input_tokens":10,"output_tokens":2}}`) + })) + defer fallback.Close() + + account := NewMockAccount() + account.AddProviderWithBaseURL(schemas.OpenAI, 1, 1, primary.URL) + account.AddProviderWithBaseURL(schemas.Anthropic, 1, 1, fallback.URL) + account.configs[schemas.OpenAI].NetworkConfig.MaxRetries = 0 + account.configs[schemas.Anthropic].NetworkConfig.MaxRetries = 0 + account.SetKeysForProvider(schemas.OpenAI, []schemas.Key{ + {ID: "primary-key", Value: *schemas.NewSecretVar("sk-primary"), Models: schemas.WhiteList{"*"}, Weight: 100}, + }) + account.SetKeysForProvider(schemas.Anthropic, []schemas.Key{ + {ID: "unpinned-key", Value: *schemas.NewSecretVar("sk-unpinned"), Models: schemas.WhiteList{"*"}, Weight: 100}, + {ID: "pinned-key", Value: *schemas.NewSecretVar("sk-pinned"), Models: schemas.WhiteList{"*"}, Weight: 0}, + }) + client := newStreamTestClient(t, account) + + ctx := schemas.NewBifrostContext(context.Background(), time.Now().Add(30*time.Second)) + _, bifrostErr := client.ChatCompletionRequest(ctx, &schemas.BifrostChatRequest{ + Provider: schemas.OpenAI, + Model: "gpt-4o-mini", + Input: []schemas.ChatMessage{ + {Role: schemas.ChatMessageRoleUser, Content: &schemas.ChatMessageContent{ContentStr: schemas.Ptr("hi")}}, + }, + Fallbacks: []schemas.Fallback{{ + Provider: schemas.Anthropic, + Model: "claude-3-5-haiku-20241022", + KeyID: "pinned-key", + }}, + }) + if bifrostErr != nil { + t.Fatalf("pinned fallback failed: %s", bifrostErr.Error.Message) + } + + if got, _ := gotAPIKey.Load().(string); got != "sk-pinned" { + t.Fatalf("fallback used api key %q, want %q", got, "sk-pinned") + } +} diff --git a/core/billingheader.go b/core/billingheader.go new file mode 100644 index 00000000000..14ba00edd36 --- /dev/null +++ b/core/billingheader.go @@ -0,0 +1,20 @@ +package bifrost + +import "github.com/capsohq/bifrost/core/schemas" + +// restoreResponsesBillingHeader runs after alias resolution so each attempt uses +// the actual destination family. Ingress already removed the metadata before +// sharing Input: non-Anthropic attempts need neither a scan nor a slice copy. +func restoreResponsesBillingHeader(ctx *schemas.BifrostContext, provider schemas.ModelProvider, r *schemas.BifrostResponsesRequest) *schemas.BifrostResponsesRequest { + if r == nil { + return r + } + if ctx != nil && ctx.Value(schemas.BifrostContextKeyUseRawRequestBody) == true && len(r.RawRequestBody) > 0 { + return r // Native passthrough already carries the original billing block. + } + family := schemas.ResolveFamily(ctx, r.Model) + if family == schemas.ModelFamilyAnthropic || (family == "" && schemas.ResolveBaseProvider(ctx, provider) == schemas.Anthropic) { + return r.WithAnthropicBillingHeader() + } + return r +} diff --git a/core/decisionemulation.go b/core/decisionemulation.go new file mode 100644 index 00000000000..443caf2906e --- /dev/null +++ b/core/decisionemulation.go @@ -0,0 +1,239 @@ +package bifrost + +import ( + "errors" + "fmt" + + "github.com/bytedance/sonic" + providerUtils "github.com/capsohq/bifrost/core/providers/utils" + schemas "github.com/capsohq/bifrost/core/schemas" +) + +// errDecisionNoContent is returned when an emulating response carries neither the +// emit_decision function call nor a JSON-object message body. +var errDecisionNoContent = errors.New("model returned no decision function call or structured output") + +// isUnsupportedOperation reports whether an error is a provider's +// "operation not supported" signal (set by NewUnsupportedOperationError). +func isUnsupportedOperation(err *schemas.BifrostError) bool { + return err != nil && err.Error != nil && err.Error.Code != nil && *err.Error.Code == "unsupported_operation" +} + +// decisionSystemPrompt frames the judgment task for an emulating LLM. +const decisionSystemPrompt = "You are a judgment engine. Read the given state and answer every question by " + + "calling the provided function exactly once. For each question emit the requested value and your " + + "confidence from 0 to 1. For every choice and score question also report the full probability " + + "distribution over its options or levels; the probabilities must sum to 1. For choice, select an option " + + "with the highest probability. Base every answer only on the state; do not invent facts." + +// emulateDecisionViaResponses answers a decision request through a general model +// when the provider has no native decision support. It runs against the provider's +// own Responses API - the richest interface every provider implements (natively on +// openai/anthropic/gemini, via chat translation elsewhere) - encoding the questions +// as a forced function tool and mapping the function-call arguments back to the +// neutral DecisionResponse shape. Used for both the primary path (an LLM named as +// the decision model) and fallbacks (an LLM after the native provider fails) - both +// flow through the same dispatch case. +func (bifrost *Bifrost) emulateDecisionViaResponses( + ctx *schemas.BifrostContext, + provider schemas.Provider, + key schemas.Key, + req *schemas.BifrostDecisionRequest, +) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + if req == nil || len(req.Questions) == 0 { + return nil, providerUtils.NewBifrostBadRequestError("decision request requires at least one question") + } + + tool, err := providerUtils.BuildDecisionResponsesTool(req.Questions) + if err != nil { + return nil, providerUtils.NewBifrostBadRequestError(err.Error()) + } + + // State as the user message: string verbatim, structured as sorted JSON. + stateText, marshalErr := decisionStateText(req.State) + if marshalErr != nil { + return nil, providerUtils.NewBifrostBadRequestError("decision state could not be serialized: " + marshalErr.Error()) + } + + instructions := decisionSystemPrompt + userRole := schemas.ResponsesInputMessageRoleUser + toolChoice := decisionToolChoice(ctx, req.Provider, req.Model) + responsesReq := &schemas.BifrostResponsesRequest{ + Provider: req.Provider, + Model: req.Model, + Input: []schemas.ResponsesMessage{ + { + Role: &userRole, + Content: &schemas.ResponsesMessageContent{ContentStr: &stateText}, + }, + }, + Params: &schemas.ResponsesParameters{ + Instructions: &instructions, + Tools: []schemas.ResponsesTool{*tool}, + ToolChoice: &schemas.ResponsesToolChoice{ResponsesToolChoiceStr: &toolChoice}, + }, + } + + resp, respErr := provider.Responses(ctx, key, responsesReq) + if respErr != nil { + return nil, respErr + } + + argsJSON, extractErr := extractDecisionToolArguments(resp) + if extractErr != nil { + return nil, decisionAnswerError(extractErr, resp) + } + + answers, parseErr := providerUtils.ParseDecisionAnswers([]byte(argsJSON), req.Questions) + if parseErr != nil { + return nil, decisionAnswerError(parseErr, resp) + } + + decision := &schemas.BifrostDecisionResponse{ + Model: modelForDecisionResponse(resp, req), + Answers: answers, + } + // Carry the underlying response's id and resolved-model/latency metadata so + // downstream pricing and integrations see the model that actually handled the + // turn, not just the requested one. RequestType is deliberately not copied - + // post-hooks run before final normalization. + if resp != nil { + if resp.ID != nil { + decision.ID = *resp.ID + } + decision.ExtraFields.ResolvedModelUsed = resp.Model + decision.ExtraFields.Latency = resp.ExtraFields.Latency + if resp.Usage != nil { + decision.Usage = resp.Usage.ToBifrostLLMUsage() + } + decision.ServiceTier = resp.ServiceTier + decision.Speed = resp.Speed + decision.InferenceGeo = resp.InferenceGeo + } + // Provider/Model on ExtraFields drive downstream cost calculation. + decision.ExtraFields.Provider = req.Provider + decision.ExtraFields.OriginalModelRequested = req.Model + return decision, nil +} + +// decisionAnswerError reports a turn whose answer could not be used. The provider +// still answered and billed it, so the error carries that usage and the served +// model for governance and logging to charge. +func decisionAnswerError(cause error, resp *schemas.BifrostResponsesResponse) *schemas.BifrostError { + bifrostErr := providerUtils.NewBifrostOperationError(cause.Error(), nil) + if resp != nil { + if resp.Usage != nil { + billed := resp.Usage.ToBifrostLLMUsage() + billed.Speed = resp.Speed + billed.InferenceGeo = resp.InferenceGeo + billed.ServiceTier = resp.ServiceTier + bifrostErr.ExtraFields.BilledUsage = billed + } + bifrostErr.ExtraFields.ResolvedModelUsed = resp.Model + } + return bifrostErr +} + +// decisionToolChoice picks the tool_choice that forces the emit_decision call. +// It prefers the "required" mode over a named-function choice: emit_decision is +// the only tool, so "required" obliges the model to call it, and the string mode +// is accepted by OpenAI, Anthropic, and OpenAI-compatible providers like +// Perplexity that reject the named-function object. Bedrock Mantle's +// OpenAI-compatible surface is the exception: it accepts only "auto" for gpt-oss +// ("Supported options: [auto]"), so the call there rests on the system prompt, +// and extraction already accepts a lone call or a JSON text body. +func decisionToolChoice(ctx *schemas.BifrostContext, provider schemas.ModelProvider, model string) string { + // A custom provider wrapping Bedrock Mantle gets Mantle's default. + if schemas.ResolveBaseProvider(ctx, provider) == schemas.BedrockMantle && !schemas.IsAnthropicModelFamily(ctx, model) { + return string(schemas.ResponsesToolChoiceTypeAuto) + } + return string(schemas.ResponsesToolChoiceTypeRequired) +} + +// decisionStateText renders the state into a message body. +func decisionStateText(state interface{}) (string, error) { + if s, ok := state.(string); ok { + return s, nil + } + raw, err := providerUtils.MarshalSorted(state) + if err != nil { + return "", err + } + return string(raw), nil +} + +// modelForDecisionResponse prefers the model the response reported (the resolved +// model after any provider-side handoff), falling back to the request. +func modelForDecisionResponse(resp *schemas.BifrostResponsesResponse, req *schemas.BifrostDecisionRequest) string { + if resp != nil && resp.Model != "" { + return resp.Model + } + return req.Model +} + +// extractDecisionToolArguments pulls the emit_decision function-call arguments from +// the Responses output items. Falls back to a lone function call, then to a plain +// output-text message for models that answered via native structured output. +func extractDecisionToolArguments(resp *schemas.BifrostResponsesResponse) (string, error) { + if resp == nil || len(resp.Output) == 0 { + return "", errDecisionNoContent + } + + // Collect every function call, counting emit_decision calls separately, before + // returning: a duplicate or ambiguous set must be rejected rather than silently + // taking the first. Providers may drop MaxToolCalls/ParallelToolCalls on the + // Responses-to-Chat fallback, so extraction cannot assume a single call. + var namedCalls []string + var allCalls []string + for i := range resp.Output { + item := resp.Output[i] + if item.Type == nil || *item.Type != schemas.ResponsesMessageTypeFunctionCall { + continue + } + if item.ResponsesToolMessage == nil || item.ResponsesToolMessage.Arguments == nil { + continue + } + allCalls = append(allCalls, *item.ResponsesToolMessage.Arguments) + if item.ResponsesToolMessage.Name != nil && *item.ResponsesToolMessage.Name == providerUtils.DecisionToolName { + namedCalls = append(namedCalls, *item.ResponsesToolMessage.Arguments) + } + } + switch { + case len(namedCalls) > 1: + return "", fmt.Errorf("model returned %d %s calls; expected exactly one", len(namedCalls), providerUtils.DecisionToolName) + case len(namedCalls) == 1: + return namedCalls[0], nil + case len(allCalls) > 1: + // No call round-tripped the name, and there is more than one - ambiguous. + return "", fmt.Errorf("model returned %d tool calls with no unambiguous %s call", len(allCalls), providerUtils.DecisionToolName) + case len(allCalls) == 1: + // A single function call is acceptable even if the name did not round-trip. + return allCalls[0], nil + } + + // Native structured-output path: the JSON object arrives as output text. + for i := range resp.Output { + item := resp.Output[i] + if item.Type != nil && *item.Type != schemas.ResponsesMessageTypeMessage { + continue + } + if item.Content == nil { + continue + } + if item.Content.ContentStr != nil && isJSONObject(*item.Content.ContentStr) { + return *item.Content.ContentStr, nil + } + for _, block := range item.Content.ContentBlocks { + if block.Text != nil && isJSONObject(*block.Text) { + return *block.Text, nil + } + } + } + return "", errDecisionNoContent +} + +// isJSONObject reports whether s parses as a JSON object. +func isJSONObject(s string) bool { + var obj map[string]interface{} + return sonic.Unmarshal([]byte(s), &obj) == nil +} diff --git a/core/decisionemulation_test.go b/core/decisionemulation_test.go new file mode 100644 index 00000000000..04525831af6 --- /dev/null +++ b/core/decisionemulation_test.go @@ -0,0 +1,366 @@ +package bifrost + +import ( + "context" + "math" + "reflect" + "strings" + "testing" + + schemas "github.com/capsohq/bifrost/core/schemas" +) + +// decisionEmulationProvider satisfies schemas.Provider for the emulation seam; +// only Responses is implemented. It records the outbound request so tests can +// assert what actually reaches the emulating model, and returns a canned +// response. +type decisionEmulationProvider struct { + schemas.Provider + lastRequest *schemas.BifrostResponsesRequest + response *schemas.BifrostResponsesResponse + err *schemas.BifrostError +} + +func (p *decisionEmulationProvider) Responses(ctx *schemas.BifrostContext, key schemas.Key, req *schemas.BifrostResponsesRequest) (*schemas.BifrostResponsesResponse, *schemas.BifrostError) { + p.lastRequest = req + return p.response, p.err +} + +// emulationFunctionCallResponse wraps emit_decision arguments in the Responses +// function-call output shape the emulation path parses. +func emulationFunctionCallResponse(args string) *schemas.BifrostResponsesResponse { + fnType := schemas.ResponsesMessageTypeFunctionCall + name := "emit_decision" + return &schemas.BifrostResponsesResponse{ + Model: "gpt-4o-mini", + Output: []schemas.ResponsesMessage{{ + Type: &fnType, + ResponsesToolMessage: &schemas.ResponsesToolMessage{Name: &name, Arguments: &args}, + }}, + } +} + +// structuredDecisionRequest carries every allowed criteria type in every slot: +// string | object | array for noul keys and score levels, plus null for choice +// options - the same matrix the typesafe converter and the harness cases pin. +func structuredDecisionRequest() *schemas.BifrostDecisionRequest { + return &schemas.BifrostDecisionRequest{ + Provider: schemas.OpenAI, + Model: "gpt-4o-mini", + State: "Customer message: I was double charged and want a refund.", + Questions: map[string]schemas.DecisionQuestion{ + "approve": { + Kind: schemas.DecisionKindNoul, + Instructions: "Approve a billing review?", + Criteria: map[string]any{ + "true": map[string]any{"meaning": "clear billing error"}, + "false": []any{"no billing issue", "general inquiry"}, + }, + }, + "category": { + Kind: schemas.DecisionKindChoice, + Instructions: "Pick the ticket category", + Criteria: map[string]any{ + "billing": map[string]any{"rubric": "money issues"}, + "bug": []any{"crash", "defect"}, + "support": "service questions", + "other": nil, + }, + }, + "urgency": { + Kind: schemas.DecisionKindScore, + Instructions: "How urgent?", + Criteria: []any{ + "low", + map[string]any{"level": "high", "examples": []any{"outage"}}, + []any{"critical", "churn risk"}, + }, + }, + }, + } +} + +// questionDescription walks the emit_decision tool schema down to the value +// description of one question property. +func questionDescription(t *testing.T, req *schemas.BifrostResponsesRequest, question, field string) string { + t.Helper() + if req == nil || req.Params == nil || len(req.Params.Tools) != 1 { + t.Fatalf("expected exactly one tool on the outbound request: %+v", req) + } + tool := req.Params.Tools[0] + if tool.ResponsesToolFunction == nil || tool.ResponsesToolFunction.Parameters == nil { + t.Fatalf("tool carries no parameters: %+v", tool) + } + prop, ok := tool.ResponsesToolFunction.Parameters.Properties.Get(question) + if !ok { + t.Fatalf("question %q missing from tool schema", question) + } + nested, ok := prop.(map[string]any)["properties"].(map[string]any)[field].(map[string]any) + if !ok { + t.Fatalf("question %q has no %q property", question, field) + } + desc, _ := nested["description"].(string) + return desc +} + +func TestEmulateDecisionForwardsStructuredCriteria(t *testing.T) { + args := `{ + "approve": {"value": 0.9, "confidence": 0.8}, + "category": {"choice": "billing", "confidence": 0.7, "probabilities": {"billing": 0.7, "bug": 0.1, "support": 0.1, "other": 0.1}}, + "urgency": {"value": 2, "confidence": 0.6, "probabilities": {"0": 0.1, "1": 0.1, "2": 0.8}} + }` + provider := &decisionEmulationProvider{response: emulationFunctionCallResponse(args)} + + var b Bifrost + resp, bifrostErr := b.emulateDecisionViaResponses(nil, provider, schemas.Key{}, structuredDecisionRequest()) + if bifrostErr != nil { + t.Fatalf("unexpected error: %v", bifrostErr) + } + + // Outbound: every criteria rubric must reach the model's tool schema, + // strings verbatim and structured values as sorted JSON. + noulDesc := questionDescription(t, provider.lastRequest, "approve", "value") + for _, want := range []string{`true={"meaning":"clear billing error"}`, `false=["no billing issue","general inquiry"]`} { + if !strings.Contains(noulDesc, want) { + t.Errorf("noul description missing %q: %q", want, noulDesc) + } + } + choiceDesc := questionDescription(t, provider.lastRequest, "category", "choice") + for _, want := range []string{`{"rubric":"money issues"}`, `["crash","defect"]`, "service questions"} { + if !strings.Contains(choiceDesc, want) { + t.Errorf("choice description missing %q: %q", want, choiceDesc) + } + } + scoreDesc := questionDescription(t, provider.lastRequest, "urgency", "value") + for _, want := range []string{"0=low", `{"examples":["outage"],"level":"high"}`, `["critical","churn risk"]`} { + if !strings.Contains(scoreDesc, want) { + t.Errorf("score description missing %q: %q", want, scoreDesc) + } + } + for _, desc := range []string{noulDesc, choiceDesc, scoreDesc} { + if strings.Contains(desc, "map[") { + t.Errorf("description leaks Go map syntax: %q", desc) + } + } + + // Outbound framing: forced tool call, system prompt, state verbatim. + if provider.lastRequest.Params.ToolChoice == nil || provider.lastRequest.Params.ToolChoice.ResponsesToolChoiceStr == nil || + *provider.lastRequest.Params.ToolChoice.ResponsesToolChoiceStr != string(schemas.ResponsesToolChoiceTypeRequired) { + t.Errorf("tool choice not forced to required: %+v", provider.lastRequest.Params.ToolChoice) + } + if provider.lastRequest.Params.Instructions == nil || *provider.lastRequest.Params.Instructions != decisionSystemPrompt { + t.Errorf("system prompt not attached: %+v", provider.lastRequest.Params.Instructions) + } + if len(provider.lastRequest.Input) != 1 || provider.lastRequest.Input[0].Content.ContentStr == nil || + *provider.lastRequest.Input[0].Content.ContentStr != "Customer message: I was double charged and want a refund." { + t.Errorf("state not forwarded verbatim: %+v", provider.lastRequest.Input) + } + + // Inbound: answers typed per kind, structured legend rendered as JSON. + if resp.Answers["approve"].Kind != schemas.DecisionKindNoul || resp.Answers["approve"].Value.(float64) != 0.9 { + t.Errorf("noul answer = %+v", resp.Answers["approve"]) + } + if resp.Answers["category"].Value.(string) != "billing" { + t.Errorf("choice answer = %+v", resp.Answers["category"]) + } + urgency := resp.Answers["urgency"] + // Score derives from the distribution: 0*0.1 + 1*0.1 + 2*0.8 = 1.7. + if math.Abs(urgency.Value.(float64)-1.7) > 1e-9 { + t.Errorf("score answer = %+v", urgency) + } + // The legend echoes each level's description verbatim, matching the native + // API's shape (values are strings, objects, or arrays - not stringified). + if !reflect.DeepEqual(urgency.Legend["1"], map[string]any{"level": "high", "examples": []any{"outage"}}) || + !reflect.DeepEqual(urgency.Legend["2"], []any{"critical", "churn risk"}) { + t.Errorf("structured legend not echoed verbatim: %+v", urgency.Legend) + } +} + +func TestEmulateDecisionStructuredOutputFallbackPath(t *testing.T) { + // A model that ignores the function tool but answers with a JSON object as + // output text takes the structured-output fallback in + // extractDecisionToolArguments; structured criteria must survive that path + // identically. + msgType := schemas.ResponsesMessageTypeMessage + content := `{ + "approve": {"value": 0.4, "confidence": 0.5}, + "category": {"choice": "support", "confidence": 0.6, "probabilities": {"billing": 0.2, "bug": 0.1, "support": 0.6, "other": 0.1}}, + "urgency": {"value": 0, "confidence": 0.7, "probabilities": {"0": 0.8, "1": 0.1, "2": 0.1}} + }` + provider := &decisionEmulationProvider{response: &schemas.BifrostResponsesResponse{ + Model: "gpt-4o-mini", + Output: []schemas.ResponsesMessage{{ + Type: &msgType, + Content: &schemas.ResponsesMessageContent{ContentStr: &content}, + }}, + }} + + var b Bifrost + resp, bifrostErr := b.emulateDecisionViaResponses(nil, provider, schemas.Key{}, structuredDecisionRequest()) + if bifrostErr != nil { + t.Fatalf("unexpected error: %v", bifrostErr) + } + if resp.Answers["category"].Value.(string) != "support" { + t.Errorf("choice answer = %+v", resp.Answers["category"]) + } + if resp.Answers["urgency"].Legend["0"] != "low" { + t.Errorf("string legend level lost: %+v", resp.Answers["urgency"].Legend) + } +} + +func TestEmulateDecisionRejectsMalformedCriteriaBeforeDispatch(t *testing.T) { + // A choice question whose criteria is not a map fails schema building; the + // request must 400 locally without ever reaching the provider. + req := structuredDecisionRequest() + question := req.Questions["category"] + question.Criteria = []any{"not", "a", "map"} + req.Questions["category"] = question + + provider := &decisionEmulationProvider{} + var b Bifrost + _, bifrostErr := b.emulateDecisionViaResponses(nil, provider, schemas.Key{}, req) + if bifrostErr == nil || bifrostErr.Error == nil || !strings.Contains(bifrostErr.Error.Message, "choice criteria must be a map of options") { + t.Fatalf("expected local criteria rejection, got %+v", bifrostErr) + } + if provider.lastRequest != nil { + t.Error("malformed criteria must be rejected before the provider is called") + } +} + +func TestEmulateDecisionRejectsAnswerOutsideStructuredOptions(t *testing.T) { + // The model answers with an option that is not in the structured criteria + // map; the emulation must reject it so fallbacks can proceed, not fabricate + // a valid-looking decision. + // Every other field is complete so the only defect is the unknown option. + args := `{ + "approve": {"value": 0.9, "confidence": 0.8}, + "category": {"choice": "nonexistent", "confidence": 0.7, "probabilities": {"billing": 0.7, "bug": 0.1, "support": 0.1, "other": 0.1}}, + "urgency": {"value": 1, "confidence": 0.6, "probabilities": {"0": 0.1, "1": 0.8, "2": 0.1}} + }` + provider := &decisionEmulationProvider{response: emulationFunctionCallResponse(args)} + var b Bifrost + _, bifrostErr := b.emulateDecisionViaResponses(nil, provider, schemas.Key{}, structuredDecisionRequest()) + if bifrostErr == nil || bifrostErr.Error == nil || !strings.Contains(bifrostErr.Error.Message, "not an allowed option") { + t.Fatalf("expected out-of-options rejection, got %+v", bifrostErr) + } +} + +// Bedrock Mantle's OpenAI-compatible surface rejects tool_choice "required" for +// gpt-oss ("Supported options: [auto]"), so emulation there must send "auto"; +// every other surface, Claude on Mantle included, keeps the forced call. +func TestEmulateDecisionToolChoicePerSurface(t *testing.T) { + args := `{ + "approve": {"value": 0.9, "confidence": 0.8}, + "category": {"choice": "billing", "confidence": 0.7, "probabilities": {"billing": 0.7, "bug": 0.1, "support": 0.1, "other": 0.1}}, + "urgency": {"value": 2, "confidence": 0.6, "probabilities": {"0": 0.1, "1": 0.1, "2": 0.8}} + }` + cases := []struct { + provider schemas.ModelProvider + model string + want schemas.ResponsesToolChoiceType + }{ + {schemas.BedrockMantle, "openai.gpt-oss-120b", schemas.ResponsesToolChoiceTypeAuto}, + {schemas.BedrockMantle, "openai.gpt-oss-20b", schemas.ResponsesToolChoiceTypeAuto}, + {schemas.BedrockMantle, "anthropic.claude-opus-4-8", schemas.ResponsesToolChoiceTypeRequired}, + {schemas.OpenAI, "gpt-4o-mini", schemas.ResponsesToolChoiceTypeRequired}, + } + for _, tc := range cases { + t.Run(string(tc.provider)+"/"+tc.model, func(t *testing.T) { + provider := &decisionEmulationProvider{response: emulationFunctionCallResponse(args)} + req := structuredDecisionRequest() + req.Provider, req.Model = tc.provider, tc.model + + var b Bifrost + if _, bifrostErr := b.emulateDecisionViaResponses(nil, provider, schemas.Key{}, req); bifrostErr != nil { + t.Fatalf("unexpected error: %v", bifrostErr) + } + choice := provider.lastRequest.Params.ToolChoice + if choice == nil || choice.ResponsesToolChoiceStr == nil || *choice.ResponsesToolChoiceStr != string(tc.want) { + t.Errorf("tool_choice = %+v, want %q", choice, tc.want) + } + }) + } +} + +// A turn whose answer cannot be used was still answered and billed by the +// provider, so the error must carry its usage and served model; otherwise +// governance and logging record the paid call as free. +func TestEmulateDecisionRejectedAnswerCarriesBilledUsage(t *testing.T) { + cases := map[string]*schemas.BifrostResponsesResponse{ + "no function call": {Model: "gpt-4o-mini-2026"}, + "answer outside options": emulationFunctionCallResponse(`{ + "approve": {"value": 0.9, "confidence": 0.8}, + "category": {"choice": "nonexistent", "confidence": 0.7, "probabilities": {"billing": 0.7, "bug": 0.1, "support": 0.1, "other": 0.1}}, + "urgency": {"value": 1, "confidence": 0.6, "probabilities": {"0": 0.1, "1": 0.8, "2": 0.1}} + }`), + } + for name, resp := range cases { + t.Run(name, func(t *testing.T) { + resp.Usage = &schemas.ResponsesResponseUsage{InputTokens: 1200, OutputTokens: 80, TotalTokens: 1280} + priority := schemas.BifrostServiceTierPriority + resp.ServiceTier = &priority + provider := &decisionEmulationProvider{response: resp} + var b Bifrost + _, bifrostErr := b.emulateDecisionViaResponses(nil, provider, schemas.Key{}, structuredDecisionRequest()) + if bifrostErr == nil { + t.Fatal("expected the unusable answer to be rejected") + } + billed := bifrostErr.ExtraFields.BilledUsage + if billed == nil || billed.PromptTokens != 1200 || billed.CompletionTokens != 80 { + t.Fatalf("billed usage = %+v, want the provider's 1200 in / 80 out", billed) + } + if billed.ServiceTier == nil || *billed.ServiceTier != schemas.BifrostServiceTierPriority { + t.Fatalf("billed usage dropped the served tier: %v", billed.ServiceTier) + } + if bifrostErr.ExtraFields.ResolvedModelUsed != resp.Model { + t.Fatalf("served model = %q, want %q", bifrostErr.ExtraFields.ResolvedModelUsed, resp.Model) + } + }) + } +} + +// A custom provider wrapping Bedrock Mantle gets Mantle's "auto" tool_choice for +// gpt-oss, which rejects "required". +func TestEmulateDecisionToolChoiceFollowsBaseProvider(t *testing.T) { + args := `{ + "approve": {"value": 0.9, "confidence": 0.8}, + "category": {"choice": "billing", "confidence": 0.7, "probabilities": {"billing": 0.7, "bug": 0.1, "support": 0.1, "other": 0.1}}, + "urgency": {"value": 2, "confidence": 0.6, "probabilities": {"0": 0.1, "1": 0.1, "2": 0.8}} + }` + provider := &decisionEmulationProvider{response: emulationFunctionCallResponse(args)} + req := structuredDecisionRequest() + req.Provider, req.Model = "my-mantle", "openai.gpt-oss-120b" + ctx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) + ctx.SetValue(schemas.BifrostContextKeyBaseProviderType, schemas.BedrockMantle) + + var b Bifrost + if _, bifrostErr := b.emulateDecisionViaResponses(ctx, provider, schemas.Key{}, req); bifrostErr != nil { + t.Fatalf("unexpected error: %v", bifrostErr) + } + choice := provider.lastRequest.Params.ToolChoice + if choice == nil || choice.ResponsesToolChoiceStr == nil || *choice.ResponsesToolChoiceStr != string(schemas.ResponsesToolChoiceTypeAuto) { + t.Fatalf("tool_choice = %+v, want auto", choice) + } +} + +// The emulating call's served tier rides on the decision, so the decision is priced +// at the rate of the service actually delivered. +func TestEmulateDecisionCarriesServedTier(t *testing.T) { + resp := emulationFunctionCallResponse(`{ + "approve": {"value": 0.9, "confidence": 0.8}, + "category": {"choice": "billing", "confidence": 0.7, "probabilities": {"billing": 0.7, "bug": 0.1, "support": 0.1, "other": 0.1}}, + "urgency": {"value": 2, "confidence": 0.6, "probabilities": {"0": 0.1, "1": 0.1, "2": 0.8}} + }`) + priority := schemas.BifrostServiceTierPriority + fast, us := "fast", "us" + resp.ServiceTier, resp.Speed, resp.InferenceGeo = &priority, &fast, &us + resp.Usage = &schemas.ResponsesResponseUsage{InputTokens: 10, OutputTokens: 2, TotalTokens: 12} + var b Bifrost + decision, bifrostErr := b.emulateDecisionViaResponses(nil, &decisionEmulationProvider{response: resp}, schemas.Key{}, structuredDecisionRequest()) + if bifrostErr != nil { + t.Fatalf("unexpected error: %v", bifrostErr) + } + if decision.ServiceTier == nil || *decision.ServiceTier != priority || decision.Speed == nil || *decision.Speed != fast || decision.InferenceGeo == nil || *decision.InferenceGeo != us { + t.Fatalf("served tier not carried: tier=%v speed=%v geo=%v", decision.ServiceTier, decision.Speed, decision.InferenceGeo) + } +} diff --git a/core/encryptedreasoning.go b/core/encryptedreasoning.go index c5b82a77b4a..15c20e70fc4 100644 --- a/core/encryptedreasoning.go +++ b/core/encryptedreasoning.go @@ -1,7 +1,7 @@ package bifrost import ( - "fmt" + "bytes" "strings" schemas "github.com/capsohq/bifrost/core/schemas" @@ -159,6 +159,13 @@ func isEncryptedReasoningRejection(err *schemas.BifrostError) bool { return false } + // Bedrock runtime's OpenAI-compatible Responses endpoint uses validation_error + // for model/account-bound reasoning replay. Match its specific refusal rather + // than treating unrelated reasoning validation errors as recoverable. + if strings.Contains(message, "encrypted reasoning was created for a different account or model") { + return true + } + if !namesEncryptedReasoningField(message) { return false } @@ -428,11 +435,18 @@ func stripRawAnthropicChatThinking(rawBody *[]byte) bool { // with tool_use present and thinking still enabled; measured against the live API on // claude-sonnet-4-5, claude-haiku-4-5 and claude-opus-5. // https://platform.claude.com/docs/en/build-with-claude/thinking#preserving-thinking-blocks + // Each message is rewritten in its own JSON and the messages array is written back + // once. Addressing through the whole body (messages..content) would reserialise + // the entire request per changed message, making this O(messages x body). + // Pinned by TestStripRawAnthropicChatThinking_AllocationScaling. changed := false - for messageIndex, message := range messages.Array() { + var rebuilt [][]byte + for _, message := range messages.Array() { + messageRaw := []byte(message.Raw) content := message.Get("content") // The shorthand string form carries no blocks, so there is nothing to rewrite. if !content.IsArray() { + rebuilt = append(rebuilt, messageRaw) continue } @@ -455,23 +469,46 @@ func stripRawAnthropicChatThinking(rawBody *[]byte) bool { } if !messageChanged || len(kept) == 0 { + rebuilt = append(rebuilt, messageRaw) continue } - updated, err := sjson.SetRawBytes(body, fmt.Sprintf("messages.%d.content", messageIndex), []byte("["+strings.Join(kept, ",")+"]")) + updated, err := sjson.SetRawBytes(messageRaw, "content", []byte("["+strings.Join(kept, ",")+"]")) if err != nil { + rebuilt = append(rebuilt, messageRaw) continue } - body = updated + rebuilt = append(rebuilt, updated) changed = true } if !changed { return false } - *rawBody = body + updated, err := sjson.SetRawBytes(body, "messages", joinRawJSONArray(rebuilt, len(body))) + if err != nil { + return false + } + *rawBody = updated return true } +// joinRawJSONArray concatenates pre-encoded JSON values into one array, so a batch of +// element-local edits costs a single write of the enclosing document instead of one per +// element. +func joinRawJSONArray(parts [][]byte, sizeHint int) []byte { + var buf bytes.Buffer + buf.Grow(sizeHint) + buf.WriteByte('[') + for i, p := range parts { + if i > 0 { + buf.WriteByte(',') + } + buf.Write(p) + } + buf.WriteByte(']') + return buf.Bytes() +} + // stripContentBlockReasoningPayloads returns the message's content blocks with every // unverifiable reasoning payload removed -- signature and encrypted_content -- and // whether anything was removed. Anthropic thinking replayed over the Responses shape @@ -683,16 +720,31 @@ func stripRawResponsesEncryptedContent(rawBody *[]byte, dropItemIDs bool) bool { // Deleting a field inside a block never reorders the content array, so the indices // read before the deletes stay valid. A string-valued content yields a one-element // array carrying neither field, and falls through untouched. - for blockIndex, block := range gjson.Get(rest, "content").Array() { - for _, field := range contentBlockReasoningCarriers { - if !gjson.Get(block.Raw, field).Exists() { - continue + // Each carrier is dropped from the block's own JSON and the content array is + // written back once. Deleting content.. through the whole item would + // reserialise it per block per carrier, making this O(blocks x item). + // Pinned by TestStripRawResponsesEncryptedContent_AllocationScaling. + if contentResult := gjson.Get(rest, "content"); contentResult.IsArray() { + var rebuiltBlocks [][]byte + contentChanged := false + for _, block := range contentResult.Array() { + blockRaw := []byte(block.Raw) + for _, field := range contentBlockReasoningCarriers { + if !gjson.GetBytes(blockRaw, field).Exists() { + continue + } + updated, err := sjson.DeleteBytes(blockRaw, field) + if err != nil { + continue + } + blockRaw, contentChanged = updated, true } - updated, err := sjson.Delete(rest, fmt.Sprintf("content.%d.%s", blockIndex, field)) - if err != nil { - continue + rebuiltBlocks = append(rebuiltBlocks, blockRaw) + } + if contentChanged { + if updated, err := sjson.SetRawBytes([]byte(rest), "content", joinRawJSONArray(rebuiltBlocks, len(rest))); err == nil { + rest, itemChanged = string(updated), true } - rest, itemChanged = updated, true } } diff --git a/core/encryptedreasoning_test.go b/core/encryptedreasoning_test.go index e530230900b..bf4ec3fdfdf 100644 --- a/core/encryptedreasoning_test.go +++ b/core/encryptedreasoning_test.go @@ -1,6 +1,7 @@ package bifrost import ( + "bytes" "context" "io" "net/http" @@ -11,7 +12,9 @@ import ( "testing" "time" + "github.com/capsohq/bifrost/core/internal/memtest" schemas "github.com/capsohq/bifrost/core/schemas" + "github.com/tidwall/gjson" ) // newEncryptedReasoningRequest builds a Responses request whose input replays a @@ -72,6 +75,17 @@ func encryptedContentError() *schemas.BifrostError { } } +func bedrockReasoningModelMismatchError() *schemas.BifrostError { + return &schemas.BifrostError{ + StatusCode: schemas.Ptr(400), + Error: &schemas.ErrorField{ + Type: schemas.Ptr("invalid_request_error"), + Code: schemas.Ptr("validation_error"), + Message: "encrypted reasoning was created for a different account or model", + }, + } +} + // TestExecuteRequestWithRetries_StripsEncryptedContentAndRetries pins the fail-soft // path for reasoning items the target upstream cannot decrypt (a different org, key, // tenancy, or provider minted them). The retry must happen even at the default @@ -183,6 +197,12 @@ func TestExecuteRequestWithRetries_HealsOnEveryProviderRejection(t *testing.T) { }, }, }, + { + name: "bedrock runtime model switch", + provider: schemas.Bedrock, + model: "us.openai.gpt-5.6-terra", + err: bedrockReasoningModelMismatchError(), + }, { // Bedrock Mantle serves OpenAI-family models over an OpenAI-compatible // /v1/responses surface, so a fallback from Azure to Mantle replays an @@ -740,6 +760,187 @@ func TestStripResponsesEncryptedContent(t *testing.T) { }) } +func TestIsEncryptedReasoningRejection_BedrockModelSwitch(t *testing.T) { + for _, tc := range []struct { + name, message string + status int + want bool + }{ + {"exact refusal", "encrypted reasoning was created for a different account or model", 400, true}, + {"wrapped mixed case", "Bedrock: Encrypted Reasoning Was Created For A Different Account Or Model.", 400, true}, + {"server error", "encrypted reasoning was created for a different account or model", 500, false}, + {"auth error", "encrypted reasoning was created for a different account or model", 403, false}, + {"unrelated validation", "invalid max_output_tokens", 400, false}, + {"different resource", "resource was created for a different account or model", 400, false}, + {"reasoning field only", "encrypted reasoning is required", 400, false}, + {"reasoning validation", "invalid encrypted reasoning length", 400, false}, + {"modified block exclusion", "encrypted reasoning was created for a different account or model; thinking blocks cannot be modified", 400, false}, + } { + t.Run(tc.name, func(t *testing.T) { + err := bedrockReasoningModelMismatchError() + err.Error.Message = tc.message + err.StatusCode = &tc.status + if got := isEncryptedReasoningRejection(err); got != tc.want { + t.Fatalf("isEncryptedReasoningRejection() = %v, want %v", got, tc.want) + } + }) + } +} + +// Exercise the real Bedrock runtime route and OpenAI error parser, including an HTTP +// rejection before SSE starts. Capturing both bodies proves recovery changes the wire +// payload without changing the model, credential, or ordinary conversation history. +func TestBedrockResponsesModelSwitchRecovery(t *testing.T) { + const model = "us.openai.gpt-5.6-terra" + const refusal = `{"error":{"type":"invalid_request_error","code":"validation_error","message":"encrypted reasoning was created for a different account or model"}}` + for _, streaming := range []bool{false, true} { + mode := "unary" + if streaming { + mode = "streaming" + } + for _, tc := range []struct { + name string + encrypted bool + keepRejecting bool + wantAttempts int + }{ + {"heals", true, false, 2}, + {"second rejection stops", true, true, 2}, + {"nothing to strip", false, true, 1}, + } { + t.Run(mode+"/"+tc.name, func(t *testing.T) { + recorder := &recordingServer{} + upstream := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + if err != nil { + t.Error(err) + } + attempt := recorder.record(string(body)) + if r.URL.Path != "/openai/v1/responses" || r.Method != http.MethodPost { + t.Errorf("unexpected route: %s %s", r.Method, r.URL.Path) + } + if r.Header.Get("Authorization") != "Bearer bedrock-test-key" { + t.Error("runtime request did not use the expected credential") + } + if attempt == 1 || tc.keepRejecting { + writeJSON(w, http.StatusBadRequest, refusal) + return + } + if streaming { + sseHandler( + `{"type":"response.output_text.delta","sequence_number":0,"output_index":0,"content_index":0,"item_id":"msg_1","delta":"tests pass"}`, + `{"type":"response.completed","sequence_number":1,"response":`+successBody+`}`, + )(w, r) + } else { + writeJSON(w, http.StatusOK, successBody) + } + })) + defer upstream.Close() + + account := NewMockAccount() + account.AddProvider(schemas.Bedrock, 1, 1) + account.configs[schemas.Bedrock].NetworkConfig.MaxRetries = 0 + account.configs[schemas.Bedrock].NetworkConfig.InsecureSkipVerify = true + account.SetKeysForProvider(schemas.Bedrock, []schemas.Key{{ + ID: "bedrock-key", Value: *schemas.NewSecretVar("bedrock-test-key"), + Models: schemas.WhiteList{"*"}, Weight: 100, UseOpenAIEndpoints: schemas.Ptr(true), + BedrockKeyConfig: &schemas.BedrockKeyConfig{ + Region: schemas.NewSecretVar("us-east-1"), + Endpoints: &schemas.BedrockEndpoints{Runtime: schemas.NewSecretVar(strings.TrimPrefix(upstream.URL, "https://"))}, + }, + }}) + client := newStreamTestClient(t, account) + req := newEncryptedReasoningRequest("foreign-luna-ciphertext").ResponsesRequest + req.Provider, req.Model = schemas.Bedrock, model + if !tc.encrypted { + req.Input[1].ResponsesReasoning.EncryptedContent = nil + } + req.Input = append(req.Input, + schemas.ResponsesMessage{ + Type: schemas.Ptr(schemas.ResponsesMessageTypeFunctionCall), + ResponsesToolMessage: &schemas.ResponsesToolMessage{ + CallID: schemas.Ptr("call_pwd"), Name: schemas.Ptr("pwd"), Arguments: schemas.Ptr("{}"), + }, + }, + schemas.ResponsesMessage{ + Type: schemas.Ptr(schemas.ResponsesMessageTypeFunctionCallOutput), + ResponsesToolMessage: &schemas.ResponsesToolMessage{ + CallID: schemas.Ptr("call_pwd"), + Output: &schemas.ResponsesToolMessageOutputStruct{ResponsesToolCallOutputStr: schemas.Ptr("/home/test")}, + }, + }, + ) + ctx := schemas.NewBifrostContext(context.Background(), time.Now().Add(10*time.Second)) + var requestErr *schemas.BifrostError + if streaming { + var stream chan *schemas.BifrostStreamChunk + stream, requestErr = client.ResponsesStreamRequest(ctx, req) + var text strings.Builder + completed := false + if stream != nil { + for chunk := range stream { + if chunk.BifrostError != nil { + t.Errorf("unexpected stream error: %v", chunk.BifrostError) + } + if response := chunk.BifrostResponsesStreamResponse; response != nil { + if response.Type == schemas.ResponsesStreamResponseTypeOutputTextDelta && response.Delta != nil { + text.WriteString(*response.Delta) + } + completed = completed || response.Type == schemas.ResponsesStreamResponseTypeCompleted + } + } + } + if !tc.keepRejecting && (text.String() != "tests pass" || !completed) { + t.Errorf("expected successful completed stream, got text=%q completed=%v", text.String(), completed) + } + } else { + var response *schemas.BifrostResponsesResponse + response, requestErr = client.ResponsesRequest(ctx, req) + if !tc.keepRejecting && (response == nil || response.ID == nil || *response.ID != "resp_healed_1") { + t.Errorf("expected healed response, got %v", response) + } + } + if tc.keepRejecting { + if requestErr == nil || requestErr.Error == nil || requestErr.Error.Message != bedrockReasoningModelMismatchError().Error.Message { + t.Errorf("expected original rejection, got %v", requestErr) + } + } else if requestErr != nil { + t.Errorf("expected recovery, got %v", requestErr) + } + bodies := recorder.snapshot() + if len(bodies) != tc.wantAttempts { + t.Fatalf("attempts = %d, want %d", len(bodies), tc.wantAttempts) + } + for _, body := range bodies { + if gjson.Get(body, "model").String() != model || gjson.Get(body, "stream").Bool() != streaming { + t.Errorf("model or stream mode changed: %s", body) + } + } + if tc.encrypted { + if gjson.Get(bodies[0], "input.1.encrypted_content").String() != "foreign-luna-ciphertext" || gjson.Get(bodies[1], "input.1.encrypted_content").Exists() { + t.Fatal("expected ciphertext only on the first attempt") + } + for _, path := range []string{"input.#", "input.0", "input.1.id", "input.1.summary", "input.2", "input.3"} { + before, after := gjson.Get(bodies[0], path), gjson.Get(bodies[1], path) + if !before.Exists() || before.Raw != after.Raw { + t.Errorf("retry changed %s: %s -> %s", path, before.Raw, after.Raw) + } + } + } + stripLogs := 0 + for _, entry := range ctx.GetRoutingEngineLogs() { + if strings.Contains(entry.Message, "Stripped unverifiable encrypted reasoning content") { + stripLogs++ + } + } + if stripLogs != tc.wantAttempts-1 { + t.Errorf("strip log count = %d, want %d", stripLogs, tc.wantAttempts-1) + } + }) + } + } +} + func TestIsEncryptedReasoningRejection(t *testing.T) { tests := []struct { name string @@ -2226,3 +2427,59 @@ func TestStripResponsesEncryptedContent_KeepsMessageItemWithSurvivingContent(t * t.Errorf("expected only the text block to survive, got %+v", kept.Content.ContentBlocks) } } + +// TestStripRawAnthropicChatThinking_AllocationScaling pins the allocation shape of the +// raw Anthropic thinking strip. +// +// The loop writes messages..content through the whole request body, once per message +// it changes, and each sjson write reserialises the entire request. A long agentic +// conversation is exactly the shape that makes that expensive. +func TestStripRawAnthropicChatThinking_AllocationScaling(t *testing.T) { + memtest.AssertAllocScaling(t, func(turns int) []byte { + var b bytes.Buffer + b.WriteString(`{"model":"claude-opus-4-8","messages":[`) + for i := range turns { + if i > 0 { + b.WriteByte(',') + } + // A signed thinking block (stripped) alongside a text block that survives + // and carries the bulk of the bytes, so both N and payload size scale. + b.WriteString(`{"role":"assistant","content":[`) + b.WriteString(`{"type":"thinking","thinking":"reasoning","signature":"sig"},`) + b.WriteString(`{"type":"text","text":"`) + b.WriteString(strings.Repeat("x", 400)) + b.WriteString(`"}]}`) + } + b.WriteString(`]}`) + return b.Bytes() + }, func(body []byte) { + scratch := append([]byte(nil), body...) + stripRawAnthropicChatThinking(&scratch) + }) +} + +// TestStripRawResponsesEncryptedContent_AllocationScaling pins the allocation shape of +// the Responses encrypted-content strip. +// +// Its inner loop deletes content.. from the item being rewritten, once per +// content block per reasoning carrier field, and each delete reserialises that whole +// item. An item carrying many reasoning blocks pays that repeatedly. +func TestStripRawResponsesEncryptedContent_AllocationScaling(t *testing.T) { + memtest.AssertAllocScaling(t, func(blocks int) []byte { + var b bytes.Buffer + b.WriteString(`{"model":"gpt-5","input":[{"type":"message","role":"assistant","content":[`) + for i := range blocks { + if i > 0 { + b.WriteByte(',') + } + b.WriteString(`{"type":"reasoning_text","text":"`) + b.WriteString(strings.Repeat("r", 400)) + b.WriteString(`","encrypted_content":"enc"}`) + } + b.WriteString(`]}]}`) + return b.Bytes() + }, func(body []byte) { + scratch := append([]byte(nil), body...) + stripRawResponsesEncryptedContent(&scratch, false) + }) +} diff --git a/core/internal/llmtests/account.go b/core/internal/llmtests/account.go index 391db959d96..a349b3037d8 100644 --- a/core/internal/llmtests/account.go +++ b/core/internal/llmtests/account.go @@ -91,6 +91,8 @@ type TestScenarios struct { ContainerFileDelete bool // Container File API delete functionality PassThroughExtraParams bool // Pass through extra params functionality Rerank bool // Rerank functionality + Decision bool // Decision functionality (annotated function-tool evaluation) + DecisionEmulation bool // Decision emulated via a general LLM (tool-calling / structured output) PassthroughAPI bool // Raw HTTP passthrough API (Passthrough + PassthroughStream) WebSocketResponses bool // WebSocket Responses API mode Realtime bool // Realtime API (bidirectional audio/text) @@ -114,6 +116,8 @@ type ComprehensiveTestConfig struct { ReasoningModel string EmbeddingModel string RerankModel string + DecisionModel string + DecisionEmulationModel string // a general LLM model used to emulate a decision TranscriptionModel string SpeechSynthesisModel string ChatAudioModel string @@ -124,6 +128,7 @@ type ComprehensiveTestConfig struct { SpeechSynthesisFallbacks []schemas.Fallback // for speech synthesis tests EmbeddingFallbacks []schemas.Fallback // for embedding tests RerankFallbacks []schemas.Fallback // for rerank tests + DecisionFallbacks []schemas.Fallback // for decision tests SkipReason string // Reason to skip certain tests ImageGenerationModel string // Model for image generation ImageGenerationFallbacks []schemas.Fallback // Fallbacks for image generation @@ -198,6 +203,7 @@ func (account *ComprehensiveTestAccount) GetConfiguredProviders() ([]schemas.Mod schemas.Wafer, schemas.Databricks, schemas.GithubCopilot, + schemas.Typesafe, ProviderOpenAICustom, }, nil } @@ -594,6 +600,14 @@ func (account *ComprehensiveTestAccount) GetKeysForProvider(ctx context.Context, UseForBatchAPI: bifrost.Ptr(true), }, }, nil + case schemas.Typesafe: + return []schemas.Key{ + { + Value: *schemas.NewSecretVar("env.TYPESAFE_API_KEY"), + Models: []string{"*"}, + Weight: 1.0, + }, + }, nil case schemas.Fireworks: return []schemas.Key{ { @@ -1064,6 +1078,19 @@ func (account *ComprehensiveTestAccount) GetConfigForProvider(providerKey schema BufferSize: 10, }, }, nil + case schemas.Typesafe: + return &schemas.ProviderConfig{ + NetworkConfig: schemas.NetworkConfig{ + DefaultRequestTimeoutInSeconds: 300, + MaxRetries: 10, + RetryBackoffInitial: 1 * time.Second, + RetryBackoffMax: 12 * time.Second, + }, + ConcurrencyAndBufferSize: schemas.ConcurrencyAndBufferSize{ + Concurrency: Concurrency, + BufferSize: 10, + }, + }, nil case schemas.Fireworks: return &schemas.ProviderConfig{ NetworkConfig: schemas.NetworkConfig{ diff --git a/core/internal/llmtests/decision.go b/core/internal/llmtests/decision.go new file mode 100644 index 00000000000..7749b104699 --- /dev/null +++ b/core/internal/llmtests/decision.go @@ -0,0 +1,167 @@ +package llmtests + +import ( + "context" + "os" + "strings" + "testing" + + bifrost "github.com/capsohq/bifrost/core" + "github.com/capsohq/bifrost/core/schemas" +) + +// BasicDecisionExpectations validates common decision invariants for provider +// tests: every requested question produces an answer of its declared kind with +// a well-typed value. +func BasicDecisionExpectations(t *testing.T, response *schemas.BifrostDecisionResponse, request *schemas.BifrostDecisionRequest) { + t.Helper() + + if response == nil { + t.Fatal("❌ Decision response is nil") + } + if len(response.Answers) == 0 { + t.Fatal("❌ Decision response carries no answers") + } + + for name, question := range request.Questions { + answer, ok := response.Answers[name] + if !ok { + t.Fatalf("❌ Question %q produced no answer", name) + } + if answer.Kind != question.Kind { + t.Fatalf("❌ Answer %q has kind %q; expected %q", name, answer.Kind, question.Kind) + } + switch answer.Kind { + case schemas.DecisionKindNoul: + number, ok := answer.Value.(float64) + if !ok || number < 0 || number > 1 { + t.Fatalf("❌ Noul answer %q is not a number in [0,1]: %v", name, answer.Value) + } + case schemas.DecisionKindChoice: + if _, ok := answer.Value.(string); !ok { + t.Fatalf("❌ Choice answer %q is not a string: %v", name, answer.Value) + } + case schemas.DecisionKindScore: + if _, ok := answer.Value.(float64); !ok { + t.Fatalf("❌ Score answer %q is not a number: %v", name, answer.Value) + } + default: + t.Fatalf("❌ Answer %q has unknown kind %q", name, answer.Kind) + } + } +} + +// RunDecisionTest executes the decision test scenario against a live provider. +func RunDecisionTest(t *testing.T, client *bifrost.Bifrost, ctx context.Context, testConfig ComprehensiveTestConfig) { + if !testConfig.Scenarios.Decision { + t.Logf("Decision not supported for provider %s", testConfig.Provider) + return + } + + if strings.TrimSpace(testConfig.DecisionModel) == "" { + t.Skipf("Decision enabled but model is not configured for provider %s; skipping", testConfig.Provider) + } + + t.Run("Decision", func(t *testing.T) { + if os.Getenv("SKIP_PARALLEL_TESTS") != "true" { + t.Parallel() + } + + state := "Customer message: I was double charged for my subscription last month and nobody has replied to my two previous emails. I want a refund today or I am cancelling." + + request := &schemas.BifrostDecisionRequest{ + Provider: testConfig.Provider, + Model: testConfig.DecisionModel, + State: state, + Questions: map[string]schemas.DecisionQuestion{ + "is_frustrated": { + Kind: schemas.DecisionKindNoul, + Instructions: "Is the customer frustrated?", + }, + "category": { + Kind: schemas.DecisionKindChoice, + Instructions: "Pick the ticket category", + Criteria: map[string]interface{}{ + "billing": "charges, refunds, invoices", + "bug": "product defects", + "other": "anything else", + }, + }, + "urgency": { + Kind: schemas.DecisionKindScore, + Instructions: "Rate how urgently this ticket needs a human reply", + Criteria: []interface{}{"can wait a week", "should be answered soon", "needs a reply today"}, + }, + }, + Fallbacks: testConfig.DecisionFallbacks, + } + + bfCtx := schemas.NewBifrostContext(ctx, schemas.NoDeadline) + response, bifrostErr := client.DecisionRequest(bfCtx, request) + + if bifrostErr != nil { + t.Fatalf("❌ Decision request failed: %v", GetErrorMessage(bifrostErr)) + } + + BasicDecisionExpectations(t, response, request) + + if response.Usage != nil { + t.Logf("📊 Usage: prompt_tokens=%d, total_tokens=%d", response.Usage.PromptTokens, response.Usage.TotalTokens) + } + t.Logf("✅ Decision test passed: model=%s, answers=%d", response.Model, len(response.Answers)) + }) +} + +// RunDecisionEmulationTest runs a decision against a general LLM model, exercising +// the emulation path (the provider has no native decision support, so Bifrost +// answers via tool-calling / structured output). Every question must come back +// as a typed answer with an LLM-estimated confidence. +func RunDecisionEmulationTest(t *testing.T, client *bifrost.Bifrost, ctx context.Context, testConfig ComprehensiveTestConfig) { + if !testConfig.Scenarios.DecisionEmulation { + t.Logf("Decision emulation not enabled for provider %s", testConfig.Provider) + return + } + if strings.TrimSpace(testConfig.DecisionEmulationModel) == "" { + t.Skipf("Decision emulation enabled but no model configured for provider %s; skipping", testConfig.Provider) + } + + t.Run("DecisionEmulation", func(t *testing.T) { + if os.Getenv("SKIP_PARALLEL_TESTS") != "true" { + t.Parallel() + } + + provider, model := schemas.ParseModelString(testConfig.DecisionEmulationModel, testConfig.Provider) + request := &schemas.BifrostDecisionRequest{ + Provider: provider, + Model: model, + State: "Customer message: I was double charged and support ignored my emails. I want a refund now or I cancel.", + Questions: map[string]schemas.DecisionQuestion{ + "is_frustrated": {Kind: schemas.DecisionKindNoul, Instructions: "Is the customer frustrated?"}, + "category": { + Kind: schemas.DecisionKindChoice, + Instructions: "Pick the ticket category", + Criteria: map[string]interface{}{"billing": "charges and refunds", "bug": "product defects", "other": "anything else"}, + }, + "urgency": { + Kind: schemas.DecisionKindScore, + Instructions: "Rate how urgently this needs a human reply", + Criteria: []interface{}{"low", "medium", "high"}, + }, + }, + } + + bfCtx := schemas.NewBifrostContext(ctx, schemas.NoDeadline) + response, bifrostErr := client.DecisionRequest(bfCtx, request) + if bifrostErr != nil { + t.Fatalf("❌ Emulated decision failed: %v", GetErrorMessage(bifrostErr)) + } + + BasicDecisionExpectations(t, response, request) + for name, answer := range response.Answers { + if answer.Confidence == nil { + t.Errorf("❌ Emulated answer %q has no confidence", name) + } + } + t.Logf("✅ Decision emulation passed via %s: %d answers", testConfig.DecisionEmulationModel, len(response.Answers)) + }) +} diff --git a/core/internal/llmtests/tests.go b/core/internal/llmtests/tests.go index 7f35c80aa02..cac55db2341 100644 --- a/core/internal/llmtests/tests.go +++ b/core/internal/llmtests/tests.go @@ -58,6 +58,8 @@ func RunAllComprehensiveTests(t *testing.T, client *bifrost.Bifrost, ctx context RunTranscriptionStreamAdvancedTest, RunEmbeddingTest, RunRerankTest, + RunDecisionTest, + RunDecisionEmulationTest, RunChatCompletionReasoningTest, RunMultiTurnReasoningTest, RunResponsesReasoningTest, @@ -178,6 +180,8 @@ func printTestSummary(t *testing.T, testConfig ComprehensiveTestConfig) { {"TranscriptionStream", testConfig.Scenarios.TranscriptionStream}, {"Embedding", testConfig.Scenarios.Embedding && testConfig.EmbeddingModel != ""}, {"Rerank", testConfig.Scenarios.Rerank && testConfig.RerankModel != ""}, + {"Decision", testConfig.Scenarios.Decision && testConfig.DecisionModel != ""}, + {"DecisionEmulation", testConfig.Scenarios.DecisionEmulation && testConfig.DecisionEmulationModel != ""}, {"ChatCompletionReasoning", testConfig.Scenarios.Reasoning && testConfig.ReasoningModel != ""}, {"MultiTurnReasoning", testConfig.Scenarios.Reasoning && testConfig.ReasoningModel != ""}, {"ResponsesReasoning", testConfig.Scenarios.Reasoning && testConfig.ReasoningModel != ""}, diff --git a/core/internal/memtest/coverage_test.go b/core/internal/memtest/coverage_test.go new file mode 100644 index 00000000000..2fea5281b1b --- /dev/null +++ b/core/internal/memtest/coverage_test.go @@ -0,0 +1,401 @@ +package memtest + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "os" + "path/filepath" + "sort" + "strings" + "testing" +) + +// rawJSONMutators are the helpers that reserialise an entire JSON document on +// every call (they wrap sjson). Called once, that is fine. Called in a loop over +// something that scales with the request payload, it is O(elements x document), +// which is the shape that made StripEmptyThinkingBlocks 23.6% of all allocation +// in production. +var rawJSONMutators = map[string]bool{ + "DeleteJSONField": true, + "SetJSONField": true, + "SetRawJSONField": true, +} + +// isRawJSONMutation reports whether a call reserialises a whole JSON document. +// +// Two forms count. The providerUtils helpers above, and raw sjson.Set*/Delete* +// calls, which bypass those helpers entirely: core/encryptedreasoning.go does +// exactly that inside a nested loop, and an earlier version of this scan could +// not see it. +func isRawJSONMutation(call *ast.CallExpr) bool { + sel, ok := call.Fun.(*ast.SelectorExpr) + if !ok { + if ident, isIdent := call.Fun.(*ast.Ident); isIdent { + return rawJSONMutators[ident.Name] + } + return false + } + if rawJSONMutators[sel.Sel.Name] { + return true + } + pkg, isIdent := sel.X.(*ast.Ident) + if !isIdent || pkg.Name != "sjson" { + return false + } + return strings.HasPrefix(sel.Sel.Name, "Set") || strings.HasPrefix(sel.Sel.Name, "Delete") +} + +// scanRoots are the trees walked for the shape. Every layer on the request path +// qualifies: the watcher leak that pinned ~80% of a production heap lived in +// transports, not in a provider. +var scanRoots = []string{ + filepath.Join("..", ".."), // core + filepath.Join("..", "..", "..", "framework"), + filepath.Join("..", "..", "..", "transports"), + filepath.Join("..", "..", "..", "plugins"), +} + +type siteStatus int + +const ( + // covered: a memtest asserts this function's allocation scaling. + covered siteStatus = iota + // bounded: the loop is over a fixed or config-sized collection, so N does + // not grow with the request payload and the rewrite cost stays constant. + bounded + // elementSized: the loop writes element-sized documents rather than + // rewriting the whole request body once per element, so it is already linear. + elementSized +) + +type siteReview struct { + status siteStatus + // test names the Test function asserting this site's allocation scaling. + // Required when status is covered, and checked to actually exist, so the + // label cannot drift into a claim nobody is backing. + test string + // reason justifies a bounded or elementSized classification. + reason string +} + +// reviewedSites records every raw-JSON mutation sitting inside a loop, and what +// was concluded about it. A site missing from this map fails the test below. +// +// The point is to invert the default: a new provider, a new loop, or a deleted +// memtest all show up as a failure rather than as silence. The scan cannot tell +// a payload-scaling loop from a bounded one, and it cannot tell a whole-document +// rewrite from an element-sized one, so that judgement is recorded here by a +// human once and re-checked automatically forever after. +var reviewedSites = map[string]siteReview{ + // A function that drops OFF this list has graduated: its loop no longer mutates + // raw JSON at all, because the edits were batched into a single write. Its + // AssertAllocScaling test stays behind as the regression guard. + // StripAutoInjectableTools and gemini.stripFunctionResponseMediaRefs both left + // this way. + // + // --- payload-scaling: N grows with the request, so these need assertions --- + "anthropic.StripEmptyThinkingBlocks": { + status: covered, + test: "TestStripEmptyThinkingBlocks_AllocationScaling", + }, + "anthropic.RemapRawToolVersionsForProvider": { + status: covered, + test: "TestRemapRawToolVersionsForProvider_AllocationScaling", + }, + "anthropic.StripUnsupportedFieldsFromRawBody": { + status: covered, + test: "TestStripUnsupportedFieldsFromRawBody_AllocationScaling", + }, + "anthropic.normalizeBase64TextSources": { + status: covered, + test: "TestNormalizeBase64TextSources_AllocationScaling", + }, + "core.stripRawAnthropicChatThinking": { + status: covered, + test: "TestStripRawAnthropicChatThinking_AllocationScaling", + }, + "core.stripRawResponsesEncryptedContent": { + status: covered, + test: "TestStripRawResponsesEncryptedContent_AllocationScaling", + }, + "databricks.normalizeReasoningBlocks": { + status: covered, + test: "TestNormalizeReasoningBlocks_AllocationScaling", + }, + + // --- bounded: the loop is over config or a fixed literal, so N is constant --- + "schemas.RawSchemaJSON": { + status: bounded, + reason: "loops over the 3-element literal {name, strict, schema}", + }, + "integrations.createAnthropicMessagesRouteConfig": { + status: bounded, + reason: "the sjson call sits in a ResponsesResponseConverter closure defined inside a " + + "2-element literal path loop; it runs once per response, not once per iteration", + }, + "anthropic.NormalizeSchemaForAnthropicRaw": { + status: elementSized, + reason: "writes accumulate into small local values (a single schema branch, an enum array), " + + "never the request body; the quadratic term is in enum cardinality, which is schema-bounded", + }, + "anthropic.BuildAnthropicChatRequestBody": { + status: bounded, + reason: "loops over cfg.ExcludeFields, which is provider config and does not grow with the request payload", + }, + "anthropic.BuildAnthropicResponsesRequestBody": { + status: bounded, + reason: "loops over cfg.ExcludeFields, which is provider config and does not grow with the request payload", + }, + "anthropic.AddMissingBetaHeadersToContextFromRawBody": { + status: bounded, + reason: "loops over the 2-element literal {messages, tools} it trims before the probe decode", + }, + "gemini.NormalizeRawGenerateContentRequestForCompatibility": { + status: bounded, + reason: "loops over a 5-element literal slice of generationConfig paths", + }, + "gemini.wrapGeminiCountTokensBody": { + status: bounded, + reason: "loops over the fixed geminiCountTokensUnsupportedFields package var", + }, + "vertex.stripVertexCountTokensUnsupportedFields": { + status: bounded, + reason: "loops over the fixed vertexCountTokensUnsupportedFields package var", + }, +} + +type foundSite struct { + key string + file string + line int + function string +} + +// scanLoopBoundJSONMutations walks core/providers and reports every raw-JSON +// mutation helper called from inside a for/range statement. +func scanLoopBoundJSONMutations(t *testing.T) []foundSite { + t.Helper() + + var sites []foundSite + fset := token.NewFileSet() + scanned := 0 + + for _, root := range scanRoots { + abs, err := filepath.Abs(root) + if err != nil { + t.Fatalf("resolving %s: %v", root, err) + } + if _, statErr := os.Stat(abs); statErr != nil { + // A module may be absent when core is consumed standalone. Skip it + // rather than fail: the roots that do exist still get scanned. + continue + } + repoRel := filepath.Dir(abs) + + walkErr := filepath.WalkDir(abs, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + // Vendored and generated trees are not ours to classify. + if name := d.Name(); name == "vendor" || name == "node_modules" || name == "testdata" { + return filepath.SkipDir + } + return nil + } + if !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") { + return nil + } + file, parseErr := parser.ParseFile(fset, path, nil, 0) + if parseErr != nil { + return fmt.Errorf("parsing %s: %w", path, parseErr) + } + scanned++ + + rel, relErr := filepath.Rel(repoRel, path) + if relErr != nil { + rel = path + } + pkg := filepath.Base(filepath.Dir(path)) + + for _, decl := range file.Decls { + fn, ok := decl.(*ast.FuncDecl) + if !ok || fn.Body == nil { + continue + } + ast.Inspect(fn.Body, func(n ast.Node) bool { + var body *ast.BlockStmt + switch loop := n.(type) { + case *ast.RangeStmt: + body = loop.Body + case *ast.ForStmt: + body = loop.Body + default: + return true + } + ast.Inspect(body, func(inner ast.Node) bool { + call, isCall := inner.(*ast.CallExpr) + if !isCall || !isRawJSONMutation(call) { + return true + } + sites = append(sites, foundSite{ + key: pkg + "." + fn.Name.Name, + file: filepath.ToSlash(rel), + line: fset.Position(call.Pos()).Line, + function: fn.Name.Name, + }) + return true + }) + return true + }) + } + return nil + }) + if walkErr != nil { + t.Fatalf("scanning %s: %v", root, walkErr) + } + } + + if scanned == 0 { + t.Fatal("scan parsed no Go files at all; the scanRoots paths have moved") + } + return sites +} + +// TestEveryLoopBoundJSONMutationIsReviewed fails when a provider gains a +// raw-JSON mutation inside a loop that nobody has classified. +// +// This is the part that makes the memtest suite survive a provider added six +// months from now. The per-provider allocation tests only cover what someone +// remembered to register; this test makes forgetting loud. +func TestEveryLoopBoundJSONMutationIsReviewed(t *testing.T) { + sites := scanLoopBoundJSONMutations(t) + if len(sites) == 0 { + t.Fatal("scan found no loop-bound raw-JSON mutations at all, which means the scan itself " + + "has broken (the helpers were renamed, or the providers path moved)") + } + + byKey := map[string][]foundSite{} + for _, s := range sites { + byKey[s.key] = append(byKey[s.key], s) + } + + var unreviewed []string + for key, group := range byKey { + if _, ok := reviewedSites[key]; ok { + continue + } + locs := make([]string, 0, len(group)) + for _, s := range group { + locs = append(locs, fmt.Sprintf("%s:%d", s.file, s.line)) + } + sort.Strings(locs) + unreviewed = append(unreviewed, fmt.Sprintf(" %q: {} // %s", key, strings.Join(locs, ", "))) + } + sort.Strings(unreviewed) + + if len(unreviewed) > 0 { + t.Errorf("%d loop-bound raw-JSON mutation site(s) are not classified in reviewedSites.\n\n"+ + "Each rewrites the whole document per iteration. Decide which it is and record it:\n"+ + " - loop scales with the request payload -> add an AssertAllocScaling test, mark {status: covered}\n"+ + " - loop is over config or a fixed list -> mark {status: bounded, reason: \"...\"}\n"+ + " - each write is element-sized, not whole-document -> mark {status: elementSized, reason: \"...\"}\n\n"+ + "Add to reviewedSites in coverage_test.go:\n%s", + len(unreviewed), strings.Join(unreviewed, "\n")) + } + + // A stale entry is as misleading as a missing one: it implies coverage that + // no longer corresponds to any code. + var stale []string + for key := range reviewedSites { + if _, ok := byKey[key]; !ok { + stale = append(stale, key) + } + } + sort.Strings(stale) + if len(stale) > 0 { + t.Errorf("reviewedSites lists %d entr(ies) with no matching code; the function was renamed "+ + "or the loop removed, so drop them: %s", len(stale), strings.Join(stale, ", ")) + } +} + +// TestCoveredSitesHaveARealTest keeps `covered` honest. +// +// Without this, marking a site covered is a claim nobody checks, and the whole +// registry degrades into a list of good intentions. Here a covered entry must +// name a Test function that actually exists in that provider's package. +func TestCoveredSitesHaveARealTest(t *testing.T) { + // package name -> set of Test function names declared in its _test.go files. + // Walks the same roots as the site scanner: a covered entry in core/ or + // transports/ must be able to find its test just as a provider one does. + testsByProvider := map[string]map[string]bool{} + fset := token.NewFileSet() + for _, root := range scanRoots { + abs, err := filepath.Abs(root) + if err != nil { + t.Fatalf("resolving %s: %v", root, err) + } + if _, statErr := os.Stat(abs); statErr != nil { + continue + } + walkErr := filepath.WalkDir(abs, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + if name := d.Name(); name == "vendor" || name == "node_modules" || name == "testdata" { + return filepath.SkipDir + } + return nil + } + if !strings.HasSuffix(path, "_test.go") { + return nil + } + file, parseErr := parser.ParseFile(fset, path, nil, 0) + if parseErr != nil { + return fmt.Errorf("parsing %s: %w", path, parseErr) + } + pkg := filepath.Base(filepath.Dir(path)) + if testsByProvider[pkg] == nil { + testsByProvider[pkg] = map[string]bool{} + } + for _, decl := range file.Decls { + if fn, ok := decl.(*ast.FuncDecl); ok && strings.HasPrefix(fn.Name.Name, "Test") { + testsByProvider[pkg][fn.Name.Name] = true + } + } + return nil + }) + if walkErr != nil { + t.Fatalf("scanning tests under %s: %v", root, walkErr) + } + } + + var problems []string + for key, review := range reviewedSites { + provider := key[:strings.Index(key, ".")] + switch review.status { + case covered: + if review.test == "" { + problems = append(problems, fmt.Sprintf("%s is marked covered but names no test", key)) + continue + } + if !testsByProvider[provider][review.test] { + problems = append(problems, fmt.Sprintf( + "%s is marked covered by %s, but no such Test function exists in package %s", + key, review.test, provider)) + } + case bounded, elementSized: + if review.reason == "" { + problems = append(problems, fmt.Sprintf( + "%s is marked bounded/elementSized but gives no reason; the next reader needs to know why", key)) + } + } + } + sort.Strings(problems) + for _, p := range problems { + t.Error(p) + } +} diff --git a/core/internal/memtest/memtest.go b/core/internal/memtest/memtest.go new file mode 100644 index 00000000000..6219b681e0c --- /dev/null +++ b/core/internal/memtest/memtest.go @@ -0,0 +1,199 @@ +// Package memtest holds assertions for memory-regression tests. +// +// It exists because two production memory bugs shipped in quick succession with +// opposite signatures, and neither was catchable by the other's technique: +// +// - StripEmptyThinkingBlocks rewrote the whole request body once per stripped +// content block. That was 23.6% of every byte the gateway allocated and 0% +// of its live heap: pure churn, invisible to any heap check. +// - The client-disconnect watcher goroutine outlived its request and pinned +// the request-scoped context. That was ~0% of allocation and ~80% of the +// live heap: pure retention, invisible to any allocation check. +// +// So the package offers both: AssertAllocScaling for churn, and +// AssertNoGoroutineLeak / AssertReleased for retention. +// +// This package must not import anything from core/providers. Provider test +// files import it, so a dependency the other way would be an import cycle. +package memtest + +import ( + "runtime" + "testing" + "time" + "weak" +) + +// Defaults for AssertAllocScaling. Base and Factor pick two input sizes; the +// assertion compares how allocation grew against how the input grew. +// +// MaxGrowth sits between the two outcomes rather than near either. A linear +// implementation lands at about Factor (4), a quadratic one at about Factor +// squared (16). Measured on the real before/after of the strip bug: 3.9x and +// 13.6x. Eight separates them with room on both sides. +const ( + DefaultBase = 50 + DefaultFactor = 4 + DefaultMaxGrowth = 8.0 +) + +// Config tunes AssertAllocScalingWith. A zero value means the defaults above. +type Config struct { + // Base is the smaller input scale handed to build. + Base int + // Factor multiplies Base for the larger input. Must be at least 2. + Factor int + // MaxGrowth is the allocation-growth ratio above which the test fails. + MaxGrowth float64 + // AllowZeroAlloc permits a function that allocates nothing. + // + // Off by default, and that default is the point. A payload that fails to + // reach the code path allocates nothing, and a growth ratio computed from + // zero would report a pass. Silently passing a test whose subject never ran + // is worse than having no test, so the zero case fails unless a caller + // deliberately opts in for a genuinely allocation-free function. + AllowZeroAlloc bool +} + +func (c Config) withDefaults() Config { + if c.Base <= 0 { + c.Base = DefaultBase + } + if c.Factor < 2 { + c.Factor = DefaultFactor + } + if c.MaxGrowth <= 0 { + c.MaxGrowth = DefaultMaxGrowth + } + return c +} + +// AllocBytesPerOp reports the bytes fn allocates per call. +// +// It runs fn under testing.Benchmark rather than diffing runtime.MemStats, so +// the figure is amortised over many iterations and is not perturbed by whatever +// else the test binary's goroutines happen to be doing. +func AllocBytesPerOp(fn func()) int64 { + result := testing.Benchmark(func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + fn() + } + }) + return result.AllocedBytesPerOp() +} + +// AssertAllocScaling fails when a function's allocation grows with the square of +// its input instead of with its input. +// +// build(n) must produce a payload whose SIZE scales with n, not merely a payload +// with n elements: the assertion is a ratio between the two measurements, so a +// builder that keeps the byte count flat makes the result meaningless. The +// helper checks this and fails loudly rather than reporting a false pass. +// +// This asserts a shape, not a byte count. An absolute threshold ("under 4 MB") +// encodes the machine and the Go version it was written on and silently rots; a +// growth ratio encodes the complexity class, which is the thing that must not +// regress. +// +// The canonical failure it catches: a loop calling sjson Set/Delete once per +// element, where each call reserialises the entire document. +func AssertAllocScaling(t *testing.T, build func(n int) []byte, fn func([]byte)) { + t.Helper() + AssertAllocScalingWith(t, Config{}, build, fn) +} + +// AssertAllocScalingWith is AssertAllocScaling with explicit sizing. +func AssertAllocScalingWith(t *testing.T, cfg Config, build func(n int) []byte, fn func([]byte)) { + t.Helper() + cfg = cfg.withDefaults() + + small := build(cfg.Base) + large := build(cfg.Base * cfg.Factor) + if len(small) == 0 { + t.Fatalf("build(%d) produced an empty payload", cfg.Base) + } + + inputGrowth := float64(len(large)) / float64(len(small)) + if inputGrowth < float64(cfg.Factor)*0.8 { + t.Fatalf("build does not scale the payload with n: %d -> %d bytes (%.2fx) for n %d -> %d (%dx). "+ + "The growth ratio below is only meaningful when the payload grows with n.", + len(small), len(large), inputGrowth, cfg.Base, cfg.Base*cfg.Factor, cfg.Factor) + } + + smallAlloc := AllocBytesPerOp(func() { fn(small) }) + if smallAlloc == 0 { + if cfg.AllowZeroAlloc { + return + } + t.Fatalf("the function allocated nothing on a %d-byte payload, so there is no growth to measure "+ + "and this test would pass without ever exercising its subject. Either the payload does not "+ + "reach the code path (check the preconditions the function gates on), or the function really "+ + "is allocation-free, in which case set Config.AllowZeroAlloc.", len(small)) + } + largeAlloc := AllocBytesPerOp(func() { fn(large) }) + + growth := float64(largeAlloc) / float64(smallAlloc) + if growth > cfg.MaxGrowth { + t.Errorf("allocation grew %.1fx for a %.1fx larger input (%d -> %d B/op), want at most %.1fx.\n"+ + "That is the O(elements x document) rewrite shape: each sjson Set/Delete reserialises the "+ + "whole document, so N edits cost N copies of it. Collect the edits and apply them in one pass.", + growth, inputGrowth, smallAlloc, largeAlloc, cfg.MaxGrowth) + } +} + +// settleAttempts and settleInterval bound how long a retention assertion waits +// for cleanup that legitimately lands after the operation returns (a deferred +// cancel, a producer goroutine finishing its own teardown). +const ( + settleAttempts = 100 + settleInterval = 10 * time.Millisecond +) + +// AssertNoGoroutineLeak fails when fn leaves goroutines running after it returns. +// +// This is the watcher-leak shape: a goroutine started per request whose exit +// depends on something the handler must remember to do. When it is forgotten the +// goroutine spins forever and pins everything its closure captured, which is how +// ~80% of a production heap became unreclaimable while GC ran perfectly. +// +// The settle loop matters. Cleanup often runs in a deferred block after the +// function under test has returned, so sampling immediately gives a false +// failure. +func AssertNoGoroutineLeak(t *testing.T, fn func()) { + t.Helper() + before := runtime.NumGoroutine() + fn() + for range settleAttempts { + if runtime.NumGoroutine() <= before { + return + } + time.Sleep(settleInterval) + } + t.Errorf("goroutine count went %d -> %d and stayed there %v after the operation completed; "+ + "something started per-operation is outliving it and pinning whatever it captured", + before, runtime.NumGoroutine(), time.Duration(settleAttempts)*settleInterval) +} + +// AssertReleased fails when the value produce returns is still reachable after +// the operation finishes. +// +// Unlike a heap-size check this proves unreachability rather than inferring it +// from a number that could have moved for any reason. Use it for objects that +// must not outlive a request: a context, an accumulator entry, a cache value. +// +// produce must both create the value and run whatever is supposed to release it, +// returning the pointer to probe. Keeping that inside a function literal is what +// lets the compiler drop the last strong reference before the GC below. +func AssertReleased[T any](t *testing.T, what string, produce func() *T) { + t.Helper() + ptr := weak.Make(produce()) + // Twice: the first cycle can leave the object queued rather than collected + // when a finalizer or cleanup is attached to it. + runtime.GC() + runtime.GC() + if ptr.Value() != nil { + t.Errorf("%s is still reachable after the operation completed, so it cannot be collected "+ + "for as long as whatever holds it stays alive", what) + } +} diff --git a/core/keyselectors/weightedrandom_bench_test.go b/core/keyselectors/weightedrandom_bench_test.go new file mode 100644 index 00000000000..9d29312980b --- /dev/null +++ b/core/keyselectors/weightedrandom_bench_test.go @@ -0,0 +1,51 @@ +package keyselectors + +// Benchmarks for key selection. +// +// WeightedRandom runs once per request attempt (and again on every fallback +// hop) to pick which provider credential serves it, so it sits directly on the +// request path. It is O(keys) twice over - once to total the weights, once to +// walk them - which is why the benchmark covers a small key set as well as the +// large pools high-throughput deployments configure. +// +// Run: +// +// go test ./core/keyselectors/ -bench WeightedRandom -benchmem + +import ( + "strconv" + "testing" + + "github.com/capsohq/bifrost/core/schemas" +) + +func benchKeys(n int) []schemas.Key { + keys := make([]schemas.Key, n) + for i := range keys { + keys[i] = schemas.Key{ + ID: "key-" + strconv.Itoa(i), + Name: "bench-key-" + strconv.Itoa(i), + Models: schemas.WhiteList{"gpt-4o", "gpt-4o-mini", "o3-mini"}, + Weight: 1.0 / float64(n), + } + } + return keys +} + +func benchmarkWeightedRandom(b *testing.B, keyCount int) { + keys := benchKeys(keyCount) + b.ReportAllocs() + for b.Loop() { + key, err := WeightedRandom(nil, keys, schemas.OpenAI, "gpt-4o") + if err != nil { + b.Fatalf("select key: %v", err) + } + if key.ID == "" { + b.Fatal("empty key selected") + } + } +} + +func BenchmarkWeightedRandom3Keys(b *testing.B) { benchmarkWeightedRandom(b, 3) } +func BenchmarkWeightedRandom25Keys(b *testing.B) { benchmarkWeightedRandom(b, 25) } +func BenchmarkWeightedRandom200Keys(b *testing.B) { benchmarkWeightedRandom(b, 200) } diff --git a/core/mcp/clientmanager.go b/core/mcp/clientmanager.go index d76389c9b9e..bf4bf783353 100644 --- a/core/mcp/clientmanager.go +++ b/core/mcp/clientmanager.go @@ -7,7 +7,9 @@ import ( "errors" "fmt" "maps" + "net" "net/http" + "net/url" "os" "slices" "strings" @@ -2639,6 +2641,12 @@ func (m *MCPManager) failConnectAttempt(entry *schemas.MCPClientState, config *s // back to the mcp-go library's own default client, which carried no guard at // all - not even the link-local/metadata block applied here. // +// The process's HTTP_PROXY / HTTPS_PROXY / NO_PROXY environment is honored +// through the http.ProxyFromEnvironment selector the cloned DefaultTransport +// carries, matching every other outbound client in the process and the +// behavior before core v1.8.5. See mcpProxySelector for how the destination +// guard stays intact on the proxied path. +// // TLS customization from MCPTLSConfig is layered on top when provided. // InsecureSkipVerify takes priority over CACertPEM when both are set. func (m *MCPManager) buildTLSHTTPClient(tlsCfg *schemas.MCPTLSConfig) (*http.Client, error) { @@ -2647,14 +2655,19 @@ func (m *MCPManager) buildTLSHTTPClient(tlsCfg *schemas.MCPTLSConfig) (*http.Cli baseTransport = &http.Transport{} } cloned := baseTransport.Clone() - // Clone() preserves DefaultTransport's http.ProxyFromEnvironment. With a - // proxy configured, http.Transport hands DialContext the proxy's address - // instead of the MCP destination, so the guard below would validate the - // proxy and the proxy would forward to the blocked target. Drop it: the - // guard is only meaningful when the dialer sees the real destination, - // which is also how every other SSRF-guarded client here is built (fresh, - // proxy-free transports in fetch.go, webhooks, and skills_serving). - cloned.Proxy = nil + // Clone() preserves DefaultTransport's http.ProxyFromEnvironment, and it + // must stay: a deployment with no direct egress (a corporate VPC, or a + // pod behind an explicit HTTP_PROXY/HTTPS_PROXY) can only reach a public + // MCP server through that proxy. Proxying does change what the dial-time + // guard sees, though: http.Transport hands DialContext the proxy's + // address, not the MCP destination, so the guard alone would validate the + // proxy and the proxy would forward to the blocked target. The selector + // wrapper closes that gap for what the gateway can check (an IP-literal + // destination, and a hostname resolved on the gateway) whenever + // a proxy is chosen for a request; on the direct path (no proxy + // configured, or a NO_PROXY match) it stays out of the way and the guard + // below sees the real target as before. + cloned.Proxy = mcpProxySelector(cloned.Proxy) cloned.DialContext = network.PrivateNetworkDialContext(mcpDialTimeout) if tlsCfg != nil { @@ -2680,6 +2693,43 @@ func (m *MCPManager) buildTLSHTTPClient(tlsCfg *schemas.MCPTLSConfig) (*http.Cli return &http.Client{Transport: cloned}, nil } +// mcpProxySelector wraps an http.Transport Proxy selector so the +// PrivateNetworkDialContext destination policy still applies to a request that +// is routed through a proxy. A direct request is validated by the dialer, +// which sees the real target. A proxied request is validated here before the +// proxy is chosen: an IP-literal host is refused if it is unspecified, +// link-local, or a cloud metadata endpoint, and a hostname must resolve on the +// gateway with every address passing the same policy. A name the gateway +// cannot resolve is refused rather than handed to the proxy, whose resolver +// could map it to an address the policy blocks; deployments whose DNS lives +// only behind the proxy must let the gateway resolve its MCP servers. The +// proxy still resolves the name again on its own side, so this binds the +// destination as far as the gateway can see it, not what the proxy dials. +// The dialer still refuses a proxy that itself sits on a blocked address. +// A nil selector is returned as nil so a transport without one is unchanged. +func mcpProxySelector(next func(*http.Request) (*url.URL, error)) func(*http.Request) (*url.URL, error) { + if next == nil { + return nil + } + return func(req *http.Request) (*url.URL, error) { + proxyURL, err := next(req) + if err != nil || proxyURL == nil { + return proxyURL, err + } + host := req.URL.Hostname() + if net.ParseIP(strings.Trim(host, "[]")) != nil { + if err := network.CheckPrivateNetworkLiteral(host); err != nil { + return nil, err + } + return proxyURL, nil + } + if _, err := network.ResolvePrivateNetworkTarget(req.Context(), host); err != nil { + return nil, err + } + return proxyURL, nil + } +} + // createHTTPConnection creates an HTTP-based MCP client connection without holding locks. // If overrides is non-nil and carries a populated ConnectionString or Headers, those values // are used instead of resolving them from config. This is how plugin PreHook mutations flow diff --git a/core/mcp/clientmanager_test.go b/core/mcp/clientmanager_test.go index 0b1b3be23b8..77ca817379e 100644 --- a/core/mcp/clientmanager_test.go +++ b/core/mcp/clientmanager_test.go @@ -3,9 +3,12 @@ package mcp import ( "context" "errors" + "io" "net" "net/http" + "net/http/httptest" "net/url" + "sync" "sync/atomic" "testing" "time" @@ -677,22 +680,109 @@ func TestBuildTLSHTTPClientAllowsLoopback(t *testing.T) { } } -// TestBuildTLSHTTPClientNeverRoutesThroughProxy proves the dial-time guard -// cannot be sidestepped by a proxy. http.DefaultTransport carries -// http.ProxyFromEnvironment, and Clone() preserves it; when a proxy is -// configured, http.Transport hands DialContext the proxy's address rather than -// the MCP destination, so PrivateNetworkDialContext would validate the proxy -// and let the proxy forward to the blocked link-local target. Every other -// SSRF-guarded client in this repo (image/document fetch, webhook delivery, -// skill URL sources) builds a fresh proxy-free transport; the MCP client must -// end up with the same property despite starting from a clone. +// TestBuildTLSHTTPClientRoutesThroughConfiguredProxy proves the client honors +// the transport's proxy selector for a permitted destination, which is what a +// deployment with no direct egress depends on: the request must reach the +// proxy, and the destination must never be dialed directly. This is the +// behavior core v1.8.5 through v1.9.1 broke by nulling the selector. // // The proxy is injected by swapping DefaultTransport.Proxy rather than via // HTTP_PROXY, because ProxyFromEnvironment reads the environment once per // process and would not see a t.Setenv made after any earlier test used it. -// This test therefore stays sequential (no t.Parallel) and restores the -// original Proxy on cleanup. -func TestBuildTLSHTTPClientNeverRoutesThroughProxy(t *testing.T) { +// Every test here that touches DefaultTransport.Proxy therefore stays +// sequential (no t.Parallel) and restores the original selector on cleanup. +func TestBuildTLSHTTPClientRoutesThroughConfiguredProxy(t *testing.T) { + // The destination must never see a connection: with a proxy selected, + // only the proxy may be dialed. + targetLn, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(func() { + if err := targetLn.Close(); err != nil { + t.Errorf("failed to close destination listener: %v", err) + } + }) + var targetHits atomic.Int32 + go func() { + for { + conn, err := targetLn.Accept() + if err != nil { + return + } + targetHits.Add(1) + _ = conn.Close() + } + }() + + // A minimal proxy: records what it was asked for, answers a plain-HTTP + // absolute-URL request itself, and refuses a CONNECT, since tunnelling + // TLS is not what this test is about; seeing the CONNECT is enough. + var ( + proxyHits atomic.Int32 + mu sync.Mutex + lastReq string + ) + proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + proxyHits.Add(1) + mu.Lock() + lastReq = r.Method + " " + r.Host + mu.Unlock() + if r.Method == http.MethodConnect { + http.Error(w, "no tunnel in this test", http.StatusBadGateway) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("via-proxy")) + })) + t.Cleanup(proxy.Close) + proxyURL, err := url.Parse(proxy.URL) + require.NoError(t, err) + seen := func() string { + mu.Lock() + defer mu.Unlock() + return lastReq + } + + base, ok := http.DefaultTransport.(*http.Transport) + require.True(t, ok) + origProxy := base.Proxy + base.Proxy = http.ProxyURL(proxyURL) + t.Cleanup(func() { base.Proxy = origProxy }) + + httpClient, err := (&MCPManager{logger: defaultLogger}).buildTLSHTTPClient(nil) + require.NoError(t, err) + httpClient.Timeout = 5 * time.Second + target := targetLn.Addr().String() + + t.Run("http", func(t *testing.T) { + resp, err := httpClient.Get("http://" + target + "/mcp") + require.NoError(t, err) + body, readErr := io.ReadAll(resp.Body) + require.NoError(t, resp.Body.Close()) + require.NoError(t, readErr) + require.Equal(t, http.StatusOK, resp.StatusCode) + require.Equal(t, "via-proxy", string(body), "the response must come from the proxy") + require.Equal(t, "GET "+target, seen(), "the proxy must be asked for the MCP destination") + }) + t.Run("https", func(t *testing.T) { + resp, err := httpClient.Get("https://" + target + "/mcp") + if resp != nil { + require.NoError(t, resp.Body.Close()) + } + require.Error(t, err, "the test proxy refuses tunnels, so the request must fail at the proxy") + require.Equal(t, "CONNECT "+target, seen(), "an https destination must be tunnelled through the proxy") + }) + require.Equal(t, int32(2), proxyHits.Load(), "one proxy request per scheme") + require.Zero(t, targetHits.Load(), "with a proxy selected the destination must never be dialed directly") +} + +// TestBuildTLSHTTPClientRefusesProxyingBlockedLiteral proves the dial-time +// guard cannot be sidestepped by a proxy. When a proxy is selected, +// http.Transport hands DialContext the proxy's address rather than the MCP +// destination, so PrivateNetworkDialContext alone would validate the proxy +// and let the proxy forward to the blocked target. The selector wrapper must +// refuse a blocked IP-literal destination (link-local, and the cloud metadata +// endpoints that live outside link-local) before the proxy is ever contacted. +func TestBuildTLSHTTPClientRefusesProxyingBlockedLiteral(t *testing.T) { proxyLn, err := net.Listen("tcp", "127.0.0.1:0") require.NoError(t, err) t.Cleanup(func() { @@ -728,25 +818,164 @@ func TestBuildTLSHTTPClientNeverRoutesThroughProxy(t *testing.T) { base.Proxy = http.ProxyURL(proxyURL) t.Cleanup(func() { base.Proxy = origProxy }) + targets := []struct { + host string + want string + }{ + {"169.254.169.254", "blocked connection to link-local address"}, + {"100.100.100.200", "blocked connection to cloud metadata endpoint"}, + {"[fd00:ec2::254]", "blocked connection to cloud metadata endpoint"}, + } for _, scheme := range []string{"http", "https"} { - t.Run(scheme, func(t *testing.T) { - httpClient, err := (&MCPManager{logger: defaultLogger}).buildTLSHTTPClient(nil) - require.NoError(t, err) - httpClient.Timeout = 5 * time.Second - - resp, err := httpClient.Get(scheme + "://169.254.169.254/latest/meta-data/") - if resp != nil { - require.NoError(t, resp.Body.Close()) - } - require.Error(t, err, "a link-local MCP target must be refused even when a proxy is configured") - require.Contains(t, err.Error(), "blocked connection to link-local address", - "the refusal must come from the dial-time guard, not from the proxy") - require.Zero(t, proxyHits.Load(), "the configured proxy must never be contacted for a guarded MCP request") - select { - case closeErr := <-proxyCloseErrs: - t.Errorf("failed to close a proxy-side connection: %v", closeErr) - default: - } - }) + for _, tc := range targets { + t.Run(scheme+"/"+tc.host, func(t *testing.T) { + httpClient, err := (&MCPManager{logger: defaultLogger}).buildTLSHTTPClient(nil) + require.NoError(t, err) + httpClient.Timeout = 5 * time.Second + + resp, err := httpClient.Get(scheme + "://" + tc.host + "/latest/meta-data/") + if resp != nil { + require.NoError(t, resp.Body.Close()) + } + require.Error(t, err, "a blocked MCP target must be refused even when a proxy is configured") + require.Contains(t, err.Error(), tc.want, + "the refusal must come from the destination policy, not from the proxy") + require.Zero(t, proxyHits.Load(), "the configured proxy must never be contacted for a guarded MCP request") + select { + case closeErr := <-proxyCloseErrs: + t.Errorf("failed to close a proxy-side connection: %v", closeErr) + default: + } + }) + } + } +} + +// TestBuildTLSHTTPClientProxiedHostnameNeedsLocalDNS pins the fork's proxied-name +// contract (upstream hands an unresolved name to the proxy): a name the gateway cannot +// resolve is refused before the proxy is contacted, since the proxy's resolver could map +// it to a blocked address, while a name that resolves to a permitted address is still sent +// to the proxy under its original hostname. +func TestBuildTLSHTTPClientProxiedHostnameNeedsLocalDNS(t *testing.T) { + var ( + mu sync.Mutex + requests []string + ) + proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + requests = append(requests, r.Method+" "+r.Host) + mu.Unlock() + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("via-proxy")) + })) + t.Cleanup(proxy.Close) + proxyURL, err := url.Parse(proxy.URL) + require.NoError(t, err) + + base, ok := http.DefaultTransport.(*http.Transport) + require.True(t, ok) + origProxy := base.Proxy + base.Proxy = http.ProxyURL(proxyURL) + t.Cleanup(func() { base.Proxy = origProxy }) + + httpClient, err := (&MCPManager{logger: defaultLogger}).buildTLSHTTPClient(nil) + require.NoError(t, err) + httpClient.Timeout = 5 * time.Second + + _, err = httpClient.Get("http://mcp.invalid/mcp") + require.Error(t, err, "a hostname the gateway cannot resolve must not be sent to the proxy") + + resp, err := httpClient.Get("http://localhost/mcp") + require.NoError(t, err) + body, readErr := io.ReadAll(resp.Body) + require.NoError(t, resp.Body.Close()) + require.NoError(t, readErr) + require.Equal(t, http.StatusOK, resp.StatusCode) + require.Equal(t, "via-proxy", string(body)) + mu.Lock() + defer mu.Unlock() + require.Equal(t, []string{"GET localhost"}, requests, "only the resolvable name reaches the proxy, under its original hostname") +} + +// TestBuildTLSHTTPClientBypassedProxyDialsDirect covers the NO_PROXY shape: the +// selector is present but returns no proxy for the request, so the transport +// dials the destination itself and the dial-time guard is what applies. A +// loopback destination is reached directly and a link-local one is still +// blocked, exactly as when no proxy is configured at all. +func TestBuildTLSHTTPClientBypassedProxyDialsDirect(t *testing.T) { + var targetHits atomic.Int32 + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + targetHits.Add(1) + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("direct")) + })) + t.Cleanup(target.Close) + + base, ok := http.DefaultTransport.(*http.Transport) + require.True(t, ok) + origProxy := base.Proxy + base.Proxy = func(*http.Request) (*url.URL, error) { return nil, nil } + t.Cleanup(func() { base.Proxy = origProxy }) + + httpClient, err := (&MCPManager{logger: defaultLogger}).buildTLSHTTPClient(nil) + require.NoError(t, err) + httpClient.Timeout = 5 * time.Second + + resp, err := httpClient.Get(target.URL + "/mcp") + require.NoError(t, err) + body, readErr := io.ReadAll(resp.Body) + require.NoError(t, resp.Body.Close()) + require.NoError(t, readErr) + require.Equal(t, "direct", string(body)) + require.Equal(t, int32(1), targetHits.Load(), "a bypassed proxy means the destination is dialed directly") + + resp, err = httpClient.Get("http://169.254.169.254/latest/meta-data/") + if resp != nil { + require.NoError(t, resp.Body.Close()) } + require.Error(t, err) + require.Contains(t, err.Error(), "blocked connection to link-local address", + "on the direct path the dial-time guard still refuses link-local") +} + +// TestMCPProxySelectorPassthrough pins the wrapper's two pass-through +// behaviors: a nil selector stays nil so a transport without one is unchanged, +// and an error from the wrapped selector is returned as-is. +func TestMCPProxySelectorPassthrough(t *testing.T) { + require.Nil(t, mcpProxySelector(nil)) + + wantErr := errors.New("selector failed") + sel := mcpProxySelector(func(*http.Request) (*url.URL, error) { return nil, wantErr }) + req, err := http.NewRequest(http.MethodGet, "http://127.0.0.1/mcp", nil) + require.NoError(t, err) + _, err = sel(req) + require.ErrorIs(t, err, wantErr) +} + +// A proxied hostname is resolved on the gateway before the proxy is chosen. A name the +// gateway cannot resolve is refused rather than handed to the proxy, whose resolver could +// map it to a blocked address; a name that resolves to a permitted address goes through. +func TestMCPProxySelectorResolvesProxiedHostnames(t *testing.T) { + proxyURL, err := url.Parse("http://127.0.0.1:3128") + require.NoError(t, err) + sel := mcpProxySelector(http.ProxyURL(proxyURL)) + + for _, target := range []string{"http://bifrost.invalid/mcp", "https://bifrost.invalid:8443/mcp"} { + req, err := http.NewRequest(http.MethodGet, target, nil) + require.NoError(t, err) + got, err := sel(req) + require.Error(t, err, "%s must not reach the proxy unresolved", target) + require.Nil(t, got) + } + + req, err := http.NewRequest(http.MethodGet, "http://localhost/mcp", nil) + require.NoError(t, err) + got, err := sel(req) + require.NoError(t, err, "a resolvable private-network name is permitted, as on the direct path") + require.Equal(t, proxyURL, got) + + req, err = http.NewRequest(http.MethodGet, "http://169.254.169.254/latest/meta-data/", nil) + require.NoError(t, err) + _, err = sel(req) + require.Error(t, err, "a link-local literal is still refused before the proxy") } diff --git a/core/mcp/codemode/starlark/starlark_test.go b/core/mcp/codemode/starlark/starlark_test.go index cf244864040..88d83a96f05 100644 --- a/core/mcp/codemode/starlark/starlark_test.go +++ b/core/mcp/codemode/starlark/starlark_test.go @@ -371,7 +371,7 @@ func TestExtractResultFromResponsesMessage(t *testing.T) { errorMsg := "Tool is not allowed by security policy: dangerous_tool" msg := &schemas.ResponsesMessage{ ResponsesToolMessage: &schemas.ResponsesToolMessage{ - Error: &errorMsg, + Error: &schemas.ResponsesToolMessageError{ResponsesToolMessageErrorStr: &errorMsg}, }, } @@ -507,9 +507,13 @@ func TestExtractResultFromResponsesMessage(t *testing.T) { t.Run("Handle empty error string (should not error)", func(t *testing.T) { emptyError := "" + output := "successful output" msg := &schemas.ResponsesMessage{ ResponsesToolMessage: &schemas.ResponsesToolMessage{ - Error: &emptyError, + Error: &schemas.ResponsesToolMessageError{ResponsesToolMessageErrorStr: &emptyError}, + Output: &schemas.ResponsesToolMessageOutputStruct{ + ResponsesToolCallOutputStr: &output, + }, }, } @@ -517,8 +521,30 @@ func TestExtractResultFromResponsesMessage(t *testing.T) { if err != nil { t.Errorf("Expected no error for empty error string, got: %v", err) } + if result != output { + t.Errorf("Expected output for empty legacy error string, got %v", result) + } + }) + + t.Run("Handle empty structured error before output", func(t *testing.T) { + output := "must not be returned" + msg := &schemas.ResponsesMessage{ + ResponsesToolMessage: &schemas.ResponsesToolMessage{ + Error: &schemas.ResponsesToolMessageError{ + ResponsesToolMessageErrorStruct: &schemas.ResponsesToolMessageErrorStruct{}, + }, + Output: &schemas.ResponsesToolMessageOutputStruct{ + ResponsesToolCallOutputStr: &output, + }, + }, + } + + result, err := extractResultFromResponsesMessage(msg) + if err == nil || err.Error() != "tool call returned an error" { + t.Fatalf("Expected fallback structured error, got result=%v err=%v", result, err) + } if result != nil { - t.Errorf("Expected nil result for empty error string, got %v", result) + t.Fatalf("Expected nil result when structured error is present, got %v", result) } }) } diff --git a/core/mcp/codemode/starlark/utils.go b/core/mcp/codemode/starlark/utils.go index 2ca8c57ff0d..28d6942b3c2 100644 --- a/core/mcp/codemode/starlark/utils.go +++ b/core/mcp/codemode/starlark/utils.go @@ -137,8 +137,12 @@ func extractResultFromResponsesMessage(msg *schemas.ResponsesMessage) (interface } if msg.ResponsesToolMessage != nil { - if msg.ResponsesToolMessage.Error != nil && *msg.ResponsesToolMessage.Error != "" { - return nil, fmt.Errorf("%s", *msg.ResponsesToolMessage.Error) + if toolError := msg.ResponsesToolMessage.Error; toolError.IsError() { + errText := toolError.Text() + if errText == "" { + errText = "tool call returned an error" + } + return nil, fmt.Errorf("%s", errText) } if msg.ResponsesToolMessage.Output != nil { diff --git a/core/network/ssrf.go b/core/network/ssrf.go index fa94b6b771e..7030d883007 100644 --- a/core/network/ssrf.go +++ b/core/network/ssrf.go @@ -28,6 +28,17 @@ var cgnat = netip.MustParsePrefix("100.64.0.0/10") // systems, so nothing is lost by refusing the range outright. var teredo = netip.MustParsePrefix("2001:0000::/32") +// metadataEndpoints are cloud instance-metadata service addresses that sit +// outside the link-local range, so the link-local block alone does not cover +// them. 169.254.169.254 (AWS, GCP, Azure, Oracle, DigitalOcean) is already +// refused as link-local. These two fall inside ranges the private-network +// policy otherwise permits: Alibaba Cloud ECS serves metadata from CGNAT +// space, and the AWS IMDS IPv6 endpoint is a unique-local address. +var metadataEndpoints = []netip.Addr{ + netip.MustParseAddr("100.100.100.200"), // Alibaba Cloud ECS + netip.MustParseAddr("fd00:ec2::254"), // AWS IMDS over IPv6 +} + // IsPublicIP reports whether ip is safe to dial from server-side code that // fetches user-controlled URLs: not loopback, private, CGNAT, link-local, // unique-local, site-local, multicast, broadcast, or unspecified. IPv6 forms @@ -301,11 +312,83 @@ func isValidHostnameLiteral(s string) bool { return true } +// checkPrivateNetworkPolicy applies the PrivateNetworkDialContext destination +// policy to one address: unspecified, link-local, and cloud metadata +// endpoints are refused; everything else, loopback and private ranges +// included, is permitted. IPv6 forms that embed an IPv4 address (IPv4-mapped, +// 6to4, NAT64) are judged by the embedded IPv4 as well, as IsPublicIP does, +// so a blocked endpoint cannot be reached through a transition +// representation. +func checkPrivateNetworkPolicy(ip net.IP, host string) error { + if ip.IsUnspecified() { + return fmt.Errorf("blocked connection to unspecified address %s (host %s)", ip, host) + } + if IsLinkLocal(ip) { + return fmt.Errorf("blocked connection to link-local address %s (host %s)", ip, host) + } + addr, ok := netip.AddrFromSlice(ip) + if !ok { + return fmt.Errorf("blocked connection to unparseable address %s (host %s)", ip, host) + } + addr = addr.Unmap() + if embedded, ok := embeddedIPv4(addr); ok { + addr = embedded + if addr.IsLinkLocalUnicast() { + return fmt.Errorf("blocked connection to link-local address %s (host %s)", ip, host) + } + } + for _, ep := range metadataEndpoints { + if addr == ep { + return fmt.Errorf("blocked connection to cloud metadata endpoint %s (host %s)", ip, host) + } + } + return nil +} + +// ResolvePrivateNetworkTarget resolves host and applies the +// PrivateNetworkDialContext destination policy to every address it resolves +// to, returning the validated addresses for the caller to dial. It is the +// resolve step of that dialer, kept separate so it can be tested on its own. +// An IP literal resolves to itself without a DNS query. +func ResolvePrivateNetworkTarget(ctx context.Context, host string) ([]net.IP, error) { + ips, err := net.DefaultResolver.LookupIP(ctx, "ip", host) + if err != nil { + return nil, fmt.Errorf("DNS lookup failed for %s: %w", host, err) + } + if len(ips) == 0 { + return nil, fmt.Errorf("DNS lookup for %s returned no addresses", host) + } + for _, ip := range ips { + if err := checkPrivateNetworkPolicy(ip, host); err != nil { + return nil, err + } + } + return ips, nil +} + +// CheckPrivateNetworkLiteral applies the PrivateNetworkDialContext destination +// policy to host without any DNS lookup: an IP literal (with or without IPv6 +// brackets) is checked, and a hostname is accepted as-is. It exists for the +// case where the connection is handed to an intermediary, such as an HTTP +// proxy, that resolves names on its own side: resolving locally there would +// make the request depend on DNS the host may not have (a proxy-only +// deployment), and would still not bind what the proxy connects to. So the +// caller enforces what it can verify without DNS, and name resolution for the +// proxied path is left to the proxy, which is operator configuration. +func CheckPrivateNetworkLiteral(host string) error { + ip := net.ParseIP(strings.Trim(host, "[]")) + if ip == nil { + return nil + } + return checkPrivateNetworkPolicy(ip, host) +} + // PrivateNetworkDialContext returns a DialContext that, unlike // SSRFSafeDialContext, permits loopback, RFC1918/unique-local, and CGNAT // destinations. It still blocks link-local addresses (including the -// 169.254.169.254 cloud metadata endpoint) and unspecified addresses, which -// have no legitimate destination under any deployment topology. Same +// 169.254.169.254 cloud metadata endpoint), the cloud metadata endpoints that +// live outside link-local (see metadataEndpoints), and unspecified addresses, +// which have no legitimate destination under any deployment topology. Same // DNS-rebinding protection as SSRFSafeDialContext: resolves once and dials // the validated IP directly. // @@ -322,20 +405,9 @@ func PrivateNetworkDialContext(dialTimeout time.Duration) func(ctx context.Conte if err != nil { return nil, fmt.Errorf("invalid dial address %q: %w", addr, err) } - ips, err := net.DefaultResolver.LookupIP(ctx, "ip", host) + ips, err := ResolvePrivateNetworkTarget(ctx, host) if err != nil { - return nil, fmt.Errorf("DNS lookup failed for %s: %w", host, err) - } - if len(ips) == 0 { - return nil, fmt.Errorf("DNS lookup for %s returned no addresses", host) - } - for _, ip := range ips { - if ip.IsUnspecified() { - return nil, fmt.Errorf("blocked connection to unspecified address %s (host %s)", ip, host) - } - if IsLinkLocal(ip) { - return nil, fmt.Errorf("blocked connection to link-local address %s (host %s)", ip, host) - } + return nil, err } // Every address resolved above passed the policy, so trying them in // turn is no weaker than dialing just the first: nothing outside this diff --git a/core/network/ssrf_test.go b/core/network/ssrf_test.go index 507b18d1550..2b61de8cef6 100644 --- a/core/network/ssrf_test.go +++ b/core/network/ssrf_test.go @@ -441,3 +441,94 @@ func TestPrivateNetworkDialContextFallsBackAcrossResolvedAddresses(t *testing.T) } conn.Close() } + +// TestResolvePrivateNetworkTargetPolicy locks in the destination policy that +// PrivateNetworkDialContext and the MCP proxy selector share: link-local and +// unspecified are refused, loopback is permitted, an IP literal resolves to +// itself, and an empty host (a request with no authority) is refused rather +// than passed through. +func TestResolvePrivateNetworkTargetPolicy(t *testing.T) { + ctx := context.Background() + if _, err := ResolvePrivateNetworkTarget(ctx, "169.254.169.254"); err == nil || !strings.Contains(err.Error(), "blocked connection to link-local address") { + t.Fatalf("expected blocked link-local error, got %v", err) + } + if _, err := ResolvePrivateNetworkTarget(ctx, "0.0.0.0"); err == nil || !strings.Contains(err.Error(), "blocked connection to unspecified address") { + t.Fatalf("expected blocked unspecified error, got %v", err) + } + ips, err := ResolvePrivateNetworkTarget(ctx, "127.0.0.1") + if err != nil || len(ips) != 1 || !ips[0].Equal(net.IPv4(127, 0, 0, 1)) { + t.Fatalf("expected loopback literal to resolve to itself, got %v, %v", ips, err) + } + if _, err := ResolvePrivateNetworkTarget(ctx, ""); err == nil { + t.Fatal("expected an empty host to be refused") + } + // Cloud metadata endpoints outside link-local: Alibaba sits in CGNAT and + // the AWS IPv6 IMDS in unique-local, both ranges this policy otherwise + // permits, so they need their own block. The transition-wrapped forms + // must be caught too, as they are for link-local in IsPublicIP. + for _, host := range []string{ + "100.100.100.200", + "fd00:ec2::254", + "::ffff:100.100.100.200", // IPv4-mapped + "2002:6464:64c8::", // 6to4 wrapping 100.100.100.200 + "64:ff9b::6464:64c8", // NAT64 wrapping 100.100.100.200 + } { + if _, err := ResolvePrivateNetworkTarget(ctx, host); err == nil || !strings.Contains(err.Error(), "blocked connection to cloud metadata endpoint") { + t.Fatalf("%s: expected blocked metadata endpoint error, got %v", host, err) + } + } + if _, err := ResolvePrivateNetworkTarget(ctx, "2002:a9fe:a9fe::"); err == nil || !strings.Contains(err.Error(), "blocked connection to link-local address") { + t.Fatalf("expected 6to4-wrapped link-local to be blocked, got %v", err) + } + // Neighbours of the blocked endpoints stay permitted: the block is on the + // endpoints themselves, not on the CGNAT or unique-local ranges. + for _, host := range []string{"100.100.100.201", "fd00:ec2::255", "10.0.0.5"} { + if _, err := ResolvePrivateNetworkTarget(ctx, host); err != nil { + t.Fatalf("%s: expected permitted private target, got %v", host, err) + } + } +} + +// TestCheckPrivateNetworkLiteral locks in the DNS-free variant used on the +// proxied path: literals are judged by the same policy, bracketed IPv6 is +// accepted, and a hostname is passed through untouched with no lookup. +func TestCheckPrivateNetworkLiteral(t *testing.T) { + for _, tc := range []struct { + host string + want string + }{ + {"169.254.169.254", "blocked connection to link-local address"}, + {"[fe80::1]", "blocked connection to link-local address"}, + {"0.0.0.0", "blocked connection to unspecified address"}, + {"100.100.100.200", "blocked connection to cloud metadata endpoint"}, + {"[fd00:ec2::254]", "blocked connection to cloud metadata endpoint"}, + {"fd00:ec2::254", "blocked connection to cloud metadata endpoint"}, + } { + if err := CheckPrivateNetworkLiteral(tc.host); err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("%s: expected %q, got %v", tc.host, tc.want, err) + } + } + for _, host := range []string{"127.0.0.1", "[::1]", "10.0.0.5", "100.100.100.201"} { + if err := CheckPrivateNetworkLiteral(host); err != nil { + t.Fatalf("%s: expected permitted literal, got %v", host, err) + } + } + // A hostname is not resolved: .invalid never resolves (RFC 6761), and a + // name that would resolve to a blocked address is the proxy's concern. + for _, host := range []string{"mcp.invalid", "metadata.google.internal", ""} { + if err := CheckPrivateNetworkLiteral(host); err != nil { + t.Fatalf("%q: expected hostname to pass through without lookup, got %v", host, err) + } + } +} + +// TestPrivateNetworkDialContextBlocksMetadataEndpoints proves the direct +// dialer refuses the non-link-local metadata endpoints before any dial. +func TestPrivateNetworkDialContextBlocksMetadataEndpoints(t *testing.T) { + dial := PrivateNetworkDialContext(time.Second) + for _, addr := range []string{"100.100.100.200:80", "[fd00:ec2::254]:80"} { + if _, err := dial(context.Background(), "tcp", addr); err == nil || !strings.Contains(err.Error(), "blocked connection to cloud metadata endpoint") { + t.Fatalf("%s: expected blocked metadata endpoint error, got %v", addr, err) + } + } +} diff --git a/core/promptcachedispatch_test.go b/core/promptcachedispatch_test.go index ef6695b196d..6a9fbf60490 100644 --- a/core/promptcachedispatch_test.go +++ b/core/promptcachedispatch_test.go @@ -2,7 +2,14 @@ package bifrost import ( "context" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" "testing" + "time" "github.com/capsohq/bifrost/core/providers/anthropic" "github.com/capsohq/bifrost/core/providers/openai" @@ -314,6 +321,54 @@ func TestPromptCacheChatRequest_PassesThroughWhenDisabled(t *testing.T) { assert.Nil(t, promptCacheChatRequest(nil, promptCacheOn(), schemas.Anthropic, nil)) } +// TestPromptCacheChatRequest_CachePointOnlyReachesBedrock covers a Bedrock -> OpenAI fallback on the same shared request. +func TestPromptCacheChatRequest_CachePointOnlyReachesBedrock(t *testing.T) { + req := chatReqWithText("stable prefix") + req.Input[0].Content.ContentBlocks = append(req.Input[0].Content.ContentBlocks, schemas.ChatContentBlock{CachePoint: &schemas.CachePoint{Type: "default"}}) + + assert.Same(t, req, promptCacheChatRequest(nil, nil, schemas.Bedrock, req)) + + out := promptCacheChatRequest(nil, nil, schemas.OpenAI, req) + require.NotSame(t, req, out) + assert.Len(t, out.Input[0].Content.ContentBlocks, 1) + assert.Len(t, req.Input[0].Content.ContentBlocks, 2, "the shared request lost its cachePoint; a Bedrock fallback would not see it") +} + +// TestPromptCacheChatRequest_CachePointGatedOnModel covers the second half of the gate: +// Converse itself rejects a cachePoint on a model that does not publish support for one, +// and the datasheet is what decides, with the model name only the fallback. +func TestPromptCacheChatRequest_CachePointGatedOnModel(t *testing.T) { + withCachePoint := func(model string) *schemas.BifrostChatRequest { + req := chatReqWithText("stable prefix") + req.Provider = schemas.Bedrock + req.Model = model + req.Input[0].Content.ContentBlocks = append(req.Input[0].Content.ContentBlocks, schemas.ChatContentBlock{CachePoint: &schemas.CachePoint{Type: "default"}}) + return req + } + + llama := withCachePoint("meta.llama3-70b-instruct-v1:0") + out := promptCacheChatRequest(nil, nil, schemas.Bedrock, llama) + require.NotSame(t, llama, out) + assert.Len(t, out.Input[0].Content.ContentBlocks, 1, "Converse rejects a cachePoint on a model without support for one") + assert.Len(t, llama.Input[0].Content.ContentBlocks, 2, "the shared request was mutated") + + schemas.SetCapabilityResolver(func(_ schemas.ModelProvider, model string) *schemas.ModelCapabilities { + if model == "meta.llama3-70b-instruct-v1:0" { + return &schemas.ModelCapabilities{SupportsCachePoint: schemas.Ptr(true)} + } + return &schemas.ModelCapabilities{SupportsCachePoint: schemas.Ptr(false)} + }) + t.Cleanup(func() { schemas.SetCapabilityResolver(nil) }) + + assert.Same(t, llama, promptCacheChatRequest(nil, nil, schemas.Bedrock, llama), + "a datasheet row saying yes must win over the name-based fallback") + + claude := withCachePoint("anthropic.claude-3-5-haiku-20241022-v1:0") + out = promptCacheChatRequest(nil, nil, schemas.Bedrock, claude) + require.NotSame(t, claude, out) + assert.Len(t, out.Input[0].Content.ContentBlocks, 1, "a datasheet row saying no must win too") +} + // TestPromptCacheDispatch_CallerMarkerSurvivesUnchanged proves the two guarantees // compose: a caller that set its own marker gets the request through untouched, and // nothing extra is added on top of it. @@ -554,3 +609,292 @@ func TestPrepareResponsesRequest_UnwrapsFunctionsNamespaceForEveryWire(t *testin assert.Equal(t, "namespace_a__js", *out.Params.Tools[1].Name) }) } + +// captureBody records each request body the server receives, then replies with status and body. +func captureBody(mu *sync.Mutex, bodies *[]string, status int, reply string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + b, _ := io.ReadAll(r.Body) + mu.Lock() + *bodies = append(*bodies, string(b)) + mu.Unlock() + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _, _ = io.WriteString(w, reply) + } +} + +const billingHeaderFixture = "x-anthropic-billing-header: cc_version=2.1.270.42c; cc_entrypoint=cli; cc_is_subagent=true;" + +func billingHeaderRequest(t testing.TB, header ...string) (*schemas.BifrostContext, *schemas.BifrostResponsesRequest) { + t.Helper() + ctx := schemas.NewBifrostContextWithValue(context.Background(), schemas.NoDeadline, schemas.BifrostContextKeyIntegrationType, "anthropic") + var incoming anthropic.AnthropicMessageRequest + require.NoError(t, schemas.Unmarshal([]byte(`{ + "model":"openai/gpt-4o-mini","max_tokens":64, + "system":[ + {"type":"text","text":"`+billingHeaderFixture+`"}, + {"type":"text","text":"Stable instructions","cache_control":{"type":"ephemeral"}} + ], + "messages":[{"role":"user","content":"Hello"}] + }`), &incoming)) + if len(header) > 0 { + incoming.System.ContentBlocks[0].Text = schemas.Ptr(header[0]) + } + req := incoming.ToBifrostResponsesRequest(ctx) + req.ExtractAnthropicBillingHeader() + return ctx, req +} + +func TestBillingHeaderNormalizedIngress(t *testing.T) { + _, req := billingHeaderRequest(t) + require.Len(t, req.Input[0].Content.ContentBlocks, 1) + assert.Equal(t, "Stable instructions", *req.Input[0].Content.ContentBlocks[0].Text) +} + +func TestBillingHeaderNonAnthropicNoCopy(t *testing.T) { + ctx, req := billingHeaderRequest(t) + out, bifrostErr := prepareResponsesRequest(ctx, nil, stubProvider{key: schemas.OpenAI}, schemas.Key{}, req) + require.Nil(t, bifrostErr) + assert.Same(t, req, out, "non-Anthropic attempts must reuse the normalized input") +} + +func TestBillingHeaderRawPassthrough(t *testing.T) { + ctx, req := billingHeaderRequest(t) + req.Model = "claude-sonnet-4" + req.RawRequestBody = []byte(`{"system":"` + billingHeaderFixture + `"}`) + ctx.SetValue(schemas.BifrostContextKeyUseRawRequestBody, true) + out := restoreResponsesBillingHeader(ctx, schemas.Anthropic, req) + assert.Same(t, req, out, "raw passthrough does not need a normalized copy") + assert.Contains(t, string(out.RawRequestBody), billingHeaderFixture) + ctx.SetValue(schemas.BifrostContextKeyUseRawRequestBody, false) + assert.Len(t, restoreResponsesBillingHeader(ctx, schemas.Anthropic, req).Input[0].Content.ContentBlocks, 2) +} + +func BenchmarkBillingHeaderNonAnthropic100MB(b *testing.B) { + ctx, req := billingHeaderRequest(b) + // Allocate 100 distinct 1 MiB payloads before timing. Filtering should neither + // inspect them nor allocate a new message slice on subsequent GPT attempts. + for i := 0; i < 100; i++ { + req.Input = append(req.Input, schemas.ResponsesMessage{ + Role: schemas.Ptr(schemas.ResponsesInputMessageRoleUser), + Content: &schemas.ResponsesMessageContent{ContentStr: schemas.Ptr(strings.Repeat("x", 1<<20))}, + }) + } + b.ReportAllocs() + var out *schemas.BifrostResponsesRequest + for b.Loop() { + out = restoreResponsesBillingHeader(ctx, schemas.OpenAI, req) + } + if out != req { + b.Fatal("non-Anthropic attempt copied the request") + } +} + +func TestPrepareResponsesBillingHeaderModelFamily(t *testing.T) { + for _, tc := range []struct { + name string + provider schemas.ModelProvider + model string + base schemas.ModelProvider + alias *schemas.AliasConfig + keep bool + }{ + {name: "GPT", provider: schemas.OpenAI, model: "gpt-4o-mini"}, + {name: "Gemini", provider: schemas.Gemini, model: "gemini-2.5-flash"}, + {name: "self hosted opaque", provider: schemas.Ollama, model: "local-model"}, + {name: "custom OpenAI", provider: customOpenAIProviderKey, base: schemas.OpenAI, model: "deployment"}, + {name: "Anthropic opaque", provider: schemas.Anthropic, model: "deployment", keep: true}, + {name: "custom Anthropic opaque", provider: customAnthropicProviderKey, base: schemas.Anthropic, model: "deployment", keep: true}, + {name: "Anthropic", provider: schemas.Anthropic, model: "claude-sonnet-4", keep: true}, + {name: "Bedrock Claude", provider: schemas.Bedrock, model: "anthropic.claude-sonnet-4", keep: true}, + {name: "Vertex Claude", provider: schemas.Vertex, model: "claude-sonnet-4", keep: true}, + {name: "Azure Claude", provider: schemas.Azure, model: "claude-sonnet-4", keep: true}, + {name: "OpenRouter Claude", provider: schemas.OpenRouter, model: "anthropic/claude-sonnet-4", keep: true}, + {name: "Claude named GPT alias", provider: schemas.OpenAI, model: "claude-alias", alias: &schemas.AliasConfig{ModelID: "opaque", ModelFamily: schemas.Ptr(schemas.ModelFamilyOpenAI)}}, + {name: "opaque Claude alias", provider: schemas.Azure, model: "deployment", alias: &schemas.AliasConfig{ModelID: "opaque", ModelFamily: schemas.Ptr(schemas.ModelFamilyAnthropic)}, keep: true}, + {name: "canonical Claude alias", provider: schemas.OpenRouter, model: "deployment", alias: &schemas.AliasConfig{ModelID: "opaque", ModelName: schemas.Ptr("claude-sonnet-4")}, keep: true}, + } { + t.Run(tc.name, func(t *testing.T) { + ctx, req := billingHeaderRequest(t) + req.Provider, req.Model = tc.provider, tc.model + if tc.base != "" { + ctx.SetValue(schemas.BifrostContextKeyBaseProviderType, tc.base) + } + if tc.alias != nil { + ctx.SetValue(schemas.BifrostContextKeyResolvedAlias, &schemas.ResolvedAlias{Key: tc.model, Config: tc.alias}) + } + before, err := providerUtils.MarshalSorted(req) + require.NoError(t, err) + out, bifrostErr := prepareResponsesRequest(ctx, nil, stubProvider{key: tc.provider}, schemas.Key{}, req) + require.Nil(t, bifrostErr) + if tc.keep { + require.Len(t, out.Input[0].Content.ContentBlocks, 2) + assert.Equal(t, billingHeaderFixture, *out.Input[0].Content.ContentBlocks[0].Text) + } else { + assert.Same(t, req, out) + require.Len(t, out.Input[0].Content.ContentBlocks, 1) + assert.Equal(t, "Stable instructions", *out.Input[0].Content.ContentBlocks[0].Text) + assert.Equal(t, req.Input[0].Content.ContentBlocks[0].CacheControl, out.Input[0].Content.ContentBlocks[0].CacheControl) + } + after, err := providerUtils.MarshalSorted(req) + require.NoError(t, err) + assert.Equal(t, string(before), string(after), "preparation mutated the original request") + }) + } +} + +func TestPrepareResponsesBillingHeaderStableWireAndFallback(t *testing.T) { + var previous []byte + for _, header := range []string{billingHeaderFixture, "x-anthropic-billing-header: cc_version=2.1.270.abc; cch=fffff;"} { + ctx, req := billingHeaderRequest(t, header) + // Both directions: Claude -> GPT -> Claude, sharing the original input. + for _, family := range []schemas.ModelFamily{schemas.ModelFamilyAnthropic, schemas.ModelFamilyOpenAI, schemas.ModelFamilyAnthropic} { + ctx.SetValue(schemas.BifrostContextKeyResolvedAlias, &schemas.ResolvedAlias{Config: &schemas.AliasConfig{ModelID: "deployment", ModelFamily: &family}}) + out, bifrostErr := prepareResponsesRequest(ctx, nil, stubProvider{key: schemas.OpenAI}, schemas.Key{}, req) + require.Nil(t, bifrostErr) + if family == schemas.ModelFamilyAnthropic { + require.Len(t, out.Input[0].Content.ContentBlocks, 2) + assert.Equal(t, header, *out.Input[0].Content.ContentBlocks[0].Text) + continue + } + wire, err := providerUtils.MarshalSorted(openai.ToOpenAIResponsesRequest(ctx, out)) + require.NoError(t, err) + assert.NotContains(t, string(wire), "x-anthropic-billing-header:") + if previous != nil { + assert.Equal(t, string(previous), string(wire)) + } + previous = wire + } + } +} + +// Capture at the provider boundary so all four dispatch paths must prepare the +// request, including Responses -> Chat conversion before streaming dispatch. +type billingHeaderCaptureProvider struct { + stubProvider + responses *schemas.BifrostResponsesRequest + chat *schemas.BifrostChatRequest +} + +func (p *billingHeaderCaptureProvider) Responses(_ *schemas.BifrostContext, _ schemas.Key, r *schemas.BifrostResponsesRequest) (*schemas.BifrostResponsesResponse, *schemas.BifrostError) { + p.responses = r + return &schemas.BifrostResponsesResponse{}, nil +} + +func (p *billingHeaderCaptureProvider) ChatCompletion(_ *schemas.BifrostContext, _ schemas.Key, r *schemas.BifrostChatRequest) (*schemas.BifrostChatResponse, *schemas.BifrostError) { + p.chat = r + return &schemas.BifrostChatResponse{}, nil +} + +func (p *billingHeaderCaptureProvider) ResponsesStream(_ *schemas.BifrostContext, _ schemas.PostHookRunner, _ func(context.Context), _ schemas.Key, r *schemas.BifrostResponsesRequest) (chan *schemas.BifrostStreamChunk, *schemas.BifrostError) { + p.responses = r + ch := make(chan *schemas.BifrostStreamChunk) + close(ch) + return ch, nil +} + +func (p *billingHeaderCaptureProvider) ChatCompletionStream(_ *schemas.BifrostContext, _ schemas.PostHookRunner, _ func(context.Context), _ schemas.Key, r *schemas.BifrostChatRequest) (chan *schemas.BifrostStreamChunk, *schemas.BifrostError) { + p.chat = r + ch := make(chan *schemas.BifrostStreamChunk) + close(ch) + return ch, nil +} + +func TestBillingHeaderDispatch(t *testing.T) { + for _, stream := range []bool{false, true} { + for _, chat := range []bool{false, true} { + t.Run(fmt.Sprintf("stream=%t/chat=%t", stream, chat), func(t *testing.T) { + ctx, req := billingHeaderRequest(t) + if chat { + ctx.SetValue(schemas.BifrostContextKeyChangeRequestType, schemas.ChatCompletionRequest) + } + message := &ChannelMessage{Context: ctx, BifrostRequest: schemas.BifrostRequest{RequestType: schemas.ResponsesRequest, ResponsesRequest: req}} + provider := &billingHeaderCaptureProvider{stubProvider: stubProvider{key: schemas.OpenAI}} + client := &Bifrost{} + if stream { + message.RequestType = schemas.ResponsesStreamRequest + _, bifrostErr := client.handleProviderStreamRequest(provider, nil, message, schemas.Key{}, nil, nil) + require.Nil(t, bifrostErr) + } else { + _, bifrostErr := client.handleProviderRequest(provider, nil, message, schemas.Key{}, nil) + require.Nil(t, bifrostErr) + } + var captured any + if chat { + require.NotNil(t, provider.chat) + captured = provider.chat + } else { + require.NotNil(t, provider.responses) + captured = provider.responses + } + body, err := providerUtils.MarshalSorted(captured) + require.NoError(t, err) + assert.NotContains(t, string(body), "x-anthropic-billing-header:") + assert.Contains(t, string(body), "Stable instructions") + require.Len(t, req.Input[0].Content.ContentBlocks, 1) + assert.Equal(t, "Stable instructions", *req.Input[0].Content.ContentBlocks[0].Text) + }) + } + } +} + +// TestChatCachePoint_StrippedForPrimaryKeptForBedrockFallback runs an OpenAI -> Bedrock fallback end to end. +func TestChatCachePoint_StrippedForPrimaryKeptForBedrockFallback(t *testing.T) { + var mu sync.Mutex + var openaiBodies, bedrockBodies []string + + primary := httptest.NewServer(captureBody(&mu, &openaiBodies, http.StatusInternalServerError, + `{"error":{"message":"boom","type":"server_error"}}`)) + defer primary.Close() + fallback := httptest.NewTLSServer(captureBody(&mu, &bedrockBodies, http.StatusOK, + `{"output":{"message":{"role":"assistant","content":[{"text":"hello"}]}},"stopReason":"end_turn","usage":{"inputTokens":1,"outputTokens":1,"totalTokens":2}}`)) + defer fallback.Close() + + account := NewMockAccount() + account.AddProviderWithBaseURL(schemas.OpenAI, 1, 1, primary.URL) + account.configs[schemas.OpenAI].NetworkConfig.MaxRetries = 0 + account.SetKeysForProvider(schemas.OpenAI, []schemas.Key{ + {ID: "openai-key", Value: *schemas.NewSecretVar("sk-test"), Models: schemas.WhiteList{"*"}, Weight: 100}, + }) + account.AddProviderWithBaseURL(schemas.Bedrock, 1, 1, "") + account.configs[schemas.Bedrock].NetworkConfig.MaxRetries = 0 + account.configs[schemas.Bedrock].NetworkConfig.InsecureSkipVerify = true + account.SetKeysForProvider(schemas.Bedrock, []schemas.Key{{ + ID: "bedrock-key", Models: schemas.WhiteList{"*"}, Weight: 100, + BedrockKeyConfig: &schemas.BedrockKeyConfig{ + AccessKey: *schemas.NewSecretVar("AKIATEST"), + SecretKey: *schemas.NewSecretVar("secret"), + Region: schemas.NewSecretVar("us-east-1"), + Endpoints: &schemas.BedrockEndpoints{Runtime: schemas.NewSecretVar(strings.TrimPrefix(fallback.URL, "https://"))}, + }, + }}) + client := newStreamTestClient(t, account) + + req := &schemas.BifrostChatRequest{ + Provider: schemas.OpenAI, + Model: "gpt-4o-mini", + Input: []schemas.ChatMessage{{ + Role: schemas.ChatMessageRoleUser, + Content: &schemas.ChatMessageContent{ContentBlocks: []schemas.ChatContentBlock{ + {Type: schemas.ChatContentBlockTypeText, Text: schemas.Ptr("stable prefix")}, + {CachePoint: &schemas.CachePoint{Type: "default"}}, + }}, + }}, + Fallbacks: []schemas.Fallback{{Provider: schemas.Bedrock, Model: "anthropic.claude-3-5-haiku-20241022-v1:0"}}, + } + + ctx := schemas.NewBifrostContext(context.Background(), time.Now().Add(10*time.Second)) + resp, bifrostErr := client.ChatCompletionRequest(ctx, req) + if bifrostErr != nil && bifrostErr.Error != nil { + t.Fatalf("fallback failed: %s", bifrostErr.Error.Message) + } + require.NotNil(t, resp) + + mu.Lock() + defer mu.Unlock() + require.Len(t, openaiBodies, 1, "primary was not attempted") + require.Len(t, bedrockBodies, 1, "fallback was not attempted") + assert.NotContains(t, openaiBodies[0], "cachePoint", "primary wire body leaked the Bedrock marker") + assert.Contains(t, bedrockBodies[0], "cachePoint", "fallback lost the caller's cachePoint") + assert.Len(t, req.Input[0].Content.ContentBlocks, 2, "caller's request was mutated") +} diff --git a/core/providers/anthropic/adaptivethinkingstrip_test.go b/core/providers/anthropic/adaptivethinkingstrip_test.go index bbd6bc01879..eae7a3fcf59 100644 --- a/core/providers/anthropic/adaptivethinkingstrip_test.go +++ b/core/providers/anthropic/adaptivethinkingstrip_test.go @@ -220,7 +220,7 @@ func TestAdaptiveOnlyThinkingStrip(t *testing.T) { // \"thinking.type.disabled\" is not supported".) func TestDisabledThinkingStrip(t *testing.T) { // Models that reject "disabled" regardless of effort. - alwaysOn := []string{"claude-fable-5", "claude-mythos-5", "claude-mythos-preview"} + alwaysOn := []string{"claude-fable-5", "claude-mythos-5", "claude-mythos-preview", "claude-opus-5-5"} // Models that accept "disabled" at any effort. disabledOK := []string{"claude-opus-4-7", "claude-opus-4-8", "claude-sonnet-5", "claude-opus-4-6", "claude-sonnet-4-5"} @@ -390,6 +390,36 @@ func TestDisabledThinkingStrip(t *testing.T) { } } }) + + // The unconditional half of the gate reads supports_reasoning_disable, so a + // row moves this passthrough strip without a release — in both directions. + t.Run("datasheet_outranks_the_name_fallback", func(t *testing.T) { + yes := true + setOverride(t, "claude-opus-5-5", schemas.ModelCapabilities{SupportsReasoningDisable: &yes}) + body := []byte(`{"model":"claude-opus-5-5","max_tokens":4096,"thinking":{"type":"disabled"}}`) + + result, err := StripUnsupportedFieldsFromRawBody(body, schemas.Anthropic, "claude-opus-5-5") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got := providerUtils.GetJSONField(result, "thinking.type").String(); got != "disabled" { + t.Errorf("thinking.type = %q, want \"disabled\" kept by the row; body: %s", got, string(result)) + } + }) + + t.Run("datasheet_disables_on_a_model_the_fallback_allows", func(t *testing.T) { + no := false + setOverride(t, "claude-opus-4-8", schemas.ModelCapabilities{SupportsReasoningDisable: &no}) + body := []byte(`{"model":"claude-opus-4-8","max_tokens":4096,"thinking":{"type":"disabled"}}`) + + result, err := StripUnsupportedFieldsFromRawBody(body, schemas.Anthropic, "claude-opus-4-8") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got := providerUtils.GetJSONField(result, "thinking.type").String(); got == "disabled" { + t.Errorf("thinking.type = \"disabled\" survived a row saying otherwise; body: %s", string(result)) + } + }) } // Claude Mythos Preview is the one Fable/Mythos model that still supports diff --git a/core/providers/anthropic/anthropic.go b/core/providers/anthropic/anthropic.go index 8740ac77c1d..80f469e7677 100644 --- a/core/providers/anthropic/anthropic.go +++ b/core/providers/anthropic/anthropic.go @@ -1790,6 +1790,14 @@ func HandleAnthropicResponsesStream( response.ExtraFields.RawResponse = eventData } + // Carry safeguard_results (Claude Code auto-mode classifier) so the + // Anthropic egress can restore it on re-rendered frames. Attached to + // the same single chunk as the raw frame, and not gated on raw capture + // so the fully typed path benefits too. + if i == rawIdx && len(event.SafeguardResults) > 0 { + response.SafeguardResults = event.SafeguardResults + } + if isLastChunk && i == len(responses)-1 { if response.Response == nil { response.Response = &schemas.BifrostResponsesResponse{} @@ -2424,6 +2432,11 @@ func (provider *AnthropicProvider) Rerank(ctx *schemas.BifrostContext, key schem return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey()) } +// Decision is not supported by the Anthropic provider. +func (provider *AnthropicProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR is not supported by the Anthropic provider. func (provider *AnthropicProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) diff --git a/core/providers/anthropic/capability_overrides_test.go b/core/providers/anthropic/capability_overrides_test.go index d5c0e514c65..c6e50887e07 100644 --- a/core/providers/anthropic/capability_overrides_test.go +++ b/core/providers/anthropic/capability_overrides_test.go @@ -186,6 +186,63 @@ func TestSupportsFastMode_OverrideAbsent_FallbackTakesOver(t *testing.T) { assert.False(t, schemas.ResolveModelCaps(schemas.Anthropic, "claude-sonnet-4-5-20250929").SupportsFastMode(DefaultSupportsFastMode("claude-sonnet-4-5-20250929"))) } +func TestSupportsSafeguards_OverrideHit(t *testing.T) { + model := "non-claude-test-model-safeguards-yes" + yes := true + setOverride(t, model, schemas.ModelCapabilities{SupportsSafeguards: &yes}) + assert.True(t, schemas.ResolveModelCaps(schemas.Anthropic, model).SupportsSafeguards(DefaultSupportsSafeguards(schemas.Anthropic, model))) +} + +func TestSupportsSafeguards_OverrideExplicitFalse(t *testing.T) { + // Even on a name the substring fallback marks supported (Sonnet 5), an + // explicit false override wins. + model := "claude-sonnet-5-test-overridden" + no := false + setOverride(t, model, schemas.ModelCapabilities{SupportsSafeguards: &no}) + assert.False(t, schemas.ResolveModelCaps(schemas.Anthropic, model).SupportsSafeguards(DefaultSupportsSafeguards(schemas.Anthropic, model))) +} + +// Catalog overrides can enable models outside the default model family gate. +func TestSupportsSafeguards_CloudOverrideEnables(t *testing.T) { + model := "claude-haiku-4-5-cloud-enabled" + yes := true + // setOverride only answers for Anthropic, so register a resolver that covers + // the cloud pairs this case is about. + providerUtils.SetCapabilityResolver(func(_ schemas.ModelProvider, m string) *schemas.ModelCapabilities { + if m == model { + return &schemas.ModelCapabilities{SupportsSafeguards: &yes} + } + return nil + }) + t.Cleanup(func() { providerUtils.SetCapabilityResolver(nil) }) + + for _, provider := range []schemas.ModelProvider{schemas.Bedrock, schemas.Vertex, schemas.Azure} { + assert.True(t, schemas.ResolveModelCaps(provider, model).SupportsSafeguards(DefaultSupportsSafeguards(provider, model)), + "datasheet override must enable %s", provider) + } +} + +func TestSupportsSafeguards_OverrideAbsent_FallbackTakesOver(t *testing.T) { + // Anthropic direct: supported on the auto-mode models (Sonnet 5, Opus 4.7+ + // covering 4.8 and 5, Fable); Haiku, Sonnet 4.6 and Opus 4.5 are out. + assert.True(t, schemas.ResolveModelCaps(schemas.Anthropic, "claude-sonnet-5").SupportsSafeguards(DefaultSupportsSafeguards(schemas.Anthropic, "claude-sonnet-5"))) + assert.True(t, schemas.ResolveModelCaps(schemas.Anthropic, "claude-opus-4-7-20260401").SupportsSafeguards(DefaultSupportsSafeguards(schemas.Anthropic, "claude-opus-4-7-20260401"))) + assert.True(t, schemas.ResolveModelCaps(schemas.Anthropic, "claude-opus-4-8").SupportsSafeguards(DefaultSupportsSafeguards(schemas.Anthropic, "claude-opus-4-8"))) + assert.True(t, schemas.ResolveModelCaps(schemas.Anthropic, "claude-opus-5").SupportsSafeguards(DefaultSupportsSafeguards(schemas.Anthropic, "claude-opus-5"))) + assert.True(t, schemas.ResolveModelCaps(schemas.Anthropic, "claude-fable-5-1").SupportsSafeguards(DefaultSupportsSafeguards(schemas.Anthropic, "claude-fable-5-1"))) + assert.False(t, schemas.ResolveModelCaps(schemas.Anthropic, "claude-haiku-4-5").SupportsSafeguards(DefaultSupportsSafeguards(schemas.Anthropic, "claude-haiku-4-5"))) + assert.False(t, schemas.ResolveModelCaps(schemas.Anthropic, "claude-sonnet-4-6").SupportsSafeguards(DefaultSupportsSafeguards(schemas.Anthropic, "claude-sonnet-4-6"))) + assert.False(t, schemas.ResolveModelCaps(schemas.Anthropic, "claude-opus-4-5-20251101").SupportsSafeguards(DefaultSupportsSafeguards(schemas.Anthropic, "claude-opus-4-5-20251101"))) + + // Claude cloud surfaces share the supported model families. + for _, provider := range []schemas.ModelProvider{schemas.Bedrock, schemas.BedrockMantle, schemas.Vertex, schemas.Azure} { + for _, model := range []string{"claude-sonnet-5", "claude-opus-4-8", "claude-fable-5-1", "claude-haiku-4-5"} { + assert.Equal(t, model != "claude-haiku-4-5", schemas.ResolveModelCaps(provider, model).SupportsSafeguards(DefaultSupportsSafeguards(provider, model)), + "%s/%s must respect the model gate", provider, model) + } + } +} + func TestSupportsAdaptiveThinking_OverrideHit(t *testing.T) { model := "non-opus-test-model-adaptive-yes" yes := true @@ -353,10 +410,10 @@ func forcedToolChoiceChatRequest(model string, tc *schemas.ChatToolChoice) *sche } } -func TestForcedToolChoice_DroppedOnFable51(t *testing.T) { +func TestForcedToolChoice_DroppedWhenUnsupported(t *testing.T) { ctx := schemas.NewBifrostContext(nil, schemas.NoDeadline) - for _, model := range []string{"claude-fable-5-1", "claude-mythos-5-1"} { + for _, model := range []string{"claude-fable-5-1", "claude-mythos-5-1", "claude-opus-5-5"} { t.Run(model+" responses any", func(t *testing.T) { req, err := ToAnthropicResponsesRequest(ctx, forcedToolChoiceResponsesRequest(model, &schemas.ResponsesToolChoice{ResponsesToolChoiceStr: schemas.Ptr("any")})) diff --git a/core/providers/anthropic/chat.go b/core/providers/anthropic/chat.go index 2e82e62f49f..0ba6f5b6fd0 100644 --- a/core/providers/anthropic/chat.go +++ b/core/providers/anthropic/chat.go @@ -154,6 +154,12 @@ func convertServerToolToAnthropic(tool schemas.ChatTool, caps schemas.ModelCaps, if toolName == "" || toolName != wantName { toolName = wantName } + // A toolset is named by its type alone, so it is the one entry that + // legitimately carries no name and must skip the guard below. + if wantName == "" { + atype := AnthropicToolType(typeStr) + return AnthropicTool{Type: &atype, CacheControl: tool.CacheControl}, true + } } } if toolName == "" { @@ -366,6 +372,25 @@ func ToAnthropicChatRequest(ctx *schemas.BifrostContext, bifrostReq *schemas.Bif // Convert parameters if bifrostReq.Params != nil { anthropicReq.ExtraParams = bifrostReq.Params.ExtraParams + if safeguards, exists := anthropicReq.ExtraParams["safeguards"]; exists { + // Copy before consuming the key: the input may be reused for a fallback. + extra := make(map[string]interface{}, len(anthropicReq.ExtraParams)) + for k, v := range anthropicReq.ExtraParams { + extra[k] = v + } + anthropicReq.ExtraParams = extra + delete(anthropicReq.ExtraParams, "safeguards") + switch v := safeguards.(type) { + case json.RawMessage: + anthropicReq.Safeguards = v + case []byte: + anthropicReq.Safeguards = json.RawMessage(v) + default: + if data, err := providerUtils.MarshalSorted(v); err == nil { + anthropicReq.Safeguards = data + } + } + } // reasoningParams is the effective reasoning config for this request. It is // normally just Params.Reasoning; when the caller used Anthropic's native @@ -679,13 +704,53 @@ func ToAnthropicChatRequest(ctx *schemas.BifrostContext, bifrostReq *schemas.Bif toolChoice.Name = bifrostReq.Params.ToolChoice.ChatToolChoiceStruct.Function.Name } case schemas.ChatToolChoiceTypeAllowedTools: - toolChoice.Type = "any" + // Anthropic has no "choose from this subset" form, so the + // restriction is expressed by removing the declarations the + // caller did not allow: the model can only call a tool it was + // given. Mapping straight to "any" kept every tool and also + // forced a call even when the caller's mode was "auto". + allowed := bifrostReq.Params.ToolChoice.ChatToolChoiceStruct.AllowedTools + if allowed != nil { + anthropicReq.Tools = filterToolsByAllowed(anthropicReq.Tools, allowed.Tools) + switch { + case countFunctionTools(anthropicReq.Tools) == 0 && len(anthropicReq.Tools) == 0: + // The allowed list matched nothing declared. "any" or + // "auto" with no tools is a request Anthropic rejects, + // so the restriction is stated as what it means. + toolChoice.Type = "none" + case countFunctionTools(anthropicReq.Tools) == 0: + // No function is allowed, but server tools remain. The + // list restricts functions only, and "none" would switch + // those off too, so the model may still use them. + toolChoice.Type = "auto" + case allowed.Mode == "required": + toolChoice.Type = "any" + default: + toolChoice.Type = "auto" + } + } else { + toolChoice.Type = "any" + } case schemas.ChatToolChoiceTypeCustom: toolChoice.Type = "auto" default: toolChoice.Type = "auto" } } + applyParallelToolUse(toolChoice, bifrostReq.Params.ParallelToolCalls) + anthropicReq.ToolChoice = toolChoice + } + + // A caller can disable parallel tool calls without expressing any + // preference about *which* tool runs. Anthropic carries that setting on + // tool_choice only, so without a tool_choice to attach it to the + // instruction had nowhere to go and was dropped. "auto" is Anthropic's + // default behaviour, so naming it here restores the flag without + // changing which tools may be called. + if anthropicReq.ToolChoice == nil && len(anthropicReq.Tools) > 0 && + bifrostReq.Params.ParallelToolCalls != nil && !*bifrostReq.Params.ParallelToolCalls { + toolChoice := &AnthropicToolChoice{Type: "auto"} + applyParallelToolUse(toolChoice, bifrostReq.Params.ParallelToolCalls) anthropicReq.ToolChoice = toolChoice } @@ -2042,3 +2107,82 @@ func ToAnthropicChatStreamError(bifrostErr *schemas.BifrostError) string { // Format as Anthropic SSE error event return fmt.Sprintf("event: error\ndata: %s\n\n", jsonData) } + +// filterToolsByAllowed narrows a converted Anthropic tool list to the names a +// caller permitted through tool_choice.allowed_tools. +// +// Anthropic's tool_choice has no subset form (auto, any, tool, none), so the +// only place the restriction can be expressed is the declaration list itself: +// a tool that is not declared cannot be called. An explicit empty allowed list +// (or one naming no function) permits no function tool, so every function +// declaration is removed; returning the full list would let the model call +// tools the caller allowed none of. +func filterToolsByAllowed(tools []AnthropicTool, allowed []schemas.ChatToolChoiceAllowedToolsTool) []AnthropicTool { + if len(tools) == 0 { + return tools + } + permitted := make(map[string]struct{}, len(allowed)) + for _, a := range allowed { + if a.Function.Name != "" { + permitted[a.Function.Name] = struct{}{} + } + } + kept := make([]AnthropicTool, 0, len(tools)) + for _, t := range tools { + // Server tools are never named in an OpenAI allowed_tools list, which + // describes function tools only. Filtering them on a name they cannot + // have would silently switch off a capability the caller enabled + // elsewhere in the same request - a stated intent quietly discarded, + // which is the harm this function exists to prevent. + // + // Two shapes carry a server tool, and testing only one of them is the + // mistake to avoid here: web search, web fetch and the computer-use + // family set Type, while an MCP toolset leaves Type nil and carries + // everything in MCPToolset with an empty Name (see AnthropicTool, whose + // own comment records this). Checking Type alone therefore matched an + // MCP toolset's empty name against the allowlist and removed it. + if t.Type != nil || t.MCPToolset != nil { + kept = append(kept, t) + continue + } + if _, ok := permitted[t.Name]; ok { + kept = append(kept, t) + } + } + // An empty result means the allowed list named only tools this request does + // not declare. Returning the originals would forward every tool the caller + // just excluded, which is the bug this function exists to prevent, so the + // intersection is honoured literally: nothing may be called. The caller sees + // a request with no tools rather than one that quietly ignored the list. + return kept +} + +// applyParallelToolUse carries an OpenAI-style parallel_tool_calls flag onto an +// Anthropic tool_choice. Anthropic expresses the same instruction as +// disable_parallel_tool_use, so the sense is inverted, and it only accepts the +// field when a tool may actually be called. +func applyParallelToolUse(toolChoice *AnthropicToolChoice, parallel *bool) { + if toolChoice == nil || parallel == nil || *parallel { + return + } + if toolChoice.Type == "none" { + return + } + disable := true + toolChoice.DisableParallelToolUse = &disable +} + +// countFunctionTools reports how many declarations a model could be told to +// call by name. Server tools are excluded in both their shapes - Type set, or +// MCPToolset set with Type nil - because they survive an allowed_tools filter +// untouched. Counting either would hide the case where every nameable tool was +// excluded, and leave the tool choice at "any" over nothing callable. +func countFunctionTools(tools []AnthropicTool) int { + n := 0 + for _, t := range tools { + if t.Type == nil && t.MCPToolset == nil { + n++ + } + } + return n +} diff --git a/core/providers/anthropic/chat_test.go b/core/providers/anthropic/chat_test.go index aa61cab9f8d..b3c3b302b3b 100644 --- a/core/providers/anthropic/chat_test.go +++ b/core/providers/anthropic/chat_test.go @@ -9,6 +9,8 @@ import ( "github.com/bytedance/sonic" "github.com/capsohq/bifrost/core/providers/openai" "github.com/capsohq/bifrost/core/schemas" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestToAnthropicChatRequest_PreservesPropertyOrder(t *testing.T) { @@ -2433,3 +2435,215 @@ func TestToAnthropicResponsesRequest_MidConversationSystem_InlinesForNonAnthropi } assertInlinedReminder(t, result, "Mid-conv instruction.") } + +// restrictionRequest builds a two-tool chat request so that a restriction has +// something to remove. The model is a current one so that no capability +// override drops the tool choice before these paths run. +func restrictionRequest(tc *schemas.ChatToolChoice, parallel *bool) *schemas.BifrostChatRequest { + return &schemas.BifrostChatRequest{ + Provider: schemas.Anthropic, + Model: "claude-sonnet-5", + Input: []schemas.ChatMessage{{ + Role: schemas.ChatMessageRoleUser, + Content: &schemas.ChatMessageContent{ContentStr: schemas.Ptr("What is the weather in Tokyo?")}, + }}, + Params: &schemas.ChatParameters{ + ToolChoice: tc, + ParallelToolCalls: parallel, + Tools: []schemas.ChatTool{ + {Type: schemas.ChatToolTypeFunction, Function: &schemas.ChatToolFunction{Name: "tool_a"}}, + {Type: schemas.ChatToolTypeFunction, Function: &schemas.ChatToolFunction{Name: "tool_b"}}, + }, + }, + } +} + +func allowedTools(mode string, names ...string) *schemas.ChatToolChoice { + tools := make([]schemas.ChatToolChoiceAllowedToolsTool, 0, len(names)) + for _, n := range names { + tools = append(tools, schemas.ChatToolChoiceAllowedToolsTool{ + Type: "function", + Function: schemas.ChatToolChoiceFunction{Name: n}, + }) + } + return &schemas.ChatToolChoice{ChatToolChoiceStruct: &schemas.ChatToolChoiceStruct{ + Type: schemas.ChatToolChoiceTypeAllowedTools, + AllowedTools: &schemas.ChatToolChoiceAllowedTools{Mode: mode, Tools: tools}, + }} +} + +func toolNames(req *AnthropicMessageRequest) []string { + names := make([]string, 0, len(req.Tools)) + for _, t := range req.Tools { + names = append(names, t.Name) + } + return names +} + +// A caller that allows only tool_a must not have tool_b forwarded. Anthropic +// has no subset form on tool_choice, so the restriction has to be carried by +// the declaration list; mapping to "any" and forwarding both tools told the +// upstream it could call either one. +func TestToAnthropicChatRequest_AllowedToolsRestrictsDeclarations(t *testing.T) { + ctx := schemas.NewBifrostContext(nil, schemas.NoDeadline) + + req, err := ToAnthropicChatRequest(ctx, restrictionRequest(allowedTools("required", "tool_a"), nil)) + require.NoError(t, err) + + assert.Equal(t, []string{"tool_a"}, toolNames(req), "tool_b was not allowed and must not be forwarded") + require.NotNil(t, req.ToolChoice) + assert.Equal(t, "any", req.ToolChoice.Type, `mode "required" means a tool must be called`) +} + +// mode "auto" means the model may call one of the allowed tools or none at all. +// Forcing "any" changed an optional call into a mandatory one. +func TestToAnthropicChatRequest_AllowedToolsAutoModeDoesNotForceACall(t *testing.T) { + ctx := schemas.NewBifrostContext(nil, schemas.NoDeadline) + + req, err := ToAnthropicChatRequest(ctx, restrictionRequest(allowedTools("auto", "tool_a"), nil)) + require.NoError(t, err) + + require.NotNil(t, req.ToolChoice) + assert.Equal(t, "auto", req.ToolChoice.Type, `mode "auto" must not force a tool call`) + assert.Equal(t, []string{"tool_a"}, toolNames(req)) +} + +// An allowed list naming only tools this request does not declare permits +// nothing. Forwarding the declarations anyway would hand the model every tool +// the caller just excluded - the same failure this conversion exists to stop, +// arrived at from the other direction. +func TestToAnthropicChatRequest_AllowedToolsUnknownNamesPermitNothing(t *testing.T) { + ctx := schemas.NewBifrostContext(nil, schemas.NoDeadline) + + req, err := ToAnthropicChatRequest(ctx, restrictionRequest(allowedTools("required", "not_declared"), nil)) + require.NoError(t, err) + + assert.Empty(t, toolNames(req), "no declared tool was allowed, so none may be forwarded") + require.NotNil(t, req.ToolChoice) + assert.Equal(t, "none", req.ToolChoice.Type, + `"any" with an empty tool list is a request Anthropic rejects`) +} + +// An explicit empty allowed list permits no function tool. Leaving the declarations in +// place would let the model call tools the caller allowed none of. +func TestToAnthropicChatRequest_AllowedToolsEmptyListPermitsNothing(t *testing.T) { + ctx := schemas.NewBifrostContext(nil, schemas.NoDeadline) + + for _, mode := range []string{"auto", "required"} { + req, err := ToAnthropicChatRequest(ctx, restrictionRequest(allowedTools(mode), nil)) + require.NoError(t, err) + assert.Empty(t, toolNames(req), "mode %q: an empty allowed list permits no function tool", mode) + require.NotNil(t, req.ToolChoice) + assert.Equal(t, "none", req.ToolChoice.Type, "mode %q", mode) + } +} + +// The allowed list restricts function tools only. When it permits none but a server +// tool is declared, the server tool stays usable: "none" would switch it off too. +func TestToAnthropicChatRequest_AllowedToolsKeepServerToolsUsable(t *testing.T) { + ctx := schemas.NewBifrostContext(nil, schemas.NoDeadline) + req := restrictionRequest(allowedTools("auto"), nil) + req.Params.Tools = append(req.Params.Tools, schemas.ChatTool{Type: schemas.ChatToolType("web_search_20260209"), Name: "web_search"}) + + converted, err := ToAnthropicChatRequest(ctx, req) + require.NoError(t, err) + assert.Equal(t, 0, countFunctionTools(converted.Tools), "no function tool was allowed") + require.Len(t, converted.Tools, 1, "the server tool stays declared") + require.NotNil(t, converted.ToolChoice) + assert.Equal(t, "auto", converted.ToolChoice.Type, `"none" would also switch off the server tool`) +} + +// parallel_tool_calls: false has a direct Anthropic equivalent that was never +// populated, so the instruction was dropped and calls stayed parallel. +func TestToAnthropicChatRequest_ParallelToolCallsFalseDisablesParallelUse(t *testing.T) { + ctx := schemas.NewBifrostContext(nil, schemas.NoDeadline) + + t.Run("with an explicit tool choice", func(t *testing.T) { + req, err := ToAnthropicChatRequest(ctx, restrictionRequest( + &schemas.ChatToolChoice{ChatToolChoiceStr: schemas.Ptr("auto")}, schemas.Ptr(false))) + require.NoError(t, err) + require.NotNil(t, req.ToolChoice) + require.NotNil(t, req.ToolChoice.DisableParallelToolUse) + assert.True(t, *req.ToolChoice.DisableParallelToolUse) + }) + + t.Run("with no tool choice of its own", func(t *testing.T) { + req, err := ToAnthropicChatRequest(ctx, restrictionRequest(nil, schemas.Ptr(false))) + require.NoError(t, err) + require.NotNil(t, req.ToolChoice, "the flag has nowhere to live without a tool_choice") + assert.Equal(t, "auto", req.ToolChoice.Type, "auto is Anthropic's default, so this adds no restriction") + require.NotNil(t, req.ToolChoice.DisableParallelToolUse) + assert.True(t, *req.ToolChoice.DisableParallelToolUse) + }) +} + +// parallel_tool_calls: true is Anthropic's default, so nothing should be sent. +func TestToAnthropicChatRequest_ParallelToolCallsTrueSendsNothing(t *testing.T) { + ctx := schemas.NewBifrostContext(nil, schemas.NoDeadline) + + req, err := ToAnthropicChatRequest(ctx, restrictionRequest(nil, schemas.Ptr(true))) + require.NoError(t, err) + if req.ToolChoice != nil { + assert.Nil(t, req.ToolChoice.DisableParallelToolUse) + } +} + +// A server tool - MCP toolset, web search, computer use - cannot be named in +// an OpenAI allowed_tools list, which describes function tools only. Filtering +// it on a name it cannot have would switch off a capability the caller enabled +// elsewhere in the same request. +func TestToAnthropicChatRequest_AllowedToolsLeavesServerToolsAlone(t *testing.T) { + ctx := schemas.NewBifrostContext(nil, schemas.NoDeadline) + + req := restrictionRequest(allowedTools("required", "tool_a"), nil) + // Stand in for a converted server tool: it carries a Type, which a + // converted function tool never does. + converted, err := ToAnthropicChatRequest(ctx, req) + require.NoError(t, err) + + // Two server-tool shapes, because they are filtered by different fields: + // web search sets Type, while an MCP toolset leaves Type nil and carries + // everything in MCPToolset with an empty Name. + webSearch := AnthropicToolType("web_search") + withServer := append([]AnthropicTool{ + {Name: "web-search", Type: &webSearch}, + {MCPToolset: &AnthropicMCPToolsetTool{Type: "mcp_toolset", MCPServerName: "docs"}}, + }, converted.Tools...) + filtered := filterToolsByAllowed(withServer, []schemas.ChatToolChoiceAllowedToolsTool{ + {Type: "function", Function: schemas.ChatToolChoiceFunction{Name: "tool_a"}}, + }) + + names := make([]string, 0, len(filtered)) + for _, tl := range filtered { + names = append(names, tl.Name) + } + assert.Contains(t, names, "web-search", "a Type-carrying server tool must survive") + assert.Contains(t, names, "tool_a") + assert.NotContains(t, names, "tool_b") + + mcpKept := false + for _, tl := range filtered { + if tl.MCPToolset != nil { + mcpKept = true + } + } + assert.True(t, mcpKept, "an MCP toolset has Type nil and an empty Name; it must not be matched against the allowlist") +} + +// With every nameable tool excluded, the choice must still be "none" even +// though server tools remain in the list. +func TestToAnthropicChatRequest_AllowedToolsNoneNameableStillMeansNone(t *testing.T) { + webSearch := AnthropicToolType("web_search") + tools := []AnthropicTool{ + {Name: "web-search", Type: &webSearch}, + {MCPToolset: &AnthropicMCPToolsetTool{Type: "mcp_toolset", MCPServerName: "docs"}}, + {Name: "tool_a"}, + } + assert.Equal(t, 1, countFunctionTools(tools), "only tool_a is nameable; neither server shape counts") + + filtered := filterToolsByAllowed(tools, []schemas.ChatToolChoiceAllowedToolsTool{ + {Type: "function", Function: schemas.ChatToolChoiceFunction{Name: "not_declared"}}, + }) + assert.Equal(t, 0, countFunctionTools(filtered), "no function tool was allowed, so the choice must be none") + assert.Len(t, filtered, 2, "both server tools are still there") +} diff --git a/core/providers/anthropic/chatconversion_bench_test.go b/core/providers/anthropic/chatconversion_bench_test.go new file mode 100644 index 00000000000..4456df3e2fe --- /dev/null +++ b/core/providers/anthropic/chatconversion_bench_test.go @@ -0,0 +1,298 @@ +package anthropic + +// Benchmarks for the Anthropic Messages translation layer. +// +// This is the most expensive provider translation in the gateway: Anthropic's +// wire format differs structurally from the neutral schema (content blocks +// instead of a content string, tool results as user turns, thinking blocks, +// server tools), so the conversion is real work rather than a field copy. It +// runs once per request on the way out and once per response - or once per +// EVENT on a stream, where the state machine below is the hot loop. +// +// The cross-API benchmark is the shape most Bifrost deployments actually serve: +// an OpenAI-format request from the caller, translated to Anthropic on the way +// to the provider. +// +// Run: +// +// go test ./core/providers/anthropic/ -bench Anthropic -benchmem + +import ( + "testing" + + "github.com/capsohq/bifrost/core/providers/openai" + "github.com/capsohq/bifrost/core/schemas" +) + +const benchAnthropicModel = "claude-sonnet-4-5-20250929" + +// An OpenAI-shaped request as a caller sends it, with a replayed tool result +// and two tool definitions. +const benchOpenAICompatRequestJSON = `{ + "model": "anthropic/claude-sonnet-4-5-20250929", + "messages": [ + {"role": "system", "content": "You are a concise assistant. Answer in one sentence."}, + {"role": "user", "content": "What's the weather in San Francisco and Bengaluru?"}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "call_9xKmNpQrStUvWxYz01", + "type": "function", + "function": {"name": "get_weather", "arguments": "{\"location\":\"San Francisco, CA\",\"unit\":\"celsius\"}"} + } + ] + }, + {"role": "tool", "tool_call_id": "call_9xKmNpQrStUvWxYz01", "content": "{\"temp_c\":14.4,\"condition\":\"foggy\"}"}, + { + "role": "user", + "content": [ + {"type": "text", "text": "And how does that compare to yesterday?"} + ] + } + ], + "temperature": 0.2, + "max_completion_tokens": 1024, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather for a location", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string", "description": "City and state, e.g. San Francisco, CA"}, + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, + "days": {"type": "integer", "minimum": 1, "maximum": 14} + }, + "required": ["location"], + "additionalProperties": false + } + } + }, + { + "type": "function", + "function": { + "name": "search_docs", + "description": "Search the internal documentation index", + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "top_k": {"type": "integer", "minimum": 1, "maximum": 50} + }, + "required": ["query"] + } + } + } + ] +}` + +// A Messages response carrying thinking, text and a tool_use block - the three +// block kinds the response converter has to demultiplex into the neutral shape. +const benchAnthropicResponseJSON = `{ + "id": "msg_01XyZaBcDeFgHiJkLmNoPqRs", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-5-20250929", + "stop_reason": "tool_use", + "stop_sequence": null, + "content": [ + { + "type": "thinking", + "thinking": "The user asked about two cities. I should call the weather tool once per city and compare with yesterday's reading.", + "signature": "EqoBCkgIARABGAIiQL2ZbW5vcHFy" + }, + { + "type": "text", + "text": "Let me check the current conditions for both cities before comparing them with yesterday." + }, + { + "type": "tool_use", + "id": "toolu_01AbCdEfGhIjKlMnOpQrStUv", + "name": "get_weather", + "input": {"location": "Bengaluru, KA", "unit": "celsius", "days": 2} + } + ], + "usage": { + "input_tokens": 1284, + "output_tokens": 96, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1024 + } +}` + +func benchBifrostChatRequest(b *testing.B, ctx *schemas.BifrostContext) *schemas.BifrostChatRequest { + b.Helper() + var openAIReq openai.OpenAIChatRequest + if err := schemas.Unmarshal([]byte(benchOpenAICompatRequestJSON), &openAIReq); err != nil { + b.Fatalf("unmarshal OpenAI-compatible request: %v", err) + } + return openAIReq.ToBifrostChatRequest(ctx) +} + +// BenchmarkToAnthropicChatRequest measures the neutral-to-Anthropic request +// conversion: message re-grouping, tool schema lowering, parameter gating. +func BenchmarkToAnthropicChatRequest(b *testing.B) { + ctx := schemas.NewBifrostContext(nil, schemas.NoDeadline) + bifrostReq := benchBifrostChatRequest(b, ctx) + b.ReportAllocs() + for b.Loop() { + out, err := ToAnthropicChatRequest(ctx, bifrostReq) + if err != nil { + b.Fatalf("convert to Anthropic request: %v", err) + } + if out == nil { + b.Fatal("nil Anthropic request") + } + } +} + +// BenchmarkAnthropicRequestCrossAPI measures the whole outbound path a +// caller-facing OpenAI request pays for when Anthropic serves it: parse, +// lower to neutral, raise to Messages, serialize. +func BenchmarkAnthropicRequestCrossAPI(b *testing.B) { + data := []byte(benchOpenAICompatRequestJSON) + ctx := schemas.NewBifrostContext(nil, schemas.NoDeadline) + b.ReportAllocs() + for b.Loop() { + var openAIReq openai.OpenAIChatRequest + if err := schemas.Unmarshal(data, &openAIReq); err != nil { + b.Fatalf("unmarshal OpenAI-compatible request: %v", err) + } + anthropicReq, err := ToAnthropicChatRequest(ctx, openAIReq.ToBifrostChatRequest(ctx)) + if err != nil { + b.Fatalf("convert to Anthropic request: %v", err) + } + payload, err := schemas.Marshal(anthropicReq) + if err != nil { + b.Fatalf("marshal Anthropic request: %v", err) + } + if len(payload) == 0 { + b.Fatal("empty payload") + } + } +} + +// BenchmarkAnthropicResponseUnmarshal measures parsing the Messages response. +func BenchmarkAnthropicResponseUnmarshal(b *testing.B) { + data := []byte(benchAnthropicResponseJSON) + b.ReportAllocs() + for b.Loop() { + var resp AnthropicMessageResponse + if err := schemas.Unmarshal(data, &resp); err != nil { + b.Fatalf("unmarshal Anthropic response: %v", err) + } + } +} + +// BenchmarkAnthropicResponseToBifrost measures demultiplexing the content +// blocks (thinking, text, tool_use) into the neutral response. +func BenchmarkAnthropicResponseToBifrost(b *testing.B) { + var resp AnthropicMessageResponse + if err := schemas.Unmarshal([]byte(benchAnthropicResponseJSON), &resp); err != nil { + b.Fatalf("unmarshal Anthropic response: %v", err) + } + ctx := schemas.NewBifrostContext(nil, schemas.NoDeadline) + b.ReportAllocs() + for b.Loop() { + if out := resp.ToBifrostChatResponse(ctx); out == nil { + b.Fatal("nil Bifrost response") + } + } +} + +// benchAnthropicStreamEvents is one complete streaming turn: message_start, +// a text block with deltas, a tool_use block whose arguments stream as JSON +// fragments, then the terminal events. +func benchAnthropicStreamEvents() []string { + events := []string{ + `{"type":"message_start","message":{"id":"msg_01XyZaBcDeFgHiJkLmNoPqRs","type":"message","role":"assistant","model":"claude-sonnet-4-5-20250929","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":1284,"output_tokens":1,"cache_read_input_tokens":1024}}}`, + `{"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}`, + } + for range 24 { + events = append(events, `{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" incrementally streamed token"}}`) + } + events = append(events, + `{"type":"content_block_stop","index":0}`, + `{"type":"content_block_start","index":1,"content_block":{"type":"tool_use","id":"toolu_01AbCdEfGhIjKlMnOpQrStUv","name":"get_weather","input":{}}}`, + `{"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"{\"location\":"}}`, + `{"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"\"Bengaluru, KA\"}"}}`, + `{"type":"content_block_stop","index":1}`, + `{"type":"message_delta","delta":{"stop_reason":"tool_use"},"usage":{"output_tokens":96}}`, + `{"type":"message_stop"}`, + ) + return events +} + +// BenchmarkAnthropicStreamTurnToBifrost measures converting a full streaming +// turn event by event, including the block-index state the converter carries +// across events. This is the per-token cost of every Anthropic stream. +func BenchmarkAnthropicStreamTurnToBifrost(b *testing.B) { + raw := benchAnthropicStreamEvents() + events := make([]AnthropicStreamEvent, len(raw)) + for i, payload := range raw { + if err := schemas.Unmarshal([]byte(payload), &events[i]); err != nil { + b.Fatalf("unmarshal stream event %d: %v", i, err) + } + } + ctx := schemas.NewBifrostContext(nil, schemas.NoDeadline) + b.ReportAllocs() + for b.Loop() { + state := NewAnthropicStreamState() + for i := range events { + _, bifrostErr, _ := events[i].ToBifrostChatCompletionStream(ctx, "", state) + if bifrostErr != nil { + b.Fatalf("convert stream event %d: %v", i, bifrostErr) + } + } + } +} + +// BenchmarkAnthropicStreamTurnParseAndConvert adds the per-event JSON parse +// that the provider loop pays before the conversion above. +func BenchmarkAnthropicStreamTurnParseAndConvert(b *testing.B) { + raw := benchAnthropicStreamEvents() + payloads := make([][]byte, len(raw)) + for i, payload := range raw { + payloads[i] = []byte(payload) + } + ctx := schemas.NewBifrostContext(nil, schemas.NoDeadline) + b.ReportAllocs() + for b.Loop() { + state := NewAnthropicStreamState() + for i, payload := range payloads { + var event AnthropicStreamEvent + if err := schemas.Unmarshal(payload, &event); err != nil { + b.Fatalf("unmarshal stream event %d: %v", i, err) + } + _, bifrostErr, _ := event.ToBifrostChatCompletionStream(ctx, "", state) + if bifrostErr != nil { + b.Fatalf("convert stream event %d: %v", i, bifrostErr) + } + } + } +} + +// BenchmarkToAnthropicChatResponse measures the reverse response conversion, +// used when a non-Anthropic provider serves a caller on the Messages surface. +func BenchmarkToAnthropicChatResponse(b *testing.B) { + var resp AnthropicMessageResponse + if err := schemas.Unmarshal([]byte(benchAnthropicResponseJSON), &resp); err != nil { + b.Fatalf("unmarshal Anthropic response: %v", err) + } + ctx := schemas.NewBifrostContext(nil, schemas.NoDeadline) + bifrostResp := resp.ToBifrostChatResponse(ctx) + if bifrostResp == nil { + b.Fatal("nil Bifrost response") + } + bifrostResp.Model = benchAnthropicModel + b.ReportAllocs() + for b.Loop() { + if out := ToAnthropicChatResponse(bifrostResp); out == nil { + b.Fatal("nil Anthropic response") + } + } +} diff --git a/core/providers/anthropic/computertoolset_test.go b/core/providers/anthropic/computertoolset_test.go new file mode 100644 index 00000000000..e750182c053 --- /dev/null +++ b/core/providers/anthropic/computertoolset_test.go @@ -0,0 +1,405 @@ +package anthropic + +import ( + "encoding/json" + "fmt" + "testing" + + providerUtils "github.com/capsohq/bifrost/core/providers/utils" + "github.com/capsohq/bifrost/core/schemas" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// roundTripAnthropicBody parses a native Anthropic body, converts it to the +// neutral Responses shape and back, and returns what would go on the wire. The +// toolset's whole contract lives on that round trip: the generation is re-derived +// from the target model, and toolset_name has to survive on both halves of every +// call/result pair. +func roundTripAnthropicBody(t *testing.T, body string) *AnthropicMessageRequest { + t.Helper() + var req AnthropicMessageRequest + require.NoError(t, json.Unmarshal([]byte(body), &req)) + + ctx := schemas.NewBifrostContext(nil, schemas.NoDeadline) + bifrostReq := req.ToBifrostResponsesRequest(ctx) + bifrostReq.Provider = schemas.Anthropic + + out, err := ToAnthropicResponsesRequest(ctx, bifrostReq) + require.NoError(t, err) + require.NotNil(t, out) + return out +} + +// A toolset entry is bare. Sending a name gets "name is not accepted on a toolset +// entry", and display_width_px/display_height_px get "Extra inputs are not +// permitted" — both verified against the live API. +func TestComputerToolset_EmittedBare(t *testing.T) { + out := roundTripAnthropicBody(t, `{"model":"claude-opus-5-5","max_tokens":64, + "messages":[{"role":"user","content":"hi"}], + "tools":[{"type":"computer_toolset_20260801"}]}`) + + require.Len(t, out.Tools, 1) + tool := out.Tools[0] + require.NotNil(t, tool.Type) + assert.Equal(t, AnthropicToolTypeComputerToolset20260801, *tool.Type) + assert.Empty(t, tool.Name, "a toolset entry rejects name outright") + assert.Nil(t, tool.AnthropicToolComputerUse, "a toolset entry rejects the display_* geometry") + + encoded, err := json.Marshal(tool) + require.NoError(t, err) + assert.NotContains(t, string(encoded), `"name"`, "name must be omitted, not sent empty: %s", encoded) +} + +// The generation is a property of the target model, not of what the caller sent, +// so a dated tool aimed at Opus 5.5 is upgraded rather than forwarded into a 400. +func TestComputerToolset_DatedToolUpgradedForOpus55(t *testing.T) { + out := roundTripAnthropicBody(t, `{"model":"claude-opus-5-5","max_tokens":64, + "messages":[{"role":"user","content":"hi"}], + "tools":[{"type":"computer_20251124","name":"computer","display_width_px":1280,"display_height_px":800}]}`) + + require.Len(t, out.Tools, 1) + require.NotNil(t, out.Tools[0].Type) + assert.Equal(t, AnthropicToolTypeComputerToolset20260801, *out.Tools[0].Type) + assert.Empty(t, out.Tools[0].Name) +} + +// The mirror of the case above: Opus 5 still accepts computer_20251124, so it +// must keep getting it rather than being moved onto the toolset. +func TestComputerToolset_Opus5KeepsDatedTool(t *testing.T) { + out := roundTripAnthropicBody(t, `{"model":"claude-opus-5","max_tokens":64, + "messages":[{"role":"user","content":"hi"}], + "tools":[{"type":"computer_20251124","name":"computer","display_width_px":1280,"display_height_px":800}]}`) + + require.Len(t, out.Tools, 1) + require.NotNil(t, out.Tools[0].Type) + assert.Equal(t, AnthropicToolTypeComputer20251124, *out.Tools[0].Type) + assert.Equal(t, "computer", out.Tools[0].Name) +} + +// toolsetName returns the toolset_name on every tool_use / tool_result block of +// the request, keyed by the id the pair shares. +func toolsetNames(req *AnthropicMessageRequest) (calls, results map[string]string) { + calls, results = map[string]string{}, map[string]string{} + for _, msg := range req.Messages { + if msg.Content.ContentBlocks == nil { + continue + } + for _, block := range msg.Content.ContentBlocks { + name := "" + if block.ToolsetName != nil { + name = *block.ToolsetName + } + switch block.Type { + case AnthropicContentBlockTypeToolUse: + if block.ID != nil { + calls[*block.ID] = name + } + case AnthropicContentBlockTypeToolResult: + if block.ToolUseID != nil { + results[*block.ToolUseID] = name + } + } + } + } + return calls, results +} + +const toolsetReplayBody = `{"model":"claude-opus-5-5","max_tokens":64, + "tools":[{"type":"computer_toolset_20260801"}], + "messages":[ + {"role":"user","content":"screenshot then click"}, + {"role":"assistant","content":[ + {"type":"tool_use","id":"toolu_aaa","name":"screenshot","input":{},"toolset_name":"computer"}, + {"type":"tool_use","id":"toolu_bbb","name":"left_click","input":{"coordinate":[100,200]},"toolset_name":"computer"} + ]}, + {"role":"user","content":[%s]} + ]}` + +// Anthropic rejects a pair whose halves disagree: a result answering a member +// call without toolset_name, and a result carrying one whose call does not, are +// both 400s. A faithful echo has to survive untouched. +func TestComputerToolset_NameSurvivesSymmetricReplay(t *testing.T) { + results := `{"type":"tool_result","tool_use_id":"toolu_aaa","toolset_name":"computer","content":"ok"}, + {"type":"tool_result","tool_use_id":"toolu_bbb","toolset_name":"computer","content":"ok"}` + calls, got := toolsetNames(roundTripAnthropicBody(t, fmt.Sprintf(toolsetReplayBody, results))) + + assert.Equal(t, map[string]string{"toolu_aaa": "computer", "toolu_bbb": "computer"}, calls) + assert.Equal(t, map[string]string{"toolu_aaa": "computer", "toolu_bbb": "computer"}, got) +} + +// A client speaking a dialect without toolset_name — an OpenAI-shaped +// function_call_output, say — returns only the call id. Forwarding that beside a +// member tool_use is the 400 above, so the value is restored from the call. +func TestComputerToolset_NameBackfilledOntoResults(t *testing.T) { + results := `{"type":"tool_result","tool_use_id":"toolu_aaa","content":"ok"}, + {"type":"tool_result","tool_use_id":"toolu_bbb","content":"ok"}` + calls, got := toolsetNames(roundTripAnthropicBody(t, fmt.Sprintf(toolsetReplayBody, results))) + + assert.Equal(t, map[string]string{"toolu_aaa": "computer", "toolu_bbb": "computer"}, calls) + assert.Equal(t, map[string]string{"toolu_aaa": "computer", "toolu_bbb": "computer"}, + got, "a member result must carry its call's toolset_name") +} + +// An ordinary function call is not a toolset member, so nothing may be invented +// for it — a toolset_name on a non-member result is equally a 400. +func TestComputerToolset_NameNotInventedForPlainToolUse(t *testing.T) { + body := `{"model":"claude-opus-5-5","max_tokens":64, + "messages":[ + {"role":"user","content":"weather?"}, + {"role":"assistant","content":[{"type":"tool_use","id":"toolu_ccc","name":"get_weather","input":{}}]}, + {"role":"user","content":[{"type":"tool_result","tool_use_id":"toolu_ccc","content":"sunny"}]} + ]}` + calls, results := toolsetNames(roundTripAnthropicBody(t, body)) + + assert.Equal(t, map[string]string{"toolu_ccc": ""}, calls) + assert.Equal(t, map[string]string{"toolu_ccc": ""}, results) +} + +// The datasheet names the exact tool type, so a row moves a model onto the +// toolset (or holds it back) without a release. +func TestComputerToolset_DatasheetOutranksFallback(t *testing.T) { + t.Run("row moves a model the fallback leaves on the dated tool", func(t *testing.T) { + setOverride(t, "claude-opus-5", schemas.ModelCapabilities{ + ServerTools: map[string]string{"computer_use": string(AnthropicToolTypeComputerToolset20260801)}, + }) + assert.Equal(t, ComputerUseGenToolset20260801, + ComputerUseGeneration(schemas.ResolveModelCaps(schemas.Anthropic, "claude-opus-5"))) + }) + + t.Run("row holds a model back on the dated tool", func(t *testing.T) { + setOverride(t, "claude-opus-5-5", schemas.ModelCapabilities{ + ServerTools: map[string]string{"computer_use": string(AnthropicToolTypeComputer20251124)}, + }) + assert.Equal(t, ComputerUseGen20251124, + ComputerUseGeneration(schemas.ResolveModelCaps(schemas.Anthropic, "claude-opus-5-5"))) + }) +} + +// The chat converter drops any tool it cannot name, which would have swallowed +// the one entry that legitimately has no name. +func TestComputerToolset_ChatPathKeepsBareEntry(t *testing.T) { + caps := schemas.ResolveModelCaps(schemas.Anthropic, "claude-opus-5-5") + tool, ok := convertServerToolToAnthropic(schemas.ChatTool{ + Type: schemas.ChatToolType(AnthropicToolTypeComputer20251124), + }, caps, schemas.Anthropic) + + require.True(t, ok, "the toolset must not be dropped for having no name") + require.NotNil(t, tool.Type) + assert.Equal(t, AnthropicToolTypeComputerToolset20260801, *tool.Type) + assert.Empty(t, tool.Name) +} + +// A streaming member call arrives with toolset_name on content_block_start. The +// gateway re-encodes the stream rather than forwarding bytes, so the field has to +// be threaded through the inbound parse and the outbound rebuild alike — dropping +// it leaves a streaming client unable to answer the call it was just handed. +func TestComputerToolset_NameSurvivesStreaming(t *testing.T) { + ctx := schemas.NewBifrostContext(nil, schemas.NoDeadline) + state := &AnthropicResponsesStreamState{ + ContentIndexToOutputIndex: make(map[int]int), + ContentIndexToBlockType: make(map[int]AnthropicContentBlockType), + ToolArgumentBuffers: make(map[int]string), + MCPCallOutputIndices: make(map[int]bool), + ItemIDs: make(map[int]string), + OutputItems: make(map[int]*schemas.ResponsesMessage), + ReasoningSignatures: make(map[int]string), + TextContentIndices: make(map[int]bool), + ReasoningContentIndices: make(map[int]bool), + CompactionContentIndices: make(map[int]*schemas.CacheControl), + CreatedAt: 1234567890, + HasEmittedCreated: true, + HasEmittedInProgress: true, + } + + inbound := &AnthropicStreamEvent{ + Type: AnthropicStreamEventTypeContentBlockStart, + Index: schemas.Ptr(0), + ContentBlock: &AnthropicContentBlock{ + Type: AnthropicContentBlockTypeToolUse, + ID: schemas.Ptr("toolu_stream1"), + Name: schemas.Ptr("screenshot"), + Input: json.RawMessage(`{}`), + ToolsetName: schemas.Ptr("computer"), + }, + } + + neutral, bifrostErr, _ := inbound.ToBifrostResponsesStream(ctx, 0, state) + require.Nil(t, bifrostErr) + require.NotEmpty(t, neutral, "content_block_start must produce a neutral event") + + var sawStart bool + for _, event := range neutral { + if event.Item != nil && event.Item.ResponsesToolMessage != nil { + require.NotNil(t, event.Item.ResponsesToolMessage.ToolsetName, + "the neutral function_call lost toolset_name") + assert.Equal(t, "computer", *event.Item.ResponsesToolMessage.ToolsetName) + } + for _, out := range ToAnthropicResponsesStreamResponse(ctx, event) { + if out.Type != AnthropicStreamEventTypeContentBlockStart || out.ContentBlock == nil { + continue + } + if out.ContentBlock.Type != AnthropicContentBlockTypeToolUse { + continue + } + sawStart = true + require.NotNil(t, out.ContentBlock.ToolsetName, + "content_block_start went back out without toolset_name") + assert.Equal(t, "computer", *out.ContentBlock.ToolsetName) + } + } + assert.True(t, sawStart, "no tool_use content_block_start was re-emitted") +} + +// On the surfaces that remap raw bodies (Vertex, Bedrock, Mantle) a dated tool +// aimed at Opus 5.5 is upgraded rather than relayed into a 400, and the name and +// display_* geometry the toolset rejects are shed. Anthropic direct does not +// remap raw bodies, so its passthrough forwards the caller's bytes untouched. +func TestComputerToolset_RawBodyUpgradedToToolset(t *testing.T) { + body := []byte(`{"model":"claude-opus-5-5","max_tokens":64,` + + `"tools":[{"type":"computer_20251124","name":"computer","display_width_px":1280,"display_height_px":800,"display_number":1}]}`) + + out, err := RemapRawToolVersionsForProvider(body, schemas.Vertex, "claude-opus-5-5") + require.NoError(t, err) + t.Logf("normalized body: %s", out) + + assert.Equal(t, string(AnthropicToolTypeComputerToolset20260801), + providerUtils.GetJSONField(out, "tools.0.type").String()) + for _, field := range []string{"name", "display_width_px", "display_height_px", "display_number"} { + assert.False(t, providerUtils.JSONFieldExists(out, "tools.0."+field), + "%s is rejected on a toolset entry but survived: %s", field, out) + } +} + +// A toolset carries no display geometry, and the dated computer tools validate +// display_*_px as >= 1, so one cannot be rewritten as the other. Where the target +// takes a toolset the caller's entry is forwarded as sent; where it takes neither +// form the tool is dropped, because a zero-sized display is a hard 400 and would +// turn a re-routed request into a failure rather than a degraded one. +func TestComputerToolset_ForwardedWhenTargetAcceptsIt(t *testing.T) { + // Every model the computer-use docs list for the toolset, on the two surfaces + // that serve it. All of these also accept the dated tool, so the generation + // says "dated" — forwarding still has to win, or the caller loses the toolset + // they explicitly asked for. + cases := []struct { + provider schemas.ModelProvider + model string + }{ + {schemas.Anthropic, "claude-opus-5"}, + {schemas.Anthropic, "claude-sonnet-5"}, + {schemas.Anthropic, "claude-opus-4-8"}, + {schemas.Anthropic, "claude-fable-5-1"}, + {schemas.Vertex, "claude-opus-5"}, + } + for _, tc := range cases { + t.Run(string(tc.provider)+"/"+tc.model, func(t *testing.T) { + caps := schemas.ResolveModelCaps(tc.provider, tc.model) + tool := convertBifrostToolToAnthropic(caps, &schemas.ResponsesTool{ + Type: schemas.ResponsesToolTypeComputerUsePreview, + ResponsesToolComputerUsePreview: &schemas.ResponsesToolComputerUsePreview{Environment: "browser"}, + }, tc.provider, false) + + require.NotNil(t, tool, "the model takes a toolset, so it must not be dropped") + require.NotNil(t, tool.Type) + assert.Equal(t, AnthropicToolTypeComputerToolset20260801, *tool.Type) + assert.Empty(t, tool.Name, "a toolset entry rejects name") + assert.Nil(t, tool.AnthropicToolComputerUse, "a toolset entry rejects the display_* geometry") + }) + } +} + +func TestComputerToolset_DroppedWhenTargetTakesNeitherForm(t *testing.T) { + // AWS and Foundry serve only the dated tool, and Sonnet 4.6 and older reject + // the toolset on every surface — so there is no shape left to send. + cases := []struct { + provider schemas.ModelProvider + model string + }{ + {schemas.Bedrock, "claude-opus-5-5"}, + {schemas.BedrockMantle, "claude-opus-5-5"}, + {schemas.Azure, "claude-opus-5-5"}, + {schemas.Anthropic, "claude-sonnet-4-6"}, + {schemas.Vertex, "claude-sonnet-4-6"}, + } + for _, tc := range cases { + t.Run(string(tc.provider)+"/"+tc.model, func(t *testing.T) { + caps := schemas.ResolveModelCaps(tc.provider, tc.model) + require.False(t, AcceptsComputerToolset(caps), "case is only meaningful where the toolset is rejected") + + tool := convertBifrostToolToAnthropic(caps, &schemas.ResponsesTool{ + Type: schemas.ResponsesToolTypeComputerUsePreview, + ResponsesToolComputerUsePreview: &schemas.ResponsesToolComputerUsePreview{Environment: "browser"}, + }, tc.provider, false) + + assert.Nil(t, tool, "a dimensionless computer tool must be dropped, not sent as a 0x0 display") + }) + } +} + +// supports_computer_toolset moves a model onto or off the toolset without a +// release, in both directions. +func TestComputerToolset_AcceptanceDatasheetOutranksFallback(t *testing.T) { + t.Run("row grants the toolset to a model the fallback withholds it from", func(t *testing.T) { + yes := true + setOverride(t, "claude-sonnet-4-6", schemas.ModelCapabilities{SupportsComputerToolset: &yes}) + assert.True(t, AcceptsComputerToolset(schemas.ResolveModelCaps(schemas.Anthropic, "claude-sonnet-4-6"))) + }) + + t.Run("row withholds it from a model the fallback grants", func(t *testing.T) { + no := false + setOverride(t, "claude-opus-5", schemas.ModelCapabilities{SupportsComputerToolset: &no}) + assert.False(t, AcceptsComputerToolset(schemas.ResolveModelCaps(schemas.Anthropic, "claude-opus-5"))) + }) + + t.Run("a row cannot grant it on a surface that does not serve it", func(t *testing.T) { + yes := true + providerUtils.SetCapabilityResolver(func(p schemas.ModelProvider, m string) *schemas.ModelCapabilities { + return &schemas.ModelCapabilities{SupportsComputerToolset: &yes} + }) + t.Cleanup(func() { providerUtils.SetCapabilityResolver(nil) }) + assert.False(t, AcceptsComputerToolset(schemas.ResolveModelCaps(schemas.Bedrock, "claude-opus-5-5")), + "AWS serves only the dated tool regardless of the row") + }) +} + +// The guard keys off the geometry, not off where the tool came from, so a genuine +// computer_use_preview still converts untouched. +func TestComputerToolset_RealGeometryStillConverts(t *testing.T) { + caps := schemas.ResolveModelCaps(schemas.Anthropic, "claude-opus-5") + tool := convertBifrostToolToAnthropic(caps, &schemas.ResponsesTool{ + Type: schemas.ResponsesToolTypeComputerUsePreview, + ResponsesToolComputerUsePreview: &schemas.ResponsesToolComputerUsePreview{ + Environment: "browser", DisplayWidth: 1280, DisplayHeight: 800, + }, + }, schemas.Anthropic, false) + + require.NotNil(t, tool, "a tool with a real display must still convert") + require.NotNil(t, tool.Type) + assert.Equal(t, AnthropicToolTypeComputer20251124, *tool.Type) + assert.Equal(t, "computer", tool.Name) + require.NotNil(t, tool.AnthropicToolComputerUse) + assert.Equal(t, 1280, *tool.AnthropicToolComputerUse.DisplayWidthPx) + assert.Equal(t, 800, *tool.AnthropicToolComputerUse.DisplayHeightPx) +} + +// The raw path downgrades by rewriting each tool in place, so it needs the same +// guard: a toolset entry there would otherwise become a dated tool with neither a +// name nor a display. +func TestComputerToolset_RawBodyDropsUndowngradableToolset(t *testing.T) { + body := []byte(`{"model":"claude-sonnet-4-6","max_tokens":64,` + + `"tools":[{"type":"computer_toolset_20260801"},` + + `{"type":"text_editor_20250728","name":"str_replace_based_edit_tool"},` + + `{"type":"bash_20250124","name":"bash"}]}`) + + out, err := RemapRawToolVersionsForProvider(body, schemas.Vertex, "claude-sonnet-4-6") + require.NoError(t, err) + + types := []string{} + for _, tool := range providerUtils.GetJSONField(out, "tools").Array() { + types = append(types, tool.Get("type").String()) + } + assert.NotContains(t, types, "computer_20251124", "undowngradable toolset must not become a dated tool: %s", out) + assert.NotContains(t, types, "computer_toolset_20260801", "the target cannot take a toolset either: %s", out) + assert.Contains(t, types, "text_editor_20250728", "sibling tools carry no geometry and must survive: %s", out) + assert.Contains(t, types, "bash_20250124", "sibling tools carry no geometry and must survive: %s", out) +} diff --git a/core/providers/anthropic/documentdataurl.go b/core/providers/anthropic/documentdataurl.go index ad1b781ca24..ab18e8f2bfa 100644 --- a/core/providers/anthropic/documentdataurl.go +++ b/core/providers/anthropic/documentdataurl.go @@ -7,7 +7,6 @@ import ( "github.com/tidwall/gjson" "github.com/tidwall/sjson" "mime" - "strconv" "strings" "unicode/utf8" ) @@ -31,36 +30,88 @@ func inlineTextDataURL(data string) *AnthropicSource { // normalizeBase64TextSources preserves base64 transport while adapting text documents to Anthropic's text source wire type. func normalizeBase64TextSources(body []byte) ([]byte, error) { - var paths []string - gjson.GetBytes(body, "messages").ForEach(func(i, message gjson.Result) bool { - message.Get("content").ForEach(func(j, block gjson.Result) bool { - if block.Get("type").String() == "document" && block.Get("source.type").String() == "base64" && isTextDocumentMediaType(block.Get("source.media_type").String()) { - paths = append(paths, "messages."+strconv.Itoa(int(i.Int()))+".content."+strconv.Itoa(int(j.Int()))+".source") + messages := gjson.GetBytes(body, "messages") + if !messages.IsArray() { + return body, nil + } + + // Each matching source is rewritten inside its own block and the messages array is + // written back once. Addressing through the whole body (messages.i.content.j.source) + // would reserialise the entire request three times per document, making this + // O(documents x body). Pinned by TestNormalizeBase64TextSources_AllocationScaling. + var rebuiltMessages [][]byte + anyChanged := false + var convErr error + + messages.ForEach(func(_, message gjson.Result) bool { + messageRaw := []byte(message.Raw) + content := message.Get("content") + if !content.IsArray() { + rebuiltMessages = append(rebuiltMessages, messageRaw) + return true + } + + var rebuiltBlocks [][]byte + messageChanged := false + content.ForEach(func(_, block gjson.Result) bool { + blockRaw := []byte(block.Raw) + if block.Get("type").String() != "document" || + block.Get("source.type").String() != "base64" || + !isTextDocumentMediaType(block.Get("source.media_type").String()) { + rebuiltBlocks = append(rebuiltBlocks, blockRaw) + return true + } + decoded, err := base64.StdEncoding.DecodeString(block.Get("source.data").String()) + if err != nil || !utf8.Valid(decoded) { + convErr = fmt.Errorf("invalid base64 UTF-8 text document") + return false + } + for _, set := range []struct { + path string + value string + }{ + {"source.type", "text"}, + {"source.media_type", "text/plain"}, + {"source.data", string(decoded)}, + } { + blockRaw, err = sjson.SetBytes(blockRaw, set.path, set.value) + if err != nil { + convErr = err + return false + } } + rebuiltBlocks = append(rebuiltBlocks, blockRaw) + messageChanged = true return true }) - return true - }) - for _, path := range paths { - source := gjson.GetBytes(body, path) - decoded, err := base64.StdEncoding.DecodeString(source.Get("data").String()) - if err != nil || !utf8.Valid(decoded) { - return nil, fmt.Errorf("invalid base64 UTF-8 text document") + if convErr != nil { + return false } - body, err = sjson.SetBytes(body, path+".type", "text") - if err != nil { - return nil, err - } - body, err = sjson.SetBytes(body, path+".media_type", "text/plain") - if err != nil { - return nil, err - } - body, err = sjson.SetBytes(body, path+".data", string(decoded)) - if err != nil { - return nil, err + + if messageChanged { + updated, err := sjson.SetRawBytes(messageRaw, "content", rawJSONArrayOf(rebuiltBlocks)) + if err != nil { + convErr = err + return false + } + messageRaw = updated + anyChanged = true } + rebuiltMessages = append(rebuiltMessages, messageRaw) + return true + }) + if convErr != nil { + return nil, convErr + } + if !anyChanged { + return body, nil + } + + updated, err := sjson.SetRawBytes(body, "messages", rawJSONArrayOf(rebuiltMessages)) + if err != nil { + return nil, err } - return body, nil + return updated, nil } // isTextDocumentMediaType recognizes textual document formats that Anthropic accepts as text sources. diff --git a/core/providers/anthropic/documentdataurl_test.go b/core/providers/anthropic/documentdataurl_test.go index 20381359d4f..8d322417c83 100644 --- a/core/providers/anthropic/documentdataurl_test.go +++ b/core/providers/anthropic/documentdataurl_test.go @@ -1,11 +1,14 @@ package anthropic import ( + "bytes" "encoding/base64" - "github.com/capsohq/bifrost/core/schemas" - "github.com/tidwall/gjson" "strings" "testing" + + "github.com/capsohq/bifrost/core/internal/memtest" + "github.com/capsohq/bifrost/core/schemas" + "github.com/tidwall/gjson" ) // TestInlineTextDocumentDataURL verifies base64 transport becomes readable text in both inference APIs. @@ -59,3 +62,32 @@ func TestNativeJSONDocument(t *testing.T) { } } } + +// TestNormalizeBase64TextSources_AllocationScaling pins the allocation shape of the +// base64 text-document rewrite. +// +// The loop performs three whole-body sjson.SetBytes calls per document source, and each +// reserialises the entire request, so N text documents cost 3N copies of the body. +func TestNormalizeBase64TextSources_AllocationScaling(t *testing.T) { + memtest.AssertAllocScaling(t, func(docs int) []byte { + // A base64 text/plain document source is what the rewrite targets; the payload + // grows with N because each document carries its own encoded body. + payload := base64.StdEncoding.EncodeToString([]byte(strings.Repeat("t", 300))) + var b bytes.Buffer + b.WriteString(`{"model":"claude-opus-4-8","messages":[{"role":"user","content":[`) + for i := range docs { + if i > 0 { + b.WriteByte(',') + } + b.WriteString(`{"type":"document","source":{"type":"base64","media_type":"text/plain","data":"`) + b.WriteString(payload) + b.WriteString(`"}}`) + } + b.WriteString(`]}]}`) + return b.Bytes() + }, func(body []byte) { + if _, err := normalizeBase64TextSources(body); err != nil { + t.Fatalf("unexpected error: %v", err) + } + }) +} diff --git a/core/providers/anthropic/messagerequest_test.go b/core/providers/anthropic/messagerequest_test.go index 7d99e05f040..ac5deb856e9 100644 --- a/core/providers/anthropic/messagerequest_test.go +++ b/core/providers/anthropic/messagerequest_test.go @@ -81,6 +81,33 @@ func TestAnthropicMessageRequestUnmarshalJSONDuplicateAndReuseSemantics(t *testi assertRawExtraParam(t, req.ExtraParams, "client_extension", `{"last":true}`) } +// Claude Code auto-mode classifier: the safeguards request field must survive +// an unmarshal→marshal round trip so the gateway forwards it unchanged (Claude +// Code gateway compatibility guide, feature pass-through). +func TestAnthropicMessageRequestSafeguardsRoundTrip(t *testing.T) { + var req AnthropicMessageRequest + if err := sonic.Unmarshal([]byte(`{ + "model":"claude-opus-4-8", + "max_tokens":1024, + "messages":[{"role":"user","content":"hello"}], + "safeguards": { "z": 1, "a": { "b": true } } + }`), &req); err != nil { + t.Fatalf("unmarshal request: %v", err) + } + + if _, ok := req.ExtraParams["safeguards"]; ok { + t.Error("safeguards must decode into the typed field, not ExtraParams") + } + + out, err := sonic.Marshal(&req) + if err != nil { + t.Fatalf("marshal request: %v", err) + } + if want := `"safeguards":{"z":1,"a":{"b":true}}`; !strings.Contains(string(out), want) { + t.Fatalf("marshaled request lost safeguards: %s", string(out)) + } +} + func TestAnthropicMessageRequestUnmarshalJSONMalformedInput(t *testing.T) { var req AnthropicMessageRequest if err := sonic.Unmarshal([]byte(`{"model":"claude-sonnet-4-20250514"`), &req); err == nil { diff --git a/core/providers/anthropic/passthroughstream_test.go b/core/providers/anthropic/passthroughstream_test.go index 3924599863a..50a2d30e18d 100644 --- a/core/providers/anthropic/passthroughstream_test.go +++ b/core/providers/anthropic/passthroughstream_test.go @@ -1,6 +1,8 @@ package anthropic import ( + "context" + "encoding/json" "fmt" "testing" "time" @@ -595,3 +597,145 @@ func TestAnthropicWebFetchPassthroughNoResultConsumesHiddenIndex(t *testing.T) { t.Fatalf("expected hidden web_fetch result index to be consumed before next block; got index %d", got) } } + +// --- Claude Code auto-mode classifier: safeguard_results stream fidelity --- + +func safeguardStreamState() *AnthropicResponsesStreamState { + return &AnthropicResponsesStreamState{ + ContentIndexToOutputIndex: make(map[int]int), + ContentIndexToBlockType: make(map[int]AnthropicContentBlockType), + ToolArgumentBuffers: make(map[int]string), + MCPCallOutputIndices: make(map[int]bool), + ItemIDs: make(map[int]string), + OutputItems: make(map[int]*schemas.ResponsesMessage), + ReasoningSignatures: make(map[int]string), + TextContentIndices: make(map[int]bool), + ReasoningContentIndices: make(map[int]bool), + CompactionContentIndices: make(map[int]*schemas.CacheControl), + CreatedAt: 1234567890, + } +} + +// A top-level safeguard_results key on a stream event must survive the typed +// decode→marshal round trip, or re-rendered frames (mustConvertInPassthrough and +// the whole non-passthrough path) strip it and Claude Code declares the session +// ineligible for server-side auto-mode checks. +func TestAnthropicStreamEventSafeguardResultsRoundTrip(t *testing.T) { + frame := `{"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":7},"safeguard_results":[{"id":"sg_1","verdict":"allow"}]}` + var event AnthropicStreamEvent + if err := sonic.Unmarshal([]byte(frame), &event); err != nil { + t.Fatalf("unmarshal stream event: %v", err) + } + out, err := sonic.Marshal(&event) + if err != nil { + t.Fatalf("marshal stream event: %v", err) + } + if got := gjson.GetBytes(out, "safeguard_results").Raw; got != `[{"id":"sg_1","verdict":"allow"}]` { + t.Fatalf("safeguard_results lost on stream event round trip: %s", string(out)) + } +} + +// safeguard_results nested in message_start.message must ride the created chunk +// through the Bifrost intermediate and back onto the rebuilt message_start. +func TestAnthropicMessageStartSafeguardResultsRoundTrip(t *testing.T) { + frame := `{"type":"message_start","message":{"model":"claude-opus-4-8","id":"msg_1","type":"message","role":"assistant","content":[],"stop_reason":null,"usage":{"input_tokens":9,"output_tokens":1},"safeguard_results":[{"id":"sg_1","verdict":"allow"}]}}` + var event AnthropicStreamEvent + if err := sonic.Unmarshal([]byte(frame), &event); err != nil { + t.Fatalf("unmarshal message_start: %v", err) + } + + chunks, bifrostErr, _ := event.ToBifrostResponsesStream(context.Background(), 0, safeguardStreamState()) + if bifrostErr != nil { + t.Fatalf("ingress conversion: %+v", bifrostErr) + } + var created *schemas.BifrostResponsesStreamResponse + for _, c := range chunks { + if c != nil && c.Type == schemas.ResponsesStreamResponseTypeCreated { + created = c + } + } + if created == nil { + t.Fatalf("no response.created chunk emitted: %#v", chunks) + } + + ctx, cancel := schemas.NewBifrostContextWithCancel(context.Background()) + defer cancel() + events := ToAnthropicResponsesStreamResponse(ctx, created) + if len(events) == 0 || events[0] == nil { + t.Fatal("no events rebuilt from response.created") + } + out, err := sonic.Marshal(events[0]) + if err != nil { + t.Fatalf("marshal rebuilt message_start: %v", err) + } + if got := gjson.GetBytes(out, "message.safeguard_results").Raw; got != `[{"id":"sg_1","verdict":"allow"}]` { + t.Fatalf("safeguard_results lost on message_start round trip: %s", string(out)) + } +} + +// A chunk-level safeguard_results carry (attached by the provider SSE loop) must +// be restored top-level on the first rebuilt frame. +func TestAnthropicStreamChunkSafeguardResultsEgressRestore(t *testing.T) { + ctx, cancel := schemas.NewBifrostContextWithCancel(context.Background()) + defer cancel() + chunk := &schemas.BifrostResponsesStreamResponse{ + Type: schemas.ResponsesStreamResponseTypeCompleted, + Response: &schemas.BifrostResponsesResponse{}, + SafeguardResults: json.RawMessage(`[{"id":"sg_1","verdict":"allow"}]`), + } + events := ToAnthropicResponsesStreamResponse(ctx, chunk) + if len(events) == 0 || events[0] == nil { + t.Fatal("no events rebuilt from response.completed") + } + out, err := sonic.Marshal(events[0]) + if err != nil { + t.Fatalf("marshal rebuilt event: %v", err) + } + if got := gjson.GetBytes(out, "safeguard_results").Raw; got != `[{"id":"sg_1","verdict":"allow"}]` { + t.Fatalf("safeguard_results not restored on rebuilt frame: %s", string(out)) + } +} + +// Safeguard updates have an explicit typed path, including empty result arrays. +func TestSafeguardsUpdateStream(t *testing.T) { + for _, results := range []string{`[]`, `[{"id":"sg_1"}]`} { + var event AnthropicStreamEvent + if err := sonic.Unmarshal([]byte(`{"type":"safeguards_update","safeguard_results":`+results+`}`), &event); err != nil { + t.Fatal(err) + } + chunks, bifrostErr, last := event.ToBifrostResponsesStream(context.Background(), 0, safeguardStreamState()) + if bifrostErr != nil || last || len(chunks) != 1 { + t.Fatalf("chunks=%d err=%v last=%v", len(chunks), bifrostErr, last) + } + ctx, cancel := schemas.NewBifrostContextWithCancel(context.Background()) + defer cancel() + out := ToAnthropicResponsesStreamResponse(ctx, chunks[0]) + if len(out) != 1 || out[0].Type != "safeguards_update" || string(out[0].SafeguardResults) != results { + t.Fatalf("unexpected round trip: %#v", out) + } + if chunks[0].WithDefaults() != nil { + t.Fatal("safeguards update leaked to OpenAI surface") + } + } +} + +func TestProviderRawEventFilteredOnTypedSurfaces(t *testing.T) { + var event AnthropicStreamEvent + if err := sonic.Unmarshal([]byte(`{"type":"future_unknown_event","safeguard_results":[{"id":"sg_1"}]}`), &event); err != nil { + t.Fatalf("unmarshal unknown event: %v", err) + } + chunks, bifrostErr, isLast := event.ToBifrostResponsesStream(context.Background(), 0, safeguardStreamState()) + if bifrostErr != nil || isLast || len(chunks) != 0 { + t.Fatalf("unknown event type: chunks=%d err=%v last=%v, want 0/nil/false", len(chunks), bifrostErr, isLast) + } + + ctx, cancel := schemas.NewBifrostContextWithCancel(context.Background()) + defer cancel() + carrier := &schemas.BifrostResponsesStreamResponse{Type: schemas.ResponsesStreamResponseTypeProviderRawEvent} + if events := ToAnthropicResponsesStreamResponse(ctx, carrier); len(events) != 0 { + t.Errorf("provider_raw_event must not render typed events, got %#v", events) + } + if carrier.WithDefaults() != nil { + t.Error("WithDefaults must filter provider_raw_event from OpenAI-shaped surfaces") + } +} diff --git a/core/providers/anthropic/requestbuilder.go b/core/providers/anthropic/requestbuilder.go index 99a1537c20c..4e502a0bb08 100644 --- a/core/providers/anthropic/requestbuilder.go +++ b/core/providers/anthropic/requestbuilder.go @@ -303,7 +303,9 @@ func BuildAnthropicResponsesRequestBody(ctx *schemas.BifrostContext, request *sc // manually. var probe AnthropicMessageRequest if unmarshalErr := schemas.Unmarshal(jsonBody, &probe); unmarshalErr == nil { - AddMissingBetaHeadersToContext(ctx, &probe, cfg.Provider) + // Cloud bodies may no longer contain model; capability checks still need it. + probe.Model = capModel + _ = AddMissingBetaHeadersToContext(ctx, &probe, cfg.Provider) } for _, field := range cfg.ExcludeFields { @@ -606,7 +608,9 @@ func BuildAnthropicChatRequestBody(ctx *schemas.BifrostContext, request *schemas var probe AnthropicMessageRequest if unmarshalErr := schemas.Unmarshal(jsonBody, &probe); unmarshalErr == nil { - AddMissingBetaHeadersToContext(ctx, &probe, cfg.Provider) + probe.Model = capModel + + _ = AddMissingBetaHeadersToContext(ctx, &probe, cfg.Provider) } for _, field := range cfg.ExcludeFields { diff --git a/core/providers/anthropic/requestbuilder_test.go b/core/providers/anthropic/requestbuilder_test.go index 0664e124514..91a19a721fa 100644 --- a/core/providers/anthropic/requestbuilder_test.go +++ b/core/providers/anthropic/requestbuilder_test.go @@ -24,6 +24,53 @@ func makeSimpleInput(text string) []schemas.ResponsesMessage { } } +func TestSafeguardsRequestBuilders(t *testing.T) { + const beta = "dangerous-tool-use-2026-09-03" + for _, provider := range []schemas.ModelProvider{schemas.Anthropic, schemas.Bedrock, schemas.BedrockMantle, schemas.Vertex, schemas.Azure} { + for _, raw := range []bool{false, true} { + for _, chat := range []bool{false, true} { + for _, streaming := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/raw=%v/chat=%v/stream=%v", provider, raw, chat, streaming), func(t *testing.T) { + ctx := schemas.NewBifrostContext(context.Background(), time.Time{}) + ctx.SetValue(schemas.BifrostContextKeyUseRawRequestBody, raw) + payload := json.RawMessage(`{"z":1,"a":{"b":true}}`) + extra := map[string]interface{}{"safeguards": payload} + body := []byte(`{"model":"claude-opus-4-8","max_tokens":32,"messages":[{"role":"user","content":"hi"}],"safeguards":{"z":1,"a":{"b":true}}}`) + cfg := AnthropicRequestBuildConfig{Provider: provider, Model: "claude-opus-4-8", IsStreaming: streaming} + var out []byte + var err *schemas.BifrostError + if chat { + out, err = BuildAnthropicChatRequestBody(ctx, &schemas.BifrostChatRequest{Provider: provider, Model: "claude-opus-4-8", RawRequestBody: body, Input: []schemas.ChatMessage{{Role: schemas.ChatMessageRoleUser, Content: &schemas.ChatMessageContent{ContentStr: schemas.Ptr("hi")}}}, Params: &schemas.ChatParameters{ExtraParams: extra}}, cfg) + } else { + out, err = BuildAnthropicResponsesRequestBody(ctx, &schemas.BifrostResponsesRequest{Provider: provider, Model: "claude-opus-4-8", RawRequestBody: body, Input: makeSimpleInput("hi"), Params: &schemas.ResponsesParameters{ExtraParams: extra}}, cfg) + } + if err != nil { + t.Fatalf("build: %v", err) + } + if got := providerUtils.GetJSONField(out, "safeguards").Raw; got != string(payload) { + t.Errorf("safeguards = %s; body=%s", got, out) + } + if _, ok := extra["safeguards"]; !ok { + t.Error("conversion consumed safeguards from the input used by fallbacks") + } + betas := FilterBetaHeadersForProvider(MergeBetaHeaders(ctx, nil), provider) + if !slices.Contains(betas, beta) { + t.Errorf("missing required beta: %v", betas) + } + if provider == schemas.Bedrock || provider == schemas.Vertex { + if !strings.Contains(providerUtils.GetJSONField(out, "anthropic_beta").Raw, beta) { + t.Errorf("missing body beta: %s", out) + } + } else if providerUtils.JSONFieldExists(out, "anthropic_beta") { + t.Errorf("unexpected body beta: %s", out) + } + }) + } + } + } + } +} + func TestBuildAnthropicResponsesRequestBody_RawBodyPath(t *testing.T) { t.Run("anthropic_native_uses_resolved_model", func(t *testing.T) { // request.Model is always the alias-resolved value by the time the provider @@ -1040,3 +1087,79 @@ func TestBuildAnthropicResponsesRequestBody_BedrockInvokeKeepsToolSearch(t *test t.Errorf("anthropic_beta = %v, want it to contain %q", betas, AnthropicToolSearchBetaHeader) } } + +// TestRawBodyBuilderKeepsToolsAndSetsBetaHeaders checks the thing that actually goes +// upstream, rather than any one step of building it. +// +// The beta probe strips input_schema and description from a local copy, and +// TestBetaProbeNeverMutatesTheOutboundBody proves that copy never touches the caller's +// bytes. But the builder does a great deal more to the body after that — strips thinking +// blocks, remaps tool versions, deletes fields, injects anthropic_version. This asserts +// the end of that pipeline: the body it returns still carries every tool intact, and the +// context carries the beta headers those tools imply. +// +// Put plainly: the final request gets all the tools AND all the headers. +func TestRawBodyBuilderKeepsToolsAndSetsBetaHeaders(t *testing.T) { + rawBody := []byte(`{"model":"claude-opus-4-8","max_tokens":1024,` + + `"tools":[` + + `{"type":"custom","name":"lookup","description":"Look something up",` + + `"input_schema":{"type":"object","properties":{"q":{"type":"string"}},"required":["q"]},"strict":true},` + + `{"type":"computer_20250124","name":"computer","description":"Use the computer",` + + `"input_schema":{"type":"object"}}` + + `],"messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}]}`) + + ctx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) + ctx.SetValue(schemas.BifrostContextKeyUseRawRequestBody, true) + + out, bErr := BuildAnthropicResponsesRequestBody(ctx, &schemas.BifrostResponsesRequest{ + Provider: schemas.Anthropic, + Model: "claude-opus-4-8", + RawRequestBody: rawBody, + }, AnthropicRequestBuildConfig{Provider: schemas.Anthropic}) + if bErr != nil { + t.Fatalf("building request body: %v", bErr) + } + + // 1. Every tool survives, with the fields the probe strips from its own copy. + tools := providerUtils.GetJSONField(out, "tools") + if !tools.IsArray() || len(tools.Array()) != 2 { + t.Fatalf("outbound body lost tools: %s", out) + } + for i, want := range []struct{ name, description string }{ + {"lookup", "Look something up"}, + {"computer", "Use the computer"}, + } { + base := fmt.Sprintf("tools.%d", i) + if got := providerUtils.GetJSONField(out, base+".name").String(); got != want.name { + t.Errorf("%s.name = %q, want %q", base, got, want.name) + } + if got := providerUtils.GetJSONField(out, base+".description").String(); got != want.description { + t.Errorf("%s.description = %q, want %q (the probe's strip reached the wire)", base, got, want.description) + } + if !providerUtils.JSONFieldExists(out, base+".input_schema") { + t.Errorf("%s.input_schema is missing from the outbound body", base) + } + } + // The nested schema must be byte-intact, not merely present. + if got := providerUtils.GetJSONField(out, "tools.0.input_schema.properties.q.type").String(); got != "string" { + t.Errorf("nested schema altered: tools.0.input_schema.properties.q.type = %q, want \"string\"", got) + } + if got := providerUtils.GetJSONField(out, "tools.0.input_schema.required.0").String(); got != "q" { + t.Errorf("nested schema altered: tools.0.input_schema.required[0] = %q, want \"q\"", got) + } + + // 2. The beta headers those tools imply are on the context, ready for the request. + extra, ok := ctx.Value(schemas.BifrostContextKeyExtraHeaders).(map[string][]string) + if !ok { + t.Fatal("no extra headers on the context; the beta probe did not run") + } + got := extra[AnthropicBetaHeader] + for _, want := range []string{ + AnthropicStructuredOutputsBetaHeader, // from tools.0.strict + AnthropicComputerUseBetaHeader20250124, // from tools.1.type + } { + if !slices.Contains(got, want) { + t.Errorf("beta header %q missing from the outbound request; got %v", want, got) + } + } +} diff --git a/core/providers/anthropic/responses.go b/core/providers/anthropic/responses.go index c359a97d4e8..13e680cadb8 100644 --- a/core/providers/anthropic/responses.go +++ b/core/providers/anthropic/responses.go @@ -870,6 +870,12 @@ func (state *AnthropicResponsesStreamState) getOrCreateOutputIndex(contentIndex // Returns a slice of responses to support cases where a single event produces multiple responses func (chunk *AnthropicStreamEvent) ToBifrostResponsesStream(ctx context.Context, sequenceNumber int, state *AnthropicResponsesStreamState) ([]*schemas.BifrostResponsesStreamResponse, *schemas.BifrostError, bool) { switch chunk.Type { + case AnthropicStreamEventTypeSafeguardsUpdate: + return []*schemas.BifrostResponsesStreamResponse{{ + Type: schemas.ResponsesStreamResponseTypeSafeguardsUpdate, + SequenceNumber: sequenceNumber, + SafeguardResults: chunk.SafeguardResults, + }}, nil, false case AnthropicStreamEventTypeMessageStart: // Message start - emit response.created and response.in_progress (OpenAI-style lifecycle) if chunk.Message != nil { @@ -895,6 +901,11 @@ func (chunk *AnthropicStreamEvent) ToBifrostResponsesStream(ctx context.Context, if chunk.Message.Diagnostics != nil { response.Diagnostics = chunk.Message.Diagnostics } + // Forward safeguard_results nested in message_start.message (Claude + // Code auto-mode classifier) so the egress can rebuild it. + if len(chunk.Message.SafeguardResults) > 0 { + response.SafeguardResults = chunk.Message.SafeguardResults + } // Forward input usage from message_start so clients see cache metrics early if chunk.Message.Usage != nil { response.Usage = &schemas.ResponsesResponseUsage{ @@ -1525,9 +1536,10 @@ func (chunk *AnthropicStreamEvent) ToBifrostResponsesStream(ctx context.Context, Type: schemas.Ptr(schemas.ResponsesMessageTypeFunctionCall), Status: &statusInProgress, ResponsesToolMessage: &schemas.ResponsesToolMessage{ - CallID: chunk.ContentBlock.ID, - Name: chunk.ContentBlock.Name, - Arguments: schemas.Ptr(""), // Arguments will be filled by deltas + CallID: chunk.ContentBlock.ID, + Name: chunk.ContentBlock.Name, + ToolsetName: chunk.ContentBlock.ToolsetName, + Arguments: schemas.Ptr(""), // Arguments will be filled by deltas }, } @@ -2888,6 +2900,12 @@ func (chunk *AnthropicStreamEvent) ToBifrostResponsesStream(ctx context.Context, // and a tool call opened as thinking both reached clients and aborted their turns. func ToAnthropicResponsesStreamResponse(ctx *schemas.BifrostContext, bifrostResp *schemas.BifrostResponsesStreamResponse) []*AnthropicStreamEvent { events := toAnthropicResponsesStreamEvents(ctx, bifrostResp) + // Restore safeguard_results (Claude Code auto-mode classifier) captured from + // the upstream event onto the first re-rendered frame, so re-rendering does + // not strip it. + if len(events) > 0 && events[0] != nil && bifrostResp != nil && len(bifrostResp.SafeguardResults) > 0 { + events[0].SafeguardResults = bifrostResp.SafeguardResults + } if len(events) == 0 || ctx == nil { return events } @@ -2948,6 +2966,8 @@ func toAnthropicResponsesStreamEvents(ctx *schemas.BifrostContext, bifrostResp * // Map ResponsesStreamResponse types to Anthropic stream events switch bifrostResp.Type { + case schemas.ResponsesStreamResponseTypeSafeguardsUpdate: + return []*AnthropicStreamEvent{{Type: AnthropicStreamEventTypeSafeguardsUpdate, SafeguardResults: bifrostResp.SafeguardResults}} case schemas.ResponsesStreamResponseTypeCreated: // Only convert response.created back to message_start (not response.in_progress to avoid duplicates) streamResp.Type = AnthropicStreamEventTypeMessageStart @@ -3008,6 +3028,11 @@ func toAnthropicResponsesStreamEvents(ctx *schemas.BifrostContext, bifrostResp * if bifrostResp.Response != nil && bifrostResp.Response.Diagnostics != nil { streamMessage.Diagnostics = bifrostResp.Response.Diagnostics } + // safeguard_results nested in message_start.message (Claude Code + // auto-mode classifier) — restore from the created chunk's carry. + if bifrostResp.Response != nil && len(bifrostResp.Response.SafeguardResults) > 0 { + streamMessage.SafeguardResults = bifrostResp.Response.SafeguardResults + } streamResp.Message = streamMessage } case schemas.ResponsesStreamResponseTypeInProgress: @@ -3250,6 +3275,7 @@ func toAnthropicResponsesStreamEvents(ctx *schemas.BifrostContext, bifrostResp * if bifrostResp.Item.ResponsesToolMessage != nil { contentBlock.ID = providerUtils.SanitizeAnthropicToolUseIDPtr(bifrostResp.Item.ResponsesToolMessage.CallID) contentBlock.Name = bifrostResp.Item.ResponsesToolMessage.Name + contentBlock.ToolsetName = bifrostResp.Item.ResponsesToolMessage.ToolsetName // Always start with empty input for streaming compatibility contentBlock.Input = json.RawMessage("{}") @@ -3828,6 +3854,13 @@ func toAnthropicResponsesStreamEvents(ctx *schemas.BifrostContext, bifrostResp * // Other code interpreter lifecycle events have no Anthropic equivalent. return nil + case schemas.ResponsesStreamResponseTypeProviderRawEvent: + // Raw-only carrier for an unmodeled provider event; it has no typed + // rendering. The integration's passthrough branch forwards the raw frame + // before this converter runs, so reaching here means the surface cannot + // use it — drop it. + return nil + case schemas.ResponsesStreamResponseTypePing: streamResp.Type = AnthropicStreamEventTypePing @@ -4051,6 +4084,11 @@ func (req *AnthropicMessageRequest) ToBifrostResponsesRequest(ctx *schemas.Bifro if req.Diagnostics != nil { params.ExtraParams["diagnostics"] = req.Diagnostics } + // Safeguards (Claude Code auto-mode classifier) — carried opaque so the + // Anthropic egress can restore it; stripped fail-closed for other providers. + if len(req.Safeguards) > 0 { + params.ExtraParams["safeguards"] = req.Safeguards + } if req.Container != nil { params.ExtraParams["container"] = req.Container } @@ -4500,6 +4538,21 @@ func ToAnthropicResponsesRequest(ctx *schemas.BifrostContext, bifrostReq *schema delete(anthropicReq.ExtraParams, "diagnostics") } } + if safeguardsRaw, exists := anthropicReq.ExtraParams["safeguards"]; exists { + // Always consume the provider-specific key; restore the opaque payload + // onto the typed field so the strip gate can apply per-provider support. + delete(anthropicReq.ExtraParams, "safeguards") + switch v := safeguardsRaw.(type) { + case json.RawMessage: + anthropicReq.Safeguards = v + case []byte: + anthropicReq.Safeguards = json.RawMessage(v) + default: + if data, err := providerUtils.MarshalSorted(v); err == nil { + anthropicReq.Safeguards = json.RawMessage(data) + } + } + } if containerRaw, exists := anthropicReq.ExtraParams["container"]; exists { parsed := false switch v := containerRaw.(type) { @@ -4958,6 +5011,12 @@ func (response *AnthropicMessageResponse) ToBifrostResponsesResponse(ctx *schema bifrostResp.Diagnostics = response.Diagnostics } + // Forward safeguard_results (Claude Code auto-mode classifier) opaquely so the + // Anthropic egress can restore it byte-for-byte. + if len(response.SafeguardResults) > 0 { + bifrostResp.SafeguardResults = response.SafeguardResults + } + return bifrostResp } @@ -5063,6 +5122,12 @@ func ToAnthropicResponsesResponse(ctx *schemas.BifrostContext, bifrostResp *sche anthropicResp.Diagnostics = bifrostResp.Diagnostics } + // Restore safeguard_results (Claude Code auto-mode classifier) forwarded + // opaquely from the provider response. + if len(bifrostResp.SafeguardResults) > 0 { + anthropicResp.SafeguardResults = bifrostResp.SafeguardResults + } + return anthropicResp } @@ -5152,6 +5217,13 @@ func ConvertBifrostMessagesToAnthropicMessages(ctx *schemas.BifrostContext, bifr // NOT be emitted as a tool_result, or Anthropic rejects the whole request. knownToolUseIDs := make(map[string]bool) + // toolsetNameByToolUseID carries each member tool_use's toolset_name so the + // matching tool_result can be tagged with it. Anthropic rejects a pair whose + // halves disagree, and a client speaking a dialect without the field (an + // OpenAI-shaped function_call_output, say) returns only the call id, so the + // value is restored here rather than trusted to survive the round trip. + toolsetNameByToolUseID := make(map[string]string) + // midConvPlacementOK reports whether a mid-conversation system message at input index i can // legally be forwarded as role:"system". Anthropic enforces two clauses and rejects a // violation of either with "messages.N: role 'system' must follow a 'user' message ...": @@ -5216,6 +5288,11 @@ func ConvertBifrostMessagesToAnthropicMessages(ctx *schemas.BifrostContext, bifr var orphaned []AnthropicContentBlock for _, block := range pendingToolResultBlocks { if block.ToolUseID != nil && knownToolUseIDs[*block.ToolUseID] { + if block.ToolsetName == nil { + if name, ok := toolsetNameByToolUseID[*block.ToolUseID]; ok { + block.ToolsetName = &name + } + } matched = append(matched, block) } else { orphaned = append(orphaned, block) @@ -5395,6 +5472,9 @@ func ConvertBifrostMessagesToAnthropicMessages(ctx *schemas.BifrostContext, bifr for _, b := range anthropicMsg.Content.ContentBlocks { if (b.Type == AnthropicContentBlockTypeToolUse || b.Type == AnthropicContentBlockTypeServerToolUse) && b.ID != nil { knownToolUseIDs[*b.ID] = true + if b.ToolsetName != nil { + toolsetNameByToolUseID[*b.ID] = *b.ToolsetName + } } } seenConversation = true @@ -5462,6 +5542,9 @@ func ConvertBifrostMessagesToAnthropicMessages(ctx *schemas.BifrostContext, bifr } if toolUseBlock.ID != nil { currentToolCallIDs[*toolUseBlock.ID] = true + if toolUseBlock.ToolsetName != nil { + toolsetNameByToolUseID[*toolUseBlock.ID] = *toolUseBlock.ToolsetName + } } // Use this message as the current one for subsequent tool calls pendingToolCalls = lastMsg.Content.ContentBlocks @@ -5480,6 +5563,9 @@ func ConvertBifrostMessagesToAnthropicMessages(ctx *schemas.BifrostContext, bifr } if toolUseBlock.ID != nil { currentToolCallIDs[*toolUseBlock.ID] = true + if toolUseBlock.ToolsetName != nil { + toolsetNameByToolUseID[*toolUseBlock.ID] = *toolUseBlock.ToolsetName + } } } @@ -6231,7 +6317,8 @@ func convertAnthropicContentBlocksToResponsesMessagesGrouped(contentBlocks []Ant Status: schemas.Ptr("completed"), CacheControl: block.CacheControl, ResponsesToolMessage: &schemas.ResponsesToolMessage{ - CallID: block.ToolUseID, + CallID: block.ToolUseID, + ToolsetName: block.ToolsetName, }, } // Initialize the nested struct before any writes @@ -6601,8 +6688,9 @@ func convertAnthropicContentBlocksToResponsesMessages(ctx *schemas.BifrostContex Status: schemas.Ptr("completed"), CacheControl: block.CacheControl, ResponsesToolMessage: &schemas.ResponsesToolMessage{ - CallID: block.ID, - Name: block.Name, + CallID: block.ID, + Name: block.Name, + ToolsetName: block.ToolsetName, }, } if isOutputMessage { @@ -6653,7 +6741,8 @@ func convertAnthropicContentBlocksToResponsesMessages(ctx *schemas.BifrostContex Status: schemas.Ptr("completed"), CacheControl: block.CacheControl, ResponsesToolMessage: &schemas.ResponsesToolMessage{ - CallID: block.ToolUseID, + CallID: block.ToolUseID, + ToolsetName: block.ToolsetName, }, } // Initialize the nested struct before any writes @@ -7163,6 +7252,7 @@ func convertBifrostFunctionCallToAnthropicToolUse(ctx *schemas.BifrostContext, m if msg.ResponsesToolMessage.Name != nil { toolUseBlock.Name = msg.ResponsesToolMessage.Name } + toolUseBlock.ToolsetName = msg.ResponsesToolMessage.ToolsetName // Parse arguments as JSON input if msg.ResponsesToolMessage.Arguments != nil && *msg.ResponsesToolMessage.Arguments != "" { @@ -7198,6 +7288,7 @@ func convertBifrostFunctionCallOutputToAnthropicToolResultBlock(msg *schemas.Res Type: AnthropicContentBlockTypeToolResult, ToolUseID: providerUtils.SanitizeAnthropicToolUseIDPtr(msg.ResponsesToolMessage.CallID), CacheControl: msg.CacheControl, + ToolsetName: msg.ResponsesToolMessage.ToolsetName, } if msg.ResponsesToolMessage.Output != nil { @@ -7205,11 +7296,15 @@ func convertBifrostFunctionCallOutputToAnthropicToolResultBlock(msg *schemas.Res } // Set is_error if there's an error message or the status indicates an error - if msg.ResponsesToolMessage.Error != nil && *msg.ResponsesToolMessage.Error != "" { + if toolError := msg.ResponsesToolMessage.Error; toolError.IsError() { toolResultBlock.IsError = schemas.Ptr(true) if toolResultBlock.Content == nil { + errText := toolError.Text() + if errText == "" { + errText = "tool call returned an error" + } toolResultBlock.Content = &AnthropicContent{ - ContentStr: msg.ResponsesToolMessage.Error, + ContentStr: &errText, } } } else if msg.Status != nil && *msg.Status == "incomplete" { @@ -7261,11 +7356,15 @@ func convertBifrostComputerCallOutputToAnthropicToolResultBlock(msg *schemas.Res } // Set is_error if there's an error message or the status indicates an error - if msg.ResponsesToolMessage.Error != nil && *msg.ResponsesToolMessage.Error != "" { + if toolError := msg.ResponsesToolMessage.Error; toolError.IsError() { toolResultBlock.IsError = schemas.Ptr(true) if toolResultBlock.Content == nil { + errText := toolError.Text() + if errText == "" { + errText = "tool call returned an error" + } toolResultBlock.Content = &AnthropicContent{ - ContentStr: msg.ResponsesToolMessage.Error, + ContentStr: &errText, } } } else if msg.Status != nil && *msg.Status == "incomplete" { @@ -7292,11 +7391,15 @@ func convertBifrostMCPCallOutputToAnthropicToolResultBlock(msg *schemas.Response } // Set is_error if there's an error message or the status indicates an error - if msg.ResponsesToolMessage.Error != nil && *msg.ResponsesToolMessage.Error != "" { + if toolError := msg.ResponsesToolMessage.Error; toolError.IsError() { toolResultBlock.IsError = schemas.Ptr(true) if toolResultBlock.Content == nil { + errText := toolError.Text() + if errText == "" { + errText = "tool call returned an error" + } toolResultBlock.Content = &AnthropicContent{ - ContentStr: msg.ResponsesToolMessage.Error, + ContentStr: &errText, } } } else if msg.Status != nil && *msg.Status == "incomplete" { @@ -8269,6 +8372,14 @@ func convertAnthropicToolToBifrost(tool *AnthropicTool) *schemas.ResponsesTool { } switch *tool.Type { + case AnthropicToolTypeComputerToolset20260801: + // The toolset carries no display geometry; the generation is re-derived + // from the target model on the way out, so nothing else needs keeping. + return &schemas.ResponsesTool{ + Type: schemas.ResponsesToolTypeComputerUsePreview, + ResponsesToolComputerUsePreview: &schemas.ResponsesToolComputerUsePreview{Environment: "browser"}, + } + case AnthropicToolTypeComputer20250124, AnthropicToolTypeComputer20251124: bifrostTool := &schemas.ResponsesTool{ Type: schemas.ResponsesToolTypeComputerUsePreview, @@ -8628,9 +8739,32 @@ func convertBifrostToolToAnthropic(caps schemas.ModelCaps, tool *schemas.Respons case schemas.ResponsesToolTypeComputerUsePreview: if tool.ResponsesToolComputerUsePreview != nil { computerToolType := AnthropicToolTypeComputer20250124 - if ComputerUseGeneration(caps) == ComputerUseGen20251124 { + switch ComputerUseGeneration(caps) { + case ComputerUseGenToolset20260801: + // Bare entry: name and display_* are rejected on a toolset. + return &AnthropicTool{ + Type: schemas.Ptr(AnthropicToolTypeComputerToolset20260801), + CacheControl: tool.CacheControl, + } + case ComputerUseGen20251124: computerToolType = AnthropicToolTypeComputer20251124 } + // No geometry means this cannot become a dated tool: those validate + // display_*_px as >= 1. A toolset carries none by design, so when the + // target takes a toolset, send that — converting it into a dated tool + // the caller never asked for, or dropping a tool the model supports, + // both lose capability. Only when neither form is reachable is the tool + // dropped, the way any unsupported tool is. + if tool.ResponsesToolComputerUsePreview.DisplayWidth <= 0 || + tool.ResponsesToolComputerUsePreview.DisplayHeight <= 0 { + if AcceptsComputerToolset(caps) { + return &AnthropicTool{ + Type: schemas.Ptr(AnthropicToolTypeComputerToolset20260801), + CacheControl: tool.CacheControl, + } + } + return nil + } return &AnthropicTool{ Type: schemas.Ptr(computerToolType), Name: string(AnthropicToolNameComputer), diff --git a/core/providers/anthropic/responses_test.go b/core/providers/anthropic/responses_test.go index 08e1ccfb5ea..c183bf1f43c 100644 --- a/core/providers/anthropic/responses_test.go +++ b/core/providers/anthropic/responses_test.go @@ -2,9 +2,11 @@ package anthropic import ( "context" + "strings" "testing" "time" + "github.com/bytedance/sonic" "github.com/capsohq/bifrost/core/schemas" ) @@ -611,3 +613,58 @@ func TestToAnthropicResponsesResponse_IncompleteReportsTruncationStopReason(t *t }) } } + +// Claude Code auto-mode classifier: safeguards must survive the typed +// (non-passthrough) ingress→egress request conversion for Anthropic direct. +func TestAnthropicSafeguardsRequestRoundTrip(t *testing.T) { + ctx, cancel := schemas.NewBifrostContextWithCancel(context.Background()) + defer cancel() + + var req AnthropicMessageRequest + if err := sonic.Unmarshal([]byte(`{"model":"claude-opus-4-8","max_tokens":32,"messages":[{"role":"user","content":"hi"}],"safeguards":{"check":"auto_mode"}}`), &req); err != nil { + t.Fatalf("unmarshal request: %v", err) + } + + bifrostReq := req.ToBifrostResponsesRequest(ctx) + if bifrostReq == nil || bifrostReq.Params == nil { + t.Fatal("nil bifrost request from ingress") + } + + out, err := ToAnthropicResponsesRequest(ctx, bifrostReq) + if err != nil { + t.Fatalf("egress conversion: %v", err) + } + body, err := sonic.Marshal(out) + if err != nil { + t.Fatalf("marshal egress request: %v", err) + } + if want := `"safeguards":{"check":"auto_mode"}`; !strings.Contains(string(body), want) { + t.Fatalf("safeguards dropped on typed request round trip: %s", string(body)) + } +} + +// Claude Code auto-mode classifier: safeguard_results must survive the typed +// unary response conversion (provider decode → Bifrost → Anthropic client shape). +func TestAnthropicSafeguardResultsUnaryRoundTrip(t *testing.T) { + ctx, cancel := schemas.NewBifrostContextWithCancel(context.Background()) + defer cancel() + + var resp AnthropicMessageResponse + if err := sonic.Unmarshal([]byte(`{"id":"msg_1","type":"message","role":"assistant","model":"claude-opus-4-8","content":[{"type":"text","text":"ok"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":1,"output_tokens":1},"safeguard_results":[{"id":"sg_1","verdict":"allow"}]}`), &resp); err != nil { + t.Fatalf("unmarshal provider response: %v", err) + } + + bifrostResp := resp.ToBifrostResponsesResponse(ctx) + if bifrostResp == nil { + t.Fatal("nil bifrost response") + } + + out := ToAnthropicResponsesResponse(ctx, bifrostResp) + body, err := sonic.Marshal(out) + if err != nil { + t.Fatalf("marshal client response: %v", err) + } + if want := `"safeguard_results":[{"id":"sg_1","verdict":"allow"}]`; !strings.Contains(string(body), want) { + t.Fatalf("safeguard_results dropped on typed unary round trip: %s", string(body)) + } +} diff --git a/core/providers/anthropic/toolresultiserror_test.go b/core/providers/anthropic/toolresultiserror_test.go index cf7b68a0c79..ee373d480f4 100644 --- a/core/providers/anthropic/toolresultiserror_test.go +++ b/core/providers/anthropic/toolresultiserror_test.go @@ -79,3 +79,61 @@ func TestToolResultIsErrorReachesAnthropicWire(t *testing.T) { t.Fatalf("tool_result without IsError must omit is_error, got %v", *ok.IsError) } } + +func TestAnthropicToolResultBuildersPreserveStructuredErrors(t *testing.T) { + callID := "call_1" + emptyLegacy := "" + output := "provider output" + + builders := []struct { + name string + build func(*schemas.ResponsesMessage) *AnthropicContentBlock + }{ + {name: "function", build: convertBifrostFunctionCallOutputToAnthropicToolResultBlock}, + {name: "computer", build: convertBifrostComputerCallOutputToAnthropicToolResultBlock}, + {name: "mcp", build: convertBifrostMCPCallOutputToAnthropicToolResultBlock}, + } + + for _, builder := range builders { + t.Run(builder.name+" structured error", func(t *testing.T) { + block := builder.build(&schemas.ResponsesMessage{ResponsesToolMessage: &schemas.ResponsesToolMessage{ + CallID: &callID, + Error: &schemas.ResponsesToolMessageError{ + ResponsesToolMessageErrorStruct: &schemas.ResponsesToolMessageErrorStruct{}, + }, + Output: &schemas.ResponsesToolMessageOutputStruct{ResponsesToolCallOutputStr: &output}, + }}) + if block == nil || block.IsError == nil || !*block.IsError { + t.Fatalf("structured error was not preserved: %#v", block) + } + if block.Content == nil || block.Content.ContentStr == nil || *block.Content.ContentStr != output { + t.Fatalf("existing output was not preserved: %#v", block.Content) + } + }) + + t.Run(builder.name+" empty structured error fallback", func(t *testing.T) { + block := builder.build(&schemas.ResponsesMessage{ResponsesToolMessage: &schemas.ResponsesToolMessage{ + CallID: &callID, + Error: &schemas.ResponsesToolMessageError{ + ResponsesToolMessageErrorStruct: &schemas.ResponsesToolMessageErrorStruct{}, + }, + }}) + if block == nil || block.Content == nil || block.Content.ContentStr == nil || *block.Content.ContentStr != "tool call returned an error" { + t.Fatalf("missing structured error fallback: %#v", block) + } + }) + + t.Run(builder.name+" empty legacy error", func(t *testing.T) { + block := builder.build(&schemas.ResponsesMessage{ResponsesToolMessage: &schemas.ResponsesToolMessage{ + CallID: &callID, + Error: &schemas.ResponsesToolMessageError{ResponsesToolMessageErrorStr: &emptyLegacy}, + }}) + if block == nil { + t.Fatal("builder returned nil") + } + if block.IsError != nil { + t.Fatalf("empty legacy error changed behavior: %#v", block.IsError) + } + }) + } +} diff --git a/core/providers/anthropic/types.go b/core/providers/anthropic/types.go index e3ed247e9c3..a95bdc44b69 100644 --- a/core/providers/anthropic/types.go +++ b/core/providers/anthropic/types.go @@ -21,6 +21,12 @@ const ( // AnthropicBetaHeader is the HTTP header name used to enable Anthropic beta features. AnthropicBetaHeader = "anthropic-beta" + // AnthropicDangerousToolUseBetaHeader activates safeguards evaluation. + AnthropicDangerousToolUseBetaHeader = "dangerous-tool-use-2026-09-03" + AnthropicDangerousToolUseBetaHeaderPrefix = "dangerous-tool-use-" + // AnthropicAutoModeClassifierBetaHeaderPrefix rides on Claude Code's auto-mode + // classifier follow-up requests; sibling of dangerous-tool-use, same feature gate. + AnthropicAutoModeClassifierBetaHeaderPrefix = "auto-mode-classifier-" // Beta headers for various Anthropic features // AnthropicFilesAPIBetaHeader is the required beta header for the Files API. @@ -195,6 +201,7 @@ type ProviderFeatureSupport struct { Diagnostics bool // diagnostics request field — cache diagnostics (cache-diagnosis-2026-04-07 beta, diagnostics.previous_message_id). Claude API only per docs ("not supported on Amazon Bedrock or Vertex AI"); stripped elsewhere fail-closed. Azure rejects it. ServerSideFallback bool // native "fallbacks" request field — server-side-fallback-2026-06-01. Claude API only per docs ("not available on Amazon Bedrock, Google Cloud, or Microsoft Foundry"). FallbackCredit bool // fallback_credit_token request field + stop_details credit fields — fallback-credit-2026-06-01 (AWS surfaces: -2026-06-09). Documented on the Claude API, Amazon Bedrock, Google Cloud and Microsoft Foundry, i.e. the inverse of ServerSideFallback. + Safeguards bool // Opaque Claude auto-mode classifier payloads; supported models require the dangerous-tool-use beta. MidConvToolChanges bool // tool_addition/tool_removal blocks — mid-conversation-tool-changes-2026-07-01. Native Anthropic surface (Claude API + Bedrock Mantle); Bedrock is Opus 5 only, enforced upstream. } @@ -217,6 +224,7 @@ var ProviderFeatures = map[schemas.ModelProvider]ProviderFeatureSupport{ Diagnostics: true, // cache-diagnosis-2026-04-07 — Claude API only; only this provider keeps diagnostics.previous_message_id. ServerSideFallback: true, // server-side-fallback-2026-06-01 — Claude API only. FallbackCredit: true, // fallback-credit-2026-06-01. + Safeguards: true, // Claude Code auto-mode classifier; model-gated with the required beta. MidConvToolChanges: true, // mid-conversation-tool-changes-2026-07-01. }, // Google Vertex AI — cite: A (overview table) and V-platform. @@ -247,6 +255,7 @@ var ProviderFeatures = map[schemas.ModelProvider]ProviderFeatureSupport{ Context1M: true, EagerInputStreaming: true, // fine-grained-tool-streaming GA per A FallbackCredit: true, // fallback credit is documented on Google Cloud + Safeguards: true, // Claude Code auto-mode classifier — Vertex shares Anthropic's native request shape via BuildAnthropicResponsesRequestBody, so this is model-gated the same way as Anthropic direct's own model list (Sonnet 5 / Opus 4.7+ / Fable) via SupportsSafeguards. }, // AWS Bedrock — cite: A + B-header (definitive beta-header list). // Notably NOT supported per docs: MCP, Skills, FilesAPI, WebFetch, @@ -275,6 +284,7 @@ var ProviderFeatures = map[schemas.ModelProvider]ProviderFeatureSupport{ // narrow tool-examples-2025-10-29 header is, gated via InputExamples above. ServiceTier: true, // Bedrock handles service_tier via its own typed conversion FallbackCredit: true, // fallback-credit-2026-06-09 (AWS date) per the Bedrock userguide + Safeguards: true, // Claude Code auto-mode classifier, model-gated via SupportsSafeguards (Sonnet 5 / Opus 4.7+ / Fable). Converse has no slot for this undocumented Anthropic-native field, so responsesUsesAnthropicInvokePath (bedrock.go) routes a safeguards-bearing request to InvokeModel — same pattern as Compaction/ToolSearch above (#6825) — which reuses this package's builder end to end and this flag applies there. }, // Bedrock Mantle — same AWS-hosted Claude models as Bedrock, reached through // the native Anthropic Messages surface (/anthropic/v1/messages) instead of @@ -317,6 +327,7 @@ var ProviderFeatures = map[schemas.ModelProvider]ProviderFeatureSupport{ ServiceTier: true, FallbackCredit: true, // fallback-credit-2026-06-09 (AWS date) per the Bedrock userguide MidConvToolChanges: true, // mid-conversation-tool-changes-2026-07-01 — Opus 5 on Bedrock, enforced upstream. + Safeguards: true, }, // Microsoft Azure AI Foundry — cite: A (most features azureAiBeta) + // Az-platform ("supports most of Claude's features"). Excluded per @@ -335,6 +346,7 @@ var ProviderFeatures = map[schemas.ModelProvider]ProviderFeatureSupport{ // FastMode, InferenceGeo, AdvisorTool, TaskBudgets — not documented on Az-platform; leave off. ServiceTier: true, FallbackCredit: true, // fallback credit is documented on Microsoft Foundry + Safeguards: true, // Claude Code auto-mode classifier — Azure shares Anthropic's native request shape via BuildAnthropicResponsesRequestBody, model-gated via SupportsSafeguards (Sonnet 5 / Opus 4.7+ / Fable), same as Vertex above. }, schemas.DeepSeek: { WebSearch: true, @@ -549,6 +561,11 @@ type AnthropicMessageRequest struct { // rejected on count_tokens. FallbackCreditToken *string `json:"fallback_credit_token,omitempty"` + // Safeguards carries Claude Code's auto-mode server-side classifier request + // (opaque shape, undocumented; see the Claude Code gateway compatibility guide's + // feature pass-through section). Claude API only; stripped elsewhere fail-closed. + Safeguards json.RawMessage `json:"safeguards,omitempty"` + // Extra params for advanced use cases ExtraParams map[string]interface{} `json:"-"` @@ -961,6 +978,7 @@ var anthropicMessageRequestKnownFields = map[string]bool{ "container": true, "diagnostics": true, "fallback_credit_token": true, + "safeguards": true, "extra_params": true, "fallbacks": true, } @@ -997,6 +1015,12 @@ func (req *AnthropicMessageRequest) UnmarshalJSON(data []byte) error { req.OutputConfig.Format = json.RawMessage(buf.Bytes()) } } + if len(req.Safeguards) > 0 { + var buf bytes.Buffer + if err := json.Compact(&buf, req.Safeguards); err == nil { + req.Safeguards = json.RawMessage(buf.Bytes()) + } + } return nil } @@ -1278,6 +1302,11 @@ type AnthropicContentBlock struct { ErrorCode *string `json:"error_code,omitempty"` // any *_tool_result_error variant StopReason *string `json:"stop_reason,omitempty"` // advisor_result / advisor_redacted_result inner block; present when advisor tool max_tokens is set Caller *AnthropicToolCaller `json:"caller,omitempty"` // tool_use, server_tool_use, every *_tool_result block + // ToolsetName names the client toolset a member call belongs to ("computer"). + // It must appear on both halves of a pair or neither: a tool_result answering + // a member tool_use without it, or carrying it when the tool_use does not, is + // a 400. Set on tool_use and tool_result only. + ToolsetName *string `json:"toolset_name,omitempty"` // search_result block: the API uses the literal key "source" with a plain // string value, which collides with the existing Source *AnthropicSource @@ -1595,6 +1624,12 @@ const ( AnthropicToolTypeTextEditor20250429 AnthropicToolType = "text_editor_20250429" AnthropicToolTypeTextEditor20250728 AnthropicToolType = "text_editor_20250728" + // Client toolsets. A toolset entry carries no name and no display_* fields — + // its members are fixed by the dated type — and every call comes back under a + // member name ("screenshot", "left_click") tagged with toolset_name. + AnthropicToolTypeComputerToolset20260801 AnthropicToolType = "computer_toolset_20260801" + AnthropicToolTypeBrowserToolset20260801 AnthropicToolType = "browser_toolset_20260801" + // Code execution AnthropicToolTypeCodeExecution20250522 AnthropicToolType = "code_execution_20250522" // Legacy Python-only AnthropicToolTypeCodeExecution AnthropicToolType = "code_execution_20250825" @@ -1711,7 +1746,9 @@ type AnthropicToolInputExample struct { // AnthropicTool represents a tool in Anthropic format type AnthropicTool struct { - Name string `json:"name"` + // Name is omitted when empty: a toolset entry rejects it outright ("name is + // not accepted on a toolset entry"), and every other tool always sets one. + Name string `json:"name,omitempty"` Type *AnthropicToolType `json:"type,omitempty"` Description *string `json:"description,omitempty"` InputSchema *schemas.ToolFunctionParameters `json:"input_schema,omitempty"` @@ -1955,6 +1992,10 @@ type AnthropicMessageResponse struct { // omitempty when absent; a present-but-null value (no divergence) is conveyed by a // non-nil pointer with a nil CacheMissReason — see schemas.CacheDiagnostics. Diagnostics *schemas.CacheDiagnostics `json:"diagnostics,omitempty"` + // SafeguardResults carries the Claude Code auto-mode server-side classifier + // verdicts (opaque, undocumented shape; the gateway compatibility guide requires + // forwarding it unchanged). Present only when the request carried safeguards. + SafeguardResults json.RawMessage `json:"safeguard_results,omitempty"` // ExtraFields carries Bifrost's own response metadata (raw_request, raw_response, // routing info, latency) on this route, mirroring the extra_fields member that @@ -2136,6 +2177,7 @@ const ( AnthropicStreamEventTypeContentBlockStop AnthropicStreamEventType = "content_block_stop" AnthropicStreamEventTypeMessageDelta AnthropicStreamEventType = "message_delta" AnthropicStreamEventTypePing AnthropicStreamEventType = "ping" + AnthropicStreamEventTypeSafeguardsUpdate AnthropicStreamEventType = "safeguards_update" AnthropicStreamEventTypeError AnthropicStreamEventType = "error" ) @@ -2149,6 +2191,11 @@ type AnthropicStreamEvent struct { Delta *AnthropicStreamDelta `json:"delta,omitempty"` Usage *AnthropicUsage `json:"usage,omitempty"` Error *AnthropicStreamError `json:"error,omitempty"` + + // SafeguardResults carries the Claude Code auto-mode server-side classifier + // verdicts on a stream event (opaque, undocumented shape; the gateway + // compatibility guide requires forwarding it unchanged). + SafeguardResults json.RawMessage `json:"safeguard_results,omitempty"` } type AnthropicStreamDeltaType string diff --git a/core/providers/anthropic/utils.go b/core/providers/anthropic/utils.go index 04a15121634..fbc79089427 100644 --- a/core/providers/anthropic/utils.go +++ b/core/providers/anthropic/utils.go @@ -306,7 +306,7 @@ func stripUnsupportedAnthropicFields(req *AnthropicMessageRequest, provider sche if req.OutputConfig != nil { effort = req.OutputConfig.Effort } - if RejectsDisabledThinking(model, effort) { + if RejectsDisabledThinking(caps, effort) { req.Thinking.Type = "adaptive" req.Thinking.BudgetTokens = nil } @@ -321,6 +321,16 @@ func stripUnsupportedAnthropicFields(req *AnthropicMessageRequest, provider sche if req.Diagnostics != nil && !features.Diagnostics { req.Diagnostics = nil } + // Safeguards (Claude Code auto-mode classifier) is gated on the + // (provider, model) pair: the provider must carry the feature at all, and + // the pair must be one that accepts the field on the wire. Both halves are + // datasheet-overridable via SupportsSafeguards, so enabling a surface that + // starts accepting it is a catalog change. Fail-closed otherwise: a surface + // that refuses the field 400s the caller's real request. + if len(req.Safeguards) > 0 && + (!features.Safeguards || !caps.SupportsSafeguards(DefaultSupportsSafeguards(provider, caps.Model()))) { + req.Safeguards = nil + } // cache_control.scope — strip on providers without PromptCachingScope // support at every slot scope can live: top-level request, tools, system // blocks, and message content blocks. Vertex additionally uses the @@ -499,6 +509,19 @@ func StripUnsupportedFieldsFromRawBody(jsonBody []byte, provider schemas.ModelPr } } + // safeguards — undocumented Claude Code auto-mode classifier field, gated on + // the (provider, model) pair exactly as the typed path above: the provider + // must carry the feature and the pair must accept the field on the wire, + // both datasheet-overridable via SupportsSafeguards. Fail-closed otherwise. + if (!features.Safeguards || + !caps.SupportsSafeguards(DefaultSupportsSafeguards(provider, caps.Model()))) && + providerUtils.JSONFieldExists(jsonBody, "safeguards") { + jsonBody, err = providerUtils.DeleteJSONField(jsonBody, "safeguards") + if err != nil { + return nil, fmt.Errorf("strip raw safeguards: %w", err) + } + } + // speed — provider AND model gate if providerUtils.JSONFieldExists(jsonBody, "speed") { if !features.FastMode || !caps.SupportsFastMode(DefaultSupportsFastMode(caps.Model())) { @@ -681,7 +704,7 @@ func StripUnsupportedFieldsFromRawBody(jsonBody []byte, provider schemas.ModelPr if e := providerUtils.GetJSONField(jsonBody, "output_config.effort"); e.Exists() { effort = new(e.String()) } - if RejectsDisabledThinking(model, effort) { + if RejectsDisabledThinking(caps, effort) { jsonBody, err = providerUtils.SetJSONField(jsonBody, "thinking.type", "adaptive") if err != nil { return nil, fmt.Errorf("rewrite raw thinking.type to adaptive: %w", err) @@ -822,49 +845,66 @@ func StripUnsupportedFieldsFromRawBody(jsonBody []byte, provider schemas.ModelPr } } - // Nested scope on system blocks (system can be a string OR array of blocks). + // Nested scope on system blocks and on messages[].content[] blocks. + // + // Both rewrite each block's own JSON and write the enclosing array back once. + // Addressing through the whole body (system..cache_control.scope) would + // reserialise the entire request per block, making this O(blocks × body). + // Pinned by TestStripUnsupportedFieldsFromRawBody_AllocationScaling. if !features.PromptCachingScope { if systemResult := providerUtils.GetJSONField(jsonBody, "system"); systemResult.Exists() && systemResult.IsArray() { - for i := range systemResult.Array() { - path := fmt.Sprintf("system.%d.cache_control.scope", i) - if providerUtils.JSONFieldExists(jsonBody, path) { - jsonBody, err = providerUtils.DeleteJSONField(jsonBody, path) - if err != nil { - return nil, fmt.Errorf("strip raw system[%d].cache_control.scope: %w", i, err) - } - parentPath := fmt.Sprintf("system.%d.cache_control", i) - if ccResult := providerUtils.GetJSONField(jsonBody, parentPath); ccResult.Exists() && ccResult.IsObject() && len(ccResult.Map()) == 0 { - jsonBody, err = providerUtils.DeleteJSONField(jsonBody, parentPath) - if err != nil { - return nil, fmt.Errorf("strip raw system[%d].cache_control empty parent: %w", i, err) - } - } + blocks := systemResult.Array() + rebuilt := make([][]byte, len(blocks)) + changed := false + for i, block := range blocks { + stripped, didStrip, stripErr := stripCacheControlScope([]byte(block.Raw)) + if stripErr != nil { + return nil, fmt.Errorf("strip raw system[%d].cache_control.scope: %w", i, stripErr) + } + rebuilt[i] = stripped + changed = changed || didStrip + } + if changed { + jsonBody, err = providerUtils.SetRawJSONField(jsonBody, "system", rawJSONArrayOf(rebuilt)) + if err != nil { + return nil, fmt.Errorf("strip raw system cache_control.scope: %w", err) } } } - // Nested scope on messages[].content[] blocks. + if messagesResult := providerUtils.GetJSONField(jsonBody, "messages"); messagesResult.Exists() && messagesResult.IsArray() { messages := messagesResult.Array() - for mi := range messages { - contentResult := providerUtils.GetJSONField(jsonBody, fmt.Sprintf("messages.%d.content", mi)) + rebuiltMessages := make([][]byte, len(messages)) + anyMessageChanged := false + for mi, msg := range messages { + rebuiltMessages[mi] = []byte(msg.Raw) + contentResult := msg.Get("content") if !contentResult.Exists() || !contentResult.IsArray() { continue } - for ci := range contentResult.Array() { - path := fmt.Sprintf("messages.%d.content.%d.cache_control.scope", mi, ci) - if providerUtils.JSONFieldExists(jsonBody, path) { - jsonBody, err = providerUtils.DeleteJSONField(jsonBody, path) - if err != nil { - return nil, fmt.Errorf("strip raw messages[%d].content[%d].cache_control.scope: %w", mi, ci, err) - } - parentPath := fmt.Sprintf("messages.%d.content.%d.cache_control", mi, ci) - if ccResult := providerUtils.GetJSONField(jsonBody, parentPath); ccResult.Exists() && ccResult.IsObject() && len(ccResult.Map()) == 0 { - jsonBody, err = providerUtils.DeleteJSONField(jsonBody, parentPath) - if err != nil { - return nil, fmt.Errorf("strip raw messages[%d].content[%d].cache_control empty parent: %w", mi, ci, err) - } - } + blocks := contentResult.Array() + rebuiltBlocks := make([][]byte, len(blocks)) + messageChanged := false + for ci, block := range blocks { + stripped, didStrip, stripErr := stripCacheControlScope([]byte(block.Raw)) + if stripErr != nil { + return nil, fmt.Errorf("strip raw messages[%d].content[%d].cache_control.scope: %w", mi, ci, stripErr) + } + rebuiltBlocks[ci] = stripped + messageChanged = messageChanged || didStrip + } + if messageChanged { + rebuiltMessages[mi], err = providerUtils.SetRawJSONField(rebuiltMessages[mi], "content", rawJSONArrayOf(rebuiltBlocks)) + if err != nil { + return nil, fmt.Errorf("strip raw messages[%d].content cache_control.scope: %w", mi, err) } + anyMessageChanged = true + } + } + if anyMessageChanged { + jsonBody, err = providerUtils.SetRawJSONField(jsonBody, "messages", rawJSONArrayOf(rebuiltMessages)) + if err != nil { + return nil, fmt.Errorf("strip raw messages cache_control.scope: %w", err) } } } @@ -873,6 +913,26 @@ func StripUnsupportedFieldsFromRawBody(jsonBody []byte, provider schemas.ModelPr return jsonBody, nil } +// stripCacheControlScope removes cache_control.scope from one content block's JSON, +// dropping cache_control entirely when that leaves it empty. The bool reports whether +// anything was removed, so callers can skip writing an unchanged array back. +func stripCacheControlScope(block []byte) ([]byte, bool, error) { + if !providerUtils.JSONFieldExists(block, "cache_control.scope") { + return block, false, nil + } + out, err := providerUtils.DeleteJSONField(block, "cache_control.scope") + if err != nil { + return nil, false, err + } + if cc := providerUtils.GetJSONField(out, "cache_control"); cc.Exists() && cc.IsObject() && len(cc.Map()) == 0 { + out, err = providerUtils.DeleteJSONField(out, "cache_control") + if err != nil { + return nil, false, err + } + } + return out, true, nil +} + // IsOpus47Plus returns true if the model is Claude Opus 4.7 or later (currently 4.7, 4.8, and 5) where: // - Extended thinking (budget_tokens) is removed — only adaptive thinking is supported. // - temperature, top_p, and top_k are not supported (setting them returns a 400). @@ -966,18 +1026,22 @@ func RejectsEnabledThinking(caps schemas.ModelCaps) bool { // to adaptive mode when not specified; use "thinking.type.enabled" with // "budget_tokens" for extended thinking. // -// Fable 5, Mythos 5 and Mythos Preview are always-on and reject it outright. -// Opus 5 (and later) accept it only at effort "high" or below - pairing it with +// Fable 5, Mythos 5, Mythos Preview and Opus 5.5 are always-on and reject it +// outright. Opus 5 accepts it only at effort "high" or below - pairing it with // "xhigh" or "max" is rejected, and that is enforced per request, which is why // effort has to be passed in rather than inferred from the model alone. Opus // 4.7/4.8 and Sonnet 5 accept "disabled" at any effort. // -// effort is nil when the caller did not set output_config.effort; the default -// sits below "xhigh", so "disabled" is accepted. +// The unconditional half reads caps.CanDisableReasoning so the datasheet's +// supports_reasoning_disable drives this passthrough gate and the converter's +// (chat.go, responses.go) from one value; the effort carve-out stays per +// request. effort is nil when the caller did not set output_config.effort; the +// default sits below "xhigh", so "disabled" is accepted. // // Source: https://platform.claude.com/docs/en/build-with-claude/thinking-troubleshooting -func RejectsDisabledThinking(model string, effort *string) bool { - if IsFableFamily(model) { +func RejectsDisabledThinking(caps schemas.ModelCaps, effort *string) bool { + model := caps.Model() + if !caps.CanDisableReasoning(DefaultCanDisableReasoning(model)) { return true } if IsOpus5Plus(model) && effort != nil { @@ -1000,6 +1064,30 @@ func DefaultSupportsFastMode(model string) bool { (strings.Contains(m, "4-6") || strings.Contains(m, "4.6")) } +// DefaultSupportsSafeguards is the name-detection fallback for the Claude Code +// auto-mode server-side classifier (`safeguards` request field), gated on the +// (provider, model) pair. It answers "would this pair accept the field on the +// wire"; the datasheet's supports_safeguards boolean overrides it per pair, so +// enabling a surface later is a catalog change rather than a code change. +// +// Claude API direct accepts it on the models that support auto mode: Sonnet 5, +// Opus 4.7 or later (covering 4.8 and 5), and the Fable family. +// +// Claude cloud surfaces require dangerous-tool-use-2026-09-03 alongside the +// field. The shared request builder injects that beta after capability filtering. +// +// Sources: +// - https://code.claude.com/docs/en/auto-mode-classifier-billing +// - https://code.claude.com/docs/en/permission-modes#enable-auto-mode-on-bedrock-agent-platform-or-foundry +func DefaultSupportsSafeguards(provider schemas.ModelProvider, model string) bool { + switch provider { + case schemas.Anthropic, schemas.Bedrock, schemas.BedrockMantle, schemas.Vertex, schemas.Azure: + default: + return false + } + return IsSonnet5Plus(model) || IsOpus47Plus(model) || IsFableFamily(model) +} + // DefaultSupportsAdaptiveThinking: thinking.type "adaptive" is accepted on Opus // 4.6, Sonnet 4.6, Sonnet 5+, Opus 4.7+ and the Fable/Mythos family. func DefaultSupportsAdaptiveThinking(model string) bool { @@ -1019,11 +1107,11 @@ func DefaultAdaptiveOnlyThinking(model string) bool { return IsOpus47Plus(model) || IsSonnet5Plus(model) || IsFableFamily(model) } -// DefaultCanDisableReasoning: the Fable/Mythos family rejects +// DefaultCanDisableReasoning: the Fable/Mythos family and Opus 5.5 reject // thinking:{type:"disabled"} — adaptive thinking is always on, so the param must // be omitted entirely rather than sent as disabled. func DefaultCanDisableReasoning(model string) bool { - return !IsFableFamily(model) + return !IsFableFamily(model) && !schemas.IsOpus55Plus(model) } // SupportsNativeEffort reports whether the model takes output_config.effort as @@ -1279,13 +1367,16 @@ const ( ) // Computer-use tool generations. +// - "20260801" — the client toolset, the only form Opus 5.5 accepts // - "20251124" — Opus 4.8, Opus 4.7, Opus 4.6, Sonnet 5, Sonnet 4.6, Opus 4.5 // - "20250124" — everything else (Sonnet 4.5, Haiku 4.5, Opus 4.1, Sonnet 4, Opus 4, Sonnet 3.7) // -// The bash tool is generation-invariant (always bash_20250124). +// The bash tool is generation-invariant (always bash_20250124). The toolset +// generation has no text_editor member, so TextEditorGeneration never returns it. const ( - ComputerUseGen20251124 = "20251124" - ComputerUseGen20250124 = "20250124" + ComputerUseGenToolset20260801 = "20260801" + ComputerUseGen20251124 = "20251124" + ComputerUseGen20250124 = "20250124" ) // ComputerUseGeneration returns the tool-version generation a Claude model @@ -1294,16 +1385,29 @@ const ( // - Which computer_*/text_editor_* type the upstream API will accept. // - Which `name` literal Anthropic's Pydantic validator demands for text_editor. // -// Prefers the datasheet's server_tools["computer_use"] ("computer_20251124" → -// new gen, anything else → old gen), falling back to substring detection. +// Prefers the datasheet's server_tools["computer_use"], which names the exact +// tool type, falling back to substring detection. +// +// The fallback only moves a model to the toolset when the dated tools are +// actually rejected, which today is Opus 5.5 and only on the surfaces that +// dropped them (the Claude API and Google Cloud; AWS and Foundry still serve +// computer_20251124). Opus 4.8 through Fable 5.1 accept both forms, so they stay +// on computer_20251124 and a row moves them when their surface is ready. func ComputerUseGeneration(caps schemas.ModelCaps) string { if computerUse, ok := caps.ServerTool(ServerToolComputerUse); ok { - if computerUse == string(AnthropicToolTypeComputer20251124) { + switch computerUse { + case string(AnthropicToolTypeComputerToolset20260801): + return ComputerUseGenToolset20260801 + case string(AnthropicToolTypeComputer20251124): return ComputerUseGen20251124 + default: + return ComputerUseGen20250124 } - return ComputerUseGen20250124 } m := strings.ToLower(caps.Model()) + if schemas.IsOpus55Plus(m) && toolsetOnlyComputerUseProvider(caps.Provider()) { + return ComputerUseGenToolset20260801 + } // Opus 4.7+, Sonnet 5+, and the Fable/Mythos family use the new generation. if IsOpus47Plus(m) || IsSonnet5Plus(m) || IsFableFamily(m) { return ComputerUseGen20251124 @@ -1323,6 +1427,47 @@ func ComputerUseGeneration(caps schemas.ModelCaps) string { return ComputerUseGen20250124 } +// toolsetOnlyComputerUseProvider reports whether a surface has dropped the dated +// computer_* tools in favour of the toolset. AWS and Microsoft Foundry still +// serve computer_20251124 on the same models, so the split is per surface rather +// than per model. +// +// Source: https://platform.claude.com/docs/en/agents-and-tools/tool-use/computer-use-tool +func toolsetOnlyComputerUseProvider(provider schemas.ModelProvider) bool { + return provider == schemas.Anthropic || provider == schemas.Vertex +} + +// AcceptsComputerToolset reports whether the (provider, model) pair takes a +// computer_toolset_20260801 entry on the wire. Wider than ComputerUseGeneration, +// which answers what to *convert* to: every model here except Opus 5.5 also +// accepts the dated computer_* tools, so a caller that already sent the toolset +// must have it forwarded rather than converted into a form it never asked for. +// +// Override-aware: the datasheet's supports_computer_toolset decides when set, so +// a model gaining the toolset is a catalog change. The provider gate stays in +// code because the toolset is GA on the Claude API and Google Cloud only. +func AcceptsComputerToolset(caps schemas.ModelCaps) bool { + if !toolsetOnlyComputerUseProvider(caps.Provider()) { + return false + } + return caps.SupportsComputerToolset(DefaultSupportsComputerToolset(caps.Model())) +} + +// DefaultSupportsComputerToolset is the name-based fallback for +// ModelCaps.SupportsComputerToolset, listing the models the computer-use docs +// publish for the toolset: the Fable/Mythos family, Opus 5.5, Opus 5, Sonnet 5 +// and Opus 4.8. Opus 4.7 and older take the dated tool only. +// +// Source: https://platform.claude.com/docs/en/agents-and-tools/tool-use/computer-use-tool +func DefaultSupportsComputerToolset(model string) bool { + m := strings.ToLower(model) + if IsFableFamily(m) || schemas.IsOpus55Plus(m) || IsOpus5Plus(m) || IsSonnet5Plus(m) { + return true + } + return strings.Contains(m, "opus") && + (strings.Contains(m, "4-8") || strings.Contains(m, "4.8")) +} + // TextEditorGeneration returns the text_editor tool-version generation for a model. // Differs from ComputerUseGeneration because Anthropic's per-tool support matrix // is not always uniform - e.g., sonnet-4-5 supports old-gen computer_20250124 but @@ -1369,7 +1514,11 @@ func TextEditorGeneration(caps schemas.ModelCaps) string { func NormalizedToolSpec(generation, baseTool string) (toolType, toolName string) { switch baseTool { case "computer": - if generation == ComputerUseGen20251124 { + switch generation { + case ComputerUseGenToolset20260801: + // Toolsets take no name — the member names are fixed by the type. + return string(AnthropicToolTypeComputerToolset20260801), "" + case ComputerUseGen20251124: return string(AnthropicToolTypeComputer20251124), "computer" } return string(AnthropicToolTypeComputer20250124), "computer" @@ -1433,12 +1582,173 @@ func setEffortOnOutputConfig(req *AnthropicMessageRequest, effort string) { req.OutputConfig.Effort = &effort } +// anthropicMessageBetaSignals holds the only two things the beta-header gating reads out +// of the (potentially very large) messages array. Keeping them in one struct lets the raw +// JSON path compute them with a gjson scan instead of materialising every message, and +// means a future messages-derived signal has exactly one place to be added for both paths. +type anthropicMessageBetaSignals struct { + // --- derived from messages[].content[] --- + + // scopedCacheControl: some content block carries cache_control.scope. + scopedCacheControl bool + // fileSource: some content block has a source of type "file" (Files API). + fileSource bool +} + +// scanMessagesForBetaSignals derives the signals from decoded messages. +func scanMessagesForBetaSignals(messages []AnthropicMessage) anthropicMessageBetaSignals { + var signals anthropicMessageBetaSignals + for _, message := range messages { + if message.Content.ContentBlocks == nil { + continue + } + for _, block := range message.Content.ContentBlocks { + if block.CacheControl != nil && block.CacheControl.Scope != nil { + signals.scopedCacheControl = true + } + if block.Source != nil && block.Source.SourceObj != nil && block.Source.SourceObj.Type == "file" { + signals.fileSource = true + } + if signals.scopedCacheControl && signals.fileSource { + return signals + } + } + } + return signals +} + +// scanRawMessagesForBetaSignals derives the same signals straight from the raw request +// body. gjson walks the bytes without building an object graph, so a multi-megabyte +// conversation costs a scan rather than a full decode plus the structs to hold it. +func scanRawMessagesForBetaSignals(jsonBody []byte) anthropicMessageBetaSignals { + var signals anthropicMessageBetaSignals + messages := providerUtils.GetJSONField(jsonBody, "messages") + if !messages.Exists() || !messages.IsArray() { + return signals + } + for _, message := range messages.Array() { + content := message.Get("content") + if !content.Exists() || !content.IsArray() { + continue + } + for _, block := range content.Array() { + // Exists() is `Type != Null || len(Raw) != 0`, so a present-but-null scope + // reports true while typed decoding leaves the *string nil. Check the type + // too, otherwise an explicit `"scope": null` injects a prompt-caching-scope + // beta header the typed path never would. + if scope := block.Get("cache_control.scope"); scope.Exists() && scope.Type != gjson.Null { + signals.scopedCacheControl = true + } + if block.Get("source.type").String() == "file" { + signals.fileSource = true + } + if signals.scopedCacheControl && signals.fileSource { + return signals + } + } + } + return signals +} + +// betaProbeDroppedFields are the arrays AddMissingBetaHeadersToContextFromRawBody always +// removes from the body before decoding it, because they are the bulk of the probe's cost. +// +// Their contents reach the gating only as anthropicMessageBetaSignals. That makes this the +// contract addMissingBetaHeadersToContext must honour: it may read any other field of the +// decoded request, but never req.Messages, which is nil on the raw path. +// TestBetaGatingNeverReadsDroppedFields enforces it against the source, so the rule cannot +// be broken by someone who has not read this comment. +// +// tools is NOT in this list. Tools are kept and decoded; only the two fields that carry +// their bulk are stripped — see betaProbeStrippedToolFields. +var betaProbeDroppedFields = []string{"messages"} + +// betaProbeStrippedToolFields are the tool fields removed before the probe decode. +// +// This is deliberately the inverse of listing the fields the gating needs. Such a list is +// fail-OPEN: a beta header added later, gated on a field nobody added to it, is silently +// lost on the passthrough path. Measured, not assumed — that exact divergence is what +// TestRawPathWithCallerBetaHeadersMatchesTypedPath caught when tools were dropped wholesale. +// +// Listing what the gating provably never reads is fail-SAFE instead: a new tool field is +// kept by default, decoded, and derived from, so a forgotten entry costs memory rather +// than correctness. These two carry essentially all of a tools array's weight (a coding +// client's input_schema blocks run to tens of kilobytes each) while no beta header depends +// on either. TestBetaGatingNeverReadsStrippedToolFields enforces that. +var betaProbeStrippedToolFields = []string{"input_schema", "description"} + +// AddMissingBetaHeadersToContextFromRawBody is the raw-body equivalent of +// AddMissingBetaHeadersToContext, for passthrough requests whose body is already JSON. +// +// It exists to avoid decoding the entire request into an AnthropicMessageRequest purely +// to read a handful of feature flags. That probe decode was holding ~526 MB live in +// production, because the messages array (the bulk of an agentic conversation) expands +// several-fold into structs that are read once and thrown away. Here the messages array +// is dropped before decoding and its two relevant signals come from a gjson scan, so the +// decode only ever sees the small top-level fields. +func AddMissingBetaHeadersToContextFromRawBody(ctx *schemas.BifrostContext, jsonBody []byte, provider schemas.ModelProvider) error { + // messages is always dropped: scanRawMessagesForBetaSignals covers everything the + // gating reads from it. + trimmed := jsonBody + for _, field := range betaProbeDroppedFields { + updated, err := providerUtils.DeleteJSONField(trimmed, field) + if err != nil { + return err + } + trimmed = updated + } + + // Tools are kept so every gated field still reaches the decode; only their bulk is + // removed. Rebuilt in one pass rather than addressed per tool, since each whole-body + // sjson write would reserialise the request. + if tools := providerUtils.GetJSONField(trimmed, "tools"); tools.Exists() && tools.IsArray() { + var slimmed [][]byte + changed := false + for _, tool := range tools.Array() { + raw := []byte(tool.Raw) + for _, field := range betaProbeStrippedToolFields { + if !providerUtils.JSONFieldExists(raw, field) { + continue + } + updated, err := providerUtils.DeleteJSONField(raw, field) + if err != nil { + return err + } + raw, changed = updated, true + } + slimmed = append(slimmed, raw) + } + if changed { + updated, err := providerUtils.SetRawJSONField(trimmed, "tools", rawJSONArrayOf(slimmed)) + if err != nil { + return err + } + trimmed = updated + } + } + var req AnthropicMessageRequest + if err := schemas.Unmarshal(trimmed, &req); err != nil { + return err + } + return addMissingBetaHeadersToContext(ctx, &req, provider, scanRawMessagesForBetaSignals(jsonBody)) +} + // AddMissingBetaHeadersToContext analyzes the Anthropic request and adds missing beta headers to the context. // The provider parameter controls which headers are included — unsupported headers for the given provider are skipped. func AddMissingBetaHeadersToContext(ctx *schemas.BifrostContext, req *AnthropicMessageRequest, provider schemas.ModelProvider) error { + return addMissingBetaHeadersToContext(ctx, req, provider, scanMessagesForBetaSignals(req.Messages)) +} + +// addMissingBetaHeadersToContext holds the gating itself. It never touches req.Messages — +// the caller supplies those signals — so the typed and raw-body entry points above stay +// in lockstep by construction. +func addMissingBetaHeadersToContext(ctx *schemas.BifrostContext, req *AnthropicMessageRequest, provider schemas.ModelProvider, signals anthropicMessageBetaSignals) error { features, hasProvider := ProviderFeatures[provider] caps := schemas.ResolveModelCaps(provider, schemas.ResolveCanonicalModel(ctx, req.Model)) headers := []string{} + if len(req.Safeguards) > 0 && (!hasProvider || (features.Safeguards && caps.SupportsSafeguards(DefaultSupportsSafeguards(provider, caps.Model())))) { + headers = appendUniqueHeader(headers, AnthropicDangerousToolUseBetaHeader) + } hasCachingScope := false if req.Tools != nil { for _, tool := range req.Tools { @@ -1620,43 +1930,22 @@ func AddMissingBetaHeadersToContext(ctx *schemas.BifrostContext, req *AnthropicM } } } - // Check for cache control with scope in messages (only if not already found) - if !hasCachingScope { - for _, message := range req.Messages { - if message.Content.ContentBlocks != nil { - for _, block := range message.Content.ContentBlocks { - if block.CacheControl != nil && block.CacheControl.Scope != nil { - if !hasProvider || features.PromptCachingScope { - headers = appendUniqueHeader(headers, AnthropicPromptCachingScopeBetaHeader) - hasCachingScope = true - } - break - } - } - if hasCachingScope { - break - } - } + // Check for cache control with scope in messages (only if not already found). + // Sourced from signals so this works identically for a decoded request and for a + // raw JSON body; see anthropicMessageBetaSignals. + // Last of the cache-control scope checks, so nothing reads hasCachingScope after + // this: setting it here would be an ineffectual assignment (caught by ineffassign + // under `make lint`). A new check added below this one has to reinstate the write. + if !hasCachingScope && signals.scopedCacheControl { + if !hasProvider || features.PromptCachingScope { + headers = appendUniqueHeader(headers, AnthropicPromptCachingScopeBetaHeader) } } // Check for file_id references (document/image blocks with a "file" // source), which require the Files API beta header. - hasFileSource := false - for _, message := range req.Messages { - if hasFileSource { - break - } - if message.Content.ContentBlocks == nil { - continue - } - for _, block := range message.Content.ContentBlocks { - if block.Source != nil && block.Source.SourceObj != nil && block.Source.SourceObj.Type == "file" { - if !hasProvider || caps.SupportsFilesAPI(features.FilesAPI) { - headers = appendUniqueHeader(headers, AnthropicFilesAPIBetaHeader) - } - hasFileSource = true - break - } + if signals.fileSource { + if !hasProvider || caps.SupportsFilesAPI(features.FilesAPI) { + headers = appendUniqueHeader(headers, AnthropicFilesAPIBetaHeader) } } if len(headers) == 0 { @@ -1697,6 +1986,8 @@ func AddMissingBetaHeadersToContext(ctx *schemas.BifrostContext, req *AnthropicM // betaHeaderPrefixKnown maps known beta header prefixes for prefix-aware dedup. var betaHeaderPrefixKnown = []string{ + AnthropicDangerousToolUseBetaHeaderPrefix, + AnthropicAutoModeClassifierBetaHeaderPrefix, "computer-use-", AnthropicStructuredOutputsBetaHeaderPrefix, AnthropicMCPClientBetaHeaderPrefix, @@ -1893,35 +2184,108 @@ func doesWebSearchOrFetchAutoInjectCodeExecution(toolType string) bool { // The predicate must stay scoped to "thinking" blocks: "redacted_thinking" // blocks carry only an encrypted "data" payload (no thinking or signature // fields) and must be replayed to Anthropic untouched. +// Deleting blocks one at a time is quadratic: sjson reserialises the entire request +// body on every DeleteJSONField call, so stripping N blocks from an S-byte body costs +// N*S. Long agentic conversations carry hundreds of unsigned thinking blocks in a +// multi-megabyte body, which made this the single largest allocator in the gateway +// (23.7% of all bytes allocated, measured in production). Instead: scan once, then +// rewrite each affected message once and the messages array once. +// Pinned by TestStripEmptyThinkingBlocks_AllocationScaling. func StripEmptyThinkingBlocks(jsonBody []byte) ([]byte, error) { messagesResult := providerUtils.GetJSONField(jsonBody, "messages") if !messagesResult.Exists() || !messagesResult.IsArray() { return jsonBody, nil } - var err error - for mi, msg := range messagesResult.Array() { + messages := messagesResult.Array() + + // Pass 1: find the messages that need rewriting and the blocks each one keeps. + // Requests with nothing to strip are the common case and return the original + // bytes untouched, so they never pay for a copy. + keptByMessage := make(map[int][]gjson.Result) + for mi, msg := range messages { contentResult := msg.Get("content") if !contentResult.Exists() || !contentResult.IsArray() { continue } - var toStrip []int - for ci, block := range contentResult.Array() { + blocks := contentResult.Array() + stripped := false + kept := make([]gjson.Result, 0, len(blocks)) + for _, block := range blocks { if block.Get("type").String() == "thinking" && (block.Get("thinking").String() == "" || block.Get("signature").String() == "") { - toStrip = append(toStrip, ci) + stripped = true + continue } + kept = append(kept, block) } - for i := len(toStrip) - 1; i >= 0; i-- { - path := fmt.Sprintf("messages.%d.content.%d", mi, toStrip[i]) - jsonBody, err = providerUtils.DeleteJSONField(jsonBody, path) - if err != nil { - return nil, fmt.Errorf("failed to strip empty thinking block at %s: %w", path, err) - } + if stripped { + keptByMessage[mi] = kept + } + } + if len(keptByMessage) == 0 { + return jsonBody, nil + } + + // Pass 2: rebuild the messages array once. Untouched messages are copied verbatim + // from their raw slice, so field order (and therefore prompt-cache byte stability) + // is preserved exactly as the per-block delete preserved it. + var rebuilt bytes.Buffer + rebuilt.Grow(len(jsonBody)) + rebuilt.WriteByte('[') + for mi, msg := range messages { + if mi > 0 { + rebuilt.WriteByte(',') + } + kept, needsRewrite := keptByMessage[mi] + if !needsRewrite { + rebuilt.WriteString(msg.Raw) + continue } + rewritten, err := providerUtils.SetRawJSONField([]byte(msg.Raw), "content", rawJSONArray(kept)) + if err != nil { + return nil, fmt.Errorf("failed to strip empty thinking blocks from messages.%d: %w", mi, err) + } + rebuilt.Write(rewritten) + } + rebuilt.WriteByte(']') + + jsonBody, err := providerUtils.SetRawJSONField(jsonBody, "messages", rebuilt.Bytes()) + if err != nil { + return nil, fmt.Errorf("failed to strip empty thinking blocks: %w", err) } return jsonBody, nil } +// rawJSONArray concatenates the raw JSON of each result into a single array, without +// reparsing or re-encoding any of them. +func rawJSONArray(results []gjson.Result) []byte { + var buf bytes.Buffer + buf.WriteByte('[') + for i, r := range results { + if i > 0 { + buf.WriteByte(',') + } + buf.WriteString(r.Raw) + } + buf.WriteByte(']') + return buf.Bytes() +} + +// rawJSONArrayOf is rawJSONArray for values that have already been edited, so they +// are held as bytes rather than as gjson results pointing into the original body. +func rawJSONArrayOf(parts [][]byte) []byte { + var buf bytes.Buffer + buf.WriteByte('[') + for i, p := range parts { + if i > 0 { + buf.WriteByte(',') + } + buf.Write(p) + } + buf.WriteByte(']') + return buf.Bytes() +} + // StripAutoInjectableTools removes code_execution tools from the raw JSON body's tools array // when web_search or web_fetch tools are also present. The Anthropic API auto-injects // code_execution when web_search_20260209 or web_fetch_20260209 is included in the request, @@ -1972,16 +2336,25 @@ func StripAutoInjectableTools(jsonBody []byte) ([]byte, error) { return providerUtils.DeleteJSONField(jsonBody, "tools") } - // Delete in reverse order to preserve indices - var err error - for i := len(indicesToStrip) - 1; i >= 0; i-- { - path := fmt.Sprintf("tools.%d", indicesToStrip[i]) - jsonBody, err = providerUtils.DeleteJSONField(jsonBody, path) - if err != nil { - return nil, fmt.Errorf("failed to strip auto-injectable tool at index %d: %w", indicesToStrip[i], err) + // Rebuild the tools array in one pass. Deleting index by index would call + // sjson once per stripped tool, and each of those reserialises the entire + // request body, making this O(tools × body). + // Pinned by TestStripAutoInjectableTools_AllocationScaling. + strip := make(map[int]bool, len(indicesToStrip)) + for _, i := range indicesToStrip { + strip[i] = true + } + kept := make([]gjson.Result, 0, len(tools)-len(indicesToStrip)) + for i, tool := range tools { + if !strip[i] { + kept = append(kept, tool) } } + jsonBody, err := providerUtils.SetRawJSONField(jsonBody, "tools", rawJSONArray(kept)) + if err != nil { + return nil, fmt.Errorf("failed to strip auto-injectable tools: %w", err) + } return jsonBody, nil } @@ -2032,7 +2405,15 @@ func RemapRawToolVersionsForProvider(jsonBody []byte, provider schemas.ModelProv caps := schemas.ResolveModelCaps(provider, model) computerGeneration := ComputerUseGeneration(caps) textEditorGeneration := TextEditorGeneration(caps) + // Edits are applied to each tool's own JSON and the array is written back once. + // Writing tools..type through the whole body would reserialise the entire + // request per tool, making this O(tools × body). + // Pinned by TestRemapRawToolVersionsForProvider_AllocationScaling. + normalized := make([][]byte, len(tools)) + dropped := make([]bool, len(tools)) + anyNormalized := false for i, tool := range tools { + normalized[i] = []byte(tool.Raw) toolType := tool.Get("type").String() baseTool := computerUseBaseTool(toolType) if baseTool == "" { @@ -2046,20 +2427,67 @@ func RemapRawToolVersionsForProvider(jsonBody []byte, provider schemas.ModelProv if wantType == "" { continue } + // The caller already sent a form this pair accepts, so leave it alone. + // Normalizing it into the generation's dated tool would strip the toolset + // the model supports and lose its member-call shape. + if toolType == string(AnthropicToolTypeComputerToolset20260801) && AcceptsComputerToolset(caps) { + continue + } + // A dated computer tool validates display_*_px as >= 1 and requires a name; + // a toolset entry has neither, by design. Rewriting its type would emit an + // entry the API rejects outright, so drop it the way any unsupported tool is + // dropped. Scoped to the computer family: text_editor and bash carry no + // geometry and are valid without it. + if baseTool == "computer" && wantName != "" { + width := providerUtils.GetJSONField(normalized[i], "display_width_px") + height := providerUtils.GetJSONField(normalized[i], "display_height_px") + if !width.Exists() || !height.Exists() || width.Int() <= 0 || height.Int() <= 0 { + dropped[i] = true + anyNormalized = true + continue + } + } if toolType != wantType { - path := fmt.Sprintf("tools.%d.type", i) - jsonBody, err = providerUtils.SetJSONField(jsonBody, path, wantType) + normalized[i], err = providerUtils.SetJSONField(normalized[i], "type", wantType) if err != nil { return nil, fmt.Errorf("failed to normalize tool type: %w", err) } + anyNormalized = true + } + if wantName == "" { + // Toolset form: name and the display_* geometry are rejected outright, + // so a dated entry being upgraded has to shed them. + for _, field := range []string{"name", "display_width_px", "display_height_px", "display_number", "enable_zoom"} { + if !providerUtils.JSONFieldExists(normalized[i], field) { + continue + } + normalized[i], err = providerUtils.DeleteJSONField(normalized[i], field) + if err != nil { + return nil, fmt.Errorf("failed to strip %s for toolset: %w", field, err) + } + anyNormalized = true + } + continue } // Only set name if the tool has one (custom tools use input_schema; computer-use family always has a name). if existingName := tool.Get("name").String(); existingName != "" && existingName != wantName { - path := fmt.Sprintf("tools.%d.name", i) - jsonBody, err = providerUtils.SetJSONField(jsonBody, path, wantName) + normalized[i], err = providerUtils.SetJSONField(normalized[i], "name", wantName) if err != nil { return nil, fmt.Errorf("failed to normalize tool name: %w", err) } + anyNormalized = true + } + } + if anyNormalized { + kept := normalized[:0:0] + for i, tool := range normalized { + if !dropped[i] { + kept = append(kept, tool) + } + } + jsonBody, err = providerUtils.SetRawJSONField(jsonBody, "tools", rawJSONArrayOf(kept)) + if err != nil { + return nil, fmt.Errorf("failed to normalize tool versions: %w", err) } } @@ -2069,21 +2497,31 @@ func RemapRawToolVersionsForProvider(jsonBody []byte, provider schemas.ModelProv return jsonBody, nil } - // Re-fetch tools array since paths may have changed via SetJSONField above + // Re-fetch tools array since the normalization above may have rewritten it. + // Same single-write treatment: edit each tool's own JSON, then one array write. tools = providerUtils.GetJSONField(jsonBody, "tools").Array() + remapped := make([][]byte, len(tools)) + anyRemapped := false for i, tool := range tools { + remapped[i] = []byte(tool.Raw) toolType := tool.Get("type").String() for _, remap := range remaps { if toolType == remap.From { - path := fmt.Sprintf("tools.%d.type", i) - jsonBody, err = providerUtils.SetJSONField(jsonBody, path, remap.To) + remapped[i], err = providerUtils.SetJSONField(remapped[i], "type", remap.To) if err != nil { return nil, fmt.Errorf("failed to remap tool type: %w", err) } + anyRemapped = true break } } } + if anyRemapped { + jsonBody, err = providerUtils.SetRawJSONField(jsonBody, "tools", rawJSONArrayOf(remapped)) + if err != nil { + return nil, fmt.Errorf("failed to remap tool versions: %w", err) + } + } return jsonBody, nil } @@ -2091,6 +2529,8 @@ func RemapRawToolVersionsForProvider(jsonBody []byte, provider schemas.ModelProv // betaHeaderPrefixToFeature maps each known beta header prefix to a function that checks // whether the feature is supported by the provider's default feature set. var betaHeaderPrefixToFeature = map[string]func(ProviderFeatureSupport) bool{ + AnthropicDangerousToolUseBetaHeaderPrefix: func(f ProviderFeatureSupport) bool { return f.Safeguards }, + AnthropicAutoModeClassifierBetaHeaderPrefix: func(f ProviderFeatureSupport) bool { return f.Safeguards }, "computer-use-": func(f ProviderFeatureSupport) bool { return f.ComputerUse }, AnthropicStructuredOutputsBetaHeaderPrefix: func(f ProviderFeatureSupport) bool { return f.StructuredOutputs }, AnthropicMCPClientBetaHeaderPrefix: func(f ProviderFeatureSupport) bool { return f.MCP }, diff --git a/core/providers/anthropic/utils_test.go b/core/providers/anthropic/utils_test.go index 8d5bea6d8ba..8f24764654c 100644 --- a/core/providers/anthropic/utils_test.go +++ b/core/providers/anthropic/utils_test.go @@ -1,13 +1,19 @@ package anthropic import ( + "bytes" "context" "encoding/json" "fmt" + "go/ast" + "go/parser" + "go/token" "net/http" "net/http/httptest" "net/url" + "os" "reflect" + "regexp" "slices" "sort" "strings" @@ -16,6 +22,7 @@ import ( "time" "github.com/bytedance/sonic" + "github.com/capsohq/bifrost/core/internal/memtest" providerUtils "github.com/capsohq/bifrost/core/providers/utils" "github.com/capsohq/bifrost/core/schemas" "github.com/tidwall/gjson" @@ -1380,6 +1387,22 @@ func TestFilterBetaHeadersForProvider(t *testing.T) { } }) } + + // Claude Code auto mode sends two betas: dangerous-tool-use-* pairs with the + // safeguards body field, and auto-mode-classifier-* rides on the classifier's + // follow-up requests. Every surface that forwards safeguards must forward both. + t.Run("auto_mode_classifier_follows_safeguards_gate", func(t *testing.T) { + const classifierBeta = "auto-mode-classifier-2026-07-16" + for _, provider := range []schemas.ModelProvider{schemas.Anthropic, schemas.Bedrock, schemas.BedrockMantle, schemas.Vertex, schemas.Azure} { + result := FilterBetaHeadersForProvider([]string{classifierBeta}, provider) + if !ProviderFeatures[provider].Safeguards { + t.Fatalf("precondition: %s is expected to support safeguards", provider) + } + if !containsHeader(result, classifierBeta) { + t.Errorf("expected %q forwarded to %s, got %v", classifierBeta, provider, result) + } + } + }) } // TestNetworkConfigBetaOverridesFlow proves the production sequence @@ -1497,6 +1520,40 @@ func TestStripUnsupportedFieldsFromRawBody(t *testing.T) { } }) + t.Run("safeguards_gated_via_feature_map", func(t *testing.T) { + // safeguards is the Claude Code auto-mode classifier request field — + // supported providers retain it for Sonnet 5 / Opus 4.7+ / Fable only. + const body = `{"model":"claude-opus-4-8","safeguards":{"check":"auto_mode"}}` + const haikuBody = `{"model":"claude-haiku-4-5","safeguards":{"check":"auto_mode"}}` + result, err := StripUnsupportedFieldsFromRawBody([]byte(body), schemas.Anthropic, "claude-opus-4-8") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !providerUtils.JSONFieldExists(result, "safeguards") { + t.Errorf("expected safeguards to be kept for Anthropic, got: %s", string(result)) + } + // Anthropic direct is model-gated too: haiku is outside the auto-mode set. + result, err = StripUnsupportedFieldsFromRawBody([]byte(haikuBody), schemas.Anthropic, "claude-haiku-4-5") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if providerUtils.JSONFieldExists(result, "safeguards") { + t.Errorf("expected safeguards to be stripped for Anthropic on haiku, got: %s", string(result)) + } + // Cloud surfaces retain safeguards on supported models with the required beta. + for _, provider := range []schemas.ModelProvider{schemas.Azure, schemas.Bedrock, schemas.BedrockMantle, schemas.Vertex} { + for _, b := range []string{body, haikuBody} { + result, err := StripUnsupportedFieldsFromRawBody([]byte(b), provider, "") + if err != nil { + t.Fatalf("unexpected error for %s: %v", provider, err) + } + if providerUtils.JSONFieldExists(result, "safeguards") != (b == body) { + t.Errorf("unexpected safeguards model gate on %s, got: %s", provider, string(result)) + } + } + } + }) + t.Run("bedrock_strips_new_request_level_fields", func(t *testing.T) { // Raw body with every new typed field. Targeting Bedrock: speed (no FastMode), // inference_geo (no InferenceGeo), mcp_servers (no MCP), container.skills @@ -3395,6 +3452,11 @@ func TestComputerUseGeneration(t *testing.T) { {"global.anthropic.claude-opus-4-7", ComputerUseGen20251124}, {"global.anthropic.claude-sonnet-4-6", ComputerUseGen20251124}, {"global.anthropic.claude-haiku-4-5-20251001-v1:0", ComputerUseGen20250124}, + // Opus 5.5 rejects every dated computer_* tool, so it resolves to the toolset. + {"claude-opus-5-5", ComputerUseGenToolset20260801}, + {"claude-opus-5.5", ComputerUseGenToolset20260801}, + {"claude-opus-5-5-20260901", ComputerUseGenToolset20260801}, + {"global.anthropic.claude-opus-5-5", ComputerUseGenToolset20260801}, } for _, tc := range cases { t.Run(tc.model, func(t *testing.T) { @@ -3406,6 +3468,23 @@ func TestComputerUseGeneration(t *testing.T) { } } +// The toolset replaced the dated computer_* tools on the Claude API and Google +// Cloud only; AWS and Microsoft Foundry still serve computer_20251124 for the +// same model, so the generation is scoped to the surface as well as the model. +func TestComputerUseGeneration_ToolsetIsProviderScoped(t *testing.T) { + const model = "claude-opus-5-5" + for _, provider := range []schemas.ModelProvider{schemas.Anthropic, schemas.Vertex} { + if got := ComputerUseGeneration(schemas.ResolveModelCaps(provider, model)); got != ComputerUseGenToolset20260801 { + t.Errorf("%s: got %q, want the toolset — the dated tools are rejected there", provider, got) + } + } + for _, provider := range []schemas.ModelProvider{schemas.Bedrock, schemas.BedrockMantle, schemas.Azure} { + if got := ComputerUseGeneration(schemas.ResolveModelCaps(provider, model)); got != ComputerUseGen20251124 { + t.Errorf("%s: got %q, want computer_20251124 — that surface still serves it", provider, got) + } + } +} + // TestNormalizedToolSpec verifies the canonical {type, name} pair returned per // (generation, base-tool) pair matches Anthropic's strict Pydantic validators. func TestNormalizedToolSpec(t *testing.T) { @@ -3421,6 +3500,10 @@ func TestNormalizedToolSpec(t *testing.T) { {ComputerUseGen20250124, "computer", "computer_20250124", "computer"}, {ComputerUseGen20250124, "text_editor", "text_editor_20250124", "str_replace_editor"}, {ComputerUseGen20250124, "bash", "bash_20250124", "bash"}, + // A toolset is named by its type alone; a name on the entry is a 400. + {ComputerUseGenToolset20260801, "computer", "computer_toolset_20260801", ""}, + // Toolsets have no text_editor/bash members, so those stay on their own generation. + {ComputerUseGenToolset20260801, "bash", "bash_20250124", "bash"}, {ComputerUseGen20251124, "web_search", "", ""}, {ComputerUseGen20250124, "", "", ""}, } @@ -3838,6 +3921,258 @@ func TestStripEmptyThinkingBlocks(t *testing.T) { } } +// TestStripEmptyThinkingBlocks_AllocationScaling pins the allocation SHAPE of the +// strip, not a byte count. +// +// The original implementation called sjson Delete once per stripped block, and each +// of those reserialises the whole request body, so stripping N blocks from an S-byte +// body allocated N*S. A production heap profile attributed 23.6% of every byte the +// gateway had ever allocated to this one loop, all of it garbage: long agentic +// conversations carry hundreds of unsigned thinking blocks in a multi-megabyte body. +// +// memtest compares allocation growth against input growth, so it fails on the +// complexity class rather than on an absolute threshold that would encode this +// machine and today's Go version. Measured on the real before/after: 13.6x vs 3.9x. +func TestStripEmptyThinkingBlocks_AllocationScaling(t *testing.T) { + memtest.AssertAllocScaling(t, func(turns int) []byte { + var b bytes.Buffer + b.WriteString(`{"model":"claude-opus-4-8","messages":[`) + for i := range turns { + if i > 0 { + b.WriteByte(',') + } + // One unsigned thinking block (stripped) plus one text block carrying + // the bulk of the bytes (kept), so both N and the payload size scale. + b.WriteString(`{"role":"assistant","content":[`) + b.WriteString(`{"type":"thinking","thinking":"cross-provider reasoning","signature":""},`) + b.WriteString(`{"type":"text","text":"` + strings.Repeat("x", 500) + `"}]}`) + } + b.WriteString(`]}`) + return b.Bytes() + }, func(body []byte) { + if _, err := StripEmptyThinkingBlocks(body); err != nil { + t.Fatalf("unexpected error: %v", err) + } + }) +} + +// TestStripAutoInjectableTools_AllocationScaling covers the same rewrite shape on the +// tools array: the loop deletes one code_execution tool per iteration, and N grows with +// however many the client sent. +func TestStripAutoInjectableTools_AllocationScaling(t *testing.T) { + memtest.AssertAllocScaling(t, func(tools int) []byte { + var b bytes.Buffer + // A web_search tool is what makes Anthropic auto-inject code_execution, which + // is the precondition for this function doing anything at all. It also keeps + // at least one tool un-stripped, avoiding the delete-the-whole-array fast path. + b.WriteString(`{"model":"claude-opus-4-8","tools":[{"type":"web_search_20260209","name":"web_search"}`) + for range tools { + b.WriteString(`,{"type":"code_execution_20250522","name":"code_execution","description":"`) + b.WriteString(strings.Repeat("d", 400)) + b.WriteString(`"}`) + } + b.WriteString(`],"messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}]}`) + return b.Bytes() + }, func(body []byte) { + if _, err := StripAutoInjectableTools(body); err != nil { + t.Fatalf("unexpected error: %v", err) + } + }) +} + +// TestRemapRawToolVersionsForProvider_AllocationScaling covers the tool-version remap, +// which writes tools.N.type and tools.N.name one tool at a time. +func TestRemapRawToolVersionsForProvider_AllocationScaling(t *testing.T) { + memtest.AssertAllocScaling(t, func(tools int) []byte { + var b bytes.Buffer + b.WriteString(`{"model":"claude-opus-4-8","tools":[`) + for i := range tools { + if i > 0 { + b.WriteByte(',') + } + // An outdated bash version, which NormalizedToolSpec remaps to + // bash_20250124, so every tool in the array triggers a write. + b.WriteString(`{"type":"bash_20241022","name":"wrong_name","description":"`) + b.WriteString(strings.Repeat("d", 400)) + b.WriteString(`"}`) + } + b.WriteString(`],"messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}]}`) + return b.Bytes() + }, func(body []byte) { + if _, err := RemapRawToolVersionsForProvider(body, schemas.Anthropic, "claude-opus-4-8"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + }) +} + +// TestStripUnsupportedFieldsFromRawBody_AllocationScaling covers the system-block +// cache_control scope strip, whose loop walks every system block in the request. +func TestStripUnsupportedFieldsFromRawBody_AllocationScaling(t *testing.T) { + memtest.AssertAllocScaling(t, func(blocks int) []byte { + var b bytes.Buffer + b.WriteString(`{"model":"claude-sonnet-4-5","system":[`) + for i := range blocks { + if i > 0 { + b.WriteByte(',') + } + b.WriteString(`{"type":"text","text":"`) + b.WriteString(strings.Repeat("s", 400)) + b.WriteString(`","cache_control":{"type":"ephemeral","scope":"organization"}}`) + } + b.WriteString(`],"messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}]}`) + return b.Bytes() + }, func(body []byte) { + // Bedrock gates PromptCachingScope off, which is what makes the strip run. + if _, err := StripUnsupportedFieldsFromRawBody(body, schemas.Bedrock, "claude-sonnet-4-5"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + }) +} + +// betaHeaderTestProviders is every provider the beta-header gating knows about, derived +// from ProviderFeatures rather than hardcoded. +// +// A hardcoded list silently goes stale: a provider added to ProviderFeatures would not be +// exercised, and a header only that provider allows through FilterBetaHeadersForProvider +// would look uncovered and get wrongly excused in notEmittedByRequestGating. Reading the +// map means a new provider is covered the day it is added. +func betaHeaderTestProviders() []schemas.ModelProvider { + out := make([]schemas.ModelProvider, 0, len(ProviderFeatures)) + for provider := range ProviderFeatures { + out = append(out, provider) + } + sort.Slice(out, func(i, j int) bool { return out[i] < out[j] }) + return out +} + +// betaHeaderCorpus is the shared set of request bodies exercising every beta-header +// signal. Shared so TestEveryBetaHeaderIsCoveredByTheCorpus can assert it reaches every +// declared header, which is what keeps a newly added header from going unasserted. +func betaHeaderCorpus() map[string]string { + const model = "claude-opus-4-8" + return map[string]string{ + "bare request": `{"model":"` + model + `","max_tokens":64,"messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}]}`, + + // --- the two messages-derived signals, which are the ones the gjson scan recovers --- + "scoped cache_control in message block": `{"model":"` + model + `","messages":[{"role":"user","content":[{"type":"text","text":"hi","cache_control":{"type":"ephemeral","scope":"organization"}}]}]}`, + "unscoped cache_control in message block (must NOT trigger)": `{"model":"` + model + `","messages":[{"role":"user","content":[{"type":"text","text":"hi","cache_control":{"type":"ephemeral"}}]}]}`, + // gjson's Exists() is `Type != Null || len(Raw) != 0`, so an explicit null is + // present-but-null and reports true, while a missing key reports false. Typed + // decoding puts nil in *string for both. Without an explicit null check the raw + // path injects a prompt-caching-scope beta header the typed path never would. + "explicit null scope in message block (must NOT trigger)": `{"model":"` + model + `","messages":[{"role":"user","content":[{"type":"text","text":"hi","cache_control":{"type":"ephemeral","scope":null}}]}]}`, + // Same present-but-null shape on the other signal. These already agree (gjson + // .String() and a Go string field both yield ""), so this pins that agreement. + "explicit null source type in message block (must NOT trigger)": `{"model":"` + model + `","messages":[{"role":"user","content":[{"type":"document","source":{"type":null}}]}]}`, + "file source in message block": `{"model":"` + model + `","messages":[{"role":"user","content":[{"type":"document","source":{"type":"file","file_id":"file_123"}}]}]}`, + "base64 source in message block (must NOT trigger)": `{"model":"` + model + `","messages":[{"role":"user","content":[{"type":"image","source":{"type":"base64","media_type":"image/png","data":"aGk="}}]}]}`, + "both message signals at once": `{"model":"` + model + `","messages":[{"role":"user","content":[{"type":"text","text":"hi","cache_control":{"type":"ephemeral","scope":"organization"}},{"type":"document","source":{"type":"file","file_id":"file_123"}}]}]}`, + "signals on a later message, not the first": `{"model":"` + model + `","messages":[{"role":"user","content":[{"type":"text","text":"one"}]},{"role":"assistant","content":[{"type":"text","text":"two"}]},{"role":"user","content":[{"type":"document","source":{"type":"file","file_id":"file_9"}}]}]}`, + "string content messages are skipped": `{"model":"` + model + `","messages":[{"role":"user","content":"plain string"}]}`, + "no messages field at all": `{"model":"` + model + `","max_tokens":64}`, + + // --- scope found elsewhere; the messages branch must not double-add or mask it --- + "scoped cache_control in system": `{"model":"` + model + `","system":[{"type":"text","text":"sys","cache_control":{"type":"ephemeral","scope":"organization"}}],"messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}]}`, + "scoped cache_control on a tool": `{"model":"` + model + `","tools":[{"name":"t","description":"d","input_schema":{"type":"object"},"cache_control":{"type":"ephemeral","scope":"organization"}}],"messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}]}`, + "scope in system AND messages": `{"model":"` + model + `","system":[{"type":"text","text":"sys","cache_control":{"type":"ephemeral","scope":"organization"}}],"messages":[{"role":"user","content":[{"type":"text","text":"hi","cache_control":{"type":"ephemeral","scope":"organization"}}]}]}`, + + // --- top-level signals: these still go through the decode, so they guard the + // "drop messages before decoding" half of the change --- + "computer use tool": `{"model":"` + model + `","tools":[{"type":"computer_20250124","name":"computer"}],"messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}]}`, + "computer use tool (newer generation)": `{"model":"` + model + `","tools":[{"type":"computer_20251124","name":"computer"}],"messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}]}`, + "advisor tool": `{"model":"` + model + `","tools":[{"type":"advisor_20260301","name":"advisor"}],"messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}]}`, + "safeguards (dangerous tool use)": `{"model":"claude-sonnet-5","safeguards":[{"type":"opaque","payload":"x"}],"messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}]}`, + "native server-side fallbacks": `{"model":"` + model + `","fallbacks":[{"model":"claude-haiku-4-5"}],"messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}]}`, + "default fallback routing": `{"model":"claude-opus-5","fallbacks":"default","messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}]}`, + "strict tool": `{"model":"` + model + `","tools":[{"name":"t","input_schema":{"type":"object"},"strict":true}],"messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}]}`, + "defer_loading tool": `{"model":"` + model + `","tools":[{"name":"t","input_schema":{"type":"object"},"defer_loading":true}],"messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}]}`, + "input_examples tool": `{"model":"` + model + `","tools":[{"name":"t","input_schema":{"type":"object"},"input_examples":[{"a":1}]}],"messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}]}`, + "allowed_callers tool": `{"model":"` + model + `","tools":[{"name":"t","input_schema":{"type":"object"},"allowed_callers":["assistant"]}],"messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}]}`, + "eager_input_streaming": `{"model":"` + model + `","tools":[{"name":"t","input_schema":{"type":"object"},"eager_input_streaming":true}],"messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}]}`, + "context_management compact": `{"model":"` + model + `","context_management":{"edits":[{"type":"compact_20260112"}]},"messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}]}`, + "context_management clear": `{"model":"` + model + `","context_management":{"edits":[{"type":"clear_tool_uses_20250919"}]},"messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}]}`, + "mcp_servers": `{"model":"` + model + `","mcp_servers":[{"type":"url","url":"https://example.com","name":"s"}],"messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}]}`, + "thinking enabled": `{"model":"` + model + `","thinking":{"type":"enabled","budget_tokens":1024},"messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}]}`, + "speed": `{"model":"` + model + `","speed":"fast","messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}]}`, + "output_config task_budget": `{"model":"` + model + `","output_config":{"task_budget":{"type":"tokens","value":100}},"messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}]}`, + "output_format": `{"model":"` + model + `","output_format":{"type":"json_schema","schema":{"type":"object"}},"messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}]}`, + "diagnostics": `{"model":"` + model + `","diagnostics":{"cache":true},"messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}]}`, + "fallback_credit_token": `{"model":"` + model + `","fallback_credit_token":"tok_1","messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}]}`, + + // --- everything at once, to catch ordering/dedup divergence --- + "kitchen sink": `{"model":"` + model + `","thinking":{"type":"enabled","budget_tokens":1024},` + + `"tools":[{"type":"computer_20250124","name":"computer"},{"name":"t","input_schema":{"type":"object"},"strict":true,"defer_loading":true,"input_examples":[{"a":1}],"allowed_callers":["assistant"],"eager_input_streaming":true}],` + + `"context_management":{"edits":[{"type":"compact_20260112"},{"type":"clear_thinking_20251015"}]},` + + `"mcp_servers":[{"type":"url","url":"https://example.com","name":"s"}],` + + `"output_format":{"type":"json_schema","schema":{"type":"object"}},"diagnostics":{"cache":true},` + + `"system":[{"type":"text","text":"sys"}],` + + `"messages":[{"role":"user","content":[{"type":"text","text":"hi","cache_control":{"type":"ephemeral","scope":"organization"}},{"type":"document","source":{"type":"file","file_id":"file_1"}}]}]}`, + } +} + +// TestAddMissingBetaHeadersFromRawBody_MatchesTypedPath is the safety net for replacing +// the raw-body probe-unmarshal with a gjson scan. +// +// BuildAnthropicResponsesRequestBody used to decode the entire request into an +// AnthropicMessageRequest purely to decide which anthropic-beta headers to inject, then +// throw the struct away. On a long agentic conversation that decode expands the messages +// array several-fold into structs; a production heap profile had it holding ~526 MB live. +// AddMissingBetaHeadersToContextFromRawBody drops the messages array before decoding and +// recovers the two signals the gating actually reads from it via gjson. +// +// Getting that wrong is not a performance bug, it is a hard 400 from Anthropic (either a +// header for an unsupported feature, or a missing header for a used one). So this asserts +// the two paths agree exactly, header-for-header, across every signal and several +// providers — rather than asserting that the new scan looks correct. +func TestAddMissingBetaHeadersFromRawBody_MatchesTypedPath(t *testing.T) { + + bodies := betaHeaderCorpus() + + providers := betaHeaderTestProviders() + + // headersVia runs one path and returns the anthropic-beta headers it put on a fresh context. + headersVia := func(t *testing.T, apply func(ctx *schemas.BifrostContext) error) []string { + t.Helper() + ctx := schemas.NewBifrostContext(nil, time.Time{}) + if err := apply(ctx); err != nil { + t.Fatalf("applying beta headers failed: %v", err) + } + extraHeaders, ok := ctx.Value(schemas.BifrostContextKeyExtraHeaders).(map[string][]string) + if !ok { + return nil + } + got := append([]string(nil), extraHeaders[AnthropicBetaHeader]...) + sort.Strings(got) + return got + } + + for name, body := range bodies { + for _, provider := range providers { + t.Run(fmt.Sprintf("%s/%s", name, provider), func(t *testing.T) { + raw := []byte(body) + + // The pre-change behaviour: full decode, then the typed entry point. + want := headersVia(t, func(ctx *schemas.BifrostContext) error { + var req AnthropicMessageRequest + if err := sonic.Unmarshal(raw, &req); err != nil { + t.Fatalf("corpus body is not a decodable AnthropicMessageRequest: %v", err) + } + return AddMissingBetaHeadersToContext(ctx, &req, provider) + }) + + got := headersVia(t, func(ctx *schemas.BifrostContext) error { + return AddMissingBetaHeadersToContextFromRawBody(ctx, raw, provider) + }) + + if !reflect.DeepEqual(got, want) { + t.Errorf("raw-body path disagrees with the typed path.\nraw: %v\ntyped: %v\nbody: %s", + got, want, body) + } + }) + } + } +} + // TestFastMode_StreamingForwardsSpeed verifies the per-event message_delta // converter surfaces the served speed on the emitted chunk (client-facing usage // visibility). NOTE: billing reads the terminal response.completed chunk, not @@ -4282,3 +4617,595 @@ func TestHandleAnthropicChatCompletionStreaming_PreservesEncodedPath(t *testing. collectTruncationChunks(t, stream) assertEncodedPathPreserved(t, rec.get()) } + +// Automatic beta injection follows the strip gate and existing header policies. +func TestSafeguardsBetaGatesAndOverrides(t *testing.T) { + for _, provider := range []schemas.ModelProvider{schemas.Anthropic, schemas.Bedrock, schemas.BedrockMantle, schemas.Vertex, schemas.Azure, schemas.DeepSeek} { + for _, model := range []string{"claude-opus-4-8", "claude-haiku-4-5"} { + for _, present := range []bool{false, true} { + ctx := schemas.NewBifrostContext(context.Background(), time.Time{}) + req := &AnthropicMessageRequest{Model: model} + if present { + req.Safeguards = json.RawMessage(`{}`) + } + stripUnsupportedAnthropicFields(req, provider, model) + if err := AddMissingBetaHeadersToContext(ctx, req, provider); err != nil { + t.Fatal(err) + } + got := FilterBetaHeadersForProvider(MergeBetaHeaders(ctx, nil), provider) + want := present && model == "claude-opus-4-8" && provider != schemas.DeepSeek + if slices.Contains(got, AnthropicDangerousToolUseBetaHeader) != want { + t.Fatalf("%s/%s present=%v: %v", provider, model, present, got) + } + } + } + } + ctx := schemas.NewBifrostContext(context.Background(), time.Time{}) + ctx.SetValue(schemas.BifrostContextKeyExtraHeaders, map[string][]string{AnthropicBetaHeader: {AnthropicDangerousToolUseBetaHeader, AnthropicCompactionBetaHeader}}) + req := &AnthropicMessageRequest{Model: "claude-opus-4-8", Safeguards: json.RawMessage(`{}`)} + for range 2 { + if err := AddMissingBetaHeadersToContext(ctx, req, schemas.Bedrock); err != nil { + t.Fatal(err) + } + } + got := FilterBetaHeadersForProvider(MergeBetaHeaders(ctx, nil), schemas.Bedrock) + if len(got) != 2 { + t.Fatalf("beta deduplication lost existing headers: %v", got) + } + got = FilterBetaHeadersForProvider(got, schemas.Bedrock, map[string]bool{AnthropicDangerousToolUseBetaHeaderPrefix: false}) + if slices.Contains(got, AnthropicDangerousToolUseBetaHeader) || !slices.Contains(got, AnthropicCompactionBetaHeader) { + t.Fatalf("explicit override not respected: %v", got) + } +} + +func TestSafeguardsStreamHandler(t *testing.T) { + const update = `{"type":"safeguards_update","safeguard_results":[{"id":"sg_1","verdict":"allow"}]}` + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get(AnthropicBetaHeader) != AnthropicDangerousToolUseBetaHeader { + t.Errorf("missing beta on wire: %q", r.Header.Get(AnthropicBetaHeader)) + } + w.Header().Set("Content-Type", "text/event-stream") + for _, frame := range []string{ptMsgStart(), update, `{"type":"future_unknown_event","safeguard_results":[]}`, ptMsgEnd()[0], ptMsgEnd()[1]} { + fmt.Fprintf(w, "event: %s\ndata: %s\n\n", gjson.Get(frame, "type").String(), frame) + } + })) + defer server.Close() + for _, raw := range []bool{false, true} { + ctx, cancel := schemas.NewBifrostContextWithCancel(context.Background()) + defer cancel() + body, buildErr := BuildAnthropicResponsesRequestBody(ctx, &schemas.BifrostResponsesRequest{ + Provider: schemas.Anthropic, Model: "claude-opus-4-8", Input: makeSimpleInput("hi"), + Params: &schemas.ResponsesParameters{ExtraParams: map[string]interface{}{"safeguards": json.RawMessage(`{}`)}}, + }, AnthropicRequestBuildConfig{Provider: schemas.Anthropic, IsStreaming: true}) + if buildErr != nil { + t.Fatal(buildErr) + } + stream, err := HandleAnthropicResponsesStream(ctx, &fasthttp.Client{}, server.URL, body, nil, nil, 30, nil, false, raw, schemas.Anthropic, truncationPassthroughPostHook, nil, nil, truncationTestLogger{}, nil) + if err != nil { + t.Fatal(err) + } + updates := 0 + for _, chunk := range collectTruncationChunks(t, stream) { + if chunk.BifrostError != nil { + t.Fatal(chunk.BifrostError) + } + response := chunk.BifrostResponsesStreamResponse + if response == nil { + continue + } + if response.Type == schemas.ResponsesStreamResponseTypeProviderRawEvent { + t.Fatal("unknown event forwarded") + } + if response.Type == schemas.ResponsesStreamResponseTypeSafeguardsUpdate { + updates++ + if raw && response.ExtraFields.RawResponse != update { + t.Fatalf("raw frame lost: %v", response.ExtraFields.RawResponse) + } + out := ToAnthropicResponsesStreamResponse(ctx, response) + if len(out) != 1 || string(out[0].SafeguardResults) != `[{"id":"sg_1","verdict":"allow"}]` { + t.Fatalf("typed update lost: %#v", out) + } + } + } + if updates != 1 { + t.Fatalf("raw=%v: got %d updates", raw, updates) + } + } +} + +func TestStripUnsupportedAnthropicFieldsSafeguards(t *testing.T) { + mk := func(model string) *AnthropicMessageRequest { + var req AnthropicMessageRequest + if err := sonic.Unmarshal([]byte(`{"model":"`+model+`","max_tokens":16,"messages":[{"role":"user","content":"hi"}],"safeguards":{"check":"auto_mode"}}`), &req); err != nil { + t.Fatalf("unmarshal request: %v", err) + } + return &req + } + + req := mk("claude-opus-4-8") + stripUnsupportedAnthropicFields(req, schemas.Anthropic, "claude-opus-4-8") + out, err := sonic.Marshal(req) + if err != nil { + t.Fatalf("marshal request: %v", err) + } + if !gjson.GetBytes(out, "safeguards").Exists() { + t.Errorf("expected safeguards to be kept for Anthropic, got: %s", string(out)) + } + + // Anthropic direct is still model-gated: Haiku is outside the auto-mode model + // set, so the field is stripped rather than sent to a model that refuses it. + req = mk("claude-haiku-4-5") + stripUnsupportedAnthropicFields(req, schemas.Anthropic, "claude-haiku-4-5") + out, err = sonic.Marshal(req) + if err != nil { + t.Fatalf("marshal request: %v", err) + } + if gjson.GetBytes(out, "safeguards").Exists() { + t.Errorf("expected safeguards to be stripped for Anthropic on haiku, got: %s", string(out)) + } + + for _, provider := range []schemas.ModelProvider{schemas.Azure, schemas.Bedrock, schemas.BedrockMantle, schemas.Vertex} { + for _, model := range []string{"claude-opus-4-8", "claude-sonnet-5", "claude-haiku-4-5"} { + req := mk(model) + stripUnsupportedAnthropicFields(req, provider, model) + out, err := sonic.Marshal(req) + if err != nil { + t.Fatalf("marshal request for %s/%s: %v", provider, model, err) + } + if gjson.GetBytes(out, "safeguards").Exists() != (model != "claude-haiku-4-5") { + t.Errorf("unexpected safeguards model gate on %s/%s, got: %s", provider, model, string(out)) + } + } + } +} + +// notEmittedByRequestGating records beta headers that AddMissingBetaHeadersToContext +// cannot produce from a request body, with the reason. Everything else must be exercised +// by the corpus in TestAddMissingBetaHeadersFromRawBody_MatchesTypedPath. +var notEmittedByRequestGating = map[string]string{ + "AnthropicFallbackCreditBetaHeaderAWS": "FilterBetaHeadersForProvider rewrites the canonical " + + "fallback-credit date to the AWS one on Bedrock/Mantle; gating emits only the canonical value", + "AnthropicMCPClientBetaHeaderDeprecated": "superseded version constant kept for inbound matching; " + + "gating emits AnthropicMCPClientBetaHeader", + "AnthropicContext1MBetaHeader": "opted into via network config or passthrough headers, not derived from the request body", + "AnthropicRedactThinkingBetaHeader": "opted into via network config or passthrough headers, not derived from the request body", + "AnthropicSkillsBetaHeader": "opted into via network config or passthrough headers, not derived from the request body", + "AnthropicMidConversationToolChangesBetaHeader": "opted into via network config or passthrough headers, not derived from the request body", +} + +// TestEveryBetaHeaderIsCoveredByTheCorpus makes a newly added beta header loud. +// +// The equivalence corpus is only as good as its coverage: a header added next quarter with +// no corpus entry would be asserted by nothing, and the raw-body path could stop emitting +// it silently. This enumerates every Anthropic*BetaHeader constant in the package and +// requires each to be either produced by the corpus or explicitly recorded as +// not-gating-emitted with a reason. +// +// Prefix constants are excluded: they are matching helpers for FilterBetaHeadersForProvider, +// not values that gating emits. +func TestEveryBetaHeaderIsCoveredByTheCorpus(t *testing.T) { + // 1. Every header value the corpus puts ON THE WIRE, across every provider. + // + // Deliberately measured after MergeBetaHeaders + FilterBetaHeadersForProvider rather + // than off the context. The gating computing a header is not the same as the request + // carrying it: the filter can drop a value the provider does not support, or rewrite + // it (that is how the AWS fallback-credit date is produced). Stopping at the context + // would pass for a header that is computed correctly and then silently discarded + // before req.Header.Set(AnthropicBetaHeader, ...) in anthropic.go. + providers := betaHeaderTestProviders() + emitted := map[string]bool{} + for _, body := range betaHeaderCorpus() { + for _, provider := range providers { + ctx := schemas.NewBifrostContext(nil, time.Time{}) + if err := AddMissingBetaHeadersToContextFromRawBody(ctx, []byte(body), provider); err != nil { + continue + } + // The exact pipeline anthropic.go runs before setting the header. + for _, h := range FilterBetaHeadersForProvider(MergeBetaHeaders(ctx, nil), provider) { + emitted[h] = true + } + } + } + if len(emitted) == 0 { + t.Fatal("the corpus put no beta headers on the wire at all, so this test is measuring nothing") + } + + // 2. Every beta-header constant declared in the package. + fset := token.NewFileSet() + pkgs, err := parser.ParseDir(fset, ".", func(fi os.FileInfo) bool { + return !strings.HasSuffix(fi.Name(), "_test.go") + }, 0) + if err != nil { + t.Fatalf("parsing package: %v", err) + } + constRe := regexp.MustCompile(`^Anthropic[A-Za-z0-9]*BetaHeader[A-Za-z0-9]*$`) + declared := map[string]string{} // ident -> value + for _, pkg := range pkgs { + for _, file := range pkg.Files { + for _, decl := range file.Decls { + gen, ok := decl.(*ast.GenDecl) + if !ok || gen.Tok != token.CONST { + continue + } + for _, spec := range gen.Specs { + vs, ok := spec.(*ast.ValueSpec) + if !ok { + continue + } + for i, name := range vs.Names { + if !constRe.MatchString(name.Name) || i >= len(vs.Values) { + continue + } + lit, ok := vs.Values[i].(*ast.BasicLit) + if !ok || lit.Kind != token.STRING { + continue + } + declared[name.Name] = strings.Trim(lit.Value, `"`) + } + } + } + } + } + if len(declared) == 0 { + t.Fatal("found no Anthropic*BetaHeader constants; the scan has broken") + } + + var uncovered []string + for ident, value := range declared { + switch { + case ident == "AnthropicBetaHeader": + continue // the HTTP header name, not a beta value + case strings.HasSuffix(ident, "Prefix"): + continue // matcher for FilterBetaHeadersForProvider, never emitted + case emitted[value]: + continue + } + if _, excused := notEmittedByRequestGating[ident]; excused { + continue + } + uncovered = append(uncovered, fmt.Sprintf(" %s = %q", ident, value)) + } + sort.Strings(uncovered) + + if len(uncovered) > 0 { + t.Errorf("%d beta header(s) never reach the wire from any corpus entry:\n%s\n\n"+ + "Either add a request body to betaHeaderCorpus() that triggers it (which also puts it "+ + "under the raw-vs-typed equivalence assertion), or record it in "+ + "notEmittedByRequestGating with the reason it cannot come from a request body.\n\n"+ + "A header that IS gated but never survives FilterBetaHeadersForProvider for any "+ + "provider is dead code on the request path, and shows up here the same way.", + len(uncovered), strings.Join(uncovered, "\n")) + } +} + +// TestBetaGatingNeverReadsDroppedFields makes dropping messages and tools safe by +// construction instead of by vigilance. +// +// AddMissingBetaHeadersToContextFromRawBody deletes betaProbeDroppedFields from the body +// before decoding, so on that path req.Messages is nil, and every tool has had its bulk +// fields (input_schema, description) stripped. Everything the +// gating needs from them arrives as anthropicMessageBetaSignals. If someone adds a beta +// header gated on a new tool or message field and reads it off req directly, the typed +// path emits the header and the raw path silently does not — a passthrough request quietly +// loses it. +// +// The equivalence test catches that only when the corpus happens to contain a triggering +// body, and TestEveryBetaHeaderIsCoveredByTheCorpus can be silenced by an entry in +// notEmittedByRequestGating. This closes that gap at the source: the gating simply may not +// reference the dropped fields, so forgetting a signal is a compile-time-shaped failure +// rather than a behavioural one nobody notices. +func TestBetaGatingNeverReadsDroppedFields(t *testing.T) { + // json field name -> Go struct field on AnthropicMessageRequest. + // Checked on the source slice, not on `forbidden`: the tool bulk fields below are + // added unconditionally, so a length check after them can never fire and an emptied + // betaProbeDroppedFields would silently stop being covered. + if len(betaProbeDroppedFields) == 0 { + t.Fatal("betaProbeDroppedFields is empty; this guard would stop covering the dropped message fields") + } + forbidden := map[string]string{} + for _, field := range betaProbeDroppedFields { + forbidden[strings.ToUpper(field[:1])+field[1:]] = field + } + // Tool bulk fields are stripped from every tool before the probe decode, so the + // gating must not read them either. Go field names for the json keys. + for goName, jsonName := range map[string]string{"InputSchema": "input_schema", "Description": "description"} { + forbidden[goName] = jsonName + } + + fset := token.NewFileSet() + pkgs, err := parser.ParseDir(fset, ".", func(fi os.FileInfo) bool { + return !strings.HasSuffix(fi.Name(), "_test.go") + }, 0) + if err != nil { + t.Fatalf("parsing package: %v", err) + } + + var violations []string + found := false + for _, pkg := range pkgs { + for _, file := range pkg.Files { + for _, decl := range file.Decls { + fn, ok := decl.(*ast.FuncDecl) + if !ok || fn.Name.Name != "addMissingBetaHeadersToContext" || fn.Body == nil { + continue + } + found = true + ast.Inspect(fn.Body, func(n ast.Node) bool { + sel, ok := n.(*ast.SelectorExpr) + if !ok { + return true + } + recv, ok := sel.X.(*ast.Ident) + if !ok || (recv.Name != "req" && recv.Name != "tool") { + return true + } + if jsonName, bad := forbidden[sel.Sel.Name]; bad { + violations = append(violations, fmt.Sprintf( + " %s reads %s.%s at %s (%q is removed on the raw-body path)", + fn.Name.Name, recv.Name, sel.Sel.Name, fset.Position(sel.Pos()), jsonName)) + } + return true + }) + } + } + } + if !found { + t.Fatal("addMissingBetaHeadersToContext not found; this guard has broken and is asserting nothing") + } + + sort.Strings(violations) + if len(violations) > 0 { + t.Errorf("the beta-header gating reads a field that the raw-body path removes before "+ + "decoding, so the raw and typed paths will disagree:\n%s\n\n"+ + "For a messages field: add it to anthropicMessageBetaSignals and compute it in BOTH "+ + "scanMessagesForBetaSignals (typed) and scanRawMessagesForBetaSignals (gjson).\n"+ + "For a tool field: remove it from betaProbeStrippedToolFields so the probe keeps it.", + strings.Join(violations, "\n")) + } +} + +// signalFieldsAssignedIn returns the anthropicMessageBetaSignals fields a function sets. +func signalFieldsAssignedIn(t *testing.T, fnName string) map[string]bool { + t.Helper() + fset := token.NewFileSet() + pkgs, err := parser.ParseDir(fset, ".", func(fi os.FileInfo) bool { + return !strings.HasSuffix(fi.Name(), "_test.go") + }, 0) + if err != nil { + t.Fatalf("parsing package: %v", err) + } + assigned := map[string]bool{} + found := false + for _, pkg := range pkgs { + for _, file := range pkg.Files { + for _, decl := range file.Decls { + fn, ok := decl.(*ast.FuncDecl) + if !ok || fn.Name.Name != fnName || fn.Body == nil { + continue + } + found = true + ast.Inspect(fn.Body, func(n ast.Node) bool { + assign, ok := n.(*ast.AssignStmt) + if !ok { + return true + } + for _, lhs := range assign.Lhs { + sel, ok := lhs.(*ast.SelectorExpr) + if !ok { + continue + } + if recv, ok := sel.X.(*ast.Ident); ok && recv.Name == "signals" { + assigned[sel.Sel.Name] = true + } + } + return true + }) + } + } + } + if !found { + t.Fatalf("%s not found; this guard has broken and is asserting nothing", fnName) + } + return assigned +} + +// TestTypedAndRawSignalScansStayInLockstep closes the last way a stale signal list can +// silently lose a beta header. +// +// TestBetaGatingNeverReadsDroppedFields stops the gating reading req.Tools/req.Messages +// directly, so every value must arrive as a signal. That leaves one gap: adding a signal +// to the TYPED scan and forgetting the RAW one. The typed path would then emit the header +// and the raw path would not, which is invisible unless the corpus happens to contain a +// body that triggers it — and TestEveryBetaHeaderIsCoveredByTheCorpus can be silenced via +// notEmittedByRequestGating. +// +// Requiring both scans to populate the identical field set makes that impossible to miss, +// independent of any corpus. +// +// Tools are deliberately absent here: they are no longer reduced to signals at all. The +// raw path keeps the tools array and strips only input_schema and description from each +// tool, so there is no list of needed tool fields that can go stale. +func TestTypedAndRawSignalScansStayInLockstep(t *testing.T) { + for _, pair := range []struct{ typed, raw string }{ + {"scanMessagesForBetaSignals", "scanRawMessagesForBetaSignals"}, + } { + typed := signalFieldsAssignedIn(t, pair.typed) + raw := signalFieldsAssignedIn(t, pair.raw) + if len(typed) == 0 { + t.Errorf("%s sets no signals at all; the guard is measuring nothing", pair.typed) + continue + } + for field := range typed { + if !raw[field] { + t.Errorf("%s sets signals.%s but %s does not: the typed path would emit the "+ + "beta header and the raw (passthrough) path would silently not. Add the "+ + "equivalent gjson check to %s.", pair.typed, field, pair.raw, pair.raw) + } + } + for field := range raw { + if !typed[field] { + t.Errorf("%s sets signals.%s but %s does not: the raw path would emit the beta "+ + "header and the typed path would silently not. Add the equivalent check to %s.", + pair.raw, field, pair.typed, pair.typed) + } + } + } +} + +// TestEveryOutboundPathSetsBetaHeadersIdentically stops the streaming and unary request +// builders drifting apart on beta headers. +// +// anthropic.go builds outbound requests in several places — unary chat, streaming chat, +// unary responses, streaming responses — and each sets anthropic-beta with its own copy of +// the same three lines. Copies drift: a fix applied to the unary path and not the streaming +// one means streaming requests silently go out with a different beta set, which surfaces as +// a feature working on one call shape and not the other. +// +// Rather than trust that they match, this extracts every block that sets the header and +// requires them to be textually identical. +func TestEveryOutboundPathSetsBetaHeadersIdentically(t *testing.T) { + source, err := os.ReadFile("anthropic.go") + if err != nil { + t.Fatalf("reading anthropic.go: %v", err) + } + lines := strings.Split(string(source), "\n") + + // Each site is the `if betaHeaders := ...` guard plus its Set/else/Del body. + var blocks []string + var at []int + for i, line := range lines { + if !strings.Contains(line, "FilterBetaHeadersForProvider(MergeBetaHeaders(") { + continue + } + end := i + // end+1, not end: the body reads lines[end] AFTER incrementing, so a matching + // call inside the final few lines of the file would index one past the slice. + for end+1 < len(lines) && end < i+6 { + end++ + if strings.TrimSpace(lines[end]) == "}" { + break + } + } + var b strings.Builder + for _, l := range lines[i : end+1] { + b.WriteString(strings.TrimSpace(l)) + b.WriteByte('\n') + } + blocks = append(blocks, b.String()) + at = append(at, i+1) + } + + if len(blocks) < 2 { + t.Fatalf("found %d beta-header blocks in anthropic.go; expected several (unary and "+ + "streaming, chat and responses). The guard has broken, or the call was renamed.", len(blocks)) + } + + for i := 1; i < len(blocks); i++ { + if blocks[i] != blocks[0] { + t.Errorf("the beta-header block at anthropic.go:%d differs from the one at "+ + "anthropic.go:%d, so these request paths can send different anthropic-beta "+ + "values for the same request.\n\nfirst:\n%s\ndiffering:\n%s", + at[i], at[0], blocks[0], blocks[i]) + } + } + t.Logf("%d outbound beta-header blocks, all identical (lines %v)", len(blocks), at) +} + +// TestRawPathWithCallerBetaHeadersMatchesTypedPath covers prefix suppression. +// +// When the caller supplies its own anthropic-beta, betaHeaderPrefixExists suppresses every +// derived header sharing a prefix with it. Both paths must reach the same answer: the raw +// path decodes a body whose tools have been stripped of input_schema and description, the +// typed path decodes them whole, and neither difference may change which headers survive. +// Checked for every corpus body and provider. +// +// A divergence here means the bet is wrong: some tool-derived header whose prefix the +// caller did not claim is emitted by the typed path and lost by the raw one. +func TestRawPathWithCallerBetaHeadersMatchesTypedPath(t *testing.T) { + // A caller header that claims one unrelated prefix, so most derived headers are still + // eligible. This is the adversarial case: if dropping tools loses anything, it shows here. + callerBeta := []string{AnthropicContext1MBetaHeader} + + for name, body := range betaHeaderCorpus() { + for _, provider := range betaHeaderTestProviders() { + t.Run(fmt.Sprintf("%s/%s", name, provider), func(t *testing.T) { + raw := []byte(body) + + headersOf := func(apply func(ctx *schemas.BifrostContext) error) []string { + ctx := schemas.NewBifrostContext(nil, time.Time{}) + ctx.SetValue(schemas.BifrostContextKeyExtraHeaders, + map[string][]string{AnthropicBetaHeader: callerBeta}) + if err := apply(ctx); err != nil { + t.Fatalf("applying beta headers: %v", err) + } + extra, _ := ctx.Value(schemas.BifrostContextKeyExtraHeaders).(map[string][]string) + got := append([]string(nil), extra[AnthropicBetaHeader]...) + sort.Strings(got) + return got + } + + want := headersOf(func(ctx *schemas.BifrostContext) error { + var req AnthropicMessageRequest + if err := sonic.Unmarshal(raw, &req); err != nil { + t.Fatalf("corpus body is not decodable: %v", err) + } + return AddMissingBetaHeadersToContext(ctx, &req, provider) + }) + got := headersOf(func(ctx *schemas.BifrostContext) error { + return AddMissingBetaHeadersToContextFromRawBody(ctx, raw, provider) + }) + + if !reflect.DeepEqual(got, want) { + t.Errorf("with a caller-supplied anthropic-beta the raw path drops tools and "+ + "disagrees with the typed path.\nraw: %v\ntyped: %v\nbody: %s", + got, want, body) + } + }) + } + } +} + +// TestBetaProbeNeverMutatesTheOutboundBody is the blast-radius check on stripping +// input_schema and description. +// +// Those fields are removed only from a local copy used for the probe decode. If the strip +// ever reached the body that goes upstream, every tool would arrive at Anthropic with no +// schema and no description: tool calling would break outright, on every request, for +// every passthrough client. That is a far worse failure than the memory it saves, so it is +// asserted rather than assumed. +func TestBetaProbeNeverMutatesTheOutboundBody(t *testing.T) { + body := []byte(`{"model":"claude-opus-4-8","tools":[` + + `{"type":"custom","name":"lookup","description":"Look something up",` + + `"input_schema":{"type":"object","properties":{"q":{"type":"string"}},"required":["q"]}},` + + `{"type":"computer_20250124","name":"computer","description":"Use the computer",` + + `"input_schema":{"type":"object"}}` + + `],"messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}]}`) + + // Byte-for-byte snapshot: aliasing bugs show up as in-place edits, not just as a + // different length. + original := append([]byte(nil), body...) + + for _, provider := range betaHeaderTestProviders() { + ctx := schemas.NewBifrostContext(nil, time.Time{}) + if err := AddMissingBetaHeadersToContextFromRawBody(ctx, body, provider); err != nil { + t.Fatalf("%s: %v", provider, err) + } + if !bytes.Equal(body, original) { + t.Fatalf("%s: the probe mutated the caller's body in place.\ngot: %s\nwant: %s", + provider, body, original) + } + } + + // Belt and braces: the fields the probe strips are still present and intact. + for i, want := range []string{"Look something up", "Use the computer"} { + if got := providerUtils.GetJSONField(body, fmt.Sprintf("tools.%d.description", i)).String(); got != want { + t.Errorf("tools.%d.description = %q, want %q", i, got, want) + } + if !providerUtils.JSONFieldExists(body, fmt.Sprintf("tools.%d.input_schema", i)) { + t.Errorf("tools.%d.input_schema was removed from the outbound body", i) + } + } + if got := providerUtils.GetJSONField(body, "tools.0.input_schema.properties.q.type").String(); got != "string" { + t.Errorf("the nested schema was altered: tools.0.input_schema.properties.q.type = %q, want \"string\"", got) + } +} diff --git a/core/providers/anthropicmsg/passthrough.go b/core/providers/anthropicmsg/passthrough.go index 96c197e6cd7..ff50b334753 100644 --- a/core/providers/anthropicmsg/passthrough.go +++ b/core/providers/anthropicmsg/passthrough.go @@ -10,6 +10,7 @@ import ( providerUtils "github.com/capsohq/bifrost/core/providers/utils" "github.com/capsohq/bifrost/core/schemas" + "github.com/tidwall/gjson" "github.com/valyala/fasthttp" ) @@ -33,6 +34,10 @@ type Config struct { NetworkConfig schemas.NetworkConfig CustomProviderConfig *schemas.CustomProviderConfig Logger schemas.Logger + // StripAnthropicOnlyFields is set by providers that are not Anthropic but serve + // its Messages API (xAI): the body loses the Claude Code fields only Anthropic + // accepts, as upstream strips them on every converted non-Anthropic request. + StripAnthropicOnlyFields bool } func (c Config) requestURL(req *schemas.BifrostPassthroughRequest) string { @@ -43,6 +48,67 @@ func (c Config) requestURL(req *schemas.BifrostPassthroughRequest) string { return url } +func (c Config) requestBody(req *schemas.BifrostPassthroughRequest) ([]byte, *schemas.BifrostError) { + if !c.StripAnthropicOnlyFields || !IsMessagesPath(req.Path) { + return req.Body, nil + } + body, err := StripAnthropicOnlyFields(req.Body) + if err != nil { + return nil, providerUtils.NewBifrostOperationError("failed to prepare native Messages request body", err) + } + return body, nil +} + +// StripAnthropicOnlyFields removes from a native Messages body what only Anthropic +// should receive: Claude Code's undocumented `safeguards` and `diagnostics` fields, +// and its per-request billing metadata line in `system`. The billing line changes on +// every request, so left in place it would defeat the provider's prompt cache +// (upstream #7393, #7243, #7431). A body without them is returned unchanged. +func StripAnthropicOnlyFields(body []byte) ([]byte, error) { + var err error + if providerUtils.JSONFieldExists(body, "safeguards") { + if body, err = providerUtils.DeleteJSONField(body, "safeguards"); err != nil { + return nil, fmt.Errorf("strip safeguards: %w", err) + } + } + if providerUtils.JSONFieldExists(body, "diagnostics") { + if body, err = providerUtils.DeleteJSONField(body, "diagnostics"); err != nil { + return nil, fmt.Errorf("strip diagnostics: %w", err) + } + } + system := providerUtils.GetJSONField(body, "system") + switch { + case system.Type == gjson.String: + if schemas.IsStandaloneAnthropicBillingHeader(system.Str) { + if body, err = providerUtils.DeleteJSONField(body, "system"); err != nil { + return nil, fmt.Errorf("strip billing header: %w", err) + } + } + case system.IsArray(): + blocks := system.Array() + kept := make([]string, 0, len(blocks)) + for _, block := range blocks { + text := block.Get("text") + if block.Get("type").Str == "text" && text.Type == gjson.String && schemas.IsStandaloneAnthropicBillingHeader(text.Str) { + continue + } + kept = append(kept, block.Raw) + } + if len(kept) == len(blocks) { + return body, nil + } + if len(kept) == 0 { + body, err = providerUtils.DeleteJSONField(body, "system") + } else { + body, err = providerUtils.SetRawJSONField(body, "system", []byte("["+strings.Join(kept, ",")+"]")) + } + if err != nil { + return nil, fmt.Errorf("strip billing header: %w", err) + } + } + return body, nil +} + func (c Config) setHeaders(ctx *schemas.BifrostContext, req *fasthttp.Request, key schemas.Key, safeHeaders map[string]string) { providerUtils.SetExtraHeaders(ctx, req, c.NetworkConfig.ExtraHeaders, nil) for name, value := range safeHeaders { @@ -84,7 +150,11 @@ func Passthrough(ctx *schemas.BifrostContext, client *fasthttp.Client, key schem providerUtils.StripCallerAuthForInsecureURL(requestURL, req.SafeHeaders) cfg.setHeaders(ctx, upstreamReq, key, req.SafeHeaders) if !providerUtils.ApplyLargePayloadRequestBodyWithModelNormalization(ctx, upstreamReq, cfg.Provider) { - upstreamReq.SetBody(req.Body) + body, bifrostErr := cfg.requestBody(req) + if bifrostErr != nil { + return nil, bifrostErr + } + upstreamReq.SetBody(body) } latency, bifrostErr, wait := providerUtils.MakeRequestWithContext(ctx, client, upstreamReq, resp) @@ -132,7 +202,12 @@ func PassthroughStream(ctx *schemas.BifrostContext, postHookRunner schemas.PostH cfg.setHeaders(ctx, upstreamReq, key, req.SafeHeaders) upstreamReq.Header.Set("Connection", "close") if !providerUtils.ApplyLargePayloadRequestBodyWithModelNormalization(ctx, upstreamReq, cfg.Provider) { - upstreamReq.SetBody(req.Body) + body, bifrostErr := cfg.requestBody(req) + if bifrostErr != nil { + fasthttp.ReleaseResponse(resp) + return nil, bifrostErr + } + upstreamReq.SetBody(body) } activeClient := providerUtils.PrepareResponseStreaming(ctx, streamingClient, resp) diff --git a/core/providers/anthropicmsg/passthrough_test.go b/core/providers/anthropicmsg/passthrough_test.go index 3cd2b05602b..3a5f1938144 100644 --- a/core/providers/anthropicmsg/passthrough_test.go +++ b/core/providers/anthropicmsg/passthrough_test.go @@ -210,3 +210,76 @@ func TestExtractUsagePreservesThinkingTokensAndInferenceGeo(t *testing.T) { func testKey(value string) schemas.Key { return schemas.Key{Value: schemas.SecretVar{Val: value}} } + +func TestStripAnthropicOnlyFields(t *testing.T) { + const billing = "x-anthropic-billing-header: cc_version=2.1.80.a1b; cc_entrypoint=cli; cch=0f3c2;" + cases := []struct { + name, body, want string + }{ + { + name: "string system that is only the billing line", + body: `{"model":"m","system":"` + billing + `","messages":[]}`, + want: `{"model":"m","messages":[]}`, + }, + { + name: "billing block removed, others kept in order with their cache markers", + body: `{"system":[{"type":"text","text":"a"},{"type":"text","text":"` + billing + `"},{"type":"text","text":"b","cache_control":{"type":"ephemeral"}}]}`, + want: `{"system":[{"type":"text","text":"a"},{"type":"text","text":"b","cache_control":{"type":"ephemeral"}}]}`, + }, + { + name: "system of only billing blocks is dropped", + body: `{"system":[{"type":"text","text":"` + billing + `"}],"messages":[]}`, + want: `{"messages":[]}`, + }, + { + name: "billing line mixed with instructions is real system content", + body: `{"system":"` + billing + ` Always answer in French."}`, + want: `{"system":"` + billing + ` Always answer in French."}`, + }, + { + name: "safeguards and diagnostics removed", + body: `{"model":"m","safeguards":{"a":1},"diagnostics":{"previous_message_id":"x"},"max_tokens":8}`, + want: `{"model":"m","max_tokens":8}`, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := StripAnthropicOnlyFields([]byte(tc.body)) + if err != nil { + t.Fatalf("StripAnthropicOnlyFields: %v", err) + } + if string(got) != tc.want { + t.Fatalf("got %s\nwant %s", got, tc.want) + } + }) + } + + clean := []byte(`{"model":"m","system":[{"type":"text","text":"keep"}],"messages":[]}`) + got, err := StripAnthropicOnlyFields(clean) + if err != nil || &got[0] != &clean[0] { + t.Fatalf("a body with nothing to strip must be returned as is (err %v)", err) + } +} + +// Anthropic itself must still receive Claude Code's fields: the strip is opt-in. +func TestPassthroughKeepsAnthropicOnlyFieldsUnlessAsked(t *testing.T) { + body := []byte(`{"model":"claude","safeguards":{"a":1},"system":"x-anthropic-billing-header: cc_version=1;","messages":[]}`) + received := make(chan []byte, 1) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + got, _ := io.ReadAll(r.Body) + received <- got + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"type":"message","content":[]}`)) + })) + defer server.Close() + + ctx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) + if _, bifrostErr := Passthrough(ctx, &fasthttp.Client{}, testKey("k"), &schemas.BifrostPassthroughRequest{ + Method: http.MethodPost, Path: "/v1/messages", Body: body, + }, Config{Provider: schemas.Anthropic, Endpoint: server.URL}); bifrostErr != nil { + t.Fatalf("Passthrough: %+v", bifrostErr) + } + if got := <-received; !bytes.Equal(got, body) { + t.Fatalf("Anthropic body changed:\n got: %s\nwant: %s", got, body) + } +} diff --git a/core/providers/azure/azure.go b/core/providers/azure/azure.go index 3f6395c960b..0f8f180aca7 100644 --- a/core/providers/azure/azure.go +++ b/core/providers/azure/azure.go @@ -734,6 +734,11 @@ func (provider *AzureProvider) Rerank(ctx *schemas.BifrostContext, key schemas.K return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey()) } +// Decision is not supported by the Azure provider. +func (provider *AzureProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR is not supported by the Azure provider. func (provider *AzureProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) diff --git a/core/providers/bedrock/bedrock.go b/core/providers/bedrock/bedrock.go index f1b42419647..e127c628c82 100644 --- a/core/providers/bedrock/bedrock.go +++ b/core/providers/bedrock/bedrock.go @@ -2415,6 +2415,11 @@ func (provider *BedrockProvider) Rerank(ctx *schemas.BifrostContext, key schemas return bifrostResponse, nil } +// Decision is not supported by the Bedrock provider. +func (provider *BedrockProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR is not supported by the Bedrock provider. func (provider *BedrockProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) @@ -4378,6 +4383,32 @@ func toolNeedsAnthropicInvokePath(toolType string, deferLoading *bool) bool { return deferLoading != nil && *deferLoading } +// safeguardsSurviveStrip reports whether the shared Anthropic strip gate would +// keep safeguards for this model. Only supported requests need InvokeModel, +// where the native field and its required beta are assembled together. +func safeguardsSurviveStrip(ctx *schemas.BifrostContext, model string) bool { + capModel := schemas.ResolveCanonicalModel(ctx, model) + caps := schemas.ResolveModelCaps(schemas.Bedrock, capModel) + return anthropic.ProviderFeatures[schemas.Bedrock].Safeguards && + caps.SupportsSafeguards(anthropic.DefaultSupportsSafeguards(schemas.Bedrock, caps.Model())) +} + +// extraParamsHasSafeguards accepts opaque JSON and programmatic parameter values. +func extraParamsHasSafeguards(extraParams map[string]interface{}) bool { + v, exists := extraParams["safeguards"] + if !exists || v == nil { + return false + } + switch val := v.(type) { + case json.RawMessage: + return len(val) > 0 + case []byte: + return len(val) > 0 + default: + return true + } +} + func chatUsesAnthropicInvokePath(ctx *schemas.BifrostContext, request *schemas.BifrostChatRequest) bool { if request == nil || request.Params == nil { return false @@ -4388,6 +4419,9 @@ func chatUsesAnthropicInvokePath(ctx *schemas.BifrostContext, request *schemas.B if !schemas.IsAnthropicModelFamily(ctx, request.Model) { return false } + if extraParamsHasSafeguards(request.Params.ExtraParams) && safeguardsSurviveStrip(ctx, request.Model) { + return true + } for _, tool := range request.Params.Tools { if toolNeedsAnthropicInvokePath(string(tool.Type), tool.DeferLoading) { return true @@ -4406,6 +4440,9 @@ func responsesUsesAnthropicInvokePath(ctx *schemas.BifrostContext, request *sche if !schemas.IsAnthropicModelFamily(ctx, request.Model) { return false } + if extraParamsHasSafeguards(request.Params.ExtraParams) && safeguardsSurviveStrip(ctx, request.Model) { + return true + } for _, tool := range request.Params.Tools { if toolNeedsAnthropicInvokePath(string(tool.Type), tool.DeferLoading) { return true diff --git a/core/providers/bedrock/bedrock_test.go b/core/providers/bedrock/bedrock_test.go index 8170cfa863a..30d1c1914aa 100644 --- a/core/providers/bedrock/bedrock_test.go +++ b/core/providers/bedrock/bedrock_test.go @@ -3002,6 +3002,46 @@ func TestToolResultJSONParsingResponsesAPI(t *testing.T) { expectedContentType: "json", expectedJSON: mustMarshalJSON(map[string]any{"results": []any{}}), }, + // Converse rejects a json document containing an empty-string object key with + // "The format of the value at ...toolResult.content.N.json is invalid" (verified + // live against us.anthropic.claude-haiku-4-5). Cursor's list_directory results + // carry such keys for extensionless files, so these payloads must fall back to a + // text block holding the original JSON string. + { + name: "EmptyKeyObjectFallsBackToText", + toolResultContent: `{"success":{"fullSubtreeExtensionCounts":{"":2,".md":1},"numFiles":3}}`, + expectedContentType: "text", + expectedText: schemas.Ptr(`{"success":{"fullSubtreeExtensionCounts":{"":2,".md":1},"numFiles":3}}`), + }, + { + name: "EmptyKeyInsideArrayFallsBackToText", + toolResultContent: `[{"path":"/repo","counts":{"":1}}]`, + expectedContentType: "text", + expectedText: schemas.Ptr(`[{"path":"/repo","counts":{"":1}}]`), + }, + { + // An empty string as a VALUE is fine; only empty keys are rejected. + name: "EmptyStringValueStaysJSON", + toolResultContent: `{"a":""}`, + expectedContentType: "json", + expectedJSON: mustMarshalJSON(map[string]any{"a": ""}), + }, + { + // Empty string value followed by an empty key: the detector must not + // confuse a value in key position with a key. + name: "EmptyValueThenEmptyKeyFallsBackToText", + toolResultContent: `{"a":"","":1}`, + expectedContentType: "text", + expectedText: schemas.Ptr(`{"a":"","":1}`), + }, + { + // Empty key appearing after a nested container in the same object: the + // detector must keep checking sibling keys after descending. + name: "EmptyKeyAfterNestedContainerFallsBackToText", + toolResultContent: `{"a":{"b":[1,2]},"":2}`, + expectedContentType: "text", + expectedText: schemas.Ptr(`{"a":{"b":[1,2]},"":2}`), + }, } for _, tt := range tests { @@ -6081,7 +6121,7 @@ func TestToBedrockChatCompletionRequest_AliasesLongMCPToolNames(t *testing.T) { assert.NotEqual(t, toolName, alias) assert.Contains(t, alias, "_list_network_requests") assert.Regexp(t, `^[A-Za-z0-9_-]{1,64}$`, alias) - assert.Regexp(t, `^[0-9a-f]{8}_`, alias) + assert.Regexp(t, `^t[0-9a-f]{8}_`, alias) require.NotNil(t, result.ToolConfig.ToolChoice) require.NotNil(t, result.ToolConfig.ToolChoice.Tool) assert.Equal(t, alias, result.ToolConfig.ToolChoice.Tool.Name) @@ -6125,7 +6165,7 @@ func TestToBedrockChatCompletionRequest_AliasesToolNamesWithInvalidChars(t *test alias := result.ToolConfig.Tools[0].ToolSpec.Name assert.NotEqual(t, toolName, alias, "name with disallowed chars must be aliased") assert.Regexp(t, `^[A-Za-z0-9_-]{1,64}$`, alias) - assert.Regexp(t, `^[0-9a-f]{8}_`, alias) + assert.Regexp(t, `^t[0-9a-f]{8}_`, alias) require.NotNil(t, result.ToolConfig.ToolChoice) require.NotNil(t, result.ToolConfig.ToolChoice.Tool) assert.Equal(t, alias, result.ToolConfig.ToolChoice.Tool.Name) @@ -6220,7 +6260,7 @@ func TestToBedrockResponsesRequest_AliasesLongMCPToolNames(t *testing.T) { assert.NotEqual(t, toolName, alias) assert.Contains(t, alias, "_notion-notion-search") assert.Regexp(t, `^[A-Za-z0-9_-]{1,64}$`, alias) - assert.Regexp(t, `^[0-9a-f]{8}_`, alias) + assert.Regexp(t, `^t[0-9a-f]{8}_`, alias) require.NotNil(t, result.ToolConfig.ToolChoice) require.NotNil(t, result.ToolConfig.ToolChoice.Tool) assert.Equal(t, alias, result.ToolConfig.ToolChoice.Tool.Name) diff --git a/core/providers/bedrock/converseresponsereasoning_test.go b/core/providers/bedrock/converseresponsereasoning_test.go index 450a631c21d..9dd03ef868d 100644 --- a/core/providers/bedrock/converseresponsereasoning_test.go +++ b/core/providers/bedrock/converseresponsereasoning_test.go @@ -2,8 +2,11 @@ package bedrock import ( "context" + "encoding/base64" + "encoding/json" "testing" + "github.com/bytedance/sonic" "github.com/capsohq/bifrost/core/schemas" "github.com/stretchr/testify/require" ) @@ -93,7 +96,7 @@ func TestConverseResponseRendersUpstreamReasoning(t *testing.T) { ResponsesReasoning: &schemas.ResponsesReasoning{ Summary: []schemas.ResponsesReasoningSummary{{Type: "summary_text", Text: "simulate night by night"}}, }, - }, converseReasoningShape("xai/grok-4-0709"), converseRequiresSignedReasoning("xai/grok-4-0709")) + }, converseReasoningShape("xai/grok-4-0709"), converseRequiresSignedReasoning("xai/grok-4-0709"), false) require.Empty(t, blocks, "a replay to Bedrock must not send a block Bedrock cannot verify") }) } @@ -144,6 +147,87 @@ func TestConverseResponseMixedReasoningRoundTrip(t *testing.T) { } } +// Converse declares reasoningContent.redactedContent a blob, so strict clients +// (aws-sdk-go-v2, boto3) base64-decode it. A non-Bedrock upstream's encrypted +// reasoning is an arbitrary token: OpenAI and Azure mint URL-safe Fernet tokens, +// whose '-' and '_' are illegal in standard base64. Rendering one raw made the +// SDK fail with "decode base64 blob: illegal base64 data" (#7514). +const foreignReasoningToken = "gAAAAABo1x-Kq2_Zr8vJ3mNcY0tW5bQpL7hEfGdA==" + +func renderRedactedReasoningWithToolCall(t *testing.T, token string) string { + t.Helper() + resp, err := ToBedrockConverseResponse(&schemas.BifrostResponsesResponse{ + Model: "gpt-5.6-luna", + Output: []schemas.ResponsesMessage{ + { + Type: schemas.Ptr(schemas.ResponsesMessageTypeReasoning), + ResponsesReasoning: &schemas.ResponsesReasoning{ + Summary: []schemas.ResponsesReasoningSummary{}, + EncryptedContent: &token, + }, + }, + { + Type: schemas.Ptr(schemas.ResponsesMessageTypeFunctionCall), + ResponsesToolMessage: &schemas.ResponsesToolMessage{ + CallID: schemas.Ptr("call_1"), + Name: schemas.Ptr("get_weather"), + Arguments: schemas.Ptr(`{"city":"Paris"}`), + }, + }, + }, + }) + require.NoError(t, err) + content := resp.Output.Message.Content + require.Len(t, content, 2, "reasoning block then toolUse block, got %+v", content) + require.NotNil(t, content[0].ReasoningContent) + require.NotNil(t, content[0].ReasoningContent.RedactedContent, "the encrypted token must render as redactedContent") + require.NotNil(t, content[1].ToolUse) + return *content[0].ReasoningContent.RedactedContent +} + +func TestConverseResponseRedactedContentIsStandardBase64(t *testing.T) { + redacted := renderRedactedReasoningWithToolCall(t, foreignReasoningToken) + _, err := base64.StdEncoding.DecodeString(redacted) + require.NoError(t, err, "redactedContent is a Converse blob and must be standard base64, got %q", redacted) +} + +// A Converse client replays redactedContent as the bytes it decoded, re-encoded +// canonically. The next turn must hand the upstream its original token back. +func TestConverseRedactedContentSurvivesSDKReplay(t *testing.T) { + for name, token := range map[string]string{ + "foreign token": foreignReasoningToken, + "native bedrock blob": "cnNuXzVaVnJpZjRKMGJYSXFtV2RsZWRqN1FJRmVGZWdz", + } { + t.Run(name, func(t *testing.T) { + redacted := renderRedactedReasoningWithToolCall(t, token) + sdkBytes, err := base64.StdEncoding.DecodeString(redacted) + require.NoError(t, err, "the SDK rejects a non-base64 blob, got %q", redacted) + replayed := base64.StdEncoding.EncodeToString(sdkBytes) + + req := &BedrockConverseRequest{ + ModelID: "azure/gpt-5.6-luna", + Messages: []BedrockMessage{ + {Role: BedrockMessageRoleUser, Content: []BedrockContentBlock{{Text: schemas.Ptr("weather in Paris?")}}}, + {Role: BedrockMessageRoleAssistant, Content: []BedrockContentBlock{ + {ReasoningContent: &BedrockReasoningContent{RedactedContent: &replayed}}, + {ToolUse: &BedrockToolUse{ToolUseID: "call_1", Name: "get_weather", Input: json.RawMessage(`{"city":"Paris"}`)}}, + }}, + }, + } + bifrostReq, err := req.ToBifrostResponsesRequest(schemas.NewBifrostContext(context.Background(), schemas.NoDeadline)) + require.NoError(t, err) + var encrypted *string + for _, msg := range bifrostReq.Input { + if msg.Type != nil && *msg.Type == schemas.ResponsesMessageTypeReasoning && msg.ResponsesReasoning != nil { + encrypted = msg.ResponsesReasoning.EncryptedContent + } + } + require.NotNil(t, encrypted, "the replayed reasoning item must carry encrypted_content") + require.Equal(t, token, *encrypted, "the upstream must get back exactly the token it minted") + }) + } +} + func TestConverseResponseSummarySignatureIsNotRedactedContent(t *testing.T) { blocks := convertBifrostReasoningToConverseResponseReasoning(&schemas.ResponsesMessage{ ResponsesReasoning: &schemas.ResponsesReasoning{ @@ -155,3 +239,42 @@ func TestConverseResponseSummarySignatureIsNotRedactedContent(t *testing.T) { require.Equal(t, "summary-signature", *blocks[0].ReasoningContent.ReasoningText.Signature) require.Nil(t, blocks[0].ReasoningContent.RedactedContent) } + +// Over HTTP, BedrockConverseRequest.UnmarshalJSON keeps unknown top-level fields as +// json.RawMessage. include and reasoning_summary were read with SafeExtract*, which +// has no RawMessage case, so a Converse client could never ask a non-Bedrock +// upstream for encrypted reasoning or a summary level: both were silently dropped. +func TestConverseRequestHonorsIncludeAndReasoningSummaryFromHTTPBody(t *testing.T) { + body := `{"messages":[{"role":"user","content":[{"text":"hi"}]}],` + + `"additionalModelRequestFields":{"reasoning_config":{"type":"enabled","budget_tokens":1024}},` + + `"include":["reasoning.encrypted_content"],"reasoning_summary":"detailed"}` + var req BedrockConverseRequest + require.NoError(t, sonic.Unmarshal([]byte(body), &req)) + req.ModelID = "azure/gpt-5.6-luna" + + bifrostReq, err := req.ToBifrostResponsesRequest(schemas.NewBifrostContext(context.Background(), schemas.NoDeadline)) + require.NoError(t, err) + require.Equal(t, []string{"reasoning.encrypted_content"}, bifrostReq.Params.Include, "include from the HTTP body must reach the upstream request") + require.NotNil(t, bifrostReq.Params.Reasoning) + require.NotNil(t, bifrostReq.Params.Reasoning.Summary) + require.Equal(t, "detailed", *bifrostReq.Params.Reasoning.Summary, "reasoning_summary from the HTTP body must win over the auto default") +} + +// A JSON null decodes into a Go string without error, so a raw `null` used to come +// back as a pointer to "": non-nil, which suppressed the "auto" summary default and +// left an OpenAI reasoning model returning no reasoning text. null means absent. +func TestConverseRequestTreatsNullReasoningSummaryAsAbsent(t *testing.T) { + body := `{"messages":[{"role":"user","content":[{"text":"hi"}]}],` + + `"additionalModelRequestFields":{"reasoning_config":{"type":"enabled","budget_tokens":1024}},` + + `"include":null,"reasoning_summary":null}` + var req BedrockConverseRequest + require.NoError(t, sonic.Unmarshal([]byte(body), &req)) + req.ModelID = "azure/gpt-5.6-luna" + + bifrostReq, err := req.ToBifrostResponsesRequest(schemas.NewBifrostContext(context.Background(), schemas.NoDeadline)) + require.NoError(t, err) + require.Nil(t, bifrostReq.Params.Include, "a null include is absent") + require.NotNil(t, bifrostReq.Params.Reasoning) + require.NotNil(t, bifrostReq.Params.Reasoning.Summary, "a null reasoning_summary must still get the auto default") + require.Equal(t, "auto", *bifrostReq.Params.Reasoning.Summary, "a null reasoning_summary must not suppress the auto default") +} diff --git a/core/providers/bedrock/invoke.go b/core/providers/bedrock/invoke.go index f917793b4e3..dd6a4f890a4 100644 --- a/core/providers/bedrock/invoke.go +++ b/core/providers/bedrock/invoke.go @@ -114,6 +114,12 @@ func (r *BedrockInvokeRequest) UnmarshalJSON(data []byte) error { } r.Messages = append(r.Messages, msg) } + // The standard path above translates cache_control into cache + // points; this one has to as well. A single message whose + // content is a bare string diverts the whole request here, so + // skipping it dropped cache_control from every other message in + // the same request too. + r.Messages = applyMessageContentCacheControl(r.Messages, aux.Messages) } } } diff --git a/core/providers/bedrock/invoke_test.go b/core/providers/bedrock/invoke_test.go index af81317ac5b..d061e8806d1 100644 --- a/core/providers/bedrock/invoke_test.go +++ b/core/providers/bedrock/invoke_test.go @@ -1131,6 +1131,69 @@ func TestUsesAnthropicInvokePath_ExtraParamsShapes(t *testing.T) { }) } +// Supported safeguards requests use InvokeModel so the native field and beta +// reach Claude together. Absent or unsupported safeguards preserve Converse. +func TestResponsesUsesAnthropicInvokePath_Safeguards(t *testing.T) { + ctx := schemas.NewBifrostContext(context.Background(), time.Time{}) + model := "us.anthropic.claude-sonnet-5" + + t.Run("nonempty safeguards diverts to invoke", func(t *testing.T) { + for _, v := range []interface{}{ + json.RawMessage(`{"check":"auto_mode"}`), + []byte(`{"check":"auto_mode"}`), + json.RawMessage(``), + nil, + } { + req := &schemas.BifrostResponsesRequest{Model: model, Params: &schemas.ResponsesParameters{ + ExtraParams: map[string]interface{}{"safeguards": v}, + }} + want := extraParamsHasSafeguards(req.Params.ExtraParams) + require.Equal(t, want, responsesUsesAnthropicInvokePath(ctx, req)) + require.Equal(t, want, chatUsesAnthropicInvokePath(ctx, &schemas.BifrostChatRequest{Model: model, Params: &schemas.ChatParameters{ExtraParams: req.Params.ExtraParams}})) + } + }) + t.Run("absent safeguards stays on converse", func(t *testing.T) { + req := &schemas.BifrostResponsesRequest{Model: model, Params: &schemas.ResponsesParameters{}} + require.False(t, responsesUsesAnthropicInvokePath(ctx, req)) + }) + t.Run("unsupported model stays on converse", func(t *testing.T) { + extra := map[string]interface{}{"safeguards": json.RawMessage(`{"check":"auto_mode"}`)} + require.False(t, responsesUsesAnthropicInvokePath(ctx, &schemas.BifrostResponsesRequest{Model: "us.anthropic.claude-haiku-4-5", Params: &schemas.ResponsesParameters{ExtraParams: extra}})) + require.False(t, chatUsesAnthropicInvokePath(ctx, &schemas.BifrostChatRequest{Model: "us.anthropic.claude-haiku-4-5", Params: &schemas.ChatParameters{ExtraParams: extra}})) + }) + t.Run("catalog can disable invoke routing", func(t *testing.T) { + schemas.SetCapabilityResolver(func(schemas.ModelProvider, string) *schemas.ModelCapabilities { + return &schemas.ModelCapabilities{SupportsSafeguards: schemas.Ptr(false)} + }) + defer schemas.SetCapabilityResolver(nil) + extra := map[string]interface{}{"safeguards": json.RawMessage(`{"check":"auto_mode"}`)} + require.False(t, responsesUsesAnthropicInvokePath(ctx, &schemas.BifrostResponsesRequest{Model: model, Params: &schemas.ResponsesParameters{ExtraParams: extra}})) + require.False(t, chatUsesAnthropicInvokePath(ctx, &schemas.BifrostChatRequest{Model: model, Params: &schemas.ChatParameters{ExtraParams: extra}})) + }) + t.Run("non-anthropic model stays on converse even with safeguards", func(t *testing.T) { + req := &schemas.BifrostResponsesRequest{Model: "us.amazon.nova-lite-v1:0", Params: &schemas.ResponsesParameters{ + ExtraParams: map[string]interface{}{"safeguards": json.RawMessage(`{"check":"auto_mode"}`)}, + }} + require.False(t, responsesUsesAnthropicInvokePath(ctx, req)) + }) + // Catalog overrides remain authoritative for model support. + t.Run("datasheet-enabled pair diverts to invoke", func(t *testing.T) { + yes := true + schemas.SetCapabilityResolver(func(p schemas.ModelProvider, m string) *schemas.ModelCapabilities { + if p == schemas.Bedrock { + return &schemas.ModelCapabilities{SupportsSafeguards: &yes} + } + return nil + }) + defer schemas.SetCapabilityResolver(nil) + + req := &schemas.BifrostResponsesRequest{Model: model, Params: &schemas.ResponsesParameters{ + ExtraParams: map[string]interface{}{"safeguards": json.RawMessage(`{"check":"auto_mode"}`)}, + }} + require.True(t, responsesUsesAnthropicInvokePath(ctx, req)) + }) +} + // The InvokeModel route (and Mantle) go through fasthttp, whose path // normalisation decodes a percent-encoded inference-profile ARN in the model // segment. The provider must build its fasthttp clients with that disabled; diff --git a/core/providers/bedrock/invokestringcontentcache_test.go b/core/providers/bedrock/invokestringcontentcache_test.go new file mode 100644 index 00000000000..ca0d1cd500e --- /dev/null +++ b/core/providers/bedrock/invokestringcontentcache_test.go @@ -0,0 +1,168 @@ +package bedrock + +import ( + "testing" + + "github.com/bytedance/sonic" +) + +// The Anthropic Messages API accepts a message's content either as an array of +// content blocks or as a plain string, and clients mix the two freely: a +// rolling cache breakpoint tends to produce block arrays for the turns that +// carry cache_control and bare strings for the rest. +// +// BedrockMessage.Content is typed []BedrockContentBlock, so one bare string +// anywhere in messages[] fails the standard unmarshal and diverts the whole +// request into the AI21 fallback. That branch rebuilt the messages but did not +// translate cache_control into cache points, so prompt caching went silently +// off for every message in the request, not only the string one. +// +// Ref: https://github.com/maximhq/bifrost/issues/7336 + +func cachePointCount(msg BedrockMessage) int { + n := 0 + for _, block := range msg.Content { + if block.CachePoint != nil { + n++ + } + } + return n +} + +func unmarshalInvoke(t *testing.T, payload string) *BedrockInvokeRequest { + t.Helper() + req := &BedrockInvokeRequest{} + if err := sonic.Unmarshal([]byte(payload), req); err != nil { + t.Fatalf("UnmarshalJSON: %v", err) + } + return req +} + +// TestInvokeUnmarshal_CacheControlSurvivesAStringContentMessage is the reported +// failure: the string message is what diverts the request, and the cache point +// belongs to a different message entirely. +func TestInvokeUnmarshal_CacheControlSurvivesAStringContentMessage(t *testing.T) { + req := unmarshalInvoke(t, `{ + "anthropic_version": "bedrock-2023-05-31", + "messages": [ + {"role": "user", "content": "an earlier turn, stored as a plain string"}, + {"role": "assistant", "content": [{"type": "text", "text": "reply"}]}, + {"role": "user", "content": [ + {"type": "text", "text": "the turn that carries the breakpoint", + "cache_control": {"type": "ephemeral"}} + ]} + ] + }`) + + if len(req.Messages) != 3 { + t.Fatalf("parsed %d messages, want 3", len(req.Messages)) + } + if got := cachePointCount(req.Messages[2]); got != 1 { + t.Errorf("cache points on the breakpoint message = %d, want 1: cache_control was dropped", got) + } + if got := cachePointCount(req.Messages[0]); got != 0 { + t.Errorf("cache points on the string message = %d, want 0", got) + } + if req.Messages[0].Content[0].Text == nil || + *req.Messages[0].Content[0].Text != "an earlier turn, stored as a plain string" { + t.Errorf("the string message lost its text: %+v", req.Messages[0].Content) + } +} + +// TestInvokeUnmarshal_CacheControlOnAStringOnlyRequest pins that a request with +// no block arrays at all still parses and grows no cache points out of nowhere. +func TestInvokeUnmarshal_CacheControlOnAStringOnlyRequest(t *testing.T) { + req := unmarshalInvoke(t, `{ + "messages": [ + {"role": "user", "content": "one"}, + {"role": "assistant", "content": "two"} + ] + }`) + + if len(req.Messages) != 2 { + t.Fatalf("parsed %d messages, want 2", len(req.Messages)) + } + for i, msg := range req.Messages { + if got := cachePointCount(msg); got != 0 { + t.Errorf("message %d has %d cache points, want 0", i, got) + } + if len(msg.Content) != 1 || msg.Content[0].Text == nil { + t.Errorf("message %d content = %+v, want one text block", i, msg.Content) + } + } +} + +// TestInvokeUnmarshal_CacheControlStillWorksWithoutAStringMessage is the +// control: the standard path was never broken, and must stay that way. +func TestInvokeUnmarshal_CacheControlStillWorksWithoutAStringMessage(t *testing.T) { + req := unmarshalInvoke(t, `{ + "messages": [ + {"role": "user", "content": [ + {"type": "text", "text": "hello", "cache_control": {"type": "ephemeral"}} + ]} + ] + }`) + + if got := cachePointCount(req.Messages[0]); got != 1 { + t.Errorf("cache points = %d, want 1", got) + } +} + +// TestInvokeUnmarshal_EveryBreakpointSurvivesAStringMessage covers the shape a +// rolling breakpoint actually produces: several marked turns, with strings +// interleaved. Losing any one of them costs the cache. +func TestInvokeUnmarshal_EveryBreakpointSurvivesAStringMessage(t *testing.T) { + req := unmarshalInvoke(t, `{ + "messages": [ + {"role": "user", "content": [ + {"type": "text", "text": "first", "cache_control": {"type": "ephemeral"}} + ]}, + {"role": "assistant", "content": "a plain string in the middle"}, + {"role": "user", "content": [ + {"type": "text", "text": "second", "cache_control": {"type": "ephemeral"}} + ]} + ] + }`) + + if len(req.Messages) != 3 { + t.Fatalf("parsed %d messages, want 3", len(req.Messages)) + } + for _, i := range []int{0, 2} { + if got := cachePointCount(req.Messages[i]); got != 1 { + t.Errorf("cache points on message %d = %d, want 1", i, got) + } + } +} + +// TestInvokeUnmarshal_ToolResultCacheControlSurvivesAStringMessage pins the +// nested case, since injectContentBlockCachePoints recurses into tool results +// and the fallback path skipped that recursion along with everything else. +func TestInvokeUnmarshal_ToolResultCacheControlSurvivesAStringMessage(t *testing.T) { + req := unmarshalInvoke(t, `{ + "messages": [ + {"role": "user", "content": "a plain string that diverts the parse"}, + {"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "call_1", "content": [ + {"type": "text", "text": "tool output", "cache_control": {"type": "ephemeral"}} + ]} + ]} + ] + }`) + + if len(req.Messages) != 2 { + t.Fatalf("parsed %d messages, want 2", len(req.Messages)) + } + blocks := req.Messages[1].Content + if len(blocks) == 0 || blocks[0].ToolResult == nil { + t.Fatalf("second message lost its tool_result: %+v", blocks) + } + found := 0 + for _, inner := range blocks[0].ToolResult.Content { + if inner.CachePoint != nil { + found++ + } + } + if found != 1 { + t.Errorf("cache points inside the tool result = %d, want 1", found) + } +} diff --git a/core/providers/bedrock/reasoning_replay_test.go b/core/providers/bedrock/reasoning_replay_test.go index a7b2d94c2a5..6f162e13918 100644 --- a/core/providers/bedrock/reasoning_replay_test.go +++ b/core/providers/bedrock/reasoning_replay_test.go @@ -199,7 +199,7 @@ func TestConvertBifrostReasoningToBedrockReasoning(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - blocks := convertBifrostReasoningToBedrockReasoning(tc.msg, schemas.BedrockReasoningShapeText, false) + blocks := convertBifrostReasoningToBedrockReasoning(tc.msg, schemas.BedrockReasoningShapeText, false, false) require.Len(t, blocks, tc.wantBlocks) reasoningTextInvariant(t, blocks) @@ -239,7 +239,7 @@ func TestConvertBifrostReasoningToBedrockReasoningEncryptedContent(t *testing.T) }, } - blocks := convertBifrostReasoningToBedrockReasoning(message, schemas.BedrockReasoningShapeText, false) + blocks := convertBifrostReasoningToBedrockReasoning(message, schemas.BedrockReasoningShapeText, false, false) require.Len(t, blocks, 1) require.NotNil(t, blocks[0].ReasoningContent) require.NotNil(t, blocks[0].ReasoningContent.ReasoningText) @@ -261,7 +261,7 @@ func TestConvertBifrostReasoningToBedrockReasoningTextAlwaysSerialized(t *testin Summary: []schemas.ResponsesReasoningSummary{}, EncryptedContent: &signature, }, - }, schemas.BedrockReasoningShapeText, false) + }, schemas.BedrockReasoningShapeText, false, false) require.Len(t, blocks, 1) raw, err := sonic.Marshal(blocks[0]) @@ -450,7 +450,7 @@ func TestConvertBifrostReasoningToBedrockReasoningRedactedShape(t *testing.T) { Summary: []schemas.ResponsesReasoningSummary{}, EncryptedContent: &blob, }, - }, schemas.BedrockReasoningShapeRedacted, false) + }, schemas.BedrockReasoningShapeRedacted, false, false) require.Len(t, blocks, 1) require.NotNil(t, blocks[0].ReasoningContent) @@ -478,7 +478,7 @@ func TestConvertBifrostReasoningToBedrockReasoningRedactedShape(t *testing.T) { ResponsesReasoning: &schemas.ResponsesReasoning{ Summary: []schemas.ResponsesReasoningSummary{}, }, - }, schemas.BedrockReasoningShapeRedacted, false) + }, schemas.BedrockReasoningShapeRedacted, false, false) require.Empty(t, blocks, "an unreplayable block must be dropped, not reshaped") }) @@ -489,7 +489,7 @@ func TestConvertBifrostReasoningToBedrockReasoningRedactedShape(t *testing.T) { ResponsesReasoning: &schemas.ResponsesReasoning{ Summary: []schemas.ResponsesReasoningSummary{{Text: "step by step"}}, }, - }, schemas.BedrockReasoningShapeRedacted, false) + }, schemas.BedrockReasoningShapeRedacted, false, false) require.Empty(t, blocks) }) diff --git a/core/providers/bedrock/responses.go b/core/providers/bedrock/responses.go index 7d5b3be33f6..31abe7845e8 100644 --- a/core/providers/bedrock/responses.go +++ b/core/providers/bedrock/responses.go @@ -2279,7 +2279,7 @@ func (request *BedrockConverseRequest) ToBifrostResponsesRequest(ctx *schemas.Bi if typeStr, ok := schemas.SafeExtractString(reasoningConfigMap["type"]); ok { if typeStr == "enabled" || typeStr == "adaptive" { var summary *string - if summaryValue, ok := schemas.SafeExtractStringPointer(request.ExtraParams["reasoning_summary"]); ok { + if summaryValue, ok := extraParamStringPointer(request.ExtraParams["reasoning_summary"]); ok { summary = summaryValue } // Converse has no reasoning-summary field, so an OpenAI reasoning @@ -2371,7 +2371,7 @@ func (request *BedrockConverseRequest) ToBifrostResponsesRequest(ctx *schemas.Bi } } - if include, ok := schemas.SafeExtractStringSlice(request.ExtraParams["include"]); ok { + if include, ok := extraParamStringSlice(request.ExtraParams["include"]); ok { bifrostReq.Params.Include = include } @@ -4047,7 +4047,8 @@ func ConvertBifrostMessagesToBedrockMessages(ctx context.Context, model string, if isConverseResponseRendering(ctx) { reasoningBlocks = convertBifrostReasoningToConverseResponseReasoning(&msg) } else { - reasoningBlocks = convertBifrostReasoningToBedrockReasoning(&msg, converseReasoningShape(model), converseRequiresSignedReasoning(model)) + requireSigned := converseRequiresSignedReasoning(model) + reasoningBlocks = convertBifrostReasoningToBedrockReasoning(&msg, converseReasoningShape(model), requireSigned, !requireSigned) } if len(reasoningBlocks) > 0 { pendingReasoningContentBlocks = append(pendingReasoningContentBlocks, reasoningBlocks...) @@ -4631,8 +4632,10 @@ func convertSingleBedrockMessageToBifrostMessages(ctx *schemas.BifrostContext, m }) } else if block.ReasoningContent.RedactedContent != nil { // Opaque blob: carried on the reasoning message rather than as a - // content block, since there is no prose for one to hold. - reasoningRedactedContent = block.ReasoningContent.RedactedContent + // content block, since there is no prose for one to hold. A blob + // Bifrost wrapped for a Converse client unwraps to the upstream's + // own token; native Bedrock blobs pass through. + reasoningRedactedContent = new(decodeRedactedContentFromConverse(*block.ReasoningContent.RedactedContent)) } } else if block.ToolUse != nil { // Tool use content @@ -5044,6 +5047,22 @@ func isConverseResponseRendering(ctx context.Context) bool { // exposed text becomes reasoningText (signed when a signature exists), while an // encrypted-only block stays redactedContent. func convertBifrostReasoningToConverseResponseReasoning(msg *schemas.ResponsesMessage) []BedrockContentBlock { + return encodeConverseRedactedBlocks(renderConverseResponseReasoning(msg)) +} + +// encodeConverseRedactedBlocks makes every redactedContent a valid Converse blob. +// Only the client-facing render does this: replays to Bedrock carry Bedrock's own +// blob, which is already one. +func encodeConverseRedactedBlocks(blocks []BedrockContentBlock) []BedrockContentBlock { + for i := range blocks { + if rc := blocks[i].ReasoningContent; rc != nil && rc.RedactedContent != nil { + rc.RedactedContent = new(encodeRedactedContentForConverse(*rc.RedactedContent)) + } + } + return blocks +} + +func renderConverseResponseReasoning(msg *schemas.ResponsesMessage) []BedrockContentBlock { if msg == nil { return nil } @@ -5051,15 +5070,15 @@ func convertBifrostReasoningToConverseResponseReasoning(msg *schemas.ResponsesMe // Converse ingress stores text blocks (with their own signatures) on // Content and a separate opaque block on EncryptedContent. Render both. // Isolate Content so summary signatures are not mistaken for opaque blocks. - textBlocks := convertBifrostReasoningToBedrockReasoning(&schemas.ResponsesMessage{Content: msg.Content}, schemas.BedrockReasoningShapeText, false) + textBlocks := convertBifrostReasoningToBedrockReasoning(&schemas.ResponsesMessage{Content: msg.Content}, schemas.BedrockReasoningShapeText, false, false) if len(textBlocks) > 0 { - return append(textBlocks, convertBifrostReasoningToBedrockReasoning(msg, schemas.BedrockReasoningShapeRedacted, false)...) + return append(textBlocks, convertBifrostReasoningToBedrockReasoning(msg, schemas.BedrockReasoningShapeRedacted, false, false)...) } } if reasoningMessageHasText(msg) { - return convertBifrostReasoningToBedrockReasoning(msg, schemas.BedrockReasoningShapeText, false) + return convertBifrostReasoningToBedrockReasoning(msg, schemas.BedrockReasoningShapeText, false, false) } - return convertBifrostReasoningToBedrockReasoning(msg, schemas.BedrockReasoningShapeRedacted, false) + return convertBifrostReasoningToBedrockReasoning(msg, schemas.BedrockReasoningShapeRedacted, false, false) } func reasoningMessageHasText(msg *schemas.ResponsesMessage) bool { @@ -5083,7 +5102,7 @@ func reasoningMessageHasText(msg *schemas.ResponsesMessage) bool { return false } -func convertBifrostReasoningToBedrockReasoning(msg *schemas.ResponsesMessage, shape schemas.BedrockReasoningShape, requireSigned bool) []BedrockContentBlock { +func convertBifrostReasoningToBedrockReasoning(msg *schemas.ResponsesMessage, shape schemas.BedrockReasoningShape, requireSigned, stripBlockSignatures bool) []BedrockContentBlock { if shape == schemas.BedrockReasoningShapeRedacted { // These models reject reasoningText in every form -- with a signature, // without one, empty or not -- with an opaque 500. Only the blob carried @@ -5113,6 +5132,9 @@ func convertBifrostReasoningToBedrockReasoning(msg *schemas.ResponsesMessage, sh for _, block := range msg.Content.ContentBlocks { if block.Type == schemas.ResponsesOutputMessageContentTypeReasoning && block.Text != nil { signature := reasoningSignatureForBedrock(block.Signature) + if stripBlockSignatures { + signature = nil + } if signature == nil && requireSigned { // Unsigned and unverifiable; skipped, and not counted as // emitted so the ResponsesReasoning fall-through below still @@ -5229,10 +5251,17 @@ func convertBifrostResponsesMessageContentBlocksToBedrockContentBlocks(ctx conte } if block.Text != nil { signature := reasoningSignatureForBedrock(block.Signature) - if !isConverseResponseRendering(ctx) && signature == nil && converseRequiresSignedReasoning(model) { - // Unsigned thinking cannot be replayed to a model that - // verifies signatures (#6624); drop the block, keep the turn. - continue + if !isConverseResponseRendering(ctx) { + requireSigned := converseRequiresSignedReasoning(model) + if !requireSigned { + // Only models that verify signatures accept the field. + signature = nil + } + if signature == nil && requireSigned { + // Unsigned thinking cannot be replayed to a model that + // verifies signatures (#6624); drop the block, keep the turn. + continue + } } bedrockBlock.ReasoningContent = &BedrockReasoningContent{ ReasoningText: &BedrockReasoningContentText{ diff --git a/core/providers/bedrock/toolresultstatus_test.go b/core/providers/bedrock/toolresultstatus_test.go index 28199c2ae58..fe8ff541eb9 100644 --- a/core/providers/bedrock/toolresultstatus_test.go +++ b/core/providers/bedrock/toolresultstatus_test.go @@ -51,3 +51,38 @@ func TestToolResultStatusFromIsError(t *testing.T) { t.Fatalf("non-error tool call must keep status \"success\", got %v", results[1].Status) } } + +// TestToolResultEmptyKeyChatSurface verifies that a string tool result whose +// JSON carries an empty-string object key converts to a text block on the +// chat-completions surface. Converse rejects such a document in the json +// field ("The format of the value at ...toolResult.content.0.json is +// invalid"), and convertToolMessages now shares tryParseJSONIntoContentBlock +// with the Responses path, so both surfaces get the same fallback. +func TestToolResultEmptyKeyChatSurface(t *testing.T) { + payload := `{"success":{"fullSubtreeExtensionCounts":{"":2,".md":1}}}` + msgs := []schemas.ChatMessage{ + { + Role: schemas.ChatMessageRoleTool, + ChatToolMessage: &schemas.ChatToolMessage{ToolCallID: schemas.Ptr("toolu_emptykey")}, + Content: &schemas.ChatMessageContent{ContentStr: schemas.Ptr(payload)}, + }, + } + + converted, err := convertToolMessages(context.Background(), "anthropic.claude-sonnet-4-5-20250929-v1:0", msgs) + if err != nil { + t.Fatalf("convert tool messages: %v", err) + } + if len(converted.Content) != 1 || converted.Content[0].ToolResult == nil { + t.Fatalf("expected a single toolResult block, got %#v", converted.Content) + } + content := converted.Content[0].ToolResult.Content + if len(content) != 1 { + t.Fatalf("expected 1 tool result content block, got %d", len(content)) + } + if content[0].JSON != nil { + t.Fatalf("empty-key payload must not be sent as a json block, got %s", string(content[0].JSON)) + } + if content[0].Text == nil || *content[0].Text != payload { + t.Fatalf("expected text fallback carrying the original payload, got %v", content[0].Text) + } +} diff --git a/core/providers/bedrock/unsignedreasoning_test.go b/core/providers/bedrock/unsignedreasoning_test.go index d96dde7aae4..aced825b2c7 100644 --- a/core/providers/bedrock/unsignedreasoning_test.go +++ b/core/providers/bedrock/unsignedreasoning_test.go @@ -133,6 +133,18 @@ func TestUnsignedReasoningReplay_Responses(t *testing.T) { require.NoError(t, err) require.Len(t, reasoningBlocksInConverse(req.Messages), 2) }) + t.Run("nova strips a foreign signature from a reasoning content block", func(t *testing.T) { + msg := &schemas.ResponsesMessage{Type: &reasoningType, + Content: &schemas.ResponsesMessageContent{ContentBlocks: []schemas.ResponsesMessageContentBlock{reasoningTextBlock("thought about it", &signature)}}, + ResponsesReasoning: &schemas.ResponsesReasoning{}, + } + req, err := ToBedrockResponsesRequest(ctx, responsesReplayWithReasoning(unsignedReasoningNova, msg)) + require.NoError(t, err) + blocks := reasoningBlocksInConverse(req.Messages) + require.Len(t, blocks, 1) + require.Nil(t, blocks[0].ReasoningContent.ReasoningText.Signature) + require.NotNil(t, msg.Content.ContentBlocks[0].Signature, "the caller's request must keep its signature for fallbacks") + }) } func chatReplayWithReasoning(model string, details []schemas.ChatReasoningDetails) *schemas.BifrostChatRequest { diff --git a/core/providers/bedrock/utils.go b/core/providers/bedrock/utils.go index 49bac67c3e7..c713ad329c3 100644 --- a/core/providers/bedrock/utils.go +++ b/core/providers/bedrock/utils.go @@ -526,7 +526,10 @@ func bedrockAliasToolName(ctx context.Context, name string) string { semanticName = "tool" } - hash := fmt.Sprintf("%08x", uint32(xxhash.Sum64String(name))) + // The "t" keeps the alias letter-first: a bare hex hash starts with a digit 10 + // times in 16, and moonshotai.kimi-k3 answers any digit-leading tool name with + // HTTP 200 and an empty stream. + hash := fmt.Sprintf("t%08x", uint32(xxhash.Sum64String(name))) maxSemanticLen := 64 - len(hash) - 1 if len(semanticName) > maxSemanticLen { semanticName = semanticName[:maxSemanticLen] @@ -1247,6 +1250,72 @@ func reasoningSignatureForBedrock(sig *string) *string { return sig } +// extraParamStringSlice reads a string-array extra param. Over HTTP, +// BedrockConverseRequest.UnmarshalJSON keeps unknown fields as json.RawMessage, +// which schemas.SafeExtractStringSlice does not decode; in-process callers pass +// Go values, which it does. A JSON null is absent, as a nil Go value would be. +func extraParamStringSlice(value any) ([]string, bool) { + if raw, ok := value.(json.RawMessage); ok { + if bytes.Equal(bytes.TrimSpace(raw), []byte("null")) { + return nil, false + } + var out []string + if err := sonic.Unmarshal(raw, &out); err != nil { + return nil, false + } + return out, true + } + return schemas.SafeExtractStringSlice(value) +} + +// extraParamStringPointer is extraParamStringSlice for a string extra param. A +// JSON null decodes into a string without error, so it is checked first: a +// pointer to "" would read as an explicit setting and suppress a caller's default. +func extraParamStringPointer(value any) (*string, bool) { + if raw, ok := value.(json.RawMessage); ok { + if bytes.Equal(bytes.TrimSpace(raw), []byte("null")) { + return nil, false + } + var out string + if err := sonic.Unmarshal(raw, &out); err != nil { + return nil, false + } + return &out, true + } + return schemas.SafeExtractStringPointer(value) +} + +// foreignRedactedContentPrefix marks a redactedContent blob Bifrost wrapped for a +// Converse client. Converse types redactedContent as a blob, so strict SDKs +// base64-decode it; a non-Bedrock upstream's token (an OpenAI Fernet token) is not +// standard base64 and fails that decode (#7514). The prefix lets the next turn +// unwrap it back to the exact token the upstream minted. +const foreignRedactedContentPrefix = "bifrost:redacted:v1:" + +// encodeRedactedContentForConverse leaves a canonical base64 blob (every native +// Bedrock blob) untouched and wraps anything else. Canonical, not merely decodable: +// SDKs replay the decoded bytes re-encoded, so only a canonical blob comes back +// byte-identical. +func encodeRedactedContentForConverse(token string) string { + if decoded, err := base64.StdEncoding.DecodeString(token); err == nil && base64.StdEncoding.EncodeToString(decoded) == token { + return token + } + return base64.StdEncoding.EncodeToString([]byte(foreignRedactedContentPrefix + token)) +} + +// decodeRedactedContentFromConverse unwraps a blob encodeRedactedContentForConverse +// wrapped and returns every other blob unchanged. +func decodeRedactedContentFromConverse(blob string) string { + decoded, err := base64.StdEncoding.DecodeString(blob) + if err != nil { + return blob + } + if token, ok := strings.CutPrefix(string(decoded), foreignRedactedContentPrefix); ok { + return token + } + return blob +} + // newBedrockCachePoint builds a default cache point, attaching the TTL only for the values // Bedrock accepts ("5m" | "1h"); anything else (e.g. Anthropic's "1m") is dropped to the default. func newBedrockCachePoint(ttl *string) *BedrockCachePoint { @@ -1508,42 +1577,11 @@ func convertToolMessages(ctx context.Context, model string, msgs []schemas.ChatM for _, msg := range msgs { var toolResultContent []BedrockContentBlock if msg.Content.ContentStr != nil { - // Bedrock expects JSON to be a parsed object, not a string - // Validate and compact JSON without parsing into Go types (preserves key ordering) - var buf bytes.Buffer - if err := json.Compact(&buf, []byte(*msg.Content.ContentStr)); err != nil { - // If it's not valid JSON, wrap it as a text block instead - toolResultContent = append(toolResultContent, BedrockContentBlock{ - Text: msg.Content.ContentStr, - }) - } else { - compacted := buf.Bytes() - // Bedrock does not accept primitives or arrays directly in the json field - if len(compacted) > 0 && compacted[0] == '{' { - // Objects are valid as-is - toolResultContent = append(toolResultContent, BedrockContentBlock{ - JSON: json.RawMessage(compacted), - }) - } else if len(compacted) > 0 && compacted[0] == '[' { - // Arrays need to be wrapped - wrapped := make([]byte, 0, len(compacted)+len(`{"results":}`)) - wrapped = append(wrapped, `{"results":`...) - wrapped = append(wrapped, compacted...) - wrapped = append(wrapped, '}') - toolResultContent = append(toolResultContent, BedrockContentBlock{ - JSON: json.RawMessage(wrapped), - }) - } else { - // Primitives (string, number, boolean, null) need to be wrapped - wrapped := make([]byte, 0, len(compacted)+len(`{"value":}`)) - wrapped = append(wrapped, `{"value":`...) - wrapped = append(wrapped, compacted...) - wrapped = append(wrapped, '}') - toolResultContent = append(toolResultContent, BedrockContentBlock{ - JSON: json.RawMessage(wrapped), - }) - } - } + // Bedrock expects JSON to be a parsed object, not a string. The helper + // validates, compacts, wraps arrays and primitives, falls back to a text + // block for non-JSON, and refuses json documents Converse rejects (such + // as objects carrying an empty-string key). + toolResultContent = append(toolResultContent, tryParseJSONIntoContentBlock(*msg.Content.ContentStr)) } else if msg.Content.ContentBlocks != nil { for _, block := range msg.Content.ContentBlocks { switch block.Type { @@ -2991,6 +3029,15 @@ func tryParseJSONIntoContentBlock(text string) BedrockContentBlock { } compacted := buf.Bytes() + // Converse rejects a json document containing an empty-string object key with + // "The format of the value at ...toolResult.content.N.json is invalid" (verified live + // against us.anthropic.claude-haiku-4-5; Cursor's list_directory results carry such + // keys for extensionless files). A text block holding the same JSON string reads + // identically to the model, so fall back to text instead of mutating the payload. + if len(compacted) > 0 && (compacted[0] == '{' || compacted[0] == '[') && jsonHasEmptyObjectKey(compacted) { + return BedrockContentBlock{Text: schemas.Ptr(text)} + } + // Bedrock does not accept primitives or arrays directly in the json field if len(compacted) > 0 && compacted[0] == '{' { // Objects are valid as-is @@ -3012,6 +3059,29 @@ func tryParseJSONIntoContentBlock(text string) BedrockContentBlock { } } +// jsonHasEmptyObjectKey reports whether the given JSON document contains an object key +// that is the empty string, at any nesting depth. Callers only reach this after +// json.Compact succeeded, so the input is known-valid and gjson's lazy parse is safe. +// The empty-key check is gated on IsObject because ForEach over an array passes +// synthetic keys that must not be mistaken for object keys. +func jsonHasEmptyObjectKey(data []byte) bool { + var walk func(v gjson.Result) bool + walk = func(v gjson.Result) bool { + found := false + isObject := v.IsObject() + v.ForEach(func(key, value gjson.Result) bool { + if isObject && key.Str == "" { + found = true + } else if value.IsObject() || value.IsArray() { + found = walk(value) + } + return !found + }) + return found + } + return walk(gjson.ParseBytes(data)) +} + // BedrockMaxCachePoints is the number of cache checkpoints Bedrock accepts in one Converse // request. Exceeding it is a hard rejection, not a degradation: verified live against // bedrock/global.anthropic.claude-haiku-4-5 with five markers, which returns diff --git a/core/providers/bedrockmantle/bedrockmantle.go b/core/providers/bedrockmantle/bedrockmantle.go index a7e1cb91581..c131f571b30 100644 --- a/core/providers/bedrockmantle/bedrockmantle.go +++ b/core/providers/bedrockmantle/bedrockmantle.go @@ -433,6 +433,11 @@ func (provider *BedrockMantleProvider) Rerank(ctx *schemas.BifrostContext, key s return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey()) } +// Decision is not supported by the BedrockMantle provider. +func (provider *BedrockMantleProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR is not supported by the Bedrock Mantle provider. func (provider *BedrockMantleProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) diff --git a/core/providers/cerebras/cerebras.go b/core/providers/cerebras/cerebras.go index b8312a12503..d46a0e00911 100644 --- a/core/providers/cerebras/cerebras.go +++ b/core/providers/cerebras/cerebras.go @@ -209,6 +209,11 @@ func (provider *CerebrasProvider) Rerank(ctx *schemas.BifrostContext, key schema return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey()) } +// Decision is not supported by the Cerebras provider. +func (provider *CerebrasProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR is not supported by the Cerebras provider. func (provider *CerebrasProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) diff --git a/core/providers/cohere/cohere.go b/core/providers/cohere/cohere.go index f89eb462a6b..d1ad5d03bce 100644 --- a/core/providers/cohere/cohere.go +++ b/core/providers/cohere/cohere.go @@ -1066,6 +1066,11 @@ func (provider *CohereProvider) Rerank(ctx *schemas.BifrostContext, key schemas. return bifrostResponse, nil } +// Decision is not supported by the Cohere provider. +func (provider *CohereProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR is not supported by the Cohere provider. func (provider *CohereProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) diff --git a/core/providers/databricks/databricks_test.go b/core/providers/databricks/databricks_test.go index dd8c0c40e77..f030a4b1bdf 100644 --- a/core/providers/databricks/databricks_test.go +++ b/core/providers/databricks/databricks_test.go @@ -880,6 +880,144 @@ func TestDatabricksChatStripsAnthropicOnlyFields(t *testing.T) { } } +// TestDatabricksGeminiSingleSystemPrompt pins the one-system-prompt rule for Gemini-backed +// endpoints. Databricks maps chat messages onto Gemini's single system_instruction and +// answers anything more with "Gemini models only support one system prompt" (400). Claude +// Code sends its system prompt as several blocks, which reach the wire as one system message +// with several text parts, and that alone trips the rule. Other model families take the +// parts as sent. +func TestDatabricksGeminiSingleSystemPrompt(t *testing.T) { + t.Parallel() + + text := func(s string) *schemas.ChatMessageContent { + return &schemas.ChatMessageContent{ContentStr: schemas.Ptr(s)} + } + parts := func(texts ...string) *schemas.ChatMessageContent { + blocks := make([]schemas.ChatContentBlock, len(texts)) + for i, s := range texts { + blocks[i] = schemas.ChatContentBlock{Type: schemas.ChatContentBlockTypeText, Text: schemas.Ptr(s)} + } + return &schemas.ChatMessageContent{ContentBlocks: blocks} + } + user := schemas.ChatMessage{Role: schemas.ChatMessageRoleUser, Content: text("hey")} + + tests := []struct { + name string + model string + input []schemas.ChatMessage + // want is the JSON the messages array must marshal to on the wire. + want string + }{ + { + name: "system parts are joined into one string", + model: "databricks-gemini-3-8-flash", + input: []schemas.ChatMessage{{Role: schemas.ChatMessageRoleSystem, Content: parts("rules", "tools")}, user}, + want: `[{"role":"system","content":"rules\n\ntools"},{"role":"user","content":"hey"}]`, + }, + { + name: "system and developer messages merge into the first", + model: "databricks-gemini-3-8-flash", + input: []schemas.ChatMessage{ + {Role: schemas.ChatMessageRoleSystem, Content: text("rules")}, + {Role: schemas.ChatMessageRoleDeveloper, Content: parts("tools", "env")}, + user, + {Role: schemas.ChatMessageRoleSystem, Content: text("reminder")}, + }, + want: `[{"role":"system","content":"rules\n\ntools\n\nenv\n\nreminder"},{"role":"user","content":"hey"}]`, + }, + { + name: "AI Gateway model name is matched too", + model: "system.ai.gemini-3-8-flash", + input: []schemas.ChatMessage{{Role: schemas.ChatMessageRoleSystem, Content: parts("rules", "tools")}, user}, + want: `[{"role":"system","content":"rules\n\ntools"},{"role":"user","content":"hey"}]`, + }, + { + name: "a single string system prompt is sent unchanged", + model: "databricks-gemini-3-8-flash", + input: []schemas.ChatMessage{{Role: schemas.ChatMessageRoleSystem, Content: text("rules")}, user}, + want: `[{"role":"system","content":"rules"},{"role":"user","content":"hey"}]`, + }, + { + name: "non-Gemini models keep the parts", + model: "databricks-claude-sonnet-4-5", + input: []schemas.ChatMessage{{Role: schemas.ChatMessageRoleSystem, Content: parts("rules", "tools")}, user}, + want: `[{"role":"system","content":[{"type":"text","text":"rules"},{"type":"text","text":"tools"}]},{"role":"user","content":"hey"}]`, + }, + } + + for _, stream := range []bool{false, true} { + for _, tt := range tests { + name := tt.name + if stream { + name += " (stream)" + } + t.Run(name, func(t *testing.T) { + t.Parallel() + + bodyCh := make(chan map[string]json.RawMessage, 1) + provider, server := newStubProvider(t, func(w http.ResponseWriter, r *http.Request) { + var body map[string]json.RawMessage + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Errorf("decode request body: %v", err) + } + bodyCh <- body + if stream { + w.Header().Set("Content-Type", "text/event-stream") + _, _ = w.Write([]byte("data: [DONE]\n\n")) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(stubChatResponse)) + }) + key := schemas.Key{ + Models: []string{"*"}, + Value: *schemas.NewSecretVar("dapi-test"), + DatabricksKeyConfig: &schemas.DatabricksKeyConfig{ + WorkspaceURL: *schemas.NewSecretVar(serverHost(t, server)), + }, + } + original, err := json.Marshal(tt.input) + if err != nil { + t.Fatalf("marshal input: %v", err) + } + request := &schemas.BifrostChatRequest{Provider: schemas.Databricks, Model: tt.model, Input: tt.input} + + ctx, cancel := schemas.NewBifrostContextWithTimeout(context.Background(), 10*time.Second) + defer cancel() + if stream { + postHookRunner := func(_ *schemas.BifrostContext, result *schemas.BifrostResponse, bErr *schemas.BifrostError) (*schemas.BifrostResponse, *schemas.BifrostError) { + return result, bErr + } + ch, bErr := provider.ChatCompletionStream(ctx, postHookRunner, nil, key, request) + if bErr != nil { + t.Fatalf("ChatCompletionStream returned an error: %v", bErr) + } + for range ch { + } + } else if _, bErr := provider.ChatCompletion(ctx, key, request); bErr != nil { + t.Fatalf("ChatCompletion returned an error: %v", bErr) + } + + select { + case body := <-bodyCh: + if got := string(body["messages"]); got != tt.want { + t.Errorf("messages on the wire:\n got %s\nwant %s", got, tt.want) + } + case <-time.After(5 * time.Second): + t.Fatal("mock server did not receive the request") + } + after, err := json.Marshal(request.Input) + if err != nil { + t.Fatalf("marshal input: %v", err) + } + if string(after) != string(original) { + t.Errorf("merging system prompts mutated the original request:\n got %s\nwant %s", after, original) + } + }) + } + } +} + // TestDatabricksGatewayRequestTags covers forwarding Bifrost governance labels to Databricks // for usage attribution. The header is opt-in, carries display names only, and must not // displace a header the caller supplied. diff --git a/core/providers/databricks/inference.go b/core/providers/databricks/inference.go index 08ba62208e2..d870048626f 100644 --- a/core/providers/databricks/inference.go +++ b/core/providers/databricks/inference.go @@ -194,6 +194,81 @@ func stripUnsupportedChatFields(ctx *schemas.BifrostContext, request *schemas.Bi return &requestCopy } +// mergeGeminiSystemPrompts folds every system and developer message into a single string +// system message for Gemini-backed endpoints. Databricks maps chat messages onto Gemini's one +// system_instruction and rejects anything more with "Gemini models only support one system +// prompt" (400), counting each text part of a system message as a prompt of its own. Claude +// Code sends its system prompt as several blocks, so it hits this on every request. +// +// The merged message takes the place of the first system message, so a system message from +// later in the conversation moves to the front: Gemini has nowhere else to put it. Only text +// parts are kept, since system_instruction takes text only. Other model families are left +// alone, and the original request is not mutated. +func mergeGeminiSystemPrompts(ctx *schemas.BifrostContext, request *schemas.BifrostChatRequest) *schemas.BifrostChatRequest { + if request == nil || !schemas.IsGeminiModelFamily(ctx, request.Model) { + return request + } + + isSystem := func(msg schemas.ChatMessage) bool { + return msg.Role == schemas.ChatMessageRoleSystem || msg.Role == schemas.ChatMessageRoleDeveloper + } + systemCount := 0 + needsMerge := false + for _, msg := range request.Input { + if !isSystem(msg) { + continue + } + systemCount++ + if msg.Role == schemas.ChatMessageRoleDeveloper || (msg.Content != nil && msg.Content.ContentStr == nil) { + needsMerge = true + } + } + if systemCount == 0 || (systemCount == 1 && !needsMerge) { + return request + } + + var texts []string + for _, msg := range request.Input { + if !isSystem(msg) || msg.Content == nil { + continue + } + if msg.Content.ContentStr != nil { + if *msg.Content.ContentStr != "" { + texts = append(texts, *msg.Content.ContentStr) + } + continue + } + for _, block := range msg.Content.ContentBlocks { + if block.Text != nil && *block.Text != "" { + texts = append(texts, *block.Text) + } + } + } + + merged := make([]schemas.ChatMessage, 0, len(request.Input)-systemCount+1) + placed := false + for _, msg := range request.Input { + if !isSystem(msg) { + merged = append(merged, msg) + continue + } + if placed { + continue + } + placed = true + joined := strings.Join(texts, "\n\n") + merged = append(merged, schemas.ChatMessage{ + Name: msg.Name, + Role: schemas.ChatMessageRoleSystem, + Content: &schemas.ChatMessageContent{ContentStr: &joined}, + }) + } + + requestCopy := *request + requestCopy.Input = merged + return &requestCopy +} + // stripUnsupportedResponsesFields is stripUnsupportedChatFields for the Model Serving // Responses surface, which carries the same fields through a different params struct — minus // the chat-only knobs (stop, the penalties, top_k) and the Anthropic betas, which the @@ -310,6 +385,7 @@ func withResponsesWireModel(ctx *schemas.BifrostContext, key schemas.Key, reques // ChatCompletion performs a chat completion request against the resolved Databricks surface. func (provider *DatabricksProvider) ChatCompletion(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostChatRequest) (*schemas.BifrostChatResponse, *schemas.BifrostError) { request = stripUnsupportedChatFields(ctx, request) + request = mergeGeminiSystemPrompts(ctx, request) request, bErr := inlineChatImageURLs(ctx, request) if bErr != nil { return nil, bErr @@ -340,6 +416,7 @@ func (provider *DatabricksProvider) ChatCompletion(ctx *schemas.BifrostContext, // Databricks surface. Both surfaces emit OpenAI-shaped Server-Sent Events. func (provider *DatabricksProvider) ChatCompletionStream(ctx *schemas.BifrostContext, postHookRunner schemas.PostHookRunner, postHookSpanFinalizer func(context.Context), key schemas.Key, request *schemas.BifrostChatRequest) (chan *schemas.BifrostStreamChunk, *schemas.BifrostError) { request = stripUnsupportedChatFields(ctx, request) + request = mergeGeminiSystemPrompts(ctx, request) request, bErr := inlineChatImageURLs(ctx, request) if bErr != nil { return nil, bErr diff --git a/core/providers/databricks/reasoning.go b/core/providers/databricks/reasoning.go index 26557def2ff..4a0b280c268 100644 --- a/core/providers/databricks/reasoning.go +++ b/core/providers/databricks/reasoning.go @@ -3,7 +3,6 @@ package databricks import ( "bytes" "encoding/json" - "fmt" "strings" "github.com/capsohq/bifrost/core/providers/anthropic" @@ -153,8 +152,15 @@ func normalizeReasoningBlocks(body []byte) []byte { return body } - out := body - choices.ForEach(func(idx, choice gjson.Result) bool { + // Each choice is edited in its own JSON and the choices array is written back + // once. Addressing through the whole body (choices...content) would + // reserialise the entire response up to four times per choice, making this + // O(choices x body). Pinned by TestNormalizeReasoningBlocks_AllocationScaling. + var rebuiltChoices [][]byte + anyChanged := false + failed := false + choices.ForEach(func(_, choice gjson.Result) bool { + choiceRaw := []byte(choice.Raw) for _, carrier := range []string{"message", "delta"} { content := choice.Get(carrier + ".content") if !content.IsArray() { @@ -186,22 +192,23 @@ func normalizeReasoningBlocks(body []byte) []byte { return true }) - base := fmt.Sprintf("choices.%d.%s", idx.Int(), carrier) + base := carrier var err error switch { case len(kept) == 0 && carrier == "delta": - out, err = sjson.DeleteBytes(out, base+".content") + choiceRaw, err = sjson.DeleteBytes(choiceRaw, base+".content") case len(kept) == 0: - out, err = sjson.SetBytes(out, base+".content", "") + choiceRaw, err = sjson.SetBytes(choiceRaw, base+".content", "") case allText: - out, err = sjson.SetBytes(out, base+".content", strings.Join(texts, "")) + choiceRaw, err = sjson.SetBytes(choiceRaw, base+".content", strings.Join(texts, "")) default: - out, err = sjson.SetRawBytes(out, base+".content", []byte("["+strings.Join(kept, ",")+"]")) + choiceRaw, err = sjson.SetRawBytes(choiceRaw, base+".content", []byte("["+strings.Join(kept, ",")+"]")) } if err != nil { - out = body + failed = true return false } + anyChanged = true if reasoning.Len() == 0 && signature == "" { continue @@ -214,17 +221,37 @@ func normalizeReasoningBlocks(body []byte) []byte { if signature != "" { detail.Signature = schemas.Ptr(signature) } - if out, err = sjson.SetBytes(out, base+".reasoning", reasoning.String()); err != nil { - out = body + if choiceRaw, err = sjson.SetBytes(choiceRaw, base+".reasoning", reasoning.String()); err != nil { + failed = true return false } - if out, err = sjson.SetBytes(out, base+".reasoning_details", []schemas.ChatReasoningDetails{detail}); err != nil { - out = body + if choiceRaw, err = sjson.SetBytes(choiceRaw, base+".reasoning_details", []schemas.ChatReasoningDetails{detail}); err != nil { + failed = true return false } } + rebuiltChoices = append(rebuiltChoices, choiceRaw) return true }) + if failed || !anyChanged { + return body + } + + var joined bytes.Buffer + joined.Grow(len(body)) + joined.WriteByte('[') + for i, c := range rebuiltChoices { + if i > 0 { + joined.WriteByte(',') + } + joined.Write(c) + } + joined.WriteByte(']') + + out, err := sjson.SetRawBytes(body, "choices", joined.Bytes()) + if err != nil { + return body + } return out } diff --git a/core/providers/databricks/reasoning_test.go b/core/providers/databricks/reasoning_test.go index 52db53a8eeb..0831722e844 100644 --- a/core/providers/databricks/reasoning_test.go +++ b/core/providers/databricks/reasoning_test.go @@ -1,8 +1,12 @@ package databricks import ( + "bytes" + "strconv" + "strings" "testing" + "github.com/capsohq/bifrost/core/internal/memtest" "github.com/tidwall/gjson" schemas "github.com/capsohq/bifrost/core/schemas" @@ -259,3 +263,34 @@ func TestNormalizeReasoningBlocks(t *testing.T) { } }) } + +// TestNormalizeReasoningBlocks_AllocationScaling pins the allocation shape of the +// reasoning-block normalization. +// +// The loop writes choices...content through the whole response body, once +// per choice per carrier, and each sjson write reserialises the entire document. An +// n>1 completion therefore costs N copies of the response. +func TestNormalizeReasoningBlocks_AllocationScaling(t *testing.T) { + memtest.AssertAllocScaling(t, func(choices int) []byte { + var b bytes.Buffer + b.WriteString(`{"id":"chatcmpl-1","choices":[`) + for i := range choices { + if i > 0 { + b.WriteByte(',') + } + // A reasoning block plus a text block: the reasoning is what triggers the + // rewrite, the text carries the bulk of the bytes so the payload scales. + b.WriteString(`{"index":`) + b.WriteString(strconv.Itoa(i)) + b.WriteString(`,"message":{"role":"assistant","content":[`) + b.WriteString(`{"type":"reasoning","summary":[{"text":"thinking about it","signature":"sig"}]},`) + b.WriteString(`{"type":"text","text":"`) + b.WriteString(strings.Repeat("x", 400)) + b.WriteString(`"}]}}`) + } + b.WriteString(`]}`) + return b.Bytes() + }, func(body []byte) { + normalizeReasoningBlocks(body) + }) +} diff --git a/core/providers/databricks/unsupported.go b/core/providers/databricks/unsupported.go index 663ead5ebf2..a99f58cc67a 100644 --- a/core/providers/databricks/unsupported.go +++ b/core/providers/databricks/unsupported.go @@ -27,6 +27,11 @@ func (provider *DatabricksProvider) Rerank(ctx *schemas.BifrostContext, key sche return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey()) } +// Decision is not supported by the Databricks provider. +func (provider *DatabricksProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + func (provider *DatabricksProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) } diff --git a/core/providers/deepseek/deepseek.go b/core/providers/deepseek/deepseek.go index 00e220f4b62..d8f8bbc037f 100644 --- a/core/providers/deepseek/deepseek.go +++ b/core/providers/deepseek/deepseek.go @@ -409,6 +409,11 @@ func (provider *DeepSeekProvider) Rerank(ctx *schemas.BifrostContext, key schema return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey()) } +// Decision is not supported by the DeepSeek provider. +func (provider *DeepSeekProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR is not supported by the DeepSeek provider. func (provider *DeepSeekProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) diff --git a/core/providers/elevenlabs/elevenlabs.go b/core/providers/elevenlabs/elevenlabs.go index 6a5369931bb..db714725a26 100644 --- a/core/providers/elevenlabs/elevenlabs.go +++ b/core/providers/elevenlabs/elevenlabs.go @@ -325,6 +325,11 @@ func (provider *ElevenlabsProvider) Rerank(ctx *schemas.BifrostContext, key sche return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey()) } +// Decision is not supported by the Elevenlabs provider. +func (provider *ElevenlabsProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR is not supported by the Elevenlabs provider. func (provider *ElevenlabsProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) diff --git a/core/providers/fireworks/fireworks.go b/core/providers/fireworks/fireworks.go index 58c905f2e20..9a1d5e472c5 100644 --- a/core/providers/fireworks/fireworks.go +++ b/core/providers/fireworks/fireworks.go @@ -355,6 +355,11 @@ func (provider *FireworksProvider) Rerank(ctx *schemas.BifrostContext, key schem return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey()) } +// Decision is not supported by the Fireworks provider. +func (provider *FireworksProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR is not supported by the Fireworks provider. func (provider *FireworksProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) diff --git a/core/providers/gemini/gemini.go b/core/providers/gemini/gemini.go index 644af3628b3..7063fcbe12c 100644 --- a/core/providers/gemini/gemini.go +++ b/core/providers/gemini/gemini.go @@ -1424,6 +1424,11 @@ func (provider *GeminiProvider) Rerank(ctx *schemas.BifrostContext, key schemas. return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey()) } +// Decision is not supported by the Gemini provider. +func (provider *GeminiProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR is not supported by the Gemini provider. func (provider *GeminiProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) diff --git a/core/providers/gemini/responses.go b/core/providers/gemini/responses.go index f32c2a45302..5f666173ac8 100644 --- a/core/providers/gemini/responses.go +++ b/core/providers/gemini/responses.go @@ -16,6 +16,7 @@ import ( "github.com/bytedance/sonic" providerUtils "github.com/capsohq/bifrost/core/providers/utils" "github.com/capsohq/bifrost/core/schemas" + "github.com/tidwall/gjson" ) // thoughtSignatureFromEncryptedContent converts a Responses-API @@ -950,6 +951,8 @@ func ToGeminiResponsesStreamResponse(bifrostResp *schemas.BifrostResponsesStream // Skip lifecycle events that don't have corresponding Gemini equivalents switch bifrostResp.Type { case schemas.ResponsesStreamResponseTypePing, + schemas.ResponsesStreamResponseTypeSafeguardsUpdate, + schemas.ResponsesStreamResponseTypeProviderRawEvent, schemas.ResponsesStreamResponseTypeCreated, schemas.ResponsesStreamResponseTypeInProgress, schemas.ResponsesStreamResponseTypeReasoningSummaryPartAdded, @@ -2581,16 +2584,41 @@ func stripFunctionResponseMediaRefs(response json.RawMessage) string { return string(response) } - cleaned := []byte(response) + // Rebuild the object in one pass rather than deleting key by key: every + // providerUtils.DeleteJSONField reserialises the whole document, so N media refs + // cost N copies of it. Pinned by TestStripFunctionResponseMediaRefs_AllocationScaling. + // + // ForEach, not Map(): Map() returns a Go map whose iteration order is randomised, + // which would make the surviving keys come out in a different order on every call + // and break byte stability for anything downstream that hashes or caches this. + // ForEach walks the document in source order, so survivors keep their positions + // exactly as the per-key delete left them. + var rebuilt bytes.Buffer + rebuilt.Grow(len(response)) + rebuilt.WriteByte('{') remaining := 0 - for key, value := range root.Map() { + dropped := 0 + root.ForEach(func(key, value gjson.Result) bool { if value.IsObject() && value.Get("$ref").Exists() { - if updated, err := providerUtils.DeleteJSONField(cleaned, key); err == nil { - cleaned = updated - } - continue + dropped++ + return true } + if remaining > 0 { + rebuilt.WriteByte(',') + } + rebuilt.WriteString(key.Raw) + rebuilt.WriteByte(':') + rebuilt.WriteString(value.Raw) remaining++ + return true + }) + rebuilt.WriteByte('}') + + // Nothing dropped means the per-key delete would have been a no-op, so hand back + // the caller's bytes untouched rather than a re-serialised equivalent. + cleaned := []byte(response) + if dropped > 0 { + cleaned = rebuilt.Bytes() } if remaining == 0 { diff --git a/core/providers/gemini/responses_test.go b/core/providers/gemini/responses_test.go new file mode 100644 index 00000000000..4eb3cb27289 --- /dev/null +++ b/core/providers/gemini/responses_test.go @@ -0,0 +1,57 @@ +package gemini + +import ( + "bytes" + "encoding/json" + "strings" + "testing" + + "github.com/capsohq/bifrost/core/internal/memtest" +) + +// TestStripFunctionResponseMediaRefs_AllocationScaling pins the allocation shape of +// the $ref strip. +// +// The loop deletes one top-level key at a time, and every providerUtils.DeleteJSONField +// reserialises the whole function-response document, so N media refs cost N copies of +// it. Today N is small in practice, which is exactly why this went unnoticed: the +// complexity is wrong but the payloads have been forgiving. A tool returning many +// media parts is all it takes for that to stop being true. +// +// memtest compares allocation growth against input growth, so this fails on the +// complexity class rather than on a byte threshold that would encode this machine. +func TestStripFunctionResponseMediaRefs_AllocationScaling(t *testing.T) { + memtest.AssertAllocScaling(t, func(refs int) []byte { + var b bytes.Buffer + b.WriteString(`{"output":"`) + b.WriteString(strings.Repeat("o", 200)) + b.WriteString(`"`) + for i := range refs { + // Each media ref is a {"$ref": ...} placeholder, which is what the strip + // targets. The long ref value is what makes the payload grow with N. + b.WriteString(`,"media_`) + b.WriteString(strings.Repeat("k", 3)) + b.WriteString(itoa(i)) + b.WriteString(`":{"$ref":"`) + b.WriteString(strings.Repeat("r", 400)) + b.WriteString(`"}`) + } + b.WriteString(`}`) + return b.Bytes() + }, func(body []byte) { + stripFunctionResponseMediaRefs(json.RawMessage(body)) + }) +} + +// itoa avoids pulling strconv in just for the payload builder above. +func itoa(n int) string { + if n == 0 { + return "0" + } + var digits []byte + for n > 0 { + digits = append([]byte{byte('0' + n%10)}, digits...) + n /= 10 + } + return string(digits) +} diff --git a/core/providers/gemini/transcription.go b/core/providers/gemini/transcription.go index c35a2f5135b..6fe5adcad7c 100644 --- a/core/providers/gemini/transcription.go +++ b/core/providers/gemini/transcription.go @@ -196,13 +196,13 @@ func (response *GenerateContentResponse) ToBifrostTranscriptionResponse() *schem if textContent != "" { bifrostResp.Text = textContent bifrostResp.Task = schemas.Ptr("transcribe") - - // Set usage information with modality details - bifrostResp.Usage = convertGeminiUsageMetadataToTranscriptionUsage(response.UsageMetadata) } } } + // Usage is billed even when the transcript is empty. + bifrostResp.Usage = convertGeminiUsageMetadataToTranscriptionUsage(response.UsageMetadata) + return bifrostResp } diff --git a/core/providers/gemini/transcription_test.go b/core/providers/gemini/transcription_test.go new file mode 100644 index 00000000000..de93f617e1d --- /dev/null +++ b/core/providers/gemini/transcription_test.go @@ -0,0 +1,31 @@ +package gemini + +import "testing" + +// Gemini bills the prompt even when it returns no transcript (for example a +// speechless clip), so usage must survive an empty-text response. +func TestToBifrostTranscriptionResponseKeepsUsageWhenTextEmpty(t *testing.T) { + resp := &GenerateContentResponse{ + Candidates: []*Candidate{{Content: &Content{Role: string(RoleModel), Parts: []*Part{{Text: ""}}}}}, + UsageMetadata: &GenerateContentResponseUsageMetadata{ + PromptTokenCount: 12, + TotalTokenCount: 12, + }, + } + + got := resp.ToBifrostTranscriptionResponse() + if got.Text != "" { + t.Fatalf("text = %q, want empty", got.Text) + } + if got.Usage == nil { + t.Fatal("usage dropped for an empty transcript") + } + if got.Usage.InputTokens == nil || *got.Usage.InputTokens != 12 { + t.Errorf("input tokens = %v, want 12", got.Usage.InputTokens) + } + + genai := ToGeminiTranscriptionResponse(got) + if genai.UsageMetadata == nil || genai.UsageMetadata.PromptTokenCount != 12 { + t.Errorf("genai usageMetadata = %+v, want promptTokenCount 12", genai.UsageMetadata) + } +} diff --git a/core/providers/githubcopilot/githubcopilot.go b/core/providers/githubcopilot/githubcopilot.go index 260e4d4cca0..1ae76bfeb8f 100644 --- a/core/providers/githubcopilot/githubcopilot.go +++ b/core/providers/githubcopilot/githubcopilot.go @@ -243,6 +243,11 @@ func (p *githubCopilotProvider) Rerank(ctx *schemas.BifrostContext, key schemas. return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, p.GetProviderKey()) } +// Decision is not supported by the githubCopilot provider. +func (p *githubCopilotProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, p.GetProviderKey()) +} + // OCR is not supported by GitHub Copilot. func (p *githubCopilotProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, p.GetProviderKey()) diff --git a/core/providers/glm/glm.go b/core/providers/glm/glm.go index 50d8392a25b..bf51f6ad4a1 100644 --- a/core/providers/glm/glm.go +++ b/core/providers/glm/glm.go @@ -235,6 +235,11 @@ func (provider *GLMProvider) Rerank(ctx *schemas.BifrostContext, key schemas.Key return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey()) } +// Decision is not supported by the GLM provider. +func (provider *GLMProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR is not supported by the GLM provider. func (provider *GLMProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) diff --git a/core/providers/groq/groq.go b/core/providers/groq/groq.go index 63e53c82036..3823669b04a 100644 --- a/core/providers/groq/groq.go +++ b/core/providers/groq/groq.go @@ -198,6 +198,11 @@ func (provider *GroqProvider) Rerank(ctx *schemas.BifrostContext, key schemas.Ke return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey()) } +// Decision is not supported by the Groq provider. +func (provider *GroqProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR is not supported by the Groq provider. func (provider *GroqProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) diff --git a/core/providers/huggingface/huggingface.go b/core/providers/huggingface/huggingface.go index 72caced8b86..1d757322d86 100644 --- a/core/providers/huggingface/huggingface.go +++ b/core/providers/huggingface/huggingface.go @@ -800,6 +800,11 @@ func (provider *HuggingFaceProvider) Rerank(ctx *schemas.BifrostContext, key sch return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey()) } +// Decision is not supported by the HuggingFace provider. +func (provider *HuggingFaceProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR is not supported by the Huggingface provider. func (provider *HuggingFaceProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) diff --git a/core/providers/minimax/minimax.go b/core/providers/minimax/minimax.go index 9eed3c27cf8..852febd072b 100644 --- a/core/providers/minimax/minimax.go +++ b/core/providers/minimax/minimax.go @@ -455,6 +455,11 @@ func (provider *MinimaxProvider) Rerank(ctx *schemas.BifrostContext, key schemas return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey()) } +// Decision is not supported by the MiniMax provider. +func (provider *MinimaxProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR is not supported by the Minimax provider. func (provider *MinimaxProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) diff --git a/core/providers/mistral/mistral.go b/core/providers/mistral/mistral.go index 06c005107f1..e67b1571b27 100644 --- a/core/providers/mistral/mistral.go +++ b/core/providers/mistral/mistral.go @@ -645,6 +645,11 @@ func (provider *MistralProvider) Rerank(ctx *schemas.BifrostContext, key schemas return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey()) } +// Decision is not supported by the Mistral provider. +func (provider *MistralProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR performs an OCR request to the Mistral API. // It sends a JSON request to Mistral's OCR endpoint and returns the extracted content. func (provider *MistralProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { diff --git a/core/providers/moonshot/moonshot.go b/core/providers/moonshot/moonshot.go index 87eb5ca5d2f..1706309eba9 100644 --- a/core/providers/moonshot/moonshot.go +++ b/core/providers/moonshot/moonshot.go @@ -229,6 +229,11 @@ func (provider *MoonshotProvider) Rerank(ctx *schemas.BifrostContext, key schema return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey()) } +// Decision is not supported by the Moonshot provider. +func (provider *MoonshotProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR is not supported by the Moonshot provider. func (provider *MoonshotProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) diff --git a/core/providers/nebius/nebius.go b/core/providers/nebius/nebius.go index 610c146d710..90f1b24ed46 100644 --- a/core/providers/nebius/nebius.go +++ b/core/providers/nebius/nebius.go @@ -237,6 +237,11 @@ func (provider *NebiusProvider) Rerank(ctx *schemas.BifrostContext, key schemas. return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey()) } +// Decision is not supported by the Nebius provider. +func (provider *NebiusProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR is not supported by the Nebius provider. func (provider *NebiusProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) diff --git a/core/providers/ollama/ollama.go b/core/providers/ollama/ollama.go index 4384d6e7ed2..73cd97c5c5d 100644 --- a/core/providers/ollama/ollama.go +++ b/core/providers/ollama/ollama.go @@ -284,6 +284,11 @@ func (provider *OllamaProvider) Rerank(ctx *schemas.BifrostContext, key schemas. return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey()) } +// Decision is not supported by the Ollama provider. +func (provider *OllamaProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR is not supported by the Ollama provider. func (provider *OllamaProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) diff --git a/core/providers/openai/chat.go b/core/providers/openai/chat.go index 8f1633c16ec..820c9f80efe 100644 --- a/core/providers/openai/chat.go +++ b/core/providers/openai/chat.go @@ -90,6 +90,7 @@ func ToOpenAIChatRequest(ctx *schemas.BifrostContext, bifrostReq *schemas.Bifros switch bifrostReq.Provider { case schemas.OpenAI, schemas.Azure: openaiReq.normalizeReasoningEffort(caps) + openaiReq.stripUnsupportedSamplingParams(caps) // URL-sourced documents are NOT inlined here. Chat Completions rejects file_url, so they // still have to be resolved before the request goes out - but that is a network fetch that // can fail, and this function has no way to report a failure. Callers invoke @@ -273,6 +274,32 @@ func (req *OpenAIChatRequest) normalizeReasoningEffort(caps schemas.ModelCaps) { // Clear max_tokens since OpenAI doesn't use it req.ChatParameters.Reasoning.MaxTokens = nil } + // A model that always reasons rejects "none"; "minimal" normalizes to its lowest level. + if e := req.ChatParameters.Reasoning.Effort; e != nil && *e == schemas.ReasoningEffortNone && + !caps.CanDisableReasoning(defaultCanDisableReasoning(caps.Model())) { + req.ChatParameters.Reasoning.Effort = schemas.Ptr(caps.NormalizeReasoningEffort(schemas.ReasoningEffortMinimal, defaultEffortControl(caps.Model()))) + } + } +} + +// stripUnsupportedSamplingParams drops sampling fields OpenAI rejects at the request's reasoning effort. +func (req *OpenAIChatRequest) stripUnsupportedSamplingParams(caps schemas.ModelCaps) { + effort := "" + if req.ChatParameters.Reasoning != nil && req.ChatParameters.Reasoning.Effort != nil { + effort = *req.ChatParameters.Reasoning.Effort + } + model := caps.Model() + if samplingParamUnsupported(caps, schemas.FieldTopP, model, effort) { + req.ChatParameters.TopP = nil + } + if samplingParamUnsupported(caps, schemas.FieldTemperature, model, effort) { + req.ChatParameters.Temperature = nil + } + if samplingParamUnsupported(caps, schemas.FieldTopLogprobs, model, effort) { + req.ChatParameters.TopLogProbs = nil + } + if samplingParamUnsupported(caps, schemas.FieldLogprobs, model, effort) { + req.ChatParameters.LogProbs = nil } } diff --git a/core/providers/openai/chat_test.go b/core/providers/openai/chat_test.go index cf3d91efac4..2a957ab2154 100644 --- a/core/providers/openai/chat_test.go +++ b/core/providers/openai/chat_test.go @@ -295,6 +295,42 @@ func TestToOpenAIChatRequest_NormalizesReasoningEffort(t *testing.T) { effort: "max", expected: "high", }, + { + name: "maps none to low for gpt-6-astra", + model: "gpt-6-astra", + effort: "none", + expected: "low", + }, + { + name: "preserves none for gpt-6-sol", + model: "gpt-6-sol", + effort: "none", + expected: "none", + }, + { + name: "preserves none for gpt-6-luna", + model: "gpt-6-luna", + effort: "none", + expected: "none", + }, + { + name: "maps none to minimal for gpt-5-mini", + model: "gpt-5-mini", + effort: "none", + expected: "minimal", + }, + { + name: "maps none to low for o4", + model: "o4", + effort: "none", + expected: "low", + }, + { + name: "preserves none for gpt-5.4", + model: "gpt-5.4", + effort: "none", + expected: "none", + }, { name: "preserves max for deepseek-v4-pro", provider: customDeepSeek, @@ -446,6 +482,67 @@ func TestToOpenAIChatRequest_VertexDropsNoneReasoningEffort(t *testing.T) { } } +// TestToOpenAIChatRequest_StripsUnsupportedSamplingParams pins that sampling fields +// OpenAI rejects at the effective reasoning effort are dropped for OpenAI and Azure. +func TestToOpenAIChatRequest_StripsUnsupportedSamplingParams(t *testing.T) { + tests := []struct { + name string + provider schemas.ModelProvider + model string + effort string + stripped bool + }{ + {name: "gpt-6-astra always strips", model: "gpt-6-astra", stripped: true}, + {name: "azure gpt-6-astra always strips", provider: schemas.Azure, model: "gpt-6-astra", effort: "low", stripped: true}, + {name: "o3 strips", model: "o3", effort: "high", stripped: true}, + {name: "gpt-5.5 omitted effort defaults to medium", model: "gpt-5.5", stripped: true}, + {name: "gpt-5.4 omitted effort defaults to none", model: "gpt-5.4", stripped: false}, + {name: "gpt-5.6 keeps while effort none", model: "gpt-5.6", effort: "none", stripped: false}, + {name: "gpt-5.6 strips once reasoning is on", model: "gpt-5.6", effort: "low", stripped: true}, + {name: "non-reasoning model keeps", model: "gpt-4o", stripped: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + provider := tt.provider + if provider == "" { + provider = schemas.OpenAI + } + params := &schemas.ChatParameters{ + Temperature: schemas.Ptr(0.2), + TopP: schemas.Ptr(0.9), + LogProbs: new(true), + TopLogProbs: schemas.Ptr(3), + } + if tt.effort != "" { + params.Reasoning = &schemas.ChatReasoning{Effort: schemas.Ptr(tt.effort)} + } + out := ToOpenAIChatRequest(schemas.NewBifrostContext(nil, schemas.NoDeadline), &schemas.BifrostChatRequest{ + Provider: provider, + Model: tt.model, + Input: []schemas.ChatMessage{{ + Role: schemas.ChatMessageRoleUser, + Content: &schemas.ChatMessageContent{ContentStr: schemas.Ptr("hello")}, + }}, + Params: params, + }) + require.NotNil(t, out) + if tt.stripped { + require.Nil(t, out.Temperature) + require.Nil(t, out.TopP) + require.Nil(t, out.LogProbs) + require.Nil(t, out.TopLogProbs) + } else { + require.NotNil(t, out.Temperature) + require.NotNil(t, out.TopP) + require.NotNil(t, out.LogProbs) + require.NotNil(t, out.TopLogProbs) + } + require.NotNil(t, params.Temperature, "caller's params must not be mutated") + }) + } +} + func TestOpenAIChatRequest_FilterOpenAISpecificParameters_NormalizesReasoningEffort(t *testing.T) { // Register the custom "deepseek" provider so ParseModelString strips its prefix. schemas.RegisterKnownProvider(schemas.ModelProvider("deepseek")) diff --git a/core/providers/openai/chatconversion_bench_test.go b/core/providers/openai/chatconversion_bench_test.go new file mode 100644 index 00000000000..51599fd0264 --- /dev/null +++ b/core/providers/openai/chatconversion_bench_test.go @@ -0,0 +1,206 @@ +package openai + +// Benchmarks for the OpenAI chat-completions translation layer. +// +// Every request that reaches the gateway on the /v1/chat/completions surface +// walks this path: parse the caller's payload into OpenAIChatRequest, lower it +// into the neutral schema, then raise it back into the wire shape for whichever +// OpenAI-family provider serves it (OpenAI, Azure, Groq, xAI, DeepSeek, ...). +// The per-provider compatibility passes run inside ToOpenAIChatRequest, so a +// regression there is charged to every single request. +// +// Run: +// +// go test ./core/providers/openai/ -bench ChatRequest -benchmem + +import ( + "testing" + + "github.com/capsohq/bifrost/core/schemas" +) + +// A payload representative of what an SDK sends: a short multi-turn +// conversation, a tool result replayed back, and two tool definitions with +// nested JSON-Schema parameters. +const benchOpenAIChatRequestJSON = `{ + "model": "openai/gpt-4o", + "messages": [ + {"role": "system", "content": "You are a concise assistant. Answer in one sentence."}, + {"role": "user", "content": "What's the weather in San Francisco and Bengaluru?"}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "call_9xKmNpQrStUvWxYz01", + "type": "function", + "function": {"name": "get_weather", "arguments": "{\"location\":\"San Francisco, CA\",\"unit\":\"celsius\"}"} + } + ] + }, + {"role": "tool", "tool_call_id": "call_9xKmNpQrStUvWxYz01", "content": "{\"temp_c\":14.4,\"condition\":\"foggy\"}"}, + { + "role": "user", + "content": [ + {"type": "text", "text": "And how does that compare to yesterday?"} + ] + } + ], + "temperature": 0.2, + "top_p": 0.95, + "max_completion_tokens": 1024, + "parallel_tool_calls": true, + "stream": false, + "user": "user_42", + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather for a location", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string", "description": "City and state, e.g. San Francisco, CA"}, + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, + "days": {"type": "integer", "minimum": 1, "maximum": 14} + }, + "required": ["location"], + "additionalProperties": false + } + } + }, + { + "type": "function", + "function": { + "name": "search_docs", + "description": "Search the internal documentation index", + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "top_k": {"type": "integer", "minimum": 1, "maximum": 50}, + "filters": { + "type": "object", + "properties": { + "product": {"type": "string"}, + "updated_after": {"type": "string"} + } + } + }, + "required": ["query"] + } + } + } + ] +}` + +func benchOpenAIChatRequest(b *testing.B) *OpenAIChatRequest { + b.Helper() + var req OpenAIChatRequest + if err := schemas.Unmarshal([]byte(benchOpenAIChatRequestJSON), &req); err != nil { + b.Fatalf("unmarshal OpenAI chat request: %v", err) + } + return &req +} + +// BenchmarkOpenAIChatRequestUnmarshal measures parsing the inbound payload, +// including the extra-params capture that keeps unknown client fields. +func BenchmarkOpenAIChatRequestUnmarshal(b *testing.B) { + data := []byte(benchOpenAIChatRequestJSON) + b.ReportAllocs() + for b.Loop() { + var req OpenAIChatRequest + if err := schemas.Unmarshal(data, &req); err != nil { + b.Fatalf("unmarshal OpenAI chat request: %v", err) + } + } +} + +// BenchmarkOpenAIChatRequestToBifrost measures lowering the provider shape into +// the neutral schema Bifrost routes on. +func BenchmarkOpenAIChatRequestToBifrost(b *testing.B) { + req := benchOpenAIChatRequest(b) + ctx := schemas.NewBifrostContext(nil, schemas.NoDeadline) + b.ReportAllocs() + for b.Loop() { + if out := req.ToBifrostChatRequest(ctx); out == nil { + b.Fatal("nil Bifrost request") + } + } +} + +// BenchmarkToOpenAIChatRequest measures raising the neutral schema back into +// the OpenAI wire shape, capability gating and tool normalization included. +func BenchmarkToOpenAIChatRequest(b *testing.B) { + ctx := schemas.NewBifrostContext(nil, schemas.NoDeadline) + bifrostReq := benchOpenAIChatRequest(b).ToBifrostChatRequest(ctx) + b.ReportAllocs() + for b.Loop() { + if out := ToOpenAIChatRequest(ctx, bifrostReq); out == nil { + b.Fatal("nil OpenAI request") + } + } +} + +// BenchmarkToOpenAIChatRequestGroq exercises a provider whose conversion runs +// the compatibility filters (unsupported parameters dropped, assistant +// reasoning renamed) rather than returning early like OpenAI itself. +func BenchmarkToOpenAIChatRequestGroq(b *testing.B) { + ctx := schemas.NewBifrostContext(nil, schemas.NoDeadline) + bifrostReq := benchOpenAIChatRequest(b).ToBifrostChatRequest(ctx) + bifrostReq.Provider = schemas.Groq + bifrostReq.Model = "llama-3.3-70b-versatile" + b.ReportAllocs() + for b.Loop() { + if out := ToOpenAIChatRequest(ctx, bifrostReq); out == nil { + b.Fatal("nil OpenAI request") + } + } +} + +// BenchmarkOpenAIChatRequestMarshal measures serializing the outbound payload. +// OpenAIChatRequest has a hand-written marshaller that preserves key order for +// provider-side prompt caching, so this is not plain reflection cost. +func BenchmarkOpenAIChatRequestMarshal(b *testing.B) { + ctx := schemas.NewBifrostContext(nil, schemas.NoDeadline) + req := ToOpenAIChatRequest(ctx, benchOpenAIChatRequest(b).ToBifrostChatRequest(ctx)) + if req == nil { + b.Fatal("nil OpenAI request") + } + b.ReportAllocs() + for b.Loop() { + out, err := schemas.Marshal(req) + if err != nil { + b.Fatalf("marshal OpenAI chat request: %v", err) + } + if len(out) == 0 { + b.Fatal("empty payload") + } + } +} + +// BenchmarkOpenAIChatRequestRoundTrip measures the full inbound-to-outbound +// pipeline for a single request: parse, lower, raise, serialize. +func BenchmarkOpenAIChatRequestRoundTrip(b *testing.B) { + data := []byte(benchOpenAIChatRequestJSON) + ctx := schemas.NewBifrostContext(nil, schemas.NoDeadline) + b.ReportAllocs() + for b.Loop() { + var req OpenAIChatRequest + if err := schemas.Unmarshal(data, &req); err != nil { + b.Fatalf("unmarshal OpenAI chat request: %v", err) + } + out := ToOpenAIChatRequest(ctx, req.ToBifrostChatRequest(ctx)) + if out == nil { + b.Fatal("nil OpenAI request") + } + payload, err := schemas.Marshal(out) + if err != nil { + b.Fatalf("marshal OpenAI chat request: %v", err) + } + if len(payload) == 0 { + b.Fatal("empty payload") + } + } +} diff --git a/core/providers/openai/openai.go b/core/providers/openai/openai.go index 787d2b88bfb..0a753c74c9d 100644 --- a/core/providers/openai/openai.go +++ b/core/providers/openai/openai.go @@ -4149,6 +4149,11 @@ func (provider *OpenAIProvider) Rerank(ctx *schemas.BifrostContext, key schemas. ) } +// Decision is not supported by the OpenAI provider. +func (provider *OpenAIProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // HandleOpenAIRerankRequest handles rerank requests for custom OpenAI-compatible APIs. func HandleOpenAIRerankRequest( ctx *schemas.BifrostContext, diff --git a/core/providers/openai/openai_test.go b/core/providers/openai/openai_test.go index 3b71eee1aab..6a43d11ec7a 100644 --- a/core/providers/openai/openai_test.go +++ b/core/providers/openai/openai_test.go @@ -46,7 +46,9 @@ func TestOpenAI(t *testing.T) { ChatAudioModel: "gpt-audio-mini", PassthroughModel: "gpt-4o", ExternalCompactionModel: "gpt-4o", + DecisionEmulationModel: "gpt-4o-mini", Scenarios: llmtests.TestScenarios{ + DecisionEmulation: true, TextCompletion: true, TextCompletionStream: true, SimpleChat: true, diff --git a/core/providers/openai/responses.go b/core/providers/openai/responses.go index 611586c0136..a14e088fa04 100644 --- a/core/providers/openai/responses.go +++ b/core/providers/openai/responses.go @@ -233,6 +233,52 @@ func hoistAdditionalTools(message schemas.ResponsesMessage) []schemas.ResponsesT return tools } +// normalizeAsyncTools keeps OpenAI's async marker only where the API accepts it: +// on function/custom tool definitions for models that support async tool calling. +// Namespace children are normalized recursively. Every modified node is copied so +// conversion never mutates the caller-owned request. +func normalizeAsyncTools(tools []schemas.ResponsesTool, supported bool) []schemas.ResponsesTool { + if len(tools) == 0 { + return tools + } + + normalized := make([]schemas.ResponsesTool, len(tools)) + for i, tool := range tools { + if !supported || (tool.Type != schemas.ResponsesToolTypeFunction && tool.Type != schemas.ResponsesToolTypeCustom) { + tool.Async = nil + } + if tool.ResponsesToolNamespace != nil && len(tool.ResponsesToolNamespace.Tools) > 0 { + namespaceCopy := *tool.ResponsesToolNamespace + namespaceCopy.Tools = normalizeAsyncTools(tool.ResponsesToolNamespace.Tools, supported) + tool.ResponsesToolNamespace = &namespaceCopy + } + normalized[i] = tool + } + return normalized +} + +// normalizeAsyncCallItems applies the same capability gate to replayed calls. +// OpenAI defines async only on function_call and custom_tool_call items. +func normalizeAsyncCallItems(messages []schemas.ResponsesMessage, supported bool) []schemas.ResponsesMessage { + for i := range messages { + message := &messages[i] + if message.ResponsesToolMessage == nil || message.ResponsesToolMessage.Async == nil { + continue + } + allowedType := message.Type != nil && + (*message.Type == schemas.ResponsesMessageTypeFunctionCall || + *message.Type == schemas.ResponsesMessageTypeCustomToolCall) + if supported && allowedType { + continue + } + + toolMessageCopy := *message.ResponsesToolMessage + toolMessageCopy.Async = nil + message.ResponsesToolMessage = &toolMessageCopy + } + return messages +} + // PromptCacheBreakpointModeExplicit is the only mode a prompt_cache_breakpoint // accepts. OpenAI defined the field for gpt-5.6+; OpenRouter reuses it as the // Responses-shaped spelling of an Anthropic cache breakpoint. @@ -255,7 +301,7 @@ const maxResponsesCacheBreakpoints = 4 // // Two unrelated families landed on the same field. OpenRouter does not expose // per-block cache_control through /v1/responses and converts a breakpoint back into -// an Anthropic one (#6290). OpenAI defined the field for gpt-5.6, where it pairs with +// an Anthropic one (#6290). OpenAI defined the field for gpt-5.6 and later, where it pairs with // request-level prompt_cache_options; Azure and Bedrock Mantle serve the same models // through the same wire format, so they inherit it (#6180). Bedrock is listed for the // same reason: its OpenAI-compatible surfaces on both hosts speak that wire format, and @@ -263,15 +309,14 @@ const maxResponsesCacheBreakpoints = 4 // // Everything else either accepts cache_control directly or caches implicitly, and for // those the serializer's existing strip is the correct behaviour. -// -// This is the name-based fallback for ModelCaps.SupportsPromptCacheBreakpoints, used -// when the datasheet row says nothing. -func responsesUsesPromptCacheBreakpoints(provider schemas.ModelProvider, model string) bool { +func responsesUsesPromptCacheBreakpoints(caps schemas.ModelCaps, provider schemas.ModelProvider, model string) bool { switch provider { case schemas.OpenRouter: - return schemas.IsAnthropicModel(model) || schemas.IsGPT56Model(model) + return caps.SupportsPromptCacheBreakpoints( + schemas.IsAnthropicModel(model) || schemas.ModelSupportsPromptCacheBreakpoint(model), + ) case schemas.OpenAI, schemas.Azure, schemas.BedrockMantle, schemas.Bedrock: - return schemas.IsGPT56Model(model) + return caps.SupportsPromptCacheBreakpoint(schemas.ModelSupportsPromptCacheBreakpoint(model)) default: return false } @@ -298,14 +343,14 @@ func responsesHasPromptCacheBreakpoint(messages []schemas.ResponsesMessage) bool // responsesUsesPromptCacheOptions reports whether the target also needs request-level // prompt_cache_options to honour an explicit breakpoint. // -// This is the gpt-5.6 half only. Those models default to IMPLICIT caching, which puts +// This is the OpenAI half only (gpt-5.6 and later). Those models default to IMPLICIT caching, which puts // the breakpoint on the latest message - so an agent loop rewrites the whole growing // prompt every turn at the cache-write rate. The block marker alone does not switch // that off; mode=explicit does. OpenRouter has no equivalent field and needs none. -func responsesUsesPromptCacheOptions(provider schemas.ModelProvider, model string) bool { +func responsesUsesPromptCacheOptions(caps schemas.ModelCaps, provider schemas.ModelProvider, model string) bool { switch provider { case schemas.OpenAI, schemas.Azure, schemas.BedrockMantle, schemas.Bedrock: - return schemas.IsGPT56Model(model) + return caps.SupportsPromptCacheBreakpoint(schemas.ModelSupportsPromptCacheBreakpoint(model)) default: return false } @@ -647,14 +692,16 @@ func ToOpenAIResponsesRequest(ctx *schemas.BifrostContext, bifrostReq *schemas.B // core/bifrost.go, and providers/utils does too; this call site was the odd one out. cachePromptProvider := schemas.ResolveBaseProvider(ctx, bifrostReq.Provider) needsExplicitPromptCacheMode := false - if caps.SupportsPromptCacheBreakpoints(responsesUsesPromptCacheBreakpoints(cachePromptProvider, capModel)) { + if responsesUsesPromptCacheBreakpoints(caps, cachePromptProvider, capModel) { applyResponsesCacheBreakpoints(messages) - needsExplicitPromptCacheMode = responsesUsesPromptCacheOptions(cachePromptProvider, capModel) && + needsExplicitPromptCacheMode = responsesUsesPromptCacheOptions(caps, cachePromptProvider, capModel) && responsesHasPromptCacheBreakpoint(messages) } // Updating params params := bifrostReq.Params + asyncToolsSupported := caps.SupportsAsyncTools(defaultSupportsAsyncTools(capModel)) + messages = normalizeAsyncCallItems(messages, asyncToolsSupported) // Create the responses request with properly mapped parameters req := &OpenAIResponsesRequest{ Model: bifrostReq.Model, @@ -715,6 +762,11 @@ func ToOpenAIResponsesRequest(ctx *schemas.BifrostContext, bifrostReq *schemas.B // Clear max_tokens since OpenAI doesn't use it req.ResponsesParameters.Reasoning.MaxTokens = nil } + // A model that always reasons rejects "none"; "minimal" normalizes to its lowest level. + if e := req.ResponsesParameters.Reasoning.Effort; e != nil && *e == schemas.ReasoningEffortNone && + !caps.CanDisableReasoning(defaultCanDisableReasoning(capModel)) { + req.ResponsesParameters.Reasoning.Effort = schemas.Ptr(caps.NormalizeReasoningEffort(schemas.ReasoningEffortMinimal, defaultEffortControl(capModel))) + } // summary:"none" is Anthropic-specific (maps to display:"omitted"); strip it for OpenAI. if req.ResponsesParameters.Reasoning.Summary != nil && *req.ResponsesParameters.Reasoning.Summary == "none" { @@ -776,9 +828,25 @@ func ToOpenAIResponsesRequest(ctx *schemas.BifrostContext, bifrostReq *schemas.B req.ResponsesParameters.Reasoning.Effort != nil { effort = *req.ResponsesParameters.Reasoning.Effort } - if topPUnsupported(caps, capModel, effort) { + if samplingParamUnsupported(caps, schemas.FieldTopP, capModel, effort) { req.ResponsesParameters.TopP = nil } + // Only OpenAI hosts gate these; third-party gpt-oss hosts accept them. + if base := schemas.ResolveBaseProvider(ctx, bifrostReq.Provider); base == schemas.OpenAI || base == schemas.Azure { + if samplingParamUnsupported(caps, schemas.FieldTemperature, capModel, effort) { + req.ResponsesParameters.Temperature = nil + } + if samplingParamUnsupported(caps, schemas.FieldTopLogprobs, capModel, effort) { + req.ResponsesParameters.TopLogProbs = nil + } + if samplingParamUnsupported(caps, schemas.FieldLogprobs, capModel, effort) && + slices.Contains(req.ResponsesParameters.Include, "message.output_text.logprobs") { + // Clone: Include shares its backing array with bifrostReq.Params. + req.ResponsesParameters.Include = slices.DeleteFunc(slices.Clone(req.ResponsesParameters.Include), func(s string) bool { + return s == "message.output_text.logprobs" + }) + } + } } // gpt-5.6 defaults to IMPLICIT caching, which anchors the breakpoint on the latest @@ -818,8 +886,7 @@ func ToOpenAIResponsesRequest(ctx *schemas.BifrostContext, bifrostReq *schemas.B // strict-pydantic upstreams (e.g. sglang) reject the explicit null outright. // We must copy the Tools slice since it shares the backing array with bifrostReq.Params.Tools. if len(req.Tools) > 0 { - normalizedTools := make([]schemas.ResponsesTool, len(req.Tools)) - copy(normalizedTools, req.Tools) + normalizedTools := normalizeAsyncTools(req.Tools, asyncToolsSupported) for i, tool := range normalizedTools { if tool.Type == schemas.ResponsesToolTypeFunction && tool.ResponsesToolFunction != nil { @@ -852,22 +919,25 @@ func ToOpenAIResponsesRequest(ctx *schemas.BifrostContext, bifrostReq *schemas.B return req } -// topPUnsupported reports whether the model rejects top_p. The datasheet can mark -// it unsupported outright, or conditionally via "when_effort_none" — accepted only +// samplingParamUnsupported reports whether the model rejects a sampling field +// (top_p, temperature, top_logprobs, logprobs). The datasheet can mark it +// unsupported outright, or conditionally via "when_effort_none" — accepted only // while reasoning is off. The fallback is name detection: OpenAI reasoning models -// (o1/o3 series) reject it, except GPT-5.x while effort is "none", which is the -// default when omitted. The -pro and -codex variants always reason, so they always -// strip. Gated on the OpenAI-family name rather than caps.SupportsReasoning: this -// asks whether the API rejects top_p, which is not the same question for xAI/Groq. -func topPUnsupported(caps schemas.ModelCaps, model, effort string) bool { - effortIsNone := effort == "" || effort == schemas.ReasoningEffortNone +// (o1/o3 series, GPT-6) reject it, except GPT-5.x while effort is "none". An +// omitted effort counts as "none" only where that is the model's default +// (GPT-5.1 through 5.4). The -pro variants always reason, so they always strip. +// Gated on the OpenAI-family name rather than caps.SupportsReasoning: +// this asks whether the API rejects the field, which is not the same question for +// xAI/Groq. +func samplingParamUnsupported(caps schemas.ModelCaps, field, model, effort string) bool { + effortIsNone := effort == schemas.ReasoningEffortNone || (effort == "" && !omittedEffortReasons(model)) // An outright unsupported_fields entry outranks the conditional label: the // row rejects the field whatever the effort. Probed with a false fallback so // only an explicit true short-circuits. - if caps.FieldUnsupported(schemas.FieldTopP, false) { + if caps.FieldUnsupported(field, false) { return true } - if caps.FieldCondition(schemas.FieldTopP) == schemas.ConditionWhenEffortNone { + if caps.FieldCondition(field) == schemas.ConditionWhenEffortNone { return !effortIsNone } @@ -878,12 +948,11 @@ func topPUnsupported(caps schemas.ModelCaps, model, effort string) bool { _, parsedModel := schemas.ParseModelString(model, schemas.OpenAI) modelLower := strings.ToLower(parsedModel) if strings.Contains(modelLower, "gpt-5.") && effortIsNone && - !strings.Contains(modelLower, "-pro") && - !strings.Contains(modelLower, "-codex") { + !strings.Contains(modelLower, "-pro") { fallback = false } } - return caps.FieldUnsupported(schemas.FieldTopP, fallback) + return caps.FieldUnsupported(field, fallback) } // filterUnsupportedTools removes tool types that OpenAI doesn't support @@ -981,6 +1050,7 @@ func (resp *OpenAIResponsesRequest) filterUnsupportedTools(webSearchContentTypes schemas.ResponsesToolTypeFunction: true, schemas.ResponsesToolTypeFileSearch: true, schemas.ResponsesToolTypeComputerUsePreview: true, + schemas.ResponsesToolTypeComputer: true, schemas.ResponsesToolTypeWebSearch: true, schemas.ResponsesToolTypeWebFetch: true, schemas.ResponsesToolTypeMCP: true, diff --git a/core/providers/openai/responses_test.go b/core/providers/openai/responses_test.go index 2596b59d28f..6f0a7a75016 100644 --- a/core/providers/openai/responses_test.go +++ b/core/providers/openai/responses_test.go @@ -576,6 +576,54 @@ func TestToOpenAIResponsesRequest_NormalizesReasoningEffort(t *testing.T) { effort: "max", expected: "high", }, + { + name: "maps none to low for gpt-6-astra", + model: "gpt-6-astra", + effort: "none", + expected: "low", + }, + { + name: "preserves none for gpt-6-sol", + model: "gpt-6-sol", + effort: "none", + expected: "none", + }, + { + name: "preserves none for gpt-6-luna", + model: "gpt-6-luna", + effort: "none", + expected: "none", + }, + { + name: "maps none to minimal for gpt-5", + model: "gpt-5", + effort: "none", + expected: "minimal", + }, + { + name: "maps none to low for gpt-5-pro", + model: "gpt-5-pro", + effort: "none", + expected: "low", + }, + { + name: "maps none to low for o3", + model: "o3", + effort: "none", + expected: "low", + }, + { + name: "preserves none for gpt-5.1", + model: "gpt-5.1", + effort: "none", + expected: "none", + }, + { + name: "preserves none for gpt-5.6", + model: "gpt-5.6", + effort: "none", + expected: "none", + }, { // DeepSeek V4 is routed via a custom OpenAI-compatible provider, so the // OpenAI-only reasoning-stripping doesn't apply and "max" passes through. @@ -2896,36 +2944,60 @@ func TestBifrostResponsesRetrieveRequest_IsStreamingRequested(t *testing.T) { } } -// TestTopPGateReadsDatasheet covers the datasheet side of the top_p strip. -// top_p is the field the schema deliberately leaves untyped because its verdict -// can be conditional, so both unsupported_fields and -// conditionally_unsupported_fields have to drive it. -func TestTopPGateReadsDatasheet(t *testing.T) { +// TestSamplingParamGateReadsDatasheet covers the datasheet side of the sampling +// strip. Their verdict can be conditional on reasoning effort, so both +// unsupported_fields and conditionally_unsupported_fields have to drive it. +func TestSamplingParamGateReadsDatasheet(t *testing.T) { t.Run("name_fallback_strips_for_reasoning_model", func(t *testing.T) { caps := schemas.ResolveModelCaps(schemas.OpenAI, "o3") - require.True(t, topPUnsupported(caps, "o3", "high")) - require.False(t, topPUnsupported(caps, "gpt-4o", "")) + require.True(t, samplingParamUnsupported(caps, schemas.FieldTopP, "o3", "high")) + require.False(t, samplingParamUnsupported(caps, schemas.FieldTopP, "gpt-4o", "")) }) t.Run("name_fallback_keeps_gpt5x_while_effort_none", func(t *testing.T) { caps := schemas.ResolveModelCaps(schemas.OpenAI, "gpt-5.4") - require.False(t, topPUnsupported(caps, "gpt-5.4", "")) - require.True(t, topPUnsupported(caps, "gpt-5.4", "high")) - require.True(t, topPUnsupported(caps, "gpt-5.4-pro", ""), "-pro always reasons") + require.False(t, samplingParamUnsupported(caps, schemas.FieldTopP, "gpt-5.4", "")) + require.True(t, samplingParamUnsupported(caps, schemas.FieldTopP, "gpt-5.4", "high")) + require.True(t, samplingParamUnsupported(caps, schemas.FieldTopP, "gpt-5.4-pro", ""), "-pro always reasons") + }) + + t.Run("omitted_effort_reasons_from_gpt55", func(t *testing.T) { + for _, model := range []string{"gpt-5.5", "gpt-5.6-sol"} { + caps := schemas.ResolveModelCaps(schemas.OpenAI, model) + require.True(t, samplingParamUnsupported(caps, schemas.FieldTemperature, model, ""), "%s defaults to medium", model) + require.False(t, samplingParamUnsupported(caps, schemas.FieldTemperature, model, "none"), model) + } + }) + + t.Run("codex_follows_its_version_default", func(t *testing.T) { + caps := schemas.ResolveModelCaps(schemas.OpenAI, "gpt-5.3-codex") + require.False(t, samplingParamUnsupported(caps, schemas.FieldTemperature, "gpt-5.3-codex", ""), "gpt-5.3-codex defaults to none") + require.True(t, samplingParamUnsupported(caps, schemas.FieldTemperature, "gpt-5.3-codex", "low")) + }) + + t.Run("gpt6_rejects_whatever_the_effort", func(t *testing.T) { + caps := schemas.ResolveModelCaps(schemas.OpenAI, "gpt-6-astra") + for _, field := range []string{schemas.FieldTopP, schemas.FieldTemperature, schemas.FieldTopLogprobs, schemas.FieldLogprobs} { + require.True(t, samplingParamUnsupported(caps, field, "gpt-6-astra", ""), field) + require.True(t, samplingParamUnsupported(caps, field, "gpt-6-astra", "low"), field) + } }) t.Run("conditional_label_drives_both_directions", func(t *testing.T) { const model = "some-conditional-reasoner" installCapabilityRecord(t, model, &schemas.ModelCapabilities{ ConditionallyUnsupportedFields: map[string]string{ - schemas.FieldTopP: schemas.ConditionWhenEffortNone, + schemas.FieldTopP: schemas.ConditionWhenEffortNone, + schemas.FieldTemperature: schemas.ConditionWhenEffortNone, }, }) caps := schemas.ResolveModelCaps(schemas.OpenAI, model) - require.False(t, topPUnsupported(caps, model, ""), "allowed while reasoning is off") - require.False(t, topPUnsupported(caps, model, "none")) - require.True(t, topPUnsupported(caps, model, "low"), "rejected once reasoning is on") + for _, field := range []string{schemas.FieldTopP, schemas.FieldTemperature} { + require.False(t, samplingParamUnsupported(caps, field, model, ""), "allowed while reasoning is off") + require.False(t, samplingParamUnsupported(caps, field, model, "none")) + require.True(t, samplingParamUnsupported(caps, field, model, "low"), "rejected once reasoning is on") + } }) t.Run("outright_entry_beats_name_fallback", func(t *testing.T) { @@ -2933,11 +3005,93 @@ func TestTopPGateReadsDatasheet(t *testing.T) { installCapabilityRecord(t, model, &schemas.ModelCapabilities{ UnsupportedFields: map[string]bool{schemas.FieldTopP: false}, }) - require.False(t, topPUnsupported(schemas.ResolveModelCaps(schemas.OpenAI, model), model, "high"), + require.False(t, samplingParamUnsupported(schemas.ResolveModelCaps(schemas.OpenAI, model), schemas.FieldTopP, model, "high"), "an explicit false must beat the reasoning-model name check") }) } +// TestToOpenAIResponsesRequest_AlwaysReasoningModels pins the request shaping for +// models that cannot turn reasoning off: "none" maps to the lowest level the model +// accepts, and sampling fields OpenAI rejects are dropped without mutating the +// caller's params. +func TestToOpenAIResponsesRequest_AlwaysReasoningModels(t *testing.T) { + convert := func(provider schemas.ModelProvider, model string, params *schemas.ResponsesParameters) *OpenAIResponsesRequest { + return ToOpenAIResponsesRequest(nil, &schemas.BifrostResponsesRequest{ + Provider: provider, + Model: model, + Input: []schemas.ResponsesMessage{{ + Role: schemas.Ptr(schemas.ResponsesInputMessageRoleUser), + Content: &schemas.ResponsesMessageContent{ContentStr: schemas.Ptr("hello")}, + }}, + Params: params, + }) + } + + t.Run("budget_zero_maps_to_lowest_level", func(t *testing.T) { + req := convert(schemas.OpenAI, "gpt-6-astra", &schemas.ResponsesParameters{ + Reasoning: &schemas.ResponsesParametersReasoning{MaxTokens: schemas.Ptr(0)}, + }) + require.Equal(t, "low", *req.Reasoning.Effort) + }) + + t.Run("datasheet_disable_flag_beats_name_fallback", func(t *testing.T) { + installCapabilityRecord(t, "gpt-6-astra", &schemas.ModelCapabilities{SupportsReasoningDisable: new(true)}) + req := convert(schemas.OpenAI, "gpt-6-astra", &schemas.ResponsesParameters{ + Reasoning: &schemas.ResponsesParametersReasoning{Effort: schemas.Ptr("none")}, + }) + require.Equal(t, "none", *req.Reasoning.Effort) + }) + + t.Run("datasheet_ladder_sets_lowest_level", func(t *testing.T) { + installCapabilityRecord(t, "gpt-5.1", &schemas.ModelCapabilities{ + SupportsReasoningDisable: new(false), + ReasoningEffortLevels: []string{"medium", "high"}, + }) + req := convert(schemas.OpenAI, "gpt-5.1", &schemas.ResponsesParameters{ + Reasoning: &schemas.ResponsesParametersReasoning{Effort: schemas.Ptr("none")}, + }) + require.Equal(t, "medium", *req.Reasoning.Effort) + }) + + t.Run("strips_sampling_fields_for_gpt6", func(t *testing.T) { + include := []string{"message.output_text.logprobs", "reasoning.encrypted_content"} + params := &schemas.ResponsesParameters{ + Temperature: schemas.Ptr(0.2), + TopP: schemas.Ptr(0.9), + TopLogProbs: schemas.Ptr(3), + Include: include, + } + for _, provider := range []schemas.ModelProvider{schemas.OpenAI, schemas.Azure} { + req := convert(provider, "gpt-6-astra", params) + require.Nil(t, req.Temperature, provider) + require.Nil(t, req.TopP, provider) + require.Nil(t, req.TopLogProbs, provider) + require.Equal(t, []string{"reasoning.encrypted_content"}, req.Include, provider) + } + require.Equal(t, []string{"message.output_text.logprobs", "reasoning.encrypted_content"}, params.Include, + "caller's include must not be mutated") + }) + + t.Run("keeps_sampling_fields_while_effort_none", func(t *testing.T) { + req := convert(schemas.OpenAI, "gpt-5.6", &schemas.ResponsesParameters{ + Temperature: schemas.Ptr(0.2), + Reasoning: &schemas.ResponsesParametersReasoning{Effort: schemas.Ptr("none")}, + }) + require.NotNil(t, req.Temperature) + + req = convert(schemas.OpenAI, "gpt-5.4", &schemas.ResponsesParameters{Temperature: schemas.Ptr(0.2)}) + require.NotNil(t, req.Temperature, "gpt-5.4 defaults to none") + + req = convert(schemas.OpenAI, "gpt-5.5", &schemas.ResponsesParameters{Temperature: schemas.Ptr(0.2)}) + require.Nil(t, req.Temperature, "gpt-5.5 defaults to medium") + }) + + t.Run("third_party_gpt_oss_keeps_temperature", func(t *testing.T) { + req := convert(schemas.Groq, "openai/gpt-oss-120b", &schemas.ResponsesParameters{Temperature: schemas.Ptr(0.2)}) + require.NotNil(t, req.Temperature) + }) +} + // TestReasoningContentBlocksGateReadsDatasheet covers the datasheet side of the // reasoning-shape gate. The gpt-oss name fallback is covered by // TestToOpenAIResponsesRequest_ReasoningOnlyMessageSkip and the Summary→blocks @@ -3334,3 +3488,95 @@ func TestToOpenAIResponsesRequest_MantleGPTOSSReplaysAssistantTextAsInput(t *tes }) } } + +// Web search action sources are sanitized for OpenAI: provider-specific fields +// (title, encrypted_content, page_age) are stripped, while the OpenAI-native +// fields survive. That includes the name of specialized API sources +// ({"type":"api","name":"oai-weather"}), which carry no URL and must not gain a +// fabricated empty one. +func TestToOpenAIResponsesRequest_StripsWebSearchSourceProviderFields(t *testing.T) { + history := `{ + "id": "ws_1", + "type": "web_search_call", + "status": "completed", + "action": { + "type": "search", + "queries": ["weather in paris"], + "sources": [ + {"type": "url", "url": "https://example.com", "title": "Example"}, + {"type": "api", "name": "oai-weather"} + ] + } + }` + var webSearchCall schemas.ResponsesMessage + require.NoError(t, schemas.Unmarshal([]byte(history), &webSearchCall)) + + bifrostReq := &schemas.BifrostResponsesRequest{ + Model: "gpt-4o", + Input: []schemas.ResponsesMessage{webSearchCall}, + } + result := ToOpenAIResponsesRequest(nil, bifrostReq) + require.NotNil(t, result) + + body, err := sonic.Marshal(result) + require.NoError(t, err) + var wire struct { + Input []struct { + Action struct { + Sources []map[string]any `json:"sources"` + } `json:"action"` + } `json:"input"` + } + require.NoError(t, json.Unmarshal(body, &wire)) + require.Len(t, wire.Input, 1, "body: %s", body) + require.Len(t, wire.Input[0].Action.Sources, 2, "body: %s", body) + + urlSource, apiSource := wire.Input[0].Action.Sources[0], wire.Input[0].Action.Sources[1] + + require.NotContains(t, urlSource, "title", "provider-specific title must be stripped: %s", body) + require.Equal(t, "https://example.com", urlSource["url"], "url source keeps its url: %s", body) + + require.Equal(t, "api", apiSource["type"], "api source keeps its type: %s", body) + require.Equal(t, "oai-weather", apiSource["name"], "api source keeps its name: %s", body) + require.NotContains(t, apiSource, "url", "api source must not gain a fabricated empty url: %s", body) + + // The caller's input must not be mutated by the strip. + require.NotNil(t, bifrostReq.Input[0].ResponsesToolMessage.Action.ResponsesWebSearchToolCallAction.Sources[0].Title, + "the caller's input must not be mutated") +} + +// TestToOpenAIResponsesRequest_ForwardsComputerTool locks in issue #7425: the +// bare `computer` tool (GPT-6 Astra / GPT-5.6 computer use) must pass the +// OpenAI tool whitelist unchanged, not be dropped or rewritten to +// computer_use_preview. +func TestToOpenAIResponsesRequest_ForwardsComputerTool(t *testing.T) { + bifrostReq := &schemas.BifrostResponsesRequest{ + Provider: schemas.OpenAI, + Model: "gpt-6-astra", + Input: []schemas.ResponsesMessage{ + { + Role: schemas.Ptr(schemas.ResponsesInputMessageRoleUser), + Content: &schemas.ResponsesMessageContent{ + ContentStr: schemas.Ptr("Click Settings."), + }, + }, + }, + Params: &schemas.ResponsesParameters{ + Tools: []schemas.ResponsesTool{{Type: schemas.ResponsesToolTypeComputer}}, + }, + } + + result := ToOpenAIResponsesRequest(nil, bifrostReq) + if result == nil { + t.Fatal("expected non-nil result") + } + if len(result.Tools) != 1 { + t.Fatalf("expected 1 tool, got %d", len(result.Tools)) + } + if result.Tools[0].Type != schemas.ResponsesToolTypeComputer { + t.Fatalf("expected tool type %q, got %q", schemas.ResponsesToolTypeComputer, result.Tools[0].Type) + } + if result.Tools[0].ResponsesToolComputerUsePreview != nil { + t.Fatal("expected no computer_use_preview fields on the bare computer tool") + } +} diff --git a/core/providers/openai/responsesmarshal_test.go b/core/providers/openai/responsesmarshal_test.go index eb487ccc6b5..ded742135d8 100644 --- a/core/providers/openai/responsesmarshal_test.go +++ b/core/providers/openai/responsesmarshal_test.go @@ -179,6 +179,8 @@ func TestNormalizeOpenAIReasoningEffort(t *testing.T) { {"gpt-6-astra keeps max", "gpt-6-astra", "max", "max"}, {"gpt-5.6 variant keeps max", "gpt-5.6-terra", "max", "max"}, {"gpt-5.6 keeps xhigh", "gpt-5.6", "xhigh", "xhigh"}, + {"grok-4.7 keeps xhigh", "grok-4.7", "xhigh", "xhigh"}, + {"grok-4.7 maps max to xhigh", "grok-4.7", "max", "xhigh"}, {"provider-prefixed gpt-5.6 keeps max", "openai/gpt-5.6", "max", "max"}, {"deepseek-v4 keeps max", "deepseek-v4", "max", "max"}, {"glm-5.2 keeps max", "glm-5.2", "max", "max"}, @@ -1684,6 +1686,9 @@ func TestToOpenAIResponsesRequest_GPT56CacheBreakpoint(t *testing.T) { {"openai", schemas.OpenAI, "gpt-5.6-sol"}, {"azure", schemas.Azure, "eu/gpt-5.6-sol"}, {"bedrock mantle", schemas.BedrockMantle, "openai.gpt-5.6-terra"}, + {"openai gpt-6", schemas.OpenAI, "gpt-6-astra"}, + {"azure gpt-6", schemas.Azure, "gpt-6-astra"}, + {"bedrock mantle gpt-6", schemas.BedrockMantle, "openai.gpt-6-astra"}, } { t.Run(tc.name, func(t *testing.T) { m, raw := marshalResponses(t, gpt56CacheReq(tc.provider, tc.model)) @@ -1738,6 +1743,33 @@ func TestToOpenAIResponsesRequest_PreGPT56Unaffected(t *testing.T) { } } +// TestToOpenAIResponsesRequest_CacheBreakpointGateReadsDatasheet pins that the +// datasheet's supports_prompt_cache_breakpoint beats the model-name fallback in both +// directions. +func TestToOpenAIResponsesRequest_CacheBreakpointGateReadsDatasheet(t *testing.T) { + t.Run("false_disables_a_named_family", func(t *testing.T) { + installCapabilityRecord(t, "gpt-5.6-sol", &schemas.ModelCapabilities{SupportsPromptCacheBreakpoint: new(false)}) + m, raw := marshalResponses(t, gpt56CacheReq(schemas.OpenAI, "gpt-5.6-sol")) + if _, present := firstBlock(t, m, 0, raw)["prompt_cache_breakpoint"]; present { + t.Errorf("datasheet false must suppress the breakpoint; raw=%s", raw) + } + if _, present := m["prompt_cache_options"]; present { + t.Errorf("datasheet false must suppress prompt_cache_options; raw=%s", raw) + } + }) + + t.Run("true_enables_an_unnamed_model", func(t *testing.T) { + installCapabilityRecord(t, "future-openai-model", &schemas.ModelCapabilities{SupportsPromptCacheBreakpoint: new(true)}) + m, raw := marshalResponses(t, gpt56CacheReq(schemas.OpenAI, "future-openai-model")) + if _, present := firstBlock(t, m, 0, raw)["prompt_cache_breakpoint"]; !present { + t.Errorf("datasheet true must enable the breakpoint; raw=%s", raw) + } + if _, present := m["prompt_cache_options"]; !present { + t.Errorf("datasheet true must enable prompt_cache_options; raw=%s", raw) + } + }) +} + // TestToOpenAIResponsesRequest_GPT56RespectsCallerCacheOptions verifies Bifrost does // not overwrite a caller that already made a caching decision. func TestToOpenAIResponsesRequest_GPT56RespectsCallerCacheOptions(t *testing.T) { @@ -2014,3 +2046,158 @@ func TestOpenAIResponsesRequest_MarshalJSON_StripsMediaResolution(t *testing.T) t.Error("sanitization must not mutate the caller's request") } } + +// TestToOpenAIResponsesRequest_ForwardsOpenAIToolFields pins that async, +// output_schema and tunnel_id reach the OpenAI wire unchanged. +func TestToOpenAIResponsesRequest_ForwardsOpenAIToolFields(t *testing.T) { + var tools []schemas.ResponsesTool + if err := sonic.Unmarshal([]byte(`[ + {"type":"function","name":"get_weather","async":true,"parameters":{"type":"object","properties":{}},"strict":true,"output_schema":{"type":"string"}}, + {"type":"custom","name":"run_job","async":true}, + {"type":"mcp","server_label":"internal","tunnel_id":"tunnel_0123456789abcdef0123456789abcdef"} + ]`), &tools); err != nil { + t.Fatalf("unmarshal tools: %v", err) + } + m, raw := marshalResponses(t, &schemas.BifrostResponsesRequest{ + Provider: schemas.OpenAI, + Model: "gpt-6-astra", + Input: []schemas.ResponsesMessage{{ + Role: schemas.Ptr(schemas.ResponsesInputMessageRoleUser), + Content: &schemas.ResponsesMessageContent{ContentStr: schemas.Ptr("hi")}, + }}, + Params: &schemas.ResponsesParameters{Tools: tools}, + }) + wireTools, _ := m["tools"].([]any) + if len(wireTools) != 3 { + t.Fatalf("expected 3 tools; raw=%s", raw) + } + fn, _ := wireTools[0].(map[string]any) + if fn["async"] != true { + t.Errorf("function async lost; raw=%s", raw) + } + if schema, _ := fn["output_schema"].(map[string]any); schema["type"] != "string" { + t.Errorf("function output_schema lost; raw=%s", raw) + } + if custom, _ := wireTools[1].(map[string]any); custom["async"] != true { + t.Errorf("custom async lost; raw=%s", raw) + } + if mcp, _ := wireTools[2].(map[string]any); mcp["tunnel_id"] != "tunnel_0123456789abcdef0123456789abcdef" { + t.Errorf("mcp tunnel_id lost; raw=%s", raw) + } +} + +// TestToOpenAIResponsesRequest_ForwardsAsyncOnReplayedCalls pins that a replayed +// pending async call keeps async on the OpenAI wire; without it OpenAI rejects the +// request with "No tool output found for function call" (live-verified on gpt-6-astra). +func TestToOpenAIResponsesRequest_ForwardsAsyncOnReplayedCalls(t *testing.T) { + var input []schemas.ResponsesMessage + if err := sonic.Unmarshal([]byte(`[ + {"role":"user","content":"Check the weather in Paris."}, + {"type":"function_call","call_id":"call_1","name":"get_weather","arguments":"{\"city\":\"Paris\"}","async":true}, + {"role":"user","content":"Thanks. Also, what is 2+2?"} + ]`), &input); err != nil { + t.Fatalf("unmarshal input: %v", err) + } + m, raw := marshalResponses(t, &schemas.BifrostResponsesRequest{ + Provider: schemas.OpenAI, + Model: "gpt-6-astra", + Input: input, + }) + items, _ := m["input"].([]any) + if len(items) != 3 { + t.Fatalf("expected 3 input items; raw=%s", raw) + } + if call, _ := items[1].(map[string]any); call["async"] != true { + t.Errorf("async dropped from replayed function_call; raw=%s", raw) + } +} + +// TestToOpenAIResponsesRequest_StripsAsyncForUnsupportedModels pins the async gate: +// models without async tool calling 400 on async in tool definitions and in replayed +// call items (live-verified on gpt-5.6), so it is stripped there and kept on gpt-6. +// The datasheet wins over the name fallback, and the caller's request is not mutated. +func TestToOpenAIResponsesRequest_StripsAsyncForUnsupportedModels(t *testing.T) { + newReq := func(t *testing.T, model string) *schemas.BifrostResponsesRequest { + t.Helper() + var tools []schemas.ResponsesTool + if err := sonic.Unmarshal([]byte(`[ + {"type":"function","name":"get_weather","async":true,"parameters":{"type":"object","properties":{}},"strict":true}, + {"type":"custom","name":"run_job","async":true}, + {"type":"namespace","name":"jobs","description":"Jobs","tools":[{"type":"function","name":"start","async":true,"parameters":{"type":"object","properties":{}},"strict":true}]} + ]`), &tools); err != nil { + t.Fatalf("unmarshal tools: %v", err) + } + var input []schemas.ResponsesMessage + if err := sonic.Unmarshal([]byte(`[ + {"role":"user","content":"Check the weather in Paris."}, + {"type":"function_call","call_id":"call_1","name":"get_weather","arguments":"{}","async":true}, + {"type":"function_call_output","call_id":"call_1","output":"{\"t\":20}"} + ]`), &input); err != nil { + t.Fatalf("unmarshal input: %v", err) + } + return &schemas.BifrostResponsesRequest{ + Provider: schemas.OpenAI, + Model: model, + Input: input, + Params: &schemas.ResponsesParameters{Tools: tools}, + } + } + + for _, tc := range []struct { + name string + model string + record *schemas.ModelCapabilities + keep bool + }{ + {name: "gpt-5.6 strips by name", model: "gpt-5.6"}, + {name: "gpt-6-astra keeps by name", model: "gpt-6-astra", keep: true}, + {name: "datasheet true keeps", model: "gpt-5.6", record: &schemas.ModelCapabilities{SupportsAsyncTools: new(true)}, keep: true}, + {name: "datasheet false strips", model: "gpt-6-astra", record: &schemas.ModelCapabilities{SupportsAsyncTools: new(false)}}, + } { + t.Run(tc.name, func(t *testing.T) { + if tc.record != nil { + installCapabilityRecord(t, tc.model, tc.record) + } + req := newReq(t, tc.model) + _, raw := marshalResponses(t, req) + if got := strings.Count(raw, `"async":true`); tc.keep && got != 4 || !tc.keep && got != 0 { + t.Fatalf("async occurrences = %d, keep=%v; raw=%s", got, tc.keep, raw) + } + + tools := req.Params.Tools + if !*tools[0].Async || !*tools[1].Async || !*tools[2].ResponsesToolNamespace.Tools[0].Async || !*req.Input[1].ResponsesToolMessage.Async { + t.Fatal("caller's request was mutated") + } + }) + } +} + +func TestToOpenAIResponsesRequest_StripsAsyncFromUnsupportedToolKinds(t *testing.T) { + var tools []schemas.ResponsesTool + if err := sonic.Unmarshal([]byte(`[ + {"type":"function","name":"run_job","async":true,"parameters":{"type":"object"}}, + {"type":"web_search","async":true} + ]`), &tools); err != nil { + t.Fatalf("unmarshal tools: %v", err) + } + var input []schemas.ResponsesMessage + if err := sonic.Unmarshal([]byte(`[ + {"type":"function_call","call_id":"call_1","name":"run_job","arguments":"{}","async":true}, + {"type":"function_call_output","call_id":"call_1","output":"done","async":true} + ]`), &input); err != nil { + t.Fatalf("unmarshal input: %v", err) + } + + _, raw := marshalResponses(t, &schemas.BifrostResponsesRequest{ + Provider: schemas.OpenAI, + Model: "gpt-6-astra", + Input: input, + Params: &schemas.ResponsesParameters{Tools: tools}, + }) + if got := strings.Count(raw, `"async":true`); got != 2 { + t.Fatalf("async occurrences = %d, want 2 on function declaration/call only; raw=%s", got, raw) + } + if tools[1].Async == nil || input[1].ResponsesToolMessage.Async == nil { + t.Fatal("caller's request was mutated") + } +} diff --git a/core/providers/openai/types.go b/core/providers/openai/types.go index bec12ab83fd..3185450b2bb 100644 --- a/core/providers/openai/types.go +++ b/core/providers/openai/types.go @@ -10,6 +10,7 @@ import ( "github.com/bytedance/sonic" providerUtils "github.com/capsohq/bifrost/core/providers/utils" "github.com/capsohq/bifrost/core/schemas" + "github.com/tidwall/gjson" "github.com/tidwall/sjson" ) @@ -560,10 +561,12 @@ func (r *OpenAIResponsesRequestInput) MarshalJSON() ([]byte, error) { webSearchActionCopy := *msg.ResponsesToolMessage.Action.ResponsesWebSearchToolCallAction strippedSources := make([]schemas.ResponsesWebSearchToolCallActionSearchSource, len(sources)) for j, source := range sources { - // Only keep Type and URL for OpenAI + // Only keep Type, URL and Name for OpenAI; Name identifies + // specialized API sources (type "api") that carry no URL. strippedSources[j] = schemas.ResponsesWebSearchToolCallActionSearchSource{ Type: source.Type, URL: source.URL, + Name: source.Name, // Title, EncryptedContent, and PageAge are omitted } } @@ -646,14 +649,47 @@ func (r *OpenAIResponsesRequestInput) MarshalJSON() ([]byte, error) { // encrypted_content rides the embedded *ResponsesReasoning, whose (no-omitempty) Summary // re-injects "summary": null. Reasoning items legitimately carry summary and are left intact. func stripCompactionItemSummary(data []byte, items []schemas.ResponsesMessage) []byte { - for i, msg := range items { - if msg.Type != nil && *msg.Type == schemas.ResponsesMessageTypeCompaction { - if updated, err := sjson.DeleteBytes(data, fmt.Sprintf("%d.summary", i)); err == nil { - data = updated + // Each item's summary is dropped from that item's own JSON and the array is written + // back once. Deleting ".summary" through the whole array would reserialise it per + // compaction item, making this O(items x payload). + // Pinned by TestStripCompactionItemSummary_AllocationScaling. + parsed := gjson.ParseBytes(data) + if !parsed.IsArray() { + return data + } + + var rebuilt [][]byte + changed := false + index := 0 + parsed.ForEach(func(_, element gjson.Result) bool { + raw := []byte(element.Raw) + if index < len(items) { + if msg := items[index]; msg.Type != nil && *msg.Type == schemas.ResponsesMessageTypeCompaction { + if updated, err := sjson.DeleteBytes(raw, "summary"); err == nil { + raw = updated + changed = true + } } } + rebuilt = append(rebuilt, raw) + index++ + return true + }) + if !changed { + return data + } + + var joined bytes.Buffer + joined.Grow(len(data)) + joined.WriteByte('[') + for i, element := range rebuilt { + if i > 0 { + joined.WriteByte(',') + } + joined.Write(element) } - return data + joined.WriteByte(']') + return joined.Bytes() } // Helper function to check if a chat message has any CacheControl fields or FileType in file blocks diff --git a/core/providers/openai/types_test.go b/core/providers/openai/types_test.go index ef6c6d92dd6..0e23a15dd62 100644 --- a/core/providers/openai/types_test.go +++ b/core/providers/openai/types_test.go @@ -1,9 +1,12 @@ package openai import ( + "bytes" + "strings" "testing" "github.com/bytedance/sonic" + "github.com/capsohq/bifrost/core/internal/memtest" "github.com/capsohq/bifrost/core/schemas" ) @@ -465,3 +468,39 @@ func TestOpenAIChatRequest_UnmarshalJSON_ValueAssertions(t *testing.T) { t.Errorf("Expected Stop value ['END', 'STOP'], got %v", req.Stop) } } + +// TestStripCompactionItemSummary_AllocationScaling pins the allocation shape of the +// compaction-summary strip. +// +// The loop deletes ".summary" through the whole input array, once per compaction +// item, and each sjson delete reserialises the entire array. A conversation carrying +// many compaction items therefore costs that many copies of it. +func TestStripCompactionItemSummary_AllocationScaling(t *testing.T) { + // items must line up with the payload, so both are built together and looked up + // by payload length when the assertion calls back. + itemsByLen := map[int][]schemas.ResponsesMessage{} + + memtest.AssertAllocScaling(t, func(n int) []byte { + var b bytes.Buffer + items := make([]schemas.ResponsesMessage, 0, n) + compaction := schemas.ResponsesMessageTypeCompaction + b.WriteByte('[') + for i := range n { + if i > 0 { + b.WriteByte(',') + } + // encrypted_content is the bulk, so the payload grows with n; summary is + // what gets stripped. + b.WriteString(`{"type":"compaction","summary":null,"encrypted_content":"`) + b.WriteString(strings.Repeat("e", 400)) + b.WriteString(`"}`) + items = append(items, schemas.ResponsesMessage{Type: &compaction}) + } + b.WriteByte(']') + payload := b.Bytes() + itemsByLen[len(payload)] = items + return payload + }, func(body []byte) { + stripCompactionItemSummary(body, itemsByLen[len(body)]) + }) +} diff --git a/core/providers/openai/utils.go b/core/providers/openai/utils.go index 1bab432d790..baae675ef3d 100644 --- a/core/providers/openai/utils.go +++ b/core/providers/openai/utils.go @@ -52,6 +52,43 @@ func IsOpenAIReasoningModel(model string) bool { return strings.Contains(modelLower, "gpt-5") || strings.Contains(modelLower, "gpt-6") } +// defaultCanDisableReasoning: reasoning.effort "none" exists from GPT-5.1 on. +// GPT-6 Sol and Luna support it, while GPT-6 Astra and unknown GPT-6 variants +// conservatively remain always-reasoning. +func defaultCanDisableReasoning(model string) bool { + m := bareModelLower(model) + switch { + case strings.Contains(m, "gpt-6-sol"), strings.Contains(m, "gpt-6-luna"): + return true + case strings.Contains(m, "gpt-6"), acceptsMinimalEffort(m): + return false + case strings.Contains(m, "gpt-5"): + return !strings.Contains(m, "-pro") + default: + return !IsOpenAIReasoningModel(m) || strings.Contains(m, "gpt-oss") + } +} + +// defaultSupportsAsyncTools: async tool calling is GPT-6 Astra and later. +func defaultSupportsAsyncTools(model string) bool { + return strings.Contains(bareModelLower(model), "gpt-6") +} + +// omittedEffortReasons reports OpenAI reasoning models that still reason when +// reasoning.effort is omitted. Only GPT-5.1 through GPT-5.4 default to "none"; +// their -pro variants always reason. +func omittedEffortReasons(model string) bool { + if !IsOpenAIReasoningModel(model) { + return false + } + m := bareModelLower(model) + if strings.Contains(m, "-pro") { + return true + } + return !strings.Contains(m, "gpt-5.1") && !strings.Contains(m, "gpt-5.2") && + !strings.Contains(m, "gpt-5.3") && !strings.Contains(m, "gpt-5.4") +} + // defaultEffortControl widens the base low/medium/high ladder with the effort // levels a model natively accepts. Only the widening is name-derived; the // datasheet's per-level booleans take precedence when a row exists. diff --git a/core/providers/opencode/opencode.go b/core/providers/opencode/opencode.go index c8b69911c75..9bb5571274a 100644 --- a/core/providers/opencode/opencode.go +++ b/core/providers/opencode/opencode.go @@ -230,6 +230,11 @@ func (p *opencodeProvider) Rerank(ctx *schemas.BifrostContext, key schemas.Key, return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, p.GetProviderKey()) } +// Decision is not supported by the opencode provider. +func (p *opencodeProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, p.GetProviderKey()) +} + // OCR is not supported by Opencode. func (p *opencodeProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, p.GetProviderKey()) diff --git a/core/providers/openrouter/openrouter.go b/core/providers/openrouter/openrouter.go index 2b5eb436672..d69169bc995 100644 --- a/core/providers/openrouter/openrouter.go +++ b/core/providers/openrouter/openrouter.go @@ -484,6 +484,11 @@ func (provider *OpenRouterProvider) Rerank(ctx *schemas.BifrostContext, key sche return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey()) } +// Decision is not supported by the OpenRouter provider. +func (provider *OpenRouterProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR is not supported by the Openrouter provider. func (provider *OpenRouterProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) diff --git a/core/providers/parasail/parasail.go b/core/providers/parasail/parasail.go index 70e192d6835..9c31d83cb81 100644 --- a/core/providers/parasail/parasail.go +++ b/core/providers/parasail/parasail.go @@ -194,6 +194,11 @@ func (provider *ParasailProvider) Rerank(ctx *schemas.BifrostContext, key schema return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey()) } +// Decision is not supported by the Parasail provider. +func (provider *ParasailProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR is not supported by the Parasail provider. func (provider *ParasailProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) diff --git a/core/providers/perplexity/perplexity.go b/core/providers/perplexity/perplexity.go index 614eb128772..193f6b8c2cc 100644 --- a/core/providers/perplexity/perplexity.go +++ b/core/providers/perplexity/perplexity.go @@ -318,6 +318,11 @@ func (provider *PerplexityProvider) Rerank(ctx *schemas.BifrostContext, key sche return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey()) } +// Decision is not supported by the Perplexity provider. +func (provider *PerplexityProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR is not supported by the Perplexity provider. func (provider *PerplexityProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) diff --git a/core/providers/qwen/qwen.go b/core/providers/qwen/qwen.go index 9fac6e254e3..1f5698e7b5a 100644 --- a/core/providers/qwen/qwen.go +++ b/core/providers/qwen/qwen.go @@ -229,6 +229,11 @@ func (provider *QwenProvider) Rerank(ctx *schemas.BifrostContext, key schemas.Ke return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey()) } +// Decision is not supported by the Qwen provider. +func (provider *QwenProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR is not supported by the Qwen provider. func (provider *QwenProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) diff --git a/core/providers/replicate/replicate.go b/core/providers/replicate/replicate.go index 52ae800400e..0af34c7b453 100644 --- a/core/providers/replicate/replicate.go +++ b/core/providers/replicate/replicate.go @@ -1749,6 +1749,11 @@ func (provider *ReplicateProvider) Rerank(ctx *schemas.BifrostContext, key schem return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey()) } +// Decision is not supported by the Replicate provider. +func (provider *ReplicateProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR is not supported by the Replicate provider. func (provider *ReplicateProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) diff --git a/core/providers/runware/runware.go b/core/providers/runware/runware.go index 26c5a55b728..04347090ba7 100644 --- a/core/providers/runware/runware.go +++ b/core/providers/runware/runware.go @@ -354,6 +354,11 @@ func (provider *RunwareProvider) Rerank(ctx *schemas.BifrostContext, key schemas return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey()) } +// Decision is not supported by the Runware provider. +func (provider *RunwareProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR is not supported by the Runware provider. func (provider *RunwareProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) diff --git a/core/providers/runway/runway.go b/core/providers/runway/runway.go index 5e6724c2cda..9e4e05ba1e5 100644 --- a/core/providers/runway/runway.go +++ b/core/providers/runway/runway.go @@ -130,6 +130,11 @@ func (provider *RunwayProvider) Rerank(ctx *schemas.BifrostContext, key schemas. return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey()) } +// Decision is not supported by the Runway provider. +func (provider *RunwayProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR is not supported by the Runway provider. func (provider *RunwayProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) diff --git a/core/providers/sarvam/sarvam.go b/core/providers/sarvam/sarvam.go index 2fc41eaa483..b1e386f8ee8 100644 --- a/core/providers/sarvam/sarvam.go +++ b/core/providers/sarvam/sarvam.go @@ -177,6 +177,11 @@ func (provider *SarvamProvider) Rerank(ctx *schemas.BifrostContext, key schemas. return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey()) } +// Decision is not supported by the Sarvam provider. +func (provider *SarvamProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR is not supported by the Sarvam provider. func (provider *SarvamProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) diff --git a/core/providers/sgl/sgl.go b/core/providers/sgl/sgl.go index eb0e8d51698..09c4f090bc6 100644 --- a/core/providers/sgl/sgl.go +++ b/core/providers/sgl/sgl.go @@ -425,6 +425,11 @@ func (provider *SGLProvider) Rerank(ctx *schemas.BifrostContext, key schemas.Key return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey()) } +// Decision is not supported by the SGL provider. +func (provider *SGLProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR is not supported by the Sgl provider. func (provider *SGLProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) diff --git a/core/providers/typesafe/decision.go b/core/providers/typesafe/decision.go new file mode 100644 index 00000000000..3deb3c55823 --- /dev/null +++ b/core/providers/typesafe/decision.go @@ -0,0 +1,426 @@ +package typesafe + +import ( + "fmt" + "reflect" + "slices" + + "github.com/tidwall/gjson" + + providerUtils "github.com/capsohq/bifrost/core/providers/utils" + schemas "github.com/capsohq/bifrost/core/schemas" +) + +// typesafeMaxChoiceOptions is Typesafe's documented cap on choice options. +const typesafeMaxChoiceOptions = 255 + +// typesafeMinScoreLevels and typesafeMaxScoreLevels bound a score criteria array. +const ( + typesafeMinScoreLevels = 2 + typesafeMaxScoreLevels = 10 +) + +// supportedDecisionKinds is the set of kinds Typesafe serves. The Bifrost kind +// vocabulary mirrors the native question types, so the mapping is identity. +var supportedDecisionKinds = map[schemas.DecisionKind]string{ + schemas.DecisionKindNoul: TypesafeQuestionTypeNoul, + schemas.DecisionKindChoice: TypesafeQuestionTypeChoice, + schemas.DecisionKindScore: TypesafeQuestionTypeScore, +} + +// validStructuredValue reports whether a state or instructions value +// serializes to one of the documented JSON shapes: string, object, or array. +// Marshaling the value answers that exactly: pointers dereference, typed +// maps/slices/structs normalize, custom marshalers speak for themselves, and +// unmarshalable values (func, chan) fail closed with a local 400 instead of a +// request-marshal 500 later. Only the value is encoded for inspection; the +// request body is serialized once, unchanged, on the send path. +func validStructuredValue(value interface{}) bool { + data, err := providerUtils.MarshalSorted(value) + if err != nil { + return false + } + for _, c := range data { + switch c { + case ' ', '\t', '\n', '\r': + continue + } + return c == '"' || c == '{' || c == '[' + } + return false +} + +// ToTypesafeDecisionRequest converts a Bifrost decision request into +// Typesafe's native systemone shape. Unsupported kinds and malformed criteria +// are rejected rather than silently approximated. +func ToTypesafeDecisionRequest(request *schemas.BifrostDecisionRequest) (*TypesafeDecisionRequest, error) { + if len(request.Questions) == 0 { + return nil, providerUtils.InvalidRequestErrorf("decision request requires at least one question") + } + if !validStructuredValue(request.State) { + return nil, providerUtils.InvalidRequestErrorf("state must be a string, object, or array") + } + + questions := make(map[string]TypesafeQuestion, len(request.Questions)) + for name, question := range request.Questions { + native, err := toTypesafeQuestion(name, question) + if err != nil { + return nil, err + } + questions[name] = *native + } + + return &TypesafeDecisionRequest{ + State: request.State, + Model: request.Model, + Questions: questions, + }, nil +} + +// toTypesafeQuestion validates one question and maps it to the native shape. +func toTypesafeQuestion(name string, question schemas.DecisionQuestion) (*TypesafeQuestion, error) { + nativeType, ok := supportedDecisionKinds[question.Kind] + if !ok { + return nil, providerUtils.InvalidRequestErrorf("question %q has unsupported kind %q; expected noul, choice, or score", name, question.Kind) + } + if question.Instructions == nil { + return nil, providerUtils.InvalidRequestErrorf("question %q has no instructions", name) + } + if !validStructuredValue(question.Instructions) { + return nil, providerUtils.InvalidRequestErrorf("question %q instructions must be a string, object, or array", name) + } + + native := TypesafeQuestion{Type: nativeType, Instructions: question.Instructions} + + switch question.Kind { + case schemas.DecisionKindNoul: + criteria, err := noulCriteria(name, question.Criteria) + if err != nil { + return nil, err + } + native.Criteria = criteria + case schemas.DecisionKindChoice: + criteria, err := choiceCriteria(name, question.Criteria) + if err != nil { + return nil, err + } + native.Criteria = criteria + case schemas.DecisionKindScore: + criteria, err := scoreCriteria(name, question.Criteria) + if err != nil { + return nil, err + } + native.Criteria = criteria + } + + return &native, nil +} + +// noulCriteria validates optional noul criteria: a map whose only keys are +// "true" and "false", each described by a string, object, or array. Passed +// through losslessly. +func noulCriteria(name string, criteria interface{}) (interface{}, error) { + if criteria == nil { + return nil, nil + } + m, err := criteriaAsMap(name, criteria, false) + if err != nil { + return nil, err + } + for key := range m { + if key != "true" && key != "false" { + return nil, providerUtils.InvalidRequestErrorf("question %q noul criteria allows only \"true\" and \"false\" keys, got %q", name, key) + } + } + return m, nil +} + +// choiceCriteria validates the required option -> description map. A +// description is a string, object, or array, or null when an option needs no +// extra detail. +func choiceCriteria(name string, criteria interface{}) (map[string]any, error) { + if criteria == nil { + return nil, providerUtils.InvalidRequestErrorf("question %q has kind choice and requires criteria options", name) + } + m, err := criteriaAsMap(name, criteria, true) + if err != nil { + return nil, err + } + if len(m) == 0 { + return nil, providerUtils.InvalidRequestErrorf("question %q has kind choice and requires criteria options", name) + } + if len(m) > typesafeMaxChoiceOptions { + return nil, providerUtils.InvalidRequestErrorf("question %q has %d choice options; Typesafe allows at most %d", name, len(m), typesafeMaxChoiceOptions) + } + return m, nil +} + +// scoreCriteria validates the ordered array of 2-10 level descriptions, each +// a string, object, or array, preserved losslessly. Go SDK callers supply +// []string; HTTP JSON decoding supplies []interface{} - both are the same +// ordered array on the wire. +func scoreCriteria(name string, criteria interface{}) (interface{}, error) { + var levels []any + switch typed := criteria.(type) { + case []any: + levels = typed + case []string: + levels = make([]any, len(typed)) + for i, level := range typed { + levels[i] = level + } + default: + // Go SDK callers pass typed slices ([]Rubric, [][]string, named slice + // types). Any slice or array is an ordered list of level descriptions; + // elements are validated by serialized shape below, exactly like the + // untyped forms, and carried losslessly. + value := reflect.ValueOf(criteria) + if !value.IsValid() || (value.Kind() != reflect.Slice && value.Kind() != reflect.Array) { + return nil, providerUtils.InvalidRequestErrorf("question %q has kind score and requires criteria as an ordered array of level descriptions", name) + } + levels = make([]any, value.Len()) + for i := range levels { + levels[i] = value.Index(i).Interface() + } + } + if len(levels) < typesafeMinScoreLevels || len(levels) > typesafeMaxScoreLevels { + return nil, providerUtils.InvalidRequestErrorf("question %q score criteria must have between %d and %d levels, got %d", name, typesafeMinScoreLevels, typesafeMaxScoreLevels, len(levels)) + } + for i, level := range levels { + if !validStructuredValue(level) { + return nil, providerUtils.InvalidRequestErrorf("question %q score criteria level %d must be a string, object, or array", name, i) + } + } + return levels, nil +} + +// criteriaAsMap normalizes a criteria value into map[string]any, rejecting +// non-map shapes. Each description must serialize to a string, object, or +// array; null is additionally allowed when allowNull is set (choice options +// that need no extra detail). Descriptions are carried losslessly. +func criteriaAsMap(name string, criteria interface{}, allowNull bool) (map[string]any, error) { + var m map[string]any + switch typed := criteria.(type) { + case map[string]string: + m = make(map[string]any, len(typed)) + for key, value := range typed { + m[key] = value + } + case map[string]any: + m = typed + default: + // Go SDK callers pass typed maps (map[string]Rubric, map[string][]string, + // ...). Any map with string keys is a map of named descriptions; its + // values are validated by serialized shape below, exactly like the + // untyped forms, and carried losslessly. + value := reflect.ValueOf(criteria) + if !value.IsValid() || value.Kind() != reflect.Map || value.Type().Key().Kind() != reflect.String { + return nil, providerUtils.InvalidRequestErrorf("question %q criteria must be a map of descriptions", name) + } + m = make(map[string]any, value.Len()) + iter := value.MapRange() + for iter.Next() { + m[iter.Key().String()] = iter.Value().Interface() + } + } + for key, value := range m { + if isJSONNull(value) { + if allowNull { + continue + } + return nil, providerUtils.InvalidRequestErrorf("question %q criteria description for %q must be a string, object, or array", name, key) + } + if !validStructuredValue(value) { + return nil, providerUtils.InvalidRequestErrorf("question %q criteria description for %q must be a string, object, or array", name, key) + } + } + return m, nil +} + +// isJSONNull reports whether a description serializes to JSON null. Typed nil +// pointers from Go SDK callers (map[string]*Rubric{"other": nil}) are non-nil +// interfaces wrapping nil pointers - value == nil misses them, but on the wire +// they are null exactly like an untyped nil, so both follow the null rules. +func isJSONNull(value any) bool { + if value == nil { + return true + } + data, err := providerUtils.MarshalSorted(value) + if err != nil { + return false + } + return gjson.ParseBytes(data).Type == gjson.Null +} + +// ToBifrostDecisionResponse converts a native systemone response back into +// the shared decision shape. Every requested question must produce an answer +// of its declared kind. A response it rejects was still answered and billed, +// so the error carries that usage for governance and logging to charge. +func ToBifrostDecisionResponse(resp *TypesafeDecisionResponse, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + response, bifrostErr := toBifrostDecisionResponse(resp, request) + if bifrostErr != nil { + bifrostErr.ExtraFields.BilledUsage = decisionUsage(resp.Usage) + if resp.Model != "" { + bifrostErr.ExtraFields.ResolvedModelUsed = resp.Model + } + return nil, bifrostErr + } + return response, nil +} + +func toBifrostDecisionResponse(resp *TypesafeDecisionResponse, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + answers := make(map[string]schemas.DecisionAnswer, len(request.Questions)) + for name, question := range request.Questions { + native, ok := resp.Answers[name] + if !ok { + return nil, providerUtils.NewBifrostOperationError(fmt.Sprintf("typesafe returned no answer for question %q", name), nil) + } + if expected := supportedDecisionKinds[question.Kind]; native.Type != expected { + return nil, providerUtils.NewBifrostOperationError(fmt.Sprintf("typesafe answered question %q as %q; expected %q", name, native.Type, expected), nil) + } + + answer := schemas.DecisionAnswer{ + Kind: question.Kind, + Confidence: native.Confidence, + Probabilities: native.Probabilities, + Legend: native.Legend, + } + switch question.Kind { + case schemas.DecisionKindNoul: + if native.Noul == nil { + return nil, providerUtils.NewBifrostOperationError(fmt.Sprintf("typesafe noul answer for question %q carries no value", name), nil) + } + if *native.Noul < 0 || *native.Noul > 1 { + return nil, providerUtils.NewBifrostOperationError(fmt.Sprintf("typesafe noul answer for question %q is outside [0,1]: %v", name, *native.Noul), nil) + } + answer.Value = *native.Noul + case schemas.DecisionKindChoice: + if native.Choice == nil { + return nil, providerUtils.NewBifrostOperationError(fmt.Sprintf("typesafe choice answer for question %q carries no value", name), nil) + } + // An answer outside the requested options is impossible, as emulation also enforces. + if opts, err := providerUtils.DecisionChoiceOptions(question.Criteria); err == nil && !slices.Contains(opts, *native.Choice) { + return nil, providerUtils.NewBifrostOperationError(fmt.Sprintf("typesafe choice answer %q for question %q is not an allowed option", *native.Choice, name), nil) + } + answer.Value = *native.Choice + case schemas.DecisionKindScore: + if native.Score == nil { + return nil, providerUtils.NewBifrostOperationError(fmt.Sprintf("typesafe score answer for question %q carries no value", name), nil) + } + // A score lies on the rubric's level indices, 0..levels-1. + if levels, err := providerUtils.DecisionScoreLevelCount(question.Criteria); err == nil && (*native.Score < 0 || *native.Score > float64(levels-1)) { + return nil, providerUtils.NewBifrostOperationError(fmt.Sprintf("typesafe score answer for question %q is outside 0..%d: %v", name, levels-1, *native.Score), nil) + } + answer.Value = *native.Score + } + answers[name] = answer + } + + response := &schemas.BifrostDecisionResponse{ + Model: resp.Model, + Answers: answers, + } + response.Usage = decisionUsage(resp.Usage) + return response, nil +} + +// decisionUsage converts systemone usage into Bifrost's, or nil when none was reported. +func decisionUsage(usage *TypesafeUsage) *schemas.BifrostLLMUsage { + if usage == nil { + return nil + } + return &schemas.BifrostLLMUsage{ + PromptTokens: usage.InputTokens, + CompletionTokens: usage.OutputTokens, + TotalTokens: usage.InputTokens + usage.OutputTokens, + } +} + +// ToBifrostDecisionRequest converts a native systemone request into the +// shared decision shape. Question identifiers become answer field names; the +// native type vocabulary is identical to the Bifrost kind vocabulary. +func (req *TypesafeDecisionRequest) ToBifrostDecisionRequest(ctx *schemas.BifrostContext) (*schemas.BifrostDecisionRequest, error) { + if req == nil { + return nil, providerUtils.InvalidRequestErrorf("request body is required") + } + if req.State == nil { + return nil, providerUtils.InvalidRequestErrorf("state is required") + } + if len(req.Questions) == 0 { + return nil, providerUtils.InvalidRequestErrorf("at least one question is required") + } + + provider, model := schemas.ParseModelString(req.Model, schemas.Typesafe) + + questions := make(map[string]schemas.DecisionQuestion, len(req.Questions)) + for name, question := range req.Questions { + questions[name] = schemas.DecisionQuestion{ + Kind: schemas.DecisionKind(question.Type), + Instructions: question.Instructions, + Criteria: question.Criteria, + } + } + + return &schemas.BifrostDecisionRequest{ + Provider: provider, + Model: model, + State: req.State, + Questions: questions, + }, nil +} + +// ToTypesafeNativeDecisionResponse converts a shared decision response back +// into Typesafe's native model, answers, and usage shape, covering all three +// answer types. +func ToTypesafeNativeDecisionResponse(resp *schemas.BifrostDecisionResponse) (*TypesafeDecisionResponse, error) { + if resp == nil { + return nil, fmt.Errorf("decision response is nil") + } + + answers := make(map[string]TypesafeAnswer, len(resp.Answers)) + for name, answer := range resp.Answers { + native := TypesafeAnswer{ + Confidence: answer.Confidence, + Probabilities: answer.Probabilities, + Legend: answer.Legend, + } + switch answer.Kind { + case schemas.DecisionKindNoul: + number, ok := answer.Value.(float64) + if !ok { + return nil, fmt.Errorf("decision answer %q is not a number", name) + } + native.Type = TypesafeQuestionTypeNoul + native.Noul = &number + case schemas.DecisionKindChoice: + choice, ok := answer.Value.(string) + if !ok { + return nil, fmt.Errorf("decision answer %q is not a string", name) + } + native.Type = TypesafeQuestionTypeChoice + native.Choice = &choice + case schemas.DecisionKindScore: + number, ok := answer.Value.(float64) + if !ok { + return nil, fmt.Errorf("decision answer %q is not a number", name) + } + native.Type = TypesafeQuestionTypeScore + native.Score = &number + default: + return nil, fmt.Errorf("decision answer %q has unknown kind %q", name, answer.Kind) + } + answers[name] = native + } + + native := &TypesafeDecisionResponse{ + Model: resp.Model, + Answers: answers, + } + if resp.Usage != nil { + native.Usage = &TypesafeUsage{ + InputTokens: resp.Usage.PromptTokens, + OutputTokens: resp.Usage.CompletionTokens, + } + } + return native, nil +} diff --git a/core/providers/typesafe/decision_test.go b/core/providers/typesafe/decision_test.go new file mode 100644 index 00000000000..85b3153d32c --- /dev/null +++ b/core/providers/typesafe/decision_test.go @@ -0,0 +1,926 @@ +package typesafe + +import ( + "encoding/json" + "fmt" + "reflect" + "strings" + "testing" + + "github.com/bytedance/sonic" + schemas "github.com/capsohq/bifrost/core/schemas" +) + +// decisionRequest builds a BifrostDecisionRequest around the given questions. +func decisionRequest(state interface{}, questions map[string]schemas.DecisionQuestion) *schemas.BifrostDecisionRequest { + return &schemas.BifrostDecisionRequest{ + Provider: schemas.Typesafe, + Model: "jev-1.13.0", + State: state, + Questions: questions, + } +} + +func TestToTypesafeDecisionRequestMixedKinds(t *testing.T) { + req := decisionRequest("the user asked for a refund", map[string]schemas.DecisionQuestion{ + "is_angry": { + Kind: schemas.DecisionKindNoul, + Instructions: "Is the user angry?", + }, + "category": { + Kind: schemas.DecisionKindChoice, + Instructions: "Pick the ticket category", + Criteria: map[string]interface{}{"billing": "money issues", "bug": "product defects", "other": "anything else"}, + }, + "severity": { + Kind: schemas.DecisionKindScore, + Instructions: "Rate the severity", + Criteria: []interface{}{"cosmetic", "annoying", "blocking"}, + }, + }) + + native, err := ToTypesafeDecisionRequest(req) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if native.Model != "jev-1.13.0" { + t.Errorf("model = %q, want bare jev-1.13.0", native.Model) + } + if native.State != "the user asked for a refund" { + t.Errorf("state not preserved: %v", native.State) + } + if len(native.Questions) != 3 { + t.Fatalf("expected 3 questions, got %d", len(native.Questions)) + } + + angry := native.Questions["is_angry"] + if angry.Type != TypesafeQuestionTypeNoul { + t.Errorf("is_angry type = %q, want noul", angry.Type) + } + if angry.Instructions != "Is the user angry?" { + t.Errorf("is_angry instructions = %v", angry.Instructions) + } + + category := native.Questions["category"] + if category.Type != TypesafeQuestionTypeChoice { + t.Errorf("category type = %q, want choice", category.Type) + } + criteria, ok := category.Criteria.(map[string]any) + if !ok || len(criteria) != 3 || criteria["bug"] != "product defects" { + t.Errorf("category criteria not preserved: %#v", category.Criteria) + } + + severity := native.Questions["severity"] + if severity.Type != TypesafeQuestionTypeScore { + t.Errorf("severity type = %q, want score", severity.Type) + } + levels, ok := severity.Criteria.([]interface{}) + if !ok || len(levels) != 3 || levels[0] != "cosmetic" { + t.Errorf("severity criteria not preserved losslessly: %#v", severity.Criteria) + } +} + +func TestToTypesafeDecisionRequestStructuredStateAndInstructions(t *testing.T) { + state := map[string]interface{}{"ticket": map[string]interface{}{"id": 42, "body": "hello"}} + structured := map[string]interface{}{"goal": "judge tone", "steps": []interface{}{"read", "decide"}} + req := decisionRequest(state, map[string]schemas.DecisionQuestion{ + "tone_ok": {Kind: schemas.DecisionKindNoul, Instructions: structured}, + }) + + native, err := ToTypesafeDecisionRequest(req) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !reflect.DeepEqual(native.Questions["tone_ok"].Instructions, interface{}(structured)) { + t.Errorf("structured instructions not lossless: %#v", native.Questions["tone_ok"].Instructions) + } + if !reflect.DeepEqual(native.State, interface{}(state)) { + t.Errorf("structured state not lossless: %#v", native.State) + } +} + +func TestToTypesafeDecisionRequestStructuredCriteria(t *testing.T) { + // The API spec types criteria descriptions as string | object | array for + // noul and score, and string | object | array | null for choice options - + // structured rubrics must pass through losslessly, not be coerced to strings. + noulCriteria := map[string]any{ + "true": map[string]any{"meaning": "approve", "signals": []any{"polite tone"}}, + "false": []any{"reject", "escalate"}, + } + choiceCriteria := map[string]any{ + "billing": map[string]any{"rubric": "money issues", "examples": []any{"refund request"}}, + "bug": []any{"crash", "defect"}, + "other": nil, // spec: null when an option needs no extra detail + } + scoreCriteria := []any{ + "cosmetic", + map[string]any{"level": "annoying", "examples": []any{"slow load"}}, + []any{"blocking", "data loss"}, + } + req := decisionRequest("state", map[string]schemas.DecisionQuestion{ + "approve": {Kind: schemas.DecisionKindNoul, Instructions: "d", Criteria: noulCriteria}, + "category": {Kind: schemas.DecisionKindChoice, Instructions: "d", Criteria: choiceCriteria}, + "severity": {Kind: schemas.DecisionKindScore, Instructions: "d", Criteria: scoreCriteria}, + }) + + native, err := ToTypesafeDecisionRequest(req) + if err != nil { + t.Fatalf("structured criteria rejected: %v", err) + } + if !reflect.DeepEqual(native.Questions["approve"].Criteria, any(noulCriteria)) { + t.Errorf("noul criteria not lossless: %#v", native.Questions["approve"].Criteria) + } + if !reflect.DeepEqual(native.Questions["category"].Criteria, any(choiceCriteria)) { + t.Errorf("choice criteria not lossless: %#v", native.Questions["category"].Criteria) + } + if !reflect.DeepEqual(native.Questions["severity"].Criteria, any(scoreCriteria)) { + t.Errorf("score criteria not lossless: %#v", native.Questions["severity"].Criteria) + } +} + +// TestToTypesafeDecisionRequestCriteriaTypeMatrix exercises every allowed +// value type in every criteria slot: string | object | array for noul keys +// and score levels, plus null for choice options, and the rejected types +// (null where not allowed, numbers, booleans, unmarshalable values) per kind. +func TestToTypesafeDecisionRequestCriteriaTypeMatrix(t *testing.T) { + str := "text description" + obj := map[string]any{"rubric": "detail", "examples": []any{"one"}} + arr := []any{"first", "second"} + allowed := map[string]any{"string": str, "object": obj, "array": arr} + rejectedNoNull := map[string]any{"number": 7, "boolean": true, "float": 3.5, "func": func() {}} + + accept := func(t *testing.T, q schemas.DecisionQuestion) *TypesafeQuestion { + t.Helper() + native, err := ToTypesafeDecisionRequest(decisionRequest("state", map[string]schemas.DecisionQuestion{"q": q})) + if err != nil { + t.Fatalf("expected acceptance, got %v", err) + } + question := native.Questions["q"] + return &question + } + reject := func(t *testing.T, q schemas.DecisionQuestion, wantSub string) { + t.Helper() + _, err := ToTypesafeDecisionRequest(decisionRequest("state", map[string]schemas.DecisionQuestion{"q": q})) + if err == nil || !strings.Contains(err.Error(), wantSub) { + t.Fatalf("expected rejection containing %q, got %v", wantSub, err) + } + } + + t.Run("noul", func(t *testing.T) { + for trueName, trueValue := range allowed { + for falseName, falseValue := range allowed { + t.Run("true="+trueName+"/false="+falseName, func(t *testing.T) { + criteria := map[string]any{"true": trueValue, "false": falseValue} + got := accept(t, schemas.DecisionQuestion{Kind: schemas.DecisionKindNoul, Instructions: "d", Criteria: criteria}) + if !reflect.DeepEqual(got.Criteria, any(criteria)) { + t.Errorf("criteria not lossless: %#v", got.Criteria) + } + }) + } + t.Run("single key true="+trueName, func(t *testing.T) { + accept(t, schemas.DecisionQuestion{Kind: schemas.DecisionKindNoul, Instructions: "d", Criteria: map[string]any{"true": trueValue}}) + }) + } + for name, value := range rejectedNoNull { + t.Run("rejects "+name, func(t *testing.T) { + reject(t, schemas.DecisionQuestion{Kind: schemas.DecisionKindNoul, Instructions: "d", Criteria: map[string]any{"true": value}}, "must be a string, object, or array") + }) + } + t.Run("rejects null", func(t *testing.T) { + reject(t, schemas.DecisionQuestion{Kind: schemas.DecisionKindNoul, Instructions: "d", Criteria: map[string]any{"false": nil}}, "must be a string, object, or array") + }) + }) + + t.Run("choice", func(t *testing.T) { + for name, value := range allowed { + t.Run("option="+name, func(t *testing.T) { + criteria := map[string]any{"opt": value, "alt": "plain"} + got := accept(t, schemas.DecisionQuestion{Kind: schemas.DecisionKindChoice, Instructions: "d", Criteria: criteria}) + if !reflect.DeepEqual(got.Criteria, any(criteria)) { + t.Errorf("criteria not lossless: %#v", got.Criteria) + } + }) + } + t.Run("option=null", func(t *testing.T) { + criteria := map[string]any{"opt": nil, "alt": "plain"} + got := accept(t, schemas.DecisionQuestion{Kind: schemas.DecisionKindChoice, Instructions: "d", Criteria: criteria}) + if !reflect.DeepEqual(got.Criteria, any(criteria)) { + t.Errorf("criteria not lossless: %#v", got.Criteria) + } + }) + t.Run("all four types in one map", func(t *testing.T) { + criteria := map[string]any{"s": str, "o": obj, "a": arr, "n": nil} + got := accept(t, schemas.DecisionQuestion{Kind: schemas.DecisionKindChoice, Instructions: "d", Criteria: criteria}) + if !reflect.DeepEqual(got.Criteria, any(criteria)) { + t.Errorf("criteria not lossless: %#v", got.Criteria) + } + }) + for name, value := range rejectedNoNull { + t.Run("rejects "+name, func(t *testing.T) { + reject(t, schemas.DecisionQuestion{Kind: schemas.DecisionKindChoice, Instructions: "d", Criteria: map[string]any{"opt": value}}, "must be a string, object, or array") + }) + } + }) + + t.Run("score", func(t *testing.T) { + // Each allowed type at each position of a three-level array. + for name, value := range allowed { + for position := 0; position < 3; position++ { + t.Run(fmt.Sprintf("%s at level %d", name, position), func(t *testing.T) { + levels := []any{"base low", "base mid", "base high"} + levels[position] = value + got := accept(t, schemas.DecisionQuestion{Kind: schemas.DecisionKindScore, Instructions: "d", Criteria: levels}) + if !reflect.DeepEqual(got.Criteria, any(levels)) { + t.Errorf("levels not lossless: %#v", got.Criteria) + } + }) + } + } + t.Run("all three types in one array", func(t *testing.T) { + levels := []any{str, obj, arr} + got := accept(t, schemas.DecisionQuestion{Kind: schemas.DecisionKindScore, Instructions: "d", Criteria: levels}) + if !reflect.DeepEqual(got.Criteria, any(levels)) { + t.Errorf("levels not lossless: %#v", got.Criteria) + } + }) + for name, value := range rejectedNoNull { + t.Run("rejects "+name, func(t *testing.T) { + reject(t, schemas.DecisionQuestion{Kind: schemas.DecisionKindScore, Instructions: "d", Criteria: []any{"low", value}}, "level 1 must be a string, object, or array") + }) + } + t.Run("rejects null level", func(t *testing.T) { + reject(t, schemas.DecisionQuestion{Kind: schemas.DecisionKindScore, Instructions: "d", Criteria: []any{"low", nil}}, "level 1 must be a string, object, or array") + }) + }) + + t.Run("typed Go maps with string keys accepted", func(t *testing.T) { + // Go SDK callers pass typed maps; any map with string keys whose values + // serialize to valid descriptions must be accepted, not only + // map[string]string and map[string]any. + type rubric struct { + Meaning string `json:"meaning"` + } + got := accept(t, schemas.DecisionQuestion{Kind: schemas.DecisionKindChoice, Instructions: "d", Criteria: map[string]rubric{ + "billing": {Meaning: "money issues"}, + "bug": {Meaning: "defects"}, + }}) + m, ok := got.Criteria.(map[string]any) + if !ok || len(m) != 2 { + t.Fatalf("typed struct map not normalized: %#v", got.Criteria) + } + if r, ok := m["billing"].(rubric); !ok || r.Meaning != "money issues" { + t.Errorf("typed value not carried losslessly: %#v", m["billing"]) + } + accept(t, schemas.DecisionQuestion{Kind: schemas.DecisionKindChoice, Instructions: "d", Criteria: map[string][]string{ + "billing": {"charges", "refunds"}, + "bug": {"crash"}, + }}) + accept(t, schemas.DecisionQuestion{Kind: schemas.DecisionKindNoul, Instructions: "d", Criteria: map[string][]string{ + "true": {"clearly upset"}, + }}) + // Non-string keys stay rejected: not a map of named descriptions. + reject(t, schemas.DecisionQuestion{Kind: schemas.DecisionKindChoice, Instructions: "d", Criteria: map[int]string{1: "first"}}, "must be a map of descriptions") + // String-keyed but invalid values still fail per value. + reject(t, schemas.DecisionQuestion{Kind: schemas.DecisionKindChoice, Instructions: "d", Criteria: map[string]int{"a": 1}}, "must be a string, object, or array") + }) + + t.Run("typed slices accepted for score criteria", func(t *testing.T) { + // Go SDK callers pass typed slices; any slice or array whose elements + // serialize to valid descriptions must be accepted, mirroring the + // typed-map handling for noul and choice. + type rubric struct { + Level string `json:"level"` + } + got := accept(t, schemas.DecisionQuestion{Kind: schemas.DecisionKindScore, Instructions: "d", Criteria: []rubric{ + {Level: "low"}, {Level: "high"}, + }}) + levels, ok := got.Criteria.([]any) + if !ok || len(levels) != 2 { + t.Fatalf("typed struct slice not normalized: %#v", got.Criteria) + } + if r, ok := levels[0].(rubric); !ok || r.Level != "low" { + t.Errorf("typed level not carried losslessly: %#v", levels[0]) + } + accept(t, schemas.DecisionQuestion{Kind: schemas.DecisionKindScore, Instructions: "d", Criteria: [][]string{ + {"can wait"}, {"needs reply", "churn risk"}, + }}) + type namedLevels []string + accept(t, schemas.DecisionQuestion{Kind: schemas.DecisionKindScore, Instructions: "d", Criteria: namedLevels{"low", "high"}}) + // Element validation still applies to typed slices. + reject(t, schemas.DecisionQuestion{Kind: schemas.DecisionKindScore, Instructions: "d", Criteria: []int{1, 2}}, "level 0 must be a string, object, or array") + reject(t, schemas.DecisionQuestion{Kind: schemas.DecisionKindScore, Instructions: "d", Criteria: []rubric{{Level: "only one"}}}, "between 2 and 10") + }) + + t.Run("typed nil pointers are JSON null descriptions", func(t *testing.T) { + // map[string]*Rubric{"other": nil} wraps a nil pointer in a non-nil + // interface; it serializes to null and must follow the null rules: + // allowed for choice options, rejected for noul. + type rubric struct { + Meaning string `json:"meaning"` + } + got := accept(t, schemas.DecisionQuestion{Kind: schemas.DecisionKindChoice, Instructions: "d", Criteria: map[string]*rubric{ + "billing": {Meaning: "money issues"}, + "other": nil, + }}) + m, ok := got.Criteria.(map[string]any) + if !ok || len(m) != 2 { + t.Fatalf("typed pointer map not normalized: %#v", got.Criteria) + } + reject(t, schemas.DecisionQuestion{Kind: schemas.DecisionKindNoul, Instructions: "d", Criteria: map[string]*rubric{ + "true": nil, + }}, "must be a string, object, or array") + }) + + t.Run("typed string maps accepted for noul and choice", func(t *testing.T) { + // Go SDK callers pass map[string]string; both map input branches must work. + accept(t, schemas.DecisionQuestion{Kind: schemas.DecisionKindNoul, Instructions: "d", Criteria: map[string]string{"true": "yes means this", "false": "no means this"}}) + got := accept(t, schemas.DecisionQuestion{Kind: schemas.DecisionKindChoice, Instructions: "d", Criteria: map[string]string{"a": "first", "b": "second"}}) + m, ok := got.Criteria.(map[string]any) + if !ok || m["a"] != "first" || m["b"] != "second" { + t.Errorf("typed string map not normalized losslessly: %#v", got.Criteria) + } + }) +} + +func TestIsJSONNull(t *testing.T) { + type rubric struct { + Meaning string `json:"meaning"` + } + var nilRubric *rubric + var nilMap map[string]string + var nilSlice []string + var nilTypedSlice []rubric + var nilIface any + + cases := []struct { + name string + value any + want bool + }{ + {"untyped nil", nil, true}, + {"nil interface variable", nilIface, true}, + {"typed nil struct pointer", nilRubric, true}, + {"nil map", nilMap, true}, + {"nil slice", nilSlice, true}, + {"nil typed slice", nilTypedSlice, true}, + {"raw json null", json.RawMessage("null"), true}, + {"raw json null with whitespace", json.RawMessage(" \n\tnull"), true}, + {"non-nil pointer", &rubric{Meaning: "x"}, false}, + {"zero struct", rubric{}, false}, + {"empty map", map[string]string{}, false}, + {"empty slice", []string{}, false}, + {"empty string", "", false}, + {"the string null", "null", false}, // serializes to "null" WITH quotes - a string, not null + {"zero int", 0, false}, + {"false", false, false}, + {"unmarshalable func", func() {}, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := isJSONNull(tc.value); got != tc.want { + t.Errorf("isJSONNull(%#v) = %v, want %v", tc.value, got, tc.want) + } + }) + } +} + +func TestToTypesafeDecisionRequestRejections(t *testing.T) { + cases := []struct { + name string + question schemas.DecisionQuestion + wantSub string + }{ + { + name: "unsupported kind", + question: schemas.DecisionQuestion{Kind: schemas.DecisionKind("ranking"), Instructions: "d"}, + wantSub: "unsupported kind", + }, + { + name: "missing instructions", + question: schemas.DecisionQuestion{Kind: schemas.DecisionKindNoul}, + wantSub: "no instructions", + }, + { + name: "noul criteria with bad key", + question: schemas.DecisionQuestion{Kind: schemas.DecisionKindNoul, Instructions: "d", Criteria: map[string]interface{}{"maybe": "x"}}, + wantSub: `allows only "true" and "false" keys`, + }, + { + name: "choice without criteria", + question: schemas.DecisionQuestion{Kind: schemas.DecisionKindChoice, Instructions: "d"}, + wantSub: "requires criteria options", + }, + { + name: "choice criteria wrong shape", + question: schemas.DecisionQuestion{Kind: schemas.DecisionKindChoice, Instructions: "d", Criteria: []interface{}{"a", "b"}}, + wantSub: "must be a map of descriptions", + }, + { + name: "choice criteria non-string description", + question: schemas.DecisionQuestion{Kind: schemas.DecisionKindChoice, Instructions: "d", Criteria: map[string]interface{}{"a": 1}}, + wantSub: "must be a string", + }, + { + name: "score criteria missing", + question: schemas.DecisionQuestion{Kind: schemas.DecisionKindScore, Instructions: "d"}, + wantSub: "ordered array", + }, + { + name: "score criteria too short", + question: schemas.DecisionQuestion{Kind: schemas.DecisionKindScore, Instructions: "d", Criteria: []interface{}{"only one"}}, + wantSub: "between 2 and 10", + }, + { + name: "score criteria non-string level", + question: schemas.DecisionQuestion{Kind: schemas.DecisionKindScore, Instructions: "d", Criteria: []interface{}{"low", 2, "high"}}, + wantSub: "level 1 must be a string", + }, + { + name: "instructions with unsupported shape", + question: schemas.DecisionQuestion{Kind: schemas.DecisionKindNoul, Instructions: 42}, + wantSub: "instructions must be a string, object, or array", + }, + { + // Only choice options may be null; the noul true/false descriptions + // are typed string | object | array. + name: "noul criteria null description", + question: schemas.DecisionQuestion{Kind: schemas.DecisionKindNoul, Instructions: "d", Criteria: map[string]any{"true": nil}}, + wantSub: "must be a string, object, or array", + }, + { + name: "noul criteria number description", + question: schemas.DecisionQuestion{Kind: schemas.DecisionKindNoul, Instructions: "d", Criteria: map[string]any{"true": 7}}, + wantSub: "must be a string, object, or array", + }, + { + name: "choice criteria unmarshalable description", + question: schemas.DecisionQuestion{Kind: schemas.DecisionKindChoice, Instructions: "d", Criteria: map[string]any{"a": func() {}}}, + wantSub: "must be a string, object, or array", + }, + { + name: "score criteria null level", + question: schemas.DecisionQuestion{Kind: schemas.DecisionKindScore, Instructions: "d", Criteria: []any{"low", nil, "high"}}, + wantSub: "level 1 must be a string, object, or array", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + req := decisionRequest("state", map[string]schemas.DecisionQuestion{"q": tc.question}) + _, err := ToTypesafeDecisionRequest(req) + if err == nil { + t.Fatalf("expected rejection containing %q, got nil error", tc.wantSub) + } + if !strings.Contains(err.Error(), tc.wantSub) { + t.Errorf("error %q does not contain %q", err.Error(), tc.wantSub) + } + }) + } + + t.Run("state with unsupported shape", func(t *testing.T) { + // Pointers to scalars marshal to JSON scalars; funcs cannot marshal at + // all - both must be caller errors, not upstream 422s or marshal 500s. + for _, bad := range []interface{}{true, 42, 3.14, nil, new(42), func() {}} { + req := decisionRequest(bad, map[string]schemas.DecisionQuestion{ + "q": {Kind: schemas.DecisionKindNoul, Instructions: "d"}, + }) + _, err := ToTypesafeDecisionRequest(req) + if err == nil || !strings.Contains(err.Error(), "state must be a string, object, or array") { + t.Fatalf("expected state shape rejection for %T, got %v", bad, err) + } + } + }) + + t.Run("instructions validated by serialized shape", func(t *testing.T) { + type rubric struct { + Goal string `json:"goal"` + } + for _, good := range []interface{}{rubric{Goal: "judge tone"}, []string{"read", "decide"}, map[string]string{"goal": "judge"}} { + req := decisionRequest("state", map[string]schemas.DecisionQuestion{ + "q": {Kind: schemas.DecisionKindNoul, Instructions: good}, + }) + if _, err := ToTypesafeDecisionRequest(req); err != nil { + t.Fatalf("expected %T instructions to be accepted, got %v", good, err) + } + } + for _, bad := range []interface{}{7, new(3.5), func() {}} { + req := decisionRequest("state", map[string]schemas.DecisionQuestion{ + "q": {Kind: schemas.DecisionKindNoul, Instructions: bad}, + }) + _, err := ToTypesafeDecisionRequest(req) + if err == nil || !strings.Contains(err.Error(), "instructions must be a string, object, or array") { + t.Fatalf("expected instructions shape rejection for %T, got %v", bad, err) + } + } + }) + + t.Run("typed string slice score criteria accepted", func(t *testing.T) { + req := decisionRequest("state", map[string]schemas.DecisionQuestion{ + "q": {Kind: schemas.DecisionKindScore, Instructions: "d", Criteria: []string{"low", "medium", "high"}}, + }) + native, err := ToTypesafeDecisionRequest(req) + if err != nil { + t.Fatalf("expected []string score criteria to be accepted, got %v", err) + } + levels, ok := native.Questions["q"].Criteria.([]interface{}) + if !ok || len(levels) != 3 || levels[0] != "low" { + t.Fatalf("score criteria not normalized to ordered array: %#v", native.Questions["q"].Criteria) + } + }) + + t.Run("typed maps, slices, and structs are valid shapes", func(t *testing.T) { + // Go SDK callers pass typed values; anything serializing to a JSON + // object or array satisfies the contract, not only the interface types + // an HTTP JSON decode produces. + type ticket struct { + ID int `json:"id"` + Body string `json:"body"` + } + for _, good := range []interface{}{ + map[string]string{"key": "value"}, + []string{"a", "b"}, + ticket{ID: 1, Body: "hello"}, + } { + req := decisionRequest(good, map[string]schemas.DecisionQuestion{ + "q": {Kind: schemas.DecisionKindNoul, Instructions: "d"}, + }) + if _, err := ToTypesafeDecisionRequest(req); err != nil { + t.Fatalf("expected %T to be a valid state shape, got %v", good, err) + } + } + }) +} + +func TestToBifrostDecisionResponseAllKindsWithZeroAndFractionalValues(t *testing.T) { + req := decisionRequest("state", map[string]schemas.DecisionQuestion{ + "is_spam": {Kind: schemas.DecisionKindNoul, Instructions: "d"}, + "lang": {Kind: schemas.DecisionKindChoice, Instructions: "d", Criteria: map[string]interface{}{"en": "", "de": ""}}, + "quality": {Kind: schemas.DecisionKindScore, Instructions: "d", Criteria: []interface{}{"bad", "ok", "good"}}, + }) + + zero := 0.0 + fractional := 1.75 + confidence := 0.9 + lang := "de" + native := &TypesafeDecisionResponse{ + Model: "jev-1.13.0", + Answers: map[string]TypesafeAnswer{ + // Zero is a legitimate evaluated value and must survive. + "is_spam": {Type: TypesafeQuestionTypeNoul, Noul: &zero, Probabilities: map[string]float64{"true": 0.0, "false": 1.0}}, + "lang": {Type: TypesafeQuestionTypeChoice, Choice: &lang, Probabilities: map[string]float64{"en": 0.1, "de": 0.9}, Confidence: &confidence}, + "quality": {Type: TypesafeQuestionTypeScore, Score: &fractional, Legend: map[string]any{"1": "bad", "2": "ok", "3": "good"}}, + }, + Usage: &TypesafeUsage{InputTokens: 120, OutputTokens: 0}, + } + + resp, bifrostErr := ToBifrostDecisionResponse(native, req) + if bifrostErr != nil { + t.Fatalf("unexpected error: %v", bifrostErr) + } + if resp.Model != "jev-1.13.0" { + t.Errorf("model = %q", resp.Model) + } + + spam := resp.Answers["is_spam"] + if spam.Kind != schemas.DecisionKindNoul || spam.Value != 0.0 { + t.Errorf("zero-valued noul answer lost: %+v", spam) + } + if spam.Probabilities["false"] != 1.0 { + t.Errorf("noul probabilities lost: %+v", spam.Probabilities) + } + + langAnswer := resp.Answers["lang"] + if langAnswer.Value != "de" { + t.Errorf("choice value = %v", langAnswer.Value) + } + if langAnswer.Confidence == nil || *langAnswer.Confidence != 0.9 { + t.Errorf("confidence lost: %+v", langAnswer) + } + + quality := resp.Answers["quality"] + if quality.Value != 1.75 { + t.Errorf("fractional score lost: %v", quality.Value) + } + if quality.Legend["3"] != "good" { + t.Errorf("legend lost: %+v", quality.Legend) + } + + if resp.Usage == nil || resp.Usage.PromptTokens != 120 || resp.Usage.CompletionTokens != 0 || resp.Usage.TotalTokens != 120 { + t.Errorf("usage not normalized: %+v", resp.Usage) + } +} + +func TestToBifrostDecisionResponseStructuredLegendDecodes(t *testing.T) { + // The legend echoes each score level verbatim - the SDKs type it as + // dict[str, str | object | array] (Python) and { [score]: T[score] } (JS) - + // so structured levels come back as objects and arrays, not strings. A + // map[string]string legend fails to decode such a body at all. + raw := []byte(`{ + "model": "jev-1.13.0", + "answers": { + "urgency": { + "type": "score", + "score": 1, + "confidence": 0.8, + "legend": {"0": "low", "1": {"examples": ["outage"], "level": "high"}, "2": ["critical", "churn risk"]} + } + }, + "usage": {"input_tokens": 10, "output_tokens": 0} + }`) + var native TypesafeDecisionResponse + if err := sonic.Unmarshal(raw, &native); err != nil { + t.Fatalf("structured legend failed to decode: %v", err) + } + + req := decisionRequest("state", map[string]schemas.DecisionQuestion{ + "urgency": {Kind: schemas.DecisionKindScore, Instructions: "d", Criteria: []any{ + "low", + map[string]any{"examples": []any{"outage"}, "level": "high"}, + []any{"critical", "churn risk"}, + }}, + }) + resp, bifrostErr := ToBifrostDecisionResponse(&native, req) + if bifrostErr != nil { + t.Fatalf("unexpected error: %v", bifrostErr) + } + legend := resp.Answers["urgency"].Legend + if legend["0"] != "low" { + t.Errorf("string legend level lost: %#v", legend["0"]) + } + if !reflect.DeepEqual(legend["1"], map[string]any{"examples": []any{"outage"}, "level": "high"}) { + t.Errorf("object legend level not lossless: %#v", legend["1"]) + } + if !reflect.DeepEqual(legend["2"], []any{"critical", "churn risk"}) { + t.Errorf("array legend level not lossless: %#v", legend["2"]) + } + + // And straight back out through the native converter without truncation. + back, err := ToTypesafeNativeDecisionResponse(resp) + if err != nil { + t.Fatalf("native round trip failed: %v", err) + } + if !reflect.DeepEqual(back.Answers["urgency"].Legend, legend) { + t.Errorf("legend lost in native round trip: %#v", back.Answers["urgency"].Legend) + } +} + +func TestToBifrostDecisionResponseFailures(t *testing.T) { + req := decisionRequest("state", map[string]schemas.DecisionQuestion{ + "is_spam": {Kind: schemas.DecisionKindNoul, Instructions: "d"}, + }) + + t.Run("missing answer", func(t *testing.T) { + native := &TypesafeDecisionResponse{Model: "jev-1.13.0", Answers: map[string]TypesafeAnswer{}} + if _, err := ToBifrostDecisionResponse(native, req); err == nil { + t.Fatal("expected error for missing answer") + } + }) + + t.Run("answer type mismatch", func(t *testing.T) { + choice := "yes" + native := &TypesafeDecisionResponse{ + Model: "jev-1.13.0", + Answers: map[string]TypesafeAnswer{"is_spam": {Type: TypesafeQuestionTypeChoice, Choice: &choice}}, + } + if _, err := ToBifrostDecisionResponse(native, req); err == nil { + t.Fatal("expected error for mismatched answer type") + } + }) + + t.Run("value missing for type", func(t *testing.T) { + native := &TypesafeDecisionResponse{ + Model: "jev-1.13.0", + Answers: map[string]TypesafeAnswer{"is_spam": {Type: TypesafeQuestionTypeNoul}}, + } + if _, err := ToBifrostDecisionResponse(native, req); err == nil { + t.Fatal("expected error for noul answer without value") + } + }) + + // A rejected response was still answered and billed by Typesafe. + t.Run("rejection carries billed usage", func(t *testing.T) { + native := &TypesafeDecisionResponse{ + Model: "jev-1.13.0", + Answers: map[string]TypesafeAnswer{}, + Usage: &TypesafeUsage{InputTokens: 5000, OutputTokens: 4}, + } + _, err := ToBifrostDecisionResponse(native, req) + if err == nil { + t.Fatal("expected error for missing answer") + } + if billed := err.ExtraFields.BilledUsage; billed == nil || billed.PromptTokens != 5000 || billed.CompletionTokens != 4 { + t.Fatalf("billed usage = %+v, want 5000 in / 4 out", billed) + } + if err.ExtraFields.ResolvedModelUsed != "jev-1.13.0" { + t.Fatalf("served model = %q", err.ExtraFields.ResolvedModelUsed) + } + }) + + t.Run("choice outside the requested options", func(t *testing.T) { + choiceReq := decisionRequest("state", map[string]schemas.DecisionQuestion{ + "category": {Kind: schemas.DecisionKindChoice, Instructions: "d", Criteria: map[string]any{"billing": "b", "bug": "g"}}, + }) + bad := "refund" + native := &TypesafeDecisionResponse{ + Model: "jev-1.13.0", + Answers: map[string]TypesafeAnswer{"category": {Type: TypesafeQuestionTypeChoice, Choice: &bad}}, + Usage: &TypesafeUsage{InputTokens: 300}, + } + _, err := ToBifrostDecisionResponse(native, choiceReq) + if err == nil { + t.Fatal("expected error for a choice outside the requested options") + } + if billed := err.ExtraFields.BilledUsage; billed == nil || billed.PromptTokens != 300 { + t.Fatalf("billed usage = %+v, want the 300 input tokens Typesafe charged", billed) + } + }) + + t.Run("score outside the rubric's levels", func(t *testing.T) { + scoreReq := decisionRequest("state", map[string]schemas.DecisionQuestion{ + "urgency": {Kind: schemas.DecisionKindScore, Instructions: "d", Criteria: []any{"low", "mid", "high"}}, + }) + for _, bad := range []float64{-0.5, 2.5} { + value := bad + native := &TypesafeDecisionResponse{ + Model: "jev-1.13.0", + Answers: map[string]TypesafeAnswer{"urgency": {Type: TypesafeQuestionTypeScore, Score: &value}}, + } + if _, err := ToBifrostDecisionResponse(native, scoreReq); err == nil { + t.Fatalf("expected error for score %v outside 0..2", bad) + } + } + edge := 2.0 + native := &TypesafeDecisionResponse{ + Model: "jev-1.13.0", + Answers: map[string]TypesafeAnswer{"urgency": {Type: TypesafeQuestionTypeScore, Score: &edge}}, + } + if _, err := ToBifrostDecisionResponse(native, scoreReq); err != nil { + t.Fatalf("the top level must be accepted: %v", err) + } + }) + + t.Run("noul value outside range", func(t *testing.T) { + // The provider response is untrusted; a noul outside [0,1] violates the + // documented contract and must not surface as a successful answer. + for _, bad := range []float64{-0.2, 1.4} { + value := bad + native := &TypesafeDecisionResponse{ + Model: "jev-1.13.0", + Answers: map[string]TypesafeAnswer{"is_spam": {Type: TypesafeQuestionTypeNoul, Noul: &value}}, + } + if _, err := ToBifrostDecisionResponse(native, req); err == nil { + t.Fatalf("expected error for noul value %v outside [0,1]", bad) + } + } + }) +} + +func TestNativeRoundTrip(t *testing.T) { + native := &TypesafeDecisionRequest{ + State: "some state", + Model: "jev-latest", + Questions: map[string]TypesafeQuestion{ + "approve": {Type: TypesafeQuestionTypeNoul, Instructions: "Approve?", Criteria: map[string]interface{}{"true": "approve it", "false": "reject it"}}, + "bucket": {Type: TypesafeQuestionTypeChoice, Instructions: "Bucket?", Criteria: map[string]interface{}{"a": "first", "b": "second"}}, + "rating": {Type: TypesafeQuestionTypeScore, Instructions: "Rate it", Criteria: []interface{}{"low", "high"}}, + }, + } + + shared, err := native.ToBifrostDecisionRequest(nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if shared.Provider != schemas.Typesafe || shared.Model != "jev-latest" { + t.Errorf("routing = %s/%s", shared.Provider, shared.Model) + } + if len(shared.Questions) != 3 { + t.Fatalf("expected 3 questions, got %d", len(shared.Questions)) + } + + // The shared shape must convert straight back to the native shape. + back, err := ToTypesafeDecisionRequest(shared) + if err != nil { + t.Fatalf("round trip failed: %v", err) + } + for name, question := range native.Questions { + got, ok := back.Questions[name] + if !ok { + t.Fatalf("question %q lost in round trip", name) + } + if got.Type != question.Type { + t.Errorf("question %q type = %q, want %q", name, got.Type, question.Type) + } + } + + // Explicit provider prefix routes the same way. + native.Model = "typesafe/jev-1.13.0" + shared, err = native.ToBifrostDecisionRequest(nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if shared.Provider != schemas.Typesafe || shared.Model != "jev-1.13.0" { + t.Errorf("prefixed routing = %s/%s", shared.Provider, shared.Model) + } +} + +func TestToTypesafeNativeDecisionResponse(t *testing.T) { + resp := &schemas.BifrostDecisionResponse{ + Model: "jev-1.13.0", + Answers: map[string]schemas.DecisionAnswer{ + "approve": {Kind: schemas.DecisionKindNoul, Value: 0.25, Probabilities: map[string]float64{"true": 0.25, "false": 0.75}}, + }, + Usage: &schemas.BifrostLLMUsage{PromptTokens: 10, CompletionTokens: 0, TotalTokens: 10}, + } + + native, err := ToTypesafeNativeDecisionResponse(resp) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + answer := native.Answers["approve"] + if answer.Type != TypesafeQuestionTypeNoul || answer.Noul == nil || *answer.Noul != 0.25 { + t.Errorf("noul answer not rebuilt: %+v", answer) + } + if answer.Probabilities["false"] != 0.75 { + t.Errorf("probabilities lost: %+v", answer.Probabilities) + } + if native.Usage == nil || native.Usage.InputTokens != 10 { + t.Errorf("usage lost: %+v", native.Usage) + } +} + +// noopLogger satisfies schemas.Logger for constructor calls in unit tests. +// schemas cannot provide one and core's DefaultLogger would import-cycle here. +type noopLogger struct{} + +func (noopLogger) Debug(string, ...any) {} +func (noopLogger) Info(string, ...any) {} +func (noopLogger) Warn(string, ...any) {} +func (noopLogger) Error(string, ...any) {} +func (noopLogger) Fatal(string, ...any) {} +func (noopLogger) SetLevel(schemas.LogLevel) {} +func (noopLogger) SetOutputType(schemas.LoggerOutputType) {} +func (noopLogger) LogHTTPRequest(schemas.LogLevel, string) schemas.LogEventBuilder { return nil } + +func TestListModelsEntriesCarryOwnerAndDescription(t *testing.T) { + provider, err := NewTypesafeProvider(&schemas.ProviderConfig{}, noopLogger{}) + if err != nil { + t.Fatalf("constructor failed: %v", err) + } + key := schemas.Key{Models: []string{"*"}} + resp, bifrostErr := provider.listModelsByKey(nil, key, &schemas.BifrostListModelsRequest{}) + if bifrostErr != nil { + t.Fatalf("unexpected error: %v", bifrostErr) + } + if len(resp.Data) != len(typesafeModels) { + t.Fatalf("expected %d models, got %d", len(typesafeModels), len(resp.Data)) + } + for _, model := range resp.Data { + if model.OwnedBy == nil || *model.OwnedBy != "typesafe" { + t.Errorf("model %s missing owned_by: %v", model.ID, model.OwnedBy) + } + if model.Description == nil || *model.Description == "" { + t.Errorf("model %s missing description", model.ID) + } + // Pricing intentionally absent here: it backfills from the datasheet in + // the list-models handler, never from the provider. + if model.Pricing != nil { + t.Errorf("model %s carries hardcoded pricing; pricing must come from the datasheet", model.ID) + } + } +} + +func TestListModelsNativeShape(t *testing.T) { + name := "Jev (latest)" + resp := &schemas.BifrostListModelsResponse{ + Data: []schemas.Model{{ID: "typesafe/jev-latest", Name: &name}}, + } + native := ToTypesafeNativeListModelsResponse(resp) + if len(native.Models) != 1 { + t.Fatalf("expected 1 model, got %d", len(native.Models)) + } + if native.Models[0].Name != "jev-latest" { + t.Errorf("native model name = %q, want bare jev-latest", native.Models[0].Name) + } + if native.Models[0].Description == "" || native.Models[0].ReleaseDate == "" { + t.Errorf("catalog metadata not attached: %+v", native.Models[0]) + } +} + +func TestToTypesafeNativeError(t *testing.T) { + errType := "invalid_request_error" + native := ToTypesafeNativeError(&schemas.BifrostError{ + Error: &schemas.ErrorField{Type: &errType, Message: "question \"q\" has unsupported kind"}, + }) + if native.Detail.ErrorType != "invalid_request_error" { + t.Errorf("error_type = %q", native.Detail.ErrorType) + } + if native.Detail.Message == "" { + t.Error("message lost") + } + + // Missing pieces fall back rather than emitting empty fields. + fallback := ToTypesafeNativeError(nil) + if fallback.Detail.ErrorType != "api_error" || fallback.Detail.Message == "" { + t.Errorf("nil error fallback = %+v", fallback.Detail) + } +} diff --git a/core/providers/typesafe/errors.go b/core/providers/typesafe/errors.go new file mode 100644 index 00000000000..0ec664cf35a --- /dev/null +++ b/core/providers/typesafe/errors.go @@ -0,0 +1,72 @@ +package typesafe + +import ( + providerUtils "github.com/capsohq/bifrost/core/providers/utils" + schemas "github.com/capsohq/bifrost/core/schemas" + "github.com/valyala/fasthttp" +) + +// parseTypesafeError parses a Typesafe error HTTP response into a BifrostError. +// Typesafe returns a JSON body detailing the issue on 401, 422, 429 and 529; +// the upstream status code and validation detail are preserved. +func parseTypesafeError(resp *fasthttp.Response) *schemas.BifrostError { + var errorResp TypesafeError + bifrostErr := providerUtils.HandleProviderAPIError(resp, &errorResp) + + message := errorResp.Message + if message == "" && errorResp.Error != nil { + message = errorResp.Error.Message + } + if message == "" { + if detail, ok := errorResp.Detail.(string); ok { + message = detail + } + } + + if bifrostErr.Error == nil { + bifrostErr.Error = &schemas.ErrorField{} + } + if message != "" { + bifrostErr.Error.Message = message + } else if bifrostErr.Error.Message == "" { + bifrostErr.Error.Message = "Typesafe API request failed" + } + + return bifrostErr +} + +// TypesafeNativeErrorDetail is the payload of Typesafe's native error body. +type TypesafeNativeErrorDetail struct { + ErrorType string `json:"error_type"` + Message string `json:"message"` +} + +// TypesafeNativeError is the error body Typesafe's API returns and its SDKs +// parse: {"detail": {"error_type": ..., "message": ...}} (observed on the live +// API; the docs describe JSON error bodies without pinning the schema). +type TypesafeNativeError struct { + Detail TypesafeNativeErrorDetail `json:"detail"` +} + +// ToTypesafeNativeError converts a Bifrost error into Typesafe's native error +// body so the /typesafe drop-in surface stays parseable by Typesafe's SDKs. +// The HTTP status code rides on the response as usual; this shapes the body. +func ToTypesafeNativeError(bifrostErr *schemas.BifrostError) *TypesafeNativeError { + native := &TypesafeNativeError{ + Detail: TypesafeNativeErrorDetail{ErrorType: "api_error"}, + } + if bifrostErr == nil { + native.Detail.Message = "unknown error" + return native + } + if bifrostErr.Error != nil { + native.Detail.Message = bifrostErr.Error.Message + if bifrostErr.Error.Type != nil && *bifrostErr.Error.Type != "" { + native.Detail.ErrorType = *bifrostErr.Error.Type + } + } + if native.Detail.Message == "" { + native.Detail.Message = "Typesafe API request failed" + } + return native +} diff --git a/core/providers/typesafe/types.go b/core/providers/typesafe/types.go new file mode 100644 index 00000000000..c515471fd42 --- /dev/null +++ b/core/providers/typesafe/types.go @@ -0,0 +1,67 @@ +package typesafe + +// Typesafe question types accepted by POST /v1/systemone. +const ( + TypesafeQuestionTypeNoul = "noul" + TypesafeQuestionTypeChoice = "choice" + TypesafeQuestionTypeScore = "score" +) + +// TypesafeQuestion is one named question in a systemone request. +// Instructions accepts a string, object, or array. Criteria is a map for noul +// (optional "true"/"false" descriptions) and choice (option -> description, +// max 255 options), and an ordered array of 2-10 level descriptions for score. +type TypesafeQuestion struct { + Type string `json:"type"` + Instructions interface{} `json:"instructions,omitempty"` + Criteria interface{} `json:"criteria,omitempty"` +} + +// TypesafeDecisionRequest is the body of POST /v1/systemone. +type TypesafeDecisionRequest struct { + State interface{} `json:"state"` + Model string `json:"model"` + Questions map[string]TypesafeQuestion `json:"questions"` +} + +// GetExtraParams implements providerUtils.RequestBodyWithExtraParams. Typesafe +// takes no passthrough params. +func (r *TypesafeDecisionRequest) GetExtraParams() map[string]interface{} { + return nil +} + +// TypesafeAnswer is one answer in a systemone response. Exactly one of Noul, +// Choice, or Score is set, matching Type. +type TypesafeAnswer struct { + Type string `json:"type"` + Noul *float64 `json:"noul,omitempty"` + Choice *string `json:"choice,omitempty"` + Score *float64 `json:"score,omitempty"` + Probabilities map[string]float64 `json:"probabilities,omitempty"` + // Legend echoes each score level's description verbatim (string, object, + // or array), keyed by level number. + Legend map[string]any `json:"legend,omitempty"` + Confidence *float64 `json:"confidence,omitempty"` +} + +// TypesafeUsage reports token consumption. Typesafe bills input tokens only. +type TypesafeUsage struct { + InputTokens int `json:"input_tokens"` + OutputTokens int `json:"output_tokens"` +} + +// TypesafeDecisionResponse is the body of a successful systemone response. +type TypesafeDecisionResponse struct { + Model string `json:"model"` + Answers map[string]TypesafeAnswer `json:"answers"` + Usage *TypesafeUsage `json:"usage,omitempty"` +} + +// TypesafeError is the JSON error body Typesafe returns on 401/422/429/529. +type TypesafeError struct { + Message string `json:"message,omitempty"` + Detail interface{} `json:"detail,omitempty"` + Error *struct { + Message string `json:"message,omitempty"` + } `json:"error,omitempty"` +} diff --git a/core/providers/typesafe/typesafe.go b/core/providers/typesafe/typesafe.go new file mode 100644 index 00000000000..5cd3d0eeb15 --- /dev/null +++ b/core/providers/typesafe/typesafe.go @@ -0,0 +1,199 @@ +// Package typesafe implements the Typesafe provider for Bifrost. +// Typesafe serves judgment models (the jev System One family) through a single +// synchronous evaluation endpoint, POST /v1/systemone. Bifrost exposes it via +// the shared decision operation; every other operation returns unsupported. +package typesafe + +import ( + "net/http" + "strings" + "time" + + providerUtils "github.com/capsohq/bifrost/core/providers/utils" + schemas "github.com/capsohq/bifrost/core/schemas" + "github.com/valyala/fasthttp" +) + +// TypesafeProvider implements the Provider interface for Typesafe's API. +type TypesafeProvider struct { + logger schemas.Logger // Logger for provider operations + client *fasthttp.Client // HTTP client for unary API requests (ReadTimeout bounds overall response) + streamingClient *fasthttp.Client // HTTP client for streaming API requests (no ReadTimeout; unused today, kept per provider pattern) + networkConfig schemas.NetworkConfig // Network configuration including extra headers + sendBackRawRequest bool // Whether to include raw request in BifrostResponse + sendBackRawResponse bool // Whether to include raw response in BifrostResponse +} + +// NewTypesafeProvider creates a new Typesafe provider instance. +func NewTypesafeProvider(config *schemas.ProviderConfig, logger schemas.Logger) (*TypesafeProvider, error) { + config.CheckAndSetDefaults() + + requestTimeout := time.Second * time.Duration(config.NetworkConfig.DefaultRequestTimeoutInSeconds) + client := &fasthttp.Client{ + ReadTimeout: requestTimeout, + WriteTimeout: requestTimeout, + MaxConnsPerHost: config.NetworkConfig.MaxConnsPerHost, + MaxIdleConnDuration: time.Second * time.Duration(config.NetworkConfig.KeepAliveTimeoutInSeconds), + MaxConnWaitTimeout: requestTimeout, + MaxConnDuration: time.Second * time.Duration(schemas.DefaultMaxConnDurationInSeconds), + ConnPoolStrategy: fasthttp.FIFO, + } + + // Configure proxy if provided + client = providerUtils.ConfigureProxy(client, config.ProxyConfig, logger) + client = providerUtils.ConfigureDialer(client, config.NetworkConfig.AllowPrivateNetwork) + client = providerUtils.ConfigureTLS(client, config.NetworkConfig, logger) + streamingClient := providerUtils.BuildStreamingClient(client) + + if config.NetworkConfig.BaseURL == "" { + config.NetworkConfig.BaseURL = typesafeDefaultBaseURL + } + config.NetworkConfig.BaseURL = strings.TrimRight(config.NetworkConfig.BaseURL, "/") + + return &TypesafeProvider{ + logger: logger, + client: client, + streamingClient: streamingClient, + networkConfig: config.NetworkConfig, + sendBackRawRequest: config.SendBackRawRequest, + sendBackRawResponse: config.SendBackRawResponse, + }, nil +} + +// GetProviderKey returns the provider identifier for Typesafe. +func (provider *TypesafeProvider) GetProviderKey() schemas.ModelProvider { + return schemas.Typesafe +} + +// ListModels serves the static jev catalog. Typesafe documents no model-listing +// endpoint, so no upstream call is made; the catalog is pinned in utils.go and +// mirrored in the hosted datasheet. +func (provider *TypesafeProvider) ListModels(ctx *schemas.BifrostContext, keys []schemas.Key, request *schemas.BifrostListModelsRequest) (*schemas.BifrostListModelsResponse, *schemas.BifrostError) { + startTime := time.Now() + + response, err := providerUtils.HandleMultipleListModelsRequests(ctx, keys, request, provider.listModelsByKey) + if err != nil { + return nil, err + } + + response.ExtraFields.Latency = time.Since(startTime).Milliseconds() + return response, nil +} + +// listModelsByKey filters the static catalog through the standard list-models +// pipeline so key whitelists, blacklists, and aliases apply. +func (provider *TypesafeProvider) listModelsByKey(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostListModelsRequest) (*schemas.BifrostListModelsResponse, *schemas.BifrostError) { + response := &schemas.BifrostListModelsResponse{ + Data: make([]schemas.Model, 0, len(typesafeModels)), + } + + pipeline := &providerUtils.ListModelsPipeline{ + AllowedModels: key.Models, + BlacklistedModels: key.BlacklistedModels, + Aliases: key.Aliases, + Unfiltered: request.Unfiltered, + ProviderKey: provider.GetProviderKey(), + MatchFns: providerUtils.DefaultMatchFns(), + } + if pipeline.ShouldEarlyExit() { + return response, nil + } + + included := make(map[string]bool) + for _, model := range typesafeModels { + for _, result := range pipeline.FilterModel(model.ID) { + name := model.Name + description := model.Description + entry := schemas.Model{ + ID: string(provider.GetProviderKey()) + "/" + result.ResolvedID, + Name: &name, + Description: &description, + OwnedBy: new("typesafe"), + } + if result.AliasValue != "" { + alias := result.AliasValue + entry.Alias = &alias + } + response.Data = append(response.Data, entry) + included[strings.ToLower(result.ResolvedID)] = true + } + } + response.Data = append(response.Data, pipeline.BackfillModels(included)...) + + return response, nil +} + +// Decision performs a synchronous evaluation against POST /v1/systemone. +func (provider *TypesafeProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + jsonData, bifrostErr := providerUtils.CheckContextAndGetRequestBody( + ctx, + request, + func() (providerUtils.RequestBodyWithExtraParams, error) { + return ToTypesafeDecisionRequest(request) + }) + if bifrostErr != nil { + return nil, bifrostErr + } + + sendBackRawRequest := providerUtils.ShouldSendBackRawRequest(ctx, provider.sendBackRawRequest) + sendBackRawResponse := providerUtils.ShouldSendBackRawResponse(ctx, provider.sendBackRawResponse) + + req := fasthttp.AcquireRequest() + resp := fasthttp.AcquireResponse() + defer fasthttp.ReleaseRequest(req) + defer fasthttp.ReleaseResponse(resp) + + providerUtils.SetExtraHeaders(ctx, req, provider.networkConfig.ExtraHeaders, nil) + req.SetRequestURI(provider.networkConfig.BaseURL + providerUtils.GetPathFromContext(ctx, typesafeSystemOnePath)) + req.Header.SetMethod(http.MethodPost) + req.Header.SetContentType("application/json") + if key.Value.GetValue() != "" { + req.Header.Set("Authorization", "Bearer "+key.Value.GetValue()) + } + req.SetBody(jsonData) + + latency, bifrostErr, wait := providerUtils.MakeRequestWithContext(ctx, provider.client, req, resp) + defer wait() + if bifrostErr != nil { + return nil, bifrostErr + } + + if resp.StatusCode() != fasthttp.StatusOK { + return nil, providerUtils.EnrichError(ctx, parseTypesafeError(resp), jsonData, nil, sendBackRawRequest, sendBackRawResponse, latency) + } + + ft, fh := providerUtils.StartPhaseSpan(ctx, "response-finalize") + respBody, err := providerUtils.CheckAndDecodeBody(resp) + if ft != nil { + if err != nil { + ft.EndSpan(fh, schemas.SpanStatusError, err.Error()) + } else { + ft.EndSpan(fh, schemas.SpanStatusOk, "") + } + } + if err != nil { + rawErrBody := append([]byte(nil), resp.Body()...) + return nil, providerUtils.EnrichError(ctx, providerUtils.NewBifrostOperationError(schemas.ErrProviderResponseDecode, err), jsonData, rawErrBody, sendBackRawRequest, sendBackRawResponse, latency) + } + + var typesafeResp TypesafeDecisionResponse + rawRequest, rawResponse, bifrostErr := providerUtils.HandleProviderResponseCtx(ctx, respBody, &typesafeResp, jsonData, sendBackRawRequest, sendBackRawResponse) + if bifrostErr != nil { + return nil, bifrostErr + } + + bifrostResp, bifrostErr := ToBifrostDecisionResponse(&typesafeResp, request) + if bifrostErr != nil { + return nil, providerUtils.EnrichError(ctx, bifrostErr, jsonData, respBody, sendBackRawRequest, sendBackRawResponse, latency) + } + + bifrostResp.ExtraFields.Latency = latency.Milliseconds() + if sendBackRawRequest { + bifrostResp.ExtraFields.RawRequest = rawRequest + } + if sendBackRawResponse { + bifrostResp.ExtraFields.RawResponse = rawResponse + } + + return bifrostResp, nil +} diff --git a/core/providers/typesafe/typesafe_test.go b/core/providers/typesafe/typesafe_test.go new file mode 100644 index 00000000000..681d9639dca --- /dev/null +++ b/core/providers/typesafe/typesafe_test.go @@ -0,0 +1,40 @@ +package typesafe_test + +import ( + "os" + "strings" + "testing" + + "github.com/capsohq/bifrost/core/internal/llmtests" + "github.com/capsohq/bifrost/core/schemas" +) + +func TestTypesafe(t *testing.T) { + t.Parallel() + if strings.TrimSpace(os.Getenv("TYPESAFE_API_KEY")) == "" { + t.Skip("Skipping Typesafe tests because TYPESAFE_API_KEY is not set") + } + + client, ctx, cancel, err := llmtests.SetupTest() + if err != nil { + t.Fatalf("Error initializing test setup: %v", err) + } + defer cancel() + defer client.Shutdown() + + testConfig := llmtests.ComprehensiveTestConfig{ + Provider: schemas.Typesafe, + DecisionModel: "jev-1.13.0", + Scenarios: llmtests.TestScenarios{ + // Typesafe serves the decision operation only; every other operation + // returns the standard unsupported-operation error. ListModels is + // served from the static catalog without an upstream call. + Decision: true, + ListModels: true, + }, + } + + t.Run("TypesafeTests", func(t *testing.T) { + llmtests.RunAllComprehensiveTests(t, client, ctx, testConfig) + }) +} diff --git a/core/providers/typesafe/unsupported.go b/core/providers/typesafe/unsupported.go new file mode 100644 index 00000000000..ed206c95ff0 --- /dev/null +++ b/core/providers/typesafe/unsupported.go @@ -0,0 +1,229 @@ +// Operations the Typesafe evaluation API does not serve. Typesafe exposes a single +// synchronous judgment endpoint (POST /v1/systemone); every other Bifrost operation +// returns the standard unsupported-operation error. +package typesafe + +import ( + "context" + + providerUtils "github.com/capsohq/bifrost/core/providers/utils" + schemas "github.com/capsohq/bifrost/core/schemas" +) + +func (provider *TypesafeProvider) TextCompletion(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostTextCompletionRequest) (*schemas.BifrostTextCompletionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.TextCompletionRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) TextCompletionStream(ctx *schemas.BifrostContext, postHookRunner schemas.PostHookRunner, postHookSpanFinalizer func(context.Context), key schemas.Key, request *schemas.BifrostTextCompletionRequest) (chan *schemas.BifrostStreamChunk, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.TextCompletionStreamRequest, provider.GetProviderKey()) +} + +// Speech is not supported by the Typesafe provider. +func (provider *TypesafeProvider) Speech(_ *schemas.BifrostContext, _ schemas.Key, _ *schemas.BifrostSpeechRequest) (*schemas.BifrostSpeechResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.SpeechRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) Rerank(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostRerankRequest) (*schemas.BifrostRerankResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) SpeechStream(ctx *schemas.BifrostContext, postHookRunner schemas.PostHookRunner, postHookSpanFinalizer func(context.Context), key schemas.Key, request *schemas.BifrostSpeechRequest) (chan *schemas.BifrostStreamChunk, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.SpeechStreamRequest, provider.GetProviderKey()) +} + +// Transcription is not supported by the Typesafe provider. +func (provider *TypesafeProvider) Transcription(_ *schemas.BifrostContext, _ schemas.Key, _ *schemas.BifrostTranscriptionRequest) (*schemas.BifrostTranscriptionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.TranscriptionRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) TranscriptionStream(ctx *schemas.BifrostContext, postHookRunner schemas.PostHookRunner, postHookSpanFinalizer func(context.Context), key schemas.Key, request *schemas.BifrostTranscriptionRequest) (chan *schemas.BifrostStreamChunk, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.TranscriptionStreamRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) ImageGeneration(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostImageGenerationRequest) (*schemas.BifrostImageGenerationResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.ImageGenerationRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) ImageGenerationStream(ctx *schemas.BifrostContext, postHookRunner schemas.PostHookRunner, postHookSpanFinalizer func(context.Context), key schemas.Key, request *schemas.BifrostImageGenerationRequest) (chan *schemas.BifrostStreamChunk, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.ImageGenerationStreamRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) ImageEdit(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostImageEditRequest) (*schemas.BifrostImageGenerationResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.ImageEditRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) ImageEditStream(ctx *schemas.BifrostContext, postHookRunner schemas.PostHookRunner, postHookSpanFinalizer func(context.Context), key schemas.Key, request *schemas.BifrostImageEditRequest) (chan *schemas.BifrostStreamChunk, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.ImageEditStreamRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) ImageVariation(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostImageVariationRequest) (*schemas.BifrostImageGenerationResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.ImageVariationRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) VideoGeneration(_ *schemas.BifrostContext, _ schemas.Key, _ *schemas.BifrostVideoGenerationRequest) (*schemas.BifrostVideoGenerationResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.VideoGenerationRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) VideoRetrieve(_ *schemas.BifrostContext, _ schemas.Key, _ *schemas.BifrostVideoRetrieveRequest) (*schemas.BifrostVideoGenerationResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.VideoRetrieveRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) VideoDownload(_ *schemas.BifrostContext, _ schemas.Key, _ *schemas.BifrostVideoDownloadRequest) (*schemas.BifrostVideoDownloadResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.VideoDownloadRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) VideoDelete(_ *schemas.BifrostContext, _ schemas.Key, _ *schemas.BifrostVideoDeleteRequest) (*schemas.BifrostVideoDeleteResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.VideoDeleteRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) VideoList(_ *schemas.BifrostContext, _ schemas.Key, _ *schemas.BifrostVideoListRequest) (*schemas.BifrostVideoListResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.VideoListRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) VideoEdit(_ *schemas.BifrostContext, _ schemas.Key, _ *schemas.BifrostVideoEditRequest) (*schemas.BifrostVideoEditResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.VideoEditRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) VideoRemix(_ *schemas.BifrostContext, _ schemas.Key, _ *schemas.BifrostVideoRemixRequest) (*schemas.BifrostVideoGenerationResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.VideoRemixRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) BatchCreate(_ *schemas.BifrostContext, _ schemas.Key, _ *schemas.BifrostBatchCreateRequest) (*schemas.BifrostBatchCreateResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.BatchCreateRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) BatchList(_ *schemas.BifrostContext, _ []schemas.Key, _ *schemas.BifrostBatchListRequest) (*schemas.BifrostBatchListResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.BatchListRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) BatchRetrieve(_ *schemas.BifrostContext, _ []schemas.Key, _ *schemas.BifrostBatchRetrieveRequest) (*schemas.BifrostBatchRetrieveResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.BatchRetrieveRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) BatchCancel(_ *schemas.BifrostContext, _ []schemas.Key, _ *schemas.BifrostBatchCancelRequest) (*schemas.BifrostBatchCancelResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.BatchCancelRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) BatchDelete(_ *schemas.BifrostContext, _ []schemas.Key, _ *schemas.BifrostBatchDeleteRequest) (*schemas.BifrostBatchDeleteResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.BatchDeleteRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) BatchResults(_ *schemas.BifrostContext, _ []schemas.Key, _ *schemas.BifrostBatchResultsRequest) (*schemas.BifrostBatchResultsResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.BatchResultsRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) FileUpload(_ *schemas.BifrostContext, _ schemas.Key, _ *schemas.BifrostFileUploadRequest) (*schemas.BifrostFileUploadResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.FileUploadRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) FileList(_ *schemas.BifrostContext, _ []schemas.Key, _ *schemas.BifrostFileListRequest) (*schemas.BifrostFileListResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.FileListRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) FileRetrieve(_ *schemas.BifrostContext, _ []schemas.Key, _ *schemas.BifrostFileRetrieveRequest) (*schemas.BifrostFileRetrieveResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.FileRetrieveRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) FileDelete(_ *schemas.BifrostContext, _ []schemas.Key, _ *schemas.BifrostFileDeleteRequest) (*schemas.BifrostFileDeleteResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.FileDeleteRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) FileContent(_ *schemas.BifrostContext, _ []schemas.Key, _ *schemas.BifrostFileContentRequest) (*schemas.BifrostFileContentResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.FileContentRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) CountTokens(_ *schemas.BifrostContext, _ schemas.Key, _ *schemas.BifrostResponsesRequest) (*schemas.BifrostCountTokensResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.CountTokensRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) Compaction(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostCompactionRequest) (*schemas.BifrostCompactionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.CompactionRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) ContainerCreate(_ *schemas.BifrostContext, _ schemas.Key, _ *schemas.BifrostContainerCreateRequest) (*schemas.BifrostContainerCreateResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.ContainerCreateRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) ContainerList(_ *schemas.BifrostContext, _ []schemas.Key, _ *schemas.BifrostContainerListRequest) (*schemas.BifrostContainerListResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.ContainerListRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) ContainerRetrieve(_ *schemas.BifrostContext, _ []schemas.Key, _ *schemas.BifrostContainerRetrieveRequest) (*schemas.BifrostContainerRetrieveResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.ContainerRetrieveRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) ContainerDelete(_ *schemas.BifrostContext, _ []schemas.Key, _ *schemas.BifrostContainerDeleteRequest) (*schemas.BifrostContainerDeleteResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.ContainerDeleteRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) ContainerFileCreate(_ *schemas.BifrostContext, _ schemas.Key, _ *schemas.BifrostContainerFileCreateRequest) (*schemas.BifrostContainerFileCreateResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.ContainerFileCreateRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) ContainerFileList(_ *schemas.BifrostContext, _ []schemas.Key, _ *schemas.BifrostContainerFileListRequest) (*schemas.BifrostContainerFileListResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.ContainerFileListRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) ContainerFileRetrieve(_ *schemas.BifrostContext, _ []schemas.Key, _ *schemas.BifrostContainerFileRetrieveRequest) (*schemas.BifrostContainerFileRetrieveResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.ContainerFileRetrieveRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) ContainerFileContent(_ *schemas.BifrostContext, _ []schemas.Key, _ *schemas.BifrostContainerFileContentRequest) (*schemas.BifrostContainerFileContentResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.ContainerFileContentRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) ContainerFileDelete(_ *schemas.BifrostContext, _ []schemas.Key, _ *schemas.BifrostContainerFileDeleteRequest) (*schemas.BifrostContainerFileDeleteResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.ContainerFileDeleteRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) Passthrough(_ *schemas.BifrostContext, _ schemas.Key, _ *schemas.BifrostPassthroughRequest) (*schemas.BifrostPassthroughResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.PassthroughRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) PassthroughStream(_ *schemas.BifrostContext, _ schemas.PostHookRunner, _ func(context.Context), _ schemas.Key, _ *schemas.BifrostPassthroughRequest) (chan *schemas.BifrostStreamChunk, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.PassthroughStreamRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) CachedContentCreate(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostCachedContentCreateRequest) (*schemas.BifrostCachedContentCreateResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.CachedContentCreateRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) CachedContentDelete(ctx *schemas.BifrostContext, keys []schemas.Key, request *schemas.BifrostCachedContentDeleteRequest) (*schemas.BifrostCachedContentDeleteResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.CachedContentDeleteRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) CachedContentList(ctx *schemas.BifrostContext, keys []schemas.Key, request *schemas.BifrostCachedContentListRequest) (*schemas.BifrostCachedContentListResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.CachedContentListRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) CachedContentRetrieve(ctx *schemas.BifrostContext, keys []schemas.Key, request *schemas.BifrostCachedContentRetrieveRequest) (*schemas.BifrostCachedContentRetrieveResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.CachedContentRetrieveRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) CachedContentUpdate(ctx *schemas.BifrostContext, keys []schemas.Key, request *schemas.BifrostCachedContentUpdateRequest) (*schemas.BifrostCachedContentUpdateResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.CachedContentUpdateRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) ChatCompletion(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostChatRequest) (*schemas.BifrostChatResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.ChatCompletionRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) ChatCompletionStream(ctx *schemas.BifrostContext, postHookRunner schemas.PostHookRunner, postHookSpanFinalizer func(context.Context), key schemas.Key, request *schemas.BifrostChatRequest) (chan *schemas.BifrostStreamChunk, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.ChatCompletionStreamRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) Embedding(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostEmbeddingRequest) (*schemas.BifrostEmbeddingResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.EmbeddingRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) Responses(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostResponsesRequest) (*schemas.BifrostResponsesResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.ResponsesRequest, provider.GetProviderKey()) +} + +func (provider *TypesafeProvider) ResponsesStream(ctx *schemas.BifrostContext, postHookRunner schemas.PostHookRunner, postHookSpanFinalizer func(context.Context), key schemas.Key, request *schemas.BifrostResponsesRequest) (chan *schemas.BifrostStreamChunk, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.ResponsesStreamRequest, provider.GetProviderKey()) +} diff --git a/core/providers/typesafe/utils.go b/core/providers/typesafe/utils.go new file mode 100644 index 00000000000..a93254d76c7 --- /dev/null +++ b/core/providers/typesafe/utils.go @@ -0,0 +1,89 @@ +package typesafe + +import ( + "strings" + + schemas "github.com/capsohq/bifrost/core/schemas" +) + +// typesafeDefaultBaseURL is the default Typesafe API host. +const typesafeDefaultBaseURL = "https://api.typesafe.ai" + +// typesafeSystemOnePath is the single evaluation endpoint for the System One +// model class; every jev model is served by it, selected via the request's +// model field. +const typesafeSystemOnePath = "/v1/systemone" + +// typesafeModel is one entry of the static model catalog. +type typesafeModel struct { + ID string + Name string + Description string + ReleaseDate string +} + +// typesafeModels is the static catalog served by ListModels. Typesafe documents +// no model-listing endpoint, so the catalog is pinned here and in the hosted +// datasheet. Aliases resolve upstream: jev-latest and jev-preview both point at +// jev-1.13.0 today. +var typesafeModels = []typesafeModel{ + { + ID: "jev-1.13.0", + Name: "Jev 1.13.0", + Description: "TypeSafe's System One Model: Jev, version 1.13.0", + // The SDKs validate release_date on every listed model; 1.13.0 shipped + // at the same instant its aliases were minted. + ReleaseDate: "2026-09-10T18:38:01.391457+00:00", + }, + { + ID: "jev-latest", + Name: "Jev (latest)", + Description: "The latest iteration of TypeSafe's System One Model: Jev", + ReleaseDate: "2026-09-10T18:38:01.391457+00:00", + }, + { + ID: "jev-preview", + Name: "Jev (preview)", + Description: "A preview version of `jev-latest`: should be better in most ways", + ReleaseDate: "2026-09-10T18:39:06.057655+00:00", + }, +} + +// TypesafeNativeModel is one entry of the native model-listing shape. +type TypesafeNativeModel struct { + Name string `json:"name"` + Description string `json:"description,omitempty"` + ReleaseDate string `json:"release_date,omitempty"` +} + +// TypesafeNativeListModelsResponse is the native model-listing shape served on +// /typesafe/v1/models. Typesafe documents no upstream listing endpoint, so +// this is synthesized from the static catalog. +type TypesafeNativeListModelsResponse struct { + Models []TypesafeNativeModel `json:"models"` +} + +// ToTypesafeNativeListModelsResponse converts a Bifrost model listing into the +// native shape, restoring bare upstream model names and re-attaching catalog +// descriptions and release dates. +func ToTypesafeNativeListModelsResponse(resp *schemas.BifrostListModelsResponse) *TypesafeNativeListModelsResponse { + native := &TypesafeNativeListModelsResponse{Models: []TypesafeNativeModel{}} + if resp == nil { + return native + } + catalog := make(map[string]typesafeModel, len(typesafeModels)) + for _, model := range typesafeModels { + catalog[model.ID] = model + } + prefix := string(schemas.Typesafe) + "/" + for _, model := range resp.Data { + name := strings.TrimPrefix(model.ID, prefix) + entry := TypesafeNativeModel{Name: name} + if known, ok := catalog[name]; ok { + entry.Description = known.Description + entry.ReleaseDate = known.ReleaseDate + } + native.Models = append(native.Models, entry) + } + return native +} diff --git a/core/providers/utils/decisionemulation.go b/core/providers/utils/decisionemulation.go new file mode 100644 index 00000000000..86352e7536a --- /dev/null +++ b/core/providers/utils/decisionemulation.go @@ -0,0 +1,708 @@ +package utils + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "math" + "sort" + "strconv" + "strings" + + "github.com/bytedance/sonic" + schemas "github.com/capsohq/bifrost/core/schemas" +) + +// scoreCriteriaLevels validates a score question's criteria and returns its +// ordered levels. The shared decision contract requires an ordered array of +// 2-10 level descriptions; missing, non-array, or out-of-range criteria are +// rejected so an emulated score is never generated or accepted without a rubric. +func scoreCriteriaLevels(criteria any) ([]any, error) { + var levels []any + switch typed := criteria.(type) { + case []any: + levels = typed + case []string: + for _, l := range typed { + levels = append(levels, l) + } + default: + return nil, fmt.Errorf("score criteria must be an ordered array of level descriptions") + } + if len(levels) < 2 || len(levels) > 10 { + return nil, fmt.Errorf("score criteria must have 2-10 levels, got %d", len(levels)) + } + return levels, nil +} + +// validateConfidence rejects a missing, non-finite, or out-of-range confidence. +// The emulation schema requires confidence in [0,1]; a model can still omit it or +// return null/NaN, which must not become a fabricated successful answer. +func validateConfidence(name string, c *float64) error { + if c == nil { + return fmt.Errorf("answer for %q is missing confidence", name) + } + if math.IsNaN(*c) || math.IsInf(*c, 0) || *c < 0 || *c > 1 { + return fmt.Errorf("confidence for %q is outside [0,1]: %v", name, *c) + } + return nil +} + +// normalizeProbabilities rejects missing options, invalid values, or a total far +// from 1. Small rounding errors are tolerated but never forwarded to callers. +func normalizeProbabilities(name string, probs map[string]float64, allowed map[string]bool) (map[string]float64, error) { + if len(probs) == 0 { + return nil, nil + } + var sum float64 + for k, v := range probs { + if allowed != nil && !allowed[k] { + return nil, fmt.Errorf("probabilities for %q contain unknown key %q", name, k) + } + if math.IsNaN(v) || math.IsInf(v, 0) || v < 0 || v > 1 { + return nil, fmt.Errorf("probability for %q key %q is outside [0,1]: %v", name, k, v) + } + sum += v + } + // The contract is a full distribution: every option or level carries a + // probability. A partial map that happens to sum to 1 (a one-hot answer + // over a subset) is incomplete, not valid. + for k := range allowed { + if _, ok := probs[k]; !ok { + return nil, fmt.Errorf("probabilities for %q are missing key %q; every option or level requires a probability", name, k) + } + } + // Decimal probabilities can land just outside the boundary in binary + // floating point (for example, 0.95 differs from 1 by slightly over 0.05). + if math.Abs(sum-1) > 0.05+1e-12 { + return nil, fmt.Errorf("probabilities for %q sum to %v, expected about 1", name, sum) + } + normalized := make(map[string]float64, len(probs)) + if math.Abs(sum-1) <= 1e-12 { + for k, v := range probs { + normalized[k] = v + } + return normalized, nil + } + for k, v := range probs { + normalized[k] = v / sum + } + return normalized, nil +} + +// Decision emulation lets any tool-capable chat model answer a decision request. +// The question set is encoded as a single function tool whose flat result object +// carries one property per question; the model fills value + confidence (and a +// probability distribution for choice). ParseDecisionAnswers maps the tool-call +// arguments back to the neutral DecisionAnswer shape, validating each value +// against its question kind the way the typesafe provider validates native +// answers (core/providers/typesafe/decision.go). + +// DecisionToolName is the synthetic function tool the model is forced to call. +const DecisionToolName = "emit_decision" + +// numberSchema builds a JSON-schema fragment for a bounded/unbounded number. +func numberSchema(desc string, min, max *float64) map[string]interface{} { + s := map[string]interface{}{"type": "number"} + if desc != "" { + s["description"] = desc + } + if min != nil { + s["minimum"] = *min + } + if max != nil { + s["maximum"] = *max + } + return s +} + +// confidenceSchema is the shared 0..1 confidence field on every answer. +func confidenceSchema() map[string]interface{} { + zero, one := 0.0, 1.0 + return numberSchema("Your confidence in this answer, from 0 to 1.", &zero, &one) +} + +// scoreIndexKeys returns the string level indexes "0".."n-1" of a score +// question - the probability-map keys shared by the schema and the parser. +func scoreIndexKeys(n int) []string { + keys := make([]string, n) + for i := range keys { + keys[i] = strconv.Itoa(i) + } + return keys +} + +// probabilitiesSchema builds a closed probability-distribution object: every +// allowed key (choice option or score index) is an enumerated required +// property bounded to [0,1], and additionalProperties is false, so a +// schema-enforcing provider rejects stray keys and out-of-range values before +// the parser ever sees them. The ~1 total is validated parser-side only - the +// schema cannot express an approximate sum. +func probabilitiesSchema(desc string, keys []string) map[string]interface{} { + zero, one := 0.0, 1.0 + keyProps := make(map[string]interface{}, len(keys)) + for _, key := range keys { + keyProps[key] = numberSchema("", &zero, &one) + } + return map[string]interface{}{ + "type": "object", + "description": desc, + "properties": keyProps, + "required": keys, + "additionalProperties": false, + } +} + +// instructionsText renders a question's instructions (string or structured) into +// a description string for the schema. +func instructionsText(instructions interface{}) string { + if instructions == nil { + return "" + } + return renderStructuredText(instructions) +} + +// renderStructuredText renders a criteria or instructions value for prompt +// text: strings pass through, structured values serialize with sorted keys so +// the emitted schema stays byte-stable, nil renders empty. +func renderStructuredText(value any) string { + if value == nil { + return "" + } + if s, ok := value.(string); ok { + return s + } + if raw, err := MarshalSorted(value); err == nil { + return string(raw) + } + return "" +} + +// choiceOptions extracts the ordered option keys from a choice question's +// criteria (a map of option -> description). Sorted for deterministic schemas. +// Descriptions may be strings, structured values (rendered as JSON), or null +// (rendered empty), matching the criteria shapes the typesafe API accepts. +func choiceOptions(criteria interface{}) ([]string, map[string]string, error) { + descs := map[string]string{} + switch typed := criteria.(type) { + case map[string]string: + for k, v := range typed { + descs[k] = v + } + case map[string]any: + for k, v := range typed { + descs[k] = renderStructuredText(v) + } + default: + return nil, nil, fmt.Errorf("choice criteria must be a map of options") + } + // A choice with no options would emit enum: [] and make every answer + // unparseable; fail locally before any model is called. + if len(descs) == 0 { + return nil, nil, fmt.Errorf("choice criteria requires at least one option") + } + opts := make([]string, 0, len(descs)) + for k := range descs { + opts = append(opts, k) + } + sort.Strings(opts) + return opts, descs, nil +} + +// DecisionScoreLevelCount returns how many levels a score question's criteria define, +// or an error when they are not a 2-10 level rubric. +func DecisionScoreLevelCount(criteria interface{}) (int, error) { + levels, err := scoreCriteriaLevels(criteria) + return len(levels), err +} + +// DecisionChoiceOptions returns a choice question's allowed options, sorted, or an error +// when its criteria name none. +func DecisionChoiceOptions(criteria interface{}) ([]string, error) { + opts, _, err := choiceOptions(criteria) + return opts, err +} + +// scoreLevels renders a score question's ordered criteria into a description. +func scoreLevels(criteria interface{}) string { + var levels []interface{} + switch typed := criteria.(type) { + case []interface{}: + levels = typed + case []string: + for _, l := range typed { + levels = append(levels, l) + } + default: + return "" + } + var out strings.Builder + out.WriteString("Score levels (index -> meaning): ") + for i, l := range levels { + if i > 0 { + out.WriteString(", ") + } + fmt.Fprintf(&out, "%d=%s", i, renderStructuredText(l)) + } + return out.String() +} + +// noulCriteriaText renders the optional true/false rubric descriptions for +// the noul value description, skipping absent keys. Like the choice and score +// rubrics, these are part of the question and must reach the model. +func noulCriteriaText(criteria any) string { + var m map[string]any + switch typed := criteria.(type) { + case map[string]string: + m = make(map[string]any, len(typed)) + for k, v := range typed { + m[k] = v + } + case map[string]any: + m = typed + default: + return "" + } + var out strings.Builder + for _, key := range []string{"true", "false"} { + desc := renderStructuredText(m[key]) + if desc == "" { + continue + } + if out.Len() == 0 { + out.WriteString(" Meaning (answer: meaning): ") + } else { + out.WriteString(", ") + } + out.WriteString(key) + out.WriteByte('=') + out.WriteString(desc) + } + return out.String() +} + +// choiceOptionsText renders the option rubric descriptions for the choice +// property description, skipping options with no description. The rubric is +// part of the question; the model must see it, not only the enum keys. +func choiceOptionsText(opts []string, descs map[string]string) string { + var out strings.Builder + for _, opt := range opts { + desc := descs[opt] + if desc == "" { + continue + } + if out.Len() == 0 { + out.WriteString(" Options (option: meaning): ") + } else { + out.WriteString(", ") + } + out.WriteString(opt) + out.WriteByte('=') + out.WriteString(desc) + } + return out.String() +} + +// scoreLegend builds the level index -> description legend from score criteria, +// so the emulated answer carries the same legend a native provider would: each +// level's description echoed verbatim (string, object, or array). +func scoreLegend(criteria interface{}) map[string]any { + var levels []interface{} + switch typed := criteria.(type) { + case []interface{}: + levels = typed + case []string: + for _, l := range typed { + levels = append(levels, l) + } + default: + return nil + } + if len(levels) == 0 { + return nil + } + legend := make(map[string]any, len(levels)) + for i, l := range levels { + legend[fmt.Sprintf("%d", i)] = l + } + return legend +} + +// BuildDecisionSchema builds the tool parameters: an object with one nested +// object property per question (value/choice + confidence, plus probabilities +// for choice). Property names are the question identifiers. +func BuildDecisionSchema(questions map[string]schemas.DecisionQuestion) (*schemas.ToolFunctionParameters, error) { + if len(questions) == 0 { + return nil, fmt.Errorf("decision emulation requires at least one question") + } + names := make([]string, 0, len(questions)) + for name := range questions { + names = append(names, name) + } + sort.Strings(names) + + props := schemas.NewOrderedMapWithCapacity(len(names)) + zero, one := 0.0, 1.0 + for _, name := range names { + q := questions[name] + desc := instructionsText(q.Instructions) + var nested map[string]interface{} + switch q.Kind { + case schemas.DecisionKindNoul: + // The native noul answer is {type, noul} only - the value near 0.5 + // is the uncertainty signal - so confidence is an optional extra + // the model may volunteer, never a required field. + nested = map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "value": numberSchema("Probability from 0 to 1. "+desc+noulCriteriaText(q.Criteria), &zero, &one), + "confidence": confidenceSchema(), + }, + "required": []string{"value"}, + "additionalProperties": false, + } + case schemas.DecisionKindChoice: + opts, descs, err := choiceOptions(q.Criteria) + if err != nil { + return nil, fmt.Errorf("question %q: %w", name, err) + } + nested = map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "choice": map[string]interface{}{"type": "string", "enum": opts, "description": desc + " Select an option with the highest probability." + choiceOptionsText(opts, descs)}, + "confidence": confidenceSchema(), + "probabilities": probabilitiesSchema("Required. Probability for each option; values must sum to 1.", opts), + }, + "required": []string{"choice", "confidence", "probabilities"}, + "additionalProperties": false, + } + case schemas.DecisionKindScore: + scoreLevelsList, err := scoreCriteriaLevels(q.Criteria) + if err != nil { + return nil, fmt.Errorf("question %q: %w", name, err) + } + // The score is the probability-weighted average of the level + // indexes, so it is bounded by the first and last index. + maxLevel := float64(len(scoreLevelsList) - 1) + nested = map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "value": numberSchema(desc+" "+scoreLevels(q.Criteria), &zero, &maxLevel), + "confidence": confidenceSchema(), + "probabilities": probabilitiesSchema( + "Required. Probability for each level index (\"0\", \"1\", ...); values must sum to 1.", + scoreIndexKeys(len(scoreLevelsList)), + ), + }, + "required": []string{"value", "confidence", "probabilities"}, + "additionalProperties": false, + } + default: + return nil, fmt.Errorf("question %q has unsupported kind %q", name, q.Kind) + } + props.Set(name, nested) + } + + return &schemas.ToolFunctionParameters{ + Type: "object", + Properties: props, + Required: names, + }, nil +} + +// DecisionToolDescription frames the forced function call for the model. +const DecisionToolDescription = "Emit the decision for every question. Fill each field from the given state." + +// BuildDecisionResponsesTool wraps the decision schema in a forced-callable +// Responses-API function tool. Decision emulation runs through provider.Responses +// (the richest interface every provider implements - natively on openai/anthropic/ +// gemini, via chat translation elsewhere), so the tool is expressed in the +// Responses tool shape rather than the chat one. +func BuildDecisionResponsesTool(questions map[string]schemas.DecisionQuestion) (*schemas.ResponsesTool, error) { + params, err := BuildDecisionSchema(questions) + if err != nil { + return nil, err + } + name := DecisionToolName + desc := DecisionToolDescription + return &schemas.ResponsesTool{ + Type: schemas.ResponsesToolTypeFunction, + Name: &name, + Description: &desc, + ResponsesToolFunction: &schemas.ResponsesToolFunction{ + Parameters: params, + }, + }, nil +} + +// emulatedAnswer is the per-question object the model returns. +type emulatedAnswer struct { + Value interface{} `json:"value"` + Choice *string `json:"choice"` + Confidence *float64 `json:"confidence"` + Probabilities map[string]float64 `json:"probabilities"` +} + +// emulatedAnswerFields are the keys an emulatedAnswer object may carry. +var emulatedAnswerFields = map[string]bool{"value": true, "choice": true, "confidence": true, "probabilities": true} + +// leakedParameterTagPrefix opens Anthropic's XML tool-call parameter syntax. +const leakedParameterTagPrefix = `billing", "confidence": 1.0, "probabilities": {...}, "urgency": ...} +// +// leaking the XML parameter syntax into the first field and repeating the rest +// as duplicate top-level keys. Keys are read in order with a streaming decoder +// because a map decode would keep only the last duplicate. Each answer field +// attaches to the question that precedes it; anything else (a field before any +// question, a repeated field, an unknown key, a string answer without the tag) +// fails recovery. The result still goes through the full answer validation, so +// recovery can only restore answers the model gave, never invent one. +func recoverFlattenedDecisionArguments(argumentsJSON []byte, questions map[string]schemas.DecisionQuestion) (map[string]emulatedAnswer, bool) { + dec := json.NewDecoder(bytes.NewReader(argumentsJSON)) + if tok, err := dec.Token(); err != nil || tok != json.Delim('{') { + return nil, false + } + fieldsByQuestion := make(map[string]map[string]json.RawMessage, len(questions)) + var current map[string]json.RawMessage + for dec.More() { + tok, err := dec.Token() + key, isKey := tok.(string) + if err != nil || !isKey { + return nil, false + } + var value json.RawMessage + if err := dec.Decode(&value); err != nil { + return nil, false + } + if _, isQuestion := questions[key]; isQuestion { + if _, seen := fieldsByQuestion[key]; seen { + return nil, false + } + if len(value) > 0 && value[0] == '{' { + // A well-formed answer object among flattened ones is complete + // on its own; no sibling field may attach to it. + var fields map[string]json.RawMessage + if err := sonic.Unmarshal(value, &fields); err != nil { + return nil, false + } + fieldsByQuestion[key] = fields + current = nil + continue + } + field, fieldValue, ok := parseLeakedParameterTag(value) + if !ok { + return nil, false + } + current = map[string]json.RawMessage{field: fieldValue} + fieldsByQuestion[key] = current + continue + } + if current == nil || !emulatedAnswerFields[key] { + return nil, false + } + if _, repeated := current[key]; repeated { + return nil, false + } + current[key] = value + } + if tok, err := dec.Token(); err != nil || tok != json.Delim('}') { + return nil, false + } + if _, err := dec.Token(); err != io.EOF { + return nil, false + } + + recovered := make(map[string]emulatedAnswer, len(fieldsByQuestion)) + for name, fields := range fieldsByQuestion { + encoded, err := MarshalSorted(fields) + if err != nil { + return nil, false + } + var answer emulatedAnswer + if err := sonic.Unmarshal(encoded, &answer); err != nil { + return nil, false + } + recovered[name] = answer + } + return recovered, true +} + +// parseLeakedParameterTag reads a `value` string (the +// closing tag is optional, and surrounding whitespace is ignored) into the +// answer field it names and that field's JSON value. A choice stays a string; +// any other value must itself be valid JSON. +func parseLeakedParameterTag(value json.RawMessage) (string, json.RawMessage, bool) { + var s string + if err := sonic.Unmarshal(value, &s); err != nil { + return "", nil, false + } + s = strings.TrimSpace(s) + if !strings.HasPrefix(s, leakedParameterTagPrefix) { + return "", nil, false + } + rest := s[len(leakedParameterTagPrefix):] + end := strings.Index(rest, `">`) + if end < 0 { + return "", nil, false + } + field := rest[:end] + if !emulatedAnswerFields[field] { + return "", nil, false + } + text := strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(rest[end+2:]), "")) + if field == "choice" { + encoded, err := sonic.Marshal(text) + if err != nil { + return "", nil, false + } + return field, encoded, true + } + if !json.Valid([]byte(text)) { + return "", nil, false + } + return field, json.RawMessage(text), true +} + +// ParseDecisionAnswers decodes the tool-call arguments and validates each answer +// against its question kind, producing the neutral DecisionAnswer map. Every +// requested question must be answered; a missing, wrong-typed, or out-of-range +// answer is an error so the request falls through to the next fallback rather +// than returning a fabricated result. +func ParseDecisionAnswers(argumentsJSON []byte, questions map[string]schemas.DecisionQuestion) (map[string]schemas.DecisionAnswer, error) { + var raw map[string]emulatedAnswer + if err := sonic.Unmarshal(argumentsJSON, &raw); err != nil { + recovered, ok := recoverFlattenedDecisionArguments(argumentsJSON, questions) + if !ok { + return nil, fmt.Errorf("decision tool-call arguments are not a JSON object: %w", err) + } + raw = recovered + } + + answers := make(map[string]schemas.DecisionAnswer, len(questions)) + for name, q := range questions { + got, ok := raw[name] + if !ok { + return nil, fmt.Errorf("model returned no answer for question %q", name) + } + // Confidence is required on choice and score answers; a noul answer + // carries none natively, so there it is validated only when the model + // volunteers one. + if q.Kind != schemas.DecisionKindNoul || got.Confidence != nil { + if err := validateConfidence(name, got.Confidence); err != nil { + return nil, err + } + } + answer := schemas.DecisionAnswer{Kind: q.Kind, Confidence: got.Confidence} + switch q.Kind { + case schemas.DecisionKindNoul: + f, ok := toFloat(got.Value) + if !ok { + return nil, fmt.Errorf("noul answer for %q is not a number", name) + } + if f < 0 || f > 1 { + return nil, fmt.Errorf("noul answer for %q is outside [0,1]: %v", name, f) + } + answer.Value = f + case schemas.DecisionKindChoice: + if got.Choice == nil { + return nil, fmt.Errorf("choice answer for %q carries no choice", name) + } + opts, _, err := choiceOptions(q.Criteria) + if err != nil { + return nil, fmt.Errorf("question %q: %w", name, err) + } + if !containsString(opts, *got.Choice) { + return nil, fmt.Errorf("choice answer %q for %q is not an allowed option", *got.Choice, name) + } + // Probabilities are a required field on native choice answers; an + // answer without them would fail strict SDK clients downstream. + if len(got.Probabilities) == 0 { + return nil, fmt.Errorf("choice answer for %q is missing probabilities", name) + } + allowed := make(map[string]bool, len(opts)) + for _, o := range opts { + allowed[o] = true + } + probabilities, err := normalizeProbabilities(name, got.Probabilities, allowed) + if err != nil { + return nil, err + } + for option, probability := range probabilities { + if probability > probabilities[*got.Choice] { + return nil, fmt.Errorf("choice answer %q for %q does not have the highest probability (option %q is higher)", *got.Choice, name, option) + } + } + answer.Value = *got.Choice + answer.Probabilities = probabilities + case schemas.DecisionKindScore: + levels, err := scoreCriteriaLevels(q.Criteria) + if err != nil { + return nil, fmt.Errorf("score question %q: %w", name, err) + } + if _, ok := toFloat(got.Value); !ok { + return nil, fmt.Errorf("score answer for %q is not a number", name) + } + // Probabilities are a required field on native score answers; an + // answer without them would fail strict SDK clients downstream. + if len(got.Probabilities) == 0 { + return nil, fmt.Errorf("score answer for %q is missing probabilities", name) + } + legend := scoreLegend(q.Criteria) + allowed := make(map[string]bool, len(levels)) + for i := range levels { + allowed[fmt.Sprintf("%d", i)] = true + } + probabilities, err := normalizeProbabilities(name, got.Probabilities, allowed) + if err != nil { + return nil, err + } + // The native contract defines score as the probability-weighted + // average of the level indexes ("Expected score ... may fall + // between integer levels"), so derive it from the validated + // distribution; the model's separately supplied value (checked + // above only for being numeric) cannot contradict its own + // probabilities. Bounded 0..len(levels)-1 by construction. + var derived float64 + for key, p := range probabilities { + index, _ := strconv.Atoi(key) // keys already validated against the "0".."n-1" set + derived += float64(index) * p + } + answer.Value = derived + answer.Legend = legend + answer.Probabilities = probabilities + default: + return nil, fmt.Errorf("question %q has unsupported kind %q", name, q.Kind) + } + answers[name] = answer + } + return answers, nil +} + +func toFloat(v interface{}) (float64, bool) { + switch n := v.(type) { + case float64: + return n, true + case float32: + return float64(n), true + case int: + return float64(n), true + case int64: + return float64(n), true + } + return 0, false +} + +func containsString(ss []string, s string) bool { + for _, x := range ss { + if x == s { + return true + } + } + return false +} diff --git a/core/providers/utils/decisionemulation_test.go b/core/providers/utils/decisionemulation_test.go new file mode 100644 index 00000000000..7a0169f666e --- /dev/null +++ b/core/providers/utils/decisionemulation_test.go @@ -0,0 +1,703 @@ +package utils + +import ( + "math" + "reflect" + "strconv" + "strings" + "testing" + + "github.com/bytedance/sonic" + schemas "github.com/capsohq/bifrost/core/schemas" +) + +func mixedQuestions() map[string]schemas.DecisionQuestion { + return map[string]schemas.DecisionQuestion{ + "is_frustrated": {Kind: schemas.DecisionKindNoul, Instructions: "Is the customer frustrated?"}, + "category": { + Kind: schemas.DecisionKindChoice, + Instructions: "Pick the ticket category", + Criteria: map[string]interface{}{"billing": "money", "bug": "defects", "other": "else"}, + }, + "urgency": { + Kind: schemas.DecisionKindScore, + Instructions: "How urgent?", + Criteria: []interface{}{"low", "medium", "high"}, + }, + } +} + +func TestBuildDecisionSchemaShape(t *testing.T) { + params, err := BuildDecisionSchema(mixedQuestions()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + // Marshal and re-read so we assert on the emitted JSON schema. + raw, err := sonic.Marshal(params) + if err != nil { + t.Fatalf("marshal: %v", err) + } + var schema map[string]interface{} + if err := sonic.Unmarshal(raw, &schema); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if schema["type"] != "object" { + t.Errorf("top type = %v", schema["type"]) + } + props := schema["properties"].(map[string]interface{}) + for _, name := range []string{"is_frustrated", "category", "urgency"} { + if _, ok := props[name]; !ok { + t.Fatalf("missing property %q", name) + } + } + + noul := props["is_frustrated"].(map[string]interface{})["properties"].(map[string]interface{}) + if _, ok := noul["value"]; !ok { + t.Error("noul missing value") + } + // A native noul answer is {type, noul} only - the value near 0.5 is the + // uncertainty signal - so confidence stays an optional extra the model may + // volunteer, never a required field. + if _, ok := noul["confidence"]; !ok { + t.Error("noul confidence should remain available as an optional field") + } + noulRequired := props["is_frustrated"].(map[string]interface{})["required"].([]interface{}) + if len(noulRequired) != 1 || noulRequired[0] != "value" { + t.Errorf("noul required = %v, want [value] only", noulRequired) + } + + choice := props["category"].(map[string]interface{})["properties"].(map[string]interface{}) + enum := choice["choice"].(map[string]interface{})["enum"].([]interface{}) + if len(enum) != 3 { + t.Errorf("choice enum = %v", enum) + } + if _, ok := choice["probabilities"]; !ok { + t.Error("choice missing probabilities") + } + // Probabilities are a required field on native choice and score answers, + // so the tool schema must require them from the emulating model too. + choiceRequired := props["category"].(map[string]interface{})["required"].([]interface{}) + if !containsAnyString(choiceRequired, "probabilities") { + t.Errorf("choice required = %v, want probabilities included", choiceRequired) + } + + score := props["urgency"].(map[string]interface{})["properties"].(map[string]interface{}) + if !strings.Contains(score["value"].(map[string]interface{})["description"].(string), "high") { + t.Error("score value description should mention the levels") + } + scoreRequired := props["urgency"].(map[string]interface{})["required"].([]interface{}) + if !containsAnyString(scoreRequired, "probabilities") { + t.Errorf("score required = %v, want probabilities included", scoreRequired) + } +} + +// containsAnyString reports whether a decoded []interface{} contains s. +func containsAnyString(list []interface{}, s string) bool { + for _, v := range list { + if v == s { + return true + } + } + return false +} + +// structuredQuestions carries the spec's richer criteria shapes: object and +// null choice descriptions, and object levels in a score array. +func structuredQuestions() map[string]schemas.DecisionQuestion { + return map[string]schemas.DecisionQuestion{ + "approve": { + Kind: schemas.DecisionKindNoul, + Instructions: "Approve the refund?", + Criteria: map[string]any{ + "true": map[string]any{"meaning": "refund it"}, + "false": []any{"deny it", "no evidence"}, + }, + }, + "category": { + Kind: schemas.DecisionKindChoice, + Instructions: "Pick the ticket category", + Criteria: map[string]any{ + "billing": map[string]any{"rubric": "money issues"}, + "bug": []any{"crash", "defect"}, + "support": "service questions", + "other": nil, + }, + }, + "urgency": { + Kind: schemas.DecisionKindScore, + Instructions: "How urgent?", + Criteria: []any{ + "low", + map[string]any{"examples": []any{"outage"}, "level": "high"}, + []any{"critical", "churn risk"}, + }, + }, + } +} + +func TestBuildDecisionSchemaStructuredCriteria(t *testing.T) { + params, err := BuildDecisionSchema(structuredQuestions()) + if err != nil { + t.Fatalf("structured criteria rejected: %v", err) + } + raw, err := sonic.Marshal(params) + if err != nil { + t.Fatalf("marshal: %v", err) + } + var schema map[string]any + if err := sonic.Unmarshal(raw, &schema); err != nil { + t.Fatalf("unmarshal: %v", err) + } + props := schema["properties"].(map[string]any) + + noulDesc := props["approve"].(map[string]any)["properties"].(map[string]any)["value"].(map[string]any)["description"].(string) + // The true/false rubric descriptions must reach the model: objects and + // arrays render as sorted JSON. + if !strings.Contains(noulDesc, `true={"meaning":"refund it"}`) || !strings.Contains(noulDesc, `false=["deny it","no evidence"]`) { + t.Errorf("noul description missing true/false criteria: %q", noulDesc) + } + + choice := props["category"].(map[string]any)["properties"].(map[string]any)["choice"].(map[string]any) + enum := choice["enum"].([]any) + if len(enum) != 4 { + t.Fatalf("choice enum should keep all 4 options including the null-described one: %v", enum) + } + choiceDesc := choice["description"].(string) + // Option rubric descriptions must reach the model: strings verbatim, + // objects and arrays as JSON, null options simply skipped. + for _, want := range []string{`{"rubric":"money issues"}`, `["crash","defect"]`, "service questions"} { + if !strings.Contains(choiceDesc, want) { + t.Errorf("choice description missing option rubric %q: %q", want, choiceDesc) + } + } + + scoreDesc := props["urgency"].(map[string]any)["properties"].(map[string]any)["value"].(map[string]any)["description"].(string) + for _, want := range []string{"0=low", `{"examples":["outage"],"level":"high"}`, `["critical","churn risk"]`} { + if !strings.Contains(scoreDesc, want) { + t.Errorf("score description missing level %q: %q", want, scoreDesc) + } + } + if strings.Contains(scoreDesc, "map[") { + t.Errorf("score description leaks Go map syntax: %q", scoreDesc) + } +} + +func TestParseDecisionAnswersStructuredScoreLegend(t *testing.T) { + args := `{ + "approve": {"value": 0.9, "confidence": 0.8}, + "category": {"choice": "other", "confidence": 0.7, "probabilities": {"billing": 0.1, "bug": 0.1, "support": 0.1, "other": 0.7}}, + "urgency": {"value": 1, "confidence": 0.6, "probabilities": {"0": 0.1, "1": 0.8, "2": 0.1}} + }` + answers, err := ParseDecisionAnswers([]byte(args), structuredQuestions()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if answers["category"].Value.(string) != "other" { + t.Errorf("null-described choice option rejected: %+v", answers["category"]) + } + // The legend echoes each level's description verbatim, matching the native + // API's shape (values are strings, objects, or arrays - not stringified). + legend := answers["urgency"].Legend + if legend["0"] != "low" { + t.Errorf("string legend level should pass through verbatim, got %#v", legend["0"]) + } + if !reflect.DeepEqual(legend["1"], map[string]any{"examples": []any{"outage"}, "level": "high"}) { + t.Errorf("object legend level should echo verbatim, got %#v", legend["1"]) + } + if !reflect.DeepEqual(legend["2"], []any{"critical", "churn risk"}) { + t.Errorf("array legend level should echo verbatim, got %#v", legend["2"]) + } +} + +// TestBuildDecisionSchemaTypedStringMapCriteria covers the map[string]string +// input branch Go SDK callers use for noul and choice criteria. +func TestBuildDecisionSchemaTypedStringMapCriteria(t *testing.T) { + questions := map[string]schemas.DecisionQuestion{ + "approve": { + Kind: schemas.DecisionKindNoul, + Instructions: "Approve?", + Criteria: map[string]string{"true": "grant it", "false": "deny it"}, + }, + "bucket": { + Kind: schemas.DecisionKindChoice, + Instructions: "Bucket?", + Criteria: map[string]string{"a": "first bucket", "b": "second bucket"}, + }, + } + params, err := BuildDecisionSchema(questions) + if err != nil { + t.Fatalf("typed string map criteria rejected: %v", err) + } + raw, err := sonic.Marshal(params) + if err != nil { + t.Fatalf("marshal: %v", err) + } + var schema map[string]any + if err := sonic.Unmarshal(raw, &schema); err != nil { + t.Fatalf("unmarshal: %v", err) + } + props := schema["properties"].(map[string]any) + + noulDesc := props["approve"].(map[string]any)["properties"].(map[string]any)["value"].(map[string]any)["description"].(string) + if !strings.Contains(noulDesc, "true=grant it") || !strings.Contains(noulDesc, "false=deny it") { + t.Errorf("noul description missing typed-map criteria: %q", noulDesc) + } + choiceDesc := props["bucket"].(map[string]any)["properties"].(map[string]any)["choice"].(map[string]any)["description"].(string) + if !strings.Contains(choiceDesc, "a=first bucket") || !strings.Contains(choiceDesc, "b=second bucket") { + t.Errorf("choice description missing typed-map rubrics: %q", choiceDesc) + } +} + +// TestBuildDecisionSchemaProbabilitiesClosed pins the probability sub-schemas: +// the actual option names / level indexes as enumerated required properties, +// additionalProperties: false, and each probability bounded to [0,1] - so a +// schema-enforcing provider rejects stray keys and out-of-range values before +// the parser ever sees them. +func TestBuildDecisionSchemaProbabilitiesClosed(t *testing.T) { + params, err := BuildDecisionSchema(mixedQuestions()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + raw, err := sonic.Marshal(params) + if err != nil { + t.Fatalf("marshal: %v", err) + } + var schema map[string]any + if err := sonic.Unmarshal(raw, &schema); err != nil { + t.Fatalf("unmarshal: %v", err) + } + props := schema["properties"].(map[string]any) + + check := func(t *testing.T, question string, wantKeys []string) { + t.Helper() + probs, ok := props[question].(map[string]any)["properties"].(map[string]any)["probabilities"].(map[string]any) + if !ok { + t.Fatalf("question %q has no probabilities schema", question) + } + if ap, ok := probs["additionalProperties"].(bool); !ok || ap { + t.Errorf("%q probabilities additionalProperties = %v, want false", question, probs["additionalProperties"]) + } + keyProps, ok := probs["properties"].(map[string]any) + if !ok || len(keyProps) != len(wantKeys) { + t.Fatalf("%q probabilities properties = %v, want keys %v", question, probs["properties"], wantKeys) + } + for _, key := range wantKeys { + entry, ok := keyProps[key].(map[string]any) + if !ok { + t.Errorf("%q probabilities missing key %q", question, key) + continue + } + if entry["minimum"] != float64(0) || entry["maximum"] != float64(1) { + t.Errorf("%q probability %q not bounded [0,1]: %v", question, key, entry) + } + } + required, ok := probs["required"].([]any) + if !ok || len(required) != len(wantKeys) { + t.Errorf("%q probabilities required = %v, want all keys", question, probs["required"]) + } + } + + check(t, "category", []string{"billing", "bug", "other"}) + check(t, "urgency", []string{"0", "1", "2"}) +} + +func TestBuildDecisionSchemaEmpty(t *testing.T) { + if _, err := BuildDecisionSchema(map[string]schemas.DecisionQuestion{}); err == nil { + t.Fatal("expected error for empty questions") + } +} + +func TestBuildDecisionSchemaRejectsEmptyChoiceCriteria(t *testing.T) { + // An empty criteria map would emit enum: [] and make every answer fail + // downstream; it must be a local error before any model is called. + questions := map[string]schemas.DecisionQuestion{ + "pick": {Kind: schemas.DecisionKindChoice, Instructions: "d", Criteria: map[string]any{}}, + } + if _, err := BuildDecisionSchema(questions); err == nil || !strings.Contains(err.Error(), "at least one option") { + t.Fatalf("empty choice criteria must be rejected, got %v", err) + } + args := `{"pick": {"choice": "anything", "confidence": 0.5, "probabilities": {"anything": 1}}}` + if _, err := ParseDecisionAnswers([]byte(args), questions); err == nil || !strings.Contains(err.Error(), "at least one option") { + t.Fatalf("empty choice criteria must be rejected in the parser too, got %v", err) + } +} + +func TestParseDecisionAnswersValid(t *testing.T) { + args := `{ + "is_frustrated": {"value": 0.9, "confidence": 0.8}, + "category": {"choice": "billing", "confidence": 0.7, "probabilities": {"billing": 0.7, "bug": 0.2, "other": 0.1}}, + "urgency": {"value": 2, "confidence": 0.6, "probabilities": {"0": 0.1, "1": 0.1, "2": 0.8}} + }` + answers, err := ParseDecisionAnswers([]byte(args), mixedQuestions()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if answers["is_frustrated"].Kind != schemas.DecisionKindNoul || answers["is_frustrated"].Value.(float64) != 0.9 { + t.Errorf("noul = %+v", answers["is_frustrated"]) + } + if *answers["is_frustrated"].Confidence != 0.8 { + t.Errorf("noul confidence = %v", answers["is_frustrated"].Confidence) + } + if answers["category"].Value.(string) != "billing" { + t.Errorf("choice = %+v", answers["category"]) + } + if answers["category"].Probabilities["billing"] != 0.7 { + t.Errorf("choice probabilities = %+v", answers["category"].Probabilities) + } + // Score is derived from the distribution (0*0.1 + 1*0.1 + 2*0.8 = 1.7), + // not taken from the supplied value. + if math.Abs(answers["urgency"].Value.(float64)-1.7) > 1e-9 { + t.Errorf("score = %+v", answers["urgency"]) + } + if answers["urgency"].Legend["2"] != "high" { + t.Errorf("score legend = %+v", answers["urgency"].Legend) + } +} + +func TestParseDecisionAnswersNoulConfidenceOptional(t *testing.T) { + // The schema no longer requires confidence on noul (the native answer is + // {type, noul} only), so a model omitting it must parse fine; one it + // volunteers is still kept and validated. + args := `{ + "is_frustrated": {"value": 0.9}, + "category": {"choice": "billing", "confidence": 0.7, "probabilities": {"billing": 0.8, "bug": 0.1, "other": 0.1}}, + "urgency": {"value": 1, "confidence": 0.5, "probabilities": {"0": 0.1, "1": 0.8, "2": 0.1}} + }` + answers, err := ParseDecisionAnswers([]byte(args), mixedQuestions()) + if err != nil { + t.Fatalf("noul answer without confidence rejected: %v", err) + } + if answers["is_frustrated"].Confidence != nil { + t.Errorf("omitted noul confidence should stay nil, got %v", *answers["is_frustrated"].Confidence) + } + + // A volunteered but invalid noul confidence is still rejected. + bad := `{ + "is_frustrated": {"value": 0.9, "confidence": 1.4}, + "category": {"choice": "billing", "confidence": 0.7, "probabilities": {"billing": 0.8, "bug": 0.1, "other": 0.1}}, + "urgency": {"value": 1, "confidence": 0.5, "probabilities": {"0": 0.1, "1": 0.8, "2": 0.1}} + }` + if _, err := ParseDecisionAnswers([]byte(bad), mixedQuestions()); err == nil { + t.Fatal("volunteered out-of-range noul confidence must be rejected") + } +} + +func TestParseDecisionAnswersRequiresProbabilities(t *testing.T) { + // Probabilities are required on native choice and score answers, so an + // emulated answer omitting them must be rejected - a response without them + // would fail strict SDK clients - while noul needs none. + missingChoice := `{ + "is_frustrated": {"value": 0.9}, + "category": {"choice": "billing", "confidence": 0.7}, + "urgency": {"value": 1, "confidence": 0.5, "probabilities": {"0": 0.1, "1": 0.8, "2": 0.1}} + }` + if _, err := ParseDecisionAnswers([]byte(missingChoice), mixedQuestions()); err == nil || !strings.Contains(err.Error(), "probabilities") { + t.Fatalf("choice answer without probabilities must be rejected, got %v", err) + } + + missingScore := `{ + "is_frustrated": {"value": 0.9}, + "category": {"choice": "billing", "confidence": 0.7, "probabilities": {"billing": 0.8, "bug": 0.1, "other": 0.1}}, + "urgency": {"value": 1, "confidence": 0.5} + }` + if _, err := ParseDecisionAnswers([]byte(missingScore), mixedQuestions()); err == nil || !strings.Contains(err.Error(), "probabilities") { + t.Fatalf("score answer without probabilities must be rejected, got %v", err) + } + + // A partial distribution that happens to sum to 1 is still incomplete: + // the contract is a probability for EVERY option and level. + partialChoice := `{ + "is_frustrated": {"value": 0.9}, + "category": {"choice": "billing", "confidence": 0.7, "probabilities": {"billing": 1}}, + "urgency": {"value": 1, "confidence": 0.5, "probabilities": {"0": 0.1, "1": 0.8, "2": 0.1}} + }` + if _, err := ParseDecisionAnswers([]byte(partialChoice), mixedQuestions()); err == nil || !strings.Contains(err.Error(), "probabilities") { + t.Fatalf("one-hot partial choice distribution must be rejected, got %v", err) + } + + partialScore := `{ + "is_frustrated": {"value": 0.9}, + "category": {"choice": "billing", "confidence": 0.7, "probabilities": {"billing": 0.8, "bug": 0.1, "other": 0.1}}, + "urgency": {"value": 1, "confidence": 0.5, "probabilities": {"1": 1}} + }` + if _, err := ParseDecisionAnswers([]byte(partialScore), mixedQuestions()); err == nil || !strings.Contains(err.Error(), "probabilities") { + t.Fatalf("partial score distribution must be rejected, got %v", err) + } + + complete := `{ + "is_frustrated": {"value": 0.9}, + "category": {"choice": "billing", "confidence": 0.7, "probabilities": {"billing": 0.8, "bug": 0.1, "other": 0.1}}, + "urgency": {"value": 1, "confidence": 0.5, "probabilities": {"0": 0.1, "1": 0.8, "2": 0.1}} + }` + answers, err := ParseDecisionAnswers([]byte(complete), mixedQuestions()) + if err != nil { + t.Fatalf("complete answers rejected: %v", err) + } + if answers["urgency"].Probabilities["1"] != 0.8 { + t.Errorf("score probabilities lost: %+v", answers["urgency"].Probabilities) + } +} + +// TestParseDecisionAnswersDerivesScoreFromDistribution pins the documented +// score contract: "Expected score: the probability-weighted average of the +// rubric levels." The emulated answer's score is derived from the validated +// distribution, not taken from the model's separately supplied value. +func TestParseDecisionAnswersDerivesScoreFromDistribution(t *testing.T) { + args := `{ + "is_frustrated": {"value": 0.9}, + "category": {"choice": "billing", "confidence": 0.7, "probabilities": {"billing": 0.8, "bug": 0.1, "other": 0.1}}, + "urgency": {"value": 2, "confidence": 0.6, "probabilities": {"0": 0.1, "1": 0.1, "2": 0.8}} + }` + answers, err := ParseDecisionAnswers([]byte(args), mixedQuestions()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + // 0*0.1 + 1*0.1 + 2*0.8 = 1.7 - the supplied 2 does not survive. + got := answers["urgency"].Value.(float64) + if math.Abs(got-1.7) > 1e-9 { + t.Errorf("score = %v, want the probability-weighted mean 1.7", got) + } +} + +func TestParseDecisionAnswersChoiceMatchesHighestProbability(t *testing.T) { + questions := map[string]schemas.DecisionQuestion{ + "route": {Kind: schemas.DecisionKindChoice, Criteria: map[string]string{"a": "Alpha", "b": "Beta"}}, + } + wrong := `{"route":{"choice":"a","confidence":0.8,"probabilities":{"a":0.1,"b":0.9}}}` + if _, err := ParseDecisionAnswers([]byte(wrong), questions); err == nil || !strings.Contains(err.Error(), "highest probability") { + t.Fatalf("choice below another option must be rejected, got %v", err) + } + tied := `{"route":{"choice":"a","confidence":0.8,"probabilities":{"a":0.5,"b":0.5}}}` + if _, err := ParseDecisionAnswers([]byte(tied), questions); err != nil { + t.Fatalf("a choice tied for highest probability must be accepted: %v", err) + } +} + +func TestParseDecisionAnswersNormalizesNearOneDistributions(t *testing.T) { + questions := map[string]schemas.DecisionQuestion{ + "route": {Kind: schemas.DecisionKindChoice, Criteria: map[string]string{"a": "Alpha", "b": "Beta"}}, + "severity": {Kind: schemas.DecisionKindScore, Criteria: []string{"low", "medium", "high"}}, + } + args := `{"route":{"choice":"a","confidence":0.8,"probabilities":{"a":0.8,"b":0.19}},"severity":{"value":2,"confidence":0.8,"probabilities":{"0":0,"1":0.05,"2":0.99}}}` + answers, err := ParseDecisionAnswers([]byte(args), questions) + if err != nil { + t.Fatalf("near-one distributions should be accepted: %v", err) + } + for _, name := range []string{"route", "severity"} { + var sum float64 + for _, probability := range answers[name].Probabilities { + sum += probability + } + if math.Abs(sum-1) > 1e-9 { + t.Errorf("%s probabilities sum to %v, want 1", name, sum) + } + } + wantScore := (0.05 + 2*0.99) / 1.04 + if score := answers["severity"].Value.(float64); math.Abs(score-wantScore) > 1e-9 { + t.Errorf("score = %v, want normalized expected value %v", score, wantScore) + } + // Exactly 5% below one is within the documented tolerance despite binary + // floating-point rounding of decimal probabilities. + boundary := `{"route":{"choice":"a","confidence":0.8,"probabilities":{"a":0.9,"b":0.05}},"severity":{"value":2,"confidence":0.8,"probabilities":{"0":0,"1":0.05,"2":0.95}}}` + if _, err := ParseDecisionAnswers([]byte(boundary), questions); err != nil { + t.Fatalf("distribution at the tolerance boundary should be accepted: %v", err) + } + tooFar := `{"route":{"choice":"a","confidence":0.8,"probabilities":{"a":0.8,"b":0.1}},"severity":{"value":2,"confidence":0.8,"probabilities":{"0":0,"1":0.05,"2":0.95}}}` + if _, err := ParseDecisionAnswers([]byte(tooFar), questions); err == nil || !strings.Contains(err.Error(), "sum") { + t.Fatalf("distribution far from one must be rejected, got %v", err) + } +} + +func TestParseDecisionAnswersRejections(t *testing.T) { + q := mixedQuestions() + // Every fixture is valid EXCEPT for the one named defect, and each case + // asserts its expected error text - otherwise Go's randomized map + // iteration can surface a different validation error first and a + // regression in the named check would leave the case green. + cases := map[string]struct { + args string + wantSub string + }{ + "noul out of range": { + args: `{"is_frustrated":{"value":1.4},"category":{"choice":"billing","confidence":0.5,"probabilities":{"billing":0.8,"bug":0.1,"other":0.1}},"urgency":{"value":1,"confidence":0.5,"probabilities":{"0":0.1,"1":0.8,"2":0.1}}}`, + wantSub: "outside [0,1]", + }, + "unknown choice": { + args: `{"is_frustrated":{"value":0.5},"category":{"choice":"nope","confidence":0.5,"probabilities":{"billing":0.8,"bug":0.1,"other":0.1}},"urgency":{"value":1,"confidence":0.5,"probabilities":{"0":0.1,"1":0.8,"2":0.1}}}`, + wantSub: "not an allowed option", + }, + "missing question": { + args: `{"is_frustrated":{"value":0.5},"category":{"choice":"billing","confidence":0.5,"probabilities":{"billing":0.8,"bug":0.1,"other":0.1}}}`, + wantSub: "no answer for question", + }, + "noul not a number": { + args: `{"is_frustrated":{"value":"high"},"category":{"choice":"billing","confidence":0.5,"probabilities":{"billing":0.8,"bug":0.1,"other":0.1}},"urgency":{"value":1,"confidence":0.5,"probabilities":{"0":0.1,"1":0.8,"2":0.1}}}`, + wantSub: "not a number", + }, + "malformed json": { + args: `not json`, + wantSub: "not a JSON object", + }, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + _, err := ParseDecisionAnswers([]byte(tc.args), q) + if err == nil { + t.Fatalf("expected rejection for %q", name) + } + if !strings.Contains(err.Error(), tc.wantSub) { + t.Errorf("error %q does not contain %q - a different validation fired first", err.Error(), tc.wantSub) + } + }) + } +} + +func TestBuildDecisionToolName(t *testing.T) { + tool, err := BuildDecisionResponsesTool(mixedQuestions()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if tool.Type != schemas.ResponsesToolTypeFunction { + t.Errorf("tool type = %v", tool.Type) + } + if tool.Name == nil || *tool.Name != DecisionToolName { + t.Errorf("tool name = %v", tool.Name) + } + if tool.ResponsesToolFunction == nil || tool.ResponsesToolFunction.Parameters == nil { + t.Errorf("tool function params missing: %+v", tool) + } +} + +// flattenedQuestions mirrors the harness decision-emulation matrix request that +// openrouter/anthropic/claude-opus-4.1 answered in flattened form. +func flattenedQuestions() map[string]schemas.DecisionQuestion { + q := map[string]schemas.DecisionQuestion{} + for _, name := range []string{ + "expresses_urgency", "has_attached_evidence", "is_angry", "is_billing_related", "is_first_time_issue", + "is_frustrated", "is_long_time_customer", "is_polite", "is_repeat_contact", "is_satisfied", + "mentions_competitor", "mentions_data_loss", "mentions_refund", "mentions_specific_amount", + "reports_product_crash", "requests_human_response", "threatens_cancellation", "threatens_chargeback", + "threatens_public_complaint", "willing_to_wait", + } { + q[name] = schemas.DecisionQuestion{Kind: schemas.DecisionKindNoul, Instructions: name} + } + choice := func(opts ...string) schemas.DecisionQuestion { + criteria := map[string]any{} + for _, o := range opts { + criteria[o] = o + } + return schemas.DecisionQuestion{Kind: schemas.DecisionKindChoice, Instructions: "pick", Criteria: criteria} + } + score := func(n int) schemas.DecisionQuestion { + levels := make([]any, n) + for i := range levels { + levels[i] = strconv.Itoa(i) + } + return schemas.DecisionQuestion{Kind: schemas.DecisionKindScore, Instructions: "rate", Criteria: levels} + } + q["category"] = choice("billing", "bug", "account", "other") + q["contact_channel"] = choice("email", "phone", "chat", "social") + q["customer_tier"] = choice("free", "pro", "enterprise") + q["primary_emotion"] = choice("anger", "confusion", "sadness", "joy") + q["recommended_action"] = choice("issue_refund", "request_more_info", "close_ticket", "ignore") + q["sentiment"] = choice("positive", "neutral", "negative") + q["churn_risk"] = score(4) + q["customer_satisfaction"] = score(5) + q["severity"] = score(4) + q["urgency"] = score(4) + return q +} + +// flattenedOpus41Args is the verbatim emit_decision arguments returned by +// openrouter/anthropic/claude-opus-4.1 (captured 2026-09-23). The model flattened +// every nested answer object into the top level, leaked its XML +// "" syntax into the first field, and repeated +// "confidence"/"probabilities" as duplicate keys. +const flattenedOpus41Args = `{"category": "billing", "confidence": 1.0, "probabilities": {"account":0.0,"billing":1.0,"bug":0.0,"other":0.0}, "churn_risk": "3", "confidence": 0.95, "probabilities": {"0":0.0,"1":0.0,"2":0.05,"3":0.95}, "contact_channel": "email", "confidence": 1.0, "probabilities": {"chat":0.0,"email":1.0,"phone":0.0,"social":0.0}, "customer_satisfaction": "0", "confidence": 1.0, "probabilities": {"0":1.0,"1":0.0,"2":0.0,"3":0.0,"4":0.0}, "customer_tier": "pro", "confidence": 1.0, "probabilities": {"enterprise":0.0,"free":0.0,"pro":1.0}, "expresses_urgency": "1.0", "confidence": 1.0, "has_attached_evidence": "1.0", "confidence": 1.0, "is_angry": "1.0", "confidence": 0.95, "is_billing_related": "1.0", "confidence": 1.0, "is_first_time_issue": "0.0", "confidence": 1.0, "is_frustrated": "1.0", "confidence": 1.0, "is_long_time_customer": "1.0", "confidence": 1.0, "is_polite": "0.0", "confidence": 0.95, "is_repeat_contact": "1.0", "confidence": 1.0, "is_satisfied": "0.0", "confidence": 1.0, "mentions_competitor": "1.0", "confidence": 1.0, "mentions_data_loss": "0.0", "confidence": 1.0, "mentions_refund": "1.0", "confidence": 1.0, "mentions_specific_amount": "1.0", "confidence": 1.0, "primary_emotion": "anger", "confidence": 0.95, "probabilities": {"anger":0.95,"confusion":0.0,"joy":0.0,"sadness":0.05}, "recommended_action": "issue_refund", "confidence": 0.95, "probabilities": {"close_ticket":0.0,"ignore":0.0,"issue_refund":0.95,"request_more_info":0.05}, "reports_product_crash": "0.0", "confidence": 1.0, "requests_human_response": "1.0", "confidence": 1.0, "sentiment": "negative", "confidence": 1.0, "probabilities": {"negative":1.0,"neutral":0.0,"positive":0.0}, "severity": "3", "confidence": 0.95, "probabilities": {"0":0.0,"1":0.0,"2":0.05,"3":0.95}, "threatens_cancellation": "1.0", "confidence": 1.0, "threatens_chargeback": "1.0", "confidence": 1.0, "threatens_public_complaint": "1.0", "confidence": 1.0, "urgency": "3", "confidence": 1.0, "probabilities": {"0":0.0,"1":0.0,"2":0.0,"3":1.0}, "willing_to_wait": "0.0", "confidence": 1.0}` + +func TestParseDecisionAnswersRecoversFlattenedParameterTags(t *testing.T) { + answers, err := ParseDecisionAnswers([]byte(flattenedOpus41Args), flattenedQuestions()) + if err != nil { + t.Fatalf("flattened answers must be recovered: %v", err) + } + if len(answers) != 30 { + t.Fatalf("expected 30 answers, got %d", len(answers)) + } + if answers["category"].Value != "billing" || answers["category"].Probabilities["billing"] != 1 { + t.Errorf("category = %+v", answers["category"]) + } + if answers["is_polite"].Value != 0.0 || *answers["is_polite"].Confidence != 0.95 { + t.Errorf("is_polite = %+v", answers["is_polite"]) + } + if answers["is_angry"].Value != 1.0 || *answers["is_angry"].Confidence != 0.95 { + t.Errorf("is_angry = %+v", answers["is_angry"]) + } + // Score derives from its own distribution: 2*0.05 + 3*0.95 = 2.95. + if math.Abs(answers["churn_risk"].Value.(float64)-2.95) > 1e-9 || *answers["churn_risk"].Confidence != 0.95 { + t.Errorf("churn_risk = %+v", answers["churn_risk"]) + } + if answers["primary_emotion"].Value != "anger" || answers["primary_emotion"].Probabilities["sadness"] != 0.05 { + t.Errorf("primary_emotion = %+v", answers["primary_emotion"]) + } +} + +// The same model sometimes pads the leaked tag with whitespace, as in +// `{"category": " billing", "confidence": 1.0, "probabilities": {"billing":1,"bug":0,"other":0}, "is_frustrated": "\n0.9", "confidence": 0.8, "urgency": "\t 2", "confidence": 0.6, "probabilities": {"0":0.1,"1":0.1,"2":0.8}}` + answers, err := ParseDecisionAnswers([]byte(args), mixedQuestions()) + if err != nil { + t.Fatalf("whitespace-padded answers must be recovered: %v", err) + } + if answers["category"].Value != "billing" || *answers["category"].Confidence != 1.0 { + t.Errorf("category = %+v", answers["category"]) + } + if answers["is_frustrated"].Value != 0.9 || *answers["is_frustrated"].Confidence != 0.8 { + t.Errorf("is_frustrated = %+v", answers["is_frustrated"]) + } + if *answers["urgency"].Confidence != 0.6 { + t.Errorf("urgency = %+v", answers["urgency"]) + } +} + +func TestParseDecisionAnswersFlattenedRejections(t *testing.T) { + q := mixedQuestions() + cases := map[string]struct { + args string + wantSub string + }{ + // The recovered answers still pass through every normal check. + "recovered choice missing probabilities": { + args: `{"category": "billing", "confidence": 1.0, "is_frustrated": "0.9", "urgency": "2", "confidence": 0.6, "probabilities": {"0":0.1,"1":0.1,"2":0.8}}`, + wantSub: "missing probabilities", + }, + "field before any question": { + args: `{"confidence": 1.0, "category": "billing", "confidence": 1.0, "probabilities": {"billing":1,"bug":0,"other":0}}`, + wantSub: "not a JSON object", + }, + "field repeated within one answer": { + args: `{"is_frustrated": "0.9", "value": 0.1}`, + wantSub: "not a JSON object", + }, + "unknown top-level key": { + args: `{"is_frustrated": "0.9", "mood": "bad"}`, + wantSub: "not a JSON object", + }, + "string answer without a parameter tag": { + args: `{"is_frustrated": "0.9"}`, + wantSub: "not a JSON object", + }, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + _, err := ParseDecisionAnswers([]byte(tc.args), q) + if err == nil { + t.Fatalf("expected rejection for %q", name) + } + if !strings.Contains(err.Error(), tc.wantSub) { + t.Errorf("error %q does not contain %q", err.Error(), tc.wantSub) + } + }) + } +} diff --git a/core/providers/utils/namespacetools.go b/core/providers/utils/namespacetools.go index d58f9218edc..ce5e269fe65 100644 --- a/core/providers/utils/namespacetools.go +++ b/core/providers/utils/namespacetools.go @@ -364,11 +364,11 @@ func ResolveToolNameLimit(ctx *schemas.BifrostContext, baseProvider schemas.Mode } // minToolNameLimit is the smallest limit the hashed alias form can honour: the -// 8-hex hash, the "_" separator and one character of the function name. +// "t"-prefixed hash, the "_" separator and one character of the function name. const minToolNameLimit = namespaceAliasHashLength + 2 -// namespaceAliasHashLength is the width of the "%08x" prefix in a hashed alias. -const namespaceAliasHashLength = 8 +// namespaceAliasHashLength is the width of the "t%08x" prefix in a hashed alias. +const namespaceAliasHashLength = 9 // namespaceToolAlias returns the flat tool name for one nested function under the // wire's limit. It is a pure function of (namespace, function, limit): history items @@ -376,10 +376,13 @@ const namespaceAliasHashLength = 8 // the same inputs must always yield the same string. // // The plain "__" form is used when it fits. Over the limit the -// name becomes "<8-hex xxhash>_", the same scheme bedrockAliasToolName uses: +// name becomes "t<8-hex xxhash>_", the same scheme bedrockAliasToolName uses: // the hash of the full alias keeps the name unique per (namespace, function), and the -// tail keeps the function readable to the model. The response-side alias map is keyed -// on whichever string was produced, so restore never needs to invert this. +// tail keeps the function readable to the model. The "t" keeps the name letter-first: +// a bare hex hash starts with a digit 10 times in 16, and moonshotai.kimi-k3 on +// Bedrock answers any digit-leading tool name with HTTP 200 and an empty stream. The +// response-side alias map is keyed on whichever string was produced, so restore never +// needs to invert this. func namespaceToolAlias(namespace, function string, limit ToolNameLimit) string { full := namespace + namespaceToolSeparator + function sanitized := limit.Sanitize(full) @@ -388,7 +391,7 @@ func namespaceToolAlias(namespace, function string, limit ToolNameLimit) string } // Hash the unsanitized alias so namespaces that differ only in a replaced // character still hash apart. - hash := fmt.Sprintf("%08x", uint32(xxhash.Sum64String(full))) + hash := fmt.Sprintf("t%08x", uint32(xxhash.Sum64String(full))) semantic := strings.Trim(limit.Sanitize(function), "_") if semantic == "" { semantic = "tool" diff --git a/core/providers/utils/promptcache.go b/core/providers/utils/promptcache.go index af2db50de24..4b76cd4f8bb 100644 --- a/core/providers/utils/promptcache.go +++ b/core/providers/utils/promptcache.go @@ -1,6 +1,8 @@ package utils import ( + "slices" + "github.com/capsohq/bifrost/core/schemas" ) @@ -149,6 +151,52 @@ func InjectChatCacheBreakpoints(cfg *schemas.PromptCacheConfig, input []schemas. return out } +// ChatCachePointsSupported reports whether this attempt accepts cachePoint markers. +// cachePoint is a Converse-only element, so the provider gate comes first: the name +// fallback matches a Claude model whatever serves it. +func ChatCachePointsSupported(provider schemas.ModelProvider, model string) bool { + if provider != schemas.Bedrock { + return false + } + return schemas.ResolveModelCaps(provider, model).SupportsCachePoint(schemas.BedrockModelSupportsCachePoints(model)) +} + +// StripChatCachePoints returns input without Bedrock cachePoint markers, copying only the messages that carry one. +func StripChatCachePoints(input []schemas.ChatMessage) ([]schemas.ChatMessage, bool) { + var out []schemas.ChatMessage + for i := range input { + content := input[i].Content + if content == nil || !slices.ContainsFunc(content.ContentBlocks, hasChatCachePoint) { + continue + } + if out == nil { + out = slices.Clone(input) + } + blocks := make([]schemas.ChatContentBlock, 0, len(content.ContentBlocks)) + for _, block := range content.ContentBlocks { + if block.CachePoint != nil { + // A standalone marker has nothing else to send. + if block.Type == "" { + continue + } + block.CachePoint = nil + } + blocks = append(blocks, block) + } + contentCopy := *content + contentCopy.ContentBlocks = blocks + out[i].Content = &contentCopy + } + if out == nil { + return input, false + } + return out, true +} + +func hasChatCachePoint(block schemas.ChatContentBlock) bool { + return block.CachePoint != nil +} + // injectionTarget names one place to write a marker. promoteStr means the message // carries a bare ContentStr that must become a block first, in which case block is // meaningless. diff --git a/core/providers/utils/promptcache_test.go b/core/providers/utils/promptcache_test.go index aa422981626..e25f0e75716 100644 --- a/core/providers/utils/promptcache_test.go +++ b/core/providers/utils/promptcache_test.go @@ -487,3 +487,63 @@ func TestResolvePromptCacheConfig(t *testing.T) { assert.Len(t, got.InjectionPoints, 1, "the header only flips auto_inject; points remain config-level") }) } + +func TestStripChatCachePoints(t *testing.T) { + cachePoint := &schemas.CachePoint{Type: "default"} + input := []schemas.ChatMessage{ + {Role: schemas.ChatMessageRoleSystem, Content: &schemas.ChatMessageContent{ContentStr: schemas.Ptr("sys")}}, + {Role: schemas.ChatMessageRoleUser, Content: &schemas.ChatMessageContent{ContentBlocks: []schemas.ChatContentBlock{ + {Type: schemas.ChatContentBlockTypeText, Text: schemas.Ptr("hi"), CachePoint: cachePoint}, + {CachePoint: cachePoint}, + {Type: schemas.ChatContentBlockTypeText, Text: schemas.Ptr("there")}, + }}}, + } + + out, stripped := StripChatCachePoints(input) + require.True(t, stripped) + require.Len(t, out[1].Content.ContentBlocks, 2) + assert.Nil(t, out[1].Content.ContentBlocks[0].CachePoint) + assert.Equal(t, "there", *out[1].Content.ContentBlocks[1].Text) + assert.Same(t, input[0].Content, out[0].Content) + + // The caller's request is shared with fallbacks, so it must keep its markers. + require.Len(t, input[1].Content.ContentBlocks, 3) + assert.NotNil(t, input[1].Content.ContentBlocks[0].CachePoint) + assert.NotNil(t, input[1].Content.ContentBlocks[1].CachePoint) + + same, stripped := StripChatCachePoints(out) + assert.False(t, stripped) + assert.Equal(t, &out[0], &same[0]) +} + +func TestChatCachePointsSupported(t *testing.T) { + t.Run("no datasheet row falls back to the name", func(t *testing.T) { + assert.True(t, ChatCachePointsSupported(schemas.Bedrock, "anthropic.claude-3-5-haiku-20241022-v1:0")) + assert.True(t, ChatCachePointsSupported(schemas.Bedrock, "amazon.nova-pro-v1:0")) + assert.False(t, ChatCachePointsSupported(schemas.Bedrock, "meta.llama3-70b-instruct-v1:0")) + }) + + t.Run("only Converse takes the marker", func(t *testing.T) { + // The name fallback matches a Claude model whatever serves it, so the + // provider gate is what keeps a cachePoint off the Anthropic wire. + for _, provider := range []schemas.ModelProvider{schemas.Anthropic, schemas.BedrockMantle, schemas.Vertex, schemas.OpenAI} { + assert.False(t, ChatCachePointsSupported(provider, "claude-3-5-haiku-20241022"), string(provider)) + } + }) + + t.Run("the datasheet overrides the name in both directions", func(t *testing.T) { + withResolver(t, func(_ schemas.ModelProvider, model string) *schemas.ModelCapabilities { + switch model { + case "meta.llama3-70b-instruct-v1:0": + return &schemas.ModelCapabilities{SupportsCachePoint: schemas.Ptr(true)} + case "anthropic.claude-3-5-haiku-20241022-v1:0": + return &schemas.ModelCapabilities{SupportsCachePoint: schemas.Ptr(false)} + } + return nil + }) + assert.True(t, ChatCachePointsSupported(schemas.Bedrock, "meta.llama3-70b-instruct-v1:0")) + assert.False(t, ChatCachePointsSupported(schemas.Bedrock, "anthropic.claude-3-5-haiku-20241022-v1:0")) + // A row saying yes still cannot put a Converse element on another wire. + assert.False(t, ChatCachePointsSupported(schemas.Anthropic, "meta.llama3-70b-instruct-v1:0")) + }) +} diff --git a/core/providers/utils/ssestream_bench_test.go b/core/providers/utils/ssestream_bench_test.go new file mode 100644 index 00000000000..552529e70b9 --- /dev/null +++ b/core/providers/utils/ssestream_bench_test.go @@ -0,0 +1,102 @@ +package utils + +// Benchmarks for the shared SSE framing readers. +// +// Every streaming response in the gateway is framed by one of these two +// readers, once per event. A stream of a few hundred tokens therefore pays this +// cost a few hundred times before the caller sees the last chunk, which makes +// the framing loop (scanner buffer splitting, prefix parsing, the copy out of +// the scanner buffer) worth tracking on its own - separate from the provider +// specific JSON conversion that follows it. +// +// Run: +// +// go test ./core/providers/utils/ -bench SSE -benchmem + +import ( + "bytes" + "errors" + "io" + "strings" + "testing" +) + +const benchSSEEventCount = 256 + +// benchSSEDataStream builds a Format A (data-only) stream the way OpenAI, +// Gemini and Cohere send it: one JSON chunk per data line, terminated by +// [DONE], with keep-alive comments interleaved. +func benchSSEDataStream() []byte { + var sb strings.Builder + for i := range benchSSEEventCount { + if i%32 == 0 { + sb.WriteString(": keep-alive\n\n") + } + sb.WriteString(`data: {"id":"chatcmpl-Bq9xK2LmNpQrStUvWxYz","object":"chat.completion.chunk","created":1742914380,"model":"gpt-4o-2024-11-20","choices":[{"index":0,"delta":{"content":" incrementally streamed token"},"finish_reason":null}]}`) + sb.WriteString("\n\n") + } + sb.WriteString("data: [DONE]\n\n") + return []byte(sb.String()) +} + +// benchSSEEventStream builds a Format B (typed event) stream the way Anthropic +// and Replicate send it: an event line plus a data line per event. +func benchSSEEventStream() []byte { + var sb strings.Builder + sb.WriteString("event: message_start\n") + sb.WriteString(`data: {"type":"message_start","message":{"id":"msg_01XyZaBcDeFgHiJkLmNoPqRs","role":"assistant","model":"claude-sonnet-4-5-20250929","content":[]}}`) + sb.WriteString("\n\n") + for range benchSSEEventCount { + sb.WriteString("event: content_block_delta\n") + sb.WriteString(`data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" incrementally streamed token"}}`) + sb.WriteString("\n\n") + } + sb.WriteString("event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n") + return []byte(sb.String()) +} + +func BenchmarkSSEDataReader(b *testing.B) { + stream := benchSSEDataStream() + b.SetBytes(int64(len(stream))) + b.ReportAllocs() + for b.Loop() { + reader := GetSSEDataReader(nil, bytes.NewReader(stream)) + lines := 0 + for { + line, err := reader.ReadDataLine() + if err != nil { + if errors.Is(err, io.EOF) { + break + } + b.Fatalf("read data line: %v", err) + } + lines += len(line) + } + if lines == 0 { + b.Fatal("no data lines read") + } + } +} + +func BenchmarkSSEEventReader(b *testing.B) { + stream := benchSSEEventStream() + b.SetBytes(int64(len(stream))) + b.ReportAllocs() + for b.Loop() { + reader := GetSSEEventReader(nil, bytes.NewReader(stream)) + events := 0 + for { + eventType, data, err := reader.ReadEvent() + if err != nil { + if errors.Is(err, io.EOF) { + break + } + b.Fatalf("read event: %v", err) + } + events += len(eventType) + len(data) + } + if events == 0 { + b.Fatal("no events read") + } + } +} diff --git a/core/providers/utils/utils.go b/core/providers/utils/utils.go index b590cf0eaf7..ac73ac34c8b 100644 --- a/core/providers/utils/utils.go +++ b/core/providers/utils/utils.go @@ -4419,3 +4419,423 @@ func ModelMatchesDenylist(denylist []string, candidates ...string) bool { } return false } + +// jsonSchemaPatternKey is the JSON Schema keyword whose value is a regex. +const jsonSchemaPatternKey = "pattern" + +// NormalizeRegexNULEscape rewrites the `\0` NUL escape to `\x00`. +// +// The two denote the same character in every engine that accepts both, but some +// model backends reject the `\0` spelling when they validate tool schemas, and +// they fail the whole request rather than reporting a bad pattern. They also fail +// differently: DeepSeek answers 400 `"^[^\0]*$" is not a "regex"`, while +// moonshotai.kimi-k3 on Bedrock answers 200 with an empty event stream and says +// nothing at all. This is those validators' behaviour, not a regex-engine rule: +// Go's own regexp accepts `\0` as a one-digit octal escape, and OpenAI, Anthropic +// and Gemini accept it unchanged. `\x00` is accepted by every backend tested, so +// the rewrite is lossless; which models receive it is decided by the caller (see +// toolSchemaPatternRewriter in core), not here. +// +// Only a `\0` that is not the start of a legacy octal escape is rewritten, so +// `\012` is left alone, and escape pairs are consumed two at a time so the `0` in +// `\\0` (a literal backslash then a zero) is never mistaken for a NUL. +func NormalizeRegexNULEscape(pattern string) string { + if !strings.Contains(pattern, `\0`) { + return pattern + } + var b strings.Builder + b.Grow(len(pattern) + 8) + for i := 0; i < len(pattern); { + if pattern[i] != '\\' || i+1 >= len(pattern) { + b.WriteByte(pattern[i]) + i++ + continue + } + // Only 0-7 are octal digits: `\012` is a legacy octal escape and stays, + // but `\08` is a NUL escape followed by a literal 8 and must be rewritten. + if pattern[i+1] == '0' && (i+2 >= len(pattern) || pattern[i+2] < '0' || pattern[i+2] > '7') { + b.WriteString(`\x00`) + } else { + b.WriteByte(pattern[i]) + b.WriteByte(pattern[i+1]) + } + i += 2 + } + return b.String() +} + +// normalizeSchemaValue normalizes any nested JSON Schema value. It returns the +// input untouched, and reports false, when nothing changed -- the common case, +// which must not allocate. +func normalizeSchemaValue(value any, rewrite PatternRewriter) (any, bool) { + switch typed := value.(type) { + case *schemas.OrderedMap: + return normalizeSchemaMap(typed, rewrite) + case map[string]any: + // Nested schemas built in code arrive as plain maps (BuildDecisionSchema + // stores each question's schema this way inside an OrderedMap), so they + // need the same pattern handling. Copy-on-write like the other cases: the + // map is duplicated only when a value changes. + var updated map[string]any + for key, nested := range typed { + normalized, changed := nested, false + if key == jsonSchemaPatternKey { + if pattern, ok := nested.(string); ok { + if rewritten := rewrite(pattern); rewritten != pattern { + normalized, changed = rewritten, true + } + } + } + if !changed { + normalized, changed = normalizeSchemaValue(nested, rewrite) + } + if !changed { + continue + } + if updated == nil { + updated = make(map[string]any, len(typed)) + for k, v := range typed { + updated[k] = v + } + } + updated[key] = normalized + } + if updated == nil { + return typed, false + } + return updated, true + case []any: + var updated []any + for i := range typed { + normalized, changed := normalizeSchemaValue(typed[i], rewrite) + if !changed { + continue + } + if updated == nil { + updated = make([]any, len(typed)) + copy(updated, typed) + } + updated[i] = normalized + } + if updated == nil { + return typed, false + } + return updated, true + } + return value, false +} + +// normalizeSchemaMap walks one schema object. OrderedMap.Clone is shallow, so a +// clone is taken only when this level actually changes and untouched subtrees stay +// shared with the caller's schema. Set replaces a value without disturbing key +// order, which the prompt cache depends on. +func normalizeSchemaMap(schema *schemas.OrderedMap, rewrite PatternRewriter) (*schemas.OrderedMap, bool) { + if schema == nil || schema.Len() == 0 { + return schema, false + } + var updated *schemas.OrderedMap + for _, key := range schema.Keys() { + value, _ := schema.Get(key) + + normalized, changed := value, false + if key == jsonSchemaPatternKey { + if pattern, ok := value.(string); ok { + if rewritten := rewrite(pattern); rewritten != pattern { + normalized, changed = rewritten, true + } + } + } + if !changed { + normalized, changed = normalizeSchemaValue(value, rewrite) + } + if !changed { + continue + } + if updated == nil { + updated = schema.Clone() + } + updated.Set(key, normalized) + } + if updated == nil { + return schema, false + } + return updated, true +} + +// RewriteToolSchemaPatterns applies rewrite to every regex `pattern` in the tool +// parameter schema, copy-on-write: a schema with nothing to rewrite is returned +// as-is and allocates nothing. +func RewriteToolSchemaPatterns(params *schemas.ToolFunctionParameters, rewrite PatternRewriter) (*schemas.ToolFunctionParameters, bool) { + if params == nil { + return params, false + } + + // Struct assignment carries the unexported keyOrder and explicitEmptyObject + // fields across, so a rewritten schema still serializes in the client's key + // order and an explicit `{}` stays `{}`. + updated := *params + changed := false + + if params.Pattern != nil { + if rewritten := rewrite(*params.Pattern); rewritten != *params.Pattern { + updated.Pattern = &rewritten + changed = true + } + } + + for _, field := range []struct { + value *schemas.OrderedMap + assign func(*schemas.OrderedMap) + }{ + {params.Properties, func(m *schemas.OrderedMap) { updated.Properties = m }}, + {params.Items, func(m *schemas.OrderedMap) { updated.Items = m }}, + {params.Defs, func(m *schemas.OrderedMap) { updated.Defs = m }}, + {params.Definitions, func(m *schemas.OrderedMap) { updated.Definitions = m }}, + } { + if normalized, fieldChanged := normalizeSchemaMap(field.value, rewrite); fieldChanged { + field.assign(normalized) + changed = true + } + } + + // additionalProperties is a schema in its own right, not just a boolean, so a + // pattern under it must be rewritten too. The struct is copied only when its + // map changes; the boolean variant is carried through untouched. + if additional := params.AdditionalProperties; additional != nil && additional.AdditionalPropertiesMap != nil { + if normalized, fieldChanged := normalizeSchemaMap(additional.AdditionalPropertiesMap, rewrite); fieldChanged { + additionalCopy := *additional + additionalCopy.AdditionalPropertiesMap = normalized + updated.AdditionalProperties = &additionalCopy + changed = true + } + } + + for _, composition := range []struct { + value []schemas.OrderedMap + assign func([]schemas.OrderedMap) + }{ + {params.AnyOf, func(s []schemas.OrderedMap) { updated.AnyOf = s }}, + {params.OneOf, func(s []schemas.OrderedMap) { updated.OneOf = s }}, + {params.AllOf, func(s []schemas.OrderedMap) { updated.AllOf = s }}, + } { + var rewritten []schemas.OrderedMap + for i := range composition.value { + normalized, elementChanged := normalizeSchemaMap(&composition.value[i], rewrite) + if !elementChanged { + continue + } + if rewritten == nil { + rewritten = make([]schemas.OrderedMap, len(composition.value)) + copy(rewritten, composition.value) + } + rewritten[i] = *normalized + } + if rewritten != nil { + composition.assign(rewritten) + changed = true + } + } + + if !changed { + return params, false + } + return &updated, true +} + +// RewriteResponsesToolSchemas applies rewrite to the regex patterns in every +// Responses tool's parameter schema, copy-on-write. Tools with nothing to rewrite +// are shared with the caller's slice rather than copied, and ResponsesToolFunction +// is embedded by pointer, so a rewritten one is replaced rather than written +// through. +func RewriteResponsesToolSchemas(tools []schemas.ResponsesTool, rewrite PatternRewriter) ([]schemas.ResponsesTool, bool) { + var updated []schemas.ResponsesTool + for i := range tools { + if tools[i].ResponsesToolFunction == nil || tools[i].ResponsesToolFunction.Parameters == nil { + continue + } + normalized, changed := RewriteToolSchemaPatterns(tools[i].ResponsesToolFunction.Parameters, rewrite) + if !changed { + continue + } + if updated == nil { + updated = make([]schemas.ResponsesTool, len(tools)) + copy(updated, tools) + } + function := *tools[i].ResponsesToolFunction + function.Parameters = normalized + updated[i].ResponsesToolFunction = &function + } + if updated == nil { + return tools, false + } + return updated, true +} + +// PatternRewriter rewrites one regex `pattern` value. It must return its input +// unchanged when it has nothing to do, so callers can detect a no-op by equality +// and keep the original schema bytes. +type PatternRewriter func(pattern string) string + +// ComposePatternRewriters applies rewriters left to right. +func ComposePatternRewriters(rewriters ...PatternRewriter) PatternRewriter { + return func(pattern string) string { + for _, rewrite := range rewriters { + pattern = rewrite(pattern) + } + return pattern + } +} + +// StripRegexLookaround removes every zero-width lookaround assertion, `(?=...)`, +// `(?!...)`, `(?<=...)` and `(?...)` +// groups are not lookaround and are left alone. An unbalanced pattern is +// returned unchanged rather than truncated. +func StripRegexLookaround(pattern string) string { + if !strings.Contains(pattern, "(?") { + return pattern + } + var b strings.Builder + b.Grow(len(pattern)) + changed := false + for i := 0; i < len(pattern); { + c := pattern[i] + switch { + case c == '\\' && i+1 < len(pattern): + b.WriteByte(c) + b.WriteByte(pattern[i+1]) + i += 2 + case c == '[': + end := regexClassEnd(pattern, i) + b.WriteString(pattern[i:end]) + i = end + case c == '(' && isRegexLookaroundStart(pattern, i): + end := regexGroupEnd(pattern, i) + if end < 0 { + return pattern + } + changed = true + // A quantifier attached to the removed assertion would otherwise land on + // the previous atom, or lead the pattern and fail to parse, so it goes too. + i = regexQuantifierEnd(pattern, end) + default: + b.WriteByte(c) + i++ + } + } + if !changed { + return pattern + } + return b.String() +} + +// regexQuantifierEnd returns the index just past a quantifier starting at +// pattern[start] (`*`, `+`, `?`, or a well-formed `{n}`, `{n,}`, `{n,m}`), plus an +// optional lazy `?`. Anything else, including a malformed brace, is left in place +// and start is returned unchanged. +func regexQuantifierEnd(pattern string, start int) int { + if start >= len(pattern) { + return start + } + i := start + switch pattern[i] { + case '*', '+', '?': + i++ + case '{': + j := i + 1 + digits := func() bool { + n := j + for j < len(pattern) && pattern[j] >= '0' && pattern[j] <= '9' { + j++ + } + return j > n + } + if !digits() { + return start + } + if j < len(pattern) && pattern[j] == ',' { + j++ + digits() + } + if j >= len(pattern) || pattern[j] != '}' { + return start + } + i = j + 1 + default: + return start + } + if i < len(pattern) && pattern[i] == '?' { + i++ + } + return i +} + +// isRegexLookaroundStart reports whether pattern[i:] opens a lookaround group. +func isRegexLookaroundStart(pattern string, i int) bool { + rest := pattern[i:] + return strings.HasPrefix(rest, "(?=") || strings.HasPrefix(rest, "(?!") || + strings.HasPrefix(rest, "(?<=") || strings.HasPrefix(rest, "(?_", so any row below 10 leaves no room for a +// alias is "t<8-hex hash>_", so any row below 11 leaves no room for a // readable tail (and a negative slice window). Such a row is ignored in favour of // the provider default; a hand-built limit still never panics or exceeds itself. func TestResolveToolNameLimit_RowBelowHashFloorKeepsDefault(t *testing.T) { @@ -3243,6 +3244,35 @@ func TestNamespaceToolAlias_TinyLimitNeverPanicsOrExceeds(t *testing.T) { } } +// moonshotai.kimi-k3 on Bedrock answers a request carrying any tool whose name starts +// with a digit with HTTP 200 and an empty stream, and a bare 8-hex hash starts with a +// digit 10 times in 16. Codex's long MCP namespaces land on the hashed form, so every +// hashed alias must start with a letter, at every limit including the tiny ones. +func TestNamespaceToolAlias_HashedFormStartsWithLetter(t *testing.T) { + letterFirst := regexp.MustCompile(`^[A-Za-z]`) + sawDigitHash := false + for i := range 64 { + namespace := fmt.Sprintf("mcp__codex_apps__codex_document_control_%02d_with_a_long_suffix", i) + function := "execute_document_command" + if full := namespace + namespaceToolSeparator + function; fmt.Sprintf("%08x", uint32(xxhash.Sum64String(full)))[0] <= '9' { + sawDigitHash = true + } + for _, max := range []int{64, 128, 12, 10, 9, 1} { + limit := ToolNameLimit{MaxLength: max, unsafe: toolNameUnsafeStrict} + got := namespaceToolAlias(namespace, function, limit) + if len(namespace)+len(namespaceToolSeparator)+len(function) <= max { + continue + } + if !letterFirst.MatchString(got) { + t.Errorf("limit %d: hashed alias %q must start with a letter", max, got) + } + } + } + if !sawDigitHash { + t.Fatal("sweep never produced a digit-leading raw hash, so it cannot prove the prefix") + } +} + // Sanitization is many-to-one on strict wires: "a:b" and "a.b" both flatten to "a_b". // Alias existence alone therefore does not prove a history call belongs to the tool // that owns the alias. A call from a namespace that merely collides must keep its own @@ -3347,7 +3377,7 @@ func TestStripCallerAuthForInsecureURL(t *testing.T) { } { t.Run(name, func(t *testing.T) { safeHeaders := map[string]string{ - tc.header: "Bearer sk-ant-oat01-token", + tc.header: "Bearer sk-ant-oat01-token", "anthropic-beta": "context-1m", } StripCallerAuthForInsecureURL(tc.url, safeHeaders) @@ -3361,3 +3391,335 @@ func TestStripCallerAuthForInsecureURL(t *testing.T) { }) } } + +func TestNormalizeRegexNULEscape(t *testing.T) { + cases := []struct { + name string + in string + want string + }{ + // The pattern Claude Code's Artifact tool ships, which DeepSeek rejects with + // a 400 and kimi-k3 on Bedrock answers with an empty 200 stream. + {"NUL escape in a negated class", `^[^\0]*$`, `^[^\x00]*$`}, + {"bare NUL escape", `^\0$`, `^\x00$`}, + {"NUL escape among other escapes", `^\d+\0\w+$`, `^\d+\x00\w+$`}, + + // Must not fire. + {"already normalized", `^[^\x00]*$`, `^[^\x00]*$`}, + {"legacy octal escape is not a NUL escape", `^\012$`, `^\012$`}, + // 8 and 9 are not octal digits, so `\08` is a NUL escape then a literal 8. + {"NUL escape followed by 8", `^\08$`, `^\x008$`}, + {"NUL escape followed by 9 inside a class", `^[^\09]*$`, `^[^\x009]*$`}, + {"octal escape inside a class", `^[\0123]$`, `^[\0123]$`}, + {"escaped backslash then a zero digit", `^\\0$`, `^\\0$`}, + {"digit and word escapes", `^\d+\w+$`, `^\d+\w+$`}, + {"no escapes at all", `^[a-z]+$`, `^[a-z]+$`}, + {"literal zero", `^0$`, `^0$`}, + {"trailing lone backslash is left intact", `^a\`, `^a\`}, + {"empty", "", ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := NormalizeRegexNULEscape(tc.in); got != tc.want { + t.Fatalf("NormalizeRegexNULEscape(%q) = %q, want %q", tc.in, got, tc.want) + } + }) + } +} + +// normalizeSchemaFromJSON round-trips a schema through ToolFunctionParameters so +// the test exercises the same nested shapes a real tool arrives as. +func normalizeSchemaFromJSON(t *testing.T, raw string) (*schemas.ToolFunctionParameters, *schemas.ToolFunctionParameters, bool) { + t.Helper() + var params schemas.ToolFunctionParameters + if err := json.Unmarshal([]byte(raw), ¶ms); err != nil { + t.Fatalf("failed to unmarshal schema: %v", err) + } + normalized, changed := RewriteToolSchemaPatterns(¶ms, NormalizeRegexNULEscape) + return ¶ms, normalized, changed +} + +func TestNormalizeToolSchemaPatterns_RewritesEveryNestedPattern(t *testing.T) { + raw := `{ + "type": "object", + "properties": { + "file_paths": {"type": "array", "items": {"type": "string", "pattern": "^[^\\0]*$"}}, + "after": {"type": "string", "pattern": "^[A-Za-z0-9_=-]{1,4096}$"}, + "nested": {"type": "object", "properties": {"deep": {"type": "string", "pattern": "^\\0$"}}} + }, + "required": ["file_paths"] + }` + _, normalized, changed := normalizeSchemaFromJSON(t, raw) + if !changed { + t.Fatal("expected the schema to be rewritten") + } + out, err := json.Marshal(normalized) + if err != nil { + t.Fatalf("marshal: %v", err) + } + rendered := string(out) + if strings.Contains(rendered, `\\0]`) || strings.Contains(rendered, `^\\0$`) { + t.Fatalf("a NUL escape survived normalization: %s", rendered) + } + for _, want := range []string{`^[^\\x00]*$`, `^\\x00$`, `^[A-Za-z0-9_=-]{1,4096}$`} { + if !strings.Contains(rendered, want) { + t.Fatalf("expected %s in the normalized schema, got %s", want, rendered) + } + } +} + +// The rewrite must not reach back into the caller's schema. OrderedMap.Clone is +// shallow, so a clone taken at the wrong level would leave nested maps shared and +// silently mutate the request the caller still holds. +func TestNormalizeToolSchemaPatterns_LeavesTheCallerSchemaUntouched(t *testing.T) { + raw := `{"type":"object","properties":{"p":{"type":"string","pattern":"^[^\\0]*$"}}}` + original, normalized, changed := normalizeSchemaFromJSON(t, raw) + if !changed { + t.Fatal("expected the schema to be rewritten") + } + if normalized == original { + t.Fatal("a rewritten schema must be a copy, not the caller's schema") + } + before, err := json.Marshal(original) + if err != nil { + t.Fatalf("marshal: %v", err) + } + if !strings.Contains(string(before), `\\0]`) { + t.Fatalf("the caller's schema was mutated in place: %s", before) + } +} + +// A schema with nothing to rewrite must come back as the very same pointer: this +// is what keeps tool bytes, and therefore prompt cache keys, identical for every +// request that already works. +func TestNormalizeToolSchemaPatterns_UnchangedSchemaIsNotCopied(t *testing.T) { + raw := `{"type":"object","properties":{"p":{"type":"string","pattern":"^[a-z]+$"}},"required":["p"]}` + original, normalized, changed := normalizeSchemaFromJSON(t, raw) + if changed { + t.Fatal("a schema with no NUL escape must not report a change") + } + if normalized != original { + t.Fatal("a schema with no NUL escape must be returned as-is, not copied") + } +} + +// Key order feeds the prompt cache key, so a rewrite must not reorder anything. +func TestNormalizeToolSchemaPatterns_PreservesKeyOrder(t *testing.T) { + raw := `{"type":"object","properties":{"zulu":{"type":"string","pattern":"^[^\\0]*$"},"alpha":{"type":"string"}},"required":["zulu"]}` + _, normalized, changed := normalizeSchemaFromJSON(t, raw) + if !changed { + t.Fatal("expected the schema to be rewritten") + } + out, err := json.Marshal(normalized) + if err != nil { + t.Fatalf("marshal: %v", err) + } + rendered := string(out) + if strings.Index(rendered, `"zulu"`) > strings.Index(rendered, `"alpha"`) { + t.Fatalf("property order changed during normalization: %s", rendered) + } +} + +func TestNormalizeResponsesToolSchemas_CopiesOnlyTheRewrittenTool(t *testing.T) { + withNUL := schemas.ToolFunctionParameters{Type: "object", Pattern: schemas.Ptr(`^[^\0]*$`)} + clean := schemas.ToolFunctionParameters{Type: "object", Pattern: schemas.Ptr(`^[a-z]+$`)} + tools := []schemas.ResponsesTool{ + {Type: schemas.ResponsesToolTypeFunction, Name: schemas.Ptr("clean"), + ResponsesToolFunction: &schemas.ResponsesToolFunction{Parameters: &clean}}, + {Type: schemas.ResponsesToolTypeFunction, Name: schemas.Ptr("dirty"), + ResponsesToolFunction: &schemas.ResponsesToolFunction{Parameters: &withNUL}}, + } + + updated, changed := RewriteResponsesToolSchemas(tools, NormalizeRegexNULEscape) + if !changed { + t.Fatal("expected the tool slice to be rewritten") + } + if *withNUL.Pattern != `^[^\0]*$` { + t.Fatalf("the caller's tool schema was mutated in place: %q", *withNUL.Pattern) + } + if got := *updated[1].ResponsesToolFunction.Parameters.Pattern; got != `^[^\x00]*$` { + t.Fatalf("dirty tool not normalized, got %q", got) + } + if updated[0].ResponsesToolFunction != tools[0].ResponsesToolFunction { + t.Fatal("a tool with no NUL escape must not be copied") + } +} + +func TestNormalizeToolSchemas_NoToolsIsANoOp(t *testing.T) { + if tools, changed := RewriteResponsesToolSchemas(nil, NormalizeRegexNULEscape); changed || tools != nil { + t.Fatal("nil responses tools must be returned unchanged") + } + if params, changed := RewriteToolSchemaPatterns(nil, NormalizeRegexNULEscape); changed || params != nil { + t.Fatal("a nil schema must be returned unchanged") + } +} + +func TestStripRegexLookaround(t *testing.T) { + cases := []struct { + name string + in string + want string + }{ + // The four ArtifactData patterns that made every Claude Code request to + // kimi-k3 come back as an empty stream. + {"ArtifactData doc_id", `^(?!\.\.?(?:\/|$))[A-Za-z0-9_\-.~:@+]{1,200}$`, `^[A-Za-z0-9_\-.~:@+]{1,200}$`}, + {"ArtifactData collection", `^(?!\.\.?(?:\/|$))[A-Za-z0-9_\-.~:@+]{1,200}(?:\/(?!\.\.?(?:\/|$))[A-Za-z0-9_\-.~:@+]{1,200}){0,14}$`, + `^[A-Za-z0-9_\-.~:@+]{1,200}(?:\/[A-Za-z0-9_\-.~:@+]{1,200}){0,14}$`}, + + {"positive lookahead", `^(?=.*\d).{8,}$`, `^.{8,}$`}, + {"positive lookbehind", `(?<=\$)\d+`, `\d+`}, + {"negative lookbehind", `(?\d{4})`, `(?\d{4})`}, + {"inline flags", `(?i)abc`, `(?i)abc`}, + {"paren literal inside a class", `[(?!]x`, `[(?!]x`}, + {"escaped lookaround-looking text", `\(\?!x`, `\(\?!x`}, + {"plain", `^[a-z]+$`, `^[a-z]+$`}, + {"empty", ``, ``}, + + // A quantifier attached to the assertion goes with it: left behind it would + // lead the pattern (RE2: "missing argument to repetition operator") or bind + // to the previous atom and change its meaning. + {"leading quantified lookahead", `(?=b)+a`, `a`}, + {"trailing quantified lookahead", `a(?=b)+`, `a`}, + {"lazy counted quantifier", `(?!x){2,3}?y`, `y`}, + {"open-ended counted quantifier", `(?!x){2,}y`, `y`}, + {"exact counted quantifier", `(?!x){2}y`, `y`}, + {"malformed brace is not a quantifier", `(?!x){2,y`, `{2,y`}, + + // Never truncate a pattern that does not balance. + {"unbalanced lookahead", `(?!abc`, `(?!abc`}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := StripRegexLookaround(tc.in); got != tc.want { + t.Fatalf("StripRegexLookaround(%q)\n got %q\nwant %q", tc.in, got, tc.want) + } + }) + } +} + +func TestComposePatternRewriters_AppliesInOrderAndStaysNoOpWhenIdle(t *testing.T) { + both := ComposePatternRewriters(NormalizeRegexNULEscape, StripRegexLookaround) + if got := both(`^(?!\0)[^\0]*$`); got != `^[^\x00]*$` { + t.Fatalf("composed rewrite = %q, want %q", got, `^[^\x00]*$`) + } + if in := `^[a-z]+$`; both(in) != in { + t.Fatal("a composed rewriter must return its input unchanged when nothing applies") + } +} + +// The lossy rewrite must go through the same copy-on-write walker as the +// lossless one: nothing copied unless a pattern actually changed, and the +// caller's schema never written through. +func TestRewriteToolSchemaPatterns_LookaroundIsCopyOnWrite(t *testing.T) { + raw := `{"type":"object","properties":{"doc_id":{"type":"string","pattern":"^(?!\\.\\.?$)[A-Za-z0-9_.]{1,200}$"},"note":{"type":"string","pattern":"^[a-z]+$"}}}` + var params schemas.ToolFunctionParameters + if err := json.Unmarshal([]byte(raw), ¶ms); err != nil { + t.Fatalf("unmarshal: %v", err) + } + updated, changed := RewriteToolSchemaPatterns(¶ms, StripRegexLookaround) + if !changed || updated == ¶ms { + t.Fatal("expected a rewritten copy") + } + before, _ := json.Marshal(¶ms) + if !strings.Contains(string(before), `(?!`) { + t.Fatalf("caller's schema was mutated in place: %s", before) + } + after, _ := json.Marshal(updated) + if strings.Contains(string(after), `(?!`) || !strings.Contains(string(after), `^[A-Za-z0-9_.]{1,200}$`) { + t.Fatalf("lookahead not stripped: %s", after) + } + // The untouched property keeps the caller's nested map: Clone is shallow and + // only the changed spine may be copied. + origNote, _ := params.Properties.Get("note") + newNote, _ := updated.Properties.Get("note") + if origNote != newNote { + t.Fatal("an unchanged nested schema was copied") + } +} + +// additionalProperties may itself be a schema (JSON Schema: "the value of the +// additionalProperties keyword is a schema"), so a pattern under it is reachable +// by the model and must be rewritten like any other. The boolean form has no +// schema and must pass through as the caller's own pointer. +func TestRewriteToolSchemaPatterns_CoversTopLevelAdditionalProperties(t *testing.T) { + extra := schemas.NewOrderedMap() + extra.Set("type", "string") + extra.Set("pattern", `^[^\0]*$`) + params := schemas.ToolFunctionParameters{ + Type: "object", + Properties: schemas.NewOrderedMap(), + AdditionalProperties: &schemas.AdditionalPropertiesStruct{AdditionalPropertiesMap: extra}, + } + + updated, changed := RewriteToolSchemaPatterns(¶ms, NormalizeRegexNULEscape) + if !changed { + t.Fatal("a pattern under additionalProperties was not rewritten") + } + if updated.AdditionalProperties == params.AdditionalProperties { + t.Fatal("AdditionalPropertiesStruct must be copied, not written through") + } + got, _ := updated.AdditionalProperties.AdditionalPropertiesMap.Get("pattern") + if got != `^[^\x00]*$` { + t.Fatalf("additionalProperties pattern = %q, want %q", got, `^[^\x00]*$`) + } + orig, _ := params.AdditionalProperties.AdditionalPropertiesMap.Get("pattern") + if orig != `^[^\0]*$` { + t.Fatalf("caller's additionalProperties schema was mutated in place: %q", orig) + } + + // Boolean variant: nothing to rewrite, nothing copied. + boolParams := schemas.ToolFunctionParameters{ + Type: "object", + Properties: schemas.NewOrderedMap(), + AdditionalProperties: &schemas.AdditionalPropertiesStruct{AdditionalPropertiesBool: schemas.Ptr(false)}, + } + if out, changed := RewriteToolSchemaPatterns(&boolParams, NormalizeRegexNULEscape); changed || out != &boolParams { + t.Fatal("boolean additionalProperties must be returned as-is") + } +} + +// Schemas built in code, not parsed from JSON, carry nested schemas as plain +// maps: BuildDecisionSchema sets each question's schema into an OrderedMap as a +// map[string]any. The walker must descend into those too, copy-on-write, and +// must hand back the caller's own map when nothing inside it changes. +func TestRewriteToolSchemaPatterns_DescendsIntoPlainMapSchemas(t *testing.T) { + dirty := map[string]any{ + "type": "object", + "properties": map[string]any{ + "id": map[string]any{"type": "string", "pattern": `^[^\0]*$`}, + }, + } + clean := map[string]any{"type": "string", "enum": []any{"a", "b"}} + props := schemas.NewOrderedMap() + props.Set("dirty", dirty) + props.Set("clean", clean) + params := schemas.ToolFunctionParameters{Type: "object", Properties: props} + + updated, changed := RewriteToolSchemaPatterns(¶ms, NormalizeRegexNULEscape) + if !changed { + t.Fatal("a pattern inside a plain-map nested schema was not rewritten") + } + v, _ := updated.Properties.Get("dirty") + got := v.(map[string]any)["properties"].(map[string]any)["id"].(map[string]any)["pattern"] + if got != `^[^\x00]*$` { + t.Fatalf("plain-map pattern = %q, want %q", got, `^[^\x00]*$`) + } + // The caller's maps are untouched at every level. + if orig := dirty["properties"].(map[string]any)["id"].(map[string]any)["pattern"]; orig != `^[^\0]*$` { + t.Fatalf("caller's plain-map schema was mutated in place: %q", orig) + } + // The untouched sibling must be the caller's very own map, not a copy: a + // write through the original has to be visible through the returned schema. + clean["sentinel"] = true + if c2, _ := updated.Properties.Get("clean"); c2.(map[string]any)["sentinel"] != true { + t.Fatal("an unchanged plain-map schema was copied instead of shared") + } +} diff --git a/core/providers/vertex/vertex.go b/core/providers/vertex/vertex.go index de903117af6..3ac1097986c 100644 --- a/core/providers/vertex/vertex.go +++ b/core/providers/vertex/vertex.go @@ -2002,6 +2002,11 @@ func (provider *VertexProvider) Rerank(ctx *schemas.BifrostContext, key schemas. return bifrostResponse, nil } +// Decision is not supported by the Vertex provider. +func (provider *VertexProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR is not supported by the Vertex provider. func (provider *VertexProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) diff --git a/core/providers/vllm/vllm.go b/core/providers/vllm/vllm.go index cf36a0092cb..df1719a7610 100644 --- a/core/providers/vllm/vllm.go +++ b/core/providers/vllm/vllm.go @@ -542,6 +542,11 @@ func (provider *VLLMProvider) Rerank(ctx *schemas.BifrostContext, key schemas.Ke return bifrostResponse, nil } +// Decision is not supported by the VLLM provider. +func (provider *VLLMProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR is not supported by the Vllm provider. func (provider *VLLMProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) diff --git a/core/providers/volcengine/volcengine.go b/core/providers/volcengine/volcengine.go index 6daaca2e8d8..ccba2ec738a 100644 --- a/core/providers/volcengine/volcengine.go +++ b/core/providers/volcengine/volcengine.go @@ -769,6 +769,11 @@ func (provider *VolcengineProvider) Rerank(ctx *schemas.BifrostContext, key sche return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey()) } +// Decision is not supported by the Volcengine provider. +func (provider *VolcengineProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR is not supported by the Volcengine provider. func (provider *VolcengineProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) diff --git a/core/providers/wafer/wafer.go b/core/providers/wafer/wafer.go index 275ad5b9efa..32b2f3702a7 100644 --- a/core/providers/wafer/wafer.go +++ b/core/providers/wafer/wafer.go @@ -214,6 +214,11 @@ func (provider *WaferProvider) Rerank(ctx *schemas.BifrostContext, key schemas.K return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey()) } +// Decision is not supported by the Wafer provider. +func (provider *WaferProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR is not supported by the Wafer provider. func (provider *WaferProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) diff --git a/core/providers/xai/anthropic_messages_test.go b/core/providers/xai/anthropic_messages_test.go index 1b562c1820a..fbe966a2d22 100644 --- a/core/providers/xai/anthropic_messages_test.go +++ b/core/providers/xai/anthropic_messages_test.go @@ -2,8 +2,10 @@ package xai_test import ( "context" + "io" "net/http" "net/http/httptest" + "strings" "testing" bifrost "github.com/capsohq/bifrost/core" @@ -64,3 +66,64 @@ func TestXAIAnthropicMessagesAttachmentDoesNotChangeOpenAIEndpoints(t *testing.T } } } + +// Claude Code sends Anthropic-only fields and a per-request billing line on every +// request. Upstream strips them for every non-Anthropic provider on its converted +// path; xAI's native Messages passthrough must strip them too, on both transports, +// or they reach xAI and the changing billing line defeats its prompt cache. +func TestXAIMessagesPassthroughStripsAnthropicOnlyFields(t *testing.T) { + const billing = "x-anthropic-billing-header: cc_version=2.1.80.a1b; cc_entrypoint=cli; cch=0f3c2;" + body := `{"model":"grok-4.5","safeguards":{"mode":"auto"},"diagnostics":{"previous_message_id":"msg_1"},` + + `"system":[{"type":"text","text":"` + billing + `"},{"type":"text","text":"You are helpful.","cache_control":{"type":"ephemeral"}}],` + + `"messages":[{"role":"user","content":"hi"}]}` + want := `{"model":"grok-4.5","system":[{"type":"text","text":"You are helpful.","cache_control":{"type":"ephemeral"}}],"messages":[{"role":"user","content":"hi"}]}` + + bodies := make(chan string, 2) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + got, _ := io.ReadAll(r.Body) + bodies <- string(got) + if strings.Contains(string(got), `"stream":true`) { + w.Header().Set("Content-Type", "text/event-stream") + _, _ = w.Write([]byte("event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n")) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"type":"message","content":[],"usage":{"input_tokens":1,"output_tokens":1}}`)) + })) + defer server.Close() + + provider, err := xai.NewXAIProvider(&schemas.ProviderConfig{NetworkConfig: schemas.NetworkConfig{ + BaseURL: server.URL, AllowPrivateNetwork: true, + }}, bifrost.NewNoOpLogger()) + if err != nil { + t.Fatalf("NewXAIProvider: %v", err) + } + ctx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) + key := schemas.Key{Value: schemas.SecretVar{Val: "test-key"}} + + if _, bifrostErr := provider.Passthrough(ctx, key, &schemas.BifrostPassthroughRequest{ + Method: http.MethodPost, Path: "/v1/messages", Body: []byte(body), + }); bifrostErr != nil { + t.Fatalf("Passthrough: %+v", bifrostErr) + } + if got := <-bodies; got != want { + t.Fatalf("buffered body sent to xAI:\n got: %s\nwant: %s", got, want) + } + + streamBody := strings.Replace(body, `"model":"grok-4.5",`, `"model":"grok-4.5","stream":true,`, 1) + streamWant := strings.Replace(want, `"model":"grok-4.5",`, `"model":"grok-4.5","stream":true,`, 1) + postHook := func(_ *schemas.BifrostContext, response *schemas.BifrostResponse, err *schemas.BifrostError) (*schemas.BifrostResponse, *schemas.BifrostError) { + return response, err + } + stream, bifrostErr := provider.PassthroughStream(ctx, postHook, nil, key, &schemas.BifrostPassthroughRequest{ + Method: http.MethodPost, Path: "/v1/messages", Body: []byte(streamBody), + }) + if bifrostErr != nil { + t.Fatalf("PassthroughStream: %+v", bifrostErr) + } + for range stream { + } + if got := <-bodies; got != streamWant { + t.Fatalf("streamed body sent to xAI:\n got: %s\nwant: %s", got, streamWant) + } +} diff --git a/core/providers/xai/xai.go b/core/providers/xai/xai.go index 089f9a6eed0..31c85594d5f 100644 --- a/core/providers/xai/xai.go +++ b/core/providers/xai/xai.go @@ -291,6 +291,11 @@ func (provider *XAIProvider) Rerank(ctx *schemas.BifrostContext, key schemas.Key return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey()) } +// Decision is not supported by the XAI provider. +func (provider *XAIProvider) Decision(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostDecisionRequest) (*schemas.BifrostDecisionResponse, *schemas.BifrostError) { + return nil, providerUtils.NewUnsupportedOperationError(schemas.DecisionRequest, provider.GetProviderKey()) +} + // OCR is not supported by the Xai provider. func (provider *XAIProvider) OCR(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostOCRRequest) (*schemas.BifrostOCRResponse, *schemas.BifrostError) { return nil, providerUtils.NewUnsupportedOperationError(schemas.OCRRequest, provider.GetProviderKey()) @@ -516,6 +521,7 @@ func (provider *XAIProvider) Passthrough(ctx *schemas.BifrostContext, key schema return anthropicmsg.Passthrough(ctx, provider.client, key, req, anthropicmsg.Config{ Provider: provider.GetProviderKey(), Endpoint: endpoint, NetworkConfig: provider.networkConfig, CustomProviderConfig: provider.customProviderConfig, Logger: provider.logger, + StripAnthropicOnlyFields: true, }) } @@ -527,5 +533,6 @@ func (provider *XAIProvider) PassthroughStream(ctx *schemas.BifrostContext, post return anthropicmsg.PassthroughStream(ctx, postHookRunner, postHookSpanFinalizer, provider.streamingClient, key, req, anthropicmsg.Config{ Provider: provider.GetProviderKey(), Endpoint: endpoint, NetworkConfig: provider.networkConfig, CustomProviderConfig: provider.customProviderConfig, Logger: provider.logger, + StripAnthropicOnlyFields: true, }) } diff --git a/core/gpt6astrae2e_test.go b/core/reasoningeffort_e2e_test.go similarity index 53% rename from core/gpt6astrae2e_test.go rename to core/reasoningeffort_e2e_test.go index 3400ccff3bc..a507fb9d202 100644 --- a/core/gpt6astrae2e_test.go +++ b/core/reasoningeffort_e2e_test.go @@ -12,7 +12,7 @@ import ( "github.com/capsohq/bifrost/core/schemas" ) -type astraUpstreamRequest struct { +type recordedUpstreamRequest struct { Method string Path string Body []byte @@ -24,10 +24,10 @@ type astraUpstreamRequest struct { // and the real HTTP client. The fake upstream makes the silently rewritten // request observable without requiring a real OpenAI API key. func TestGPT6AstraMaxReasoningEffortReachesOpenAIUpstream(t *testing.T) { - requests := make(chan astraUpstreamRequest, 1) + requests := make(chan recordedUpstreamRequest, 1) upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { body, err := io.ReadAll(r.Body) - requests <- astraUpstreamRequest{ + requests <- recordedUpstreamRequest{ Method: r.Method, Path: r.URL.Path, Body: body, @@ -108,3 +108,82 @@ func TestGPT6AstraMaxReasoningEffortReachesOpenAIUpstream(t *testing.T) { wire.Reasoning.Effort, schemas.ReasoningEffortMax) } } + +// TestGrok47XHighReasoningEffortReachesXAIUpstream exercises the public +// Responses API through provider selection, xAI/OpenAI conversion, JSON +// encoding, and the real HTTP client. The local upstream makes a silent effort +// downgrade observable without requiring an xAI API key. +func TestGrok47XHighReasoningEffortReachesXAIUpstream(t *testing.T) { + requests := make(chan recordedUpstreamRequest, 1) + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + requests <- recordedUpstreamRequest{ + Method: r.Method, + Path: r.URL.Path, + Body: body, + ReadErr: err, + } + writeJSON(w, http.StatusOK, `{"id":"resp_grok_47_1","object":"response","created_at":1,"status":"completed","model":"grok-4.7","output":[],"usage":{"input_tokens":1,"output_tokens":1,"total_tokens":2}}`) + })) + t.Cleanup(upstream.Close) + + account := NewMockAccount() + account.AddProviderWithBaseURL(schemas.XAI, 1, 1, upstream.URL) + account.configs[schemas.XAI].NetworkConfig.MaxRetries = 0 + account.SetKeysForProvider(schemas.XAI, []schemas.Key{{ + ID: "grok-47-repro-key", + Value: *schemas.NewSecretVar("sk-local-grok-47-repro"), + Models: schemas.WhiteList{"grok-4.7"}, + Weight: 100, + }}) + client := newStreamTestClient(t, account) + + ctx := schemas.NewBifrostContext(context.Background(), time.Now().Add(5*time.Second)) + response, bifrostErr := client.ResponsesRequest(ctx, &schemas.BifrostResponsesRequest{ + Provider: schemas.XAI, + Model: "grok-4.7", + Input: []schemas.ResponsesMessage{{ + Type: schemas.Ptr(schemas.ResponsesMessageTypeMessage), + Role: schemas.Ptr(schemas.ResponsesInputMessageRoleUser), + Content: &schemas.ResponsesMessageContent{ + ContentStr: schemas.Ptr("Say hello"), + }, + }}, + Params: &schemas.ResponsesParameters{ + Reasoning: &schemas.ResponsesParametersReasoning{ + Effort: schemas.Ptr(schemas.ReasoningEffortXHigh), + }, + }, + }) + if bifrostErr != nil { + t.Fatalf("ResponsesRequest failed before reaching the upstream: %v", bifrostErr) + } + if response == nil { + t.Fatal("ResponsesRequest returned a nil response") + } + + observed := <-requests + if observed.ReadErr != nil { + t.Fatalf("read upstream request body: %v", observed.ReadErr) + } + if observed.Method != http.MethodPost || observed.Path != "/v1/responses" { + t.Fatalf("upstream request = %s %s, want POST /v1/responses", observed.Method, observed.Path) + } + + var wire struct { + Model string `json:"model"` + Reasoning struct { + Effort string `json:"effort"` + } `json:"reasoning"` + } + if err := json.Unmarshal(observed.Body, &wire); err != nil { + t.Fatalf("decode upstream request body %q: %v", observed.Body, err) + } + if wire.Model != "grok-4.7" { + t.Fatalf("upstream model = %q, want grok-4.7", wire.Model) + } + if wire.Reasoning.Effort != schemas.ReasoningEffortXHigh { + t.Fatalf("upstream reasoning.effort = %q, want %q; Bifrost silently changed the requested effort on the wire", + wire.Reasoning.Effort, schemas.ReasoningEffortXHigh) + } +} diff --git a/core/schemas/account_test.go b/core/schemas/account_test.go index 4de9a991483..b34096e29aa 100644 --- a/core/schemas/account_test.go +++ b/core/schemas/account_test.go @@ -689,3 +689,37 @@ func TestModelFamilyIsValid(t *testing.T) { t.Fatal("nil ModelFamily should be invalid") } } + +func TestIsMoonshotModel(t *testing.T) { + cases := []struct { + model string + want bool + }{ + // Every id Moonshot publishes starts with "kimi-"; Bedrock and Groq wrap it + // in a "moonshotai" namespace. + {"kimi-k3", true}, + {"kimi-k2.7-code", true}, + {"KIMI-K3", true}, + {"moonshotai.kimi-k3", true}, + {"global.moonshotai.kimi-k3", true}, + {"us.moonshotai.kimi-k2.5", true}, + {"moonshotai/kimi-k2-instruct", true}, + + // Near misses that a substring match would wrongly claim. The predicate + // gates a lossy tool-schema rewrite, so these must stay false. + {"kimina-prover", false}, + {"mykimi", false}, + {"kimi", false}, + {"akimi-k3", false}, + {"deepseek-chat", false}, + {"gpt-4o-mini", false}, + {"", false}, + } + for _, tc := range cases { + t.Run(tc.model, func(t *testing.T) { + if got := IsMoonshotModel(tc.model); got != tc.want { + t.Fatalf("IsMoonshotModel(%q) = %v, want %v", tc.model, got, tc.want) + } + }) + } +} diff --git a/core/schemas/bifrost.go b/core/schemas/bifrost.go index af888c4d34f..1c13fa30eea 100644 --- a/core/schemas/bifrost.go +++ b/core/schemas/bifrost.go @@ -29,13 +29,14 @@ type BifrostConfig struct { OAuth2Provider OAuth2Provider MCPHeadersProvider MCPHeadersProvider // Backend for MCPAuthTypePerUserHeaders credential storage; nil disables per-user-headers auth (resolver errors at use) Logger Logger - Tracer Tracer // Tracer for distributed tracing (nil = NoOpTracer) - InitialPoolSize int // Initial pool size for sync pools in Bifrost. Higher values will reduce memory allocations but will increase memory usage. - DropExcessRequests bool // If true, in cases where the queue is full, requests will not wait for the queue to be empty and will be dropped instead. - MCPConfig *MCPConfig // MCP (Model Context Protocol) configuration for tool integration - KeySelector KeySelector // Custom key selector function - KeyPoolFilter KeyPoolFilter // Optional hook to filter available keys before selection; nil = all keys eligible - KVStore KVStore // shared KV store for clustering/session stickiness; nil = disabled + Tracer Tracer // Tracer for distributed tracing (nil = NoOpTracer) + InitialPoolSize int // Initial pool size for sync pools in Bifrost. Higher values will reduce memory allocations but will increase memory usage. + DropExcessRequests bool // If true, in cases where the queue is full, requests will not wait for the queue to be empty and will be dropped instead. + MCPConfig *MCPConfig // MCP (Model Context Protocol) configuration for tool integration + KeySelector KeySelector // Custom key selector function + KeyPoolFilter KeyPoolFilter // Optional hook to filter available keys before selection; nil = all keys eligible + KVStore KVStore // shared KV store for clustering/session stickiness; nil = disabled + SessionAffinity SessionAffinity // Decides which key a session stays on; nil = Bifrost's own default, which needs KVStore to bind anything ModelCatalog ModelInfoProvider } @@ -81,6 +82,7 @@ const ( Wafer ModelProvider = "wafer" GithubCopilot ModelProvider = "github-copilot" Databricks ModelProvider = "databricks" + Typesafe ModelProvider = "typesafe" ) // SupportedBaseProviders is the list of base providers allowed for custom providers. @@ -142,6 +144,7 @@ var StandardProviders = []ModelProvider{ Wafer, GithubCopilot, Databricks, + Typesafe, } // RequestType represents the type of request being made to a provider. @@ -209,6 +212,7 @@ const ( ContainerFileContentRequest RequestType = "container_file_content" ContainerFileDeleteRequest RequestType = "container_file_delete" RerankRequest RequestType = "rerank" + DecisionRequest RequestType = "decisions" OCRRequest RequestType = "ocr" CountTokensRequest RequestType = "count_tokens" CompactionRequest RequestType = "compaction" @@ -432,6 +436,7 @@ const ( BifrostContextKeySSEReaderFactory BifrostContextKey = "bifrost-sse-reader-factory" // *providerUtils.SSEReaderFactory (set by enterprise — replaces default bufio.Scanner SSE readers with streaming readers) BifrostContextKeySessionID BifrostContextKey = "bifrost-session-id" // string session ID for the request (session stickiness) BifrostContextKeySessionTTL BifrostContextKey = "bifrost-session-ttl" // time.Duration session TTL for the request (session stickiness) + BifrostContextKeySessionAffinity BifrostContextKey = "bifrost-session-affinity" // bool: whether the request lets its session decide where it goes (default true) BifrostContextKeyMCPExtraHeaders BifrostContextKey = "bifrost-mcp-extra-headers" // map[string][]string (these headers are forwarded only to the MCP while tool execution if they are in the allowlist of the MCP client) BifrostContextKeyMCPLogID BifrostContextKey = "bifrost-mcp-log-id" // string (unique UUID for each MCP tool log entry - set per goroutine by agent executor - DO NOT SET THIS MANUALLY) BifrostContextKeyMCPHealthCheckRequest BifrostContextKey = "bifrost-mcp-health-check-request" // bool (set by bifrost - DO NOT SET THIS MANUALLY) - true when the MCP connect/ping/list-tools request was generated by bifrost itself (periodic health checks, or the admin connection-verification probe) rather than originating from a caller @@ -490,6 +495,11 @@ const ( RoutingEngineLoadbalancing = "loadbalancing" RoutingEngineModelCatalog = "model-catalog" RoutingEngineCircuitBreaker = "circuit-breaker" + // RoutingEngineSessionAffinity marks decisions a session made for the request: + // staying on the provider or key that served it before, ahead of what the + // routing engines proposed. Recorded so the request's trail says why it went + // where it went. + RoutingEngineSessionAffinity = "session-affinity" // RoutingEngineCore represents the Bifrost core orchestrator's own // routing decisions — primarily fallback transitions. Emitted when the // primary attempt fails and core advances through the fallback chain so @@ -573,6 +583,7 @@ type LargePayloadMetadata struct { type Fallback struct { Provider ModelProvider `json:"provider"` Model string `json:"model"` + KeyID string `json:"key_id,omitempty"` // pins a provider key for this attempt; empty means normal key selection } // BifrostRequest is the request struct for all bifrost requests. @@ -603,6 +614,7 @@ type BifrostRequest struct { CompactionRequest *BifrostCompactionRequest EmbeddingRequest *BifrostEmbeddingRequest RerankRequest *BifrostRerankRequest + DecisionRequest *BifrostDecisionRequest OCRRequest *BifrostOCRRequest SpeechRequest *BifrostSpeechRequest TranscriptionRequest *BifrostTranscriptionRequest @@ -671,6 +683,8 @@ func (br *BifrostRequest) GetRequestFields() (provider ModelProvider, model stri return br.EmbeddingRequest.Provider, br.EmbeddingRequest.Model, br.EmbeddingRequest.Fallbacks case br.RerankRequest != nil: return br.RerankRequest.Provider, br.RerankRequest.Model, br.RerankRequest.Fallbacks + case br.DecisionRequest != nil: + return br.DecisionRequest.Provider, br.DecisionRequest.Model, br.DecisionRequest.Fallbacks case br.OCRRequest != nil: return br.OCRRequest.Provider, br.OCRRequest.Model, br.OCRRequest.Fallbacks case br.SpeechRequest != nil: @@ -824,6 +838,8 @@ func (br *BifrostRequest) SetProvider(provider ModelProvider) { br.EmbeddingRequest.Provider = provider case br.RerankRequest != nil: br.RerankRequest.Provider = provider + case br.DecisionRequest != nil: + br.DecisionRequest.Provider = provider case br.OCRRequest != nil: br.OCRRequest.Provider = provider case br.SpeechRequest != nil: @@ -879,6 +895,8 @@ func (br *BifrostRequest) SetModel(model string) { br.EmbeddingRequest.Model = model case br.RerankRequest != nil: br.RerankRequest.Model = model + case br.DecisionRequest != nil: + br.DecisionRequest.Model = model case br.OCRRequest != nil: br.OCRRequest.Model = model case br.SpeechRequest != nil: @@ -936,6 +954,8 @@ func (br *BifrostRequest) SetFallbacks(fallbacks []Fallback) { br.EmbeddingRequest.Fallbacks = fallbacks case br.RerankRequest != nil: br.RerankRequest.Fallbacks = fallbacks + case br.DecisionRequest != nil: + br.DecisionRequest.Fallbacks = fallbacks case br.OCRRequest != nil: br.OCRRequest.Fallbacks = fallbacks case br.SpeechRequest != nil: @@ -979,6 +999,8 @@ func (br *BifrostRequest) SetRawRequestBody(rawRequestBody []byte) { br.EmbeddingRequest.RawRequestBody = rawRequestBody case br.RerankRequest != nil: br.RerankRequest.RawRequestBody = rawRequestBody + case br.DecisionRequest != nil: + br.DecisionRequest.RawRequestBody = rawRequestBody case br.OCRRequest != nil: br.OCRRequest.RawRequestBody = rawRequestBody case br.SpeechRequest != nil: @@ -1154,6 +1176,7 @@ type BifrostResponse struct { CompactionResponse *BifrostCompactionResponse EmbeddingResponse *BifrostEmbeddingResponse RerankResponse *BifrostRerankResponse + DecisionResponse *BifrostDecisionResponse OCRResponse *BifrostOCRResponse SpeechResponse *BifrostSpeechResponse SpeechStreamResponse *BifrostSpeechStreamResponse @@ -1217,6 +1240,8 @@ func (r *BifrostResponse) GetExtraFields() *BifrostResponseExtraFields { return &r.EmbeddingResponse.ExtraFields case r.RerankResponse != nil: return &r.RerankResponse.ExtraFields + case r.DecisionResponse != nil: + return &r.DecisionResponse.ExtraFields case r.OCRResponse != nil: return &r.OCRResponse.ExtraFields case r.SpeechResponse != nil: @@ -1485,6 +1510,11 @@ func (r *BifrostResponse) PopulateExtraFields(requestType RequestType, provider r.RerankResponse.ExtraFields.Provider = provider r.RerankResponse.ExtraFields.OriginalModelRequested = originalModelRequested r.RerankResponse.ExtraFields.ResolvedModelUsed = resolvedModel + case r.DecisionResponse != nil: + r.DecisionResponse.ExtraFields.RequestType = requestType + r.DecisionResponse.ExtraFields.Provider = provider + r.DecisionResponse.ExtraFields.OriginalModelRequested = originalModelRequested + r.DecisionResponse.ExtraFields.ResolvedModelUsed = resolvedModel case r.SpeechResponse != nil: r.SpeechResponse.ExtraFields.RequestType = requestType r.SpeechResponse.ExtraFields.Provider = provider diff --git a/core/schemas/billingheader.go b/core/schemas/billingheader.go new file mode 100644 index 00000000000..7d30000150a --- /dev/null +++ b/core/schemas/billingheader.go @@ -0,0 +1,131 @@ +package schemas + +import ( + "slices" + "strings" + "unicode" +) + +type anthropicBillingHeader struct { + system *ResponsesMessageContent + text *string + blocks []billingHeaderBlock +} + +type billingHeaderBlock struct { + index int + block ResponsesMessageContentBlock +} + +// ExtractAnthropicBillingHeader normalizes a freshly converted Messages request. +// Call only before sharing Input with plugins or attempts: compacting the owned +// system block slice here avoids cloning N messages on every non-Anthropic attempt. +// Only removed metadata is retained (O(H) space); prompt strings are never copied. +// RawRequestBody is untouched, so native passthrough still has the original body. +func (r *BifrostResponsesRequest) ExtractAnthropicBillingHeader() { + if r == nil || r.anthropicBillingHeader != nil || len(r.Input) == 0 { + return + } + message := &r.Input[0] // Messages ingress puts top-level system content first. + content := message.Content + if message.Role == nil || *message.Role != ResponsesInputMessageRoleSystem || content == nil { + return + } + if content.ContentStr != nil { + if isStandaloneBillingHeader(*content.ContentStr) { + r.anthropicBillingHeader = &anthropicBillingHeader{text: content.ContentStr} + r.Input = r.Input[1:] + } + return + } + kept := content.ContentBlocks[:0] + for i, block := range content.ContentBlocks { + if block.Type == ResponsesInputMessageContentBlockTypeText && block.Text != nil && isStandaloneBillingHeader(*block.Text) { + if r.anthropicBillingHeader == nil { + r.anthropicBillingHeader = &anthropicBillingHeader{system: content} + } + r.anthropicBillingHeader.blocks = append(r.anthropicBillingHeader.blocks, billingHeaderBlock{i, block}) + } else { + kept = append(kept, block) + } + } + if r.anthropicBillingHeader != nil { + clear(content.ContentBlocks[len(kept):]) + content.ContentBlocks = kept + if len(kept) == 0 { + r.anthropicBillingHeader.system = nil + r.Input = r.Input[1:] + } + } +} + +// WithAnthropicBillingHeader restores attribution on an attempt-local copy. +// Only normalized Anthropic attempts pay O(N+B+H) space for message/block structs; +// non-Anthropic attempts reuse the header-free request. The shared input and text +// bytes remain unchanged, including across retries and cross-family fallbacks. +func (r *BifrostResponsesRequest) WithAnthropicBillingHeader() *BifrostResponsesRequest { + if r == nil || r.anthropicBillingHeader == nil { + return r + } + header := r.anthropicBillingHeader + index := -1 + var blocks []ResponsesMessageContentBlock + if header.system != nil { + index = slices.IndexFunc(r.Input, func(m ResponsesMessage) bool { return m.Content == header.system }) + if index < 0 { + return r // A plugin replaced the system message; do not resurrect it. + } + blocks = r.Input[index].Content.ContentBlocks + } + content := &ResponsesMessageContent{ContentStr: header.text} + if header.text == nil { + content.ContentBlocks = make([]ResponsesMessageContentBlock, 0, len(blocks)+len(header.blocks)) + start := 0 + for i, saved := range header.blocks { + end := min(saved.index-i, len(blocks)) + content.ContentBlocks = append(content.ContentBlocks, blocks[start:end]...) + content.ContentBlocks = append(content.ContentBlocks, saved.block) + start = end + } + content.ContentBlocks = append(content.ContentBlocks, blocks[start:]...) + } + copy := *r + copy.anthropicBillingHeader = nil // Restoring an already prepared copy is a no-op. + if index >= 0 { + copy.Input = slices.Clone(r.Input) + copy.Input[index].Content = content + } else { + copy.Input = make([]ResponsesMessage, 0, len(r.Input)+1) + copy.Input = append(copy.Input, ResponsesMessage{Role: Ptr(ResponsesInputMessageRoleSystem), Content: content}) + copy.Input = append(copy.Input, r.Input...) + } + return © +} + +// IsStandaloneAnthropicBillingHeader reports whether text is exactly the billing +// metadata line ExtractAnthropicBillingHeader removes, for byte-level callers that +// never build a BifrostResponsesRequest. +func IsStandaloneAnthropicBillingHeader(text string) bool { + return isStandaloneBillingHeader(text) +} + +// Match an entire metadata line so mixed instruction blocks are left intact. +func isStandaloneBillingHeader(text string) bool { + metadata, ok := strings.CutPrefix(strings.TrimSpace(text), "x-anthropic-billing-header:") + if !ok || strings.ContainsAny(metadata, "\r\n") { + return false + } + found := false + for field := range strings.SplitSeq(metadata, ";") { + field = strings.TrimSpace(field) + if field == "" { + continue + } + key, value, ok := strings.Cut(field, "=") + if !ok || key == "" || value == "" || strings.IndexFunc(field, unicode.IsSpace) >= 0 { + return false + } + found = true + } + return found +} diff --git a/core/schemas/chatcompletions.go b/core/schemas/chatcompletions.go index 22c5f343cac..dfae6c7fafd 100644 --- a/core/schemas/chatcompletions.go +++ b/core/schemas/chatcompletions.go @@ -1944,6 +1944,9 @@ type BifrostLLMUsage struct { // the tier multiplier. json:"-" keeps them out of every serialized usage payload. Speed *string `json:"-"` InferenceGeo *string `json:"-"` + // ServiceTier is the served processing tier, carried for the same bare-usage + // billing path; json:"-" for the same reason. + ServiceTier *BifrostServiceTier `json:"-"` // Model that actually served the turn after a server-side fallback handoff. // Carried here for the same reason as the two above: the bare-usage billing path // (CalculateCostForUsage) never sees RoutingInfo, so without it a fallback-served @@ -2413,6 +2416,10 @@ func (u *BifrostLLMUsage) DeepCopy() *BifrostLLMUsage { g := *u.InferenceGeo c.InferenceGeo = &g } + if u.ServiceTier != nil { + st := *u.ServiceTier + c.ServiceTier = &st + } if u.ServerSideFallbackModel != nil { m := *u.ServerSideFallbackModel c.ServerSideFallbackModel = &m diff --git a/core/schemas/chatpayload_bench_test.go b/core/schemas/chatpayload_bench_test.go new file mode 100644 index 00000000000..9685d653d68 --- /dev/null +++ b/core/schemas/chatpayload_bench_test.go @@ -0,0 +1,288 @@ +package schemas + +// Benchmarks for the JSON payload handling every chat request pays for. +// +// Bifrost sits on the request path of every LLM call, so the cost of turning a +// provider payload into the neutral schema (and back) is charged once per +// request - and once per CHUNK on a stream, where a single response can carry +// several hundred of them. These benchmarks pin that cost: +// +// - response/chunk unmarshal: what the provider sends us +// - response/chunk marshal: what we send back to the caller +// - tool schema round-trip: OrderedMap key-order preservation, which is the +// most expensive part of a tools-heavy request +// - message marshal: ChatMessage's hand-written marshaller, which splices +// embedded tool/assistant fragments textually for every message on the wire +// +// Run: +// +// go test ./core/schemas/ -bench 'Chat|Tool' -benchmem + +import ( + "testing" +) + +// A non-streaming chat completion as an OpenAI-family provider returns it. +// OpenAI's response shape IS the neutral shape, so this is the payload the +// gateway parses on every non-streaming request to those providers. +const benchChatResponseJSON = `{ + "id": "chatcmpl-Bq9xK2LmNpQrStUvWxYz", + "object": "chat.completion", + "created": 1742914380, + "model": "gpt-4o-2024-11-20", + "system_fingerprint": "fp_9b1c2d3e4f", + "choices": [ + { + "index": 0, + "finish_reason": "tool_calls", + "logprobs": null, + "message": { + "role": "assistant", + "content": "I'll look up the current weather for both cities before answering.", + "tool_calls": [ + { + "id": "call_9xKmNpQrStUvWxYz01", + "type": "function", + "function": { + "name": "get_weather", + "arguments": "{\"location\":\"San Francisco, CA\",\"unit\":\"celsius\"}" + } + }, + { + "id": "call_9xKmNpQrStUvWxYz02", + "type": "function", + "function": { + "name": "get_weather", + "arguments": "{\"location\":\"Bengaluru, KA\",\"unit\":\"celsius\"}" + } + } + ] + } + } + ], + "usage": { + "prompt_tokens": 1284, + "completion_tokens": 96, + "total_tokens": 1380, + "prompt_tokens_details": {"cached_tokens": 1024}, + "completion_tokens_details": {"reasoning_tokens": 0} + } +}` + +// One streaming delta, the unit of work a streaming response repeats for every +// token it emits. +const benchChatStreamChunkJSON = `{ + "id": "chatcmpl-Bq9xK2LmNpQrStUvWxYz", + "object": "chat.completion.chunk", + "created": 1742914380, + "model": "gpt-4o-2024-11-20", + "choices": [ + { + "index": 0, + "finish_reason": null, + "logprobs": null, + "delta": {"content": " incrementally streamed token"} + } + ] +}` + +// A tool schema with nested properties and $defs. Key order matters here (LLMs +// are sensitive to it), which is why these go through OrderedMap rather than a +// plain Go map - and why the round-trip is worth tracking. +const benchToolSchemaJSON = `{ + "type": "object", + "properties": { + "location": {"type": "string", "description": "City and state, e.g. San Francisco, CA"}, + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, + "days": {"type": "integer", "minimum": 1, "maximum": 14}, + "include": { + "type": "array", + "items": {"type": "string", "enum": ["hourly", "daily", "alerts"]} + }, + "caller": {"$ref": "#/$defs/caller"} + }, + "required": ["location"], + "additionalProperties": false, + "$defs": { + "caller": { + "type": "object", + "properties": { + "id": {"type": "string"}, + "tier": {"type": "string", "enum": ["free", "pro", "enterprise"]} + }, + "required": ["id"] + } + } +}` + +func BenchmarkChatResponseUnmarshal(b *testing.B) { + data := []byte(benchChatResponseJSON) + b.ReportAllocs() + for b.Loop() { + var resp BifrostChatResponse + if err := Unmarshal(data, &resp); err != nil { + b.Fatalf("unmarshal chat response: %v", err) + } + } +} + +func BenchmarkChatResponseMarshal(b *testing.B) { + var resp BifrostChatResponse + if err := Unmarshal([]byte(benchChatResponseJSON), &resp); err != nil { + b.Fatalf("unmarshal chat response: %v", err) + } + b.ReportAllocs() + for b.Loop() { + out, err := Marshal(&resp) + if err != nil { + b.Fatalf("marshal chat response: %v", err) + } + if len(out) == 0 { + b.Fatal("empty payload") + } + } +} + +func BenchmarkChatStreamChunkUnmarshal(b *testing.B) { + data := []byte(benchChatStreamChunkJSON) + b.ReportAllocs() + for b.Loop() { + var chunk BifrostChatResponse + if err := Unmarshal(data, &chunk); err != nil { + b.Fatalf("unmarshal stream chunk: %v", err) + } + } +} + +func BenchmarkChatStreamChunkMarshal(b *testing.B) { + var chunk BifrostChatResponse + if err := Unmarshal([]byte(benchChatStreamChunkJSON), &chunk); err != nil { + b.Fatalf("unmarshal stream chunk: %v", err) + } + b.ReportAllocs() + for b.Loop() { + out, err := Marshal(&chunk) + if err != nil { + b.Fatalf("marshal stream chunk: %v", err) + } + if len(out) == 0 { + b.Fatal("empty payload") + } + } +} + +func BenchmarkToolSchemaUnmarshal(b *testing.B) { + data := []byte(benchToolSchemaJSON) + b.ReportAllocs() + for b.Loop() { + var params ToolFunctionParameters + if err := Unmarshal(data, ¶ms); err != nil { + b.Fatalf("unmarshal tool schema: %v", err) + } + } +} + +func BenchmarkToolSchemaMarshal(b *testing.B) { + var params ToolFunctionParameters + if err := Unmarshal([]byte(benchToolSchemaJSON), ¶ms); err != nil { + b.Fatalf("unmarshal tool schema: %v", err) + } + b.ReportAllocs() + for b.Loop() { + out, err := Marshal(¶ms) + if err != nil { + b.Fatalf("marshal tool schema: %v", err) + } + if len(out) == 0 { + b.Fatal("empty payload") + } + } +} + +// BenchmarkChatSchemaNormalized measures the deterministic re-ordering applied +// to tool schemas before they go out to a provider. It runs on every tools +// request and exists to keep provider-side prompt caches hitting. +func BenchmarkToolSchemaNormalized(b *testing.B) { + var params ToolFunctionParameters + if err := Unmarshal([]byte(benchToolSchemaJSON), ¶ms); err != nil { + b.Fatalf("unmarshal tool schema: %v", err) + } + b.ReportAllocs() + for b.Loop() { + if got := params.Normalized(); got == nil { + b.Fatal("nil normalized params") + } + } +} + +// benchChatMessages is a short multi-turn conversation with a tool call and its +// result - the shape ChatMessage's hand-written marshaller has to splice. +func benchChatMessages() []ChatMessage { + return []ChatMessage{ + { + Role: ChatMessageRoleSystem, + Content: &ChatMessageContent{ContentStr: Ptr("You are a concise assistant. Answer in one sentence.")}, + }, + { + Role: ChatMessageRoleUser, + Content: &ChatMessageContent{ContentStr: Ptr("What's the weather in San Francisco and Bengaluru?")}, + }, + { + Role: ChatMessageRoleAssistant, + Content: &ChatMessageContent{ContentStr: Ptr("")}, + ChatAssistantMessage: &ChatAssistantMessage{ + ToolCalls: []ChatAssistantMessageToolCall{ + { + ID: Ptr("call_9xKmNpQrStUvWxYz01"), + Type: Ptr("function"), + Function: ChatAssistantMessageToolCallFunction{ + Name: Ptr("get_weather"), + Arguments: `{"location":"San Francisco, CA","unit":"celsius"}`, + }, + }, + }, + }, + }, + { + Role: ChatMessageRoleTool, + Content: &ChatMessageContent{ContentStr: Ptr(`{"temp_c":14.4,"condition":"foggy"}`)}, + ChatToolMessage: &ChatToolMessage{ + ToolCallID: Ptr("call_9xKmNpQrStUvWxYz01"), + }, + }, + { + Role: ChatMessageRoleUser, + Content: &ChatMessageContent{ContentBlocks: []ChatContentBlock{ + {Type: ChatContentBlockTypeText, Text: Ptr("And how does that compare to yesterday?")}, + }}, + }, + } +} + +func BenchmarkChatMessagesMarshal(b *testing.B) { + messages := benchChatMessages() + b.ReportAllocs() + for b.Loop() { + out, err := Marshal(messages) + if err != nil { + b.Fatalf("marshal messages: %v", err) + } + if len(out) == 0 { + b.Fatal("empty payload") + } + } +} + +func BenchmarkChatMessagesUnmarshal(b *testing.B) { + data, err := Marshal(benchChatMessages()) + if err != nil { + b.Fatalf("marshal messages: %v", err) + } + b.ReportAllocs() + for b.Loop() { + var messages []ChatMessage + if err := Unmarshal(data, &messages); err != nil { + b.Fatalf("unmarshal messages: %v", err) + } + } +} diff --git a/core/schemas/context.go b/core/schemas/context.go index a9ff125d9da..798d27dadf0 100644 --- a/core/schemas/context.go +++ b/core/schemas/context.go @@ -517,6 +517,11 @@ func (bc *BifrostContext) SetRoutingInfoSnapshot(ri RoutingInfo) { bc.setReservedValue(BifrostContextKeyRoutingInfo, ri) } +// SetFallbackPinnedAPIKeyID pins a fallback's provider key, bypassing the restricted-writes guard (set by core - DO NOT SET THIS MANUALLY). +func (bc *BifrostContext) SetFallbackPinnedAPIKeyID(keyID string) { + bc.setReservedValue(BifrostContextKeyAPIKeyID, keyID) +} + // ClearValue clears a value from the internal userValues map. // For scoped contexts, delegates to the root context via valueDelegate. func (bc *BifrostContext) ClearValue(key any) { diff --git a/core/schemas/decision.go b/core/schemas/decision.go new file mode 100644 index 00000000000..8c0f65947aa --- /dev/null +++ b/core/schemas/decision.go @@ -0,0 +1,69 @@ +package schemas + +// DecisionKind identifies how a single question is decided. The vocabulary +// mirrors Typesafe's System One question types. +type DecisionKind string + +const ( + // DecisionKindNoul yields a probability between 0 and 1. + DecisionKindNoul DecisionKind = "noul" + // DecisionKindChoice yields one option from the question's criteria. + DecisionKindChoice DecisionKind = "choice" + // DecisionKindScore yields a numeric rubric score, including fractions. + DecisionKindScore DecisionKind = "score" +) + +// DecisionQuestion is one named question in a decision request. Instructions +// accepts a string, object, or array, carried losslessly. Criteria is a map of +// descriptions for noul (only "true"/"false" keys) and choice (option -> +// description), and an ordered array of level descriptions for score. +type DecisionQuestion struct { + Kind DecisionKind `json:"kind"` + Instructions interface{} `json:"instructions,omitempty"` + Criteria interface{} `json:"criteria,omitempty"` +} + +// BifrostDecisionRequest represents a request to evaluate state against a map +// of named questions. The shape mirrors Typesafe's System One endpoint. +type BifrostDecisionRequest struct { + Provider ModelProvider `json:"provider"` + Model string `json:"model"` + State interface{} `json:"state"` // string, object, or array + Questions map[string]DecisionQuestion `json:"questions"` + Fallbacks []Fallback `json:"fallbacks,omitempty"` + RawRequestBody []byte `json:"-"` +} + +// GetRawRequestBody returns the raw request body for the decision request. +func (r *BifrostDecisionRequest) GetRawRequestBody() []byte { + return r.RawRequestBody +} + +// DecisionAnswer is one evaluated answer. Value carries the decided value for +// the question's kind: a number in [0,1] for noul, an option string for +// choice, a numeric rubric score for score. Confidence, Probabilities, and +// Legend carry per-field metadata when the provider supplies it. Legend echoes +// each score level's description verbatim, so its values are strings, objects, +// or arrays - whatever the criteria carried. +type DecisionAnswer struct { + Kind DecisionKind `json:"kind"` + Value interface{} `json:"value"` + Confidence *float64 `json:"confidence,omitempty"` + Probabilities map[string]float64 `json:"probabilities,omitempty"` + Legend map[string]any `json:"legend,omitempty"` +} + +// BifrostDecisionResponse represents the response from a decision request. +// Answers is keyed by question identifier; every requested question produces +// an answer. +type BifrostDecisionResponse struct { + ID string `json:"id,omitempty"` + Model string `json:"model"` + Answers map[string]DecisionAnswer `json:"answers"` + Usage *BifrostLLMUsage `json:"usage,omitempty"` + // Tier the emulating text call was served on, which prices the decision as that call. + ServiceTier *BifrostServiceTier `json:"service_tier,omitempty"` + Speed *string `json:"speed,omitempty"` + InferenceGeo *string `json:"inference_geo,omitempty"` + ExtraFields BifrostResponseExtraFields `json:"extra_fields"` +} diff --git a/core/schemas/errors.go b/core/schemas/errors.go index 9ec540ceb3f..9d0d8548aa7 100644 --- a/core/schemas/errors.go +++ b/core/schemas/errors.go @@ -105,6 +105,7 @@ var modelAddressedRequestTypes = map[RequestType]struct{}{ ImageGenerationRequest: {}, ImageGenerationStreamRequest: {}, RerankRequest: {}, + DecisionRequest: {}, CountTokensRequest: {}, } diff --git a/core/schemas/errors_test.go b/core/schemas/errors_test.go index d7cfdcf89fc..25840c253c5 100644 --- a/core/schemas/errors_test.go +++ b/core/schemas/errors_test.go @@ -417,6 +417,7 @@ var modelAddressingByRequestType = map[string]bool{ "ImageVariationRequest": false, "VideoGenerationRequest": false, "RerankRequest": true, + "DecisionRequest": true, "OCRRequest": false, "CountTokensRequest": true, diff --git a/core/schemas/modelcapabilities.go b/core/schemas/modelcapabilities.go index e6ab4a0d671..857bdaca6ce 100644 --- a/core/schemas/modelcapabilities.go +++ b/core/schemas/modelcapabilities.go @@ -33,6 +33,7 @@ type ModelCapabilities struct { SupportsCodeExecution *bool `json:"supports_code_execution,omitempty"` SupportsBashTool *bool `json:"supports_bash_tool,omitempty"` SupportsTextEditorTool *bool `json:"supports_text_editor_tool,omitempty"` + SupportsComputerToolset *bool `json:"supports_computer_toolset,omitempty"` // accepts the computer_toolset_20260801 client toolset; absent ⇒ name detection SupportsMemoryTool *bool `json:"supports_memory_tool,omitempty"` SupportsToolSearch *bool `json:"supports_tool_search,omitempty"` ToolNameMaxLength *int `json:"tool_name_max_length,omitempty"` // longest tool name the wire accepts; absent falls back to the per-provider default in core/providers/utils (64 for OpenAI-compatible wires and Bedrock, 128 for Anthropic and Gemini) @@ -53,13 +54,16 @@ type ModelCapabilities struct { SupportsInputExamples *bool `json:"supports_input_examples,omitempty"` SupportsAdvisorTool *bool `json:"supports_advisor_tool,omitempty"` SupportsInferenceGeo *bool `json:"supports_inference_geo,omitempty"` + SupportsSafeguards *bool `json:"supports_safeguards,omitempty"` // Claude Code auto-mode classifier (safeguards/safeguard_results), model-gated on Anthropic and cloud surfaces (Sonnet 5, Opus 4.7+, Fable). SupportsPromptCachingScope *bool `json:"supports_prompt_caching_scope,omitempty"` SupportsExtendedCacheTTL *bool `json:"supports_extended_cache_ttl,omitempty"` + SupportsPromptCacheBreakpoint *bool `json:"supports_prompt_cache_breakpoint,omitempty"` // accepts OpenAI prompt_cache_breakpoint + prompt_cache_options SupportsReasoningContentBlocks *bool `json:"supports_reasoning_content_blocks,omitempty"` SupportsMultimodalToolOutput *bool `json:"supports_multimodal_tool_output,omitempty"` SupportsResponseSchemaWithTools *bool `json:"supports_response_schema_with_tools,omitempty"` SupportsForcedToolChoice *bool `json:"supports_forced_tool_choice,omitempty"` // false ⇒ tool_choice any/tool rejected (Fable 5.1+) SupportsPromptCacheBreakpoints *bool `json:"supports_prompt_cache_breakpoints,omitempty"` // Responses input_text accepts prompt_cache_breakpoint (Claude via OpenRouter, gpt-5.6+) + SupportsAsyncTools *bool `json:"supports_async_tools,omitempty"` // accepts OpenAI async on tools and replayed call items // Baseline request-surface flags. These drive the compat plugin's // parameter allowlist rather than provider request shaping, so they are diff --git a/core/schemas/modelcaps.go b/core/schemas/modelcaps.go index ce4da885c1c..4ab0c2a2fef 100644 --- a/core/schemas/modelcaps.go +++ b/core/schemas/modelcaps.go @@ -105,11 +105,35 @@ func (c ModelCaps) SupportsFastMode(fallback bool) bool { return fallback } +// SupportsSafeguards returns true if the model supports the Claude Code +// auto-mode server-side classifier (`safeguards` request field / +// `safeguard_results` response field) on surfaces where the feature is +// model-gated. Auto mode on Amazon Bedrock, Google Cloud's Agent Platform, +// Microsoft Foundry, and Claude apps gateway sessions is supported only on +// Sonnet 5, Opus 4.7 or later, and the Fable models. Anthropic direct uses the +// same model gate. Payloads are forwarded opaquely after capability filtering. +// +// Sources: +// - https://code.claude.com/docs/en/auto-mode-classifier-billing +// - https://code.claude.com/docs/en/permission-modes#enable-auto-mode-on-bedrock-agent-platform-or-foundry +// +// Prefers the datasheet's supports_safeguards boolean when set, falling back to +// name detection when no record is registered. +func (c ModelCaps) SupportsSafeguards(fallback bool) bool { + if c.record != nil && c.record.SupportsSafeguards != nil { + return *c.record.SupportsSafeguards + } + return fallback +} + // Wire field names used as UnsupportedFields and ConditionallyUnsupportedFields // keys. Call sites pass these rather than string literals so a typo fails to // compile instead of silently reading as "supported". const ( FieldTopP = "top_p" + FieldTemperature = "temperature" + FieldTopLogprobs = "top_logprobs" + FieldLogprobs = "logprobs" FieldPresencePenalty = "presence_penalty" FieldFrequencyPenalty = "frequency_penalty" FieldStop = "stop" @@ -254,6 +278,24 @@ func (c ModelCaps) SupportsCachePoint(fallback bool) bool { return fallback } +// SupportsPromptCacheBreakpoint reports whether the model accepts OpenAI's +// per-block prompt_cache_breakpoint and request-level prompt_cache_options. +func (c ModelCaps) SupportsPromptCacheBreakpoint(fallback bool) bool { + if c.record != nil && c.record.SupportsPromptCacheBreakpoint != nil { + return *c.record.SupportsPromptCacheBreakpoint + } + return fallback +} + +// SupportsAsyncTools reports whether the model accepts OpenAI's async flag on tool +// definitions and on replayed function/custom call items. +func (c ModelCaps) SupportsAsyncTools(fallback bool) bool { + if c.record != nil && c.record.SupportsAsyncTools != nil { + return *c.record.SupportsAsyncTools + } + return fallback +} + // SupportsPromptCaching reports whether the model supports explicit prompt caching // at all. It is the base feature that SupportsPromptCachingScope and // SupportsExtendedCacheTTL refine, and it is what gates breakpoint injection: a @@ -609,6 +651,17 @@ func (c ModelCaps) SupportsFilesAPI(fallback bool) bool { return fallback } +// SupportsComputerToolset reports whether the model accepts the +// computer_toolset_20260801 client toolset. Distinct from the dated computer_* +// tools: most models that take the toolset still accept the dated form too, and +// Opus 5.5 on the Claude API and Google Cloud takes only the toolset. +func (c ModelCaps) SupportsComputerToolset(fallback bool) bool { + if c.record != nil && c.record.SupportsComputerToolset != nil { + return *c.record.SupportsComputerToolset + } + return fallback +} + // SupportsTextEditorTool reports whether the model accepts the text_editor client tool. func (c ModelCaps) SupportsTextEditorTool(fallback bool) bool { if c.record != nil && c.record.SupportsTextEditorTool != nil { diff --git a/core/schemas/mux.go b/core/schemas/mux.go index 76fcc15c4ee..01a5c71e43b 100644 --- a/core/schemas/mux.go +++ b/core/schemas/mux.go @@ -585,8 +585,12 @@ func (cm *ChatMessage) ToResponsesMessages() []ResponsesMessage { if messageType == ResponsesMessageTypeFunctionCallOutput { // Don't set content for function_call_output - it will be set in ResponsesToolMessage.Output } else { - responseBlocks := make([]ResponsesMessageContentBlock, len(cm.Content.ContentBlocks)) - for i, block := range cm.Content.ContentBlocks { + responseBlocks := make([]ResponsesMessageContentBlock, 0, len(cm.Content.ContentBlocks)) + for _, block := range cm.Content.ContentBlocks { + // Responses blocks have no cachePoint equivalent; a standalone marker would serialize as an empty-type block. + if block.Type == "" && block.CachePoint != nil { + continue + } blockType := ResponsesMessageContentBlockType(block.Type) switch block.Type { @@ -604,33 +608,34 @@ func (cm *ChatMessage) ToResponsesMessages() []ResponsesMessage { blockType = ResponsesInputMessageContentBlockTypeAudio } - responseBlocks[i] = ResponsesMessageContentBlock{ + responseBlocks = append(responseBlocks, ResponsesMessageContentBlock{ Type: blockType, Text: block.Text, - } + }) + rb := &responseBlocks[len(responseBlocks)-1] // Convert specific block types if block.ImageURLStruct != nil { - responseBlocks[i].ResponsesInputMessageContentBlockImage = &ResponsesInputMessageContentBlockImage{ + rb.ResponsesInputMessageContentBlockImage = &ResponsesInputMessageContentBlockImage{ ImageURL: &block.ImageURLStruct.URL, Detail: block.ImageURLStruct.Detail, } } if block.File != nil { - responseBlocks[i].ResponsesInputMessageContentBlockFile = &ResponsesInputMessageContentBlockFile{ + rb.ResponsesInputMessageContentBlockFile = &ResponsesInputMessageContentBlockFile{ FileData: block.File.FileData, FileURL: block.File.FileURL, Filename: block.File.Filename, FileType: block.File.FileType, } - responseBlocks[i].FileID = block.File.FileID + rb.FileID = block.File.FileID } if block.InputAudio != nil { format := "" if block.InputAudio.Format != nil { format = *block.InputAudio.Format } - responseBlocks[i].Audio = &ResponsesInputMessageContentBlockAudio{ + rb.Audio = &ResponsesInputMessageContentBlockAudio{ Data: block.InputAudio.Data, Format: format, } @@ -854,7 +859,7 @@ func ToChatMessages(rms []ResponsesMessage) []ChatMessage { // result collapse to the chat surface's single bool. Mirrors // the same pair the Anthropic Responses converter treats as // is_error (providers/anthropic/responses.go). - if (rm.ResponsesToolMessage.Error != nil && *rm.ResponsesToolMessage.Error != "") || + if rm.ResponsesToolMessage.Error.IsError() || (rm.Status != nil && *rm.Status == "incomplete") { cm.ChatToolMessage.IsError = Ptr(true) } diff --git a/core/schemas/provider.go b/core/schemas/provider.go index 585f85afb93..44215367be8 100644 --- a/core/schemas/provider.go +++ b/core/schemas/provider.go @@ -349,6 +349,7 @@ type AllowedRequests struct { Compaction bool `json:"compaction"` Embedding bool `json:"embedding"` Rerank bool `json:"rerank"` + Decision bool `json:"decisions"` OCR bool `json:"ocr"` Speech bool `json:"speech"` SpeechStream bool `json:"speech_stream"` @@ -434,6 +435,8 @@ func (ar *AllowedRequests) IsOperationAllowed(operation RequestType) bool { return ar.Embedding case RerankRequest: return ar.Rerank + case DecisionRequest: + return ar.Decision case OCRRequest: return ar.OCR case SpeechRequest: @@ -706,6 +709,8 @@ type Provider interface { Embedding(ctx *BifrostContext, key Key, request *BifrostEmbeddingRequest) (*BifrostEmbeddingResponse, *BifrostError) // Rerank performs a rerank request to reorder documents by relevance to a query Rerank(ctx *BifrostContext, key Key, request *BifrostRerankRequest) (*BifrostRerankResponse, *BifrostError) + // Decision performs an decision request against an annotated function-tool definition (Typesafe-only; other providers return unsupported) + Decision(ctx *BifrostContext, key Key, request *BifrostDecisionRequest) (*BifrostDecisionResponse, *BifrostError) // OCR performs an optical character recognition request on a document OCR(ctx *BifrostContext, key Key, request *BifrostOCRRequest) (*BifrostOCRResponse, *BifrostError) // Speech performs a text to speech request diff --git a/core/schemas/responses.go b/core/schemas/responses.go index 1fd0b7cb725..2ef0c1db672 100644 --- a/core/schemas/responses.go +++ b/core/schemas/responses.go @@ -52,6 +52,10 @@ type BifrostResponsesRequest struct { // target wire does not support namespace tools, and the response path reads it to // restore function_call items. Never serialized; the shared request never has it. NamespaceToolAliases map[string]NamespaceToolAlias `json:"-"` + + // Removed at Messages ingress, before Input is shared. Shallow fallback copies + // retain this private metadata; only Anthropic attempts restore it. + anthropicBillingHeader *anthropicBillingHeader } func (r *BifrostResponsesRequest) GetRawRequestBody() []byte { @@ -249,11 +253,12 @@ type BifrostResponsesResponse struct { Reasoning *ResponsesParametersReasoning `json:"reasoning"` // Configuration options for reasoning models SafetyIdentifier *string `json:"safety_identifier"` // Safety identifier ServiceTier *BifrostServiceTier `json:"service_tier"` - Speed *string `json:"speed,omitempty"` // "fast" | "standard" — speed actually served (Anthropic fast mode); drives fast-mode billing - InferenceGeo *string `json:"inference_geo,omitempty"` // "us" | "global" — inference geography served (Anthropic data residency); drives the 1.1x US multiplier - Diagnostics *CacheDiagnostics `json:"diagnostics,omitempty"` // Anthropic cache diagnostics (cache-diagnosis-2026-04-07); first prompt-cache prefix divergence point - Container *ResponsesResponseContainer `json:"container,omitempty"` // Code-execution sandbox container (Anthropic surfaces it on the response / final streaming message_delta). The neutral per-call id also lives on ResponsesCodeInterpreterToolCall.ContainerID. - Status *string `json:"status,omitempty"` // completed, failed, in_progress, cancelled, queued, or incomplete + Speed *string `json:"speed,omitempty"` // "fast" | "standard" — speed actually served (Anthropic fast mode); drives fast-mode billing + InferenceGeo *string `json:"inference_geo,omitempty"` // "us" | "global" — inference geography served (Anthropic data residency); drives the 1.1x US multiplier + Diagnostics *CacheDiagnostics `json:"diagnostics,omitempty"` // Anthropic cache diagnostics (cache-diagnosis-2026-04-07); first prompt-cache prefix divergence point + SafeguardResults json.RawMessage `json:"safeguard_results,omitempty"` // Claude Code auto-mode classifier verdicts (opaque; forwarded unchanged per the gateway compatibility guide). Not copied by WithDefaults, so OpenAI-shaped surfaces never see it. + Container *ResponsesResponseContainer `json:"container,omitempty"` // Code-execution sandbox container (Anthropic surfaces it on the response / final streaming message_delta). The neutral per-call id also lives on ResponsesCodeInterpreterToolCall.ContainerID. + Status *string `json:"status,omitempty"` // completed, failed, in_progress, cancelled, queued, or incomplete StreamOptions *ResponsesStreamOptions `json:"stream_options,omitempty"` StopReason *string `json:"stop_reason,omitempty"` // Not in OpenAI's spec, but sent by other providers StopDetails *ResponsesStopDetails `json:"stop_details,omitempty"` // Anthropic refusal detail; null unless stop_reason is "refusal" @@ -505,31 +510,31 @@ type PromptCacheBreakpoint struct { } type ResponsesParameters struct { - Background *bool `json:"background,omitempty"` - Conversation *string `json:"conversation,omitempty"` - Include []string `json:"include,omitempty"` // Supported values: "web_search_call.action.sources", "code_interpreter_call.outputs", "computer_call_output.output.image_url", "file_search_call.results", "message.input_image.image_url", "message.output_text.logprobs", "reasoning.encrypted_content" - Instructions *string `json:"instructions,omitempty"` - MaxOutputTokens *int `json:"max_output_tokens,omitempty"` - MaxToolCalls *int `json:"max_tool_calls,omitempty"` - Metadata *map[string]any `json:"metadata,omitempty"` - ParallelToolCalls *bool `json:"parallel_tool_calls,omitempty"` - PreviousResponseID *string `json:"previous_response_id,omitempty"` - PromptCacheKey *string `json:"prompt_cache_key,omitempty"` // Prompt cache key - PromptCacheRetention *string `json:"prompt_cache_retention,omitempty"` - PromptCacheOptions *PromptCacheOptions `json:"prompt_cache_options,omitempty"` // Request-wide prompt cache options (OpenAI gpt-5.6+) - Reasoning *ResponsesParametersReasoning `json:"reasoning,omitempty"` // Configuration options for reasoning models - SafetyIdentifier *string `json:"safety_identifier,omitempty"` // Safety identifier - ServiceTier *BifrostServiceTier `json:"service_tier,omitempty"` - StreamOptions *ResponsesStreamOptions `json:"stream_options,omitempty"` - Store *bool `json:"store,omitempty"` - Temperature *float64 `json:"temperature,omitempty"` - Text *ResponsesTextConfig `json:"text,omitempty"` - TopLogProbs *int `json:"top_logprobs,omitempty"` - TopP *float64 `json:"top_p,omitempty"` // Controls diversity via nucleus sampling - ToolChoice *ResponsesToolChoice `json:"tool_choice,omitempty"` // Whether to call a tool - Tools []ResponsesTool `json:"tools,omitempty"` // Tools to use - Truncation *string `json:"truncation,omitempty"` - User *string `json:"user,omitempty"` + Background *bool `json:"background,omitempty"` + Conversation *ResponsesResponseConversation `json:"conversation,omitempty"` + Include []string `json:"include,omitempty"` // Supported values: "web_search_call.action.sources", "code_interpreter_call.outputs", "computer_call_output.output.image_url", "file_search_call.results", "message.input_image.image_url", "message.output_text.logprobs", "reasoning.encrypted_content" + Instructions *string `json:"instructions,omitempty"` + MaxOutputTokens *int `json:"max_output_tokens,omitempty"` + MaxToolCalls *int `json:"max_tool_calls,omitempty"` + Metadata *map[string]any `json:"metadata,omitempty"` + ParallelToolCalls *bool `json:"parallel_tool_calls,omitempty"` + PreviousResponseID *string `json:"previous_response_id,omitempty"` + PromptCacheKey *string `json:"prompt_cache_key,omitempty"` // Prompt cache key + PromptCacheRetention *string `json:"prompt_cache_retention,omitempty"` + PromptCacheOptions *PromptCacheOptions `json:"prompt_cache_options,omitempty"` // Request-wide prompt cache options (OpenAI gpt-5.6+) + Reasoning *ResponsesParametersReasoning `json:"reasoning,omitempty"` // Configuration options for reasoning models + SafetyIdentifier *string `json:"safety_identifier,omitempty"` // Safety identifier + ServiceTier *BifrostServiceTier `json:"service_tier,omitempty"` + StreamOptions *ResponsesStreamOptions `json:"stream_options,omitempty"` + Store *bool `json:"store,omitempty"` + Temperature *float64 `json:"temperature,omitempty"` + Text *ResponsesTextConfig `json:"text,omitempty"` + TopLogProbs *int `json:"top_logprobs,omitempty"` + TopP *float64 `json:"top_p,omitempty"` // Controls diversity via nucleus sampling + ToolChoice *ResponsesToolChoice `json:"tool_choice,omitempty"` // Whether to call a tool + Tools []ResponsesTool `json:"tools,omitempty"` // Tools to use + Truncation *string `json:"truncation,omitempty"` + User *string `json:"user,omitempty"` // Opts into running built-in server-side tools (e.g. Google Search) in the same // turn as function declarations. Required by Gemini 3+, which otherwise rejects @@ -1179,14 +1184,14 @@ func (rc *ResponsesResponseConversation) UnmarshalJSON(data []byte) error { // First, try to unmarshal as a direct string var stringContent string if err := Unmarshal(data, &stringContent); err == nil { - rc.ResponsesResponseConversationStr = &stringContent + *rc = ResponsesResponseConversation{ResponsesResponseConversationStr: &stringContent} return nil } // Try to unmarshal as a direct array of ContentBlock var structContent ResponsesResponseConversationStruct if err := Unmarshal(data, &structContent); err == nil { - rc.ResponsesResponseConversationStruct = &structContent + *rc = ResponsesResponseConversation{ResponsesResponseConversationStruct: &structContent} return nil } @@ -1454,7 +1459,7 @@ const ( ResponsesMessageTypeImageGenerationCall ResponsesMessageType = "image_generation_call" ResponsesMessageTypeMCPListTools ResponsesMessageType = "mcp_list_tools" ResponsesMessageTypeMCPApprovalRequest ResponsesMessageType = "mcp_approval_request" - ResponsesMessageTypeMCPApprovalResponses ResponsesMessageType = "mcp_approval_responses" + ResponsesMessageTypeMCPApprovalResponses ResponsesMessageType = "mcp_approval_response" ResponsesMessageTypeReasoning ResponsesMessageType = "reasoning" ResponsesMessageTypeItemReference ResponsesMessageType = "item_reference" ResponsesMessageTypeRefusal ResponsesMessageType = "refusal" @@ -1735,20 +1740,40 @@ func (rc ResponsesMessageContent) MarshalJSON() ([]byte, error) { // It determines whether "content" is a string or array and assigns to the appropriate field. // It also handles direct string/array content without a wrapper object. func (rc *ResponsesMessageContent) UnmarshalJSON(data []byte) error { - // First, try to unmarshal as a direct string - var stringContent string - if err := Unmarshal(data, &stringContent); err == nil { - rc.ContentStr = &stringContent - return nil - } - - // Try to unmarshal as a direct array of ContentBlock - var arrayContent []ResponsesMessageContentBlock - if err := Unmarshal(data, &arrayContent); err == nil { - rc.ContentBlocks = arrayContent - return nil + // Peek the first non-whitespace byte to pick the decode path directly: a + // failed whole-value unmarshal attempt still builds and discards a full DOM, + // and content is the largest field in a multimodal request. + for _, b := range data { + switch b { + case ' ', '\t', '\r', '\n': + continue + case '"': + var stringContent string + if err := Unmarshal(data, &stringContent); err != nil { + return fmt.Errorf("content field is neither a string nor an array of Content blocks") + } + rc.ContentStr = &stringContent + return nil + case '[': + var arrayContent []ResponsesMessageContentBlock + if err := Unmarshal(data, &arrayContent); err != nil { + return fmt.Errorf("content field is neither a string nor an array of Content blocks") + } + rc.ContentBlocks = arrayContent + return nil + case 'n': + // A null content is valid per the OpenAI spec. Decoding null into a + // string yields "", matching what the previous try-string-first + // implementation produced. + var nullContent string + if err := Unmarshal(data, &nullContent); err != nil { + return fmt.Errorf("content field is neither a string nor an array of Content blocks") + } + rc.ContentStr = &nullContent + return nil + } + break } - return fmt.Errorf("content field is neither a string nor an array of Content blocks") } @@ -1910,11 +1935,16 @@ type ResponsesToolMessage struct { Namespace *string `json:"namespace,omitempty"` // Namespace for function_call items (set by OpenAI when namespace tools are used) Arguments *string `json:"arguments,omitempty"` Execution *string `json:"execution,omitempty"` // "client" on deferred calls (e.g. tool_search_call); Codex needs it to dispatch the call + Async *bool `json:"async,omitempty"` // true on function/custom calls to async tools; must survive replay or OpenAI 400s Output *ResponsesToolMessageOutputStruct `json:"output,omitempty"` Action *ResponsesToolMessageActionStruct `json:"action,omitempty"` - Error *string `json:"error,omitempty"` + Error *ResponsesToolMessageError `json:"error,omitempty"` // Caller is the neutral form of Anthropic's "caller" union on server-tool blocks Caller *ResponsesToolCaller `json:"tool_caller,omitempty"` + // ToolsetName is the client toolset a member call belongs to ("computer" for + // computer_toolset_20260801). Anthropic requires it on both halves of a + // call/result pair or neither, so it rides the call and the output alike. + ToolsetName *string `json:"toolset_name,omitempty"` // Tool calls and outputs *ResponsesFileSearchToolCall @@ -2200,6 +2230,86 @@ func (output *ResponsesToolMessageOutputStruct) UnmarshalJSON(data []byte) error return fmt.Errorf("responses tool message output struct is neither a string nor an array of responses message content blocks nor a computer tool call output data nor an image generation call output") } +// ResponsesToolMessageError is a tool item's error: a plain string (mcp_list_tools, +// and mcp_call before OpenAI structured it) or a structured mcp_call error object. +type ResponsesToolMessageError struct { + ResponsesToolMessageErrorStr *string + ResponsesToolMessageErrorStruct *ResponsesToolMessageErrorStruct +} + +// ResponsesToolMessageErrorStruct is the structured mcp_call error: mcp_protocol_error +// and http_error carry code and message, mcp_tool_execution_error carries content. +type ResponsesToolMessageErrorStruct struct { + Type string `json:"type"` + Code *int `json:"code,omitempty"` + Message *string `json:"message,omitempty"` + Content json.RawMessage `json:"content,omitempty"` // untyped in OpenAI's spec, kept verbatim +} + +func (e ResponsesToolMessageError) MarshalJSON() ([]byte, error) { + if e.ResponsesToolMessageErrorStr != nil && e.ResponsesToolMessageErrorStruct != nil { + return nil, fmt.Errorf("both ResponsesToolMessageErrorStr and ResponsesToolMessageErrorStruct are set; only one should be non-nil") + } + if e.ResponsesToolMessageErrorStr != nil { + return MarshalSorted(*e.ResponsesToolMessageErrorStr) + } + if e.ResponsesToolMessageErrorStruct != nil { + return MarshalSorted(e.ResponsesToolMessageErrorStruct) + } + return MarshalSorted(nil) +} + +func (e *ResponsesToolMessageError) UnmarshalJSON(data []byte) error { + var str string + if err := Unmarshal(data, &str); err == nil { + *e = ResponsesToolMessageError{ResponsesToolMessageErrorStr: &str} + return nil + } + var errStruct ResponsesToolMessageErrorStruct + if err := Unmarshal(data, &errStruct); err == nil { + *e = ResponsesToolMessageError{ResponsesToolMessageErrorStruct: &errStruct} + return nil + } + return fmt.Errorf("responses tool message error is neither a string nor an error object") +} + +// IsError reports whether the union represents a failed tool call. A structured +// error is a failure even when it carries no renderable text. The legacy string +// form preserves its historical empty-string-is-success behavior. +func (e *ResponsesToolMessageError) IsError() bool { + if e == nil { + return false + } + if e.ResponsesToolMessageErrorStruct != nil { + return true + } + return e.ResponsesToolMessageErrorStr != nil && *e.ResponsesToolMessageErrorStr != "" +} + +// Text returns the error as plain text; nil-safe, "" when there is no error. +func (e *ResponsesToolMessageError) Text() string { + if e == nil { + return "" + } + if e.ResponsesToolMessageErrorStr != nil { + return *e.ResponsesToolMessageErrorStr + } + if s := e.ResponsesToolMessageErrorStruct; s != nil { + if s.Message != nil { + return *s.Message + } + if len(s.Content) > 0 && string(s.Content) != "null" { + var str string + if err := Unmarshal(s.Content, &str); err == nil { + return str + } + return string(s.Content) + } + return s.Type + } + return "" +} + // ============================================================================= // 4. TOOL CALL STRUCTURES (organized by tool type) // ============================================================================= @@ -2290,8 +2400,9 @@ type ResponsesWebSearchToolCallAction struct { // ResponsesWebSearchToolCallActionSearchSource represents a web search action search source type ResponsesWebSearchToolCallActionSearchSource struct { - Type string `json:"type"` // always "url" - URL string `json:"url"` + Type string `json:"type"` // "url" for web pages, "api" for specialized API sources + URL string `json:"url,omitempty"` + Name string `json:"name,omitempty"` // Identifies specialized API sources (type "api"), which carry no URL // Anthropic specific fields Title *string `json:"title,omitempty"` @@ -2553,9 +2664,9 @@ type ResponsesMCPApprovalRequestAction struct { // ResponsesMCPApprovalResponse represents a MCP approval response type ResponsesMCPApprovalResponse struct { - ApprovalResponseID string `json:"approval_response_id"` - Approve bool `json:"approve"` - Reason *string `json:"reason,omitempty"` + ApprovalRequestID string `json:"approval_request_id"` + Approve bool `json:"approve"` + Reason *string `json:"reason,omitempty"` } // ResponsesMCPToolCall represents a MCP tool call @@ -2738,6 +2849,7 @@ const ( ResponsesToolTypeFunction ResponsesToolType = "function" ResponsesToolTypeFileSearch ResponsesToolType = "file_search" ResponsesToolTypeComputerUsePreview ResponsesToolType = "computer_use_preview" + ResponsesToolTypeComputer ResponsesToolType = "computer" // OpenAI computer tool for GPT-6 Astra / GPT-5.6 (no display or environment fields) ResponsesToolTypeWebSearch ResponsesToolType = "web_search" ResponsesToolTypeWebFetch ResponsesToolType = "web_fetch" ResponsesToolTypeMCP ResponsesToolType = "mcp" @@ -2778,6 +2890,8 @@ func normalizeResponsesToolType(t ResponsesToolType) ResponsesToolType { return t case strings.HasPrefix(s, "web_fetch"): return ResponsesToolTypeWebFetch + case t == ResponsesToolTypeComputer: + return t case strings.HasPrefix(s, "computer") && t != ResponsesToolTypeComputerUsePreview: // Covers "computer_20250124", "computer_20251124", etc. return ResponsesToolTypeComputerUsePreview @@ -2848,6 +2962,7 @@ type ResponsesTool struct { Type ResponsesToolType `json:"type"` // "function" | "file_search" | "computer_use_preview" | "web_search" | "web_search_2025_08_26" | "mcp" | "code_interpreter" | "image_generation" | "local_shell" | "custom" | "web_search_preview" | "web_search_preview_2025_03_11" | "x_search" Name *string `json:"name,omitempty"` // Common name field (Function, Custom tools) Description *string `json:"description,omitempty"` // Common description field (Function, Custom tools) + Async *bool `json:"async,omitempty"` // OpenAI async tool calling (Function, Custom tools) // Not in OpenAI's schemas, but sent by a few providers (Anthropic, Bedrock are some of them) CacheControl *CacheControl `json:"cache_control,omitempty"` @@ -2912,6 +3027,11 @@ func (t ResponsesTool) MarshalJSON() ([]byte, error) { return nil, err } } + if t.Async != nil { + if data, err = sjson.SetBytes(data, "async", *t.Async); err != nil { + return nil, err + } + } if t.CacheControl != nil { ccBytes, ccErr := MarshalSorted(t.CacheControl) if ccErr != nil { @@ -3057,6 +3177,7 @@ func (t *ResponsesTool) UnmarshalJSON(data []byte) error { "input_examples", // 6 "eager_input_streaming", // 7 "function", // 8 — Chat Completions wrapper, lifted below + "async", // 9 ) // Extract type field @@ -3080,6 +3201,9 @@ func (t *ResponsesTool) UnmarshalJSON(data []byte) error { if v := fields[2]; v.Type == gjson.String { t.Description = new(v.String()) } + if v := fields[9]; v.IsBool() { + t.Async = new(v.Bool()) + } if v := fields[3]; v.Exists() { var cc CacheControl if err := Unmarshal([]byte(v.Raw), &cc); err != nil { @@ -3262,8 +3386,9 @@ func (t *ResponsesTool) UnmarshalJSON(data []byte) error { // ResponsesToolFunction represents a tool function type ResponsesToolFunction struct { - Parameters *ToolFunctionParameters `json:"parameters,omitempty"` // A JSON schema object describing the parameters - Strict *bool `json:"strict"` // Whether to enforce strict parameter validation + Parameters *ToolFunctionParameters `json:"parameters,omitempty"` // A JSON schema object describing the parameters + Strict *bool `json:"strict"` // Whether to enforce strict parameter validation + OutputSchema *OrderedMap `json:"output_schema,omitempty"` // JSON schema of the value encoded in string outputs (OpenAI) } // ResponsesToolFileSearch represents a tool file search @@ -3276,7 +3401,7 @@ type ResponsesToolFileSearch struct { // ResponsesToolFileSearchFilter represents a file search filter type ResponsesToolFileSearchFilter struct { - Type string `json:"type"` // "eq" | "ne" | "gt" | "gte" | "lt" | "lte" | "and" | "or" + Type string `json:"type"` // "eq" | "ne" | "gt" | "gte" | "lt" | "lte" | "in" | "nin" | "and" | "or" // Filter types - only one should be set *ResponsesToolFileSearchComparisonFilter @@ -3302,7 +3427,7 @@ func (f *ResponsesToolFileSearchFilter) MarshalJSON() ([]byte, error) { } switch f.Type { - case "eq", "ne", "gt", "gte", "lt", "lte": + case "eq", "ne", "gt", "gte", "lt", "lte", "in", "nin": if f.ResponsesToolFileSearchComparisonFilter == nil { return nil, fmt.Errorf("comparison filter is nil but type is %s", f.Type) } @@ -3353,7 +3478,7 @@ func (f *ResponsesToolFileSearchFilter) UnmarshalJSON(data []byte) error { // Initialize the appropriate embedded struct based on type switch typeStr { - case "eq", "ne", "gt", "gte", "lt", "lte": + case "eq", "ne", "gt", "gte", "lt", "lte", "in", "nin": // This is a comparison filter f.ResponsesToolFileSearchComparisonFilter = &ResponsesToolFileSearchComparisonFilter{} f.ResponsesToolFileSearchCompoundFilter = nil @@ -3390,7 +3515,7 @@ func (f *ResponsesToolFileSearchFilter) UnmarshalJSON(data []byte) error { } default: - return fmt.Errorf("unknown filter type: %s (supported types: eq, ne, gt, gte, lt, lte, and, or)", typeStr) + return fmt.Errorf("unknown filter type: %s (supported types: eq, ne, gt, gte, lt, lte, in, nin, and, or)", typeStr) } return nil @@ -3400,7 +3525,7 @@ func (f *ResponsesToolFileSearchFilter) UnmarshalJSON(data []byte) error { type ResponsesToolFileSearchComparisonFilter struct { Key string `json:"key"` // The key to compare against the value Type string `json:"type"` // - Value interface{} `json:"value"` // The value to compare (string, number, or boolean) + Value interface{} `json:"value"` // The value to compare (string, number, boolean, or an array for in/nin) } // ResponsesToolFileSearchCompoundFilter represents a file search compound filter @@ -3527,13 +3652,41 @@ type ResponsesToolMCP struct { RequireApproval *ResponsesToolMCPAllowedToolsApprovalSetting `json:"require_approval,omitempty"` // Tool approval settings ServerDescription *string `json:"server_description,omitempty"` // Optional server description ServerURL *string `json:"server_url,omitempty"` // The URL for the MCP server + TunnelID *string `json:"tunnel_id,omitempty"` // Secure MCP Tunnel ID used instead of server_url } // ResponsesToolMCPAllowedTools - List of allowed tool names or a filter object type ResponsesToolMCPAllowedTools struct { // Either a simple array of tool names or a filter object - ToolNames []string `json:",omitempty"` - Filter *ResponsesToolMCPAllowedToolsFilter `json:",omitempty"` + ToolNames []string + Filter *ResponsesToolMCPAllowedToolsFilter +} + +func (a ResponsesToolMCPAllowedTools) MarshalJSON() ([]byte, error) { + if a.ToolNames != nil && a.Filter != nil { + return nil, fmt.Errorf("both ToolNames and Filter are set; only one should be non-nil") + } + if a.ToolNames != nil { + return MarshalSorted(a.ToolNames) + } + if a.Filter != nil { + return MarshalSorted(a.Filter) + } + return MarshalSorted(nil) +} + +func (a *ResponsesToolMCPAllowedTools) UnmarshalJSON(data []byte) error { + var toolNames []string + if err := Unmarshal(data, &toolNames); err == nil { + *a = ResponsesToolMCPAllowedTools{ToolNames: toolNames} + return nil + } + var filter ResponsesToolMCPAllowedToolsFilter + if err := Unmarshal(data, &filter); err == nil { + *a = ResponsesToolMCPAllowedTools{Filter: &filter} + return nil + } + return fmt.Errorf("mcp allowed_tools is neither an array of tool names nor a filter object") } // ResponsesToolMCPAllowedToolsFilter - A filter object to specify which tools are allowed @@ -3746,6 +3899,12 @@ const ( // Ping events are just keepalive (sent by very few providers, Anthropic is one of them) ResponsesStreamResponseTypePing ResponsesStreamResponseType = "response.ping" + // Deprecated: retained for source compatibility. Providers no longer synthesize + // generic raw events; supported provider events have explicit types. + ResponsesStreamResponseTypeProviderRawEvent ResponsesStreamResponseType = "response.provider_raw_event" + // SafeguardsUpdate carries an Anthropic classifier event, omitted on OpenAI surfaces. + ResponsesStreamResponseTypeSafeguardsUpdate ResponsesStreamResponseType = "response.safeguards_update" + ResponsesStreamResponseTypeCreated ResponsesStreamResponseType = "response.created" ResponsesStreamResponseTypeInProgress ResponsesStreamResponseType = "response.in_progress" ResponsesStreamResponseTypeCompleted ResponsesStreamResponseType = "response.completed" @@ -3866,6 +4025,13 @@ type BifrostResponsesStreamResponse struct { ExtraFields BifrostResponseExtraFields `json:"extra_fields"` + // SafeguardResults carries the Claude Code auto-mode classifier verdicts found + // top-level on a provider stream event (opaque; forwarded unchanged per the + // gateway compatibility guide), so the provider-native egress can restore them + // on the re-rendered frame. Deliberately NOT copied by WithDefaults: OpenAI-shaped + // surfaces never see it. + SafeguardResults json.RawMessage `json:"safeguard_results,omitempty"` + // Perplexity-specific fields SearchResults []SearchResult `json:"search_results,omitempty"` Videos []VideoResult `json:"videos,omitempty"` @@ -3893,7 +4059,7 @@ func (resp *BifrostResponsesStreamResponse) WithDefaults() *BifrostResponsesStre } // Filter out non-OpenAI response types - if resp.Type == ResponsesStreamResponseTypePing { + if resp.Type == ResponsesStreamResponseTypePing || resp.Type == ResponsesStreamResponseTypeProviderRawEvent || resp.Type == ResponsesStreamResponseTypeSafeguardsUpdate { return nil } diff --git a/core/schemas/responses_test.go b/core/schemas/responses_test.go index b91a97049fb..5573a0a77a2 100644 --- a/core/schemas/responses_test.go +++ b/core/schemas/responses_test.go @@ -2,10 +2,94 @@ package schemas import ( "encoding/json" + "reflect" "strings" "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) +func TestAnthropicBillingHeaderExtraction(t *testing.T) { + header := "x-anthropic-billing-header: cc_version=2.1.270.42c; cc_entrypoint=cli; cch=12345;" + for _, tc := range []struct { + name, text string + strip bool + }{ + {"metadata", header, true}, + {"whitespace", " \n" + header + "\n", true}, + {"future field", "x-anthropic-billing-header: cc_version=2.1.270.abc; future=value;", true}, + {"ordinary text", "Keep these instructions.", false}, + {"quoted header", "Discuss " + header, false}, + {"mixed multiline", header + "\nKeep these instructions.", false}, + {"mixed same line", header + " Keep these instructions.", false}, + } { + for _, shape := range []string{"string", "blocks"} { + t.Run(tc.name+"/"+shape, func(t *testing.T) { + content := &ResponsesMessageContent{ContentStr: &tc.text} + if shape == "blocks" { + content = &ResponsesMessageContent{ContentBlocks: []ResponsesMessageContentBlock{{Type: ResponsesInputMessageContentBlockTypeText, Text: &tc.text}}} + } + r := &BifrostResponsesRequest{Input: []ResponsesMessage{ + {Role: Ptr(ResponsesInputMessageRoleSystem), Content: content}, + {Role: Ptr(ResponsesInputMessageRoleUser), Content: &ResponsesMessageContent{ContentStr: &tc.text}}, + }, RawRequestBody: []byte("original raw body")} + before, err := MarshalSorted(r) + require.NoError(t, err) + r.ExtractAnthropicBillingHeader() + r.ExtractAnthropicBillingHeader() // Repeated normalization is harmless. + if tc.strip { + require.Len(t, r.Input, 1) + assert.Equal(t, ResponsesInputMessageRoleUser, *r.Input[0].Role) + assert.Equal(t, tc.text, *r.Input[0].Content.ContentStr) + } else { + assert.Nil(t, r.anthropicBillingHeader) + require.Len(t, r.Input, 2) + } + assert.Equal(t, "original raw body", string(r.RawRequestBody)) + restored := r.WithAnthropicBillingHeader() + after, err := MarshalSorted(restored) + require.NoError(t, err) + assert.Equal(t, string(before), string(after)) + assert.Same(t, restored, restored.WithAnthropicBillingHeader()) + }) + } + } +} + +func TestAnthropicBillingHeaderRestoresPositionsAndCacheMarkers(t *testing.T) { + header := "x-anthropic-billing-header: cc_version=2.1.270.42c;" + block := func(text string) ResponsesMessageContentBlock { + return ResponsesMessageContentBlock{Type: ResponsesInputMessageContentBlockTypeText, Text: Ptr(text)} + } + for _, onlyHeaders := range []bool{false, true} { + blocks := []ResponsesMessageContentBlock{block(header), block(header)} + if !onlyHeaders { + blocks = []ResponsesMessageContentBlock{block("first"), block(header), block("last"), block(header)} + } + blocks[1].CacheControl = &CacheControl{Type: CacheControlTypeEphemeral} + r := &BifrostResponsesRequest{Input: []ResponsesMessage{ + {Role: Ptr(ResponsesInputMessageRoleSystem), Content: &ResponsesMessageContent{ContentBlocks: blocks}}, + {Role: Ptr(ResponsesInputMessageRoleUser), Content: &ResponsesMessageContent{ContentStr: Ptr("hello")}}, + }} + before, err := MarshalSorted(r) + require.NoError(t, err) + r.ExtractAnthropicBillingHeader() + normalized, err := MarshalSorted(r) + require.NoError(t, err) + assert.NotContains(t, string(normalized), "x-anthropic-billing-header:") + // Core fallbacks shallow-copy the request; the private metadata must survive. + fallback := *r + restored := fallback.WithAnthropicBillingHeader() + after, err := MarshalSorted(restored) + require.NoError(t, err) + assert.Equal(t, string(before), string(after)) + unchanged, err := MarshalSorted(r) + require.NoError(t, err) + assert.Equal(t, string(normalized), string(unchanged)) + } +} + // TestBifrostResponsesStreamResponseOmitsEmptyItem verifies that events without // an item object (response.created, output_text.delta, response.completed, ...) // do not serialize "item": null. Strict Responses API clients (e.g. opencode's @@ -344,6 +428,63 @@ func TestBifrostResponsesResponseUnmarshalTimestamps(t *testing.T) { }) } +func TestResponsesMessageContentUnmarshalJSONBoundaries(t *testing.T) { + tests := []struct { + name string + data string + want ResponsesMessageContent + }{ + {name: "empty string", data: `""`, want: ResponsesMessageContent{ContentStr: Ptr("")}}, + {name: "string with whitespace", data: " \t\r\n\"hello\" \t\r\n", want: ResponsesMessageContent{ContentStr: Ptr("hello")}}, + {name: "escaped string", data: `"line\n\"quote\"\u4e16\u754c"`, want: ResponsesMessageContent{ContentStr: Ptr("line\n\"quote\"\u4e16\u754c")}}, + {name: "null", data: " \t\r\nnull \t\r\n", want: ResponsesMessageContent{ContentStr: Ptr("")}}, + {name: "empty array", data: " \t\r\n[] \t\r\n", want: ResponsesMessageContent{ContentBlocks: []ResponsesMessageContentBlock{}}}, + { + name: "content blocks", + data: `[{"type":"input_text","text":"hello"}]`, + want: ResponsesMessageContent{ContentBlocks: []ResponsesMessageContentBlock{ + {Type: ResponsesInputMessageContentBlockTypeText, Text: Ptr("hello")}, + }}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var got ResponsesMessageContent + require.NoError(t, got.UnmarshalJSON([]byte(tt.data))) + assert.Equal(t, tt.want, got) + }) + } + + invalid := []struct { + name string + data string + }{ + {name: "empty", data: ""}, + {name: "whitespace only", data: " \t\r\n"}, + {name: "object", data: `{}`}, + {name: "number", data: `123`}, + {name: "boolean", data: `true`}, + {name: "non JSON whitespace", data: "\vnull"}, + {name: "truncated null", data: `nul`}, + {name: "truncated string", data: `"unterminated`}, + {name: "invalid escape", data: `"\q"`}, + {name: "truncated array", data: `[`}, + {name: "invalid array item", data: `[{"type":"input_text","text":"new"},42]`}, + {name: "trailing comma", data: `[{},]`}, + {name: "trailing value", data: `"text" false`}, + } + for _, tt := range invalid { + t.Run(tt.name, func(t *testing.T) { + before := ResponsesMessageContent{ContentStr: Ptr("previous")} + got := before + err := got.UnmarshalJSON([]byte(tt.data)) + const wantError = "content field is neither a string nor an array of Content blocks" + require.EqualError(t, err, wantError) + assert.Equal(t, before, got, "failed decode changed receiver") + }) + } +} + // TestResponsesMessageContentEmptyMarshalsToEmptyString verifies that empty // content serializes as "" rather than null, since the OpenAI Responses API // rejects null content. @@ -675,6 +816,227 @@ func TestDeepCopyResponsesMessagePreservesRawPreserved(t *testing.T) { } } +// TestDeepCopyResponsesMessagePreservesCacheControls verifies cache breakpoints survive copy-on-write request transforms. +func TestDeepCopyResponsesMessagePreservesCacheControls(t *testing.T) { + ttl := "1h" + scope := "user" + original := ResponsesMessage{ + Type: Ptr(ResponsesMessageTypeFunctionCallOutput), + CacheControl: &CacheControl{Type: CacheControlTypeEphemeral, TTL: &ttl, Scope: &scope}, + Content: &ResponsesMessageContent{ContentBlocks: []ResponsesMessageContentBlock{{ + Type: ResponsesInputMessageContentBlockTypeText, + Text: Ptr("cacheable content"), + CacheControl: &CacheControl{Type: CacheControlTypeEphemeral, TTL: &ttl, Scope: &scope}, + }}}, + } + + copied := DeepCopyResponsesMessage(original) + cacheControls := [][2]*CacheControl{ + {original.CacheControl, copied.CacheControl}, + {original.Content.ContentBlocks[0].CacheControl, copied.Content.ContentBlocks[0].CacheControl}, + } + for _, pair := range cacheControls { + originalCacheControl, copiedCacheControl := pair[0], pair[1] + if copiedCacheControl == nil { + t.Fatal("deep copy dropped cache control") + } + if copiedCacheControl.Type != originalCacheControl.Type || copiedCacheControl.TTL == nil || originalCacheControl.TTL == nil || *copiedCacheControl.TTL != *originalCacheControl.TTL || copiedCacheControl.Scope == nil || originalCacheControl.Scope == nil || *copiedCacheControl.Scope != *originalCacheControl.Scope { + t.Fatalf("cache control = %#v, want %#v", copiedCacheControl, originalCacheControl) + } + if copiedCacheControl == originalCacheControl { + t.Error("copy aliases the original cache control struct") + } + if copiedCacheControl.TTL == originalCacheControl.TTL { + t.Error("copy aliases the original cache control TTL") + } + if copiedCacheControl.Scope == originalCacheControl.Scope { + t.Error("copy aliases the original cache control scope") + } + } +} + +// TestDeepCopyResponsesMessagePreservesExtendedFields verifies newly supported Responses fields survive the shared copy path. +func TestDeepCopyResponsesMessagePreservesExtendedFields(t *testing.T) { + fileType := "application/pdf" + breakpointMode := "explicit" + annotationIndex := 1 + annotationPage := 2 + annotationSource := "anthropic" + annotationEncryptedIndex := "ciphertext" + category := "safety" + original := ResponsesMessage{ + ProviderNativeParts: json.RawMessage(`{"thoughtSignature":"opaque"}`), + Content: &ResponsesMessageContent{ContentBlocks: []ResponsesMessageContentBlock{{ + Type: ResponsesOutputMessageContentTypeText, + ResponsesInputMessageContentBlockFile: &ResponsesInputMessageContentBlockFile{FileType: &fileType}, + Citations: &Citations{Enabled: Ptr(true)}, + PromptCacheBreakpoint: &PromptCacheBreakpoint{Mode: &breakpointMode}, + ResponsesOutputMessageContentText: &ResponsesOutputMessageContentText{Annotations: []ResponsesOutputMessageContentTextAnnotation{{ + Index: &annotationIndex, + StartCharIndex: &annotationIndex, + EndCharIndex: &annotationIndex, + StartPageNumber: &annotationPage, + EndPageNumber: &annotationPage, + StartBlockIndex: &annotationIndex, + EndBlockIndex: &annotationIndex, + Source: &annotationSource, + EncryptedIndex: &annotationEncryptedIndex, + }}}, + ResponsesOutputMessageContentRenderedContent: &ResponsesOutputMessageContentRenderedContent{RenderedContent: "rendered"}, + ResponsesOutputMessageContentCompaction: &ResponsesOutputMessageContentCompaction{Summary: "summary"}, + ResponsesOutputMessageContentFallback: &ResponsesOutputMessageContentFallback{ + FromModel: "claude-opus-5", + ToModel: "claude-sonnet-5", + TriggerType: "refusal", + TriggerCategory: &category, + }, + }}}, + ResponsesToolMessage: &ResponsesToolMessage{ + Action: &ResponsesToolMessageActionStruct{ResponsesToolCallActionStr: Ptr("generate")}, + ResponsesComputerToolCall: &ResponsesComputerToolCall{PendingSafetyChecks: []ResponsesComputerToolCallPendingSafetyCheck{{ + ID: "check_1", Code: "confirm", Message: "confirm action", + }}}, + ResponsesComputerToolCallOutput: &ResponsesComputerToolCallOutput{AcknowledgedSafetyChecks: []ResponsesComputerToolCallAcknowledgedSafetyCheck{{ + ID: "check_1", Code: Ptr("confirm"), Message: Ptr("approved"), + }}}, + ResponsesCodeInterpreterToolCall: &ResponsesCodeInterpreterToolCall{ + Code: Ptr("print('hello')"), + ContainerID: "container_1", + Outputs: []ResponsesCodeInterpreterOutput{{ + ResponsesCodeInterpreterOutputLogs: &ResponsesCodeInterpreterOutputLogs{Type: "logs", Logs: "hello"}, + }}, + }, + ResponsesMCPToolCall: &ResponsesMCPToolCall{ServerLabel: "repo"}, + ResponsesImageGenerationCall: &ResponsesImageGenerationCall{ + Result: "image", + Background: Ptr("transparent"), + OutputFormat: Ptr("png"), + Quality: Ptr("high"), + RevisedPrompt: Ptr("draw a cat"), + Size: Ptr("1024x1024"), + }, + ResponsesMCPListTools: &ResponsesMCPListTools{ServerLabel: "repo", Tools: []ResponsesMCPTool{{ + Name: "read_file", + InputSchema: map[string]any{"type": "object", "properties": map[string]any{"path": "string"}}, + Description: Ptr("read a file"), + Annotations: &map[string]any{"readOnlyHint": true}, + }}}, + ResponsesMCPApprovalResponse: &ResponsesMCPApprovalResponse{ + ApprovalRequestID: "approval_1", Approve: true, Reason: Ptr("allowed"), + }, + ResponsesAdvisorCall: &ResponsesAdvisorCall{ + ResultType: "advisor_result", + Text: Ptr("advice"), + EncryptedContent: Ptr("encrypted"), + ErrorCode: Ptr("none"), + StopReason: Ptr("end_turn"), + }, + ResponsesToolSearchCall: &ResponsesToolSearchCall{ToolReferences: []string{"read_file"}}, + ResponsesCodeExecutionCall: &ResponsesCodeExecutionCall{ + ToolName: "bash_code_execution", + Input: Ptr(`{"command":"pwd"}`), + Lines: []string{"line one"}, + Files: []ResponsesCodeExecutionFileOutput{{FileID: "file_1"}}, + Caller: &ResponsesToolCaller{Type: "code_execution_20260120", ToolID: Ptr("tool_1")}, + }, + }, + } + + copied := DeepCopyResponsesMessage(original) + if !reflect.DeepEqual(original, copied) { + t.Fatalf("deep copy lost Responses fields\noriginal: %#v\ncopied: %#v", original, copied) + } + if &original.ProviderNativeParts[0] == &copied.ProviderNativeParts[0] { + t.Fatal("copy aliases provider native parts") + } + if original.Content.ContentBlocks[0].ResponsesInputMessageContentBlockFile.FileType == copied.Content.ContentBlocks[0].ResponsesInputMessageContentBlockFile.FileType { + t.Fatal("copy aliases file type") + } + if original.ResponsesToolMessage.Action.ResponsesToolCallActionStr == copied.ResponsesToolMessage.Action.ResponsesToolCallActionStr { + t.Fatal("copy aliases bare tool action") + } + + copied.ProviderNativeParts[0] = '[' + copied.ResponsesToolMessage.ResponsesMCPListTools.Tools[0].InputSchema["type"] = "array" + copied.ResponsesToolMessage.ResponsesCodeExecutionCall.Lines[0] = "changed" + if string(original.ProviderNativeParts) != `{"thoughtSignature":"opaque"}` { + t.Fatal("mutating copied provider native parts changed the original") + } + if original.ResponsesToolMessage.ResponsesMCPListTools.Tools[0].InputSchema["type"] != "object" { + t.Fatal("mutating copied MCP schema changed the original") + } + if original.ResponsesToolMessage.ResponsesCodeExecutionCall.Lines[0] != "line one" { + t.Fatal("mutating copied code execution lines changed the original") + } +} + +// TestDeepCopyResponsesMessagePreservesNilMCPAnnotations verifies a non-nil annotations pointer to a nil map is copied without panicking or aliasing. +func TestDeepCopyResponsesMessagePreservesNilMCPAnnotations(t *testing.T) { + annotations := map[string]any(nil) + original := ResponsesMessage{ + ResponsesToolMessage: &ResponsesToolMessage{ + ResponsesMCPListTools: &ResponsesMCPListTools{Tools: []ResponsesMCPTool{{Annotations: &annotations}}}, + }, + } + + copied := DeepCopyResponsesMessage(original) + copyAnnotations := copied.ResponsesToolMessage.ResponsesMCPListTools.Tools[0].Annotations + if copyAnnotations == nil { + t.Fatal("copy dropped annotations pointer") + } + if copyAnnotations == original.ResponsesToolMessage.ResponsesMCPListTools.Tools[0].Annotations { + t.Fatal("copy aliases annotations pointer") + } + if *copyAnnotations != nil { + t.Fatal("copy changed nil annotations map") + } +} + +// TestDeepCopyResponsesMessageCopiesCodeExecutionPointers verifies code-execution pointer fields do not alias the original message. +func TestDeepCopyResponsesMessageCopiesCodeExecutionPointers(t *testing.T) { + originalCall := &ResponsesCodeExecutionCall{ + Input: Ptr(`{"command":"pwd"}`), + Stdout: Ptr("output"), + Stderr: Ptr("error"), + ReturnCode: Ptr(1), + EncryptedStdout: Ptr("encrypted"), + FileType: Ptr("text"), + FileContent: Ptr("contents"), + StartLine: Ptr(1), + NumLines: Ptr(2), + TotalLines: Ptr(3), + IsFileUpdate: Ptr(true), + OldStart: Ptr(4), + OldLines: Ptr(5), + NewStart: Ptr(6), + NewLines: Ptr(7), + Lines: []string{"before", "after"}, + ErrorCode: Ptr("unavailable"), + Files: []ResponsesCodeExecutionFileOutput{{FileID: "file_1"}}, + ContainerExpiresAt: Ptr("2026-10-01T00:00:00Z"), + Caller: &ResponsesToolCaller{Type: "code_execution_20260120", ToolID: Ptr("tool_1")}, + } + original := ResponsesMessage{ResponsesToolMessage: &ResponsesToolMessage{ResponsesCodeExecutionCall: originalCall}} + + copied := DeepCopyResponsesMessage(original) + copiedCall := copied.ResponsesToolMessage.ResponsesCodeExecutionCall + if !reflect.DeepEqual(originalCall, copiedCall) { + t.Fatalf("deep copy changed code-execution call\noriginal: %#v\ncopied: %#v", originalCall, copiedCall) + } + + for _, field := range []string{ + "Input", "Stdout", "Stderr", "ReturnCode", "EncryptedStdout", "FileType", "FileContent", + "StartLine", "NumLines", "TotalLines", "IsFileUpdate", "OldStart", "OldLines", "NewStart", + "NewLines", "ErrorCode", "ContainerExpiresAt", + } { + originalPointer := reflect.ValueOf(originalCall).Elem().FieldByName(field) + copiedPointer := reflect.ValueOf(copiedCall).Elem().FieldByName(field) + if originalPointer.Pointer() == copiedPointer.Pointer() { + t.Fatalf("copy aliases code-execution %s", field) + } + } +} + func TestDeepCopyResponsesMessagePreservesToolSearchFields(t *testing.T) { toolSearchOutputType := ResponsesMessageTypeToolSearchOutput callID := "call_1" @@ -945,3 +1307,304 @@ func TestDeepCopyResponsesMessagePreservesMediaResolution(t *testing.T) { t.Fatalf("numTokens = %d, want 512", *got.NumTokens) } } + +// TestResponsesWebSearchSourceRoundTrip pins web_search_call action sources +// through a decode -> re-encode cycle. OpenAI's hosted web search can return +// specialized API sources ({"type":"api","name":"oai-weather"}) that carry a +// name and no URL; they must survive the round-trip without losing the name or +// fabricating an empty url. +func TestResponsesWebSearchSourceRoundTrip(t *testing.T) { + roundTripSource := func(t *testing.T, raw string) map[string]any { + t.Helper() + var msg ResponsesMessage + if err := Unmarshal([]byte(raw), &msg); err != nil { + t.Fatalf("unmarshal web_search_call: %v", err) + } + encoded, err := MarshalSorted(msg) + if err != nil { + t.Fatalf("marshal web_search_call: %v", err) + } + var out struct { + Action struct { + Sources []map[string]any `json:"sources"` + } `json:"action"` + } + if err := json.Unmarshal(encoded, &out); err != nil { + t.Fatalf("unmarshal encoded web_search_call: %v", err) + } + if len(out.Action.Sources) != 1 { + t.Fatalf("expected 1 source after round-trip, got %d (encoded: %s)", len(out.Action.Sources), encoded) + } + return out.Action.Sources[0] + } + + t.Run("api source keeps name and gains no url", func(t *testing.T) { + raw := `{"id":"ws_1","type":"web_search_call","status":"completed","action":{"type":"search","queries":["weather in paris"],"sources":[{"type":"api","name":"oai-weather"}]}}` + + source := roundTripSource(t, raw) + if source["type"] != "api" { + t.Fatalf("expected source type %q, got %v", "api", source["type"]) + } + if source["name"] != "oai-weather" { + t.Fatalf("expected source name %q, got %v", "oai-weather", source["name"]) + } + if _, ok := source["url"]; ok { + t.Fatalf("expected no url key on an api source, got %v", source["url"]) + } + }) + + t.Run("url source round-trips unchanged", func(t *testing.T) { + raw := `{"id":"ws_1","type":"web_search_call","status":"completed","action":{"type":"search","queries":["weather in paris"],"sources":[{"type":"url","url":"https://example.com"}]}}` + + source := roundTripSource(t, raw) + if source["type"] != "url" { + t.Fatalf("expected source type %q, got %v", "url", source["type"]) + } + if source["url"] != "https://example.com" { + t.Fatalf("expected source url %q, got %v", "https://example.com", source["url"]) + } + if _, ok := source["name"]; ok { + t.Fatalf("expected no name key on a plain url source, got %v", source["name"]) + } + }) +} + +// TestResponsesOpenAIWireShapes pins OpenAI Responses shapes that previously failed +// to decode or re-encoded lossily: each input must survive an unmarshal/marshal +// round trip unchanged. +func TestResponsesOpenAIWireShapes(t *testing.T) { + assertRoundTrip := func(t *testing.T, v any, in string) { + t.Helper() + if err := Unmarshal([]byte(in), v); err != nil { + t.Fatalf("unmarshal: %v", err) + } + out, err := MarshalSorted(v) + if err != nil { + t.Fatalf("marshal: %v", err) + } + var got, want any + if err := json.Unmarshal(out, &got); err != nil { + t.Fatalf("decode output: %v", err) + } + if err := json.Unmarshal([]byte(in), &want); err != nil { + t.Fatalf("decode input: %v", err) + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("round trip mismatch\n got: %s\nwant: %s", out, in) + } + } + + t.Run("mcp_approval_response_keeps_approval_request_id", func(t *testing.T) { + assertRoundTrip(t, &ResponsesMessage{}, `{"type":"mcp_approval_response","approval_request_id":"mcpr_1","approve":true,"reason":"ok"}`) + }) + + t.Run("mcp_call_structured_errors", func(t *testing.T) { + for _, tc := range []struct{ in, text string }{ + {`{"type":"mcp_call","id":"mcp_1","name":"search","arguments":"{}","error":{"type":"mcp_protocol_error","code":-32602,"message":"bad params"}}`, "bad params"}, + {`{"type":"mcp_call","id":"mcp_2","name":"search","arguments":"{}","error":{"type":"http_error","code":502,"message":"upstream down"}}`, "upstream down"}, + {`{"type":"mcp_call","id":"mcp_3","name":"search","arguments":"{}","error":{"type":"mcp_tool_execution_error","content":[{"type":"text","text":"boom"}]}}`, `[{"type":"text","text":"boom"}]`}, + {`{"type":"mcp_call","id":"mcp_4","name":"search","arguments":"{}","error":"legacy string"}`, "legacy string"}, + } { + msg := &ResponsesMessage{} + assertRoundTrip(t, msg, tc.in) + if got := msg.ResponsesToolMessage.Error.Text(); got != tc.text { + t.Fatalf("Text() = %q, want %q", got, tc.text) + } + } + }) + + t.Run("conversation_accepts_string_and_object", func(t *testing.T) { + assertRoundTrip(t, &ResponsesParameters{}, `{"conversation":"conv_1"}`) + assertRoundTrip(t, &ResponsesParameters{}, `{"conversation":{"id":"conv_1"}}`) + }) + + t.Run("reused_conversation_clears_previous_union_arm", func(t *testing.T) { + var conversation ResponsesResponseConversation + if err := Unmarshal([]byte(`{"id":"conv_1"}`), &conversation); err != nil { + t.Fatalf("unmarshal object conversation: %v", err) + } + if err := Unmarshal([]byte(`"conv_2"`), &conversation); err != nil { + t.Fatalf("unmarshal string conversation into reused receiver: %v", err) + } + if conversation.ResponsesResponseConversationStruct != nil || + conversation.ResponsesResponseConversationStr == nil || + *conversation.ResponsesResponseConversationStr != "conv_2" { + t.Fatalf("reused conversation retained stale object arm: %#v", conversation) + } + + if err := Unmarshal([]byte(`{"id":"conv_3"}`), &conversation); err != nil { + t.Fatalf("unmarshal object conversation into reused receiver: %v", err) + } + if conversation.ResponsesResponseConversationStr != nil || + conversation.ResponsesResponseConversationStruct == nil || + conversation.ResponsesResponseConversationStruct.ID != "conv_3" { + t.Fatalf("reused conversation retained stale string arm: %#v", conversation) + } + }) + + t.Run("mcp_allowed_tools_accepts_array_and_filter", func(t *testing.T) { + assertRoundTrip(t, &ResponsesTool{}, `{"type":"mcp","server_label":"docs","server_url":"https://mcp.example.com","allowed_tools":["search","fetch"]}`) + assertRoundTrip(t, &ResponsesTool{}, `{"type":"mcp","server_label":"docs","server_url":"https://mcp.example.com","allowed_tools":{"read_only":true,"tool_names":["search"]}}`) + }) + + t.Run("reused_tool_error_clears_previous_union_arm", func(t *testing.T) { + var toolError ResponsesToolMessageError + if err := Unmarshal([]byte(`{"type":"http_error","code":502}`), &toolError); err != nil { + t.Fatalf("unmarshal structured error: %v", err) + } + if err := Unmarshal([]byte(`"legacy"`), &toolError); err != nil { + t.Fatalf("unmarshal string error into reused receiver: %v", err) + } + if toolError.ResponsesToolMessageErrorStruct != nil || toolError.ResponsesToolMessageErrorStr == nil || *toolError.ResponsesToolMessageErrorStr != "legacy" { + t.Fatalf("reused error retained stale union state: %#v", toolError) + } + + if err := Unmarshal([]byte(`{"type":"mcp_protocol_error"}`), &toolError); err != nil { + t.Fatalf("unmarshal structured error into reused receiver: %v", err) + } + if toolError.ResponsesToolMessageErrorStr != nil || toolError.ResponsesToolMessageErrorStruct == nil { + t.Fatalf("reused error retained stale string arm: %#v", toolError) + } + }) + + t.Run("reused_allowed_tools_clears_previous_union_arm", func(t *testing.T) { + var allowed ResponsesToolMCPAllowedTools + if err := Unmarshal([]byte(`{"read_only":true}`), &allowed); err != nil { + t.Fatalf("unmarshal filter: %v", err) + } + if err := Unmarshal([]byte(`["search"]`), &allowed); err != nil { + t.Fatalf("unmarshal names into reused receiver: %v", err) + } + if allowed.Filter != nil || !reflect.DeepEqual(allowed.ToolNames, []string{"search"}) { + t.Fatalf("reused allowed_tools retained stale union state: %#v", allowed) + } + + if err := Unmarshal([]byte(`{"tool_names":["fetch"]}`), &allowed); err != nil { + t.Fatalf("unmarshal filter into reused receiver: %v", err) + } + if allowed.ToolNames != nil || allowed.Filter == nil || !reflect.DeepEqual(allowed.Filter.ToolNames, []string{"fetch"}) { + t.Fatalf("reused allowed_tools retained stale names arm: %#v", allowed) + } + }) + + t.Run("file_search_in_and_nin_filters", func(t *testing.T) { + assertRoundTrip(t, &ResponsesTool{}, `{"type":"file_search","vector_store_ids":["vs_1"],"filters":{"type":"and","filters":[{"type":"in","key":"region","value":["us","eu"]},{"type":"nin","key":"year","value":[2023,2024]}]}}`) + }) +} + +func TestResponsesToolMessageErrorClassification(t *testing.T) { + tests := []struct { + name string + err *ResponsesToolMessageError + want bool + }{ + {name: "absent", err: nil, want: false}, + {name: "empty legacy string", err: &ResponsesToolMessageError{ResponsesToolMessageErrorStr: Ptr("")}, want: false}, + {name: "legacy string", err: &ResponsesToolMessageError{ResponsesToolMessageErrorStr: Ptr("failed")}, want: true}, + {name: "empty structured error", err: &ResponsesToolMessageError{ResponsesToolMessageErrorStruct: &ResponsesToolMessageErrorStruct{}}, want: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.err.IsError(); got != tt.want { + t.Fatalf("IsError() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestDeepCopyResponsesMessageCopiesStructuredErrorPointers(t *testing.T) { + original := ResponsesMessage{ResponsesToolMessage: &ResponsesToolMessage{Error: &ResponsesToolMessageError{ + ResponsesToolMessageErrorStruct: &ResponsesToolMessageErrorStruct{ + Type: "http_error", Code: Ptr(502), Message: Ptr("upstream failed"), Content: json.RawMessage(`{"retryable":true}`), + }, + }}} + + copied := DeepCopyResponsesMessage(original) + originalError := original.ResponsesToolMessage.Error.ResponsesToolMessageErrorStruct + copiedError := copied.ResponsesToolMessage.Error.ResponsesToolMessageErrorStruct + if copiedError == nil { + t.Fatal("deep copy dropped structured error") + } + if copiedError == originalError || copiedError.Code == originalError.Code || copiedError.Message == originalError.Message { + t.Fatal("deep copy aliases structured error fields") + } + if len(copiedError.Content) > 0 && &copiedError.Content[0] == &originalError.Content[0] { + t.Fatal("deep copy aliases structured error content") + } + + *copiedError.Code = 503 + *copiedError.Message = "changed" + copiedError.Content[2] = 'x' + if *originalError.Code != 502 || *originalError.Message != "upstream failed" || string(originalError.Content) != `{"retryable":true}` { + t.Fatalf("mutating copied error changed original: %#v", originalError) + } +} + +// TestResponsesToolOpenAIFields pins the OpenAI tool fields async (function and +// custom), output_schema (function) and tunnel_id (MCP) through a round trip, +// including function tools nested in a namespace. +func TestResponsesToolOpenAIFields(t *testing.T) { + for _, in := range []string{ + `{"type":"function","name":"get_weather","async":true,"parameters":{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]},"strict":true,"output_schema":{"type":"object","properties":{"temp":{"type":"number","exclusiveMinimum":-273}},"const":"x"}}`, + `{"type":"custom","name":"run_job","async":false,"format":{"type":"text"}}`, + `{"type":"mcp","server_label":"internal","tunnel_id":"tunnel_0123456789abcdef0123456789abcdef","require_approval":"never"}`, + `{"type":"namespace","name":"jobs","description":"Job tools","tools":[{"type":"function","name":"start","async":true,"parameters":{"type":"object","properties":{}},"strict":false,"output_schema":{"type":"string"}}]}`, + } { + var tool ResponsesTool + if err := Unmarshal([]byte(in), &tool); err != nil { + t.Fatalf("unmarshal %s: %v", in, err) + } + out, err := MarshalSorted(tool) + if err != nil { + t.Fatalf("marshal: %v", err) + } + var got, want any + if err := json.Unmarshal(out, &got); err != nil { + t.Fatalf("decode output: %v", err) + } + if err := json.Unmarshal([]byte(in), &want); err != nil { + t.Fatalf("decode input: %v", err) + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("round trip mismatch\n got: %s\nwant: %s", out, in) + } + } +} + +// TestResponsesToolCallAsyncSurvives pins async on function_call and +// custom_tool_call items. A replayed pending async call without it is rejected by +// OpenAI with "No tool output found for function call". +func TestResponsesToolCallAsyncSurvives(t *testing.T) { + for _, in := range []string{ + `{"type":"function_call","id":"fc_1","call_id":"call_1","name":"get_weather","arguments":"{\"city\":\"Paris\"}","async":true,"status":"completed"}`, + `{"type":"custom_tool_call","call_id":"call_2","name":"run_job","input":"go","async":true}`, + } { + var msg ResponsesMessage + if err := Unmarshal([]byte(in), &msg); err != nil { + t.Fatalf("unmarshal: %v", err) + } + out, err := MarshalSorted(msg) + if err != nil { + t.Fatalf("marshal: %v", err) + } + var got, want any + if err := json.Unmarshal(out, &got); err != nil { + t.Fatalf("decode output: %v", err) + } + if err := json.Unmarshal([]byte(in), &want); err != nil { + t.Fatalf("decode input: %v", err) + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("round trip mismatch\n got: %s\nwant: %s", out, in) + } + + copied := DeepCopyResponsesMessage(msg) + if copied.ResponsesToolMessage.Async == nil || !*copied.ResponsesToolMessage.Async { + t.Fatalf("deep copy lost async: %s", in) + } + if copied.ResponsesToolMessage.Async == msg.ResponsesToolMessage.Async { + t.Fatalf("deep copy aliases the async pointer: %s", in) + } + } +} diff --git a/core/schemas/responsestoolunmarshal_test.go b/core/schemas/responsestoolunmarshal_test.go index 3d91ea134cd..cafd712daea 100644 --- a/core/schemas/responsestoolunmarshal_test.go +++ b/core/schemas/responsestoolunmarshal_test.go @@ -280,6 +280,7 @@ func TestResponsesToolMarshalUnmarshalRoundTrip(t *testing.T) { {name: "file_search", in: `{"type":"file_search","vector_store_ids":["vs_1"]}`}, {name: "mcp", in: `{"type":"mcp","server_label":"srv","server_url":"https://example.com"}`}, {name: "local_shell", in: `{"type":"local_shell"}`}, + {name: "computer", in: `{"type":"computer"}`}, {name: "image_generation", in: `{"type":"image_generation"}`}, {name: "web_search canonical", in: `{"type":"web_search","max_uses":3}`}, { @@ -325,6 +326,25 @@ func TestResponsesToolMarshalUnmarshalRoundTrip(t *testing.T) { } } +// TestResponsesToolComputerRoundTripsBare locks in issue #7425: OpenAI's +// computer tool for GPT-6 Astra / GPT-5.6 is exactly {"type":"computer"}. It +// used to be folded into computer_use_preview by the "computer" prefix match +// (meant for Anthropic's dated computer_20250124 variants), and OpenAI then +// rejected the request with "Tool 'computer_use_preview' is not supported". +// Byte-equality matters: OpenAI 400s on any extra field (display_*, environment). +func TestResponsesToolComputerRoundTripsBare(t *testing.T) { + const in = `{"type":"computer"}` + + var tool ResponsesTool + require.NoError(t, Unmarshal([]byte(in), &tool)) + assert.Equal(t, ResponsesToolTypeComputer, tool.Type) + assert.Nil(t, tool.ResponsesToolComputerUsePreview) + + out, err := MarshalSorted(tool) + require.NoError(t, err) + assert.Equal(t, in, string(out)) +} + // ============================================================================= // image_generation_call — OpenAI sends `action` as a bare string, not an object // ============================================================================= diff --git a/core/schemas/serialization_test.go b/core/schemas/serialization_test.go index b3074c7cf7a..bf3fbc7b4ec 100644 --- a/core/schemas/serialization_test.go +++ b/core/schemas/serialization_test.go @@ -1211,6 +1211,8 @@ func TestNormalizeResponsesToolType(t *testing.T) { {"web_fetch_20260209", ResponsesToolTypeWebFetch}, {"web_fetch_20260309", ResponsesToolTypeWebFetch}, + // bare "computer" is OpenAI's own tool (GPT-6 Astra / GPT-5.6) and must not fold into the preview type + {ResponsesToolTypeComputer, ResponsesToolTypeComputer}, // computer versioned aliases {"computer_20250124", ResponsesToolTypeComputerUsePreview}, {"computer_20251124", ResponsesToolTypeComputerUsePreview}, diff --git a/core/schemas/sessionaffinity.go b/core/schemas/sessionaffinity.go new file mode 100644 index 00000000000..f96aecef437 --- /dev/null +++ b/core/schemas/sessionaffinity.go @@ -0,0 +1,65 @@ +package schemas + +// Route is one provider and model a request can be sent to, with the provider key pinned for +// that attempt when a routing rule named one. An empty KeyID leaves the key to selection. +// The pin belongs to the route: an implementation that reorders a chain moves each route +// whole, so a pin stays with the provider it was decided for wherever that provider ends up. +type Route struct { + Provider ModelProvider + Model string + KeyID string +} + +// RouteOutcome is how a request ended. +type RouteOutcome struct { + // Served is the route that answered, nil when no route did. + Served *Route + // KeyID is the key Served used, when key selection ran for it. + KeyID string + // Fallback reports that Served was not the first route of the dispatched chain. + Fallback bool + // Err is the failure the request ended in, nil when it was served. + Err *BifrostError +} + +// IsSessionAffinityActive reports whether a request takes part in session affinity: it carries +// a session id and has not switched affinity off (BifrostContextKeySessionAffinity). Core +// consults it before every SessionAffinity call, so an implementation never sees a request +// that does not take part. +func IsSessionAffinityActive(ctx *BifrostContext) bool { + if ctx == nil { + return false + } + if enabled, ok := ctx.Value(BifrostContextKeySessionAffinity).(bool); ok && !enabled { + return false + } + sessionID, _ := ctx.Value(BifrostContextKeySessionID).(string) + return sessionID != "" +} + +// SessionAffinity keeps a request that carries a session id on what served that session +// before. Core asks it at three points and applies the answers; the policy behind them +// belongs to the implementation. The one Bifrost ships keeps a session on the provider and +// key that last served it, as far as the request's own outcome shows, forgets a binding the +// request followed into a failure, and is installed when nothing else is. A deployment that knows more, such as the health of its providers and +// keys, registers its own through BifrostConfig.SessionAffinity. +// +// Core asks only for requests IsSessionAffinityActive reports as taking part: ones that carry +// a session id and have not switched affinity off. Every method receives the request context, +// which carries the session id (BifrostContextKeySessionID), the session TTL +// (BifrostContextKeySessionTTL) and the identity that scopes the session to its caller. +type SessionAffinity interface { + // ResolveRoute is asked once every routing hook has run, with the route the caller asked + // for and the chain the hooks produced: chain[0] is the primary, the rest its fallbacks in + // order. It answers with the chain to dispatch. The chain unchanged, or nil, leaves the + // request as the hooks left it. + ResolveRoute(ctx *BifrostContext, requested Route, chain []Route) []Route + // ResolveKey is asked once the eligible key pool for a provider attempt has been built + // with two or more keys in it. It answers with the key the session's state settles on, or + // with ok=false to leave the choice to ordinary key selection. A returned key is used for + // every attempt of the request. + ResolveKey(ctx *BifrostContext, provider ModelProvider, model string, eligible []Key) (key Key, ok bool) + // Observe is told how the request ended, once: for a unary request when its response is + // in, for a stream once the upstream has accepted it and started streaming. + Observe(ctx *BifrostContext, requested Route, outcome RouteOutcome) +} diff --git a/core/schemas/toolerrormux_test.go b/core/schemas/toolerrormux_test.go index 6577d4f5a2d..c6c492cdbba 100644 --- a/core/schemas/toolerrormux_test.go +++ b/core/schemas/toolerrormux_test.go @@ -59,7 +59,7 @@ func TestResponsesToChatCarriesToolError(t *testing.T) { Type: Ptr(ResponsesMessageTypeFunctionCallOutput), ResponsesToolMessage: &ResponsesToolMessage{ CallID: Ptr("call_1"), - Error: Ptr("ENOENT: no such file or directory"), + Error: &ResponsesToolMessageError{ResponsesToolMessageErrorStr: Ptr("ENOENT: no such file or directory")}, }, }, }, @@ -76,6 +76,18 @@ func TestResponsesToChatCarriesToolError(t *testing.T) { }, }, }, + { + name: "empty structured error", + msg: ResponsesMessage{ + Type: Ptr(ResponsesMessageTypeFunctionCallOutput), + ResponsesToolMessage: &ResponsesToolMessage{ + CallID: Ptr("call_1"), + Error: &ResponsesToolMessageError{ + ResponsesToolMessageErrorStruct: &ResponsesToolMessageErrorStruct{}, + }, + }, + }, + }, } for _, tt := range tests { @@ -99,6 +111,7 @@ func TestResponsesToChatCarriesToolError(t *testing.T) { Status: Ptr("completed"), ResponsesToolMessage: &ResponsesToolMessage{ CallID: Ptr("call_2"), + Error: &ResponsesToolMessageError{ResponsesToolMessageErrorStr: Ptr("")}, Output: &ResponsesToolMessageOutputStruct{ResponsesToolCallOutputStr: Ptr("15 degrees")}, }, }}) diff --git a/core/schemas/utils.go b/core/schemas/utils.go index 86d258a136d..092398b0477 100644 --- a/core/schemas/utils.go +++ b/core/schemas/utils.go @@ -1315,6 +1315,18 @@ func DeepCopyResponsesMessage(original ResponsesMessage) ResponsesMessage { copyRole := *original.Role copy.Role = ©Role } + if original.CacheControl != nil { + copyCacheControl := &CacheControl{Type: original.CacheControl.Type} + if original.CacheControl.TTL != nil { + copyTTL := *original.CacheControl.TTL + copyCacheControl.TTL = ©TTL + } + if original.CacheControl.Scope != nil { + copyScope := *original.CacheControl.Scope + copyCacheControl.Scope = ©Scope + } + copy.CacheControl = copyCacheControl + } // Deep copy Author and Recipient (multi-agent collab_tool_call items). // json.RawMessage is a []byte slice; copy the bytes so callers don't share @@ -1331,6 +1343,10 @@ func DeepCopyResponsesMessage(original ResponsesMessage) ResponsesMessage { if original.AdditionalTools != nil { copy.AdditionalTools = append(json.RawMessage(nil), original.AdditionalTools...) } + // Deep copy ProviderNativeParts. + if original.ProviderNativeParts != nil { + copy.ProviderNativeParts = append(json.RawMessage(nil), original.ProviderNativeParts...) + } // Raw-preserved items re-marshal from these bytes; without them the copy // falls back to a field-by-field encode and loses the item's payload. if original.rawPreserved != nil { @@ -1373,6 +1389,11 @@ func DeepCopyResponsesMessage(original ResponsesMessage) ResponsesMessage { copy.ResponsesToolMessage.Arguments = ©Arguments } + if original.ResponsesToolMessage.ToolsetName != nil { + copyToolsetName := *original.ResponsesToolMessage.ToolsetName + copy.ResponsesToolMessage.ToolsetName = ©ToolsetName + } + if original.ResponsesToolMessage.ResponsesCustomToolCall != nil { copyCustomToolCall := *original.ResponsesToolMessage.ResponsesCustomToolCall copy.ResponsesToolMessage.ResponsesCustomToolCall = ©CustomToolCall @@ -1388,8 +1409,29 @@ func DeepCopyResponsesMessage(original ResponsesMessage) ResponsesMessage { copy.ResponsesToolMessage.Execution = ©Execution } + if original.ResponsesToolMessage.Async != nil { + copy.ResponsesToolMessage.Async = new(*original.ResponsesToolMessage.Async) + } + if original.ResponsesToolMessage.Error != nil { - copyError := *original.ResponsesToolMessage.Error + copyError := ResponsesToolMessageError{} + if original.ResponsesToolMessage.Error.ResponsesToolMessageErrorStr != nil { + copyErrorStr := *original.ResponsesToolMessage.Error.ResponsesToolMessageErrorStr + copyError.ResponsesToolMessageErrorStr = ©ErrorStr + } + if original.ResponsesToolMessage.Error.ResponsesToolMessageErrorStruct != nil { + copyErrorStruct := *original.ResponsesToolMessage.Error.ResponsesToolMessageErrorStruct + if copyErrorStruct.Code != nil { + copyCode := *copyErrorStruct.Code + copyErrorStruct.Code = ©Code + } + if copyErrorStruct.Message != nil { + copyMessage := *copyErrorStruct.Message + copyErrorStruct.Message = ©Message + } + copyErrorStruct.Content = append(json.RawMessage(nil), copyErrorStruct.Content...) + copyError.ResponsesToolMessageErrorStruct = ©ErrorStruct + } copy.ResponsesToolMessage.Error = ©Error } @@ -1503,6 +1545,67 @@ func DeepCopyResponsesMessage(original ResponsesMessage) ResponsesMessage { copy.ResponsesToolMessage.ResponsesWebFetchCall = ©Call } + // Deep copy ResponsesComputerToolCall. + if original.ResponsesToolMessage.ResponsesComputerToolCall != nil { + copyToolCall := *original.ResponsesToolMessage.ResponsesComputerToolCall + if original.ResponsesToolMessage.ResponsesComputerToolCall.PendingSafetyChecks != nil { + copyToolCall.PendingSafetyChecks = append([]ResponsesComputerToolCallPendingSafetyCheck(nil), original.ResponsesToolMessage.ResponsesComputerToolCall.PendingSafetyChecks...) + } + copy.ResponsesToolMessage.ResponsesComputerToolCall = ©ToolCall + } + + // Deep copy ResponsesComputerToolCallOutput. + if original.ResponsesToolMessage.ResponsesComputerToolCallOutput != nil { + copyToolCallOutput := *original.ResponsesToolMessage.ResponsesComputerToolCallOutput + if original.ResponsesToolMessage.ResponsesComputerToolCallOutput.AcknowledgedSafetyChecks != nil { + copyToolCallOutput.AcknowledgedSafetyChecks = make([]ResponsesComputerToolCallAcknowledgedSafetyCheck, len(original.ResponsesToolMessage.ResponsesComputerToolCallOutput.AcknowledgedSafetyChecks)) + for i, safetyCheck := range original.ResponsesToolMessage.ResponsesComputerToolCallOutput.AcknowledgedSafetyChecks { + copySafetyCheck := ResponsesComputerToolCallAcknowledgedSafetyCheck{ID: safetyCheck.ID} + if safetyCheck.Code != nil { + copyCode := *safetyCheck.Code + copySafetyCheck.Code = ©Code + } + if safetyCheck.Message != nil { + copyMessage := *safetyCheck.Message + copySafetyCheck.Message = ©Message + } + copyToolCallOutput.AcknowledgedSafetyChecks[i] = copySafetyCheck + } + } + copy.ResponsesToolMessage.ResponsesComputerToolCallOutput = ©ToolCallOutput + } + + // Deep copy ResponsesCodeInterpreterToolCall. + if original.ResponsesToolMessage.ResponsesCodeInterpreterToolCall != nil { + copyToolCall := *original.ResponsesToolMessage.ResponsesCodeInterpreterToolCall + if original.ResponsesToolMessage.ResponsesCodeInterpreterToolCall.Code != nil { + copyCode := *original.ResponsesToolMessage.ResponsesCodeInterpreterToolCall.Code + copyToolCall.Code = ©Code + } + if original.ResponsesToolMessage.ResponsesCodeInterpreterToolCall.Outputs != nil { + copyToolCall.Outputs = make([]ResponsesCodeInterpreterOutput, len(original.ResponsesToolMessage.ResponsesCodeInterpreterToolCall.Outputs)) + for i, output := range original.ResponsesToolMessage.ResponsesCodeInterpreterToolCall.Outputs { + copyOutput := ResponsesCodeInterpreterOutput{} + if output.ResponsesCodeInterpreterOutputLogs != nil { + copyLogs := *output.ResponsesCodeInterpreterOutputLogs + copyOutput.ResponsesCodeInterpreterOutputLogs = ©Logs + } + if output.ResponsesCodeInterpreterOutputImage != nil { + copyImage := *output.ResponsesCodeInterpreterOutputImage + copyOutput.ResponsesCodeInterpreterOutputImage = ©Image + } + copyToolCall.Outputs[i] = copyOutput + } + } + copy.ResponsesToolMessage.ResponsesCodeInterpreterToolCall = ©ToolCall + } + + // Deep copy ResponsesMCPToolCall. + if original.ResponsesToolMessage.ResponsesMCPToolCall != nil { + copyToolCall := *original.ResponsesToolMessage.ResponsesMCPToolCall + copy.ResponsesToolMessage.ResponsesMCPToolCall = ©ToolCall + } + // Deep copy ResponsesImageGenerationCall. if original.ResponsesToolMessage.ResponsesImageGenerationCall != nil { copyToolCall := *original.ResponsesToolMessage.ResponsesImageGenerationCall @@ -1529,6 +1632,160 @@ func DeepCopyResponsesMessage(original ResponsesMessage) ResponsesMessage { copy.ResponsesToolMessage.ResponsesImageGenerationCall = ©ToolCall } + // Deep copy ResponsesMCPListTools. + if original.ResponsesToolMessage.ResponsesMCPListTools != nil { + copyToolList := *original.ResponsesToolMessage.ResponsesMCPListTools + if original.ResponsesToolMessage.ResponsesMCPListTools.Tools != nil { + copyToolList.Tools = make([]ResponsesMCPTool, len(original.ResponsesToolMessage.ResponsesMCPListTools.Tools)) + for i, tool := range original.ResponsesToolMessage.ResponsesMCPListTools.Tools { + copyTool := tool + if tool.InputSchema != nil { + copyTool.InputSchema = DeepCopy(tool.InputSchema).(map[string]any) + } + if tool.Description != nil { + copyDescription := *tool.Description + copyTool.Description = ©Description + } + if tool.Annotations != nil { + var copyAnnotations map[string]any + if *tool.Annotations != nil { + copyAnnotations = DeepCopy(*tool.Annotations).(map[string]any) + } + copyTool.Annotations = ©Annotations + } + copyToolList.Tools[i] = copyTool + } + } + copy.ResponsesToolMessage.ResponsesMCPListTools = ©ToolList + } + + // Deep copy ResponsesMCPApprovalResponse. + if original.ResponsesToolMessage.ResponsesMCPApprovalResponse != nil { + copyApprovalResponse := *original.ResponsesToolMessage.ResponsesMCPApprovalResponse + if original.ResponsesToolMessage.ResponsesMCPApprovalResponse.Reason != nil { + copyReason := *original.ResponsesToolMessage.ResponsesMCPApprovalResponse.Reason + copyApprovalResponse.Reason = ©Reason + } + copy.ResponsesToolMessage.ResponsesMCPApprovalResponse = ©ApprovalResponse + } + + // Deep copy ResponsesAdvisorCall. + if original.ResponsesToolMessage.ResponsesAdvisorCall != nil { + copyAdvisorCall := *original.ResponsesToolMessage.ResponsesAdvisorCall + if original.ResponsesToolMessage.ResponsesAdvisorCall.Text != nil { + copyText := *original.ResponsesToolMessage.ResponsesAdvisorCall.Text + copyAdvisorCall.Text = ©Text + } + if original.ResponsesToolMessage.ResponsesAdvisorCall.EncryptedContent != nil { + copyEncryptedContent := *original.ResponsesToolMessage.ResponsesAdvisorCall.EncryptedContent + copyAdvisorCall.EncryptedContent = ©EncryptedContent + } + if original.ResponsesToolMessage.ResponsesAdvisorCall.ErrorCode != nil { + copyErrorCode := *original.ResponsesToolMessage.ResponsesAdvisorCall.ErrorCode + copyAdvisorCall.ErrorCode = ©ErrorCode + } + if original.ResponsesToolMessage.ResponsesAdvisorCall.StopReason != nil { + copyStopReason := *original.ResponsesToolMessage.ResponsesAdvisorCall.StopReason + copyAdvisorCall.StopReason = ©StopReason + } + copy.ResponsesToolMessage.ResponsesAdvisorCall = ©AdvisorCall + } + + // Deep copy ResponsesToolSearchCall. + if original.ResponsesToolMessage.ResponsesToolSearchCall != nil { + copyToolSearchCall := *original.ResponsesToolMessage.ResponsesToolSearchCall + copyToolSearchCall.ToolReferences = append([]string(nil), original.ResponsesToolMessage.ResponsesToolSearchCall.ToolReferences...) + copy.ResponsesToolMessage.ResponsesToolSearchCall = ©ToolSearchCall + } + + // Deep copy ResponsesCodeExecutionCall. + if original.ResponsesToolMessage.ResponsesCodeExecutionCall != nil { + copyCodeExecutionCall := *original.ResponsesToolMessage.ResponsesCodeExecutionCall + if original.ResponsesToolMessage.ResponsesCodeExecutionCall.Input != nil { + copyInput := *original.ResponsesToolMessage.ResponsesCodeExecutionCall.Input + copyCodeExecutionCall.Input = ©Input + } + if original.ResponsesToolMessage.ResponsesCodeExecutionCall.Stdout != nil { + copyStdout := *original.ResponsesToolMessage.ResponsesCodeExecutionCall.Stdout + copyCodeExecutionCall.Stdout = ©Stdout + } + if original.ResponsesToolMessage.ResponsesCodeExecutionCall.Stderr != nil { + copyStderr := *original.ResponsesToolMessage.ResponsesCodeExecutionCall.Stderr + copyCodeExecutionCall.Stderr = ©Stderr + } + if original.ResponsesToolMessage.ResponsesCodeExecutionCall.ReturnCode != nil { + copyReturnCode := *original.ResponsesToolMessage.ResponsesCodeExecutionCall.ReturnCode + copyCodeExecutionCall.ReturnCode = ©ReturnCode + } + if original.ResponsesToolMessage.ResponsesCodeExecutionCall.EncryptedStdout != nil { + copyEncryptedStdout := *original.ResponsesToolMessage.ResponsesCodeExecutionCall.EncryptedStdout + copyCodeExecutionCall.EncryptedStdout = ©EncryptedStdout + } + if original.ResponsesToolMessage.ResponsesCodeExecutionCall.FileType != nil { + copyFileType := *original.ResponsesToolMessage.ResponsesCodeExecutionCall.FileType + copyCodeExecutionCall.FileType = ©FileType + } + if original.ResponsesToolMessage.ResponsesCodeExecutionCall.FileContent != nil { + copyFileContent := *original.ResponsesToolMessage.ResponsesCodeExecutionCall.FileContent + copyCodeExecutionCall.FileContent = ©FileContent + } + if original.ResponsesToolMessage.ResponsesCodeExecutionCall.StartLine != nil { + copyStartLine := *original.ResponsesToolMessage.ResponsesCodeExecutionCall.StartLine + copyCodeExecutionCall.StartLine = ©StartLine + } + if original.ResponsesToolMessage.ResponsesCodeExecutionCall.NumLines != nil { + copyNumLines := *original.ResponsesToolMessage.ResponsesCodeExecutionCall.NumLines + copyCodeExecutionCall.NumLines = ©NumLines + } + if original.ResponsesToolMessage.ResponsesCodeExecutionCall.TotalLines != nil { + copyTotalLines := *original.ResponsesToolMessage.ResponsesCodeExecutionCall.TotalLines + copyCodeExecutionCall.TotalLines = ©TotalLines + } + if original.ResponsesToolMessage.ResponsesCodeExecutionCall.IsFileUpdate != nil { + copyIsFileUpdate := *original.ResponsesToolMessage.ResponsesCodeExecutionCall.IsFileUpdate + copyCodeExecutionCall.IsFileUpdate = ©IsFileUpdate + } + if original.ResponsesToolMessage.ResponsesCodeExecutionCall.OldStart != nil { + copyOldStart := *original.ResponsesToolMessage.ResponsesCodeExecutionCall.OldStart + copyCodeExecutionCall.OldStart = ©OldStart + } + if original.ResponsesToolMessage.ResponsesCodeExecutionCall.OldLines != nil { + copyOldLines := *original.ResponsesToolMessage.ResponsesCodeExecutionCall.OldLines + copyCodeExecutionCall.OldLines = ©OldLines + } + if original.ResponsesToolMessage.ResponsesCodeExecutionCall.NewStart != nil { + copyNewStart := *original.ResponsesToolMessage.ResponsesCodeExecutionCall.NewStart + copyCodeExecutionCall.NewStart = ©NewStart + } + if original.ResponsesToolMessage.ResponsesCodeExecutionCall.NewLines != nil { + copyNewLines := *original.ResponsesToolMessage.ResponsesCodeExecutionCall.NewLines + copyCodeExecutionCall.NewLines = ©NewLines + } + if original.ResponsesToolMessage.ResponsesCodeExecutionCall.Lines != nil { + copyCodeExecutionCall.Lines = append([]string(nil), original.ResponsesToolMessage.ResponsesCodeExecutionCall.Lines...) + } + if original.ResponsesToolMessage.ResponsesCodeExecutionCall.ErrorCode != nil { + copyErrorCode := *original.ResponsesToolMessage.ResponsesCodeExecutionCall.ErrorCode + copyCodeExecutionCall.ErrorCode = ©ErrorCode + } + if original.ResponsesToolMessage.ResponsesCodeExecutionCall.Files != nil { + copyCodeExecutionCall.Files = append([]ResponsesCodeExecutionFileOutput(nil), original.ResponsesToolMessage.ResponsesCodeExecutionCall.Files...) + } + if original.ResponsesToolMessage.ResponsesCodeExecutionCall.ContainerExpiresAt != nil { + copyContainerExpiresAt := *original.ResponsesToolMessage.ResponsesCodeExecutionCall.ContainerExpiresAt + copyCodeExecutionCall.ContainerExpiresAt = ©ContainerExpiresAt + } + if original.ResponsesToolMessage.ResponsesCodeExecutionCall.Caller != nil { + copyCaller := *original.ResponsesToolMessage.ResponsesCodeExecutionCall.Caller + if original.ResponsesToolMessage.ResponsesCodeExecutionCall.Caller.ToolID != nil { + copyToolID := *original.ResponsesToolMessage.ResponsesCodeExecutionCall.Caller.ToolID + copyCaller.ToolID = ©ToolID + } + copyCodeExecutionCall.Caller = ©Caller + } + copy.ResponsesToolMessage.ResponsesCodeExecutionCall = ©CodeExecutionCall + } + // Add other embedded tool calls as needed... } @@ -1547,6 +1804,38 @@ func deepCopyResponsesMessageContentBlock(original ResponsesMessageContentBlock) Type: original.Type, } + // Deep copy CacheControl if present + if original.CacheControl != nil { + copyCacheControl := &CacheControl{Type: original.CacheControl.Type} + if original.CacheControl.TTL != nil { + copyTTL := *original.CacheControl.TTL + copyCacheControl.TTL = ©TTL + } + if original.CacheControl.Scope != nil { + copyScope := *original.CacheControl.Scope + copyCacheControl.Scope = ©Scope + } + copy.CacheControl = copyCacheControl + } + // Deep copy Citations. + if original.Citations != nil { + copyCitations := &Citations{} + if original.Citations.Enabled != nil { + copyEnabled := *original.Citations.Enabled + copyCitations.Enabled = ©Enabled + } + copy.Citations = copyCitations + } + // Deep copy PromptCacheBreakpoint. + if original.PromptCacheBreakpoint != nil { + copyPromptCacheBreakpoint := &PromptCacheBreakpoint{} + if original.PromptCacheBreakpoint.Mode != nil { + copyMode := *original.PromptCacheBreakpoint.Mode + copyPromptCacheBreakpoint.Mode = ©Mode + } + copy.PromptCacheBreakpoint = copyPromptCacheBreakpoint + } + // Copy FileID if present if original.FileID != nil { copyFileID := *original.FileID @@ -1608,6 +1897,11 @@ func deepCopyResponsesMessageContentBlock(original ResponsesMessageContentBlock) copyFilename := *original.ResponsesInputMessageContentBlockFile.Filename copyFile.Filename = ©Filename } + // Deep copy FileType. + if original.ResponsesInputMessageContentBlockFile.FileType != nil { + copyFileType := *original.ResponsesInputMessageContentBlockFile.FileType + copyFile.FileType = ©FileType + } copy.ResponsesInputMessageContentBlockFile = copyFile } @@ -1667,6 +1961,39 @@ func deepCopyResponsesMessageContentBlock(original ResponsesMessageContentBlock) copyContainerID := *annotation.ContainerID copyAnnotation.ContainerID = ©ContainerID } + // Deep copy Anthropic-specific annotation fields. + if annotation.StartCharIndex != nil { + copyStartCharIndex := *annotation.StartCharIndex + copyAnnotation.StartCharIndex = ©StartCharIndex + } + if annotation.EndCharIndex != nil { + copyEndCharIndex := *annotation.EndCharIndex + copyAnnotation.EndCharIndex = ©EndCharIndex + } + if annotation.StartPageNumber != nil { + copyStartPageNumber := *annotation.StartPageNumber + copyAnnotation.StartPageNumber = ©StartPageNumber + } + if annotation.EndPageNumber != nil { + copyEndPageNumber := *annotation.EndPageNumber + copyAnnotation.EndPageNumber = ©EndPageNumber + } + if annotation.StartBlockIndex != nil { + copyStartBlockIndex := *annotation.StartBlockIndex + copyAnnotation.StartBlockIndex = ©StartBlockIndex + } + if annotation.EndBlockIndex != nil { + copyEndBlockIndex := *annotation.EndBlockIndex + copyAnnotation.EndBlockIndex = ©EndBlockIndex + } + if annotation.Source != nil { + copySource := *annotation.Source + copyAnnotation.Source = ©Source + } + if annotation.EncryptedIndex != nil { + copyEncryptedIndex := *annotation.EncryptedIndex + copyAnnotation.EncryptedIndex = ©EncryptedIndex + } copyText.Annotations[i] = copyAnnotation } } @@ -1718,6 +2045,29 @@ func deepCopyResponsesMessageContentBlock(original ResponsesMessageContentBlock) } copy.ResponsesOutputMessageContentRefusal = copyRefusal } + // Deep copy extended output content variants. + if original.ResponsesOutputMessageContentRenderedContent != nil { + copy.ResponsesOutputMessageContentRenderedContent = &ResponsesOutputMessageContentRenderedContent{ + RenderedContent: original.ResponsesOutputMessageContentRenderedContent.RenderedContent, + } + } + if original.ResponsesOutputMessageContentCompaction != nil { + copy.ResponsesOutputMessageContentCompaction = &ResponsesOutputMessageContentCompaction{ + Summary: original.ResponsesOutputMessageContentCompaction.Summary, + } + } + if original.ResponsesOutputMessageContentFallback != nil { + copyFallback := &ResponsesOutputMessageContentFallback{ + FromModel: original.ResponsesOutputMessageContentFallback.FromModel, + ToModel: original.ResponsesOutputMessageContentFallback.ToModel, + TriggerType: original.ResponsesOutputMessageContentFallback.TriggerType, + } + if original.ResponsesOutputMessageContentFallback.TriggerCategory != nil { + copyTriggerCategory := *original.ResponsesOutputMessageContentFallback.TriggerCategory + copyFallback.TriggerCategory = ©TriggerCategory + } + copy.ResponsesOutputMessageContentFallback = copyFallback + } return copy } @@ -1742,6 +2092,27 @@ func IsDeepSeekModel(model string) bool { return strings.Contains(strings.ToLower(model), "deepseek") } +// IsMoonshotModel reports whether the model is a Moonshot (Kimi) model, under +// any of the ids it is reached by: "moonshotai.kimi-k3" on Bedrock (with or +// without an inference-profile prefix), "moonshotai/kimi-k2-instruct" on Groq, +// or a bare "kimi-k3" behind an OpenAI-compatible gateway. +// +// Matching is boundary-aware: a "moonshotai" path segment, or a segment that +// starts with "kimi-", the prefix every model on Moonshot's own list carries. +// A bare substring match would also claim unrelated ids such as +// "kimina-prover", and this predicate gates a lossy schema rewrite, so a false +// positive quietly degrades a model that was fine. +func IsMoonshotModel(model string) bool { + for _, segment := range strings.FieldsFunc(strings.ToLower(model), func(r rune) bool { + return r == '/' || r == '.' || r == ':' + }) { + if segment == "moonshotai" || strings.HasPrefix(segment, "kimi-") { + return true + } + } + return false +} + // IsGPT56Model reports whether the model belongs to the gpt-5.6 family, which is the // first OpenAI generation to accept prompt_cache_options / prompt_cache_breakpoint. // @@ -1772,6 +2143,12 @@ func IsGPT56Model(model string) bool { return false } +// ModelSupportsPromptCacheBreakpoint is the name-based fallback for +// ModelCaps.SupportsPromptCacheBreakpoint: the gpt-5.6 and gpt-6 families. +func ModelSupportsPromptCacheBreakpoint(model string) bool { + return IsGPT56Model(model) || strings.Contains(strings.ToLower(model), "gpt-6") +} + // IsAnthropicModel checks if the model is an Anthropic model. func IsAnthropicModel(model string) bool { return strings.Contains(model, "anthropic.") || strings.Contains(model, "claude") @@ -1824,6 +2201,9 @@ func (r *BifrostResponse) ServedModel() string { return r.ResponsesStreamResponse.Response.Model case r.TextCompletionResponse != nil: return r.TextCompletionResponse.Model + case r.DecisionResponse != nil: + // An emulated decision names the model its text call was served by. + return r.DecisionResponse.Model } return "" } @@ -1884,13 +2264,13 @@ func ModelSupportsPromptCaching(provider ModelProvider, model string) bool { case Anthropic, OpenRouter: return IsAnthropicModel(model) case Bedrock, BedrockMantle: - return BedrockModelSupportsCachePoints(model) || IsGPT56Model(model) + return BedrockModelSupportsCachePoints(model) || ModelSupportsPromptCacheBreakpoint(model) case Vertex: // Vertex serves Claude (cache_control) and Gemini (cachedContent) side by // side; only the former is markable. return IsAnthropicModel(model) case Azure, OpenAI: - return IsGPT56Model(model) + return ModelSupportsPromptCacheBreakpoint(model) default: return false } @@ -1953,10 +2333,29 @@ func IsFable51(model string) bool { return strings.Contains(m, "5-1") || strings.Contains(m, "5.1") } +// IsOpus55Plus checks if the model is Claude Opus 5.5 or later, which dropped +// forced tool use and made thinking always-on: tool_choice "any"/"tool" and +// thinking:{"type":"disabled"} each return a 400 at every effort level. Matches +// the Bedrock/Vertex/date-suffixed forms. +// +// Only the versions known to have dropped them are matched here; a later model +// that also drops them is carried by the datasheet rather than this fallback. +// The sibling thinking:{"type":"enabled"} rejection is already covered by the +// adaptive-only gate these models share with Opus 4.7+. +// +// Source: https://platform.claude.com/docs/en/models/opus-5-5/whats-new-opus-5-5 +func IsOpus55Plus(model string) bool { + m := strings.ToLower(model) + if !strings.Contains(m, "opus") { + return false + } + return strings.Contains(m, "5-5") || strings.Contains(m, "5.5") +} + // DefaultSupportsForcedToolChoice is the name-based fallback for // ModelCaps.SupportsForcedToolChoice, used when the datasheet says nothing. func DefaultSupportsForcedToolChoice(model string) bool { - return !IsFable51(model) + return !IsFable51(model) && !IsOpus55Plus(model) } // IsLlamaModel checks if the model is a Meta Llama model. @@ -2072,6 +2471,7 @@ func SupportsGrokReasoningEffort(model string) bool { // while keeping the request honest about what it asked for. var grokModelsWithXHighReasoningEffort = map[string]struct{}{ "grok-4.6": {}, + "grok-4.7": {}, "grok-4.20-multi-agent": {}, } diff --git a/core/schemas/utils_test.go b/core/schemas/utils_test.go index 3fe9e1e911c..a2a47121ae0 100644 --- a/core/schemas/utils_test.go +++ b/core/schemas/utils_test.go @@ -161,3 +161,43 @@ func TestIsGPT56ModelRequiresRevisionBoundary(t *testing.T) { assert.False(t, IsGPT56Model(model), "expected %q NOT to resolve as gpt-5.6 family", model) } } + +// TestModelSupportsPromptCacheBreakpointCoversGPT6 pins that the breakpoint fallback, +// and the prompt-caching fallback built on it, include the gpt-6 family. +func TestModelSupportsPromptCacheBreakpointCoversGPT6(t *testing.T) { + for _, model := range []string{"gpt-5.6-sol", "gpt-6-astra", "azure/gpt-6-astra", "openai.gpt-6-astra"} { + assert.True(t, ModelSupportsPromptCacheBreakpoint(model), model) + } + for _, model := range []string{"gpt-5.5", "gpt-5", "gpt-4o"} { + assert.False(t, ModelSupportsPromptCacheBreakpoint(model), model) + } + assert.True(t, ModelSupportsPromptCaching(OpenAI, "gpt-6-astra")) + assert.True(t, ModelSupportsPromptCaching(BedrockMantle, "openai.gpt-6-astra")) + assert.False(t, ModelSupportsPromptCaching(OpenAI, "gpt-5.5")) +} + +// DeepCopyResponsesMessage keeps a computer-toolset member call's toolset name. +func TestDeepCopyResponsesMessageCopiesToolsetName(t *testing.T) { + original := ResponsesMessage{ + Type: Ptr(ResponsesMessageTypeFunctionCall), + ResponsesToolMessage: &ResponsesToolMessage{ + CallID: Ptr("call_1"), + ToolsetName: Ptr("computer"), + }, + } + copied := DeepCopyResponsesMessage(original) + if copied.ResponsesToolMessage == nil || copied.ResponsesToolMessage.ToolsetName == nil || *copied.ResponsesToolMessage.ToolsetName != "computer" { + t.Fatalf("toolset name not copied: %#v", copied.ResponsesToolMessage) + } + if copied.ResponsesToolMessage.ToolsetName == original.ResponsesToolMessage.ToolsetName { + t.Fatal("toolset name pointer was aliased") + } +} + +// An emulated decision names the model its text call was served by. +func TestServedModelIncludesDecisionResponses(t *testing.T) { + r := &BifrostResponse{DecisionResponse: &BifrostDecisionResponse{Model: "gpt-5.5-2026-09"}} + if got := r.ServedModel(); got != "gpt-5.5-2026-09" { + t.Fatalf("ServedModel() = %q", got) + } +} diff --git a/core/sessionaffinity.go b/core/sessionaffinity.go new file mode 100644 index 00000000000..787cef7f9dd --- /dev/null +++ b/core/sessionaffinity.go @@ -0,0 +1,489 @@ +package bifrost + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "slices" + "strings" + "time" + + "github.com/bytedance/sonic" + "github.com/capsohq/bifrost/core/schemas" +) + +// Session state keeps a request that carries a session id on what served that session before. +// The session id is chosen by the caller, so two callers can send the same one; every piece of +// session state is therefore keyed under who the request's grant identity attributes it to, +// the virtual key and user, or under the deployment as a whole when neither is known. State +// lives in the schemas.KVStore the Bifrost instance was given, so it is shared wherever that +// store is. + +const ( + sessionStateKeyPrefix = "session:v2:" + + // SessionStateKindRoute is the state kind of a session's route binding: the provider and + // model that served the session for what the caller asked for. + SessionStateKindRoute = "route" + // SessionStateKindKey is the state kind of a session's key binding within one provider and + // model. + SessionStateKindKey = "key" + + // sessionAffinityResolvedKey is where the shipped affinity remembers, for the life of one + // request, what it settled at resolve time, so that the outcome can tell a move from a bind + // that lost a race. + sessionAffinityResolvedKey schemas.BifrostContextKey = "bifrost-session-affinity-resolved" +) + +// sessionResolution is what sessionAffinity settled for a request before it was dispatched. +type sessionResolution struct { + route string // the route binding the request followed, as routeString spells it + keyRoute schemas.Route // the route ResolveKey answered for + keyID string // the key binding the request followed on keyRoute +} + +// sessionAffinity is the session affinity Bifrost ships, installed when the configuration +// registers none. It knows nothing beyond what requests show it: a session stays on the +// provider that last served it, when routing still offers that provider, and on the key that +// last served it within a provider, while that key stays eligible. Bindings are written when +// a request is served, so a fallback that served becomes the session's new home and a key that +// only ever failed is never bound. A binding the request followed into a failure is dropped: +// a session is never held on a provider or key that just failed it for the rest of the TTL. +type sessionAffinity struct { + kv schemas.KVStore + logger schemas.Logger +} + +// NewSessionAffinity builds the affinity Bifrost ships over kv, the one Init installs when the +// configuration registers none. It is returned as the interface so a deployment that registers +// its own can embed it and delegate: decide first from what it knows, then let this one keep +// the bindings. Without a store it declines every request. The logger may not be nil. +func NewSessionAffinity(kv schemas.KVStore, logger schemas.Logger) schemas.SessionAffinity { + return &sessionAffinity{kv: kv, logger: logger} +} + +// ResolveRoute implements schemas.SessionAffinity: for a request that left the provider to +// routing, the route that last served the session for that model, provider and model together, +// moves to the front of the chain when the chain offers it. A request that named its provider +// is left as it is. +func (a *sessionAffinity) ResolveRoute(ctx *schemas.BifrostContext, requested schemas.Route, chain []schemas.Route) []schemas.Route { + if a == nil || a.kv == nil || ctx == nil { + return chain + } + // A context can carry what an earlier request on it followed, so this request starts clean: + // Observe must never take a stale resolution for its own. + ctx.ClearValue(sessionAffinityResolvedKey) + if len(chain) == 0 { + return chain + } + // A caller that named the provider is asking for it, not for whatever served the session + // last. Such a request is never reordered; only its key within that provider is the + // session's to keep. + if requested.Provider != "" { + return chain + } + key := SessionStateKey(ctx, SessionStateKindRoute, string(requested.Provider), requested.Model) + bound, found := SessionStateString(a.kv, key) + if !found { + return chain + } + boundRoute, ok := ParseRouteState(bound) + if !ok { + // Nothing can follow a binding that names no route, and keeping it would refuse the + // next served request's first-writer bind until it expired. + if _, err := a.kv.Delete(key); err != nil { + a.logger.Warn("error dropping unreadable session route binding: %s", err.Error()) + } + return chain + } + // A binding names a route, provider and model together, and only that route counts as the + // session's. A chain offering the provider on another model does not offer it: following the + // provider alone would move a session bound to vertex/opus-4-8 onto a vertex/opus-4-7 + // fallback entry, an older model than the primary routing proposed, and keep it there. + at := slices.IndexFunc(chain, func(route schemas.Route) bool { + return route.Provider == boundRoute.Provider && route.Model == boundRoute.Model + }) + if at < 0 { + // The chain is what this request may use, so a binding outside it is stale. Dropping it + // lets the next served request bind afresh and the session converge there, instead of + // scattering across routing's picks until the binding expires. + if _, err := a.kv.Delete(key); err != nil { + a.logger.Warn("error dropping session route binding to %s: %s", bound, err.Error()) + } + ctx.AppendRoutingEngineLog(schemas.RoutingEngineSessionAffinity, schemas.LogLevelInfo, fmt.Sprintf("Session was last served by %s for %s, which this request cannot use, so the routing decision stands and the session rebinds on its next served request", bound, requested.Model)) + // Refusing a stale binding, and dropping it, is as much a decision as following one: the + // engine is listed so a trail entry is never attributed to an engine the request does not + // record. The paths that write no entry, an unreadable binding and no binding at all, stay + // silent at both levels. + schemas.AppendToContextList(ctx, schemas.BifrostContextKeyRoutingEnginesUsed, schemas.RoutingEngineSessionAffinity) + return chain + } + a.remember(ctx, func(r *sessionResolution) { r.route = RouteStateValue(chain[at]) }) + schemas.AppendToContextList(ctx, schemas.BifrostContextKeyRoutingEnginesUsed, schemas.RoutingEngineSessionAffinity) + if at == 0 { + // The session and routing agree, so the chain is left as it is. Say so anyway: a trail that + // falls silent here cannot be told apart from one where the session was never consulted, + // and the key level below reports every reuse whether or not it changed anything. + ctx.AppendRoutingEngineLog(schemas.RoutingEngineSessionAffinity, schemas.LogLevelInfo, fmt.Sprintf("Session stays on %s for %s, which routing also proposed", bound, requested.Model)) + return chain + } + resolved := make([]schemas.Route, 0, len(chain)) + resolved = append(resolved, chain[at]) + resolved = append(resolved, chain[:at]...) + resolved = append(resolved, chain[at+1:]...) + ctx.AppendRoutingEngineLog(schemas.RoutingEngineSessionAffinity, schemas.LogLevelInfo, fmt.Sprintf("Session stays on %s for %s; routing proposed %s", bound, requested.Model, RouteStateValue(chain[0]))) + return resolved +} + +// ResolveKey implements schemas.SessionAffinity: the key that last served the session on this +// provider and model is used while it is still eligible. Only the primary provider of a +// request answers from a binding; a fallback provider serves because the primary could not, +// and what it serves with is settled by the outcome. +func (a *sessionAffinity) ResolveKey(ctx *schemas.BifrostContext, provider schemas.ModelProvider, model string, eligible []schemas.Key) (schemas.Key, bool) { + if a == nil || a.kv == nil || ctx == nil || len(eligible) < 2 { + return schemas.Key{}, false + } + if fallbackIndex, _ := ctx.Value(schemas.BifrostContextKeyFallbackIndex).(int); fallbackIndex > 0 { + return schemas.Key{}, false + } + key := SessionStateKey(ctx, SessionStateKindKey, string(provider), model) + boundID, found := SessionStateString(a.kv, key) + if !found { + return schemas.Key{}, false + } + for _, candidate := range eligible { + if candidate.ID != boundID { + continue + } + a.remember(ctx, func(r *sessionResolution) { + r.keyRoute = schemas.Route{Provider: provider, Model: model} + r.keyID = candidate.ID + }) + ctx.AppendRoutingEngineLog(schemas.RoutingEngineSessionAffinity, schemas.LogLevelInfo, fmt.Sprintf("Session reused key %s for %s/%s", candidate.Name, provider, model)) + schemas.AppendToContextList(ctx, schemas.BifrostContextKeyRoutingEnginesUsed, schemas.RoutingEngineSessionAffinity) + return candidate, true + } + if _, err := a.kv.Delete(key); err != nil { + a.logger.Warn("error deleting stale session key binding for provider=%s: %s", provider, err.Error()) + } + ctx.AppendRoutingEngineLog(schemas.RoutingEngineSessionAffinity, schemas.LogLevelInfo, fmt.Sprintf("The key this session last used for %s/%s is no longer eligible, so one is being picked", provider, model)) + return schemas.Key{}, false +} + +// Observe implements schemas.SessionAffinity: a served request binds the session to the route +// and key that served it. A binding the request followed is refreshed when it served and +// replaced when something else did; a session with no binding takes the first that lands. A +// fallback that served replaces the key binding on its provider, since it picked that key +// without the session. A binding the request followed and did not serve on is dropped, whether +// the request failed outright or a fallback rescued it: the provider or key it named just +// failed this session, and keeping it would send every request for the rest of the TTL back +// there first. A failure writes nothing else, a binding the request did not follow says +// nothing about the failure and is left alone, and a request the caller cancelled keeps +// everything, since that failure is not the provider's. +func (a *sessionAffinity) Observe(ctx *schemas.BifrostContext, requested schemas.Route, outcome schemas.RouteOutcome) { + if a == nil || a.kv == nil || ctx == nil { + return + } + ttl := sessionTTLFromContext(ctx) + resolved, _ := ctx.Value(sessionAffinityResolvedKey).(sessionResolution) + + // A caller that gave up on the request says nothing about the provider or key it followed, + // so a cancellation keeps both bindings. A deadline is the provider's failure to answer in + // time, and is treated like any other failure below. + if outcome.Served == nil && cancelledByCaller(outcome.Err) { + return + } + if resolved.keyID != "" && (outcome.Served == nil || *outcome.Served != resolved.keyRoute) { + a.forget(SessionStateKey(ctx, SessionStateKindKey, string(resolved.keyRoute.Provider), resolved.keyRoute.Model), fmt.Sprintf("key binding to %s on %s/%s", resolved.keyID, resolved.keyRoute.Provider, resolved.keyRoute.Model)) + ctx.AppendRoutingEngineLog(schemas.RoutingEngineSessionAffinity, schemas.LogLevelInfo, fmt.Sprintf("The key this session followed for %s/%s failed, so the session forgets it and a key is picked normally next time", resolved.keyRoute.Provider, resolved.keyRoute.Model)) + } + if outcome.Served == nil { + if resolved.route != "" && requested.Provider == "" { + a.forget(SessionStateKey(ctx, SessionStateKindRoute, string(requested.Provider), requested.Model), fmt.Sprintf("route binding to %s", resolved.route)) + ctx.AppendRoutingEngineLog(schemas.RoutingEngineSessionAffinity, schemas.LogLevelInfo, fmt.Sprintf("The provider this session followed for %s failed, so the session forgets it and the next request follows the routing decision", requested.Model)) + } + return + } + served := *outcome.Served + + // Only a request that left the provider to routing binds a route; one that named it has + // nothing to remember at that level, even when a fallback served. + if requested.Provider == "" { + a.settle(SessionStateKey(ctx, SessionStateKindRoute, string(requested.Provider), requested.Model), RouteStateValue(served), resolved.route, ttl) + } + if outcome.KeyID != "" { + key := SessionStateKey(ctx, SessionStateKindKey, string(served.Provider), served.Model) + followed := "" + if resolved.keyRoute == served { + followed = resolved.keyID + } + if followed == "" && outcome.Fallback { + // A fallback attempt gets no key from the session, so a key its provider already held + // for the session is older than the one that just served there, and gives way to it. + a.replace(key, outcome.KeyID, ttl) + } else { + a.settle(key, outcome.KeyID, followed, ttl) + } + } +} + +// settle writes what served under key. A request that followed no binding binds only if +// nothing has landed first, so parallel first requests of one session converge on whichever +// finished first. A request that followed a binding refreshes it when it served and moves it +// to what served instead. +func (a *sessionAffinity) settle(key, value, followed string, ttl time.Duration) { + if followed == "" { + if _, err := a.kv.SetNXWithTTL(key, value, ttl); err != nil { + a.logger.Warn("error binding session to %s: %s", value, err.Error()) + } + return + } + if followed != value { + a.logger.Debug("session moved from %s to %s", followed, value) + } + a.replace(key, value, ttl) +} + +// cancelledByCaller reports whether the request ended because the caller abandoned it, which +// core records as schemas.RequestCancelled, as opposed to a deadline (schemas.RequestTimedOut). +func cancelledByCaller(err *schemas.BifrostError) bool { + return err != nil && err.Error != nil && err.Error.Type != nil && *err.Error.Type == schemas.RequestCancelled +} + +// forget drops the binding under key, and says so if the store refuses. +func (a *sessionAffinity) forget(key, what string) { + if _, err := a.kv.Delete(key); err != nil { + a.logger.Warn("error dropping failed session %s: %s", what, err.Error()) + } +} + +// replace binds the session to value under key over whatever was there. +func (a *sessionAffinity) replace(key, value string, ttl time.Duration) { + if err := a.kv.SetWithTTL(key, value, ttl); err != nil { + a.logger.Warn("error rebinding session to %s: %s", value, err.Error()) + } +} + +// remember records part of what this request resolved, for Observe. +func (a *sessionAffinity) remember(ctx *schemas.BifrostContext, update func(*sessionResolution)) { + resolution, _ := ctx.Value(sessionAffinityResolvedKey).(sessionResolution) + update(&resolution) + ctx.SetValue(sessionAffinityResolvedKey, resolution) +} + +// RouteStateValue spells a route as a route binding stores it. Provider names carry no slash, +// so the first one separates the provider from the model. +func RouteStateValue(route schemas.Route) string { + return string(route.Provider) + "/" + route.Model +} + +// callerKeySelection describes the key the caller chose for this request's provider, or +// returns "" when the caller chose none: a direct key, a key name, or a key id other than +// the routing-rule pin core committed into the same context value. +func callerKeySelection(ctx *schemas.BifrostContext) string { + if _, ok := ctx.Value(schemas.BifrostContextKeyDirectKey).(schemas.Key); ok { + return "a direct key" + } + if name, _ := ctx.Value(schemas.BifrostContextKeyAPIKeyName).(string); strings.TrimSpace(name) != "" { + return "an explicit key name" + } + if id, _ := ctx.Value(schemas.BifrostContextKeyAPIKeyID).(string); strings.TrimSpace(id) != "" && strings.TrimSpace(id) != routingKeyPinFromContext(ctx) { + return "an explicit key id" + } + return "" +} + +// ParseRouteState reads a route back from what RouteStateValue wrote. ok is false when the +// value names no provider. +func ParseRouteState(value string) (schemas.Route, bool) { + provider, model, found := strings.Cut(value, "/") + if !found || provider == "" { + return schemas.Route{}, false + } + return schemas.Route{Provider: schemas.ModelProvider(provider), Model: model}, true +} + +// resolveSessionRoute asks the session affinity to settle the chain the routing hooks +// produced for req and applies its answer: the first route becomes the primary, the rest its +// fallbacks. A request no hook could route is left for validation to refuse. +func (bifrost *Bifrost) resolveSessionRoute(ctx *schemas.BifrostContext, requested schemas.Route, req *schemas.BifrostRequest) { + if !schemas.IsSessionAffinityActive(ctx) { + // A request that carries a session but switched affinity off is routed as if it had none. + // Say so: the session id is on the log record either way, so a trail that stayed silent + // here reads as though affinity was consulted and had nothing to add. + if sessionIDFromContext(ctx) != "" { + ctx.AppendRoutingEngineLog(schemas.RoutingEngineSessionAffinity, schemas.LogLevelInfo, fmt.Sprintf("Request carries a session but asked not to follow it, so the routing decision stands for %s", requested.Model)) + } + return + } + provider, model, fallbacks := req.GetRequestFields() + if provider == "" { + return + } + // A direct key, or an explicit key selector (x-bf-api-key / x-bf-api-key-id), names + // the caller's credential for this provider and stays on the context for the primary + // attempt (only fallbacks clear it). Moving the primary to another provider would send + // that credential, or a same-named key of another account, to the wrong upstream, so + // the route the caller chose the key for stands. A routing-rule pin is not the + // caller's: it moves with its route below. + if reason := callerKeySelection(ctx); reason != "" { + ctx.AppendRoutingEngineLog(schemas.RoutingEngineSessionAffinity, schemas.LogLevelInfo, fmt.Sprintf("Request carries %s for %s, so the session does not move it", reason, provider)) + return + } + // Every route carries the key pinned for it: the primary's is the routing pin + // RunPreRequestHooks committed, each fallback's is its own. A pin names a key of one + // provider, so it must follow that provider through the reorder: looked up among another + // provider's keys it can only fail, and the attempt would die before the provider was called. + primaryKeyPin := routingKeyPinFromContext(ctx) + chain := make([]schemas.Route, 0, 1+len(fallbacks)) + chain = append(chain, schemas.Route{Provider: provider, Model: model, KeyID: primaryKeyPin}) + for _, fallback := range fallbacks { + chain = append(chain, schemas.Route{Provider: fallback.Provider, Model: fallback.Model, KeyID: strings.TrimSpace(fallback.KeyID)}) + } + resolved := bifrost.sessionAffinity.ResolveRoute(ctx, requested, chain) + if len(resolved) == 0 || slices.Equal(resolved, chain) { + return + } + req.SetProvider(resolved[0].Provider) + req.SetModel(resolved[0].Model) + // A demoted primary keeps its pin as a fallback pin, which the fallback loop re-applies + // after clearing the previous attempt's context; a promoted fallback brings its own. + next := make([]schemas.Fallback, 0, len(resolved)-1) + for _, route := range resolved[1:] { + next = append(next, schemas.Fallback{Provider: route.Provider, Model: route.Model, KeyID: route.KeyID}) + } + req.SetFallbacks(next) + if resolved[0].KeyID == primaryKeyPin { + return + } + setRoutingPin(ctx, resolved[0].KeyID) + moved := fmt.Sprintf("Session moved the request from %s to %s", provider, resolved[0].Provider) + if resolved[0].KeyID != "" { + moved += ", which brings the key pinned for it" + } else { + moved += ", so a key is picked normally there" + } + if primaryKeyPin != "" { + moved += fmt.Sprintf("; the key pinned for %s applies if %s is tried as a fallback", provider, provider) + } + ctx.AppendRoutingEngineLog(schemas.RoutingEngineSessionAffinity, schemas.LogLevelInfo, moved) +} + +// routingKeyPinFromContext returns the key a matched routing rule pinned for the primary attempt, +// as the routing plugin recorded it and RunPreRequestHooks committed it, or "" when none did. +func routingKeyPinFromContext(ctx *schemas.BifrostContext) string { + pin, _ := ctx.Value(schemas.BifrostContextKeyRoutingPinnedAPIKeyID).(string) + return strings.TrimSpace(pin) +} + +// setRoutingPin makes keyID the pin the primary attempt reads, in the routing pin and in the +// api-key-id it is committed into, or clears both when keyID is empty so the attempt selects a +// key normally. It runs after the hooks' blocked phase, where the reserved key is writable, as +// it is when RunPreRequestHooks commits the pin. A caller's own pins are not touched: with no +// routing pin the api-key-id is the caller's, and it is left as it is. +func setRoutingPin(ctx *schemas.BifrostContext, keyID string) { + if keyID == "" { + ctx.ClearValue(schemas.BifrostContextKeyRoutingPinnedAPIKeyID) + ctx.ClearValue(schemas.BifrostContextKeyAPIKeyID) + return + } + ctx.SetValue(schemas.BifrostContextKeyRoutingPinnedAPIKeyID, keyID) + ctx.SetValue(schemas.BifrostContextKeyAPIKeyID, keyID) +} + +// observeSessionOutcome tells the session affinity how a request ended: the route that served +// it and the key that route used, or nothing served and the error it ended in. +func (bifrost *Bifrost) observeSessionOutcome(ctx *schemas.BifrostContext, requested schemas.Route, served *schemas.Route, fallback bool, err *schemas.BifrostError) { + if !schemas.IsSessionAffinityActive(ctx) { + return + } + outcome := schemas.RouteOutcome{Served: served, Fallback: fallback, Err: err} + if served != nil { + outcome.KeyID, _ = ctx.Value(schemas.BifrostContextKeySelectedKeyID).(string) + } + bifrost.sessionAffinity.Observe(ctx, requested, outcome) +} + +// SessionStateKey builds the store key for one piece of session state: its kind, the request's +// session, who that session belongs to, and whatever further parts tell instances of that kind +// apart (a provider and model for a key binding). The session and its owner come from ctx: the +// session id, and the virtual key and user the grant identity attributes the request to. A +// request nothing governs, or whose identity is not settled, is scoped to the deployment. +// Everything but the kind is hashed, so the key is bounded in size and carries no +// caller-supplied text. Any component that keeps per-session state keys it this way, so one +// session id means one thing everywhere. +func SessionStateKey(ctx *schemas.BifrostContext, kind string, parts ...string) string { + virtualKeyID, userID := sessionIdentityParts(ctx) + h := sha256.New() + for _, part := range append([]string{virtualKeyID, userID, sessionIDFromContext(ctx)}, parts...) { + _, _ = h.Write([]byte(part)) + _, _ = h.Write([]byte{0}) + } + return sessionStateKeyPrefix + kind + ":" + hex.EncodeToString(h.Sum(nil)) +} + +// sessionIdentityParts is who the request is attributed to, as far as its grant identity has +// settled it: the virtual key it was made with and the user behind it, either possibly empty. +func sessionIdentityParts(ctx *schemas.BifrostContext) (virtualKeyID, userID string) { + if ctx == nil { + return "", "" + } + grant := ctx.Grant() + if grant == nil { + return "", "" + } + identity := grant.Identity() + if identity == nil { + return "", "" + } + if key := identity.VirtualKey(); key != nil { + virtualKeyID = key.ID + } + if user := identity.User(); user != nil { + userID = user.ID + } + return virtualKeyID, userID +} + +// sessionIDFromContext returns the request's session id, or "" when it carries none. +func sessionIDFromContext(ctx *schemas.BifrostContext) string { + if ctx == nil { + return "" + } + sessionID, _ := ctx.Value(schemas.BifrostContextKeySessionID).(string) + return sessionID +} + +// sessionTTLFromContext returns how long session state written for this request stays alive: +// the request's own TTL when it set one, otherwise schemas.DefaultSessionStickyTTL. +func sessionTTLFromContext(ctx *schemas.BifrostContext) time.Duration { + if ctx != nil { + if ttl, ok := ctx.Value(schemas.BifrostContextKeySessionTTL).(time.Duration); ok && ttl > 0 { + return ttl + } + } + return schemas.DefaultSessionStickyTTL +} + +// SessionStateString reads the string under key. A missing key, a read error and an empty +// value all read as not found. Values come back as strings from a local store and as JSON +// bytes from one that replicated them, so both are accepted. +func SessionStateString(kv schemas.KVStore, key string) (string, bool) { + raw, err := kv.Get(key) + if err != nil { + return "", false + } + var value string + switch v := raw.(type) { + case string: + value = v + case []byte: + if err := sonic.Unmarshal(v, &value); err != nil { + value = string(v) + } + } + if value == "" { + return "", false + } + return value, true +} diff --git a/core/sessionaffinity_test.go b/core/sessionaffinity_test.go new file mode 100644 index 00000000000..5e16151c2a7 --- /dev/null +++ b/core/sessionaffinity_test.go @@ -0,0 +1,1253 @@ +package bifrost + +import ( + "context" + "slices" + "strings" + "sync" + "testing" + "time" + + "github.com/capsohq/bifrost/core/schemas" +) + +func sessionCtx(sessionID string) *schemas.BifrostContext { + ctx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) + if sessionID != "" { + ctx.SetValue(schemas.BifrostContextKeySessionID, sessionID) + } + return ctx +} + +// stubIdentity is the least an installing layer could settle: which key and user the request +// is attributed to. Everything else answers as unknown. +type stubIdentity struct { + virtualKeyID string + userID string +} + +func (i *stubIdentity) Credential() schemas.Credential { return schemas.Credential{} } +func (i *stubIdentity) Presented() bool { return i.virtualKeyID != "" || i.userID != "" } +func (i *stubIdentity) User() *schemas.UserRef { + if i.userID == "" { + return nil + } + return &schemas.UserRef{ID: i.userID} +} +func (i *stubIdentity) VirtualKey() *schemas.EntityRef { + if i.virtualKeyID == "" { + return nil + } + return &schemas.EntityRef{ID: i.virtualKeyID} +} +func (i *stubIdentity) Teams() []schemas.EntityRef { return nil } +func (i *stubIdentity) Customers() []schemas.EntityRef { return nil } +func (i *stubIdentity) BusinessUnits() []schemas.EntityRef { return nil } +func (i *stubIdentity) Project() *schemas.EntityRef { return nil } + +// stubGrant carries only an identity, which is all session state reads. +type stubGrant struct { + identity schemas.Identity +} + +func (g *stubGrant) Identity() schemas.Identity { return g.identity } +func (g *stubGrant) Access() schemas.Access { return nil } +func (g *stubGrant) Limits() schemas.Limits { return nil } +func (g *stubGrant) SetIdentity(i schemas.Identity) bool { g.identity = i; return i != nil } +func (g *stubGrant) SetAccess(schemas.Access) bool { return false } +func (g *stubGrant) SetLimits(schemas.Limits) bool { return false } + +// attributed installs a grant whose identity names the given key and user on ctx. +func attributed(ctx *schemas.BifrostContext, virtualKeyID, userID string) *schemas.BifrostContext { + ctx.SetGrant(&stubGrant{identity: &stubIdentity{virtualKeyID: virtualKeyID, userID: userID}}) + return ctx +} + +var sessionTestPool = []schemas.Key{ + {ID: "key-a", Name: "Key A"}, + {ID: "key-b", Name: "Key B"}, + {ID: "key-c", Name: "Key C"}, +} + +func routeOf(provider schemas.ModelProvider, model string) schemas.Route { + return schemas.Route{Provider: provider, Model: model} +} + +func servedBy(route schemas.Route, keyID string, fallback bool) schemas.RouteOutcome { + return schemas.RouteOutcome{Served: &route, KeyID: keyID, Fallback: fallback} +} + +func testAffinity(kv schemas.KVStore) *sessionAffinity { + return NewSessionAffinity(kv, NewDefaultLogger(schemas.LogLevelError)).(*sessionAffinity) +} + +func enginesUsed(ctx *schemas.BifrostContext) []string { + engines, _ := ctx.Value(schemas.BifrostContextKeyRoutingEnginesUsed).([]string) + return engines +} + +func trailMentions(ctx *schemas.BifrostContext, text string) bool { + for _, entry := range ctx.GetRoutingEngineLogs() { + if entry.Engine == schemas.RoutingEngineSessionAffinity && strings.Contains(entry.Message, text) { + return true + } + } + return false +} + +// recordingAffinity answers what a test tells it and records what core asked. +type recordingAffinity struct { + mu sync.Mutex + routeAnswer func(chain []schemas.Route) []schemas.Route + routeCalls int + keyCalls int + requested []schemas.Route + outcomes []schemas.RouteOutcome +} + +func (r *recordingAffinity) ResolveRoute(_ *schemas.BifrostContext, _ schemas.Route, chain []schemas.Route) []schemas.Route { + r.mu.Lock() + defer r.mu.Unlock() + r.routeCalls++ + if r.routeAnswer != nil { + return r.routeAnswer(chain) + } + return chain +} + +func (r *recordingAffinity) ResolveKey(*schemas.BifrostContext, schemas.ModelProvider, string, []schemas.Key) (schemas.Key, bool) { + r.mu.Lock() + defer r.mu.Unlock() + r.keyCalls++ + return schemas.Key{}, false +} + +func (r *recordingAffinity) Observe(_ *schemas.BifrostContext, requested schemas.Route, outcome schemas.RouteOutcome) { + r.mu.Lock() + defer r.mu.Unlock() + r.requested = append(r.requested, requested) + r.outcomes = append(r.outcomes, outcome) +} + +func TestSessionStateKeyScopesBySessionAndIdentity(t *testing.T) { + base := SessionStateKey(attributed(sessionCtx("session-1"), "vk-1", ""), SessionStateKindKey, "openai", "gpt-4o") + if !strings.HasPrefix(base, "session:v2:key:") { + t.Fatalf("key %q lacks the kind prefix", base) + } + if strings.Contains(base, "session-1") || strings.Contains(base, "vk-1") || strings.Contains(base, "gpt-4o") { + t.Fatalf("key %q leaks a hashed part", base) + } + if base != SessionStateKey(attributed(sessionCtx("session-1"), "vk-1", ""), SessionStateKindKey, "openai", "gpt-4o") { + t.Fatal("key is not deterministic") + } + + variants := map[string]string{ + "other virtual key": SessionStateKey(attributed(sessionCtx("session-1"), "vk-2", ""), SessionStateKindKey, "openai", "gpt-4o"), + "same id as a user": SessionStateKey(attributed(sessionCtx("session-1"), "", "vk-1"), SessionStateKindKey, "openai", "gpt-4o"), + "key and user": SessionStateKey(attributed(sessionCtx("session-1"), "vk-1", "u-1"), SessionStateKindKey, "openai", "gpt-4o"), + "other session": SessionStateKey(attributed(sessionCtx("session-2"), "vk-1", ""), SessionStateKindKey, "openai", "gpt-4o"), + "other provider": SessionStateKey(attributed(sessionCtx("session-1"), "vk-1", ""), SessionStateKindKey, "azure", "gpt-4o"), + "other model": SessionStateKey(attributed(sessionCtx("session-1"), "vk-1", ""), SessionStateKindKey, "openai", "gpt-4o-mini"), + "route kind": SessionStateKey(attributed(sessionCtx("session-1"), "vk-1", ""), SessionStateKindRoute, "openai", "gpt-4o"), + } + for name, v := range variants { + if v == base { + t.Fatalf("%s collides with the base key", name) + } + } + + // A request nothing governs, one whose grant has no identity, and one whose identity names + // nothing all scope to the deployment and share a key. + deployment := SessionStateKey(sessionCtx("session-1"), SessionStateKindKey, "openai", "gpt-4o") + unsettled := sessionCtx("session-1") + unsettled.SetGrant(&stubGrant{}) + if got := SessionStateKey(unsettled, SessionStateKindKey, "openai", "gpt-4o"); got != deployment { + t.Fatal("a grant without identity should scope to the deployment") + } + if got := SessionStateKey(attributed(sessionCtx("session-1"), "", ""), SessionStateKindKey, "openai", "gpt-4o"); got != deployment { + t.Fatal("an identity naming no key and no user should scope to the deployment") + } + if deployment == base { + t.Fatal("deployment scope collides with a virtual key scope") + } + if SessionStateKey(nil, SessionStateKindKey, "openai", "gpt-4o") == "" { + t.Fatal("nil context should still produce a key") + } +} + +func TestSessionTTLFromContext(t *testing.T) { + if got := sessionTTLFromContext(nil); got != schemas.DefaultSessionStickyTTL { + t.Fatalf("nil context TTL = %v, want the default", got) + } + if got := sessionTTLFromContext(sessionCtx("s")); got != schemas.DefaultSessionStickyTTL { + t.Fatalf("unset TTL = %v, want the default", got) + } + ctx := sessionCtx("s") + ctx.SetValue(schemas.BifrostContextKeySessionTTL, 3*time.Minute) + if got := sessionTTLFromContext(ctx); got != 3*time.Minute { + t.Fatalf("set TTL = %v, want 3m", got) + } + ctx.SetValue(schemas.BifrostContextKeySessionTTL, time.Duration(0)) + if got := sessionTTLFromContext(ctx); got != schemas.DefaultSessionStickyTTL { + t.Fatalf("zero TTL = %v, want the default", got) + } +} + +func TestIsSessionAffinityActive(t *testing.T) { + if schemas.IsSessionAffinityActive(nil) { + t.Fatal("nil context takes part") + } + if schemas.IsSessionAffinityActive(sessionCtx("")) { + t.Fatal("a request without a session takes part") + } + if !schemas.IsSessionAffinityActive(sessionCtx("s")) { + t.Fatal("a request with a session and an unset switch does not take part") + } + ctx := sessionCtx("s") + ctx.SetValue(schemas.BifrostContextKeySessionAffinity, false) + if schemas.IsSessionAffinityActive(ctx) { + t.Fatal("a request that switched affinity off takes part") + } + ctx.SetValue(schemas.BifrostContextKeySessionAffinity, true) + if !schemas.IsSessionAffinityActive(ctx) { + t.Fatal("a request that switched affinity on does not take part") + } +} + +func TestSessionAffinityResolveKeyReadsReplicatedBindings(t *testing.T) { + cases := []struct { + name string + value any + want string + }{ + {"plain string", "key-b", "key-b"}, + {"json bytes", []byte(`"key-c"`), "key-c"}, + {"raw bytes", []byte("key-b"), "key-b"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + kv := newMockKVStore() + ctx := sessionCtx("session-1") + _ = kv.SetWithTTL(SessionStateKey(ctx, SessionStateKindKey, "openai", "gpt-4o"), tc.value, time.Minute) + key, ok := testAffinity(kv).ResolveKey(ctx, schemas.OpenAI, "gpt-4o", sessionTestPool) + if !ok || key.ID != tc.want { + t.Fatalf("got %q ok=%v, want the stored binding %q", key.ID, ok, tc.want) + } + }) + } + + // An empty stored value is no binding. + kv := newMockKVStore() + ctx := sessionCtx("session-1") + _ = kv.SetWithTTL(SessionStateKey(ctx, SessionStateKindKey, "openai", "gpt-4o"), "", time.Minute) + if _, ok := testAffinity(kv).ResolveKey(ctx, schemas.OpenAI, "gpt-4o", sessionTestPool); ok { + t.Fatal("empty binding produced a key") + } +} + +func TestSessionAffinityBindsOnOutcomeThenReuses(t *testing.T) { + kv := newMockKVStore() + a := testAffinity(kv) + requested := routeOf("", "gpt-4o") + served := routeOf(schemas.OpenAI, "gpt-4o") + + // Nothing bound yet: the pool builder gets no answer and nothing is written. + ctx := sessionCtx("session-1") + ctx.SetValue(schemas.BifrostContextKeySessionTTL, 5*time.Minute) + if _, ok := a.ResolveKey(ctx, schemas.OpenAI, "gpt-4o", sessionTestPool); ok { + t.Fatal("an unbound session got a key before anything served it") + } + if len(kv.data) != 0 { + t.Fatalf("resolving wrote state: %v", kv.data) + } + + // Served by key-b: the route and the key are bound with the request's TTL. + a.Observe(ctx, requested, servedBy(served, "key-b", false)) + routeKey := SessionStateKey(ctx, SessionStateKindRoute, "", "gpt-4o") + keyKey := SessionStateKey(ctx, SessionStateKindKey, "openai", "gpt-4o") + if entry := kv.data[routeKey]; entry.value != "openai/gpt-4o" || entry.ttl != 5*time.Minute { + t.Fatalf("route binding after first request: %+v", entry) + } + if entry := kv.data[keyKey]; entry.value != "key-b" || entry.ttl != 5*time.Minute { + t.Fatalf("key binding after first request: %+v", entry) + } + + // The next request reuses key-b and the trail says so. + next := sessionCtx("session-1") + next.SetValue(schemas.BifrostContextKeySessionTTL, 7*time.Minute) + key, ok := a.ResolveKey(next, schemas.OpenAI, "gpt-4o", sessionTestPool) + if !ok || key.ID != "key-b" { + t.Fatalf("second request: got %q ok=%v, want key-b", key.ID, ok) + } + if !slices.Contains(enginesUsed(next), schemas.RoutingEngineSessionAffinity) || !trailMentions(next, "reused key Key B") { + t.Fatalf("second request did not record the reuse: engines=%v", enginesUsed(next)) + } + + // Served again by the key it followed: the binding is refreshed, not replaced. + a.Observe(next, requested, servedBy(served, "key-b", false)) + if entry := kv.data[keyKey]; entry.value != "key-b" || entry.ttl != 7*time.Minute { + t.Fatalf("reuse did not refresh the key binding: %+v", entry) + } + + // A fallback provider that served binds its own key, and does not take the route from a + // request that followed no route binding. + a.Observe(next, requested, servedBy(routeOf(schemas.Azure, "gpt-4o"), "az-1", true)) + if entry := kv.data[SessionStateKey(next, SessionStateKindKey, "azure", "gpt-4o")]; entry.value != "az-1" { + t.Fatalf("fallback key binding: %+v", entry) + } + if entry := kv.data[routeKey]; entry.value != "openai/gpt-4o" { + t.Fatalf("route binding overwritten by a request that followed none: %+v", entry) + } +} + +// A fallback picks its key without consulting the session, so a key binding its provider already +// held for the session is older than the key that just served there, and gives way to it. +func TestSessionAffinityFallbackReplacesAnOlderKeyBindingOnItsProvider(t *testing.T) { + kv := newMockKVStore() + a := testAffinity(kv) + requested := routeOf("", "gpt-4o") + openai, azure := routeOf(schemas.OpenAI, "gpt-4o"), routeOf(schemas.Azure, "gpt-4o") + + // Earlier turns: openai served with key-a, then the session moved to azure on key-b. + seed := sessionCtx("session-1") + openaiKey := SessionStateKey(seed, SessionStateKindKey, "openai", "gpt-4o") + azureKey := SessionStateKey(seed, SessionStateKindKey, "azure", "gpt-4o") + routeKey := SessionStateKey(seed, SessionStateKindRoute, "", "gpt-4o") + _ = kv.SetWithTTL(openaiKey, "key-a", time.Hour) + _ = kv.SetWithTTL(azureKey, "key-b", time.Hour) + _ = kv.SetWithTTL(routeKey, "azure/gpt-4o", time.Hour) + + // This turn follows azure and its key, azure fails on that key, and the openai fallback serves + // on key-c, picked freely because a fallback attempt gets no key from the session. + ctx := sessionCtx("session-1") + if got := a.ResolveRoute(ctx, requested, []schemas.Route{openai, azure}); got[0] != azure { + t.Fatalf("session did not follow azure: %v", got) + } + if key, ok := a.ResolveKey(ctx, schemas.Azure, "gpt-4o", sessionTestPool); !ok || key.ID != "key-b" { + t.Fatalf("primary attempt: got %q ok=%v, want key-b", key.ID, ok) + } + ctx.SetValue(schemas.BifrostContextKeyFallbackIndex, 1) + if _, ok := a.ResolveKey(ctx, schemas.OpenAI, "gpt-4o", sessionTestPool); ok { + t.Fatal("fallback attempt got a key from the session") + } + a.Observe(ctx, requested, servedBy(openai, "key-c", true)) + + if entry := kv.data[routeKey]; entry.value != "openai/gpt-4o" { + t.Fatalf("route binding after the fallback served: %+v, want openai/gpt-4o", entry) + } + if entry := kv.data[openaiKey]; entry.value != "key-c" { + t.Fatalf("openai key binding after its fallback served on key-c: %+v, want key-c", entry) + } + // Azure failed on the key the session followed, so that binding is dropped rather than kept + // for the rest of its TTL: the next time azure serves this session, a key is picked afresh. + if entry, bound := kv.data[azureKey]; bound { + t.Fatalf("azure key binding survived azure failing on it: %+v", entry) + } +} + +func TestSessionAffinityRebindsWhenBoundKeyLeavesThePool(t *testing.T) { + kv := newMockKVStore() + ctx := sessionCtx("session-1") + stateKey := SessionStateKey(ctx, SessionStateKindKey, "openai", "gpt-4o") + _ = kv.SetWithTTL(stateKey, "key-gone", time.Minute) + + if _, ok := testAffinity(kv).ResolveKey(ctx, schemas.OpenAI, "gpt-4o", sessionTestPool); ok { + t.Fatal("a binding to a key outside the pool produced a key") + } + if _, present := kv.data[stateKey]; present { + t.Fatal("stale binding was not deleted") + } + if !trailMentions(ctx, "no longer eligible") { + t.Fatal("stale binding was not explained in the trail") + } +} + +func TestSessionAffinityResolveRoute(t *testing.T) { + chain := []schemas.Route{routeOf(schemas.Groq, "openai/gpt-4o"), routeOf(schemas.OpenAI, "gpt-4o"), routeOf(schemas.Azure, "gpt-4o")} + requested := routeOf("", "gpt-4o") + bind := func(kv *mockKVStore, ctx *schemas.BifrostContext, value string) { + _ = kv.SetWithTTL(SessionStateKey(ctx, SessionStateKindRoute, "", "gpt-4o"), value, time.Minute) + } + + t.Run("no binding leaves the chain", func(t *testing.T) { + kv := newMockKVStore() + ctx := sessionCtx("session-1") + if got := testAffinity(kv).ResolveRoute(ctx, requested, chain); !slices.Equal(got, chain) { + t.Fatalf("got %v, want the chain unchanged", got) + } + if _, recorded := ctx.Value(sessionAffinityResolvedKey).(sessionResolution); recorded { + t.Fatal("nothing was followed, nothing should be recorded") + } + }) + + t.Run("bound provider moves to the front and the rest keep their order", func(t *testing.T) { + kv := newMockKVStore() + ctx := sessionCtx("session-1") + bind(kv, ctx, "azure/gpt-4o") + got := testAffinity(kv).ResolveRoute(ctx, requested, chain) + want := []schemas.Route{chain[2], chain[0], chain[1]} + if !slices.Equal(got, want) { + t.Fatalf("got %v, want %v", got, want) + } + if !slices.Contains(enginesUsed(ctx), schemas.RoutingEngineSessionAffinity) || !trailMentions(ctx, "Session stays on azure") { + t.Fatalf("decision not recorded: engines=%v", enginesUsed(ctx)) + } + if res, _ := ctx.Value(sessionAffinityResolvedKey).(sessionResolution); res.route != "azure/gpt-4o" { + t.Fatalf("followed route not recorded: %+v", res) + } + }) + + // A binding names a route, provider and model together. A chain that offers the provider on + // another model does not offer that route: a session served on azure/gpt-4o-old must not be + // moved onto azure/gpt-4o just because both are azure, the routing decision stands instead. + t.Run("a bound route whose model the chain does not offer is dropped, provider or not", func(t *testing.T) { + kv := newMockKVStore() + a := testAffinity(kv) + ctx := sessionCtx("session-1") + bind(kv, ctx, "azure/gpt-4o-old") + if got := a.ResolveRoute(ctx, requested, chain); !slices.Equal(got, chain) { + t.Fatalf("got %v, want the chain unchanged", got) + } + if _, bound := kv.data[SessionStateKey(ctx, SessionStateKindRoute, "", "gpt-4o")]; bound { + t.Fatal("a binding to a route the chain does not offer was kept") + } + if !trailMentions(ctx, "cannot use") { + t.Fatal("the dropped binding was not explained in the trail") + } + if _, recorded := ctx.Value(sessionAffinityResolvedKey).(sessionResolution); recorded { + t.Fatal("a binding outside the chain was recorded as followed") + } + }) + + t.Run("bound provider already first is left alone but still reported", func(t *testing.T) { + kv := newMockKVStore() + ctx := sessionCtx("session-1") + bind(kv, ctx, "groq/openai/gpt-4o") + if got := testAffinity(kv).ResolveRoute(ctx, requested, chain); !slices.Equal(got, chain) { + t.Fatalf("got %v, want the chain unchanged", got) + } + // Agreeing with routing is a decision the session made, and a trail that omitted it could + // not be told apart from one where the session was never consulted. + if !slices.Contains(enginesUsed(ctx), schemas.RoutingEngineSessionAffinity) || !trailMentions(ctx, "which routing also proposed") { + t.Fatalf("agreement not recorded: engines=%v", enginesUsed(ctx)) + } + if res, _ := ctx.Value(sessionAffinityResolvedKey).(sessionResolution); res.route != "groq/openai/gpt-4o" { + t.Fatalf("followed route not recorded: %+v", res) + } + }) + + t.Run("bound provider the request cannot use leaves the chain and is dropped", func(t *testing.T) { + kv := newMockKVStore() + a := testAffinity(kv) + ctx := sessionCtx("session-1") + bind(kv, ctx, "anthropic/claude-sonnet") + if got := a.ResolveRoute(ctx, requested, chain); !slices.Equal(got, chain) { + t.Fatalf("got %v, want the chain unchanged", got) + } + if !trailMentions(ctx, "cannot use") { + t.Fatal("stale binding was not explained in the trail") + } + // Refusing a stale binding is a decision, so it is listed among the engines used: a trail + // entry attributed to an engine the request does not record cannot be filtered for. + if !slices.Contains(enginesUsed(ctx), schemas.RoutingEngineSessionAffinity) { + t.Fatalf("refusing a stale binding was not counted as a session decision: engines=%v", enginesUsed(ctx)) + } + if _, recorded := ctx.Value(sessionAffinityResolvedKey).(sessionResolution); recorded { + t.Fatal("a binding outside the chain was recorded as followed") + } + routeKey := SessionStateKey(ctx, SessionStateKindRoute, "", "gpt-4o") + if _, present := kv.data[routeKey]; present { + t.Fatal("a binding outside the chain was kept") + } + // The request that then serves binds the session afresh, so it converges there. + a.Observe(ctx, requested, servedBy(routeOf(schemas.OpenAI, "gpt-4o"), "oa-1", false)) + if entry := kv.data[routeKey]; entry.value != "openai/gpt-4o" { + t.Fatalf("session did not rebind to what served: %+v", entry) + } + }) + + t.Run("a binding that cannot be read is dropped so the session can rebind", func(t *testing.T) { + for _, unreadable := range []struct{ name, value string }{ + {"no separator between provider and model", "azure"}, + {"no provider", "/gpt-4o"}, + } { + t.Run(unreadable.name, func(t *testing.T) { + kv := newMockKVStore() + a := testAffinity(kv) + ctx := sessionCtx("session-1") + bind(kv, ctx, unreadable.value) + if got := a.ResolveRoute(ctx, requested, chain); !slices.Equal(got, chain) { + t.Fatalf("got %v, want the chain unchanged", got) + } + if _, recorded := ctx.Value(sessionAffinityResolvedKey).(sessionResolution); recorded { + t.Fatal("a binding that names no route was recorded as followed") + } + // Nothing was decided and nothing is said, at either level: a binding that names no + // route is dropped as corrupt, not refused on the request's behalf. + if len(ctx.GetRoutingEngineLogs()) != 0 || slices.Contains(enginesUsed(ctx), schemas.RoutingEngineSessionAffinity) { + t.Fatalf("an unreadable binding was reported as a decision: engines=%v trail=%v", enginesUsed(ctx), ctx.GetRoutingEngineLogs()) + } + routeKey := SessionStateKey(ctx, SessionStateKindRoute, "", "gpt-4o") + if _, present := kv.data[routeKey]; present { + t.Fatal("a binding that names no route was kept") + } + // Kept, it would refuse the first-writer bind below and strand the session on + // routing's pick until the binding expired. + a.Observe(ctx, requested, servedBy(routeOf(schemas.OpenAI, "gpt-4o"), "oa-1", false)) + if entry := kv.data[routeKey]; entry.value != "openai/gpt-4o" { + t.Fatalf("session did not rebind to what served: %+v", entry) + } + }) + } + }) + + t.Run("empty chain, nil policy and no store leave the chain", func(t *testing.T) { + kv := newMockKVStore() + bind(kv, sessionCtx("session-1"), "azure/gpt-4o") + if got := testAffinity(kv).ResolveRoute(sessionCtx("session-1"), requested, nil); got != nil { + t.Fatalf("empty chain: got %v", got) + } + var none *sessionAffinity + if got := none.ResolveRoute(sessionCtx("session-1"), requested, chain); !slices.Equal(got, chain) { + t.Fatalf("nil policy: got %v", got) + } + if got := testAffinity(nil).ResolveRoute(sessionCtx("session-1"), requested, chain); !slices.Equal(got, chain) { + t.Fatalf("no store: got %v", got) + } + }) +} + +func TestSessionAffinityObserveRoute(t *testing.T) { + chain := []schemas.Route{routeOf(schemas.OpenAI, "gpt-4o"), routeOf(schemas.Azure, "gpt-4o")} + requested := routeOf("", "gpt-4o") + routeKeyOf := func(ctx *schemas.BifrostContext) string { + return SessionStateKey(ctx, SessionStateKindRoute, "", "gpt-4o") + } + + t.Run("a followed route that served is refreshed", func(t *testing.T) { + kv := newMockKVStore() + a := testAffinity(kv) + ctx := sessionCtx("session-1") + _ = kv.SetWithTTL(routeKeyOf(ctx), "azure/gpt-4o", time.Minute) + ctx.SetValue(schemas.BifrostContextKeySessionTTL, 9*time.Minute) + a.ResolveRoute(ctx, requested, chain) + a.Observe(ctx, requested, servedBy(routeOf(schemas.Azure, "gpt-4o"), "az-1", false)) + if entry := kv.data[routeKeyOf(ctx)]; entry.value != "azure/gpt-4o" || entry.ttl != 9*time.Minute { + t.Fatalf("route binding after reuse: %+v", entry) + } + }) + + t.Run("a fallback that served moves the route", func(t *testing.T) { + kv := newMockKVStore() + a := testAffinity(kv) + ctx := sessionCtx("session-1") + _ = kv.SetWithTTL(routeKeyOf(ctx), "azure/gpt-4o", time.Minute) + a.ResolveRoute(ctx, requested, chain) + a.Observe(ctx, requested, servedBy(routeOf(schemas.OpenAI, "gpt-4o"), "oa-1", true)) + if entry := kv.data[routeKeyOf(ctx)]; entry.value != "openai/gpt-4o" { + t.Fatalf("route binding after a fallback served: %+v", entry) + } + }) + + t.Run("a request that followed no route binding does not overwrite one", func(t *testing.T) { + kv := newMockKVStore() + a := testAffinity(kv) + ctx := sessionCtx("session-1") + _ = kv.SetWithTTL(routeKeyOf(ctx), "azure/gpt-4o", time.Minute) + a.Observe(ctx, requested, servedBy(routeOf(schemas.OpenAI, "gpt-4o"), "oa-1", false)) + if entry := kv.data[routeKeyOf(ctx)]; entry.value != "azure/gpt-4o" { + t.Fatalf("route binding overwritten: %+v", entry) + } + }) + + t.Run("a failure that followed nothing writes nothing", func(t *testing.T) { + kv := newMockKVStore() + ctx := sessionCtx("session-1") + testAffinity(kv).Observe(ctx, requested, schemas.RouteOutcome{Err: &schemas.BifrostError{}}) + if len(kv.data) != 0 { + t.Fatalf("a failure wrote state: %v", kv.data) + } + }) + + t.Run("a served route without a key binds only the route", func(t *testing.T) { + kv := newMockKVStore() + ctx := sessionCtx("session-1") + testAffinity(kv).Observe(ctx, requested, servedBy(routeOf(schemas.OpenAI, "gpt-4o"), "", false)) + if len(kv.data) != 1 { + t.Fatalf("a served route without a key should bind only the route: %v", kv.data) + } + }) +} + +// A binding the request followed into a failure is dropped, at both levels, so the next request +// is routed and keyed afresh instead of returning to what just failed for up to the TTL. A binding +// the request did not follow says nothing about the failure and is left alone. +func TestSessionAffinityFailureDropsWhatTheRequestFollowed(t *testing.T) { + requested := routeOf("", "gpt-4o") + chain := []schemas.Route{routeOf(schemas.OpenAI, "gpt-4o"), routeOf(schemas.Azure, "gpt-4o")} + failed := schemas.RouteOutcome{Err: &schemas.BifrostError{}} + keys := func(ctx *schemas.BifrostContext) (route, openaiKey, azureKey string) { + return SessionStateKey(ctx, SessionStateKindRoute, "", "gpt-4o"), + SessionStateKey(ctx, SessionStateKindKey, "openai", "gpt-4o"), + SessionStateKey(ctx, SessionStateKindKey, "azure", "gpt-4o") + } + + t.Run("a failed request drops the route and the key it followed", func(t *testing.T) { + kv := newMockKVStore() + a := testAffinity(kv) + ctx := sessionCtx("session-1") + route, openaiKey, azureKey := keys(ctx) + _ = kv.SetWithTTL(route, "openai/gpt-4o", time.Minute) + _ = kv.SetWithTTL(openaiKey, "key-b", time.Minute) + _ = kv.SetWithTTL(azureKey, "az-1", time.Minute) + a.ResolveRoute(ctx, requested, chain) + if key, ok := a.ResolveKey(ctx, schemas.OpenAI, "gpt-4o", sessionTestPool); !ok || key.ID != "key-b" { + t.Fatalf("setup: the session did not follow key-b: %q ok=%v", key.ID, ok) + } + a.Observe(ctx, requested, failed) + if _, bound := kv.data[route]; bound { + t.Fatalf("the route binding survived the failure of the provider it named: %+v", kv.data[route]) + } + if _, bound := kv.data[openaiKey]; bound { + t.Fatalf("the key binding survived the failure of the key it named: %+v", kv.data[openaiKey]) + } + if entry := kv.data[azureKey]; entry.value != "az-1" { + t.Fatalf("a key binding the request never followed was touched: %+v", entry) + } + if !trailMentions(ctx, "failed") { + t.Fatalf("dropping the bindings left no trace in the trail: %v", ctx.GetRoutingEngineLogs()) + } + }) + + // A caller that gives up on a request says nothing about the provider or key it followed, so + // a cancellation keeps both bindings. A deadline is the provider's failure to answer and drops. + t.Run("a request the caller cancelled keeps what it followed, a timeout does not", func(t *testing.T) { + for _, tc := range []struct { + errType string + kept bool + }{{schemas.RequestCancelled, true}, {schemas.RequestTimedOut, false}} { + kv := newMockKVStore() + a := testAffinity(kv) + ctx := sessionCtx("session-1") + route, openaiKey, _ := keys(ctx) + _ = kv.SetWithTTL(route, "openai/gpt-4o", time.Minute) + _ = kv.SetWithTTL(openaiKey, "key-b", time.Minute) + a.ResolveRoute(ctx, requested, chain) + a.ResolveKey(ctx, schemas.OpenAI, "gpt-4o", sessionTestPool) + errType := tc.errType + a.Observe(ctx, requested, schemas.RouteOutcome{Err: &schemas.BifrostError{Error: &schemas.ErrorField{Type: &errType}}}) + _, routeBound := kv.data[route] + _, keyBound := kv.data[openaiKey] + if routeBound != tc.kept || keyBound != tc.kept { + t.Fatalf("%s: route bound=%v key bound=%v, want both %v", tc.errType, routeBound, keyBound, tc.kept) + } + } + }) + + t.Run("a failed request that followed nothing leaves bindings alone", func(t *testing.T) { + kv := newMockKVStore() + a := testAffinity(kv) + ctx := sessionCtx("session-1") + route, openaiKey, _ := keys(ctx) + _ = kv.SetWithTTL(route, "openai/gpt-4o", time.Minute) + _ = kv.SetWithTTL(openaiKey, "key-b", time.Minute) + // The caller named its provider, so the route binding was never consulted, and the key + // pool had one key, so neither was the key binding. + named := routeOf(schemas.OpenAI, "gpt-4o") + a.ResolveRoute(ctx, named, chain[:1]) + a.ResolveKey(ctx, schemas.OpenAI, "gpt-4o", sessionTestPool[:1]) + a.Observe(ctx, named, failed) + if entry := kv.data[route]; entry.value != "openai/gpt-4o" { + t.Fatalf("route binding touched by a request that never followed it: %+v", entry) + } + if entry := kv.data[openaiKey]; entry.value != "key-b" { + t.Fatalf("key binding touched by a request that never followed it: %+v", entry) + } + }) + + t.Run("a fallback that served drops the key the primary followed and binds its own", func(t *testing.T) { + kv := newMockKVStore() + a := testAffinity(kv) + ctx := sessionCtx("session-1") + route, openaiKey, azureKey := keys(ctx) + _ = kv.SetWithTTL(route, "openai/gpt-4o", time.Minute) + _ = kv.SetWithTTL(openaiKey, "key-b", time.Minute) + a.ResolveRoute(ctx, requested, chain) + a.ResolveKey(ctx, schemas.OpenAI, "gpt-4o", sessionTestPool) + a.Observe(ctx, requested, servedBy(routeOf(schemas.Azure, "gpt-4o"), "az-1", true)) + if _, bound := kv.data[openaiKey]; bound { + t.Fatalf("the key the primary followed into its failure survived: %+v", kv.data[openaiKey]) + } + if entry := kv.data[azureKey]; entry.value != "az-1" { + t.Fatalf("the fallback's key was not bound: %+v", entry) + } + if entry := kv.data[route]; entry.value != "azure/gpt-4o" { + t.Fatalf("the route did not move to the fallback that served: %+v", entry) + } + }) + + t.Run("a failed request that followed only the route keeps a key binding it never used", func(t *testing.T) { + kv := newMockKVStore() + a := testAffinity(kv) + ctx := sessionCtx("session-1") + route, openaiKey, _ := keys(ctx) + _ = kv.SetWithTTL(route, "openai/gpt-4o", time.Minute) + _ = kv.SetWithTTL(openaiKey, "key-b", time.Minute) + // The route was followed; the key was decided by a pin, so ResolveKey was never asked. + a.ResolveRoute(ctx, requested, chain) + a.Observe(ctx, requested, failed) + if _, bound := kv.data[route]; bound { + t.Fatalf("the route binding survived the failure: %+v", kv.data[route]) + } + if entry := kv.data[openaiKey]; entry.value != "key-b" { + t.Fatalf("a key binding a pinned request never consulted was touched: %+v", entry) + } + }) +} + +func TestSessionAffinityResolveKeyDeclines(t *testing.T) { + kv := newMockKVStore() + a := testAffinity(kv) + bound := sessionCtx("session-1") + _ = kv.SetWithTTL(SessionStateKey(bound, SessionStateKindKey, "openai", "gpt-4o"), "key-a", time.Minute) + + if _, ok := a.ResolveKey(nil, schemas.OpenAI, "gpt-4o", sessionTestPool); ok { + t.Fatal("nil context got a key") + } + if _, ok := a.ResolveKey(sessionCtx("session-1"), schemas.OpenAI, "gpt-4o", sessionTestPool[:1]); ok { + t.Fatal("single-key pool got a key") + } + fallback := sessionCtx("session-1") + fallback.SetValue(schemas.BifrostContextKeyFallbackIndex, 1) + if _, ok := a.ResolveKey(fallback, schemas.OpenAI, "gpt-4o", sessionTestPool); ok { + t.Fatal("fallback attempt got a key") + } + if key, ok := a.ResolveKey(sessionCtx("session-1"), schemas.OpenAI, "gpt-4o", sessionTestPool); !ok || key.ID != "key-a" { + t.Fatalf("bound session: got %q ok=%v, want key-a", key.ID, ok) + } + var none *sessionAffinity + if _, ok := none.ResolveKey(sessionCtx("session-1"), schemas.OpenAI, "gpt-4o", sessionTestPool); ok { + t.Fatal("nil policy got a key") + } + if _, ok := testAffinity(nil).ResolveKey(sessionCtx("session-1"), schemas.OpenAI, "gpt-4o", sessionTestPool); ok { + t.Fatal("policy without a store got a key") + } +} + +func TestSessionAffinityConcurrentFirstRequestsConverge(t *testing.T) { + kv := newMockKVStore() + a := testAffinity(kv) + requested := routeOf("", "gpt-4o") + served := routeOf(schemas.OpenAI, "gpt-4o") + + // Eight first requests of one session resolve before any of them is served: none finds a + // binding, and each is then served by whatever key selection gave it. + contexts := make([]*schemas.BifrostContext, 8) + for i := range contexts { + contexts[i] = sessionCtx("session-1") + if _, ok := a.ResolveKey(contexts[i], schemas.OpenAI, "gpt-4o", sessionTestPool); ok { + t.Fatal("a first request found a binding") + } + } + var wg sync.WaitGroup + for i, ctx := range contexts { + wg.Add(1) + go func(i int, ctx *schemas.BifrostContext) { + defer wg.Done() + a.Observe(ctx, requested, servedBy(served, sessionTestPool[i%len(sessionTestPool)].ID, false)) + }(i, ctx) + } + wg.Wait() + + bound, _ := kv.data[SessionStateKey(sessionCtx("session-1"), SessionStateKindKey, "openai", "gpt-4o")].value.(string) + if bound == "" { + t.Fatal("no key binding after eight served requests") + } + key, ok := a.ResolveKey(sessionCtx("session-1"), schemas.OpenAI, "gpt-4o", sessionTestPool) + if !ok || key.ID != bound { + t.Fatalf("session resolves to %q ok=%v, want the first bound key %q", key.ID, ok, bound) + } +} + +func TestSessionAffinityScopesSessionsByCaller(t *testing.T) { + kv := newMockKVStore() + a := testAffinity(kv) + requested := routeOf("", "gpt-4o") + served := routeOf(schemas.OpenAI, "gpt-4o") + a.Observe(attributed(sessionCtx("shared-session"), "vk-a", ""), requested, servedBy(served, "key-a", false)) + a.Observe(attributed(sessionCtx("shared-session"), "vk-b", ""), requested, servedBy(served, "key-b", false)) + if len(kv.data) != 4 { + t.Fatalf("two callers sharing a session id should hold two route and two key bindings, got %d", len(kv.data)) + } + if key, ok := a.ResolveKey(attributed(sessionCtx("shared-session"), "vk-b", ""), schemas.OpenAI, "gpt-4o", sessionTestPool); !ok || key.ID != "key-b" { + t.Fatalf("caller b resolves to %q ok=%v, want its own key-b", key.ID, ok) + } +} + +func TestResolveSessionRouteAppliesTheAnswer(t *testing.T) { + reverse := func(chain []schemas.Route) []schemas.Route { + out := slices.Clone(chain) + slices.Reverse(out) + return out + } + newRequest := func(provider schemas.ModelProvider) *schemas.BifrostRequest { + req := &schemas.BifrostRequest{ + RequestType: schemas.ChatCompletionRequest, + ChatRequest: &schemas.BifrostChatRequest{Provider: provider, Model: "gpt-4o"}, + } + req.SetFallbacks([]schemas.Fallback{{Provider: schemas.Azure, Model: "gpt-4o"}, {Provider: schemas.Groq, Model: "openai/gpt-4o"}}) + return req + } + setup := func(t *testing.T, answer func([]schemas.Route) []schemas.Route) (*Bifrost, *recordingAffinity) { + t.Helper() + fake := &recordingAffinity{routeAnswer: answer} + client, err := Init(context.Background(), schemas.BifrostConfig{ + Account: NewMockAccount(), + Logger: NewDefaultLogger(schemas.LogLevelError), + SessionAffinity: fake, + }) + if err != nil { + t.Fatalf("Init: %v", err) + } + t.Cleanup(client.Shutdown) + return client, fake + } + + t.Run("the answer becomes primary and fallbacks", func(t *testing.T) { + client, fake := setup(t, reverse) + req := newRequest(schemas.OpenAI) + client.resolveSessionRoute(sessionCtx("s"), routeOf("", "gpt-4o"), req) + provider, model, fallbacks := req.GetRequestFields() + if provider != schemas.Groq || model != "openai/gpt-4o" { + t.Fatalf("primary = %s/%s, want groq/openai/gpt-4o", provider, model) + } + want := []schemas.Fallback{{Provider: schemas.Azure, Model: "gpt-4o"}, {Provider: schemas.OpenAI, Model: "gpt-4o"}} + if !slices.Equal(fallbacks, want) { + t.Fatalf("fallbacks = %v, want %v", fallbacks, want) + } + if fake.routeCalls != 1 { + t.Fatalf("ResolveRoute called %d times, want 1", fake.routeCalls) + } + }) + + t.Run("the same chain, or no answer, leaves the request untouched", func(t *testing.T) { + for name, answer := range map[string]func([]schemas.Route) []schemas.Route{ + "same": nil, + "empty": func([]schemas.Route) []schemas.Route { return nil }, + } { + client, _ := setup(t, answer) + req := newRequest(schemas.OpenAI) + before, beforeModel, beforeFallbacks := req.GetRequestFields() + client.resolveSessionRoute(sessionCtx("s"), routeOf("", "gpt-4o"), req) + provider, model, fallbacks := req.GetRequestFields() + if provider != before || model != beforeModel || !slices.Equal(fallbacks, beforeFallbacks) { + t.Fatalf("%s: request changed to %s/%s %v", name, provider, model, fallbacks) + } + } + }) + + // A direct key belongs to the request's own provider; a session binding must not move + // the primary to another provider while that credential is still on the context. + t.Run("a request with a direct key is not moved", func(t *testing.T) { + client, fake := setup(t, reverse) + ctx := sessionCtx("s") + ctx.SetValue(schemas.BifrostContextKeyDirectKey, schemas.Key{ID: "caller", Value: schemas.SecretVar{Val: "sk-caller"}}) + req := newRequest(schemas.OpenAI) + client.resolveSessionRoute(ctx, routeOf("", "gpt-4o"), req) + if provider, _, _ := req.GetRequestFields(); provider != schemas.OpenAI { + t.Fatalf("direct-key request moved to %s", provider) + } + if fake.routeCalls != 0 { + t.Fatal("ResolveRoute was asked about a direct-key request") + } + }) + + // An explicit key selector names a key of the request's own provider, so it holds the + // route too; a routing-rule pin does not, since it moves with its route. + t.Run("a request with an explicit key selector is not moved", func(t *testing.T) { + for name, set := range map[string]func(*schemas.BifrostContext){ + "key name": func(ctx *schemas.BifrostContext) { ctx.SetValue(schemas.BifrostContextKeyAPIKeyName, "prod") }, + "key id": func(ctx *schemas.BifrostContext) { ctx.SetValue(schemas.BifrostContextKeyAPIKeyID, "k-caller") }, + } { + client, fake := setup(t, reverse) + ctx := sessionCtx("s") + set(ctx) + req := newRequest(schemas.OpenAI) + client.resolveSessionRoute(ctx, routeOf("", "gpt-4o"), req) + if provider, _, _ := req.GetRequestFields(); provider != schemas.OpenAI { + t.Fatalf("%s: request moved to %s", name, provider) + } + if fake.routeCalls != 0 { + t.Fatalf("%s: ResolveRoute was asked", name) + } + } + client, fake := setup(t, reverse) + ctx := sessionCtx("s") + ctx.SetValue(schemas.BifrostContextKeyRoutingPinnedAPIKeyID, "k-rule") + ctx.SetValue(schemas.BifrostContextKeyAPIKeyID, "k-rule") + client.resolveSessionRoute(ctx, routeOf("", "gpt-4o"), newRequest(schemas.OpenAI)) + if fake.routeCalls != 1 { + t.Fatal("a routing-rule pin must not hold the route") + } + }) + + t.Run("a request no hook could route is not offered", func(t *testing.T) { + client, fake := setup(t, reverse) + client.resolveSessionRoute(sessionCtx("s"), routeOf("", "gpt-4o"), newRequest("")) + if fake.routeCalls != 0 { + t.Fatal("ResolveRoute was asked about a request with no provider") + } + }) + + t.Run("a request that takes no part is not offered", func(t *testing.T) { + client, fake := setup(t, reverse) + off := sessionCtx("s") + off.SetValue(schemas.BifrostContextKeySessionAffinity, false) + noSession := sessionCtx("") + for _, ctx := range []*schemas.BifrostContext{noSession, off} { + req := newRequest(schemas.OpenAI) + client.resolveSessionRoute(ctx, routeOf("", "gpt-4o"), req) + if provider, _, _ := req.GetRequestFields(); provider != schemas.OpenAI { + t.Fatalf("request changed to %s", provider) + } + } + if fake.routeCalls != 0 { + t.Fatal("ResolveRoute was asked about a request that takes no part") + } + // A request that carries a session and switched affinity off says so, because its session + // id is on the log record and silence would read as affinity having had nothing to add. + if !trailMentions(off, "asked not to follow") { + t.Fatal("a request that switched affinity off did not say so in the trail") + } + // One with no session at all has nothing to explain. + if len(noSession.GetRoutingEngineLogs()) != 0 { + t.Fatalf("a request with no session wrote a trail: %v", noSession.GetRoutingEngineLogs()) + } + }) +} + +// TestResolveSessionRouteMovesKeyPinsWithTheirRoutes pins the bug where a routing rule's key +// pin stayed on the request when the session moved the primary to another provider: a Vertex +// key looked up among Anthropic's keys can only fail, and the attempt died with "no supported +// key found" before Anthropic was ever called. A pin belongs to the route it was decided for, +// so it follows that route wherever the session puts it in the chain, and a fallback the +// session promotes brings its own pin along. +func TestResolveSessionRouteMovesKeyPinsWithTheirRoutes(t *testing.T) { + const vertexPin, anthropicPin = "vertex-key-id", "anthropic-key-id" + reverse := func(chain []schemas.Route) []schemas.Route { + out := slices.Clone(chain) + slices.Reverse(out) + return out + } + newRequest := func(fallbackPin string) *schemas.BifrostRequest { + req := &schemas.BifrostRequest{ + RequestType: schemas.ChatCompletionRequest, + ChatRequest: &schemas.BifrostChatRequest{Provider: schemas.Vertex, Model: "claude-opus-5"}, + } + req.SetFallbacks([]schemas.Fallback{{Provider: schemas.Anthropic, Model: "claude-opus-5", KeyID: fallbackPin}}) + return req + } + setup := func(t *testing.T, answer func([]schemas.Route) []schemas.Route) *Bifrost { + t.Helper() + client, err := Init(context.Background(), schemas.BifrostConfig{ + Account: NewMockAccount(), + Logger: NewDefaultLogger(schemas.LogLevelError), + SessionAffinity: &recordingAffinity{routeAnswer: answer}, + }) + if err != nil { + t.Fatalf("Init: %v", err) + } + t.Cleanup(client.Shutdown) + return client + } + // pinned is a context as RunPreRequestHooks leaves it once a rule pinned a key for the + // primary: the routing pin recorded, and committed into the api-key-id key selection reads. + pinned := func() *schemas.BifrostContext { + ctx := sessionCtx("s") + ctx.SetValue(schemas.BifrostContextKeyRoutingPinnedAPIKeyID, vertexPin) + ctx.SetValue(schemas.BifrostContextKeyAPIKeyID, vertexPin) + return ctx + } + // primaryPin is the pin the primary attempt will read, checking that the routing pin and + // the committed api-key-id never disagree. + primaryPin := func(t *testing.T, ctx *schemas.BifrostContext) string { + t.Helper() + routing, _ := ctx.Value(schemas.BifrostContextKeyRoutingPinnedAPIKeyID).(string) + apiKey, _ := ctx.Value(schemas.BifrostContextKeyAPIKeyID).(string) + if routing != apiKey { + t.Fatalf("routing pin %q and api-key-id %q disagree", routing, apiKey) + } + return apiKey + } + fallbacksOf := func(req *schemas.BifrostRequest) []schemas.Fallback { + _, _, fallbacks := req.GetRequestFields() + return fallbacks + } + + t.Run("the primary's pin follows it when the session demotes it to a fallback", func(t *testing.T) { + client := setup(t, reverse) + ctx := pinned() + req := newRequest("") + client.resolveSessionRoute(ctx, routeOf("", "claude-opus-5"), req) + if provider, _, _ := req.GetRequestFields(); provider != schemas.Anthropic { + t.Fatalf("primary = %s, want anthropic", provider) + } + if pin := primaryPin(t, ctx); pin != "" { + t.Fatalf("the moved primary still reads pin %q, which names a vertex key", pin) + } + want := []schemas.Fallback{{Provider: schemas.Vertex, Model: "claude-opus-5", KeyID: vertexPin}} + if got := fallbacksOf(req); !slices.Equal(got, want) { + t.Fatalf("fallbacks = %+v, want the demoted vertex route carrying its pin %+v", got, want) + } + if !trailMentions(ctx, "pinned") { + t.Fatalf("moving the pin left no trace in the trail: %v", ctx.GetRoutingEngineLogs()) + } + }) + + t.Run("a promoted fallback's own pin becomes the primary pin", func(t *testing.T) { + client := setup(t, reverse) + ctx := pinned() + req := newRequest(anthropicPin) + client.resolveSessionRoute(ctx, routeOf("", "claude-opus-5"), req) + if pin := primaryPin(t, ctx); pin != anthropicPin { + t.Fatalf("primary pin = %q, want the promoted route's own %q", pin, anthropicPin) + } + want := []schemas.Fallback{{Provider: schemas.Vertex, Model: "claude-opus-5", KeyID: vertexPin}} + if got := fallbacksOf(req); !slices.Equal(got, want) { + t.Fatalf("fallbacks = %+v, want %+v", got, want) + } + }) + + t.Run("an unpinned primary demoted behind a pinned fallback claims no pin of its own", func(t *testing.T) { + client := setup(t, reverse) + ctx := sessionCtx("s") + req := newRequest(anthropicPin) + client.resolveSessionRoute(ctx, routeOf("", "claude-opus-5"), req) + if pin := primaryPin(t, ctx); pin != anthropicPin { + t.Fatalf("primary pin = %q, want the promoted route's own %q", pin, anthropicPin) + } + want := []schemas.Fallback{{Provider: schemas.Vertex, Model: "claude-opus-5"}} + if got := fallbacksOf(req); !slices.Equal(got, want) { + t.Fatalf("fallbacks = %+v, want the demoted vertex route with no pin %+v", got, want) + } + if trailMentions(ctx, "applies if") { + t.Fatalf("the trail claims a pin for vertex that the rule never set: %v", ctx.GetRoutingEngineLogs()) + } + }) + + t.Run("pins stay where they are when the session agrees with routing", func(t *testing.T) { + for name, answer := range map[string]func([]schemas.Route) []schemas.Route{ + "same": nil, + "empty": func([]schemas.Route) []schemas.Route { return nil }, + } { + client := setup(t, answer) + ctx := pinned() + req := newRequest(anthropicPin) + client.resolveSessionRoute(ctx, routeOf("", "claude-opus-5"), req) + if pin := primaryPin(t, ctx); pin != vertexPin { + t.Fatalf("%s: primary pin = %q, want %q untouched", name, pin, vertexPin) + } + want := []schemas.Fallback{{Provider: schemas.Anthropic, Model: "claude-opus-5", KeyID: anthropicPin}} + if got := fallbacksOf(req); !slices.Equal(got, want) { + t.Fatalf("%s: fallbacks = %+v, want %+v untouched", name, got, want) + } + if trailMentions(ctx, "pinned") { + t.Fatalf("%s: the trail claims a pin moved: %v", name, ctx.GetRoutingEngineLogs()) + } + } + }) + + t.Run("a caller's own pins are left alone", func(t *testing.T) { + client := setup(t, reverse) + ctx := sessionCtx("s") + ctx.SetValue(schemas.BifrostContextKeyAPIKeyID, "caller-key") + ctx.SetValue(schemas.BifrostContextKeyAPIKeyName, "caller-name") + client.resolveSessionRoute(ctx, routeOf("", "claude-opus-5"), newRequest("")) + if got, _ := ctx.Value(schemas.BifrostContextKeyAPIKeyID).(string); got != "caller-key" { + t.Fatalf("api-key-id = %q after a session move, want the caller's own pin kept", got) + } + if got, _ := ctx.Value(schemas.BifrostContextKeyAPIKeyName).(string); got != "caller-name" { + t.Fatalf("api-key name = %q after a session move, want the caller's own pin kept", got) + } + if trailMentions(ctx, "pinned") { + t.Fatalf("the trail mentions a routing pin the request never had: %v", ctx.GetRoutingEngineLogs()) + } + }) +} + +func TestObserveSessionOutcomeReportsServedRouteAndKey(t *testing.T) { + fake := &recordingAffinity{} + client, err := Init(context.Background(), schemas.BifrostConfig{ + Account: NewMockAccount(), + Logger: NewDefaultLogger(schemas.LogLevelError), + SessionAffinity: fake, + }) + if err != nil { + t.Fatalf("Init: %v", err) + } + t.Cleanup(client.Shutdown) + + ctx := sessionCtx("s") + ctx.SetValue(schemas.BifrostContextKeySelectedKeyID, "key-a") + requested := routeOf("", "gpt-4o") + client.observeSessionOutcome(ctx, requested, &schemas.Route{Provider: schemas.Azure, Model: "gpt-4o"}, true, nil) + client.observeSessionOutcome(ctx, requested, nil, false, &schemas.BifrostError{}) + + if len(fake.outcomes) != 2 || fake.requested[0] != requested { + t.Fatalf("outcomes recorded: %+v for %v", fake.outcomes, fake.requested) + } + served := fake.outcomes[0] + if served.Served == nil || served.Served.Provider != schemas.Azure || served.KeyID != "key-a" || !served.Fallback || served.Err != nil { + t.Fatalf("served outcome: %+v", served) + } + failed := fake.outcomes[1] + if failed.Served != nil || failed.KeyID != "" || failed.Err == nil { + t.Fatalf("failed outcome: %+v", failed) + } + + // A request that takes no part is not reported. + off := sessionCtx("s") + off.SetValue(schemas.BifrostContextKeySessionAffinity, false) + client.observeSessionOutcome(off, requested, &schemas.Route{Provider: schemas.Azure, Model: "gpt-4o"}, false, nil) + client.observeSessionOutcome(sessionCtx(""), requested, &schemas.Route{Provider: schemas.Azure, Model: "gpt-4o"}, false, nil) + if len(fake.outcomes) != 2 { + t.Fatalf("a request that takes no part was reported: %d outcomes", len(fake.outcomes)) + } +} + +func TestKeyPoolAsksAffinityOnlyForRequestsThatTakePart(t *testing.T) { + account := NewMockAccount() + account.AddProvider(schemas.OpenAI, 5, 1000) + account.SetKeysForProvider(schemas.OpenAI, []schemas.Key{ + {ID: "key-a", Name: "Key A", Value: *schemas.NewSecretVar("sk-a"), Models: schemas.WhiteList{"*"}, Weight: 1}, + {ID: "key-b", Name: "Key B", Value: *schemas.NewSecretVar("sk-b"), Models: schemas.WhiteList{"*"}, Weight: 1}, + }) + fake := &recordingAffinity{} + client, err := Init(context.Background(), schemas.BifrostConfig{ + Account: account, + Logger: NewDefaultLogger(schemas.LogLevelError), + KVStore: newMockKVStore(), + SessionAffinity: fake, + }) + if err != nil { + t.Fatalf("Init: %v", err) + } + t.Cleanup(client.Shutdown) + + off := sessionCtx("s") + off.SetValue(schemas.BifrostContextKeySessionAffinity, false) + for name, ctx := range map[string]*schemas.BifrostContext{"no session": sessionCtx(""), "switched off": off} { + keys, canRotate, err := client.selectKeyFromProviderForModelWithPool(ctx, schemas.ChatCompletionRequest, schemas.OpenAI, "gpt-4", schemas.OpenAI) + if err != nil || !canRotate || len(keys) != 2 { + t.Fatalf("%s: got %d keys canRotate=%v err=%v, want the full rotating pool", name, len(keys), canRotate, err) + } + } + if fake.keyCalls != 0 { + t.Fatalf("ResolveKey was asked %d times about requests that take no part", fake.keyCalls) + } + if _, _, err := client.selectKeyFromProviderForModelWithPool(sessionCtx("s"), schemas.ChatCompletionRequest, schemas.OpenAI, "gpt-4", schemas.OpenAI); err != nil { + t.Fatalf("pool build: %v", err) + } + if fake.keyCalls != 1 { + t.Fatalf("ResolveKey was asked %d times about a request that takes part, want 1", fake.keyCalls) + } +} + +// foreignKeyAffinity answers with a key core never offered it, as a misbehaving custom +// SessionAffinity could. +type foreignKeyAffinity struct{} + +func (foreignKeyAffinity) ResolveRoute(_ *schemas.BifrostContext, _ schemas.Route, chain []schemas.Route) []schemas.Route { + return chain +} + +func (foreignKeyAffinity) ResolveKey(*schemas.BifrostContext, schemas.ModelProvider, string, []schemas.Key) (schemas.Key, bool) { + return schemas.Key{ID: "key-foreign", Name: "Foreign"}, true +} + +func (foreignKeyAffinity) Observe(*schemas.BifrostContext, schemas.Route, schemas.RouteOutcome) {} + +func TestKeyPoolIgnoresAnAffinityKeyOutsideThePool(t *testing.T) { + account := NewMockAccount() + account.AddProvider(schemas.OpenAI, 5, 1000) + account.SetKeysForProvider(schemas.OpenAI, []schemas.Key{ + {ID: "key-a", Name: "Key A", Value: *schemas.NewSecretVar("sk-a"), Models: schemas.WhiteList{"*"}, Weight: 1}, + {ID: "key-b", Name: "Key B", Value: *schemas.NewSecretVar("sk-b"), Models: schemas.WhiteList{"*"}, Weight: 1}, + }) + client, err := Init(context.Background(), schemas.BifrostConfig{ + Account: account, + Logger: NewDefaultLogger(schemas.LogLevelError), + KVStore: newMockKVStore(), + SessionAffinity: foreignKeyAffinity{}, + }) + if err != nil { + t.Fatalf("Init: %v", err) + } + t.Cleanup(client.Shutdown) + + keys, canRotate, err := client.selectKeyFromProviderForModelWithPool(sessionCtx("session-1"), schemas.ChatCompletionRequest, schemas.OpenAI, "gpt-4", schemas.OpenAI) + if err != nil { + t.Fatalf("pool build: %v", err) + } + if !canRotate || len(keys) != 2 { + t.Fatalf("got %d keys canRotate=%v, want the full rotating pool when the affinity names a key outside it", len(keys), canRotate) + } + for _, key := range keys { + if key.ID == "key-foreign" { + t.Fatal("a key the pool never held was handed to the request") + } + } +} + +func TestSessionAffinityForgetsWhatAnEarlierRequestFollowed(t *testing.T) { + kv := newMockKVStore() + a := testAffinity(kv) + requested := routeOf("", "gpt-4o") + chain := []schemas.Route{routeOf(schemas.OpenAI, "gpt-4o"), routeOf(schemas.Azure, "gpt-4o")} + ctx := sessionCtx("session-1") + routeKey := SessionStateKey(ctx, SessionStateKindRoute, "", "gpt-4o") + + // An earlier request on this context followed a binding to azure. + _ = kv.SetWithTTL(routeKey, "azure/gpt-4o", time.Minute) + a.ResolveRoute(ctx, requested, chain) + + // The binding expires. The next request on the same context finds none, and while it is + // in flight another request of the session binds openai first. + _, _ = kv.Delete(routeKey) + a.ResolveRoute(ctx, requested, chain) + _ = kv.SetWithTTL(routeKey, "openai/gpt-4o", time.Minute) + + a.Observe(ctx, requested, servedBy(routeOf(schemas.Azure, "gpt-4o"), "az-1", false)) + if entry := kv.data[routeKey]; entry.value != "openai/gpt-4o" { + t.Fatalf("a request that followed nothing overwrote another request's binding: %+v", entry) + } +} + +func TestSessionAffinityLeavesAnExplicitProviderAlone(t *testing.T) { + kv := newMockKVStore() + a := testAffinity(kv) + // The caller named openai; routing kept it first and added azure as a fallback. + requested := routeOf(schemas.OpenAI, "gpt-4o") + chain := []schemas.Route{routeOf(schemas.OpenAI, "gpt-4o"), routeOf(schemas.Azure, "gpt-4o")} + ctx := sessionCtx("session-1") + routeKey := SessionStateKey(ctx, SessionStateKindRoute, "openai", "gpt-4o") + + // A fallback that served does not become the session's home when the caller named the + // provider: no route binding is written, only the key the fallback used. + a.Observe(ctx, requested, servedBy(routeOf(schemas.Azure, "gpt-4o"), "az-1", true)) + if _, present := kv.data[routeKey]; present { + t.Fatal("a route binding was written for a request that named its provider") + } + if entry := kv.data[SessionStateKey(ctx, SessionStateKindKey, "azure", "gpt-4o")]; entry.value != "az-1" { + t.Fatalf("the fallback's key binding was not written: %+v", entry) + } + + // Even with a binding in the store, a request that named its provider is not reordered. + _ = kv.SetWithTTL(routeKey, "azure/gpt-4o", time.Minute) + if got := a.ResolveRoute(ctx, requested, chain); !slices.Equal(got, chain) { + t.Fatalf("a request that named openai was sent to %v", got) + } + if slices.Contains(enginesUsed(ctx), schemas.RoutingEngineSessionAffinity) { + t.Fatal("the session was recorded as deciding a route the caller named") + } +} diff --git a/core/streamfallback_test.go b/core/streamfallback_test.go index ce9e0b175df..e10328c2be7 100644 --- a/core/streamfallback_test.go +++ b/core/streamfallback_test.go @@ -185,19 +185,35 @@ func TestStreamFallbackAfterFirstChunkError(t *testing.T) { } func TestAzureChatFallbackAfterPreamble(t *testing.T) { - testAzureFallbackAfterPreamble(t, false) + testFallbackAfterPreamble(t, schemas.Azure, false, + `{"error":{"message":"rate limit exceeded","type":"rate_limit_error"}}`) } func TestAzureResponsesFallbackAfterPreamble(t *testing.T) { - testAzureFallbackAfterPreamble(t, true) + testFallbackAfterPreamble(t, schemas.Azure, true, + `{"type":"response.failed","response":{"id":"failed-attempt","error":{"code":"rate_limit_exceeded","message":"rate limit exceeded"}}}`) } -func testAzureFallbackAfterPreamble(t *testing.T, responses bool) { +// Direct OpenAI under overload: the stream opens, emits startup events, then +// fails with server_is_overloaded (no HTTP status). The fallback must serve it. +func TestOpenAIChatFallbackAfterPreamble(t *testing.T) { + testFallbackAfterPreamble(t, schemas.OpenAI, false, + `{"error":{"message":"The server is overloaded. Please try again later.","type":"server_error","code":"server_is_overloaded"}}`) +} + +func TestOpenAIResponsesFallbackAfterPreamble(t *testing.T) { + testFallbackAfterPreamble(t, schemas.OpenAI, true, + `{"type":"response.failed","response":{"id":"failed-attempt","status":"failed","error":{"code":"server_is_overloaded","message":"The server is overloaded. Please try again later."}}}`) +} + +// testFallbackAfterPreamble serves startup events followed by failure from the +// primary and asserts the Anthropic fallback serves the whole stream. +func testFallbackAfterPreamble(t *testing.T, primaryProvider schemas.ModelProvider, responses bool, failure string) { t.Helper() payloads := []string{ `{"choices":[],"prompt_filter_results":[]}`, `{"id":"failed-attempt","choices":[{"index":0,"delta":{"role":"assistant"}}]}`, - `{"error":{"message":"rate limit exceeded","type":"rate_limit_error"}}`, + failure, } if responses { payloads = []string{ @@ -205,7 +221,7 @@ func testAzureFallbackAfterPreamble(t *testing.T, responses bool) { `{"type":"response.in_progress","response":{"id":"failed-attempt","status":"in_progress","output":[]}}`, `{"type":"response.output_item.added","item":{"id":"failed-item","type":"message","role":"assistant","status":"in_progress","content":[]}}`, `{"type":"response.content_part.added","part":{"type":"output_text","text":"","annotations":[]}}`, - `{"type":"response.failed","response":{"id":"failed-attempt","error":{"code":"rate_limit_exceeded","message":"rate limit exceeded"}}}`, + failure, } } primary := httptest.NewServer(sseHandler(payloads...)) @@ -218,17 +234,20 @@ func testAzureFallbackAfterPreamble(t *testing.T, responses bool) { defer fallback.Close() account := NewMockAccount() - account.AddProviderWithBaseURL(schemas.Azure, 1, 1, primary.URL) + account.AddProviderWithBaseURL(primaryProvider, 1, 1, primary.URL) account.AddProviderWithBaseURL(schemas.Anthropic, 1, 1, fallback.URL) - account.configs[schemas.Azure].NetworkConfig.MaxRetries = 0 + account.configs[primaryProvider].NetworkConfig.MaxRetries = 0 account.configs[schemas.Anthropic].NetworkConfig.MaxRetries = 0 - account.SetKeysForProvider(schemas.Azure, []schemas.Key{{ - ID: "azure-key", Value: *schemas.NewSecretVar("test-key"), + primaryKey := schemas.Key{ + ID: "primary-key", Value: *schemas.NewSecretVar("test-key"), Models: schemas.WhiteList{"*"}, Weight: 100, - AzureKeyConfig: &schemas.AzureKeyConfig{ + } + if primaryProvider == schemas.Azure { + primaryKey.AzureKeyConfig = &schemas.AzureKeyConfig{ Endpoint: *schemas.NewSecretVar(primary.URL), - }, - }}) + } + } + account.SetKeysForProvider(primaryProvider, []schemas.Key{primaryKey}) account.SetKeysForProvider(schemas.Anthropic, []schemas.Key{{ ID: "fallback-key", Value: *schemas.NewSecretVar("test-key"), Models: schemas.WhiteList{"*"}, Weight: 100, @@ -237,7 +256,7 @@ func testAzureFallbackAfterPreamble(t *testing.T, responses bool) { ctx := schemas.NewBifrostContext(context.Background(), time.Now().Add(5*time.Second)) request := &schemas.BifrostChatRequest{ - Provider: schemas.Azure, + Provider: primaryProvider, Model: "gpt-4o-mini", Input: []schemas.ChatMessage{{ Role: schemas.ChatMessageRoleUser, diff --git a/core/utils.go b/core/utils.go index 67e47108429..e1e1b021a2d 100644 --- a/core/utils.go +++ b/core/utils.go @@ -3,8 +3,6 @@ package bifrost import ( "bytes" "context" - "crypto/sha256" - "encoding/hex" "encoding/json" "errors" "fmt" @@ -840,22 +838,6 @@ func IsCodemodeTool(toolName string) bool { return mcp.IsCodeModeTool(toolName) } -// hashSHA256 returns a deterministic hex-encoded SHA-256 hash of the input. -func hashSHA256(value string) string { - h := sha256.Sum256([]byte(value)) - return hex.EncodeToString(h[:]) -} - -func buildSessionKey(providerKey schemas.ModelProvider, sessionID string, model string) string { - // Hash session ID to prevent PII leakage and ensure bounded key size - hashedSessionID := hashSHA256(sessionID) - discriminator := model - if discriminator == "" { - discriminator = "__modelless__" - } - return "session:" + string(providerKey) + ":" + hashedSessionID + ":" + hashSHA256(discriminator) -} - // isPromptOptionalImageEditType returns true for edit task types that do not require a text prompt. // It normalises hyphenated variants (e.g. "erase-object") to underscore form before matching. func isPromptOptionalImageEditType(t *string) bool { diff --git a/core/version b/core/version index 9ab8337f396..5ad2491cf88 100644 --- a/core/version +++ b/core/version @@ -1 +1 @@ -1.9.1 +1.10.2 diff --git a/docs/changelogs/edge-v0.9.0.mdx b/docs/changelogs/edge-v0.9.0.mdx new file mode 100644 index 00000000000..fb8905d4ad9 --- /dev/null +++ b/docs/changelogs/edge-v0.9.0.mdx @@ -0,0 +1,31 @@ +--- +title: "v0.9.0" +description: "Edge v0.9.0 changelog - 2026-09-19" +--- + + + +## Changelog + +v0.9.0 makes pass-through mode a first-class routing state. A single per-application routing policy now governs every request path, and applications left in pass-through are continuously cleaned of stale Edge credentials instead of only once at startup. Cursor gains context-limit recovery through tool-output compaction and rolling conversation summarization. On macOS, administrators get a fleet-level capture kill switch that can be toggled at runtime without restarting anything. Across all supported coding tools, unknown provider event types now pass through as-is instead of failing the request, so new provider features keep working before Edge learns about them. + +## ✨ Features + +- **Per-Application Routing Policy** - A single immutable policy snapshot now decides gateway versus pass-through mode for every request path: HTTP, HTTP/2, WebSocket, model catalog, and Claude Cowork. Application aliases are folded consistently (`codex` resolves to `codex-desktop`, `claude-code-cli` to `claude-code`), an in-flight turn keeps the policy it started with even if the policy changes mid-turn, and the tray and diagnostics now show the applied routing state with its version, last sync time, and any reconciliation error. +- **Continuous Pass-Through Credential Cleanup** - Applications in pass-through mode (Claude Code, Codex, OpenCode, Gemini CLI) are now watched continuously, so a stale Edge credential restored by a client after the policy switch is removed on the next reconcile cycle rather than persisting until the next restart. Cleanup strips only the Edge-planted credential and never claims ownership of, alters, or deletes the user's own settings files. +- **Fleet Capture Kill Switch on macOS** - Administrators can now disable Edge's macOS network capture fleet-wide at runtime. The daemon pushes the policy to the capture extension live, flows are declined and passed through normally while capture is disabled, and re-enabling requires no reinstall, restart, or re-approval. Older agent versions ignore the new field and behave as before. +- **Cursor Context-Limit Recovery** - When a Cursor conversation grows past the model's input limit, Edge now recovers instead of failing the turn: older completed research tool outputs are compacted into explicitly incomplete previews and the request is retried once, and if that is not enough a bounded rolling summary of the conversation is generated through the same governed model. Summaries are checkpointed atomically, so cold resumes stay durable, and accepted streams and partial tool calls are never retried. +- **Gemini CLI Tool Delivery Gate with Incremental Streaming** - Gemini CLI requests now flow through the shared tool delivery gate: text, reasoning, and lifecycle events stream to the client immediately, while function calls are held until they pass policy inspection. A heartbeat keeps the connection alive during upstream and policy waits, and fixed response deadlines across the agent are replaced by request cancellation so long-running turns no longer hit arbitrary cutoffs. +- **Forward Compatibility with Unknown Provider Events** - Unknown streaming event types, content block variants, output item types, and finish reasons are now passed through as opaque data instead of being rejected, across all supported coding tools. Only recognized tool-call shapes enter policy inspection, and an unknown event can never complete or approve a pending tool call. +- **Unknown MCP Calls Continue as Observations** - A tool call for an MCP server that cannot be matched to the device inventory (stale inventory, removed server, unrecognized name) is now logged as an observation with full attribution instead of being hard-denied, and the rest of the batch continues under normal enforcement. Explicit policy denials are still enforced strictly. + +## 🐞 Fixed + +- **Cursor Short Model Aliases Failed to Resolve** - Selecting a short alias from Cursor's model picker could fail model resolution; unresolved names are now matched against the live model catalog's alias lists instead of guessing a provider. +- **Malformed Policy Messages Could Re-Enable Capture** - A capture policy message with a missing or wrongly typed field could unintentionally re-enable capture, and a policy update racing a connection handshake could leave a stale policy state; both paths are now hardened and covered by conformance tests. +- **Unknown Cursor Tool Calls Ended the Conversation** - A tool call Edge could not resolve terminated the Cursor run; it now returns an inline `tool_unavailable` result so the conversation continues, ambiguous tool names are no longer resolved by guessing, and an unconfirmed mutating tool on resume receives an explicit unknown-outcome message instead of failing closed. +- **Cursor Error Messages Buried Under Diagnostics** - Upstream error responses that append large diagnostic payloads after the structured message are now parsed for the message first, so the actual explanation is preserved without draining multi-megabyte bodies. +- **Model Catalog Fetch Timeouts Too Aggressive** - Model catalog fetches now allow 30 seconds instead of 5 to 10, fixing empty model pickers on slow networks. +- **Startup Log Noise** - Removed repetitive daemon log bursts emitted during startup on macOS and Linux. + + diff --git a/docs/changelogs/ent-v2.2.1.mdx b/docs/changelogs/ent-v2.2.1.mdx new file mode 100644 index 00000000000..c79ed10d00d --- /dev/null +++ b/docs/changelogs/ent-v2.2.1.mdx @@ -0,0 +1,497 @@ +--- +title: "v2.2.1" +description: "v2.2.1 changelog - 2026-09-19" +--- + + + +## Changelog + +v2.2.1 moves the enterprise gateway onto OSS transports v2.2.1. API key scopes are now clamped to the caller's own permissions at create and update. Virtual key search now matches the assigned user's name or email, and the assignee column is resolved in one batched query per page instead of one request per row. The OSS release also carries three security fixes: resource IDs are validated before they enter outbound URL paths, Bedrock S3 bucket names can no longer redirect signed requests to a caller-chosen host, and the billing idempotency key can no longer be forged through the `x-request-id` header. + +## ✨ Features + +- **API Key Scope Ceiling** - Creating or updating an API key now clamps the granted scopes to the permissions the caller holds. Unknown permission ids are rejected with 400, scopes the caller does not hold are rejected with 403 naming the offending permissions, and a key's own scopes win the permission check even when the creator's role is broader. Local-admin and auth-disabled requests keep their existing bypass. +- **Scope Ceiling in the API Key Picker** - The scope picker shows the ceiling before save: permissions above the caller's own are rendered locked with a lock icon and cannot be toggled on, while a locked scope already on the key stays removable so access can always be lowered. If the server still returns a 403 because the permission map went stale, the denied scopes are highlighted inline and the caller's permissions are re-fetched. The picker is built on a shared `PermissionMatrix` component with search and per-resource bulk select, now used by both API key scopes and role permissions. +- **Virtual Key Assignee Search and Batched Resolution** - Virtual key search now matches the assigned user's name or email through the enterprise store, which previously delegated search to the base store and silently dropped the assignee filter. The "Assigned To" column on the list page and in CSV export is resolved with a single batched query per page instead of one request per key. +- **Access-Profile MCP Grants in DAC Scopes** - Own-data and team-data scoped reads now treat access-profile grants as a first-class visibility path for MCP servers and Virtual MCPs. An MCP server granted exclusively through a user's access profile was previously invisible to that user under DAC scoping, because access-profile-managed virtual keys intentionally carry no VK-to-MCP assignment rows. +- **Unknown MCP Calls Recorded as Observations** - When an MCP tool call cannot be matched to any known installation in the device inventory (stale inventory, removed server, unrecognized name), the gateway now records it as an observation with `status=unknown` and full attribution metadata instead of issuing a hard denial, and the rest of the batch continues under normal enforcement. True policy denials and guardrail failures are still enforced strictly. + +## 🌎 Open Source Features + +- **Virtual Key Assignees and Expanded Search** - The virtual key list resolves `assigned_user` for every row via a single batch lookup, and VK search matches team, customer and user names in addition to key fields, so keys can be found by who owns them. +- **User Attribution in Prometheus Metrics** - The telemetry plugin's metrics export includes user id and user name labels, so per-user usage and error rates can be graphed and alerted on directly. +- **Prompt Cache Breakpoints Capability** - A `SupportsPromptCacheBreakpoints` model capability with a name-based fallback for OpenRouter gates `prompt_cache_breakpoint` forwarding, so OpenRouter's non-Claude models stop rejecting requests that carry it. +- **MCP Observed Latency** - Native (observed) MCP tool-call latency from `observed_latency_ms` is surfaced in the logs duration column and detail sheet, kept clearly distinct from true execution time and never fabricating a synthetic start timestamp on the timeline. + +## 🐞 Fixed + +- **Access Profile Role-Detach Revocation Scope** - Detaching a role from an access profile now revokes the profile only from the detached role's users, sparing users whose remaining role still grants the profile. +- **Resource ID Path Injection (OSS)** - Caller-supplied resource IDs (batch, file, video, container, response IDs, cached content names) were interpolated into outbound URL paths unvalidated; a crafted ID could redirect the request to an unintended upstream endpoint. A central `EscapeResourceID` helper validates and escapes every such ID. +- **Bedrock S3 SSRF (OSS)** - A caller-supplied `s3://` file ID or `s3_bucket` param could control the upstream host of a SigV4-signed connection; bucket names are now validated with a DNS-compatible regex. +- **Caller-Forged Billing Idempotency (OSS)** - The billing idempotency key included the caller-controllable `x-request-id`, so two requests sharing a chosen ID collided and the second was never charged. An internally minted `BillingNonce` is mixed into the key, making it unforgeable. +- **Abandoned Non-Streaming Request Hung Forever (OSS)** - The worker kept a `ctx.Done()` arm on an already-claimed delivery send, so a non-streaming caller could hang indefinitely. +- **Repeated Empty Thinking Blocks (OSS)** - Streaming chat chunks emitted empty reasoning/message fields on every content delta, which clients rendered as repeated empty thinking blocks. +- **Claude Code Thread Continuations Behind Key Rotation (OSS)** - Server-side conversation threads are bound to the upstream account that creates them, which per-request key selection cannot guarantee; the Anthropic integration now declares itself stateless, refusing continuations with a 400 `thread_unsupported_request` so the client resends the turn in full, and stripping `thread` from create requests so no orphaned thread state accumulates upstream. +- **Gemini Flash-Lite Minimal Thinking Promoted to Low (OSS)** - `gemini-3.1-flash-lite` requests silently promoted `minimal` thinking to `low` on Gemini and Vertex; the text model's four supported levels are now registered. +- **Config.json Virtual Key Limits Broke Under UI Edits (OSS)** - VK rate limits and budgets created via the config.json standalone-limits flow and then edited through the UI produced duplicate, conflicting ownership records; a migration consolidates ownership into VK-scoped model configs while preserving usage counters. +- **Bedrock Service Tier Rejections (OSS)** - Service tier forwarding for Bedrock is now gated on explicit model capability metadata, failing closed when none exists. +- **Anthropic Root-Level Tool Schema Compositions (OSS)** - Root-level `oneOf`/`anyOf`/`allOf` in a tool's `input_schema` are rewritten into a flat object schema before dispatch, unblocking tools that Anthropic would otherwise reject. +- **GenAI Streaming TTS (OSS)** - Speech stream chunks routed through the `/genai` integration had no converter registered; streaming TTS now works, and a missing stream converter returns a clean error instead of panicking. +- **xAI Usage and Cost (OSS)** - `completion_tokens` now folds reasoning tokens in so token totals add up, and streaming cost normalization preserves xAI's authoritative cost ticks instead of falling back to catalog pricing. +- **OpenAI-Only `search_content_types` Forwarded Everywhere (OSS)** - The field is gated behind a per-provider capability check so Bedrock and other OpenAI-compatible backends receive a clean `web_search` tool. +- **Claude Code `diagnostics` Field Rejected by Non-Native Providers (OSS)** - `diagnostics.previous_message_id` is stripped for providers that do not support it, fixing 400s on Bedrock, Vertex and Azure. +- **Unreadable Bedrock Error Logs (OSS)** - AWS's flat `{"message", "__type"}` error shape now seeds the error message, and the logs UI falls back to the raw provider error body when no message could be extracted. +- **Anthropic `container` Param Dropped (OSS)** - The string-form `container` param on `/anthropic/v1/messages` is now carried through the round trip, so container reuse stops provisioning a fresh container every time. +- **Responses API Finish Reason Missing From OTEL Spans (OSS)** - `gen_ai.response.finish_reason` is now emitted on the `llm.call` span for `/v1/responses` requests, matching the chat path. +- **Cohere Fallback Response Shape (OSS)** - Responses served by a fallback from a non-Cohere provider are converted to the Cohere v2 shape instead of returning the raw normalized response. +- **Skill Serving Race (OSS)** - Fixed a race condition in the skill serving handler. +- **Log and Dashboard Label Truncation (OSS)** - Long model and provider labels truncate from the start so the distinctive suffix stays visible, the logs model column is wider, and key picker options are keyed by id so duplicate labels stop highlighting together. + +## 🗄️ Database Migrations + +- **migrate_vk_standalone_limits_to_model_configs** - Consolidates virtual key standalone budgets and rate limits into VK-scoped model configs: standalone budgets are re-pointed to the VK's top-level model config (created if missing, usage preserved), orphaned duplicate UUID rate-limit rows created by the UI are deleted with the model config re-pointed to the canonical config.json row, and `vk.rate_limit_id` is cleared. + +## 🐙 Closed OSS Issues + +- [#123](https://github.com/maximhq/bifrost/issues/123) - Files API Support +- [#5707](https://github.com/maximhq/bifrost/issues/5707) - string-form `container` param on /anthropic/v1/messages is silently dropped - container reuse provisions a fresh container +- [#7204](https://github.com/maximhq/bifrost/issues/7204) - Responses API path never sets gen_ai.response.finish_reason in OTEL traces (refusals invisible to observability) +- [#7287](https://github.com/maximhq/bifrost/issues/7287) - Gemini 3.1 Flash-Lite minimal thinking is silently promoted to low +- [#7294](https://github.com/maximhq/bifrost/issues/7294) - Streaming chat chunks emit empty reasoning/message fields on every content delta, causing repeated thinking blocks +- [#7308](https://github.com/maximhq/bifrost/issues/7308) - non-streaming caller hangs forever - worker keeps a ctx.Done() arm on a claimed delivery send (regression from #6972) + +## 📀 Base OSS version + +`transports/v2.2.1` (pinned as `github.com/maximhq/bifrost/transports v1.6.12-0.20260918214249-f44b5410fb7b`), with core `v1.9.1`, framework `v1.7.2`, governance `v1.8.1`, and logging `v1.8.1`. + +## 🔌 If you are compiling plugin against this release - use following deps + +```go +module github.com/maximhq/bifrost-enterprise/transports + +go 1.27.0 + +require ( + github.com/bytedance/sonic v1.15.3-0.20260730064818-2a36d6da63e2 + github.com/coreos/go-oidc/v3 v3.18.0 + github.com/fasthttp/router v1.5.4 + github.com/google/cel-go v0.30.0 + github.com/google/uuid v1.6.0 + github.com/maximhq/bifrost-enterprise/core v0.0.0 + github.com/maximhq/bifrost-enterprise/framework v0.0.0 + github.com/maximhq/bifrost-enterprise/plugins v0.0.0 + github.com/maximhq/bifrost/core v1.9.1 + github.com/maximhq/bifrost/framework v1.7.2 + github.com/maximhq/bifrost/plugins/governance v1.8.1 + github.com/maximhq/bifrost/plugins/logging v1.8.1 + github.com/maximhq/bifrost/plugins/routing v1.1.1 + github.com/maximhq/bifrost/plugins/semanticcache v1.6.4 + github.com/maximhq/bifrost/transports v1.6.12-0.20260918214249-f44b5410fb7b + github.com/stretchr/testify v1.11.1 + github.com/valyala/fasthttp v1.74.0 + golang.org/x/sync v0.22.0 + gorm.io/driver/sqlite v1.6.0 + gorm.io/gorm v1.31.1 +) + +require ( + cel.dev/expr v0.25.2 // indirect + cloud.google.com/go v0.123.0 // indirect + cloud.google.com/go/auth v0.20.0 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect + cloud.google.com/go/bigquery v1.74.0 // indirect + cloud.google.com/go/compute/metadata v0.9.0 // indirect + cloud.google.com/go/iam v1.7.0 // indirect + cloud.google.com/go/monitoring v1.24.3 // indirect + cloud.google.com/go/pubsub/v2 v2.4.0 // indirect + cloud.google.com/go/secretmanager v1.16.0 // indirect + cloud.google.com/go/storage v1.62.1 // indirect + dario.cat/mergo v1.0.2 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect + github.com/BobuSumisu/aho-corasick v1.0.3 // indirect + github.com/ClickHouse/ch-go v0.65.0 // indirect + github.com/ClickHouse/clickhouse-go/v2 v2.32.0 // indirect + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.77.0 // indirect + github.com/DataDog/datadog-agent/pkg/obfuscate v0.77.0 // indirect + github.com/DataDog/datadog-agent/pkg/opentelemetry-mapping-go/otlp/attributes v0.77.0 // indirect + github.com/DataDog/datadog-agent/pkg/proto v0.77.0 // indirect + github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.77.0 // indirect + github.com/DataDog/datadog-agent/pkg/template v0.77.0 // indirect + github.com/DataDog/datadog-agent/pkg/trace v0.77.0 // indirect + github.com/DataDog/datadog-agent/pkg/trace/log v0.77.0 // indirect + github.com/DataDog/datadog-agent/pkg/trace/otel v0.77.0 // indirect + github.com/DataDog/datadog-agent/pkg/trace/stats v0.77.0 // indirect + github.com/DataDog/datadog-agent/pkg/trace/traceutil v0.77.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/log v0.77.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/scrubber v0.77.0 // indirect + github.com/DataDog/datadog-agent/pkg/version v0.77.0 // indirect + github.com/DataDog/datadog-go/v5 v5.8.3 // indirect + github.com/DataDog/dd-trace-go/v2 v2.8.2 // indirect + github.com/DataDog/go-libddwaf/v4 v4.9.0 // indirect + github.com/DataDog/go-runtime-metrics-internal v0.0.4-0.20260217080614-b0f4edc38a6d // indirect + github.com/DataDog/go-sqllexer v0.1.13 // indirect + github.com/DataDog/go-tuf v1.1.1-0.5.2 // indirect + github.com/DataDog/sketches-go v1.4.8 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.33.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 // indirect + github.com/Masterminds/goutils v1.1.1 // indirect + github.com/Masterminds/semver/v3 v3.4.0 // indirect + github.com/Masterminds/sprig/v3 v3.3.0 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/ProtonMail/go-crypto v1.1.6 // indirect + github.com/STARRY-S/zip v0.2.1 // indirect + github.com/andybalholm/brotli v1.2.2 // indirect + github.com/antlr4-go/antlr/v4 v4.13.1 // indirect + github.com/apache/arrow/go/v15 v15.0.2 // indirect + github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect + github.com/armon/go-metrics v0.4.1 // indirect + github.com/aws/aws-sdk-go-v2 v1.42.0 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect + github.com/aws/aws-sdk-go-v2/config v1.32.14 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.19.14 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.29 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.29 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect + github.com/aws/aws-sdk-go-v2/service/bedrockruntime v1.50.6 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 // indirect + github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0 // indirect + github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.42.3 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 // indirect + github.com/aws/smithy-go v1.27.1 // indirect + github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect + github.com/bahlo/generic-list-go v0.2.0 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/blevesearch/go-porterstemmer v1.0.3 // indirect + github.com/bodgit/plumbing v1.3.0 // indirect + github.com/bodgit/sevenzip v1.6.0 // indirect + github.com/bodgit/windows v1.0.1 // indirect + github.com/buger/jsonparser v1.2.0 // indirect + github.com/bytedance/gopkg v0.1.3 // indirect + github.com/bytedance/sonic/loader v0.5.2 // indirect + github.com/cenkalti/backoff v2.2.1+incompatible // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/cenkalti/backoff/v5 v5.0.3 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/charmbracelet/colorprofile v0.3.1 // indirect + github.com/charmbracelet/lipgloss v1.1.0 // indirect + github.com/charmbracelet/x/ansi v0.10.1 // indirect + github.com/charmbracelet/x/cellbuf v0.0.13 // indirect + github.com/charmbracelet/x/term v0.2.1 // indirect + github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect + github.com/cloudflare/circl v1.6.3 // indirect + github.com/cloudwego/base64x v0.1.6 // indirect + github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect + github.com/coreos/go-semver v0.3.1 // indirect + github.com/coreos/go-systemd/v22 v22.6.0 // indirect + github.com/cyphar/filepath-securejoin v0.6.1 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/ebitengine/purego v0.10.0 // indirect + github.com/emicklei/go-restful/v3 v3.13.0 // indirect + github.com/emirpasic/gods v1.18.1 // indirect + github.com/envoyproxy/go-control-plane/envoy v1.37.0 // indirect + github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect + github.com/fasthttp/websocket v1.5.12 // indirect + github.com/fatih/color v1.18.0 // indirect + github.com/fatih/semgroup v1.2.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/gitleaks/go-gitdiff v0.9.1 // indirect + github.com/go-faster/city v1.0.1 // indirect + github.com/go-faster/errors v0.7.1 // indirect + github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect + github.com/go-git/go-billy/v5 v5.9.0 // indirect + github.com/go-git/go-git/v5 v5.19.2 // indirect + github.com/go-jose/go-jose/v4 v4.1.4 // indirect + github.com/go-logr/logr v1.4.4 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-ole/go-ole v1.3.0 // indirect + github.com/go-openapi/analysis v0.24.2 // indirect + github.com/go-openapi/errors v0.22.5 // indirect + github.com/go-openapi/jsonpointer v0.22.4 // indirect + github.com/go-openapi/jsonreference v0.21.4 // indirect + github.com/go-openapi/loads v0.23.2 // indirect + github.com/go-openapi/runtime v0.29.2 // indirect + github.com/go-openapi/spec v0.22.3 // indirect + github.com/go-openapi/strfmt v0.25.0 // indirect + github.com/go-openapi/swag v0.25.4 // indirect + github.com/go-openapi/swag/cmdutils v0.25.4 // indirect + github.com/go-openapi/swag/conv v0.25.4 // indirect + github.com/go-openapi/swag/fileutils v0.25.4 // indirect + github.com/go-openapi/swag/jsonname v0.25.4 // indirect + github.com/go-openapi/swag/jsonutils v0.25.4 // indirect + github.com/go-openapi/swag/loading v0.25.4 // indirect + github.com/go-openapi/swag/mangling v0.25.4 // indirect + github.com/go-openapi/swag/netutils v0.25.4 // indirect + github.com/go-openapi/swag/stringutils v0.25.4 // indirect + github.com/go-openapi/swag/typeutils v0.25.4 // indirect + github.com/go-openapi/swag/yamlutils v0.25.4 // indirect + github.com/go-openapi/validate v0.25.1 // indirect + github.com/go-viper/mapstructure/v2 v2.5.0 // indirect + github.com/goccy/go-json v0.10.5 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang-jwt/jwt/v5 v5.3.1 // indirect + github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/btree v1.1.3 // indirect + github.com/google/flatbuffers v23.5.26+incompatible // indirect + github.com/google/gnostic-models v0.7.0 // indirect + github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 // indirect + github.com/google/s2a-go v0.1.9 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.16 // indirect + github.com/googleapis/gax-go/v2 v2.22.0 // indirect + github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect + github.com/grandcat/zeroconf v1.0.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect + github.com/h2non/filetype v1.1.3 // indirect + github.com/hashicorp/consul/api v1.34.3 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-hclog v1.6.3 // indirect + github.com/hashicorp/go-immutable-radix v1.3.1 // indirect + github.com/hashicorp/go-metrics v0.5.4 // indirect + github.com/hashicorp/go-msgpack/v2 v2.1.5 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/hashicorp/go-retryablehttp v0.7.8 // indirect + github.com/hashicorp/go-rootcerts v1.0.2 // indirect + github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0 // indirect + github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect + github.com/hashicorp/go-sockaddr v1.0.7 // indirect + github.com/hashicorp/go-version v1.8.0 // indirect + github.com/hashicorp/golang-lru v1.0.2 // indirect + github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect + github.com/hashicorp/hcl v1.0.1-vault-7 // indirect + github.com/hashicorp/memberlist v0.5.4 // indirect + github.com/hashicorp/serf v0.10.1 // indirect + github.com/hashicorp/vault/api v1.23.0 // indirect + github.com/huandu/xstrings v1.5.0 // indirect + github.com/invopop/jsonschema v0.13.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/pgx/v5 v5.9.2 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/jaswdr/faker/v2 v2.8.0 // indirect + github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect + github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jinzhu/now v1.1.5 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/kevinburke/ssh_config v1.2.0 // indirect + github.com/klauspost/compress v1.20.0 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect + github.com/klauspost/pgzip v1.2.6 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect + github.com/linkdata/deadlock v0.5.5 // indirect + github.com/lucasb-eyer/go-colorful v1.2.0 // indirect + github.com/lufia/plan9stats v0.0.0-20260216142805-b3301c5f2a88 // indirect + github.com/magiconair/properties v1.8.10 // indirect + github.com/mailru/easyjson v0.9.1 // indirect + github.com/mark3labs/mcp-go v0.43.2 // indirect + github.com/mattn/go-colorable v0.1.14 // indirect + github.com/mattn/go-isatty v0.0.24 // indirect + github.com/mattn/go-runewidth v0.0.17 // indirect + github.com/mattn/go-sqlite3 v1.14.32 // indirect + github.com/maximhq/bifrost/plugins/compat v0.3.1 // indirect + github.com/maximhq/bifrost/plugins/maxim v1.7.4 // indirect + github.com/maximhq/bifrost/plugins/mocker v1.6.4 // indirect + github.com/maximhq/bifrost/plugins/modelcatalogresolver v1.1.4 // indirect + github.com/maximhq/bifrost/plugins/otel v1.5.4 // indirect + github.com/maximhq/bifrost/plugins/prompts v1.1.4 // indirect + github.com/maximhq/bifrost/plugins/telemetry v1.8.0 // indirect + github.com/maximhq/maxim-go v0.2.1 // indirect + github.com/mholt/archives v0.1.2 // indirect + github.com/miekg/dns v1.1.68 // indirect + github.com/minio/minlz v1.0.0 // indirect + github.com/minio/simdjson-go v0.4.5 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect + github.com/molecule-man/go-brrr v1.0.1 // indirect + github.com/muesli/termenv v0.16.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/nakabonne/tstorage v0.3.6 // indirect + github.com/nwaples/rardecode/v2 v2.2.2 // indirect + github.com/oapi-codegen/runtime v1.1.1 // indirect + github.com/oklog/ulid v1.3.1 // indirect + github.com/outcaste-io/ristretto v0.2.3 // indirect + github.com/paulmach/orb v0.11.1 // indirect + github.com/pelletier/go-toml/v2 v2.2.3 // indirect + github.com/petermattis/goid v0.0.0-20260226131333-17d1149c6ac6 // indirect + github.com/philhofer/fwd v1.2.0 // indirect + github.com/philippgille/chromem-go v0.7.0 // indirect + github.com/pierrec/lz4/v4 v4.1.22 // indirect + github.com/pinecone-io/go-pinecone/v5 v5.3.0 // indirect + github.com/pion/datachannel v1.6.0 // indirect + github.com/pion/dtls/v3 v3.1.5 // indirect + github.com/pion/ice/v4 v4.2.1 // indirect + github.com/pion/interceptor v0.1.44 // indirect + github.com/pion/logging v0.2.4 // indirect + github.com/pion/mdns/v2 v2.1.0 // indirect + github.com/pion/randutil v0.1.0 // indirect + github.com/pion/rtcp v1.2.16 // indirect + github.com/pion/rtp v1.10.1 // indirect + github.com/pion/sctp v1.9.2 // indirect + github.com/pion/sdp/v3 v3.0.18 // indirect + github.com/pion/srtp/v3 v3.0.10 // indirect + github.com/pion/stun/v3 v3.1.6 // indirect + github.com/pion/transport/v4 v4.0.2 // indirect + github.com/pion/turn/v4 v4.1.4 // indirect + github.com/pion/webrtc/v4 v4.2.9 // indirect + github.com/pjbgf/sha1cd v0.6.0 // indirect + github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect + github.com/prometheus/client_golang v1.23.2 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.67.5 // indirect + github.com/prometheus/procfs v0.19.2 // indirect + github.com/puzpuzpuz/xsync/v3 v3.5.1 // indirect + github.com/qdrant/go-client v1.16.2 // indirect + github.com/redis/go-redis/v9 v9.17.2 // indirect + github.com/rivo/uniseg v0.4.7 // indirect + github.com/rs/zerolog v1.34.0 // indirect + github.com/ryanuber/go-glob v1.0.0 // indirect + github.com/sagikazarmark/locafero v0.7.0 // indirect + github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect + github.com/savsgio/gotils v0.0.0-20250408102913-196191ec6287 // indirect + github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect + github.com/secure-systems-lab/go-securesystemslib v0.10.0 // indirect + github.com/segmentio/asm v1.2.0 // indirect + github.com/segmentio/kafka-go v0.4.51 // indirect + github.com/sergi/go-diff v1.4.0 // indirect + github.com/shirou/gopsutil/v4 v4.26.3 // indirect + github.com/shopspring/decimal v1.4.0 // indirect + github.com/skeema/knownhosts v1.3.1 // indirect + github.com/sorairolake/lzip-go v0.3.5 // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spf13/afero v1.15.0 // indirect + github.com/spf13/cast v1.10.0 // indirect + github.com/spf13/pflag v1.0.10 // indirect + github.com/spf13/viper v1.19.0 // indirect + github.com/spiffe/go-spiffe/v2 v2.7.0 // indirect + github.com/stretchr/objx v0.5.3 // indirect + github.com/subosito/gotenv v1.6.0 // indirect + github.com/tetratelabs/wazero v1.11.0 // indirect + github.com/therootcompany/xz v1.0.1 // indirect + github.com/tidwall/gjson v1.18.0 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.1 // indirect + github.com/tidwall/sjson v1.2.5 // indirect + github.com/tinylib/msgp v1.6.3 // indirect + github.com/tklauser/go-sysconf v0.3.16 // indirect + github.com/tklauser/numcpus v0.11.0 // indirect + github.com/trailofbits/go-mutexasserts v0.0.0-20250514102930-c1f3d2e37561 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/ulikunitz/xz v0.5.15 // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect + github.com/wasilibs/go-re2 v1.9.0 // indirect + github.com/wasilibs/wazero-helpers v0.0.0-20240620070341-3dff1577cd52 // indirect + github.com/weaviate/weaviate v1.38.0 // indirect + github.com/weaviate/weaviate-go-client/v5 v5.7.1 // indirect + github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect + github.com/wlynxg/anet v0.0.5 // indirect + github.com/x448/float16 v0.8.4 // indirect + github.com/xanzy/ssh-agent v0.3.3 // indirect + github.com/xdg-go/pbkdf2 v1.0.0 // indirect + github.com/xdg-go/scram v1.1.2 // indirect + github.com/xdg-go/stringprep v1.0.4 // indirect + github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect + github.com/yosida95/uritemplate/v3 v3.0.2 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect + github.com/zeebo/xxh3 v1.1.0 // indirect + github.com/zricethezav/gitleaks/v8 v8.30.1 // indirect + go.etcd.io/etcd/api/v3 v3.6.11 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.6.11 // indirect + go.etcd.io/etcd/client/v3 v3.6.11 // indirect + go.mongodb.org/mongo-driver v1.17.7 // indirect + go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/collector/component v1.51.1-0.20260205185216-81bc641f26c0 // indirect + go.opentelemetry.io/collector/featuregate v1.51.1-0.20260205185216-81bc641f26c0 // indirect + go.opentelemetry.io/collector/pdata v1.51.1-0.20260205185216-81bc641f26c0 // indirect + go.opentelemetry.io/collector/pdata/pprofile v0.145.1-0.20260205185216-81bc641f26c0 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.44.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect + go.opentelemetry.io/otel v1.45.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.44.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.44.0 // indirect + go.opentelemetry.io/otel/metric v1.45.0 // indirect + go.opentelemetry.io/otel/sdk v1.45.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.45.0 // indirect + go.opentelemetry.io/otel/trace v1.45.0 // indirect + go.opentelemetry.io/proto/otlp v1.10.0 // indirect + go.starlark.net v0.0.0-20260102030733-3fee463870c9 // indirect + go.uber.org/atomic v1.11.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.1 // indirect + go.yaml.in/yaml/v2 v2.4.3 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + go4.org v0.0.0-20230225012048-214862532bf5 // indirect + golang.org/x/arch v0.23.0 // indirect + golang.org/x/crypto v0.55.0 // indirect + golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect + golang.org/x/mod v0.38.0 // indirect + golang.org/x/net v0.58.0 // indirect + golang.org/x/oauth2 v0.36.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/telemetry v0.0.0-20260708182218-49f421fb7959 // indirect + golang.org/x/term v0.45.0 // indirect + golang.org/x/text v0.41.0 // indirect + golang.org/x/time v0.15.0 // indirect + golang.org/x/tools v0.48.0 // indirect + golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect + google.golang.org/api v0.282.0 // indirect + google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect + google.golang.org/grpc v1.83.2 // indirect + google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect + gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/ini.v1 v1.67.1 // indirect + gopkg.in/warnings.v0 v0.1.2 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + gorm.io/driver/clickhouse v0.7.0 // indirect + gorm.io/driver/postgres v1.6.0 // indirect + k8s.io/api v0.36.1 // indirect + k8s.io/apimachinery v0.36.1 // indirect + k8s.io/client-go v0.36.1 // indirect + k8s.io/klog/v2 v2.140.0 // indirect + k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a // indirect + k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 // indirect + sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.2 // indirect + sigs.k8s.io/yaml v1.6.0 // indirect +) + +replace github.com/maximhq/bifrost-enterprise/core => ../core + +replace github.com/maximhq/bifrost-enterprise/framework => ../framework + +replace github.com/maximhq/bifrost-enterprise/plugins => ../plugins +``` + + diff --git a/docs/changelogs/ent-v2.2.2.mdx b/docs/changelogs/ent-v2.2.2.mdx new file mode 100644 index 00000000000..4c82f5b6d0e --- /dev/null +++ b/docs/changelogs/ent-v2.2.2.mdx @@ -0,0 +1,492 @@ +--- +title: "v2.2.2" +description: "v2.2.2 changelog - 2026-09-23" +--- + + + +## Changelog + +v2.2.2 moves the enterprise gateway onto OSS transports v2.2.2. Session affinity is now health-aware in enterprise: a session bound to a provider or key that is failing on the node, or that the circuit breaker holds back, is dropped and re-routed instead of being sent back to the failing route. The OSS release adds the typesafe provider with a `/v1/decisions` endpoint, provider-level session affinity for Claude Code, Codex CLI and OpenCode sessions, and Claude Opus 5.5 support. + +## ✨ Features + +- **Health-Aware Session Affinity** - Enterprise registers its own session affinity on top of the one OSS ships. A route binding to a provider whose direction is Failed on this node is dropped and the routing decision stands, and a key binding is dropped when the key's route is Failed or the circuit breaker removes it from the eligible pool, so key selection picks again. Each drop is written to the routing engine log. Health is node-local while bindings are shared across the cluster, so the node that sees the failure drops the binding and the next served request rebinds it. Without the adaptive load balancer or the circuit breaker, behaviour is identical to OSS. [Docs](https://docs.getbifrost.ai/providers/session-affinity) + +## 🌎 Open Source Features + +- **Typesafe Provider and Decisions API** - A new `typesafe` provider, a `/v1/decisions` endpoint and a `/typesafe` integration. Providers without native decision support answer decision requests through forced tool-calling on their chat model, as primary or as fallback; probabilities are normalized to sum to exactly 1 and the chosen option must be the most likely one. Decision requests are priced from the datasheet under a new `decisions` usage type and logged with their answers. [Docs](https://docs.getbifrost.ai/quickstart/gateway/decisions) +- **Provider-Level Session Affinity** - A session, identified by `x-bf-session-id` or the session header Claude Code, Codex CLI and OpenCode already send, stays on the provider and key that last served it. Bindings are updated from the request outcome, affinity only reorders the chain that routing built and never restores a provider that routing excluded, and the `x-bf-session-affinity` header opts a request out. The logs UI shows affinity as a routing engine. [Docs](https://docs.getbifrost.ai/providers/session-affinity) +- **Claude Opus 5.5 Support** - Computer use sends `computer_toolset_20260801` on the Anthropic API and Vertex, while Bedrock and Azure keep `computer_20251124`, and `toolset_name` is carried on both halves of each call/result pair across typed, raw passthrough and streaming paths. Disabled thinking and forced tool choice are rejected for Opus 5.5 and later, which the datasheet `supports_reasoning_disable` field can override. +- **Claude Code Auto-Mode Safeguards Passthrough** - `safeguards` and `safeguard_results` are forwarded byte-for-byte on requests, responses and stream events to the direct Anthropic provider and stripped for every other provider. The `dangerous-tool-use` and `auto-mode-classifier` betas are gated the same way, and unknown Anthropic SSE events are forwarded raw on the Anthropic passthrough. + +## 🐞 Fixed + +- **Allow-All Providers on Access Profiles and Projects** - Access profiles and projects with allow-all providers now create a provider permit for every configured provider, so requests are no longer refused for providers without an explicit entry. +- **Held Responses Stream Raw Payloads** - When guardrails rewrite held Responses stream chunks, the raw provider response is dropped instead of patched, so a raw copy can no longer carry unredacted tool arguments or fail the stream on a shape mismatch. +- **Blocked Models With All Models Allowed** - The access profile grant details view renders the Blocked Models row correctly when the profile allows all models. +- **Allow-All Provider Access on Virtual Keys (OSS)** - Virtual keys that allow every provider now list models from, and route to, every configured provider; the governance routing log names providers excluded for having no weight. +- **OpenAI-Family Stream Errors After Startup Events (OSS)** - Stream errors that arrive after `response.created`, `in_progress` or an empty role delta now reach retry and fallback for OpenAI models on every host (OpenAI, Bedrock, Bedrock Mantle, Vertex, custom providers), not only Azure. +- **Kimi and DeepSeek With Claude Code (OSS)** - Tool-schema regex patterns are rewritten for Moonshot and DeepSeek models only, and kimi-k3 on Bedrock no longer returns an empty stream. +- **Anthropic Billing Header Leak (OSS)** - Claude Code's `x-anthropic-billing-header` system block is stripped at ingress and restored only for Anthropic-family attempts, so it no longer pollutes GPT or Gemini prompts. +- **Anthropic `allowed_tools` and `parallel_tool_calls` (OSS)** - OpenAI-style `allowed_tools` restrictions and `parallel_tool_calls` are now carried into Anthropic requests instead of being silently dropped. +- **MCP Egress Proxy (OSS)** - MCP HTTP/SSE connections honor `HTTP_PROXY`, `HTTPS_PROXY` and `NO_PROXY` again, and link-local or unspecified destinations are refused before the proxy is dialed. +- **OpenAI `computer` Tool (OSS)** - The bare `{"type":"computer"}` tool is no longer rewritten to `computer_use_preview`, fixing computer use on GPT-6 Astra and GPT-5.6. +- **Streaming Memory Leaks (OSS)** - The request context is cancelled on every stream exit path, and empty-thinking-block stripping and Anthropic beta-header gating no longer re-encode or fully decode the body. +- **Streaming Gate Replay Buffer (OSS)** - Full-hold replay buffer size accounting uses cached estimates instead of marshalling every chunk. +- **Bedrock cachePoint Leak (OSS)** - Bedrock `cachePoint` markers are stripped copy-on-write for non-Bedrock providers and kept for Bedrock fallbacks, and the compat plugin no longer mutates the shared request. +- **Bedrock Empty JSON Keys (OSS)** - Tool results containing an empty-string object key are sent as text, so Converse no longer rejects them. +- **Bedrock cache_control on String Content (OSS)** - InvokeModel keeps every `cache_control` when any message's content is a plain string. +- **Web Search Source Names (OSS)** - Responses web search API sources keep their `name` and no longer emit an empty `url`. +- **Grok 4.7 xhigh Reasoning (OSS)** - `xhigh` reasoning effort is no longer downgraded to `high`. +- **OpenAI Chat Stream Framing (OSS)** - Bundled raw finish and usage frames on the OpenAI chat stream passthrough each get their own `data:` prefix. +- **Responses Deep Copy (OSS)** - `DeepCopyResponsesMessage` deep copies cache controls, provider-native parts, tool fields and annotations, so copies no longer share pointers with the original. +- **Request Preparation Performance (OSS)** - Responses requests are decoded once instead of several times, and the compat plugin clones only the fields it writes. + +## 🗄️ Database Migrations + +- No new database migrations in this release. + +## 🐙 Closed OSS Issues + +- [#7223](https://github.com/maximhq/bifrost/issues/7223) - MCP client HTTP transport ignores HTTP_PROXY/HTTPS_PROXY and fails on any deployment behind an egress proxy +- [#7336](https://github.com/maximhq/bifrost/issues/7336) - Bedrock InvokeModel drops message-level cache_control when any historical message content is a JSON string +- [#7356](https://github.com/maximhq/bifrost/issues/7356) - Responses web search API source name is dropped during round-trip +- [#7402](https://github.com/maximhq/bifrost/issues/7402) - Grok 4.7 xhigh reasoning effort is silently downgraded to high +- [#7411](https://github.com/maximhq/bifrost/issues/7411) - Redundant JSON decoding in Responses request preparation +- [#7425](https://github.com/maximhq/bifrost/issues/7425) - Responses: `{"type":"computer"}` is rewritten to `computer_use_preview`, breaking GPT-6 Astra / GPT-5.6 computer use + +## 📀 Base OSS version + +`transports/v2.2.2` (pinned as `github.com/maximhq/bifrost/transports v1.6.12-0.20260923045635-fdeef8e3f31a`), with core `v1.10.1`, framework `v1.7.3`, governance `v1.8.2`, and logging `v1.8.2`. + +## 🔌 If you are compiling plugin against this release - use following deps + +```go +module github.com/maximhq/bifrost-enterprise/transports + +go 1.27.0 + +require ( + github.com/bytedance/sonic v1.15.3-0.20260730064818-2a36d6da63e2 + github.com/coreos/go-oidc/v3 v3.18.0 + github.com/fasthttp/router v1.5.4 + github.com/google/cel-go v0.30.0 + github.com/google/uuid v1.6.0 + github.com/maximhq/bifrost-enterprise/core v0.0.0 + github.com/maximhq/bifrost-enterprise/framework v0.0.0 + github.com/maximhq/bifrost-enterprise/plugins v0.0.0 + github.com/maximhq/bifrost/core v1.10.1 + github.com/maximhq/bifrost/framework v1.7.3 + github.com/maximhq/bifrost/plugins/governance v1.8.2 + github.com/maximhq/bifrost/plugins/logging v1.8.2 + github.com/maximhq/bifrost/plugins/routing v1.1.2 + github.com/maximhq/bifrost/plugins/semanticcache v1.6.5 + github.com/maximhq/bifrost/transports v1.6.12-0.20260923045635-fdeef8e3f31a + github.com/stretchr/testify v1.11.1 + github.com/valyala/fasthttp v1.74.0 + golang.org/x/sync v0.22.0 + gorm.io/driver/sqlite v1.6.0 + gorm.io/gorm v1.31.1 +) + +require ( + cel.dev/expr v0.25.2 // indirect + cloud.google.com/go v0.123.0 // indirect + cloud.google.com/go/auth v0.20.0 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect + cloud.google.com/go/bigquery v1.74.0 // indirect + cloud.google.com/go/compute/metadata v0.9.0 // indirect + cloud.google.com/go/iam v1.7.0 // indirect + cloud.google.com/go/monitoring v1.24.3 // indirect + cloud.google.com/go/pubsub/v2 v2.4.0 // indirect + cloud.google.com/go/secretmanager v1.16.0 // indirect + cloud.google.com/go/storage v1.62.1 // indirect + dario.cat/mergo v1.0.2 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect + github.com/BobuSumisu/aho-corasick v1.0.3 // indirect + github.com/ClickHouse/ch-go v0.65.0 // indirect + github.com/ClickHouse/clickhouse-go/v2 v2.32.0 // indirect + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.77.0 // indirect + github.com/DataDog/datadog-agent/pkg/obfuscate v0.77.0 // indirect + github.com/DataDog/datadog-agent/pkg/opentelemetry-mapping-go/otlp/attributes v0.77.0 // indirect + github.com/DataDog/datadog-agent/pkg/proto v0.77.0 // indirect + github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.77.0 // indirect + github.com/DataDog/datadog-agent/pkg/template v0.77.0 // indirect + github.com/DataDog/datadog-agent/pkg/trace v0.77.0 // indirect + github.com/DataDog/datadog-agent/pkg/trace/log v0.77.0 // indirect + github.com/DataDog/datadog-agent/pkg/trace/otel v0.77.0 // indirect + github.com/DataDog/datadog-agent/pkg/trace/stats v0.77.0 // indirect + github.com/DataDog/datadog-agent/pkg/trace/traceutil v0.77.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/log v0.77.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/scrubber v0.77.0 // indirect + github.com/DataDog/datadog-agent/pkg/version v0.77.0 // indirect + github.com/DataDog/datadog-go/v5 v5.8.3 // indirect + github.com/DataDog/dd-trace-go/v2 v2.8.2 // indirect + github.com/DataDog/go-libddwaf/v4 v4.9.0 // indirect + github.com/DataDog/go-runtime-metrics-internal v0.0.4-0.20260217080614-b0f4edc38a6d // indirect + github.com/DataDog/go-sqllexer v0.1.13 // indirect + github.com/DataDog/go-tuf v1.1.1-0.5.2 // indirect + github.com/DataDog/sketches-go v1.4.8 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.33.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 // indirect + github.com/Masterminds/goutils v1.1.1 // indirect + github.com/Masterminds/semver/v3 v3.4.0 // indirect + github.com/Masterminds/sprig/v3 v3.3.0 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/ProtonMail/go-crypto v1.1.6 // indirect + github.com/STARRY-S/zip v0.2.1 // indirect + github.com/andybalholm/brotli v1.2.2 // indirect + github.com/antlr4-go/antlr/v4 v4.13.1 // indirect + github.com/apache/arrow/go/v15 v15.0.2 // indirect + github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect + github.com/armon/go-metrics v0.4.1 // indirect + github.com/aws/aws-sdk-go-v2 v1.42.0 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect + github.com/aws/aws-sdk-go-v2/config v1.32.14 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.19.14 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.29 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.29 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect + github.com/aws/aws-sdk-go-v2/service/bedrockruntime v1.50.6 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 // indirect + github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0 // indirect + github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.42.3 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 // indirect + github.com/aws/smithy-go v1.27.1 // indirect + github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect + github.com/bahlo/generic-list-go v0.2.0 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/blevesearch/go-porterstemmer v1.0.3 // indirect + github.com/bodgit/plumbing v1.3.0 // indirect + github.com/bodgit/sevenzip v1.6.0 // indirect + github.com/bodgit/windows v1.0.1 // indirect + github.com/buger/jsonparser v1.2.0 // indirect + github.com/bytedance/gopkg v0.1.3 // indirect + github.com/bytedance/sonic/loader v0.5.2 // indirect + github.com/cenkalti/backoff v2.2.1+incompatible // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/cenkalti/backoff/v5 v5.0.3 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/charmbracelet/colorprofile v0.3.1 // indirect + github.com/charmbracelet/lipgloss v1.1.0 // indirect + github.com/charmbracelet/x/ansi v0.10.1 // indirect + github.com/charmbracelet/x/cellbuf v0.0.13 // indirect + github.com/charmbracelet/x/term v0.2.1 // indirect + github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect + github.com/cloudflare/circl v1.6.3 // indirect + github.com/cloudwego/base64x v0.1.6 // indirect + github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect + github.com/coreos/go-semver v0.3.1 // indirect + github.com/coreos/go-systemd/v22 v22.6.0 // indirect + github.com/cyphar/filepath-securejoin v0.6.1 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/ebitengine/purego v0.10.0 // indirect + github.com/emicklei/go-restful/v3 v3.13.0 // indirect + github.com/emirpasic/gods v1.18.1 // indirect + github.com/envoyproxy/go-control-plane/envoy v1.37.0 // indirect + github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect + github.com/fasthttp/websocket v1.5.12 // indirect + github.com/fatih/color v1.18.0 // indirect + github.com/fatih/semgroup v1.2.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/gitleaks/go-gitdiff v0.9.1 // indirect + github.com/go-faster/city v1.0.1 // indirect + github.com/go-faster/errors v0.7.1 // indirect + github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect + github.com/go-git/go-billy/v5 v5.9.0 // indirect + github.com/go-git/go-git/v5 v5.19.2 // indirect + github.com/go-jose/go-jose/v4 v4.1.4 // indirect + github.com/go-logr/logr v1.4.4 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-ole/go-ole v1.3.0 // indirect + github.com/go-openapi/analysis v0.24.2 // indirect + github.com/go-openapi/errors v0.22.5 // indirect + github.com/go-openapi/jsonpointer v0.22.4 // indirect + github.com/go-openapi/jsonreference v0.21.4 // indirect + github.com/go-openapi/loads v0.23.2 // indirect + github.com/go-openapi/runtime v0.29.2 // indirect + github.com/go-openapi/spec v0.22.3 // indirect + github.com/go-openapi/strfmt v0.25.0 // indirect + github.com/go-openapi/swag v0.25.4 // indirect + github.com/go-openapi/swag/cmdutils v0.25.4 // indirect + github.com/go-openapi/swag/conv v0.25.4 // indirect + github.com/go-openapi/swag/fileutils v0.25.4 // indirect + github.com/go-openapi/swag/jsonname v0.25.4 // indirect + github.com/go-openapi/swag/jsonutils v0.25.4 // indirect + github.com/go-openapi/swag/loading v0.25.4 // indirect + github.com/go-openapi/swag/mangling v0.25.4 // indirect + github.com/go-openapi/swag/netutils v0.25.4 // indirect + github.com/go-openapi/swag/stringutils v0.25.4 // indirect + github.com/go-openapi/swag/typeutils v0.25.4 // indirect + github.com/go-openapi/swag/yamlutils v0.25.4 // indirect + github.com/go-openapi/validate v0.25.1 // indirect + github.com/go-viper/mapstructure/v2 v2.5.0 // indirect + github.com/goccy/go-json v0.10.5 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang-jwt/jwt/v5 v5.3.1 // indirect + github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/btree v1.1.3 // indirect + github.com/google/flatbuffers v23.5.26+incompatible // indirect + github.com/google/gnostic-models v0.7.0 // indirect + github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 // indirect + github.com/google/s2a-go v0.1.9 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.16 // indirect + github.com/googleapis/gax-go/v2 v2.22.0 // indirect + github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect + github.com/grandcat/zeroconf v1.0.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect + github.com/h2non/filetype v1.1.3 // indirect + github.com/hashicorp/consul/api v1.34.3 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-hclog v1.6.3 // indirect + github.com/hashicorp/go-immutable-radix v1.3.1 // indirect + github.com/hashicorp/go-metrics v0.5.4 // indirect + github.com/hashicorp/go-msgpack/v2 v2.1.5 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/hashicorp/go-retryablehttp v0.7.8 // indirect + github.com/hashicorp/go-rootcerts v1.0.2 // indirect + github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0 // indirect + github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect + github.com/hashicorp/go-sockaddr v1.0.7 // indirect + github.com/hashicorp/go-version v1.8.0 // indirect + github.com/hashicorp/golang-lru v1.0.2 // indirect + github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect + github.com/hashicorp/hcl v1.0.1-vault-7 // indirect + github.com/hashicorp/memberlist v0.5.4 // indirect + github.com/hashicorp/serf v0.10.1 // indirect + github.com/hashicorp/vault/api v1.23.0 // indirect + github.com/huandu/xstrings v1.5.0 // indirect + github.com/invopop/jsonschema v0.13.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/pgx/v5 v5.9.2 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/jaswdr/faker/v2 v2.8.0 // indirect + github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect + github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jinzhu/now v1.1.5 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/kevinburke/ssh_config v1.2.0 // indirect + github.com/klauspost/compress v1.20.0 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect + github.com/klauspost/pgzip v1.2.6 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect + github.com/linkdata/deadlock v0.5.5 // indirect + github.com/lucasb-eyer/go-colorful v1.2.0 // indirect + github.com/lufia/plan9stats v0.0.0-20260216142805-b3301c5f2a88 // indirect + github.com/magiconair/properties v1.8.10 // indirect + github.com/mailru/easyjson v0.9.1 // indirect + github.com/mark3labs/mcp-go v0.43.2 // indirect + github.com/mattn/go-colorable v0.1.14 // indirect + github.com/mattn/go-isatty v0.0.24 // indirect + github.com/mattn/go-runewidth v0.0.17 // indirect + github.com/mattn/go-sqlite3 v1.14.32 // indirect + github.com/maximhq/bifrost/plugins/compat v0.3.2 // indirect + github.com/maximhq/bifrost/plugins/maxim v1.7.5 // indirect + github.com/maximhq/bifrost/plugins/mocker v1.6.5 // indirect + github.com/maximhq/bifrost/plugins/modelcatalogresolver v1.1.5 // indirect + github.com/maximhq/bifrost/plugins/otel v1.5.5 // indirect + github.com/maximhq/bifrost/plugins/prompts v1.1.5 // indirect + github.com/maximhq/bifrost/plugins/telemetry v1.8.1 // indirect + github.com/maximhq/maxim-go v0.2.1 // indirect + github.com/mholt/archives v0.1.2 // indirect + github.com/miekg/dns v1.1.68 // indirect + github.com/minio/minlz v1.0.0 // indirect + github.com/minio/simdjson-go v0.4.5 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect + github.com/molecule-man/go-brrr v1.0.1 // indirect + github.com/muesli/termenv v0.16.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/nakabonne/tstorage v0.3.6 // indirect + github.com/nwaples/rardecode/v2 v2.2.2 // indirect + github.com/oapi-codegen/runtime v1.1.1 // indirect + github.com/oklog/ulid v1.3.1 // indirect + github.com/outcaste-io/ristretto v0.2.3 // indirect + github.com/paulmach/orb v0.11.1 // indirect + github.com/pelletier/go-toml/v2 v2.2.3 // indirect + github.com/petermattis/goid v0.0.0-20260226131333-17d1149c6ac6 // indirect + github.com/philhofer/fwd v1.2.0 // indirect + github.com/philippgille/chromem-go v0.7.0 // indirect + github.com/pierrec/lz4/v4 v4.1.22 // indirect + github.com/pinecone-io/go-pinecone/v5 v5.3.0 // indirect + github.com/pion/datachannel v1.6.0 // indirect + github.com/pion/dtls/v3 v3.1.5 // indirect + github.com/pion/ice/v4 v4.2.1 // indirect + github.com/pion/interceptor v0.1.44 // indirect + github.com/pion/logging v0.2.4 // indirect + github.com/pion/mdns/v2 v2.1.0 // indirect + github.com/pion/randutil v0.1.0 // indirect + github.com/pion/rtcp v1.2.16 // indirect + github.com/pion/rtp v1.10.1 // indirect + github.com/pion/sctp v1.9.2 // indirect + github.com/pion/sdp/v3 v3.0.18 // indirect + github.com/pion/srtp/v3 v3.0.10 // indirect + github.com/pion/stun/v3 v3.1.6 // indirect + github.com/pion/transport/v4 v4.0.2 // indirect + github.com/pion/turn/v4 v4.1.4 // indirect + github.com/pion/webrtc/v4 v4.2.9 // indirect + github.com/pjbgf/sha1cd v0.6.0 // indirect + github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect + github.com/prometheus/client_golang v1.23.2 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.67.5 // indirect + github.com/prometheus/procfs v0.19.2 // indirect + github.com/puzpuzpuz/xsync/v3 v3.5.1 // indirect + github.com/qdrant/go-client v1.16.2 // indirect + github.com/redis/go-redis/v9 v9.17.2 // indirect + github.com/rivo/uniseg v0.4.7 // indirect + github.com/rs/zerolog v1.34.0 // indirect + github.com/ryanuber/go-glob v1.0.0 // indirect + github.com/sagikazarmark/locafero v0.7.0 // indirect + github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect + github.com/savsgio/gotils v0.0.0-20250408102913-196191ec6287 // indirect + github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect + github.com/secure-systems-lab/go-securesystemslib v0.10.0 // indirect + github.com/segmentio/asm v1.2.0 // indirect + github.com/segmentio/kafka-go v0.4.51 // indirect + github.com/sergi/go-diff v1.4.0 // indirect + github.com/shirou/gopsutil/v4 v4.26.3 // indirect + github.com/shopspring/decimal v1.4.0 // indirect + github.com/skeema/knownhosts v1.3.1 // indirect + github.com/sorairolake/lzip-go v0.3.5 // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spf13/afero v1.15.0 // indirect + github.com/spf13/cast v1.10.0 // indirect + github.com/spf13/pflag v1.0.10 // indirect + github.com/spf13/viper v1.19.0 // indirect + github.com/spiffe/go-spiffe/v2 v2.7.0 // indirect + github.com/stretchr/objx v0.5.3 // indirect + github.com/subosito/gotenv v1.6.0 // indirect + github.com/tetratelabs/wazero v1.11.0 // indirect + github.com/therootcompany/xz v1.0.1 // indirect + github.com/tidwall/gjson v1.18.0 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.1 // indirect + github.com/tidwall/sjson v1.2.5 // indirect + github.com/tinylib/msgp v1.6.3 // indirect + github.com/tklauser/go-sysconf v0.3.16 // indirect + github.com/tklauser/numcpus v0.11.0 // indirect + github.com/trailofbits/go-mutexasserts v0.0.0-20250514102930-c1f3d2e37561 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/ulikunitz/xz v0.5.15 // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect + github.com/wasilibs/go-re2 v1.9.0 // indirect + github.com/wasilibs/wazero-helpers v0.0.0-20240620070341-3dff1577cd52 // indirect + github.com/weaviate/weaviate v1.38.0 // indirect + github.com/weaviate/weaviate-go-client/v5 v5.7.1 // indirect + github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect + github.com/wlynxg/anet v0.0.5 // indirect + github.com/x448/float16 v0.8.4 // indirect + github.com/xanzy/ssh-agent v0.3.3 // indirect + github.com/xdg-go/pbkdf2 v1.0.0 // indirect + github.com/xdg-go/scram v1.1.2 // indirect + github.com/xdg-go/stringprep v1.0.4 // indirect + github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect + github.com/yosida95/uritemplate/v3 v3.0.2 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect + github.com/zeebo/xxh3 v1.1.0 // indirect + github.com/zricethezav/gitleaks/v8 v8.30.1 // indirect + go.etcd.io/etcd/api/v3 v3.6.11 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.6.11 // indirect + go.etcd.io/etcd/client/v3 v3.6.11 // indirect + go.mongodb.org/mongo-driver v1.17.7 // indirect + go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/collector/component v1.51.1-0.20260205185216-81bc641f26c0 // indirect + go.opentelemetry.io/collector/featuregate v1.51.1-0.20260205185216-81bc641f26c0 // indirect + go.opentelemetry.io/collector/pdata v1.51.1-0.20260205185216-81bc641f26c0 // indirect + go.opentelemetry.io/collector/pdata/pprofile v0.145.1-0.20260205185216-81bc641f26c0 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.44.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect + go.opentelemetry.io/otel v1.45.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.44.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.44.0 // indirect + go.opentelemetry.io/otel/metric v1.45.0 // indirect + go.opentelemetry.io/otel/sdk v1.45.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.45.0 // indirect + go.opentelemetry.io/otel/trace v1.45.0 // indirect + go.opentelemetry.io/proto/otlp v1.10.0 // indirect + go.starlark.net v0.0.0-20260102030733-3fee463870c9 // indirect + go.uber.org/atomic v1.11.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.1 // indirect + go.yaml.in/yaml/v2 v2.4.3 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + go4.org v0.0.0-20230225012048-214862532bf5 // indirect + golang.org/x/arch v0.23.0 // indirect + golang.org/x/crypto v0.55.0 // indirect + golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect + golang.org/x/mod v0.38.0 // indirect + golang.org/x/net v0.58.0 // indirect + golang.org/x/oauth2 v0.36.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/telemetry v0.0.0-20260708182218-49f421fb7959 // indirect + golang.org/x/term v0.45.0 // indirect + golang.org/x/text v0.41.0 // indirect + golang.org/x/time v0.15.0 // indirect + golang.org/x/tools v0.48.0 // indirect + golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect + google.golang.org/api v0.282.0 // indirect + google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect + google.golang.org/grpc v1.83.2 // indirect + google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect + gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/ini.v1 v1.67.1 // indirect + gopkg.in/warnings.v0 v0.1.2 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + gorm.io/driver/clickhouse v0.7.0 // indirect + gorm.io/driver/postgres v1.6.0 // indirect + k8s.io/api v0.36.1 // indirect + k8s.io/apimachinery v0.36.1 // indirect + k8s.io/client-go v0.36.1 // indirect + k8s.io/klog/v2 v2.140.0 // indirect + k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a // indirect + k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 // indirect + sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.2 // indirect + sigs.k8s.io/yaml v1.6.0 // indirect +) + +replace github.com/maximhq/bifrost-enterprise/core => ../core + +replace github.com/maximhq/bifrost-enterprise/framework => ../framework + +replace github.com/maximhq/bifrost-enterprise/plugins => ../plugins +``` + + diff --git a/docs/changelogs/v2.2.1.mdx b/docs/changelogs/v2.2.1.mdx new file mode 100644 index 00000000000..3560ee2c778 --- /dev/null +++ b/docs/changelogs/v2.2.1.mdx @@ -0,0 +1,220 @@ +--- +title: "v2.2.1" +description: "v2.2.1 changelog - 2026-09-18" +--- + + + ```bash + npx -y @maximhq/bifrost --transport-version v2.2.1 + ``` + + + ```bash + docker pull maximhq/bifrost:v2.2.1 + docker run -p 8080:8080 maximhq/bifrost:v2.2.1 + ``` + + + + +## ✨ Features + +- **Virtual Key Assignees and Expanded Search** - The virtual key list resolves `assigned_user` for every row via a single batch lookup, and VK search matches team, customer and user names in addition to key fields, so keys can be found by who owns them (#7225, #7226) +- **User Attribution in Prometheus Metrics** - The telemetry plugin's metrics export includes user id and user name labels, so per-user usage and error rates can be graphed and alerted on directly (#7267) +- **Prompt Cache Breakpoints Capability** - A `SupportsPromptCacheBreakpoints` model cap with a name-based fallback for OpenRouter gates `prompt_cache_breakpoint` forwarding, so OpenRouter's non-Claude models stop rejecting requests that carry it (#7261, #7262) +- **MCP Observed Latency** - Native (observed) MCP tool-call latency from `observed_latency_ms` is surfaced in the logs duration column and detail sheet, kept clearly distinct from true execution time and never fabricating a synthetic start timestamp on the timeline (#7271) + +## 🐞 Fixed + +- **Resource ID Path Injection** - Caller-supplied resource IDs (batch, file, video, container, response IDs, cached content names) were interpolated into outbound URL paths unvalidated; a crafted ID with `../`, `?`, `#`, percent-encoded bytes or control characters could redirect the request to an unintended upstream endpoint. A central `EscapeResourceID` helper validates and escapes every such ID (#7307, #7310) +- **Bedrock S3 SSRF** - A caller-supplied `s3://` file ID or `s3_bucket` param could control the upstream host, since S3 virtual-hosted URLs are built as `https://{bucket}.{s3host}/{key}`; a bucket like `127.0.0.1:PORT#` opened a SigV4-signed TLS connection to a caller-chosen host. Bucket names are now validated with a DNS-compatible regex (#7315) +- **Caller-Forged Billing Idempotency** - The billing-idempotency key was `(RequestID, AttemptNumber)`, and `RequestID` may come from the caller's `x-request-id` header, so two unrelated requests sharing a deliberately chosen ID collided and the second was silently never charged. An internally minted `BillingNonce` is mixed into the key, making it unforgeable (#7303) +- **Abandoned Non-Streaming Request Hung Forever** - The worker kept a `ctx.Done()` arm on an already-claimed delivery send, a regression from the #6972 delivery fix, so a non-streaming caller could hang indefinitely (#7313) +- **Repeated Empty Thinking Blocks** - Streaming chat chunks emitted empty reasoning/message fields on every content delta, which clients rendered as repeated empty thinking blocks (#7318) +- **Claude Code Thread Continuations Failed Behind Key Rotation** - Claude Code's server-side conversation threads are bound to the upstream account that creates them, and Bifrost's per-request key selection, retries and fallbacks cannot keep a continuation on that account, so `thread: {"type": "continue"}` requests failed with `thread_not_found` (about half the time on a two-key config). The Anthropic integration now declares itself stateless: continuations are refused before the provider call with a 400 whose `details.error_code` is `thread_unsupported_request`, which makes the client resend the turn in full and stop sending the thread field for the rest of the session, and the provider raw-body path strips `thread` from create requests so no orphaned thread state accumulates upstream. Token counting is never refused (#7274) +- **Gemini Flash-Lite Minimal Thinking Promoted to Low** - Normalized `gemini-3.1-flash-lite` requests silently promoted `minimal` thinking to `low` on both Gemini and Vertex; the text model's four supported levels are now registered, preserving the image variant's separate restrictions (#7288) (thanks [@Javtor](https://github.com/Javtor)!) +- **Config.json Virtual Key Limits Broke Under UI Edits** - VK rate limits and budgets created via the config.json standalone-limits flow and then edited through the UI produced duplicate, conflicting ownership records (standalone budgets owned directly by the VK, plus orphaned UUID rate-limit rows created by the UI). Migration `migrate_vk_standalone_limits_to_model_configs` consolidates ownership into VK-scoped model configs while preserving usage counters, and the write paths stop creating the divergent rows (#7291) +- **Bedrock Service Tier Rejections** - Service tier forwarding for Bedrock (Converse and Mantle paths) sent whatever tier was requested; it is now gated on explicit model capability metadata, failing closed when none exists, so models that do not support the requested tier stop rejecting the request (#7266) +- **Anthropic Root-Level Tool Schema Compositions** - Anthropic rejects `oneOf`/`anyOf`/`allOf` at the root of a tool's `input_schema` but accepts them inside properties; root-level compositions are now rewritten into a flat object schema before dispatch, unblocking tools like Codex's `automation_update` (#7265) +- **GenAI Streaming TTS** - Speech stream chunks routed through the `/genai` integration had no converter registered; `ToGeminiSpeechStreamResponse` now serves streaming TTS, and the streaming router returns a clean error instead of panicking when any stream converter is missing (#7248) +- **xAI Usage and Cost** - xAI reports visible and reasoning completion tokens separately; `completion_tokens` now folds reasoning in so `prompt_tokens + completion_tokens = total_tokens` holds, and streaming cost normalization preserves xAI's authoritative `cost_in_usd_ticks` to the terminal usage chunk instead of falling back to catalog pricing (#7245, #7250) +- **OpenAI-Only `search_content_types` Forwarded Everywhere** - `search_content_types` on `web_search` tools is an OpenAI-specific extension; it is now gated behind a per-provider capability check so Bedrock and other OpenAI-compatible backends receive a clean `web_search` tool without the field (#7244) +- **Claude Code `diagnostics` Field Rejected by Non-Native Providers** - Claude Code sends `diagnostics.previous_message_id` on every request; on the typed-sanitizer path Bedrock, Vertex and Azure returned 400 `diagnostics: Extra inputs are not permitted`. The field is stripped for providers that do not support it (#7243) +- **Unreadable Bedrock Error Logs** - AWS returns errors in a flat `{"message": ..., "__type": ...}` shape the shared Anthropic/OpenAI parsers never looked at, so Bedrock errors were logged with no human-readable reason; the root-level message now seeds `BifrostError`, and the logs UI falls back to showing the raw provider error body when no message could be extracted (#7221, #7222) +- **Anthropic `container` Param Dropped** - The string-form `container` param on `/anthropic/v1/messages` was silently dropped, so container reuse provisioned a fresh container every time; it is now carried through the round trip (#5829) (thanks [@AdityaPainuli](https://github.com/AdityaPainuli)!) +- **Responses API Finish Reason Missing From OTEL Spans** - `gen_ai.response.finish_reason` / `finish_reasons` are now emitted on the `llm.call` span for `/v1/responses` requests, so refusals are visible to OTEL consumers, matching the chat path (#7205) (thanks [@bkfl-notai](https://github.com/bkfl-notai)!) +- **Cohere Fallback Response Shape** - The Cohere-compatible route returned the raw Bifrost normalized response when a fallback served the request from a non-Cohere provider, which the Cohere SDK failed to parse; responses are now converted to the Cohere v2 shape (`message`, `finish_reason`) (#7198) +- **Skill Serving Race** - Fixed a race condition in the skill serving handler +- **Log and Dashboard Label Truncation** - Long model and provider labels in the logs table and dashboard charts truncate from the start so the distinctive suffix stays visible, the logs model column is wider, the search field border and icon spacing are cleaned up, and key picker options are keyed by id so duplicate labels stop highlighting together (#7215, #7216, #7217, #7269, #7297) + +## 🗄️ Database Migrations + +- **migrate_vk_standalone_limits_to_model_configs** - Consolidates virtual key standalone budgets and rate limits into VK-scoped model configs: standalone budgets are re-pointed to the VK's top-level model config (created if missing, usage preserved), orphaned duplicate UUID rate-limit rows created by the UI are deleted with the model config re-pointed to the canonical config.json row, and `vk.rate_limit_id` is cleared. + + +## 🐙 Closed GitHub Issues + +- [#123](https://github.com/maximhq/bifrost/issues/123) - Files API Support +- [#5707](https://github.com/maximhq/bifrost/issues/5707) - string-form `container` param on /anthropic/v1/messages is silently dropped - container reuse provisions a fresh container +- [#7204](https://github.com/maximhq/bifrost/issues/7204) - Responses API path never sets gen_ai.response.finish_reason in OTEL traces (refusals invisible to observability) +- [#7287](https://github.com/maximhq/bifrost/issues/7287) - Gemini 3.1 Flash-Lite minimal thinking is silently promoted to low +- [#7294](https://github.com/maximhq/bifrost/issues/7294) - Streaming chat chunks emit empty reasoning/message fields on every content delta, causing repeated thinking blocks +- [#7308](https://github.com/maximhq/bifrost/issues/7308) - non-streaming caller hangs forever - worker keeps a ctx.Done() arm on a claimed delivery send (regression from #6972) + + + +- fix: `POST /v1/realtime/client_secrets` mints a Bifrost-issued ephemeral token instead of returning the provider's token (e.g. OpenAI `ek_...`) verbatim, which let a client connect to the provider directly and bypass governance, metering and attribution; virtual key credentials are validated before a realtime session is admitted, so a nonexistent VK can no longer open an upstream provider session (#7317) +- fix: streaming chat chunks no longer emit empty reasoning/message fields on every content delta, which made clients render repeated empty thinking blocks (#7318) +- fix: an abandoned non-streaming request no longer hangs its caller forever; the worker kept a `ctx.Done()` arm on an already-claimed delivery send, a regression from the #6972 delivery fix (#7313) +- fix: caller-supplied `s3://` file IDs and `s3_bucket` params on the Bedrock provider are validated with a DNS-compatible bucket regex before URL construction, closing an SSRF where a bucket like `127.0.0.1:PORT#` steered the SigV4-signed request to a caller-chosen host (#7315) +- fix: resource IDs (batch, file, video, container, response IDs and cached content names) are validated and escaped through a central `EscapeResourceID` helper before being interpolated into outbound URL paths, so IDs carrying `../`, `?`, `#`, percent-encoded bytes or control characters can no longer redirect requests to unintended upstream endpoints (#7307) +- fix: an internally minted `BillingNonce` is mixed into the billing-idempotency key, which was previously `(RequestID, AttemptNumber)` alone; since `x-request-id` is caller-supplied, two unrelated requests sharing a forged ID collided on the key and the second was never charged (#7303) +- fix: Bedrock service tier forwarding is gated on explicit model capability metadata (failing closed when none exists) on both Converse and Mantle paths, instead of sending whatever tier was requested and getting rejected by models that do not support it (#7266) +- fix: root-level `oneOf`/`anyOf`/`allOf` in a custom tool's `input_schema` are normalized into a flat object schema before reaching Anthropic, which rejects composition keywords at the schema root but accepts them inside properties; unblocks tools like Codex's `automation_update` (#7265) +- fix: speech stream chunks have a GenAI converter (`ToGeminiSpeechStreamResponse`), enabling streaming TTS through the `/genai` integration, and the streaming router returns a clean error instead of panicking when a stream converter is missing (#7248) +- feat: `SupportsPromptCacheBreakpoints` model cap with a name-based fallback for OpenRouter, so `prompt_cache_breakpoint` is only forwarded to models that accept it (#7261, #7262) +- fix: xAI `completion_tokens` folds `reasoning_tokens` into the total so `prompt_tokens + completion_tokens = total_tokens` holds, and streaming cost normalization preserves xAI's authoritative `cost_in_usd_ticks` through to the terminal usage chunk instead of falling back to catalog pricing (#7245, #7250) +- fix: `search_content_types` on `web_search` tools is an OpenAI-specific extension and is now gated behind a per-provider capability check, so Bedrock and other OpenAI-compatible backends receive a clean `web_search` tool without the field (#7244) +- fix: the `diagnostics` field Claude Code sends on every request is stripped for providers that reject it (Bedrock, Vertex, Azure return 400 `diagnostics: Extra inputs are not permitted` on the typed sanitizer path) (#7243) +- fix: AWS Bedrock's flat error shape `{"message": ..., "__type": ...}` seeds the root-level `message` into `BifrostError`, so Bedrock errors served through the shared Anthropic/OpenAI handlers are logged with a human-readable reason (#7222) +- [fix]: the string-form `container` param is carried through the /v1/messages round trip instead of being silently dropped, so container reuse stops provisioning a fresh container (#5829) (thanks [@AdityaPainuli](https://github.com/AdityaPainuli)!) +- fix: the Cohere-compatible route converts fallback responses from non-Cohere providers into the Cohere v2 shape (`message`, `finish_reason`) via `ToCohereChatResponse`, instead of returning the raw Bifrost normalized response the Cohere SDK cannot parse (#7198) +- fix: clearer error message in the Anthropic integration (#7264) +- [fix]: preserve Gemini 3.1 Flash-Lite minimal thinking on Gemini and Vertex [@Javtor](https://github.com/Javtor) +- fix: the Anthropic raw-body request path strips the server-side `thread` field before the wire on both messages and count_tokens (thread state is bound to the creating account and cannot survive Bifrost's per-request key selection, retries, or fallbacks; the HTTP transport refuses continuations outright), `AnthropicMessageErrorStruct` gains an optional `details` object with `error_code` so error envelopes can carry the machine-readable codes clients key their recovery on, and `LargePayloadMetadata` gains `ThreadType` so the refusal also works when large-payload mode skips body parsing (populated by the enterprise metadata extractor) +- [fix]: MCP tool schemas keep one property order across tool syncs [@dougcalobrisi](https://github.com/dougcalobrisi) +- feat: `WhiteList` and `BlackList` entries that start with `regex:` are RE2 patterns. `MatchEntry` compiles them once as a case-insensitive full match, `Validate` refuses a pattern that is empty, `*` or does not compile, and the list-models pipeline never surfaces a pattern as a model. Plain entries keep their exact, case-insensitive match +- fix: fasthttp-level stale-connection retries no longer multiply `max_retries`. `contextTransport.RoundTrip` reports a failure before response headers on a freshly dialed socket with `retry=false`, so `network.StaleConnectionRetryIfErr` only walks past pooled keep-alive sockets the upstream closed while idle; an upstream that closes a fresh connection without answering now costs exactly one attempt of Bifrost's own retry budget instead of up to four. The retry backoff also ends as soon as the request context is cancelled, so a worker is freed immediately instead of after up to `retry_backoff_max` (#7035) +- [fix]: Gemini provider - preserve inline image and audio data in chat completions responses [@Atharva-Kanherkar](https://github.com/Atharva-Kanherkar) +- [fix]: route opencode-zen Responses calls through /v1/chat/completions, which its upstream does not serve on /v1/responses [@miguelchico](https://github.com/miguelchico) +- feat: the namespace-tool names a provider reserves for its own server tools (`web`, `image_gen`, `browser`, `python` on Bedrock and Bedrock Mantle) are now read from the datasheet row `reserved_tool_namespaces` (new `ModelCapabilities` field and `ModelCaps.ReservedToolNamespaces`) for the base provider and canonical model; a non-empty row replaces the hardcoded list, an absent or empty row keeps it, so any OpenAI-wire provider can reserve names through a row without a code change +- fix: Codex >= 0.147 wraps its default tools in a namespace literally named `functions` (openai/codex#37022), which Bedrock Mantle reserves and rejects with `User-defined namespace 'functions' collides with an existing tool namespace`. Core now unwraps that namespace to top-level tools, unprefixed, for every provider before dispatch; Codex treats the bare name and the explicit `functions` namespace as the same tool, so no mapping back is needed; a description on the `functions` namespace is prepended to each hoisted member, as flattening does +- fix: Responses `namespace` tools are now flattened in core for every provider whose wire does not support the type (Anthropic, Gemini, Vertex, Bedrock Converse, DeepSeek, and every OpenAI-compatible third party), with nested functions renamed to `__` so two namespaces that share a function name no longer collide into an upstream `Tool names must be unique` 400; returned `function_call` items are mapped back to the bare `name` plus `namespace` (unary and streaming), prior-turn calls carrying `namespace` and `tool_choice` names are re-aliased to match, and a name that is still duplicated after flattening or a `tool_choice` that matches several namespaces is rejected with a clear 400 before reaching the provider. Bedrock answers namespace support from its own surface resolver via the new optional `ResponsesNamespaceToolProvider` interface. A datasheet row `supports_namespace_tools` (new `ModelCapabilities` field and `ModelCaps.SupportsNamespaceTools`) overrides the per-provider default for a (provider, model) pair; with no row the default applies. A row can only narrow within what the wire can carry: the Anthropic Messages API (Anthropic, Claude on Azure or Bedrock Mantle), the Gemini API (Gemini, Vertex) and Bedrock Converse have no namespace container, so they answer false regardless of the row and always flatten. Flattened names honour the target wire's documented tool-name limit, 64 characters of `[A-Za-z0-9_-]` for OpenAI-compatible wires, Bedrock Converse and Fireworks, 128 for Anthropic, and 128 with `.` and `:` allowed for Gemini and Vertex, overridable per model through the datasheet row `tool_name_max_length` (a row below 10 cannot hold the hashed form and is ignored); a longer alias becomes an 8-hex hash prefix plus the function name, deterministically, so history and `tool_choice` re-alias to the same string. The alias map travels on the prepared request (`BifrostResponsesRequest.NamespaceToolAliases`) and is applied to that attempt's response, unary and streaming; nothing is kept on the request context or in process-wide state (#7048) +- fix: drop namespace tools whose name Amazon Bedrock reserves (`web`, `image_gen`, `browser`, `python`) on the Bedrock and Bedrock Mantle Responses paths instead of forwarding them into a `tools.namespace` collision 400; a dropped Codex `web` namespace becomes the hosted `web_search` tool on Bedrock Mantle +- fix: Bedrock Mantle chat streaming no longer drops the usage-only chunk that arrives after `finish_reason`; `ProviderSendsDoneMarker` now treats `bedrock_mantle` (and the legacy Mantle route under the `bedrock` key) as sending `[DONE]`, so streamed usage and cost are recorded (#7065) +- fix: `default_request_timeout_in_seconds` now bounds the wait for response headers on streaming requests, and cancelling a request now closes the upstream socket. Every fasthttp client is driven through a Bifrost-owned `RoundTripper` (`core/providers/utils/roundtripper.go`) that applies the client's read/write timeouts and the request context to the request write and the header wait, then lifts the socket deadline once headers are parsed so `stream_idle_timeout_in_seconds` remains the only bound on the body. `BuildStreamingClient` and `BuildLargeResponseClient` no longer zero the timeouts. An upstream that accepts the connection and never answers now fails with 504 `RequestTimedOut` after the request timeout and the request's fallbacks are used, instead of pinning the provider worker until the upstream closes the socket; `MakeRequestWithContext` no longer leaves a background `client.Do` running until `ReadTimeout` after its context is cancelled (#7034) +- fix: unary large-response downloads (`FinalizeResponseWithLargeDetection`) now bound every body read with `stream_idle_timeout_in_seconds` and close the upstream socket on request cancellation, mirroring the streamed passthrough path; a stall during the prefetch surfaces as 504 `RequestTimedOut` instead of pinning the provider worker, and a stall while the transport drains the `LargeResponseReader` fails the read with the idle-timeout error instead of blocking the writer indefinitely (#7104) +- fix: gzip-encoded unary responses are now classified for large-response mode by decompressed size; `Content-Length` describes the compressed bytes, so a body that was small on the wire but large once decompressed used to be materialized in full past `large_response_threshold` (#7104) +- fix: an OpenAI-compatible upstream that omits `[DONE]` and then goes silent after `finish_reason` no longer fails the stream when `stream_idle_timeout_in_seconds` fires. The chat and text completion read loops now treat an idle timeout after a terminal signal as a parked upstream, mark the stream parked so the connection is abandoned rather than drained, and synthesize the final chunk with the buffered `finish_reason`; a stall before `finish_reason` still surfaces as the idle-timeout error (#7108) +- fix: a streamed upstream that drops the connection mid-body is again reported as the retryable 502 `provider closed the stream before sending a completion marker` error instead of a generic `Error reading stream: unexpected EOF`. The Bifrost round tripper's chunked decoder surfaced the drop as `io.ErrUnexpectedEOF`, which no provider read loop treats as end of stream; it now reports the plain `io.EOF` fasthttp always did and discards the half-read connection (#7104 follow-up) +- fix: fallbacks for image edit, image variation and video edit requests now reach the configured fallback provider and model. `prepareFallbackRequest` had no arm for those three types, so the shallow request copy kept the primary's sub-request pointer and the "fallback" attempt was routed back to the primary while `RoutingInfo`, the `x-bifrost-routing-info-*` headers, the log row and the `fallback_index` metric label reported it as a fallback. The helper now also verifies the prepared request targets the fallback provider and model and skips the fallback with a warning otherwise, so a future request type added without an arm fails loudly instead of silently re-running the primary (#6966) +- fix: route Bedrock Claude requests that carry a `compact_20260112` edit to InvokeModel / InvokeModelWithResponseStream with the native Anthropic Messages body, so server-side compaction works on `bedrock/` models, including keys that pin an inference-profile ARN. AWS documents compaction as unsupported on the Converse API, which previously received the edit and silently ignored it (#6825) +- feat: serve Anthropic tool search (`tool_search_tool_*`, `defer_loading`) on `bedrock/` Claude models by routing those requests to InvokeModel / InvokeModelWithResponseStream, the only Bedrock API AWS allows it on; CountTokens counts such requests with the same InvokeModel body via the `invokeModel` input +- fix: a `reasoning.context` value the target model does not accept is now dropped on the OpenAI and Azure Responses path, so `all_turns` on the original gpt-5 family incl. gpt-5-pro, gpt-5.1 to gpt-5.3, and the o-series runs under the model's own `current_turn` default instead of failing with `Unsupported value: 'all_turns' is not supported with the 'gpt-5-pro' model`; gpt-5.4, gpt-5.5 and gpt-5.6 keep it. The accepted values come from the datasheet row `supported_reasoning_contexts` (new `ModelCapabilities` field and `ModelCaps.SupportedReasoningContexts`), falling back to a name-based default of `auto` and `current_turn` everywhere plus `all_turns` on gpt-5.4+ +- fix: mid-conversation `role: "system"` messages are now inlined in place as `` user turns for every model family on every converter that has a top-level system field: Bedrock Converse (Responses and Chat Completions), Gemini Chat Completions (`systemInstruction`), and the Anthropic wire shape used by DeepSeek, Fireworks and SGL. Previously only Claude on Bedrock/Anthropic inlined; everything else hoisted each reminder into the top-level system block. Claude Code appends a trailing `` system reminder after every turn, so the hoisted block grew the front of the prompt each turn and prefix-based caches (Bedrock implicit caching for `global.openai.gpt-5.6-luna` on Converse, Gemini implicit caching, DeepSeek context caching) reported a full cache write and zero cache reads on every turn +- [fix]: vLLM key selection resolves the requested alias through `key.Aliases` per key before comparing it with `vllm_key_config.model_name`, so a valid key is no longer rejected when the request used an alias; the same public alias can map to different physical model IDs across vLLM instances, and allow and block checks keep matching the original user-facing alias [@Constantine3](https://github.com/Constantine3) +- [fix]: Gemini Responses requests keep `top_k`, `frequency_penalty`, `presence_penalty`, `stop_sequences` and `media_resolution` across retries and fallbacks. `convertParamsToGenerationConfigResponses` deleted those keys from `ExtraParams` while mapping them into `generationConfig`, and the conversion runs once per attempt on the same request, so every attempt after the first reached the provider without them; the outbound `ExtraParams` is now built through a filtered copy instead of mutating the request [@VictorRequenaMaisa](https://github.com/VictorRequenaMaisa) +- feat: `wait_for_usage` on `custom_provider_config` keeps the OpenAI-compatible chat and text completion read loops open past `finish_reason` until the trailing usage-only frame arrives, so a custom provider that also sets `does_not_send_done_marker` no longer synthesizes a zero-usage terminal chunk and records the request at zero tokens and zero cost. `WaitForUsage` is carried on `BifrostContextKeyWaitForUsage` with the same set-or-clear discipline as `DoesNotSendDoneMarker`, so it never leaks onto a fallback provider that did not declare it, and termination stays bounded by the usage chunk, two consecutive post-finish heartbeat comments, EOF, or `stream_idle_timeout_in_seconds` (#7143) +- fix: `ExtraFields.RawResponse` now carries role-only, finish-only and usage-only SSE frames on the OpenAI-compatible chat stream. Those frames never entered the chunk-forwarding branch, so their bytes were discarded from the reconstructed `raw_response`, which made the captured audit trail irreconcilable against a provider invoice because the usage frame carries the token counts Bifrost bills from. A `pendingRawFrames` buffer accumulates them and drains onto the next forwarded chunk or the synthetic terminal chunk; the Responses-over-Chat fallback no longer stamps one upstream frame onto every derived event, and `delta.refusal` and `delta.annotations` are added to the forwarding predicate so refusals and streamed URL citations are no longer dropped entirely (#7144) +- fix: Anthropic server-side tool search survives the Bedrock-native invoke ingress end to end. `tool_search_tool_*` was dropped during the Converse-shaped intermediate conversion, leaving the egress predicate blind to the tool and routing the request to Converse where server-side tool search is unavailable; the tool is now carried as an ingress-only marker (`json:"-"`, never reaching a Converse body) and the neutral tool is rebuilt in `ToBifrostResponsesRequest` along with `defer_loading`. Results are returned as a `server_tool_use` plus `tool_search_tool_result` pair rather than a client `tool_use`, which Anthropic rejects when the caller echoes a `tool_result` for the `srvtoolu_` ID on the next turn, on both the unary and streaming paths; `BedrockContentBlock.UnmarshalJSON` gained `server_tool_use` and `tool_search_tool_result` cases so replayed turns round-trip unchanged instead of falling through to empty structs; and the Anthropic-native response path carries the same blocks (#7155) +- fix: native Gemini `generateContent` image requests routed through `/genai` no longer return HTTP 200 while silently ignoring part of the request. `isImageEditRequest` only inspected `contents[0].parts[0]`, so a request with the prompt text before the `inlineData` part, the ordering in Google's own REST edit sample, was misclassified as image generation: the image never reached Vertex and the model invented a picture from the prompt alone. Detection now scans every part across every content. `imageConfig.aspectRatio` is preserved as a typed `aspect_ratio` param that both outbound converters prefer, instead of being folded into a `WxH` size string that collapsed any ratio outside `1:1`, `3:4`, `4:3`, `9:16` and `16:9` back to square (#7173) +- fix: Gemini `Part` gained `mediaResolution`, the per-part override for `generationConfig.mediaResolution`. `Part.UnmarshalJSON` decodes into a closed alias, so the key was discarded before any conversion ran and per-part image and PDF tokenization fell back to the model default; an `ULTRA_HIGH` image billed about 21k prompt tokens through `/genai` instead of about 22.1k direct. Both the `mediaResolution` and `media_resolution` spellings are parsed, the value round-trips through `ResponsesMessageContentBlock` and is reconstructed on the outbound part, and it is stripped on the OpenAI wire path where the field is unknown (#7156) +- fix: image blocks inside a Bedrock Converse `toolResult` are hoisted out and re-inserted immediately after the last tool result in the same message, with a placeholder text block left behind so the now image-free result is not rejected for being empty. Some Bedrock-hosted models, notably the OpenAI and Grok families, reject nested tool-result images on Converse even though they accept images in tool output via the Responses API. `BedrockModelSupportsToolResultImages` supplies the name-based default and the datasheet field `supports_converse_tool_result_images` (new `ModelCapabilities` field and `ModelCaps.SupportsConverseToolResultImages`) overrides it per row (#7150) +- feat: `bifrost_error_requests_total` gains a normalized `error_type` label drawn from a closed, prefix-structured vocabulary (`caller_*`, `policy_*`, `provider_*`, `bifrost_*`, `_OTHER`), so a 429 from a governance rate limit is distinguishable from a 429 from an upstream provider, a 403 from a blocked model from one from a bad key, and a 503 Bifrost shed under queue pressure from an upstream overload. `ClassifyErrorType` resolves a declared `ExtraFields.ErrorType` first, then Bifrost's own string markers, then the upstream status code; it deliberately ignores the provider's own `error.type` and `error.code` strings, which disagree across providers for the same condition. Values are prefixed by fault domain so an alarm expression can match a family with a single regex (#7141) +- fix: Azure Foundry deployments of Fireworks-hosted models are no longer silently capped at 4096 output tokens on `/openai/v1/responses`, where Microsoft routes those models through chat completions internally. The Azure provider checks the model's datasheet `supported_endpoints` and, when `/v1/responses` is absent, transparently serves both `Responses` and `ResponsesStream` through `/openai/v1/chat/completions` and converts the result back to the Responses shape; a model with an explicit `/v1/responses` entry or no datasheet row at all is unchanged. Separately, `ToAnthropicResponsesResponse` now follows `StopReason > IncompleteDetails > tool_use inference > end_turn`, so a turn truncated by the output-token cap on any OpenAI-shaped Responses provider reports `stop_reason: max_tokens` instead of hiding the truncation as `end_turn` (#6782) +- fix: DeepSeek chat requests move `max_completion_tokens` into `max_tokens` on the wire. DeepSeek's chat-completions endpoint only recognizes the legacy field and silently ignores `max_completion_tokens`, so the limit had no effect; this matches the behaviour already in place for Opencode and Ollama (#7131) +- feat: guardrail redaction gains an identity-based transformation path for provider-native request and response bodies, so exact provider-managed rewrites such as Model Armor and Bedrock land in the correct native JSON field even when the same text value appears in several fields. `TextTargetID`, `TextTargetIDForIndex` and `TextRewrite` identify each guardrail-visible field by a stable ID rather than by content value; `RawRequestBodyTextTransformer` and `RawResponseTextTransformer` let integrations register handlers on the request context; and `rewriteRawJSONTextTargets` validates target uniqueness, verifies each `Original` value before patching and re-reads the written value after, so stale or mismatched normalized text cannot silently rewrite the wrong field (#7049) +- feat: guardrail redaction covers LLM tool-call arguments, Chat function arguments, Responses function arguments and custom-tool input, across the Anthropic streaming and non-streaming paths. `input_json_delta` events are inspected and rewritten alongside `text_delta`, reading and writing `delta.partial_json`; `collectAnthropicArgumentStringPaths` recursively collects string-valued paths inside a `tool_use` block's `input` without touching tool names, IDs or definitions; `BifrostResponsesStreamResponse` gained `Input` to carry the full custom-tool payload on `custom_tool_call_input.done`; and `DeepCopyResponsesMessage` deep-copies `ResponsesCustomToolCall` so a copied message cannot mutate the original (#6977) +- fix: an abandoned non-streaming request, one whose caller context was already cancelled by the time the upstream finished, is billed and logged deterministically instead of about half the time. The worker's delivery `select` had two simultaneously ready cases, a send into a cap-1 channel and `ctx.Done()`, and Go picks uniformly among ready cases, so the terminal post-hooks that record billing and finalize the log row were skipped roughly 50% of the time. `requestWorker` now checks `req.Context.Err()` before the select on both the error and success paths and calls `billAbandonedTerminal` directly, keeping the 5-second timer guard inside the select for a caller that leaves between the check and the send (#6972) +- fix: provider response headers are filtered by classifier, not only by a fixed map of 28 exact names, so a credential-named header outside that map is no longer re-served to the inference caller in the HTTP response and `extra_fields.provider_response_headers`. The extractors now also consult `schemas.IsSensitiveHeader`, which matches credential names by substring and suffix and already recognized `cf-access-*` and `x-amzn-oidc-*`; a fixed list cannot enumerate the space when `network_config.extra_headers` exists to carry custom authentication headers and some upstreams echo request headers back (#7120) [@Atharva-Kanherkar](https://github.com/Atharva-Kanherkar) +- [fix]: a plugin that returns an incomplete `BifrostError` whose nested `Error` field is nil no longer panics the request worker. The fallback decision helper nil-checks `fallbackErr`, guards access to `Error.Type` and reads the message through the nil-safe `GetErrorString()` helper, continuing to the next fallback when allowed; behaviour for cancelled requests and `AllowFallbacks == false` is unchanged (#6967) [@Constantine3](https://github.com/Constantine3) +- [fix]: `clearCtxForFallback` now clears `BifrostContextKeyProviderResponseHeaders`. Providers set that key from their own HTTP response before the status check so error paths can forward it, and when a fallback attempt failed pre-flight, key selection failing for the fallback provider, a plugin short-circuiting it, or the queue retiring, nothing overwrote the key and the primary's headers survived onto the fallback's error response. A client could receive a response attributed to provider B carrying provider A's `Retry-After` and `x-ratelimit-remaining-*`, and wait according to a limit belonging to a provider that never served the request (#6973) [@Huang-404-Q](https://github.com/Huang-404-Q) +- [fix]: untitled document blocks get unique names on the Bedrock Converse path. Every untitled block was named the literal `document`, and Converse rejects duplicate document names, so any request carrying two or more untitled documents failed unconditionally with `ValidationException: Messages can't contain duplicate document names`. A per-request namer threaded through the content-block conversion disambiguates with numeric suffixes (`document`, `document-2`, `document-3`) and suffixes explicitly titled documents only on an actual collision; the Responses input replay path gets the same treatment scoped to its content-block list (#7003) [@Huang-404-Q](https://github.com/Huang-404-Q) +- [fix]: `BifrostResponsesStreamResponse.Item` is `omitempty`, so Responses stream events that carry no item payload no longer serialize `"item": null`. Strict OpenAI Responses clients reject those as invalid stream frames, which broke streamed `/v1/responses` usage entirely; `response.output_item.added` and `response.output_item.done` still emit the item object [@ReStranger](https://github.com/ReStranger) +- feat: `SecretVar.RedactedIfSecret()` returns a plain clone when the value is a literal and delegates to `Redacted()` when it is env-var or vault-backed, so regions and service URLs (Azure endpoint, Vertex/Bedrock/Bedrock Mantle region, vLLM/Ollama/SGL/Databricks URL, MCP connection string) are readable in the UI instead of being masked as if they were credentials. It always returns a fresh pointer, so the redacted copy never aliases the live config, and it replaces the ad-hoc `IsFromSecret()` guard blocks that tried to replicate this inline (#7085) +- feat: GA realtime transcription is served for OpenAI and Azure over both WebSocket and WebRTC with normal Bifrost authentication, routing, governance, guardrails, logging and transcription-aware pricing. Unlike a normal realtime session, which names its routing model in the connect URL, a GA transcription session carries only `intent=transcription` and delivers the model later in `session.update` under `audio.input.transcription.model` (or in the initial multipart `/v1/realtime/calls` request on WebRTC), so routing now resolves from the nested transcription model while realtime connection and turn semantics are preserved (#7089) +- fix: Claude Code replaying a prior assistant message through `POST /anthropic/v1/messages` at a Bedrock Mantle model is no longer rejected with hundreds of validation errors. The Bedrock-grouped ingress converter tagged user and system input text as `output_text`, though only `input_text` is valid on input messages, and omitted the required `status` on replayed assistant output messages, so Mantle's strict OpenAI-compatible validator refused the whole request. Replayed assistant messages now carry `Status: "completed"` and input text blocks are tagged `input_text`. Bedrock requests with no explicit `max_tokens` also populate it from the model's known capacity instead of truncating silently on large context windows (#7074) +- fix: Anthropic server tools such as `web_search_20250305` are stripped before a request reaches Fireworks, vLLM or SGLang on their Anthropic-compatible endpoints, which return 400 for tools executed by Anthropic-operated infrastructure that does not exist on third-party hosts; clients whose built-in web search is always on, such as Codex, hit this on every request. `ProviderFeatures` entries for `Fireworks`, `VLLM` and `SGL` declare all server-tool flags off so the existing validators strip them, `StripUnsupportedServerToolsFromRawBody` mirrors that on the raw-body passthrough path, the caller's function tools are kept, and the drops are reported on the response's `DroppedUnsupportedTools` instead of failing the call (#7090) +- fix: `guardrailConfig` reaches Bedrock's OpenAI-compatible `chat/completions` and `responses` surfaces, streaming and non-streaming, which apply guardrails through request headers rather than the body field Converse uses, so a configured guardrail was previously ignored there with no error. `withGuardrailHeaders` maps `guardrailIdentifier`, `guardrailVersion` and optional `trace` to `X-Amzn-Bedrock-GuardrailIdentifier`, `X-Amzn-Bedrock-GuardrailVersion` and `X-Amzn-Bedrock-Trace`, then deletes the key from `ExtraParams` so it is not also emitted into the body. A half-formed config with only one of identifier or version is left untouched rather than sent, since both are required upstream, and the base header map is cloned rather than mutated (#7095) +- feat: `use_openai_endpoints` on Bedrock keys and aliases routes chat completions and responses through Bedrock's OpenAI-compatible `/openai/v1` endpoints instead of Converse, for models that support them, mirroring the existing `use_anthropic_endpoints` pattern. It is opt-in by design: Converse carries Bedrock Guardrails, `performanceConfig` and `requestMetadata` that the OpenAI-compatible surface silently ignores, so diverting automatically could stop a guardrail from being enforced with no visible error. `ResolveUseOpenAIEndpoints` gives the alias value precedence over the key, and the narrow `runtimeServesResponses` check is replaced by a general `runtimeServesOpenAIAPI` that takes a `BedrockAPI` discriminator; chat completions, streaming and non-streaming, gained the runtime path that previously existed only for Responses (#7073) +- fix: Bedrock runtime models that serve the Responses API are routed to it through a dedicated surface resolver rather than falling back to Converse (#7071) +- fix: Bedrock Converse document blocks always ship their content base64-encoded through `source.bytes`, never through `source.text`, which Converse rejects unless citations are explicitly enabled. Plain text formats (`text/plain`, `text/markdown`, `text/csv`, `text/html`) previously took the text branch and failed with `DocumentSource must set one of the following keys: bytes, s3Location`; the branch is removed for data URLs, percent-encoded payloads and the non-data-URL `file_data` path alike (#7072) +- fix: the Bedrock Mantle base path is resolved from the model capabilities datasheet instead of hard-coded string matching in two packages. Mantle serves each model on exactly one of `v1` or `openai/v1` and returns a 400 on the other, and the previous matching covered only generations up to GPT-5 and Gemma 4, so GPT-6 and any future closed-generation model silently fell through to the wrong path. `ResolveBedrockMantleBasePath` applies family-name detection as a fallback and defers to the new `BedrockMantleBasePath` field on `ModelCapabilities`, so a new generation needs a datasheet row rather than a code change (#7077) +- fix: `image_generation_call` items where OpenAI emits `action` as a bare JSON string, such as `"generate"`, decode correctly. `ResponsesToolMessageActionStruct.UnmarshalJSON` immediately peeked at a `.type` field, which cannot be read from a JSON string, so the whole decode failed with `failed to peek at type field`; that silently dropped the `response.output_item.done` and `response.completed` events carrying the image, leaving the stream without a terminal event and surfacing as a bogus `provider closed the stream` truncation error. The action is now tried as a bare string before falling back to the object type-peek, and `ResponsesImageGenerationCall` keeps the `background`, `output_format`, `quality`, `revised_prompt` and `size` settings OpenAI echoes back on completed items (#7060) + + + +- fix: virtual key rate limits and budgets created via the config.json standalone-limits flow and later edited through the UI no longer produce duplicate, conflicting ownership records; migration `migrate_vk_standalone_limits_to_model_configs` consolidates ownership into VK-scoped model configs, preserving usage counters, and the write paths stop creating the divergent rows (#7291) +- feat: virtual key listing batch-resolves assignees (`assigned_user`) instead of resolving per row, and VK search matches team and customer names in addition to key fields (#7225) +- chore: OTEL dependency updates (#7311) +- fix: Responses API requests emit `gen_ai.response.finish_reason` / `gen_ai.response.finish_reasons` on the OTEL `llm.call` span. `PopulateResponsesResponseAttributes` never copied the top-level `stop_reason` into the span attributes, so refusals on `/v1/responses` were invisible to OTEL consumers even though the stop reason reached the Logs DB; it now emits `finish_reasons` like the chat path and the tracer derives the singular key from it (#7204) +- feat: model allow and block lists accept `regex:` entries. The provider-key aggregate keeps an exact name next to a pattern that also covers it, the catalog allow check tries a pattern against the bare name and `provider/model`, and listings skip pattern entries +- fix: ClickHouse log store deletes no longer run as heavyweight `ALTER TABLE ... DELETE` mutations. The retention cleaner issued one such mutation per 100 rows, each rewriting the whole current-month part, and the once-a-minute stale-`processing` sweeps issued one per table unconditionally, filling replica disks in minutes. Every delete on the ClickHouse store (retention sweep, `Flush`/`FlushMCPToolLogs`, UI log deletes, async job and webhook delivery expiry) is now a single lightweight `DELETE FROM ... WHERE` per run, skipped entirely when nothing matches. The table TTL derived from `logs_store.retention_days` is now reconciled on every startup with `MODIFY TTL` (metadata only), so changing the value reaches existing tables; `0` leaves an existing TTL untouched (#7098) +- feat: access profiles and governance projects reference Virtual MCPs through `virtual_mcp_name`, so config files are portable across environments instead of carrying database-assigned integer IDs. Names resolve to stored records on startup and a name that matches nothing is refused; `virtual_mcp_id` is deprecated, still accepted, and wins when both are set. `mcp_configs` (`{ mcp_client_id, tools_to_execute }`) replaces the `mcp_servers` and `mcp_tool_overrides` include-exclude model with a single allowlist, where `["*"]` grants all tools including future ones, `[]` grants none, and a named list grants only those; the old keys are deprecated, still accepted, and folded into the new shape at load time (#7181) +- feat: `MCPToolLog` records governance entity names beside their IDs, so MCP tool logs carry the same attribution shape the `logs` table has instead of rendering raw UUIDs in the dashboard. `user_name`, `team_name`, `customer_name` and `business_unit_name` stop being `gorm:"-"` transients and become storage; the multi-valued `team_ids`/`team_names`, `customer_ids`/`customer_names` and `business_unit_ids`/`business_unit_names` sets are stored as index-aligned JSON arrays; and `budget_ids` and `rate_limit_ids` are recorded id-only, as in `logs`. Names are written from the request context at ingestion through the new `MCPToolLog.ApplyGovernance` in `framework/logstore/governance.go`, with nothing resolved on read. Added by migration `mcp_tool_logs_add_governance_snapshots` (#7154) +- feat: endpoint-attributed MCP observations are carried in the standard logging pipeline, so inspected MCP tool calls are logged with bounded identity (device, app key, server label, tool name, decision) sourced from the gateway rather than from payload-supplied headers. `SetMCPObservation` attaches the attribution to a `BifrostContext` and `applyMCPObservation` writes it onto the `MCPToolLog` from both `PreMCPHook` and `PostMCPHook`, snapshotting the observation so it cannot alias across async log entries (#6959) +- feat: the `error_type` classification vocabulary is threaded through the framework so `bifrost_error_requests_total` can carry a normalized fault-domain label alongside `status_code` (#7141) +- feat: model pricing supports time-of-day peak and off-peak rates. `TableModelPricing` gains `off_peak_cost_multiplier` (a nullable float) and `peak_hours` (a JSON-serialized `PeakHoursSchedule` of recurring weekly windows using IANA timezone names, weekday numbers and half-open `HH:MM` intervals that may wrap past midnight), added by migration `add_time_of_day_pricing_columns` and aliased into the datasheet package so the JSON shape stays self-contained. The cost engine evaluates the schedule against the request start time and scales usage-based charges by the multiplier when the request falls outside every peak window, applied once in `computeCostFromInput` so every modality is covered; flat `CostPerRequest` and `SearchQueriesCost` fees are excluded, as is `AdditionalCost`, which is discounted independently through its own pricing rows. Both fields are exposed on `PricingPatch` in the OpenAPI and governance schemas, with `off_peak_cost_multiplier` bounded to `(0, 1]` (#6574, #6575, #6576) +- feat: `SecretVar.RedactedIfSecret()` is used for non-credential fields in `ProviderConfig.Redacted()`, `Config.GetAllKeys()` and `Config.RedactMCPClientConfig()`, so regions, endpoints, service URLs and MCP connection strings surface as plaintext while anything env-var or vault-backed stays masked (#7085) +- feat: GA realtime transcription sessions are routed, governed, logged and priced through the normal framework pipeline, resolving the routing model from the nested `audio.input.transcription.model` that arrives in `session.update` rather than from the connect URL (#7089) +- feat: a `use_openai_endpoints` column on the provider keys table, added by migration `add_use_openai_endpoints_column`, opts a Bedrock key or alias into Bedrock's OpenAI-compatible endpoints instead of Converse (#7073) +- feat: the reserved tool-namespace list a provider keeps for its own server tools is read from the datasheet row `reserved_tool_namespaces`, so a namespace collision check no longer requires a code change (#7084) +- fix: `exchangeRefreshToken` includes `client_secret` only when the secret is non-empty, matching `exchangeCodeForTokensWithPKCE`. Public OAuth2 clients registered against servers that support only `token_endpoint_auth_method: none` have no secret, and unconditionally setting `client_secret=` sent an empty `client_secret_post` attempt that strict authorization servers answered with `invalid_client`, flipping the token row to `needs_reauth` even though the refresh token was valid (#7042) +- fix: the virtual key `allowed_models: ["*"]` handling for governance added in #6767 is reverted, returning the wildcard-with-empty-synced-catalog case to its previous behaviour (#7053) + + + +- chore: upgraded core to v1.9.1 and framework to v1.7.1 +- fix: removed the namespace-tool flattening that ran under `should_convert_params`; it spliced nested functions into the top-level list without a namespace prefix, so two namespaces sharing a function name produced duplicate tool names and an upstream 400. Flattening now lives in Bifrost core for every provider whose wire lacks the `namespace` type, with unique `__` names and response-side mapping back. `should_convert_params` is still accepted so existing configs load, but it no longer changes any request (#7048) + + + +- fix: the billing-idempotency key mixes in the internally minted `BillingNonce`, so caller-forged `x-request-id` duplicates can no longer suppress charges (#7303) +- fix: virtual key credentials are validated before realtime session admission (#7317) +- chore: upgraded core to v1.9.1 and framework to v1.7.1 +- feat: MCP tool logs record governance entity names beside their IDs, so the plugin stamps `user_name`, `team_name`, `customer_name`, `business_unit_name` and the multi-valued team, customer and business-unit sets onto the log entry at ingestion instead of leaving the dashboard to render raw UUIDs (#7154) +- feat: endpoint-attributed MCP inspections carry bounded identity (device, app key, server label, tool name, decision) from the gateway into the MCP authorization path, so an inspected tool call is attributed without trusting payload-supplied headers (#6959) +- feat: governance errors are classified into the normalized `error_type` vocabulary, so a 429 raised by a governance rate limit is distinguishable from an upstream 429 on `bifrost_error_requests_total` (#7141) +- feat: model allow and block lists accept `regex:` entries. The `*_patterns` fields added earlier in this release window were withdrawn in favour of the prefix form, which needs no new schema fields (#6988, #7133, #7134) +- fix: `UsageTracker.Cleanup()` cancels and waits for the periodic reset worker before taking its final budget and rate-limit snapshots. It previously dumped first, so `trackerCancel()` could cancel an in-flight rate-limit dump and fail with `failed to dump rate limits to database: context canceled`, and the final dump was not guaranteed to be the tracker's last database writer. A queued ticker event can no longer start another reset cycle during shutdown, the current cycle stops when the tracker context is cancelled, and `context.Canceled` is treated as an expected result only when that context was actually cancelled (#7099) [@Constantine3](https://github.com/Constantine3) + + + +- chore: upgraded core to v1.9.1 and framework to v1.7.1 + + + +- chore: upgraded core to v1.9.1 and framework to v1.7.1 +- feat: MCP tool log rows carry governance entity names alongside their IDs, written from the request context at ingestion, so nothing is resolved on read (#7154) +- feat: endpoint-attributed MCP observations are written onto both the pending and the final MCP tool log entry, with the observation snapshotted so it cannot alias across async entries (#6959) +- feat: cost recalculation honours time-of-day peak and off-peak pricing, scaling usage-based charges by `off_peak_cost_multiplier` when a request falls outside every declared peak window (#6575) +- feat: GA realtime transcription sessions are logged and priced through the standard pipeline with transcription-aware pricing (#7089) +- fix: a request whose caller disconnected before the upstream finished is logged and finalized deterministically rather than roughly half the time, so abandoned requests no longer leave a log row stuck in its pending state (#6972) +- docs: clarified that `CountRecalcTargets` reads a materialized view that can lag, so its `Total` is an approximation rather than an exact count (#7078) + + + +- chore: upgraded core to v1.9.1 and framework to v1.7.1 + + + +- chore: upgraded core to v1.9.1 and framework to v1.7.1 + + + +- chore: upgraded core to v1.9.1 and framework to v1.7.1 + + + +- chore: upgraded core to v1.9.1 and framework to v1.7.1 + + + +- chore: upgraded core to v1.9.1 and framework to v1.7.1 + + + +- chore: upgraded core to v1.9.1 and framework to v1.7.1 +- fix: a continuation turn, such as a tool result following a prior user message, is classified instead of skipped when no active session state is found. `BuildInputWithDisposition` now returns the populated `ComplexityInput` including `LastUserText` for trailing-continuation turns in both the chat and responses paths, and `computeComplexity` falls back to classifying that recovered text, so new or recovered sessions get a tier assignment; the skip path applies only when `LastUserText` is also empty (#7122) + + + +- chore: upgraded core to v1.9.1 and framework to v1.7.1 + + + +- feat: metrics export includes user id and user name labels, so per-user attribution is available in Prometheus (#7267) +- chore: upgraded core to v1.9.1 and framework to v1.7.1 +- feat: `bifrost_error_requests_total` carries a normalized `error_type` label alongside `status_code`, drawn from a closed prefix-structured vocabulary (`caller_*`, `policy_*`, `provider_*`, `bifrost_*`, `_OTHER`), so alarm expressions can separate fault domains with a single regex instead of enumerating status codes (#7141) + + diff --git a/docs/changelogs/v2.2.2.mdx b/docs/changelogs/v2.2.2.mdx new file mode 100644 index 00000000000..7cd8355e783 --- /dev/null +++ b/docs/changelogs/v2.2.2.mdx @@ -0,0 +1,139 @@ +--- +title: "v2.2.2" +description: "v2.2.2 changelog - 2026-09-23" +--- + + + ```bash + npx -y @maximhq/bifrost --transport-version v2.2.2 + ``` + + + ```bash + docker pull maximhq/bifrost:v2.2.2 + docker run -p 8080:8080 maximhq/bifrost:v2.2.2 + ``` + + + + +## ✨ Features + +- **Typesafe Provider and Decisions API** - New typesafe provider, `/v1/decisions` endpoint and `/typesafe` integration. Providers without native decision support now answer decision requests through forced tool-calling on their chat model, whether used as the primary or as a fallback. Probabilities are normalized to sum to exactly 1 and the chosen option must be the most likely one. Decision requests are priced from the datasheet and logged with their answers (#7355, #7361, #7384, #7440) +- **Provider-Level Session Affinity** - A session (from `x-bf-session-id` or the session header that Claude Code, Codex CLI or OpenCode already send) stays on the provider and key that last served it. Affinity only reorders the chain routing built and never restores a provider routing excluded. The logs UI shows it as a routing engine +- **Claude Opus 5.5 Support** - Computer use sends `computer_toolset_20260801` on the Anthropic API and Vertex, while Bedrock and Azure keep `computer_20251124`. `toolset_name` is carried on both halves of each call/result pair across typed, raw passthrough and streaming paths. Disabled thinking and forced tool choice are rejected for Opus 5.5+, and the datasheet `supports_reasoning_disable` field can override this (#7433, #7434, #7441) +- **Claude Code Auto-Mode Safeguards Passthrough** - `safeguards` and `safeguard_results` are forwarded byte-for-byte on requests, responses and stream events to the direct Anthropic provider and stripped on every other provider. The `dangerous-tool-use` and `auto-mode-classifier` betas are gated the same way. Unknown Anthropic SSE events are forwarded raw on the Anthropic passthrough (#7393, #7440) + +## 🐞 Fixed + +- **Kimi and DeepSeek with Claude Code** - Tool-schema regex patterns are rewritten (`\0` to `\x00`, lookaround assertions stripped) for Moonshot and DeepSeek models only. kimi-k3 on Bedrock no longer returns an empty stream, and every other model gets byte-identical schemas (#7430) +- **Anthropic Billing Header Leak** - Claude Code's `x-anthropic-billing-header` system block is removed at Messages ingress and restored only for Anthropic-family attempts, including fallbacks and alias targets, so it no longer pollutes GPT or Gemini prompts (#7431) +- **MCP Egress Proxy** - MCP HTTP/SSE connections honor `HTTP_PROXY`, `HTTPS_PROXY` and `NO_PROXY` again (broken since core v1.8.5). Link-local and unspecified destinations are refused before the proxy is dialed (#7437) +- **OpenAI `computer` Tool** - The bare `{"type":"computer"}` tool is no longer rewritten to `computer_use_preview`, which fixes computer use on GPT-6 Astra and GPT-5.6 (#7426) (thanks [@abhishekgahlot2](https://github.com/abhishekgahlot2)!) +- **Streaming Memory Leaks** - The request context is cancelled on every stream exit path, not only on write errors, which stops leaked disconnect watchers from pinning request contexts. `StripEmptyThinkingBlocks` rewrites the body once instead of once per block, and Anthropic beta-header gating no longer decodes the full request body (#7406) +- **Bedrock cachePoint Leak** - Bedrock `cachePoint` markers are stripped copy-on-write for non-Bedrock providers and kept for Bedrock fallbacks. The compat plugin no longer mutates the shared request. Nova reasoning signatures are stripped on the wire only (#7182) +- **Bedrock Empty JSON Keys** - Tool results containing an empty-string object key (such as Cursor's `list_directory`) are sent as text, so Converse no longer rejects them (#7396) +- **Bedrock cache_control on String Content** - InvokeModel keeps every `cache_control` when any message's content is a plain string (#7360) (thanks [@basil-k-aji-dev](https://github.com/basil-k-aji-dev)!) +- **Web Search Source Names** - Responses web search API sources keep their `name` and no longer emit an empty `url` (#7358) (thanks [@g-yixuan](https://github.com/g-yixuan)!) +- **Grok 4.7 xhigh Reasoning** - `xhigh` reasoning effort is no longer downgraded to `high` (#7403) (thanks [@nettee](https://github.com/nettee)!) +- **Allow-All Provider Access** - Virtual keys that allow every provider now list models from, and route to, every configured provider. The governance routing log names providers excluded for having no weight (#7375) +- **OpenAI Chat Stream Framing** - Bundled raw finish and usage frames on the OpenAI chat stream passthrough each get their own `data:` prefix (#7440) +- **Responses Deep Copy** - `DeepCopyResponsesMessage` now deep copies cache controls, provider-native parts, computer/MCP/code-interpreter tool fields and annotations, so copies no longer share pointers with the original (#7422) +- **Request Preparation Performance** - Responses requests are decoded once instead of several times, and the compat plugin clones only the fields it writes (#7412, #7097) (thanks [@G-XD](https://github.com/G-XD)!) + +## 🗄️ Database Migrations + +- No new database migrations in this release. + +## 🐙 Closed GitHub Issues + +- [#7223](https://github.com/maximhq/bifrost/issues/7223) - MCP client HTTP transport ignores HTTP_PROXY/HTTPS_PROXY and fails on any deployment behind an egress proxy +- [#7336](https://github.com/maximhq/bifrost/issues/7336) - Bedrock InvokeModel drops message-level cache_control when any historical message content is a JSON string +- [#7356](https://github.com/maximhq/bifrost/issues/7356) - Responses web search API source name is dropped during round-trip +- [#7402](https://github.com/maximhq/bifrost/issues/7402) - Grok 4.7 xhigh reasoning effort is silently downgraded to high +- [#7411](https://github.com/maximhq/bifrost/issues/7411) - Redundant JSON decoding in Responses request preparation +- [#7425](https://github.com/maximhq/bifrost/issues/7425) - Responses: `{"type":"computer"}` is rewritten to `computer_use_preview`, breaking GPT-6 Astra / GPT-5.6 computer use + + + +- feat: typesafe provider, /v1/decisions endpoint, and decision emulation via forced tool-calling for providers without native decision support (#7355, #7361, #7384, #7440) +- feat: provider-level session affinity through the SessionAffinity seam, bound on request outcome +- feat: Claude Opus 5.5 computer_toolset_20260801 support with toolset_name round-trip, and disabled-thinking/forced-tool-choice gating overridable from the datasheet (#7433, #7434, #7441) +- feat: safeguards/safeguard_results passthrough for Claude Code auto-mode on direct Anthropic, stripped elsewhere; raw carrier for unknown Anthropic stream events (#7393, #7440) +- fix: rewrite tool-schema regex NUL escapes and lookarounds for Moonshot and DeepSeek models only (#7430) +- fix: strip Anthropic billing header at ingress and restore it only for Anthropic-family attempts (#7431) +- fix: honor HTTP_PROXY/HTTPS_PROXY/NO_PROXY for MCP connections with a pre-proxy link-local guard (#7437) +- fix: stop rewriting OpenAI `computer` tool to computer_use_preview (#7426) (thanks [@abhishekgahlot2](https://github.com/abhishekgahlot2)!) +- fix: linear StripEmptyThinkingBlocks and beta-header gating without full body decode (#7406) +- fix: strip Bedrock cachePoint markers copy-on-write for non-Bedrock providers, preserved for fallbacks (#7182) +- fix: Bedrock tool results with empty-string JSON keys sent as text (#7396) +- fix: deep copy extended Responses fields in DeepCopyResponsesMessage (#7422) +- fix: frame bundled OpenAI chat stream raw frames with a data prefix each (#7440) +- fix: stream errors after startup events (response.created, in_progress, empty role delta) now reach retry and fallback for OpenAI models on every host (OpenAI, Bedrock, Bedrock Mantle, Vertex, custom providers), not only Azure; an overloaded stream no longer reaches the client as an error when a fallback is configured +- [perf]: avoid redundant JSON decoding in Responses request preparation +- [fix]: preserve xhigh reasoning effort for Grok 4.7 [@nettee](https://github.com/nettee) +- [fix]: Bedrock InvokeModel keeps cache_control when any message's content is a plain string. BedrockMessage.Content is typed as content blocks, so one bare string anywhere in messages[] failed the standard unmarshal and diverted the whole request into the AI21 string fallback, which rebuilt the messages without calling applyMessageContentCacheControl. Every cache_control in the request was dropped rather than only the one on the string message, so prompt caching went off silently with just the system cachePoint surviving. The fallback now makes the same translation the standard path does (#7336) [@basil-k-aji-dev](https://github.com/basil-k-aji-dev) +- [fix]: web search action sources round-trip their name and no longer fabricate an empty url. OpenAI Responses web search can return specialized API sources (`{"type":"api","name":"oai-weather"}`) that carry a name and no URL; the typed source schema only modeled type and a required url, so decode dropped name and re-encode emitted "url":"", and the OpenAI request-side source sanitization rebuilt sources without name. url is now omitempty and name survives both the schema round-trip and the sanitization path (#7356) [@g-yixuan](https://github.com/g-yixuan) + + + +- feat: decision request pricing and "decisions" usage type in the datasheet +- chore: upgraded core to v1.10.0 +- fix: replace streaming gate replay-buffer size accounting with cached zero-marshal estimates (eliminates per-chunk MarshalJSON on the full-hold path) + + + +- fix: Bedrock cachePoint handling moved to core dispatch; plugin no longer mutates the shared request (#7182) +- fix: clone only Reasoning, ToolChoice and Tools in PreLLMHook (#7097) +- chore: upgraded core to v1.10.0 and framework to v1.7.3 + + + +- fix: allow-all virtual keys list and route every configured provider (#7375) +- fix: routing log names providers excluded for having no weight +- chore: upgraded core to v1.10.0 and framework to v1.7.3 + + + +- chore: upgraded core to v1.10.0 and framework to v1.7.3 + + + +- feat: log decision requests with usage, cost and answers (#7355) +- chore: upgraded core to v1.10.0 and framework to v1.7.3 + + + +- chore: upgraded core to v1.10.0 and framework to v1.7.3 + + + +- chore: upgraded core to v1.10.0 and framework to v1.7.3 + + + +- chore: upgraded core to v1.10.0 and framework to v1.7.3 + + + +- chore: upgraded core to v1.10.0 and framework to v1.7.3 + + + +- chore: upgraded core to v1.10.0 and framework to v1.7.3 + + + +- chore: complexity routing session keys built through the shared session-state builder +- chore: upgraded core to v1.10.0 and framework to v1.7.3 + + + +- chore: upgraded core to v1.10.0 and framework to v1.7.3 + + + +- chore: shared label-value splice helper for metric labels (#7378) +- chore: upgraded core to v1.10.0 and framework to v1.7.3 + + diff --git a/docs/cli-agents/claude-code.mdx b/docs/cli-agents/claude-code.mdx index af77848c48c..b2a2e905af8 100644 --- a/docs/cli-agents/claude-code.mdx +++ b/docs/cli-agents/claude-code.mdx @@ -61,6 +61,10 @@ Set `ANTHROPIC_CUSTOM_HEADERS` to `x-bf-vk: your-virtual-key`. This passes the v ## Configuring Claude Code to work with Bifrost + +Claude Code sends `x-claude-code-session-id` on every request, so a session sticks to the provider and key that served it and keeps hitting the same prompt cache, with nothing to configure. See [Session Affinity](/providers/session-affinity). + + **To avoid caching issues in Claude Code**, follow these steps: diff --git a/docs/cli-agents/codex-cli.mdx b/docs/cli-agents/codex-cli.mdx index 7ad6b9ba6a8..5101f36f27f 100644 --- a/docs/cli-agents/codex-cli.mdx +++ b/docs/cli-agents/codex-cli.mdx @@ -18,6 +18,10 @@ npm install -g @openai/codex ## Configuring Codex CLI with Bifrost + +Codex CLI sends a `session-id` header on every request, so a session sticks to the provider and key that served it and keeps hitting the same prompt cache, with nothing to configure. See [Session Affinity](/providers/session-affinity). + + Codex CLI always prefers OAuth over custom API keys. Make sure you run `/logout` before configuring the Bifrost gateway with Codex. diff --git a/docs/cli-agents/opencode.mdx b/docs/cli-agents/opencode.mdx index b6929853409..f13eb6a4a9d 100644 --- a/docs/cli-agents/opencode.mdx +++ b/docs/cli-agents/opencode.mdx @@ -14,6 +14,10 @@ If your Allowed Headers are already set to `*`, you can skip this note. If not a ## Setup + +OpenCode sends `x-session-affinity` and `x-session-id` on every request, so a session sticks to the provider and key that served it and keeps hitting the same prompt cache, with nothing to configure. See [Session Affinity](/providers/session-affinity). + + ### 1. Configure OpenCode to work with Bifrost OpenCode uses a JSON config file (`opencode.json`) to configure providers. Point your provider's `baseURL` to Bifrost. diff --git a/docs/deployment-guides/config-json/governance.mdx b/docs/deployment-guides/config-json/governance.mdx index 04d37bfafcb..7f38fa238c7 100644 --- a/docs/deployment-guides/config-json/governance.mdx +++ b/docs/deployment-guides/config-json/governance.mdx @@ -259,7 +259,7 @@ Routing rules dynamically select the provider and model for each request based o | `scope` | No | `"global"` (default), `"team"`, `"customer"`, `"virtual_key"` | | `scope_id` | Conditional | Required when `scope` is not `"global"` | | `chain_rule` | No | If `true`, re-evaluates the chain after this rule matches | -| `fallbacks` | No | Ordered fallback provider list if primary target fails | +| `fallbacks` | No | Ordered fallback list if the primary target fails. See [Fallback Fields](#fallback-fields) | ### Target Fields @@ -268,7 +268,26 @@ Routing rules dynamically select the provider and model for each request based o | `weight` | Yes | Fraction of traffic (all weights in a rule must sum to `1.0`) | | `provider` | No | Target provider. Omit to keep the incoming request's provider | | `model` | No | Target model. Omit to keep the incoming request's model | -| `key_id` | No | Pin a specific API key by name | +| `provider_key_name` | No | Pin a specific provider key by name. Requires `provider`, and is resolved to an internal key ID at load time | + +### Fallback Fields + +Each entry is either the string `"provider/model"` (use `"provider/"` to keep the incoming model), or an object when it needs to pin a key: + +| Field | Required | Description | +|-------|----------|-------------| +| `provider` | Yes | Fallback provider | +| `model` | No | Fallback model. Omit to keep the incoming request's model | +| `provider_key_name` | No | Pin a specific provider key by name, resolved to an internal key ID at load time. Omit for load-balanced key selection | + +```json +"fallbacks": [ + "anthropic/claude-sonnet-4-5", + { "provider": "vertex", "provider_key_name": "vertex-prod" } +] +``` + +A fallback whose pinned key is missing from that provider's pool is skipped, and the chain moves to the next entry. --- @@ -461,7 +480,7 @@ Budgets and rate limits inside a project are declared without ids. "name": "Fallback on error", "cel_expression": "true", "targets": [{ "provider": "openai", "weight": 1.0 }], - "fallbacks": ["anthropic"] + "fallbacks": ["anthropic/"] } ] }, diff --git a/docs/docs.json b/docs/docs.json index 5e012c1f430..3836240c376 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -60,7 +60,9 @@ { "group": "Overview", "icon": "book-open-cover", - "pages": ["overview"] + "pages": [ + "overview" + ] }, { "group": "Quick Start", @@ -77,6 +79,7 @@ "quickstart/gateway/tool-calling", "quickstart/gateway/multimodal", "quickstart/gateway/reranking", + "quickstart/gateway/decisions", "quickstart/gateway/integrations" ] }, @@ -92,6 +95,7 @@ "quickstart/go-sdk/tool-calling", "quickstart/go-sdk/multimodal", "quickstart/go-sdk/reranking", + "quickstart/go-sdk/decisions", "quickstart/go-sdk/logger" ] } @@ -100,22 +104,31 @@ { "group": "Release Cadence", "icon": "calendar", - "pages": ["release-cadence"] + "pages": [ + "release-cadence" + ] }, { "group": "Migration Guides", "icon": "arrow-up-right-dots", - "pages": ["migration-guides/v2.0.0", "migration-guides/v1.5.0"] + "pages": [ + "migration-guides/v2.0.0", + "migration-guides/v1.5.0" + ] }, { "group": "Migrate from other Gateway", "icon": "arrow-up", - "pages": ["migration-guides/litellm"] + "pages": [ + "migration-guides/litellm" + ] }, { "group": "SDK Integrations", "icon": "plug", - "pages": ["integrations/what-is-an-integration"] + "pages": [ + "integrations/what-is-an-integration" + ] }, { "group": "Providers & Guides", @@ -159,6 +172,7 @@ "providers/supported-providers/runway", "providers/supported-providers/sarvam", "providers/supported-providers/sgl", + "providers/supported-providers/typesafe", "providers/supported-providers/vertex", "providers/supported-providers/volcengine", "providers/supported-providers/vllm", @@ -168,6 +182,7 @@ }, "providers/routing-rules", "providers/provider-routing", + "providers/session-affinity", "providers/aliasing-models", "providers/reasoning", "providers/performance", @@ -217,7 +232,10 @@ { "group": "Writing Plugins", "icon": "code", - "pages": ["plugins/writing-go-plugin", "plugins/writing-wasm-plugin"] + "pages": [ + "plugins/writing-go-plugin", + "plugins/writing-wasm-plugin" + ] }, "plugins/migration-guide" ] @@ -263,7 +281,10 @@ { "group": "Plugins", "icon": "puzzle-piece", - "pages": ["features/plugins/mocker", "features/plugins/jsonparser"] + "pages": [ + "features/plugins/mocker", + "features/plugins/jsonparser" + ] } ] } @@ -276,7 +297,9 @@ { "group": "Getting Started", "icon": "rocket", - "pages": ["enterprise/overview"] + "pages": [ + "enterprise/overview" + ] }, { "group": "Moving from OSS", @@ -292,12 +315,17 @@ { "group": "Release Cadence", "icon": "calendar", - "pages": ["enterprise/release-cadence"] + "pages": [ + "enterprise/release-cadence" + ] }, { "group": "Migration Guides", "icon": "arrow-up-right-dots", - "pages": ["enterprise/migration-guides/v2.0.0", "enterprise/migration-guides/v1.4.0"] + "pages": [ + "enterprise/migration-guides/v2.0.0", + "enterprise/migration-guides/v1.4.0" + ] }, { "group": "Features", @@ -310,6 +338,7 @@ "enterprise/data-access-control", "enterprise/virtual-mcps", "enterprise/projects", + "enterprise/prompt-deployments", { "group": "Guardrails", "icon": "road-barrier", @@ -360,7 +389,9 @@ { "group": "Security", "icon": "shield", - "pages": ["security"] + "pages": [ + "security" + ] } ] }, @@ -370,22 +401,35 @@ "groups": [ { "group": "Overview", - "pages": ["edge/overview", "edge/how-it-works"] + "pages": [ + "edge/overview", + "edge/how-it-works" + ] }, { "group": "Governance & Control", "icon": "shield-halved", - "pages": ["edge/app-governance", "edge/mcp-governance", "edge/security"] + "pages": [ + "edge/app-governance", + "edge/mcp-governance", + "edge/security" + ] }, { "group": "Admin controls", "icon": "sliders", - "pages": ["edge/admin-devices", "edge/admin-approvals", "edge/admin-configurations"] + "pages": [ + "edge/admin-devices", + "edge/admin-approvals", + "edge/admin-configurations" + ] }, { "group": "Deployment", "icon": "cloud-arrow-up", - "pages": ["edge/deployment-mdm"] + "pages": [ + "edge/deployment-mdm" + ] }, { "group": "Onboarding Instructions", @@ -402,7 +446,9 @@ { "group": "Reference", "icon": "list", - "pages": ["edge/supported-applications"] + "pages": [ + "edge/supported-applications" + ] } ] }, @@ -413,7 +459,10 @@ { "group": "Runbooks", "icon": "book-open", - "pages": ["runbooks/claude-code-bedrock", "runbooks/codex-bedrock"] + "pages": [ + "runbooks/claude-code-bedrock", + "runbooks/codex-bedrock" + ] }, { "group": "CLI Agents & Editors", @@ -466,11 +515,14 @@ { "group": "GenAI SDK", "icon": "diamond", - "pages": ["integrations/genai-sdk/overview"] + "pages": [ + "integrations/genai-sdk/overview" + ] }, "integrations/litellm-sdk", "integrations/langchain-sdk", "integrations/pydanticai-sdk", + "integrations/typesafe-sdk", "integrations/passthrough" ] }, @@ -482,37 +534,50 @@ "group": "Okta", "icon": "/media/okta-icon.svg", "collapsed": true, - "pages": ["enterprise/setting-up-okta/oidc", "enterprise/setting-up-okta/scim"] + "pages": [ + "enterprise/setting-up-okta/oidc", + "enterprise/setting-up-okta/scim" + ] }, { "group": "Microsoft Entra", "icon": "microsoft", "collapsed": true, - "pages": ["enterprise/setting-up-entra/oidc"] + "pages": [ + "enterprise/setting-up-entra/oidc" + ] }, { "group": "Zitadel", "icon": "/media/zitadel-icon.svg", "collapsed": true, - "pages": ["enterprise/setting-up-zitadel/oidc"] + "pages": [ + "enterprise/setting-up-zitadel/oidc" + ] }, { "group": "Keycloak", "icon": "/media/keycloak-icon.svg", "collapsed": true, - "pages": ["enterprise/setting-up-keycloak/oidc"] + "pages": [ + "enterprise/setting-up-keycloak/oidc" + ] }, { "group": "Google Workspace", "icon": "google", "collapsed": true, - "pages": ["enterprise/setting-up-google-workspace"] + "pages": [ + "enterprise/setting-up-google-workspace" + ] }, { "group": "Auth0", "icon": "/media/auth0-icon.svg", "collapsed": true, - "pages": ["enterprise/setting-up-auth0/oidc"] + "pages": [ + "enterprise/setting-up-auth0/oidc" + ] }, { "group": "Generic OIDC", @@ -683,7 +748,9 @@ { "group": "Authentication", "icon": "key", - "pages": ["api/procuring-api-keys"] + "pages": [ + "api/procuring-api-keys" + ] }, { "group": "Inference", @@ -691,17 +758,23 @@ { "group": "Models", "expanded": false, - "pages": ["GET /v1/models"] + "pages": [ + "GET /v1/models" + ] }, { "group": "Chat Completions", "expanded": false, - "pages": ["POST /v1/chat/completions"] + "pages": [ + "POST /v1/chat/completions" + ] }, { "group": "Text Completions", "expanded": false, - "pages": ["POST /v1/completions"] + "pages": [ + "POST /v1/completions" + ] }, { "group": "Responses", @@ -718,22 +791,38 @@ { "group": "OCR", "expanded": false, - "pages": ["POST /v1/ocr"] + "pages": [ + "POST /v1/ocr" + ] }, { "group": "Rerank", "expanded": false, - "pages": ["POST /v1/rerank"] + "pages": [ + "POST /v1/rerank" + ] + }, + { + "group": "Decisions", + "expanded": false, + "pages": [ + "POST /v1/decisions" + ] }, { "group": "Embeddings", "expanded": false, - "pages": ["POST /v1/embeddings"] + "pages": [ + "POST /v1/embeddings" + ] }, { "group": "Audio", "expanded": false, - "pages": ["POST /v1/audio/speech", "POST /v1/audio/transcriptions"] + "pages": [ + "POST /v1/audio/speech", + "POST /v1/audio/transcriptions" + ] }, { "group": "Images", @@ -759,12 +848,16 @@ { "group": "Count Tokens", "expanded": false, - "pages": ["POST /v1/responses/input_tokens"] + "pages": [ + "POST /v1/responses/input_tokens" + ] }, { "group": "Compaction", "expanded": false, - "pages": ["POST /v1/responses/compact"] + "pages": [ + "POST /v1/responses/compact" + ] }, { "group": "Batch", @@ -980,6 +1073,14 @@ "POST /cohere/v1/tokenize" ] }, + { + "group": "Typesafe Integration", + "expanded": false, + "pages": [ + "POST /typesafe/v1/systemone", + "GET /typesafe/v1/models" + ] + }, { "group": "Cursor Integration", "expanded": false, @@ -1084,7 +1185,9 @@ { "group": "Health", "expanded": false, - "pages": ["GET /health"] + "pages": [ + "GET /health" + ] }, { "group": "Configuration", @@ -1299,12 +1402,20 @@ { "group": "Vault", "expanded": false, - "pages": ["POST /api/vault/flush-cache"] + "pages": [ + "POST /api/vault/flush-cache" + ] }, { "group": "Infrastructure", "expanded": false, - "pages": ["GET /ws", "GET /mcp", "POST /mcp", "GET /metrics", "POST /inspect"] + "pages": [ + "GET /ws", + "GET /mcp", + "POST /mcp", + "GET /metrics", + "POST /inspect" + ] }, { "group": "Webhooks", @@ -1325,7 +1436,10 @@ { "group": "Notifications", "expanded": false, - "pages": ["GET /api/notifications", "POST /api/notifications"] + "pages": [ + "GET /api/notifications", + "POST /api/notifications" + ] } ] }, @@ -1496,7 +1610,10 @@ { "group": "Budgets & Rate Limits", "expanded": false, - "pages": ["GET /api/governance/budgets", "GET /api/governance/rate-limits"] + "pages": [ + "GET /api/governance/budgets", + "GET /api/governance/rate-limits" + ] }, { "group": "Model Configs", @@ -1584,7 +1701,9 @@ { "group": "Customers", "expanded": false, - "pages": ["GET /api/customers/{customer_id}/teams"] + "pages": [ + "GET /api/customers/{customer_id}/teams" + ] }, { "group": "RBAC", @@ -1732,6 +1851,8 @@ "item": "Open Source", "icon": "rocket", "pages": [ + "changelogs/v2.2.2", + "changelogs/v2.2.1", "changelogs/v2.2.0", "changelogs/v2.1.1", "changelogs/v2.1.0", @@ -1941,7 +2062,10 @@ }, { "group": "September 2025", - "pages": ["changelogs/v1.2.22", "changelogs/v1.2.21"] + "pages": [ + "changelogs/v1.2.22", + "changelogs/v1.2.21" + ] } ] }, @@ -1949,6 +2073,8 @@ "item": "Enterprise", "icon": "building", "pages": [ + "changelogs/ent-v2.2.2", + "changelogs/ent-v2.2.1", "changelogs/ent-v2.2.0", "changelogs/ent-v2.1.1", "changelogs/ent-v2.1.0", @@ -2043,12 +2169,16 @@ "item": "Edge", "icon": "grip", "pages": [ + "changelogs/edge-v0.9.0", "changelogs/edge-v0.8.0", "changelogs/edge-v0.7.0", "changelogs/edge-v0.6.0", { "group": "August 2026", - "pages": ["changelogs/edge-v0.5.6", "changelogs/edge-v0.4.0"] + "pages": [ + "changelogs/edge-v0.5.6", + "changelogs/edge-v0.4.0" + ] }, { "group": "July 2026", @@ -2067,7 +2197,10 @@ "changelogs/cli-v0.10.6", { "group": "May 2026", - "pages": ["changelogs/cli-v0.10.5", "changelogs/cli-v0.10.4"] + "pages": [ + "changelogs/cli-v0.10.5", + "changelogs/cli-v0.10.4" + ] }, { "group": "March 2026", @@ -2406,4 +2539,4 @@ "linkedin": "https://linkedin.com/company/maxim-ai" } } -} \ No newline at end of file +} diff --git a/docs/enterprise/adaptive-load-balancing.mdx b/docs/enterprise/adaptive-load-balancing.mdx index 7b2f928e48f..5cad1e3fede 100644 --- a/docs/enterprise/adaptive-load-balancing.mdx +++ b/docs/enterprise/adaptive-load-balancing.mdx @@ -230,7 +230,7 @@ Unlike the API, this block is presence-aware: omitted fields keep their current - **Per-node weights**: Each node load-balances on its own observed metrics. The only signal shared across nodes is a rate-limit (TPM) backoff, and only within the same region - there is no global weight consensus or cross-region coordination. This is deliberate: latency and error profiles differ per region, so importing another region's metrics would pollute a node's view of route health. - **~5-second adaptation**: Weight and state changes lag live traffic by up to one recompute cycle. Immediate per-request resilience (key rotation and fallback failover) is handled separately and is not subject to this delay. - **Optimistic cold start**: A brand-new key or provider enters at full weight and competes at roughly fair share before it has been measured, then self-corrects within a cycle or two. -- **Relative, not absolute**: Routes are ranked against their peers, not against a fixed latency or cost target. The system is not cost-, org-, or session-aware, and does not accept manual per-key weights for the adaptive path - those concerns are handled by [governance routing](/providers/provider-routing). +- **Relative, not absolute**: Routes are ranked against their peers, not against a fixed latency or cost target. The system is not cost- or org-aware, and does not accept manual per-key weights for the adaptive path - those concerns are handled by [governance routing](/providers/provider-routing). It is not session-aware either: [Session Affinity](/providers/session-affinity) runs after it and keeps a session on its provider and key only while the load balancer still offers them and has not marked them as failed, and the circuit breaker is not holding that key back. --- diff --git a/docs/enterprise/prompt-deployments.mdx b/docs/enterprise/prompt-deployments.mdx new file mode 100644 index 00000000000..4347fa3d0ea --- /dev/null +++ b/docs/enterprise/prompt-deployments.mdx @@ -0,0 +1,285 @@ +--- +title: "Prompt Deployments" +description: "Enterprise version routing for the Prompt Repository: CEL matching rules and weighted traffic distribution decide which committed prompt version serves each request." +icon: "route" +--- + +## Overview + +A **prompt deployment** is a rule attached to a single prompt in the [Prompt Repository](/features/prompt-repository/playground) that decides **which committed version** of that prompt serves a request. Each deployment pairs a **matching rule** (a CEL expression over request attributes) with a **traffic distribution** (a weighted split across version numbers). + +This lets you ship a prompt change without changing any client code: callers keep sending the same prompt ID, and the gateway picks the version. + +**What it enables:** + +- **A/B testing** - split traffic 50/50 (or any ratio) across two committed versions and compare them in logs. +- **Staged rollout** - start a new version at 10%, widen as it proves out. +- **Conditional versions** - serve a different version per model, per provider, per request type, or per request header. + + +Prompt deployments are part of the Bifrost **enterprise** license. Open-source Bifrost ships the Prompt Repository and the [prompts plugin](/features/prompt-repository/prompts-plugin), where the caller picks the version explicitly with the `x-bf-prompt-version` header. Server-side version selection is what enterprise adds. + + +--- + +## How it works + +Deployments do not change how a prompt is selected - the request still names the prompt with the `x-bf-prompt-id` header. They only change **which version** of that prompt is injected. + +```mermaid +flowchart TB + Client([Client sends x-bf-prompt-id]) --> PreHook["HTTP pre-hook:
capture headers and query params"] + PreHook --> Match{"Evaluate this prompt's
deployments, newest first"} + Match -->|First CEL match| Split["Weighted random pick from
traffic_distribution"] + Match -->|No match| Fallback["x-bf-prompt-version header,
else latest version"] + Split --> Inject["Prepend that version's messages
and merge its model params"] + Fallback --> Inject + Inject --> Provider[Provider] +``` + +Resolution, in order: + +1. **No `x-bf-prompt-id` header** - the plugin does nothing and the request passes through untouched. +2. **Deployments for that prompt are evaluated newest-first** (by creation time, descending). A deployment with an empty traffic distribution is skipped entirely. +3. **The first deployment whose CEL expression matches wins.** Evaluation stops there - later deployments are not consulted. +4. **A version is drawn from that deployment's traffic distribution** by weighted random selection. Entries at `0%` are never drawn. +5. **If no deployment matches**, the plugin falls back to the `x-bf-prompt-version` header, and if that is absent, to the prompt's latest committed version. + + +A matching deployment **overrides** the `x-bf-prompt-version` header. The header is only honoured when no deployment matches the request. + + +--- + +## Prerequisites + +- An enterprise Bifrost deployment with a **config store that holds the Prompt Repository tables** (typically PostgreSQL). +- A prompt with **at least two committed versions** - deployments route between version numbers, so there is nothing to split until you have committed more than one. +- The `enterprise-prompts` plugin, which is initialized automatically by the enterprise gateway. There is no `config.json` section for deployments; they are stored in the database and managed through the UI or the API. + +--- + +## Creating a deployment + + + + +1. Open the **Prompt Repository** and select the prompt you want to route. Deployments are scoped to one prompt, so the panel is empty until a prompt is open. +2. In the right-hand settings panel, expand the **Deployments** section. +3. Click **Add** to open the deployment sheet. +4. Enter a **Name** (required, and unique across all prompts) and an optional **Description**. +5. Under **Matching Rules**, build the condition that selects this deployment with **Add Rule** and **Add Rule Group**, combining clauses with **AND** or **OR**. The **CEL Expression Preview** below the builder shows the expression that will actually be saved. Leave the builder empty to match every request for this prompt. +6. Under **Traffic distribution**, click **Add version** for each version you want in the split, pick the version number, and drag the grip handles on the bar to set percentages. The total must be exactly **100%**, and a version can appear only once. +7. Click **Create**. + +![Create Deployment sheet with matching rules and traffic distribution](../media/ui-prompt-deployments-sheet.png) + +Existing deployments are listed in the panel with their traffic bar; expand a row to see its full expression, or use the pencil and trash icons to edit or delete it. + + + + +**Create a deployment:** + +```bash +curl -X POST http://localhost:8080/api/prompt-repo/deployments \ + -H "Content-Type: application/json" \ + -d '{ + "prompt_id": "9f1c1b6e-1f2a-4a1e-9a77-2b6b5e0f4d21", + "name": "gpt-4o canary", + "description": "Send 10% of gpt-4o chat traffic to v3", + "cel_expression": "model == \"gpt-4o\" && request_type == \"chat_completion\"", + "traffic_distribution": [ + { "version": "2", "percentage": 90 }, + { "version": "3", "percentage": 10 } + ] + }' +``` + +**Response (`201 Created`):** + +```json +{ + "id": "1f7a5c2c-9d1b-4f0e-b3a4-7c6a2f9e8d10", + "prompt_id": "9f1c1b6e-1f2a-4a1e-9a77-2b6b5e0f4d21", + "name": "gpt-4o canary", + "description": "Send 10% of gpt-4o chat traffic to v3", + "cel_expression": "model == \"gpt-4o\" && request_type == \"chat_completion\"", + "traffic_distribution": [ + { "version": "2", "percentage": 90 }, + { "version": "3", "percentage": 10 } + ], + "created_at": "2026-09-22T10:04:11Z", + "updated_at": "2026-09-22T10:04:11Z" +} +``` + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `prompt_id` | string | Yes | ID of the prompt this deployment routes. Create only. | +| `name` | string | Yes | Display name. Unique across **all** deployments, not just this prompt. | +| `description` | string | No | Free text shown in the deployments list. | +| `cel_expression` | string | No | Matching rule. Empty or `true` matches every request for the prompt. Validated and compiled on save. | +| `query` | object | No | The rule-builder representation of the expression. The UI sends it so it can re-open the visual builder; it is not evaluated at request time. | +| `traffic_distribution` | array | No | Entries of `{ "version": "", "percentage": }`. Percentages must sum to exactly `100`. A deployment without one never matches. | + +**Endpoints:** + +| Method | Path | Description | +|--------|------|-------------| +| `GET` | `/api/prompt-repo/deployments?prompt_id=` | List deployments. Omit `prompt_id` to list all. | +| `GET` | `/api/prompt-repo/deployments/{id}` | Fetch one deployment. | +| `POST` | `/api/prompt-repo/deployments` | Create a deployment. | +| `PUT` | `/api/prompt-repo/deployments/{id}` | Replace name, description, expression, and distribution. | +| `DELETE` | `/api/prompt-repo/deployments/{id}` | Delete a deployment. | + +**Errors:** + +| Status | Body | Cause | +|--------|------|-------| +| `400` | `prompt_id is required` / `name is required` | Missing required field. | +| `400` | `CEL compile error: ...` | The expression does not compile against the available variables. | +| `400` | `traffic distribution percentages must sum to 100` | Percentages do not total 100. | +| `400` | `traffic distribution percentage must be between 0 and 100` | An entry is out of range. | +| `409` | `a deployment with that name already exists` | The name is taken. | +| `404` | `deployment not found` | Unknown ID, or a prompt outside your [data access control](/enterprise/data-access-control) scope. | + + + + +--- + +## Matching rules + +Matching rules are [CEL](https://github.com/google/cel-spec) expressions, the same language used by [routing rules](/features/governance/routing). An expression must evaluate to a boolean. These variables are available: + +| Variable | Type | Description | +|----------|------|-------------| +| `model` | string | Model on the request, e.g. `gpt-4o`. | +| `provider` | string | Provider on the request, e.g. `openai`. | +| `request_type` | string | Normalized request type, e.g. `chat_completion`, `chat_completion_stream`, `responses`, `text_completion`. Streaming and non-streaming are distinct values. | +| `headers` | map(string, string) | Request headers. Keys are lowercased. | +| `params` | map(string, string) | Query parameters. Keys are lowercased. | +| `tokens_used` | double | Token usage as a percentage of the applicable limit. | +| `request` | double | Request-count usage as a percentage of the applicable limit. | +| `budget_used` | double | Budget usage as a percentage of the applicable limit. | + +Examples: + +```javascript +// Everything for this prompt +true + +// One model, non-streaming chat only +model == "gpt-4o" && request_type == "chat_completion" + +// Internal callers, identified by a header +headers["x-team"] == "growth" + +// Anthropic traffic when the budget is nearly exhausted +provider == "anthropic" && budget_used >= 90.0 +``` + + +Header and query-parameter keys are lowercased both when the expression is saved and when it is evaluated, so `headers["X-Team"]` and `headers["x-team"]` behave identically. Values are matched as-is and are case-sensitive. + + + +`tokens_used`, `request`, and `budget_used` are `0.0` unless the [governance](/features/governance/budget-and-limits) plugin is active to supply live usage. Without it, a rule like `budget_used >= 90.0` never matches. + + +--- + +## Traffic distribution + +The traffic distribution is the weighted split that decides the version once a deployment matches. + +- Percentages must sum to **exactly 100**; each entry must be between 0 and 100. +- Each version may appear **at most once** in a distribution. +- Entries at `0%` are excluded from the draw, so you can park a version without removing it. +- The version is chosen by **weighted random selection on every request**. + + +Selection is per request and stateless - it is **not** sticky per user, session, or conversation. Two consecutive calls from the same client can land on different versions. If a flow depends on the same version across several turns, pin it explicitly with `x-bf-prompt-version` and leave that traffic out of the deployment's matching rule. + + +A deployment with **no** traffic distribution is inert: it is skipped during resolution even if its expression would match. Give every deployment a distribution, or delete it. + +--- + +## Confirming which version served a request + +When a prompt is resolved, the gateway echoes the decision on the response: + +| Header | Description | +|--------|-------------| +| `bf-resolved-prompt-id` | The prompt that was injected. | +| `bf-resolved-prompt-version` | The version number that was selected. `0` means the prompt's latest committed version was used. | + +These are the fastest way to verify a split is behaving - send a batch of identical requests and count the versions that come back. + +--- + +## Ordering and overlap + +Deployments for a prompt are evaluated **newest-first**, and the first match wins. Two consequences are worth planning around: + +- A newly created deployment takes precedence over older ones whose rules overlap with it. +- A catch-all deployment (`true`, or an empty expression) shadows every deployment created **before** it. Create catch-alls first, narrow rules after. + +--- + +## Permissions and visibility + +Deployments are governed by the `PromptDeploymentStrategy` [RBAC](/enterprise/rbac) resource, with `Create`, `View`, `Update`, and `Delete` operations. In the default roles: + +| Role | Permissions | +|------|-------------| +| Admin | Create, View, Update, Delete | +| Developer | Create, View, Update, Delete | +| Viewer | View | + +[Data access control](/enterprise/data-access-control) scope is **inherited from the parent prompt**: a deployment attached to a prompt outside your scope is not listed, and fetching it by ID returns `404`. + +--- + +## Clustering + +In a [cluster](/enterprise/clustering), every create, update, and delete reloads the local deployment cache, gossips to the other nodes under the `prompt_deployment` entity type, and notifies connected UI clients. CEL expressions are compiled once at cache load, so matching stays off the request path. + +--- + +## Troubleshooting + +### My deployment never serves its new version + +**Symptom:** `bf-resolved-prompt-version` always comes back as the old version, or as `0`. +**Causes:** the deployment has no traffic distribution; a newer deployment with an overlapping rule is matching first; or the expression does not match the request. +**Fix:** confirm the distribution totals 100%, check whether a more recently created deployment shadows this one, and simplify the expression to `true` temporarily to confirm the rest of the path works. + +### Saving the deployment returns a CEL compile error + +**Symptom:** `400 CEL compile error: undeclared reference to ''`. +**Cause:** the expression references a variable that is not in the deployment CEL environment. +**Fix:** rewrite the rule using the variables listed under [Matching rules](#matching-rules). + +### Versions alternate between requests from the same user + +**Symptom:** a multi-turn conversation gets different prompt versions on different turns. +**Cause:** weighted selection is per request and not sticky. +**Fix:** pin the version with `x-bf-prompt-version` for those requests, and exclude them from the deployment's matching rule. + +### Budget or usage rules never match + +**Symptom:** a rule on `budget_used`, `tokens_used`, or `request` never fires. +**Cause:** governance is not supplying usage data, so all three evaluate to `0.0`. +**Fix:** enable the governance plugin and configure budgets or rate limits for the provider and model in question. + +--- + +## Next steps + +- **[Prompt Repository](/features/prompt-repository/playground)** - author prompts, commit versions, and find a prompt's ID. +- **[Prompts plugin](/features/prompt-repository/prompts-plugin)** - how committed versions are injected into requests, and the header contract. +- **[Routing rules](/features/governance/routing)** - the same CEL builder applied to provider and model selection. +- **[RBAC](/enterprise/rbac)** - grant or restrict deployment management per role. diff --git a/docs/features/governance/complexity-router.mdx b/docs/features/governance/complexity-router.mdx index 836c48a09e0..20045a1e3a7 100644 --- a/docs/features/governance/complexity-router.mdx +++ b/docs/features/governance/complexity-router.mdx @@ -70,7 +70,7 @@ Complexity Router does not currently classify Codex requests sent through native -Session-aware routing keeps the **complexity tier** stable; it does not pin a weighted routing target, provider key, or provider prompt-cache entry. Provider cache TTLs remain provider-owned and independent of the 24-hour routing-state lifetime. +Session-aware routing keeps the **complexity tier** stable; it does not pin a weighted routing target, provider key, or provider prompt-cache entry. Provider cache TTLs remain provider-owned and independent of the 24-hour routing-state lifetime. Keeping a session on one provider and key is the job of [Session Affinity](/providers/session-affinity), which uses the same session identity and runs alongside the router. --- diff --git a/docs/features/prompt-repository/playground.mdx b/docs/features/prompt-repository/playground.mdx index c5970a7b27c..da5930dd4fa 100644 --- a/docs/features/prompt-repository/playground.mdx +++ b/docs/features/prompt-repository/playground.mdx @@ -90,7 +90,7 @@ The playground uses a simple **three-panel layout**: The settings panel is organized into collapsible sections: - **Configuration** - Provider, model, API key, variables, and model parameters -- **Deployments** - Prompt deployment strategies and traffic routing (enterprise) +- **[Deployments](/enterprise/prompt-deployments)** - Prompt deployment strategies and traffic routing (enterprise) ![Workspace Layout](../../media/prompt-repo-layout.png) diff --git a/docs/features/prompt-repository/prompts-plugin.mdx b/docs/features/prompt-repository/prompts-plugin.mdx index 70c989a51a3..584e54574cf 100644 --- a/docs/features/prompt-repository/prompts-plugin.mdx +++ b/docs/features/prompt-repository/prompts-plugin.mdx @@ -52,6 +52,10 @@ If the prompt ID is missing, the plugin does nothing and the request passes thro Invalid or unknown IDs / versions are logged as warnings; the request is **not** failed by the plugin (it proceeds without template injection). + +On enterprise, a matching [prompt deployment](/enterprise/prompt-deployments) selects the version server-side and overrides `x-bf-prompt-version`. The header is honoured only when no deployment matches. + + --- ## Example: Chat Completions @@ -144,5 +148,6 @@ After injection, the plugin sets the following context keys (read by the logging ## Related - [Playground](/features/prompt-repository/playground) - create folders, prompts, sessions, and committed versions. +- [Prompt Deployments](/enterprise/prompt-deployments) - enterprise server-side version selection with CEL rules and traffic splits. - [Writing Go plugins](/plugins/writing-go-plugin) - plugin interfaces and lifecycle. - Built-in plugin name in code: `prompts` (`github.com/capsohq/bifrost/plugins/prompts`). diff --git a/docs/integrations/typesafe-sdk.mdx b/docs/integrations/typesafe-sdk.mdx new file mode 100644 index 00000000000..2636a3318a3 --- /dev/null +++ b/docs/integrations/typesafe-sdk.mdx @@ -0,0 +1,95 @@ +--- +title: "TypeSafe SDK" +description: "Use TypeSafe's official Python and JavaScript SDKs with Bifrost by changing one base URL." +icon: "scale-balanced" +--- + +Bifrost exposes TypeSafe's native API 1:1 under the `/typesafe` prefix, so the official TypeSafe SDKs work by pointing their base URL at Bifrost. You keep the SDK's question builders, retries, and types - and gain Bifrost's key management, governance, logging, and cost tracking. + +## Endpoints + +| Native TypeSafe | Via Bifrost | +|-----------------|-------------| +| `POST https://api.typesafe.ai/v1/systemone` | `POST http://localhost:8080/typesafe/v1/systemone` | +| Model listing (not offered upstream) | `GET http://localhost:8080/typesafe/v1/models` | + +## Python SDK + +Both SDKs read `TYPESAFE_BASE_URL` from the environment, so the zero-code-change setup is: + +```bash +export TYPESAFE_BASE_URL="http://localhost:8080/typesafe" +``` + +Or pass it explicitly (`typesafe-sdk` package): + +```python +from typesafe_sdk import TypeSafeClient, Noul + +with TypeSafeClient(base_url="http://localhost:8080/typesafe") as client: + response = client.system_one( + state={"document": "I was charged twice. Please fix this ASAP."}, + questions={"billing": Noul(instructions="Is this ticket about billing?")}, + ) + print(response.nouls["billing"].noul) +``` + +## JavaScript / TypeScript SDK + +`@typesafe-ai/sdk` takes `baseURL` (falls back to `TYPESAFE_BASE_URL`): + +```ts +import { TypeSafeClient, choice } from "@typesafe-ai/sdk"; + +const client = new TypeSafeClient({ baseURL: "http://localhost:8080/typesafe" }); + +const response = await client.systemOne({ + state: { document: "I was charged twice. Please fix this ASAP." }, + questions: { + category: choice("What is this ticket about?", { + billing: null, + technical: null, + other: null, + }), + }, +}); +``` + +## Authentication + +The SDK's `Authorization: Bearer` header authenticates the request to Bifrost when Bifrost authentication is enabled. Configure the TypeSafe provider with its upstream keys - Bifrost selects and injects the provider key on the upstream call, so clients can use Bifrost virtual keys instead of raw TypeSafe keys. + +**Using a virtual key** - pass it as the SDK's API key; governance (budgets, rate limits, model restrictions) applies per key: + +```python +from typesafe_sdk import TypeSafeClient, Noul + +with TypeSafeClient( + base_url="http://localhost:8080/typesafe", + api_key="sk-bf-v1-your-virtual-key", # Bifrost virtual key, not a TypeSafe key +) as client: + response = client.system_one( + state={"document": "I was charged twice."}, + questions={"billing": Noul(instructions="Is this ticket about billing?")}, + ) +``` + +```ts +import { TypeSafeClient } from "@typesafe-ai/sdk"; + +const client = new TypeSafeClient({ + baseURL: "http://localhost:8080/typesafe", + apiKey: "sk-bf-v1-your-virtual-key", // Bifrost virtual key +}); +``` + +## Behavior Notes + +- **Success responses are shape-compatible** with TypeSafe's own API: native `answers` with `noul`/`choice`/`score` value fields and `input_tokens`/`output_tokens` usage. Bytes are identical only on the raw-response passthrough path; otherwise the response is rebuilt and re-encoded, preserving the shape but not exact JSON byte ordering. +- **Models**: bare IDs (`jev-1.13.0`, `jev-latest`, `jev-preview`) work exactly as upstream; `typesafe/`-prefixed IDs are additionally accepted. +- **Errors**: returned in TypeSafe's native shape (`{"detail": {"error_type", "message"}}`) with upstream status codes (401, 422, 429, 529) preserved, so SDK exception parsing works unchanged. Requests rejected by Bifrost's local validation return `400` where TypeSafe's own validation would return `422`. +- **Retries**: Bifrost retries 429 and 529 with backoff per the provider's `network_config` in addition to any SDK-side retry policy. + + +For provider-routed access with fallbacks and Bifrost's normalized response shape, use [`POST /v1/decisions`](/quickstart/gateway/decisions) instead - same questions, `kind` instead of `type`, and a unified `value` per answer. + diff --git a/docs/mcp/connecting-to-servers.mdx b/docs/mcp/connecting-to-servers.mdx index cd4c5fa26e1..c7b91c51faa 100644 --- a/docs/mcp/connecting-to-servers.mdx +++ b/docs/mcp/connecting-to-servers.mdx @@ -578,6 +578,14 @@ Environment variables are: - Redacted in API responses and UI for security - Validated at startup to ensure all required variables are set +## Egress Proxies + +HTTP and SSE MCP connections honor the standard `HTTP_PROXY`, `HTTPS_PROXY`, and `NO_PROXY` variables from the Bifrost process environment, the same way every other outbound HTTP client in the process does. This is how a deployment with no direct internet route (a locked-down VPC, or a pod behind a corporate egress proxy) reaches a public MCP server. Set the variables on the container or task definition; a `NO_PROXY` entry keeps an internal MCP server on the direct path. + +The `network_config.proxy_config` block in `config.json` does not apply to MCP connections. Its `enable_for_*` flags cover inference, SCIM, and API clients only. + +The link-local and cloud-metadata block still applies when a proxy is in use: a connection string whose host is a link-local or metadata IP literal is refused before the proxy is contacted. A hostname on the proxied path must resolve on the Bifrost host, and every address it resolves to must pass the same block, before Bifrost sends the request to the proxy. A name Bifrost cannot resolve is refused, so in a deployment whose DNS lives only behind the proxy, Bifrost must still be able to resolve the MCP servers it connects to. + --- ## Forwarding Request Headers to MCP Servers diff --git a/docs/media/ui-prompt-deployments-sheet.png b/docs/media/ui-prompt-deployments-sheet.png new file mode 100644 index 00000000000..27a5965ea81 Binary files /dev/null and b/docs/media/ui-prompt-deployments-sheet.png differ diff --git a/docs/openapi/openapi.json b/docs/openapi/openapi.json index e26887f38ac..827803f132e 100644 --- a/docs/openapi/openapi.json +++ b/docs/openapi/openapi.json @@ -61,6 +61,10 @@ "name": "Rerank", "description": "Document reranking by relevance to a query" }, + { + "name": "Decisions", + "description": "Structured decisions evaluated against annotated function-tool definitions" + }, { "name": "Embeddings", "description": "Text embedding generation" @@ -125,6 +129,10 @@ "name": "Cohere Integration", "description": "Cohere compatible API endpoints (/cohere/*)" }, + { + "name": "Typesafe Integration", + "description": "Typesafe compatible API endpoints (/typesafe/*)" + }, { "name": "LiteLLM Integration", "description": "LiteLLM proxy endpoints with multi-provider support (/litellm/*)" @@ -1394,11 +1402,21 @@ } ] }, + "async": { + "type": "boolean" + }, "action": { "type": "object" }, "error": { - "type": "string" + "oneOf": [ + { + "type": "string" + }, + { + "type": "object" + } + ] }, "queries": { "type": "array", @@ -1935,6 +1953,69 @@ ] } }, + "/v1/decisions": { + "post": { + "operationId": "createDecision", + "summary": "Evaluate a decision", + "description": "Evaluates state against a map of named questions and returns one typed\nanswer per question, with per-answer confidence, probabilities, and\nlegend metadata. Served by decision providers such as Typesafe's jev models.\n", + "tags": [ + "Decisions" + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DecisionRequest" + } + } + } + }, + "responses": { + "200": { + "description": "Successful response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DecisionResponse" + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BifrostError" + } + } + } + }, + "500": { + "description": "Internal server error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BifrostError" + } + } + } + } + }, + "security": [ + { + "BearerAuth": [] + }, + { + "VirtualKeyAuth": [] + }, + { + "ApiKeyAuth": [] + } + ] + } + }, "/v1/embeddings": { "post": { "operationId": "createEmbedding", @@ -5584,11 +5665,21 @@ } ] }, + "async": { + "type": "boolean" + }, "action": { "type": "object" }, "error": { - "type": "string" + "oneOf": [ + { + "type": "string" + }, + { + "type": "object" + } + ] }, "queries": { "type": "array", @@ -6096,11 +6187,21 @@ } ] }, + "async": { + "type": "boolean" + }, "action": { "type": "object" }, "error": { - "type": "string" + "oneOf": [ + { + "type": "string" + }, + { + "type": "object" + } + ] }, "queries": { "type": "array", @@ -6805,11 +6906,21 @@ } ] }, + "async": { + "type": "boolean" + }, "action": { "type": "object" }, "error": { - "type": "string" + "oneOf": [ + { + "type": "string" + }, + { + "type": "object" + } + ] }, "queries": { "type": "array", @@ -13699,11 +13810,21 @@ } ] }, + "async": { + "type": "boolean" + }, "action": { "type": "object" }, "error": { - "type": "string" + "oneOf": [ + { + "type": "string" + }, + { + "type": "object" + } + ] }, "queries": { "type": "array", @@ -14643,11 +14764,21 @@ } ] }, + "async": { + "type": "boolean" + }, "action": { "type": "object" }, "error": { - "type": "string" + "oneOf": [ + { + "type": "string" + }, + { + "type": "object" + } + ] }, "queries": { "type": "array", @@ -15624,11 +15755,21 @@ } ] }, + "async": { + "type": "boolean" + }, "action": { "type": "object" }, "error": { - "type": "string" + "oneOf": [ + { + "type": "string" + }, + { + "type": "object" + } + ] }, "queries": { "type": "array", @@ -16720,11 +16861,21 @@ } ] }, + "async": { + "type": "boolean" + }, "action": { "type": "object" }, "error": { - "type": "string" + "oneOf": [ + { + "type": "string" + }, + { + "type": "object" + } + ] }, "queries": { "type": "array", @@ -17232,11 +17383,21 @@ } ] }, + "async": { + "type": "boolean" + }, "action": { "type": "object" }, "error": { - "type": "string" + "oneOf": [ + { + "type": "string" + }, + { + "type": "object" + } + ] }, "queries": { "type": "array", @@ -27638,6 +27799,300 @@ ] } }, + "/typesafe/v1/systemone": { + "post": { + "operationId": "typesafeSystemOne", + "summary": "Evaluate state (Typesafe format)", + "description": "Evaluates state against a map of named questions using Typesafe's native\nSystem One API. The request body matches Typesafe's native format; question\nidentifiers become result field names.\n", + "tags": [ + "Typesafe Integration" + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "state", + "model", + "questions" + ], + "properties": { + "state": { + "description": "A plain string, or structured data (object or array) to evaluate.", + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + "additionalProperties": true + }, + { + "type": "array", + "items": {} + } + ] + }, + "model": { + "type": "string", + "description": "Typesafe model identifier (e.g. `jev-1.13.0`, `jev-latest`)." + }, + "questions": { + "type": "object", + "minProperties": 1, + "description": "Map of named questions.", + "additionalProperties": { + "type": "object", + "required": [ + "type", + "instructions" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "noul", + "choice", + "score" + ], + "description": "Question type. `noul` returns a probability from 0 to 1, `choice` a selection with probabilities, `score` a probability-weighted rubric value." + }, + "instructions": { + "description": "Question instructions. A string, or structured data.", + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + "additionalProperties": true + }, + { + "type": "array", + "items": {} + } + ] + }, + "criteria": { + "description": "Optional `true`/`false` descriptions for `noul`; a map of option\nto description for `choice` (at most 255 options); an ordered\narray of 2-10 level descriptions for `score`.\n", + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + } + } + } + } + } + } + } + } + }, + "responses": { + "200": { + "description": "Successful response in Typesafe's native shape", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "model", + "answers" + ], + "properties": { + "model": { + "type": "string", + "description": "Model version that performed the evaluation" + }, + "answers": { + "type": "object", + "description": "Map matching question identifiers to answers", + "additionalProperties": { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "noul", + "choice", + "score" + ] + }, + "noul": { + "type": "number", + "description": "Probability from 0 to 1 (noul answers)" + }, + "choice": { + "type": "string", + "description": "Selected option (choice answers)" + }, + "score": { + "type": "number", + "description": "Probability-weighted value (score answers)" + }, + "probabilities": { + "type": "object", + "additionalProperties": { + "type": "number" + } + }, + "legend": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "confidence": { + "type": "number" + } + } + } + }, + "usage": { + "type": "object", + "properties": { + "input_tokens": { + "type": "integer" + }, + "output_tokens": { + "type": "integer" + } + } + } + } + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BifrostError" + } + } + } + }, + "500": { + "description": "Internal server error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BifrostError" + } + } + } + } + }, + "security": [ + { + "BearerAuth": [] + }, + { + "VirtualKeyAuth": [] + }, + { + "ApiKeyAuth": [] + } + ] + } + }, + "/typesafe/v1/models": { + "get": { + "operationId": "typesafeListModels", + "summary": "List models (Typesafe format)", + "description": "Lists available Typesafe models in the native shape. Typesafe documents no\nupstream listing endpoint, so the catalog is served from Bifrost's model\ndatasheet.\n", + "tags": [ + "Typesafe Integration" + ], + "responses": { + "200": { + "description": "Successful response", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "models" + ], + "properties": { + "models": { + "type": "array", + "items": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "example": "jev-latest" + }, + "description": { + "type": "string" + }, + "release_date": { + "type": "string", + "format": "date-time" + } + } + } + } + } + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BifrostError" + } + } + } + }, + "500": { + "description": "Internal server error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BifrostError" + } + } + } + } + }, + "security": [ + { + "BearerAuth": [] + }, + { + "VirtualKeyAuth": [] + }, + { + "ApiKeyAuth": [] + } + ] + } + }, "/cohere/v1/tokenize": { "post": { "operationId": "cohereTokenize", @@ -30402,11 +30857,21 @@ } ] }, + "async": { + "type": "boolean" + }, "action": { "type": "object" }, "error": { - "type": "string" + "oneOf": [ + { + "type": "string" + }, + { + "type": "object" + } + ] }, "queries": { "type": "array", @@ -33490,11 +33955,21 @@ } ] }, + "async": { + "type": "boolean" + }, "action": { "type": "object" }, "error": { - "type": "string" + "oneOf": [ + { + "type": "string" + }, + { + "type": "object" + } + ] }, "queries": { "type": "array", @@ -36689,11 +37164,21 @@ } ] }, + "async": { + "type": "boolean" + }, "action": { "type": "object" }, "error": { - "type": "string" + "oneOf": [ + { + "type": "string" + }, + { + "type": "object" + } + ] }, "queries": { "type": "array", @@ -43893,11 +44378,21 @@ } ] }, + "async": { + "type": "boolean" + }, "action": { "type": "object" }, "error": { - "type": "string" + "oneOf": [ + { + "type": "string" + }, + { + "type": "object" + } + ] }, "queries": { "type": "array", @@ -99026,11 +99521,21 @@ } ] }, + "async": { + "type": "boolean" + }, "action": { "type": "object" }, "error": { - "type": "string" + "oneOf": [ + { + "type": "string" + }, + { + "type": "object" + } + ] }, "queries": { "type": "array", @@ -99087,7 +99592,19 @@ "type": "boolean" }, "conversation": { - "type": "string" + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + } + } + } + ] }, "context_management": { "$ref": "#/components/schemas/ContextManagement" @@ -99319,6 +99836,9 @@ "description": { "type": "string" }, + "async": { + "type": "boolean" + }, "cache_control": { "$ref": "#/components/schemas/CacheControl" }, @@ -99355,6 +99875,9 @@ "strict": { "type": "boolean" }, + "output_schema": { + "type": "object" + }, "vector_store_ids": { "type": "array", "items": { @@ -99394,8 +99917,21 @@ "server_url": { "type": "string" }, + "tunnel_id": { + "type": "string" + }, "allowed_tools": { - "type": "object" + "oneOf": [ + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "type": "object" + } + ] }, "authorization": { "type": "string" @@ -99931,11 +100467,21 @@ } ] }, + "async": { + "type": "boolean" + }, "action": { "type": "object" }, "error": { - "type": "string" + "oneOf": [ + { + "type": "string" + }, + { + "type": "object" + } + ] }, "queries": { "type": "array", @@ -100193,6 +100739,9 @@ "description": { "type": "string" }, + "async": { + "type": "boolean" + }, "cache_control": { "$ref": "#/components/schemas/CacheControl" }, @@ -100229,6 +100778,9 @@ "strict": { "type": "boolean" }, + "output_schema": { + "type": "object" + }, "vector_store_ids": { "type": "array", "items": { @@ -100268,8 +100820,21 @@ "server_url": { "type": "string" }, + "tunnel_id": { + "type": "string" + }, "allowed_tools": { - "type": "object" + "oneOf": [ + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "type": "object" + } + ] }, "authorization": { "type": "string" @@ -100854,11 +101419,21 @@ } ] }, + "async": { + "type": "boolean" + }, "action": { "type": "object" }, "error": { - "type": "string" + "oneOf": [ + { + "type": "string" + }, + { + "type": "object" + } + ] }, "queries": { "type": "array", @@ -101040,6 +101615,180 @@ } } }, + "DecisionRequest": { + "type": "object", + "required": [ + "model", + "state", + "questions" + ], + "properties": { + "model": { + "type": "string", + "description": "Model in provider/model format", + "example": "typesafe/jev-1.13.0" + }, + "state": { + "description": "State to evaluate. A plain string, or structured data (object or array).", + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + "additionalProperties": true + }, + { + "type": "array", + "items": {} + } + ] + }, + "questions": { + "type": "object", + "minProperties": 1, + "description": "Map of named questions. Question identifiers become answer field names.", + "additionalProperties": { + "$ref": "#/components/schemas/DecisionQuestion" + } + }, + "fallbacks": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Fallback models in provider/model format" + } + } + }, + "DecisionResponse": { + "type": "object", + "required": [ + "model", + "answers" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique identifier for the decision response" + }, + "model": { + "type": "string", + "description": "Model that performed the decision" + }, + "answers": { + "type": "object", + "description": "Map matching question identifiers to answers. Every requested question produces an answer.", + "additionalProperties": { + "$ref": "#/components/schemas/DecisionAnswer" + } + }, + "usage": { + "$ref": "#/components/schemas/BifrostLLMUsage" + }, + "extra_fields": { + "$ref": "#/components/schemas/BifrostResponseExtraFields" + } + } + }, + "DecisionQuestion": { + "type": "object", + "required": [ + "kind", + "instructions" + ], + "properties": { + "kind": { + "type": "string", + "enum": [ + "noul", + "choice", + "score" + ], + "description": "How the question is decided. `noul` yields a probability between 0 and 1, `choice` yields one option from the criteria, `score` yields a numeric rubric score including fractions." + }, + "instructions": { + "description": "Instructions for this question. A string, or structured data preserved losslessly.", + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + "additionalProperties": true + }, + { + "type": "array", + "items": {} + } + ] + }, + "criteria": { + "description": "Applicable criteria. Optional `true`/`false` descriptions for `noul`; a\nrequired map of option to description for `choice` (at most 255 options);\na required ordered array of 2-10 level descriptions for `score`.\n", + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + } + } + }, + "DecisionAnswer": { + "type": "object", + "required": [ + "kind", + "value" + ], + "properties": { + "kind": { + "type": "string", + "enum": [ + "noul", + "choice", + "score" + ], + "description": "The question's decision kind" + }, + "value": { + "description": "The decided value. A number in [0,1] for `noul`, an option string for `choice`, a numeric rubric score for `score`.", + "oneOf": [ + { + "type": "number" + }, + { + "type": "string" + } + ] + }, + "confidence": { + "type": "number", + "description": "Model confidence for this answer, when supplied" + }, + "probabilities": { + "type": "object", + "additionalProperties": { + "type": "number" + }, + "description": "Probability distribution over options or levels, when supplied" + }, + "legend": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "description": "Legend mapping levels to descriptions, when supplied" + } + } + }, "EmbeddingRequest": { "type": "object", "required": [ @@ -101934,11 +102683,21 @@ } ] }, + "async": { + "type": "boolean" + }, "action": { "type": "object" }, "error": { - "type": "string" + "oneOf": [ + { + "type": "string" + }, + { + "type": "object" + } + ] }, "queries": { "type": "array", @@ -102017,6 +102776,9 @@ "description": { "type": "string" }, + "async": { + "type": "boolean" + }, "cache_control": { "$ref": "#/components/schemas/CacheControl" }, @@ -102053,6 +102815,9 @@ "strict": { "type": "boolean" }, + "output_schema": { + "type": "object" + }, "vector_store_ids": { "type": "array", "items": { @@ -102092,8 +102857,21 @@ "server_url": { "type": "string" }, + "tunnel_id": { + "type": "string" + }, "allowed_tools": { - "type": "object" + "oneOf": [ + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "type": "object" + } + ] }, "authorization": { "type": "string" @@ -104158,11 +104936,21 @@ } ] }, + "async": { + "type": "boolean" + }, "action": { "type": "object" }, "error": { - "type": "string" + "oneOf": [ + { + "type": "string" + }, + { + "type": "object" + } + ] }, "queries": { "type": "array", @@ -104408,6 +105196,9 @@ "description": { "type": "string" }, + "async": { + "type": "boolean" + }, "cache_control": { "$ref": "#/components/schemas/CacheControl" }, @@ -104444,6 +105235,9 @@ "strict": { "type": "boolean" }, + "output_schema": { + "type": "object" + }, "vector_store_ids": { "type": "array", "items": { @@ -104483,8 +105277,21 @@ "server_url": { "type": "string" }, + "tunnel_id": { + "type": "string" + }, "allowed_tools": { - "type": "object" + "oneOf": [ + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "type": "object" + } + ] }, "authorization": { "type": "string" @@ -112962,9 +113769,43 @@ "fallbacks": { "type": "array", "items": { - "type": "string" + "description": "A fallback in a routing rule's chain. Either the \"provider/model\" string (\"provider/\" keeps the incoming model), or an object that additionally pins a provider key.", + "oneOf": [ + { + "type": "string", + "minLength": 1 + }, + { + "type": "object", + "required": [ + "provider" + ], + "properties": { + "provider": { + "type": "string", + "minLength": 1, + "description": "Fallback provider" + }, + "model": { + "type": [ + "string", + "null" + ], + "description": "Fallback model (omit or empty to use the incoming request model)" + }, + "key_id": { + "type": [ + "string", + "null" + ], + "description": "UUID of the API key to pin for this fallback (omit for load-balanced key selection). A fallback whose pinned key is missing from the provider's pool is skipped." + } + }, + "additionalProperties": false + } + ] }, - "description": "Fallback providers in format \"provider/model\"" + "description": "Ordered fallback chain, tried after the selected target fails" }, "scope": { "type": "string", @@ -113147,9 +113988,43 @@ "fallbacks": { "type": "array", "items": { - "type": "string" + "description": "A fallback in a routing rule's chain. Either the \"provider/model\" string (\"provider/\" keeps the incoming model), or an object that additionally pins a provider key.", + "oneOf": [ + { + "type": "string", + "minLength": 1 + }, + { + "type": "object", + "required": [ + "provider" + ], + "properties": { + "provider": { + "type": "string", + "minLength": 1, + "description": "Fallback provider" + }, + "model": { + "type": [ + "string", + "null" + ], + "description": "Fallback model (omit or empty to use the incoming request model)" + }, + "key_id": { + "type": [ + "string", + "null" + ], + "description": "UUID of the API key to pin for this fallback (omit for load-balanced key selection). A fallback whose pinned key is missing from the provider's pool is skipped." + } + }, + "additionalProperties": false + } + ] }, - "description": "Fallback providers in format \"provider/model\"" + "description": "Ordered fallback chain, tried after the selected target fails" }, "scope": { "type": "string", @@ -113285,8 +114160,43 @@ "fallbacks": { "type": "array", "items": { - "type": "string" - } + "description": "A fallback in a routing rule's chain. Either the \"provider/model\" string (\"provider/\" keeps the incoming model), or an object that additionally pins a provider key.", + "oneOf": [ + { + "type": "string", + "minLength": 1 + }, + { + "type": "object", + "required": [ + "provider" + ], + "properties": { + "provider": { + "type": "string", + "minLength": 1, + "description": "Fallback provider" + }, + "model": { + "type": [ + "string", + "null" + ], + "description": "Fallback model (omit or empty to use the incoming request model)" + }, + "key_id": { + "type": [ + "string", + "null" + ], + "description": "UUID of the API key to pin for this fallback (omit for load-balanced key selection). A fallback whose pinned key is missing from the provider's pool is skipped." + } + }, + "additionalProperties": false + } + ] + }, + "description": "Ordered fallback chain, tried after the selected target fails" }, "priority": { "type": "integer" @@ -115904,11 +116814,21 @@ } ] }, + "async": { + "type": "boolean" + }, "action": { "type": "object" }, "error": { - "type": "string" + "oneOf": [ + { + "type": "string" + }, + { + "type": "object" + } + ] }, "queries": { "type": "array", @@ -116365,11 +117285,21 @@ } ] }, + "async": { + "type": "boolean" + }, "action": { "type": "object" }, "error": { - "type": "string" + "oneOf": [ + { + "type": "string" + }, + { + "type": "object" + } + ] }, "queries": { "type": "array", diff --git a/docs/openapi/openapi.yaml b/docs/openapi/openapi.yaml index 3e015f4da29..4371d727a6d 100644 --- a/docs/openapi/openapi.yaml +++ b/docs/openapi/openapi.yaml @@ -86,6 +86,8 @@ tags: description: Optical character recognition for documents and images - name: Rerank description: Document reranking by relevance to a query + - name: Decisions + description: Structured decisions evaluated against annotated function-tool definitions - name: Embeddings description: Text embedding generation - name: Images @@ -119,6 +121,8 @@ tags: description: AWS Bedrock compatible API endpoints (/bedrock/*) - name: Cohere Integration description: Cohere compatible API endpoints (/cohere/*) + - name: Typesafe Integration + description: Typesafe compatible API endpoints (/typesafe/*) - name: LiteLLM Integration description: LiteLLM proxy endpoints with multi-provider support (/litellm/*) - name: LangChain Integration @@ -171,6 +175,8 @@ paths: $ref: "./paths/inference/ocr.yaml#/ocr" /v1/rerank: $ref: "./paths/inference/rerank.yaml#/rerank" + /v1/decisions: + $ref: "./paths/inference/decisions.yaml#/decisions" /v1/embeddings: $ref: "./paths/inference/embeddings.yaml#/embeddings" /v1/audio/speech: @@ -519,6 +525,12 @@ paths: /cohere/v2/rerank: $ref: "./paths/integrations/cohere/rerank.yaml#/rerank" + # Typesafe Integration + /typesafe/v1/systemone: + $ref: "./paths/integrations/typesafe/systemone.yaml#/systemone" + /typesafe/v1/models: + $ref: "./paths/integrations/typesafe/systemone.yaml#/models" + # Tokenize (v1) /cohere/v1/tokenize: $ref: "./paths/integrations/cohere/tokenize.yaml#/tokenize" @@ -1515,6 +1527,16 @@ components: RerankResult: $ref: "./schemas/inference/rerank.yaml#/RerankResult" + # ==================== Decisions ==================== + DecisionRequest: + $ref: "./schemas/inference/decisions.yaml#/DecisionRequest" + DecisionResponse: + $ref: "./schemas/inference/decisions.yaml#/DecisionResponse" + DecisionQuestion: + $ref: "./schemas/inference/decisions.yaml#/DecisionQuestion" + DecisionAnswer: + $ref: "./schemas/inference/decisions.yaml#/DecisionAnswer" + # ==================== Embeddings ==================== EmbeddingRequest: $ref: "./schemas/inference/embeddings.yaml#/EmbeddingRequest" diff --git a/docs/openapi/paths/inference/decisions.yaml b/docs/openapi/paths/inference/decisions.yaml new file mode 100644 index 00000000000..fd2c171b13d --- /dev/null +++ b/docs/openapi/paths/inference/decisions.yaml @@ -0,0 +1,31 @@ +decisions: + post: + operationId: createDecision + summary: Evaluate a decision + description: | + Evaluates state against a map of named questions and returns one typed + answer per question, with per-answer confidence, probabilities, and + legend metadata. Served by decision providers such as Typesafe's jev models. + tags: + - Decisions + requestBody: + required: true + content: + application/json: + schema: + $ref: '../../schemas/inference/decisions.yaml#/DecisionRequest' + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '../../schemas/inference/decisions.yaml#/DecisionResponse' + '400': + $ref: '../../openapi.yaml#/components/responses/BadRequest' + '500': + $ref: '../../openapi.yaml#/components/responses/InternalError' + security: + - BearerAuth: [] + - VirtualKeyAuth: [] + - ApiKeyAuth: [] diff --git a/docs/openapi/paths/integrations/typesafe/systemone.yaml b/docs/openapi/paths/integrations/typesafe/systemone.yaml new file mode 100644 index 00000000000..0631196fb14 --- /dev/null +++ b/docs/openapi/paths/integrations/typesafe/systemone.yaml @@ -0,0 +1,171 @@ +# Typesafe Integration - System One Endpoint + +systemone: + post: + operationId: typesafeSystemOne + summary: Evaluate state (Typesafe format) + description: | + Evaluates state against a map of named questions using Typesafe's native + System One API. The request body matches Typesafe's native format; question + identifiers become result field names. + tags: + - Typesafe Integration + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - state + - model + - questions + properties: + state: + description: A plain string, or structured data (object or array) to evaluate. + oneOf: + - type: string + - type: object + additionalProperties: true + - type: array + items: {} + model: + type: string + description: Typesafe model identifier (e.g. `jev-1.13.0`, `jev-latest`). + questions: + type: object + minProperties: 1 + description: Map of named questions. + additionalProperties: + type: object + required: + - type + - instructions + properties: + type: + type: string + enum: [noul, choice, score] + description: Question type. `noul` returns a probability from 0 to 1, `choice` a selection with probabilities, `score` a probability-weighted rubric value. + instructions: + description: Question instructions. A string, or structured data. + oneOf: + - type: string + - type: object + additionalProperties: true + - type: array + items: {} + criteria: + description: | + Optional `true`/`false` descriptions for `noul`; a map of option + to description for `choice` (at most 255 options); an ordered + array of 2-10 level descriptions for `score`. + oneOf: + - type: object + additionalProperties: + type: string + - type: array + items: + type: string + responses: + '200': + description: Successful response in Typesafe's native shape + content: + application/json: + schema: + type: object + required: + - model + - answers + properties: + model: + type: string + description: Model version that performed the evaluation + answers: + type: object + description: Map matching question identifiers to answers + additionalProperties: + type: object + required: + - type + properties: + type: + type: string + enum: [noul, choice, score] + noul: + type: number + description: Probability from 0 to 1 (noul answers) + choice: + type: string + description: Selected option (choice answers) + score: + type: number + description: Probability-weighted value (score answers) + probabilities: + type: object + additionalProperties: + type: number + legend: + type: object + additionalProperties: + type: string + confidence: + type: number + usage: + type: object + properties: + input_tokens: + type: integer + output_tokens: + type: integer + '400': + $ref: '../../../openapi.yaml#/components/responses/BadRequest' + '500': + $ref: '../../../openapi.yaml#/components/responses/InternalError' + security: + - BearerAuth: [] + - VirtualKeyAuth: [] + - ApiKeyAuth: [] + +models: + get: + operationId: typesafeListModels + summary: List models (Typesafe format) + description: | + Lists available Typesafe models in the native shape. Typesafe documents no + upstream listing endpoint, so the catalog is served from Bifrost's model + datasheet. + tags: + - Typesafe Integration + responses: + '200': + description: Successful response + content: + application/json: + schema: + type: object + required: + - models + properties: + models: + type: array + items: + type: object + required: + - name + properties: + name: + type: string + example: jev-latest + description: + type: string + release_date: + type: string + format: date-time + '400': + $ref: '../../../openapi.yaml#/components/responses/BadRequest' + '500': + $ref: '../../../openapi.yaml#/components/responses/InternalError' + security: + - BearerAuth: [] + - VirtualKeyAuth: [] + - ApiKeyAuth: [] diff --git a/docs/openapi/schemas/inference/decisions.yaml b/docs/openapi/schemas/inference/decisions.yaml new file mode 100644 index 00000000000..d8838d7744a --- /dev/null +++ b/docs/openapi/schemas/inference/decisions.yaml @@ -0,0 +1,114 @@ +# Decisions API schemas + +DecisionRequest: + type: object + required: + - model + - state + - questions + properties: + model: + type: string + description: Model in provider/model format + example: typesafe/jev-1.13.0 + state: + description: State to evaluate. A plain string, or structured data (object or array). + oneOf: + - type: string + - type: object + additionalProperties: true + - type: array + items: {} + questions: + type: object + minProperties: 1 + description: Map of named questions. Question identifiers become answer field names. + additionalProperties: + $ref: '#/DecisionQuestion' + fallbacks: + type: array + items: + type: string + description: Fallback models in provider/model format + +DecisionQuestion: + type: object + required: + - kind + - instructions + properties: + kind: + type: string + enum: [noul, choice, score] + description: How the question is decided. `noul` yields a probability between 0 and 1, `choice` yields one option from the criteria, `score` yields a numeric rubric score including fractions. + instructions: + description: Instructions for this question. A string, or structured data preserved losslessly. + oneOf: + - type: string + - type: object + additionalProperties: true + - type: array + items: {} + criteria: + description: | + Applicable criteria. Optional `true`/`false` descriptions for `noul`; a + required map of option to description for `choice` (at most 255 options); + a required ordered array of 2-10 level descriptions for `score`. + oneOf: + - type: object + additionalProperties: + type: string + - type: array + items: + type: string + +DecisionResponse: + type: object + required: + - model + - answers + properties: + id: + type: string + description: Unique identifier for the decision response + model: + type: string + description: Model that performed the decision + answers: + type: object + description: Map matching question identifiers to answers. Every requested question produces an answer. + additionalProperties: + $ref: '#/DecisionAnswer' + usage: + $ref: './usage.yaml#/BifrostLLMUsage' + extra_fields: + $ref: './common.yaml#/BifrostResponseExtraFields' + +DecisionAnswer: + type: object + required: + - kind + - value + properties: + kind: + type: string + enum: [noul, choice, score] + description: The question's decision kind + value: + description: The decided value. A number in [0,1] for `noul`, an option string for `choice`, a numeric rubric score for `score`. + oneOf: + - type: number + - type: string + confidence: + type: number + description: Model confidence for this answer, when supplied + probabilities: + type: object + additionalProperties: + type: number + description: Probability distribution over options or levels, when supplied + legend: + type: object + additionalProperties: + type: string + description: Legend mapping levels to descriptions, when supplied diff --git a/docs/openapi/schemas/inference/responses.yaml b/docs/openapi/schemas/inference/responses.yaml index bbb199ad280..99f9c30688a 100644 --- a/docs/openapi/schemas/inference/responses.yaml +++ b/docs/openapi/schemas/inference/responses.yaml @@ -50,7 +50,12 @@ ResponsesRequest: background: type: boolean conversation: - type: string + oneOf: + - type: string + - type: object + properties: + id: + type: string context_management: $ref: "#/ContextManagement" include: @@ -148,10 +153,14 @@ ResponsesMessage: type: string image_url: type: string + async: + type: boolean action: type: object error: - type: string + oneOf: + - type: string + - type: object queries: type: array items: @@ -423,12 +432,16 @@ ResponsesTool: type: string description: type: string + async: + type: boolean cache_control: $ref: "./common.yaml#/CacheControl" parameters: $ref: "./chat.yaml#/ToolFunctionParameters" strict: type: boolean + output_schema: + type: object vector_store_ids: type: array items: @@ -455,8 +468,14 @@ ResponsesTool: type: string server_url: type: string + tunnel_id: + type: string allowed_tools: - type: object + oneOf: + - type: array + items: + type: string + - type: object authorization: type: string connector_id: diff --git a/docs/openapi/schemas/management/governance.yaml b/docs/openapi/schemas/management/governance.yaml index 7db543217d2..4254a7888ce 100644 --- a/docs/openapi/schemas/management/governance.yaml +++ b/docs/openapi/schemas/management/governance.yaml @@ -1344,6 +1344,27 @@ TableKey: # Routing Rules schemas +RoutingFallback: + description: A fallback in a routing rule's chain. Either the "provider/model" string ("provider/" keeps the incoming model), or an object that additionally pins a provider key. + oneOf: + - type: string + minLength: 1 + - type: object + required: + - provider + properties: + provider: + type: string + minLength: 1 + description: Fallback provider + model: + type: [string, 'null'] + description: Fallback model (omit or empty to use the incoming request model) + key_id: + type: [string, 'null'] + description: UUID of the API key to pin for this fallback (omit for load-balanced key selection). A fallback whose pinned key is missing from the provider's pool is skipped. + additionalProperties: false + RoutingTarget: type: object description: A single weighted routing target within a routing rule @@ -1396,8 +1417,8 @@ RoutingRule: fallbacks: type: array items: - type: string - description: Fallback providers in format "provider/model" + $ref: '#/RoutingFallback' + description: Ordered fallback chain, tried after the selected target fails scope: type: string enum: [global, team, customer, virtual_key] @@ -1469,8 +1490,8 @@ CreateRoutingRuleRequest: fallbacks: type: array items: - type: string - description: Fallback providers in format "provider/model" + $ref: '#/RoutingFallback' + description: Ordered fallback chain, tried after the selected target fails scope: type: string enum: [global, team, customer, virtual_key] @@ -1526,7 +1547,8 @@ UpdateRoutingRuleRequest: fallbacks: type: array items: - type: string + $ref: '#/RoutingFallback' + description: Ordered fallback chain, tried after the selected target fails priority: type: integer query: diff --git a/docs/providers/provider-routing.mdx b/docs/providers/provider-routing.mdx index c7ff56eef12..543355a38e4 100644 --- a/docs/providers/provider-routing.mdx +++ b/docs/providers/provider-routing.mdx @@ -1288,6 +1288,15 @@ All three routing layers (governance, enterprise LB Level 1, model-catalog-resol *layer* where Level 1 runs has moved from a middleware to PreRequestHook. + + **Session affinity** sits on top of this order. After PreRequestHook it moves the + provider that last served the session to the front of the fallback chain. A request + that names its provider (`openai/gpt-4o`) is never reordered. Inside Level 2 it reuses + the key that last served the session while that key stays in the pool, including for a + request that named its provider. Routing decides which providers and keys are offered; + the session only decides their order. See [Session Affinity](/providers/session-affinity). + + ### Example Scenarios diff --git a/docs/providers/request-options.mdx b/docs/providers/request-options.mdx index 26363e10a81..1f823db7e11 100644 --- a/docs/providers/request-options.mdx +++ b/docs/providers/request-options.mdx @@ -14,8 +14,9 @@ Bifrost provides request options that control behavior, enable features, and pas | `BifrostContextKeyAPIKeyName` | `x-bf-api-key` | `string` | Explicit API key name selection | | `BifrostContextKeyAPIKeyID` | `x-bf-api-key-id` | `string` | Explicit API key ID selection (takes priority over name) | | `BifrostContextKeyDirectKey` | `x-bf-direct-key` (+ `Authorization` / `x-api-key` / `x-goog-api-key`) | `schemas.Key` | Use a caller-supplied raw provider key directly, bypassing the registered key pool. On the gateway, requires `allow_direct_keys` server-side | -| `BifrostContextKeySessionID` | `x-bf-session-id` (falls back to coding-harness session headers) | `string` | Session ID for key stickiness (requires KV store) | -| `BifrostContextKeySessionTTL` | `x-bf-session-ttl` | `time.Duration` | Session-to-key cache TTL (duration string or seconds) | +| `BifrostContextKeySessionID` | `x-bf-session-id` (falls back to coding-harness session headers) | `string` | Session ID for provider and key session affinity (requires KV store) | +| `BifrostContextKeySessionTTL` | `x-bf-session-ttl` | `time.Duration` | Session binding TTL (duration string or seconds) | +| `BifrostContextKeySessionAffinity` | `x-bf-session-affinity` | `bool` | Whether the session decides routing for this request: `on` (default) or `off` | | `BifrostContextKeyRequestID` | `x-request-id` | `string` | Custom request ID for tracking | | `BifrostContextKeySendBackRawRequest` | `x-bf-send-back-raw-request` | `bool` | Include raw provider request in the response | | `BifrostContextKeySendBackRawResponse` | `x-bf-send-back-raw-response` | `bool` | Include raw provider response in the response | @@ -229,13 +230,20 @@ response, err := client.ChatCompletionRequest(ctx, &schemas.BifrostChatRequest{ **Type:** `string` **Required:** No -Bind a session to a specific API key so that requests with the same session ID consistently use the same key. Useful for predictable rate-limit buckets, cost attribution per user, and consistent model routing per session. +Keep a session on what served it before, so a conversation keeps hitting the same provider prompt cache. Affinity works at two levels: -On the first request for a session ID, Bifrost selects a key and caches the binding in the KV store. Subsequent requests with the same session ID reuse the cached key as long as it remains valid. +- **Provider**: when routing offers more than one provider for what the caller asked for (virtual key weights, load balancing, model catalog candidates), the provider that last served the session is tried first, as long as routing still offers it. Routing decides which providers are allowed; the session only decides their order. A request that names its provider (`openai/gpt-4o`) is never reordered: the caller asked for that provider, so only the key level applies to it, even when a fallback served an earlier turn. +- **Key**: within a provider, the key that last served the session is used while it stays eligible. + +Bindings are written when a request is served, never when it fails, and are scoped to the virtual key and user the request is attributed to. A fallback that serves becomes the session's new home. Bindings live in the KV store for the session TTL, refreshed by every served request. **Retry and Fallback Behavior:** -- **Retries**: Session stickiness persists across retry attempts. If a request fails and is retried, the same sticky key is used. -- **Fallbacks**: When falling back to a different provider (e.g., from OpenAI to Anthropic), session stickiness is disabled and a new key is selected for the fallback provider. This ensures availability when the primary provider's keys are exhausted or failing. +- **Retries**: A bound key is used for every retry of a request. A session with no key binding yet rotates keys on rate limits like any other request, and the key that serves is the one that gets bound. +- **Fallbacks**: Fallback providers select keys freely. When a fallback serves, the session is rebound to that provider and its key, so the next request starts there. +- **Switch**: `x-bf-session-affinity: off` turns affinity off for a request, at both levels; `on` is the default. +- **Routing trail**: Decisions the session made appear in the request's routing logs under the `session-affinity` engine, which is also listed among the routing engines used. The UI labels it **Session**. + +The [Session Affinity](/providers/session-affinity) guide covers the full lifecycle of a binding, how affinity composes with each routing engine, and the health checks Bifrost Enterprise runs before following one. **Coding Harness Headers:** @@ -294,7 +302,7 @@ Input: messages, **Type:** `time.Duration` (header value: duration string like `"30m"` or `"1h"`, or seconds as integer) **Required:** No -Optional. Controls how long the session-to-key binding is cached. If not set, Bifrost uses 1 hour. The TTL is refreshed on each request so active sessions do not expire. +Optional. Controls how long the session bindings are kept. If not set, Bifrost uses 1 hour. The TTL is refreshed by each served request so active sessions do not expire. Accepts duration strings (`"30s"`, `"5m"`, `"1h"`) or plain numbers (treated as seconds). diff --git a/docs/providers/routing-rules.mdx b/docs/providers/routing-rules.mdx index 9b20381e447..b695e7d0f07 100644 --- a/docs/providers/routing-rules.mdx +++ b/docs/providers/routing-rules.mdx @@ -466,7 +466,10 @@ Define routing rules in your `config.json` file under the governance configurati - `model` (string, optional): Target model - omit to use the incoming request model - `key_id` (string, optional): UUID of the API key to pin - requires `provider` to be present; omit for load-balanced key selection - `weight` (number, required): Probability weight - all weights in a rule must sum to 1 (e.g. 0.7 + 0.3 = 1.0) -- **fallbacks** (array[string]): Fallback providers in "provider/model" format +- **fallbacks** (array, optional): Ordered fallback chain, tried after every target fails. Each entry is either the string `"provider/model"` (use `"provider/"` to keep the incoming model), or an object: + - `provider` (string, required): Fallback provider + - `model` (string, optional): Fallback model - omit to use the incoming request model + - `key_id` (string, optional): UUID of the API key to pin - omit for load-balanced key selection. In `config.json` use `provider_key_name` instead, which resolves to a key ID at load time. A fallback whose pinned key is missing from that provider's pool is skipped, and the chain moves on - **scope** (string): Scope level - "global", "customer", "team", or "virtual_key" - **scope_id** (string, optional): ID of scoped entity (null for global scope) - **priority** (number): Rule evaluation order within scope (lower = evaluated first) @@ -793,6 +796,22 @@ If OpenAI fails: 1. Level 2 load balancing evaluates Azure keys 2. If all Azure keys fail, tries Groq +A fallback can pin its own provider key, which skips load balancing for that attempt: + +```json +{ + "provider": "openai", + "fallbacks": [ + "azure/gpt-4o", + { "provider": "vertex", "model": "gemini-2.5-pro", "key_id": "8f3c1d2e-..." } + ] +} +``` + +Each attempt is pinned independently, so an unpinned fallback later in the chain still load balances normally. If the pinned key is not in that provider's pool, the attempt is skipped and the chain moves to the next entry. + +A pin stays with its provider when [session affinity](/providers/session-affinity) reorders the chain. A session that moves the primary to another provider carries the target's pin onto that provider's fallback entry, and a fallback the session promotes brings its own pin to the primary attempt. + --- ## Execution & Performance diff --git a/docs/providers/session-affinity.mdx b/docs/providers/session-affinity.mdx new file mode 100644 index 00000000000..3fd46e492fb --- /dev/null +++ b/docs/providers/session-affinity.mdx @@ -0,0 +1,208 @@ +--- +title: "Session Affinity" +description: "Keep a conversation on the provider and key that served it before, so multi-turn sessions and agent runs keep hitting the same provider prompt cache and rate-limit bucket, whatever routing engine picked the provider." +icon: "link" +--- + +Session affinity, also called session stickiness, keeps a session on what served it before. A session is a conversation, an agent run, or any group of requests that share a session ID. Routing still decides which providers and keys a request may use; the session only decides which of them is tried first. + +It runs for every request that carries a session ID and works with every routing engine: routing rules, virtual key load balancing, the model catalog, and Adaptive Load Balancing. Coding harnesses such as Claude Code, Codex CLI, and OpenCode get it with no configuration, because Bifrost adopts the session header they already send. + +## Why it matters + +- **Provider prompt caches** are per provider, and usually per API key or organization. A conversation that hops between OpenAI and Azure, or between two OpenAI keys, misses the cache it warmed a turn ago. +- **Rate-limit buckets** are per key. A session that stays on one key uses one bucket predictably instead of spreading across several. +- **Consistency** across a run: the same provider serves every turn, so tool behaviour and response style do not drift mid-conversation. + +Affinity does not create or manage provider caches. It makes sure a session keeps landing where its cache lives. + +## What counts as a session + +| Source | How the session ID is set | +| ----------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Explicit header | `x-bf-session-id` on the request. Always wins when present. | +| Coding harness headers | When `x-bf-session-id` is absent, the first non-empty harness session header is adopted. See the [header list](/providers/request-options#session-stickiness-session-id) for the full priority order. | +| Go SDK | `BifrostContextKeySessionID` on the request context. | + +A session ID may be at most 255 runes. An explicit `x-bf-session-id` that is longer gives the request no session at all rather than falling back to a harness header, because grouping requests under a session the caller did not ask for is worse than leaving them ungrouped. + +Bindings are scoped to who the request is attributed to: the virtual key and the user behind it, as resolved by governance, plus the session ID. The same session ID under two different virtual keys is two independent sessions. A request with no virtual key and no user is scoped by its session ID alone. + + +Claude Code subagents send the same session ID as their parent, so an agent run with parallel subagents stays on one provider and one key. + + +## Two levels of affinity + +### Provider level + +When the caller asks for a bare model (`gpt-4o`) and routing offers more than one provider for it, the route that last served the session, provider and model together, is tried first, as long as routing still offers that exact route. A chain that offers the same provider on a different model does not offer it: the binding is dropped and the routing decision stands. Routing engines build the chain of providers; affinity reorders that chain, and nothing else. A provider that routing excluded, for budget, rate limits, model allowance, or health, is never brought back by a session. + +The binding is keyed by what the caller asked for, so a session that asks for `gpt-4o` and later for `gpt-4o-mini` holds one binding per model. + + +A request that names its provider (`openai/gpt-4o`) is never reordered and writes no provider binding. The caller asked for that provider, so only the key level applies, even when a fallback served an earlier turn of the same session. + + +### Key level + +Within the provider that serves the request, the key that last served the session for that provider and model is used again while it stays in the eligible pool. The pool is what key selection would otherwise choose from: the keys enabled for the provider that allow the model. A provider with a single key has nothing to choose, so the key level is skipped. + +A key that leaves the pool, because it was disabled, removed, or stopped allowing the model, loses its binding and a key is picked normally. + +## Lifecycle of a binding + +Bindings are written only when a request is served. A failed request writes nothing, so a session is never bound to a provider or key that did not deliver. + +| Event | Provider binding | Key binding | +| ------------------------------------------------------ | ---------------------------------------------------------------------- | -------------------------------------------------------------------- | +| First served request of the session | Written to the provider that served | Written to the key that served | +| Served request that followed the binding | TTL refreshed | TTL refreshed | +| Served by a fallback provider | Moved to the fallback provider | The key the primary followed is deleted; moved to the key the fallback used | +| Failed after following the binding | Deleted, the next request follows the routing decision | Deleted, a key is picked normally on the next request | +| Cancelled by the caller after following the binding | Unchanged, the failure is not the provider's | Unchanged | +| Request failed on every attempt | Untouched | Untouched | +| Request refused by governance before routing | Untouched | Untouched | +| Bound route not in the chain routing built (provider and model) | Deleted, the routing decision stands, the next served request rebinds | Unchanged | +| Bound key not in the eligible pool | Unchanged | Deleted, a key is picked normally, the next served request rebinds | +| TTL expires with no served request | Gone | Gone | +| Request sent with `x-bf-session-affinity: off` | Neither read nor written | Neither read nor written | + +A request refused by governance outright, because its credential is inactive or expired, or because the provider or model it asked for is not allowed, is refused before any routing engine builds a chain. Affinity never sees it, so the session keeps whatever binding it had. Revoking a session's access to the provider it is bound to therefore leaves that binding in the trail until it expires, even though no request can follow it. Narrowing access instead, so routing still offers some other provider, deletes the binding on the next request and rebinds the session to whatever serves it. + +The TTL defaults to one hour and can be set per request with `x-bf-session-ttl` (a duration string such as `30m`, or a number of seconds). Every served request that follows a binding refreshes it, so an active session does not expire mid-conversation. + +When several requests of a brand-new session arrive in parallel, they can land on different providers or keys before any of them is served. The first one to be served writes the binding, the rest of that burst is already in flight, and every request after it follows the winner. + +## How it fits with routing + +Affinity runs after every routing engine has had its say, and again inside each provider attempt when a key is chosen: + +1. Governance evaluates routing rules and, for virtual keys with provider configs, load balances across the weighted providers. +2. Adaptive Load Balancing (Enterprise) ranks the eligible providers by measured performance. +3. The model catalog resolver fills in a provider for a bare model that nothing above resolved, with the other catalog providers as fallbacks. +4. **Session affinity, provider level**: the bound provider moves to the front of the chain if it is in it. +5. For each attempt, the key pool is built for the provider and filtered, then **session affinity, key level** reuses the bound key if it is in the pool. +6. Fallbacks run in the resulting order when an attempt fails. + +| Routing engine | What it decides | What the session changes | +| ------------------------------------- | -------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Routing rules | A target and optional fallbacks per matching request | If the rule lists fallbacks, the session's provider is moved first among them. A rule that picks one weighted target and lists no fallbacks leaves a one-provider chain, so there is nothing to reorder and the weights are rolled on every request. | +| Virtual key load balancing | A weighted random provider, with the other weighted providers as fallbacks | After the first served request the session skips the dice and goes to its provider, for as long as budgets, rate limits, and model allowances keep that provider in the chain. | +| Adaptive Load Balancing (Enterprise) | The best-scoring healthy provider, with the healthy rest as fallbacks | The session's provider comes first while the load balancer still offers it. A provider the load balancer marks as failed is dropped, see [Health-aware invalidation](#health-aware-invalidation-enterprise). | +| Model catalog | Any provider whose catalog lists the model, when nothing else picked one | The session's provider comes first among the catalog candidates. | +| Circuit breaker (Enterprise) | Which keys are held back from the pool | A bound key that the circuit breaker removes from the pool loses its binding; a key is picked normally. | +| Complexity Router | The complexity tier a routing rule can match on | Independent. The router keeps the tier stable per session; affinity keeps the provider and key stable. Both use the same session identity and neither reads the other's state. | +| Fallbacks and retries | Which provider to try next, and how often to retry | Fallback attempts pick their keys freely unless the rule pinned a key for that entry. A pin belongs to the provider it was decided for, so when the session moves that provider elsewhere in the chain the pin moves with it. Retries of one attempt reuse the bound key. Whatever finally serves becomes the session's new binding at both levels. | + +## Health-aware invalidation (Enterprise) + + + **Enterprise Feature**: Health-aware invalidation is part of Bifrost + Enterprise. [Contact us](https://www.getmaxim.ai/bifrost/enterprise) to enable + it. + + +Without health signals, a session follows its binding until the request actually fails. Bifrost Enterprise checks the binding against what this node's Adaptive Load Balancer and circuit breaker know before following it: + +- **Provider level**: if the load balancer marks the bound provider as failed for the requested model on this node (every key in that direction has failed), the binding is dropped and the routing decision stands. The next served request rebinds the session. +- **Key level**: if the load balancer marks the bound key's route as failed on this node, or the circuit breaker holds the key back, the binding is dropped and a key is picked normally. The next served request rebinds the session to the key that served it. + +Bindings live in the shared KV store and replicate across the cluster, while health is measured per node. A node that sees a provider failing drops the binding for every node. A node that has not observed the failure yet still follows the binding until its own load balancer marks the route as failed or the request fails there. + +## Controls + +| Control | Header | Go SDK context key | Values | +| ------------------------ | ------------------------- | ----------------------------------- | ---------------------------------------------------------------------------------- | +| Session ID | `x-bf-session-id` | `BifrostContextKeySessionID` | Any string up to 255 runes. Falls back to coding-harness headers in the gateway. | +| Binding TTL | `x-bf-session-ttl` | `BifrostContextKeySessionTTL` | Duration string (`30m`, `2h`) or seconds. Default one hour. | +| Affinity switch | `x-bf-session-affinity` | `BifrostContextKeySessionAffinity` | `on`, `true`, `1` (default) or `off`, `false`, `0`. Applies to both levels. | + +A request sent with the switch off is routed as if it had no session: it neither follows nor updates any binding. Any other value is ignored with a warning and the default applies. There is no stored default; the switch is per request. + + +Bindings live in Bifrost's KV store. The gateway always creates one, so nothing needs to be configured there. Go SDK users must set `KVStore` on `BifrostConfig`, otherwise affinity has nowhere to keep a binding and every request is routed as if it had no session. In an Enterprise cluster the KV store replicates, so a session can land on any node and still find its binding. + + +```bash Gateway +curl -X POST http://localhost:8080/v1/chat/completions \ + --header 'Content-Type: application/json' \ + --header 'x-bf-session-id: user-123-session-abc' \ + --header 'x-bf-session-ttl: 30m' \ + --data '{"model": "gpt-4o", "messages": [{"role": "user", "content": "Hello"}]}' +``` + +```go Go SDK +ctx = context.WithValue(ctx, schemas.BifrostContextKeySessionID, "user-123-session-abc") +ctx = context.WithValue(ctx, schemas.BifrostContextKeySessionTTL, 30*time.Minute) +``` + +## Seeing what the session did + +Every decision affinity makes is written to the request's routing trail under the `session-affinity` engine, and every decision also lists that engine among the routing engines used for the request, whether the session followed a binding or refused a stale one. A request that took no part, because it carries no session or asked not to follow one, lists no engine. In the logs explorer, filter by session ID to see one session's requests together, and by routing engine to find requests a session had a say in. The UI labels this engine **Session**; `session-affinity` is the identifier used in the API and in exported logs. + +| Trail entry | Meaning | +| ------------------------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------- | +| `Session stays on azure/gpt-4o for gpt-4o; routing proposed openai/gpt-4o` | The bound route was moved ahead of the one routing put first. | +| `Session stays on azure/gpt-4o for gpt-4o, which routing also proposed` | The bound route was already first, so the chain was left as it is. | +| `Session reused key prod-eu-2 for azure/gpt-4o` | The bound key was used instead of running key selection. | +| `Session was last served by azure/gpt-4o for gpt-4o, which this request cannot use, so the routing decision stands and the session rebinds on its next served request` | Routing no longer offers the bound route, provider and model together, so the binding was deleted. | +| `Session moved the request from vertex to anthropic, so a key is picked normally there; the key pinned for vertex applies if vertex is tried as a fallback` | The session promoted another provider; the routing rule's pin stayed with the provider it was set for, now a fallback entry. | +| `The key this session followed for azure/gpt-4o failed, so the session forgets it and a key is picked normally next time` | The request followed the key binding and did not serve on it, so the binding was deleted. | +| `The provider this session followed for gpt-4o failed, so the session forgets it and the next request follows the routing decision` | The request followed the route binding and failed outright, so the binding was deleted. | +| `The key this session last used for azure/gpt-4o is no longer eligible, so one is being picked` | The bound key left the pool, so the binding was deleted. | +| `Request carries a session but asked not to follow it, so the routing decision stands for gpt-4o` | The request sent `x-bf-session-affinity: off`, so no binding was read or written. | +| `Session was last served by azure for gpt-4o, which is failing on this node, so the routing decision stands` | Enterprise only: the load balancer marks the provider as failed here, so the binding was dropped. | +| `The key this session last used for azure/gpt-4o is held back by the circuit breaker on this node, so one is being picked` | Enterprise only: the circuit breaker removed the bound key from the pool, so the binding was dropped. | + +The last two entries are emitted only by Bifrost Enterprise; an open-source deployment never writes them, because it follows a binding until the request itself fails, and only then forgets it, which the two entries before them record. + +Both levels record every binding they follow, whether or not it changed the outcome, so a session that agrees with routing is distinguishable in the trail from one that was never consulted. + +In OpenTelemetry, the trace's root span carries the session ID as the `session.id` attribute, and the `group_traces_by_session` setting puts a whole session into one trace. See [OpenTelemetry](/features/observability/otel#grouping-traces-by-session). + +## Walk-throughs + +### A virtual key that load balances + +A virtual key allows `gpt-4o` on OpenAI with weight 70 and Azure with weight 30. Azure has two keys. A Claude Code session sends its session header on every request. + +1. **Turn 1**: governance rolls the weights and picks Azure, with OpenAI as the fallback. Key selection picks Azure key `prod-eu-2`. The request is served, and the session is bound to Azure for `gpt-4o` and to `prod-eu-2` on Azure. +2. **Turn 2**: governance rolls OpenAI this time and builds the chain `[openai, azure]`. Affinity moves Azure to the front and the trail says `Session stays on azure for gpt-4o; routing proposed openai`. Key selection reuses `prod-eu-2`. The Azure prompt cache warmed by turn 1 is hit. +3. **Turn 3**: the virtual key's Azure budget is exhausted, so governance excludes Azure and offers only OpenAI. The bound provider is not in the chain, so the binding is deleted with `…which this request cannot use…` in the trail. OpenAI serves and the session rebinds to OpenAI and to the OpenAI key that served. + +### A fallback that serves + +The same session is bound to Azure and `prod-eu-2`. + +1. Azure returns a server error on every retry. Retries reuse `prod-eu-2`, because the session is bound to it. +2. The fallback attempt on OpenAI selects a key freely, say `openai-main`, and serves. +3. The session moves: the provider binding now points at OpenAI, and a key binding for OpenAI and `gpt-4o` points at `openai-main`. The stale Azure key binding is left to expire; it is only consulted if Azure serves this session again. +4. The next turn's chain, whatever governance rolls, is reordered to put OpenAI first. + +### A provider that fails on one node (Enterprise) + +A three-node cluster. The session is bound to Azure and `prod-eu-2`. + +1. Azure starts timing out. Node B's Adaptive Load Balancer marks the Azure direction for `gpt-4o` as failed after every key in it has failed. +2. The next request lands on node B. Before following the binding, affinity asks the load balancer and learns the direction is failed. The binding is dropped, replicated to nodes A and C, and the trail says `…which is failing on this node, so the routing decision stands`. The load balancer's own choice, OpenAI, serves, and the session rebinds to OpenAI. +3. Had the request landed on node A before A observed the failure, A would have followed the Azure binding. The request would then fail on Azure, the fallback would serve, and the session would move exactly as in the fallback walk-through. + +## Limits to know + +- **Explicit providers get no provider stickiness.** `openai/gpt-4o` is honoured as written. Use a bare model name and let routing offer several providers if you want the session to choose among them. +- **Weighted routing-rule targets with no fallbacks** are rolled on every request. Add fallbacks to the rule if the session should stick to the target that served it. +- **A bound key is used for every retry.** A session with a key binding does not rotate keys on a rate limit; the retry policy runs on the bound key, and only a fallback provider moves the session. A session with no key binding yet rotates like any other request. +- **A new session's first parallel burst scatters once.** Requests already in flight when the first one is served keep the provider and key they were given. +- **Health checks are node-local (Enterprise).** A binding is refused where a node has seen the failure. A node that has not seen it follows the binding until the request fails or its own load balancer catches up. +- **A node that joins the cluster starts with an empty binding store.** It receives bindings written after it joined. A session that lands on it before then is treated as new, rebinds there, and that binding replicates to the other nodes. +- **Realtime and WebSocket Responses key selection reads but never writes.** These connection paths reuse a key bound by earlier requests of the session, but they do not create or refresh a binding themselves. +- **The switch is per request.** There is no configuration that turns affinity off for a deployment; send `x-bf-session-affinity: off` on the requests that should not take part. + +## Next Steps + +- **[Request Options](/providers/request-options#session-stickiness-session-id)** - Every session header, the coding-harness header list, and the Go SDK context keys +- **[Provider Routing](/providers/provider-routing)** - How governance, Adaptive Load Balancing, and the model catalog build the chain affinity reorders +- **[Adaptive Load Balancing](/enterprise/adaptive-load-balancing)** - The health signals Enterprise affinity checks before following a binding +- **[Circuit Breaker](/enterprise/circuit-breaker)** - Key-level sub-circuits that take a bound key out of the pool +- **[Complexity Router](/features/governance/complexity-router#session-aware-routing)** - Session-stable complexity tiers that compose with affinity diff --git a/docs/providers/supported-providers/overview.mdx b/docs/providers/supported-providers/overview.mdx index e80281c0661..5cc3d26c47d 100644 --- a/docs/providers/supported-providers/overview.mdx +++ b/docs/providers/supported-providers/overview.mdx @@ -14,44 +14,45 @@ Bifrost can also act as a provider-compatible gateway (for example, [Anthropi The following table summarizes which operations are supported by each provider via Bifrost’s unified interface. -| Provider | Models | Text | Text (stream) | Chat | Chat (stream) | Responses | Responses (stream) | Images | Images (stream) | Image Edit | Image Edit (stream) | Image Variation | Embeddings | TTS | TTS (stream) | STT | STT (stream) | Files | Batch | Count tokens | Rerank | OCR | Video | Video Remix | Containers | Passthrough | Passthrough (stream) | -| ------------------------------------ | ------ | ---- | ------------- | ---- | ------------- | --------- | ------------------ | ------ | --------------- | ---------- | ------------------- | --------------- | ---------- | --- | ------------ | --- | ------------ | ----- | ----- | ------------ | ------ | --- | ----- | ----------- | ---------- | ----------- | -------------------- | -| Anthropic (`anthropic/`) | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | -| Azure (`azure/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | -| Bedrock (`bedrock/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| Bedrock Mantle (`bedrock_mantle/`) | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| Cerebras (`cerebras/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| Cohere (`cohere/`) | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| Databricks (`databricks/`) | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| DeepSeek (`deepseek/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| Elevenlabs (`elevenlabs/`) | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| Fireworks (`fireworks/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| Gemini (`gemini/`) | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | -| GLM (`glm/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| GitHub Copilot (`github-copilot/`) | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| Groq (`groq/`) | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| Hugging Face (`huggingface/`) | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| MiniMax (`minimax/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| Mistral (`mistral/`) | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | -| ModelArk (`modelark/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| Moonshot (`moonshot/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| Nebius (`nebius/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| Ollama (`ollama/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| OpenAI (`openai/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | -| OpenRouter (`openrouter/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| Parasail (`parasail/`) | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| Perplexity (`perplexity/`) | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| Qwen (`qwen/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| Replicate (`replicate/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | -| Runware (`runware/`) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | -| Runway (`runway/`) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | -| Sarvam AI (`sarvam/`) | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| SGL (`sgl/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| Vertex AI (`vertex/`) | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | -| Volcengine (`volcengine/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| vLLM (`vllm/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| Wafer (`wafer/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| xAI (`xai/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| Provider | Models | Text | Text (stream) | Chat | Chat (stream) | Responses | Responses (stream) | Images | Images (stream) | Image Edit | Image Edit (stream) | Image Variation | Embeddings | TTS | TTS (stream) | STT | STT (stream) | Files | Batch | Count tokens | Rerank | Decisions | OCR | Video | Video Remix | Containers | Passthrough | Passthrough (stream) | +| ------------------------------------ | ------ | ---- | ------------- | ---- | ------------- | --------- | ------------------ | ------ | --------------- | ---------- | ------------------- | --------------- | ---------- | --- | ------------ | --- | ------------ | ----- | ----- | ------------ | ------ | --------- | --- | ----- | ----------- | ---------- | ----------- | -------------------- | +| Anthropic (`anthropic/`) | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | +| Azure (`azure/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | +| Bedrock (`bedrock/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| Bedrock Mantle (`bedrock_mantle/`) | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| Cerebras (`cerebras/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| Cohere (`cohere/`) | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| Databricks (`databricks/`) | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| DeepSeek (`deepseek/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| Elevenlabs (`elevenlabs/`) | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| Fireworks (`fireworks/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| Gemini (`gemini/`) | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | +| GLM (`glm/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| GitHub Copilot (`github-copilot/`) | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| Groq (`groq/`) | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| Hugging Face (`huggingface/`) | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| MiniMax (`minimax/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| Mistral (`mistral/`) | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | +| ModelArk (`modelark/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| Moonshot (`moonshot/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| Nebius (`nebius/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| Ollama (`ollama/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| OpenAI (`openai/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | +| OpenRouter (`openrouter/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| Parasail (`parasail/`) | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| Perplexity (`perplexity/`) | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| Qwen (`qwen/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| Replicate (`replicate/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | +| Runware (`runware/`) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | +| Runway (`runway/`) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | +| Sarvam AI (`sarvam/`) | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| SGL (`sgl/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| TypeSafe (`typesafe/`) | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| Vertex AI (`vertex/`) | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | +| Volcengine (`volcengine/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| vLLM (`vllm/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| Wafer (`wafer/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| xAI (`xai/`) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | - 🟡 Not supported by the downstream provider, but internally implemented by Bifrost as a fallback. - ❌ Not supported by the downstream provider, hence not supported by Bifrost. diff --git a/docs/providers/supported-providers/typesafe.mdx b/docs/providers/supported-providers/typesafe.mdx new file mode 100644 index 00000000000..3eca4940b09 --- /dev/null +++ b/docs/providers/supported-providers/typesafe.mdx @@ -0,0 +1,189 @@ +--- +title: "TypeSafe" +description: "TypeSafe API conversion guide - jev judgment models, decisions endpoint, question types, and model management" +icon: "scale-balanced" +--- + +## Overview + +TypeSafe is a specialized decision provider serving the jev family of System One judgment models. A decision request carries a `state` (what to evaluate) and a map of named `questions`; the response carries one typed answer per question. Bifrost performs conversions including: + +- **Model ID mapping** - Uses provider model identifier directly (`jev-1.13.0`, `jev-latest`, `jev-preview`) +- **Question conversion** - Maps Bifrost decision kinds (`noul`, `choice`, `score`) to TypeSafe's native System One question types 1:1 +- **Answer normalization** - Native `noul`/`choice`/`score` value fields normalize to a unified `value`, with `confidence`, `probabilities`, and `legend` metadata preserved +- **Usage normalization** - `input_tokens`/`output_tokens` map to Bifrost's `prompt_tokens`/`completion_tokens` (TypeSafe bills input tokens only) +- **Strict validation** - Unsupported kinds, missing instructions, and malformed criteria are rejected locally with a 400 rather than silently approximated + +### Supported Operations + +| Operation | Non-Streaming | Streaming | Endpoint | +|-----------|---------------|-----------|----------| +| Decisions | ✅ | - | `/v1/decisions` | +| List Models | ✅ | - | `/v1/models` | +| Chat Completions | ❌ | ❌ | - | +| Responses API | ❌ | ❌ | - | +| Text Completions | ❌ | ❌ | - | +| Embeddings | ❌ | ❌ | - | + + +**Unsupported Operations** (❌): Chat Completions, Responses API, Text Completions, Embeddings, and every other operation return `UnsupportedOperationError` - TypeSafe serves judgment models only. + +**Model listing**: TypeSafe documents no upstream models endpoint. Bifrost serves the jev catalog from its model datasheet, including pricing and context length. + + +## Setup & Configuration + +Configure TypeSafe as a provider with a bearer API key: + +```json +{ + "providers": { + "typesafe": { + "keys": [ + { + "name": "TypeSafe API Key", + "value": "env.TYPESAFE_API_KEY", + "weight": 1, + "models": ["*"] + } + ] + } + } +} +``` + +## Decisions API + +`POST /v1/decisions` evaluates state against named questions. Each question has a `kind`, `instructions` (string or structured data), and kind-specific `criteria`: + +| Kind | Answer value | Criteria | +|------|--------------|----------| +| `noul` | Probability between 0 and 1 | Optional `true`/`false` descriptions | +| `choice` | One option string | Required map of option to description (max 255) | +| `score` | Numeric rubric score, including fractions | Required ordered array of 2-10 level descriptions | + +```bash +curl http://localhost:8080/v1/decisions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "typesafe/jev-1.13.0", + "state": "Customer message: I was double charged and nobody replied. I want a refund today.", + "questions": { + "is_frustrated": { "kind": "noul", "instructions": "Is the customer frustrated?" }, + "category": { + "kind": "choice", + "instructions": "Pick the ticket category", + "criteria": { "billing": "charges and refunds", "bug": "product defects", "other": "anything else" } + }, + "urgency": { + "kind": "score", + "instructions": "Rate how urgently this needs a human reply", + "criteria": ["can wait a week", "should be answered soon", "needs a reply today"] + } + } + }' +``` + +Response: + +```json +{ + "model": "jev-1.13.0", + "answers": { + "is_frustrated": { "kind": "noul", "value": 0.98 }, + "category": { + "kind": "choice", + "value": "billing", + "confidence": 1, + "probabilities": { "billing": 1, "bug": 0, "other": 0 } + }, + "urgency": { + "kind": "score", + "value": 2, + "confidence": 1, + "probabilities": { "0": 0, "1": 0, "2": 1 }, + "legend": { "0": "can wait a week", "1": "should be answered soon", "2": "needs a reply today" } + } + }, + "usage": { "prompt_tokens": 437, "completion_tokens": 72, "total_tokens": 509 } +} +``` + +## Field Mapping Reference + +Complete mapping between Bifrost's `/v1/decisions` contract and TypeSafe's native `/v1/systemone` API. + +### Request + +| Bifrost field | Native field | Accepted shapes | Validation | +|---------------|--------------|-----------------|------------| +| `model` | `model` | `typesafe/` (routing prefix stripped) | Must resolve to the typesafe provider | +| `state` | `state` | string, object, array (lossless) | Any other shape (number, boolean, null) rejected with 400 | +| `questions` | `questions` | Map of 1+ named questions | Empty map rejected with 400 | +| `questions..kind` | `questions..type` | `noul`, `choice`, `score` | Any other kind rejected with 400 | +| `questions..instructions` | `instructions` | string, object, array (lossless) | Required; other shapes rejected with 400 | +| `questions..criteria` | `criteria` | Kind-specific, see below | Kind-specific, see below | +| `fallbacks` | - | `["provider/model", ...]` | Bifrost-only; consumed by fallback routing, never sent upstream | + +### Criteria by kind + +| Kind | Bifrost / native shape (identical) | Validation | +|------|------------------------------------|------------| +| `noul` | Optional map with only `"true"` and `"false"` keys, string descriptions | Other keys or non-string descriptions rejected with 400 | +| `choice` | Required map of option → description (string) | Missing, empty, non-map, non-string descriptions, or more than 255 options rejected with 400 | +| `score` | Required ordered array of level descriptions | Fewer than 2 or more than 10 levels, or any non-string level, rejected with 400 | + +In the converted request, validated criteria values pass through to TypeSafe unchanged - descriptions are never rewritten. The conversion path re-encodes JSON, so whitespace and object-key ordering can differ; raw-request passthrough bypasses conversion and preserves the original request bytes. + +### Response + +| Bifrost field | Built from native | Notes | +|---------------|-------------------|-------| +| `model` | `model` | Aliases resolve upstream: request `jev-latest`, response reports `jev-1.13.0` | +| `answers..kind` | `answers..type` | Identical vocabulary (`noul`/`choice`/`score`) | +| `answers..value` | `answers..noul` \| `.choice` \| `.score` | Unified value field: number in [0,1] for noul (out-of-range provider values rejected), option string for choice, fractional rubric score for score | +| `answers..confidence` | `confidence` | Present on choice/score answers when supplied | +| `answers..probabilities` | `probabilities` | Distribution over options (choice) or level indices (score) | +| `answers..legend` | `legend` | Level index → description map for score answers | +| `usage.prompt_tokens` | `usage.input_tokens` | TypeSafe bills input tokens only | +| `usage.completion_tokens` | `usage.output_tokens` | Output rate is $0 | +| `usage.total_tokens` | - | Computed: input + output | +| `extra_fields` | - | Bifrost-only: request type, routing info, latency, raw request/response capture | + +Every requested question must produce an answer of its declared kind; a missing answer, a kind mismatch, or a missing value fails the request rather than returning partial results. + +### Errors + +| Upstream status | Meaning | Bifrost behavior | +|-----------------|---------|------------------| +| 401 | Invalid or missing API key | Status and message preserved; key marked failed, other keys rotate in | +| 422 | Request validation failed upstream | Status and validation detail preserved | +| 429 | Rate limited | Retried with backoff; keys rotate (per-key limit) | +| 529 | Service overloaded | Retried with backoff on the same key (capacity, not credential) | + +Requests rejected by Bifrost's own validation (see tables above) return `400` with `caller_invalid_request` before any upstream call is made. + +## Native Integration + +Bifrost also exposes TypeSafe's native API 1:1 under the `/typesafe` prefix, for clients written against TypeSafe directly: + +| Native Endpoint | Method | Notes | +|-----------------|--------|-------| +| `/typesafe/v1/systemone` | POST | Native request/response shape; questions use `type` instead of `kind` | +| `/typesafe/v1/models` | GET | Native `{"models": [{"name", "description", "release_date"}]}` shape, served from the datasheet | + +The native route accepts bare model IDs (`jev-1.13.0`) exactly as TypeSafe does, and additionally accepts `typesafe/`-prefixed IDs. + + +Success responses are shape-compatible with TypeSafe's own API - identical byte-for-byte only on the raw-response passthrough path, otherwise rebuilt and re-encoded (native shape preserved, exact JSON byte ordering not guaranteed). Error responses use TypeSafe's native `{"detail": {"error_type", "message"}}` shape with upstream status codes preserved. Requests rejected by Bifrost's local validation return `400` where TypeSafe's own validation would return `422`. + + +## Model Naming + +| Model | Notes | +|-------|-------| +| `jev-1.13.0` | Current versioned release | +| `jev-latest` | Alias, currently resolves to `jev-1.13.0` | +| `jev-preview` | Preview alias, currently resolves to `jev-1.13.0` | + +Responses always report the resolved versioned model. TypeSafe bills input tokens only; the jev pricing, 64k context window, and rate limits are documented at [docs.typesafe.ai/models](https://docs.typesafe.ai/models). diff --git a/docs/quickstart/gateway/decisions.mdx b/docs/quickstart/gateway/decisions.mdx new file mode 100644 index 00000000000..c8b1190dd4c --- /dev/null +++ b/docs/quickstart/gateway/decisions.mdx @@ -0,0 +1,94 @@ +--- +title: "Decisions" +description: "Evaluate state against named questions using /v1/decisions." +icon: "scale-balanced" +--- + +Use decisions to get structured judgments - probabilities, choices, and rubric scores - from judgment models like TypeSafe's jev family. A decision request carries a `state` (what to evaluate) and a map of named `questions`; every question produces one typed answer. + +## Provider Model Examples + +- TypeSafe: `typesafe/jev-1.13.0`, `typesafe/jev-latest`, `typesafe/jev-preview` + +## Basic Request + +```bash +curl --location 'http://localhost:8080/v1/decisions' \ +--header 'Content-Type: application/json' \ +--data '{ + "model": "typesafe/jev-1.13.0", + "state": "Customer message: I was double charged and nobody replied to my two emails. I want a refund today.", + "questions": { + "is_frustrated": { + "kind": "noul", + "instructions": "Is the customer frustrated?" + }, + "category": { + "kind": "choice", + "instructions": "Pick the ticket category", + "criteria": { "billing": "charges and refunds", "bug": "product defects", "other": "anything else" } + }, + "urgency": { + "kind": "score", + "instructions": "Rate how urgently this needs a human reply", + "criteria": ["can wait a week", "should be answered soon", "needs a reply today"] + } + } +}' +``` + +## Question Kinds + +| Kind | Answer value | Criteria | +|------|--------------|----------| +| `noul` | Probability between 0 and 1 | Optional `true`/`false` descriptions | +| `choice` | One option string | Required map of option to description (max 255 options) | +| `score` | Probability-weighted rubric score, including fractions | Required ordered array of 2-10 level descriptions | + +`state` and `instructions` accept a plain string or structured data (object or array), preserved losslessly. + +## Response + +```json +{ + "model": "jev-1.13.0", + "answers": { + "is_frustrated": { "kind": "noul", "value": 0.98 }, + "category": { + "kind": "choice", + "value": "billing", + "confidence": 1, + "probabilities": { "billing": 1, "bug": 0, "other": 0 } + }, + "urgency": { + "kind": "score", + "value": 2, + "confidence": 1, + "probabilities": { "0": 0, "1": 0, "2": 1 }, + "legend": { "0": "can wait a week", "1": "should be answered soon", "2": "needs a reply today" } + } + }, + "usage": { "prompt_tokens": 437, "completion_tokens": 72, "total_tokens": 509 } +} +``` + +Every declared question produces an answer of its declared kind. `confidence`, `probabilities`, and `legend` are included when the provider supplies them. + +## Fallbacks + +Standard fallback syntax applies: + +```json +{ + "model": "typesafe/jev-preview", + "fallbacks": ["typesafe/jev-1.13.0"] +} +``` + +## Validation + +Requests are validated before dispatch and rejected with a `400` rather than silently approximated: unsupported kinds, missing instructions, missing choice criteria, score criteria outside 2-10 levels or with non-string entries, and states or instructions that are neither string, object, nor array. + + +TypeSafe's native API is also available 1:1 at `POST /typesafe/v1/systemone` - see the [TypeSafe provider page](/providers/supported-providers/typesafe) for the mapping between the two. + diff --git a/docs/quickstart/go-sdk/context-keys.mdx b/docs/quickstart/go-sdk/context-keys.mdx index df2d45ed8fe..0e09ac8af62 100644 --- a/docs/quickstart/go-sdk/context-keys.mdx +++ b/docs/quickstart/go-sdk/context-keys.mdx @@ -98,12 +98,12 @@ A direct key bypasses the **key pool** (no weighted selection, no rotation or fa ### Session Stickiness (Session ID) -Bind a session to a specific API key so that requests with the same session ID consistently use the same key. Useful for predictable rate-limit buckets, cost attribution per user, and consistent model routing per session. +Keep a session on what served it before: the provider that last served it, when routing still offers that provider, and the key that last served it within that provider. A request that names its provider is never reordered; only the key level applies to it. Useful for keeping provider prompt caches warm across a conversation, predictable rate-limit buckets, and cost attribution per user. -On the first request for a session ID, Bifrost selects a key (via weighted random) and caches the binding in the KV store. Subsequent requests with the same session ID reuse the cached key as long as it remains valid. If the cached key is no longer in the supported set (disabled, removed, or model support changed), Bifrost re-selects and overwrites the cache. +Bindings are written when a request is served, never when it fails. A later request with the same session ID puts the bound provider first among the routes the hooks produced and uses the bound key while it remains eligible; a key that is no longer in the supported set (disabled, removed, or model support changed) is dropped and the next served request rebinds. A fallback that serves becomes the session's new home. Decisions the session made are recorded in the request's routing logs under the `session-affinity` engine. -Session stickiness requires a `KVStore` to be configured in `BifrostConfig`. +The policy that picks and keeps the key is `BifrostConfig.SessionAffinity`. When it is unset, Bifrost's own default applies: it needs a `KVStore` in `BifrostConfig`, binds the session to the key selected for its first request, and scopes bindings to the virtual key and user the request's grant identity is attributed to, so a session id reused by another caller does not share them. A custom `SessionAffinity` may keep its state elsewhere and scope it differently. ```go @@ -112,12 +112,20 @@ bfCtx.SetValue(schemas.BifrostContextKeySessionID, "user-123-session-abc") ### Session TTL -Controls how long the session-to-key binding is cached. If not set, Bifrost uses a default TTL of 1 hour. The TTL is refreshed on each request so active sessions do not expire. +Controls how long the session bindings are kept. If not set, Bifrost uses a default TTL of 1 hour. The TTL is refreshed by each served request so active sessions do not expire. ```go bfCtx.SetValue(schemas.BifrostContextKeySessionTTL, 30*time.Minute) ``` +### Session Affinity Switch + +Whether a request lets its session decide where it goes. On by default; `false` leaves the provider to routing and the key to key selection, and writes no session state for the request. + +```go +bfCtx.SetValue(schemas.BifrostContextKeySessionAffinity, false) +``` + ### Request ID Set a custom request ID for tracking and correlation. @@ -366,8 +374,9 @@ func makeRequest(client *bifrost.Bifrost) { | `BifrostContextKeyRequestID` | `string` | Set | Custom request ID for tracking | | `BifrostContextKeyFallbackRequestID` | `string` | Read | Request ID used for fallback attempt | | `BifrostContextKeySkipKeySelection` | `bool` | Set | Skip key selection entirely | -| `BifrostContextKeySessionID` | `string` | Set | Session ID for key stickiness (requires KV store) | -| `BifrostContextKeySessionTTL` | `time.Duration` | Set | TTL for session-to-key cache (default: 1 hour) | +| `BifrostContextKeySessionID` | `string` | Set | Session ID for provider and key session affinity (requires KV store) | +| `BifrostContextKeySessionTTL` | `time.Duration` | Set | TTL for session bindings (default: 1 hour) | +| `BifrostContextKeySessionAffinity` | `bool` | Set | Whether the session decides routing for the request (default: true) | | `BifrostContextKeyExtraHeaders` | `map[string][]string` | Set | Custom headers forwarded to the provider | | `BifrostContextKeyURLPath` | `string` | Set | Custom URL path appended to provider base URL | | `BifrostContextKeyStreamIdleTimeout` | `time.Duration` | Set | Per-chunk idle timeout for streaming responses | diff --git a/docs/quickstart/go-sdk/decisions.mdx b/docs/quickstart/go-sdk/decisions.mdx new file mode 100644 index 00000000000..81fadfee681 --- /dev/null +++ b/docs/quickstart/go-sdk/decisions.mdx @@ -0,0 +1,107 @@ +--- +title: "Decisions" +description: "Evaluate state against named questions with Bifrost Go SDK using client.DecisionRequest." +icon: "scale-balanced" +--- + +Use the Go SDK to get structured judgments - probabilities, choices, and rubric scores - from judgment models like TypeSafe's jev family. + +Provider/model examples: +- TypeSafe: `Provider: schemas.Typesafe`, `Model: "jev-1.13.0"` (aliases: `jev-latest`, `jev-preview`) + +## Basic Example + +The account must serve keys for `schemas.Typesafe` - the OpenAI-only account from the setup guide returns "provider not supported" for this example. A minimal TypeSafe-capable account: + +```go +type MyAccount struct{} + +func (a *MyAccount) GetConfiguredProviders() ([]schemas.ModelProvider, error) { + return []schemas.ModelProvider{schemas.Typesafe}, nil +} + +func (a *MyAccount) GetKeysForProvider(ctx context.Context, provider schemas.ModelProvider) ([]schemas.Key, error) { + return []schemas.Key{ + { + Value: *schemas.NewSecretVar("env.TYPESAFE_API_KEY"), + Models: []string{"*"}, + Weight: 1.0, + }, + }, nil +} + +func (a *MyAccount) GetConfigForProvider(provider schemas.ModelProvider) (*schemas.ProviderConfig, error) { + return &schemas.ProviderConfig{}, nil +} +``` + +```go +package main + +import ( + "context" + "fmt" + + bifrost "github.com/maximhq/bifrost/core" + "github.com/maximhq/bifrost/core/schemas" +) + +func main() { + client, err := bifrost.Init(context.Background(), schemas.BifrostConfig{ + Account: &MyAccount{}, + }) + if err != nil { + panic(err) + } + defer client.Shutdown() + + ctx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) + + response, bifrostErr := client.DecisionRequest(ctx, &schemas.BifrostDecisionRequest{ + Provider: schemas.Typesafe, + Model: "jev-1.13.0", + State: "Customer message: I was double charged and nobody replied. I want a refund today.", + Questions: map[string]schemas.DecisionQuestion{ + "is_frustrated": { + Kind: schemas.DecisionKindNoul, + Instructions: "Is the customer frustrated?", + }, + "category": { + Kind: schemas.DecisionKindChoice, + Instructions: "Pick the ticket category", + Criteria: map[string]interface{}{ + "billing": "charges and refunds", + "bug": "product defects", + "other": "anything else", + }, + }, + "urgency": { + Kind: schemas.DecisionKindScore, + Instructions: "Rate how urgently this needs a human reply", + Criteria: []interface{}{"can wait a week", "should be answered soon", "needs a reply today"}, + }, + }, + }) + if bifrostErr != nil { + panic(bifrostErr.Error.Message) + } + + for name, answer := range response.Answers { + fmt.Printf("%s (%s): %v\n", name, answer.Kind, answer.Value) + } +} +``` + +## Answers + +Every question produces one `schemas.DecisionAnswer`: + +| Field | Meaning | +|-------|---------| +| `Kind` | The question's kind (`noul`, `choice`, `score`) | +| `Value` | `float64` in [0,1] for noul, option `string` for choice, `float64` rubric score for score | +| `Confidence` | Model confidence, when supplied | +| `Probabilities` | Distribution over options or levels, when supplied | +| `Legend` | Level index to description map for score answers, when supplied | + +`State` and `Instructions` accept a string, `map[string]interface{}`, or `[]interface{}`, preserved losslessly. Fallbacks work like every other request type via the `Fallbacks` field. diff --git a/examples/plugins/hello-world/go.mod b/examples/plugins/hello-world/go.mod index 1d8fae23c2f..94caf9664e6 100644 --- a/examples/plugins/hello-world/go.mod +++ b/examples/plugins/hello-world/go.mod @@ -2,7 +2,7 @@ module github.com/capsohq/bifrost/examples/plugins/hello-world go 1.27.0 -require github.com/capsohq/bifrost/core v1.9.1 +require github.com/capsohq/bifrost/core v1.10.2 require ( github.com/bahlo/generic-list-go v0.2.0 // indirect diff --git a/examples/plugins/hello-world/go.sum b/examples/plugins/hello-world/go.sum index 33e2e8da6a0..5bdb7784309 100644 --- a/examples/plugins/hello-world/go.sum +++ b/examples/plugins/hello-world/go.sum @@ -37,8 +37,8 @@ github.com/mailru/easyjson v0.9.1 h1:LbtsOm5WAswyWbvTEOqhypdPeZzHavpZx96/n553mR8 github.com/mailru/easyjson v0.9.1/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= github.com/mark3labs/mcp-go v0.43.2 h1:21PUSlWWiSbUPQwXIJ5WKlETixpFpq+WBpbMGDSVy/I= github.com/mark3labs/mcp-go v0.43.2/go.mod h1:YnJfOL382MIWDx1kMY+2zsRHU/q78dBg9aFb8W6Thdw= -github.com/maximhq/bifrost/core v1.9.1 h1:CF7TTj63htKQbuBL7eAGzou5yoe4XiQ0YMWpAO/atnw= -github.com/maximhq/bifrost/core v1.9.1/go.mod h1:bgTt5CsqB7duKiy/pC0uUMTxIRnvjj0t2OvH7Tlvuz0= +github.com/maximhq/bifrost/core v1.10.2 h1:jRvvZN1KJVaX2Pxn9crTrmL3kW2QgDh7d1jsuz6b72k= +github.com/maximhq/bifrost/core v1.10.2/go.mod h1:bgTt5CsqB7duKiy/pC0uUMTxIRnvjj0t2OvH7Tlvuz0= github.com/molecule-man/go-brrr v1.0.1 h1:cEjgx8hgNw6UGdhQ94SPDbPkKuRbkUcxBO3IzbGpA/o= github.com/molecule-man/go-brrr v1.0.1/go.mod h1:7ybW6/7gA3oKY45jOfVNjSJDtrr6ea4tzbsTkjmQDC4= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= diff --git a/framework/configstore/frameworkconfighash_test.go b/framework/configstore/frameworkconfighash_test.go index 37717957281..587c7fd8c1c 100644 --- a/framework/configstore/frameworkconfighash_test.go +++ b/framework/configstore/frameworkconfighash_test.go @@ -1,6 +1,10 @@ package configstore import ( + "github.com/bytedance/sonic" + "github.com/capsohq/bifrost/framework/configstore/tables" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "testing" ) @@ -84,3 +88,55 @@ func TestGenerateFrameworkConfigHash_NilLiveIntervalPreservesLegacyDigest(t *tes } }) } + +// ruleWithFallbacks decodes a routing fixture through the persisted fallback wire format. +func ruleWithFallbacks(t *testing.T, fallbacksJSON string) tables.TableRoutingRule { + t.Helper() + var parsed []tables.RoutingFallback + require.NoError(t, sonic.Unmarshal([]byte(fallbacksJSON), &parsed)) + return tables.TableRoutingRule{ + ID: "rule-1", + Name: "route gpt-4o", + CelExpression: `model == "gpt-4o"`, + Scope: "global", + Targets: []tables.TableRoutingTarget{{RuleID: "rule-1", Weight: 1}}, + ParsedFallbacks: parsed, + } +} + +// TestGenerateRoutingRuleHash_LegacyFallbacksPreserveByteShape proves a config-origin rule with legacy-string fallbacks still hashes to its pre-change digest. +func TestGenerateRoutingRuleHash_LegacyFallbacksPreserveByteShape(t *testing.T) { + for _, fallbacksJSON := range []string{ + `["openai/gpt-4o"]`, + `["azure/"]`, + `["anthropic"]`, + `["openai/gpt-4o","azure/","vertex/gemini-2.5-pro"]`, + } { + t.Run(fallbacksJSON, func(t *testing.T) { + rule := ruleWithFallbacks(t, fallbacksJSON) + + fromParsed, err := GenerateRoutingRuleHash(rule) + require.NoError(t, err) + + // The DB-origin path hashes the raw column verbatim; the config-origin path marshals ParsedFallbacks, and the two must agree. + raw := fallbacksJSON + dbRule := rule + dbRule.Fallbacks = &raw + fromRaw, err := GenerateRoutingRuleHash(dbRule) + require.NoError(t, err) + + assert.Equal(t, fromRaw, fromParsed, "config-origin hash drifted from the DB-origin hash") + }) + } +} + +// TestGenerateRoutingRuleHash_PinnedFallbackChangesHash detects changes to a configured key pin. +func TestGenerateRoutingRuleHash_PinnedFallbackChangesHash(t *testing.T) { + unpinned, err := GenerateRoutingRuleHash(ruleWithFallbacks(t, `["azure/gpt-4o"]`)) + require.NoError(t, err) + + pinned, err := GenerateRoutingRuleHash(ruleWithFallbacks(t, `[{"provider":"azure","model":"gpt-4o","key_id":"k1"}]`)) + require.NoError(t, err) + + assert.NotEqual(t, unpinned, pinned, "pinning a key must change the rule hash") +} diff --git a/framework/configstore/rdb.go b/framework/configstore/rdb.go index 3ba21f45cbf..3a5361720a1 100644 --- a/framework/configstore/rdb.go +++ b/framework/configstore/rdb.go @@ -5822,6 +5822,11 @@ func (s *RDBConfigStore) UpdateRoutingRule(ctx context.Context, rule *tables.Tab // created_at is immutable: Save writes every column, so a caller passing a rule it // didn't read from the DB would otherwise zero it out. Always keep the persisted value. rule.CreatedAt = existing.CreatedAt + // enabled is NOT NULL with a DB default, and Save writes a nil Enabled as NULL. + // An omitted value keeps the persisted state. + if rule.Enabled == nil { + rule.Enabled = existing.Enabled + } if err := tx.Omit("Targets").Save(rule).Error; err != nil { return err } @@ -5919,6 +5924,11 @@ func (s *RDBConfigStore) SyncRoutingRules(ctx context.Context, toAdd []tables.Ta // selects every column, so an unset CreatedAt would overwrite the original insert // timestamp with the zero time. Carry the persisted value forward. rule.CreatedAt = existing.CreatedAt + // config.json rules usually omit "enabled"; keep the persisted value instead of + // letting Save write NULL into the NOT NULL column. + if rule.Enabled == nil { + rule.Enabled = existing.Enabled + } if err := tx.Omit("Targets").Save(rule).Error; err != nil { return err } diff --git a/framework/configstore/rdb_test.go b/framework/configstore/rdb_test.go index 5317547e245..e6d296ac526 100644 --- a/framework/configstore/rdb_test.go +++ b/framework/configstore/rdb_test.go @@ -4427,6 +4427,42 @@ func TestRDBConfigStore_SyncRoutingRules(t *testing.T) { } } +// TestRDBConfigStore_RoutingRuleUpdateOmittedEnabled pins the update path for a rule whose +// enabled field is omitted, as every config.json rule without "enabled" is. Save writes every +// column, so a nil Enabled used to write NULL into the NOT NULL column and fail startup. +func TestRDBConfigStore_RoutingRuleUpdateOmittedEnabled(t *testing.T) { + ctx := context.Background() + + updaters := map[string]func(store *RDBConfigStore, rule *tables.TableRoutingRule) error{ + "SyncRoutingRules": func(store *RDBConfigStore, rule *tables.TableRoutingRule) error { + return store.SyncRoutingRules(ctx, nil, []tables.TableRoutingRule{*rule}) + }, + "UpdateRoutingRule": func(store *RDBConfigStore, rule *tables.TableRoutingRule) error { + return store.UpdateRoutingRule(ctx, rule) + }, + } + + for name, update := range updaters { + for _, stored := range []bool{true, false} { + t.Run(fmt.Sprintf("%s/stored=%t", name, stored), func(t *testing.T) { + store := setupRDBTestStore(t) + created := routingRuleFixture("rule-a", 0, "openai") + created.Enabled = new(stored) + require.NoError(t, store.CreateRoutingRule(ctx, created)) + + incoming := routingRuleFixture("rule-a", 0, "openai") + incoming.Enabled = nil + require.NoError(t, update(store, incoming)) + + got, err := store.GetRoutingRule(ctx, "rule-a") + require.NoError(t, err) + require.NotNil(t, got.Enabled) + require.Equal(t, stored, *got.Enabled, "omitted enabled must keep the stored value") + }) + } + } +} + // TestUpsertModelPricesBatch_InputCostPerQuerySurvivesResync guards the ON CONFLICT DO UPDATE // column list. Create() writes every column, so a first sync looks correct even when a field is // missing from pricingSyncUpdateColumns - the value only disappears on the next resync of an diff --git a/framework/configstore/tables/routingrules.go b/framework/configstore/tables/routingrules.go index f08499c5fcf..f02dee55ea1 100644 --- a/framework/configstore/tables/routingrules.go +++ b/framework/configstore/tables/routingrules.go @@ -1,11 +1,13 @@ package tables import ( + "bytes" "strings" "time" "github.com/bytedance/sonic" bifrost "github.com/capsohq/bifrost/core" + "github.com/capsohq/bifrost/core/schemas" "gorm.io/gorm" ) @@ -21,8 +23,8 @@ type TableRoutingRule struct { // Routing Targets (output) — 1:many relationship; weights must sum to 1 Targets []TableRoutingTarget `gorm:"foreignKey:RuleID;constraint:OnDelete:CASCADE" json:"targets"` - Fallbacks *string `gorm:"type:text" json:"-"` // JSON array of fallback chains - ParsedFallbacks []string `gorm:"-" json:"fallbacks,omitempty"` // Parsed fallbacks from JSON + Fallbacks *string `gorm:"type:text" json:"-"` // JSON array of fallback chains + ParsedFallbacks []RoutingFallback `gorm:"-" json:"fallbacks,omitempty"` // Parsed fallbacks from JSON Query *string `gorm:"type:text" json:"-"` ParsedQuery map[string]any `gorm:"-" json:"query,omitempty"` @@ -94,6 +96,70 @@ func (r *TableRoutingRule) AfterFind(tx *gorm.DB) error { return nil } +// RoutingFallback is one entry in a routing rule's fallback chain, decoded from either the legacy "provider/model" string or an object that pins a provider key. +type RoutingFallback struct { + schemas.Fallback // provider, model and key_id; the same fields core routes on + + ProviderKeyName *string `json:"provider_key_name,omitempty"` // config-only alias; resolved to key_id during load + + raw string // verbatim legacy string, replayed by MarshalJSON so unpinned entries round-trip byte-identically +} + +// IsKeyPinned reports whether this fallback names a specific provider key. +func (f RoutingFallback) IsKeyPinned() bool { + return strings.TrimSpace(f.KeyID) != "" || (f.ProviderKeyName != nil && strings.TrimSpace(*f.ProviderKeyName) != "") +} + +// String renders the legacy "provider/model" form. +func (f RoutingFallback) String() string { + if f.raw != "" { + return f.raw + } + if f.Provider == "" { + return f.Model + } + return string(f.Provider) + "/" + f.Model +} + +// MarshalJSON emits the legacy string unless a key is pinned, so unpinned rules keep their config hash. +func (f RoutingFallback) MarshalJSON() ([]byte, error) { + if !f.IsKeyPinned() { + return sonic.Marshal(f.String()) + } + type alias RoutingFallback + return sonic.Marshal(alias(f)) +} + +// UnmarshalJSON accepts the legacy "provider/model" string and the object form. +func (f *RoutingFallback) UnmarshalJSON(data []byte) error { + trimmed := bytes.TrimSpace(data) + if len(trimmed) > 0 && trimmed[0] == '"' { + var raw string + if err := sonic.Unmarshal(trimmed, &raw); err != nil { + return err + } + provider, model := schemas.ParseModelString(raw, "") + *f = RoutingFallback{Fallback: schemas.Fallback{Provider: provider, Model: model}, raw: raw} + return nil + } + type alias RoutingFallback + var decoded alias + if err := sonic.Unmarshal(trimmed, &decoded); err != nil { + return err + } + *f = RoutingFallback(decoded) + return nil +} + +// RoutingFallbackStrings renders a fallback slice in its legacy string form, for logs. +func RoutingFallbackStrings(fallbacks []RoutingFallback) []string { + out := make([]string, 0, len(fallbacks)) + for _, fb := range fallbacks { + out = append(out, fb.String()) + } + return out +} + // TableRoutingTarget represents a weighted routing target for probabilistic routing. // Multiple targets can be associated with a single routing rule; weights determine // the probability of each target being selected and must sum to 1 across all targets in a rule. diff --git a/framework/configstore/tables/routingrules_test.go b/framework/configstore/tables/routingrules_test.go new file mode 100644 index 00000000000..02cd0ff655c --- /dev/null +++ b/framework/configstore/tables/routingrules_test.go @@ -0,0 +1,122 @@ +package tables + +import ( + "testing" + + "github.com/bytedance/sonic" + "github.com/capsohq/bifrost/core/schemas" +) + +// TestRoutingFallback_UnpinnedRoundTripsByteIdentically guards GenerateRoutingRuleHash: a changed byte shape rewrites every rule on the next boot. +func TestRoutingFallback_UnpinnedRoundTripsByteIdentically(t *testing.T) { + cases := []string{ + `["openai/gpt-4o"]`, + `["azure/"]`, + `["anthropic"]`, + `["openai/ft:gpt-4o:org::abc/v2"]`, + `["meta-llama/Llama-3.1-8B"]`, + `[]`, + `["openai/gpt-4o","azure/","vertex/gemini-2.5-pro"]`, + } + for _, input := range cases { + t.Run(input, func(t *testing.T) { + var decoded []RoutingFallback + if err := sonic.Unmarshal([]byte(input), &decoded); err != nil { + t.Fatalf("unmarshal: %v", err) + } + out, err := sonic.Marshal(decoded) + if err != nil { + t.Fatalf("marshal: %v", err) + } + if string(out) != input { + t.Fatalf("round-trip changed bytes: got %s, want %s", out, input) + } + }) + } +} + +// TestRoutingFallback_ParsesLegacyString preserves the existing provider/model parser semantics. +func TestRoutingFallback_ParsesLegacyString(t *testing.T) { + cases := []struct { + input string + provider string + model string + }{ + {`"openai/gpt-4o"`, "openai", "gpt-4o"}, + {`"azure/"`, "azure", ""}, + {`"anthropic"`, "", "anthropic"}, + {`"meta-llama/Llama-3.1-8B"`, "", "meta-llama/Llama-3.1-8B"}, + } + for _, tc := range cases { + t.Run(tc.input, func(t *testing.T) { + var fb RoutingFallback + if err := sonic.Unmarshal([]byte(tc.input), &fb); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if string(fb.Provider) != tc.provider || fb.Model != tc.model { + t.Fatalf("got provider=%q model=%q, want provider=%q model=%q", fb.Provider, fb.Model, tc.provider, tc.model) + } + if fb.IsKeyPinned() { + t.Fatal("legacy string must not be pinned") + } + }) + } +} + +// TestRoutingFallback_ObjectFormKeepsKeyID preserves mixed pinned and legacy fallback chains. +func TestRoutingFallback_ObjectFormKeepsKeyID(t *testing.T) { + input := `["openai/gpt-4o",{"provider":"azure","model":"gpt-4o","key_id":"k1"}]` + var decoded []RoutingFallback + if err := sonic.Unmarshal([]byte(input), &decoded); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if len(decoded) != 2 { + t.Fatalf("got %d entries, want 2", len(decoded)) + } + if decoded[0].IsKeyPinned() { + t.Fatal("first entry must not be pinned") + } + if decoded[1].KeyID != "k1" || !decoded[1].IsKeyPinned() { + t.Fatalf("second entry lost its key: %+v", decoded[1]) + } + out, err := sonic.Marshal(decoded) + if err != nil { + t.Fatalf("marshal: %v", err) + } + if string(out) != input { + t.Fatalf("mixed round-trip: got %s, want %s", out, input) + } +} + +// TestRoutingFallback_ProgrammaticMarshalsAsString covers entries built in code, which have no raw literal to replay. +func TestRoutingFallback_ProgrammaticMarshalsAsString(t *testing.T) { + out, err := sonic.Marshal([]RoutingFallback{ + {Fallback: schemas.Fallback{Provider: "openai", Model: "gpt-4o"}}, + {Fallback: schemas.Fallback{Provider: "azure"}}, + }) + if err != nil { + t.Fatalf("marshal: %v", err) + } + if string(out) != `["openai/gpt-4o","azure/"]` { + t.Fatalf("got %s", out) + } +} + +// TestRoutingFallback_ProviderOnlyObjectSurvivesPersistence keeps incoming-model fallbacks valid after serialization. +func TestRoutingFallback_ProviderOnlyObjectSurvivesPersistence(t *testing.T) { + var fallback RoutingFallback + if err := sonic.Unmarshal([]byte(`{"provider":"azure"}`), &fallback); err != nil { + t.Fatal(err) + } + encoded, err := sonic.Marshal(fallback) + if err != nil { + t.Fatal(err) + } + var restored RoutingFallback + if err := sonic.Unmarshal(encoded, &restored); err != nil { + t.Fatal(err) + } + if restored.Provider != "azure" || restored.Model != "" || restored.IsKeyPinned() { + t.Fatalf("provider-only fallback changed after persistence: %+v (wire %s)", restored, encoded) + } +} diff --git a/framework/go.mod b/framework/go.mod index 3af88e31ca8..64cd9d17825 100644 --- a/framework/go.mod +++ b/framework/go.mod @@ -12,7 +12,7 @@ require ( github.com/bytedance/sonic v1.15.3-0.20260730064818-2a36d6da63e2 github.com/google/uuid v1.6.0 github.com/jackc/pgx/v5 v5.9.2 - github.com/capsohq/bifrost/core v1.9.1 + github.com/capsohq/bifrost/core v1.10.2 github.com/philippgille/chromem-go v0.7.0 github.com/pinecone-io/go-pinecone/v5 v5.3.0 github.com/qdrant/go-client v1.16.2 diff --git a/framework/go.sum b/framework/go.sum index 5c8b7ab204e..8273871bf3e 100644 --- a/framework/go.sum +++ b/framework/go.sum @@ -262,8 +262,8 @@ github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsRe github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= github.com/mattn/go-sqlite3 v1.14.32 h1:JD12Ag3oLy1zQA+BNn74xRgaBbdhbNIDYvQUEuuErjs= github.com/mattn/go-sqlite3 v1.14.32/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= -github.com/maximhq/bifrost/core v1.9.1 h1:CF7TTj63htKQbuBL7eAGzou5yoe4XiQ0YMWpAO/atnw= -github.com/maximhq/bifrost/core v1.9.1/go.mod h1:bgTt5CsqB7duKiy/pC0uUMTxIRnvjj0t2OvH7Tlvuz0= +github.com/maximhq/bifrost/core v1.10.2 h1:jRvvZN1KJVaX2Pxn9crTrmL3kW2QgDh7d1jsuz6b72k= +github.com/maximhq/bifrost/core v1.10.2/go.mod h1:bgTt5CsqB7duKiy/pC0uUMTxIRnvjj0t2OvH7Tlvuz0= github.com/molecule-man/go-brrr v1.0.1 h1:cEjgx8hgNw6UGdhQ94SPDbPkKuRbkUcxBO3IzbGpA/o= github.com/molecule-man/go-brrr v1.0.1/go.mod h1:7ybW6/7gA3oKY45jOfVNjSJDtrr6ea4tzbsTkjmQDC4= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= diff --git a/framework/modelcatalog/datasheet/cost.go b/framework/modelcatalog/datasheet/cost.go index a5688a6ca55..58048b41e60 100644 --- a/framework/modelcatalog/datasheet/cost.go +++ b/framework/modelcatalog/datasheet/cost.go @@ -210,7 +210,7 @@ func (s *Store) CalculateCostBreakdownForUsage(usage *schemas.BifrostLLMUsage, p // Apply the served tier (fast mode / data residency) carried on the usage so // cancelled/failed fast or US-residency streams keep their multiplier. input := costInput{usage: usage} - input.tier = tierFromResponse(nil, usage.Speed, usage.InferenceGeo) + input.tier = tierFromResponse(usage.ServiceTier, usage.Speed, usage.InferenceGeo) return s.computeCostFromInput( input, @@ -654,6 +654,8 @@ func (s *Store) computeCostFromInput(input costInput, routingInfo schemas.Routin cost = computeEmbeddingCost(pricing, input.usage, input.tier) case schemas.RerankRequest: cost = computeRerankCost(pricing, input.usage, input.tier) + case schemas.DecisionRequest: + cost = computeDecisionCost(pricing, input.usage, input.tier) case schemas.SpeechRequest: cost = computeSpeechCost(pricing, input.usage, input.audioSeconds, input.audioTextInputChars, input.tier) case schemas.TranscriptionRequest: @@ -759,6 +761,10 @@ func extractCostInput(result *schemas.BifrostResponse) costInput { case result.RerankResponse != nil: input.usage = result.RerankResponse.Usage + case result.DecisionResponse != nil && result.DecisionResponse.Usage != nil: + input.usage = result.DecisionResponse.Usage + input.tier = tierFromResponse(result.DecisionResponse.ServiceTier, result.DecisionResponse.Speed, result.DecisionResponse.InferenceGeo) + case result.SpeechResponse != nil && result.SpeechResponse.Usage != nil: input.usage = speechUsageToBifrostUsage(result.SpeechResponse.Usage) input.audioTextInputChars = result.SpeechResponse.Usage.InputChars @@ -1256,6 +1262,26 @@ func computeRerankCost(pricing *configstoreTables.TableModelPricing, usage *sche return newInputOutputCostWithDetails(inputTokensCost, outputTokensCost, inputDetails, outputDetails) } +// computeDecisionCost prices a decision request on token usage. Typesafe bills +// input tokens only, so the output rate is normally zero, but both sides are +// honored from the datasheet so a provider that later charges output stays +// correct without a code change. +// +// A decision priced from a chat or responses row was answered by emulation, the +// text call it made, so it is priced as one, with its cache read and write rates. +func computeDecisionCost(pricing *configstoreTables.TableModelPricing, usage *schemas.BifrostLLMUsage, tier serviceTier) *schemas.BifrostCost { + if usage == nil { + return nil + } + if pricing.Mode != "" && pricing.Mode != normalizeRequestType(schemas.DecisionRequest) { + return computeTextCost(pricing, usage, tier) + } + tierTokens := usage.PromptTokens + inputCost := float64(usage.PromptTokens) * tieredInputRate(pricing, tierTokens, tier) + outputCost := float64(usage.CompletionTokens) * tieredOutputRate(pricing, tierTokens, tier) + return newInputOutputCost(inputCost, outputCost) +} + // newInputOutputCost builds a BifrostCost from separate input and output costs, // or returns nil when both are zero (nothing to record). func newInputOutputCost(inputCost, outputCost float64) *schemas.BifrostCost { @@ -2169,7 +2195,13 @@ func (s *Store) resolvePricing(routingInfo schemas.RoutingInfo, requestType sche } base, exists := s.getBasePricing(candidate, catalogProvider, requestType) if exists && base != nil { - result, _ := s.applyPricingOverrides(overrideKey, requestType, *base, scopes) + result, matched := s.applyPricingOverrides(overrideKey, requestType, *base, scopes) + // A decision priced from a text row was answered by emulation, one Responses + // call, so an override written for that call applies when no decision-specific + // one does. + if !matched && requestType == schemas.DecisionRequest && base.Mode != normalizeRequestType(schemas.DecisionRequest) { + result, _ = s.applyPricingOverrides(overrideKey, schemas.ResponsesRequest, *base, scopes) + } return &result } s.logger.Debug("pricing not found for %s, trying next candidate", candidate) @@ -2210,6 +2242,22 @@ func (s *Store) getBasePricing(model, provider string, requestType schemas.Reque s.mu.RLock() defer s.mu.RUnlock() + pricing, ok := s.getBasePricingLocked(model, provider, requestType) + // A model with no decisions row answers a decision by emulation, which is a + // Responses call: price it from the model's chat row, or its responses row, + // rather than at zero (see computeDecisionCost). + if !ok && requestType == schemas.DecisionRequest { + pricing, ok = s.getBasePricingLocked(model, provider, schemas.ChatCompletionRequest) + if !ok { + pricing, ok = s.getBasePricingLocked(model, provider, schemas.ResponsesRequest) + } + } + return pricing, ok +} + +// getBasePricingLocked is getBasePricing's lookup for one request type; the caller +// holds s.mu for reading. +func (s *Store) getBasePricingLocked(model, provider string, requestType schemas.RequestType) (*configstoreTables.TableModelPricing, bool) { mode := normalizeRequestType(requestType) fallbackMode, hasFallbackMode := chatResponsesFallbackMode(requestType) diff --git a/framework/modelcatalog/datasheet/cost_test.go b/framework/modelcatalog/datasheet/cost_test.go index ee24297a04d..ad30a202ba6 100644 --- a/framework/modelcatalog/datasheet/cost_test.go +++ b/framework/modelcatalog/datasheet/cost_test.go @@ -260,6 +260,19 @@ func makeRerankResponse(provider schemas.ModelProvider, model string, usage *sch } } +// makeDecisionResponse builds a minimal BifrostResponse for a decision request. +func makeDecisionResponse(provider schemas.ModelProvider, model string, usage *schemas.BifrostLLMUsage) *schemas.BifrostResponse { + return &schemas.BifrostResponse{ + DecisionResponse: &schemas.BifrostDecisionResponse{ + Usage: usage, + ExtraFields: schemas.BifrostResponseExtraFields{ + RequestType: schemas.DecisionRequest, + RoutingInfo: routingInfoFor(provider, model), + }, + }, + } +} + // makeImageResponse builds a minimal BifrostResponse for an image generation request. func makeImageResponse(provider schemas.ModelProvider, model string, usage *schemas.ImageUsage) *schemas.BifrostResponse { return &schemas.BifrostResponse{ @@ -303,6 +316,10 @@ func computeRerankCostTotal(pricing *configstoreTables.TableModelPricing, usage return bcTotal(computeRerankCost(pricing, usage, tier)) } +func computeDecisionCostTotal(pricing *configstoreTables.TableModelPricing, usage *schemas.BifrostLLMUsage, tier serviceTier) float64 { + return bcTotal(computeDecisionCost(pricing, usage, tier)) +} + func computeSpeechCostTotal(pricing *configstoreTables.TableModelPricing, usage *schemas.BifrostLLMUsage, audioSeconds *float64, audioTextInputChars int, tier serviceTier) float64 { return bcTotal(computeSpeechCost(pricing, usage, audioSeconds, audioTextInputChars, tier)) } @@ -5373,6 +5390,154 @@ func TestCalculateCost_RerankPerQuery(t *testing.T) { }) } +// ========================================================================= +// computeDecisionCost — unit tests +// ========================================================================= + +func TestComputeDecisionCost_InputOnlyBilling(t *testing.T) { + // Typesafe's jev pricing shape: input tokens billed, output tokens free. + p := configstoreTables.TableModelPricing{ + InputCostPerToken: bifrost.Ptr(0.000000042), + OutputCostPerToken: bifrost.Ptr(0.0), + } + usage := &schemas.BifrostLLMUsage{ + PromptTokens: 50000, + CompletionTokens: 40, + TotalTokens: 50040, + } + cost := computeDecisionCostTotal(&p, usage, serviceTier{}) + assert.InDelta(t, 50000*0.000000042, cost, 1e-12) +} + +func TestComputeDecisionCost_OutputRateHonored(t *testing.T) { + p := configstoreTables.TableModelPricing{ + InputCostPerToken: bifrost.Ptr(0.000001), + OutputCostPerToken: bifrost.Ptr(0.000002), + } + usage := &schemas.BifrostLLMUsage{ + PromptTokens: 1000, + CompletionTokens: 500, + TotalTokens: 1500, + } + cost := computeDecisionCostTotal(&p, usage, serviceTier{}) + assert.InDelta(t, 1000*0.000001+500*0.000002, cost, 1e-12) +} + +func TestComputeDecisionCost_NilUsage(t *testing.T) { + p := configstoreTables.TableModelPricing{InputCostPerToken: new(0.000000042)} + assert.Equal(t, 0.0, computeDecisionCostTotal(&p, nil, serviceTier{})) +} + +func TestCalculateCost_DecisionTokenBilling(t *testing.T) { + // Pins the dispatch arm: an unhandled request type silently falls into + // CalculateCost's default nil branch and every decision would cost zero. + s := testStoreWithPricing(map[string]configstoreTables.TableModelPricing{ + makeKey("jev-1.13.0", "typesafe", "decisions"): { + Model: "jev-1.13.0", Provider: "typesafe", Mode: "decisions", + InputCostPerToken: bifrost.Ptr(0.000000042), + OutputCostPerToken: bifrost.Ptr(0.0), + }, + }) + + resp := makeDecisionResponse(schemas.Typesafe, "jev-1.13.0", &schemas.BifrostLLMUsage{ + PromptTokens: 50000, + TotalTokens: 50000, + }) + + assert.InDelta(t, 50000*0.000000042, s.CalculateCost(resp, nil), 1e-12) +} + +// A model with no decisions row answers a decision by emulation: one Responses call +// to the provider, which must be billed as that text call, cache reads and writes +// included, from its chat row or, failing that, its responses row. Without the +// fallback the decision found no row and cost nothing, so every emulated decision +// was a free paid call outside every budget. +func TestCalculateCost_EmulatedDecisionBilledAsTextCall(t *testing.T) { + usage := &schemas.BifrostLLMUsage{ + PromptTokens: 1000, + CompletionTokens: 100, + TotalTokens: 1100, + PromptTokensDetails: &schemas.ChatPromptTokensDetails{ + CachedReadTokens: 800, + CachedWriteTokens: 100, + }, + } + row := configstoreTables.TableModelPricing{ + Model: "gpt-5.5", Provider: "openai", + InputCostPerToken: bifrost.Ptr(0.000003), + OutputCostPerToken: bifrost.Ptr(0.00001), + CacheReadInputTokenCost: bifrost.Ptr(0.0000003), + CacheCreationInputTokenCost: bifrost.Ptr(0.00000375), + } + want := computeTextCostTotal(&row, usage, serviceTier{}) + if want <= 0 { + t.Fatalf("text cost of the emulated call = %v, want > 0", want) + } + + for _, mode := range []string{"chat", "responses"} { + t.Run(mode, func(t *testing.T) { + r := row + r.Mode = mode + s := testStoreWithPricing(map[string]configstoreTables.TableModelPricing{ + makeKey("gpt-5.5", "openai", mode): r, + }) + got := s.CalculateCost(makeDecisionResponse(schemas.OpenAI, "gpt-5.5", usage), nil) + assert.InDelta(t, want, got, 1e-12) + }) + } +} + +// An emulated decision served at priority is billed at the priority rate, as its text +// call would be. +func TestCalculateCost_EmulatedDecisionUsesServedTier(t *testing.T) { + s := testStoreWithPricing(map[string]configstoreTables.TableModelPricing{ + makeKey("gpt-5.5", "openai", "chat"): { + Model: "gpt-5.5", Provider: "openai", Mode: "chat", + InputCostPerToken: bifrost.Ptr(0.000001), OutputCostPerToken: bifrost.Ptr(0.000002), + InputCostPerTokenPriority: bifrost.Ptr(0.000005), + }, + }) + resp := makeDecisionResponse(schemas.OpenAI, "gpt-5.5", &schemas.BifrostLLMUsage{PromptTokens: 1000, TotalTokens: 1000}) + priority := schemas.BifrostServiceTierPriority + resp.DecisionResponse.ServiceTier = &priority + assert.InDelta(t, 1000*0.000005, s.CalculateCost(resp, nil), 1e-12) +} + +// A failed request billed from its bare usage keeps the served processing tier. +func TestCalculateCostForUsage_UsesServedServiceTier(t *testing.T) { + s := testStoreWithPricing(map[string]configstoreTables.TableModelPricing{ + makeKey("gpt-5.5", "openai", "responses"): { + Model: "gpt-5.5", Provider: "openai", Mode: "responses", + InputCostPerToken: bifrost.Ptr(0.000001), OutputCostPerToken: bifrost.Ptr(0.000002), + InputCostPerTokenPriority: bifrost.Ptr(0.000005), + }, + }) + priority := schemas.BifrostServiceTierPriority + usage := &schemas.BifrostLLMUsage{PromptTokens: 1000, TotalTokens: 1000, ServiceTier: &priority} + assert.InDelta(t, 1000*0.000005, s.CalculateCostForUsage(usage, schemas.OpenAI, "gpt-5.5", schemas.ResponsesRequest, nil), 1e-12) +} + +// A native decisions row keeps the token-only decision pricing even when the model +// also has a chat row. +func TestCalculateCost_NativeDecisionRowWinsOverChatRow(t *testing.T) { + s := testStoreWithPricing(map[string]configstoreTables.TableModelPricing{ + makeKey("jev-1.13.0", "typesafe", "decisions"): { + Model: "jev-1.13.0", Provider: "typesafe", Mode: "decisions", + InputCostPerToken: bifrost.Ptr(0.000000042), + OutputCostPerToken: bifrost.Ptr(0.0), + }, + makeKey("jev-1.13.0", "typesafe", "chat"): { + Model: "jev-1.13.0", Provider: "typesafe", Mode: "chat", + InputCostPerToken: bifrost.Ptr(0.001), + OutputCostPerToken: bifrost.Ptr(0.001), + }, + }) + resp := makeDecisionResponse(schemas.Typesafe, "jev-1.13.0", &schemas.BifrostLLMUsage{ + PromptTokens: 50000, CompletionTokens: 40, TotalTokens: 50040, + }) + assert.InDelta(t, 50000*0.000000042, s.CalculateCost(resp, nil), 1e-12) +} + func TestCalculateCost_RerankPerTokenStillWorks(t *testing.T) { // Voyage and Jina style rerankers bill per token and carry no per-query rate; the two // pricing shapes must not interfere. diff --git a/framework/modelcatalog/datasheet/overrides_test.go b/framework/modelcatalog/datasheet/overrides_test.go index ba002711d20..ec8fad4d044 100644 --- a/framework/modelcatalog/datasheet/overrides_test.go +++ b/framework/modelcatalog/datasheet/overrides_test.go @@ -1132,3 +1132,35 @@ func TestPatchPricing_TimeOfDayFields(t *testing.T) { assert.Same(t, baseSchedule, base.PeakHours) }) } + +// An emulated decision is one Responses call priced from the model's text row, so a +// Responses override for that model applies to it; a decision-specific override still +// wins when there is one. +func TestGetPricing_EmulatedDecisionUsesResponsesOverride(t *testing.T) { + s := newTestStore() + s.pricingData[makeKey("gpt-4o", "openai", "chat")] = configstoreTables.TableModelPricing{ + Model: "gpt-4o", Provider: "openai", Mode: "chat", + InputCostPerToken: bifrost.Ptr(1.0), OutputCostPerToken: bifrost.Ptr(2.0), + } + providerID := "openai" + responsesOverride := configstoreTables.TablePricingOverride{ + ID: "responses", ScopeKind: string(ScopeKindProvider), ProviderID: &providerID, + MatchType: string(MatchTypeExact), Pattern: "gpt-4o", + RequestTypes: []schemas.RequestType{schemas.ResponsesRequest}, + PricingPatchJSON: `{"input_cost_per_token":7}`, + } + require.NoError(t, s.SetOverrides([]configstoreTables.TablePricingOverride{responsesOverride})) + pricing := s.resolvePricing(schemas.RoutingInfo{Provider: "openai", Model: "gpt-4o"}, schemas.DecisionRequest, LookupScopes{Provider: "openai"}) + require.NotNil(t, pricing) + assert.Equal(t, 7.0, *pricing.InputCostPerToken) + + require.NoError(t, s.SetOverrides([]configstoreTables.TablePricingOverride{responsesOverride, { + ID: "decisions", ScopeKind: string(ScopeKindProvider), ProviderID: &providerID, + MatchType: string(MatchTypeExact), Pattern: "gpt-4o", + RequestTypes: []schemas.RequestType{schemas.DecisionRequest}, + PricingPatchJSON: `{"input_cost_per_token":3}`, + }})) + pricing = s.resolvePricing(schemas.RoutingInfo{Provider: "openai", Model: "gpt-4o"}, schemas.DecisionRequest, LookupScopes{Provider: "openai"}) + require.NotNil(t, pricing) + assert.Equal(t, 3.0, *pricing.InputCostPerToken) +} diff --git a/framework/modelcatalog/datasheet/store.go b/framework/modelcatalog/datasheet/store.go index a5895879e8d..852a22be217 100644 --- a/framework/modelcatalog/datasheet/store.go +++ b/framework/modelcatalog/datasheet/store.go @@ -184,6 +184,7 @@ func (s *Store) GetPricingEntryForModel(model string, provider schemas.ModelProv schemas.ResponsesRequest, schemas.EmbeddingRequest, schemas.RerankRequest, + schemas.DecisionRequest, schemas.SpeechRequest, schemas.TranscriptionRequest, schemas.ImageGenerationRequest, diff --git a/framework/modelcatalog/datasheet/types.go b/framework/modelcatalog/datasheet/types.go index 6e68b37607d..9d7cdfbe272 100644 --- a/framework/modelcatalog/datasheet/types.go +++ b/framework/modelcatalog/datasheet/types.go @@ -445,6 +445,8 @@ func normalizeRequestType(reqType schemas.RequestType) string { return "embedding" case schemas.RerankRequest: return "rerank" + case schemas.DecisionRequest: + return "decisions" case schemas.SpeechRequest, schemas.SpeechStreamRequest: return "audio_speech" case schemas.TranscriptionRequest, schemas.TranscriptionStreamRequest: diff --git a/framework/streaming/gate.go b/framework/streaming/gate.go index 19c84687fc4..6cef32b54bb 100644 --- a/framework/streaming/gate.go +++ b/framework/streaming/gate.go @@ -1,9 +1,11 @@ package streaming import ( + "encoding/json" "fmt" "time" + "github.com/bytedance/sonic" "github.com/capsohq/bifrost/core/schemas" ) @@ -12,19 +14,199 @@ import ( // consumer is notified and memory is released. const gateReplayBufMaxBytes int64 = 100 * 1024 * 1024 -// chunkBytes returns the approximate wire size of a BifrostStreamChunk via -// MarshalJSON. Used only on the paused replay path to bound buffer growth. -func chunkBytes(chunk *schemas.BifrostStreamChunk) int64 { - if chunk == nil { +// gateChunkSizeFloor is the per-chunk baseline for replay accounting: struct +// overhead plus the small scalar fields the estimator does not walk. Measured +// heap overhead for a passthrough chunk is ~500B; 512 keeps the floor honest +// without tripping the 100MB cap on large-but-legitimate streams (a 60MB +// stream in 1.4KB events carries ~43k chunks — a 1KB floor alone would have +// added 43MB to the estimate). +const gateChunkSizeFloor int64 = 512 + +// gateReplayEntry pairs a buffered chunk with its size estimate, computed once +// at append so cap accounting never re-serializes held chunks. +// +// Ownership contract: a chunk handed to GateSend belongs to the delivery +// pipeline from that point on. Producers must hand over freshly built, +// right-sized chunks (no shared scratch buffers, no sub-slices of larger +// arrays) and must not retain or mutate them afterward — the same invariant +// the non-gated channel send already requires, since the chunk outlives the +// producer's loop either way. The only sanctioned post-append modification is +// TransformPausedBuffer, whose copy-on-write replacements are re-estimated at +// install. Violating this contract cannot corrupt delivery or bypass guardrail +// inspection; it can only make the cached size (and thus the +// gateReplayBufMaxBytes accounting) under- or over-read. +type gateReplayEntry struct { + chunk *schemas.BifrostStreamChunk + size int64 +} + +// gateRawBytes sizes the raw request/response carriers without serializing: +// they typically hold json.RawMessage, []byte, or string. An atypical carrier +// (e.g. a pre-parsed map) is sonic-sized rather than riding the buffer +// uncounted; that path only exists for providers that deviate from +// compactRawJSON's json.RawMessage contract. +func gateRawBytes(v interface{}) int64 { + switch b := v.(type) { + case nil: + return 0 + case *interface{}: + // Unwrap a pointer-boxed value so its payload hits the fast len() + // cases below instead of the sonic fallback. + if b == nil { + return 0 + } + return gateRawBytes(*b) + case json.RawMessage: + return int64(len(b)) + case []byte: + return int64(len(b)) + case string: + return int64(len(b)) + } + return gateCompositeBytes(v) +} + +func gateStrPtrBytes(s *string) int64 { + if s == nil { return 0 } - b, err := chunk.MarshalJSON() + return int64(len(*s)) +} + +// gateCompositeBytes sizes a composite carrier (full item, part, or terminal +// message copy) with a plain sonic marshal. Composites ride only item-boundary +// events, never per-delta chunks, so this stays off the hot append path. +func gateCompositeBytes(v interface{}) int64 { + b, err := sonic.Marshal(v) if err != nil { return 0 } return int64(len(b)) } +// estimateChunkBytes estimates the heap held by one buffered chunk without +// serializing it. It feeds only the gateReplayBufMaxBytes safety cap, so +// exactness is not required: it counts the carriers that grow with response +// size — passthrough bodies, raw request/response bytes, delta/text strings, +// tool-call arguments, and composite item copies — and covers everything else +// with a fixed per-chunk floor. +func estimateChunkBytes(chunk *schemas.BifrostStreamChunk) int64 { + if chunk == nil { + return 0 + } + size := gateChunkSizeFloor + switch { + case chunk.BifrostPassthroughResponse != nil: + r := chunk.BifrostPassthroughResponse + size += int64(len(r.Body)) + size += gateRawBytes(r.ExtraFields.RawRequest) + gateRawBytes(r.ExtraFields.RawResponse) + case chunk.BifrostChatResponse != nil: + r := chunk.BifrostChatResponse + size += int64(len(r.ID)) + gateChoiceBytes(r.Choices) + size += gateRawBytes(r.ExtraFields.RawRequest) + gateRawBytes(r.ExtraFields.RawResponse) + case chunk.BifrostTextCompletionResponse != nil: + r := chunk.BifrostTextCompletionResponse + size += int64(len(r.ID)) + gateChoiceBytes(r.Choices) + size += gateRawBytes(r.ExtraFields.RawRequest) + gateRawBytes(r.ExtraFields.RawResponse) + case chunk.BifrostResponsesStreamResponse != nil: + r := chunk.BifrostResponsesStreamResponse + size += gateStrPtrBytes(r.Delta) + gateStrPtrBytes(r.Text) + gateStrPtrBytes(r.Arguments) + + gateStrPtrBytes(r.Input) + gateStrPtrBytes(r.Refusal) + gateStrPtrBytes(r.Obfuscation) + + gateStrPtrBytes(r.PartialImageB64) + gateStrPtrBytes(r.Signature) + + gateStrPtrBytes(r.Code) + gateStrPtrBytes(r.Message) + gateStrPtrBytes(r.Param) + + int64(len(r.SafeguardResults)) + for _, citation := range r.Citations { + size += int64(len(citation)) + } + if r.Annotation != nil { + size += gateCompositeBytes(r.Annotation) + } + if r.Error != nil { + size += gateCompositeBytes(r.Error) + } + if len(r.SearchResults) > 0 { + size += gateCompositeBytes(r.SearchResults) + } + if len(r.Videos) > 0 { + size += gateCompositeBytes(r.Videos) + } + if r.Item != nil { + size += gateCompositeBytes(r.Item) + } + if r.Part != nil { + size += gateCompositeBytes(r.Part) + } + if r.Response != nil { + size += gateCompositeBytes(r.Response) + } + if len(r.LogProbs) > 0 { + // Same treatment as chat choice logprobs: only present when the + // client requested them, several KB per chunk at high top_logprobs. + size += gateCompositeBytes(r.LogProbs) + } + size += gateRawBytes(r.ExtraFields.RawRequest) + gateRawBytes(r.ExtraFields.RawResponse) + case chunk.BifrostSpeechStreamResponse != nil: + r := chunk.BifrostSpeechStreamResponse + size += int64(len(r.Audio)) + size += gateRawBytes(r.ExtraFields.RawRequest) + gateRawBytes(r.ExtraFields.RawResponse) + case chunk.BifrostTranscriptionStreamResponse != nil: + r := chunk.BifrostTranscriptionStreamResponse + size += gateStrPtrBytes(r.Delta) + int64(len(r.Text)) + size += gateRawBytes(r.ExtraFields.RawRequest) + gateRawBytes(r.ExtraFields.RawResponse) + case chunk.BifrostImageGenerationStreamResponse != nil: + r := chunk.BifrostImageGenerationStreamResponse + size += int64(len(r.B64JSON)) + int64(len(r.URL)) + // The image variant carries direct raw strings separate from ExtraFields. + size += int64(len(r.RawRequest)) + int64(len(r.RawResponse)) + size += gateRawBytes(r.ExtraFields.RawRequest) + gateRawBytes(r.ExtraFields.RawResponse) + case chunk.BifrostError != nil: + e := chunk.BifrostError + if e.Error != nil { + size += int64(len(e.Error.Message)) + } + size += gateRawBytes(e.ExtraFields.RawRequest) + gateRawBytes(e.ExtraFields.RawResponse) + } + return size +} + +// gateChoiceBytes sizes the text carriers of chat/text-completion choices: +// stream deltas (content, reasoning, refusal, tool-call arguments), terminal +// message copies, and text-completion text. +func gateChoiceBytes(choices []schemas.BifrostResponseChoice) int64 { + var size int64 + for i := range choices { + c := &choices[i] + if c.ChatStreamResponseChoice != nil && c.ChatStreamResponseChoice.Delta != nil { + d := c.ChatStreamResponseChoice.Delta + size += gateStrPtrBytes(d.Content) + gateStrPtrBytes(d.Reasoning) + gateStrPtrBytes(d.Refusal) + size += int64(len(d.ExtraContent)) + for j := range d.ToolCalls { + size += int64(len(d.ToolCalls[j].Function.Arguments)) + } + if d.Audio != nil { + size += int64(len(d.Audio.Data)) + int64(len(d.Audio.Transcript)) + } + for j := range d.ReasoningDetails { + rd := &d.ReasoningDetails[j] + size += gateStrPtrBytes(rd.Text) + gateStrPtrBytes(rd.Data) + + gateStrPtrBytes(rd.Signature) + gateStrPtrBytes(rd.Summary) + } + } + if c.LogProbs != nil { + // Only present when the client requested logprobs; several KB per + // chunk at top_logprobs=20, far past the floor. + size += gateCompositeBytes(c.LogProbs) + } + if c.ChatNonStreamResponseChoice != nil && c.ChatNonStreamResponseChoice.Message != nil { + size += gateCompositeBytes(c.ChatNonStreamResponseChoice.Message) + } + if c.TextCompletionResponseChoice != nil { + size += gateStrPtrBytes(c.TextCompletionResponseChoice.Text) + } + } + return size +} + // PauseStream is the Tracer-level entry point for pausing a stream. // Forwards to the per-stream accumulator entry keyed by traceID. func (a *Accumulator) PauseStream(traceID string) { @@ -360,7 +542,12 @@ func (sa *StreamAccumulator) TransformPausedBuffer(transform schemas.PausedStrea sa.mu.Unlock() return fmt.Errorf("paused stream buffer transform scope is invalid") } - original := append([]*schemas.BifrostStreamChunk(nil), sa.gateReplayBuf[start:]...) + original := make([]*schemas.BifrostStreamChunk, len(sa.gateReplayBuf)-start) + var originalBytes int64 + for i, entry := range sa.gateReplayBuf[start:] { + original[i] = entry.chunk + originalBytes += entry.size + } sa.mu.Unlock() result, err := transform(original) @@ -374,10 +561,12 @@ func (sa *StreamAccumulator) TransformPausedBuffer(transform schemas.PausedStrea return fmt.Errorf("paused stream buffer transform release count %d is outside [0,%d]", result.ReleaseCount, len(original)) } - var originalBytes, transformedBytes int64 - for i := range original { - originalBytes += chunkBytes(original[i]) - transformedBytes += chunkBytes(result.Chunks[i]) + // Size the replacements once; cached alongside each installed entry below. + transformedSizes := make([]int64, len(result.Chunks)) + var transformedBytes int64 + for i := range result.Chunks { + transformedSizes[i] = estimateChunkBytes(result.Chunks[i]) + transformedBytes += transformedSizes[i] } sa.mu.Lock() @@ -390,7 +579,7 @@ func (sa *StreamAccumulator) TransformPausedBuffer(transform schemas.PausedStrea return fmt.Errorf("paused stream buffer changed during transformation") } for i := range original { - if sa.gateReplayBuf[currentStart+i] != original[i] { + if sa.gateReplayBuf[currentStart+i].chunk != original[i] { return fmt.Errorf("paused stream buffer changed during transformation") } } @@ -401,7 +590,9 @@ func (sa *StreamAccumulator) TransformPausedBuffer(transform schemas.PausedStrea if nextBytes > gateReplayBufMaxBytes { return fmt.Errorf("transformed paused stream buffer exceeds %d bytes", gateReplayBufMaxBytes) } - copy(sa.gateReplayBuf[currentStart:], result.Chunks) + for i := range result.Chunks { + sa.gateReplayBuf[currentStart+i] = gateReplayEntry{chunk: result.Chunks[i], size: transformedSizes[i]} + } sa.gateReplayBufBytes = nextBytes nextBoundary := currentStart + result.ReleaseCount sa.gateTransformStart = nextBoundary @@ -479,11 +670,14 @@ func (sa *StreamAccumulator) GateSend(chunk *schemas.BifrostStreamChunk, isFinal // or until EndStream is called explicitly. Enforces a 100 MB cap to // prevent unbounded heap growth from a paused-and-forgotten stream. if sa.gateState == StreamStatePaused { - size := chunkBytes(chunk) + size := estimateChunkBytes(chunk) if sa.gateReplayBufBytes+size > gateReplayBufMaxBytes { // Overflow: force-end the gate with a synthetic error so the // consumer is notified and memory is released. Drops this chunk - // and any further chunks for this stream. + // and any further chunks for this stream, and every held chunk + // past the approved prefix: those were never approved, so ending + // the gate must not release them. + sa.truncateReplayToApprovedPrefixLocked() sa.gateState = StreamStateEnded // Replay cannot continue after overflow forces the gate to end. sa.gateReplayEventInterval = 0 @@ -505,7 +699,7 @@ func (sa *StreamAccumulator) GateSend(chunk *schemas.BifrostStreamChunk, isFinal sa.mu.Unlock() return false } - sa.gateReplayBuf = append(sa.gateReplayBuf, chunk) + sa.gateReplayBuf = append(sa.gateReplayBuf, gateReplayEntry{chunk: chunk, size: size}) sa.gateReplayBufBytes += size if isFinal || isHardErr { sa.gatePendingTerminal = true @@ -530,8 +724,9 @@ func (sa *StreamAccumulator) GateSend(chunk *schemas.BifrostStreamChunk, isFinal // If there's already a flusher or a non-empty buffer, append the final // chunk so it's delivered after pending chunks. Otherwise fast-path send. if sa.gateFlusherOn || len(sa.gateReplayBuf) > 0 { - sa.gateReplayBuf = append(sa.gateReplayBuf, chunk) - sa.gateReplayBufBytes += chunkBytes(chunk) + size := estimateChunkBytes(chunk) + sa.gateReplayBuf = append(sa.gateReplayBuf, gateReplayEntry{chunk: chunk, size: size}) + sa.gateReplayBufBytes += size sa.gateState = StreamStateEnded if sa.gateCond != nil { sa.gateCond.Broadcast() @@ -548,8 +743,9 @@ func (sa *StreamAccumulator) GateSend(chunk *schemas.BifrostStreamChunk, isFinal // so order is preserved between buffered and live chunks. Otherwise pure // passthrough. if len(sa.gateReplayBuf) > 0 { - sa.gateReplayBuf = append(sa.gateReplayBuf, chunk) - sa.gateReplayBufBytes += chunkBytes(chunk) + size := estimateChunkBytes(chunk) + sa.gateReplayBuf = append(sa.gateReplayBuf, gateReplayEntry{chunk: chunk, size: size}) + sa.gateReplayBufBytes += size sa.gateCond.Broadcast() sa.mu.Unlock() return true @@ -558,13 +754,33 @@ func (sa *StreamAccumulator) GateSend(chunk *schemas.BifrostStreamChunk, isFinal return sendOrCancel(ctx, ch, chunk) } +// truncateReplayToApprovedPrefixLocked drops the held chunks past the approved +// prefix, keeping only what a decision released for delivery. Caller holds sa.mu. +func (sa *StreamAccumulator) truncateReplayToApprovedPrefixLocked() { + keep := min(max(sa.gateApprovedPrefix, 0), len(sa.gateReplayBuf)) + for _, entry := range sa.gateReplayBuf[keep:] { + sa.gateReplayBufBytes -= entry.size + } + if sa.gateReplayBufBytes < 0 || keep == 0 { + sa.gateReplayBufBytes = 0 + } + if keep == 0 { + sa.gateReplayBuf = nil + } else { + sa.gateReplayBuf = sa.gateReplayBuf[:keep:keep] + } + sa.gateTransformStart = min(sa.gateTransformStart, keep) + // A terminal chunk among the dropped ones is gone with them. + sa.gatePendingTerminal = false +} + // drainBufferLocked drains gateReplayBuf to gateFlusherCh in order. MUST be // called with sa.mu held. Releases sa.mu while sending; reacquires before // returning. While paused, it drains only the approved prefix and leaves the // current unevaluated suffix held. It also stops when ctx is done. func (sa *StreamAccumulator) drainBufferLocked() { for len(sa.gateReplayBuf) > 0 && (sa.gateState != StreamStatePaused || sa.gateApprovedPrefix > 0) { - chunk := sa.gateReplayBuf[0] + entry := sa.gateReplayBuf[0] sa.gateReplayBuf = sa.gateReplayBuf[1:] if sa.gateTransformStart > 0 { sa.gateTransformStart-- @@ -573,17 +789,16 @@ func (sa *StreamAccumulator) drainBufferLocked() { sa.gateApprovedPrefix-- } // Decrement bytes per-chunk so the counter stays accurate even if - // the loop exits mid-drain (Pause). Clamp at zero to absorb any - // rounding drift between MarshalJSON sizes captured on append vs. - // re-computed here. - sa.gateReplayBufBytes -= chunkBytes(chunk) + // the loop exits mid-drain (Pause). Sizes are cached at append, so + // this is exact; the clamp is pure defense. + sa.gateReplayBufBytes -= entry.size if sa.gateReplayBufBytes < 0 { sa.gateReplayBufBytes = 0 } ch := sa.gateFlusherCh ctx := sa.gateFlusherCtx sa.mu.Unlock() - ok := sendOrCancel(ctx, ch, chunk) + ok := sendOrCancel(ctx, ch, entry.chunk) sa.mu.Lock() if ok && len(sa.gateReplayBuf) > 0 { ok = sa.waitForReplayIntervalLocked() diff --git a/framework/streaming/gate_test.go b/framework/streaming/gate_test.go index 81739a2b30c..6635dc690d2 100644 --- a/framework/streaming/gate_test.go +++ b/framework/streaming/gate_test.go @@ -1,7 +1,9 @@ package streaming import ( + "bytes" "context" + "encoding/json" "fmt" "strings" "sync" @@ -32,6 +34,15 @@ func makeChunks(n int) []*schemas.BifrostStreamChunk { return out } +// gateEntries wraps chunks as replay entries with cached sizes, mirroring production appends. +func gateEntries(chunks ...*schemas.BifrostStreamChunk) []gateReplayEntry { + out := make([]gateReplayEntry, len(chunks)) + for i, c := range chunks { + out[i] = gateReplayEntry{chunk: c, size: estimateChunkBytes(c)} + } + return out +} + // recorder consumes chunks off a channel, recording each with its arrival timestamp. type recorder struct { mu sync.Mutex @@ -606,7 +617,7 @@ func TestGate_TransformPausedStreamBufferRejectsConcurrentAppend(t *testing.T) { func TestGate_TransformPausedStreamBufferScopesLatestPause(t *testing.T) { sa := &StreamAccumulator{ gateState: StreamStateActive, - gateReplayBuf: makeChunks(2), + gateReplayBuf: gateEntries(makeChunks(2)...), gateReplayBufBytes: 1, } sa.Pause() @@ -614,7 +625,7 @@ func TestGate_TransformPausedStreamBufferScopesLatestPause(t *testing.T) { t.Fatalf("new pause boundaries = (%d, %d), want transform start 2 and approved prefix 0", sa.gateTransformStart, sa.gateApprovedPrefix) } latest := makeChunks(1)[0] - sa.gateReplayBuf = append(sa.gateReplayBuf, latest) + sa.gateReplayBuf = append(sa.gateReplayBuf, gateEntries(latest)...) seen := 0 err := sa.TransformPausedBuffer(func(buffered []*schemas.BifrostStreamChunk) (schemas.PausedStreamBufferTransformResult, error) { @@ -684,8 +695,8 @@ func TestGate_TransformPausedStreamBufferEnforcesCap(t *testing.T) { sa := &StreamAccumulator{gateState: StreamStateActive} sa.Pause() original := makeChunks(1)[0] - sa.gateReplayBuf = append(sa.gateReplayBuf, original) - originalBytes := chunkBytes(original) + sa.gateReplayBuf = append(sa.gateReplayBuf, gateEntries(original)...) + originalBytes := estimateChunkBytes(original) sa.gateReplayBufBytes = gateReplayBufMaxBytes - 1 err := sa.TransformPausedBuffer(func(buffered []*schemas.BifrostStreamChunk) (schemas.PausedStreamBufferTransformResult, error) { @@ -698,7 +709,7 @@ func TestGate_TransformPausedStreamBufferEnforcesCap(t *testing.T) { if err == nil { t.Fatal("TransformPausedBuffer() error = nil for oversized replacement") } - if sa.gateReplayBuf[0] != original { + if sa.gateReplayBuf[0].chunk != original { t.Fatal("oversized replacement changed the buffered chunk") } } @@ -1775,3 +1786,549 @@ func TestGate_GetAccumulatedResponse_DoesNotEngageGate(t *testing.T) { t.Fatalf("GetAccumulatedResponse must not engage the gate (StreamGated should remain unset)") } } + +// TestGate_EstimateChunkBytesCountsBulkCarriers: the estimator must count the +// fields that grow with response size — passthrough bodies, raw request/response +// bytes, and flat delta/text carriers — without serializing the chunk. Small +// metadata rides the per-chunk floor. +func TestGate_EstimateChunkBytesCountsBulkCarriers(t *testing.T) { + if got := estimateChunkBytes(nil); got != 0 { + t.Fatalf("estimateChunkBytes(nil) = %d, want 0", got) + } + + body := make([]byte, 10*1024) + pt := &schemas.BifrostStreamChunk{ + BifrostPassthroughResponse: &schemas.BifrostPassthroughResponse{Body: body}, + } + if got := estimateChunkBytes(pt); got < int64(len(body)) { + t.Fatalf("passthrough estimate = %d, want >= %d (body length)", got, len(body)) + } + + content := strings.Repeat("c", 8*1024) + args := strings.Repeat("a", 4*1024) + chat := &schemas.BifrostStreamChunk{ + BifrostChatResponse: &schemas.BifrostChatResponse{ + Choices: []schemas.BifrostResponseChoice{{ + ChatStreamResponseChoice: &schemas.ChatStreamResponseChoice{ + Delta: &schemas.ChatStreamResponseChoiceDelta{ + Content: &content, + ToolCalls: []schemas.ChatAssistantMessageToolCall{{ + Function: schemas.ChatAssistantMessageToolCallFunction{Arguments: args}, + }}, + }, + }, + }}, + }, + } + if got := estimateChunkBytes(chat); got < int64(len(content)+len(args)) { + t.Fatalf("chat estimate = %d, want >= %d (delta content + tool args)", got, len(content)+len(args)) + } + + raw := json.RawMessage(make([]byte, 6*1024)) + withRaw := &schemas.BifrostStreamChunk{ + BifrostChatResponse: &schemas.BifrostChatResponse{ + ExtraFields: schemas.BifrostResponseExtraFields{RawResponse: raw}, + }, + } + if got := estimateChunkBytes(withRaw); got < int64(len(raw)) { + t.Fatalf("raw-response estimate = %d, want >= %d", got, len(raw)) + } + + delta := strings.Repeat("d", 5*1024) + resp := &schemas.BifrostStreamChunk{ + BifrostResponsesStreamResponse: &schemas.BifrostResponsesStreamResponse{Delta: &delta}, + } + if got := estimateChunkBytes(resp); got < int64(len(delta)) { + t.Fatalf("responses estimate = %d, want >= %d (delta)", got, len(delta)) + } + + // Every variable-sized Responses carrier counts, not only the text deltas: a large + // event carried in any one of them must not be charged the 512-byte floor. + big := strings.Repeat("v", 64*1024) + for name, r := range map[string]*schemas.BifrostResponsesStreamResponse{ + "safeguard_results": {SafeguardResults: json.RawMessage(`"` + big + `"`)}, + "search_results": {SearchResults: []schemas.SearchResult{{Title: "t", URL: "u", Snippet: &big}}}, + "videos": {Videos: []schemas.VideoResult{{URL: big}}}, + "citations": {Citations: []string{big}}, + "annotation": {Annotation: &schemas.ResponsesOutputMessageContentTextAnnotation{Text: &big}}, + "error_message": {Message: &big}, + "error_code": {Code: &big}, + } { + chunk := &schemas.BifrostStreamChunk{BifrostResponsesStreamResponse: r} + if got := estimateChunkBytes(chunk); got < int64(len(big)) { + t.Errorf("responses %s estimate = %d, want >= %d", name, got, len(big)) + } + } + + audioData := strings.Repeat("A", 16*1024) + audio := &schemas.BifrostStreamChunk{ + BifrostChatResponse: &schemas.BifrostChatResponse{ + Choices: []schemas.BifrostResponseChoice{{ + ChatStreamResponseChoice: &schemas.ChatStreamResponseChoice{ + Delta: &schemas.ChatStreamResponseChoiceDelta{ + Audio: &schemas.ChatAudioMessageAudio{Data: audioData}, + }, + }, + }}, + }, + } + if got := estimateChunkBytes(audio); got < int64(len(audioData)) { + t.Fatalf("audio-delta estimate = %d, want >= %d (b64 audio data)", got, len(audioData)) + } + + rawImage := strings.Repeat("r", 8*1024) + img := &schemas.BifrostStreamChunk{ + BifrostImageGenerationStreamResponse: &schemas.BifrostImageGenerationStreamResponse{ + RawRequest: rawImage, + RawResponse: rawImage, + }, + } + if got := estimateChunkBytes(img); got < int64(2*len(rawImage)) { + t.Fatalf("image estimate = %d, want >= %d (direct raw request+response strings)", got, 2*len(rawImage)) + } + + lp := make([]schemas.ResponsesOutputMessageContentTextLogProb, 256) + for i := range lp { + lp[i] = schemas.ResponsesOutputMessageContentTextLogProb{Token: strings.Repeat("t", 32)} + } + respLP := &schemas.BifrostStreamChunk{ + BifrostResponsesStreamResponse: &schemas.BifrostResponsesStreamResponse{LogProbs: lp}, + } + if got := estimateChunkBytes(respLP); got < int64(256*32) { + t.Fatalf("responses logprobs estimate = %d, want >= %d", got, 256*32) + } + + structuredRaw := map[string]interface{}{"blob": strings.Repeat("m", 4*1024)} + mapRaw := &schemas.BifrostStreamChunk{ + BifrostChatResponse: &schemas.BifrostChatResponse{ + ExtraFields: schemas.BifrostResponseExtraFields{RawResponse: structuredRaw}, + }, + } + if got := estimateChunkBytes(mapRaw); got < int64(4*1024) { + t.Fatalf("structured raw estimate = %d, want >= %d (sonic fallback for non-bytes carriers)", got, 4*1024) + } + + var boxed interface{} = json.RawMessage(make([]byte, 3*1024)) + boxedRaw := &schemas.BifrostStreamChunk{ + BifrostChatResponse: &schemas.BifrostChatResponse{ + ExtraFields: schemas.BifrostResponseExtraFields{RawResponse: &boxed}, + }, + } + if got := estimateChunkBytes(boxedRaw); got < int64(3*1024) { + t.Fatalf("pointer-boxed raw estimate = %d, want >= %d (unwrapped *interface{})", got, 3*1024) + } + var nilBoxed *interface{} + if got := gateRawBytes(nilBoxed); got != 0 { + t.Fatalf("nil *interface{} raw = %d, want 0", got) + } +} + +// TestGate_PausedBufferAccountingUsesCachedSizes: sizes are computed once at +// append and drained exactly, so the byte counter returns to zero without any +// re-serialization of held chunks. +func TestGate_PausedBufferAccountingUsesCachedSizes(t *testing.T) { + a := newTestAccumulator(t) + traceID := "trace-cached-sizes" + ctx := schemas.NewBifrostContext(context.Background(), time.Time{}) + r := newRecorder(8) + defer r.close() + + chunk := &schemas.BifrostStreamChunk{ + BifrostPassthroughResponse: &schemas.BifrostPassthroughResponse{Body: make([]byte, 2048)}, + } + want := estimateChunkBytes(chunk) + + a.PauseStream(traceID) + if !a.GateSend(traceID, chunk, false, false, r.ch, ctx) { + t.Fatal("paused GateSend returned false") + } + sa := mustGet(t, a, traceID) + sa.mu.Lock() + gotBytes := sa.gateReplayBufBytes + gotCached := sa.gateReplayBuf[0].size + sa.mu.Unlock() + if gotBytes != want || gotCached != want { + t.Fatalf("buffered bytes = %d, cached size = %d, want both %d", gotBytes, gotCached, want) + } + + a.ResumeStream(traceID) + if !r.waitFor(1, time.Second) { + t.Fatal("chunk was not replayed after resume") + } + sa.mu.Lock() + after := sa.gateReplayBufBytes + sa.mu.Unlock() + if after != 0 { + t.Fatalf("buffered bytes after drain = %d, want 0", after) + } + a.EndStream(traceID, nil) + sa.WaitForFlusher() +} + +// benchChunks builds a realistic full-hold workload: passthrough chunks with +// SSE-sized bodies, as buffered on the native Anthropic (claude-cli) path. +func benchChunks(n, bodyBytes int) []*schemas.BifrostStreamChunk { + out := make([]*schemas.BifrostStreamChunk, n) + for i := range out { + out[i] = &schemas.BifrostStreamChunk{ + BifrostPassthroughResponse: &schemas.BifrostPassthroughResponse{Body: make([]byte, bodyBytes)}, + } + } + return out +} + +// BenchmarkChunkSizeEstimator measures the new zero-marshal size accounting. +func BenchmarkChunkSizeEstimator(b *testing.B) { + chunks := benchChunks(1, 1400) + b.ReportAllocs() + for i := 0; i < b.N; i++ { + _ = estimateChunkBytes(chunks[0]) + } +} + +// BenchmarkChunkSizeMarshal measures the pre-change size accounting +// (MarshalJSON per lookup), kept here as the comparison baseline. +func BenchmarkChunkSizeMarshal(b *testing.B) { + chunks := benchChunks(1, 1400) + b.ReportAllocs() + for i := 0; i < b.N; i++ { + data, err := chunks[0].MarshalJSON() + if err != nil { + b.Fatal(err) + } + _ = int64(len(data)) + } +} + +// BenchmarkGatePauseHoldDrain exercises the full production cycle: pause, +// buffer n chunks, resume, drain to a consumer. Allocation count here is what +// a held stream pays end to end under the gate. +func BenchmarkGatePauseHoldDrain(b *testing.B) { + const n, bodyBytes = 500, 1400 + ctx := schemas.NewBifrostContext(context.Background(), time.Time{}) + b.ReportAllocs() + for i := 0; i < b.N; i++ { + b.StopTimer() + a := NewAccumulator(nil, bifrost.NewDefaultLogger(schemas.LogLevelError)) + traceID := fmt.Sprintf("bench-%d", i) + chunks := benchChunks(n, bodyBytes) + ch := make(chan *schemas.BifrostStreamChunk, n+1) + done := make(chan struct{}) + go func() { + for range ch { + } + close(done) + }() + b.StartTimer() + + a.PauseStream(traceID) + for _, c := range chunks { + if !a.GateSend(traceID, c, false, false, ch, ctx) { + b.Fatal("GateSend returned false") + } + } + a.ResumeStream(traceID) + a.EndStream(traceID, nil) + a.WaitForFlusher(traceID) + + b.StopTimer() + close(ch) + <-done + a.Cleanup() + b.StartTimer() + } +} + +// ssePassthroughChunk wraps one SSE event as a passthrough stream chunk — the +// shape the native Anthropic path buffers under guardrails full-hold. +func ssePassthroughChunk(event, data string) *schemas.BifrostStreamChunk { + return &schemas.BifrostStreamChunk{ + BifrostPassthroughResponse: &schemas.BifrostPassthroughResponse{ + StatusCode: 200, + Body: []byte("event: " + event + "\ndata: " + data + "\n\n"), + }, + } +} + +// toolCarryingSSEStream builds a realistic tool-carrying Anthropic SSE stream +// whose input_json_delta fragments carry a secret a guardrail would redact. +func toolCarryingSSEStream() []*schemas.BifrostStreamChunk { + return []*schemas.BifrostStreamChunk{ + ssePassthroughChunk("message_start", `{"type":"message_start","message":{"id":"msg_01","role":"assistant"}}`), + ssePassthroughChunk("content_block_start", `{"type":"content_block_start","index":0,"content_block":{"type":"tool_use","id":"toolu_01","name":"bash","input":{}}}`), + ssePassthroughChunk("content_block_delta", `{"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"{\"command\":\"curl -H "}}`), + ssePassthroughChunk("content_block_delta", `{"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"'Authorization: Bearer sk-ant-SECRET123'"}}`), + ssePassthroughChunk("content_block_delta", `{"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":" https://api.example.com\"}"}}`), + ssePassthroughChunk("content_block_stop", `{"type":"content_block_stop","index":0}`), + ssePassthroughChunk("message_delta", `{"type":"message_delta","delta":{"stop_reason":"tool_use"}}`), + ssePassthroughChunk("message_stop", `{"type":"message_stop"}`), + } +} + +func passthroughBody(c *schemas.BifrostStreamChunk) []byte { + if c != nil && c.BifrostPassthroughResponse != nil { + return c.BifrostPassthroughResponse.Body + } + return nil +} + +// TestGate_FullHoldRedactReplaySSE walks the guardrails full-hold seam with a +// realistic claude-cli stream: pause before the first chunk, copy-on-write +// redaction via TransformPausedStreamBuffer, then replay. Nothing may escape +// while held, replay must preserve SSE order, the secret must be gone from the +// wire, and the originally buffered chunks must be untouched. +func TestGate_FullHoldRedactReplaySSE(t *testing.T) { + a := newTestAccumulator(t) + traceID := "trace-full-hold-redact" + ctx := schemas.NewBifrostContext(context.Background(), time.Time{}) + r := newRecorder(32) + defer r.close() + chunks := toolCarryingSSEStream() + + a.PauseStream(traceID) + for i, c := range chunks { + if !a.GateSend(traceID, c, i == len(chunks)-1, false, r.ch, ctx) { + t.Fatalf("paused GateSend returned false at %d", i) + } + } + if got, _ := r.snapshot(); len(got) != 0 { + t.Fatalf("%d chunks escaped while held", len(got)) + } + + err := a.TransformPausedStreamBuffer(traceID, func(buffered []*schemas.BifrostStreamChunk) (schemas.PausedStreamBufferTransformResult, error) { + out := append([]*schemas.BifrostStreamChunk(nil), buffered...) + for i, c := range buffered { + body := passthroughBody(c) + if !bytes.Contains(body, []byte("sk-ant-SECRET123")) { + continue + } + cp := *c + pt := *c.BifrostPassthroughResponse + pt.Body = bytes.ReplaceAll(body, []byte("sk-ant-SECRET123"), []byte("[REDACTED:api-key]")) + cp.BifrostPassthroughResponse = &pt + out[i] = &cp + } + return schemas.PausedStreamBufferTransformResult{Chunks: out, ReleaseCount: len(out)}, nil + }) + if err != nil { + t.Fatalf("TransformPausedStreamBuffer() error = %v", err) + } + + a.ResumeStream(traceID) + if !r.waitFor(len(chunks), time.Second) { + got, _ := r.snapshot() + t.Fatalf("replay delivered %d of %d chunks", len(got), len(chunks)) + } + sa := mustGet(t, a, traceID) + sa.WaitForFlusher() + + got, _ := r.snapshot() + var wire []byte + for i, c := range got { + wantEvent := bytes.SplitN(passthroughBody(chunks[i]), []byte("\n"), 2)[0] + if !bytes.HasPrefix(passthroughBody(c), wantEvent) { + t.Fatalf("replay order broken at %d", i) + } + wire = append(wire, passthroughBody(c)...) + } + if bytes.Contains(wire, []byte("sk-ant-SECRET123")) { + t.Fatal("secret leaked to the wire") + } + if !bytes.Contains(wire, []byte("[REDACTED:api-key]")) { + t.Fatal("redaction placeholder missing from the wire") + } + if !bytes.Contains(passthroughBody(chunks[3]), []byte("sk-ant-SECRET123")) { + t.Fatal("copy-on-write violated: original buffered chunk was mutated") + } + if !a.IsStreamEnded(traceID) { + t.Fatal("gate not ended after terminal replay") + } +} + +// TestGate_BlockVerdictClearsBufferAndDeliversTerminal mirrors a guardrail +// block: held content is dropped via ClearPausedStreamBuffer and the client +// receives only the synthetic terminal error; later sends are rejected. +func TestGate_BlockVerdictClearsBufferAndDeliversTerminal(t *testing.T) { + a := newTestAccumulator(t) + traceID := "trace-block-verdict" + ctx := schemas.NewBifrostContext(context.Background(), time.Time{}) + r := newRecorder(32) + defer r.close() + + a.PauseStream(traceID) + for _, c := range toolCarryingSSEStream() { + if !a.GateSend(traceID, c, false, false, r.ch, ctx) { + t.Fatal("paused GateSend returned false") + } + } + if err := a.ClearPausedStreamBuffer(traceID); err != nil { + t.Fatalf("ClearPausedStreamBuffer() error = %v", err) + } + a.EndStream(traceID, &schemas.BifrostError{ + IsBifrostError: true, + Error: &schemas.ErrorField{Message: "blocked by guardrail policy"}, + }) + sa := mustGet(t, a, traceID) + sa.WaitForFlusher() + if !r.waitFor(1, time.Second) { + t.Fatal("block terminal was not delivered") + } + + got, _ := r.snapshot() + if len(got) != 1 || got[0].BifrostError == nil || + !strings.Contains(got[0].BifrostError.Error.Message, "blocked by guardrail policy") { + t.Fatalf("want exactly the block terminal, got %d chunks", len(got)) + } + if a.GateSend(traceID, ssePassthroughChunk("ping", `{}`), false, false, r.ch, ctx) { + t.Fatal("send accepted after block terminal") + } +} + +// TestGate_OverflowCapWithSSEPayloads drives 5MB SSE events into a paused gate +// and asserts the cap trips right at the 100MB boundary, the client is told via +// paused_replay_buffer_overflow, and the gate is terminally closed. +func TestGate_OverflowCapWithSSEPayloads(t *testing.T) { + a := newTestAccumulator(t) + traceID := "trace-overflow-sse" + ctx := schemas.NewBifrostContext(context.Background(), time.Time{}) + r := newRecorder(64) + defer r.close() + + a.PauseStream(traceID) + big := strings.Repeat("x", 5*1024*1024) + refusedAt := -1 + for i := 0; i < 40; i++ { + if !a.GateSend(traceID, ssePassthroughChunk("content_block_delta", big), false, false, r.ch, ctx) { + refusedAt = i + break + } + } + // ~5MB payload + 512B floor per chunk: refusal expected at the 100MB line. + if refusedAt < 18 || refusedAt > 21 { + t.Fatalf("cap refused at chunk %d (~%dMB), want ~20 (100MB)", refusedAt, refusedAt*5) + } + sa := mustGet(t, a, traceID) + sa.WaitForFlusher() + // On overflow the flusher delivers only an approved prefix (none here) and + // then the synthetic terminal, asynchronously, so poll until it lands. + sawOverflow := false + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) && !sawOverflow { + got, _ := r.snapshot() + for _, c := range got { + if c.BifrostError != nil && c.BifrostError.Error != nil && c.BifrostError.Error.Type != nil && + *c.BifrostError.Error.Type == "paused_replay_buffer_overflow" { + sawOverflow = true + } + } + if !sawOverflow { + time.Sleep(2 * time.Millisecond) + } + } + if !sawOverflow { + t.Fatal("client never received paused_replay_buffer_overflow") + } + if !a.IsStreamEnded(traceID) { + t.Fatal("gate not ended after overflow") + } + if a.GateSend(traceID, ssePassthroughChunk("ping", `{}`), false, false, r.ch, ctx) { + t.Fatal("send accepted after overflow") + } +} + +// TestGate_TransformInPlaceMutationIsRepriced pins the accounting guarantee +// for a transform flow that modifies held chunks IN PLACE (through the original +// pointers) instead of copy-on-write: because sizes are recomputed from +// result.Chunks after the callback returns, in-place growth inside the +// callback is still re-priced before install and delivered mutated on replay. +func TestGate_TransformInPlaceMutationIsRepriced(t *testing.T) { + a := newTestAccumulator(t) + traceID := "trace-inplace-reprice" + ctx := schemas.NewBifrostContext(context.Background(), time.Time{}) + r := newRecorder(16) + defer r.close() + + a.PauseStream(traceID) + chunk := ssePassthroughChunk("content_block_delta", `{"delta":{"partial_json":"{\"cmd\":\"ls\"}"}}`) + if !a.GateSend(traceID, chunk, false, false, r.ch, ctx) { + t.Fatal("paused GateSend returned false") + } + sa := mustGet(t, a, traceID) + sa.mu.Lock() + before := sa.gateReplayBufBytes + sa.mu.Unlock() + + const grow = 64 * 1024 + err := a.TransformPausedStreamBuffer(traceID, func(buffered []*schemas.BifrostStreamChunk) (schemas.PausedStreamBufferTransformResult, error) { + pt := buffered[0].BifrostPassthroughResponse + pt.Body = append(pt.Body, bytes.Repeat([]byte("z"), grow)...) // in-place, same pointer + return schemas.PausedStreamBufferTransformResult{Chunks: buffered, ReleaseCount: len(buffered)}, nil + }) + if err != nil { + t.Fatalf("in-place transform rejected: %v", err) + } + + sa.mu.Lock() + after := sa.gateReplayBufBytes + entrySize := sa.gateReplayBuf[0].size + entryEstimate := estimateChunkBytes(sa.gateReplayBuf[0].chunk) + sa.mu.Unlock() + if after < before+grow { + t.Fatalf("in-place growth not re-priced: before=%d after=%d grow=%d", before, after, grow) + } + if entrySize != entryEstimate { + t.Fatalf("cached size %d diverges from fresh estimate %d after in-place transform", entrySize, entryEstimate) + } + + a.ResumeStream(traceID) + if !r.waitFor(1, time.Second) { + t.Fatal("mutated chunk was not replayed") + } + got, _ := r.snapshot() + if !bytes.Contains(passthroughBody(got[0]), bytes.Repeat([]byte("z"), 16)) { + t.Fatal("replayed chunk does not carry the in-place mutation") + } + a.EndStream(traceID, nil) + sa.WaitForFlusher() +} + +// TestGate_OverflowNeverReleasesUnapprovedChunks: chunks held while a stream is +// paused for approval were never approved, so when the replay cap forces the gate +// to end, the client gets the overflow error and none of the held content. +func TestGate_OverflowNeverReleasesUnapprovedChunks(t *testing.T) { + a := newTestAccumulator(t) + traceID := "trace-overflow-unapproved" + ctx := schemas.NewBifrostContext(context.Background(), time.Time{}) + r := newRecorder(64) + defer r.close() + + a.PauseStream(traceID) + big := strings.Repeat("x", 5*1024*1024) + for i := 0; i < 40; i++ { + if !a.GateSend(traceID, ssePassthroughChunk("content_block_delta", big), false, false, r.ch, ctx) { + break + } + } + sa := mustGet(t, a, traceID) + sa.WaitForFlusher() + deadline := time.Now().Add(2 * time.Second) + for { + got, _ := r.snapshot() + sawOverflow := false + for _, c := range got { + if c.BifrostError == nil { + t.Fatal("a chunk held for approval was released by the overflow") + } + if c.BifrostError.Error != nil && c.BifrostError.Error.Type != nil && *c.BifrostError.Error.Type == "paused_replay_buffer_overflow" { + sawOverflow = true + } + } + if sawOverflow { + return + } + if time.Now().After(deadline) { + t.Fatal("client never received paused_replay_buffer_overflow") + } + time.Sleep(2 * time.Millisecond) + } +} diff --git a/framework/streaming/responses.go b/framework/streaming/responses.go index b344b086fea..1104f7c1e9f 100644 --- a/framework/streaming/responses.go +++ b/framework/streaming/responses.go @@ -282,6 +282,11 @@ func deepCopyResponsesMessage(original schemas.ResponsesMessage) schemas.Respons copy.ResponsesToolMessage.Arguments = ©Arguments } + if original.ResponsesToolMessage.ToolsetName != nil { + copyToolsetName := *original.ResponsesToolMessage.ToolsetName + copy.ResponsesToolMessage.ToolsetName = ©ToolsetName + } + if original.ResponsesToolMessage.Namespace != nil { copyNamespace := *original.ResponsesToolMessage.Namespace copy.ResponsesToolMessage.Namespace = ©Namespace @@ -289,8 +294,28 @@ func deepCopyResponsesMessage(original schemas.ResponsesMessage) schemas.Respons copyOptionalStringFieldByName(copy.ResponsesToolMessage, original.ResponsesToolMessage, "Execution") + if original.ResponsesToolMessage.Async != nil { + copy.ResponsesToolMessage.Async = new(*original.ResponsesToolMessage.Async) + } + if original.ResponsesToolMessage.Error != nil { - copyError := *original.ResponsesToolMessage.Error + copyError := schemas.ResponsesToolMessageError{} + if original.ResponsesToolMessage.Error.ResponsesToolMessageErrorStr != nil { + copyError.ResponsesToolMessageErrorStr = new(*original.ResponsesToolMessage.Error.ResponsesToolMessageErrorStr) + } + if original.ResponsesToolMessage.Error.ResponsesToolMessageErrorStruct != nil { + copyErrorStruct := *original.ResponsesToolMessage.Error.ResponsesToolMessageErrorStruct + if copyErrorStruct.Code != nil { + copyCode := *copyErrorStruct.Code + copyErrorStruct.Code = ©Code + } + if copyErrorStruct.Message != nil { + copyMessage := *copyErrorStruct.Message + copyErrorStruct.Message = ©Message + } + copyErrorStruct.Content = append(json.RawMessage(nil), copyErrorStruct.Content...) + copyError.ResponsesToolMessageErrorStruct = ©ErrorStruct + } copy.ResponsesToolMessage.Error = ©Error } diff --git a/framework/streaming/responses_test.go b/framework/streaming/responses_test.go index bf191076ff9..c933828c1b3 100644 --- a/framework/streaming/responses_test.go +++ b/framework/streaming/responses_test.go @@ -19,6 +19,27 @@ func testResponsesAccumulator(tb testing.TB) *Accumulator { return acc } +func TestDeepCopyResponsesMessageCopiesStructuredErrorPointers(t *testing.T) { + original := schemas.ResponsesMessage{ResponsesToolMessage: &schemas.ResponsesToolMessage{Error: &schemas.ResponsesToolMessageError{ + ResponsesToolMessageErrorStruct: &schemas.ResponsesToolMessageErrorStruct{ + Type: "http_error", Code: schemas.Ptr(502), Message: schemas.Ptr("upstream failed"), + }, + }}} + + copied := deepCopyResponsesMessage(original) + originalError := original.ResponsesToolMessage.Error.ResponsesToolMessageErrorStruct + copiedError := copied.ResponsesToolMessage.Error.ResponsesToolMessageErrorStruct + require.NotNil(t, copiedError) + require.NotSame(t, originalError, copiedError) + require.NotSame(t, originalError.Code, copiedError.Code) + require.NotSame(t, originalError.Message, copiedError.Message) + + *copiedError.Code = 503 + *copiedError.Message = "changed" + require.Equal(t, 502, *originalError.Code) + require.Equal(t, "upstream failed", *originalError.Message) +} + func TestAccumulatedResponsesStreamPreservesServiceTierBeforeUsageOnlyChunk(t *testing.T) { acc := testResponsesAccumulator(t) requestID := "responses-service-tier" @@ -155,6 +176,25 @@ func TestDeepCopyResponsesStreamResponseCopiesToolCaller(t *testing.T) { } } +func TestDeepCopyResponsesStreamResponseCopiesAsync(t *testing.T) { + original := &schemas.BifrostResponsesStreamResponse{ + Type: schemas.ResponsesStreamResponseTypeOutputItemDone, + Item: &schemas.ResponsesMessage{ + ID: schemas.Ptr("fc_1"), + Type: schemas.Ptr(schemas.ResponsesMessageTypeFunctionCall), + ResponsesToolMessage: &schemas.ResponsesToolMessage{ + CallID: schemas.Ptr("call_1"), + Async: new(true), + }, + }, + } + + copied := deepCopyResponsesStreamResponse(original) + require.NotNil(t, copied.Item.ResponsesToolMessage.Async) + require.True(t, *copied.Item.ResponsesToolMessage.Async) + require.NotSame(t, original.Item.ResponsesToolMessage.Async, copied.Item.ResponsesToolMessage.Async) +} + // TestBuildResponsesMessageAccumulatesReasoningSummary verifies reasoning // summary deltas (no content index) concatenate into a single summary entry. func TestBuildResponsesMessageAccumulatesReasoningSummary(t *testing.T) { @@ -588,3 +628,23 @@ func TestProcessStreamingResponseKeepsImageGenerationCall(t *testing.T) { require.Equal(t, "generate", *out.Action.ResponsesToolCallActionStr) require.Equal(t, "1024x1024", *out.ResponsesImageGenerationCall.Size) } + +// A computer-toolset member call carries its toolset name on output_item.added; the +// accumulator's copies must keep it, or a replayed snapshot loses the toolset identity. +func TestDeepCopyResponsesMessageCopiesToolsetName(t *testing.T) { + original := schemas.ResponsesMessage{ + Type: schemas.Ptr(schemas.ResponsesMessageTypeFunctionCall), + ResponsesToolMessage: &schemas.ResponsesToolMessage{ + CallID: schemas.Ptr("call_1"), + Name: schemas.Ptr("screenshot"), + ToolsetName: schemas.Ptr("computer"), + }, + } + copied := deepCopyResponsesMessage(original) + if copied.ResponsesToolMessage == nil || copied.ResponsesToolMessage.ToolsetName == nil || *copied.ResponsesToolMessage.ToolsetName != "computer" { + t.Fatalf("toolset name not copied: %#v", copied.ResponsesToolMessage) + } + if copied.ResponsesToolMessage.ToolsetName == original.ResponsesToolMessage.ToolsetName { + t.Fatal("toolset name pointer was aliased") + } +} diff --git a/framework/streaming/types.go b/framework/streaming/types.go index 0df6f8ccf15..52ea7e8b1a9 100644 --- a/framework/streaming/types.go +++ b/framework/streaming/types.go @@ -180,8 +180,8 @@ type StreamAccumulator struct { // buffer and transitions the gate to Ended. gatePendingTerminal bool gateSeq int // monotonic, bumped on every GateSend - gateReplayBuf []*schemas.BifrostStreamChunk // wire-format chunks captured while paused - gateReplayBufBytes int64 // sum of MarshalJSON sizes of chunks in gateReplayBuf; capped by gateReplayBufMaxBytes + gateReplayBuf []gateReplayEntry // wire-format chunks captured while paused, each with its MarshalJSON size cached at append + gateReplayBufBytes int64 // sum of cached sizes in gateReplayBuf; capped by gateReplayBufMaxBytes gateReplayEventInterval time.Duration // delay between buffered events after paced resume is armed gateCond *sync.Cond // wakes flusher on Resume / End / append-while-active gateEndError *schemas.BifrostError // delivered as terminal chunk if EndStream(err) was called with non-nil diff --git a/framework/version b/framework/version index 943f9cbc4ec..10c088013f8 100644 --- a/framework/version +++ b/framework/version @@ -1 +1 @@ -1.7.1 +1.7.4 diff --git a/go.work b/go.work index 249467d3b67..cff2909e00c 100644 --- a/go.work +++ b/go.work @@ -33,6 +33,7 @@ replace ( github.com/capsohq/bifrost/core v1.8.6 => ./core github.com/capsohq/bifrost/core v1.9.0 => ./core github.com/capsohq/bifrost/core v1.9.1 => ./core + github.com/capsohq/bifrost/core v1.10.2 => ./core ) replace ( @@ -45,6 +46,7 @@ replace ( github.com/capsohq/bifrost/framework v1.6.2 => ./framework github.com/capsohq/bifrost/framework v1.7.0 => ./framework github.com/capsohq/bifrost/framework v1.7.1 => ./framework + github.com/capsohq/bifrost/framework v1.7.4 => ./framework ) replace ( @@ -56,6 +58,7 @@ replace ( github.com/capsohq/bifrost/plugins/compat v0.2.2 => ./plugins/compat github.com/capsohq/bifrost/plugins/compat v0.3.0 => ./plugins/compat github.com/capsohq/bifrost/plugins/compat v0.3.1 => ./plugins/compat + github.com/capsohq/bifrost/plugins/compat v0.3.3 => ./plugins/compat ) replace ( @@ -67,6 +70,7 @@ replace ( github.com/capsohq/bifrost/plugins/governance v1.7.2 => ./plugins/governance github.com/capsohq/bifrost/plugins/governance v1.8.0 => ./plugins/governance github.com/capsohq/bifrost/plugins/governance v1.8.1 => ./plugins/governance + github.com/capsohq/bifrost/plugins/governance v1.8.3 => ./plugins/governance ) replace ( @@ -78,6 +82,7 @@ replace ( github.com/capsohq/bifrost/plugins/logging v1.7.2 => ./plugins/logging github.com/capsohq/bifrost/plugins/logging v1.8.0 => ./plugins/logging github.com/capsohq/bifrost/plugins/logging v1.8.1 => ./plugins/logging + github.com/capsohq/bifrost/plugins/logging v1.8.3 => ./plugins/logging ) replace ( @@ -89,6 +94,7 @@ replace ( github.com/capsohq/bifrost/plugins/maxim v1.7.2 => ./plugins/maxim github.com/capsohq/bifrost/plugins/maxim v1.7.3 => ./plugins/maxim github.com/capsohq/bifrost/plugins/maxim v1.7.4 => ./plugins/maxim + github.com/capsohq/bifrost/plugins/maxim v1.7.6 => ./plugins/maxim ) replace ( @@ -101,6 +107,7 @@ replace ( github.com/capsohq/bifrost/plugins/mocker v1.6.2 => ./plugins/mocker github.com/capsohq/bifrost/plugins/mocker v1.6.3 => ./plugins/mocker github.com/capsohq/bifrost/plugins/mocker v1.6.4 => ./plugins/mocker + github.com/capsohq/bifrost/plugins/mocker v1.6.6 => ./plugins/mocker ) replace ( @@ -112,6 +119,7 @@ replace ( github.com/capsohq/bifrost/plugins/modelcatalogresolver v1.1.2 => ./plugins/modelcatalogresolver github.com/capsohq/bifrost/plugins/modelcatalogresolver v1.1.3 => ./plugins/modelcatalogresolver github.com/capsohq/bifrost/plugins/modelcatalogresolver v1.1.4 => ./plugins/modelcatalogresolver + github.com/capsohq/bifrost/plugins/modelcatalogresolver v1.1.6 => ./plugins/modelcatalogresolver ) replace ( @@ -123,6 +131,7 @@ replace ( github.com/capsohq/bifrost/plugins/otel v1.5.2 => ./plugins/otel github.com/capsohq/bifrost/plugins/otel v1.5.3 => ./plugins/otel github.com/capsohq/bifrost/plugins/otel v1.5.4 => ./plugins/otel + github.com/capsohq/bifrost/plugins/otel v1.5.6 => ./plugins/otel ) replace ( @@ -134,6 +143,7 @@ replace ( github.com/capsohq/bifrost/plugins/prompts v1.1.2 => ./plugins/prompts github.com/capsohq/bifrost/plugins/prompts v1.1.3 => ./plugins/prompts github.com/capsohq/bifrost/plugins/prompts v1.1.4 => ./plugins/prompts + github.com/capsohq/bifrost/plugins/prompts v1.1.6 => ./plugins/prompts ) replace ( @@ -145,6 +155,7 @@ replace ( github.com/capsohq/bifrost/plugins/semanticcache v1.6.2 => ./plugins/semanticcache github.com/capsohq/bifrost/plugins/semanticcache v1.6.3 => ./plugins/semanticcache github.com/capsohq/bifrost/plugins/semanticcache v1.6.4 => ./plugins/semanticcache + github.com/capsohq/bifrost/plugins/semanticcache v1.6.6 => ./plugins/semanticcache ) replace ( @@ -156,8 +167,10 @@ replace ( github.com/capsohq/bifrost/plugins/telemetry v1.6.2 => ./plugins/telemetry github.com/capsohq/bifrost/plugins/telemetry v1.7.0 => ./plugins/telemetry github.com/capsohq/bifrost/plugins/telemetry v1.8.0 => ./plugins/telemetry + github.com/capsohq/bifrost/plugins/telemetry v1.8.2 => ./plugins/telemetry ) replace github.com/capsohq/bifrost/plugins/routing v1.0.2 => ./plugins/routing replace github.com/capsohq/bifrost/plugins/routing v1.1.0 => ./plugins/routing replace github.com/capsohq/bifrost/plugins/routing v1.1.1 => ./plugins/routing +replace github.com/capsohq/bifrost/plugins/routing v1.1.3 => ./plugins/routing diff --git a/plugins/compat/dropparams.go b/plugins/compat/dropparams.go index 58d70b413e4..c40e073c965 100644 --- a/plugins/compat/dropparams.go +++ b/plugins/compat/dropparams.go @@ -137,16 +137,6 @@ func dropUnsupportedParams(ctx *schemas.BifrostContext, req *schemas.BifrostRequ } } - if req.ChatRequest != nil && req.ChatRequest.Input != nil { - if req.ChatRequest.Provider != schemas.Bedrock || !isSupported["cachePoint"] { - droppedKeys := dropCachePoint(req.ChatRequest) - if len(droppedKeys) > 0 { - ctx.Log(schemas.LogLevelWarn, fmt.Sprintf("dropped %d cache_point block(s) - cachePoint is only supported on Bedrock models that list it: %s", len(droppedKeys), strings.Join(droppedKeys, ", "))) - } - dropped = append(dropped, droppedKeys...) - } - } - if req.ResponsesRequest != nil && req.ResponsesRequest.Params != nil { params := req.ResponsesRequest.Params @@ -227,28 +217,6 @@ func dropUnsupportedParams(ctx *schemas.BifrostContext, req *schemas.BifrostRequ } } - if req.ResponsesRequest != nil && req.ResponsesRequest.Input != nil { - if req.ResponsesRequest.Provider == schemas.Bedrock && !schemas.IsAnthropicModel(req.ResponsesRequest.Model) { - droppedKeys := applyBedrockResponsesCompatibility(req.ResponsesRequest) - if len(droppedKeys) > 0 { - ctx.Log(schemas.LogLevelWarn, fmt.Sprintf("applied Bedrock compatibility for a non-Anthropic model - removed %d empty text block(s)/reasoning signature(s): %s", len(droppedKeys), strings.Join(droppedKeys, ", "))) - } - dropped = append(dropped, droppedKeys...) - } - } - - if req.ResponsesRequest != nil { - // all anthropic models support cache_control - // for bedrock models cache_control is converted to cachePoint - if req.ResponsesRequest.Provider == schemas.Bedrock && !isSupported["cache_control"] { - droppedKeys := dropCacheControlFromResponsesMessages(req.ResponsesRequest) - if len(droppedKeys) > 0 { - ctx.Log(schemas.LogLevelWarn, fmt.Sprintf("dropped %d cache_control field(s) - the model does not support cache_control: %s", len(droppedKeys), strings.Join(droppedKeys, ", "))) - } - dropped = append(dropped, droppedKeys...) - } - } - if req.TextCompletionRequest != nil && req.TextCompletionRequest.Params != nil { params := req.TextCompletionRequest.Params @@ -317,84 +285,3 @@ func dropWebsearchToolCalls(req *schemas.BifrostRequest) []string { req.ResponsesRequest.Params.Tools = kept return dropped } - -// dropCachePoint drops cache point (only supported by bedrock) from the request -func dropCachePoint(req *schemas.BifrostChatRequest) []string { - dropped := []string{} - for i := range req.Input { - if req.Input[i].Content != nil && req.Input[i].Content.ContentBlocks != nil { - blocks := req.Input[i].Content.ContentBlocks - kept := blocks[:0] - for j, block := range blocks { - if block.CachePoint != nil { - dropped = append(dropped, fmt.Sprintf("input[%d].content.content_blocks[%d].cache_point", i, j)) - } else { - kept = append(kept, block) - } - } - req.Input[i].Content.ContentBlocks = kept - } - } - return dropped -} - -// dropCacheControlFromResponsesMessages clears cache_control from all content blocks. -func dropCacheControlFromResponsesMessages(req *schemas.BifrostResponsesRequest) []string { - var dropped []string - - if req.Input != nil { - for i := range req.Input { - msg := &req.Input[i] - if msg.CacheControl != nil { - msg.CacheControl = nil - dropped = append(dropped, fmt.Sprintf("input[%d].cache_control", i)) - } - if msg.Content == nil || msg.Content.ContentBlocks == nil { - continue - } - for j := range msg.Content.ContentBlocks { - if msg.Content.ContentBlocks[j].CacheControl != nil { - msg.Content.ContentBlocks[j].CacheControl = nil - dropped = append(dropped, fmt.Sprintf("input[%d].content.content_blocks[%d].cache_control", i, j)) - } - } - } - } - - if req.Params != nil { - for i := range req.Params.Tools { - if req.Params.Tools[i].CacheControl != nil { - req.Params.Tools[i].CacheControl = nil - dropped = append(dropped, fmt.Sprintf("tools[%d].cache_control", i)) - } - } - } - return dropped -} - -// applyBedrockResponsesCompatibility sanitizes messages for OpenAI-compatible Bedrock models: -// - drops empty text content blocks -// - strips reasoning signatures (Anthropic-specific, not supported by OpenAI models) -func applyBedrockResponsesCompatibility(req *schemas.BifrostResponsesRequest) []string { - var dropped []string - for i := range req.Input { - msg := &req.Input[i] - if msg.Content == nil || msg.Content.ContentBlocks == nil { - continue - } - kept := msg.Content.ContentBlocks[:0] - for j, block := range msg.Content.ContentBlocks { - if block.Text != nil && *block.Text == "" { - dropped = append(dropped, fmt.Sprintf("input[%d].content.content_blocks[%d]", i, j)) - continue - } - if block.Signature != nil { - msg.Content.ContentBlocks[j].Signature = nil - dropped = append(dropped, fmt.Sprintf("input[%d].content.content_blocks[%d].signature", i, j)) - } - kept = append(kept, msg.Content.ContentBlocks[j]) - } - msg.Content.ContentBlocks = kept - } - return dropped -} diff --git a/plugins/compat/go.mod b/plugins/compat/go.mod index f1f8df68f67..bc43ef3e495 100644 --- a/plugins/compat/go.mod +++ b/plugins/compat/go.mod @@ -4,8 +4,8 @@ go 1.27.0 require ( github.com/bytedance/sonic v1.15.3-0.20260730064818-2a36d6da63e2 - github.com/capsohq/bifrost/core v1.9.1 - github.com/capsohq/bifrost/framework v1.7.1 + github.com/capsohq/bifrost/core v1.10.2 + github.com/capsohq/bifrost/framework v1.7.4 ) require ( diff --git a/plugins/compat/go.sum b/plugins/compat/go.sum index eac4db919c9..233a6b06f1c 100644 --- a/plugins/compat/go.sum +++ b/plugins/compat/go.sum @@ -262,10 +262,10 @@ github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsRe github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= github.com/mattn/go-sqlite3 v1.14.32 h1:JD12Ag3oLy1zQA+BNn74xRgaBbdhbNIDYvQUEuuErjs= github.com/mattn/go-sqlite3 v1.14.32/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= -github.com/maximhq/bifrost/core v1.9.1 h1:CF7TTj63htKQbuBL7eAGzou5yoe4XiQ0YMWpAO/atnw= -github.com/maximhq/bifrost/core v1.9.1/go.mod h1:bgTt5CsqB7duKiy/pC0uUMTxIRnvjj0t2OvH7Tlvuz0= -github.com/maximhq/bifrost/framework v1.7.1 h1:pGCdT7ksXmcnCylBpQqSHlQ071QAqgJ1XS7JwI8uV9U= -github.com/maximhq/bifrost/framework v1.7.1/go.mod h1:Acd4YNI+K8SjMPdghUQbjw0plJpqIDOP0s01iQr51Go= +github.com/maximhq/bifrost/core v1.10.2 h1:jRvvZN1KJVaX2Pxn9crTrmL3kW2QgDh7d1jsuz6b72k= +github.com/maximhq/bifrost/core v1.10.2/go.mod h1:bgTt5CsqB7duKiy/pC0uUMTxIRnvjj0t2OvH7Tlvuz0= +github.com/maximhq/bifrost/framework v1.7.4 h1:aM4MM3H3dXlPpkvyqRvt337WV8jl3m7DfBehAzs8HNU= +github.com/maximhq/bifrost/framework v1.7.4/go.mod h1:yHmcJhKvvaeoNUJYd5yV/wsNb3jZBwnjw2ny246C55Y= github.com/molecule-man/go-brrr v1.0.1 h1:cEjgx8hgNw6UGdhQ94SPDbPkKuRbkUcxBO3IzbGpA/o= github.com/molecule-man/go-brrr v1.0.1/go.mod h1:7ybW6/7gA3oKY45jOfVNjSJDtrr6ea4tzbsTkjmQDC4= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= diff --git a/plugins/compat/requestcopy.go b/plugins/compat/requestcopy.go index 2bc7758c0f5..8f2e66284a0 100644 --- a/plugins/compat/requestcopy.go +++ b/plugins/compat/requestcopy.go @@ -1,13 +1,13 @@ package compat import ( - "encoding/json" - "maps" "slices" "github.com/capsohq/bifrost/core/schemas" ) +// cloneBifrostReq only deep clones the fields that are going to be changed, rest +// remains shallow copied to reduce allocations func cloneBifrostReq(req *schemas.BifrostRequest) *schemas.BifrostRequest { if req == nil { return nil @@ -19,7 +19,8 @@ func cloneBifrostReq(req *schemas.BifrostRequest) *schemas.BifrostRequest { textReq := *req.TextCompletionRequest cloned.TextCompletionRequest = &textReq if req.TextCompletionRequest.Params != nil { - cloned.TextCompletionRequest.Params = cloneTextCompletionParameters(req.TextCompletionRequest.Params) + params := *req.TextCompletionRequest.Params + cloned.TextCompletionRequest.Params = ¶ms } } if req.ChatRequest != nil { @@ -40,368 +41,39 @@ func cloneBifrostReq(req *schemas.BifrostRequest) *schemas.BifrostRequest { return &cloned } -func cloneTextCompletionParameters(params *schemas.TextCompletionParameters) *schemas.TextCompletionParameters { - if params == nil { - return nil - } - cloned := *params - if params.LogitBias != nil { - logitBias := cloneStringFloat64Map(*params.LogitBias) - cloned.LogitBias = &logitBias - } - if params.Stop != nil { - cloned.Stop = slices.Clone(params.Stop) - } - if params.StreamOptions != nil { - streamOptions := *params.StreamOptions - cloned.StreamOptions = &streamOptions - } - if params.ExtraParams != nil { - cloned.ExtraParams = cloneAnyMap(params.ExtraParams) - } - return &cloned -} - func cloneChatParameters(params *schemas.ChatParameters) *schemas.ChatParameters { - if params == nil { - return nil - } - cloned := *params - if params.Audio != nil { - audio := *params.Audio - cloned.Audio = &audio - } - if params.LogitBias != nil { - logitBias := cloneStringFloat64Map(*params.LogitBias) - cloned.LogitBias = &logitBias - } - if params.Metadata != nil { - metadata := cloneAnyMap(*params.Metadata) - cloned.Metadata = &metadata - } - if params.Modalities != nil { - cloned.Modalities = slices.Clone(params.Modalities) - } - if params.Prediction != nil { - prediction := *params.Prediction - prediction.Content = cloneAnyValue(params.Prediction.Content) - cloned.Prediction = &prediction - } if params.Reasoning != nil { reasoning := *params.Reasoning cloned.Reasoning = &reasoning } - if params.ResponseFormat != nil { - responseFormat := cloneAnyValue(*params.ResponseFormat) - cloned.ResponseFormat = &responseFormat - } - if params.StreamOptions != nil { - streamOptions := *params.StreamOptions - cloned.StreamOptions = &streamOptions - } - if params.Stop != nil { - cloned.Stop = slices.Clone(params.Stop) - } if params.ToolChoice != nil { - cloned.ToolChoice = cloneChatToolChoice(params.ToolChoice) + toolChoice := *params.ToolChoice + cloned.ToolChoice = &toolChoice } if params.Tools != nil { - cloned.Tools = make([]schemas.ChatTool, len(params.Tools)) - for i, tool := range params.Tools { - cloned.Tools[i] = schemas.DeepCopyChatTool(tool) - } - } - if params.WebSearchOptions != nil { - cloned.WebSearchOptions = cloneChatWebSearchOptions(params.WebSearchOptions) - } - if params.ExtraParams != nil { - cloned.ExtraParams = cloneAnyMap(params.ExtraParams) - } - return &cloned -} - -func cloneChatToolChoice(choice *schemas.ChatToolChoice) *schemas.ChatToolChoice { - if choice == nil { - return nil - } - - cloned := &schemas.ChatToolChoice{} - if choice.ChatToolChoiceStr != nil { - value := *choice.ChatToolChoiceStr - cloned.ChatToolChoiceStr = &value - } - if choice.ChatToolChoiceStruct != nil { - choiceStruct := *choice.ChatToolChoiceStruct - if choice.ChatToolChoiceStruct.Function != nil { - function := *choice.ChatToolChoiceStruct.Function - choiceStruct.Function = &function - } - if choice.ChatToolChoiceStruct.Custom != nil { - custom := *choice.ChatToolChoiceStruct.Custom - choiceStruct.Custom = &custom - } - if choice.ChatToolChoiceStruct.AllowedTools != nil { - allowedTools := *choice.ChatToolChoiceStruct.AllowedTools - allowedTools.Tools = slices.Clone(choice.ChatToolChoiceStruct.AllowedTools.Tools) - choiceStruct.AllowedTools = &allowedTools - } - cloned.ChatToolChoiceStruct = &choiceStruct - } - return cloned -} - -func cloneChatWebSearchOptions(options *schemas.ChatWebSearchOptions) *schemas.ChatWebSearchOptions { - if options == nil { - return nil - } - - cloned := *options - if options.UserLocation != nil { - userLocation := *options.UserLocation - if options.UserLocation.Approximate != nil { - approximate := *options.UserLocation.Approximate - userLocation.Approximate = &approximate - } - cloned.UserLocation = &userLocation + cloned.Tools = slices.Clone(params.Tools) } return &cloned } func cloneResponsesParameters(params *schemas.ResponsesParameters) *schemas.ResponsesParameters { - if params == nil { - return nil - } - cloned := *params - if params.Include != nil { - cloned.Include = slices.Clone(params.Include) - } - if params.Metadata != nil { - metadata := cloneAnyMap(*params.Metadata) - cloned.Metadata = &metadata - } if params.Reasoning != nil { reasoning := *params.Reasoning cloned.Reasoning = &reasoning } - if params.StreamOptions != nil { - streamOptions := *params.StreamOptions - cloned.StreamOptions = &streamOptions - } - if params.Text != nil { - cloned.Text = cloneResponsesTextConfig(params.Text) - } if params.ToolChoice != nil { - cloned.ToolChoice = cloneResponsesToolChoice(params.ToolChoice) - } - if params.Tools != nil { - cloned.Tools = make([]schemas.ResponsesTool, len(params.Tools)) - for i, tool := range params.Tools { - cloned.Tools[i] = cloneResponsesTool(tool) + toolChoice := *params.ToolChoice + if params.ToolChoice.ResponsesToolChoiceStruct != nil { + choiceStruct := *params.ToolChoice.ResponsesToolChoiceStruct + choiceStruct.Tools = slices.Clone(choiceStruct.Tools) + toolChoice.ResponsesToolChoiceStruct = &choiceStruct } + cloned.ToolChoice = &toolChoice } - if params.ExtraParams != nil { - cloned.ExtraParams = cloneAnyMap(params.ExtraParams) - } - return &cloned -} - -func cloneResponsesTextConfig(text *schemas.ResponsesTextConfig) *schemas.ResponsesTextConfig { - if text == nil { - return nil - } - - cloned := *text - if text.Format != nil { - format := *text.Format - if text.Format.JSONSchema != nil { - jsonSchema := *text.Format.JSONSchema - if text.Format.JSONSchema.Schema != nil { - schemaCopy := *text.Format.JSONSchema.Schema - if schemaCopy.SchemaBool != nil { - schemaCopy.SchemaBool = schemas.Ptr(*schemaCopy.SchemaBool) - } - schemaCopy.SchemaMap = cloneOrderedMap(schemaCopy.SchemaMap) - jsonSchema.Schema = &schemaCopy - } - if text.Format.JSONSchema.Properties != nil { - jsonSchema.Properties = cloneOrderedMap(text.Format.JSONSchema.Properties) - } - if text.Format.JSONSchema.Required != nil { - jsonSchema.Required = slices.Clone(text.Format.JSONSchema.Required) - } - if text.Format.JSONSchema.Defs != nil { - jsonSchema.Defs = cloneOrderedMap(text.Format.JSONSchema.Defs) - } - if text.Format.JSONSchema.Definitions != nil { - jsonSchema.Definitions = cloneOrderedMap(text.Format.JSONSchema.Definitions) - } - if text.Format.JSONSchema.Items != nil { - jsonSchema.Items = cloneOrderedMap(text.Format.JSONSchema.Items) - } - if text.Format.JSONSchema.AnyOf != nil { - jsonSchema.AnyOf = cloneOrderedMapSlice(text.Format.JSONSchema.AnyOf) - } - if text.Format.JSONSchema.OneOf != nil { - jsonSchema.OneOf = cloneOrderedMapSlice(text.Format.JSONSchema.OneOf) - } - if text.Format.JSONSchema.AllOf != nil { - jsonSchema.AllOf = cloneOrderedMapSlice(text.Format.JSONSchema.AllOf) - } - if text.Format.JSONSchema.Default != nil { - jsonSchema.Default = cloneAnyValue(text.Format.JSONSchema.Default) - } - if text.Format.JSONSchema.Enum != nil { - jsonSchema.Enum = slices.Clone(text.Format.JSONSchema.Enum) - } - if text.Format.JSONSchema.PropertyOrdering != nil { - jsonSchema.PropertyOrdering = slices.Clone(text.Format.JSONSchema.PropertyOrdering) - } - format.JSONSchema = &jsonSchema - } - cloned.Format = &format + if params.Tools != nil { + cloned.Tools = slices.Clone(params.Tools) } return &cloned } - -func cloneResponsesToolChoice(choice *schemas.ResponsesToolChoice) *schemas.ResponsesToolChoice { - if choice == nil { - return nil - } - - cloned := &schemas.ResponsesToolChoice{} - if choice.ResponsesToolChoiceStr != nil { - value := *choice.ResponsesToolChoiceStr - cloned.ResponsesToolChoiceStr = &value - } - if choice.ResponsesToolChoiceStruct != nil { - choiceStruct := *choice.ResponsesToolChoiceStruct - if choice.ResponsesToolChoiceStruct.Tools != nil { - choiceStruct.Tools = slices.Clone(choice.ResponsesToolChoiceStruct.Tools) - } - cloned.ResponsesToolChoiceStruct = &choiceStruct - } - return cloned -} - -func cloneResponsesTool(tool schemas.ResponsesTool) schemas.ResponsesTool { - data, err := schemas.MarshalSorted(tool) - if err != nil { - return tool - } - - var cloned schemas.ResponsesTool - if err := schemas.Unmarshal(data, &cloned); err != nil { - return tool - } - - return cloned -} - -func cloneStringFloat64Map(input map[string]float64) map[string]float64 { - if input == nil { - return nil - } - - cloned := make(map[string]float64, len(input)) - maps.Copy(cloned, input) - return cloned -} - -func cloneAnyMap(input map[string]any) map[string]any { - if input == nil { - return nil - } - - cloned := make(map[string]any, len(input)) - for key, value := range input { - cloned[key] = cloneAnyValue(value) - } - return cloned -} - -func cloneAnyMapSlice(input []map[string]any) []map[string]any { - if input == nil { - return nil - } - - cloned := make([]map[string]any, len(input)) - for i, value := range input { - cloned[i] = cloneAnyMap(value) - } - return cloned -} - -func cloneAnySlice(input []any) []any { - if input == nil { - return nil - } - - cloned := make([]any, len(input)) - for i, value := range input { - cloned[i] = cloneAnyValue(value) - } - return cloned -} - -func cloneAnyValue(value any) any { - switch typed := value.(type) { - case nil: - return nil - case map[string]any: - return cloneAnyMap(typed) - case []any: - return cloneAnySlice(typed) - case []string: - return slices.Clone(typed) - case json.RawMessage: - // Raw JSON (e.g. response_format off the wire) is a byte slice: copy it - // so the clone does not share a backing array with the original request. - return json.RawMessage(slices.Clone(typed)) - case map[string]string: - cloned := make(map[string]string, len(typed)) - maps.Copy(cloned, typed) - return cloned - case *schemas.OrderedMap: - return cloneOrderedMap(typed) - case schemas.OrderedMap: - if cloned := cloneOrderedMap(&typed); cloned != nil { - return *cloned - } - return typed - case []schemas.OrderedMap: - return cloneOrderedMapSlice(typed) - default: - return typed - } -} - -// cloneOrderedMap deep-copies an OrderedMap, preserving key order and -// recursively cloning nested values (including nested *OrderedMap). -func cloneOrderedMap(input *schemas.OrderedMap) *schemas.OrderedMap { - if input == nil { - return nil - } - - cloned := schemas.NewOrderedMapWithCapacity(input.Len()) - input.Range(func(key string, value any) bool { - cloned.Set(key, cloneAnyValue(value)) - return true - }) - return cloned -} - -func cloneOrderedMapSlice(input []schemas.OrderedMap) []schemas.OrderedMap { - if input == nil { - return nil - } - - cloned := make([]schemas.OrderedMap, len(input)) - for i := range input { - if c := cloneOrderedMap(&input[i]); c != nil { - cloned[i] = *c - } - } - return cloned -} diff --git a/plugins/compat/requestcopy_test.go b/plugins/compat/requestcopy_test.go new file mode 100644 index 00000000000..ad2659b89df --- /dev/null +++ b/plugins/compat/requestcopy_test.go @@ -0,0 +1,136 @@ +package compat + +import ( + "testing" + + bifrost "github.com/capsohq/bifrost/core" + "github.com/capsohq/bifrost/core/schemas" + "github.com/capsohq/bifrost/framework/modelcatalog" + "github.com/capsohq/bifrost/framework/modelcatalog/datasheet" +) + +// TestCloneBifrostReq_OriginalUntouched guards the copy-on-write clone: core +// rebuilds fallback attempts from the caller's request, so every write the +// plugin makes must land on the clone. The clone shares tool schemas and +// messages with the original, so a new in-place write into those shows up here. +func TestCloneBifrostReq_OriginalUntouched(t *testing.T) { + p := newTestPlugin(t, map[string][]string{"model-router": {"tools"}}) + + tools := []schemas.ResponsesTool{ + {Type: schemas.ResponsesToolTypeFunction, Name: schemas.Ptr("lookup"), CacheControl: &schemas.CacheControl{Type: "ephemeral"}}, + {Type: schemas.ResponsesToolTypeWebSearch}, + } + req := newResponsesRequest(schemas.Azure, "model-router", &schemas.ResponsesParameters{ + Temperature: schemas.Ptr(0.3), + TopP: schemas.Ptr(0.9), + MaxOutputTokens: schemas.Ptr(100), + Metadata: &map[string]any{"k": "v"}, + Reasoning: &schemas.ResponsesParametersReasoning{Effort: schemas.Ptr("high"), Summary: schemas.Ptr("detailed")}, + ToolChoice: &schemas.ResponsesToolChoice{ResponsesToolChoiceStr: schemas.Ptr("auto")}, + Tools: tools, + }) + req.ResponsesRequest.Input = []schemas.ResponsesMessage{{Role: schemas.Ptr(schemas.ResponsesInputMessageRoleUser)}} + + before, err := schemas.MarshalSorted(req) + if err != nil { + t.Fatalf("marshal: %v", err) + } + modified, _, err := p.PreLLMHook(newTestContext(), req) + if err != nil { + t.Fatalf("PreLLMHook: %v", err) + } + after, err := schemas.MarshalSorted(req) + if err != nil { + t.Fatalf("marshal: %v", err) + } + if string(before) != string(after) { + t.Fatalf("original request mutated by PreLLMHook\nbefore: %s\nafter: %s", before, after) + } + got, err := schemas.MarshalSorted(modified) + if err != nil { + t.Fatalf("marshal: %v", err) + } + if string(got) == string(before) { + t.Fatalf("PreLLMHook dropped nothing, so the guard proved nothing: %s", got) + } + if n := len(modified.ResponsesRequest.Params.Tools); n != 1 { + t.Errorf("clone tools = %d, want web_search dropped", n) + } + if len(tools) != 2 || tools[0].CacheControl == nil { + t.Errorf("original tools slice mutated: %+v", tools) + } +} + +// TestCloneBifrostReq_FallbackSeesOriginalTools mirrors core's fallback path: +// prepareFallbackRequest shallow-copies the caller's request, so a fallback +// attempt runs PreLLMHook against the same Params the primary attempt saw. The +// tool rewrites made for the primary provider must not reach the fallback. +func TestCloneBifrostReq_FallbackSeesOriginalTools(t *testing.T) { + ds := datasheet.NewTestStore(nil) + ds.SetSupportedParamsForTest(map[string][]string{ + "primary-model": {"tools"}, + "fallback-model": {"tools", "web_search", "cache_control", "tool_choice"}, + }) + p, err := Init(Config{ShouldDropParams: true, ShouldConvertParams: true}, bifrost.NewNoOpLogger(), modelcatalog.NewTestCatalogWithDatasheet(ds)) + if err != nil { + t.Fatalf("Init: %v", err) + } + + original := newResponsesRequest(schemas.Bedrock, "primary-model", &schemas.ResponsesParameters{ + ToolChoice: &schemas.ResponsesToolChoice{ResponsesToolChoiceStr: schemas.Ptr("required")}, + Tools: []schemas.ResponsesTool{ + {Type: schemas.ResponsesToolTypeFunction, Name: schemas.Ptr("lookup"), CacheControl: &schemas.CacheControl{Type: "ephemeral"}}, + {Type: schemas.ResponsesToolTypeWebSearch}, + {Type: schemas.ResponsesToolTypeNamespace, Name: schemas.Ptr("github"), ResponsesToolNamespace: &schemas.ResponsesToolNamespace{ + Tools: []schemas.ResponsesTool{{Type: schemas.ResponsesToolTypeFunction, Name: schemas.Ptr("create_issue")}}, + }}, + }, + }) + + ctx := newTestContext() + primary, _, err := p.PreLLMHook(ctx, original) + if err != nil { + t.Fatalf("PreLLMHook (primary): %v", err) + } + // Primary attempt: web_search dropped, tool_choice dropped. + // Namespace flattening happens later in core dispatch, so the namespace stays here. + if got := toolTypes(primary.ResponsesRequest.Params.Tools); len(got) != 2 || got[0] != schemas.ResponsesToolTypeFunction || got[1] != schemas.ResponsesToolTypeNamespace { + t.Fatalf("primary tools = %v, want [function namespace]", got) + } + if primary.ResponsesRequest.Params.ToolChoice != nil { + t.Fatalf("primary attempt kept tool_choice, so the fallback check below proves nothing") + } + + // Build the fallback request the way core's prepareFallbackRequest does. + fallbackReq := *original + tmp := *original.ResponsesRequest + tmp.Provider = schemas.OpenAI + tmp.Model = "fallback-model" + fallbackReq.ResponsesRequest = &tmp + + fallback, _, err := p.PreLLMHook(ctx, &fallbackReq) + if err != nil { + t.Fatalf("PreLLMHook (fallback): %v", err) + } + tools := fallback.ResponsesRequest.Params.Tools + if got := toolTypes(tools); len(got) != 3 || got[1] != schemas.ResponsesToolTypeWebSearch || got[2] != schemas.ResponsesToolTypeNamespace { + t.Errorf("fallback tools = %v, want the caller's [function web_search namespace]", got) + } + if len(tools) > 0 && tools[0].CacheControl == nil { + t.Errorf("fallback lost cache_control on tools[0]") + } + if fallback.ResponsesRequest.Params.ToolChoice == nil { + t.Errorf("fallback lost tool_choice") + } + if len(tools) > 2 && tools[2].ResponsesToolNamespace != nil && len(tools[2].ResponsesToolNamespace.Tools) != 1 { + t.Errorf("fallback namespace tool lost its nested tools") + } +} + +func toolTypes(tools []schemas.ResponsesTool) []schemas.ResponsesToolType { + types := make([]schemas.ResponsesToolType, len(tools)) + for i, tool := range tools { + types[i] = tool.Type + } + return types +} diff --git a/plugins/compat/version b/plugins/compat/version index 9e11b32fcaa..1c09c74e221 100644 --- a/plugins/compat/version +++ b/plugins/compat/version @@ -1 +1 @@ -0.3.1 +0.3.3 diff --git a/plugins/governance/allowbydefault_test.go b/plugins/governance/allowbydefault_test.go index 3ee876fe9a6..4a912952233 100644 --- a/plugins/governance/allowbydefault_test.go +++ b/plugins/governance/allowbydefault_test.go @@ -23,6 +23,14 @@ func (m *mockInMemoryStore) GetConfiguredProviders() map[schemas.ModelProvider]c return m.configuredProviders } +func (m *mockInMemoryStore) GetConfiguredProviderNames() []string { + names := make([]string, 0, len(m.configuredProviders)) + for provider := range m.configuredProviders { + names = append(names, string(provider)) + } + return names +} + func (m *mockInMemoryStore) GetMCPClientsAllowedByDefault() map[string]string { return m.allowedByDefaultClients } @@ -196,3 +204,91 @@ func TestAppendMCPPermitsAllowedByDefault(t *testing.T) { assert.Equal(t, own, AppendMCPPermitsAllowedByDefault(own, configured, nil), "nothing allowed by default leaves the list as it was") } + +// A permit that grants every provider names none, so the providers it grants by the flag alone are +// materialised onto it at construction. Every consumer then reads one list and cannot disagree with +// the permit about what it grants — which is what let a listing refuse a provider the request path +// admitted. +func TestAppendAllProviderPermits(t *testing.T) { + configured := []string{"openai", "anthropic", "bedrock", ""} + + t.Run("materialises the providers no config names", func(t *testing.T) { + permits := AppendAllProviderPermits([]schemas.ProviderPermit{ + {Provider: "openai", AllowedModels: schemas.WhiteList{"gpt-4o"}, KeyIDs: schemas.WhiteList{"key-1"}}, + }, configured) + + byProvider := map[string]schemas.ProviderPermit{} + for _, permit := range permits { + byProvider[permit.Provider] = permit + } + if len(byProvider) != 3 { + t.Fatalf("expected openai, anthropic and bedrock, got %v", permits) + } + // The empty name the deployment reported is not a provider and must not become a permit. + if _, ok := byProvider[""]; ok { + t.Error("an empty provider name must not be materialised") + } + // A provider the permit already named keeps its own rules: the flag widens the set, it does + // not relax the overrides. + if got := byProvider["openai"].AllowedModels; len(got) != 1 || got[0] != "gpt-4o" { + t.Errorf("openai override was replaced: %v", got) + } + // One it did not name is granted with nothing narrowed, and no weight, since a weight is a + // routing preference a provider config expresses and this one expresses none. + anthropic := byProvider["anthropic"] + if !anthropic.AllowedModels.IsUnrestricted() { + t.Errorf("expected all models allowed, got %v", anthropic.AllowedModels) + } + if !schemas.WhiteList(anthropic.KeyIDs).IsUnrestricted() { + t.Errorf("expected all keys allowed, got %v", anthropic.KeyIDs) + } + if len(anthropic.BlacklistedModels) != 0 { + t.Errorf("expected nothing blocked, got %v", anthropic.BlacklistedModels) + } + if anthropic.Weight != nil { + t.Errorf("expected no weight, got %v", *anthropic.Weight) + } + }) + + t.Run("a deployment with no providers adds nothing", func(t *testing.T) { + permits := AppendAllProviderPermits(nil, nil) + if len(permits) != 0 { + t.Fatalf("expected no permits, got %v", permits) + } + }) +} + +// The end the fix is for: a key that grants every provider must answer for one it holds no config +// for, in the same list every consumer reads. Asked through the access, because that is what the +// listing routes and the routing allowlist ask. +func TestPermitForVirtualKeyGrantsEveryConfiguredProvider(t *testing.T) { + store := &LocalGovernanceStore{inMemoryStore: &mockInMemoryStore{ + configuredProviders: map[schemas.ModelProvider]configstore.ProviderConfig{ + schemas.OpenAI: {}, schemas.Anthropic: {}, schemas.Bedrock: {}, + }, + }} + // The customer's shape: allow-all, plus one provider config carrying an override. + vk := &configstoreTables.TableVirtualKey{ + ID: "vk-1", Name: "allow-all", AllowAllProviders: true, + ProviderConfigs: []configstoreTables.TableVirtualKeyProviderConfig{{ + Provider: string(schemas.Bedrock), AllowedModels: schemas.WhiteList{"anthropic.claude-haiku-4-5"}, AllowAllKeys: true, + }}, + } + access := grant.NewAccess([]schemas.Permit{store.permitForVirtualKey(emptyCtx(), vk)}, nil, "", nil) + + assert.ElementsMatch(t, []string{"openai", "anthropic", "bedrock"}, access.GrantedProvidersForModel(""), + "a provider the key names no config for is still granted") + // The override survives: the flag widens the provider set, it does not relax what a config says. + assert.True(t, access.IsModelAllowed("bedrock", "anthropic.claude-haiku-4-5")) + assert.False(t, access.IsModelAllowed("bedrock", "amazon.titan-embed-text-v2:0")) + // And a provider granted by the flag alone narrows nothing. + assert.True(t, access.IsModelAllowed("anthropic", "claude-haiku-4-5")) + + t.Run("without the flag only the named providers are granted", func(t *testing.T) { + vk.AllowAllProviders = false + access := grant.NewAccess([]schemas.Permit{store.permitForVirtualKey(emptyCtx(), vk)}, nil, "", nil) + + assert.Equal(t, []string{"bedrock"}, access.GrantedProvidersForModel("")) + assert.False(t, access.IsModelAllowed("anthropic", "claude-haiku-4-5")) + }) +} diff --git a/plugins/governance/go.mod b/plugins/governance/go.mod index a8e2c226e37..aae8d717995 100644 --- a/plugins/governance/go.mod +++ b/plugins/governance/go.mod @@ -4,8 +4,8 @@ go 1.27.0 require ( github.com/google/uuid v1.6.0 - github.com/capsohq/bifrost/core v1.9.1 - github.com/capsohq/bifrost/framework v1.7.1 + github.com/capsohq/bifrost/core v1.10.2 + github.com/capsohq/bifrost/framework v1.7.4 github.com/stretchr/testify v1.11.1 github.com/valyala/fasthttp v1.74.0 gorm.io/gorm v1.31.1 diff --git a/plugins/governance/go.sum b/plugins/governance/go.sum index eac4db919c9..233a6b06f1c 100644 --- a/plugins/governance/go.sum +++ b/plugins/governance/go.sum @@ -262,10 +262,10 @@ github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsRe github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= github.com/mattn/go-sqlite3 v1.14.32 h1:JD12Ag3oLy1zQA+BNn74xRgaBbdhbNIDYvQUEuuErjs= github.com/mattn/go-sqlite3 v1.14.32/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= -github.com/maximhq/bifrost/core v1.9.1 h1:CF7TTj63htKQbuBL7eAGzou5yoe4XiQ0YMWpAO/atnw= -github.com/maximhq/bifrost/core v1.9.1/go.mod h1:bgTt5CsqB7duKiy/pC0uUMTxIRnvjj0t2OvH7Tlvuz0= -github.com/maximhq/bifrost/framework v1.7.1 h1:pGCdT7ksXmcnCylBpQqSHlQ071QAqgJ1XS7JwI8uV9U= -github.com/maximhq/bifrost/framework v1.7.1/go.mod h1:Acd4YNI+K8SjMPdghUQbjw0plJpqIDOP0s01iQr51Go= +github.com/maximhq/bifrost/core v1.10.2 h1:jRvvZN1KJVaX2Pxn9crTrmL3kW2QgDh7d1jsuz6b72k= +github.com/maximhq/bifrost/core v1.10.2/go.mod h1:bgTt5CsqB7duKiy/pC0uUMTxIRnvjj0t2OvH7Tlvuz0= +github.com/maximhq/bifrost/framework v1.7.4 h1:aM4MM3H3dXlPpkvyqRvt337WV8jl3m7DfBehAzs8HNU= +github.com/maximhq/bifrost/framework v1.7.4/go.mod h1:yHmcJhKvvaeoNUJYd5yV/wsNb3jZBwnjw2ny246C55Y= github.com/molecule-man/go-brrr v1.0.1 h1:cEjgx8hgNw6UGdhQ94SPDbPkKuRbkUcxBO3IzbGpA/o= github.com/molecule-man/go-brrr v1.0.1/go.mod h1:7ybW6/7gA3oKY45jOfVNjSJDtrr6ea4tzbsTkjmQDC4= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= diff --git a/plugins/governance/main.go b/plugins/governance/main.go index 3c691299587..718a2647386 100644 --- a/plugins/governance/main.go +++ b/plugins/governance/main.go @@ -40,6 +40,7 @@ type Config struct { type InMemoryStore interface { GetConfiguredProviders() map[schemas.ModelProvider]configstore.ProviderConfig + GetConfiguredProviderNames() []string GetMCPClientsAllowedByDefault() map[string]string // clientID → clientName GetMCPClientNames() map[string]string // clientID → clientName, every client // GetMCPClientBySlug resolves a client by its endpoint slug (for serving one client at /mcp/). @@ -458,11 +459,16 @@ func (p *GovernancePlugin) LoadBalanceProvider(ctx *schemas.BifrostContext, req return nil } + // Only weighted candidates can be selected or offered as fallbacks, so a candidate without a + // weight is as excluded as one a budget refused. Name it for the same reason every other + // exclusion above is named: the counts below are otherwise impossible to reconcile from the trail. weighted := make([]schemas.ProviderCandidate, 0, len(eligible)) for _, candidate := range eligible { - if candidate.Weight != nil { - weighted = append(weighted, candidate) + if candidate.Weight == nil { + ctx.AppendRoutingEngineLog(schemas.RoutingEngineGovernance, schemas.LogLevelInfo, fmt.Sprintf("Provider %s excluded: no weight assigned for model %s", candidate.Provider, modelStr)) + continue } + weighted = append(weighted, candidate) } if len(weighted) == 0 { @@ -495,8 +501,16 @@ func (p *GovernancePlugin) LoadBalanceProvider(ctx *schemas.BifrostContext, req selectedProvider = schemas.ModelProvider(weighted[0].Provider) } + var weightedProviders []string + for _, candidate := range weighted { + weightedProviders = append(weightedProviders, candidate.Provider) + } + p.logger.Debug("[governance] Selected provider: %s", selectedProvider) - ctx.AppendRoutingEngineLog(schemas.RoutingEngineGovernance, schemas.LogLevelInfo, fmt.Sprintf("Selected provider %s for model %s (from %d eligible: %v)", selectedProvider, modelStr, len(eligible), eligibleProviders)) + // The pool selection actually ran over, which is `weighted` and not `eligible`: reporting the + // wider set would describe candidates the roll could never have landed on, and would not + // account for the fallbacks added below. + ctx.AppendRoutingEngineLog(schemas.RoutingEngineGovernance, schemas.LogLevelInfo, fmt.Sprintf("Selected provider %s for model %s (from %d weighted: %v)", selectedProvider, modelStr, len(weighted), weightedProviders)) refinedModel := modelStr // Refine the model for the selected provider @@ -1657,7 +1671,7 @@ func (p *GovernancePlugin) reportBatchModelUsage(ctx context.Context, usage joba if len(usage.ModelUsage) == 0 { return nil } - alreadyCharged := make(map[string]bool, len(usage.BudgetIDs)+len(usage.RateLimitIDs)) + alreadyCharged := make(map[string]bool) for _, id := range usage.BudgetIDs { alreadyCharged["budget:"+id] = true } diff --git a/plugins/governance/store.go b/plugins/governance/store.go index 082980fe98b..d41a53e040d 100644 --- a/plugins/governance/store.go +++ b/plugins/governance/store.go @@ -1451,6 +1451,9 @@ func (gs *LocalGovernanceStore) permitForVirtualKey(ctx context.Context, vk *con Weight: config.Weight, }) } + if vk.AllowAllProviders && gs.inMemoryStore != nil { + providerPermits = AppendAllProviderPermits(providerPermits, gs.inMemoryStore.GetConfiguredProviderNames()) + } // A key's own MCP configs and its Virtual MCPs build one accumulator, so a Virtual MCP grants // exactly like a config: owns the clients it names, unions per client, blocks allowed-by-default. diff --git a/plugins/governance/utils.go b/plugins/governance/utils.go index 0f4f090bda4..870e3229e4f 100644 --- a/plugins/governance/utils.go +++ b/plugins/governance/utils.go @@ -258,3 +258,38 @@ func PresentedCredentialResolved(ctx *schemas.BifrostContext) bool { access := g.Access() return access != nil && unusablePermit(access) == nil } + +// AppendAllProviderPermits completes a permit that grants every provider. Such a permit names none, +// so the providers it grants by the flag alone are materialised here, from what the deployment has +// configured, and the permit then carries its whole grant in one readable list. +// +// Doing it at permit construction rather than where a consumer reads the permit is what keeps every +// consumer honest: enumerating provider permits and asking whether the permit allows a provider give +// the same answer, so a listing cannot refuse what the request path admits. Built per request, so a +// provider added after the permit was last written is granted by the same rule. +// +// A provider the permit already names keeps its own entry: those are overrides, and the flag widens +// the set rather than relaxing them. A materialised entry narrows nothing - every model, every key, +// nothing blocked - and carries no weight, because a weight is a routing preference a provider +// config expresses and this one expresses none. +func AppendAllProviderPermits(permits []schemas.ProviderPermit, configured []string) []schemas.ProviderPermit { + named := make(map[string]struct{}, len(permits)) + for i := range permits { + named[permits[i].Provider] = struct{}{} + } + for _, provider := range configured { + if provider == "" { + continue + } + if _, dup := named[provider]; dup { + continue + } + named[provider] = struct{}{} + permits = append(permits, schemas.ProviderPermit{ + Provider: provider, + AllowedModels: schemas.WhiteList{"*"}, + KeyIDs: schemas.WhiteList{"*"}, + }) + } + return permits +} diff --git a/plugins/governance/version b/plugins/governance/version index a8fdfda1c78..a7ee35a3ea7 100644 --- a/plugins/governance/version +++ b/plugins/governance/version @@ -1 +1 @@ -1.8.1 +1.8.3 diff --git a/plugins/jsonparser/go.mod b/plugins/jsonparser/go.mod index fe6b0dbe78e..c80d47aebab 100644 --- a/plugins/jsonparser/go.mod +++ b/plugins/jsonparser/go.mod @@ -2,7 +2,7 @@ module github.com/capsohq/bifrost/plugins/jsonparser go 1.27.0 -require github.com/capsohq/bifrost/core v1.9.1 +require github.com/capsohq/bifrost/core v1.10.2 require ( cloud.google.com/go v0.123.0 // indirect diff --git a/plugins/jsonparser/go.sum b/plugins/jsonparser/go.sum index fc4abca6a75..5f43a96a0c6 100644 --- a/plugins/jsonparser/go.sum +++ b/plugins/jsonparser/go.sum @@ -111,8 +111,8 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/ github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= -github.com/maximhq/bifrost/core v1.9.1 h1:CF7TTj63htKQbuBL7eAGzou5yoe4XiQ0YMWpAO/atnw= -github.com/maximhq/bifrost/core v1.9.1/go.mod h1:bgTt5CsqB7duKiy/pC0uUMTxIRnvjj0t2OvH7Tlvuz0= +github.com/maximhq/bifrost/core v1.10.2 h1:jRvvZN1KJVaX2Pxn9crTrmL3kW2QgDh7d1jsuz6b72k= +github.com/maximhq/bifrost/core v1.10.2/go.mod h1:bgTt5CsqB7duKiy/pC0uUMTxIRnvjj0t2OvH7Tlvuz0= github.com/molecule-man/go-brrr v1.0.1 h1:cEjgx8hgNw6UGdhQ94SPDbPkKuRbkUcxBO3IzbGpA/o= github.com/molecule-man/go-brrr v1.0.1/go.mod h1:7ybW6/7gA3oKY45jOfVNjSJDtrr6ea4tzbsTkjmQDC4= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= diff --git a/plugins/jsonparser/version b/plugins/jsonparser/version index 9edc58bb1dd..ec70f755607 100644 --- a/plugins/jsonparser/version +++ b/plugins/jsonparser/version @@ -1 +1 @@ -1.6.4 +1.6.6 diff --git a/plugins/logging/go.mod b/plugins/logging/go.mod index 0a160692156..64945612864 100644 --- a/plugins/logging/go.mod +++ b/plugins/logging/go.mod @@ -5,8 +5,8 @@ go 1.27.0 require ( github.com/bytedance/sonic v1.15.3-0.20260730064818-2a36d6da63e2 github.com/google/uuid v1.6.0 - github.com/capsohq/bifrost/core v1.9.1 - github.com/capsohq/bifrost/framework v1.7.1 + github.com/capsohq/bifrost/core v1.10.2 + github.com/capsohq/bifrost/framework v1.7.4 github.com/stretchr/testify v1.11.1 ) diff --git a/plugins/logging/go.sum b/plugins/logging/go.sum index eac4db919c9..233a6b06f1c 100644 --- a/plugins/logging/go.sum +++ b/plugins/logging/go.sum @@ -262,10 +262,10 @@ github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsRe github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= github.com/mattn/go-sqlite3 v1.14.32 h1:JD12Ag3oLy1zQA+BNn74xRgaBbdhbNIDYvQUEuuErjs= github.com/mattn/go-sqlite3 v1.14.32/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= -github.com/maximhq/bifrost/core v1.9.1 h1:CF7TTj63htKQbuBL7eAGzou5yoe4XiQ0YMWpAO/atnw= -github.com/maximhq/bifrost/core v1.9.1/go.mod h1:bgTt5CsqB7duKiy/pC0uUMTxIRnvjj0t2OvH7Tlvuz0= -github.com/maximhq/bifrost/framework v1.7.1 h1:pGCdT7ksXmcnCylBpQqSHlQ071QAqgJ1XS7JwI8uV9U= -github.com/maximhq/bifrost/framework v1.7.1/go.mod h1:Acd4YNI+K8SjMPdghUQbjw0plJpqIDOP0s01iQr51Go= +github.com/maximhq/bifrost/core v1.10.2 h1:jRvvZN1KJVaX2Pxn9crTrmL3kW2QgDh7d1jsuz6b72k= +github.com/maximhq/bifrost/core v1.10.2/go.mod h1:bgTt5CsqB7duKiy/pC0uUMTxIRnvjj0t2OvH7Tlvuz0= +github.com/maximhq/bifrost/framework v1.7.4 h1:aM4MM3H3dXlPpkvyqRvt337WV8jl3m7DfBehAzs8HNU= +github.com/maximhq/bifrost/framework v1.7.4/go.mod h1:yHmcJhKvvaeoNUJYd5yV/wsNb3jZBwnjw2ny246C55Y= github.com/molecule-man/go-brrr v1.0.1 h1:cEjgx8hgNw6UGdhQ94SPDbPkKuRbkUcxBO3IzbGpA/o= github.com/molecule-man/go-brrr v1.0.1/go.mod h1:7ybW6/7gA3oKY45jOfVNjSJDtrr6ea4tzbsTkjmQDC4= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= diff --git a/plugins/logging/main.go b/plugins/logging/main.go index 9c53d99ec88..c86c54d6299 100644 --- a/plugins/logging/main.go +++ b/plugins/logging/main.go @@ -1645,6 +1645,8 @@ func (p *LoggerPlugin) PreLLMHook(ctx *schemas.BifrostContext, req *schemas.Bifr initialData.Params = req.EmbeddingRequest.Params case schemas.RerankRequest: initialData.Params = req.RerankRequest.Params + case schemas.DecisionRequest: + initialData.Params = req.DecisionRequest.Questions case schemas.OCRRequest: initialData.Params = req.OCRRequest.Params initialData.OCRInput = &req.OCRRequest.Document diff --git a/plugins/logging/operations.go b/plugins/logging/operations.go index fb616b603f9..acccee2f189 100644 --- a/plugins/logging/operations.go +++ b/plugins/logging/operations.go @@ -590,6 +590,10 @@ func (p *LoggerPlugin) applyNonStreamingOutputToEntry(entry *logstore.Log, resul usage = result.CompactionResponse.Usage.ToBifrostLLMUsage() case result.EmbeddingResponse != nil && result.EmbeddingResponse.Usage != nil: usage = result.EmbeddingResponse.Usage + case result.RerankResponse != nil && result.RerankResponse.Usage != nil: + usage = result.RerankResponse.Usage + case result.DecisionResponse != nil && result.DecisionResponse.Usage != nil: + usage = result.DecisionResponse.Usage case result.TranscriptionResponse != nil && result.TranscriptionResponse.Usage != nil: usage = &schemas.BifrostLLMUsage{} if result.TranscriptionResponse.Usage.InputTokens != nil { @@ -704,6 +708,17 @@ func (p *LoggerPlugin) applyNonStreamingOutputToEntry(entry *logstore.Log, resul if result.RerankResponse != nil && len(result.RerankResponse.Results) > 0 { entry.RerankOutputParsed = result.RerankResponse.Results } + if result.DecisionResponse != nil && len(result.DecisionResponse.Answers) > 0 { + if answersJSON, err := sonic.Marshal(result.DecisionResponse.Answers); err == nil { + answers := string(answersJSON) + entry.OutputMessageParsed = &schemas.ChatMessage{ + Role: schemas.ChatMessageRoleAssistant, + Content: &schemas.ChatMessageContent{ + ContentStr: &answers, + }, + } + } + } if result.OCRResponse != nil { entry.OCROutputParsed = result.OCRResponse } @@ -2170,6 +2185,16 @@ func buildResponseForRequestType(requestType schemas.RequestType, usage *schemas ExtraFields: extra, }, } + case schemas.DecisionRequest: + return &schemas.BifrostResponse{ + DecisionResponse: &schemas.BifrostDecisionResponse{ + Usage: usage, + ServiceTier: tier.serviceTier, + Speed: tier.speed, + InferenceGeo: tier.inferenceGeo, + ExtraFields: extra, + }, + } case schemas.OCRRequest: return &schemas.BifrostResponse{ OCRResponse: &schemas.BifrostOCRResponse{ diff --git a/plugins/logging/operations_test.go b/plugins/logging/operations_test.go index 24ea44fe7c4..3f94be974db 100644 --- a/plugins/logging/operations_test.go +++ b/plugins/logging/operations_test.go @@ -3671,3 +3671,19 @@ func TestStoreOrEnqueueBeforeInjectStillParks(t *testing.T) { t.Fatalf("Inject must drain the parked entry, writeQueue has %d entries", got) } } + +// A logged decision's served tier survives cost recalculation: the rebuilt response +// carries it, as the Responses rebuild does. +func TestBuildResponseForRequestTypeKeepsDecisionTier(t *testing.T) { + priority := schemas.BifrostServiceTierPriority + fast, us := "fast", "us" + resp := buildResponseForRequestType(schemas.DecisionRequest, &schemas.BifrostLLMUsage{PromptTokens: 10}, schemas.BifrostResponseExtraFields{}, + servedTier{serviceTier: &priority, speed: &fast, inferenceGeo: &us}) + if resp == nil || resp.DecisionResponse == nil { + t.Fatal("no decision response rebuilt") + } + d := resp.DecisionResponse + if d.ServiceTier == nil || *d.ServiceTier != priority || d.Speed == nil || *d.Speed != fast || d.InferenceGeo == nil || *d.InferenceGeo != us { + t.Fatalf("decision tier not restored: tier=%v speed=%v geo=%v", d.ServiceTier, d.Speed, d.InferenceGeo) + } +} diff --git a/plugins/logging/utils.go b/plugins/logging/utils.go index e4f80404776..37d6a4fc617 100644 --- a/plugins/logging/utils.go +++ b/plugins/logging/utils.go @@ -693,6 +693,22 @@ func (p *LoggerPlugin) extractInputHistory(request *schemas.BifrostRequest) ([]s }, }, []schemas.ResponsesMessage{} } + if request.DecisionRequest != nil { + var state string + if s, ok := request.DecisionRequest.State.(string); ok { + state = s + } else if raw, err := sonic.Marshal(request.DecisionRequest.State); err == nil { + state = string(raw) + } + return []schemas.ChatMessage{ + { + Role: schemas.ChatMessageRoleUser, + Content: &schemas.ChatMessageContent{ + ContentStr: &state, + }, + }, + }, []schemas.ResponsesMessage{} + } if request.RerankRequest != nil { query := request.RerankRequest.Query return []schemas.ChatMessage{ diff --git a/plugins/logging/version b/plugins/logging/version index a8fdfda1c78..a7ee35a3ea7 100644 --- a/plugins/logging/version +++ b/plugins/logging/version @@ -1 +1 @@ -1.8.1 +1.8.3 diff --git a/plugins/maxim/go.mod b/plugins/maxim/go.mod index 9c9d011b1d2..af075ab0368 100644 --- a/plugins/maxim/go.mod +++ b/plugins/maxim/go.mod @@ -5,8 +5,8 @@ go 1.27.0 require ( github.com/bytedance/sonic v1.15.3-0.20260730064818-2a36d6da63e2 github.com/google/uuid v1.6.0 - github.com/capsohq/bifrost/core v1.9.1 - github.com/capsohq/bifrost/framework v1.7.1 + github.com/capsohq/bifrost/core v1.10.2 + github.com/capsohq/bifrost/framework v1.7.4 github.com/maximhq/maxim-go v0.2.1 ) diff --git a/plugins/maxim/go.sum b/plugins/maxim/go.sum index 2834023e730..a8f4ea1ba72 100644 --- a/plugins/maxim/go.sum +++ b/plugins/maxim/go.sum @@ -262,10 +262,10 @@ github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsRe github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= github.com/mattn/go-sqlite3 v1.14.32 h1:JD12Ag3oLy1zQA+BNn74xRgaBbdhbNIDYvQUEuuErjs= github.com/mattn/go-sqlite3 v1.14.32/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= -github.com/maximhq/bifrost/core v1.9.1 h1:CF7TTj63htKQbuBL7eAGzou5yoe4XiQ0YMWpAO/atnw= -github.com/maximhq/bifrost/core v1.9.1/go.mod h1:bgTt5CsqB7duKiy/pC0uUMTxIRnvjj0t2OvH7Tlvuz0= -github.com/maximhq/bifrost/framework v1.7.1 h1:pGCdT7ksXmcnCylBpQqSHlQ071QAqgJ1XS7JwI8uV9U= -github.com/maximhq/bifrost/framework v1.7.1/go.mod h1:Acd4YNI+K8SjMPdghUQbjw0plJpqIDOP0s01iQr51Go= +github.com/maximhq/bifrost/core v1.10.2 h1:jRvvZN1KJVaX2Pxn9crTrmL3kW2QgDh7d1jsuz6b72k= +github.com/maximhq/bifrost/core v1.10.2/go.mod h1:bgTt5CsqB7duKiy/pC0uUMTxIRnvjj0t2OvH7Tlvuz0= +github.com/maximhq/bifrost/framework v1.7.4 h1:aM4MM3H3dXlPpkvyqRvt337WV8jl3m7DfBehAzs8HNU= +github.com/maximhq/bifrost/framework v1.7.4/go.mod h1:yHmcJhKvvaeoNUJYd5yV/wsNb3jZBwnjw2ny246C55Y= github.com/maximhq/maxim-go v0.2.1 h1:hCp8dQ4HsyyNC+y5HCUuY/HFD0sOnGkjL5MdYCHkgEQ= github.com/maximhq/maxim-go v0.2.1/go.mod h1:nwFznXy0Dn4mxXGU4X+BCnE3VP68L+FPEaW0yUgk96o= github.com/molecule-man/go-brrr v1.0.1 h1:cEjgx8hgNw6UGdhQ94SPDbPkKuRbkUcxBO3IzbGpA/o= diff --git a/plugins/maxim/version b/plugins/maxim/version index 10c088013f8..de28578affc 100644 --- a/plugins/maxim/version +++ b/plugins/maxim/version @@ -1 +1 @@ -1.7.4 +1.7.6 diff --git a/plugins/mocker/go.mod b/plugins/mocker/go.mod index eea0fe9497c..975b1cae579 100644 --- a/plugins/mocker/go.mod +++ b/plugins/mocker/go.mod @@ -4,7 +4,7 @@ go 1.27.0 require ( github.com/jaswdr/faker/v2 v2.8.0 - github.com/capsohq/bifrost/core v1.9.1 + github.com/capsohq/bifrost/core v1.10.2 ) require ( diff --git a/plugins/mocker/go.sum b/plugins/mocker/go.sum index 355d76e9420..1184b395f19 100644 --- a/plugins/mocker/go.sum +++ b/plugins/mocker/go.sum @@ -113,8 +113,8 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/ github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= -github.com/maximhq/bifrost/core v1.9.1 h1:CF7TTj63htKQbuBL7eAGzou5yoe4XiQ0YMWpAO/atnw= -github.com/maximhq/bifrost/core v1.9.1/go.mod h1:bgTt5CsqB7duKiy/pC0uUMTxIRnvjj0t2OvH7Tlvuz0= +github.com/maximhq/bifrost/core v1.10.2 h1:jRvvZN1KJVaX2Pxn9crTrmL3kW2QgDh7d1jsuz6b72k= +github.com/maximhq/bifrost/core v1.10.2/go.mod h1:bgTt5CsqB7duKiy/pC0uUMTxIRnvjj0t2OvH7Tlvuz0= github.com/molecule-man/go-brrr v1.0.1 h1:cEjgx8hgNw6UGdhQ94SPDbPkKuRbkUcxBO3IzbGpA/o= github.com/molecule-man/go-brrr v1.0.1/go.mod h1:7ybW6/7gA3oKY45jOfVNjSJDtrr6ea4tzbsTkjmQDC4= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= diff --git a/plugins/mocker/version b/plugins/mocker/version index 9edc58bb1dd..ec70f755607 100644 --- a/plugins/mocker/version +++ b/plugins/mocker/version @@ -1 +1 @@ -1.6.4 +1.6.6 diff --git a/plugins/modelcatalogresolver/go.mod b/plugins/modelcatalogresolver/go.mod index 8565187efa7..134c490205a 100644 --- a/plugins/modelcatalogresolver/go.mod +++ b/plugins/modelcatalogresolver/go.mod @@ -3,8 +3,8 @@ module github.com/capsohq/bifrost/plugins/modelcatalogresolver go 1.27.0 require ( - github.com/capsohq/bifrost/core v1.9.1 - github.com/capsohq/bifrost/framework v1.7.1 + github.com/capsohq/bifrost/core v1.10.2 + github.com/capsohq/bifrost/framework v1.7.4 ) require ( diff --git a/plugins/modelcatalogresolver/go.sum b/plugins/modelcatalogresolver/go.sum index eac4db919c9..233a6b06f1c 100644 --- a/plugins/modelcatalogresolver/go.sum +++ b/plugins/modelcatalogresolver/go.sum @@ -262,10 +262,10 @@ github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsRe github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= github.com/mattn/go-sqlite3 v1.14.32 h1:JD12Ag3oLy1zQA+BNn74xRgaBbdhbNIDYvQUEuuErjs= github.com/mattn/go-sqlite3 v1.14.32/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= -github.com/maximhq/bifrost/core v1.9.1 h1:CF7TTj63htKQbuBL7eAGzou5yoe4XiQ0YMWpAO/atnw= -github.com/maximhq/bifrost/core v1.9.1/go.mod h1:bgTt5CsqB7duKiy/pC0uUMTxIRnvjj0t2OvH7Tlvuz0= -github.com/maximhq/bifrost/framework v1.7.1 h1:pGCdT7ksXmcnCylBpQqSHlQ071QAqgJ1XS7JwI8uV9U= -github.com/maximhq/bifrost/framework v1.7.1/go.mod h1:Acd4YNI+K8SjMPdghUQbjw0plJpqIDOP0s01iQr51Go= +github.com/maximhq/bifrost/core v1.10.2 h1:jRvvZN1KJVaX2Pxn9crTrmL3kW2QgDh7d1jsuz6b72k= +github.com/maximhq/bifrost/core v1.10.2/go.mod h1:bgTt5CsqB7duKiy/pC0uUMTxIRnvjj0t2OvH7Tlvuz0= +github.com/maximhq/bifrost/framework v1.7.4 h1:aM4MM3H3dXlPpkvyqRvt337WV8jl3m7DfBehAzs8HNU= +github.com/maximhq/bifrost/framework v1.7.4/go.mod h1:yHmcJhKvvaeoNUJYd5yV/wsNb3jZBwnjw2ny246C55Y= github.com/molecule-man/go-brrr v1.0.1 h1:cEjgx8hgNw6UGdhQ94SPDbPkKuRbkUcxBO3IzbGpA/o= github.com/molecule-man/go-brrr v1.0.1/go.mod h1:7ybW6/7gA3oKY45jOfVNjSJDtrr6ea4tzbsTkjmQDC4= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= diff --git a/plugins/modelcatalogresolver/version b/plugins/modelcatalogresolver/version index 65087b4f5ec..0664a8fd291 100644 --- a/plugins/modelcatalogresolver/version +++ b/plugins/modelcatalogresolver/version @@ -1 +1 @@ -1.1.4 +1.1.6 diff --git a/plugins/otel/go.mod b/plugins/otel/go.mod index 564d24a8740..2d24a639f21 100644 --- a/plugins/otel/go.mod +++ b/plugins/otel/go.mod @@ -4,8 +4,8 @@ go 1.27.0 require ( github.com/bytedance/sonic v1.15.3-0.20260730064818-2a36d6da63e2 - github.com/capsohq/bifrost/core v1.9.1 - github.com/capsohq/bifrost/framework v1.7.1 + github.com/capsohq/bifrost/core v1.10.2 + github.com/capsohq/bifrost/framework v1.7.4 go.opentelemetry.io/otel v1.45.0 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.44.0 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.44.0 diff --git a/plugins/otel/go.sum b/plugins/otel/go.sum index 2e2e15040e6..299e06bec09 100644 --- a/plugins/otel/go.sum +++ b/plugins/otel/go.sum @@ -266,10 +266,10 @@ github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsRe github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= github.com/mattn/go-sqlite3 v1.14.32 h1:JD12Ag3oLy1zQA+BNn74xRgaBbdhbNIDYvQUEuuErjs= github.com/mattn/go-sqlite3 v1.14.32/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= -github.com/maximhq/bifrost/core v1.9.1 h1:CF7TTj63htKQbuBL7eAGzou5yoe4XiQ0YMWpAO/atnw= -github.com/maximhq/bifrost/core v1.9.1/go.mod h1:bgTt5CsqB7duKiy/pC0uUMTxIRnvjj0t2OvH7Tlvuz0= -github.com/maximhq/bifrost/framework v1.7.1 h1:pGCdT7ksXmcnCylBpQqSHlQ071QAqgJ1XS7JwI8uV9U= -github.com/maximhq/bifrost/framework v1.7.1/go.mod h1:Acd4YNI+K8SjMPdghUQbjw0plJpqIDOP0s01iQr51Go= +github.com/maximhq/bifrost/core v1.10.2 h1:jRvvZN1KJVaX2Pxn9crTrmL3kW2QgDh7d1jsuz6b72k= +github.com/maximhq/bifrost/core v1.10.2/go.mod h1:bgTt5CsqB7duKiy/pC0uUMTxIRnvjj0t2OvH7Tlvuz0= +github.com/maximhq/bifrost/framework v1.7.4 h1:aM4MM3H3dXlPpkvyqRvt337WV8jl3m7DfBehAzs8HNU= +github.com/maximhq/bifrost/framework v1.7.4/go.mod h1:yHmcJhKvvaeoNUJYd5yV/wsNb3jZBwnjw2ny246C55Y= github.com/molecule-man/go-brrr v1.0.1 h1:cEjgx8hgNw6UGdhQ94SPDbPkKuRbkUcxBO3IzbGpA/o= github.com/molecule-man/go-brrr v1.0.1/go.mod h1:7ybW6/7gA3oKY45jOfVNjSJDtrr6ea4tzbsTkjmQDC4= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= diff --git a/plugins/otel/version b/plugins/otel/version index 94fe62c2740..eac1e0ada6d 100644 --- a/plugins/otel/version +++ b/plugins/otel/version @@ -1 +1 @@ -1.5.4 +1.5.6 diff --git a/plugins/prompts/go.mod b/plugins/prompts/go.mod index 72c09ce951a..9568d793178 100644 --- a/plugins/prompts/go.mod +++ b/plugins/prompts/go.mod @@ -3,8 +3,8 @@ module github.com/capsohq/bifrost/plugins/prompts go 1.27.0 require ( - github.com/capsohq/bifrost/core v1.9.1 - github.com/capsohq/bifrost/framework v1.7.1 + github.com/capsohq/bifrost/core v1.10.2 + github.com/capsohq/bifrost/framework v1.7.4 github.com/stretchr/testify v1.11.1 ) diff --git a/plugins/prompts/go.sum b/plugins/prompts/go.sum index 850e10c11d2..02c62a66d37 100644 --- a/plugins/prompts/go.sum +++ b/plugins/prompts/go.sum @@ -125,10 +125,10 @@ github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsRe github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= github.com/mattn/go-sqlite3 v1.14.32 h1:JD12Ag3oLy1zQA+BNn74xRgaBbdhbNIDYvQUEuuErjs= github.com/mattn/go-sqlite3 v1.14.32/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= -github.com/maximhq/bifrost/core v1.9.1 h1:CF7TTj63htKQbuBL7eAGzou5yoe4XiQ0YMWpAO/atnw= -github.com/maximhq/bifrost/core v1.9.1/go.mod h1:bgTt5CsqB7duKiy/pC0uUMTxIRnvjj0t2OvH7Tlvuz0= -github.com/maximhq/bifrost/framework v1.7.1 h1:pGCdT7ksXmcnCylBpQqSHlQ071QAqgJ1XS7JwI8uV9U= -github.com/maximhq/bifrost/framework v1.7.1/go.mod h1:Acd4YNI+K8SjMPdghUQbjw0plJpqIDOP0s01iQr51Go= +github.com/maximhq/bifrost/core v1.10.2 h1:jRvvZN1KJVaX2Pxn9crTrmL3kW2QgDh7d1jsuz6b72k= +github.com/maximhq/bifrost/core v1.10.2/go.mod h1:bgTt5CsqB7duKiy/pC0uUMTxIRnvjj0t2OvH7Tlvuz0= +github.com/maximhq/bifrost/framework v1.7.4 h1:aM4MM3H3dXlPpkvyqRvt337WV8jl3m7DfBehAzs8HNU= +github.com/maximhq/bifrost/framework v1.7.4/go.mod h1:yHmcJhKvvaeoNUJYd5yV/wsNb3jZBwnjw2ny246C55Y= github.com/molecule-man/go-brrr v1.0.1 h1:cEjgx8hgNw6UGdhQ94SPDbPkKuRbkUcxBO3IzbGpA/o= github.com/molecule-man/go-brrr v1.0.1/go.mod h1:7ybW6/7gA3oKY45jOfVNjSJDtrr6ea4tzbsTkjmQDC4= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= diff --git a/plugins/prompts/version b/plugins/prompts/version index 65087b4f5ec..0664a8fd291 100644 --- a/plugins/prompts/version +++ b/plugins/prompts/version @@ -1 +1 @@ -1.1.4 +1.1.6 diff --git a/plugins/routing/complexityrouting.go b/plugins/routing/complexityrouting.go index f36467f7a9d..85de1392c13 100644 --- a/plugins/routing/complexityrouting.go +++ b/plugins/routing/complexityrouting.go @@ -25,7 +25,6 @@ type complexityProposal struct { func (p *RoutingPlugin) computeComplexity( ctx *schemas.BifrostContext, req *schemas.BifrostRequest, - virtualKeyID string, ) *complexity.ComplexityResult { input, disposition := complexity.BuildInputWithDisposition(ctx, req) sessionID, _ := ctx.Value(schemas.BifrostContextKeySessionID).(string) @@ -33,7 +32,7 @@ func (p *RoutingPlugin) computeComplexity( if disposition == complexity.InputContinuation { if sessionActive { - key := buildComplexitySessionKey(ctx, virtualKeyID, sessionID) + key := complexitySessionKey(ctx) tier, found, err := p.sessionStore.load(key, true) if err != nil { p.logComplexitySessionStoreError("refresh continuation", err) @@ -74,7 +73,7 @@ func (p *RoutingPlugin) computeComplexity( return proposal.Result } - key := buildComplexitySessionKey(ctx, virtualKeyID, sessionID) + key := complexitySessionKey(ctx) priorTier, priorFound, loadErr := p.sessionStore.load(key, false) if loadErr != nil { p.logComplexitySessionStoreError("inspect", loadErr) diff --git a/plugins/routing/complexitysession.go b/plugins/routing/complexitysession.go index 6c253f2c0ba..ec99e60878b 100644 --- a/plugins/routing/complexitysession.go +++ b/plugins/routing/complexitysession.go @@ -1,8 +1,6 @@ package routing import ( - "crypto/sha256" - "encoding/hex" "encoding/json" "errors" "fmt" @@ -15,7 +13,8 @@ import ( ) const ( - complexitySessionKeyPrefix = "complexity-session:v1:" + // complexitySessionKind is the session state kind under which the effective tier is kept. + complexitySessionKind = "complexity" complexitySessionInactivityTTL = 24 * time.Hour ) @@ -143,29 +142,9 @@ func decodeStoredComplexityTier(value any) (string, error) { return tier, nil } -// buildComplexitySessionKey isolates equal caller session IDs across virtual -// keys and authenticated users, then hashes the complete tuple so the in-memory -// key has bounded size and reveals no caller-provided identifier. -func buildComplexitySessionKey( - ctx *schemas.BifrostContext, - virtualKeyID string, - sessionID string, -) string { - userID := bifrost.GetStringFromContext(ctx, schemas.BifrostContextKeyUserID) - scopeKind := "deployment" - switch { - case virtualKeyID != "" && userID != "": - scopeKind = "virtual-key-user" - case virtualKeyID != "": - scopeKind = "virtual-key" - case userID != "": - scopeKind = "user" - } - - hash := sha256.New() - for _, part := range []string{scopeKind, virtualKeyID, userID, sessionID} { - _, _ = hash.Write([]byte(part)) - _, _ = hash.Write([]byte{0}) - } - return complexitySessionKeyPrefix + hex.EncodeToString(hash.Sum(nil)) +// complexitySessionKey is where the request's session keeps its effective tier: the shared +// session state key, scoped to the virtual key and user the request is attributed to, under +// the complexity kind. +func complexitySessionKey(ctx *schemas.BifrostContext) string { + return bifrost.SessionStateKey(ctx, complexitySessionKind) } diff --git a/plugins/routing/complexitysession_test.go b/plugins/routing/complexitysession_test.go index 892a685f303..0694346d3e8 100644 --- a/plugins/routing/complexitysession_test.go +++ b/plugins/routing/complexitysession_test.go @@ -10,6 +10,7 @@ import ( "github.com/capsohq/bifrost/core/schemas" "github.com/capsohq/bifrost/framework/configstore" + "github.com/capsohq/bifrost/framework/grant" "github.com/capsohq/bifrost/framework/kvstore" "github.com/capsohq/bifrost/plugins/routing/complexity" "github.com/capsohq/bifrost/plugins/routing/rules" @@ -62,7 +63,7 @@ func (s *recordingSessionKVStore) SetWithTTL(key string, value any, ttl time.Dur func TestComplexitySessionStoreMonotonicLadder(t *testing.T) { sessions, _ := newTestComplexitySessionStore(t, time.Minute) - key := complexitySessionKeyPrefix + "ladder" + key := "complexity-test:ladder" resolution, err := sessions.resolve(key, complexity.TierSimple) require.NoError(t, err) @@ -91,7 +92,7 @@ func TestComplexitySessionStoreMonotonicLadder(t *testing.T) { func TestComplexitySessionStoreEmptyProposalOnlyRefreshesExisting(t *testing.T) { sessions, store := newTestComplexitySessionStore(t, time.Minute) - key := complexitySessionKeyPrefix + "continuation" + key := "complexity-test:continuation" resolution, err := sessions.resolve(key, "") require.NoError(t, err) @@ -114,7 +115,7 @@ func TestComplexitySessionStoreRefreshesTTLWhenTierDoesNotChange(t *testing.T) { const sessionTTL = 45 * time.Minute recordingStore := &recordingSessionKVStore{KVStore: store} sessions := newComplexitySessionStore(recordingStore, sessionTTL) - key := complexitySessionKeyPrefix + "refresh" + key := "complexity-test:refresh" _, err = sessions.resolve(key, complexity.TierMedium) require.NoError(t, err) @@ -131,7 +132,7 @@ func TestComplexitySessionStoreRefreshesTTLWhenTierDoesNotChange(t *testing.T) { func TestComplexitySessionStoreExpiryStartsNewEpoch(t *testing.T) { sessions, _ := newTestComplexitySessionStore(t, 10*time.Millisecond) - key := complexitySessionKeyPrefix + "expiry" + key := "complexity-test:expiry" _, err := sessions.resolve(key, complexity.TierComplex) require.NoError(t, err) @@ -150,7 +151,7 @@ func TestComplexitySessionStoreExpiryStartsNewEpoch(t *testing.T) { func TestComplexitySessionStoreRejectsCorruptTier(t *testing.T) { sessions, store := newTestComplexitySessionStore(t, time.Minute) - key := complexitySessionKeyPrefix + "corrupt" + key := "complexity-test:corrupt" require.NoError(t, store.SetWithTTL(key, "UNKNOWN", time.Minute)) _, _, err := sessions.load(key, false) @@ -161,7 +162,7 @@ func TestComplexitySessionStoreRejectsCorruptTier(t *testing.T) { func TestComplexitySessionStoreDecodesReplicatedString(t *testing.T) { sessions, store := newTestComplexitySessionStore(t, time.Minute) - key := complexitySessionKeyPrefix + "remote" + key := "complexity-test:remote" now := time.Now() require.NoError(t, store.SetRemote(key, []byte(`"MEDIUM"`), now.UnixNano(), now.Add(time.Minute).UnixNano())) @@ -171,22 +172,40 @@ func TestComplexitySessionStoreDecodesReplicatedString(t *testing.T) { require.Equal(t, complexity.TierMedium, tier) } -func TestBuildComplexitySessionKeyScopesAndHidesIdentity(t *testing.T) { - ctx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) - ctx.SetValue(schemas.BifrostContextKeyUserID, "user-1") +func TestComplexitySessionKeyScopesByIdentityAndHidesIt(t *testing.T) { sessionID := "caller-session-secret" + attributed := func(sessionID, virtualKeyID, userID string) *schemas.BifrostContext { + ctx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) + ctx.SetValue(schemas.BifrostContextKeySessionID, sessionID) + var user *schemas.UserRef + if userID != "" { + user = &schemas.UserRef{ID: userID} + } + var virtualKey *schemas.EntityRef + if virtualKeyID != "" { + virtualKey = &schemas.EntityRef{ID: virtualKeyID} + } + g := grant.New() + g.SetIdentity(grant.NewIdentity(schemas.Credential{}, user, virtualKey, nil, nil, nil, nil)) + ctx.SetGrant(g) + return ctx + } - base := buildComplexitySessionKey(ctx, "vk-1", sessionID) - require.True(t, strings.HasPrefix(base, complexitySessionKeyPrefix)) + base := complexitySessionKey(attributed(sessionID, "vk-1", "user-1")) + require.True(t, strings.HasPrefix(base, "session:v2:complexity:")) require.NotContains(t, base, sessionID) require.NotContains(t, base, "user-1") require.NotContains(t, base, "vk-1") - require.Equal(t, base, buildComplexitySessionKey(ctx, "vk-1", sessionID)) + require.Equal(t, base, complexitySessionKey(attributed(sessionID, "vk-1", "user-1"))) + + require.NotEqual(t, base, complexitySessionKey(attributed(sessionID, "vk-2", "user-1"))) + require.NotEqual(t, base, complexitySessionKey(attributed(sessionID, "vk-1", "user-2"))) + require.NotEqual(t, base, complexitySessionKey(attributed("another-session", "vk-1", "user-1"))) - require.NotEqual(t, base, buildComplexitySessionKey(ctx, "vk-2", sessionID)) - ctx.SetValue(schemas.BifrostContextKeyUserID, "user-2") - require.NotEqual(t, base, buildComplexitySessionKey(ctx, "vk-1", sessionID)) - require.NotEqual(t, base, buildComplexitySessionKey(ctx, "vk-1", "another-session")) + // A request nothing governs keys under the deployment, apart from any caller's scope. + ungoverned := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) + ungoverned.SetValue(schemas.BifrostContextKeySessionID, sessionID) + require.NotEqual(t, base, complexitySessionKey(ungoverned)) } func TestPublishLocalSessionFallbackPreservesProposalEvidence(t *testing.T) { diff --git a/plugins/routing/go.mod b/plugins/routing/go.mod index 9013417c1d3..6a3f1383c69 100644 --- a/plugins/routing/go.mod +++ b/plugins/routing/go.mod @@ -6,9 +6,9 @@ require ( github.com/blevesearch/go-porterstemmer v1.0.3 github.com/google/cel-go v0.30.0 github.com/google/uuid v1.6.0 - github.com/capsohq/bifrost/core v1.9.1 - github.com/capsohq/bifrost/framework v1.7.1 - github.com/capsohq/bifrost/plugins/governance v1.8.1 + github.com/capsohq/bifrost/core v1.10.2 + github.com/capsohq/bifrost/framework v1.7.4 + github.com/capsohq/bifrost/plugins/governance v1.8.3 github.com/stretchr/testify v1.11.1 gorm.io/gorm v1.31.1 ) diff --git a/plugins/routing/go.sum b/plugins/routing/go.sum index 6bfd0d8a268..e0e590925f7 100644 --- a/plugins/routing/go.sum +++ b/plugins/routing/go.sum @@ -268,12 +268,12 @@ github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsRe github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= github.com/mattn/go-sqlite3 v1.14.32 h1:JD12Ag3oLy1zQA+BNn74xRgaBbdhbNIDYvQUEuuErjs= github.com/mattn/go-sqlite3 v1.14.32/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= -github.com/maximhq/bifrost/core v1.9.1 h1:CF7TTj63htKQbuBL7eAGzou5yoe4XiQ0YMWpAO/atnw= -github.com/maximhq/bifrost/core v1.9.1/go.mod h1:bgTt5CsqB7duKiy/pC0uUMTxIRnvjj0t2OvH7Tlvuz0= -github.com/maximhq/bifrost/framework v1.7.1 h1:pGCdT7ksXmcnCylBpQqSHlQ071QAqgJ1XS7JwI8uV9U= -github.com/maximhq/bifrost/framework v1.7.1/go.mod h1:Acd4YNI+K8SjMPdghUQbjw0plJpqIDOP0s01iQr51Go= -github.com/maximhq/bifrost/plugins/governance v1.8.1 h1:PNOy5o2fo+R2g8bY6tr4LCx8CJv/8JmOTtNg2SIKUy4= -github.com/maximhq/bifrost/plugins/governance v1.8.1/go.mod h1:T+FoyatNH+qqqziXmGNiKqXZw8DB6neKZfu3tXiWojo= +github.com/maximhq/bifrost/core v1.10.2 h1:jRvvZN1KJVaX2Pxn9crTrmL3kW2QgDh7d1jsuz6b72k= +github.com/maximhq/bifrost/core v1.10.2/go.mod h1:bgTt5CsqB7duKiy/pC0uUMTxIRnvjj0t2OvH7Tlvuz0= +github.com/maximhq/bifrost/framework v1.7.4 h1:aM4MM3H3dXlPpkvyqRvt337WV8jl3m7DfBehAzs8HNU= +github.com/maximhq/bifrost/framework v1.7.4/go.mod h1:yHmcJhKvvaeoNUJYd5yV/wsNb3jZBwnjw2ny246C55Y= +github.com/maximhq/bifrost/plugins/governance v1.8.3 h1:LNnhrzjsEYGyFyR07nXou9XxIzSAP0RuIn0N/ovdEpk= +github.com/maximhq/bifrost/plugins/governance v1.8.3/go.mod h1:yr3Pmp1wnF/AETjUki7AZx8m3NQme1OPDpxF9ux7E0c= github.com/molecule-man/go-brrr v1.0.1 h1:cEjgx8hgNw6UGdhQ94SPDbPkKuRbkUcxBO3IzbGpA/o= github.com/molecule-man/go-brrr v1.0.1/go.mod h1:7ybW6/7gA3oKY45jOfVNjSJDtrr6ea4tzbsTkjmQDC4= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= diff --git a/plugins/routing/main.go b/plugins/routing/main.go index d711b8f90a3..237ba585ebb 100644 --- a/plugins/routing/main.go +++ b/plugins/routing/main.go @@ -19,6 +19,7 @@ import ( bifrost "github.com/capsohq/bifrost/core" "github.com/capsohq/bifrost/core/schemas" "github.com/capsohq/bifrost/framework/configstore" + configstoreTables "github.com/capsohq/bifrost/framework/configstore/tables" "github.com/capsohq/bifrost/framework/vectorstore" "github.com/capsohq/bifrost/plugins/routing/complexity" "github.com/capsohq/bifrost/plugins/routing/rules" @@ -482,7 +483,7 @@ func (p *RoutingPlugin) applyRoutingRules(ctx *schemas.BifrostContext, req *sche var computeComplexity func() *complexity.ComplexityResult if p.complexityAnalyzer.Load() != nil { computeComplexity = func() *complexity.ComplexityResult { - return p.computeComplexity(ctx, req, scope.VirtualKeyID) + return p.computeComplexity(ctx, req) } } @@ -526,19 +527,17 @@ func (p *RoutingPlugin) applyRoutingRules(ctx *schemas.BifrostContext, req *sche if len(decision.Fallbacks) > 0 { resolvedFallbacks := make([]schemas.Fallback, 0, len(decision.Fallbacks)) for _, fb := range decision.Fallbacks { - fbProvider, fbModel := schemas.ParseModelString(fb, "") - trimmedFbProvider := strings.TrimSpace(string(fbProvider)) - trimmedFbModel := strings.TrimSpace(fbModel) - if trimmedFbProvider == "" { + resolved := fb.Fallback + resolved.Provider = schemas.ModelProvider(strings.TrimSpace(string(resolved.Provider))) + resolved.Model = strings.TrimSpace(resolved.Model) + resolved.KeyID = strings.TrimSpace(resolved.KeyID) + if resolved.Provider == "" { continue } - if trimmedFbModel == "" && model != "" { - trimmedFbModel = model + if resolved.Model == "" && model != "" { + resolved.Model = model } - resolvedFallbacks = append(resolvedFallbacks, schemas.Fallback{ - Provider: schemas.ModelProvider(trimmedFbProvider), - Model: trimmedFbModel, - }) + resolvedFallbacks = append(resolvedFallbacks, resolved) } req.SetFallbacks(resolvedFallbacks) } @@ -552,7 +551,7 @@ func (p *RoutingPlugin) applyRoutingRules(ctx *schemas.BifrostContext, req *sche ctx.SetValue(schemas.BifrostContextKeyRoutingPinnedAPIKeyID, decision.KeyID) } - p.logger.Debug("[Routing] Applied routing decision: provider=%s, model=%s, keyID=%s, fallbacks=%v", decision.Provider, decision.Model, decision.KeyID, decision.Fallbacks) + p.logger.Debug("[Routing] Applied routing decision: provider=%s, model=%s, keyID=%s, fallbacks=%v", decision.Provider, decision.Model, decision.KeyID, configstoreTables.RoutingFallbackStrings(decision.Fallbacks)) return decision, nil } diff --git a/plugins/routing/routingpin_test.go b/plugins/routing/routingpin_test.go index c64c2369440..e83d003d69c 100644 --- a/plugins/routing/routingpin_test.go +++ b/plugins/routing/routingpin_test.go @@ -66,6 +66,57 @@ func TestApplyRoutingRules_PinnedKeyReachesContext(t *testing.T) { "routing-rule pinned key_id must reach BifrostContextKeyRoutingPinnedAPIKeyID that selectKeyFromProviderForModelWithPool reads") } +// TestApplyRoutingRules_FallbackKeyPinReachesRequest covers the fallback pin, which travels on the request's fallback list because core clears the context per attempt. +func TestApplyRoutingRules_FallbackKeyPinReachesRequest(t *testing.T) { + const fallbackKeyID = "fallback-key-xyz-789" + + store, err := rules.NewLocalStore(context.Background(), rules.NewMockLogger(), nil) + require.NoError(t, err) + require.NoError(t, store.UpsertRule(context.Background(), &configstoreTables.TableRoutingRule{ + ID: "fb-pin-1", + Name: "Pinned Fallback Rule", + CelExpression: "model == 'gpt-4o'", + Targets: []configstoreTables.TableRoutingTarget{ + {Provider: bifrost.Ptr("azure"), Model: bifrost.Ptr("gpt-4-turbo"), Weight: 1.0}, + }, + ParsedFallbacks: []configstoreTables.RoutingFallback{ + {Fallback: schemas.Fallback{Provider: "vertex", Model: "gemini-2.5-pro", KeyID: fallbackKeyID}}, + {Fallback: schemas.Fallback{Provider: "anthropic"}}, + }, + Enabled: bifrost.Ptr(true), + Scope: "global", + Priority: 0, + })) + + plugin, err := InitFromStore(context.Background(), nil, rules.NewMockLogger(), nil, store, NewMockGovernance()) + require.NoError(t, err) + + req := &schemas.BifrostRequest{ + RequestType: schemas.ChatCompletionRequest, + ChatRequest: &schemas.BifrostChatRequest{Provider: schemas.OpenAI, Model: "gpt-4o"}, + } + + root := schemas.NewBifrostContext(context.Background(), time.Now()) + root.BlockRestrictedWrites() + pluginName := PluginName + scoped := root.WithPluginScope(&pluginName) + + decision, err := plugin.applyRoutingRules(scoped, req, rules.GovernanceScope{}) + require.NoError(t, err) + require.NotNil(t, decision) + + fallbacks := req.ChatRequest.Fallbacks + require.Len(t, fallbacks, 2) + assert.Equal(t, schemas.ModelProvider("vertex"), fallbacks[0].Provider) + assert.Equal(t, fallbackKeyID, fallbacks[0].KeyID, + "a routing rule's fallback key_id must reach schemas.Fallback.KeyID, which core re-pins per attempt") + + // An unpinned fallback inherits the incoming model and stays load-balanced. + assert.Equal(t, schemas.ModelProvider("anthropic"), fallbacks[1].Provider) + assert.Equal(t, "gpt-4o", fallbacks[1].Model) + assert.Empty(t, fallbacks[1].KeyID) +} + // TestPreRequestHook_MaterializesVirtualKeyRoutingAfterRules pins the ordering this plugin // exists to guarantee: a matched rule rewrites the model, and both the provider allowlist and // the load balancer must then run against the rewritten model, not the one the caller sent. diff --git a/plugins/routing/rules/engine.go b/plugins/routing/rules/engine.go index c7d311ae603..df38f4aa678 100644 --- a/plugins/routing/rules/engine.go +++ b/plugins/routing/rules/engine.go @@ -28,12 +28,12 @@ type ScopeLevel struct { // Decision is the output of routing rule evaluation // Represents which provider/model to route to and fallback chain type Decision struct { - Provider string // Primary provider (e.g., "openai", "azure") - Model string // Model to use (or empty to use original) - KeyID string // Optional: pin a specific API key by UUID ("" = no pin) - Fallbacks []string // Fallback chain: ["provider/model", ...] - MatchedRuleID string // ID of the rule that matched - MatchedRuleName string // Name of the rule that matched + Provider string // Primary provider (e.g., "openai", "azure") + Model string // Model to use (or empty to use original) + KeyID string // Optional: pin a specific API key by UUID ("" = no pin) + Fallbacks []configstoreTables.RoutingFallback // Fallback chain, each optionally pinning a provider key + MatchedRuleID string // ID of the rule that matched + MatchedRuleName string // Name of the rule that matched } // GovernanceScope is who a request is governed as: the identifiers the access it carries was resolved @@ -60,7 +60,7 @@ type EvaluationContext struct { Provider schemas.ModelProvider // Current provider Model string // Current model RequestType string // Request type (e.g., "chat_completion", "embedding"); streaming requests carry a distinct "_stream" suffix (e.g., "chat_completion_stream") - Fallbacks []string // Fallback chain: ["provider/model", ...] + Fallbacks []configstoreTables.RoutingFallback // Fallback chain, each optionally pinning a provider key Headers map[string]string // Request headers for dynamic routing QueryParams map[string]string // Query parameters for dynamic routing BudgetAndRateLimitStatus *governance.BudgetAndRateLimitStatus // Budget and rate limit status by provider/model @@ -312,8 +312,8 @@ func (re *Engine) EvaluateRoutingRules(ctx *schemas.BifrostContext, routingCtx * if matchedRule.ChainRule { chainSuffix = " [chain_rule=true, continuing]" } - re.logger.Debug("[Engine] Rule matched! Selected target (weight=%.2f): provider=%s, model=%s, fallbacks=%v%s", matchedTargetWeight, stepDecision.Provider, stepDecision.Model, stepDecision.Fallbacks, chainSuffix) - ctx.AppendRoutingEngineLog(schemas.RoutingEngineRoutingRule, schemas.LogLevelInfo, fmt.Sprintf("Rule '%s' [%s] → matched, selected target (weight=%.2f): provider=%s, model=%s, fallbacks=%v%s", matchedRule.Name, matchedRule.CelExpression, matchedTargetWeight, stepDecision.Provider, stepDecision.Model, stepDecision.Fallbacks, chainSuffix)) + re.logger.Debug("[Engine] Rule matched! Selected target (weight=%.2f): provider=%s, model=%s, fallbacks=%v%s", matchedTargetWeight, stepDecision.Provider, stepDecision.Model, configstoreTables.RoutingFallbackStrings(stepDecision.Fallbacks), chainSuffix) + ctx.AppendRoutingEngineLog(schemas.RoutingEngineRoutingRule, schemas.LogLevelInfo, fmt.Sprintf("Rule '%s' [%s] → matched, selected target (weight=%.2f): provider=%s, model=%s, fallbacks=%v%s", matchedRule.Name, matchedRule.CelExpression, matchedTargetWeight, stepDecision.Provider, stepDecision.Model, configstoreTables.RoutingFallbackStrings(stepDecision.Fallbacks), chainSuffix)) // TERMINATION 2: Rule is terminal (chain_rule=false, the default). if !matchedRule.ChainRule { diff --git a/plugins/routing/rules/store_test.go b/plugins/routing/rules/store_test.go index b7a609f8e4c..9a0872aa9ed 100644 --- a/plugins/routing/rules/store_test.go +++ b/plugins/routing/rules/store_test.go @@ -6,6 +6,7 @@ import ( "time" bifrost "github.com/capsohq/bifrost/core" + "github.com/capsohq/bifrost/core/schemas" configstoreTables "github.com/capsohq/bifrost/framework/configstore/tables" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -28,7 +29,7 @@ func TestRuleStore_CreateAndRetrieve(t *testing.T) { {Provider: bifrost.Ptr("openai"), Model: bifrost.Ptr("gpt-4"), Weight: 1.0}, }, Fallbacks: nil, - ParsedFallbacks: []string{"azure/gpt-4-turbo"}, + ParsedFallbacks: []configstoreTables.RoutingFallback{{Fallback: schemas.Fallback{Provider: "azure", Model: "gpt-4-turbo"}}}, Scope: "global", ScopeID: nil, Priority: 10, @@ -48,7 +49,7 @@ func TestRuleStore_CreateAndRetrieve(t *testing.T) { {Provider: bifrost.Ptr("azure"), Weight: 1.0}, }, Fallbacks: nil, - ParsedFallbacks: []string{"groq/mixtral-8x7b"}, + ParsedFallbacks: []configstoreTables.RoutingFallback{{Fallback: schemas.Fallback{Provider: "groq", Model: "mixtral-8x7b"}}}, Scope: "team", ScopeID: &teamID, Priority: 20, diff --git a/plugins/routing/version b/plugins/routing/version index 524cb55242b..781dcb07cd8 100644 --- a/plugins/routing/version +++ b/plugins/routing/version @@ -1 +1 @@ -1.1.1 +1.1.3 diff --git a/plugins/semanticcache/go.mod b/plugins/semanticcache/go.mod index e24ec9ec06b..b6847fda3c2 100644 --- a/plugins/semanticcache/go.mod +++ b/plugins/semanticcache/go.mod @@ -5,9 +5,9 @@ go 1.27.0 require ( github.com/cespare/xxhash/v2 v2.3.0 github.com/google/uuid v1.6.0 - github.com/capsohq/bifrost/core v1.9.1 - github.com/capsohq/bifrost/framework v1.7.1 - github.com/capsohq/bifrost/plugins/mocker v1.6.4 + github.com/capsohq/bifrost/core v1.10.2 + github.com/capsohq/bifrost/framework v1.7.4 + github.com/capsohq/bifrost/plugins/mocker v1.6.6 ) require ( diff --git a/plugins/semanticcache/go.sum b/plugins/semanticcache/go.sum index 92fb4cca69a..14476503d70 100644 --- a/plugins/semanticcache/go.sum +++ b/plugins/semanticcache/go.sum @@ -181,12 +181,12 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/ github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= -github.com/maximhq/bifrost/core v1.9.1 h1:CF7TTj63htKQbuBL7eAGzou5yoe4XiQ0YMWpAO/atnw= -github.com/maximhq/bifrost/core v1.9.1/go.mod h1:bgTt5CsqB7duKiy/pC0uUMTxIRnvjj0t2OvH7Tlvuz0= -github.com/maximhq/bifrost/framework v1.7.1 h1:pGCdT7ksXmcnCylBpQqSHlQ071QAqgJ1XS7JwI8uV9U= -github.com/maximhq/bifrost/framework v1.7.1/go.mod h1:Acd4YNI+K8SjMPdghUQbjw0plJpqIDOP0s01iQr51Go= -github.com/maximhq/bifrost/plugins/mocker v1.6.4 h1:EYiF8Q0gULpgdlnmVECfrbteC1CJgkiHD88IvDNLWPY= -github.com/maximhq/bifrost/plugins/mocker v1.6.4/go.mod h1:Y+oZP7YAloPuznjJRwqWs6hUkNE26moCdl+xHbc25wg= +github.com/maximhq/bifrost/core v1.10.2 h1:jRvvZN1KJVaX2Pxn9crTrmL3kW2QgDh7d1jsuz6b72k= +github.com/maximhq/bifrost/core v1.10.2/go.mod h1:bgTt5CsqB7duKiy/pC0uUMTxIRnvjj0t2OvH7Tlvuz0= +github.com/maximhq/bifrost/framework v1.7.4 h1:aM4MM3H3dXlPpkvyqRvt337WV8jl3m7DfBehAzs8HNU= +github.com/maximhq/bifrost/framework v1.7.4/go.mod h1:yHmcJhKvvaeoNUJYd5yV/wsNb3jZBwnjw2ny246C55Y= +github.com/maximhq/bifrost/plugins/mocker v1.6.6 h1:oZjsCScTXLMRuGRxiztdpSzR3xO7xniC2phTEs/83Xo= +github.com/maximhq/bifrost/plugins/mocker v1.6.6/go.mod h1:iuR83+IPA4U54vs6fnrpooJH98pNXZCHBT1m3L/Wkz8= github.com/molecule-man/go-brrr v1.0.1 h1:cEjgx8hgNw6UGdhQ94SPDbPkKuRbkUcxBO3IzbGpA/o= github.com/molecule-man/go-brrr v1.0.1/go.mod h1:7ybW6/7gA3oKY45jOfVNjSJDtrr6ea4tzbsTkjmQDC4= github.com/oapi-codegen/runtime v1.1.1 h1:EXLHh0DXIJnWhdRPN2w4MXAzFyE4CskzhNLUmtpMYro= diff --git a/plugins/semanticcache/utils.go b/plugins/semanticcache/utils.go index 91f68a0e109..d41933c9213 100644 --- a/plugins/semanticcache/utils.go +++ b/plugins/semanticcache/utils.go @@ -827,7 +827,13 @@ func (plugin *Plugin) extractResponsesParametersToMetadata(params *schemas.Respo putIfSet(metadata, "max_tokens", params.MaxOutputTokens) putIfSet(metadata, "parallel_tool_calls", params.ParallelToolCalls) putIfSet(metadata, "background", params.Background) - putIfSet(metadata, "conversation", params.Conversation) + if params.Conversation != nil { + if params.Conversation.ResponsesResponseConversationStr != nil { + metadata["conversation"] = *params.Conversation.ResponsesResponseConversationStr + } else if params.Conversation.ResponsesResponseConversationStruct != nil { + metadata["conversation"] = params.Conversation.ResponsesResponseConversationStruct.ID + } + } putSortedSetIfNonEmpty(metadata, "include", params.Include) putIfSet(metadata, "instructions", params.Instructions) putIfSet(metadata, "max_tool_calls", params.MaxToolCalls) diff --git a/plugins/semanticcache/version b/plugins/semanticcache/version index 9edc58bb1dd..ec70f755607 100644 --- a/plugins/semanticcache/version +++ b/plugins/semanticcache/version @@ -1 +1 @@ -1.6.4 +1.6.6 diff --git a/plugins/telemetry/go.mod b/plugins/telemetry/go.mod index 2cde9e53773..79bf24a4b36 100644 --- a/plugins/telemetry/go.mod +++ b/plugins/telemetry/go.mod @@ -4,8 +4,8 @@ go 1.27.0 require ( github.com/bytedance/sonic v1.15.3-0.20260730064818-2a36d6da63e2 - github.com/capsohq/bifrost/core v1.9.1 - github.com/capsohq/bifrost/framework v1.7.1 + github.com/capsohq/bifrost/core v1.10.2 + github.com/capsohq/bifrost/framework v1.7.4 github.com/prometheus/client_golang v1.23.2 github.com/valyala/fasthttp v1.74.0 ) diff --git a/plugins/telemetry/go.sum b/plugins/telemetry/go.sum index 814f1fe3e21..8c99b5c54a5 100644 --- a/plugins/telemetry/go.sum +++ b/plugins/telemetry/go.sum @@ -264,10 +264,10 @@ github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsRe github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= github.com/mattn/go-sqlite3 v1.14.32 h1:JD12Ag3oLy1zQA+BNn74xRgaBbdhbNIDYvQUEuuErjs= github.com/mattn/go-sqlite3 v1.14.32/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= -github.com/maximhq/bifrost/core v1.9.1 h1:CF7TTj63htKQbuBL7eAGzou5yoe4XiQ0YMWpAO/atnw= -github.com/maximhq/bifrost/core v1.9.1/go.mod h1:bgTt5CsqB7duKiy/pC0uUMTxIRnvjj0t2OvH7Tlvuz0= -github.com/maximhq/bifrost/framework v1.7.1 h1:pGCdT7ksXmcnCylBpQqSHlQ071QAqgJ1XS7JwI8uV9U= -github.com/maximhq/bifrost/framework v1.7.1/go.mod h1:Acd4YNI+K8SjMPdghUQbjw0plJpqIDOP0s01iQr51Go= +github.com/maximhq/bifrost/core v1.10.2 h1:jRvvZN1KJVaX2Pxn9crTrmL3kW2QgDh7d1jsuz6b72k= +github.com/maximhq/bifrost/core v1.10.2/go.mod h1:bgTt5CsqB7duKiy/pC0uUMTxIRnvjj0t2OvH7Tlvuz0= +github.com/maximhq/bifrost/framework v1.7.4 h1:aM4MM3H3dXlPpkvyqRvt337WV8jl3m7DfBehAzs8HNU= +github.com/maximhq/bifrost/framework v1.7.4/go.mod h1:yHmcJhKvvaeoNUJYd5yV/wsNb3jZBwnjw2ny246C55Y= github.com/molecule-man/go-brrr v1.0.1 h1:cEjgx8hgNw6UGdhQ94SPDbPkKuRbkUcxBO3IzbGpA/o= github.com/molecule-man/go-brrr v1.0.1/go.mod h1:7ybW6/7gA3oKY45jOfVNjSJDtrr6ea4tzbsTkjmQDC4= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= diff --git a/plugins/telemetry/main.go b/plugins/telemetry/main.go index 743772af4cd..dfcb1090f36 100644 --- a/plugins/telemetry/main.go +++ b/plugins/telemetry/main.go @@ -1324,10 +1324,8 @@ func (p *PrometheusPlugin) PostLLMHook(ctx *schemas.BifrostContext, result *sche // Record latency duration := time.Since(startTime).Seconds() - latencyLabelValues := make([]string, 0, len(promLabelValues)+1) - latencyLabelValues = append(latencyLabelValues, promLabelValues[:len(p.defaultBifrostLabels)]...) // all default labels - latencyLabelValues = append(latencyLabelValues, strconv.FormatBool(bifrostErr == nil)) // is_success - latencyLabelValues = append(latencyLabelValues, promLabelValues[len(p.defaultBifrostLabels):]...) // then custom labels + // Default labels, then is_success, then custom labels. + latencyLabelValues := spliceLabelValues(promLabelValues, len(p.defaultBifrostLabels), strconv.FormatBool(bifrostErr == nil)) p.UpstreamLatencySeconds.WithLabelValues(latencyLabelValues...).Observe(duration) // SDK caller: no transport hooks fire, so this LLM-hook window is all there @@ -1352,11 +1350,8 @@ func (p *PrometheusPlugin) PostLLMHook(ctx *schemas.BifrostContext, result *sche // cannot disagree. Never empty: bifrostErr is non-nil in this branch. errorType := schemas.ClassifyErrorType(bifrostErr, requestType) - errorPromLabelValues := make([]string, 0, len(promLabelValues)+2) - errorPromLabelValues = append(errorPromLabelValues, promLabelValues[:len(p.defaultBifrostLabels)]...) // all default labels - errorPromLabelValues = append(errorPromLabelValues, statusCode) // status_code - errorPromLabelValues = append(errorPromLabelValues, string(errorType)) // error_type - errorPromLabelValues = append(errorPromLabelValues, promLabelValues[len(p.defaultBifrostLabels):]...) // then custom labels + // Default labels, then status_code and error_type, then custom labels. + errorPromLabelValues := spliceLabelValues(promLabelValues, len(p.defaultBifrostLabels), statusCode, string(errorType)) p.ErrorRequestsTotal.WithLabelValues(errorPromLabelValues...).Inc() } else { @@ -1473,11 +1468,8 @@ func (p *PrometheusPlugin) PostLLMHook(ctx *schemas.BifrostContext, result *sche cacheType = *extraFields.CacheDebug.HitType } - // Add cache_type to label values (create new slice to avoid modifying original) - cacheHitLabelValues := make([]string, 0, len(promLabelValues)+1) - cacheHitLabelValues = append(cacheHitLabelValues, promLabelValues[:len(p.defaultBifrostLabels)]...) // all default labels - cacheHitLabelValues = append(cacheHitLabelValues, cacheType) // cache_type - cacheHitLabelValues = append(cacheHitLabelValues, promLabelValues[len(p.defaultBifrostLabels):]...) // then custom labels + // Default labels, then cache_type, then custom labels (clone so the original is untouched). + cacheHitLabelValues := spliceLabelValues(promLabelValues, len(p.defaultBifrostLabels), cacheType) p.CacheHitsTotal.WithLabelValues(cacheHitLabelValues...).Inc() } diff --git a/plugins/telemetry/main_test.go b/plugins/telemetry/main_test.go index d299b0a9093..17c78d18bf2 100644 --- a/plugins/telemetry/main_test.go +++ b/plugins/telemetry/main_test.go @@ -6,6 +6,9 @@ import ( "io" "net/http" "net/http/httptest" + "reflect" + "slices" + "strings" "testing" "time" @@ -374,22 +377,36 @@ func TestMetricsEnabledGating(t *testing.T) { // TestMarshalConfigForStorageKeepsToggles guards the hand-maintained storage whitelist // (Config.MarshalForStorage's configStorage struct): a toggle added to Config must be -// added there too, or it is silently dropped on save and the UI reverts it. Regression -// test for overhead_breakdown_enabled, which was initially dropped this way. +// added there too, or it is silently dropped on save and the UI reverts it. Every *bool +// field on Config is enumerated by reflection so a new toggle cannot escape this test, +// which is how overhead_breakdown_enabled and later user_labels_enabled both regressed. func TestMarshalConfigForStorageKeepsToggles(t *testing.T) { p := newTestPlugin(t) - out, err := p.MarshalConfigForStorage(map[string]any{ - "overhead_breakdown_enabled": true, - "metrics_enabled": false, - }) - if err != nil { - t.Fatalf("MarshalConfigForStorage: %v", err) - } - if v, ok := out["overhead_breakdown_enabled"].(bool); !ok || !v { - t.Errorf("overhead_breakdown_enabled dropped by storage: got %v (%T), want true", out["overhead_breakdown_enabled"], out["overhead_breakdown_enabled"]) - } - if v, ok := out["metrics_enabled"].(bool); !ok || v { - t.Errorf("metrics_enabled = %v, want false to survive storage round-trip", out["metrics_enabled"]) + ct := reflect.TypeOf(Config{}) + for i := 0; i < ct.NumField(); i++ { + f := ct.Field(i) + if f.Type.Kind() != reflect.Ptr || f.Type.Elem().Kind() != reflect.Bool { + continue + } + key := strings.Split(f.Tag.Get("json"), ",")[0] + if key == "" || key == "-" { + t.Fatalf("Config.%s is a toggle with no json tag", f.Name) + } + // Both values: omitempty would hide a dropped field if only false were sent. + for _, want := range []bool{true, false} { + out, err := p.MarshalConfigForStorage(map[string]any{key: want}) + if err != nil { + t.Fatalf("MarshalConfigForStorage(%s=%v): %v", key, want, err) + } + got, ok := out[key].(bool) + if !ok { + t.Errorf("%s dropped by storage whitelist (Config.%s): got %v (%T), want %v", key, f.Name, out[key], out[key], want) + continue + } + if got != want { + t.Errorf("%s = %v after storage round-trip, want %v", key, got, want) + } + } } // user_labels_enabled (upstream #7267): true and false survive, omitted stays omitted. @@ -402,7 +419,7 @@ func TestMarshalConfigForStorageKeepsToggles(t *testing.T) { t.Errorf("user_labels_enabled = %v (%T), want %v to survive storage", out["user_labels_enabled"], out["user_labels_enabled"], want) } } - out, err = p.MarshalConfigForStorage(map[string]any{"metrics_enabled": true}) + out, err := p.MarshalConfigForStorage(map[string]any{"metrics_enabled": true}) if err != nil { t.Fatalf("MarshalConfigForStorage: %v", err) } @@ -622,3 +639,64 @@ func TestApplyCustomLabels(t *testing.T) { }) } } + +// TestSpliceLabelValues pins the final label-value ordering produced for the latency, +// error, and cache-hit metrics: default labels first, then the metric's extra values +// (is_success / status_code+error_type / cache_type), then the custom labels. Metric +// vectors match values to label names purely by position, so any reordering here +// silently attributes values to the wrong labels. +func TestSpliceLabelValues(t *testing.T) { + defaults := []string{"openai", "gpt-4o", "chat"} // provider, model, method + custom := []string{"team-a", "env-prod"} // custom dimension labels + base := append(append([]string{}, defaults...), custom...) + + tests := []struct { + name string + in []string + extras []string + want []string + }{ + { + name: "latency is_success between defaults and custom labels", + in: base, + extras: []string{"true"}, + want: []string{"openai", "gpt-4o", "chat", "true", "team-a", "env-prod"}, + }, + { + name: "error status_code and error_type keep their order", + in: base, + extras: []string{"429", "rate_limit"}, + want: []string{"openai", "gpt-4o", "chat", "429", "rate_limit", "team-a", "env-prod"}, + }, + { + name: "cache_type between defaults and custom labels", + in: base, + extras: []string{"semantic"}, + want: []string{"openai", "gpt-4o", "chat", "semantic", "team-a", "env-prod"}, + }, + { + name: "no custom labels appends extras at the end", + in: defaults, + extras: []string{"true"}, + want: []string{"openai", "gpt-4o", "chat", "true"}, + }, + { + name: "no extras returns the input unchanged", + in: base, + extras: nil, + want: []string{"openai", "gpt-4o", "chat", "team-a", "env-prod"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + before := slices.Clone(tt.in) + got := spliceLabelValues(tt.in, len(defaults), tt.extras...) + if !slices.Equal(got, tt.want) { + t.Errorf("spliceLabelValues() = %v, want %v", got, tt.want) + } + if !slices.Equal(tt.in, before) { + t.Errorf("input mutated: %v, was %v", tt.in, before) + } + }) + } +} diff --git a/plugins/telemetry/utils.go b/plugins/telemetry/utils.go index 56a72b6dde3..b8d28c1226c 100644 --- a/plugins/telemetry/utils.go +++ b/plugins/telemetry/utils.go @@ -6,6 +6,7 @@ package telemetry import ( "log" "math" + "slices" "strings" schemas "github.com/capsohq/bifrost/core/schemas" @@ -62,6 +63,14 @@ func collectPrometheusKeyValues(ctx *fasthttp.RequestCtx) map[string]string { return labelValues } +// spliceLabelValues returns promLabelValues with extras inserted between the default +// labels (the first nDefault values) and the custom labels that follow them, matching +// metric definitions that declare default labels, then extras, then custom labels. +// The input slice is never mutated. +func spliceLabelValues(promLabelValues []string, nDefault int, extras ...string) []string { + return slices.Insert(slices.Clone(promLabelValues), nDefault, extras...) +} + // safeObserve safely records a value in a Prometheus histogram. // It prevents recording invalid values (negative or infinite) that could cause issues. func safeObserve(histogram *prometheus.HistogramVec, value float64, labels ...string) { diff --git a/plugins/telemetry/version b/plugins/telemetry/version index 27f9cd322bb..53adb84c822 100644 --- a/plugins/telemetry/version +++ b/plugins/telemetry/version @@ -1 +1 @@ -1.8.0 +1.8.2 diff --git a/tests/cmd/e2eseed/go.mod b/tests/cmd/e2eseed/go.mod index 74eb29e7d9a..7f5d4693056 100644 --- a/tests/cmd/e2eseed/go.mod +++ b/tests/cmd/e2eseed/go.mod @@ -53,7 +53,7 @@ require ( github.com/bytedance/gopkg v0.1.3 // indirect github.com/bytedance/sonic v1.15.3-0.20260730064818-2a36d6da63e2 // indirect github.com/bytedance/sonic/loader v0.5.2 // indirect - github.com/capsohq/bifrost/core v1.9.1 // indirect + github.com/capsohq/bifrost/core v1.10.2 // indirect github.com/capsohq/bifrost/framework v1.3.16 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudwego/base64x v0.1.6 // indirect diff --git a/tests/cmd/seed/go.mod b/tests/cmd/seed/go.mod index 872a9e74787..fdbba4d1f1e 100644 --- a/tests/cmd/seed/go.mod +++ b/tests/cmd/seed/go.mod @@ -8,7 +8,7 @@ replace ( ) require ( - github.com/capsohq/bifrost/core v1.9.1 + github.com/capsohq/bifrost/core v1.10.2 github.com/capsohq/bifrost/framework v1.3.16 gorm.io/driver/postgres v1.6.0 gorm.io/driver/sqlite v1.6.0 diff --git a/tests/cmd/seedvks/go.mod b/tests/cmd/seedvks/go.mod index 5c36f9d00f6..dcc02963f5b 100644 --- a/tests/cmd/seedvks/go.mod +++ b/tests/cmd/seedvks/go.mod @@ -8,7 +8,7 @@ replace ( ) require ( - github.com/capsohq/bifrost/core v1.9.1 + github.com/capsohq/bifrost/core v1.10.2 github.com/capsohq/bifrost/framework v1.3.16 github.com/google/uuid v1.6.0 gorm.io/driver/postgres v1.6.0 diff --git a/tests/e2e/api/README.md b/tests/e2e/api/README.md index 31dd6e81ce1..6c45b293dc8 100644 --- a/tests/e2e/api/README.md +++ b/tests/e2e/api/README.md @@ -31,8 +31,11 @@ newman run tests/e2e/api/collections/provider-harness.json \ ``` The fixture emits metadata followed by an error for `preamble-error`, then returns -`hello` for the configured fallback, `preamble-success`. Both Chat Completions and -Responses must return only the successful attempt's events and a terminal result. +`hello` for the configured fallback, `preamble-success`. Azure paths (`/openai/v1/*`) +fail with a rate limit; direct OpenAI paths (`/v1/*`) fail with `server_is_overloaded` +and no HTTP status, as a real overloaded OpenAI stream does. For both providers, Chat +Completions and Responses must return only the successful attempt's events and a +terminal result. The cases are skipped unless `azureStreamPreambleFixture=1`. Stop the fixture and isolated gateway with Ctrl+C after testing. diff --git a/tests/e2e/api/collections/bifrost-api-management.postman_collection.json b/tests/e2e/api/collections/bifrost-api-management.postman_collection.json index 7abdd2e0c5a..6a80c7b38c8 100644 --- a/tests/e2e/api/collections/bifrost-api-management.postman_collection.json +++ b/tests/e2e/api/collections/bifrost-api-management.postman_collection.json @@ -3783,6 +3783,271 @@ } ] }, + { + "name": "Governance - VK Allow All Providers", + "description": "A virtual key with allow_all_providers grants every provider, including ones it holds no provider_config for; the configs it does hold are overrides rather than the whole grant. The listing routes used to answer from the provider_config rows alone, so such a key saw only the providers it happened to override and every other provider vanished from /api/models*, /v1/models and the VK model pickers, while inference on them kept working.", + "item": [ + { + "name": "Baseline - catalog sizes with no VK", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('Baseline listing returns 200', function () { pm.response.to.have.status(200); });", + "// Parse defensively: an SPA fallback, a 401/403 or any non-JSON body makes an unguarded", + "// pm.response.json() throw a script error, which aborts the run instead of failing a", + "// named assertion and strands every request after this one.", + "var body = null;", + "try { body = pm.response.json(); } catch (e) { body = null; }", + "var models = (body && body.models) || [];", + "var counts = {};", + "models.forEach(function (m) { if (m && m.provider) counts[m.provider] = (counts[m.provider] || 0) + 1; });", + "pm.collectionVariables.set('vkall_base_anthropic', String(counts.anthropic || 0));", + "pm.collectionVariables.set('vkall_base_openai', String(counts.openai || 0));", + "// The scenario needs a provider to leave unnamed and one to override; without both", + "// configured there is nothing to tell apart and the assertions below would pass vacuously.", + "pm.test('anthropic is configured on this server', function () { pm.expect(counts.anthropic || 0).to.be.above(0); });", + "pm.test('openai is configured on this server', function () { pm.expect(counts.openai || 0).to.be.above(1); });" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{base_url}}/api/models/details?limit=5000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models", + "details" + ], + "query": [ + { + "key": "limit", + "value": "5000" + } + ] + } + } + }, + { + "name": "Create VK (allow_all_providers, openai overridden)", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var ts = Date.now();", + "var body = {", + " name: 'VK AllowAllProviders ' + ts,", + " is_active: true,", + " allow_all_providers: true,", + " // The only provider_config: openai is narrowed to one model. Every other provider is", + " // granted by the flag alone and names no config at all.", + " provider_configs: [", + " { provider: 'openai', weight: 1, allowed_models: ['gpt-4o-mini'], key_ids: ['*'] }", + " ]", + "};", + "pm.request.body.raw = JSON.stringify(body);" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var code = pm.response.code;", + "pm.test('Create VK returns 2xx', function () { pm.expect(code).to.be.within(200, 299); });", + "if (code >= 200 && code <= 299) {", + " var json = null;", + " try { json = pm.response.json(); } catch (e) { json = null; }", + " var vk = (json && (json.virtual_key || json)) || {};", + " if (vk && vk.id) pm.collectionVariables.set('vkall_id', vk.id);", + " if (vk && vk.value) pm.collectionVariables.set('vkall_value', vk.value);", + " pm.test('allow_all_providers round-trips', function () { pm.expect(vk.allow_all_providers).to.eql(true); });", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{}" + }, + "url": { + "raw": "{{base_url}}/api/governance/virtual-keys", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "governance", + "virtual-keys" + ] + } + } + }, + { + "name": "/api/models/details - unnamed provider is granted in full, overridden one stays narrowed", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('GET /api/models/details with VK returns 200', function () { pm.response.to.have.status(200); });", + "// Parse defensively: an SPA fallback, a 401/403 or any non-JSON body makes an unguarded", + "// pm.response.json() throw a script error, which aborts the run instead of failing a", + "// named assertion and strands every request after this one.", + "var body = null;", + "try { body = pm.response.json(); } catch (e) { body = null; }", + "var models = (body && body.models) || [];", + "var counts = {};", + "models.forEach(function (m) { if (m && m.provider) counts[m.provider] = (counts[m.provider] || 0) + 1; });", + "var baseAnthropic = Number(pm.collectionVariables.get('vkall_base_anthropic'));", + "var baseOpenai = Number(pm.collectionVariables.get('vkall_base_openai'));", + "// The regression: anthropic has no provider_config, so it is granted by the flag alone.", + "// Answering from the configs alone dropped it entirely.", + "pm.test('anthropic is listed though the VK names no config for it', function () {", + " pm.expect(counts.anthropic || 0).to.be.above(0);", + "});", + "pm.test('anthropic is granted in full, not narrowed', function () {", + " pm.expect(counts.anthropic).to.eql(baseAnthropic);", + "});", + "// And the other half: a config the VK does hold still narrows its provider, so the fix", + "// widens nothing it should not.", + "pm.test('openai stays narrowed by its own config', function () {", + " pm.expect(counts.openai || 0).to.be.below(baseOpenai);", + "});" + ] + } + } + ], + "request": { + "method": "GET", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkall_value}}" + } + ], + "url": { + "raw": "{{base_url}}/api/models/details?limit=5000", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "models", + "details" + ], + "query": [ + { + "key": "limit", + "value": "5000" + } + ] + } + } + }, + { + "name": "/v1/models - fan-out reaches the unnamed provider", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('GET /v1/models with VK returns 200', function () { pm.response.to.have.status(200); });", + "// Parse defensively: an SPA fallback, a 401/403 or any non-JSON body makes an unguarded", + "// pm.response.json() throw a script error, which aborts the run instead of failing a", + "// named assertion and strands every request after this one.", + "var body = null;", + "try { body = pm.response.json(); } catch (e) { body = null; }", + "var data = (body && body.data) || [];", + "var seen = {};", + "data.forEach(function (m) {", + " var id = (m && m.id) || '';", + " var idx = id.indexOf('/');", + " if (idx > 0) seen[id.substring(0, idx)] = true;", + "});", + "// The fan-out narrows to the same answer, so an allow-all key must not be pinned to the", + "// providers it happens to override.", + "pm.test('/v1/models includes the unnamed provider', function () {", + " pm.expect(Object.keys(seen)).to.include('anthropic');", + "});" + ] + } + } + ], + "request": { + "method": "GET", + "header": [ + { + "key": "x-bf-vk", + "value": "{{vkall_value}}" + } + ], + "url": { + "raw": "{{base_url}}/v1/models", + "host": [ + "{{base_url}}" + ], + "path": [ + "v1", + "models" + ] + } + } + }, + { + "name": "Teardown - Delete VK", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('Delete VK returns 2xx', function () { pm.expect(pm.response.code).to.be.within(200, 299); });" + ] + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{base_url}}/api/governance/virtual-keys/{{vkall_id}}", + "host": [ + "{{base_url}}" + ], + "path": [ + "api", + "governance", + "virtual-keys", + "{{vkall_id}}" + ] + } + } + } + ] + }, { "name": "Governance - VK Regex Entries", "description": "regex: entries in the existing model lists (allowed_models / blacklisted_models on a VK provider config, models / blacklisted_models on a provider key): the text after the prefix is an RE2 pattern matched case-insensitively as a full match against the model name, next to exact names in the same list. Sections: write-time validation (400 on an unparsable pattern or regex:* on VK and provider-key endpoints), a base VK exercised on /v1/models plus real completions (family allowed, preview blocked, other family denied), a PUT that mixes an exact name with a regex entry, and a provider-key round-trip.", diff --git a/tests/e2e/api/collections/provider-harness.json b/tests/e2e/api/collections/provider-harness.json index 6d5888a99de..6499ad48b1b 100644 --- a/tests/e2e/api/collections/provider-harness.json +++ b/tests/e2e/api/collections/provider-harness.json @@ -59,7 +59,8 @@ "// Only register the content-shape test when the request actually succeeded.", "// Otherwise newman reports it as passing (because of an early return) which is misleading next to a failing status.", "var __u = (pm.request && pm.request.url && pm.request.url.toString()) || '';", - "var __skipShape = (__u.indexOf('/files/') !== -1 && (__u.indexOf('/content?') !== -1 || __u.slice(-8) === '/content')) || __u.indexOf('storage.googleapis.com') !== -1 || __u.indexOf('batchPredictionJobs') !== -1 || (__u.indexOf('/genai/v1beta/batches/') !== -1 && __u.indexOf(':cancel') !== -1);", + "// Management API responses (/api/...) carry no inference content, so the shape check does not apply to them; those rows assert their own bodies.", + "var __skipShape = __u.indexOf('/api/') !== -1 || (__u.indexOf('/files/') !== -1 && (__u.indexOf('/content?') !== -1 || __u.slice(-8) === '/content')) || __u.indexOf('storage.googleapis.com') !== -1 || __u.indexOf('batchPredictionJobs') !== -1 || (__u.indexOf('/genai/v1beta/batches/') !== -1 && __u.indexOf(':cancel') !== -1);", "if (pm.response.code < 400 && !__skipShape && (pm.response.headers.get('content-type') || '').indexOf('text/event-stream') === -1 && (pm.response.headers.get('content-type') || '').indexOf('vnd.amazon.eventstream') === -1 && (pm.response.headers.get('content-type') || '').indexOf('audio/') === -1) {", " pm.test('Response has content (text or tool_use)', function () {", " var j;", @@ -72,7 +73,10 @@ " }", " var hasContent = false;", " var shape = 'unknown';", - " if (j.message && typeof j.message === 'object' && (j.message.content !== undefined || Array.isArray(j.message.tool_calls))) {", + " if (j.answers && typeof j.answers === 'object' && !Array.isArray(j.answers)) {", + " shape = 'decisions';", + " hasContent = Object.keys(j.answers).length > 0;", + " } else if (j.message && typeof j.message === 'object' && (j.message.content !== undefined || Array.isArray(j.message.tool_calls))) {", " shape = 'cohere-v2-chat';", " var cohereContent = j.message.content;", " var cohereHasText = typeof cohereContent === 'string' ? cohereContent.length > 0 : Array.isArray(cohereContent) && cohereContent.some(function (c) { return c && ((c.type === 'text' && c.text) || (c.type === 'thinking' && c.thinking)); });", @@ -16764,6 +16768,1517 @@ { "name": "Anthropic Beta Headers", "item": [ + { + "name": "12.B15 Beta header derived from request -> cache-diagnosis-2026-04-07", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var txt = pm.response.text() || '';", + "var body = {};", + "try { body = pm.response.json() || {}; } catch (e) { body = {}; }", + "var rr = (body.extra_fields || {}).raw_request;", + "if (typeof rr === 'string') { try { rr = JSON.parse(rr); } catch (e) { rr = null; } }", + "var wire = JSON.stringify(rr || null);", + "", + "// cache-diagnosis-2026-04-07 had ZERO coverage anywhere in this collection before this row.", + "//", + "// The client sends no anthropic-beta: Bifrost must derive the header from", + "// the diagnostics field and inject it. Anthropic refuses the gated field without it,", + "// so the status is the provider confirming what went out. Outbound headers are not", + "// visible here (ExtraFields carries raw_request only), which makes the status the", + "// strongest signal available on a header-based provider.", + "pm.test('cache-diagnosis-2026-04-07 is injected by Bifrost (200 = Anthropic accepted the gated field)', function () {", + " pm.expect(pm.response.code, 'want 200; a 4xx means the beta header was not injected, and any 5xx is a real failure too: ' + txt.slice(0, 400)).to.eql(200);", + "});", + "", + "// The field the header exists for must itself survive to the provider.", + "pm.test('cache-diagnosis-2026-04-07: the gated field reaches the provider', function () {", + " pm.expect(rr, 'raw_request missing or unparseable - x-bf-send-back-raw-request not honored: ' + txt.slice(0, 300)).to.be.an('object');", + " pm.expect(rr.diagnostics, 'the diagnostics field was dropped before the provider: ' + wire.slice(0, 400)).to.be.an('object');", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-api-key", + "value": "{{anthropicKey}}" + }, + { + "key": "anthropic-version", + "value": "2023-06-01" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"claude-haiku-4-5\",\n \"max_tokens\": 256,\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"diagnostics\": {\n \"previous_message_id\": null\n }\n}" + }, + "url": { + "raw": "{{baseUrl}}/anthropic/v1/messages", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "anthropic", + "v1", + "messages" + ] + } + } + }, + { + "name": "12.B16 Beta header derived from request -> context-management-2025-06-27", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var txt = pm.response.text() || '';", + "var body = {};", + "try { body = pm.response.json() || {}; } catch (e) { body = {}; }", + "var rr = (body.extra_fields || {}).raw_request;", + "if (typeof rr === 'string') { try { rr = JSON.parse(rr); } catch (e) { rr = null; } }", + "var wire = JSON.stringify(rr || null);", + "", + "// context-management-2025-06-27 had ZERO coverage anywhere in this collection before this row.", + "//", + "// The client sends no anthropic-beta: Bifrost must derive the header from", + "// context_management.edits[].type = clear_tool_uses_20250919 and inject it. Anthropic refuses the gated field without it,", + "// so the status is the provider confirming what went out. Outbound headers are not", + "// visible here (ExtraFields carries raw_request only), which makes the status the", + "// strongest signal available on a header-based provider.", + "pm.test('context-management-2025-06-27 is injected by Bifrost (200 = Anthropic accepted the gated field)', function () {", + " pm.expect(pm.response.code, 'want 200; a 4xx means the beta header was not injected, and any 5xx is a real failure too: ' + txt.slice(0, 400)).to.eql(200);", + "});", + "", + "// The field the header exists for must itself survive to the provider.", + "pm.test('context-management-2025-06-27: the gated field reaches the provider', function () {", + " pm.expect(rr, 'raw_request missing or unparseable - x-bf-send-back-raw-request not honored: ' + txt.slice(0, 300)).to.be.an('object');", + " pm.expect(rr.context_management, 'context_management was dropped before the provider: ' + wire.slice(0, 400)).to.be.an('object');", + " pm.expect(rr.context_management.edits[0].type, 'the clear-tool-uses edit was altered on the wire: ' + JSON.stringify(rr.context_management).slice(0, 300)).to.eql('clear_tool_uses_20250919');", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-api-key", + "value": "{{anthropicKey}}" + }, + { + "key": "anthropic-version", + "value": "2023-06-01" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"claude-haiku-4-5\",\n \"max_tokens\": 256,\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"context_management\": {\n \"edits\": [\n {\n \"type\": \"clear_tool_uses_20250919\"\n }\n ]\n }\n}" + }, + "url": { + "raw": "{{baseUrl}}/anthropic/v1/messages", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "anthropic", + "v1", + "messages" + ] + } + } + }, + { + "name": "12.B17 Beta header derived from request -> mcp-client-2025-11-20", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var txt = pm.response.text() || '';", + "var body = {};", + "try { body = pm.response.json() || {}; } catch (e) { body = {}; }", + "var rr = (body.extra_fields || {}).raw_request;", + "if (typeof rr === 'string') { try { rr = JSON.parse(rr); } catch (e) { rr = null; } }", + "var wire = JSON.stringify(rr || null);", + "", + "// mcp-client-2025-11-20 had ZERO coverage anywhere in this collection before this row.", + "//", + "// The client sends no anthropic-beta: Bifrost must derive the header from", + "// a non-empty mcp_servers array and inject it. Anthropic refuses the gated field without it,", + "// so the status is the provider confirming what went out. Outbound headers are not", + "// visible here (ExtraFields carries raw_request only), which makes the status the", + "// strongest signal available on a header-based provider.", + "pm.test('mcp-client-2025-11-20 is injected by Bifrost (200 = Anthropic accepted the gated field)', function () {", + " pm.expect(pm.response.code, 'want 200; a 4xx means the beta header was not injected, and any 5xx is a real failure too: ' + txt.slice(0, 400)).to.eql(200);", + "});", + "", + "// The field the header exists for must itself survive to the provider.", + "pm.test('mcp-client-2025-11-20: the gated field reaches the provider', function () {", + " pm.expect(rr, 'raw_request missing or unparseable - x-bf-send-back-raw-request not honored: ' + txt.slice(0, 300)).to.be.an('object');", + " pm.expect(rr.mcp_servers, 'mcp_servers was dropped before the provider: ' + wire.slice(0, 400)).to.be.an('array');", + " pm.expect(rr.mcp_servers[0].name, 'the MCP server definition was altered on the wire: ' + JSON.stringify(rr.mcp_servers).slice(0, 300)).to.eql('harness_mcp');", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-api-key", + "value": "{{anthropicKey}}" + }, + { + "key": "anthropic-version", + "value": "2023-06-01" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"claude-haiku-4-5\",\n \"max_tokens\": 256,\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"mcp_servers\": [\n {\n \"type\": \"url\",\n \"url\": \"https://mcp.deepwiki.com/mcp\",\n \"name\": \"harness_mcp\"\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/anthropic/v1/messages", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "anthropic", + "v1", + "messages" + ] + } + } + }, + { + "name": "12.B18 Beta header derived from request -> server-side-fallback-2026-07-01 (default routing)", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var txt = pm.response.text() || '';", + "var body = {};", + "try { body = pm.response.json() || {}; } catch (e) { body = {}; }", + "var rr = (body.extra_fields || {}).raw_request;", + "if (typeof rr === 'string') { try { rr = JSON.parse(rr); } catch (e) { rr = null; } }", + "var wire = JSON.stringify(rr || null);", + "", + "// server-side-fallback-2026-07-01 had ZERO coverage anywhere in this collection before this row.", + "//", + "// The client sends no anthropic-beta: Bifrost must derive the header from", + "// fallbacks set to the bare string \"default\", which needs the superset header rather than server-side-fallback-2026-06-01 and inject it. Anthropic refuses the gated field without it,", + "// so the status is the provider confirming what went out. Outbound headers are not", + "// visible here (ExtraFields carries raw_request only), which makes the status the", + "// strongest signal available on a header-based provider.", + "pm.test('server-side-fallback-2026-07-01 is injected by Bifrost (200 = Anthropic accepted the gated field)', function () {", + " pm.expect(pm.response.code, 'want 200; a 4xx means the beta header was not injected, and any 5xx is a real failure too: ' + txt.slice(0, 400)).to.eql(200);", + "});", + "", + "// The field the header exists for must itself survive to the provider.", + "pm.test('server-side-fallback-2026-07-01: the gated field reaches the provider', function () {", + " pm.expect(rr, 'raw_request missing or unparseable - x-bf-send-back-raw-request not honored: ' + txt.slice(0, 300)).to.be.an('object');", + " pm.expect(rr.fallbacks, 'the fallbacks preset was dropped before the provider: ' + wire.slice(0, 400)).to.eql('default');", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-api-key", + "value": "{{anthropicKey}}" + }, + { + "key": "anthropic-version", + "value": "2023-06-01" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"claude-opus-5\",\n \"max_tokens\": 256,\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"fallbacks\": \"default\"\n}" + }, + "url": { + "raw": "{{baseUrl}}/anthropic/v1/messages", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "anthropic", + "v1", + "messages" + ] + } + } + }, + { + "name": "12.B9 Beta header derived from tools -> bedrock invoke path (anthropic_beta asserted in body)", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var txt = pm.response.text() || '';", + "var body = {};", + "try { body = pm.response.json() || {}; } catch (e) { body = {}; }", + "var rr = (body.extra_fields || {}).raw_request;", + "if (typeof rr === 'string') { try { rr = JSON.parse(rr); } catch (e) { rr = null; } }", + "var wire = JSON.stringify(rr || null);", + "", + "// Bedrock normally serves Anthropic models through CONVERSE, whose outbound body is", + "// {messages, inferenceConfig, toolConfig.tools[].toolSpec} and carries no anthropic_beta", + "// at all -- an earlier version of this row asserted against Converse and failed for that", + "// reason, not because the header logic was wrong.", + "//", + "// tools[].defer_loading is one of the triggers in responsesUsesAnthropicInvokePath", + "// (bedrock.go:4446), which routes the request to InvokeModel instead. That path reuses", + "// the Anthropic builder end to end, so the body is Anthropic-shaped AND carries the", + "// derived beta list as anthropic_beta (requestbuilder.go:488). This row therefore", + "// asserts the DERIVED HEADER ITSELF on the wire, which no header-based provider allows,", + "// since ExtraFields exposes raw_request and never the outbound HTTP headers.", + "pm.test('bedrock invoke path: request succeeds', function () {", + " pm.expect(pm.response.code, 'want 200; a 4xx means a gated field went out without its beta header, and any 5xx is a real failure too: ' + txt.slice(0, 400)).to.eql(200);", + "});", + "", + "pm.test('bedrock: anthropic_beta on the wire carries tool-search-tool-2025-10-19', function () {", + " pm.expect(rr, 'raw_request missing or unparseable - x-bf-send-back-raw-request not honored: ' + txt.slice(0, 300)).to.be.an('object');", + " pm.expect(rr.anthropic_beta, 'anthropic_beta is absent, which means the request went via Converse rather than InvokeModel: ' + wire.slice(0, 500)).to.be.an('array');", + " pm.expect(rr.anthropic_beta, 'Bifrost did not derive tool-search-tool-2025-10-19 from tools[].defer_loading: ' + JSON.stringify(rr.anthropic_beta)).to.include('tool-search-tool-2025-10-19');", + "});", + "", + "// The probe strips input_schema and description from a LOCAL copy before decoding. If that", + "// ever reached the wire, tools would arrive schemaless and tool calling would break.", + "pm.test('bedrock: tool feature, schema and description reach the provider intact', function () {", + " pm.expect(rr, 'raw_request missing: ' + txt.slice(0, 300)).to.be.an('object');", + " var tools = rr.tools || [];", + " pm.expect(tools.length, 'expected 2 tools on the Anthropic-shaped body: ' + wire.slice(0, 500)).to.eql(2);", + " var probe = null;", + " tools.forEach(function (t) { if (t && t.name === 'harness_probe') { probe = t; } });", + " pm.expect(probe, 'the probe tool vanished from the outbound body: ' + wire.slice(0, 400)).to.be.an('object');", + " pm.expect(probe.defer_loading, 'tools[].defer_loading was dropped before the provider: ' + JSON.stringify(probe).slice(0, 400)).to.eql(true);", + " pm.expect(probe.description, 'tools[].description was stripped from the outbound body: ' + wire.slice(0, 400)).to.eql('Harness probe tool');", + " pm.expect(probe.input_schema, 'tools[].input_schema was stripped from the outbound body: ' + wire.slice(0, 400)).to.be.an('object');", + " pm.expect(probe.input_schema.properties.q.type, 'the nested schema was altered on the wire: ' + JSON.stringify(probe.input_schema).slice(0, 300)).to.eql('string');", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-api-key", + "value": "{{anthropicKey}}" + }, + { + "key": "anthropic-version", + "value": "2023-06-01" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"bedrock/global.anthropic.claude-sonnet-4-6\",\n \"max_tokens\": 256,\n \"tools\": [\n {\n \"name\": \"harness_probe\",\n \"description\": \"Harness probe tool\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"q\": {\n \"type\": \"string\"\n }\n }\n },\n \"defer_loading\": true\n },\n {\n \"name\": \"plain_tool\",\n \"description\": \"Plain tool\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"q\": {\n \"type\": \"string\"\n }\n }\n }\n }\n ],\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/anthropic/v1/messages", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "anthropic", + "v1", + "messages" + ] + } + } + }, + { + "name": "12.B13 Beta header derived from tools -> tool-examples-2025-10-29 (Bedrock narrow variant, invoke path)", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var txt = pm.response.text() || '';", + "var body = {};", + "try { body = pm.response.json() || {}; } catch (e) { body = {}; }", + "var rr = (body.extra_fields || {}).raw_request;", + "if (typeof rr === 'string') { try { rr = JSON.parse(rr); } catch (e) { rr = null; } }", + "var wire = JSON.stringify(rr || null);", + "", + "// Bedrock normally serves Anthropic models through CONVERSE, whose outbound body is", + "// {messages, inferenceConfig, toolConfig.tools[].toolSpec} and carries no anthropic_beta", + "// at all -- an earlier version of this row asserted against Converse and failed for that", + "// reason, not because the header logic was wrong.", + "//", + "// tools[].defer_loading is one of the triggers in responsesUsesAnthropicInvokePath", + "// (bedrock.go:4446), which routes the request to InvokeModel instead. That path reuses", + "// the Anthropic builder end to end, so the body is Anthropic-shaped AND carries the", + "// derived beta list as anthropic_beta (requestbuilder.go:488). This row therefore", + "// asserts the DERIVED HEADER ITSELF on the wire, which no header-based provider allows,", + "// since ExtraFields exposes raw_request and never the outbound HTTP headers.", + "pm.test('bedrock invoke path: request succeeds', function () {", + " pm.expect(pm.response.code, 'want 200; a 4xx means a gated field went out without its beta header, and any 5xx is a real failure too: ' + txt.slice(0, 400)).to.eql(200);", + "});", + "", + "pm.test('bedrock: anthropic_beta on the wire carries tool-search-tool-2025-10-19, tool-examples-2025-10-29', function () {", + " pm.expect(rr, 'raw_request missing or unparseable - x-bf-send-back-raw-request not honored: ' + txt.slice(0, 300)).to.be.an('object');", + " pm.expect(rr.anthropic_beta, 'anthropic_beta is absent, which means the request went via Converse rather than InvokeModel: ' + wire.slice(0, 500)).to.be.an('array');", + " pm.expect(rr.anthropic_beta, 'Bifrost did not derive tool-search-tool-2025-10-19 from tools[].defer_loading and tools[].input_examples: ' + JSON.stringify(rr.anthropic_beta)).to.include('tool-search-tool-2025-10-19');", + " pm.expect(rr.anthropic_beta, 'Bifrost did not derive tool-examples-2025-10-29 from tools[].defer_loading and tools[].input_examples: ' + JSON.stringify(rr.anthropic_beta)).to.include('tool-examples-2025-10-29');", + "});", + "", + "// The probe strips input_schema and description from a LOCAL copy before decoding. If that", + "// ever reached the wire, tools would arrive schemaless and tool calling would break.", + "pm.test('bedrock: tool feature, schema and description reach the provider intact', function () {", + " pm.expect(rr, 'raw_request missing: ' + txt.slice(0, 300)).to.be.an('object');", + " var tools = rr.tools || [];", + " pm.expect(tools.length, 'expected 2 tools on the Anthropic-shaped body: ' + wire.slice(0, 500)).to.eql(2);", + " var probe = null;", + " tools.forEach(function (t) { if (t && t.name === 'harness_probe') { probe = t; } });", + " pm.expect(probe, 'the probe tool vanished from the outbound body: ' + wire.slice(0, 400)).to.be.an('object');", + " pm.expect(probe.defer_loading, 'tools[].defer_loading was dropped before the provider: ' + JSON.stringify(probe).slice(0, 400)).to.eql(true);", + " pm.expect(probe.input_examples, 'tools[].input_examples was dropped before the provider: ' + JSON.stringify(probe).slice(0, 400)).to.be.an('array');", + " pm.expect(probe.description, 'tools[].description was stripped from the outbound body: ' + wire.slice(0, 400)).to.eql('Harness probe tool');", + " pm.expect(probe.input_schema, 'tools[].input_schema was stripped from the outbound body: ' + wire.slice(0, 400)).to.be.an('object');", + " pm.expect(probe.input_schema.properties.q.type, 'the nested schema was altered on the wire: ' + JSON.stringify(probe.input_schema).slice(0, 300)).to.eql('string');", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-api-key", + "value": "{{anthropicKey}}" + }, + { + "key": "anthropic-version", + "value": "2023-06-01" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"bedrock/global.anthropic.claude-sonnet-4-6\",\n \"max_tokens\": 256,\n \"tools\": [\n {\n \"name\": \"harness_probe\",\n \"description\": \"Harness probe tool\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"q\": {\n \"type\": \"string\"\n }\n }\n },\n \"defer_loading\": true,\n \"input_examples\": [\n {\n \"input\": {\n \"q\": \"hello\"\n }\n }\n ]\n },\n {\n \"name\": \"plain_tool\",\n \"description\": \"Plain tool\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"q\": {\n \"type\": \"string\"\n }\n }\n }\n }\n ],\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/anthropic/v1/messages", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "anthropic", + "v1", + "messages" + ] + } + } + }, + { + "name": "12.B12 Beta header derived from tools -> computer-use-2025-11-24", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var txt = pm.response.text() || '';", + "var body = {};", + "try { body = pm.response.json() || {}; } catch (e) { body = {}; }", + "var rr = (body.extra_fields || {}).raw_request;", + "if (typeof rr === 'string') { try { rr = JSON.parse(rr); } catch (e) { rr = null; } }", + "var wire = JSON.stringify(rr || null);", + "", + "// No anthropic-beta from the client: Bifrost must derive computer-use-2025-11-24", + "// from tools[].type computer_20251124. The upstream refuses the gated field without that header,", + "// so the status is the provider confirming what went out.", + "pm.test('computer-use-2025-11-24 is injected by Bifrost (200 = upstream accepted the gated field)', function () {", + " pm.expect(pm.response.code, 'want 200; a 4xx means the beta header was not injected, and any 5xx is a real failure too: ' + txt.slice(0, 400)).to.eql(200);", + "});", + "", + "pm.test('computer-use-2025-11-24: the gated tool feature and its schema reach the provider', function () {", + " pm.expect(rr, 'raw_request missing or unparseable - x-bf-send-back-raw-request not honored: ' + txt.slice(0, 300)).to.be.an('object');", + " var tools = rr.tools || [];", + " pm.expect(tools.length, 'expected 2 tool(s) on the wire: ' + wire.slice(0, 500)).to.eql(2);", + " var probe = null;", + " tools.forEach(function (t) { if (t && t.name === 'harness_probe') { probe = t; } });", + " pm.expect(probe, 'the probe tool vanished from the outbound body: ' + wire.slice(0, 400)).to.be.an('object');", + " var feature = null;", + " tools.forEach(function (t) { if (t && t.type === 'computer_20251124') { feature = t; } });", + " pm.expect(feature, 'the newer-generation computer-use tool was dropped before the provider: ' + wire.slice(0, 400)).to.be.an('object');", + " pm.expect(feature.display_width_px, 'the computer tool lost its parameters: ' + JSON.stringify(feature).slice(0, 300)).to.eql(1024);", + " pm.expect(probe.description, 'tools[].description was stripped from the outbound body: ' + wire.slice(0, 400)).to.eql('Harness probe tool');", + " pm.expect(probe.input_schema, 'tools[].input_schema was stripped from the outbound body: ' + wire.slice(0, 400)).to.be.an('object');", + " pm.expect(probe.input_schema.properties.q.type, 'the nested schema was altered on the wire: ' + JSON.stringify(probe.input_schema).slice(0, 300)).to.eql('string');", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-api-key", + "value": "{{anthropicKey}}" + }, + { + "key": "anthropic-version", + "value": "2023-06-01" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"claude-sonnet-4-6\",\n \"max_tokens\": 256,\n \"tools\": [\n {\n \"name\": \"harness_probe\",\n \"description\": \"Harness probe tool\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"q\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"q\"\n ]\n }\n },\n {\n \"type\": \"computer_20251124\",\n \"name\": \"computer\",\n \"display_width_px\": 1024,\n \"display_height_px\": 768\n }\n ],\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/anthropic/v1/messages", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "anthropic", + "v1", + "messages" + ] + } + } + }, + { + "name": "12.B14 Beta header derived from tools -> advisor-tool-2026-03-01 (tool and header must agree)", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var txt = pm.response.text() || '';", + "var body = {};", + "try { body = pm.response.json() || {}; } catch (e) { body = {}; }", + "var rr = (body.extra_fields || {}).raw_request;", + "if (typeof rr === 'string') { try { rr = JSON.parse(rr); } catch (e) { rr = null; } }", + "var wire = JSON.stringify(rr || null);", + "", + "// advisor-tool-2026-03-01 has ZERO coverage anywhere else in this collection.", + "//", + "// Whether the advisor tool survives is datasheet-driven: SupportsAdvisorTool falls back to", + "// the provider flag (true for Anthropic) unless the model record overrides it, and the", + "// datasheet is fetched at runtime, so this row cannot assume either outcome. Asserting", + "// 'the tool survives' would fail on any model whose record says otherwise, which is", + "// correct behaviour, not a bug.", + "//", + "// What must hold on EVERY model is that the tool and its beta header agree. A kept tool", + "// with no header is a 400 from Anthropic; a dropped tool is a deliberate strip. Both are", + "// fine, one at a time. The failure this catches is the tool surviving while the header", + "// does not, which is the exact shape of a stale derivation.", + "pm.test('advisor-tool-2026-03-01: advisor tool and its beta header agree', function () {", + " pm.expect(rr, 'raw_request missing or unparseable - x-bf-send-back-raw-request not honored: ' + txt.slice(0, 300)).to.be.an('object');", + " var advisor = null;", + " (rr.tools || []).forEach(function (t) { if (t && t.type === 'advisor_20260301') { advisor = t; } });", + " if (advisor) {", + " pm.expect(pm.response.code, 'the advisor tool reached the provider but the request was refused, which means the advisor beta header was not injected alongside it: ' + txt.slice(0, 400)).to.eql(200);", + " } else {", + " pm.expect(pm.response.code, 'the advisor tool was stripped (model datasheet says unsupported), so the request must still succeed: ' + txt.slice(0, 400)).to.eql(200);", + " }", + "});", + "", + "// The probe tool is never gated, so it must survive either way, schema and all.", + "pm.test('advisor-tool-2026-03-01: the ungated probe tool survives regardless of the advisor outcome', function () {", + " var probe = null;", + " (rr.tools || []).forEach(function (t) { if (t && t.name === 'harness_probe') { probe = t; } });", + " pm.expect(probe, 'the probe tool vanished from the outbound body: ' + wire.slice(0, 400)).to.be.an('object');", + " pm.expect(probe.description, 'tools[].description was stripped from the outbound body: ' + wire.slice(0, 400)).to.eql('Harness probe tool');", + " pm.expect(probe.input_schema.properties.q.type, 'the nested schema was altered on the wire: ' + JSON.stringify(probe.input_schema).slice(0, 300)).to.eql('string');", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-api-key", + "value": "{{anthropicKey}}" + }, + { + "key": "anthropic-version", + "value": "2023-06-01" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"claude-opus-4-8\",\n \"max_tokens\": 256,\n \"tools\": [\n {\n \"name\": \"harness_probe\",\n \"description\": \"Harness probe tool\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"q\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"q\"\n ]\n }\n },\n {\n \"type\": \"advisor_20260301\",\n \"name\": \"advisor\"\n }\n ],\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/anthropic/v1/messages", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "anthropic", + "v1", + "messages" + ] + } + } + }, + { + "name": "12.B8 Beta header derived from tools -> vertex (anthropic_beta asserted in body)", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var txt = pm.response.text() || '';", + "var body = {};", + "try { body = pm.response.json() || {}; } catch (e) { body = {}; }", + "var rr = (body.extra_fields || {}).raw_request;", + "if (typeof rr === 'string') { try { rr = JSON.parse(rr); } catch (e) { rr = null; } }", + "var wire = JSON.stringify(rr || null);", + "", + "// No anthropic-beta from the client: Bifrost must derive fine-grained-tool-streaming-2025-05-14", + "// from tools[].eager_input_streaming and inject it. The upstream refuses the gated field without", + "// that header, so the status is the provider confirming what went out.", + "pm.test('fine-grained-tool-streaming-2025-05-14 is injected by Bifrost (200 = upstream accepted the gated field)', function () {", + " pm.expect(pm.response.code, 'want 200; a 4xx means the beta header was not injected, and any 5xx is a real failure too: ' + txt.slice(0, 400)).to.eql(200);", + "});", + "", + "// This provider sets InjectBetaHeadersIntoBody, so the derived beta list is written into", + "// the JSON body as anthropic_beta (requestbuilder.go:488) rather than an HTTP header.", + "// That makes this the one place the harness can assert the DERIVED BETA ITSELF on the", + "// wire instead of inferring it, since ExtraFields exposes raw_request and no headers.", + "pm.test('fine-grained-tool-streaming-2025-05-14: anthropic_beta on the wire carries the tool-derived header', function () {", + " pm.expect(rr, 'raw_request missing or unparseable - x-bf-send-back-raw-request not honored: ' + txt.slice(0, 300)).to.be.an('object');", + " pm.expect(rr.anthropic_beta, 'anthropic_beta is absent from the outbound body: ' + wire.slice(0, 500)).to.be.an('array');", + " pm.expect(rr.anthropic_beta, 'Bifrost did not derive the header from the tool: ' + JSON.stringify(rr.anthropic_beta)).to.include('fine-grained-tool-streaming-2025-05-14');", + "});", + "", + "// The gated feature must itself survive, and the tool COUNT is what catches a whole tool", + "// being dropped rather than a field being stripped.", + "pm.test('fine-grained-tool-streaming-2025-05-14: the gated tool feature and its schema reach the provider', function () {", + " pm.expect(rr, 'raw_request missing: ' + txt.slice(0, 300)).to.be.an('object');", + " var tools = rr.tools || [];", + " pm.expect(tools.length, 'expected 1 tool(s) on the wire: ' + wire.slice(0, 500)).to.eql(1);", + " var probe = null;", + " tools.forEach(function (t) { if (t && t.name === 'harness_probe') { probe = t; } });", + " pm.expect(probe, 'the probe tool vanished from the outbound body: ' + wire.slice(0, 400)).to.be.an('object');", + " pm.expect(probe.eager_input_streaming, 'tools[].eager_input_streaming was dropped before the provider: ' + JSON.stringify(probe).slice(0, 400)).to.eql(true);", + " pm.expect(probe.description, 'tools[].description was stripped from the outbound body: ' + wire.slice(0, 400)).to.eql('Harness probe tool');", + " pm.expect(probe.input_schema, 'tools[].input_schema was stripped from the outbound body: ' + wire.slice(0, 400)).to.be.an('object');", + " pm.expect(probe.input_schema.properties.q.type, 'the nested schema was altered on the wire: ' + JSON.stringify(probe.input_schema).slice(0, 300)).to.eql('string');", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-api-key", + "value": "{{anthropicKey}}" + }, + { + "key": "anthropic-version", + "value": "2023-06-01" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"vertex/claude-opus-4-7\",\n \"max_tokens\": 256,\n \"tools\": [\n {\n \"name\": \"harness_probe\",\n \"description\": \"Harness probe tool\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"q\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"q\"\n ]\n },\n \"eager_input_streaming\": true\n }\n ],\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/anthropic/v1/messages", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "anthropic", + "v1", + "messages" + ] + } + } + }, + { + "name": "12.B10 Beta header derived from tools -> azure (header path, status-inferred)", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var txt = pm.response.text() || '';", + "var body = {};", + "try { body = pm.response.json() || {}; } catch (e) { body = {}; }", + "var rr = (body.extra_fields || {}).raw_request;", + "if (typeof rr === 'string') { try { rr = JSON.parse(rr); } catch (e) { rr = null; } }", + "var wire = JSON.stringify(rr || null);", + "", + "// No anthropic-beta from the client: Bifrost must derive fine-grained-tool-streaming-2025-05-14", + "// from tools[].eager_input_streaming and inject it. The upstream refuses the gated field without", + "// that header, so the status is the provider confirming what went out.", + "pm.test('fine-grained-tool-streaming-2025-05-14 is injected by Bifrost (200 = upstream accepted the gated field)', function () {", + " pm.expect(pm.response.code, 'want 200; a 4xx means the beta header was not injected, and any 5xx is a real failure too: ' + txt.slice(0, 400)).to.eql(200);", + "});", + "", + "// The gated feature must itself survive, and the tool COUNT is what catches a whole tool", + "// being dropped rather than a field being stripped.", + "pm.test('fine-grained-tool-streaming-2025-05-14: the gated tool feature and its schema reach the provider', function () {", + " pm.expect(rr, 'raw_request missing: ' + txt.slice(0, 300)).to.be.an('object');", + " var tools = rr.tools || [];", + " pm.expect(tools.length, 'expected 1 tool(s) on the wire: ' + wire.slice(0, 500)).to.eql(1);", + " var probe = null;", + " tools.forEach(function (t) { if (t && t.name === 'harness_probe') { probe = t; } });", + " pm.expect(probe, 'the probe tool vanished from the outbound body: ' + wire.slice(0, 400)).to.be.an('object');", + " pm.expect(probe.eager_input_streaming, 'tools[].eager_input_streaming was dropped before the provider: ' + JSON.stringify(probe).slice(0, 400)).to.eql(true);", + " pm.expect(probe.description, 'tools[].description was stripped from the outbound body: ' + wire.slice(0, 400)).to.eql('Harness probe tool');", + " pm.expect(probe.input_schema, 'tools[].input_schema was stripped from the outbound body: ' + wire.slice(0, 400)).to.be.an('object');", + " pm.expect(probe.input_schema.properties.q.type, 'the nested schema was altered on the wire: ' + JSON.stringify(probe.input_schema).slice(0, 300)).to.eql('string');", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-api-key", + "value": "{{anthropicKey}}" + }, + { + "key": "anthropic-version", + "value": "2023-06-01" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"azure/claude-haiku-4-5\",\n \"max_tokens\": 256,\n \"tools\": [\n {\n \"name\": \"harness_probe\",\n \"description\": \"Harness probe tool\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"q\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"q\"\n ]\n },\n \"eager_input_streaming\": true\n }\n ],\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/anthropic/v1/messages", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "anthropic", + "v1", + "messages" + ] + } + } + }, + { + "name": "12.B11 Beta header derived from tools -> anthropic passthrough route", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var txt = pm.response.text() || '';", + "", + "// The passthrough route forwards the upstream response verbatim, so Bifrost attaches no", + "// extra_fields and raw_request is unavailable here -- unlike every other row in this set.", + "// The status is still a genuine signal: the client sends no anthropic-beta, the body", + "// carries tools[].eager_input_streaming, and Anthropic refuses that field without", + "// fine-grained-tool-streaming-2025-05-14. A 200 therefore means Bifrost derived and sent", + "// the header on the passthrough surface too.", + "pm.test('fine-grained-tool-streaming-2025-05-14 is injected on the passthrough route', function () {", + " pm.expect(pm.response.code, 'want 200; a 4xx means the beta header was not injected on the passthrough surface, and any 5xx is a real failure too: ' + txt.slice(0, 400)).to.eql(200);", + "});", + "", + "// A passthrough response must still be a well-formed Anthropic message, which rules out", + "// the request having been mangled on the way through.", + "pm.test('passthrough returns a well-formed Anthropic message', function () {", + " var body = {};", + " try { body = pm.response.json() || {}; } catch (e) { body = {}; }", + " pm.expect(body.type, 'expected an Anthropic message envelope: ' + txt.slice(0, 300)).to.eql('message');", + " pm.expect(body.content, 'no content in the passthrough response: ' + txt.slice(0, 300)).to.be.an('array');", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "anthropic-version", + "value": "2023-06-01" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"claude-haiku-4-5\",\n \"max_tokens\": 256,\n \"tools\": [\n {\n \"name\": \"harness_probe\",\n \"description\": \"Harness probe tool\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"q\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"q\"\n ]\n },\n \"eager_input_streaming\": true\n }\n ],\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/anthropic_passthrough/v1/messages", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "anthropic_passthrough", + "v1", + "messages" + ] + } + } + }, + { + "name": "12.B1 Beta header derived from tools -> structured-outputs-2025-11-13", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var txt = pm.response.text() || '';", + "var body = {};", + "try { body = pm.response.json() || {}; } catch (e) { body = {}; }", + "var rr = (body.extra_fields || {}).raw_request;", + "if (typeof rr === 'string') { try { rr = JSON.parse(rr); } catch (e) { rr = null; } }", + "var wire = JSON.stringify(rr || null);", + "", + "// The client deliberately sends NO anthropic-beta header: Bifrost must derive", + "// structured-outputs-2025-11-13 from tools[].strict and inject it. Anthropic refuses the gated field", + "// when the header is missing, so the upstream status is the provider itself confirming", + "// what went out. Outbound headers are not visible here (ExtraFields carries raw_request", + "// only, no raw headers), which makes that status the strongest signal available.", + "pm.test('structured-outputs-2025-11-13 is injected by Bifrost (200 = Anthropic accepted the gated field)', function () {", + " pm.expect(pm.response.code, 'want 200; a 4xx means the beta header was not injected, and any 5xx is a real failure too: ' + txt.slice(0, 400)).to.eql(200);", + "});", + "", + "// The gated feature itself must survive. Asserting only that some tool arrived would pass", + "// while the field the header exists for was silently dropped. The tool COUNT is what", + "// catches a whole tool being removed, which is not hypothetical: an earlier run of this", + "// row caught an unsupported tool being dropped before it reached the provider.", + "pm.test('structured-outputs-2025-11-13: the gated tool feature reaches the provider', function () {", + " pm.expect(rr, 'raw_request missing or unparseable - x-bf-send-back-raw-request not honored: ' + txt.slice(0, 300)).to.be.an('object');", + " var tools = rr.tools || [];", + " pm.expect(tools.length, 'expected 1 tool(s) on the wire: ' + wire.slice(0, 500)).to.eql(1);", + " var probe = null;", + " tools.forEach(function (t) { if (t && t.name === 'harness_probe') { probe = t; } });", + " pm.expect(probe, 'the probe tool vanished from the outbound body: ' + wire.slice(0, 400)).to.be.an('object');", + " pm.expect(probe.strict, 'tools[].strict was dropped before the provider: ' + JSON.stringify(probe).slice(0, 400)).to.eql(true);", + "});", + "", + "// The beta probe strips input_schema and description from a LOCAL copy before decoding.", + "// Were that strip ever to reach the wire, every tool would arrive with no schema and tool", + "// calling would break outright on every passthrough request.", + "pm.test('structured-outputs-2025-11-13: tool schema and description reach the provider intact', function () {", + " pm.expect(rr, 'raw_request missing: ' + txt.slice(0, 300)).to.be.an('object');", + " var probe = null;", + " (rr.tools || []).forEach(function (t) { if (t && t.name === 'harness_probe') { probe = t; } });", + " pm.expect(probe, 'the probe tool vanished: ' + wire.slice(0, 400)).to.be.an('object');", + " pm.expect(probe.description, 'tools[].description was stripped from the outbound body: ' + wire.slice(0, 400)).to.eql('Harness probe tool');", + " pm.expect(probe.input_schema, 'tools[].input_schema was stripped from the outbound body: ' + wire.slice(0, 400)).to.be.an('object');", + " pm.expect(probe.input_schema.properties.q.type, 'the nested schema was altered on the wire: ' + JSON.stringify(probe.input_schema).slice(0, 300)).to.eql('string');", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-api-key", + "value": "{{anthropicKey}}" + }, + { + "key": "anthropic-version", + "value": "2023-06-01" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"claude-haiku-4-5\",\n \"max_tokens\": 256,\n \"tools\": [\n {\n \"name\": \"harness_probe\",\n \"description\": \"Harness probe tool\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"q\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"q\"\n ],\n \"additionalProperties\": false\n },\n \"strict\": true\n }\n ],\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/anthropic/v1/messages", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "anthropic", + "v1", + "messages" + ] + } + } + }, + { + "name": "12.B2 Beta header derived from tools -> tool-search-tool-2025-10-19", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var txt = pm.response.text() || '';", + "var body = {};", + "try { body = pm.response.json() || {}; } catch (e) { body = {}; }", + "var rr = (body.extra_fields || {}).raw_request;", + "if (typeof rr === 'string') { try { rr = JSON.parse(rr); } catch (e) { rr = null; } }", + "var wire = JSON.stringify(rr || null);", + "", + "// The client deliberately sends NO anthropic-beta header: Bifrost must derive", + "// tool-search-tool-2025-10-19 from tools[].defer_loading and inject it. Anthropic refuses the gated field", + "// when the header is missing, so the upstream status is the provider itself confirming", + "// what went out. Outbound headers are not visible here (ExtraFields carries raw_request", + "// only, no raw headers), which makes that status the strongest signal available.", + "pm.test('tool-search-tool-2025-10-19 is injected by Bifrost (200 = Anthropic accepted the gated field)', function () {", + " pm.expect(pm.response.code, 'want 200; a 4xx means the beta header was not injected, and any 5xx is a real failure too: ' + txt.slice(0, 400)).to.eql(200);", + "});", + "", + "// The gated feature itself must survive. Asserting only that some tool arrived would pass", + "// while the field the header exists for was silently dropped. The tool COUNT is what", + "// catches a whole tool being removed, which is not hypothetical: an earlier run of this", + "// row caught an unsupported tool being dropped before it reached the provider.", + "pm.test('tool-search-tool-2025-10-19: the gated tool feature reaches the provider', function () {", + " pm.expect(rr, 'raw_request missing or unparseable - x-bf-send-back-raw-request not honored: ' + txt.slice(0, 300)).to.be.an('object');", + " var tools = rr.tools || [];", + " pm.expect(tools.length, 'expected 2 tool(s) on the wire: ' + wire.slice(0, 500)).to.eql(2);", + " var probe = null;", + " tools.forEach(function (t) { if (t && t.name === 'harness_probe') { probe = t; } });", + " pm.expect(probe, 'the probe tool vanished from the outbound body: ' + wire.slice(0, 400)).to.be.an('object');", + " pm.expect(probe.defer_loading, 'tools[].defer_loading was dropped before the provider: ' + JSON.stringify(probe).slice(0, 400)).to.eql(true);", + "});", + "", + "// The beta probe strips input_schema and description from a LOCAL copy before decoding.", + "// Were that strip ever to reach the wire, every tool would arrive with no schema and tool", + "// calling would break outright on every passthrough request.", + "pm.test('tool-search-tool-2025-10-19: tool schema and description reach the provider intact', function () {", + " pm.expect(rr, 'raw_request missing: ' + txt.slice(0, 300)).to.be.an('object');", + " var probe = null;", + " (rr.tools || []).forEach(function (t) { if (t && t.name === 'harness_probe') { probe = t; } });", + " pm.expect(probe, 'the probe tool vanished: ' + wire.slice(0, 400)).to.be.an('object');", + " pm.expect(probe.description, 'tools[].description was stripped from the outbound body: ' + wire.slice(0, 400)).to.eql('Harness probe tool');", + " pm.expect(probe.input_schema, 'tools[].input_schema was stripped from the outbound body: ' + wire.slice(0, 400)).to.be.an('object');", + " pm.expect(probe.input_schema.properties.q.type, 'the nested schema was altered on the wire: ' + JSON.stringify(probe.input_schema).slice(0, 300)).to.eql('string');", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-api-key", + "value": "{{anthropicKey}}" + }, + { + "key": "anthropic-version", + "value": "2023-06-01" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"claude-haiku-4-5\",\n \"max_tokens\": 256,\n \"tools\": [\n {\n \"name\": \"harness_probe\",\n \"description\": \"Harness probe tool\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"q\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"q\"\n ],\n \"additionalProperties\": false\n },\n \"defer_loading\": true\n },\n {\n \"name\": \"plain_tool\",\n \"description\": \"Plain tool\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"q\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"q\"\n ],\n \"additionalProperties\": false\n }\n }\n ],\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/anthropic/v1/messages", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "anthropic", + "v1", + "messages" + ] + } + } + }, + { + "name": "12.B3 Beta header derived from tools -> advanced-tool-use-2025-11-20", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var txt = pm.response.text() || '';", + "var body = {};", + "try { body = pm.response.json() || {}; } catch (e) { body = {}; }", + "var rr = (body.extra_fields || {}).raw_request;", + "if (typeof rr === 'string') { try { rr = JSON.parse(rr); } catch (e) { rr = null; } }", + "var wire = JSON.stringify(rr || null);", + "", + "// The client deliberately sends NO anthropic-beta header: Bifrost must derive", + "// advanced-tool-use-2025-11-20 from tools[].input_examples and inject it. Anthropic refuses the gated field", + "// when the header is missing, so the upstream status is the provider itself confirming", + "// what went out. Outbound headers are not visible here (ExtraFields carries raw_request", + "// only, no raw headers), which makes that status the strongest signal available.", + "pm.test('advanced-tool-use-2025-11-20 is injected by Bifrost (200 = Anthropic accepted the gated field)', function () {", + " pm.expect(pm.response.code, 'want 200; a 4xx means the beta header was not injected, and any 5xx is a real failure too: ' + txt.slice(0, 400)).to.eql(200);", + "});", + "", + "// The gated feature itself must survive. Asserting only that some tool arrived would pass", + "// while the field the header exists for was silently dropped. The tool COUNT is what", + "// catches a whole tool being removed, which is not hypothetical: an earlier run of this", + "// row caught an unsupported tool being dropped before it reached the provider.", + "pm.test('advanced-tool-use-2025-11-20: the gated tool feature reaches the provider', function () {", + " pm.expect(rr, 'raw_request missing or unparseable - x-bf-send-back-raw-request not honored: ' + txt.slice(0, 300)).to.be.an('object');", + " var tools = rr.tools || [];", + " pm.expect(tools.length, 'expected 1 tool(s) on the wire: ' + wire.slice(0, 500)).to.eql(1);", + " var probe = null;", + " tools.forEach(function (t) { if (t && t.name === 'harness_probe') { probe = t; } });", + " pm.expect(probe, 'the probe tool vanished from the outbound body: ' + wire.slice(0, 400)).to.be.an('object');", + " pm.expect(probe.input_examples, 'tools[].input_examples was dropped before the provider: ' + JSON.stringify(probe).slice(0, 400)).to.be.an('array');", + " pm.expect(probe.input_examples.length, 'input_examples arrived empty: ' + JSON.stringify(probe).slice(0, 400)).to.be.above(0);", + "});", + "", + "// The beta probe strips input_schema and description from a LOCAL copy before decoding.", + "// Were that strip ever to reach the wire, every tool would arrive with no schema and tool", + "// calling would break outright on every passthrough request.", + "pm.test('advanced-tool-use-2025-11-20: tool schema and description reach the provider intact', function () {", + " pm.expect(rr, 'raw_request missing: ' + txt.slice(0, 300)).to.be.an('object');", + " var probe = null;", + " (rr.tools || []).forEach(function (t) { if (t && t.name === 'harness_probe') { probe = t; } });", + " pm.expect(probe, 'the probe tool vanished: ' + wire.slice(0, 400)).to.be.an('object');", + " pm.expect(probe.description, 'tools[].description was stripped from the outbound body: ' + wire.slice(0, 400)).to.eql('Harness probe tool');", + " pm.expect(probe.input_schema, 'tools[].input_schema was stripped from the outbound body: ' + wire.slice(0, 400)).to.be.an('object');", + " pm.expect(probe.input_schema.properties.q.type, 'the nested schema was altered on the wire: ' + JSON.stringify(probe.input_schema).slice(0, 300)).to.eql('string');", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-api-key", + "value": "{{anthropicKey}}" + }, + { + "key": "anthropic-version", + "value": "2023-06-01" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"claude-haiku-4-5\",\n \"max_tokens\": 256,\n \"tools\": [\n {\n \"name\": \"harness_probe\",\n \"description\": \"Harness probe tool\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"q\": {\n \"type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"input_examples\": [\n {\n \"input\": {\n \"q\": \"hello\"\n }\n }\n ]\n }\n ],\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/anthropic/v1/messages", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "anthropic", + "v1", + "messages" + ] + } + } + }, + { + "name": "12.B4 Beta header derived from tools -> advanced-tool-use-2025-11-20", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var txt = pm.response.text() || '';", + "var body = {};", + "try { body = pm.response.json() || {}; } catch (e) { body = {}; }", + "var rr = (body.extra_fields || {}).raw_request;", + "if (typeof rr === 'string') { try { rr = JSON.parse(rr); } catch (e) { rr = null; } }", + "var wire = JSON.stringify(rr || null);", + "", + "// The client deliberately sends NO anthropic-beta header: Bifrost must derive", + "// advanced-tool-use-2025-11-20 from tools[].allowed_callers and inject it. Anthropic refuses the gated field", + "// when the header is missing, so the upstream status is the provider itself confirming", + "// what went out. Outbound headers are not visible here (ExtraFields carries raw_request", + "// only, no raw headers), which makes that status the strongest signal available.", + "pm.test('advanced-tool-use-2025-11-20 is injected by Bifrost (200 = Anthropic accepted the gated field)', function () {", + " pm.expect(pm.response.code, 'want 200; a 4xx means the beta header was not injected, and any 5xx is a real failure too: ' + txt.slice(0, 400)).to.eql(200);", + "});", + "", + "// The gated feature itself must survive. Asserting only that some tool arrived would pass", + "// while the field the header exists for was silently dropped. The tool COUNT is what", + "// catches a whole tool being removed, which is not hypothetical: an earlier run of this", + "// row caught an unsupported tool being dropped before it reached the provider.", + "pm.test('advanced-tool-use-2025-11-20: the gated tool feature reaches the provider', function () {", + " pm.expect(rr, 'raw_request missing or unparseable - x-bf-send-back-raw-request not honored: ' + txt.slice(0, 300)).to.be.an('object');", + " var tools = rr.tools || [];", + " pm.expect(tools.length, 'expected 1 tool(s) on the wire: ' + wire.slice(0, 500)).to.eql(1);", + " var probe = null;", + " tools.forEach(function (t) { if (t && t.name === 'harness_probe') { probe = t; } });", + " pm.expect(probe, 'the probe tool vanished from the outbound body: ' + wire.slice(0, 400)).to.be.an('object');", + " pm.expect(probe.allowed_callers, 'tools[].allowed_callers was dropped before the provider: ' + JSON.stringify(probe).slice(0, 400)).to.be.an('array');", + " pm.expect(probe.allowed_callers).to.include('direct');", + "});", + "", + "// The beta probe strips input_schema and description from a LOCAL copy before decoding.", + "// Were that strip ever to reach the wire, every tool would arrive with no schema and tool", + "// calling would break outright on every passthrough request.", + "pm.test('advanced-tool-use-2025-11-20: tool schema and description reach the provider intact', function () {", + " pm.expect(rr, 'raw_request missing: ' + txt.slice(0, 300)).to.be.an('object');", + " var probe = null;", + " (rr.tools || []).forEach(function (t) { if (t && t.name === 'harness_probe') { probe = t; } });", + " pm.expect(probe, 'the probe tool vanished: ' + wire.slice(0, 400)).to.be.an('object');", + " pm.expect(probe.description, 'tools[].description was stripped from the outbound body: ' + wire.slice(0, 400)).to.eql('Harness probe tool');", + " pm.expect(probe.input_schema, 'tools[].input_schema was stripped from the outbound body: ' + wire.slice(0, 400)).to.be.an('object');", + " pm.expect(probe.input_schema.properties.q.type, 'the nested schema was altered on the wire: ' + JSON.stringify(probe.input_schema).slice(0, 300)).to.eql('string');", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-api-key", + "value": "{{anthropicKey}}" + }, + { + "key": "anthropic-version", + "value": "2023-06-01" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"claude-haiku-4-5\",\n \"max_tokens\": 256,\n \"tools\": [\n {\n \"name\": \"harness_probe\",\n \"description\": \"Harness probe tool\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"q\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"q\"\n ],\n \"additionalProperties\": false\n },\n \"allowed_callers\": [\n \"direct\"\n ]\n }\n ],\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/anthropic/v1/messages", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "anthropic", + "v1", + "messages" + ] + } + } + }, + { + "name": "12.B5 Beta header derived from tools -> fine-grained-tool-streaming-2025-05-14", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var txt = pm.response.text() || '';", + "var body = {};", + "try { body = pm.response.json() || {}; } catch (e) { body = {}; }", + "var rr = (body.extra_fields || {}).raw_request;", + "if (typeof rr === 'string') { try { rr = JSON.parse(rr); } catch (e) { rr = null; } }", + "var wire = JSON.stringify(rr || null);", + "", + "// The client deliberately sends NO anthropic-beta header: Bifrost must derive", + "// fine-grained-tool-streaming-2025-05-14 from tools[].eager_input_streaming and inject it. Anthropic refuses the gated field", + "// when the header is missing, so the upstream status is the provider itself confirming", + "// what went out. Outbound headers are not visible here (ExtraFields carries raw_request", + "// only, no raw headers), which makes that status the strongest signal available.", + "pm.test('fine-grained-tool-streaming-2025-05-14 is injected by Bifrost (200 = Anthropic accepted the gated field)', function () {", + " pm.expect(pm.response.code, 'want 200; a 4xx means the beta header was not injected, and any 5xx is a real failure too: ' + txt.slice(0, 400)).to.eql(200);", + "});", + "", + "// The gated feature itself must survive. Asserting only that some tool arrived would pass", + "// while the field the header exists for was silently dropped. The tool COUNT is what", + "// catches a whole tool being removed, which is not hypothetical: an earlier run of this", + "// row caught an unsupported tool being dropped before it reached the provider.", + "pm.test('fine-grained-tool-streaming-2025-05-14: the gated tool feature reaches the provider', function () {", + " pm.expect(rr, 'raw_request missing or unparseable - x-bf-send-back-raw-request not honored: ' + txt.slice(0, 300)).to.be.an('object');", + " var tools = rr.tools || [];", + " pm.expect(tools.length, 'expected 1 tool(s) on the wire: ' + wire.slice(0, 500)).to.eql(1);", + " var probe = null;", + " tools.forEach(function (t) { if (t && t.name === 'harness_probe') { probe = t; } });", + " pm.expect(probe, 'the probe tool vanished from the outbound body: ' + wire.slice(0, 400)).to.be.an('object');", + " pm.expect(probe.eager_input_streaming, 'tools[].eager_input_streaming was dropped before the provider: ' + JSON.stringify(probe).slice(0, 400)).to.eql(true);", + "});", + "", + "// The beta probe strips input_schema and description from a LOCAL copy before decoding.", + "// Were that strip ever to reach the wire, every tool would arrive with no schema and tool", + "// calling would break outright on every passthrough request.", + "pm.test('fine-grained-tool-streaming-2025-05-14: tool schema and description reach the provider intact', function () {", + " pm.expect(rr, 'raw_request missing: ' + txt.slice(0, 300)).to.be.an('object');", + " var probe = null;", + " (rr.tools || []).forEach(function (t) { if (t && t.name === 'harness_probe') { probe = t; } });", + " pm.expect(probe, 'the probe tool vanished: ' + wire.slice(0, 400)).to.be.an('object');", + " pm.expect(probe.description, 'tools[].description was stripped from the outbound body: ' + wire.slice(0, 400)).to.eql('Harness probe tool');", + " pm.expect(probe.input_schema, 'tools[].input_schema was stripped from the outbound body: ' + wire.slice(0, 400)).to.be.an('object');", + " pm.expect(probe.input_schema.properties.q.type, 'the nested schema was altered on the wire: ' + JSON.stringify(probe.input_schema).slice(0, 300)).to.eql('string');", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-api-key", + "value": "{{anthropicKey}}" + }, + { + "key": "anthropic-version", + "value": "2023-06-01" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"claude-haiku-4-5\",\n \"max_tokens\": 256,\n \"tools\": [\n {\n \"name\": \"harness_probe\",\n \"description\": \"Harness probe tool\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"q\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"q\"\n ],\n \"additionalProperties\": false\n },\n \"eager_input_streaming\": true\n }\n ],\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/anthropic/v1/messages", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "anthropic", + "v1", + "messages" + ] + } + } + }, + { + "name": "12.B6 Beta header derived from tools -> prompt-caching-scope-2026-01-05", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var txt = pm.response.text() || '';", + "var body = {};", + "try { body = pm.response.json() || {}; } catch (e) { body = {}; }", + "var rr = (body.extra_fields || {}).raw_request;", + "if (typeof rr === 'string') { try { rr = JSON.parse(rr); } catch (e) { rr = null; } }", + "var wire = JSON.stringify(rr || null);", + "", + "// The client deliberately sends NO anthropic-beta header: Bifrost must derive", + "// prompt-caching-scope-2026-01-05 from tools[].cache_control.scope and inject it. Anthropic refuses the gated field", + "// when the header is missing, so the upstream status is the provider itself confirming", + "// what went out. Outbound headers are not visible here (ExtraFields carries raw_request", + "// only, no raw headers), which makes that status the strongest signal available.", + "pm.test('prompt-caching-scope-2026-01-05 is injected by Bifrost (200 = Anthropic accepted the gated field)', function () {", + " pm.expect(pm.response.code, 'want 200; a 4xx means the beta header was not injected, and any 5xx is a real failure too: ' + txt.slice(0, 400)).to.eql(200);", + "});", + "", + "// The gated feature itself must survive. Asserting only that some tool arrived would pass", + "// while the field the header exists for was silently dropped. The tool COUNT is what", + "// catches a whole tool being removed, which is not hypothetical: an earlier run of this", + "// row caught an unsupported tool being dropped before it reached the provider.", + "pm.test('prompt-caching-scope-2026-01-05: the gated tool feature reaches the provider', function () {", + " pm.expect(rr, 'raw_request missing or unparseable - x-bf-send-back-raw-request not honored: ' + txt.slice(0, 300)).to.be.an('object');", + " var tools = rr.tools || [];", + " pm.expect(tools.length, 'expected 1 tool(s) on the wire: ' + wire.slice(0, 500)).to.eql(1);", + " var probe = null;", + " tools.forEach(function (t) { if (t && t.name === 'harness_probe') { probe = t; } });", + " pm.expect(probe, 'the probe tool vanished from the outbound body: ' + wire.slice(0, 400)).to.be.an('object');", + " pm.expect(probe.cache_control, 'tools[].cache_control was dropped before the provider: ' + JSON.stringify(probe).slice(0, 400)).to.be.an('object');", + " pm.expect(probe.cache_control.scope, 'cache_control.scope was dropped before the provider: ' + JSON.stringify(probe).slice(0, 400)).to.eql('org');", + "});", + "", + "// The beta probe strips input_schema and description from a LOCAL copy before decoding.", + "// Were that strip ever to reach the wire, every tool would arrive with no schema and tool", + "// calling would break outright on every passthrough request.", + "pm.test('prompt-caching-scope-2026-01-05: tool schema and description reach the provider intact', function () {", + " pm.expect(rr, 'raw_request missing: ' + txt.slice(0, 300)).to.be.an('object');", + " var probe = null;", + " (rr.tools || []).forEach(function (t) { if (t && t.name === 'harness_probe') { probe = t; } });", + " pm.expect(probe, 'the probe tool vanished: ' + wire.slice(0, 400)).to.be.an('object');", + " pm.expect(probe.description, 'tools[].description was stripped from the outbound body: ' + wire.slice(0, 400)).to.eql('Harness probe tool');", + " pm.expect(probe.input_schema, 'tools[].input_schema was stripped from the outbound body: ' + wire.slice(0, 400)).to.be.an('object');", + " pm.expect(probe.input_schema.properties.q.type, 'the nested schema was altered on the wire: ' + JSON.stringify(probe.input_schema).slice(0, 300)).to.eql('string');", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-api-key", + "value": "{{anthropicKey}}" + }, + { + "key": "anthropic-version", + "value": "2023-06-01" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"claude-haiku-4-5\",\n \"max_tokens\": 256,\n \"tools\": [\n {\n \"name\": \"harness_probe\",\n \"description\": \"Harness probe tool\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"q\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"q\"\n ],\n \"additionalProperties\": false\n },\n \"cache_control\": {\n \"type\": \"ephemeral\",\n \"scope\": \"org\"\n }\n }\n ],\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/anthropic/v1/messages", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "anthropic", + "v1", + "messages" + ] + } + } + }, + { + "name": "12.B7 Beta header derived from tools -> computer-use-2025-01-24", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var txt = pm.response.text() || '';", + "var body = {};", + "try { body = pm.response.json() || {}; } catch (e) { body = {}; }", + "var rr = (body.extra_fields || {}).raw_request;", + "if (typeof rr === 'string') { try { rr = JSON.parse(rr); } catch (e) { rr = null; } }", + "var wire = JSON.stringify(rr || null);", + "", + "// The client deliberately sends NO anthropic-beta header: Bifrost must derive", + "// computer-use-2025-01-24 from tools[].type computer_20250124 and inject it. Anthropic refuses the gated field", + "// when the header is missing, so the upstream status is the provider itself confirming", + "// what went out. Outbound headers are not visible here (ExtraFields carries raw_request", + "// only, no raw headers), which makes that status the strongest signal available.", + "pm.test('computer-use-2025-01-24 is injected by Bifrost (200 = Anthropic accepted the gated field)', function () {", + " pm.expect(pm.response.code, 'want 200; a 4xx means the beta header was not injected, and any 5xx is a real failure too: ' + txt.slice(0, 400)).to.eql(200);", + "});", + "", + "// The gated feature itself must survive. Asserting only that some tool arrived would pass", + "// while the field the header exists for was silently dropped. The tool COUNT is what", + "// catches a whole tool being removed, which is not hypothetical: an earlier run of this", + "// row caught an unsupported tool being dropped before it reached the provider.", + "pm.test('computer-use-2025-01-24: the gated tool feature reaches the provider', function () {", + " pm.expect(rr, 'raw_request missing or unparseable - x-bf-send-back-raw-request not honored: ' + txt.slice(0, 300)).to.be.an('object');", + " var tools = rr.tools || [];", + " pm.expect(tools.length, 'expected 2 tool(s) on the wire: ' + wire.slice(0, 500)).to.eql(2);", + " var probe = null;", + " tools.forEach(function (t) { if (t && t.name === 'harness_probe') { probe = t; } });", + " pm.expect(probe, 'the probe tool vanished from the outbound body: ' + wire.slice(0, 400)).to.be.an('object');", + " var feature = null;", + " tools.forEach(function (t) { if (t && t.type && String(t.type).indexOf('computer_') === 0) { feature = t; } });", + " pm.expect(feature, 'the computer-use tool was dropped before the provider: ' + wire.slice(0, 400)).to.be.an('object');", + " pm.expect(feature.display_width_px, 'the computer tool lost its parameters: ' + JSON.stringify(feature).slice(0, 300)).to.eql(1024);", + "});", + "", + "// The beta probe strips input_schema and description from a LOCAL copy before decoding.", + "// Were that strip ever to reach the wire, every tool would arrive with no schema and tool", + "// calling would break outright on every passthrough request.", + "pm.test('computer-use-2025-01-24: tool schema and description reach the provider intact', function () {", + " pm.expect(rr, 'raw_request missing: ' + txt.slice(0, 300)).to.be.an('object');", + " var probe = null;", + " (rr.tools || []).forEach(function (t) { if (t && t.name === 'harness_probe') { probe = t; } });", + " pm.expect(probe, 'the probe tool vanished: ' + wire.slice(0, 400)).to.be.an('object');", + " pm.expect(probe.description, 'tools[].description was stripped from the outbound body: ' + wire.slice(0, 400)).to.eql('Harness probe tool');", + " pm.expect(probe.input_schema, 'tools[].input_schema was stripped from the outbound body: ' + wire.slice(0, 400)).to.be.an('object');", + " pm.expect(probe.input_schema.properties.q.type, 'the nested schema was altered on the wire: ' + JSON.stringify(probe.input_schema).slice(0, 300)).to.eql('string');", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-api-key", + "value": "{{anthropicKey}}" + }, + { + "key": "anthropic-version", + "value": "2023-06-01" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"claude-sonnet-4-5\",\n \"max_tokens\": 256,\n \"tools\": [\n {\n \"name\": \"harness_probe\",\n \"description\": \"Harness probe tool\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"q\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"q\"\n ],\n \"additionalProperties\": false\n }\n },\n {\n \"type\": \"computer_20250124\",\n \"name\": \"computer\",\n \"display_width_px\": 1024,\n \"display_height_px\": 768\n }\n ],\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/anthropic/v1/messages", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "anthropic", + "v1", + "messages" + ] + } + } + }, { "name": "Beta: token-efficient-tools", "request": { @@ -20380,7 +21895,7 @@ } }, { - "name": "[PREVIEW] Azure: reasoning_effort (o3 deployment)", + "name": "[PREVIEW] Azure: reasoning_effort (gpt-5.4-mini deployment)", "request": { "method": "POST", "header": [ @@ -20398,7 +21913,7 @@ "raw": "{\n \"messages\": [{\"role\":\"user\",\"content\":\"Hi\"}],\n \"reasoning_effort\": \"high\"\n}" }, "url": { - "raw": "{{baseUrl}}/openai/openai/deployments/o3/chat/completions?api-version={{azureApiVersion}}", + "raw": "{{baseUrl}}/openai/openai/deployments/gpt-5.4-mini/chat/completions?api-version={{azureApiVersion}}", "host": [ "{{baseUrl}}" ], @@ -20789,6 +22304,24 @@ }, { "name": "Gemini: audio input (inline)", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('audio input request succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.equal(200);", + "});", + "var body = pm.response.json();", + "var usage = body && body.usageMetadata;", + "pm.test('audio input reports usageMetadata.promptTokenCount', function () {", + " pm.expect(usage && usage.promptTokenCount, 'usageMetadata missing: ' + pm.response.text()).to.be.above(0);", + "});" + ] + } + } + ], "request": { "method": "POST", "header": [ @@ -20803,7 +22336,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"contents\": [{\"parts\":[{\"text\":\"Transcribe\"},{\"inlineData\":{\"mimeType\":\"audio/wav\",\"data\":\"UklGRkQDAABXQVZFZm10IBAAAAABAAEAQB8AAIA+AAACABAAZGF0YSADAAAAAJUK6xPlGrIe3R5iG6oUgAv7AFj22eye5YHhAOEo5Jzql/MK/rcIXhLYGUQeHB9GHBgWTg3wAjv4cO645v7h0OBS4znp0fEW/NEGvhCxGLgdPB8OHXAXDg/jBCX6GPDs55niwOCZ4uznGPAl+uMEDg9wFw4dPB+4HbEYvhDRBhb80fE56VLj0OD+4bjmcO47+PACTg0YFkYcHB9EHtgZXhK3CAr+l/Oc6ijkAOGB4Z7l2exY9vsAgAuqFGIb3R6yHuUa6xOVCgAAa/UV7BvlTuEj4Z7kVuuA9AX/qAknE2Iafx4AH9gbZBVpDPYBSfei7SjmvOHk4Lrj6Omy8hD9xQeQEUgZAh4wH64cxxYvDuoDL/lC70/nSOLE4PLikOjy8B372wXoDxQYZx1AH2cdFBjoD9sFHfvy8JDo8uLE4EjiT+dC7y/56gMvDscWrhwwHwIeSBmQEcUHEP2y8ujpuuPk4LzhKOai7Un39gFpDGQV2BsAH38eYhonE6gJBf+A9FbrnuQj4U7hG+UV7Gv1AACVCusT5RqyHt0eYhuqFIAL+wBY9tnsnuWB4QDhKOSc6pfzCv63CF4S2BlEHhwfRhwYFk4N8AI7+HDuuOb+4dDgUuM56dHxFvzRBr4QsRi4HTwfDh1wFw4P4wQl+hjw7OeZ4sDgmeLs5xjwJfrjBA4PcBcOHTwfuB2xGL4Q0QYW/NHxOelS49Dg/uG45nDuO/jwAk4NGBZGHBwfRB7YGV4StwgK/pfznOoo5ADhgeGe5dnsWPb7AIALqhRiG90esh7lGusTlQoAAGv1Fewb5U7hI+Ge5FbrgPQF/6gJJxNiGn8eAB/YG2QVaQz2AUn3ou0o5rzh5OC64+jpsvIQ/cUHkBFIGQIeMB+uHMcWLw7qAy/5Qu9P50jixODy4pDo8vAd+9sF6A8UGGcdQB9nHRQY6A/bBR378vCQ6PLixOBI4k/nQu8v+eoDLw7HFq4cMB8CHkgZkBHFBxD9svLo6brj5OC84Sjmou1J9/YBaQxkFdgbAB9/HmIaJxOoCQX/gPRW657kI+FO4RvlFexr9Q==\"}}]}]\n}" + "raw": "{\n \"contents\": [{\"parts\":[{\"text\":\"Describe the sound in this audio in a few words.\"},{\"inlineData\":{\"mimeType\":\"audio/wav\",\"data\":\"UklGRkQDAABXQVZFZm10IBAAAAABAAEAQB8AAIA+AAACABAAZGF0YSADAAAAAJUK6xPlGrIe3R5iG6oUgAv7AFj22eye5YHhAOEo5Jzql/MK/rcIXhLYGUQeHB9GHBgWTg3wAjv4cO645v7h0OBS4znp0fEW/NEGvhCxGLgdPB8OHXAXDg/jBCX6GPDs55niwOCZ4uznGPAl+uMEDg9wFw4dPB+4HbEYvhDRBhb80fE56VLj0OD+4bjmcO47+PACTg0YFkYcHB9EHtgZXhK3CAr+l/Oc6ijkAOGB4Z7l2exY9vsAgAuqFGIb3R6yHuUa6xOVCgAAa/UV7BvlTuEj4Z7kVuuA9AX/qAknE2Iafx4AH9gbZBVpDPYBSfei7SjmvOHk4Lrj6Omy8hD9xQeQEUgZAh4wH64cxxYvDuoDL/lC70/nSOLE4PLikOjy8B372wXoDxQYZx1AH2cdFBjoD9sFHfvy8JDo8uLE4EjiT+dC7y/56gMvDscWrhwwHwIeSBmQEcUHEP2y8ujpuuPk4LzhKOai7Un39gFpDGQV2BsAH38eYhonE6gJBf+A9FbrnuQj4U7hG+UV7Gv1AACVCusT5RqyHt0eYhuqFIAL+wBY9tnsnuWB4QDhKOSc6pfzCv63CF4S2BlEHhwfRhwYFk4N8AI7+HDuuOb+4dDgUuM56dHxFvzRBr4QsRi4HTwfDh1wFw4P4wQl+hjw7OeZ4sDgmeLs5xjwJfrjBA4PcBcOHTwfuB2xGL4Q0QYW/NHxOelS49Dg/uG45nDuO/jwAk4NGBZGHBwfRB7YGV4StwgK/pfznOoo5ADhgeGe5dnsWPb7AIALqhRiG90esh7lGusTlQoAAGv1Fewb5U7hI+Ge5FbrgPQF/6gJJxNiGn8eAB/YG2QVaQz2AUn3ou0o5rzh5OC64+jpsvIQ/cUHkBFIGQIeMB+uHMcWLw7qAy/5Qu9P50jixODy4pDo8vAd+9sF6A8UGGcdQB9nHRQY6A/bBR378vCQ6PLixOBI4k/nQu8v+eoDLw7HFq4cMB8CHkgZkBHFBxD9svLo6brj5OC84Sjmou1J9/YBaQxkFdgbAB9/HmIaJxOoCQX/gPRW657kI+FO4RvlFexr9Q==\"}}]}]\n}" }, "url": { "raw": "{{baseUrl}}/genai/v1beta/models/{{genaiModel}}:generateContent", @@ -21900,6 +23433,24 @@ }, { "name": "Vertex Gemini: audio input", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('audio input request succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.equal(200);", + "});", + "var body = pm.response.json();", + "var usage = body && body.usageMetadata;", + "pm.test('audio input reports usageMetadata.promptTokenCount', function () {", + " pm.expect(usage && usage.promptTokenCount, 'usageMetadata missing: ' + pm.response.text()).to.be.above(0);", + "});" + ] + } + } + ], "request": { "method": "POST", "header": [ @@ -21914,7 +23465,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"contents\": [{\"parts\":[{\"text\":\"Transcribe\"},{\"inlineData\":{\"mimeType\":\"audio/wav\",\"data\":\"UklGRkQDAABXQVZFZm10IBAAAAABAAEAQB8AAIA+AAACABAAZGF0YSADAAAAAJUK6xPlGrIe3R5iG6oUgAv7AFj22eye5YHhAOEo5Jzql/MK/rcIXhLYGUQeHB9GHBgWTg3wAjv4cO645v7h0OBS4znp0fEW/NEGvhCxGLgdPB8OHXAXDg/jBCX6GPDs55niwOCZ4uznGPAl+uMEDg9wFw4dPB+4HbEYvhDRBhb80fE56VLj0OD+4bjmcO47+PACTg0YFkYcHB9EHtgZXhK3CAr+l/Oc6ijkAOGB4Z7l2exY9vsAgAuqFGIb3R6yHuUa6xOVCgAAa/UV7BvlTuEj4Z7kVuuA9AX/qAknE2Iafx4AH9gbZBVpDPYBSfei7SjmvOHk4Lrj6Omy8hD9xQeQEUgZAh4wH64cxxYvDuoDL/lC70/nSOLE4PLikOjy8B372wXoDxQYZx1AH2cdFBjoD9sFHfvy8JDo8uLE4EjiT+dC7y/56gMvDscWrhwwHwIeSBmQEcUHEP2y8ujpuuPk4LzhKOai7Un39gFpDGQV2BsAH38eYhonE6gJBf+A9FbrnuQj4U7hG+UV7Gv1AACVCusT5RqyHt0eYhuqFIAL+wBY9tnsnuWB4QDhKOSc6pfzCv63CF4S2BlEHhwfRhwYFk4N8AI7+HDuuOb+4dDgUuM56dHxFvzRBr4QsRi4HTwfDh1wFw4P4wQl+hjw7OeZ4sDgmeLs5xjwJfrjBA4PcBcOHTwfuB2xGL4Q0QYW/NHxOelS49Dg/uG45nDuO/jwAk4NGBZGHBwfRB7YGV4StwgK/pfznOoo5ADhgeGe5dnsWPb7AIALqhRiG90esh7lGusTlQoAAGv1Fewb5U7hI+Ge5FbrgPQF/6gJJxNiGn8eAB/YG2QVaQz2AUn3ou0o5rzh5OC64+jpsvIQ/cUHkBFIGQIeMB+uHMcWLw7qAy/5Qu9P50jixODy4pDo8vAd+9sF6A8UGGcdQB9nHRQY6A/bBR378vCQ6PLixOBI4k/nQu8v+eoDLw7HFq4cMB8CHkgZkBHFBxD9svLo6brj5OC84Sjmou1J9/YBaQxkFdgbAB9/HmIaJxOoCQX/gPRW657kI+FO4RvlFexr9Q==\"}}]}]\n}" + "raw": "{\n \"contents\": [{\"parts\":[{\"text\":\"Describe the sound in this audio in a few words.\"},{\"inlineData\":{\"mimeType\":\"audio/wav\",\"data\":\"UklGRkQDAABXQVZFZm10IBAAAAABAAEAQB8AAIA+AAACABAAZGF0YSADAAAAAJUK6xPlGrIe3R5iG6oUgAv7AFj22eye5YHhAOEo5Jzql/MK/rcIXhLYGUQeHB9GHBgWTg3wAjv4cO645v7h0OBS4znp0fEW/NEGvhCxGLgdPB8OHXAXDg/jBCX6GPDs55niwOCZ4uznGPAl+uMEDg9wFw4dPB+4HbEYvhDRBhb80fE56VLj0OD+4bjmcO47+PACTg0YFkYcHB9EHtgZXhK3CAr+l/Oc6ijkAOGB4Z7l2exY9vsAgAuqFGIb3R6yHuUa6xOVCgAAa/UV7BvlTuEj4Z7kVuuA9AX/qAknE2Iafx4AH9gbZBVpDPYBSfei7SjmvOHk4Lrj6Omy8hD9xQeQEUgZAh4wH64cxxYvDuoDL/lC70/nSOLE4PLikOjy8B372wXoDxQYZx1AH2cdFBjoD9sFHfvy8JDo8uLE4EjiT+dC7y/56gMvDscWrhwwHwIeSBmQEcUHEP2y8ujpuuPk4LzhKOai7Un39gFpDGQV2BsAH38eYhonE6gJBf+A9FbrnuQj4U7hG+UV7Gv1AACVCusT5RqyHt0eYhuqFIAL+wBY9tnsnuWB4QDhKOSc6pfzCv63CF4S2BlEHhwfRhwYFk4N8AI7+HDuuOb+4dDgUuM56dHxFvzRBr4QsRi4HTwfDh1wFw4P4wQl+hjw7OeZ4sDgmeLs5xjwJfrjBA4PcBcOHTwfuB2xGL4Q0QYW/NHxOelS49Dg/uG45nDuO/jwAk4NGBZGHBwfRB7YGV4StwgK/pfznOoo5ADhgeGe5dnsWPb7AIALqhRiG90esh7lGusTlQoAAGv1Fewb5U7hI+Ge5FbrgPQF/6gJJxNiGn8eAB/YG2QVaQz2AUn3ou0o5rzh5OC64+jpsvIQ/cUHkBFIGQIeMB+uHMcWLw7qAy/5Qu9P50jixODy4pDo8vAd+9sF6A8UGGcdQB9nHRQY6A/bBR378vCQ6PLixOBI4k/nQu8v+eoDLw7HFq4cMB8CHkgZkBHFBxD9svLo6brj5OC84Sjmou1J9/YBaQxkFdgbAB9/HmIaJxOoCQX/gPRW657kI+FO4RvlFexr9Q==\"}}]}]\n}" }, "url": { "raw": "{{baseUrl}}/genai/v1beta/models/{{vertexModel}}:generateContent", @@ -22888,46 +24439,6 @@ } } }, - { - "name": "[PREVIEW] Azure: audio (gpt-4o-audio deployment)", - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - }, - { - "key": "api-key", - "value": "{{openaiKey}}" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"messages\": [{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Describe\"},{\"type\":\"input_audio\",\"input_audio\":{\"data\":\"REPLACE_BASE64\",\"format\":\"wav\"}}]}]\n}" - }, - "url": { - "raw": "{{baseUrl}}/openai/openai/deployments/gpt-4o-audio-preview/chat/completions?api-version={{azureApiVersion}}", - "host": [ - "{{baseUrl}}" - ], - "path": [ - "openai", - "openai", - "deployments", - "gpt-4o-audio-preview", - "chat", - "completions" - ], - "query": [ - { - "key": "api-version", - "value": "{{azureApiVersion}}" - } - ] - } - } - }, { "name": "[PREVIEW] Azure: skills/container", "request": { @@ -36729,7 +38240,7 @@ ], "body": { "mode": "raw", - "raw": "{\"model\": \"openrouter/google/gemini-2.5-flash\", \"max_tokens\": 64, \"messages\": [{\"role\": \"system\", \"content\": [{\"type\": \"text\", \"text\": \"You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant.\", \"cache_control\": {\"type\": \"ephemeral\"}}]}, {\"role\": \"user\", \"content\": \"Reply with one word.\"}]}" + "raw": "{\"model\": \"openrouter/google/gemini-3.8-flash\", \"max_tokens\": 64, \"messages\": [{\"role\": \"system\", \"content\": [{\"type\": \"text\", \"text\": \"You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant. You are a meticulous assistant.\", \"cache_control\": {\"type\": \"ephemeral\"}}]}, {\"role\": \"user\", \"content\": \"Reply with one word.\"}]}" }, "url": { "raw": "{{baseUrl}}/v1/chat/completions", @@ -40985,14 +42496,14 @@ "script": { "type": "text/javascript", "exec": [ - "pm.test('Chat->Responses conversion actually fired - #5101', function () {", - " // markForConversion is datasheet-driven: x-bf-compat only ENABLES the check, it cannot", - " // force a conversion. On a chat-capable model (this row used openai/gpt-4o-mini until", - " // #6831 testing caught it) nothing converts and the assertions below pass vacuously.", - " var j = pm.response.json();", - " var ef = j.extra_fields || {};", - " pm.expect(ef.converted_request_type, 'chat->responses conversion did not fire, so the replayed tool call never reached ToResponsesRequest and this row proves nothing').to.eql('responses');", - "});", + "pm.test('Chat->Responses conversion actually fired - #5101', function () {", + " // markForConversion is datasheet-driven: x-bf-compat only ENABLES the check, it cannot", + " // force a conversion. On a chat-capable model (this row used openai/gpt-4o-mini until", + " // #6831 testing caught it) nothing converts and the assertions below pass vacuously.", + " var j = pm.response.json();", + " var ef = j.extra_fields || {};", + " pm.expect(ef.converted_request_type, 'chat->responses conversion did not fire, so the replayed tool call never reached ToResponsesRequest and this row proves nothing').to.eql('responses');", + "});", "pm.test('No unknown_parameter input[N].role error - #5101', function () {", " var body = pm.response.text() || '';", " pm.expect(body, 'OpenAI rejected a replayed function_call input item (regression of PR #5102)').to.not.match(/input\\[\\d+\\]\\.role/);", @@ -41047,31 +42558,31 @@ "script": { "type": "text/javascript", "exec": [ - "pm.test('Chat->Responses conversion actually fired - #5101', function () {", - " // markForConversion is datasheet-driven: x-bf-compat only ENABLES the check, it cannot", - " // force a conversion. On a chat-capable model (this row used openai/gpt-4o-mini until", - " // #6831 testing caught it) nothing converts and the assertions below pass vacuously.", - " //", - " // This row streams, so the body is SSE frames rather than one JSON document and", - " // pm.response.json() throws on it. The marker rides extra_fields on the chunks (compat's", - " // PostLLMHook runs per chunk) and is omitempty, so one frame carrying it is enough.", - " var converted = [];", - " (pm.response.text() || '').split('\\n').forEach(function (line) {", - " var t = line.replace(/\\r$/, '');", - " if (t.indexOf('data:') !== 0) { return; }", - " var payload = t.slice(5).trim();", - " if (!payload || payload === '[DONE]') { return; }", - " try {", - " var frame = JSON.parse(payload);", - " var ef = frame.extra_fields || {};", - " if (ef.converted_request_type) { converted.push(ef.converted_request_type); }", - " } catch (e) {}", - " });", - " pm.expect(converted, 'chat->responses conversion did not fire on any streamed chunk, so the replayed tool call never reached ToResponsesRequest and this row proves nothing').to.not.be.empty;", - " converted.forEach(function (marker) {", - " pm.expect(marker, 'a streamed chunk reported a different conversion target').to.eql('responses');", - " });", - "});", + "pm.test('Chat->Responses conversion actually fired - #5101', function () {", + " // markForConversion is datasheet-driven: x-bf-compat only ENABLES the check, it cannot", + " // force a conversion. On a chat-capable model (this row used openai/gpt-4o-mini until", + " // #6831 testing caught it) nothing converts and the assertions below pass vacuously.", + " //", + " // This row streams, so the body is SSE frames rather than one JSON document and", + " // pm.response.json() throws on it. The marker rides extra_fields on the chunks (compat's", + " // PostLLMHook runs per chunk) and is omitempty, so one frame carrying it is enough.", + " var converted = [];", + " (pm.response.text() || '').split('\\n').forEach(function (line) {", + " var t = line.replace(/\\r$/, '');", + " if (t.indexOf('data:') !== 0) { return; }", + " var payload = t.slice(5).trim();", + " if (!payload || payload === '[DONE]') { return; }", + " try {", + " var frame = JSON.parse(payload);", + " var ef = frame.extra_fields || {};", + " if (ef.converted_request_type) { converted.push(ef.converted_request_type); }", + " } catch (e) {}", + " });", + " pm.expect(converted, 'chat->responses conversion did not fire on any streamed chunk, so the replayed tool call never reached ToResponsesRequest and this row proves nothing').to.not.be.empty;", + " converted.forEach(function (marker) {", + " pm.expect(marker, 'a streamed chunk reported a different conversion target').to.eql('responses');", + " });", + "});", "pm.test('No unknown_parameter input[N].role error - #5101', function () {", " var body = pm.response.text() || '';", " pm.expect(body, 'OpenAI rejected a replayed function_call input item (regression of PR #5102)').to.not.match(/input\\[\\d+\\]\\.role/);", @@ -46755,7 +48266,7 @@ "});", "if (pm.response.code >= 400) { return; }", "var body = pm.response.json();", - "pm.test('invoke turn 1 (Haiku 4.5): cache_control nested inside a tool_result block's own content creates a Bedrock cache entry - #5560 / #5629 (turn 1: creates a cache entry)', function () {", + "pm.test('invoke turn 1 (Haiku 4.5): cache_control nested inside a tool_result block\\'s own content creates a Bedrock cache entry - #5560 / #5629 (turn 1: creates a cache entry)', function () {", " var usage = body.usage || {};", " var cacheTokens = (usage.cache_creation_input_tokens || 0) + (usage.cache_read_input_tokens || 0);", " pm.expect(cacheTokens, 'expected a cache breakpoint to be honored, usage=' + JSON.stringify(usage)).to.be.above(0);", @@ -46955,7 +48466,7 @@ "});", "if (pm.response.code >= 400) { return; }", "var body = pm.response.json();", - "pm.test('invoke turn 1 (Sonnet 4.6): cache_control nested inside a tool_result block's own content creates a Bedrock cache entry - #5560 / #5629 (turn 1: creates a cache entry)', function () {", + "pm.test('invoke turn 1 (Sonnet 4.6): cache_control nested inside a tool_result block\\'s own content creates a Bedrock cache entry - #5560 / #5629 (turn 1: creates a cache entry)', function () {", " var usage = body.usage || {};", " var cacheTokens = (usage.cache_creation_input_tokens || 0) + (usage.cache_read_input_tokens || 0);", " pm.expect(cacheTokens, 'expected a cache breakpoint to be honored, usage=' + JSON.stringify(usage)).to.be.above(0);", @@ -47155,7 +48666,7 @@ "});", "if (pm.response.code >= 400) { return; }", "var body = pm.response.json();", - "pm.test('invoke turn 1 (Opus 4.7): cache_control nested inside a tool_result block's own content creates a Bedrock cache entry - #5560 / #5629 (turn 1: creates a cache entry)', function () {", + "pm.test('invoke turn 1 (Opus 4.7): cache_control nested inside a tool_result block\\'s own content creates a Bedrock cache entry - #5560 / #5629 (turn 1: creates a cache entry)', function () {", " var usage = body.usage || {};", " var cacheTokens = (usage.cache_creation_input_tokens || 0) + (usage.cache_read_input_tokens || 0);", " pm.expect(cacheTokens, 'expected a cache breakpoint to be honored, usage=' + JSON.stringify(usage)).to.be.above(0);", @@ -47454,7 +48965,7 @@ "});", "if (pm.response.code >= 400) { return; }", "var body = pm.response.json();", - "pm.test('unified turn 1 (Bedrock Mantle, Opus 4.8): cache_control nested inside a tool_result block's own content creates a cache entry (cross-provider sanity) - #5560 / #5629 (turn 1: creates a cache entry)', function () {", + "pm.test('unified turn 1 (Bedrock Mantle, Opus 4.8): cache_control nested inside a tool_result block\\'s own content creates a cache entry (cross-provider sanity) - #5560 / #5629 (turn 1: creates a cache entry)', function () {", " var details = (body.usage || {}).prompt_tokens_details || {};", " var cacheTokens = (details.cached_tokens || 0) + (details.cache_write_tokens || 0);", " pm.expect(cacheTokens, 'expected a cache breakpoint to be honored, usage=' + JSON.stringify(body.usage)).to.be.above(0);", @@ -47677,7 +49188,7 @@ "}).filter(function (e) { return e; });", "var messageDelta = events.filter(function (e) { return e.type === 'message_delta'; })[0];", "pm.expect(messageDelta, 'expected a message_delta event in the decoded stream: ' + JSON.stringify(events).slice(0, 500)).to.be.an('object');", - "pm.test('invoke-with-response-stream turn 1 (Haiku 4.5): cache_control nested inside a tool_result block's own content creates a Bedrock cache entry - #5560 / #5629 (turn 1: creates a cache entry)', function () {", + "pm.test('invoke-with-response-stream turn 1 (Haiku 4.5): cache_control nested inside a tool_result block\\'s own content creates a Bedrock cache entry - #5560 / #5629 (turn 1: creates a cache entry)', function () {", " var usage = (messageDelta && messageDelta.usage) || {};", " var cacheTokens = (usage.cache_creation_input_tokens || 0) + (usage.cache_read_input_tokens || 0);", " pm.expect(cacheTokens, 'expected a cache breakpoint to be honored, usage=' + JSON.stringify(usage)).to.be.above(0);", @@ -47891,7 +49402,7 @@ "});", "if (pm.response.code >= 400) { return; }", "var body = pm.response.json();", - "pm.test('converse turn 1 (Haiku 4.5): cachePoint nested inside a toolResult's own content creates a Bedrock cache entry (turn 1: creates a cache entry)', function () {", + "pm.test('converse turn 1 (Haiku 4.5): cachePoint nested inside a toolResult\\'s own content creates a Bedrock cache entry (turn 1: creates a cache entry)', function () {", " var usage = body.usage || {};", " var cacheTokens = (usage.cacheWriteInputTokens || 0) + (usage.cacheReadInputTokens || 0);", " pm.expect(cacheTokens, 'expected a cache breakpoint to be honored, usage=' + JSON.stringify(usage)).to.be.above(0);", @@ -48091,7 +49602,7 @@ "});", "if (pm.response.code >= 400) { return; }", "var body = pm.response.json();", - "pm.test('converse turn 1 (Sonnet 4.6): cachePoint nested inside a toolResult's own content creates a Bedrock cache entry (turn 1: creates a cache entry)', function () {", + "pm.test('converse turn 1 (Sonnet 4.6): cachePoint nested inside a toolResult\\'s own content creates a Bedrock cache entry (turn 1: creates a cache entry)', function () {", " var usage = body.usage || {};", " var cacheTokens = (usage.cacheWriteInputTokens || 0) + (usage.cacheReadInputTokens || 0);", " pm.expect(cacheTokens, 'expected a cache breakpoint to be honored, usage=' + JSON.stringify(usage)).to.be.above(0);", @@ -48291,7 +49802,7 @@ "});", "if (pm.response.code >= 400) { return; }", "var body = pm.response.json();", - "pm.test('converse turn 1 (Opus 4.7): cachePoint nested inside a toolResult's own content creates a Bedrock cache entry (turn 1: creates a cache entry)', function () {", + "pm.test('converse turn 1 (Opus 4.7): cachePoint nested inside a toolResult\\'s own content creates a Bedrock cache entry (turn 1: creates a cache entry)', function () {", " var usage = body.usage || {};", " var cacheTokens = (usage.cacheWriteInputTokens || 0) + (usage.cacheReadInputTokens || 0);", " pm.expect(cacheTokens, 'expected a cache breakpoint to be honored, usage=' + JSON.stringify(usage)).to.be.above(0);", @@ -48593,7 +50104,7 @@ "var rawText = (pm.response.stream ? Buffer.from(pm.response.stream).toString('binary') : pm.response.text()) || '';", "var cw = (rawText.match(/\"cacheWriteInputTokens\":(\\d+)/) || [])[1];", "var cr = (rawText.match(/\"cacheReadInputTokens\":(\\d+)/) || [])[1];", - "pm.test('converse-stream turn 1 (Haiku 4.5): cachePoint nested inside a toolResult's own content creates a Bedrock cache entry (turn 1: creates a cache entry)', function () {", + "pm.test('converse-stream turn 1 (Haiku 4.5): cachePoint nested inside a toolResult\\'s own content creates a Bedrock cache entry (turn 1: creates a cache entry)', function () {", " var cacheTokens = (parseInt(cw, 10) || 0) + (parseInt(cr, 10) || 0);", " pm.expect(cacheTokens, 'expected a cache breakpoint to be honored: ' + rawText.slice(-500)).to.be.above(0);", "});" @@ -48923,7 +50434,7 @@ "});", "if (pm.response.code >= 400) { return; }", "var body = pm.response.json();", - "pm.test('unified turn 1 (Vertex Claude Opus 4.7): cache_control nested inside a tool_result block's own content creates a cache entry (turn 1: creates a cache entry)', function () {", + "pm.test('unified turn 1 (Vertex Claude Opus 4.7): cache_control nested inside a tool_result block\\'s own content creates a cache entry (turn 1: creates a cache entry)', function () {", " var details = (body.usage || {}).prompt_tokens_details || {};", " var cacheTokens = (details.cached_tokens || 0) + (details.cache_write_tokens || 0);", " pm.expect(cacheTokens, 'expected a cache breakpoint to be honored, usage=' + JSON.stringify(body.usage)).to.be.above(0);", @@ -49136,7 +50647,7 @@ "});", "if (pm.response.code >= 400) { return; }", "var body = pm.response.json();", - "pm.test('anthropic native turn 1 (Haiku 4.5): cache_control nested inside a tool_result block's own content creates a cache entry (turn 1: creates a cache entry)', function () {", + "pm.test('anthropic native turn 1 (Haiku 4.5): cache_control nested inside a tool_result block\\'s own content creates a cache entry (turn 1: creates a cache entry)', function () {", " var usage = body.usage || {};", " var cacheTokens = (usage.cache_creation_input_tokens || 0) + (usage.cache_read_input_tokens || 0);", " pm.expect(cacheTokens, 'expected a cache breakpoint to be honored, usage=' + JSON.stringify(usage)).to.be.above(0);", @@ -60826,7 +62337,7 @@ ] }, { - "name": "[PREVIEW] 47.5.A native /v1/chat/completions -> azure/o3-mini (o3-mini deployment is account-scoped)", + "name": "[PREVIEW] 47.5.A native /v1/chat/completions -> azure/gpt-5.4-mini", "request": { "method": "POST", "header": [ @@ -60837,7 +62348,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"model\": \"azure/o3-mini\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"A farmer has 47 sheep. Each night, one third of the flock (rounded down) escapes, and each morning he recovers 5 plus the number that escaped two nights prior. Starting from night 1, how many sheep does he have after the 4th morning? Show your working.\"\n }\n ],\n \"max_tokens\": 2048,\n \"reasoning_effort\": \"low\"\n}" + "raw": "{\n \"model\": \"azure/gpt-5.4-mini\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"A farmer has 47 sheep. Each night, one third of the flock (rounded down) escapes, and each morning he recovers 5 plus the number that escaped two nights prior. Starting from night 1, how many sheep does he have after the 4th morning? Show your working.\"\n }\n ],\n \"max_tokens\": 2048,\n \"reasoning_effort\": \"low\"\n}" }, "url": { "raw": "{{baseUrl}}/v1/chat/completions", @@ -61254,7 +62765,7 @@ ] }, { - "name": "[PREVIEW] 47.5.B native /v1/responses -> azure/o3-mini (o3-mini deployment is account-scoped)", + "name": "[PREVIEW] 47.5.B native /v1/responses -> azure/gpt-5.4-mini", "request": { "method": "POST", "header": [ @@ -61265,7 +62776,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"model\": \"azure/o3-mini\",\n \"input\": \"A farmer has 47 sheep. Each night, one third of the flock (rounded down) escapes, and each morning he recovers 5 plus the number that escaped two nights prior. Starting from night 1, how many sheep does he have after the 4th morning? Show your working.\",\n \"max_output_tokens\": 2048,\n \"reasoning\": {\n \"effort\": \"low\"\n }\n}" + "raw": "{\n \"model\": \"azure/gpt-5.4-mini\",\n \"input\": \"A farmer has 47 sheep. Each night, one third of the flock (rounded down) escapes, and each morning he recovers 5 plus the number that escaped two nights prior. Starting from night 1, how many sheep does he have after the 4th morning? Show your working.\",\n \"max_output_tokens\": 2048,\n \"reasoning\": {\n \"effort\": \"low\"\n }\n}" }, "url": { "raw": "{{baseUrl}}/v1/responses", @@ -61647,7 +63158,7 @@ ] }, { - "name": "[PREVIEW] 47.5.C /openai/v1/chat/completions -> azure/o3-mini (o3-mini deployment is account-scoped)", + "name": "[PREVIEW] 47.5.C /openai/v1/chat/completions -> azure/gpt-5.4-mini", "request": { "method": "POST", "header": [ @@ -61658,7 +63169,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"model\": \"azure/o3-mini\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"A farmer has 47 sheep. Each night, one third of the flock (rounded down) escapes, and each morning he recovers 5 plus the number that escaped two nights prior. Starting from night 1, how many sheep does he have after the 4th morning? Show your working.\"\n }\n ],\n \"max_tokens\": 2048,\n \"reasoning_effort\": \"low\"\n}" + "raw": "{\n \"model\": \"azure/gpt-5.4-mini\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"A farmer has 47 sheep. Each night, one third of the flock (rounded down) escapes, and each morning he recovers 5 plus the number that escaped two nights prior. Starting from night 1, how many sheep does he have after the 4th morning? Show your working.\"\n }\n ],\n \"max_tokens\": 2048,\n \"reasoning_effort\": \"low\"\n}" }, "url": { "raw": "{{baseUrl}}/openai/v1/chat/completions", @@ -62037,7 +63548,7 @@ ] }, { - "name": "[PREVIEW] 47.5.D /openai/v1/responses -> azure/o3-mini (o3-mini deployment is account-scoped)", + "name": "[PREVIEW] 47.5.D /openai/v1/responses -> azure/gpt-5.4-mini", "request": { "method": "POST", "header": [ @@ -62048,7 +63559,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"model\": \"azure/o3-mini\",\n \"input\": \"A farmer has 47 sheep. Each night, one third of the flock (rounded down) escapes, and each morning he recovers 5 plus the number that escaped two nights prior. Starting from night 1, how many sheep does he have after the 4th morning? Show your working.\",\n \"max_output_tokens\": 2048,\n \"reasoning\": {\n \"effort\": \"low\"\n }\n}" + "raw": "{\n \"model\": \"azure/gpt-5.4-mini\",\n \"input\": \"A farmer has 47 sheep. Each night, one third of the flock (rounded down) escapes, and each morning he recovers 5 plus the number that escaped two nights prior. Starting from night 1, how many sheep does he have after the 4th morning? Show your working.\",\n \"max_output_tokens\": 2048,\n \"reasoning\": {\n \"effort\": \"low\"\n }\n}" }, "url": { "raw": "{{baseUrl}}/openai/v1/responses", @@ -62418,7 +63929,7 @@ ] }, { - "name": "[PREVIEW] 47.5.E /anthropic/v1/messages -> azure/o3-mini (o3-mini deployment is account-scoped)", + "name": "[PREVIEW] 47.5.E /anthropic/v1/messages -> azure/gpt-5.4-mini", "request": { "method": "POST", "header": [ @@ -62429,7 +63940,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"model\": \"azure/o3-mini\",\n \"max_tokens\": 2048,\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"A farmer has 47 sheep. Each night, one third of the flock (rounded down) escapes, and each morning he recovers 5 plus the number that escaped two nights prior. Starting from night 1, how many sheep does he have after the 4th morning? Show your working.\"\n }\n ],\n \"thinking\": {\n \"type\": \"enabled\",\n \"budget_tokens\": 1024\n }\n}" + "raw": "{\n \"model\": \"azure/gpt-5.4-mini\",\n \"max_tokens\": 2048,\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"A farmer has 47 sheep. Each night, one third of the flock (rounded down) escapes, and each morning he recovers 5 plus the number that escaped two nights prior. Starting from night 1, how many sheep does he have after the 4th morning? Show your working.\"\n }\n ],\n \"thinking\": {\n \"type\": \"enabled\",\n \"budget_tokens\": 1024\n }\n}" }, "url": { "raw": "{{baseUrl}}/anthropic/v1/messages", @@ -62814,7 +64325,7 @@ ] }, { - "name": "[PREVIEW] 47.5.F /genai :generateContent -> azure/o3-mini (o3-mini deployment is account-scoped)", + "name": "[PREVIEW] 47.5.F /genai :generateContent -> azure/gpt-5.4-mini", "request": { "method": "POST", "header": [ @@ -62828,7 +64339,7 @@ "raw": "{\n \"contents\": [\n {\n \"role\": \"user\",\n \"parts\": [\n {\n \"text\": \"A farmer has 47 sheep. Each night, one third of the flock (rounded down) escapes, and each morning he recovers 5 plus the number that escaped two nights prior. Starting from night 1, how many sheep does he have after the 4th morning? Show your working.\"\n }\n ]\n }\n ],\n \"generationConfig\": {\n \"maxOutputTokens\": 2048,\n \"thinkingConfig\": {\n \"thinkingBudget\": 1024,\n \"includeThoughts\": true\n }\n }\n}" }, "url": { - "raw": "{{baseUrl}}/genai/v1beta/models/azure/o3-mini:generateContent", + "raw": "{{baseUrl}}/genai/v1beta/models/azure/gpt-5.4-mini:generateContent", "host": [ "{{baseUrl}}" ], @@ -62837,7 +64348,7 @@ "v1beta", "models", "azure", - "o3-mini:generateContent" + "gpt-5.4-mini:generateContent" ] } }, @@ -63209,7 +64720,7 @@ ] }, { - "name": "[PREVIEW] 47.5.G /bedrock converse -> azure/o3-mini (o3-mini deployment is account-scoped)", + "name": "[PREVIEW] 47.5.G /bedrock converse -> azure/gpt-5.4-mini", "request": { "method": "POST", "header": [ @@ -63223,14 +64734,14 @@ "raw": "{\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": [\n {\n \"text\": \"A farmer has 47 sheep. Each night, one third of the flock (rounded down) escapes, and each morning he recovers 5 plus the number that escaped two nights prior. Starting from night 1, how many sheep does he have after the 4th morning? Show your working.\"\n }\n ]\n }\n ],\n \"inferenceConfig\": {\n \"maxTokens\": 4096\n },\n \"additionalModelRequestFields\": {\n \"reasoning_config\": {\n \"type\": \"enabled\",\n \"budget_tokens\": 1024\n }\n }\n}" }, "url": { - "raw": "{{baseUrl}}/bedrock/model/azure%2Fo3-mini/converse", + "raw": "{{baseUrl}}/bedrock/model/azure%2Fgpt-5.4-mini/converse", "host": [ "{{baseUrl}}" ], "path": [ "bedrock", "model", - "azure%2Fo3-mini", + "azure%2Fgpt-5.4-mini", "converse" ] } @@ -66165,7 +67676,7 @@ ] }, { - "name": "[PREVIEW] 47.6.2.A native /v1/chat/completions -> azure/o3-mini (o3-mini deployment is account-scoped)", + "name": "[PREVIEW] 47.6.2.A native /v1/chat/completions -> azure/gpt-5.4-mini", "request": { "method": "POST", "header": [ @@ -66176,7 +67687,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"model\": \"azure/o3-mini\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"A farmer has 47 sheep. Each night, one third of the flock (rounded down) escapes, and each morning he recovers 5 plus the number that escaped two nights prior. Starting from night 1, how many sheep does he have after the 4th morning? Show your working.\"\n }\n ],\n \"max_tokens\": 2048,\n \"reasoning_effort\": \"low\",\n \"stream\": true\n}" + "raw": "{\n \"model\": \"azure/gpt-5.4-mini\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"A farmer has 47 sheep. Each night, one third of the flock (rounded down) escapes, and each morning he recovers 5 plus the number that escaped two nights prior. Starting from night 1, how many sheep does he have after the 4th morning? Show your working.\"\n }\n ],\n \"max_tokens\": 2048,\n \"reasoning_effort\": \"low\",\n \"stream\": true\n}" }, "url": { "raw": "{{baseUrl}}/v1/chat/completions", @@ -66542,7 +68053,7 @@ ] }, { - "name": "[PREVIEW] 47.6.2.B native /v1/responses -> azure/o3-mini (o3-mini deployment is account-scoped)", + "name": "[PREVIEW] 47.6.2.B native /v1/responses -> azure/gpt-5.4-mini", "request": { "method": "POST", "header": [ @@ -66553,7 +68064,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"model\": \"azure/o3-mini\",\n \"input\": \"A farmer has 47 sheep. Each night, one third of the flock (rounded down) escapes, and each morning he recovers 5 plus the number that escaped two nights prior. Starting from night 1, how many sheep does he have after the 4th morning? Show your working.\",\n \"max_output_tokens\": 2048,\n \"reasoning\": {\n \"effort\": \"low\"\n },\n \"stream\": true\n}" + "raw": "{\n \"model\": \"azure/gpt-5.4-mini\",\n \"input\": \"A farmer has 47 sheep. Each night, one third of the flock (rounded down) escapes, and each morning he recovers 5 plus the number that escaped two nights prior. Starting from night 1, how many sheep does he have after the 4th morning? Show your working.\",\n \"max_output_tokens\": 2048,\n \"reasoning\": {\n \"effort\": \"low\"\n },\n \"stream\": true\n}" }, "url": { "raw": "{{baseUrl}}/v1/responses", @@ -66885,7 +68396,7 @@ ] }, { - "name": "[PREVIEW] 47.6.2.C /openai/v1/chat/completions -> azure/o3-mini (o3-mini deployment is account-scoped)", + "name": "[PREVIEW] 47.6.2.C /openai/v1/chat/completions -> azure/gpt-5.4-mini", "request": { "method": "POST", "header": [ @@ -66896,7 +68407,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"model\": \"azure/o3-mini\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"A farmer has 47 sheep. Each night, one third of the flock (rounded down) escapes, and each morning he recovers 5 plus the number that escaped two nights prior. Starting from night 1, how many sheep does he have after the 4th morning? Show your working.\"\n }\n ],\n \"max_tokens\": 2048,\n \"reasoning_effort\": \"low\",\n \"stream\": true\n}" + "raw": "{\n \"model\": \"azure/gpt-5.4-mini\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"A farmer has 47 sheep. Each night, one third of the flock (rounded down) escapes, and each morning he recovers 5 plus the number that escaped two nights prior. Starting from night 1, how many sheep does he have after the 4th morning? Show your working.\"\n }\n ],\n \"max_tokens\": 2048,\n \"reasoning_effort\": \"low\",\n \"stream\": true\n}" }, "url": { "raw": "{{baseUrl}}/openai/v1/chat/completions", @@ -67229,7 +68740,7 @@ ] }, { - "name": "[PREVIEW] 47.6.2.D /openai/v1/responses -> azure/o3-mini (o3-mini deployment is account-scoped)", + "name": "[PREVIEW] 47.6.2.D /openai/v1/responses -> azure/gpt-5.4-mini", "request": { "method": "POST", "header": [ @@ -67240,7 +68751,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"model\": \"azure/o3-mini\",\n \"input\": \"A farmer has 47 sheep. Each night, one third of the flock (rounded down) escapes, and each morning he recovers 5 plus the number that escaped two nights prior. Starting from night 1, how many sheep does he have after the 4th morning? Show your working.\",\n \"max_output_tokens\": 2048,\n \"reasoning\": {\n \"effort\": \"low\"\n },\n \"stream\": true\n}" + "raw": "{\n \"model\": \"azure/gpt-5.4-mini\",\n \"input\": \"A farmer has 47 sheep. Each night, one third of the flock (rounded down) escapes, and each morning he recovers 5 plus the number that escaped two nights prior. Starting from night 1, how many sheep does he have after the 4th morning? Show your working.\",\n \"max_output_tokens\": 2048,\n \"reasoning\": {\n \"effort\": \"low\"\n },\n \"stream\": true\n}" }, "url": { "raw": "{{baseUrl}}/openai/v1/responses", @@ -67570,7 +69081,7 @@ ] }, { - "name": "[PREVIEW] 47.6.2.E /anthropic/v1/messages -> azure/o3-mini (o3-mini deployment is account-scoped)", + "name": "[PREVIEW] 47.6.2.E /anthropic/v1/messages -> azure/gpt-5.4-mini", "request": { "method": "POST", "header": [ @@ -67581,7 +69092,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"model\": \"azure/o3-mini\",\n \"max_tokens\": 2048,\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"A farmer has 47 sheep. Each night, one third of the flock (rounded down) escapes, and each morning he recovers 5 plus the number that escaped two nights prior. Starting from night 1, how many sheep does he have after the 4th morning? Show your working.\"\n }\n ],\n \"thinking\": {\n \"type\": \"enabled\",\n \"budget_tokens\": 1024\n },\n \"stream\": true\n}" + "raw": "{\n \"model\": \"azure/gpt-5.4-mini\",\n \"max_tokens\": 2048,\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"A farmer has 47 sheep. Each night, one third of the flock (rounded down) escapes, and each morning he recovers 5 plus the number that escaped two nights prior. Starting from night 1, how many sheep does he have after the 4th morning? Show your working.\"\n }\n ],\n \"thinking\": {\n \"type\": \"enabled\",\n \"budget_tokens\": 1024\n },\n \"stream\": true\n}" }, "url": { "raw": "{{baseUrl}}/anthropic/v1/messages", @@ -67951,7 +69462,7 @@ ] }, { - "name": "[PREVIEW] 47.6.2.F /genai :generateContent -> azure/o3-mini (o3-mini deployment is account-scoped)", + "name": "[PREVIEW] 47.6.2.F /genai :generateContent -> azure/gpt-5.4-mini", "request": { "method": "POST", "header": [ @@ -67965,7 +69476,7 @@ "raw": "{\n \"contents\": [\n {\n \"role\": \"user\",\n \"parts\": [\n {\n \"text\": \"A farmer has 47 sheep. Each night, one third of the flock (rounded down) escapes, and each morning he recovers 5 plus the number that escaped two nights prior. Starting from night 1, how many sheep does he have after the 4th morning? Show your working.\"\n }\n ]\n }\n ],\n \"generationConfig\": {\n \"maxOutputTokens\": 2048,\n \"thinkingConfig\": {\n \"thinkingBudget\": 1024,\n \"includeThoughts\": true\n }\n }\n}" }, "url": { - "raw": "{{baseUrl}}/genai/v1beta/models/azure/o3-mini:streamGenerateContent?alt=sse", + "raw": "{{baseUrl}}/genai/v1beta/models/azure/gpt-5.4-mini:streamGenerateContent?alt=sse", "host": [ "{{baseUrl}}" ], @@ -67974,7 +69485,7 @@ "v1beta", "models", "azure", - "o3-mini:streamGenerateContent" + "gpt-5.4-mini:streamGenerateContent" ], "query": [ { @@ -68351,7 +69862,7 @@ ] }, { - "name": "[PREVIEW] 47.6.2.G /bedrock converse -> azure/o3-mini (o3-mini deployment is account-scoped)", + "name": "[PREVIEW] 47.6.2.G /bedrock converse -> azure/gpt-5.4-mini", "request": { "method": "POST", "header": [ @@ -68365,14 +69876,14 @@ "raw": "{\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": [\n {\n \"text\": \"A farmer has 47 sheep. Each night, one third of the flock (rounded down) escapes, and each morning he recovers 5 plus the number that escaped two nights prior. Starting from night 1, how many sheep does he have after the 4th morning? Show your working.\"\n }\n ]\n }\n ],\n \"inferenceConfig\": {\n \"maxTokens\": 4096\n },\n \"additionalModelRequestFields\": {\n \"reasoning_config\": {\n \"type\": \"enabled\",\n \"budget_tokens\": 1024\n }\n }\n}" }, "url": { - "raw": "{{baseUrl}}/bedrock/model/azure%2Fo3-mini/converse-stream", + "raw": "{{baseUrl}}/bedrock/model/azure%2Fgpt-5.4-mini/converse-stream", "host": [ "{{baseUrl}}" ], "path": [ "bedrock", "model", - "azure%2Fo3-mini", + "azure%2Fgpt-5.4-mini", "converse-stream" ] } @@ -86125,7 +87636,7 @@ ] }, { - "name": "[PREVIEW] 48.1.O /langchain reasoning -> azure/o3-mini (o3-mini deployment is account-scoped)", + "name": "[PREVIEW] 48.1.O /langchain reasoning -> azure/gpt-5.4-mini", "request": { "method": "POST", "header": [ @@ -86136,7 +87647,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"model\": \"azure/o3-mini\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"A farmer has 47 sheep. Each night, one third of the flock (rounded down) escapes, and each morning he recovers 5 plus the number that escaped two nights prior. Starting from night 1, how many sheep does he have after the 4th morning? Show your working.\"\n }\n ],\n \"max_tokens\": 4096,\n \"reasoning_effort\": \"low\"\n}" + "raw": "{\n \"model\": \"azure/gpt-5.4-mini\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"A farmer has 47 sheep. Each night, one third of the flock (rounded down) escapes, and each morning he recovers 5 plus the number that escaped two nights prior. Starting from night 1, how many sheep does he have after the 4th morning? Show your working.\"\n }\n ],\n \"max_tokens\": 4096,\n \"reasoning_effort\": \"low\"\n}" }, "url": { "raw": "{{baseUrl}}/langchain/v1/chat/completions", @@ -88202,7 +89713,7 @@ ] }, { - "name": "[PREVIEW] 48.1.R /langchain reasoning -> azure/o3-mini (o3-mini deployment is account-scoped)", + "name": "[PREVIEW] 48.1.R /langchain reasoning -> azure/gpt-5.4-mini", "request": { "method": "POST", "header": [ @@ -88213,7 +89724,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"model\": \"azure/o3-mini\",\n \"input\": \"A farmer has 47 sheep. Each night, one third of the flock (rounded down) escapes, and each morning he recovers 5 plus the number that escaped two nights prior. Starting from night 1, how many sheep does he have after the 4th morning? Show your working.\",\n \"max_output_tokens\": 2048,\n \"reasoning\": {\n \"effort\": \"low\"\n }\n}" + "raw": "{\n \"model\": \"azure/gpt-5.4-mini\",\n \"input\": \"A farmer has 47 sheep. Each night, one third of the flock (rounded down) escapes, and each morning he recovers 5 plus the number that escaped two nights prior. Starting from night 1, how many sheep does he have after the 4th morning? Show your working.\",\n \"max_output_tokens\": 2048,\n \"reasoning\": {\n \"effort\": \"low\"\n }\n}" }, "url": { "raw": "{{baseUrl}}/langchain/v1/responses", @@ -90319,7 +91830,7 @@ ] }, { - "name": "[PREVIEW] 48.1.A /langchain reasoning -> azure/o3-mini (o3-mini deployment is account-scoped)", + "name": "[PREVIEW] 48.1.A /langchain reasoning -> azure/gpt-5.4-mini", "request": { "method": "POST", "header": [ @@ -90330,7 +91841,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"model\": \"azure/o3-mini\",\n \"max_tokens\": 4096,\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"A farmer has 47 sheep. Each night, one third of the flock (rounded down) escapes, and each morning he recovers 5 plus the number that escaped two nights prior. Starting from night 1, how many sheep does he have after the 4th morning? Show your working.\"\n }\n ],\n \"thinking\": {\n \"type\": \"enabled\",\n \"budget_tokens\": 1024\n }\n}" + "raw": "{\n \"model\": \"azure/gpt-5.4-mini\",\n \"max_tokens\": 4096,\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"A farmer has 47 sheep. Each night, one third of the flock (rounded down) escapes, and each morning he recovers 5 plus the number that escaped two nights prior. Starting from night 1, how many sheep does he have after the 4th morning? Show your working.\"\n }\n ],\n \"thinking\": {\n \"type\": \"enabled\",\n \"budget_tokens\": 1024\n }\n}" }, "url": { "raw": "{{baseUrl}}/langchain/v1/messages", @@ -92499,7 +94010,7 @@ ] }, { - "name": "[PREVIEW] 48.1.G /langchain reasoning -> azure/o3-mini (o3-mini deployment is account-scoped)", + "name": "[PREVIEW] 48.1.G /langchain reasoning -> azure/gpt-5.4-mini", "request": { "method": "POST", "header": [ @@ -92513,7 +94024,7 @@ "raw": "{\n \"contents\": [\n {\n \"role\": \"user\",\n \"parts\": [\n {\n \"text\": \"A farmer has 47 sheep. Each night, one third of the flock (rounded down) escapes, and each morning he recovers 5 plus the number that escaped two nights prior. Starting from night 1, how many sheep does he have after the 4th morning? Show your working.\"\n }\n ]\n }\n ],\n \"generationConfig\": {\n \"maxOutputTokens\": 2048,\n \"thinkingConfig\": {\n \"thinkingBudget\": 1024,\n \"includeThoughts\": true\n }\n }\n}" }, "url": { - "raw": "{{baseUrl}}/langchain/v1beta/models/azure/o3-mini:generateContent", + "raw": "{{baseUrl}}/langchain/v1beta/models/azure/gpt-5.4-mini:generateContent", "host": [ "{{baseUrl}}" ], @@ -92522,7 +94033,7 @@ "v1beta", "models", "azure", - "o3-mini:generateContent" + "gpt-5.4-mini:generateContent" ] } }, @@ -94742,7 +96253,7 @@ ] }, { - "name": "[PREVIEW] 48.1.B /langchain reasoning -> azure/o3-mini (o3-mini deployment is account-scoped)", + "name": "[PREVIEW] 48.1.B /langchain reasoning -> azure/gpt-5.4-mini", "request": { "method": "POST", "header": [ @@ -94756,14 +96267,14 @@ "raw": "{\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": [\n {\n \"text\": \"A farmer has 47 sheep. Each night, one third of the flock (rounded down) escapes, and each morning he recovers 5 plus the number that escaped two nights prior. Starting from night 1, how many sheep does he have after the 4th morning? Show your working.\"\n }\n ]\n }\n ],\n \"inferenceConfig\": {\n \"maxTokens\": 4096\n },\n \"additionalModelRequestFields\": {\n \"reasoning_config\": {\n \"type\": \"enabled\",\n \"budget_tokens\": 1024\n }\n }\n}" }, "url": { - "raw": "{{baseUrl}}/langchain/model/azure%2Fo3-mini/converse", + "raw": "{{baseUrl}}/langchain/model/azure%2Fgpt-5.4-mini/converse", "host": [ "{{baseUrl}}" ], "path": [ "langchain", "model", - "azure%2Fo3-mini", + "azure%2Fgpt-5.4-mini", "converse" ] } @@ -97099,7 +98610,7 @@ ] }, { - "name": "[PREVIEW] 48.1.C /langchain reasoning -> azure/o3-mini (o3-mini deployment is account-scoped)", + "name": "[PREVIEW] 48.1.C /langchain reasoning -> azure/gpt-5.4-mini", "request": { "method": "POST", "header": [ @@ -97110,7 +98621,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"model\": \"azure/o3-mini\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"A farmer has 47 sheep. Each night, one third of the flock (rounded down) escapes, and each morning he recovers 5 plus the number that escaped two nights prior. Starting from night 1, how many sheep does he have after the 4th morning? Show your working.\"\n }\n ],\n \"max_tokens\": 4096,\n \"thinking\": {\n \"type\": \"enabled\",\n \"token_budget\": 1024\n }\n}" + "raw": "{\n \"model\": \"azure/gpt-5.4-mini\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"A farmer has 47 sheep. Each night, one third of the flock (rounded down) escapes, and each morning he recovers 5 plus the number that escaped two nights prior. Starting from night 1, how many sheep does he have after the 4th morning? Show your working.\"\n }\n ],\n \"max_tokens\": 4096,\n \"thinking\": {\n \"type\": \"enabled\",\n \"token_budget\": 1024\n }\n}" }, "url": { "raw": "{{baseUrl}}/langchain/v2/chat", @@ -99233,7 +100744,7 @@ ] }, { - "name": "[PREVIEW] 48.2.O /litellm reasoning -> azure/o3-mini (o3-mini deployment is account-scoped)", + "name": "[PREVIEW] 48.2.O /litellm reasoning -> azure/gpt-5.4-mini", "request": { "method": "POST", "header": [ @@ -99244,7 +100755,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"model\": \"azure/o3-mini\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"A farmer has 47 sheep. Each night, one third of the flock (rounded down) escapes, and each morning he recovers 5 plus the number that escaped two nights prior. Starting from night 1, how many sheep does he have after the 4th morning? Show your working.\"\n }\n ],\n \"max_tokens\": 4096,\n \"reasoning_effort\": \"low\"\n}" + "raw": "{\n \"model\": \"azure/gpt-5.4-mini\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"A farmer has 47 sheep. Each night, one third of the flock (rounded down) escapes, and each morning he recovers 5 plus the number that escaped two nights prior. Starting from night 1, how many sheep does he have after the 4th morning? Show your working.\"\n }\n ],\n \"max_tokens\": 4096,\n \"reasoning_effort\": \"low\"\n}" }, "url": { "raw": "{{baseUrl}}/litellm/v1/chat/completions", @@ -101310,7 +102821,7 @@ ] }, { - "name": "[PREVIEW] 48.2.R /litellm reasoning -> azure/o3-mini (o3-mini deployment is account-scoped)", + "name": "[PREVIEW] 48.2.R /litellm reasoning -> azure/gpt-5.4-mini", "request": { "method": "POST", "header": [ @@ -101321,7 +102832,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"model\": \"azure/o3-mini\",\n \"input\": \"A farmer has 47 sheep. Each night, one third of the flock (rounded down) escapes, and each morning he recovers 5 plus the number that escaped two nights prior. Starting from night 1, how many sheep does he have after the 4th morning? Show your working.\",\n \"max_output_tokens\": 2048,\n \"reasoning\": {\n \"effort\": \"low\"\n }\n}" + "raw": "{\n \"model\": \"azure/gpt-5.4-mini\",\n \"input\": \"A farmer has 47 sheep. Each night, one third of the flock (rounded down) escapes, and each morning he recovers 5 plus the number that escaped two nights prior. Starting from night 1, how many sheep does he have after the 4th morning? Show your working.\",\n \"max_output_tokens\": 2048,\n \"reasoning\": {\n \"effort\": \"low\"\n }\n}" }, "url": { "raw": "{{baseUrl}}/litellm/v1/responses", @@ -103427,7 +104938,7 @@ ] }, { - "name": "[PREVIEW] 48.2.A /litellm reasoning -> azure/o3-mini (o3-mini deployment is account-scoped)", + "name": "[PREVIEW] 48.2.A /litellm reasoning -> azure/gpt-5.4-mini", "request": { "method": "POST", "header": [ @@ -103438,7 +104949,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"model\": \"azure/o3-mini\",\n \"max_tokens\": 4096,\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"A farmer has 47 sheep. Each night, one third of the flock (rounded down) escapes, and each morning he recovers 5 plus the number that escaped two nights prior. Starting from night 1, how many sheep does he have after the 4th morning? Show your working.\"\n }\n ],\n \"thinking\": {\n \"type\": \"enabled\",\n \"budget_tokens\": 1024\n }\n}" + "raw": "{\n \"model\": \"azure/gpt-5.4-mini\",\n \"max_tokens\": 4096,\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"A farmer has 47 sheep. Each night, one third of the flock (rounded down) escapes, and each morning he recovers 5 plus the number that escaped two nights prior. Starting from night 1, how many sheep does he have after the 4th morning? Show your working.\"\n }\n ],\n \"thinking\": {\n \"type\": \"enabled\",\n \"budget_tokens\": 1024\n }\n}" }, "url": { "raw": "{{baseUrl}}/litellm/v1/messages", @@ -105607,7 +107118,7 @@ ] }, { - "name": "[PREVIEW] 48.2.G /litellm reasoning -> azure/o3-mini (o3-mini deployment is account-scoped)", + "name": "[PREVIEW] 48.2.G /litellm reasoning -> azure/gpt-5.4-mini", "request": { "method": "POST", "header": [ @@ -105621,7 +107132,7 @@ "raw": "{\n \"contents\": [\n {\n \"role\": \"user\",\n \"parts\": [\n {\n \"text\": \"A farmer has 47 sheep. Each night, one third of the flock (rounded down) escapes, and each morning he recovers 5 plus the number that escaped two nights prior. Starting from night 1, how many sheep does he have after the 4th morning? Show your working.\"\n }\n ]\n }\n ],\n \"generationConfig\": {\n \"maxOutputTokens\": 2048,\n \"thinkingConfig\": {\n \"thinkingBudget\": 1024,\n \"includeThoughts\": true\n }\n }\n}" }, "url": { - "raw": "{{baseUrl}}/litellm/v1beta/models/azure/o3-mini:generateContent", + "raw": "{{baseUrl}}/litellm/v1beta/models/azure/gpt-5.4-mini:generateContent", "host": [ "{{baseUrl}}" ], @@ -105630,7 +107141,7 @@ "v1beta", "models", "azure", - "o3-mini:generateContent" + "gpt-5.4-mini:generateContent" ] } }, @@ -107850,7 +109361,7 @@ ] }, { - "name": "[PREVIEW] 48.2.B /litellm reasoning -> azure/o3-mini (o3-mini deployment is account-scoped)", + "name": "[PREVIEW] 48.2.B /litellm reasoning -> azure/gpt-5.4-mini", "request": { "method": "POST", "header": [ @@ -107864,14 +109375,14 @@ "raw": "{\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": [\n {\n \"text\": \"A farmer has 47 sheep. Each night, one third of the flock (rounded down) escapes, and each morning he recovers 5 plus the number that escaped two nights prior. Starting from night 1, how many sheep does he have after the 4th morning? Show your working.\"\n }\n ]\n }\n ],\n \"inferenceConfig\": {\n \"maxTokens\": 4096\n },\n \"additionalModelRequestFields\": {\n \"reasoning_config\": {\n \"type\": \"enabled\",\n \"budget_tokens\": 1024\n }\n }\n}" }, "url": { - "raw": "{{baseUrl}}/litellm/model/azure%2Fo3-mini/converse", + "raw": "{{baseUrl}}/litellm/model/azure%2Fgpt-5.4-mini/converse", "host": [ "{{baseUrl}}" ], "path": [ "litellm", "model", - "azure%2Fo3-mini", + "azure%2Fgpt-5.4-mini", "converse" ] } @@ -110207,7 +111718,7 @@ ] }, { - "name": "[PREVIEW] 48.2.C /litellm reasoning -> azure/o3-mini (o3-mini deployment is account-scoped)", + "name": "[PREVIEW] 48.2.C /litellm reasoning -> azure/gpt-5.4-mini", "request": { "method": "POST", "header": [ @@ -110218,7 +111729,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"model\": \"azure/o3-mini\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"A farmer has 47 sheep. Each night, one third of the flock (rounded down) escapes, and each morning he recovers 5 plus the number that escaped two nights prior. Starting from night 1, how many sheep does he have after the 4th morning? Show your working.\"\n }\n ],\n \"max_tokens\": 4096,\n \"thinking\": {\n \"type\": \"enabled\",\n \"token_budget\": 1024\n }\n}" + "raw": "{\n \"model\": \"azure/gpt-5.4-mini\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"A farmer has 47 sheep. Each night, one third of the flock (rounded down) escapes, and each morning he recovers 5 plus the number that escaped two nights prior. Starting from night 1, how many sheep does he have after the 4th morning? Show your working.\"\n }\n ],\n \"max_tokens\": 4096,\n \"thinking\": {\n \"type\": \"enabled\",\n \"token_budget\": 1024\n }\n}" }, "url": { "raw": "{{baseUrl}}/litellm/v2/chat", @@ -112341,7 +113852,7 @@ ] }, { - "name": "[PREVIEW] 48.3.O /pydanticai reasoning -> azure/o3-mini (o3-mini deployment is account-scoped)", + "name": "[PREVIEW] 48.3.O /pydanticai reasoning -> azure/gpt-5.4-mini", "request": { "method": "POST", "header": [ @@ -112352,7 +113863,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"model\": \"azure/o3-mini\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"A farmer has 47 sheep. Each night, one third of the flock (rounded down) escapes, and each morning he recovers 5 plus the number that escaped two nights prior. Starting from night 1, how many sheep does he have after the 4th morning? Show your working.\"\n }\n ],\n \"max_tokens\": 4096,\n \"reasoning_effort\": \"low\"\n}" + "raw": "{\n \"model\": \"azure/gpt-5.4-mini\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"A farmer has 47 sheep. Each night, one third of the flock (rounded down) escapes, and each morning he recovers 5 plus the number that escaped two nights prior. Starting from night 1, how many sheep does he have after the 4th morning? Show your working.\"\n }\n ],\n \"max_tokens\": 4096,\n \"reasoning_effort\": \"low\"\n}" }, "url": { "raw": "{{baseUrl}}/pydanticai/v1/chat/completions", @@ -114418,7 +115929,7 @@ ] }, { - "name": "[PREVIEW] 48.3.R /pydanticai reasoning -> azure/o3-mini (o3-mini deployment is account-scoped)", + "name": "[PREVIEW] 48.3.R /pydanticai reasoning -> azure/gpt-5.4-mini", "request": { "method": "POST", "header": [ @@ -114429,7 +115940,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"model\": \"azure/o3-mini\",\n \"input\": \"A farmer has 47 sheep. Each night, one third of the flock (rounded down) escapes, and each morning he recovers 5 plus the number that escaped two nights prior. Starting from night 1, how many sheep does he have after the 4th morning? Show your working.\",\n \"max_output_tokens\": 2048,\n \"reasoning\": {\n \"effort\": \"low\"\n }\n}" + "raw": "{\n \"model\": \"azure/gpt-5.4-mini\",\n \"input\": \"A farmer has 47 sheep. Each night, one third of the flock (rounded down) escapes, and each morning he recovers 5 plus the number that escaped two nights prior. Starting from night 1, how many sheep does he have after the 4th morning? Show your working.\",\n \"max_output_tokens\": 2048,\n \"reasoning\": {\n \"effort\": \"low\"\n }\n}" }, "url": { "raw": "{{baseUrl}}/pydanticai/v1/responses", @@ -116535,7 +118046,7 @@ ] }, { - "name": "[PREVIEW] 48.3.A /pydanticai reasoning -> azure/o3-mini (o3-mini deployment is account-scoped)", + "name": "[PREVIEW] 48.3.A /pydanticai reasoning -> azure/gpt-5.4-mini", "request": { "method": "POST", "header": [ @@ -116546,7 +118057,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"model\": \"azure/o3-mini\",\n \"max_tokens\": 4096,\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"A farmer has 47 sheep. Each night, one third of the flock (rounded down) escapes, and each morning he recovers 5 plus the number that escaped two nights prior. Starting from night 1, how many sheep does he have after the 4th morning? Show your working.\"\n }\n ],\n \"thinking\": {\n \"type\": \"enabled\",\n \"budget_tokens\": 1024\n }\n}" + "raw": "{\n \"model\": \"azure/gpt-5.4-mini\",\n \"max_tokens\": 4096,\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"A farmer has 47 sheep. Each night, one third of the flock (rounded down) escapes, and each morning he recovers 5 plus the number that escaped two nights prior. Starting from night 1, how many sheep does he have after the 4th morning? Show your working.\"\n }\n ],\n \"thinking\": {\n \"type\": \"enabled\",\n \"budget_tokens\": 1024\n }\n}" }, "url": { "raw": "{{baseUrl}}/pydanticai/v1/messages", @@ -118715,7 +120226,7 @@ ] }, { - "name": "[PREVIEW] 48.3.G /pydanticai reasoning -> azure/o3-mini (o3-mini deployment is account-scoped)", + "name": "[PREVIEW] 48.3.G /pydanticai reasoning -> azure/gpt-5.4-mini", "request": { "method": "POST", "header": [ @@ -118729,7 +120240,7 @@ "raw": "{\n \"contents\": [\n {\n \"role\": \"user\",\n \"parts\": [\n {\n \"text\": \"A farmer has 47 sheep. Each night, one third of the flock (rounded down) escapes, and each morning he recovers 5 plus the number that escaped two nights prior. Starting from night 1, how many sheep does he have after the 4th morning? Show your working.\"\n }\n ]\n }\n ],\n \"generationConfig\": {\n \"maxOutputTokens\": 2048,\n \"thinkingConfig\": {\n \"thinkingBudget\": 1024,\n \"includeThoughts\": true\n }\n }\n}" }, "url": { - "raw": "{{baseUrl}}/pydanticai/v1beta/models/azure/o3-mini:generateContent", + "raw": "{{baseUrl}}/pydanticai/v1beta/models/azure/gpt-5.4-mini:generateContent", "host": [ "{{baseUrl}}" ], @@ -118738,7 +120249,7 @@ "v1beta", "models", "azure", - "o3-mini:generateContent" + "gpt-5.4-mini:generateContent" ] } }, @@ -120958,7 +122469,7 @@ ] }, { - "name": "[PREVIEW] 48.3.B /pydanticai reasoning -> azure/o3-mini (o3-mini deployment is account-scoped)", + "name": "[PREVIEW] 48.3.B /pydanticai reasoning -> azure/gpt-5.4-mini", "request": { "method": "POST", "header": [ @@ -120972,14 +122483,14 @@ "raw": "{\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": [\n {\n \"text\": \"A farmer has 47 sheep. Each night, one third of the flock (rounded down) escapes, and each morning he recovers 5 plus the number that escaped two nights prior. Starting from night 1, how many sheep does he have after the 4th morning? Show your working.\"\n }\n ]\n }\n ],\n \"inferenceConfig\": {\n \"maxTokens\": 4096\n },\n \"additionalModelRequestFields\": {\n \"reasoning_config\": {\n \"type\": \"enabled\",\n \"budget_tokens\": 1024\n }\n }\n}" }, "url": { - "raw": "{{baseUrl}}/pydanticai/model/azure%2Fo3-mini/converse", + "raw": "{{baseUrl}}/pydanticai/model/azure%2Fgpt-5.4-mini/converse", "host": [ "{{baseUrl}}" ], "path": [ "pydanticai", "model", - "azure%2Fo3-mini", + "azure%2Fgpt-5.4-mini", "converse" ] } @@ -123315,7 +124826,7 @@ ] }, { - "name": "[PREVIEW] 48.3.C /pydanticai reasoning -> azure/o3-mini (o3-mini deployment is account-scoped)", + "name": "[PREVIEW] 48.3.C /pydanticai reasoning -> azure/gpt-5.4-mini", "request": { "method": "POST", "header": [ @@ -123326,7 +124837,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"model\": \"azure/o3-mini\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"A farmer has 47 sheep. Each night, one third of the flock (rounded down) escapes, and each morning he recovers 5 plus the number that escaped two nights prior. Starting from night 1, how many sheep does he have after the 4th morning? Show your working.\"\n }\n ],\n \"max_tokens\": 4096,\n \"thinking\": {\n \"type\": \"enabled\",\n \"token_budget\": 1024\n }\n}" + "raw": "{\n \"model\": \"azure/gpt-5.4-mini\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"A farmer has 47 sheep. Each night, one third of the flock (rounded down) escapes, and each morning he recovers 5 plus the number that escaped two nights prior. Starting from night 1, how many sheep does he have after the 4th morning? Show your working.\"\n }\n ],\n \"max_tokens\": 4096,\n \"thinking\": {\n \"type\": \"enabled\",\n \"token_budget\": 1024\n }\n}" }, "url": { "raw": "{{baseUrl}}/pydanticai/v2/chat", @@ -125719,7 +127230,7 @@ ] }, { - "name": "[PREVIEW] 48.5.C /cohere/v2/chat reasoning -> azure/o3-mini (o3-mini deployment is account-scoped)", + "name": "[PREVIEW] 48.5.C /cohere/v2/chat reasoning -> azure/gpt-5.4-mini", "request": { "method": "POST", "header": [ @@ -125730,7 +127241,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"model\": \"azure/o3-mini\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"A farmer has 47 sheep. Each night, one third of the flock (rounded down) escapes, and each morning he recovers 5 plus the number that escaped two nights prior. Starting from night 1, how many sheep does he have after the 4th morning? Show your working.\"\n }\n ],\n \"max_tokens\": 4096,\n \"thinking\": {\n \"type\": \"enabled\",\n \"token_budget\": 1024\n }\n}" + "raw": "{\n \"model\": \"azure/gpt-5.4-mini\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"A farmer has 47 sheep. Each night, one third of the flock (rounded down) escapes, and each morning he recovers 5 plus the number that escaped two nights prior. Starting from night 1, how many sheep does he have after the 4th morning? Show your working.\"\n }\n ],\n \"max_tokens\": 4096,\n \"thinking\": {\n \"type\": \"enabled\",\n \"token_budget\": 1024\n }\n}" }, "url": { "raw": "{{baseUrl}}/cohere/v2/chat", @@ -138618,6 +140129,94 @@ } } }, + { + "name": "databricks/databricks-gemini-3-8-flash merges multi-part system prompts on chat", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('Gemini endpoints receive a single system prompt', function () {", + " var body = pm.response.text() || '';", + " pm.expect(body, 'Databricks rejected several system prompts instead of Bifrost merging them').to.not.include('only support one system prompt');", + " pm.expect(pm.response.code, 'request failed: ' + body).to.be.below(400);", + " var json = pm.response.json();", + " pm.expect(json.choices, 'choices missing: ' + body).to.be.an('array').that.is.not.empty;", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\"model\": \"databricks/databricks-gemini-3-8-flash\", \"max_tokens\": 32, \"messages\": [{\"role\": \"system\", \"content\": [{\"type\": \"text\", \"text\": \"You are a terse assistant.\"}, {\"type\": \"text\", \"text\": \"Answer with just the number.\"}]}, {\"role\": \"developer\", \"content\": \"Never add punctuation.\"}, {\"role\": \"user\", \"content\": \"What is 2+2?\"}]}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "databricks/databricks-gemini-3-8-flash /anthropic/v1/messages Claude Code multi-block system", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('Claude Code system blocks reach Gemini as one system prompt', function () {", + " var body = pm.response.text() || '';", + " pm.expect(body, 'Databricks rejected several system prompts instead of Bifrost merging them').to.not.include('only support one system prompt');", + " pm.expect(pm.response.code, 'request failed: ' + body).to.be.below(400);", + " pm.expect(body, 'no SSE data frames').to.include('data:');", + " pm.expect(body, 'stream did not finish').to.include('message_stop');", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\"model\": \"databricks/databricks-gemini-3-8-flash\", \"max_tokens\": 64, \"stream\": true, \"system\": [{\"type\": \"text\", \"text\": \"You are Claude Code, a terse assistant.\"}, {\"type\": \"text\", \"text\": \"Answer with just the number.\", \"cache_control\": {\"type\": \"ephemeral\"}}], \"messages\": [{\"role\": \"user\", \"content\": \"What is 2+2?\"}]}" + }, + "url": { + "raw": "{{baseUrl}}/anthropic/v1/messages", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "anthropic", + "v1", + "messages" + ] + } + } + }, { "name": "databricks/databricks-claude-sonnet-4-5 /v1/chat/completions streaming", "event": [ @@ -139127,15 +140726,9 @@ } } ] - } - ] - }, - { - "name": "68. Replayed thinking must stay in its own assistant turn (PR #6854)", - "description": "Two defects in the Anthropic -> Bifrost Responses -> Bedrock Converse replay path, both\nsurfacing as the same client-visible 400:\n\n messages.N.content.M: `thinking` or `redacted_thinking` blocks in the latest assistant\n message cannot be modified. These blocks must remain as they were in the original response.\n\nMeasured against Bedrock Converse with real signed blocks, that refusal fires on exactly one\nmutation: an EXTRA thinking block appearing in an assistant turn that is a tool continuation.\nRemoving a block, reordering blocks, and fusing turns are all accepted; duplication is not.\n\n1. Reasoning migration (core/providers/bedrock/responses.go). pendingReasoningContentBlocks\n was consumed by an assistant message or a tool-call flush and by nothing else, so reasoning\n left buffered when a user turn arrived was prepended to the NEXT assistant message. That\n happens whenever an assistant turn ends on a thinking block - adaptive thinking, a\n max_tokens truncation, or a trailing block the ingress dropped - and it shifts every later\n turn one turn forward. The three chained cases replay a turn whose last block is the\n thinking block; turn 3 is the one that 400s without the fix.\n\n2. Encrypted-reasoning fail-soft (core/encryptedreasoning.go). When an upstream refuses a\n replayed signature the strip retries once. It used to blank the signature and keep the\n thinking prose, which is legal on OpenAI but refused by Anthropic on every turn -\n \"Invalid `signature` in `thinking` block\" - so the retry turned a request the API would\n have accepted into one it refused. The two bogus-signature cases carry a signature no\n model minted and assert the retry heals to a 2xx instead of forwarding a second 400.\n\nThe last case covers Anthropic direct on the typed chat shape: the removal is gated on the\nmodel family rather than the provider, and stripChatUnverifiableReasoning is a separate\ncode path from the Responses one, so Bedrock coverage alone would not pin it.\n\nFolder 46 replays thinking in model order (thinking first) and folder 51 covers the raw\nAnthropic passthrough; neither exercises the Bedrock typed path with a trailing thinking\nblock, and `Invalid `signature`` appeared nowhere in the collection before this folder.", - "item": [ + }, { - "name": "bedrock/global.anthropic.claude-sonnet-4-5 /anthropic/v1/messages turn 1: signed thinking + tool_use captured - PR #6854", + "name": "azure/gpt-5.6-luna /bedrock converse turn 1: foreign redactedContent is standard base64 beside a toolUse (#7514) - converse-redacted-reasoning", "request": { "method": "POST", "header": [ @@ -139144,25 +140737,110 @@ "value": "application/json" } ], + "url": { + "raw": "{{baseUrl}}/bedrock/model/azure%2Fgpt-5.6-luna/converse", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "bedrock", + "model", + "azure%2Fgpt-5.6-luna", + "converse" + ] + }, "body": { "mode": "raw", - "raw": "{\"model\":\"bedrock/global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"max_tokens\":2048,\"thinking\":{\"type\":\"enabled\",\"budget_tokens\":1024},\"tools\":[{\"name\":\"get_weather\",\"description\":\"Current weather for a city\",\"input_schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}],\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"What is the weather in Tokyo right now? Think it through, then use the get_weather tool.\"}]}]}", + "raw": "{\"messages\":[{\"role\":\"user\",\"content\":[{\"text\":\"Find the capital of the country whose largest city is Istanbul, then get that capital's current weather.\"}]}],\"inferenceConfig\":{\"maxTokens\":2048},\"additionalModelRequestFields\":{\"reasoning_config\":{\"type\":\"enabled\",\"budget_tokens\":1024},\"output_config\":{\"effort\":\"high\"}},\"include\":[\"reasoning.encrypted_content\"],\"reasoning_summary\":\"none\",\"toolConfig\":{\"tools\":[{\"toolSpec\":{\"name\":\"get_weather\",\"description\":\"Get the current weather for a city.\",\"inputSchema\":{\"json\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}}}],\"toolChoice\":{\"any\":{}}}}", "options": { "raw": { "language": "json" } } - }, + } + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "// Converse types redactedContent as a blob: strict SDKs (aws-sdk-go-v2, boto3)", + "// base64-decode it. Azure mints URL-safe Fernet tokens ('-', '_'), which fail that", + "// decode with \"illegal base64 data\" unless Bifrost wraps them (#7514).", + "var STD_B64 = /^(?:[A-Za-z0-9+\\/]{4})*(?:[A-Za-z0-9+\\/]{2}==|[A-Za-z0-9+\\/]{3}=)?$/;", + "var body = pm.response.text() || '';", + "var j = null;", + "var captured = false;", + "pm.test('capture turn returns reasoning and a tool call', function () {", + " pm.expect(pm.response.code, 'capture request failed: ' + body.slice(0, 500)).to.equal(200);", + " j = pm.response.json();", + " var content = ((j.output || {}).message || {}).content || [];", + " pm.expect(content.some(function (c) { return c && c.toolUse; }), 'no toolUse block - the defect needs reasoning and a tool call in one turn: ' + body.slice(0, 500)).to.be.true;", + " var redacted = content.filter(function (c) { return c && c.reasoningContent && c.reasoningContent.redactedContent; });", + " pm.expect(redacted.length, 'no redactedContent came back, so this row would test nothing - encrypted reasoning was not requested or was dropped: ' + body.slice(0, 500)).to.be.above(0);", + " captured = true;", + "});", + "pm.test('every redactedContent is standard base64 an AWS SDK can decode', function () {", + " var content = ((j || {}).output || {}).message ? j.output.message.content : [];", + " content.forEach(function (c) {", + " if (!c || !c.reasoningContent || c.reasoningContent.redactedContent === undefined) { return; }", + " var blob = c.reasoningContent.redactedContent;", + " pm.expect(STD_B64.test(blob), 'redactedContent is not standard base64, so aws-sdk-go-v2 fails with \"decode base64 blob: illegal base64 data\": ' + String(blob).slice(0, 80)).to.be.true;", + " });", + "});", + "// Save the conversation only when the capture proved what turn 2 needs, so a", + "// failed capture fails here instead of cascading into a misleading replay error.", + "try {", + " var msg = j && j.output && j.output.message;", + " var toolUses = msg ? msg.content.filter(function (c) { return c && c.toolUse; }) : [];", + " if (captured && toolUses.length) {", + " pm.collectionVariables.set(", + " 'bedrockConverseForeignRedactedMessages',", + " JSON.stringify([", + " { role: \"user\", content: [{ text: \"Find the capital of the country whose largest city is Istanbul, then get that capital's current weather.\" }] },", + " msg,", + " // Answer every toolUse: a parallel call left unanswered is a 400.", + " { role: \"user\", content: toolUses.map(function (c) { return { toolResult: { toolUseId: c.toolUse.toolUseId, content: [{ text: \"Ankara: 18C, clear\" }] } }; }) }", + " ])", + " );", + " }", + "} catch (e) {}" + ] + } + } + ] + }, + { + "name": "azure/gpt-5.6-luna /bedrock converse turn 2: foreign redactedContent replays to Azure (#7514) - converse-redacted-reasoning", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], "url": { - "raw": "{{baseUrl}}/anthropic/v1/messages", + "raw": "{{baseUrl}}/bedrock/model/azure%2Fgpt-5.6-luna/converse", "host": [ "{{baseUrl}}" ], "path": [ - "anthropic", - "v1", - "messages" + "bedrock", + "model", + "azure%2Fgpt-5.6-luna", + "converse" ] + }, + "body": { + "mode": "raw", + "raw": "{\"messages\":{{bedrockConverseForeignRedactedMessages}},\"inferenceConfig\":{\"maxTokens\":2048},\"additionalModelRequestFields\":{\"reasoning_config\":{\"type\":\"enabled\",\"budget_tokens\":1024},\"output_config\":{\"effort\":\"high\"}},\"include\":[\"reasoning.encrypted_content\"],\"reasoning_summary\":\"none\",\"toolConfig\":{\"tools\":[{\"toolSpec\":{\"name\":\"get_weather\",\"description\":\"Get the current weather for a city.\",\"inputSchema\":{\"json\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}}}],\"toolChoice\":{\"auto\":{}}}}", + "options": { + "raw": { + "language": "json" + } + } } }, "event": [ @@ -139171,50 +140849,33 @@ "script": { "type": "text/javascript", "exec": [ - "pm.test('turn 1 returns a signed thinking block and a tool_use to build the replay on', function () {", - " pm.expect(pm.response.code, 'request failed: ' + pm.response.text()).to.be.below(400);", - " var j = pm.response.json();", - " pm.expect(j.content, 'expected a content array').to.be.an('array').that.is.not.empty;", - " var kinds = j.content.map(function (b) { return b.type; });", - " pm.expect(kinds, 'no tool_use came back, so turns 2 and 3 would skip and this chain would pass having tested nothing').to.include('tool_use');", - " var thinking = j.content.filter(function (b) { return b.type === 'thinking'; });", - " // Without a captured thinking block there is nothing to move to the end of the turn,", - " // so turns 2 and 3 would replay an ordinary history and pass having tested nothing.", - " pm.expect(thinking, 'no thinking block came back, so the migration path is never exercised').to.not.be.empty;", - " thinking.forEach(function (b) {", - " pm.expect(b.signature, 'a thinking block came back unsigned, so the replay below would prove nothing').to.be.a('string').that.is.not.empty;", - " });", + "// The client replays the blob exactly as turn 1 rendered it. Bifrost must unwrap", + "// it back to the token Azure minted. The unit test", + "// TestConverseRedactedContentSurvivesSDKReplay pins the exact bytes; this row pins", + "// that the full stack accepts the replayed turn and answers it.", + "var body = pm.response.text() || '';", + "pm.test('capture turn produced a conversation to replay', function () {", + " pm.expect(pm.request.body.raw.indexOf('{{'), 'turn 1 did not set bedrockConverseForeignRedactedMessages, so nothing was replayed').to.equal(-1);", "});", - "pm.collectionVariables.unset('bedrockTrailingThinkingTurn2_6854');", - "pm.collectionVariables.unset('bedrockTrailingThinkingTurn3_6854');", - "var trailingThinking6854 = function (content) {", - " var blocks = content || [];", - " var think = blocks.filter(function (b) { return b.type === 'thinking' || b.type === 'redacted_thinking'; });", - " var rest = blocks.filter(function (b) { return b.type !== 'thinking' && b.type !== 'redacted_thinking'; });", - " return rest.concat(think);", - "};", - "var stub6854 = \"{\\\"city\\\":\\\"Tokyo\\\",\\\"temp_c\\\":18,\\\"condition\\\":\\\"rain\\\"}\";", - "try {", - " var body = pm.response.json();", - " var uses = (body.content || []).filter(function (b) { return b.type === 'tool_use' && b.id; });", - " var thought = (body.content || []).some(function (b) { return b.type === 'thinking' || b.type === 'redacted_thinking'; });", - " if (uses.length && thought) {", - " var messages = [", - " { role: 'user', content: [{ type: 'text', text: 'What is the weather in Tokyo right now? Think it through, then use the get_weather tool.' }] },", - " { role: 'assistant', content: trailingThinking6854(body.content) },", - " { role: 'user', content: uses.map(function (u) { return { type: 'tool_result', tool_use_id: u.id, content: stub6854 }; })", - " .concat([{ type: 'text', text: 'Now check the weather in Osaka with the same tool.' }]) }", - " ];", - " pm.collectionVariables.set('bedrockTrailingThinkingTurn2_6854', JSON.stringify(messages));", - " }", - "} catch (e) {}" + "pm.test('replayed foreign redactedContent is accepted', function () {", + " pm.expect(pm.response.code, 'replay request failed: ' + body.slice(0, 500)).to.equal(200);", + " pm.expect(body, 'Azure refused the replayed reasoning token').to.not.include('invalid_encrypted_content');", + " var content = ((pm.response.json().output || {}).message || {}).content || [];", + " pm.expect(content, 'no content returned on the replayed turn: ' + body.slice(0, 500)).to.be.an('array').that.is.not.empty;", + "});" ] } } ] - }, + } + ] + }, + { + "name": "68. Replayed thinking must stay in its own assistant turn (PR #6854)", + "description": "Two defects in the Anthropic -> Bifrost Responses -> Bedrock Converse replay path, both\nsurfacing as the same client-visible 400:\n\n messages.N.content.M: `thinking` or `redacted_thinking` blocks in the latest assistant\n message cannot be modified. These blocks must remain as they were in the original response.\n\nMeasured against Bedrock Converse with real signed blocks, that refusal fires on exactly one\nmutation: an EXTRA thinking block appearing in an assistant turn that is a tool continuation.\nRemoving a block, reordering blocks, and fusing turns are all accepted; duplication is not.\n\n1. Reasoning migration (core/providers/bedrock/responses.go). pendingReasoningContentBlocks\n was consumed by an assistant message or a tool-call flush and by nothing else, so reasoning\n left buffered when a user turn arrived was prepended to the NEXT assistant message. That\n happens whenever an assistant turn ends on a thinking block - adaptive thinking, a\n max_tokens truncation, or a trailing block the ingress dropped - and it shifts every later\n turn one turn forward. The three chained cases replay a turn whose last block is the\n thinking block; turn 3 is the one that 400s without the fix.\n\n2. Encrypted-reasoning fail-soft (core/encryptedreasoning.go). When an upstream refuses a\n replayed signature the strip retries once. It used to blank the signature and keep the\n thinking prose, which is legal on OpenAI but refused by Anthropic on every turn -\n \"Invalid `signature` in `thinking` block\" - so the retry turned a request the API would\n have accepted into one it refused. The two bogus-signature cases carry a signature no\n model minted and assert the retry heals to a 2xx instead of forwarding a second 400.\n\nThe last case covers Anthropic direct on the typed chat shape: the removal is gated on the\nmodel family rather than the provider, and stripChatUnverifiableReasoning is a separate\ncode path from the Responses one, so Bedrock coverage alone would not pin it.\n\nFolder 46 replays thinking in model order (thinking first) and folder 51 covers the raw\nAnthropic passthrough; neither exercises the Bedrock typed path with a trailing thinking\nblock, and `Invalid `signature`` appeared nowhere in the collection before this folder.", + "item": [ { - "name": "bedrock/global.anthropic.claude-sonnet-4-5 /anthropic/v1/messages turn 2: trailing thinking block replayed across a tool result - PR #6854", + "name": "bedrock/global.anthropic.claude-sonnet-4-5 /anthropic/v1/messages turn 1: signed thinking + tool_use captured - PR #6854", "request": { "method": "POST", "header": [ @@ -139225,7 +140886,86 @@ ], "body": { "mode": "raw", - "raw": "{\"model\":\"bedrock/global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"max_tokens\":2048,\"thinking\":{\"type\":\"enabled\",\"budget_tokens\":1024},\"tools\":[{\"name\":\"get_weather\",\"description\":\"Current weather for a city\",\"input_schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}],\"messages\":{{bedrockTrailingThinkingTurn2_6854}}}", + "raw": "{\"model\":\"bedrock/global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"max_tokens\":2048,\"thinking\":{\"type\":\"enabled\",\"budget_tokens\":1024},\"tools\":[{\"name\":\"get_weather\",\"description\":\"Current weather for a city\",\"input_schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}],\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"What is the weather in Tokyo right now? Think it through, then use the get_weather tool.\"}]}]}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{baseUrl}}/anthropic/v1/messages", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "anthropic", + "v1", + "messages" + ] + } + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('turn 1 returns a signed thinking block and a tool_use to build the replay on', function () {", + " pm.expect(pm.response.code, 'request failed: ' + pm.response.text()).to.be.below(400);", + " var j = pm.response.json();", + " pm.expect(j.content, 'expected a content array').to.be.an('array').that.is.not.empty;", + " var kinds = j.content.map(function (b) { return b.type; });", + " pm.expect(kinds, 'no tool_use came back, so turns 2 and 3 would skip and this chain would pass having tested nothing').to.include('tool_use');", + " var thinking = j.content.filter(function (b) { return b.type === 'thinking'; });", + " // Without a captured thinking block there is nothing to move to the end of the turn,", + " // so turns 2 and 3 would replay an ordinary history and pass having tested nothing.", + " pm.expect(thinking, 'no thinking block came back, so the migration path is never exercised').to.not.be.empty;", + " thinking.forEach(function (b) {", + " pm.expect(b.signature, 'a thinking block came back unsigned, so the replay below would prove nothing').to.be.a('string').that.is.not.empty;", + " });", + "});", + "pm.collectionVariables.unset('bedrockTrailingThinkingTurn2_6854');", + "pm.collectionVariables.unset('bedrockTrailingThinkingTurn3_6854');", + "var trailingThinking6854 = function (content) {", + " var blocks = content || [];", + " var think = blocks.filter(function (b) { return b.type === 'thinking' || b.type === 'redacted_thinking'; });", + " var rest = blocks.filter(function (b) { return b.type !== 'thinking' && b.type !== 'redacted_thinking'; });", + " return rest.concat(think);", + "};", + "var stub6854 = \"{\\\"city\\\":\\\"Tokyo\\\",\\\"temp_c\\\":18,\\\"condition\\\":\\\"rain\\\"}\";", + "try {", + " var body = pm.response.json();", + " var uses = (body.content || []).filter(function (b) { return b.type === 'tool_use' && b.id; });", + " var thought = (body.content || []).some(function (b) { return b.type === 'thinking' || b.type === 'redacted_thinking'; });", + " if (uses.length && thought) {", + " var messages = [", + " { role: 'user', content: [{ type: 'text', text: 'What is the weather in Tokyo right now? Think it through, then use the get_weather tool.' }] },", + " { role: 'assistant', content: trailingThinking6854(body.content) },", + " { role: 'user', content: uses.map(function (u) { return { type: 'tool_result', tool_use_id: u.id, content: stub6854 }; })", + " .concat([{ type: 'text', text: 'Now check the weather in Osaka with the same tool.' }]) }", + " ];", + " pm.collectionVariables.set('bedrockTrailingThinkingTurn2_6854', JSON.stringify(messages));", + " }", + "} catch (e) {}" + ] + } + } + ] + }, + { + "name": "bedrock/global.anthropic.claude-sonnet-4-5 /anthropic/v1/messages turn 2: trailing thinking block replayed across a tool result - PR #6854", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\"model\":\"bedrock/global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"max_tokens\":2048,\"thinking\":{\"type\":\"enabled\",\"budget_tokens\":1024},\"tools\":[{\"name\":\"get_weather\",\"description\":\"Current weather for a city\",\"input_schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}],\"messages\":{{bedrockTrailingThinkingTurn2_6854}}}", "options": { "raw": { "language": "json" @@ -144088,7 +145828,7 @@ " pm.expect(pm.response.code, 'stream request failed: ' + pm.response.text()).to.be.below(400);", " var lines = (pm.response.text() || '').split('\\n');", " var reasons = [];", - " var converted = null;", + " var converted = null;", " for (var i = 0; i < lines.length; i++) {", " var ln = lines[i];", " if (ln.indexOf('data:') !== 0) { continue; }", @@ -144096,16 +145836,16 @@ " if (!payload || payload === '[DONE]') { continue; }", " var obj = null; try { obj = JSON.parse(payload); } catch (e) { continue; }", " var ef = obj && obj.extra_fields;", - " if (ef && ef.converted_request_type) { converted = ef.converted_request_type; }", - " var ch = obj && obj.choices && obj.choices[0];", + " if (ef && ef.converted_request_type) { converted = ef.converted_request_type; }", + " var ch = obj && obj.choices && obj.choices[0];", " if (ch && ch.finish_reason !== null && ch.finish_reason !== undefined) { reasons.push(ch.finish_reason); }", " }", " // Anti-vacuous guard, the streaming counterpart of the converted_request_type check the", - " // non-streaming rows make via pm.response.json(). An SSE body cannot be parsed that way,", - " // so read the marker off the frames instead: on a chat-capable model nothing converts, the", - " // marker is absent from every frame, and this row would otherwise pass on a native stream.", - " pm.expect(converted, 'no streamed frame carried extra_fields.converted_request_type=responses, so the chat->responses conversion did not fire and the finish_reason below proves nothing').to.eql('responses');", - " pm.expect(reasons.length, 'expected exactly one chunk carrying finish_reason, got ' + reasons.length + ' - the streaming converter emits it only on the terminal completed/incomplete event').to.eql(1);", + " // non-streaming rows make via pm.response.json(). An SSE body cannot be parsed that way,", + " // so read the marker off the frames instead: on a chat-capable model nothing converts, the", + " // marker is absent from every frame, and this row would otherwise pass on a native stream.", + " pm.expect(converted, 'no streamed frame carried extra_fields.converted_request_type=responses, so the chat->responses conversion did not fire and the finish_reason below proves nothing').to.eql('responses');", + " pm.expect(reasons.length, 'expected exactly one chunk carrying finish_reason, got ' + reasons.length + ' - the streaming converter emits it only on the terminal completed/incomplete event').to.eql(1);", " pm.expect(reasons[0], 'streaming terminal reason must match what the non-streaming row derives').to.eql('stop');", "});" ] @@ -145318,7 +147058,7 @@ }, { "name": "74. Azure streaming preamble fallback", - "description": "Local fixture regression. Configure an isolated gateway's Azure endpoint to the azure-stream-preamble-fixture.mjs server with max_retries=0. Set azureStreamPreambleFixture=1 to enable these cases. The primary model emits startup metadata and then fails; the fallback must return hello without leaking failed-attempt events.", + "description": "Local fixture regression. Configure an isolated gateway's Azure endpoint and OpenAI base_url to the azure-stream-preamble-fixture.mjs server with max_retries=0. Set azureStreamPreambleFixture=1 to enable these cases. The primary model emits startup metadata and then fails (Azure: rate limit; OpenAI: server_is_overloaded with no HTTP status); the fallback must return hello without leaking failed-attempt events.", "event": [ { "listen": "prerequest", @@ -145372,7 +147112,10 @@ "request": { "method": "POST", "header": [ - { "key": "Content-Type", "value": "application/json" } + { + "key": "Content-Type", + "value": "application/json" + } ], "body": { "mode": "raw", @@ -145386,7 +147129,10 @@ "request": { "method": "POST", "header": [ - { "key": "Content-Type", "value": "application/json" } + { + "key": "Content-Type", + "value": "application/json" + } ], "body": { "mode": "raw", @@ -145394,6 +147140,40 @@ }, "url": "{{baseUrl}}/v1/responses" } + }, + { + "name": "OpenAI streaming preamble overload fallback - Chat Completions", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\"model\":\"openai/preamble-error\",\"messages\":[{\"role\":\"user\",\"content\":\"hi\"}],\"stream\":true,\"fallbacks\":[\"openai/preamble-success\"]}" + }, + "url": "{{baseUrl}}/v1/chat/completions" + } + }, + { + "name": "OpenAI streaming preamble overload fallback - Responses", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\"model\":\"openai/preamble-error\",\"input\":\"hi\",\"stream\":true,\"fallbacks\":[\"openai/preamble-success\"]}" + }, + "url": "{{baseUrl}}/v1/responses" + } } ] }, @@ -148062,7 +149842,7 @@ "for (var i = 0; i < lines.length; i++) {", " var line = lines[i];", " if (line.indexOf('data: ') !== 0) { continue; }", - " if (line === 'data: [DONE]') { seen++; if (doneIndex === -1) { doneIndex = seen; } continue; }", + " if (line === 'data: [DONE]') { seen++; if (doneIndex === -1) { doneIndex = seen; } continue; }", " var chunk;", " try { chunk = JSON.parse(line.slice(6)); } catch (e) { continue; }", " seen++;", @@ -148077,8 +149857,8 @@ " pm.expect(usage.completion_tokens, 'usage: ' + JSON.stringify(usage)).to.be.above(0);", " pm.expect(usage.total_tokens, 'usage: ' + JSON.stringify(usage)).to.be.above(0);", " pm.expect(usageIndex, 'usage must ride the finish_reason chunk or a later one (finish=' + finishIndex + ', usage=' + usageIndex + ')').to.be.at.least(finishIndex);", - " pm.expect(doneIndex, 'expected exactly one [DONE] frame after the usage chunk (usage=' + usageIndex + ', done=' + doneIndex + ')').to.be.above(usageIndex);", - " pm.expect(seen, '[DONE] must be the last frame; frames after it are invisible to clients (done=' + doneIndex + ', frames=' + seen + ')').to.equal(doneIndex);", + " pm.expect(doneIndex, 'expected exactly one [DONE] frame after the usage chunk (usage=' + usageIndex + ', done=' + doneIndex + ')').to.be.above(usageIndex);", + " pm.expect(seen, '[DONE] must be the last frame; frames after it are invisible to clients (done=' + doneIndex + ', frames=' + seen + ')').to.equal(doneIndex);", "});" ] } @@ -148136,7 +149916,7 @@ "for (var i = 0; i < lines.length; i++) {", " var line = lines[i];", " if (line.indexOf('data: ') !== 0) { continue; }", - " if (line === 'data: [DONE]') { seen++; if (doneIndex === -1) { doneIndex = seen; } continue; }", + " if (line === 'data: [DONE]') { seen++; if (doneIndex === -1) { doneIndex = seen; } continue; }", " var chunk;", " try { chunk = JSON.parse(line.slice(6)); } catch (e) { continue; }", " seen++;", @@ -148151,8 +149931,8 @@ " pm.expect(usage.completion_tokens, 'usage: ' + JSON.stringify(usage)).to.be.above(0);", " pm.expect(usage.total_tokens, 'usage: ' + JSON.stringify(usage)).to.be.above(0);", " pm.expect(usageIndex, 'usage must ride the finish_reason chunk or a later one (finish=' + finishIndex + ', usage=' + usageIndex + ')').to.be.at.least(finishIndex);", - " pm.expect(doneIndex, 'expected exactly one [DONE] frame after the usage chunk (usage=' + usageIndex + ', done=' + doneIndex + ')').to.be.above(usageIndex);", - " pm.expect(seen, '[DONE] must be the last frame; frames after it are invisible to clients (done=' + doneIndex + ', frames=' + seen + ')').to.equal(doneIndex);", + " pm.expect(doneIndex, 'expected exactly one [DONE] frame after the usage chunk (usage=' + usageIndex + ', done=' + doneIndex + ')').to.be.above(usageIndex);", + " pm.expect(seen, '[DONE] must be the last frame; frames after it are invisible to clients (done=' + doneIndex + ', frames=' + seen + ')').to.equal(doneIndex);", "});" ] } @@ -148903,7 +150683,7 @@ " for (var i = 0; i < tools.length; i++) {", " var name = (tools[i].function || {}).name || '';", " pm.expect(name, 'tool[' + i + '] name ' + JSON.stringify(name) + ' (len ' + name.length + ') breaks the 64-char contract').to.match(/^[A-Za-z0-9_-]{1,64}$/);", - " pm.expect(name, 'a 98-char alias must take the <8-hex hash>_ form on a 64-char wire').to.match(/^[0-9a-f]{8}_/);", + " pm.expect(name, 'a 98-char alias must take the letter-first t<8-hex hash>_ form on a 64-char wire').to.match(/^t[0-9a-f]{8}_/);", " }", " pm.expect(tools[0].function.name, 'the function name must stay readable after the hash prefix').to.include('read_repository_file');", "});" @@ -148944,6 +150724,86 @@ } }, "response": [] + }, + { + "name": "bedrock/us.moonshotai.kimi-k3 /v1/responses over-64-char namespace alias: hashed name is letter-first so kimi answers (was digit-leading, empty stream) (raw_request) - namespace-alias-length", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "// Bifrost hashes an over-64-char namespace alias to _. A bare 8-hex", + "// hash starts with a digit 10 times in 16, and moonshotai.kimi-k3 on Bedrock answers", + "// any digit-leading tool name with HTTP 200 and an empty stream (Codex: 0 tokens, no", + "// text). This namespace hashes to 7e4a44cb under the old scheme, so pre-fix the wire", + "// carries '7e4a44cb_execute_document_command' and the model never answers.", + "var txt = pm.response.text() || '';", + "var body = {};", + "try { body = pm.response.json() || {}; } catch (e) { body = {}; }", + "var rr = (body.extra_fields || {}).raw_request;", + "if (typeof rr === 'string') { try { rr = JSON.parse(rr); } catch (e) { rr = null; } }", + "var tools = ((rr || {}).toolConfig || {}).tools || [];", + "var names = tools.map(function (t) { return (t.toolSpec || {}).name || ''; });", + "pm.test('bedrock kimi-k3: request accepted - namespace-alias-length', function () {", + " pm.expect(pm.response.code, 'want 200: ' + txt.slice(0, 600)).to.eql(200);", + " pm.expect(rr, 'raw_request missing or unparseable: ' + txt.slice(0, 300)).to.be.an('object');", + "});", + "pm.test('bedrock kimi-k3: hashed alias is letter-first on the wire - namespace-alias-length', function () {", + " pm.expect(names, 'outbound tool names: ' + JSON.stringify(names)).to.eql(['t7e4a44cb_execute_document_command']);", + " pm.expect(names[0], 'a digit-leading tool name makes kimi-k3 return an empty stream').to.match(/^[A-Za-z]/);", + "});", + "pm.test('bedrock kimi-k3: the model answers - namespace-alias-length', function () {", + " // Positive usage alone is not an answer. The model must either reply in text or", + " // call the tool, and a call must come back under the caller's own namespace and", + " // name: the hashed alias is Bifrost's wire name and must never leak to the client.", + " var NS = 'mcp__codex_apps__codex_document_control_harness_alias_00';", + " var output = body.output || [];", + " var text = output.filter(function (o) { return o && o.type === 'message'; }).map(function (o) {", + " return (o.content || []).filter(function (c) { return c && c.type === 'output_text'; }).map(function (c) { return c.text || ''; }).join('');", + " }).join('');", + " var calls = output.filter(function (o) { return o && o.type === 'function_call'; });", + " var restored = calls.filter(function (o) { return o.name === 'execute_document_command' && o.namespace === NS; });", + " pm.expect(text.trim().length > 0 || restored.length > 0, 'want output_text or a function_call restored to ' + NS + '/execute_document_command, got output: ' + JSON.stringify(output).slice(0, 400) + ' body: ' + txt.slice(0, 300)).to.be.true;", + " pm.expect(calls.length, 'every function_call must be restored to the caller namespace and name: ' + JSON.stringify(calls).slice(0, 300)).to.equal(restored.length);", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"bedrock/us.moonshotai.kimi-k3\",\n \"input\": \"Reply with the single word OK.\",\n \"max_output_tokens\": 256,\n \"tools\": [\n {\n \"type\": \"namespace\",\n \"name\": \"mcp__codex_apps__codex_document_control_harness_alias_00\",\n \"description\": \"A long Codex-style MCP namespace\",\n \"tools\": [\n {\n \"type\": \"function\",\n \"name\": \"execute_document_command\",\n \"description\": \"Return OK. Takes no arguments.\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {},\n \"additionalProperties\": false\n }\n }\n ]\n }\n ]\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{baseUrl}}/v1/responses", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "responses" + ] + } + }, + "response": [] } ] }, @@ -158176,6 +160036,10397 @@ "response": [] } ] + }, + { + "name": "104. Web Search API Source Round-Trip (#7356)", + "description": "OpenAI Responses web search can return specialized API sources in web_search_call.action.sources ({\"type\":\"api\",\"name\":\"oai-weather\"}) that carry a name and no URL; the published OpenAPI schema does not model them yet (openai/openai-openapi#573). Bifrost's typed source schema had no name field and a required url, so a decode/re-encode round-trip dropped name and fabricated an empty url, and the OpenAI request-side source sanitization rebuilt sources without name. The case pins the client-visible path: replaying a completed web_search_call in input through POST /v1/responses must be accepted with the api source intact. Red/green note: pre-fix the replayed api source reached OpenAI as {\"type\":\"api\",\"url\":\"\"}; whether OpenAI rejects that shape depends on server-side validation this case cannot control, so the primary pin is post-fix acceptance of the corrected shape.", + "item": [ + { + "name": "openai/gpt-4o responses api source replay - #7356", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "if ([401, 403, 429, 500, 502, 503, 504].indexOf(pm.response.code) !== -1) { return; }", + "pm.test('web search api source replay accepted - #7356', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "pm.test('response body is valid json - #7356', function () {", + " pm.response.json();", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\"model\":\"openai/gpt-4o\",\"max_output_tokens\":16,\"input\":[{\"type\":\"web_search_call\",\"id\":\"ws_7356\",\"status\":\"completed\",\"action\":{\"type\":\"search\",\"queries\":[\"weather in paris\"],\"sources\":[{\"type\":\"url\",\"url\":\"https://example.com/weather\"},{\"type\":\"api\",\"name\":\"oai-weather\"}]}},{\"role\":\"user\",\"content\":\"Summarize the search results above in one sentence.\"}]}" + }, + "url": { + "raw": "{{baseUrl}}/v1/responses", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "responses" + ] + } + } + } + ] + }, + { + "name": "105. Routing fallback key pinning", + "description": "Creates a request-header-scoped routing rule using a pinned fallback followed by a legacy unpinned fallback. Before the change, the object fallback is rejected by the routing API. After the change, a missing pinned key must skip its attempt rather than silently selecting another key: success must come from fallback index 2. Covers JSON and SSE through the real routing and core retry paths. The temporary rule is deleted after each request. Requires management access (optional managementApiKey environment variable when authentication is enabled) and the configured OpenAI gpt-4o-mini provider. Known-key selection and restricted-write races have deterministic Go coverage.", + "item": [ + { + "name": "openai gpt-4o-mini fallback key pinning non-streaming", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var marker = pm.variables.replaceIn('{{$guid}}');", + "pm.variables.set('fallbackPinMarker', marker);", + "pm.variables.unset('fallbackPinRuleID');", + "var headers = {'Content-Type': 'application/json'};", + "var managementToken = pm.environment.get('managementApiKey') || pm.collectionVariables.get('managementApiKey');", + "if (managementToken) { headers.Authorization = 'Bearer ' + managementToken; }", + "var rule = {", + " name: 'harness-fallback-key-pin-' + marker,", + " enabled: true,", + " scope: 'global',", + " priority: Date.now() % 1000000000,", + " cel_expression: '\"x-fallback-pin-test\" in headers && headers[\"x-fallback-pin-test\"] == \"' + marker + '\"',", + " targets: [{provider: 'openai', model: 'bifrost-harness-nonexistent-primary', weight: 1}],", + " fallbacks: [", + " {provider: 'openai', model: 'gpt-4o-mini', key_id: '00000000-0000-4000-8000-000000000001'},", + " 'openai/gpt-4o-mini'", + " ]", + "};", + "pm.sendRequest({url: pm.variables.replaceIn('{{baseUrl}}') + '/api/routing/rules', method: 'POST', header: headers, body: {mode: 'raw', raw: JSON.stringify(rule)}}, function (err, response) {", + " if (response && [401, 403].indexOf(response.code) !== -1) {", + " pm.test.skip('fallback key pinning requires management access; set managementApiKey if needed');", + " pm.execution.skipRequest();", + " return;", + " }", + " pm.test('routing API accepts mixed pinned and legacy fallback entries', function () {", + " pm.expect(err).to.equal(null);", + " pm.expect(response.code, response.text()).to.equal(200);", + " });", + " if (err || response.code !== 200) { pm.execution.skipRequest(); return; }", + " var created = response.json().rule;", + " pm.variables.set('fallbackPinRuleID', created.id);", + " pm.test('routing API preserves the explicit fallback key ID', function () {", + " pm.expect(created.fallbacks[0].key_id).to.equal('00000000-0000-4000-8000-000000000001');", + " pm.expect(created.fallbacks[1]).to.equal('openai/gpt-4o-mini');", + " });", + "});" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var ruleID = pm.variables.get('fallbackPinRuleID');", + "if (ruleID) {", + " var headers = {};", + " var managementToken = pm.environment.get('managementApiKey') || pm.collectionVariables.get('managementApiKey');", + " if (managementToken) { headers.Authorization = 'Bearer ' + managementToken; }", + " pm.sendRequest({url: pm.variables.replaceIn('{{baseUrl}}') + '/api/routing/rules/' + ruleID, method: 'DELETE', header: headers}, function (err, response) {", + " pm.test('temporary fallback key pinning rule is removed', function () {", + " pm.expect(err).to.equal(null);", + " pm.expect(response.code, response.text()).to.be.below(300);", + " });", + " });", + "}", + "if ([401, 403, 429, 500, 502, 503, 504].indexOf(pm.response.code) !== -1) { return; }", + "pm.test('unknown pinned fallback is skipped and the explicit unpinned fallback succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.equal(200);", + " pm.expect(pm.response.headers.get('x-bifrost-routing-info-is-fallback')).to.equal('true');", + " pm.expect(pm.response.headers.get('x-bifrost-fallback-index')).to.equal('2');", + "});", + "pm.test(\"chat completion returned\", function () { pm.expect(pm.response.json().choices).to.be.an(\"array\").that.is.not.empty; });" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-fallback-pin-test", + "value": "{{fallbackPinMarker}}" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"openai/gpt-4o-mini\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply OK.\"\n }\n ],\n \"max_tokens\": 8,\n \"stream\": false\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "openai gpt-4o-mini fallback key pinning streaming", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "var marker = pm.variables.replaceIn('{{$guid}}');", + "pm.variables.set('fallbackPinMarker', marker);", + "pm.variables.unset('fallbackPinRuleID');", + "var headers = {'Content-Type': 'application/json'};", + "var managementToken = pm.environment.get('managementApiKey') || pm.collectionVariables.get('managementApiKey');", + "if (managementToken) { headers.Authorization = 'Bearer ' + managementToken; }", + "var rule = {", + " name: 'harness-fallback-key-pin-' + marker,", + " enabled: true,", + " scope: 'global',", + " priority: Date.now() % 1000000000,", + " cel_expression: '\"x-fallback-pin-test\" in headers && headers[\"x-fallback-pin-test\"] == \"' + marker + '\"',", + " targets: [{provider: 'openai', model: 'bifrost-harness-nonexistent-primary', weight: 1}],", + " fallbacks: [", + " {provider: 'openai', model: 'gpt-4o-mini', key_id: '00000000-0000-4000-8000-000000000001'},", + " 'openai/gpt-4o-mini'", + " ]", + "};", + "pm.sendRequest({url: pm.variables.replaceIn('{{baseUrl}}') + '/api/routing/rules', method: 'POST', header: headers, body: {mode: 'raw', raw: JSON.stringify(rule)}}, function (err, response) {", + " if (response && [401, 403].indexOf(response.code) !== -1) {", + " pm.test.skip('fallback key pinning requires management access; set managementApiKey if needed');", + " pm.execution.skipRequest();", + " return;", + " }", + " pm.test('routing API accepts mixed pinned and legacy fallback entries', function () {", + " pm.expect(err).to.equal(null);", + " pm.expect(response.code, response.text()).to.equal(200);", + " });", + " if (err || response.code !== 200) { pm.execution.skipRequest(); return; }", + " var created = response.json().rule;", + " pm.variables.set('fallbackPinRuleID', created.id);", + " pm.test('routing API preserves the explicit fallback key ID', function () {", + " pm.expect(created.fallbacks[0].key_id).to.equal('00000000-0000-4000-8000-000000000001');", + " pm.expect(created.fallbacks[1]).to.equal('openai/gpt-4o-mini');", + " });", + "});" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var ruleID = pm.variables.get('fallbackPinRuleID');", + "if (ruleID) {", + " var headers = {};", + " var managementToken = pm.environment.get('managementApiKey') || pm.collectionVariables.get('managementApiKey');", + " if (managementToken) { headers.Authorization = 'Bearer ' + managementToken; }", + " pm.sendRequest({url: pm.variables.replaceIn('{{baseUrl}}') + '/api/routing/rules/' + ruleID, method: 'DELETE', header: headers}, function (err, response) {", + " pm.test('temporary fallback key pinning rule is removed', function () {", + " pm.expect(err).to.equal(null);", + " pm.expect(response.code, response.text()).to.be.below(300);", + " });", + " });", + "}", + "if ([401, 403, 429, 500, 502, 503, 504].indexOf(pm.response.code) !== -1) { return; }", + "pm.test('unknown pinned fallback is skipped and the explicit unpinned fallback succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.equal(200);", + " pm.expect(pm.response.headers.get('x-bifrost-routing-info-is-fallback')).to.equal('true');", + " pm.expect(pm.response.headers.get('x-bifrost-fallback-index')).to.equal('2');", + "});", + "pm.test(\"stream completes\", function () { pm.expect(pm.response.text()).to.include(\"data: [DONE]\"); });" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-fallback-pin-test", + "value": "{{fallbackPinMarker}}" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"openai/gpt-4o-mini\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply OK.\"\n }\n ],\n \"max_tokens\": 8,\n \"stream\": true\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + } + } + ] + }, + { + "name": "106. Typesafe decisions provider (typesafe-decisions)", + "item": [ + { + "name": "/v1/decisions - mixed kinds return typed answers - typesafe/jev-1.13.0", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "pm.test('every question produced a typed answer', function () {", + " pm.expect(body.answers, 'missing answers: ' + pm.response.text()).to.be.an('object');", + " pm.expect(body.answers.is_frustrated.kind).to.eql('noul');", + " pm.expect(body.answers.is_frustrated.value).to.be.a('number');", + " pm.expect(body.answers.is_frustrated.value).to.be.at.least(0);", + " pm.expect(body.answers.is_frustrated.value).to.be.at.most(1);", + " pm.expect(body.answers.category.kind).to.eql('choice');", + " pm.expect(['billing', 'bug', 'other']).to.include(body.answers.category.value);", + " pm.expect(body.answers.urgency.kind).to.eql('score');", + " pm.expect(body.answers.urgency.value).to.be.a('number');", + "});", + "pm.test('usage is normalized', function () {", + " pm.expect(body.usage.prompt_tokens, 'usage: ' + JSON.stringify(body.usage)).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"typesafe/jev-1.13.0\",\n \"state\": \"Customer message: I was double charged last month and nobody replied to my two emails. I want a refund today or I am cancelling.\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges and refunds\",\n \"bug\": \"product defects\",\n \"other\": \"anything else\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"Rate how urgently this needs a human reply\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered soon\",\n \"needs a reply today\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - structured object, array, and null criteria accepted - typesafe/jev-1.13.0", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('every allowed criteria type is accepted, not rejected as non-string', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "pm.test('every question still produced a typed answer', function () {", + " pm.expect(body.answers, 'missing answers: ' + pm.response.text()).to.be.an('object');", + " ['is_frustrated', 'asks_refund', 'threatens_cancel'].forEach(function (name) {", + " pm.expect(body.answers[name].kind, name).to.eql('noul');", + " pm.expect(body.answers[name].value, name).to.be.at.least(0);", + " pm.expect(body.answers[name].value, name).to.be.at.most(1);", + " });", + " pm.expect(body.answers.category.kind).to.eql('choice');", + " pm.expect(['billing', 'bug', 'support', 'other']).to.include(body.answers.category.value);", + " pm.expect(body.answers.category.probabilities, 'choice probabilities are a required response field').to.be.an('object');", + " pm.expect(body.answers.urgency.kind).to.eql('score');", + " pm.expect(body.answers.urgency.value).to.be.a('number');", + " // The legend echoes each level's description verbatim - structured", + " // levels come back as objects/arrays, not stringified.", + " pm.expect(body.answers.urgency.legend['0']).to.eql('can wait a week');", + " pm.expect(body.answers.urgency.legend['1']).to.deep.eql({level: 'should be answered soon'});", + " pm.expect(body.answers.urgency.legend['2']).to.deep.eql(['needs a reply today', 'churn risk']);", + " pm.expect(body.answers.urgency.probabilities, 'score probabilities are a required response field').to.be.an('object');", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"typesafe/jev-1.13.0\",\n \"state\": \"Customer message: I was double charged last month and nobody replied to my two emails. I want a refund today or I am cancelling.\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\",\n \"criteria\": {\n \"true\": {\n \"meaning\": \"clearly upset\",\n \"signals\": [\n \"threats to cancel\",\n \"repeated contact\"\n ]\n },\n \"false\": [\n \"calm\",\n \"neutral tone\"\n ]\n }\n },\n \"asks_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\",\n \"criteria\": {\n \"true\": [\n \"asks for money back\",\n \"mentions refund\"\n ],\n \"false\": {\n \"meaning\": \"no refund language\"\n }\n }\n },\n \"threatens_cancel\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel?\",\n \"criteria\": {\n \"true\": \"cancellation is threatened\",\n \"false\": \"no cancellation language\"\n }\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": {\n \"rubric\": \"charges and refunds\",\n \"examples\": [\n \"double charge\",\n \"refund request\"\n ]\n },\n \"bug\": [\n \"crash\",\n \"product defect\"\n ],\n \"support\": \"service questions\",\n \"other\": null\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"Rate how urgently this needs a human reply\",\n \"criteria\": [\n \"can wait a week\",\n {\n \"level\": \"should be answered soon\"\n },\n [\n \"needs a reply today\",\n \"churn risk\"\n ]\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - unsupported kind rejected with 400 - typesafe/jev-1.13.0", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('unsupported kind is rejected, not silently approximated', function () {", + " pm.expect(pm.response.code, 'expected 400, got: ' + pm.response.text()).to.eql(400);", + " pm.expect(pm.response.text()).to.include('unsupported kind');", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"typesafe/jev-1.13.0\",\n \"state\": \"Customer message: I was double charged last month and nobody replied to my two emails. I want a refund today or I am cancelling.\",\n \"questions\": {\n \"plain\": {\n \"kind\": \"ranking\",\n \"instructions\": \"rank it\"\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/typesafe/v1/systemone - native questions round trip - jev-1.13.0", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('native systemone succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "pm.test('all three native answer types come back', function () {", + " pm.expect(body.answers.is_frustrated.type).to.eql('noul');", + " pm.expect(body.answers.is_frustrated.noul).to.be.a('number');", + " pm.expect(body.answers.category.type).to.eql('choice');", + " pm.expect(['billing', 'bug', 'other']).to.include(body.answers.category.choice);", + " pm.expect(body.answers.urgency.type).to.eql('score');", + " pm.expect(body.answers.urgency.score).to.be.a('number');", + "});", + "pm.test('native model and usage shape retained', function () {", + " pm.expect(body.model, 'model: ' + body.model).to.be.a('string');", + " pm.expect(body.model).to.not.include('typesafe/');", + " pm.expect(body.usage.input_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/typesafe/v1/systemone", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "typesafe", + "v1", + "systemone" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"state\": \"Customer message: I was double charged last month and nobody replied to my two emails. I want a refund today or I am cancelling.\",\n \"model\": \"jev-1.13.0\",\n \"questions\": {\n \"is_frustrated\": {\n \"type\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"category\": {\n \"type\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges and refunds\",\n \"bug\": \"product defects\",\n \"other\": \"anything else\"\n }\n },\n \"urgency\": {\n \"type\": \"score\",\n \"instructions\": \"Rate how urgently this needs a human reply\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered soon\",\n \"needs a reply today\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/typesafe/v1/models - native catalog shape - jev models", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('models list succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "pm.test('native shape lists bare jev model names', function () {", + " pm.expect(body.models, 'body: ' + pm.response.text()).to.be.an('array');", + " var names = body.models.map(function (m) { return m.name; });", + " pm.expect(names).to.include('jev-1.13.0');", + " pm.expect(names).to.include('jev-latest');", + " names.forEach(function (n) { pm.expect(n).to.not.include('typesafe/'); });", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "GET", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/typesafe/v1/models", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "typesafe", + "v1", + "models" + ] + } + }, + "response": [] + } + ] + }, + { + "name": "107. Decision LLM emulation and fallback (decision-llm-emulation)", + "item": [ + { + "name": "/v1/decisions - LLM emulates decision as primary - openai/gpt-4o-mini", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "pm.test('every question emulated into a typed answer with confidence', function () {", + " pm.expect(body.answers.is_frustrated.kind).to.eql('noul');", + " pm.expect(body.answers.is_frustrated.value).to.be.a('number');", + " pm.expect(body.answers.is_frustrated.value).to.be.at.least(0);", + " pm.expect(body.answers.is_frustrated.value).to.be.at.most(1);", + " pm.expect(body.answers.is_frustrated.confidence).to.be.a('number');", + " pm.expect(['billing', 'bug', 'other']).to.include(body.answers.category.value);", + " pm.expect(body.answers.urgency.kind).to.eql('score');", + " pm.expect(body.answers.urgency.value).to.be.a('number');", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('openai');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"openai/gpt-4o-mini\",\n \"state\": \"Customer: I was double charged and support ignored my emails. I want a refund now or I cancel.\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges and refunds\",\n \"bug\": \"product defects\",\n \"other\": \"anything else\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"Rate how urgently this needs a human reply\",\n \"criteria\": [\n \"low\",\n \"medium\",\n \"high\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - emulation forwards criteria descriptions to the model (opaque option codes) - openai/gpt-4o-mini", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision with structured criteria succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "pm.test('criteria descriptions reached the model: opaque code resolved via its rubric', function () {", + " // The option keys are meaningless codes; only the criteria descriptions", + " // say what they mean. Without the descriptions on the wire the model can", + " // only guess among opt_a/opt_b/opt_c/opt_d, so this pins the forwarding.", + " pm.expect(body.answers.pick.kind, 'body: ' + pm.response.text()).to.eql('choice');", + " pm.expect(body.answers.pick.value).to.eql('opt_a');", + " pm.expect(body.answers.pick.probabilities, 'choice probabilities are a required response field').to.be.an('object');", + "});", + "pm.test('noul and structured-score questions answered in range', function () {", + " pm.expect(body.answers.approve_review.kind).to.eql('noul');", + " pm.expect(body.answers.approve_review.value).to.be.at.least(0.5);", + " pm.expect(body.answers.urgency.kind).to.eql('score');", + " pm.expect(body.answers.urgency.value).to.be.a('number');", + " // The legend echoes each level's description verbatim - structured", + " // levels come back as objects/arrays, not stringified.", + " pm.expect(body.answers.urgency.legend['0']).to.eql('can wait a week');", + " pm.expect(body.answers.urgency.legend['1']).to.deep.eql({level: 'should be answered soon'});", + " pm.expect(body.answers.urgency.legend['2']).to.deep.eql(['needs a reply today', 'churn risk']);", + " pm.expect(body.answers.urgency.probabilities, 'score probabilities are a required response field').to.be.an('object');", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"openai/gpt-4o-mini\",\n \"state\": \"Customer message: I was double charged last month and nobody replied to my two emails. I want a refund today or I am cancelling.\",\n \"questions\": {\n \"approve_review\": {\n \"kind\": \"noul\",\n \"instructions\": \"Should this ticket get a billing review?\",\n \"criteria\": {\n \"true\": \"the customer reports a concrete billing error\",\n \"false\": [\n \"no billing problem is described\",\n \"general inquiry\"\n ]\n }\n },\n \"pick\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category. The option names are opaque codes; use the criteria descriptions to decide.\",\n \"criteria\": {\n \"opt_a\": {\n \"rubric\": \"charges and refunds\"\n },\n \"opt_b\": \"product defects\",\n \"opt_c\": null,\n \"opt_d\": [\n \"installation help\",\n \"how-to setup questions\"\n ]\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"Rate how urgently this needs a human reply\",\n \"criteria\": [\n \"can wait a week\",\n {\n \"level\": \"should be answered soon\"\n },\n [\n \"needs a reply today\",\n \"churn risk\"\n ]\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - failing primary falls through to LLM fallback - openai", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('fallback LLM emulated the decision', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "pm.test('answer came back from the fallback', function () {", + " pm.expect(body.answers.is_confirmation.kind).to.eql('noul');", + " pm.expect(body.answers.is_confirmation.value).to.be.a('number');", + " pm.expect(body.extra_fields.provider).to.eql('openai');", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"openai/nonexistent-model-xyz\",\n \"fallbacks\": [\n \"openai/gpt-4o-mini\"\n ],\n \"state\": \"Reply: sounds good, see you at 3pm.\",\n \"questions\": {\n \"is_confirmation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does this confirm the meeting?\"\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/typesafe/v1/systemone - LLM emulates in native shape - openai/gpt-4o-mini", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('native systemone emulation succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "pm.test('native answer shape from an emulated turn', function () {", + " pm.expect(body.answers.is_phishing.type).to.eql('noul');", + " pm.expect(body.answers.is_phishing.noul).to.be.a('number');", + " pm.expect(body.usage.input_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/typesafe/v1/systemone", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "typesafe", + "v1", + "systemone" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"openai/gpt-4o-mini\",\n \"state\": \"Email: verify your password at bank-secure.xyz within 24h.\",\n \"questions\": {\n \"is_phishing\": {\n \"type\": \"noul\",\n \"instructions\": \"Is this phishing?\"\n }\n }\n}" + } + }, + "response": [] + } + ] + }, + { + "name": "108. Decision emulation matrix - all providers (decision-emulation-matrix)", + "item": [ + { + "name": "/v1/decisions - matches jev anchors - openai/gpt-5.6", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('openai');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"openai/gpt-5.6\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - openai/gpt-5.1", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('openai');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"openai/gpt-5.1\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - openai/gpt-5-mini", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('openai');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"openai/gpt-5-mini\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - openai/gpt-5.6-luna", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('openai');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"openai/gpt-5.6-luna\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - openai/gpt-5.6-terra", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('openai');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"openai/gpt-5.6-terra\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - openai/gpt-5.4-nano", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('openai');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"openai/gpt-5.4-nano\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - anthropic/claude-opus-5", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('anthropic');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"anthropic/claude-opus-5\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - anthropic/claude-sonnet-5", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('anthropic');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"anthropic/claude-sonnet-5\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - anthropic/claude-fable-5-1", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('anthropic');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"anthropic/claude-fable-5-1\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - anthropic/claude-haiku-4-5-20251001", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('anthropic');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"anthropic/claude-haiku-4-5-20251001\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - gemini/gemini-3.1-pro-preview", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('gemini');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"gemini/gemini-3.1-pro-preview\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - gemini/gemini-3.6-flash", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('gemini');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"gemini/gemini-3.6-flash\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - gemini/gemini-3.7-flash", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('gemini');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"gemini/gemini-3.7-flash\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - gemini/gemini-2.5-pro", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('gemini');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"gemini/gemini-2.5-pro\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - vertex/gemini-3.7-flash", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('vertex');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"vertex/gemini-3.7-flash\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - vertex/gemini-3.6-flash", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('vertex');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"vertex/gemini-3.6-flash\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - vertex/claude-opus-4-8", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('vertex');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"vertex/claude-opus-4-8\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - vertex/claude-fable-5-1", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('vertex');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"vertex/claude-fable-5-1\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('bedrock');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - bedrock/global.anthropic.claude-sonnet-4-20250514-v1:0", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('bedrock');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"bedrock/global.anthropic.claude-sonnet-4-20250514-v1:0\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - azure/gpt-5.4", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('azure');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"azure/gpt-5.4\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - azure/gpt-5.4-mini", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('azure');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"azure/gpt-5.4-mini\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - azure/gpt-4.1", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('azure');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"azure/gpt-4.1\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - azure/gpt-4o", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('azure');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"azure/gpt-4o\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - cohere/command-a-03-2025", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('cohere');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"cohere/command-a-03-2025\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - cohere/command-a-reasoning-08-2025", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('cohere');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"cohere/command-a-reasoning-08-2025\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - cohere/command-a-vision-07-2025", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('cohere');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"cohere/command-a-vision-07-2025\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - cohere/command-a-translate-08-2025", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('cohere');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"cohere/command-a-translate-08-2025\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - mistral/mistral-large-latest", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('mistral');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"mistral/mistral-large-latest\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - mistral/mistral-medium-latest", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('mistral');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"mistral/mistral-medium-latest\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - mistral/mistral-small-latest", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('mistral');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"mistral/mistral-small-latest\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - mistral/magistral-medium-latest", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('mistral');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"mistral/magistral-medium-latest\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - groq/qwen/qwen3.8-27b", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('groq');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"groq/qwen/qwen3.8-27b\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - deepseek/deepseek-v4-pro", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('deepseek');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"deepseek/deepseek-v4-pro\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - deepseek/deepseek-v4-flash", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('deepseek');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"deepseek/deepseek-v4-flash\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - deepseek/deepseek-reasoner", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('deepseek');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"deepseek/deepseek-reasoner\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - deepseek/deepseek-chat", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('deepseek');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"deepseek/deepseek-chat\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - cerebras/gpt-oss-120b", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('cerebras');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"cerebras/gpt-oss-120b\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - cerebras/llama-3.3-70b", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('cerebras');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"cerebras/llama-3.3-70b\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - cerebras/qwen-3-32b", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('cerebras');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"cerebras/qwen-3-32b\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - cerebras/llama-4-scout-17b-16e-instruct", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('cerebras');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"cerebras/llama-4-scout-17b-16e-instruct\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - xai/grok-4.5", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('xai');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"xai/grok-4.5\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - xai/grok-4", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('xai');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"xai/grok-4\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - xai/grok-3", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('xai');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"xai/grok-3\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - xai/grok-code-fast-1", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('xai');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"xai/grok-code-fast-1\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - openrouter/openai/gpt-5", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('openrouter');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"openrouter/openai/gpt-5\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - openrouter/anthropic/claude-opus-4.1", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('openrouter');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"openrouter/anthropic/claude-opus-4.1\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - openrouter/google/gemini-3.8-flash", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('openrouter');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"openrouter/google/gemini-3.8-flash\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - openrouter/meta-llama/llama-3.3-70b-instruct", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('openrouter');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"openrouter/meta-llama/llama-3.3-70b-instruct\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - perplexity/sonar-pro", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('perplexity');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"perplexity/sonar-pro\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - perplexity/sonar-reasoning-pro", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('perplexity');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"perplexity/sonar-reasoning-pro\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - perplexity/sonar", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('perplexity');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"perplexity/sonar\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - perplexity/sonar-deep-research", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('perplexity');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"perplexity/sonar-deep-research\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - parasail/parasail-gpt-oss-120b", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('parasail');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"parasail/parasail-gpt-oss-120b\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - parasail/parasail-qwen3-235b-a22b", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('parasail');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"parasail/parasail-qwen3-235b-a22b\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - parasail/parasail-deepseek-v3", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('parasail');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"parasail/parasail-deepseek-v3\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - parasail/parasail-kimi-k2-instruct", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('parasail');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"parasail/parasail-kimi-k2-instruct\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - bedrock_mantle/openai.gpt-oss-120b", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('bedrock_mantle');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"bedrock_mantle/openai.gpt-oss-120b\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - bedrock_mantle/openai.gpt-oss-20b", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('bedrock_mantle');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"bedrock_mantle/openai.gpt-oss-20b\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - bedrock_mantle/anthropic.claude-opus-4-8", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('bedrock_mantle');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"bedrock_mantle/anthropic.claude-opus-4-8\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + }, + { + "name": "/v1/decisions - matches jev anchors - bedrock_mantle/anthropic.claude-sonnet-5", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('emulated decision succeeds', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "var body = pm.response.json();", + "var expected = 30;", + "pm.test('every question answered with a confidence', function () {", + " pm.expect(Object.keys(body.answers).length).to.eql(expected);", + " Object.keys(body.answers).forEach(function (k) {", + " pm.expect(body.answers[k].confidence, k + ' confidence').to.be.a('number');", + " });", + "});", + "pm.test('anchor verdicts match jev', function () {", + " pm.expect(Math.round(body.answers['is_frustrated'].value), 'is_frustrated').to.eql(1);", + " pm.expect(Math.round(body.answers['is_angry'].value), 'is_angry').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_refund'].value), 'mentions_refund').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_chargeback'].value), 'threatens_chargeback').to.eql(1);", + " pm.expect(Math.round(body.answers['threatens_cancellation'].value), 'threatens_cancellation').to.eql(1);", + " pm.expect(Math.round(body.answers['mentions_competitor'].value), 'mentions_competitor').to.eql(1);", + " pm.expect(Math.round(body.answers['is_satisfied'].value), 'is_satisfied').to.eql(0);", + " pm.expect(body.answers['category'].value, 'category').to.eql(\"billing\");", + " pm.expect(body.answers['sentiment'].value, 'sentiment').to.eql(\"negative\");", + " pm.expect(body.answers['customer_tier'].value, 'customer_tier').to.eql(\"pro\");", + "});", + "pm.test('resolved by the LLM provider with usage', function () {", + " pm.expect(body.extra_fields.provider).to.eql('bedrock_mantle');", + " pm.expect(body.usage.prompt_tokens).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/v1/decisions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "decisions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"bedrock_mantle/anthropic.claude-sonnet-5\",\n \"state\": \"Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'\",\n \"questions\": {\n \"is_frustrated\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer frustrated?\"\n },\n \"is_angry\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer angry?\"\n },\n \"is_satisfied\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer satisfied with the service?\"\n },\n \"is_polite\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the message written in a calm, polite tone?\"\n },\n \"mentions_refund\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer ask for a refund?\"\n },\n \"threatens_chargeback\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten a bank chargeback?\"\n },\n \"threatens_cancellation\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to cancel their subscription?\"\n },\n \"threatens_public_complaint\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer threaten to complain publicly (social media, review sites, or a regulator)?\"\n },\n \"mentions_competitor\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer mention a competitor?\"\n },\n \"is_billing_related\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the issue about billing or charges?\"\n },\n \"is_repeat_contact\": {\n \"kind\": \"noul\",\n \"instructions\": \"Has the customer contacted support about this before?\"\n },\n \"has_attached_evidence\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer say they have evidence such as screenshots?\"\n },\n \"mentions_specific_amount\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer cite a specific dollar amount?\"\n },\n \"is_long_time_customer\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this a long-time customer (multiple years)?\"\n },\n \"requests_human_response\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer explicitly ask to speak to a real person?\"\n },\n \"is_first_time_issue\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is this the first time the customer is raising this issue?\"\n },\n \"willing_to_wait\": {\n \"kind\": \"noul\",\n \"instructions\": \"Is the customer willing to wait patiently for a resolution?\"\n },\n \"expresses_urgency\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer express that this is urgent?\"\n },\n \"mentions_data_loss\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report lost data?\"\n },\n \"reports_product_crash\": {\n \"kind\": \"noul\",\n \"instructions\": \"Does the customer report the product crashing or a software defect?\"\n },\n \"category\": {\n \"kind\": \"choice\",\n \"instructions\": \"Pick the ticket category\",\n \"criteria\": {\n \"billing\": \"charges, refunds, invoices\",\n \"bug\": \"product defects or crashes\",\n \"account\": \"login or account access\",\n \"other\": \"anything else\"\n }\n },\n \"sentiment\": {\n \"kind\": \"choice\",\n \"instructions\": \"Overall sentiment of the message\",\n \"criteria\": {\n \"positive\": \"happy\",\n \"neutral\": \"matter of fact\",\n \"negative\": \"unhappy or hostile\"\n }\n },\n \"primary_emotion\": {\n \"kind\": \"choice\",\n \"instructions\": \"The dominant emotion\",\n \"criteria\": {\n \"anger\": \"angry or hostile\",\n \"confusion\": \"confused\",\n \"sadness\": \"sad or disappointed\",\n \"joy\": \"happy\"\n }\n },\n \"customer_tier\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which plan is the customer on?\",\n \"criteria\": {\n \"free\": \"free plan\",\n \"pro\": \"pro plan\",\n \"enterprise\": \"enterprise plan\"\n }\n },\n \"recommended_action\": {\n \"kind\": \"choice\",\n \"instructions\": \"The single best next action\",\n \"criteria\": {\n \"issue_refund\": \"refund the extra charges\",\n \"request_more_info\": \"ask for more details\",\n \"close_ticket\": \"close with no action\",\n \"ignore\": \"take no action\"\n }\n },\n \"contact_channel\": {\n \"kind\": \"choice\",\n \"instructions\": \"Which channel did the customer use?\",\n \"criteria\": {\n \"email\": \"email or support ticket\",\n \"phone\": \"phone call\",\n \"chat\": \"live chat\",\n \"social\": \"social media\"\n }\n },\n \"urgency\": {\n \"kind\": \"score\",\n \"instructions\": \"How urgently does this need a human reply?\",\n \"criteria\": [\n \"can wait a week\",\n \"should be answered this week\",\n \"should be answered today\",\n \"needs an immediate reply\"\n ]\n },\n \"severity\": {\n \"kind\": \"score\",\n \"instructions\": \"How severe is the customer's problem?\",\n \"criteria\": [\n \"minor\",\n \"moderate\",\n \"major\",\n \"severe\"\n ]\n },\n \"churn_risk\": {\n \"kind\": \"score\",\n \"instructions\": \"How likely is this customer to cancel?\",\n \"criteria\": [\n \"not at all likely\",\n \"somewhat likely\",\n \"very likely\",\n \"almost certain\"\n ]\n },\n \"customer_satisfaction\": {\n \"kind\": \"score\",\n \"instructions\": \"How satisfied is the customer right now?\",\n \"criteria\": [\n \"very unsatisfied\",\n \"unsatisfied\",\n \"neutral\",\n \"satisfied\",\n \"very satisfied\"\n ]\n }\n }\n}" + } + }, + "response": [] + } + ] + }, + { + "name": "108. Safeguards pass-through (Claude Code auto-mode classifier) (safeguards)", + "description": "Safeguards are retained for supported Claude models on Anthropic, Bedrock Invoke/Mantle, Vertex, and Azure, with dangerous-tool-use-2026-09-03 automatically enabled. These synthetic probes check forwarding, not evaluated verdicts: the valid classifier payload will be supplied separately. Only HTTP 200 or a safeguards-specific HTTP 400 is allowed; auth, rate limits, and unrelated errors fail. Haiku retains the negative model-gate case.", + "item": [ + { + "name": "[EXPECT-4XX] Anthropic /v1/messages - safeguards forwarded on typed path, provider rejects it by name (claude-sonnet-5) - safeguards", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('safeguards forwarded with required beta where body carries it', function () {", + " pm.expect([200, 400], 'unexpected status: ' + pm.response.text()).to.include(pm.response.code);", + " if (pm.response.code === 400) {", + " pm.expect(pm.response.text(), 'synthetic payload must be rejected by name: ' + pm.response.text()).to.include('safeguards');", + " pm.expect(pm.response.text(), 'required safeguards beta was not activated').to.not.include('safeguards: Extra inputs are not permitted');", + " } else {", + " var body = pm.response.json();", + " pm.expect(body.extra_fields, 'raw request capture missing: ' + pm.response.text()).to.exist;", + " var raw = body.extra_fields.raw_request;", + " if (typeof raw === 'string') { raw = JSON.parse(raw); }", + " pm.expect(raw, 'outbound safeguards missing').to.have.property('safeguards');", + " if (raw.anthropic_version) {", + " pm.expect(raw.anthropic_beta, 'required body beta missing').to.include('dangerous-tool-use-2026-09-03');", + " }", + " }", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-api-key", + "value": "{{anthropicKey}}" + }, + { + "key": "anthropic-version", + "value": "2023-06-01" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"anthropic/claude-sonnet-5\",\n \"max_tokens\": 32,\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"safeguards\": {\n \"probe\": \"auto_mode_classifier\"\n }\n}" + }, + "url": { + "raw": "{{baseUrl}}/anthropic/v1/messages", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "anthropic", + "v1", + "messages" + ] + } + } + }, + { + "name": "[EXPECT-4XX] Anthropic /v1/messages stream - safeguards forwarded on claude-cli passthrough, provider rejects it by name (claude-sonnet-5) - safeguards", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var text = pm.response.text();", + "pm.test('safeguards forwarded on the claude-cli passthrough', function () {", + " pm.expect([200, 400], 'unexpected status: ' + text).to.include(pm.response.code);", + " if (pm.response.code === 400) {", + " pm.expect(text, 'a 4xx must be the upstream rejecting the forwarded field by name: ' + text).to.include('safeguards');", + " pm.expect(text, 'required safeguards beta was not activated').to.not.include('safeguards: Extra inputs are not permitted');", + " return;", + " }", + " pm.expect(text, 'missing message_start: ' + text.slice(0, 500)).to.include('event: message_start');", + " pm.expect(text, 'missing message_stop: ' + text.slice(-500)).to.include('event: message_stop');", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-api-key", + "value": "{{anthropicKey}}" + }, + { + "key": "anthropic-version", + "value": "2023-06-01" + }, + { + "key": "User-Agent", + "value": "claude-cli/2.1.278 (external)" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"anthropic/claude-sonnet-5\",\n \"max_tokens\": 32,\n \"stream\": true,\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"safeguards\": {\n \"probe\": \"auto_mode_classifier\"\n }\n}" + }, + "url": { + "raw": "{{baseUrl}}/anthropic/v1/messages", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "anthropic", + "v1", + "messages" + ] + } + } + }, + { + "name": "Anthropic /v1/messages - safeguards stripped for a model outside the auto-mode set (claude-haiku-4-5) - safeguards", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('safeguards is stripped fail-closed from the outbound request', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + " var body = pm.response.json();", + " pm.expect(body.extra_fields, 'extra_fields missing - raw request capture not enabled: ' + pm.response.text()).to.exist;", + " var rawReq = JSON.stringify(body.extra_fields.raw_request);", + " pm.expect(rawReq, 'haiku is outside the Sonnet 5 / Opus 4.7+ / Fable auto-mode set, so SupportsSafeguards strips it rather than 400 the real request: ' + rawReq).to.not.include('\"safeguards\"');", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-api-key", + "value": "{{anthropicKey}}" + }, + { + "key": "anthropic-version", + "value": "2023-06-01" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"anthropic/claude-haiku-4-5\",\n \"max_tokens\": 32,\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"safeguards\": {\n \"probe\": \"auto_mode_classifier\"\n }\n}" + }, + "url": { + "raw": "{{baseUrl}}/anthropic/v1/messages", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "anthropic", + "v1", + "messages" + ] + } + } + }, + { + "name": "Bedrock /v1/messages - safeguards forwarded with automatic beta (global.anthropic.claude-opus-4-8) - safeguards", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('safeguards forwarded with the required beta', function () {", + " pm.expect(pm.response.code, 'expected 200 for a valid safeguards array: ' + pm.response.text()).to.equal(200);", + " var body = pm.response.json();", + " pm.expect(body.extra_fields, 'raw request capture missing: ' + pm.response.text()).to.exist;", + " var raw = body.extra_fields.raw_request;", + " if (typeof raw === 'string') { raw = JSON.parse(raw); }", + " pm.expect(raw.safeguards, 'outbound safeguards missing or reshaped: ' + JSON.stringify(raw)).to.be.an('array');", + " pm.expect(raw.safeguards[0].type, 'outbound safeguards entry reshaped: ' + JSON.stringify(raw.safeguards)).to.equal('dangerous_tool_use');", + " // Bedrock InvokeModel and Vertex rawPredict carry betas in the body. Azure and Mantle send the", + " // anthropic-beta header instead; there a 200 on the recognized safeguards field is the proof,", + " // because without dangerous-tool-use the provider answers 400 'Extra inputs are not permitted'.", + " if (raw.anthropic_version) {", + " pm.expect(raw.anthropic_beta, 'required body beta missing: ' + JSON.stringify(raw)).to.include('dangerous-tool-use-2026-09-03');", + " }", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-api-key", + "value": "{{anthropicKey}}" + }, + { + "key": "anthropic-version", + "value": "2023-06-01" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"bedrock/global.anthropic.claude-opus-4-8\",\n \"max_tokens\": 32,\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"safeguards\": [\n {\n \"type\": \"dangerous_tool_use\",\n \"classifier_context\": {\n \"v\": 1,\n \"permission_mode\": \"auto\"\n }\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/anthropic/v1/messages", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "anthropic", + "v1", + "messages" + ] + } + } + }, + { + "name": "Vertex /v1/messages - safeguards forwarded with automatic beta (claude-opus-4-8) - safeguards", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('safeguards forwarded with the required beta', function () {", + " pm.expect(pm.response.code, 'expected 200 for a valid safeguards array: ' + pm.response.text()).to.equal(200);", + " var body = pm.response.json();", + " pm.expect(body.extra_fields, 'raw request capture missing: ' + pm.response.text()).to.exist;", + " var raw = body.extra_fields.raw_request;", + " if (typeof raw === 'string') { raw = JSON.parse(raw); }", + " pm.expect(raw.safeguards, 'outbound safeguards missing or reshaped: ' + JSON.stringify(raw)).to.be.an('array');", + " pm.expect(raw.safeguards[0].type, 'outbound safeguards entry reshaped: ' + JSON.stringify(raw.safeguards)).to.equal('dangerous_tool_use');", + " // Bedrock InvokeModel and Vertex rawPredict carry betas in the body. Azure and Mantle send the", + " // anthropic-beta header instead; there a 200 on the recognized safeguards field is the proof,", + " // because without dangerous-tool-use the provider answers 400 'Extra inputs are not permitted'.", + " if (raw.anthropic_version) {", + " pm.expect(raw.anthropic_beta, 'required body beta missing: ' + JSON.stringify(raw)).to.include('dangerous-tool-use-2026-09-03');", + " }", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-api-key", + "value": "{{anthropicKey}}" + }, + { + "key": "anthropic-version", + "value": "2023-06-01" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"vertex/claude-opus-4-8\",\n \"max_tokens\": 32,\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"safeguards\": [\n {\n \"type\": \"dangerous_tool_use\",\n \"classifier_context\": {\n \"v\": 1,\n \"permission_mode\": \"auto\"\n }\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/anthropic/v1/messages", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "anthropic", + "v1", + "messages" + ] + } + } + }, + { + "name": "Azure /v1/messages - safeguards forwarded with automatic beta (claude-opus-4-8) - safeguards", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('safeguards forwarded with the required beta', function () {", + " pm.expect(pm.response.code, 'expected 200 for a valid safeguards array: ' + pm.response.text()).to.equal(200);", + " var body = pm.response.json();", + " pm.expect(body.extra_fields, 'raw request capture missing: ' + pm.response.text()).to.exist;", + " var raw = body.extra_fields.raw_request;", + " if (typeof raw === 'string') { raw = JSON.parse(raw); }", + " pm.expect(raw.safeguards, 'outbound safeguards missing or reshaped: ' + JSON.stringify(raw)).to.be.an('array');", + " pm.expect(raw.safeguards[0].type, 'outbound safeguards entry reshaped: ' + JSON.stringify(raw.safeguards)).to.equal('dangerous_tool_use');", + " // Bedrock InvokeModel and Vertex rawPredict carry betas in the body. Azure and Mantle send the", + " // anthropic-beta header instead; there a 200 on the recognized safeguards field is the proof,", + " // because without dangerous-tool-use the provider answers 400 'Extra inputs are not permitted'.", + " if (raw.anthropic_version) {", + " pm.expect(raw.anthropic_beta, 'required body beta missing: ' + JSON.stringify(raw)).to.include('dangerous-tool-use-2026-09-03');", + " }", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-api-key", + "value": "{{anthropicKey}}" + }, + { + "key": "anthropic-version", + "value": "2023-06-01" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"azure/claude-opus-4-8\",\n \"max_tokens\": 32,\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"safeguards\": [\n {\n \"type\": \"dangerous_tool_use\",\n \"classifier_context\": {\n \"v\": 1,\n \"permission_mode\": \"auto\"\n }\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/anthropic/v1/messages", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "anthropic", + "v1", + "messages" + ] + } + } + }, + { + "name": "Bedrock Mantle /v1/messages - safeguards forwarded with automatic beta (claude-opus-4-8) - safeguards", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('safeguards forwarded with the required beta', function () {", + " pm.expect(pm.response.code, 'expected 200 for a valid safeguards array: ' + pm.response.text()).to.equal(200);", + " var body = pm.response.json();", + " pm.expect(body.extra_fields, 'raw request capture missing: ' + pm.response.text()).to.exist;", + " var raw = body.extra_fields.raw_request;", + " if (typeof raw === 'string') { raw = JSON.parse(raw); }", + " pm.expect(raw.safeguards, 'outbound safeguards missing or reshaped: ' + JSON.stringify(raw)).to.be.an('array');", + " pm.expect(raw.safeguards[0].type, 'outbound safeguards entry reshaped: ' + JSON.stringify(raw.safeguards)).to.equal('dangerous_tool_use');", + " // Bedrock InvokeModel and Vertex rawPredict carry betas in the body. Azure and Mantle send the", + " // anthropic-beta header instead; there a 200 on the recognized safeguards field is the proof,", + " // because without dangerous-tool-use the provider answers 400 'Extra inputs are not permitted'.", + " if (raw.anthropic_version) {", + " pm.expect(raw.anthropic_beta, 'required body beta missing: ' + JSON.stringify(raw)).to.include('dangerous-tool-use-2026-09-03');", + " }", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-api-key", + "value": "{{anthropicKey}}" + }, + { + "key": "anthropic-version", + "value": "2023-06-01" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"bedrock_mantle/anthropic.claude-opus-4-8\",\n \"max_tokens\": 32,\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"safeguards\": [\n {\n \"type\": \"dangerous_tool_use\",\n \"classifier_context\": {\n \"v\": 1,\n \"permission_mode\": \"auto\"\n }\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/anthropic/v1/messages", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "anthropic", + "v1", + "messages" + ] + } + } + }, + { + "name": "Vertex /v1/messages - auto-mode-classifier beta forwarded to cloud surface (claude-opus-4-8) - safeguards", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('auto-mode-classifier beta forwarded alongside the safeguards gate', function () {", + " pm.expect(pm.response.code, 'expected 200: ' + pm.response.text()).to.equal(200);", + " var body = pm.response.json();", + " pm.expect(body.extra_fields, 'raw request capture missing: ' + pm.response.text()).to.exist;", + " var raw = body.extra_fields.raw_request;", + " if (typeof raw === 'string') { raw = JSON.parse(raw); }", + " // Vertex rawPredict carries betas in the body, so a dropped header shows up as a missing value here.", + " pm.expect(raw.anthropic_beta, 'anthropic_beta missing from the outbound body, so the client beta never reached Bifrost: ' + JSON.stringify(raw)).to.be.an('array');", + " pm.expect(raw.anthropic_beta, 'auto-mode-classifier beta dropped: ' + JSON.stringify(raw)).to.include('auto-mode-classifier-2026-07-16');", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-api-key", + "value": "{{anthropicKey}}" + }, + { + "key": "anthropic-version", + "value": "2023-06-01" + }, + { + "key": "x-bf-eh-anthropic-beta", + "value": "auto-mode-classifier-2026-07-16" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"vertex/claude-opus-4-8\",\n \"max_tokens\": 32,\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/anthropic/v1/messages", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "anthropic", + "v1", + "messages" + ] + } + } + } + ] + }, + { + "name": "109. Claude Code billing header filtering (billing-header)", + "description": "Claude Code billing metadata changes the prompt prefix. Messages ingress must strip standalone billing blocks for non-Anthropic model families while preserving real system text and Anthropic requests. Raw outbound request assertions fail before the fix without relying on nondeterministic cache-hit rates. Streaming and Responses-to-Chat dispatch are covered by Go provider-boundary tests.", + "item": [ + { + "name": "openai gpt-4o-mini version metadata - billing-header", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "anthropic-version", + "value": "2023-06-01" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"openai/gpt-4o-mini\",\n \"max_tokens\": 64,\n \"system\": [\n {\n \"type\": \"text\",\n \"text\": \"x-anthropic-billing-header: cc_version=2.1.270.42c; cc_entrypoint=cli; cc_is_subagent=true;\"\n },\n {\n \"type\": \"text\",\n \"text\": \"Reply briefly.\"\n }\n ],\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Say OK.\"\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/anthropic/v1/messages", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "anthropic", + "v1", + "messages" + ] + } + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var text = pm.response.text();", + "pm.test('Messages request succeeds', function () { pm.expect(pm.response.code, text).to.eql(200); });", + "var body = pm.response.json();", + "pm.test('Messages response is valid', function () { pm.expect(body.type, text).to.eql('message'); pm.expect(body.content, text).to.be.an('array').that.is.not.empty; });", + "var raw = (body.extra_fields || {}).raw_request;", + "if (typeof raw === 'string') { raw = JSON.parse(raw); }", + "pm.test('outbound prompt preserves only the intended system content', function () {", + " pm.expect(raw, 'raw request capture missing: ' + text).to.be.an('object');", + " pm.expect(raw.input, JSON.stringify(raw)).to.be.an('array');", + " pm.expect(raw.input[0].role).to.eql('system');", + " pm.expect(raw.input[0].content).to.eql([{ type: 'input_text', text: 'Reply briefly.' }]);", + " pm.expect(JSON.stringify(raw.input)).not.to.include('x-anthropic-billing-header:');", + "});" + ] + } + } + ] + }, + { + "name": "openai gpt-4o-mini nonce metadata - billing-header", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "anthropic-version", + "value": "2023-06-01" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"openai/gpt-4o-mini\",\n \"max_tokens\": 64,\n \"system\": [\n {\n \"type\": \"text\",\n \"text\": \"x-anthropic-billing-header: cc_version=2.1.270.abc; cc_entrypoint=cli; cch=12345;\"\n },\n {\n \"type\": \"text\",\n \"text\": \"Reply briefly.\"\n }\n ],\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Say OK.\"\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/anthropic/v1/messages", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "anthropic", + "v1", + "messages" + ] + } + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var text = pm.response.text();", + "pm.test('Messages request succeeds', function () { pm.expect(pm.response.code, text).to.eql(200); });", + "var body = pm.response.json();", + "pm.test('Messages response is valid', function () { pm.expect(body.type, text).to.eql('message'); pm.expect(body.content, text).to.be.an('array').that.is.not.empty; });", + "var raw = (body.extra_fields || {}).raw_request;", + "if (typeof raw === 'string') { raw = JSON.parse(raw); }", + "pm.test('outbound prompt preserves only the intended system content', function () {", + " pm.expect(raw, 'raw request capture missing: ' + text).to.be.an('object');", + " pm.expect(raw.input, JSON.stringify(raw)).to.be.an('array');", + " pm.expect(raw.input[0].role).to.eql('system');", + " pm.expect(raw.input[0].content).to.eql([{ type: 'input_text', text: 'Reply briefly.' }]);", + " pm.expect(JSON.stringify(raw.input)).not.to.include('x-anthropic-billing-header:');", + "});" + ] + } + } + ] + }, + { + "name": "anthropic claude-haiku-4-5 preserves metadata - billing-header", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "anthropic-version", + "value": "2023-06-01" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"anthropic/claude-haiku-4-5\",\n \"max_tokens\": 64,\n \"system\": [\n {\n \"type\": \"text\",\n \"text\": \"x-anthropic-billing-header: cc_version=2.1.270.42c; cc_entrypoint=cli;\"\n },\n {\n \"type\": \"text\",\n \"text\": \"Reply briefly.\"\n }\n ],\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Say OK.\"\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/anthropic/v1/messages", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "anthropic", + "v1", + "messages" + ] + } + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var text = pm.response.text();", + "pm.test('Messages request succeeds', function () { pm.expect(pm.response.code, text).to.eql(200); });", + "var body = pm.response.json();", + "pm.test('Messages response is valid', function () { pm.expect(body.type, text).to.eql('message'); pm.expect(body.content, text).to.be.an('array').that.is.not.empty; });", + "var raw = (body.extra_fields || {}).raw_request;", + "if (typeof raw === 'string') { raw = JSON.parse(raw); }", + "pm.test('outbound prompt preserves only the intended system content', function () {", + " pm.expect(raw, 'raw request capture missing: ' + text).to.be.an('object');", + " pm.expect(raw.system, JSON.stringify(raw)).to.be.an('array');", + " pm.expect(raw.system[0].text).to.eql(\"x-anthropic-billing-header: cc_version=2.1.270.42c; cc_entrypoint=cli;\");", + " pm.expect(raw.system[1].text).to.eql('Reply briefly.');", + "});" + ] + } + } + ] + } + ] + }, + { + "name": "110. Tool-schema regex relaxation for Moonshot and DeepSeek only (tool-schema-regex)", + "item": [ + { + "name": "110.1 deepseek/deepseek-chat (openai-compatible wire, runs under the openai fork) /v1/responses: \\0 rewritten and lookahead stripped (was 400 'is not a regex') - tool-schema-regex", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var txt = pm.response.text() || '';", + "var body = {};", + "try { body = pm.response.json() || {}; } catch (e) { body = {}; }", + "var rr = (body.extra_fields || {}).raw_request;", + "if (typeof rr === 'string') { try { rr = JSON.parse(rr); } catch (e) { rr = null; } }", + "var wire = JSON.stringify(rr || null);", + "", + "// Every row asserts the OUTBOUND tool schema via x-bf-send-back-raw-request. The", + "// rewrite (providerUtils.NormalizeRegexNULEscape + StripRegexLookaround) runs in", + "// prepareResponsesRequest (core/bifrost.go) ONLY for Moonshot and DeepSeek models;", + "// every other model's schema must reach the wire byte-identical. The comparison", + "// is against the exact expected pattern, not fragments, so a rewrite that", + "// changed any other part of the pattern would fail here.", + "function collectPatterns(node, out) {", + " if (!node || typeof node !== 'object') { return out; }", + " if (Array.isArray(node)) { node.forEach(function (n) { collectPatterns(n, out); }); return out; }", + " Object.keys(node).forEach(function (k) {", + " if (k === 'pattern' && typeof node[k] === 'string') { out.push(node[k]); } else { collectPatterns(node[k], out); }", + " });", + " return out;", + "}", + "var pats = collectPatterns(rr, []);", + "var SENT = '^(?!\\\\.\\\\.?$)[^\\\\0]{1,200}$';", + "var RELAXED = '^[^\\\\x00]{1,200}$';", + "", + "pm.test('request is accepted', function () {", + " pm.expect(pm.response.code, 'want 200: ' + txt.slice(0, 400)).to.eql(200);", + " pm.expect(rr, 'raw_request missing or unparseable: ' + txt.slice(0, 300)).to.be.an('object');", + "});", + "", + "pm.test('pattern on the wire is exactly the relaxed form', function () {", + " pm.expect(pats, 'want exactly [' + RELAXED + '], got: ' + JSON.stringify(pats) + ' wire: ' + wire.slice(0, 300)).to.eql([RELAXED]);", + "});", + "pm.test('the model answers', function () {", + " var u = body.usage || {};", + " pm.expect(u.output_tokens, 'model produced no output: ' + txt.slice(0, 300)).to.be.above(0);", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"deepseek/deepseek-chat\",\n \"max_output_tokens\": 256,\n \"input\": [\n {\n \"role\": \"user\",\n \"content\": \"Say hi\"\n }\n ],\n \"tools\": [\n {\n \"type\": \"function\",\n \"name\": \"probe\",\n \"description\": \"short\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"p\": {\n \"type\": \"string\",\n \"pattern\": \"^(?!\\\\.\\\\.?$)[^\\\\0]{1,200}$\"\n }\n },\n \"required\": [\n \"p\"\n ]\n }\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/responses", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "responses" + ] + } + } + }, + { + "name": "110.2 bedrock/global.moonshotai.kimi-k3 /v1/responses: \\0 rewritten and lookahead stripped so the model answers (was empty stream) - tool-schema-regex", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var txt = pm.response.text() || '';", + "var body = {};", + "try { body = pm.response.json() || {}; } catch (e) { body = {}; }", + "var rr = (body.extra_fields || {}).raw_request;", + "if (typeof rr === 'string') { try { rr = JSON.parse(rr); } catch (e) { rr = null; } }", + "var wire = JSON.stringify(rr || null);", + "", + "// Every row asserts the OUTBOUND tool schema via x-bf-send-back-raw-request. The", + "// rewrite (providerUtils.NormalizeRegexNULEscape + StripRegexLookaround) runs in", + "// prepareResponsesRequest (core/bifrost.go) ONLY for Moonshot and DeepSeek models;", + "// every other model's schema must reach the wire byte-identical. The comparison", + "// is against the exact expected pattern, not fragments, so a rewrite that", + "// changed any other part of the pattern would fail here.", + "function collectPatterns(node, out) {", + " if (!node || typeof node !== 'object') { return out; }", + " if (Array.isArray(node)) { node.forEach(function (n) { collectPatterns(n, out); }); return out; }", + " Object.keys(node).forEach(function (k) {", + " if (k === 'pattern' && typeof node[k] === 'string') { out.push(node[k]); } else { collectPatterns(node[k], out); }", + " });", + " return out;", + "}", + "var pats = collectPatterns(rr, []);", + "var SENT = '^(?!\\\\.\\\\.?$)[^\\\\0]{1,200}$';", + "var RELAXED = '^[^\\\\x00]{1,200}$';", + "", + "pm.test('request is accepted', function () {", + " pm.expect(pm.response.code, 'want 200: ' + txt.slice(0, 400)).to.eql(200);", + " pm.expect(rr, 'raw_request missing or unparseable: ' + txt.slice(0, 300)).to.be.an('object');", + "});", + "", + "pm.test('pattern on the wire is exactly the relaxed form', function () {", + " pm.expect(pats, 'want exactly [' + RELAXED + '], got: ' + JSON.stringify(pats) + ' wire: ' + wire.slice(0, 300)).to.eql([RELAXED]);", + "});", + "pm.test('the model answers', function () {", + " var u = body.usage || {};", + " pm.expect(u.output_tokens, 'model produced no output: ' + txt.slice(0, 300)).to.be.above(0);", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"bedrock/global.moonshotai.kimi-k3\",\n \"max_output_tokens\": 256,\n \"input\": [\n {\n \"role\": \"user\",\n \"content\": \"Say hi\"\n }\n ],\n \"tools\": [\n {\n \"type\": \"function\",\n \"name\": \"probe\",\n \"description\": \"short\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"p\": {\n \"type\": \"string\",\n \"pattern\": \"^(?!\\\\.\\\\.?$)[^\\\\0]{1,200}$\"\n }\n },\n \"required\": [\n \"p\"\n ]\n }\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/responses", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "responses" + ] + } + } + }, + { + "name": "110.3 openai/gpt-4o-mini /v1/responses: pattern reaches the provider byte-identical - tool-schema-regex", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var txt = pm.response.text() || '';", + "var body = {};", + "try { body = pm.response.json() || {}; } catch (e) { body = {}; }", + "var rr = (body.extra_fields || {}).raw_request;", + "if (typeof rr === 'string') { try { rr = JSON.parse(rr); } catch (e) { rr = null; } }", + "var wire = JSON.stringify(rr || null);", + "", + "// Every row asserts the OUTBOUND tool schema via x-bf-send-back-raw-request. The", + "// rewrite (providerUtils.NormalizeRegexNULEscape + StripRegexLookaround) runs in", + "// prepareResponsesRequest (core/bifrost.go) ONLY for Moonshot and DeepSeek models;", + "// every other model's schema must reach the wire byte-identical. The comparison", + "// is against the exact expected pattern, not fragments, so a rewrite that", + "// changed any other part of the pattern would fail here.", + "function collectPatterns(node, out) {", + " if (!node || typeof node !== 'object') { return out; }", + " if (Array.isArray(node)) { node.forEach(function (n) { collectPatterns(n, out); }); return out; }", + " Object.keys(node).forEach(function (k) {", + " if (k === 'pattern' && typeof node[k] === 'string') { out.push(node[k]); } else { collectPatterns(node[k], out); }", + " });", + " return out;", + "}", + "var pats = collectPatterns(rr, []);", + "var SENT = '^(?!\\\\.\\\\.?$)[^\\\\0]{1,200}$';", + "var RELAXED = '^[^\\\\x00]{1,200}$';", + "", + "pm.test('request is accepted', function () {", + " pm.expect(pm.response.code, 'want 200: ' + txt.slice(0, 400)).to.eql(200);", + " pm.expect(rr, 'raw_request missing or unparseable: ' + txt.slice(0, 300)).to.be.an('object');", + "});", + "", + "// Not Moonshot, not DeepSeek: the schema must be byte-identical to what the client", + "// sent. This is what keeps prompt-cache keys stable for every other model.", + "pm.test('pattern reaches the provider byte-identical', function () {", + " pm.expect(pats, 'want exactly [' + SENT + '], got: ' + JSON.stringify(pats) + ' wire: ' + wire.slice(0, 300)).to.eql([SENT]);", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Authorization", + "value": "Bearer {{openaiKey}}" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"openai/gpt-4o-mini\",\n \"max_output_tokens\": 256,\n \"input\": [\n {\n \"role\": \"user\",\n \"content\": \"Say hi\"\n }\n ],\n \"tools\": [\n {\n \"type\": \"function\",\n \"name\": \"probe\",\n \"description\": \"short\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"p\": {\n \"type\": \"string\",\n \"pattern\": \"^(?!\\\\.\\\\.?$)[^\\\\0]{1,200}$\"\n }\n },\n \"required\": [\n \"p\"\n ]\n }\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/responses", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "responses" + ] + } + } + }, + { + "name": "110.4 anthropic/claude-haiku-4-5 /v1/responses: pattern reaches the provider byte-identical - tool-schema-regex", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var txt = pm.response.text() || '';", + "var body = {};", + "try { body = pm.response.json() || {}; } catch (e) { body = {}; }", + "var rr = (body.extra_fields || {}).raw_request;", + "if (typeof rr === 'string') { try { rr = JSON.parse(rr); } catch (e) { rr = null; } }", + "var wire = JSON.stringify(rr || null);", + "", + "// Every row asserts the OUTBOUND tool schema via x-bf-send-back-raw-request. The", + "// rewrite (providerUtils.NormalizeRegexNULEscape + StripRegexLookaround) runs in", + "// prepareResponsesRequest (core/bifrost.go) ONLY for Moonshot and DeepSeek models;", + "// every other model's schema must reach the wire byte-identical. The comparison", + "// is against the exact expected pattern, not fragments, so a rewrite that", + "// changed any other part of the pattern would fail here.", + "function collectPatterns(node, out) {", + " if (!node || typeof node !== 'object') { return out; }", + " if (Array.isArray(node)) { node.forEach(function (n) { collectPatterns(n, out); }); return out; }", + " Object.keys(node).forEach(function (k) {", + " if (k === 'pattern' && typeof node[k] === 'string') { out.push(node[k]); } else { collectPatterns(node[k], out); }", + " });", + " return out;", + "}", + "var pats = collectPatterns(rr, []);", + "var SENT = '^(?!\\\\.\\\\.?$)[^\\\\0]{1,200}$';", + "var RELAXED = '^[^\\\\x00]{1,200}$';", + "", + "pm.test('request is accepted', function () {", + " pm.expect(pm.response.code, 'want 200: ' + txt.slice(0, 400)).to.eql(200);", + " pm.expect(rr, 'raw_request missing or unparseable: ' + txt.slice(0, 300)).to.be.an('object');", + "});", + "", + "// Not Moonshot, not DeepSeek: the schema must be byte-identical to what the client", + "// sent. This is what keeps prompt-cache keys stable for every other model.", + "pm.test('pattern reaches the provider byte-identical', function () {", + " pm.expect(pats, 'want exactly [' + SENT + '], got: ' + JSON.stringify(pats) + ' wire: ' + wire.slice(0, 300)).to.eql([SENT]);", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Authorization", + "value": "Bearer {{anthropicKey}}" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"anthropic/claude-haiku-4-5\",\n \"max_output_tokens\": 256,\n \"input\": [\n {\n \"role\": \"user\",\n \"content\": \"Say hi\"\n }\n ],\n \"tools\": [\n {\n \"type\": \"function\",\n \"name\": \"probe\",\n \"description\": \"short\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"p\": {\n \"type\": \"string\",\n \"pattern\": \"^(?!\\\\.\\\\.?$)[^\\\\0]{1,200}$\"\n }\n },\n \"required\": [\n \"p\"\n ]\n }\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/responses", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "responses" + ] + } + } + }, + { + "name": "110.5 gemini/gemini-3.6-flash /v1/responses: pattern reaches the provider byte-identical - tool-schema-regex", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var txt = pm.response.text() || '';", + "var body = {};", + "try { body = pm.response.json() || {}; } catch (e) { body = {}; }", + "var rr = (body.extra_fields || {}).raw_request;", + "if (typeof rr === 'string') { try { rr = JSON.parse(rr); } catch (e) { rr = null; } }", + "var wire = JSON.stringify(rr || null);", + "", + "// Every row asserts the OUTBOUND tool schema via x-bf-send-back-raw-request. The", + "// rewrite (providerUtils.NormalizeRegexNULEscape + StripRegexLookaround) runs in", + "// prepareResponsesRequest (core/bifrost.go) ONLY for Moonshot and DeepSeek models;", + "// every other model's schema must reach the wire byte-identical. The comparison", + "// is against the exact expected pattern, not fragments, so a rewrite that", + "// changed any other part of the pattern would fail here.", + "function collectPatterns(node, out) {", + " if (!node || typeof node !== 'object') { return out; }", + " if (Array.isArray(node)) { node.forEach(function (n) { collectPatterns(n, out); }); return out; }", + " Object.keys(node).forEach(function (k) {", + " if (k === 'pattern' && typeof node[k] === 'string') { out.push(node[k]); } else { collectPatterns(node[k], out); }", + " });", + " return out;", + "}", + "var pats = collectPatterns(rr, []);", + "var SENT = '^(?!\\\\.\\\\.?$)[^\\\\0]{1,200}$';", + "var RELAXED = '^[^\\\\x00]{1,200}$';", + "", + "pm.test('request is accepted', function () {", + " pm.expect(pm.response.code, 'want 200: ' + txt.slice(0, 400)).to.eql(200);", + " pm.expect(rr, 'raw_request missing or unparseable: ' + txt.slice(0, 300)).to.be.an('object');", + "});", + "", + "// Not Moonshot, not DeepSeek: the schema must be byte-identical to what the client", + "// sent. This is what keeps prompt-cache keys stable for every other model.", + "pm.test('pattern reaches the provider byte-identical', function () {", + " pm.expect(pats, 'want exactly [' + SENT + '], got: ' + JSON.stringify(pats) + ' wire: ' + wire.slice(0, 300)).to.eql([SENT]);", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"gemini/gemini-3.6-flash\",\n \"max_output_tokens\": 256,\n \"input\": [\n {\n \"role\": \"user\",\n \"content\": \"Say hi\"\n }\n ],\n \"tools\": [\n {\n \"type\": \"function\",\n \"name\": \"probe\",\n \"description\": \"short\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"p\": {\n \"type\": \"string\",\n \"pattern\": \"^(?!\\\\.\\\\.?$)[^\\\\0]{1,200}$\"\n }\n },\n \"required\": [\n \"p\"\n ]\n }\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/responses", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "responses" + ] + } + } + } + ] + }, + { + "name": "111. Claude Opus 5.5 request surface (opus-5-5)", + "description": "Claude Opus 5.5 changed the request surface in ways that 400 if forwarded unchanged, verified against the live Claude API: thinking:{\"type\":\"disabled\"} and thinking:{\"type\":\"enabled\"} are rejected at every effort level (Opus 5 accepts disabled at effort high or below); forced tool_choice \"any\"/\"tool\" is rejected, on count_tokens too; and computer use accepts only computer_toolset_20260801 - both computer_20251124 and computer_20250124 are rejected on the Claude API and Google Cloud, while AWS and Foundry still serve the dated tool. A toolset entry carries no name and no display_* geometry, its calls return under member names (screenshot, left_click) tagged with toolset_name, and that tag must appear on both halves of a call/result pair or neither - a half-preserved pair is a 400 in either direction. These cases assert the OUTBOUND body via x-bf-send-back-raw-request, so they pin Bifrost's rewrite rather than merely that upstream accepted the request.", + "item": [ + { + "name": "Anthropic /v1/messages - thinking disabled is never forwarded (claude-opus-5-5) - opus-5-5", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('request succeeded', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + "});", + "var body = pm.response.json();", + "var raw = (body.extra_fields || {}).raw_request;", + "if (typeof raw === 'string') { raw = JSON.parse(raw); }", + "pm.test('thinking:disabled never reaches Opus 5.5 at any effort', function () {", + " pm.expect(raw, 'raw request capture missing: ' + pm.response.text()).to.exist;", + " if (raw.thinking) {", + " pm.expect(raw.thinking.type, 'disabled thinking was forwarded and upstream 400s on it').to.not.eql('disabled');", + " }", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-api-key", + "value": "{{anthropicKey}}" + }, + { + "key": "anthropic-version", + "value": "2023-06-01" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"anthropic/claude-opus-5-5\",\n \"max_tokens\": 16,\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"thinking\": {\n \"type\": \"disabled\"\n }\n}" + }, + "url": { + "raw": "{{baseUrl}}/anthropic/v1/messages", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "anthropic", + "v1", + "messages" + ] + } + } + }, + { + "name": "Anthropic /v1/messages - thinking disabled dropped at effort max, effort preserved (claude-opus-5-5) - opus-5-5", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('request succeeded', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + "});", + "var body = pm.response.json();", + "var raw = (body.extra_fields || {}).raw_request;", + "if (typeof raw === 'string') { raw = JSON.parse(raw); }", + "pm.test('disabled thinking dropped but the requested effort survives', function () {", + " pm.expect(raw, 'raw request capture missing: ' + pm.response.text()).to.exist;", + " if (raw.thinking) { pm.expect(raw.thinking.type).to.not.eql('disabled'); }", + " pm.expect(raw.output_config, 'output_config dropped').to.exist;", + " pm.expect(raw.output_config.effort, 'caller effort must not be lost').to.eql('max');", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-api-key", + "value": "{{anthropicKey}}" + }, + { + "key": "anthropic-version", + "value": "2023-06-01" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"anthropic/claude-opus-5-5\",\n \"max_tokens\": 16,\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"thinking\": {\n \"type\": \"disabled\"\n },\n \"output_config\": {\n \"effort\": \"max\"\n }\n}" + }, + "url": { + "raw": "{{baseUrl}}/anthropic/v1/messages", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "anthropic", + "v1", + "messages" + ] + } + } + }, + { + "name": "Anthropic /v1/messages - thinking enabled with budget_tokens becomes adaptive (claude-opus-5-5) - opus-5-5", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('request succeeded', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + "});", + "var body = pm.response.json();", + "var raw = (body.extra_fields || {}).raw_request;", + "if (typeof raw === 'string') { raw = JSON.parse(raw); }", + "pm.test('extended thinking is rewritten to adaptive and the budget dropped', function () {", + " pm.expect(raw, 'raw request capture missing: ' + pm.response.text()).to.exist;", + " pm.expect(raw.thinking, 'thinking dropped entirely').to.exist;", + " pm.expect(raw.thinking.type, 'Opus 5.5 rejects thinking.type.enabled').to.eql('adaptive');", + " pm.expect(raw.thinking.budget_tokens, 'budget_tokens is inert under adaptive and must not ride along').to.be.undefined;", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-api-key", + "value": "{{anthropicKey}}" + }, + { + "key": "anthropic-version", + "value": "2023-06-01" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"anthropic/claude-opus-5-5\",\n \"max_tokens\": 2048,\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"thinking\": {\n \"type\": \"enabled\",\n \"budget_tokens\": 1024\n }\n}" + }, + "url": { + "raw": "{{baseUrl}}/anthropic/v1/messages", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "anthropic", + "v1", + "messages" + ] + } + } + }, + { + "name": "Anthropic /v1/messages - forced tool_choice any is dropped (claude-opus-5-5) - opus-5-5", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('request succeeded', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + "});", + "var body = pm.response.json();", + "var raw = (body.extra_fields || {}).raw_request;", + "if (typeof raw === 'string') { raw = JSON.parse(raw); }", + "pm.test('forced tool choice is dropped, tools survive', function () {", + " pm.expect(raw, 'raw request capture missing: ' + pm.response.text()).to.exist;", + " if (raw.tool_choice) {", + " pm.expect(['any', 'tool'], 'Opus 5.5 rejects forced tool_choice').to.not.include(raw.tool_choice.type);", + " }", + " pm.expect(raw.tools, 'dropping the pin must not drop the tools').to.exist;", + " pm.expect(raw.tools.length).to.eql(1);", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-api-key", + "value": "{{anthropicKey}}" + }, + { + "key": "anthropic-version", + "value": "2023-06-01" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"anthropic/claude-opus-5-5\",\n \"max_tokens\": 16,\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"tools\": [\n {\n \"name\": \"noop\",\n \"description\": \"does nothing\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {}\n }\n }\n ],\n \"tool_choice\": {\n \"type\": \"any\"\n }\n}" + }, + "url": { + "raw": "{{baseUrl}}/anthropic/v1/messages", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "anthropic", + "v1", + "messages" + ] + } + } + }, + { + "name": "Anthropic /v1/messages/count_tokens - forced named tool_choice is dropped (claude-opus-5-5) - opus-5-5", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('count_tokens applies the same forced tool_choice validation', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + " var body = pm.response.json();", + " pm.expect(body.input_tokens, 'count_tokens returned no token count: ' + pm.response.text()).to.be.a('number');", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-api-key", + "value": "{{anthropicKey}}" + }, + { + "key": "anthropic-version", + "value": "2023-06-01" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"anthropic/claude-opus-5-5\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"tools\": [\n {\n \"name\": \"noop\",\n \"description\": \"does nothing\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {}\n }\n }\n ],\n \"tool_choice\": {\n \"type\": \"tool\",\n \"name\": \"noop\"\n }\n}" + }, + "url": { + "raw": "{{baseUrl}}/anthropic/v1/messages/count_tokens", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "anthropic", + "v1", + "messages", + "count_tokens" + ] + } + } + }, + { + "name": "Anthropic /v1/messages - computer toolset sent bare alongside text_editor and bash (claude-opus-5-5) - opus-5-5", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('request succeeded', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + "});", + "var body = pm.response.json();", + "var raw = (body.extra_fields || {}).raw_request;", + "if (typeof raw === 'string') { raw = JSON.parse(raw); }", + "pm.test('toolset entry goes out bare - name and display_* are rejected on it', function () {", + " pm.expect(raw, 'raw request capture missing: ' + pm.response.text()).to.exist;", + " var ct = null;", + " for (var i = 0; i < raw.tools.length; i++) {", + " if (String(raw.tools[i].type || '').indexOf('computer') === 0) { ct = raw.tools[i]; }", + " }", + " pm.expect(ct, 'computer tool dropped from the outbound body').to.exist;", + " pm.expect(ct.type, 'Opus 5.5 accepts only the toolset form').to.eql('computer_toolset_20260801');", + " pm.expect(ct.name, 'name is not accepted on a toolset entry').to.be.undefined;", + " pm.expect(ct.display_width_px, 'display_* is not accepted on a toolset entry').to.be.undefined;", + " pm.expect(ct.display_height_px).to.be.undefined;", + "});", + "pm.test('member calls come back tagged with their toolset', function () {", + " var calls = [];", + " for (var i = 0; i < body.content.length; i++) {", + " if (body.content[i].type === 'tool_use') { calls.push(body.content[i]); }", + " }", + " pm.expect(calls.length, 'model made no tool call: ' + pm.response.text()).to.be.above(0);", + " for (var j = 0; j < calls.length; j++) {", + " pm.expect(calls[j].toolset_name, 'member call lost toolset_name: ' + JSON.stringify(calls[j])).to.eql('computer');", + " pm.expect(calls[j].name, 'member name must be the member, not the family').to.not.eql('computer');", + " }", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-api-key", + "value": "{{anthropicKey}}" + }, + { + "key": "anthropic-version", + "value": "2023-06-01" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"anthropic/claude-opus-5-5\",\n \"max_tokens\": 512,\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Take a screenshot, then click at 100,200. Do it now.\"\n }\n ],\n \"tools\": [\n {\n \"type\": \"computer_toolset_20260801\"\n },\n {\n \"type\": \"text_editor_20250728\",\n \"name\": \"str_replace_based_edit_tool\"\n },\n {\n \"type\": \"bash_20250124\",\n \"name\": \"bash\"\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/anthropic/v1/messages", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "anthropic", + "v1", + "messages" + ] + } + } + }, + { + "name": "Anthropic /v1/messages stream - computer toolset member calls carry toolset_name (claude-opus-5-5) - opus-5-5", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('stream completed without an upstream rejection', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + " pm.expect(pm.response.text(), 'stream carried an error frame').to.not.include('invalid_request_error');", + "});", + "pm.test('content_block_start for a member call carries toolset_name', function () {", + " var lines = pm.response.text().split('\\n');", + " var seen = 0;", + " for (var i = 0; i < lines.length; i++) {", + " if (lines[i].indexOf('data: ') !== 0) { continue; }", + " var payload = lines[i].slice(6).trim();", + " if (!payload || payload === '[DONE]') { continue; }", + " var ev;", + " try { ev = JSON.parse(payload); } catch (e) { continue; }", + " if (ev.type !== 'content_block_start') { continue; }", + " var cb = ev.content_block || {};", + " if (cb.type !== 'tool_use') { continue; }", + " seen++;", + " pm.expect(cb.toolset_name, 'streamed member call lost toolset_name: ' + payload).to.eql('computer');", + " }", + " pm.expect(seen, 'no tool_use content_block_start in the stream: ' + pm.response.text().slice(0, 400)).to.be.above(0);", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-api-key", + "value": "{{anthropicKey}}" + }, + { + "key": "anthropic-version", + "value": "2023-06-01" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"anthropic/claude-opus-5-5\",\n \"max_tokens\": 512,\n \"stream\": true,\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Take a screenshot, then click at 100,200. Do it now.\"\n }\n ],\n \"tools\": [\n {\n \"type\": \"computer_toolset_20260801\"\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/anthropic/v1/messages", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "anthropic", + "v1", + "messages" + ] + } + } + }, + { + "name": "Anthropic /v1/messages - dated computer_20251124 upgraded to the toolset (claude-opus-5-5) - opus-5-5", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('request succeeded', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + "});", + "var body = pm.response.json();", + "var raw = (body.extra_fields || {}).raw_request;", + "if (typeof raw === 'string') { raw = JSON.parse(raw); }", + "pm.test('the dated tool is upgraded, not relayed into a 400', function () {", + " pm.expect(raw, 'raw request capture missing: ' + pm.response.text()).to.exist;", + " pm.expect(raw.tools[0].type, 'Opus 5.5 rejects every dated computer_* type').to.eql('computer_toolset_20260801');", + " pm.expect(raw.tools[0].name, 'name must be shed when upgrading to a toolset').to.be.undefined;", + " pm.expect(raw.tools[0].display_width_px, 'display_* must be shed when upgrading to a toolset').to.be.undefined;", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-api-key", + "value": "{{anthropicKey}}" + }, + { + "key": "anthropic-version", + "value": "2023-06-01" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"anthropic/claude-opus-5-5\",\n \"max_tokens\": 64,\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"tools\": [\n {\n \"type\": \"computer_20251124\",\n \"name\": \"computer\",\n \"display_width_px\": 1280,\n \"display_height_px\": 800\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/anthropic/v1/messages", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "anthropic", + "v1", + "messages" + ] + } + } + }, + { + "name": "Anthropic /v1/messages - tool_result toolset_name backfilled for a member call (claude-opus-5-5) - opus-5-5", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('request succeeded', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + "});", + "var body = pm.response.json();", + "var raw = (body.extra_fields || {}).raw_request;", + "if (typeof raw === 'string') { raw = JSON.parse(raw); }", + "pm.test('a member result is tagged even when the caller omits toolset_name', function () {", + " pm.expect(raw, 'raw request capture missing: ' + pm.response.text()).to.exist;", + " var call = null, result = null;", + " for (var i = 0; i < raw.messages.length; i++) {", + " var c = raw.messages[i].content;", + " if (Object.prototype.toString.call(c) !== '[object Array]') { continue; }", + " for (var j = 0; j < c.length; j++) {", + " if (c[j].type === 'tool_use') { call = c[j]; }", + " if (c[j].type === 'tool_result') { result = c[j]; }", + " }", + " }", + " pm.expect(call, 'tool_use missing from the outbound body').to.exist;", + " pm.expect(result, 'tool_result missing from the outbound body').to.exist;", + " pm.expect(call.toolset_name, 'member call lost toolset_name').to.eql('computer');", + " pm.expect(result.toolset_name, 'half-preserved pair: upstream 400s when only one side carries toolset_name').to.eql('computer');", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-api-key", + "value": "{{anthropicKey}}" + }, + { + "key": "anthropic-version", + "value": "2023-06-01" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"anthropic/claude-opus-5-5\",\n \"max_tokens\": 64,\n \"tools\": [\n {\n \"type\": \"computer_toolset_20260801\"\n }\n ],\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Take a screenshot.\"\n },\n {\n \"role\": \"assistant\",\n \"content\": [\n {\n \"type\": \"tool_use\",\n \"id\": \"toolu_01HarnessScreenshotAAA\",\n \"name\": \"screenshot\",\n \"input\": {},\n \"toolset_name\": \"computer\"\n }\n ]\n },\n {\n \"role\": \"user\",\n \"content\": [\n {\n \"type\": \"tool_result\",\n \"tool_use_id\": \"toolu_01HarnessScreenshotAAA\",\n \"content\": \"ok\"\n }\n ]\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/anthropic/v1/messages", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "anthropic", + "v1", + "messages" + ] + } + } + }, + { + "name": "Vertex /v1/messages - thinking disabled is never forwarded (vertex claude-opus-5-5) - opus-5-5", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('request succeeded', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + "});", + "var body = pm.response.json();", + "var raw = (body.extra_fields || {}).raw_request;", + "if (typeof raw === 'string') { raw = JSON.parse(raw); }", + "pm.test('thinking:disabled never reaches Opus 5.5 at any effort', function () {", + " pm.expect(raw, 'raw request capture missing: ' + pm.response.text()).to.exist;", + " if (raw.thinking) {", + " pm.expect(raw.thinking.type, 'disabled thinking was forwarded and upstream 400s on it').to.not.eql('disabled');", + " }", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-api-key", + "value": "{{anthropicKey}}" + }, + { + "key": "anthropic-version", + "value": "2023-06-01" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"vertex/claude-opus-5-5\",\n \"max_tokens\": 16,\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"thinking\": {\n \"type\": \"disabled\"\n }\n}" + }, + "url": { + "raw": "{{baseUrl}}/anthropic/v1/messages", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "anthropic", + "v1", + "messages" + ] + } + } + }, + { + "name": "Vertex /v1/messages - thinking disabled dropped at effort max, effort preserved (vertex claude-opus-5-5) - opus-5-5", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('request succeeded', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + "});", + "var body = pm.response.json();", + "var raw = (body.extra_fields || {}).raw_request;", + "if (typeof raw === 'string') { raw = JSON.parse(raw); }", + "pm.test('disabled thinking dropped but the requested effort survives', function () {", + " pm.expect(raw, 'raw request capture missing: ' + pm.response.text()).to.exist;", + " if (raw.thinking) { pm.expect(raw.thinking.type).to.not.eql('disabled'); }", + " pm.expect(raw.output_config, 'output_config dropped').to.exist;", + " pm.expect(raw.output_config.effort, 'caller effort must not be lost').to.eql('max');", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-api-key", + "value": "{{anthropicKey}}" + }, + { + "key": "anthropic-version", + "value": "2023-06-01" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"vertex/claude-opus-5-5\",\n \"max_tokens\": 16,\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"thinking\": {\n \"type\": \"disabled\"\n },\n \"output_config\": {\n \"effort\": \"max\"\n }\n}" + }, + "url": { + "raw": "{{baseUrl}}/anthropic/v1/messages", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "anthropic", + "v1", + "messages" + ] + } + } + }, + { + "name": "Vertex /v1/messages - thinking enabled with budget_tokens becomes adaptive (vertex claude-opus-5-5) - opus-5-5", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('request succeeded', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + "});", + "var body = pm.response.json();", + "var raw = (body.extra_fields || {}).raw_request;", + "if (typeof raw === 'string') { raw = JSON.parse(raw); }", + "pm.test('extended thinking is rewritten to adaptive and the budget dropped', function () {", + " pm.expect(raw, 'raw request capture missing: ' + pm.response.text()).to.exist;", + " pm.expect(raw.thinking, 'thinking dropped entirely').to.exist;", + " pm.expect(raw.thinking.type, 'Opus 5.5 rejects thinking.type.enabled').to.eql('adaptive');", + " pm.expect(raw.thinking.budget_tokens, 'budget_tokens is inert under adaptive and must not ride along').to.be.undefined;", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-api-key", + "value": "{{anthropicKey}}" + }, + { + "key": "anthropic-version", + "value": "2023-06-01" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"vertex/claude-opus-5-5\",\n \"max_tokens\": 2048,\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"thinking\": {\n \"type\": \"enabled\",\n \"budget_tokens\": 1024\n }\n}" + }, + "url": { + "raw": "{{baseUrl}}/anthropic/v1/messages", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "anthropic", + "v1", + "messages" + ] + } + } + }, + { + "name": "Vertex /v1/messages - forced tool_choice any is dropped (vertex claude-opus-5-5) - opus-5-5", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('request succeeded', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + "});", + "var body = pm.response.json();", + "var raw = (body.extra_fields || {}).raw_request;", + "if (typeof raw === 'string') { raw = JSON.parse(raw); }", + "pm.test('forced tool choice is dropped, tools survive', function () {", + " pm.expect(raw, 'raw request capture missing: ' + pm.response.text()).to.exist;", + " if (raw.tool_choice) {", + " pm.expect(['any', 'tool'], 'Opus 5.5 rejects forced tool_choice').to.not.include(raw.tool_choice.type);", + " }", + " pm.expect(raw.tools, 'dropping the pin must not drop the tools').to.exist;", + " pm.expect(raw.tools.length).to.eql(1);", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-api-key", + "value": "{{anthropicKey}}" + }, + { + "key": "anthropic-version", + "value": "2023-06-01" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"vertex/claude-opus-5-5\",\n \"max_tokens\": 16,\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"tools\": [\n {\n \"name\": \"noop\",\n \"description\": \"does nothing\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {}\n }\n }\n ],\n \"tool_choice\": {\n \"type\": \"any\"\n }\n}" + }, + "url": { + "raw": "{{baseUrl}}/anthropic/v1/messages", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "anthropic", + "v1", + "messages" + ] + } + } + }, + { + "name": "Vertex /v1/messages - computer toolset sent bare alongside text_editor and bash (vertex claude-opus-5-5) - opus-5-5", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('request succeeded', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + "});", + "var body = pm.response.json();", + "var raw = (body.extra_fields || {}).raw_request;", + "if (typeof raw === 'string') { raw = JSON.parse(raw); }", + "pm.test('toolset entry goes out bare - name and display_* are rejected on it', function () {", + " pm.expect(raw, 'raw request capture missing: ' + pm.response.text()).to.exist;", + " var ct = null;", + " for (var i = 0; i < raw.tools.length; i++) {", + " if (String(raw.tools[i].type || '').indexOf('computer') === 0) { ct = raw.tools[i]; }", + " }", + " pm.expect(ct, 'computer tool dropped from the outbound body').to.exist;", + " pm.expect(ct.type, 'Opus 5.5 accepts only the toolset form').to.eql('computer_toolset_20260801');", + " pm.expect(ct.name, 'name is not accepted on a toolset entry').to.be.undefined;", + " pm.expect(ct.display_width_px, 'display_* is not accepted on a toolset entry').to.be.undefined;", + " pm.expect(ct.display_height_px).to.be.undefined;", + "});", + "pm.test('member calls come back tagged with their toolset', function () {", + " var calls = [];", + " for (var i = 0; i < body.content.length; i++) {", + " if (body.content[i].type === 'tool_use') { calls.push(body.content[i]); }", + " }", + " pm.expect(calls.length, 'model made no tool call: ' + pm.response.text()).to.be.above(0);", + " for (var j = 0; j < calls.length; j++) {", + " pm.expect(calls[j].toolset_name, 'member call lost toolset_name: ' + JSON.stringify(calls[j])).to.eql('computer');", + " pm.expect(calls[j].name, 'member name must be the member, not the family').to.not.eql('computer');", + " }", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-api-key", + "value": "{{anthropicKey}}" + }, + { + "key": "anthropic-version", + "value": "2023-06-01" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"vertex/claude-opus-5-5\",\n \"max_tokens\": 512,\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Take a screenshot, then click at 100,200. Do it now.\"\n }\n ],\n \"tools\": [\n {\n \"type\": \"computer_toolset_20260801\"\n },\n {\n \"type\": \"text_editor_20250728\",\n \"name\": \"str_replace_based_edit_tool\"\n },\n {\n \"type\": \"bash_20250124\",\n \"name\": \"bash\"\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/anthropic/v1/messages", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "anthropic", + "v1", + "messages" + ] + } + } + }, + { + "name": "Vertex /v1/messages stream - computer toolset member calls carry toolset_name (vertex claude-opus-5-5) - opus-5-5", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('stream completed without an upstream rejection', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + " pm.expect(pm.response.text(), 'stream carried an error frame').to.not.include('invalid_request_error');", + "});", + "pm.test('content_block_start for a member call carries toolset_name', function () {", + " var lines = pm.response.text().split('\\n');", + " var seen = 0;", + " for (var i = 0; i < lines.length; i++) {", + " if (lines[i].indexOf('data: ') !== 0) { continue; }", + " var payload = lines[i].slice(6).trim();", + " if (!payload || payload === '[DONE]') { continue; }", + " var ev;", + " try { ev = JSON.parse(payload); } catch (e) { continue; }", + " if (ev.type !== 'content_block_start') { continue; }", + " var cb = ev.content_block || {};", + " if (cb.type !== 'tool_use') { continue; }", + " seen++;", + " pm.expect(cb.toolset_name, 'streamed member call lost toolset_name: ' + payload).to.eql('computer');", + " }", + " pm.expect(seen, 'no tool_use content_block_start in the stream: ' + pm.response.text().slice(0, 400)).to.be.above(0);", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-api-key", + "value": "{{anthropicKey}}" + }, + { + "key": "anthropic-version", + "value": "2023-06-01" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"vertex/claude-opus-5-5\",\n \"max_tokens\": 512,\n \"stream\": true,\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Take a screenshot, then click at 100,200. Do it now.\"\n }\n ],\n \"tools\": [\n {\n \"type\": \"computer_toolset_20260801\"\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/anthropic/v1/messages", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "anthropic", + "v1", + "messages" + ] + } + } + }, + { + "name": "Vertex /v1/messages - dated computer_20251124 upgraded to the toolset (vertex claude-opus-5-5) - opus-5-5", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('request succeeded', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + "});", + "var body = pm.response.json();", + "var raw = (body.extra_fields || {}).raw_request;", + "if (typeof raw === 'string') { raw = JSON.parse(raw); }", + "pm.test('the dated tool is upgraded, not relayed into a 400', function () {", + " pm.expect(raw, 'raw request capture missing: ' + pm.response.text()).to.exist;", + " pm.expect(raw.tools[0].type, 'Opus 5.5 rejects every dated computer_* type').to.eql('computer_toolset_20260801');", + " pm.expect(raw.tools[0].name, 'name must be shed when upgrading to a toolset').to.be.undefined;", + " pm.expect(raw.tools[0].display_width_px, 'display_* must be shed when upgrading to a toolset').to.be.undefined;", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-api-key", + "value": "{{anthropicKey}}" + }, + { + "key": "anthropic-version", + "value": "2023-06-01" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"vertex/claude-opus-5-5\",\n \"max_tokens\": 64,\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"tools\": [\n {\n \"type\": \"computer_20251124\",\n \"name\": \"computer\",\n \"display_width_px\": 1280,\n \"display_height_px\": 800\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/anthropic/v1/messages", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "anthropic", + "v1", + "messages" + ] + } + } + }, + { + "name": "Vertex /v1/messages - tool_result toolset_name backfilled for a member call (vertex claude-opus-5-5) - opus-5-5", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('request succeeded', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + "});", + "var body = pm.response.json();", + "var raw = (body.extra_fields || {}).raw_request;", + "if (typeof raw === 'string') { raw = JSON.parse(raw); }", + "pm.test('a member result is tagged even when the caller omits toolset_name', function () {", + " pm.expect(raw, 'raw request capture missing: ' + pm.response.text()).to.exist;", + " var call = null, result = null;", + " for (var i = 0; i < raw.messages.length; i++) {", + " var c = raw.messages[i].content;", + " if (Object.prototype.toString.call(c) !== '[object Array]') { continue; }", + " for (var j = 0; j < c.length; j++) {", + " if (c[j].type === 'tool_use') { call = c[j]; }", + " if (c[j].type === 'tool_result') { result = c[j]; }", + " }", + " }", + " pm.expect(call, 'tool_use missing from the outbound body').to.exist;", + " pm.expect(result, 'tool_result missing from the outbound body').to.exist;", + " pm.expect(call.toolset_name, 'member call lost toolset_name').to.eql('computer');", + " pm.expect(result.toolset_name, 'half-preserved pair: upstream 400s when only one side carries toolset_name').to.eql('computer');", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-api-key", + "value": "{{anthropicKey}}" + }, + { + "key": "anthropic-version", + "value": "2023-06-01" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"vertex/claude-opus-5-5\",\n \"max_tokens\": 64,\n \"tools\": [\n {\n \"type\": \"computer_toolset_20260801\"\n }\n ],\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Take a screenshot.\"\n },\n {\n \"role\": \"assistant\",\n \"content\": [\n {\n \"type\": \"tool_use\",\n \"id\": \"toolu_01HarnessScreenshotAAA\",\n \"name\": \"screenshot\",\n \"input\": {},\n \"toolset_name\": \"computer\"\n }\n ]\n },\n {\n \"role\": \"user\",\n \"content\": [\n {\n \"type\": \"tool_result\",\n \"tool_use_id\": \"toolu_01HarnessScreenshotAAA\",\n \"content\": \"ok\"\n }\n ]\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/anthropic/v1/messages", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "anthropic", + "v1", + "messages" + ] + } + } + }, + { + "name": "Anthropic /v1/responses - reasoning effort none never becomes thinking disabled (anthropic/claude-opus-5-5) - opus-5-5", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('request succeeded', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + "});", + "var body = pm.response.json();", + "var raw = (body.extra_fields || {}).raw_request;", + "if (typeof raw === 'string') { raw = JSON.parse(raw); }", + "pm.test('the OpenAI-shaped reasoning-off maps to omitted thinking, not disabled', function () {", + " pm.expect(raw, 'raw request capture missing: ' + pm.response.text()).to.exist;", + " if (raw.thinking) { pm.expect(raw.thinking.type, 'Opus 5.5 rejects thinking.type.disabled').to.not.eql('disabled'); }", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Authorization", + "value": "Bearer {{anthropicKey}}" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"anthropic/claude-opus-5-5\",\n \"max_output_tokens\": 16,\n \"input\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"reasoning\": {\n \"effort\": \"none\"\n }\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/responses", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "responses" + ] + } + } + }, + { + "name": "Anthropic /v1/responses - forced tool_choice required is dropped (anthropic/claude-opus-5-5) - opus-5-5", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('request succeeded', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + "});", + "var body = pm.response.json();", + "var raw = (body.extra_fields || {}).raw_request;", + "if (typeof raw === 'string') { raw = JSON.parse(raw); }", + "pm.test('the OpenAI-shaped forced pin is dropped, tools survive', function () {", + " pm.expect(raw, 'raw request capture missing: ' + pm.response.text()).to.exist;", + " if (raw.tool_choice) { pm.expect(['any', 'tool'], 'Opus 5.5 rejects forced tool_choice').to.not.include(raw.tool_choice.type); }", + " pm.expect(raw.tools, 'dropping the pin must not drop the tools').to.exist;", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Authorization", + "value": "Bearer {{anthropicKey}}" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"anthropic/claude-opus-5-5\",\n \"max_output_tokens\": 16,\n \"input\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"tools\": [\n {\n \"type\": \"function\",\n \"name\": \"noop\",\n \"description\": \"does nothing\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {}\n }\n }\n ],\n \"tool_choice\": \"required\"\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/responses", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "responses" + ] + } + } + }, + { + "name": "Anthropic /v1/responses - computer_use_preview becomes the bare toolset (anthropic/claude-opus-5-5) - opus-5-5", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('request succeeded', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + "});", + "var body = pm.response.json();", + "var raw = (body.extra_fields || {}).raw_request;", + "if (typeof raw === 'string') { raw = JSON.parse(raw); }", + "pm.test('the neutral computer tool is sent as a bare toolset', function () {", + " pm.expect(raw, 'raw request capture missing: ' + pm.response.text()).to.exist;", + " pm.expect(raw.tools[0].type, 'Opus 5.5 accepts only the toolset form').to.eql('computer_toolset_20260801');", + " pm.expect(raw.tools[0].name, 'name is rejected on a toolset entry').to.be.undefined;", + " pm.expect(raw.tools[0].display_width_px, 'a toolset takes no geometry, so the caller dims are dropped').to.be.undefined;", + "});", + "pm.test('member calls surface with toolset_name on the Responses shape', function () {", + " var calls = [];", + " for (var i = 0; i < body.output.length; i++) {", + " if (body.output[i].type === 'function_call') { calls.push(body.output[i]); }", + " }", + " pm.expect(calls.length, 'model made no tool call: ' + pm.response.text()).to.be.above(0);", + " for (var j = 0; j < calls.length; j++) {", + " pm.expect(calls[j].toolset_name, 'member call lost toolset_name: ' + JSON.stringify(calls[j])).to.eql('computer');", + " }", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Authorization", + "value": "Bearer {{anthropicKey}}" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"anthropic/claude-opus-5-5\",\n \"max_output_tokens\": 512,\n \"input\": [\n {\n \"role\": \"user\",\n \"content\": \"Take a screenshot, then click at 100,200. Do it now.\"\n }\n ],\n \"tools\": [\n {\n \"type\": \"computer_use_preview\",\n \"display_width\": 1280,\n \"display_height\": 800,\n \"environment\": \"browser\"\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/responses", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "responses" + ] + } + } + }, + { + "name": "Anthropic /v1/responses stream - computer toolset member calls carry toolset_name (anthropic/claude-opus-5-5) - opus-5-5", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('stream completed without an upstream rejection', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + " pm.expect(pm.response.text(), 'stream carried an error frame').to.not.include('invalid_request_error');", + "});", + "pm.test('output_item.added for a member call carries toolset_name', function () {", + " var lines = pm.response.text().split('\\n');", + " var seen = 0;", + " for (var i = 0; i < lines.length; i++) {", + " if (lines[i].indexOf('data: ') !== 0) { continue; }", + " var payload = lines[i].slice(6).trim();", + " if (!payload || payload === '[DONE]') { continue; }", + " var ev;", + " try { ev = JSON.parse(payload); } catch (e) { continue; }", + " if (ev.type !== 'response.output_item.added') { continue; }", + " var item = ev.item || {};", + " if (item.type !== 'function_call') { continue; }", + " seen++;", + " pm.expect(item.toolset_name, 'streamed member call lost toolset_name: ' + payload).to.eql('computer');", + " }", + " pm.expect(seen, 'no function_call item in the stream: ' + pm.response.text().slice(0, 400)).to.be.above(0);", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Authorization", + "value": "Bearer {{anthropicKey}}" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"anthropic/claude-opus-5-5\",\n \"max_output_tokens\": 512,\n \"stream\": true,\n \"input\": [\n {\n \"role\": \"user\",\n \"content\": \"Take a screenshot, then click at 100,200. Do it now.\"\n }\n ],\n \"tools\": [\n {\n \"type\": \"computer_use_preview\",\n \"display_width\": 1280,\n \"display_height\": 800,\n \"environment\": \"browser\"\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/responses", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "responses" + ] + } + } + }, + { + "name": "Anthropic /v1/responses - dimensionless computer tool dropped for a model that takes neither form (anthropic/claude-sonnet-4-6) - opus-5-5", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('request succeeded', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + "});", + "var body = pm.response.json();", + "var raw = (body.extra_fields || {}).raw_request;", + "if (typeof raw === 'string') { raw = JSON.parse(raw); }", + "pm.test('no toolset support and no geometry leaves no shape to send - dropped', function () {", + " pm.expect(raw, 'raw request capture missing: ' + pm.response.text()).to.exist;", + " var tools = raw.tools || [];", + " for (var i = 0; i < tools.length; i++) {", + " pm.expect(String(tools[i].type || ''), 'this model takes neither the toolset nor a dated tool without geometry').to.not.include('computer');", + " }", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Authorization", + "value": "Bearer {{anthropicKey}}" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"anthropic/claude-sonnet-4-6\",\n \"max_output_tokens\": 16,\n \"input\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"tools\": [\n {\n \"type\": \"computer_use_preview\",\n \"environment\": \"browser\"\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/responses", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "responses" + ] + } + } + }, + { + "name": "Vertex /v1/responses - reasoning effort none never becomes thinking disabled (vertex/claude-opus-5-5) - opus-5-5", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('request succeeded', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + "});", + "var body = pm.response.json();", + "var raw = (body.extra_fields || {}).raw_request;", + "if (typeof raw === 'string') { raw = JSON.parse(raw); }", + "pm.test('the OpenAI-shaped reasoning-off maps to omitted thinking, not disabled', function () {", + " pm.expect(raw, 'raw request capture missing: ' + pm.response.text()).to.exist;", + " if (raw.thinking) { pm.expect(raw.thinking.type, 'Opus 5.5 rejects thinking.type.disabled').to.not.eql('disabled'); }", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Authorization", + "value": "Bearer {{anthropicKey}}" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"vertex/claude-opus-5-5\",\n \"max_output_tokens\": 16,\n \"input\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"reasoning\": {\n \"effort\": \"none\"\n }\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/responses", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "responses" + ] + } + } + }, + { + "name": "Vertex /v1/responses - forced tool_choice required is dropped (vertex/claude-opus-5-5) - opus-5-5", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('request succeeded', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + "});", + "var body = pm.response.json();", + "var raw = (body.extra_fields || {}).raw_request;", + "if (typeof raw === 'string') { raw = JSON.parse(raw); }", + "pm.test('the OpenAI-shaped forced pin is dropped, tools survive', function () {", + " pm.expect(raw, 'raw request capture missing: ' + pm.response.text()).to.exist;", + " if (raw.tool_choice) { pm.expect(['any', 'tool'], 'Opus 5.5 rejects forced tool_choice').to.not.include(raw.tool_choice.type); }", + " pm.expect(raw.tools, 'dropping the pin must not drop the tools').to.exist;", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Authorization", + "value": "Bearer {{anthropicKey}}" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"vertex/claude-opus-5-5\",\n \"max_output_tokens\": 16,\n \"input\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"tools\": [\n {\n \"type\": \"function\",\n \"name\": \"noop\",\n \"description\": \"does nothing\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {}\n }\n }\n ],\n \"tool_choice\": \"required\"\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/responses", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "responses" + ] + } + } + }, + { + "name": "Vertex /v1/responses - computer_use_preview becomes the bare toolset (vertex/claude-opus-5-5) - opus-5-5", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('request succeeded', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + "});", + "var body = pm.response.json();", + "var raw = (body.extra_fields || {}).raw_request;", + "if (typeof raw === 'string') { raw = JSON.parse(raw); }", + "pm.test('the neutral computer tool is sent as a bare toolset', function () {", + " pm.expect(raw, 'raw request capture missing: ' + pm.response.text()).to.exist;", + " pm.expect(raw.tools[0].type, 'Opus 5.5 accepts only the toolset form').to.eql('computer_toolset_20260801');", + " pm.expect(raw.tools[0].name, 'name is rejected on a toolset entry').to.be.undefined;", + " pm.expect(raw.tools[0].display_width_px, 'a toolset takes no geometry, so the caller dims are dropped').to.be.undefined;", + "});", + "pm.test('member calls surface with toolset_name on the Responses shape', function () {", + " var calls = [];", + " for (var i = 0; i < body.output.length; i++) {", + " if (body.output[i].type === 'function_call') { calls.push(body.output[i]); }", + " }", + " pm.expect(calls.length, 'model made no tool call: ' + pm.response.text()).to.be.above(0);", + " for (var j = 0; j < calls.length; j++) {", + " pm.expect(calls[j].toolset_name, 'member call lost toolset_name: ' + JSON.stringify(calls[j])).to.eql('computer');", + " }", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Authorization", + "value": "Bearer {{anthropicKey}}" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"vertex/claude-opus-5-5\",\n \"max_output_tokens\": 512,\n \"input\": [\n {\n \"role\": \"user\",\n \"content\": \"Take a screenshot, then click at 100,200. Do it now.\"\n }\n ],\n \"tools\": [\n {\n \"type\": \"computer_use_preview\",\n \"display_width\": 1280,\n \"display_height\": 800,\n \"environment\": \"browser\"\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/responses", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "responses" + ] + } + } + }, + { + "name": "Vertex /v1/responses stream - computer toolset member calls carry toolset_name (vertex/claude-opus-5-5) - opus-5-5", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('stream completed without an upstream rejection', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + " pm.expect(pm.response.text(), 'stream carried an error frame').to.not.include('invalid_request_error');", + "});", + "pm.test('output_item.added for a member call carries toolset_name', function () {", + " var lines = pm.response.text().split('\\n');", + " var seen = 0;", + " for (var i = 0; i < lines.length; i++) {", + " if (lines[i].indexOf('data: ') !== 0) { continue; }", + " var payload = lines[i].slice(6).trim();", + " if (!payload || payload === '[DONE]') { continue; }", + " var ev;", + " try { ev = JSON.parse(payload); } catch (e) { continue; }", + " if (ev.type !== 'response.output_item.added') { continue; }", + " var item = ev.item || {};", + " if (item.type !== 'function_call') { continue; }", + " seen++;", + " pm.expect(item.toolset_name, 'streamed member call lost toolset_name: ' + payload).to.eql('computer');", + " }", + " pm.expect(seen, 'no function_call item in the stream: ' + pm.response.text().slice(0, 400)).to.be.above(0);", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Authorization", + "value": "Bearer {{anthropicKey}}" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"vertex/claude-opus-5-5\",\n \"max_output_tokens\": 512,\n \"stream\": true,\n \"input\": [\n {\n \"role\": \"user\",\n \"content\": \"Take a screenshot, then click at 100,200. Do it now.\"\n }\n ],\n \"tools\": [\n {\n \"type\": \"computer_use_preview\",\n \"display_width\": 1280,\n \"display_height\": 800,\n \"environment\": \"browser\"\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/responses", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "responses" + ] + } + } + }, + { + "name": "Vertex /v1/responses - dimensionless computer tool dropped for a model that takes neither form (vertex/claude-sonnet-4-6) - opus-5-5", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('request succeeded', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + "});", + "var body = pm.response.json();", + "var raw = (body.extra_fields || {}).raw_request;", + "if (typeof raw === 'string') { raw = JSON.parse(raw); }", + "pm.test('no toolset support and no geometry leaves no shape to send - dropped', function () {", + " pm.expect(raw, 'raw request capture missing: ' + pm.response.text()).to.exist;", + " var tools = raw.tools || [];", + " for (var i = 0; i < tools.length; i++) {", + " pm.expect(String(tools[i].type || ''), 'this model takes neither the toolset nor a dated tool without geometry').to.not.include('computer');", + " }", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Authorization", + "value": "Bearer {{anthropicKey}}" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"vertex/claude-sonnet-4-6\",\n \"max_output_tokens\": 16,\n \"input\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"tools\": [\n {\n \"type\": \"computer_use_preview\",\n \"environment\": \"browser\"\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/responses", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "responses" + ] + } + } + } + ] + }, + { + "name": "115. Session affinity keeps a Vertex key pin with Vertex when the session stays on Anthropic (session-pin)", + "description": "A routing rule can pin a provider key on its target (the production rule was 'Claude Models -> Vertex' with key btaiops-prod-project-global and an Anthropic fallback on the incoming model). Core committed that pin to the request-wide api-key-id, and only then did session affinity run: a session last served by Anthropic moved Anthropic to the front of the chain, so the Anthropic attempt looked the Vertex key up among Anthropic's keys and died with 'no supported key found with id ... for provider: anthropic' before Anthropic was ever called. The Vertex fallback then served unpinned when it could, and when it could not the caller got that key error as a 400. Fixed in core/sessionaffinity.go: a pin belongs to the route it was decided for, so resolveSessionRoute carries pins through the session's reorder. The demoted Vertex entry keeps the rule's pin as its fallback key, which the fallback loop re-applies when Vertex is tried, and the promoted Anthropic attempt selects a key normally (or brings its own pin when the rule gave its fallback one). The trail records the move. Only requests that carry a session id take part, which is why Edge, Claude Code and Codex traffic saw it and plain API callers did not.\n\nThese cases rely on two rules in tests/integrations/python/config.json (governance.routing_rules) scoped to the harness-only header x-harness-session-pin, so they match nothing else in the suite. Each pair shares one fresh x-bf-session-id: the bind request is routed to Anthropic by the first rule, which binds the session there; the pinned request then matches the second rule (Vertex target with provider_key_name 'Vertex Global (Claude)', fallback anthropic/), affinity moves Anthropic first, and the case asserts Anthropic served. Before the fix the answer came from vertex (fallback rescue) or was the key error itself; after it, anthropic. Pinned on /v1/chat/completions and on /v1/responses, the route the report came in on.", + "item": [ + { + "name": "Session pin: bind - anthropic/claude-sonnet-4-6 serves the session first (chat) - session-pin", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "// One fresh session per run: the pair below must share it, and a stale binding from an", + "// earlier run would make the pinned request pass or fail for reasons unrelated to this fix.", + "pm.collectionVariables.set('sessionPinChatId', 'session-pin-' + Date.now() + '-' + Math.floor(Math.random() * 1e9));" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('bind request succeeded', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + "});", + "var body = pm.response.json();", + "pm.test('the session was bound by an Anthropic response', function () {", + " pm.expect((body.extra_fields || {}).provider, 'the bind rule did not route to anthropic, so the session is not bound where the pinned request expects it: ' + pm.response.text()).to.eql('anthropic');", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-bf-session-id", + "value": "{{sessionPinChatId}}" + }, + { + "key": "x-harness-session-pin", + "value": "bind" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"claude-sonnet-4-6\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"max_tokens\": 8\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "Session pin: pinned Vertex key - anthropic/claude-sonnet-4-6 keeps serving instead of no supported key found (chat) - session-pin", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('pinned request succeeded', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + "});", + "pm.test('the Vertex key pin did not follow the request onto Anthropic', function () {", + " pm.expect(pm.response.text(), 'key selection looked the pinned Vertex key up among Anthropic keys').to.not.include('no supported key found');", + "});", + "var body = pm.response.json();", + "pm.test('Anthropic served the session, not the Vertex fallback', function () {", + " pm.expect((body.extra_fields || {}).provider, 'a vertex answer means the Anthropic attempt died on the pin and only the fallback rescued the request: ' + pm.response.text()).to.eql('anthropic');", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-bf-session-id", + "value": "{{sessionPinChatId}}" + }, + { + "key": "x-harness-session-pin", + "value": "vertex" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"claude-sonnet-4-6\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"max_tokens\": 8\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "Session pin: bind - anthropic/claude-sonnet-4-6 serves the session first (responses) - session-pin", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "// One fresh session per run: the pair below must share it, and a stale binding from an", + "// earlier run would make the pinned request pass or fail for reasons unrelated to this fix.", + "pm.collectionVariables.set('sessionPinResponsesId', 'session-pin-' + Date.now() + '-' + Math.floor(Math.random() * 1e9));" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('bind request succeeded', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + "});", + "var body = pm.response.json();", + "pm.test('the session was bound by an Anthropic response', function () {", + " pm.expect((body.extra_fields || {}).provider, 'the bind rule did not route to anthropic, so the session is not bound where the pinned request expects it: ' + pm.response.text()).to.eql('anthropic');", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-bf-session-id", + "value": "{{sessionPinResponsesId}}" + }, + { + "key": "x-harness-session-pin", + "value": "bind" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"claude-sonnet-4-6\",\n \"input\": \"Reply with the single word: ok\",\n \"max_output_tokens\": 16\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/responses", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "responses" + ] + } + } + }, + { + "name": "Session pin: pinned Vertex key - anthropic/claude-sonnet-4-6 keeps serving instead of no supported key found (responses) - session-pin", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('pinned request succeeded', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + "});", + "pm.test('the Vertex key pin did not follow the request onto Anthropic', function () {", + " pm.expect(pm.response.text(), 'key selection looked the pinned Vertex key up among Anthropic keys').to.not.include('no supported key found');", + "});", + "var body = pm.response.json();", + "pm.test('Anthropic served the session, not the Vertex fallback', function () {", + " pm.expect((body.extra_fields || {}).provider, 'a vertex answer means the Anthropic attempt died on the pin and only the fallback rescued the request: ' + pm.response.text()).to.eql('anthropic');", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-bf-session-id", + "value": "{{sessionPinResponsesId}}" + }, + { + "key": "x-harness-session-pin", + "value": "vertex" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"claude-sonnet-4-6\",\n \"input\": \"Reply with the single word: ok\",\n \"max_output_tokens\": 16\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/responses", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "responses" + ] + } + } + } + ] + }, + { + "name": "116. Session affinity matrix on Vertex and Anthropic (session-affinity)", + "description": "Wire-level pins for session affinity across every lever that can change where a session lands: what makes a request take part (x-bf-session-id, the Claude Code and Codex session headers, x-bf-session-affinity: off, a provider-prefixed model), what builds the chain the session reorders (routing rules with targets, pins and fallbacks, a real upstream failure that moves the session to a fallback, a one-provider chain that drops a binding), what pins outrank the session's key binding (a rule's pin changed mid-session), and what governance takes away (a virtual key whose key_ids no longer include the bound key). Every sequence uses one fresh session id and asserts extra_fields.provider, routing_info.key and routing_info.is_fallback.\n\nStatic rules live in tests/integrations/python/config.json (governance.routing_rules) behind the harness-only header x-harness-session-pin: bind, vertex, rebind, rescue, vertex-only, anthropic-first, anthropic-first-other-model and vertex-badkey. Two of them pin keys that cannot serve the model on purpose: the rescue rule's Anthropic fallback (so the promoted Anthropic attempt fails on the same model, which exact route matching requires) and the vertex-badkey rule (so a request fails at key selection without leaving the bound route). Case 12 pins that a binding names provider and model together, and case 13 that a binding followed into a failure is forgotten. The last three cases are single rows that do their own management work through pm.sendRequest: the pre-request script creates a second Anthropic key from the same env credential, a run-scoped routing rule keyed by x-harness-session-run, and for case 11 a virtual key; the test script makes the follow-up inference call with the same session id and deletes everything it created. One row per case is deliberate: the recipe forks a newman process per provider and per modality class and sub-shards a class by row position, and collection variables never cross a process, so state set up in one row cannot be consumed in another. Leftovers of an aborted run carry their run timestamp in their names and are swept by the next run once they are ten minutes old; younger objects belong to a shard that may still be running. The whole folder is attributed to the Vertex fork.", + "item": [ + { + "name": "Session matrix 1: Vertex fails on a bogus model and Anthropic rescues - anthropic/claude-sonnet-4-6 - session-affinity", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "// One run id for the dynamic rules and keys below, one fresh session per sequence.", + "pm.collectionVariables.set('sessionMatrixRun', 'run-' + Date.now());", + "pm.collectionVariables.set('smSession1', 'session-affinity-' + Date.now() + '-' + Math.floor(Math.random() * 1e9));" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('rescued request succeeded', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + "});", + "var body = pm.response.json();", + "var ri = (body.extra_fields || {}).routing_info || {};", + "pm.test('served by anthropic', function () {", + " pm.expect((body.extra_fields || {}).provider, 'the Vertex attempt on a bogus model was expected to fail and Anthropic to rescue: ' + pm.response.text()).to.eql('anthropic');", + "});", + "pm.test('is_fallback is true', function () {", + " pm.expect(!!ri.is_fallback, 'Anthropic must have served as the fallback, which is what binds the session there: ' + pm.response.text()).to.eql(true);", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-bf-session-id", + "value": "{{smSession1}}" + }, + { + "key": "x-harness-session-pin", + "value": "rebind" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"claude-sonnet-4-6\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"max_tokens\": 8\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "Session matrix 1: the rescued session stays on Anthropic through the pinned Vertex rule - anthropic/claude-sonnet-4-6 - session-affinity", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('follow-up succeeded', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + "});", + "var body = pm.response.json();", + "var ri = (body.extra_fields || {}).routing_info || {};", + "pm.test('no key pin followed the request onto the wrong provider', function () {", + " pm.expect(pm.response.text()).to.not.include('no supported key found');", + "});", + "pm.test('served by anthropic', function () {", + " pm.expect((body.extra_fields || {}).provider, 'a session rescued by Anthropic must be served by Anthropic as primary, with the Vertex pin left on the demoted Vertex entry: ' + pm.response.text()).to.eql('anthropic');", + "});", + "pm.test('is_fallback is false', function () {", + " pm.expect(!!ri.is_fallback, 'Anthropic must be the primary attempt here, not a rescue: ' + pm.response.text()).to.eql(false);", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-bf-session-id", + "value": "{{smSession1}}" + }, + { + "key": "x-harness-session-pin", + "value": "vertex" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"claude-sonnet-4-6\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"max_tokens\": 8\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "Session matrix 2: bind - anthropic/claude-sonnet-4-6 serves the session first - session-affinity", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "pm.collectionVariables.set('smSession2a', 'session-affinity-' + Date.now() + '-' + Math.floor(Math.random() * 1e9));" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('bind succeeded', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + "});", + "var body = pm.response.json();", + "var ri = (body.extra_fields || {}).routing_info || {};", + "pm.test('served by anthropic', function () {", + " pm.expect((body.extra_fields || {}).provider, 'the bind rule did not route to anthropic, so the session is not bound where the next request expects it: ' + pm.response.text()).to.eql('anthropic');", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-bf-session-id", + "value": "{{smSession2a}}" + }, + { + "key": "x-harness-session-pin", + "value": "bind" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"claude-sonnet-4-6\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"max_tokens\": 8\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "Session matrix 2: Anthropic fails as the promoted primary and the demoted Vertex entry serves on its pinned key - vertex/claude-sonnet-4-6 - session-affinity", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('rescue succeeded', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + "});", + "var body = pm.response.json();", + "var ri = (body.extra_fields || {}).routing_info || {};", + "pm.test('served by vertex', function () {", + " pm.expect((body.extra_fields || {}).provider, 'the promoted Anthropic attempt is pinned to a key that cannot serve this model and must fail at key selection, leaving Vertex to serve: ' + pm.response.text()).to.eql('vertex');", + "});", + "pm.test('is_fallback is true', function () {", + " pm.expect(!!ri.is_fallback, 'Vertex must have served as the fallback behind the promoted Anthropic entry: ' + pm.response.text()).to.eql(true);", + "});", + "pm.test('served on the expected key', function () {", + " pm.expect(ri.key, 'the pin the rule set for Vertex did not travel with it when the session demoted Vertex to a fallback: ' + pm.response.text()).to.eql('Vertex Global (Claude)');", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-bf-session-id", + "value": "{{smSession2a}}" + }, + { + "key": "x-harness-session-pin", + "value": "rescue" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"claude-sonnet-4-6\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"max_tokens\": 8\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "Session matrix 2: bind - anthropic/claude-sonnet-4-6 serves the session first - session-affinity", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "pm.collectionVariables.set('smSession2b', 'session-affinity-' + Date.now() + '-' + Math.floor(Math.random() * 1e9));" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('bind succeeded', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + "});", + "var body = pm.response.json();", + "var ri = (body.extra_fields || {}).routing_info || {};", + "pm.test('served by anthropic', function () {", + " pm.expect((body.extra_fields || {}).provider, 'the bind rule did not route to anthropic, so the session is not bound where the next request expects it: ' + pm.response.text()).to.eql('anthropic');", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-bf-session-id", + "value": "{{smSession2b}}" + }, + { + "key": "x-harness-session-pin", + "value": "bind" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"claude-sonnet-4-6\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"max_tokens\": 8\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "Session matrix 2: Anthropic fails as the promoted primary and the demoted Vertex entry serves on its pinned key - vertex/claude-sonnet-4-6 - session-affinity", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('rescue succeeded', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + "});", + "var body = pm.response.json();", + "var ri = (body.extra_fields || {}).routing_info || {};", + "pm.test('served by vertex', function () {", + " pm.expect((body.extra_fields || {}).provider, 'the promoted Anthropic attempt is pinned to a key that cannot serve this model and must fail at key selection, leaving Vertex to serve: ' + pm.response.text()).to.eql('vertex');", + "});", + "pm.test('is_fallback is true', function () {", + " pm.expect(!!ri.is_fallback, 'Vertex must have served as the fallback behind the promoted Anthropic entry: ' + pm.response.text()).to.eql(true);", + "});", + "pm.test('served on the expected key', function () {", + " pm.expect(ri.key, 'the pin the rule set for Vertex did not travel with it when the session demoted Vertex to a fallback: ' + pm.response.text()).to.eql('Vertex Global (Claude)');", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-bf-session-id", + "value": "{{smSession2b}}" + }, + { + "key": "x-harness-session-pin", + "value": "rescue" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"claude-sonnet-4-6\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"max_tokens\": 8\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "Session matrix 3: bind - anthropic/claude-sonnet-4-6 serves the session first - session-affinity", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "pm.collectionVariables.set('smSession3', 'session-affinity-' + Date.now() + '-' + Math.floor(Math.random() * 1e9));" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('bind succeeded', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + "});", + "var body = pm.response.json();", + "var ri = (body.extra_fields || {}).routing_info || {};", + "pm.test('served by anthropic', function () {", + " pm.expect((body.extra_fields || {}).provider, 'the bind rule did not route to anthropic, so the session is not bound where the next request expects it: ' + pm.response.text()).to.eql('anthropic');", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-bf-session-id", + "value": "{{smSession3}}" + }, + { + "key": "x-harness-session-pin", + "value": "bind" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"claude-sonnet-4-6\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"max_tokens\": 8\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "Session matrix 3: x-bf-session-affinity off lets the pinned Vertex rule stand - vertex/claude-sonnet-4-6 - session-affinity", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('opt-out succeeded', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + "});", + "var body = pm.response.json();", + "var ri = (body.extra_fields || {}).routing_info || {};", + "pm.test('served by vertex', function () {", + " pm.expect((body.extra_fields || {}).provider, 'with affinity switched off the routing decision must stand: ' + pm.response.text()).to.eql('vertex');", + "});", + "pm.test('is_fallback is false', function () {", + " pm.expect(!!ri.is_fallback, 'Vertex must be the primary attempt: ' + pm.response.text()).to.eql(false);", + "});", + "pm.test('served on the expected key', function () {", + " pm.expect(ri.key, 'the target pin must apply to the primary attempt: ' + pm.response.text()).to.eql('Vertex Global (Claude)');", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-bf-session-id", + "value": "{{smSession3}}" + }, + { + "key": "x-harness-session-pin", + "value": "vertex" + }, + { + "key": "x-bf-session-affinity", + "value": "off" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"claude-sonnet-4-6\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"max_tokens\": 8\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "Session matrix 4: bind - anthropic/claude-sonnet-4-6 serves the session first - session-affinity", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "pm.collectionVariables.set('smSession4', 'session-affinity-' + Date.now() + '-' + Math.floor(Math.random() * 1e9));" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('bind succeeded', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + "});", + "var body = pm.response.json();", + "var ri = (body.extra_fields || {}).routing_info || {};", + "pm.test('served by anthropic', function () {", + " pm.expect((body.extra_fields || {}).provider, 'the bind rule did not route to anthropic, so the session is not bound where the next request expects it: ' + pm.response.text()).to.eql('anthropic');", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-bf-session-id", + "value": "{{smSession4}}" + }, + { + "key": "x-harness-session-pin", + "value": "bind" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"claude-sonnet-4-6\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"max_tokens\": 8\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "Session matrix 4: a Vertex-only rule cannot follow the Anthropic binding, so it is dropped - vertex/claude-sonnet-4-6 - session-affinity", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('one-provider chain succeeded', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + "});", + "var body = pm.response.json();", + "var ri = (body.extra_fields || {}).routing_info || {};", + "pm.test('served by vertex', function () {", + " pm.expect((body.extra_fields || {}).provider, 'a chain without Anthropic must be served by Vertex: ' + pm.response.text()).to.eql('vertex');", + "});", + "pm.test('is_fallback is false', function () {", + " pm.expect(!!ri.is_fallback, 'Vertex is the only entry: ' + pm.response.text()).to.eql(false);", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-bf-session-id", + "value": "{{smSession4}}" + }, + { + "key": "x-harness-session-pin", + "value": "vertex-only" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"claude-sonnet-4-6\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"max_tokens\": 8\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "Session matrix 4: the session rebound to Vertex and stays there through the pinned rule - vertex/claude-sonnet-4-6 - session-affinity", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('rebound succeeded', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + "});", + "var body = pm.response.json();", + "var ri = (body.extra_fields || {}).routing_info || {};", + "pm.test('served by vertex', function () {", + " pm.expect((body.extra_fields || {}).provider, 'the previous served request rebound the session to Vertex, so the Anthropic fallback must not be promoted: ' + pm.response.text()).to.eql('vertex');", + "});", + "pm.test('is_fallback is false', function () {", + " pm.expect(!!ri.is_fallback, 'Vertex must be the primary attempt: ' + pm.response.text()).to.eql(false);", + "});", + "pm.test('served on the expected key', function () {", + " pm.expect(ri.key, 'the target pin must apply to the primary attempt: ' + pm.response.text()).to.eql('Vertex Global (Claude)');", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-bf-session-id", + "value": "{{smSession4}}" + }, + { + "key": "x-harness-session-pin", + "value": "vertex" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"claude-sonnet-4-6\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"max_tokens\": 8\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "Session matrix 5: bind - anthropic/claude-sonnet-4-6 serves the session first - session-affinity", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "pm.collectionVariables.set('smSession5', 'session-affinity-' + Date.now() + '-' + Math.floor(Math.random() * 1e9));" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('bind succeeded', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + "});", + "var body = pm.response.json();", + "var ri = (body.extra_fields || {}).routing_info || {};", + "pm.test('served by anthropic', function () {", + " pm.expect((body.extra_fields || {}).provider, 'the bind rule did not route to anthropic, so the session is not bound where the next request expects it: ' + pm.response.text()).to.eql('anthropic');", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-bf-session-id", + "value": "{{smSession5}}" + }, + { + "key": "x-harness-session-pin", + "value": "bind" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"claude-sonnet-4-6\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"max_tokens\": 8\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "Session matrix 5: a prefixed vertex/claude-sonnet-4-6 request bypasses the Anthropic binding - session-affinity", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('prefixed request succeeded', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + "});", + "var body = pm.response.json();", + "var ri = (body.extra_fields || {}).routing_info || {};", + "pm.test('served by vertex', function () {", + " pm.expect((body.extra_fields || {}).provider, 'a request that names its provider must go there regardless of the session: ' + pm.response.text()).to.eql('vertex');", + "});", + "pm.test('is_fallback is false', function () {", + " pm.expect(!!ri.is_fallback, 'no fallback is involved: ' + pm.response.text()).to.eql(false);", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-bf-session-id", + "value": "{{smSession5}}" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"vertex/claude-sonnet-4-6\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"max_tokens\": 8\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "Session matrix 5: the bare-model request is still served by Anthropic afterwards - anthropic/claude-sonnet-4-6 - session-affinity", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('bare-model follow-up succeeded', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + "});", + "var body = pm.response.json();", + "var ri = (body.extra_fields || {}).routing_info || {};", + "pm.test('served by anthropic', function () {", + " pm.expect((body.extra_fields || {}).provider, 'the prefixed request must not have rewritten the route binding: ' + pm.response.text()).to.eql('anthropic');", + "});", + "pm.test('is_fallback is false', function () {", + " pm.expect(!!ri.is_fallback, 'Anthropic must be the primary attempt: ' + pm.response.text()).to.eql(false);", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-bf-session-id", + "value": "{{smSession5}}" + }, + { + "key": "x-harness-session-pin", + "value": "vertex" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"claude-sonnet-4-6\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"max_tokens\": 8\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "Session matrix 6: bind - anthropic/claude-sonnet-4-6 serves the session first - session-affinity", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "pm.collectionVariables.set('smSession6', 'session-affinity-' + Date.now() + '-' + Math.floor(Math.random() * 1e9));" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('bind succeeded', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + "});", + "var body = pm.response.json();", + "var ri = (body.extra_fields || {}).routing_info || {};", + "pm.test('served by anthropic', function () {", + " pm.expect((body.extra_fields || {}).provider, 'the bind rule did not route to anthropic, so the session is not bound where the next request expects it: ' + pm.response.text()).to.eql('anthropic');", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-bf-session-id", + "value": "{{smSession6}}" + }, + { + "key": "x-harness-session-pin", + "value": "bind" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"claude-sonnet-4-6\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"max_tokens\": 8\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "Session matrix 6: without a session header the pinned Vertex rule stands - vertex/claude-sonnet-4-6 - session-affinity", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('sessionless request succeeded', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + "});", + "var body = pm.response.json();", + "var ri = (body.extra_fields || {}).routing_info || {};", + "pm.test('served by vertex', function () {", + " pm.expect((body.extra_fields || {}).provider, 'a request with no session must not follow any binding: ' + pm.response.text()).to.eql('vertex');", + "});", + "pm.test('is_fallback is false', function () {", + " pm.expect(!!ri.is_fallback, 'Vertex must be the primary attempt: ' + pm.response.text()).to.eql(false);", + "});", + "pm.test('served on the expected key', function () {", + " pm.expect(ri.key, 'the target pin must apply to the primary attempt: ' + pm.response.text()).to.eql('Vertex Global (Claude)');", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-harness-session-pin", + "value": "vertex" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"claude-sonnet-4-6\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"max_tokens\": 8\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "Session matrix 7: Claude Code session header (x-claude-code-session-id): bind - anthropic/claude-sonnet-4-6 serves the session first - session-affinity", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "pm.collectionVariables.set('smSession7a', 'session-affinity-' + Date.now() + '-' + Math.floor(Math.random() * 1e9));" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('bind succeeded', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + "});", + "var body = pm.response.json();", + "var ri = (body.extra_fields || {}).routing_info || {};", + "pm.test('served by anthropic', function () {", + " pm.expect((body.extra_fields || {}).provider, 'the bind rule did not route to anthropic, so the session is not bound where the next request expects it: ' + pm.response.text()).to.eql('anthropic');", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-claude-code-session-id", + "value": "{{smSession7a}}" + }, + { + "key": "x-harness-session-pin", + "value": "bind" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"claude-sonnet-4-6\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"max_tokens\": 8\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "Session matrix 7: Claude Code session header (x-claude-code-session-id) keeps the session on Anthropic - anthropic/claude-sonnet-4-6 - session-affinity", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('Claude Code follow-up succeeded', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + "});", + "var body = pm.response.json();", + "var ri = (body.extra_fields || {}).routing_info || {};", + "pm.test('no key pin followed the request onto the wrong provider', function () {", + " pm.expect(pm.response.text()).to.not.include('no supported key found');", + "});", + "pm.test('served by anthropic', function () {", + " pm.expect((body.extra_fields || {}).provider, 'the x-claude-code-session-id header must carry the session id, so the binding must be followed: ' + pm.response.text()).to.eql('anthropic');", + "});", + "pm.test('is_fallback is false', function () {", + " pm.expect(!!ri.is_fallback, 'Anthropic must be the primary attempt: ' + pm.response.text()).to.eql(false);", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-claude-code-session-id", + "value": "{{smSession7a}}" + }, + { + "key": "x-harness-session-pin", + "value": "vertex" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"claude-sonnet-4-6\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"max_tokens\": 8\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "Session matrix 7: Codex session header (session_id): bind - anthropic/claude-sonnet-4-6 serves the session first - session-affinity", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "pm.collectionVariables.set('smSession7b', 'session-affinity-' + Date.now() + '-' + Math.floor(Math.random() * 1e9));" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('bind succeeded', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + "});", + "var body = pm.response.json();", + "var ri = (body.extra_fields || {}).routing_info || {};", + "pm.test('served by anthropic', function () {", + " pm.expect((body.extra_fields || {}).provider, 'the bind rule did not route to anthropic, so the session is not bound where the next request expects it: ' + pm.response.text()).to.eql('anthropic');", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "session_id", + "value": "{{smSession7b}}" + }, + { + "key": "x-harness-session-pin", + "value": "bind" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"claude-sonnet-4-6\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"max_tokens\": 8\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "Session matrix 7: Codex session header (session_id) keeps the session on Anthropic - anthropic/claude-sonnet-4-6 - session-affinity", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('Codex follow-up succeeded', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + "});", + "var body = pm.response.json();", + "var ri = (body.extra_fields || {}).routing_info || {};", + "pm.test('no key pin followed the request onto the wrong provider', function () {", + " pm.expect(pm.response.text()).to.not.include('no supported key found');", + "});", + "pm.test('served by anthropic', function () {", + " pm.expect((body.extra_fields || {}).provider, 'the session_id header must carry the session id, so the binding must be followed: ' + pm.response.text()).to.eql('anthropic');", + "});", + "pm.test('is_fallback is false', function () {", + " pm.expect(!!ri.is_fallback, 'Anthropic must be the primary attempt: ' + pm.response.text()).to.eql(false);", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "session_id", + "value": "{{smSession7b}}" + }, + { + "key": "x-harness-session-pin", + "value": "vertex" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"claude-sonnet-4-6\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"max_tokens\": 8\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "Session matrix 8: bind with x-bf-session-id and a different Claude Code session id - anthropic/claude-sonnet-4-6 - session-affinity", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "pm.collectionVariables.set('smSession8x', 'session-affinity-' + Date.now() + '-' + Math.floor(Math.random() * 1e9));", + "pm.collectionVariables.set('smSession8y', 'session-affinity-' + Date.now() + '-' + Math.floor(Math.random() * 1e9));" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('bind succeeded', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + "});", + "var body = pm.response.json();", + "var ri = (body.extra_fields || {}).routing_info || {};", + "pm.test('served by anthropic', function () {", + " pm.expect((body.extra_fields || {}).provider, 'the bind rule did not route to anthropic: ' + pm.response.text()).to.eql('anthropic');", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-bf-session-id", + "value": "{{smSession8x}}" + }, + { + "key": "x-claude-code-session-id", + "value": "{{smSession8y}}" + }, + { + "key": "x-harness-session-pin", + "value": "bind" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"claude-sonnet-4-6\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"max_tokens\": 8\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "Session matrix 8: the alias id alone has no binding, so the pinned Vertex rule stands - vertex/claude-sonnet-4-6 - session-affinity", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('alias-only request succeeded', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + "});", + "var body = pm.response.json();", + "var ri = (body.extra_fields || {}).routing_info || {};", + "pm.test('served by vertex', function () {", + " pm.expect((body.extra_fields || {}).provider, 'the explicit header won at bind time, so the alias id must have no binding: ' + pm.response.text()).to.eql('vertex');", + "});", + "pm.test('is_fallback is false', function () {", + " pm.expect(!!ri.is_fallback, 'Vertex must be the primary attempt: ' + pm.response.text()).to.eql(false);", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-claude-code-session-id", + "value": "{{smSession8y}}" + }, + { + "key": "x-harness-session-pin", + "value": "vertex" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"claude-sonnet-4-6\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"max_tokens\": 8\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "Session matrix 8: the explicit id alone is still bound to Anthropic - anthropic/claude-sonnet-4-6 - session-affinity", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('explicit-only request succeeded', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + "});", + "var body = pm.response.json();", + "var ri = (body.extra_fields || {}).routing_info || {};", + "pm.test('served by anthropic', function () {", + " pm.expect((body.extra_fields || {}).provider, 'the explicit session id is the one that was bound: ' + pm.response.text()).to.eql('anthropic');", + "});", + "pm.test('is_fallback is false', function () {", + " pm.expect(!!ri.is_fallback, 'Anthropic must be the primary attempt: ' + pm.response.text()).to.eql(false);", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-bf-session-id", + "value": "{{smSession8x}}" + }, + { + "key": "x-harness-session-pin", + "value": "vertex" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"claude-sonnet-4-6\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"max_tokens\": 8\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "Session matrix 12: bind the session to vertex/claude-sonnet-4-6 through the pinned rule - session-affinity", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "pm.collectionVariables.set('smSession12', 'session-affinity-' + Date.now() + '-' + Math.floor(Math.random() * 1e9));" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('bind succeeded', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + "});", + "var body = pm.response.json();", + "var ri = (body.extra_fields || {}).routing_info || {};", + "pm.test('served by vertex', function () {", + " pm.expect((body.extra_fields || {}).provider, 'the pinned Vertex rule must serve as primary: ' + pm.response.text()).to.eql('vertex');", + "});", + "pm.test('is_fallback is false', function () {", + " pm.expect(!!ri.is_fallback, 'Vertex must be the primary attempt: ' + pm.response.text()).to.eql(false);", + "});", + "pm.test('served on the expected key', function () {", + " pm.expect(ri.key, 'the target pin must apply: ' + pm.response.text()).to.eql('Vertex Global (Claude)');", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-bf-session-id", + "value": "{{smSession12}}" + }, + { + "key": "x-harness-session-pin", + "value": "vertex" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"claude-sonnet-4-6\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"max_tokens\": 8\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "Session matrix 12: a Vertex fallback on claude-opus-4-8 is not the bound route, so Anthropic serves - anthropic/claude-sonnet-4-6 - session-affinity", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('other-model request succeeded', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + "});", + "var body = pm.response.json();", + "var ri = (body.extra_fields || {}).routing_info || {};", + "pm.test('served by anthropic', function () {", + " pm.expect((body.extra_fields || {}).provider, 'a Vertex entry on another model must not be promoted for a session bound to Vertex on this model: ' + pm.response.text()).to.eql('anthropic');", + "});", + "pm.test('is_fallback is false', function () {", + " pm.expect(!!ri.is_fallback, 'Anthropic must be the primary attempt: ' + pm.response.text()).to.eql(false);", + "});", + "pm.test('served on the model the caller asked for', function () {", + " pm.expect(ri.model, 'the session must not have been moved onto the fallback model: ' + pm.response.text()).to.eql(\"claude-sonnet-4-6\");", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-bf-session-id", + "value": "{{smSession12}}" + }, + { + "key": "x-harness-session-pin", + "value": "anthropic-first-other-model" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"claude-sonnet-4-6\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"max_tokens\": 8\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "Session matrix 12: the session rebound on Anthropic and now moves the pinned Vertex rule aside - anthropic/claude-sonnet-4-6 - session-affinity", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('follow-up succeeded', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + "});", + "var body = pm.response.json();", + "var ri = (body.extra_fields || {}).routing_info || {};", + "pm.test('no key pin followed the request onto the wrong provider', function () {", + " pm.expect(pm.response.text()).to.not.include('no supported key found');", + "});", + "pm.test('served by anthropic', function () {", + " pm.expect((body.extra_fields || {}).provider, 'the previous served request rebound the session to Anthropic, so the pinned Vertex rule must be moved aside: ' + pm.response.text()).to.eql('anthropic');", + "});", + "pm.test('is_fallback is false', function () {", + " pm.expect(!!ri.is_fallback, 'Anthropic must be the primary attempt: ' + pm.response.text()).to.eql(false);", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-bf-session-id", + "value": "{{smSession12}}" + }, + { + "key": "x-harness-session-pin", + "value": "vertex" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"claude-sonnet-4-6\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"max_tokens\": 8\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "Session matrix 13: bind the session to vertex/claude-sonnet-4-6 through the pinned rule - session-affinity", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "pm.collectionVariables.set('smSession13', 'session-affinity-' + Date.now() + '-' + Math.floor(Math.random() * 1e9));" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('bind succeeded', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + "});", + "var body = pm.response.json();", + "var ri = (body.extra_fields || {}).routing_info || {};", + "pm.test('served by vertex', function () {", + " pm.expect((body.extra_fields || {}).provider, 'the pinned Vertex rule must serve as primary: ' + pm.response.text()).to.eql('vertex');", + "});", + "pm.test('is_fallback is false', function () {", + " pm.expect(!!ri.is_fallback, 'Vertex must be the primary attempt: ' + pm.response.text()).to.eql(false);", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-bf-session-id", + "value": "{{smSession13}}" + }, + { + "key": "x-harness-session-pin", + "value": "vertex" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"claude-sonnet-4-6\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"max_tokens\": 8\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "[EXPECT-4XX] Session matrix 13: the session follows its Vertex binding into a key-selection failure - vertex/claude-sonnet-4-6 - session-affinity", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('the pinned key cannot serve the model, so the request fails', function () {", + " pm.expect(pm.response.code, 'expected a 4xx, got ' + pm.response.code + ': ' + pm.response.text()).to.be.within(400, 499);", + " pm.expect(pm.response.text(), 'the failure must come from key selection, not from upstream: ' + pm.response.text()).to.include('no supported key found');", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-bf-session-id", + "value": "{{smSession13}}" + }, + { + "key": "x-harness-session-pin", + "value": "vertex-badkey" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"claude-sonnet-4-6\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"max_tokens\": 8\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "Session matrix 13: the Vertex binding was forgotten, so the Anthropic-first rule stands - anthropic/claude-sonnet-4-6 - session-affinity", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('post-failure request succeeded', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + "});", + "var body = pm.response.json();", + "var ri = (body.extra_fields || {}).routing_info || {};", + "pm.test('served by anthropic', function () {", + " pm.expect((body.extra_fields || {}).provider, 'a binding followed into a failure must be dropped, so the routing decision must stand instead of promoting Vertex again: ' + pm.response.text()).to.eql('anthropic');", + "});", + "pm.test('is_fallback is false', function () {", + " pm.expect(!!ri.is_fallback, 'Anthropic must be the primary attempt: ' + pm.response.text()).to.eql(false);", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-bf-session-id", + "value": "{{smSession13}}" + }, + { + "key": "x-harness-session-pin", + "value": "anthropic-first" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"claude-sonnet-4-6\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"max_tokens\": 8\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "Session matrix 9: a rule pin moved mid-session outranks the key the session bound - anthropic/claude-sonnet-4-6 - session-affinity", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "// ---- in-script management calls (see the folder description) ----\nvar BASE = pm.variables.get('baseUrl');\nvar CHAT_BODY = {\"model\":\"claude-sonnet-4-6\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok\"}],\"max_tokens\":8};\nfunction call(method, path, body, cb) {\n var req = { url: BASE + path, method: method, header: { 'Content-Type': 'application/json' } };\n if (body !== null && body !== undefined) { req.body = { mode: 'raw', raw: JSON.stringify(body) }; }\n pm.sendRequest(req, function (err, res) {\n if (err) { pm.test(method + ' ' + path + ' reached the gateway', function () { pm.expect.fail(String(err)); }); return cb(null, null); }\n var json = null; try { json = res.json(); } catch (e) { json = null; }\n cb(res, json);\n });\n}\nfunction expectOk(label, res) {\n pm.test(label, function () { pm.expect(res && res.code, 'unexpected status: ' + (res ? res.text() : 'no response')).to.eql(200); });\n return !!(res && res.code === 200);\n}\nfunction chat(headers, model, cb) {\n var h = { 'Content-Type': 'application/json' }; for (var k in headers) { h[k] = headers[k]; }\n var b = JSON.parse(JSON.stringify(CHAT_BODY)); if (model) { b.model = model; }\n pm.sendRequest({ url: BASE + '/v1/chat/completions', method: 'POST', header: h, body: { mode: 'raw', raw: JSON.stringify(b) } }, function (err, res) {\n if (err) { pm.test('follow-up chat request reached the gateway', function () { pm.expect.fail(String(err)); }); return cb(null, null, {}); }\n var json = null; try { json = res.json(); } catch (e) { json = {}; }\n cb(res, json, ((json || {}).extra_fields || {}).routing_info || {});\n });\n}\n// Leftovers of an aborted earlier run carry their run timestamp in their names. Only objects older\n// than ten minutes are swept, so a parallel shard's live objects are never touched.\nfunction isStale(name, prefix) {\n var m = String(name || '').indexOf(prefix) === 0 && String(name).match(/run-(\\d+)/);\n return !!(m && (Date.now() - Number(m[1])) > 10 * 60 * 1000);\n}\nfunction deleteEach(items, pathOf, done) {\n var i = 0; (function next() { if (i >= items.length) { return done(); } var it = items[i++]; call('DELETE', pathOf(it), null, function () { next(); }); })();\n}\nfunction sweepStale(done) {\n call('GET', '/api/governance/routing-rules', null, function (r1, j1) {\n var rules = ((j1 && j1.rules) || []).filter(function (r) { return isStale(r.name, 'Harness session matrix '); });\n deleteEach(rules, function (r) { return '/api/governance/routing-rules/' + r.id; }, function () {\n call('GET', '/api/governance/virtual-keys', null, function (r2, j2) {\n var vks = ((j2 && j2.virtual_keys) || []).filter(function (v) { return isStale(v.name, 'harness-session-matrix-'); });\n deleteEach(vks, function (v) { return '/api/governance/virtual-keys/' + v.id; }, function () {\n call('GET', '/api/providers/anthropic/keys', null, function (r3, j3) {\n var keys = ((j3 && j3.keys) || []).filter(function (k) { return isStale(k.name, 'Harness Session Key B '); });\n deleteEach(keys, function (k) { return '/api/providers/anthropic/keys/' + k.id; }, done);\n });\n });\n });\n });\n });\n}\nfunction anthropicKeyA(cb) {\n call('GET', '/api/providers/anthropic/keys', null, function (res, j) {\n if (!expectOk('setup: listed anthropic keys', res)) { return cb(null); }\n var a = ((j && j.keys) || []).filter(function (k) { return k.name === 'Anthropic API Key'; })[0];\n pm.test('setup: the profile key Anthropic API Key exists', function () { pm.expect(a, 'expected a key named Anthropic API Key: ' + res.text()).to.exist; });\n cb(a || null);\n });\n}\nfunction createKeyB(run, cb) {\n call('POST', '/api/providers/anthropic/keys', { name: 'Harness Session Key B ' + run, value: 'env.ANTHROPIC_API_KEY', models: ['*'], weight: 1 }, function (res, k) {\n if (!expectOk('setup: created a second anthropic key from the same credential', res)) { return cb(null); }\n cb(k);\n });\n}\nfunction createRule(run, suffix, priority, targets, fallbacks, cb) {\n var body = { name: 'Harness session matrix ' + suffix + ' ' + run, description: 'Provider harness, created at run time and deleted by the same row.', priority: priority, cel_expression: '\"x-harness-session-run\" in headers && headers[\"x-harness-session-run\"] == \"' + run + '-' + suffix + '\"', targets: targets, scope: 'global' };\n if (fallbacks) { body.fallbacks = fallbacks; }\n call('POST', '/api/governance/routing-rules', body, function (res, j) {\n if (!expectOk('setup: created the ' + suffix + ' rule', res)) { return cb(null); }\n cb(((j || {}).rule || {}).id || null);\n });\n}\nfunction del(label, path, cb) { if (!path) { return cb(); } call('DELETE', path, null, function (res) { expectOk('cleanup: ' + label, res); cb(); }); }\n// Priorities must be distinct within a scope; derive them from the clock so two runs, or two\n// shards of one run, never collide on a fixed number.\nvar PRIO = 900000 + (Date.now() % 90000);", + "var run = 'run-' + Date.now(); pm.collectionVariables.set('smRun9', run);", + "pm.collectionVariables.set('smSession9', 'session-affinity-' + Date.now() + '-' + Math.floor(Math.random() * 1e9));", + "sweepStale(function () {", + " anthropicKeyA(function (a) {", + " if (!a) { return; }", + " pm.collectionVariables.set('sm9KeyAName', a.name);", + " createKeyB(run, function (k) {", + " if (!k) { return; }", + " pm.collectionVariables.set('sm9KeyBId', k.id); pm.collectionVariables.set('sm9KeyBName', k.name);", + " createRule(run, 'pin', PRIO + 1, [{ provider: 'anthropic', key_id: a.id, weight: 1 }], null, function (id) {", + " if (id) { pm.collectionVariables.set('sm9RuleId', id); }", + " });", + " });", + " });", + "});" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "// ---- in-script management calls (see the folder description) ----\nvar BASE = pm.variables.get('baseUrl');\nvar CHAT_BODY = {\"model\":\"claude-sonnet-4-6\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok\"}],\"max_tokens\":8};\nfunction call(method, path, body, cb) {\n var req = { url: BASE + path, method: method, header: { 'Content-Type': 'application/json' } };\n if (body !== null && body !== undefined) { req.body = { mode: 'raw', raw: JSON.stringify(body) }; }\n pm.sendRequest(req, function (err, res) {\n if (err) { pm.test(method + ' ' + path + ' reached the gateway', function () { pm.expect.fail(String(err)); }); return cb(null, null); }\n var json = null; try { json = res.json(); } catch (e) { json = null; }\n cb(res, json);\n });\n}\nfunction expectOk(label, res) {\n pm.test(label, function () { pm.expect(res && res.code, 'unexpected status: ' + (res ? res.text() : 'no response')).to.eql(200); });\n return !!(res && res.code === 200);\n}\nfunction chat(headers, model, cb) {\n var h = { 'Content-Type': 'application/json' }; for (var k in headers) { h[k] = headers[k]; }\n var b = JSON.parse(JSON.stringify(CHAT_BODY)); if (model) { b.model = model; }\n pm.sendRequest({ url: BASE + '/v1/chat/completions', method: 'POST', header: h, body: { mode: 'raw', raw: JSON.stringify(b) } }, function (err, res) {\n if (err) { pm.test('follow-up chat request reached the gateway', function () { pm.expect.fail(String(err)); }); return cb(null, null, {}); }\n var json = null; try { json = res.json(); } catch (e) { json = {}; }\n cb(res, json, ((json || {}).extra_fields || {}).routing_info || {});\n });\n}\n// Leftovers of an aborted earlier run carry their run timestamp in their names. Only objects older\n// than ten minutes are swept, so a parallel shard's live objects are never touched.\nfunction isStale(name, prefix) {\n var m = String(name || '').indexOf(prefix) === 0 && String(name).match(/run-(\\d+)/);\n return !!(m && (Date.now() - Number(m[1])) > 10 * 60 * 1000);\n}\nfunction deleteEach(items, pathOf, done) {\n var i = 0; (function next() { if (i >= items.length) { return done(); } var it = items[i++]; call('DELETE', pathOf(it), null, function () { next(); }); })();\n}\nfunction sweepStale(done) {\n call('GET', '/api/governance/routing-rules', null, function (r1, j1) {\n var rules = ((j1 && j1.rules) || []).filter(function (r) { return isStale(r.name, 'Harness session matrix '); });\n deleteEach(rules, function (r) { return '/api/governance/routing-rules/' + r.id; }, function () {\n call('GET', '/api/governance/virtual-keys', null, function (r2, j2) {\n var vks = ((j2 && j2.virtual_keys) || []).filter(function (v) { return isStale(v.name, 'harness-session-matrix-'); });\n deleteEach(vks, function (v) { return '/api/governance/virtual-keys/' + v.id; }, function () {\n call('GET', '/api/providers/anthropic/keys', null, function (r3, j3) {\n var keys = ((j3 && j3.keys) || []).filter(function (k) { return isStale(k.name, 'Harness Session Key B '); });\n deleteEach(keys, function (k) { return '/api/providers/anthropic/keys/' + k.id; }, done);\n });\n });\n });\n });\n });\n}\nfunction anthropicKeyA(cb) {\n call('GET', '/api/providers/anthropic/keys', null, function (res, j) {\n if (!expectOk('setup: listed anthropic keys', res)) { return cb(null); }\n var a = ((j && j.keys) || []).filter(function (k) { return k.name === 'Anthropic API Key'; })[0];\n pm.test('setup: the profile key Anthropic API Key exists', function () { pm.expect(a, 'expected a key named Anthropic API Key: ' + res.text()).to.exist; });\n cb(a || null);\n });\n}\nfunction createKeyB(run, cb) {\n call('POST', '/api/providers/anthropic/keys', { name: 'Harness Session Key B ' + run, value: 'env.ANTHROPIC_API_KEY', models: ['*'], weight: 1 }, function (res, k) {\n if (!expectOk('setup: created a second anthropic key from the same credential', res)) { return cb(null); }\n cb(k);\n });\n}\nfunction createRule(run, suffix, priority, targets, fallbacks, cb) {\n var body = { name: 'Harness session matrix ' + suffix + ' ' + run, description: 'Provider harness, created at run time and deleted by the same row.', priority: priority, cel_expression: '\"x-harness-session-run\" in headers && headers[\"x-harness-session-run\"] == \"' + run + '-' + suffix + '\"', targets: targets, scope: 'global' };\n if (fallbacks) { body.fallbacks = fallbacks; }\n call('POST', '/api/governance/routing-rules', body, function (res, j) {\n if (!expectOk('setup: created the ' + suffix + ' rule', res)) { return cb(null); }\n cb(((j || {}).rule || {}).id || null);\n });\n}\nfunction del(label, path, cb) { if (!path) { return cb(); } call('DELETE', path, null, function (res) { expectOk('cleanup: ' + label, res); cb(); }); }\n// Priorities must be distinct within a scope; derive them from the clock so two runs, or two\n// shards of one run, never collide on a fixed number.\nvar PRIO = 900000 + (Date.now() % 90000);", + "pm.test('first request succeeded', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + "});", + "var body = pm.response.json();", + "var ri = (body.extra_fields || {}).routing_info || {};", + "pm.test('served by anthropic', function () {", + " pm.expect((body.extra_fields || {}).provider, 'the run-scoped rule must route to anthropic: ' + pm.response.text()).to.eql('anthropic');", + "});", + "pm.test('served on key A', function () {", + " pm.expect(ri.key, 'the rule pins key A: ' + pm.response.text()).to.eql(pm.collectionVariables.get('sm9KeyAName'));", + "});", + "var ruleId = pm.collectionVariables.get('sm9RuleId'); var keyBId = pm.collectionVariables.get('sm9KeyBId');", + "var session = pm.collectionVariables.get('smSession9'); var run = pm.collectionVariables.get('smRun9');", + "function cleanup() { del('pin rule deleted', ruleId && '/api/governance/routing-rules/' + ruleId, function () { del('second anthropic key deleted', keyBId && '/api/providers/anthropic/keys/' + keyBId, function () {}); }); }", + "if (!ruleId || !keyBId) { cleanup(); return; }", + "call('PUT', '/api/governance/routing-rules/' + ruleId, { targets: [{ provider: 'anthropic', key_id: keyBId, weight: 1 }] }, function (res) {", + " if (!expectOk('the rule pin moved to key B', res)) { return cleanup(); }", + " chat({ 'x-bf-session-id': session, 'x-harness-session-run': run + '-pin' }, null, function (res2, j2, ri2) {", + " pm.test('second request succeeded', function () { pm.expect(res2 && res2.code, 'unexpected status: ' + (res2 ? res2.text() : 'no response')).to.eql(200); });", + " pm.test('the same session now serves on key B, since a pin outranks the key binding', function () {", + " pm.expect(ri2.key, 'the updated pin must win over the key the session used before: ' + (res2 ? res2.text() : '')).to.eql(pm.collectionVariables.get('sm9KeyBName'));", + " });", + " cleanup();", + " });", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-bf-session-id", + "value": "{{smSession9}}" + }, + { + "key": "x-harness-session-run", + "value": "{{smRun9}}-pin" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"claude-sonnet-4-6\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"max_tokens\": 8\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "Session matrix 10: a session binding outlives the rule that made it - anthropic/claude-sonnet-4-6 - session-affinity", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "// ---- in-script management calls (see the folder description) ----\nvar BASE = pm.variables.get('baseUrl');\nvar CHAT_BODY = {\"model\":\"claude-sonnet-4-6\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok\"}],\"max_tokens\":8};\nfunction call(method, path, body, cb) {\n var req = { url: BASE + path, method: method, header: { 'Content-Type': 'application/json' } };\n if (body !== null && body !== undefined) { req.body = { mode: 'raw', raw: JSON.stringify(body) }; }\n pm.sendRequest(req, function (err, res) {\n if (err) { pm.test(method + ' ' + path + ' reached the gateway', function () { pm.expect.fail(String(err)); }); return cb(null, null); }\n var json = null; try { json = res.json(); } catch (e) { json = null; }\n cb(res, json);\n });\n}\nfunction expectOk(label, res) {\n pm.test(label, function () { pm.expect(res && res.code, 'unexpected status: ' + (res ? res.text() : 'no response')).to.eql(200); });\n return !!(res && res.code === 200);\n}\nfunction chat(headers, model, cb) {\n var h = { 'Content-Type': 'application/json' }; for (var k in headers) { h[k] = headers[k]; }\n var b = JSON.parse(JSON.stringify(CHAT_BODY)); if (model) { b.model = model; }\n pm.sendRequest({ url: BASE + '/v1/chat/completions', method: 'POST', header: h, body: { mode: 'raw', raw: JSON.stringify(b) } }, function (err, res) {\n if (err) { pm.test('follow-up chat request reached the gateway', function () { pm.expect.fail(String(err)); }); return cb(null, null, {}); }\n var json = null; try { json = res.json(); } catch (e) { json = {}; }\n cb(res, json, ((json || {}).extra_fields || {}).routing_info || {});\n });\n}\n// Leftovers of an aborted earlier run carry their run timestamp in their names. Only objects older\n// than ten minutes are swept, so a parallel shard's live objects are never touched.\nfunction isStale(name, prefix) {\n var m = String(name || '').indexOf(prefix) === 0 && String(name).match(/run-(\\d+)/);\n return !!(m && (Date.now() - Number(m[1])) > 10 * 60 * 1000);\n}\nfunction deleteEach(items, pathOf, done) {\n var i = 0; (function next() { if (i >= items.length) { return done(); } var it = items[i++]; call('DELETE', pathOf(it), null, function () { next(); }); })();\n}\nfunction sweepStale(done) {\n call('GET', '/api/governance/routing-rules', null, function (r1, j1) {\n var rules = ((j1 && j1.rules) || []).filter(function (r) { return isStale(r.name, 'Harness session matrix '); });\n deleteEach(rules, function (r) { return '/api/governance/routing-rules/' + r.id; }, function () {\n call('GET', '/api/governance/virtual-keys', null, function (r2, j2) {\n var vks = ((j2 && j2.virtual_keys) || []).filter(function (v) { return isStale(v.name, 'harness-session-matrix-'); });\n deleteEach(vks, function (v) { return '/api/governance/virtual-keys/' + v.id; }, function () {\n call('GET', '/api/providers/anthropic/keys', null, function (r3, j3) {\n var keys = ((j3 && j3.keys) || []).filter(function (k) { return isStale(k.name, 'Harness Session Key B '); });\n deleteEach(keys, function (k) { return '/api/providers/anthropic/keys/' + k.id; }, done);\n });\n });\n });\n });\n });\n}\nfunction anthropicKeyA(cb) {\n call('GET', '/api/providers/anthropic/keys', null, function (res, j) {\n if (!expectOk('setup: listed anthropic keys', res)) { return cb(null); }\n var a = ((j && j.keys) || []).filter(function (k) { return k.name === 'Anthropic API Key'; })[0];\n pm.test('setup: the profile key Anthropic API Key exists', function () { pm.expect(a, 'expected a key named Anthropic API Key: ' + res.text()).to.exist; });\n cb(a || null);\n });\n}\nfunction createKeyB(run, cb) {\n call('POST', '/api/providers/anthropic/keys', { name: 'Harness Session Key B ' + run, value: 'env.ANTHROPIC_API_KEY', models: ['*'], weight: 1 }, function (res, k) {\n if (!expectOk('setup: created a second anthropic key from the same credential', res)) { return cb(null); }\n cb(k);\n });\n}\nfunction createRule(run, suffix, priority, targets, fallbacks, cb) {\n var body = { name: 'Harness session matrix ' + suffix + ' ' + run, description: 'Provider harness, created at run time and deleted by the same row.', priority: priority, cel_expression: '\"x-harness-session-run\" in headers && headers[\"x-harness-session-run\"] == \"' + run + '-' + suffix + '\"', targets: targets, scope: 'global' };\n if (fallbacks) { body.fallbacks = fallbacks; }\n call('POST', '/api/governance/routing-rules', body, function (res, j) {\n if (!expectOk('setup: created the ' + suffix + ' rule', res)) { return cb(null); }\n cb(((j || {}).rule || {}).id || null);\n });\n}\nfunction del(label, path, cb) { if (!path) { return cb(); } call('DELETE', path, null, function (res) { expectOk('cleanup: ' + label, res); cb(); }); }\n// Priorities must be distinct within a scope; derive them from the clock so two runs, or two\n// shards of one run, never collide on a fixed number.\nvar PRIO = 900000 + (Date.now() % 90000);", + "var run = 'run-' + Date.now(); pm.collectionVariables.set('smRun10', run);", + "pm.collectionVariables.set('smSession10', 'session-affinity-' + Date.now() + '-' + Math.floor(Math.random() * 1e9));", + "sweepStale(function () {", + " createRule(run, 'del', PRIO + 2, [{ provider: 'anthropic', weight: 1 }], ['vertex/'], function (id) {", + " if (id) { pm.collectionVariables.set('sm10RuleId', id); }", + " });", + "});" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "// ---- in-script management calls (see the folder description) ----\nvar BASE = pm.variables.get('baseUrl');\nvar CHAT_BODY = {\"model\":\"claude-sonnet-4-6\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok\"}],\"max_tokens\":8};\nfunction call(method, path, body, cb) {\n var req = { url: BASE + path, method: method, header: { 'Content-Type': 'application/json' } };\n if (body !== null && body !== undefined) { req.body = { mode: 'raw', raw: JSON.stringify(body) }; }\n pm.sendRequest(req, function (err, res) {\n if (err) { pm.test(method + ' ' + path + ' reached the gateway', function () { pm.expect.fail(String(err)); }); return cb(null, null); }\n var json = null; try { json = res.json(); } catch (e) { json = null; }\n cb(res, json);\n });\n}\nfunction expectOk(label, res) {\n pm.test(label, function () { pm.expect(res && res.code, 'unexpected status: ' + (res ? res.text() : 'no response')).to.eql(200); });\n return !!(res && res.code === 200);\n}\nfunction chat(headers, model, cb) {\n var h = { 'Content-Type': 'application/json' }; for (var k in headers) { h[k] = headers[k]; }\n var b = JSON.parse(JSON.stringify(CHAT_BODY)); if (model) { b.model = model; }\n pm.sendRequest({ url: BASE + '/v1/chat/completions', method: 'POST', header: h, body: { mode: 'raw', raw: JSON.stringify(b) } }, function (err, res) {\n if (err) { pm.test('follow-up chat request reached the gateway', function () { pm.expect.fail(String(err)); }); return cb(null, null, {}); }\n var json = null; try { json = res.json(); } catch (e) { json = {}; }\n cb(res, json, ((json || {}).extra_fields || {}).routing_info || {});\n });\n}\n// Leftovers of an aborted earlier run carry their run timestamp in their names. Only objects older\n// than ten minutes are swept, so a parallel shard's live objects are never touched.\nfunction isStale(name, prefix) {\n var m = String(name || '').indexOf(prefix) === 0 && String(name).match(/run-(\\d+)/);\n return !!(m && (Date.now() - Number(m[1])) > 10 * 60 * 1000);\n}\nfunction deleteEach(items, pathOf, done) {\n var i = 0; (function next() { if (i >= items.length) { return done(); } var it = items[i++]; call('DELETE', pathOf(it), null, function () { next(); }); })();\n}\nfunction sweepStale(done) {\n call('GET', '/api/governance/routing-rules', null, function (r1, j1) {\n var rules = ((j1 && j1.rules) || []).filter(function (r) { return isStale(r.name, 'Harness session matrix '); });\n deleteEach(rules, function (r) { return '/api/governance/routing-rules/' + r.id; }, function () {\n call('GET', '/api/governance/virtual-keys', null, function (r2, j2) {\n var vks = ((j2 && j2.virtual_keys) || []).filter(function (v) { return isStale(v.name, 'harness-session-matrix-'); });\n deleteEach(vks, function (v) { return '/api/governance/virtual-keys/' + v.id; }, function () {\n call('GET', '/api/providers/anthropic/keys', null, function (r3, j3) {\n var keys = ((j3 && j3.keys) || []).filter(function (k) { return isStale(k.name, 'Harness Session Key B '); });\n deleteEach(keys, function (k) { return '/api/providers/anthropic/keys/' + k.id; }, done);\n });\n });\n });\n });\n });\n}\nfunction anthropicKeyA(cb) {\n call('GET', '/api/providers/anthropic/keys', null, function (res, j) {\n if (!expectOk('setup: listed anthropic keys', res)) { return cb(null); }\n var a = ((j && j.keys) || []).filter(function (k) { return k.name === 'Anthropic API Key'; })[0];\n pm.test('setup: the profile key Anthropic API Key exists', function () { pm.expect(a, 'expected a key named Anthropic API Key: ' + res.text()).to.exist; });\n cb(a || null);\n });\n}\nfunction createKeyB(run, cb) {\n call('POST', '/api/providers/anthropic/keys', { name: 'Harness Session Key B ' + run, value: 'env.ANTHROPIC_API_KEY', models: ['*'], weight: 1 }, function (res, k) {\n if (!expectOk('setup: created a second anthropic key from the same credential', res)) { return cb(null); }\n cb(k);\n });\n}\nfunction createRule(run, suffix, priority, targets, fallbacks, cb) {\n var body = { name: 'Harness session matrix ' + suffix + ' ' + run, description: 'Provider harness, created at run time and deleted by the same row.', priority: priority, cel_expression: '\"x-harness-session-run\" in headers && headers[\"x-harness-session-run\"] == \"' + run + '-' + suffix + '\"', targets: targets, scope: 'global' };\n if (fallbacks) { body.fallbacks = fallbacks; }\n call('POST', '/api/governance/routing-rules', body, function (res, j) {\n if (!expectOk('setup: created the ' + suffix + ' rule', res)) { return cb(null); }\n cb(((j || {}).rule || {}).id || null);\n });\n}\nfunction del(label, path, cb) { if (!path) { return cb(); } call('DELETE', path, null, function (res) { expectOk('cleanup: ' + label, res); cb(); }); }\n// Priorities must be distinct within a scope; derive them from the clock so two runs, or two\n// shards of one run, never collide on a fixed number.\nvar PRIO = 900000 + (Date.now() % 90000);", + "pm.test('first request succeeded', function () { pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200); });", + "var body = pm.response.json(); var ri = (body.extra_fields || {}).routing_info || {};", + "pm.test('served by anthropic as primary', function () {", + " pm.expect((body.extra_fields || {}).provider, 'the run-scoped rule must route to anthropic: ' + pm.response.text()).to.eql('anthropic');", + " pm.expect(!!ri.is_fallback, 'Anthropic must be the primary attempt: ' + pm.response.text()).to.eql(false);", + "});", + "var ruleId = pm.collectionVariables.get('sm10RuleId'); var session = pm.collectionVariables.get('smSession10');", + "if (!ruleId) { return; }", + "del('run-scoped rule deleted mid-session', '/api/governance/routing-rules/' + ruleId, function () {", + " chat({ 'x-bf-session-id': session, 'x-harness-session-pin': 'vertex' }, null, function (res2, j2, ri2) {", + " pm.test('post-delete request succeeded', function () { pm.expect(res2 && res2.code, 'unexpected status: ' + (res2 ? res2.text() : 'no response')).to.eql(200); });", + " pm.test('the binding written under the deleted rule still moves the pinned Vertex rule aside', function () {", + " pm.expect(((j2 || {}).extra_fields || {}).provider, 'a session bound to Anthropic must be served by Anthropic as primary: ' + (res2 ? res2.text() : '')).to.eql('anthropic');", + " pm.expect(!!ri2.is_fallback, 'Anthropic must be the primary attempt').to.eql(false);", + " pm.expect(res2 ? res2.text() : '').to.not.include('no supported key found');", + " });", + " });", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-bf-session-id", + "value": "{{smSession10}}" + }, + { + "key": "x-harness-session-run", + "value": "{{smRun10}}-del" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"claude-sonnet-4-6\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"max_tokens\": 8\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "Session matrix 11: a virtual key shrunk mid-session drops the binding to the removed key - anthropic/claude-sonnet-4-6 - session-affinity", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "// ---- in-script management calls (see the folder description) ----\nvar BASE = pm.variables.get('baseUrl');\nvar CHAT_BODY = {\"model\":\"claude-sonnet-4-6\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok\"}],\"max_tokens\":8};\nfunction call(method, path, body, cb) {\n var req = { url: BASE + path, method: method, header: { 'Content-Type': 'application/json' } };\n if (body !== null && body !== undefined) { req.body = { mode: 'raw', raw: JSON.stringify(body) }; }\n pm.sendRequest(req, function (err, res) {\n if (err) { pm.test(method + ' ' + path + ' reached the gateway', function () { pm.expect.fail(String(err)); }); return cb(null, null); }\n var json = null; try { json = res.json(); } catch (e) { json = null; }\n cb(res, json);\n });\n}\nfunction expectOk(label, res) {\n pm.test(label, function () { pm.expect(res && res.code, 'unexpected status: ' + (res ? res.text() : 'no response')).to.eql(200); });\n return !!(res && res.code === 200);\n}\nfunction chat(headers, model, cb) {\n var h = { 'Content-Type': 'application/json' }; for (var k in headers) { h[k] = headers[k]; }\n var b = JSON.parse(JSON.stringify(CHAT_BODY)); if (model) { b.model = model; }\n pm.sendRequest({ url: BASE + '/v1/chat/completions', method: 'POST', header: h, body: { mode: 'raw', raw: JSON.stringify(b) } }, function (err, res) {\n if (err) { pm.test('follow-up chat request reached the gateway', function () { pm.expect.fail(String(err)); }); return cb(null, null, {}); }\n var json = null; try { json = res.json(); } catch (e) { json = {}; }\n cb(res, json, ((json || {}).extra_fields || {}).routing_info || {});\n });\n}\n// Leftovers of an aborted earlier run carry their run timestamp in their names. Only objects older\n// than ten minutes are swept, so a parallel shard's live objects are never touched.\nfunction isStale(name, prefix) {\n var m = String(name || '').indexOf(prefix) === 0 && String(name).match(/run-(\\d+)/);\n return !!(m && (Date.now() - Number(m[1])) > 10 * 60 * 1000);\n}\nfunction deleteEach(items, pathOf, done) {\n var i = 0; (function next() { if (i >= items.length) { return done(); } var it = items[i++]; call('DELETE', pathOf(it), null, function () { next(); }); })();\n}\nfunction sweepStale(done) {\n call('GET', '/api/governance/routing-rules', null, function (r1, j1) {\n var rules = ((j1 && j1.rules) || []).filter(function (r) { return isStale(r.name, 'Harness session matrix '); });\n deleteEach(rules, function (r) { return '/api/governance/routing-rules/' + r.id; }, function () {\n call('GET', '/api/governance/virtual-keys', null, function (r2, j2) {\n var vks = ((j2 && j2.virtual_keys) || []).filter(function (v) { return isStale(v.name, 'harness-session-matrix-'); });\n deleteEach(vks, function (v) { return '/api/governance/virtual-keys/' + v.id; }, function () {\n call('GET', '/api/providers/anthropic/keys', null, function (r3, j3) {\n var keys = ((j3 && j3.keys) || []).filter(function (k) { return isStale(k.name, 'Harness Session Key B '); });\n deleteEach(keys, function (k) { return '/api/providers/anthropic/keys/' + k.id; }, done);\n });\n });\n });\n });\n });\n}\nfunction anthropicKeyA(cb) {\n call('GET', '/api/providers/anthropic/keys', null, function (res, j) {\n if (!expectOk('setup: listed anthropic keys', res)) { return cb(null); }\n var a = ((j && j.keys) || []).filter(function (k) { return k.name === 'Anthropic API Key'; })[0];\n pm.test('setup: the profile key Anthropic API Key exists', function () { pm.expect(a, 'expected a key named Anthropic API Key: ' + res.text()).to.exist; });\n cb(a || null);\n });\n}\nfunction createKeyB(run, cb) {\n call('POST', '/api/providers/anthropic/keys', { name: 'Harness Session Key B ' + run, value: 'env.ANTHROPIC_API_KEY', models: ['*'], weight: 1 }, function (res, k) {\n if (!expectOk('setup: created a second anthropic key from the same credential', res)) { return cb(null); }\n cb(k);\n });\n}\nfunction createRule(run, suffix, priority, targets, fallbacks, cb) {\n var body = { name: 'Harness session matrix ' + suffix + ' ' + run, description: 'Provider harness, created at run time and deleted by the same row.', priority: priority, cel_expression: '\"x-harness-session-run\" in headers && headers[\"x-harness-session-run\"] == \"' + run + '-' + suffix + '\"', targets: targets, scope: 'global' };\n if (fallbacks) { body.fallbacks = fallbacks; }\n call('POST', '/api/governance/routing-rules', body, function (res, j) {\n if (!expectOk('setup: created the ' + suffix + ' rule', res)) { return cb(null); }\n cb(((j || {}).rule || {}).id || null);\n });\n}\nfunction del(label, path, cb) { if (!path) { return cb(); } call('DELETE', path, null, function (res) { expectOk('cleanup: ' + label, res); cb(); }); }\n// Priorities must be distinct within a scope; derive them from the clock so two runs, or two\n// shards of one run, never collide on a fixed number.\nvar PRIO = 900000 + (Date.now() % 90000);", + "var run = 'run-' + Date.now(); pm.collectionVariables.set('smRun11', run);", + "pm.collectionVariables.set('smSession11', 'session-affinity-' + Date.now() + '-' + Math.floor(Math.random() * 1e9));", + "sweepStale(function () {", + " anthropicKeyA(function (a) {", + " if (!a) { return; }", + " pm.collectionVariables.set('sm11KeyAId', a.id); pm.collectionVariables.set('sm11KeyAName', a.name);", + " createKeyB(run, function (k) {", + " if (!k) { return; }", + " pm.collectionVariables.set('sm11KeyBId', k.id); pm.collectionVariables.set('sm11KeyBName', k.name);", + " call('POST', '/api/governance/virtual-keys', { name: 'harness-session-matrix-' + run, provider_configs: [{ provider: 'anthropic', weight: 1, allowed_models: ['*'], key_ids: [a.id, k.id] }] }, function (res, j) {", + " if (!expectOk('setup: created a virtual key granting both anthropic keys', res)) { return; }", + " var vk = (j || {}).virtual_key || {}; var value = (vk.value && typeof vk.value === 'object') ? vk.value.value : vk.value; var pc = (vk.provider_configs || [])[0] || {};", + " pm.test('setup: the virtual key carries a value and its provider config id', function () { pm.expect(value, res.text()).to.be.a('string').and.not.empty; pm.expect(pc.id, res.text()).to.exist; });", + " pm.collectionVariables.set('sm11VkId', vk.id); pm.collectionVariables.set('smVkValue11', value); pm.collectionVariables.set('sm11VkPcId', String(pc.id));", + " createRule(run, 'vk', PRIO + 3, [{ provider: 'anthropic', key_id: k.id, weight: 1 }], null, function (id) {", + " if (id) { pm.collectionVariables.set('sm11RuleId', id); }", + " });", + " });", + " });", + " });", + "});" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "// ---- in-script management calls (see the folder description) ----\nvar BASE = pm.variables.get('baseUrl');\nvar CHAT_BODY = {\"model\":\"claude-sonnet-4-6\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word: ok\"}],\"max_tokens\":8};\nfunction call(method, path, body, cb) {\n var req = { url: BASE + path, method: method, header: { 'Content-Type': 'application/json' } };\n if (body !== null && body !== undefined) { req.body = { mode: 'raw', raw: JSON.stringify(body) }; }\n pm.sendRequest(req, function (err, res) {\n if (err) { pm.test(method + ' ' + path + ' reached the gateway', function () { pm.expect.fail(String(err)); }); return cb(null, null); }\n var json = null; try { json = res.json(); } catch (e) { json = null; }\n cb(res, json);\n });\n}\nfunction expectOk(label, res) {\n pm.test(label, function () { pm.expect(res && res.code, 'unexpected status: ' + (res ? res.text() : 'no response')).to.eql(200); });\n return !!(res && res.code === 200);\n}\nfunction chat(headers, model, cb) {\n var h = { 'Content-Type': 'application/json' }; for (var k in headers) { h[k] = headers[k]; }\n var b = JSON.parse(JSON.stringify(CHAT_BODY)); if (model) { b.model = model; }\n pm.sendRequest({ url: BASE + '/v1/chat/completions', method: 'POST', header: h, body: { mode: 'raw', raw: JSON.stringify(b) } }, function (err, res) {\n if (err) { pm.test('follow-up chat request reached the gateway', function () { pm.expect.fail(String(err)); }); return cb(null, null, {}); }\n var json = null; try { json = res.json(); } catch (e) { json = {}; }\n cb(res, json, ((json || {}).extra_fields || {}).routing_info || {});\n });\n}\n// Leftovers of an aborted earlier run carry their run timestamp in their names. Only objects older\n// than ten minutes are swept, so a parallel shard's live objects are never touched.\nfunction isStale(name, prefix) {\n var m = String(name || '').indexOf(prefix) === 0 && String(name).match(/run-(\\d+)/);\n return !!(m && (Date.now() - Number(m[1])) > 10 * 60 * 1000);\n}\nfunction deleteEach(items, pathOf, done) {\n var i = 0; (function next() { if (i >= items.length) { return done(); } var it = items[i++]; call('DELETE', pathOf(it), null, function () { next(); }); })();\n}\nfunction sweepStale(done) {\n call('GET', '/api/governance/routing-rules', null, function (r1, j1) {\n var rules = ((j1 && j1.rules) || []).filter(function (r) { return isStale(r.name, 'Harness session matrix '); });\n deleteEach(rules, function (r) { return '/api/governance/routing-rules/' + r.id; }, function () {\n call('GET', '/api/governance/virtual-keys', null, function (r2, j2) {\n var vks = ((j2 && j2.virtual_keys) || []).filter(function (v) { return isStale(v.name, 'harness-session-matrix-'); });\n deleteEach(vks, function (v) { return '/api/governance/virtual-keys/' + v.id; }, function () {\n call('GET', '/api/providers/anthropic/keys', null, function (r3, j3) {\n var keys = ((j3 && j3.keys) || []).filter(function (k) { return isStale(k.name, 'Harness Session Key B '); });\n deleteEach(keys, function (k) { return '/api/providers/anthropic/keys/' + k.id; }, done);\n });\n });\n });\n });\n });\n}\nfunction anthropicKeyA(cb) {\n call('GET', '/api/providers/anthropic/keys', null, function (res, j) {\n if (!expectOk('setup: listed anthropic keys', res)) { return cb(null); }\n var a = ((j && j.keys) || []).filter(function (k) { return k.name === 'Anthropic API Key'; })[0];\n pm.test('setup: the profile key Anthropic API Key exists', function () { pm.expect(a, 'expected a key named Anthropic API Key: ' + res.text()).to.exist; });\n cb(a || null);\n });\n}\nfunction createKeyB(run, cb) {\n call('POST', '/api/providers/anthropic/keys', { name: 'Harness Session Key B ' + run, value: 'env.ANTHROPIC_API_KEY', models: ['*'], weight: 1 }, function (res, k) {\n if (!expectOk('setup: created a second anthropic key from the same credential', res)) { return cb(null); }\n cb(k);\n });\n}\nfunction createRule(run, suffix, priority, targets, fallbacks, cb) {\n var body = { name: 'Harness session matrix ' + suffix + ' ' + run, description: 'Provider harness, created at run time and deleted by the same row.', priority: priority, cel_expression: '\"x-harness-session-run\" in headers && headers[\"x-harness-session-run\"] == \"' + run + '-' + suffix + '\"', targets: targets, scope: 'global' };\n if (fallbacks) { body.fallbacks = fallbacks; }\n call('POST', '/api/governance/routing-rules', body, function (res, j) {\n if (!expectOk('setup: created the ' + suffix + ' rule', res)) { return cb(null); }\n cb(((j || {}).rule || {}).id || null);\n });\n}\nfunction del(label, path, cb) { if (!path) { return cb(); } call('DELETE', path, null, function (res) { expectOk('cleanup: ' + label, res); cb(); }); }\n// Priorities must be distinct within a scope; derive them from the clock so two runs, or two\n// shards of one run, never collide on a fixed number.\nvar PRIO = 900000 + (Date.now() % 90000);", + "pm.test('first request succeeded', function () {", + " pm.expect(pm.response.code, 'unexpected status: ' + pm.response.text()).to.eql(200);", + "});", + "var body = pm.response.json();", + "var ri = (body.extra_fields || {}).routing_info || {};", + "pm.test('served by anthropic', function () {", + " pm.expect((body.extra_fields || {}).provider, 'the run-scoped rule must route to anthropic: ' + pm.response.text()).to.eql('anthropic');", + "});", + "pm.test('served on key B', function () {", + " pm.expect(ri.key, 'the rule pins key B: ' + pm.response.text()).to.eql(pm.collectionVariables.get('sm11KeyBName'));", + "});", + "var ruleId = pm.collectionVariables.get('sm11RuleId'); var vkId = pm.collectionVariables.get('sm11VkId'); var keyBId = pm.collectionVariables.get('sm11KeyBId');", + "var keyAId = pm.collectionVariables.get('sm11KeyAId'); var pcId = pm.collectionVariables.get('sm11VkPcId'); var vkValue = pm.collectionVariables.get('smVkValue11'); var session = pm.collectionVariables.get('smSession11');", + "function cleanup() { del('virtual key rule deleted', ruleId && '/api/governance/routing-rules/' + ruleId, function () { del('virtual key deleted', vkId && '/api/governance/virtual-keys/' + vkId, function () { del('second anthropic key deleted', keyBId && '/api/providers/anthropic/keys/' + keyBId, function () {}); }); }); }", + "if (!ruleId || !vkId || !keyBId || !pcId) { cleanup(); return; }", + "call('PUT', '/api/governance/virtual-keys/' + vkId, { provider_configs: [{ id: Number(pcId), provider: 'anthropic', weight: 1, allowed_models: ['*'], key_ids: [keyAId] }] }, function (res) {", + " if (!expectOk('the virtual key was shrunk to key A only', res)) { return cleanup(); }", + " chat({ 'x-bf-session-id': session, 'x-bf-vk': vkValue }, 'anthropic/' + \"claude-sonnet-4-6\", function (res2, j2, ri2) {", + " pm.test('post-shrink request succeeded', function () { pm.expect(res2 && res2.code, 'unexpected status: ' + (res2 ? res2.text() : 'no response')).to.eql(200); });", + " pm.test('the session cannot follow its binding to the removed key and serves on key A', function () {", + " pm.expect(ri2.key, 'the binding to key B must be dropped once governance no longer grants it: ' + (res2 ? res2.text() : '')).to.eql(pm.collectionVariables.get('sm11KeyAName'));", + " });", + " cleanup();", + " });", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-bf-session-id", + "value": "{{smSession11}}" + }, + { + "key": "x-harness-session-run", + "value": "{{smRun11}}-vk" + }, + { + "key": "x-bf-vk", + "value": "{{smVkValue11}}" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"claude-sonnet-4-6\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with the single word: ok\"\n }\n ],\n \"max_tokens\": 8\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + } + } + ] + }, + { + "name": "115. OpenAI GPT-6 compatibility stack (Astra / Sol / Luna)", + "description": "Live regression coverage for the GPT-6 compatibility stack.\n\n115.1/115.2 pin Astra's always-reasoning fallback on both OpenAI request converters: caller effort none becomes low, and sampling/logprob fields that Astra rejects are removed from the outbound body.\n115.3/115.4 pin the inverse for the newly released Sol and Luna models. Both can disable reasoning, so none must survive; the requests include forced function calling because Chat tool use on these models depends on reasoning being disabled.\n115.5 covers the GPT-6 max-effort ladder and the prompt-cache translation added for the family: cache_control becomes prompt_cache_breakpoint and opts the request into explicit cache mode.\n115.6/115.7 are a real async-tool chain. The first response must return async on its function_call; the second replays that pending call without a tool output. Dropping async on replay makes OpenAI reject the request with \"No tool output found for function call\".\n115.8 proves async is retained only on the supported function/custom tool kinds when they are mixed with another tool.\n\nEvery shaping assertion reads extra_fields.raw_request via x-bf-send-back-raw-request, so these rows verify what Bifrost sent rather than inferring it from a 2xx response.", + "item": [ + { + "name": "115.1 openai/gpt-6-astra chat: none becomes low and unsupported sampling fields are stripped", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('115.1 exact success status', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { return; }", + "var body = pm.response.json() || {};", + "var rr = (body.extra_fields || {}).raw_request;", + "if (typeof rr === 'string') { try { rr = JSON.parse(rr); } catch (e) { rr = null; } }", + "pm.test('115.1 outbound request captured', function () {", + " pm.expect(rr, 'raw_request missing').to.be.an('object');", + "});", + "if (!rr) { return; }", + "pm.test('115.1 Astra normalizes reasoning_effort none to low', function () {", + " pm.expect(rr.reasoning_effort).to.eql('low');", + "});", + "pm.test('115.1 Astra sampling and logprob fields are absent on the wire', function () {", + " ['temperature', 'top_p', 'logprobs', 'top_logprobs'].forEach(function (key) {", + " pm.expect(rr[key], key + ' leaked: ' + JSON.stringify(rr).slice(0, 800)).to.equal(undefined);", + " });", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"openai/gpt-6-astra\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Reply with only OK.\"\n }\n ],\n \"reasoning_effort\": \"none\",\n \"temperature\": 0.2,\n \"top_p\": 0.9,\n \"logprobs\": true,\n \"top_logprobs\": 2,\n \"max_completion_tokens\": 32\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "115.2 openai/gpt-6-astra responses: none becomes low and unsupported sampling fields are stripped", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('115.2 exact success status', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { return; }", + "var body = pm.response.json() || {};", + "var rr = (body.extra_fields || {}).raw_request;", + "if (typeof rr === 'string') { try { rr = JSON.parse(rr); } catch (e) { rr = null; } }", + "pm.test('115.2 outbound request captured', function () {", + " pm.expect(rr, 'raw_request missing').to.be.an('object');", + "});", + "if (!rr) { return; }", + "pm.test('115.2 Astra normalizes reasoning effort none to low', function () {", + " pm.expect(rr.reasoning).to.be.an('object');", + " pm.expect(rr.reasoning.effort).to.eql('low');", + "});", + "pm.test('115.2 Astra Responses sampling and logprob fields are absent on the wire', function () {", + " ['temperature', 'top_p', 'top_logprobs'].forEach(function (key) {", + " pm.expect(rr[key], key + ' leaked: ' + JSON.stringify(rr).slice(0, 800)).to.equal(undefined);", + " });", + " pm.expect(JSON.stringify(rr.include || [])).to.not.include('message.output_text.logprobs');", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"openai/gpt-6-astra\",\n \"input\": \"Reply with only OK.\",\n \"reasoning\": {\n \"effort\": \"none\"\n },\n \"temperature\": 0.2,\n \"top_p\": 0.9,\n \"top_logprobs\": 2,\n \"include\": [\n \"message.output_text.logprobs\"\n ],\n \"max_output_tokens\": 32\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/responses", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "responses" + ] + } + } + }, + { + "name": "115.3 openai/gpt-6-sol chat: reasoning_effort none survives and enables forced function calling", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('115.3 exact success status', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { return; }", + "var body = pm.response.json() || {};", + "var rr = (body.extra_fields || {}).raw_request;", + "if (typeof rr === 'string') { try { rr = JSON.parse(rr); } catch (e) { rr = null; } }", + "pm.test('115.3 outbound request captured', function () {", + " pm.expect(rr, 'raw_request missing').to.be.an('object');", + "});", + "if (!rr) { return; }", + "pm.test('115.3 Sol preserves reasoning_effort none on the wire', function () {", + " pm.expect(rr.reasoning_effort).to.eql('none');", + "});", + "pm.test('115.3 Sol returns the required function call while reasoning is disabled', function () {", + " var msg = body.choices && body.choices[0] && body.choices[0].message;", + " pm.expect(msg, 'assistant message missing').to.be.an('object');", + " pm.expect(msg.tool_calls, 'tool_calls missing: ' + pm.response.text()).to.be.an('array').that.is.not.empty;", + " pm.expect(msg.tool_calls[0].function.name).to.eql('get_weather');", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"openai/gpt-6-sol\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Call get_weather for Paris. Do not answer directly.\"\n }\n ],\n \"reasoning_effort\": \"none\",\n \"tools\": [\n {\n \"type\": \"function\",\n \"function\": {\n \"name\": \"get_weather\",\n \"description\": \"Get current weather for a city\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"city\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"city\"\n ],\n \"additionalProperties\": false\n },\n \"strict\": true\n }\n }\n ],\n \"tool_choice\": \"required\",\n \"max_completion_tokens\": 64\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "115.4 openai/gpt-6-luna responses streaming: effort none survives through a forced function call", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var txt = pm.response.text() || '';", + "pm.test('115.4 exact success status', function () { pm.expect(pm.response.code, 'failed: ' + txt).to.equal(200); });", + "pm.test('115.4 response is SSE and reaches response.completed', function () {", + " pm.expect(pm.response.headers.get('content-type') || '').to.include('event-stream');", + " pm.expect(txt, txt.slice(-800)).to.include('response.completed');", + "});", + "if (pm.response.code !== 200) { return; }", + "var terminal = null;", + "txt.split('\\n').forEach(function (line) {", + " if (line.indexOf('data:') !== 0) { return; }", + " var ev; try { ev = JSON.parse(line.slice(5)); } catch (e) { return; }", + " if (ev && ev.type === 'response.completed') { terminal = ev; }", + "});", + "var rr = terminal && terminal.extra_fields && terminal.extra_fields.raw_request;", + "if (typeof rr === 'string') { try { rr = JSON.parse(rr); } catch (e) { rr = null; } }", + "pm.test('115.4 Luna preserves reasoning effort none on the streaming wire', function () {", + " pm.expect(rr, 'raw_request missing on response.completed').to.be.an('object');", + " pm.expect(rr.reasoning).to.be.an('object');", + " pm.expect(rr.reasoning.effort).to.eql('none');", + "});", + "pm.test('115.4 stream contains the required function_call', function () {", + " pm.expect(txt, 'function_call absent: ' + txt.slice(0, 1200)).to.include('function_call');", + " pm.expect(txt).to.include('get_weather');", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"openai/gpt-6-luna\",\n \"input\": \"Call get_weather for Paris. Do not answer directly.\",\n \"reasoning\": {\n \"effort\": \"none\"\n },\n \"tools\": [\n {\n \"type\": \"function\",\n \"name\": \"get_weather\",\n \"description\": \"Get current weather for a city\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"city\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"city\"\n ],\n \"additionalProperties\": false\n },\n \"strict\": true\n }\n ],\n \"tool_choice\": \"required\",\n \"max_output_tokens\": 64,\n \"stream\": true\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/responses", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "responses" + ] + } + } + }, + { + "name": "115.5 openai/gpt-6-sol responses: max effort and explicit prompt-cache breakpoint reach OpenAI", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('115.5 exact success status', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { return; }", + "var body = pm.response.json() || {};", + "var rr = (body.extra_fields || {}).raw_request;", + "if (typeof rr === 'string') { try { rr = JSON.parse(rr); } catch (e) { rr = null; } }", + "pm.test('115.5 outbound request captured', function () {", + " pm.expect(rr, 'raw_request missing').to.be.an('object');", + "});", + "if (!rr) { return; }", + "pm.test('115.5 GPT-6 Sol keeps max reasoning effort', function () {", + " pm.expect(rr.reasoning).to.be.an('object');", + " pm.expect(rr.reasoning.effort).to.eql('max');", + "});", + "pm.test('115.5 cache_control becomes an explicit prompt_cache_breakpoint', function () {", + " var first = (((rr.input || [])[0] || {}).content || [])[0] || {};", + " pm.expect(first.prompt_cache_breakpoint, 'first input block: ' + JSON.stringify(first)).to.be.an('object');", + " pm.expect(first.prompt_cache_breakpoint.mode).to.eql('explicit');", + " pm.expect(JSON.stringify(rr.input || [])).to.not.include('cache_control');", + " pm.expect(rr.prompt_cache_options, 'prompt_cache_options missing').to.be.an('object');", + " pm.expect(rr.prompt_cache_options.mode).to.eql('explicit');", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"openai/gpt-6-sol\",\n \"input\": [\n {\n \"role\": \"user\",\n \"content\": [\n {\n \"type\": \"input_text\",\n \"text\": \"Stable instructions: answer tersely.\",\n \"cache_control\": {\n \"type\": \"ephemeral\"\n }\n },\n {\n \"type\": \"input_text\",\n \"text\": \"Reply with only OK.\"\n }\n ]\n }\n ],\n \"reasoning\": {\n \"effort\": \"max\"\n },\n \"max_output_tokens\": 128\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/responses", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "responses" + ] + } + } + }, + { + "name": "115.6 openai/gpt-6-astra responses async turn 1: definition and returned call retain async", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.collectionVariables.unset('gpt6AsyncReplayInput115');", + "pm.test('115.6 exact success status', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { return; }", + "var body = pm.response.json() || {};", + "var rr = (body.extra_fields || {}).raw_request;", + "if (typeof rr === 'string') { try { rr = JSON.parse(rr); } catch (e) { rr = null; } }", + "pm.test('115.6 outbound request captured', function () {", + " pm.expect(rr, 'raw_request missing').to.be.an('object');", + "});", + "if (!rr) { return; }", + "pm.test('115.6 async definition and output_schema reach OpenAI', function () {", + " var tool = (rr.tools || [])[0] || {};", + " pm.expect(tool.type).to.eql('function');", + " pm.expect(tool.async).to.eql(true);", + " pm.expect(tool.output_schema, 'output_schema missing').to.be.an('object');", + " pm.expect(tool.output_schema.type).to.eql('object');", + "});", + "var call = (body.output || []).filter(function (item) { return item && item.type === 'function_call'; })[0];", + "pm.test('115.6 returned function_call retains async true', function () {", + " pm.expect(call, 'function_call missing: ' + pm.response.text()).to.be.an('object');", + " pm.expect(call.async).to.eql(true);", + " pm.expect(call.call_id).to.be.a('string').that.is.not.empty;", + "});", + "if (call && call.async === true) {", + " pm.collectionVariables.set('gpt6AsyncReplayInput115', JSON.stringify([", + " { role: 'user', content: 'Start job report-1156 with the start_job function.' },", + " call,", + " { role: 'user', content: 'While that job is still running, reply with only the number 4.' }", + " ]));", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"openai/gpt-6-astra\",\n \"input\": \"Start job report-1156 with the start_job function. Do not answer directly.\",\n \"reasoning\": {\n \"effort\": \"low\"\n },\n \"tools\": [\n {\n \"type\": \"function\",\n \"name\": \"start_job\",\n \"description\": \"Start a background job\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"job\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"job\"\n ],\n \"additionalProperties\": false\n },\n \"strict\": true,\n \"async\": true,\n \"output_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"job_id\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"job_id\"\n ],\n \"additionalProperties\": false\n }\n }\n ],\n \"tool_choice\": \"required\",\n \"max_output_tokens\": 64\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/responses", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "responses" + ] + } + } + }, + { + "name": "115.7 openai/gpt-6-astra responses async turn 2: pending call replay is accepted without output", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var requestRaw = (pm.request.body && pm.request.body.raw) || '';", + "if (requestRaw.indexOf('{{gpt6AsyncReplayInput115}}') !== -1) {", + " pm.test('115.7 turn 1 produced replay input', function () { pm.expect.fail('gpt6AsyncReplayInput115 is unresolved; run 115.6 first'); });", + " return;", + "}", + "pm.test('115.7 exact success status', function () {", + " pm.expect(pm.response.code, 'dropping async causes OpenAI to demand a tool output: ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { return; }", + "var body = pm.response.json() || {};", + "var rr = (body.extra_fields || {}).raw_request;", + "if (typeof rr === 'string') { try { rr = JSON.parse(rr); } catch (e) { rr = null; } }", + "pm.test('115.7 replayed pending function_call keeps async true on the wire', function () {", + " pm.expect(rr, 'raw_request missing').to.be.an('object');", + " var call = (rr.input || []).filter(function (item) { return item && item.type === 'function_call'; })[0];", + " pm.expect(call, 'replayed function_call missing: ' + JSON.stringify(rr).slice(0, 1000)).to.be.an('object');", + " pm.expect(call.async).to.eql(true);", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"openai/gpt-6-astra\",\n \"input\": {{gpt6AsyncReplayInput115}},\n \"reasoning\": {\n \"effort\": \"low\"\n },\n \"max_output_tokens\": 64\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/responses", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "responses" + ] + } + } + }, + { + "name": "115.8 openai/gpt-6-sol responses: mixed async tools retain markers only on function and custom", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('115.8 exact success status', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.equal(200);", + "});", + "if (pm.response.code !== 200) { return; }", + "var body = pm.response.json() || {};", + "var rr = (body.extra_fields || {}).raw_request;", + "if (typeof rr === 'string') { try { rr = JSON.parse(rr); } catch (e) { rr = null; } }", + "pm.test('115.8 outbound request captured', function () {", + " pm.expect(rr, 'raw_request missing').to.be.an('object');", + "});", + "if (!rr) { return; }", + "var byType = {};", + "(rr.tools || []).forEach(function (tool) { byType[tool.type] = tool; });", + "pm.test('115.8 function and custom tools keep async true', function () {", + " pm.expect(byType.function, 'function tool missing').to.be.an('object');", + " pm.expect(byType.function.async).to.eql(true);", + " pm.expect(byType.custom, 'custom tool missing').to.be.an('object');", + " pm.expect(byType.custom.async).to.eql(true);", + "});", + "pm.test('115.8 unsupported web_search async marker is stripped', function () {", + " pm.expect(byType.web_search, 'web_search tool missing').to.be.an('object');", + " pm.expect(byType.web_search.async, 'async leaked to web_search: ' + JSON.stringify(byType.web_search)).to.equal(undefined);", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"openai/gpt-6-sol\",\n \"input\": \"Reply with only OK and do not call any tool.\",\n \"reasoning\": {\n \"effort\": \"none\"\n },\n \"tools\": [\n {\n \"type\": \"function\",\n \"name\": \"start_job\",\n \"description\": \"Start a background job\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"job\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"job\"\n ],\n \"additionalProperties\": false\n },\n \"strict\": true,\n \"async\": true\n },\n {\n \"type\": \"custom\",\n \"name\": \"format_report\",\n \"description\": \"Format a report from free-form text\",\n \"format\": {\n \"type\": \"text\"\n },\n \"async\": true\n },\n {\n \"type\": \"web_search\",\n \"async\": true\n }\n ],\n \"tool_choice\": \"none\",\n \"max_output_tokens\": 32\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/responses", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "responses" + ] + } + } + } + ] + }, + { + "name": "115. Bedrock encrypted reasoning model switch", + "description": "Regression for Codex switching from Luna to Terra in one conversation: Bedrock runtime rejects replayed encrypted reasoning with HTTP 400 / validation_error / \"encrypted reasoning was created for a different account or model\". Bifrost must recognize that refusal and retry once without the incompatible encrypted state. The producer captures real Luna output; both consumers independently replay that same history to Terra and require completed responses. A missing encrypted payload fails the producer, and the harness chained-variable guard fails dependent requests. Requires the existing bedrock_openai_ep provider (use_openai_endpoints: true) and access to both models; the default bedrock provider can route through Converse and would miss the reported endpoint.", + "item": [ + { + "name": "bedrock_openai_ep/us.openai.gpt-5.6-luna capture encrypted reasoning - Bedrock encrypted reasoning model switch", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "pm.collectionVariables.unset('bedrockModelSwitchHistory');" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('Luna produces real encrypted reasoning for cross-model replay', function () {", + " var body = pm.response.text();", + " pm.expect(pm.response.code, 'producer failed: ' + body).to.equal(200);", + " var response = pm.response.json();", + " pm.expect(response.status, 'producer did not complete: ' + body).to.equal('completed');", + " pm.expect(response.output, 'missing producer output: ' + body).to.be.an('array').that.is.not.empty;", + " var reasoning = response.output.filter(function (item) {", + " return item.type === 'reasoning' && typeof item.encrypted_content === 'string' && item.encrypted_content.length > 0;", + " });", + " pm.expect(reasoning, 'producer returned no encrypted reasoning: ' + body).to.not.be.empty;", + " var history = [{role: 'user', content: \"Find the smallest positive integer that leaves remainder 2 when divided by 3, remainder 3 when divided by 5, and remainder 2 when divided by 7. Give only the integer.\"}];", + " response.output.forEach(function (item) { history.push(item); });", + " history.push({role: 'user', content: 'Now say OK.'});", + " pm.collectionVariables.set('bedrockModelSwitchHistory', JSON.stringify(history));", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\"model\":\"bedrock_openai_ep/us.openai.gpt-5.6-luna\",\"store\":false,\"max_output_tokens\":2048,\"reasoning\":{\"effort\":\"low\"},\"include\":[\"reasoning.encrypted_content\"],\"input\":[{\"role\":\"user\",\"content\":\"Find the smallest positive integer that leaves remainder 2 when divided by 3, remainder 3 when divided by 5, and remainder 2 when divided by 7. Give only the integer.\"}]}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{baseUrl}}/v1/responses", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "responses" + ] + } + } + }, + { + "name": "bedrock_openai_ep/us.openai.gpt-5.6-terra unary replay - Bedrock encrypted reasoning model switch", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('Terra completes after replaying Luna encrypted reasoning', function () {", + " var body = pm.response.text();", + " pm.expect(pm.response.code, 'model-switch replay failed: ' + body).to.equal(200);", + " pm.expect(body.toLowerCase()).to.not.include('encrypted reasoning was created for a different account or model');", + " var response = pm.response.json();", + " pm.expect(response.error, 'response error: ' + body).to.not.be.ok;", + " pm.expect(response.status, 'response did not complete: ' + body).to.equal('completed');", + " var text = (response.output || []).filter(function (item) { return item.type === 'message'; }).map(function (item) {", + " return (item.content || []).filter(function (block) { return block.type === 'output_text'; }).map(function (block) { return block.text; }).join('');", + " }).join('');", + " pm.expect(text, 'no assistant text: ' + body).to.be.a('string').and.not.empty;", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\"model\":\"bedrock_openai_ep/us.openai.gpt-5.6-terra\",\"store\":false,\"max_output_tokens\":2048,\"reasoning\":{\"effort\":\"low\"},\"input\":{{bedrockModelSwitchHistory}}}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{baseUrl}}/v1/responses", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "responses" + ] + } + } + }, + { + "name": "bedrock_openai_ep/us.openai.gpt-5.6-terra streaming replay - Bedrock encrypted reasoning model switch", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('Terra stream completes without replay errors', function () {", + " var body = pm.response.text();", + " pm.expect(pm.response.code, 'streaming model-switch replay failed: ' + body).to.equal(200);", + " pm.expect(body.toLowerCase()).to.not.include('encrypted reasoning was created for a different account or model');", + " var events = body.split(/\\r?\\n/).filter(function (line) { return line.indexOf('data:') === 0; }).map(function (line) {", + " return line.slice(5).trim();", + " }).filter(function (data) { return data && data !== '[DONE]'; }).map(function (data) { return JSON.parse(data); });", + " var errors = events.filter(function (event) {", + " return event.error || event.type === 'error' || event.type === 'response.failed' || event.type === 'response.incomplete' || (event.response && event.response.error);", + " });", + " pm.expect(errors, 'SSE errors: ' + body).to.be.empty;", + " var completed = events.filter(function (event) { return event.type === 'response.completed'; });", + " pm.expect(completed, 'no completion event: ' + body).to.have.lengthOf(1);", + " pm.expect(completed[0].response.status, 'response did not complete: ' + body).to.equal('completed');", + " var text = events.filter(function (event) { return event.type === 'response.output_text.delta'; }).map(function (event) { return event.delta || ''; }).join('');", + " pm.expect(text, 'no assistant text deltas: ' + body).to.be.a('string').and.not.empty;", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\"model\":\"bedrock_openai_ep/us.openai.gpt-5.6-terra\",\"store\":false,\"stream\":true,\"max_output_tokens\":2048,\"reasoning\":{\"effort\":\"low\"},\"input\":{{bedrockModelSwitchHistory}}}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{baseUrl}}/v1/responses", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "responses" + ] + } + } + } + ] } ] } diff --git a/tests/e2e/api/provider_config/azure-stream-preamble.config.json b/tests/e2e/api/provider_config/azure-stream-preamble.config.json index 2fd6bf3c5ba..732c4271f35 100644 --- a/tests/e2e/api/provider_config/azure-stream-preamble.config.json +++ b/tests/e2e/api/provider_config/azure-stream-preamble.config.json @@ -17,6 +17,22 @@ "default_request_timeout_in_seconds": 10, "stream_idle_timeout_in_seconds": 5 } + }, + "openai": { + "keys": [ + { + "name": "local-stream-fixture", + "value": "test-key", + "models": ["preamble-error", "preamble-success"], + "weight": 1 + } + ], + "network_config": { + "base_url": "http://127.0.0.1:8789", + "max_retries": 0, + "default_request_timeout_in_seconds": 10, + "stream_idle_timeout_in_seconds": 5 + } } } } diff --git a/tests/e2e/api/runners/azure-stream-preamble-fixture.mjs b/tests/e2e/api/runners/azure-stream-preamble-fixture.mjs index 57b9359894a..f30919965ca 100644 --- a/tests/e2e/api/runners/azure-stream-preamble-fixture.mjs +++ b/tests/e2e/api/runners/azure-stream-preamble-fixture.mjs @@ -32,13 +32,35 @@ const responsesSuccess = [ '{"type":"response.completed","sequence_number":7,"response":{"id":"azure-preamble-success","object":"response","status":"completed","model":"preamble-success","output":[{"id":"success-item","type":"message","role":"assistant","status":"completed","content":[{"type":"output_text","text":"hello","annotations":[]}]}],"usage":{"input_tokens":10,"output_tokens":1,"total_tokens":11}}}', ]; +// Direct OpenAI (and every host serving OpenAI models) fails an overloaded +// stream after the same startup events, with server_is_overloaded and no HTTP +// status, so the failure is fallback-eligible but not retryable. +const openaiChatFailure = [ + chatFailure[1], + '{"error":{"message":"The server is overloaded. Please try again later.","type":"server_error","code":"server_is_overloaded"}}', +]; + +const openaiResponsesFailure = [ + ...responsesFailure.slice(0, 4), + '{"type":"response.failed","sequence_number":4,"response":{"id":"azure-preamble-failed","status":"failed","error":{"code":"server_is_overloaded","message":"The server is overloaded. Please try again later."}}}', +]; + +// Azure deployments are served under /openai/v1, direct OpenAI under /v1. +const routes = { + "/openai/v1/chat/completions": { responses: false, openai: false }, + "/openai/v1/responses": { responses: true, openai: false }, + "/v1/chat/completions": { responses: false, openai: true }, + "/v1/responses": { responses: true, openai: true }, +}; + const server = http.createServer(async (req, res) => { const path = new URL(req.url, "http://localhost").pathname; - const responses = path === "/openai/v1/responses"; - if (req.method !== "POST" || (!responses && path !== "/openai/v1/chat/completions")) { + const route = routes[path]; + if (req.method !== "POST" || !route) { res.writeHead(404).end(); return; } + const { responses, openai } = route; try { req.setEncoding("utf8"); let body = ""; @@ -49,9 +71,10 @@ const server = http.createServer(async (req, res) => { return; } const failed = model === "preamble-error"; - const events = responses - ? (failed ? responsesFailure : responsesSuccess) - : (failed ? chatFailure : chatSuccess); + const failure = responses + ? (openai ? openaiResponsesFailure : responsesFailure) + : (openai ? openaiChatFailure : chatFailure); + const events = failed ? failure : (responses ? responsesSuccess : chatSuccess); res.writeHead(200, { "Content-Type": "text/event-stream" }); res.flushHeaders(); for (const event of events) res.write(`data: ${event}\n\n`); @@ -62,7 +85,7 @@ const server = http.createServer(async (req, res) => { }); server.listen(port, "127.0.0.1", () => { - console.log(`Azure streaming fixture: http://127.0.0.1:${port}`); + console.log(`Azure/OpenAI streaming fixture: http://127.0.0.1:${port}`); }); for (const signal of ["SIGINT", "SIGTERM"]) { process.on(signal, () => server.close(() => process.exit(0))); diff --git a/tests/e2e/api/runners/filter-collection.mjs b/tests/e2e/api/runners/filter-collection.mjs index 095f48ff7db..5d1596bcc50 100644 --- a/tests/e2e/api/runners/filter-collection.mjs +++ b/tests/e2e/api/runners/filter-collection.mjs @@ -152,6 +152,7 @@ const PROVIDER_KEYWORDS = { xai: ["xai", "grok"], replicate: ["replicate", "/replicate", "flux", "black-forest-labs"], runware: ["runware", "runware/"], + typesafe: ["typesafe", "/typesafe", "jev-"], }; // Haystack = item JSON + ancestor folder names. Folder names encode the harness @@ -437,8 +438,22 @@ const filterTree = (items, keep) => { // --provider/--feature slicing whenever the two ends of a chain match differently. So after the // user's predicates decide the initial set, pull in each selected item's producers transitively. // Collection order is untouched, and producers already precede consumers there. +// +// Earlier consumers of the same variable are pulled in too. A chain's middle step can change +// server state (a session binding, say) without setting any variable, so it is never anybody's +// producer, and cost slicing would otherwise run the final step in a shard without it - against +// state the middle step never touched. Consumer counts stay small (at most a handful per variable +// in this collection), so this never balloons a shard. const expandWithProducers = (selected, entries) => { const producerIndex = buildProducerIndex(entries); + const position = new Map(entries.map(({ item }, i) => [item, i])); + const consumersOf = new Map(); + for (const { item } of entries) { + for (const { variable } of chainedDependencies(item, producerIndex)) { + if (!consumersOf.has(variable)) consumersOf.set(variable, []); + consumersOf.get(variable).push(item); + } + } const keep = new Set(selected); const queue = [...selected]; @@ -451,6 +466,12 @@ const expandWithProducers = (selected, entries) => { // leaving the consumer to fail on an unsubstituted {{var}}, which is the // failure this whole function exists to prevent. for (const { producer, producerItem: dep, variable } of chainedDependencies(item, producerIndex)) { + for (const step of consumersOf.get(variable) || []) { + if (keep.has(step) || position.get(step) >= position.get(item)) continue; + keep.add(step); + queue.push(step); + pulled.push(`${item.name} <- ${variable} <- ${step.name} (earlier chain step)`); + } if (!dep || keep.has(dep)) continue; keep.add(dep); queue.push(dep); diff --git a/tests/e2e/api/runners/lib/chained-vars.mjs b/tests/e2e/api/runners/lib/chained-vars.mjs index d3bfe93807f..5cbfafda44b 100644 --- a/tests/e2e/api/runners/lib/chained-vars.mjs +++ b/tests/e2e/api/runners/lib/chained-vars.mjs @@ -7,6 +7,10 @@ // or in its URL (the Responses lifecycle rows, whose create captures a resp_ id that the // retrieve/stream-retrieve/delete rows carry in the path; those are GET/DELETE with no body // at all, so a body-only scan sees no dependency and leaves the chain unguarded and splittable). +// A header is a third place (the session-affinity rows: a bind request computes an x-bf-session-id +// value that its follow-up sends in that header, and a virtual key value rides in x-bf-vk), and it +// matters most under sharding, where a consumer cut off from its producer runs with the template +// unsubstituted and answers a question nobody asked. // // Postman has no notion of that link, and an unset variable is not an error. In a body the // literal "{{var}}" stays put and the provider answers 400 "Invalid JSON" in ~1ms; in a URL it @@ -57,9 +61,23 @@ export function rawBodyOf(item) { // caller keys a Map) is cheaper than deciding which one to trust. export function templateSourcesOf(item) { const url = item.request?.url; - if (typeof url === "string") return [rawBodyOf(item), url]; + // Postman resolves templates in header names and values too. A session id or a credential a + // producer computed travels there (x-bf-session-id, x-bf-vk), and a consumer that names it only + // in a header used to be invisible here: no producer pulled into its shard, no guard. A disabled + // header is never sent, so a template in one is not a dependency: counting it would pull a + // producer the request does not need and let the guard skip an independent request. The + // collection format also allows the whole header block as one raw string, like the URL: that + // string is a single source, scanned as it is. + const header = item.request?.header; + const headers = + typeof header === "string" + ? [header] + : (Array.isArray(header) ? header : []) + .filter((h) => h && h.disabled !== true) + .flatMap((h) => [h.key, h.value]); + if (typeof url === "string") return [rawBodyOf(item), url, ...headers].filter((s) => typeof s === "string"); const query = (url?.query || []).flatMap((q) => [q?.key, q?.value]); - return [rawBodyOf(item), url?.raw, ...(url?.host || []), ...(url?.path || []), ...query].filter( + return [rawBodyOf(item), url?.raw, ...(url?.host || []), ...(url?.path || []), ...query, ...headers].filter( (s) => typeof s === "string" ); } diff --git a/tests/e2e/api/runners/lib/chained-vars.test.mjs b/tests/e2e/api/runners/lib/chained-vars.test.mjs index af2721346d9..62287a23fc1 100644 --- a/tests/e2e/api/runners/lib/chained-vars.test.mjs +++ b/tests/e2e/api/runners/lib/chained-vars.test.mjs @@ -137,6 +137,85 @@ test("a {{var}} in an exploded path or query is found without url.raw", () => { } }); +// Headers resolve templates as well. The session-affinity rows carry the session id their bind +// request computed in x-bf-session-id, and nothing else on the follow-up names it, so a scan that +// ignored headers left those pairs unchained: under sub-sharding the follow-up could run without +// its bind and send the literal template as its session id. +test("a {{var}} in a header links to its producer", () => { + const withHeader = { + item: [ + req("bind", "{}", 'pm.collectionVariables.set("sessionId", "s_1");'), + { + name: "follow-up", + request: { + method: "POST", + header: [{ key: "x-bf-session-id", value: "{{sessionId}}" }], + body: { mode: "raw", raw: "{}" }, + url: { raw: "http://x/y" }, + }, + }, + ], + }; + const entries = walkRequests(withHeader.item); + const byName = Object.fromEntries(entries.map(({ item }) => [item.name, item])); + assert.deepStrictEqual( + chainedDependencies(byName["follow-up"], buildProducerIndex(entries)).map((d) => d.variable), + ["sessionId"] + ); + injectChainedVarGuards(withHeader); + assert.match(guardOf(byName["follow-up"]), /sessionId/); + assert.strictEqual(guardOf(byName.bind), ""); +}); + +// The collection format also allows request.header as one raw string. It resolves templates like +// any other request string, so it is one source, and it must not crash the scan. +test("a {{var}} in a string-form header links to its producer", () => { + const stringHeader = { + item: [ + req("bind", "{}", 'pm.collectionVariables.set("sessionId", "s_1");'), + { + name: "follow-up", + request: { + method: "POST", + header: "x-bf-session-id: {{sessionId}}\nContent-Type: application/json", + body: { mode: "raw", raw: "{}" }, + url: { raw: "http://x/y" }, + }, + }, + ], + }; + const entries = walkRequests(stringHeader.item); + const byName = Object.fromEntries(entries.map(({ item }) => [item.name, item])); + assert.deepStrictEqual( + chainedDependencies(byName["follow-up"], buildProducerIndex(entries)).map((d) => d.variable), + ["sessionId"] + ); +}); + +// A disabled header is never sent, so a template in it must not chain the request: the guard +// would otherwise skip a request that needs nothing from the producer. +test("a {{var}} in a disabled header is not a dependency", () => { + const disabled = { + item: [ + req("bind", "{}", 'pm.collectionVariables.set("sessionId", "s_1");'), + { + name: "independent", + request: { + method: "POST", + header: [{ key: "x-bf-session-id", value: "{{sessionId}}", disabled: true }], + body: { mode: "raw", raw: "{}" }, + url: { raw: "http://x/y" }, + }, + }, + ], + }; + const entries = walkRequests(disabled.item); + const byName = Object.fromEntries(entries.map(({ item }) => [item.name, item])); + assert.deepStrictEqual(chainedDependencies(byName.independent, buildProducerIndex(entries)), []); + injectChainedVarGuards(disabled); + assert.strictEqual(guardOf(byName.independent), ""); +}); + test("each URL consumer is guarded, and the producer is not", () => { const c = lifecycle(); assert.strictEqual(injectChainedVarGuards(c), 3); diff --git a/tests/e2e/api/runners/lib/decision-content-shape.test.mjs b/tests/e2e/api/runners/lib/decision-content-shape.test.mjs new file mode 100644 index 00000000000..895e4bc7c02 --- /dev/null +++ b/tests/e2e/api/runners/lib/decision-content-shape.test.mjs @@ -0,0 +1,33 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import test from 'node:test'; + +const collection = JSON.parse(fs.readFileSync(new URL('../../collections/provider-harness.json', import.meta.url), 'utf8')); +const collectionScript = collection.event.find(event => event.listen === 'test').script.exec.join('\n'); +const contentScript = collectionScript.slice( + collectionScript.indexOf('// Only register the content-shape test'), + collectionScript.indexOf('// Routed-identity response headers'), +); + +function contentFailures(body) { + const failures = []; + new Function('pm', contentScript)({ + request: { url: { toString: () => 'http://localhost:8080/v1/decisions' } }, + response: { + code: 200, + headers: { get: () => 'application/json' }, + json: () => body, + }, + test: (_name, fn) => { try { fn(); } catch (error) { failures.push(error); } }, + expect: (value) => ({ to: { be: { get true() { assert.equal(value, true); } } } }), + }); + return failures; +} + +test('decision answers count as response content', () => { + assert.equal(contentFailures({ answers: { category: { kind: 'choice', value: 'billing' } } }).length, 0); +}); + +test('empty decision answers still fail the content check', () => { + assert.equal(contentFailures({ answers: {} }).length, 1); +}); diff --git a/tests/e2e/features/routing-rules/pages/routing-rules.page.ts b/tests/e2e/features/routing-rules/pages/routing-rules.page.ts index 7408855ca9c..486964950c0 100644 --- a/tests/e2e/features/routing-rules/pages/routing-rules.page.ts +++ b/tests/e2e/features/routing-rules/pages/routing-rules.page.ts @@ -15,8 +15,10 @@ export interface RoutingRuleConfig { enabled?: boolean scope?: 'global' | 'team' | 'customer' | 'virtual_key' scopeId?: string - // Fallback providers - fallbacks?: string[] + // Fallback providers, as "provider/model" strings or objects that pin a provider key. + // The key is named, not identified: the key select renders key_id as the option value + // and the key's name as its text, and pinFallbackKey locates the option by that text. + fallbacks?: Array } /** @@ -123,7 +125,7 @@ export class RoutingRulesPage extends BasePage { const toast = this.page.locator('[data-sonner-toast]:not([data-removed="true"])').first() await expect(toast).toBeVisible({ timeout: 10000 }) const toastText = await toast.textContent() - if (toastText?.toLowerCase().includes('error') || toastText?.toLowerCase().includes('failed')) { + if (await toast.getAttribute('data-type') === 'error' || toastText?.toLowerCase().includes('error') || toastText?.toLowerCase().includes('failed')) { throw new Error(`Failed to ${action}: ${toastText}`) } await this.dismissToasts() @@ -175,7 +177,7 @@ export class RoutingRulesPage extends BasePage { // Select provider if provided (in the Routing Target section) if (config.provider) { - const providerCombo = this.sheet.getByRole('combobox').filter({ hasText: /Select provider/i }).first() + const providerCombo = this.sheet.getByTestId('routing-target-0-provider-select') if (await providerCombo.isVisible().catch(() => false)) { await providerCombo.click() await this.page.waitForSelector('[role="listbox"]', { timeout: 5000 }) @@ -209,6 +211,17 @@ export class RoutingRulesPage extends BasePage { } } + // Add fallbacks if provided + for (const fallback of config.fallbacks ?? []) { + const spec = typeof fallback === 'string' + ? { provider: fallback.split('/')[0], model: fallback.split('/').slice(1).join('/') || undefined, key_name: undefined } + : fallback + const index = await this.addFallbackProvider(spec.provider, spec.model) + if (spec.key_name) { + await this.pinFallbackKey(index, spec.key_name) + } + } + // Save. Duplicate priorities can happen in shared CI environments, so retry with a fresh // priority instead of failing the whole test on a non-deterministic collision. await this.saveBtn.waitFor({ state: 'visible' }) @@ -255,11 +268,8 @@ export class RoutingRulesPage extends BasePage { const row = this.getRuleRow(name) await row.scrollIntoViewIfNeeded() - const editBtn = row.locator('button').filter({ has: this.page.locator('svg.lucide-pencil') }).or( - row.getByRole('button', { name: /Edit/i }) - ) - await editBtn.waitFor({ state: 'visible' }) - await editBtn.click() + await row.getByRole('button', { name: `Actions for routing rule ${name}`, exact: true }).click() + await this.page.getByRole('menuitem', { name: 'Edit', exact: true }).click() await expect(this.sheet).toBeVisible({ timeout: 5000 }) await this.waitForSheetAnimation() @@ -274,11 +284,8 @@ export class RoutingRulesPage extends BasePage { await row.scrollIntoViewIfNeeded() // Find edit button - const editBtn = row.locator('button').filter({ has: this.page.locator('svg.lucide-pencil') }).or( - row.getByRole('button', { name: /Edit/i }) - ) - await editBtn.waitFor({ state: 'visible' }) - await editBtn.click() + await row.getByRole('button', { name: `Actions for routing rule ${name}`, exact: true }).click() + await this.page.getByRole('menuitem', { name: 'Edit', exact: true }).click() await expect(this.sheet).toBeVisible({ timeout: 5000 }) await this.waitForSheetAnimation() @@ -321,12 +328,8 @@ export class RoutingRulesPage extends BasePage { const row = this.getRuleRow(name) await row.scrollIntoViewIfNeeded() - // Find delete button (may have lucide-trash or lucide-trash-2 icon) - const deleteBtn = row.locator('button').filter({ - has: this.page.locator('svg.lucide-trash, svg.lucide-trash-2') - }).first() - await deleteBtn.waitFor({ state: 'visible' }) - await deleteBtn.click() + await row.getByRole('button', { name: `Actions for routing rule ${name}`, exact: true }).click() + await this.page.getByRole('menuitem', { name: 'Delete', exact: true }).click() // Wait for confirmation dialog (AlertDialog uses role="alertdialog") const alertDialog = this.page.locator('[role="alertdialog"]') @@ -633,23 +636,81 @@ export class RoutingRulesPage extends BasePage { } /** - * Add a fallback provider + * Add a fallback row and pick its provider, optionally its model. + * Returns the index of the row that was added. */ - async addFallbackProvider(provider: string, model?: string): Promise { - // Find the "Add Fallback" button - const addFallbackBtn = this.sheet.getByRole('button', { name: /Add Fallback/i }).or( - this.sheet.locator('button').filter({ hasText: /Fallback/i }) - ) + async addFallbackProvider(provider: string, model?: string): Promise { + const existing = await this.sheet.locator('[data-testid^="routing-fallback-"][data-testid$="-provider-select"]').count() + const index = existing - const isVisible = await addFallbackBtn.isVisible().catch(() => false) - if (isVisible) { - await addFallbackBtn.click() + await this.sheet.getByRole('button', { name: /Add Fallback/i }).click() + + const providerCombo = this.sheet.getByTestId(`routing-fallback-${index}-provider-select`) + await providerCombo.waitFor({ state: 'visible', timeout: 5000 }) + await providerCombo.click() + await this.page.waitForSelector('[role="listbox"]', { timeout: 5000 }) + const option = this.page.getByRole('option', { name: new RegExp(provider, 'i') }).first() + await option.scrollIntoViewIfNeeded() + await option.click({ force: true }) + await this.page.waitForSelector('[role="listbox"]', { state: 'hidden', timeout: 5000 }).catch(() => {}) - // Fill in provider/model - typically in format "provider/model" - const fallbackInput = this.sheet.locator('input[placeholder*="fallback" i], input[placeholder*="provider" i]').first() - const value = model ? `${provider}/${model}` : provider - await fallbackInput.fill(value) + if (model) { + const modelSelect = this.sheet.getByTestId(`routing-fallback-${index}-model-select`) + await modelSelect.click() + await modelSelect.getByRole('combobox').or(modelSelect.locator('input')).first().fill(model) + await this.page.keyboard.press('Enter') } + + return index + } + + /** + * The provider currently selected on a fallback row. + */ + async getFallbackProvider(index: number): Promise { + const combo = this.sheet.getByTestId(`routing-fallback-${index}-provider-select`) + if (!(await combo.isVisible().catch(() => false))) { + return null + } + return (await combo.textContent())?.trim() ?? null + } + + /** + * Pin a provider key on a fallback row. Returns false when the row offers no key select, + * which is what happens when the chosen provider has no keys configured in this environment. + */ + async pinFallbackKey(index: number, keyName?: string): Promise { + const keySelect = this.sheet.getByTestId(`routing-fallback-${index}-apikey-select`) + if (!(await keySelect.isVisible().catch(() => false))) { + return false + } + + await keySelect.click() + await this.page.waitForSelector('[role="listbox"]', { timeout: 5000 }) + const option = keyName + ? this.page.getByRole('option', { name: new RegExp(keyName, 'i') }).first() + : this.page.getByRole('option').first() + await option.click({ force: true }) + await this.page.waitForSelector('[role="listbox"]', { state: 'hidden', timeout: 5000 }).catch(() => {}) + return true + } + + /** + * The key currently pinned on a fallback row, or null when the row has no key select. + */ + async getPinnedFallbackKey(index: number): Promise { + const keySelect = this.sheet.getByTestId(`routing-fallback-${index}-apikey-select`) + if (!(await keySelect.isVisible().catch(() => false))) { + return null + } + return (await keySelect.textContent())?.trim() ?? null + } + + /** + * Clear the pinned key on a fallback row, returning it to load-balanced selection. + */ + async clearFallbackKey(index: number): Promise { + await this.sheet.getByTestId(`routing-fallback-${index}-apikey-clear`).click() } /** diff --git a/tests/e2e/features/routing-rules/routing-rules.spec.ts b/tests/e2e/features/routing-rules/routing-rules.spec.ts index 00d37eb3252..04faff024de 100644 --- a/tests/e2e/features/routing-rules/routing-rules.spec.ts +++ b/tests/e2e/features/routing-rules/routing-rules.spec.ts @@ -15,6 +15,7 @@ test.describe('Routing Rules', () => { }) test.afterEach(async ({ routingRulesPage }) => { + await routingRulesPage.cancelRule() // Clean up any rules created during tests for (const ruleName of [...createdRules]) { try { @@ -220,6 +221,66 @@ test.describe('Routing Rules', () => { expect(exists).toBe(true) }) + test('should create rule with a fallback', async ({ routingRulesPage }) => { + const ruleData = createRoutingRuleData({ + name: `Fallback Rule ${Date.now()}`, + enabled: false, + provider: 'openai', + fallbacks: [{ provider: 'anthropic' }], + }) + createdRules.push(ruleData.name) + + await routingRulesPage.createRoutingRule(ruleData) + + expect(await routingRulesPage.ruleExists(ruleData.name)).toBe(true) + + // The fallback must survive the round-trip through the API's legacy string form. + await routingRulesPage.openEditSheet(ruleData.name) + expect(await routingRulesPage.getFallbackProvider(0)).toMatch(/anthropic/i) + await routingRulesPage.cancelRule() + }) + + test('should pin and clear a provider key on a fallback', async ({ routingRulesPage }) => { + const ruleData = createRoutingRuleData({ + name: `Pinned Fallback Rule ${Date.now()}`, + enabled: false, + provider: 'openai', + }) + createdRules.push(ruleData.name) + + await routingRulesPage.createRoutingRule(ruleData) + await routingRulesPage.openEditSheet(ruleData.name) + + const index = await routingRulesPage.addFallbackProvider('openai') + const pinned = await routingRulesPage.pinFallbackKey(index) + // The key select only renders once the chosen provider has keys configured. + test.skip(!pinned, 'no provider keys configured in this environment') + + const selectedKey = await routingRulesPage.getPinnedFallbackKey(index) + expect(selectedKey).toBeTruthy() + expect(selectedKey).not.toMatch(/Select key/i) + + await routingRulesPage.saveBtn.click() + // The rule already exists, so waiting for its row alone can pass while the sheet is + // still closing, and the reopen below would then act on the sheet that is on its way out. + await expect(routingRulesPage.sheet).not.toBeVisible() + await routingRulesPage.waitForRuleToAppear(ruleData.name) + + // Reopening must preselect the same key rather than falling back to the placeholder. + await routingRulesPage.openEditSheet(ruleData.name) + expect(await routingRulesPage.getPinnedFallbackKey(index)).toBe(selectedKey) + + // Clearing returns the fallback to load-balanced key selection. + await routingRulesPage.clearFallbackKey(index) + expect(await routingRulesPage.getPinnedFallbackKey(index)).toMatch(/Select key/i) + await routingRulesPage.saveBtn.click() + await expect(routingRulesPage.sheet).not.toBeVisible() + await routingRulesPage.openEditSheet(ruleData.name) + expect(await routingRulesPage.getPinnedFallbackKey(index)).toMatch(/Select key/i) + expect(await routingRulesPage.getFallbackProvider(index)).toMatch(/openai/i) + await routingRulesPage.cancelRule() + }) + test('should reorder rules by changing priority', async ({ routingRulesPage }) => { // Create two rules with unique priorities (avoid fixed 500/600 so parallel workers don't collide) const rule1 = createRoutingRuleData({ name: `Reorder Test Rule 1 ${Date.now()}` }) diff --git a/tests/integrations/decision_emulation_matrix.json b/tests/integrations/decision_emulation_matrix.json new file mode 100644 index 00000000000..30e0f2c85c6 --- /dev/null +++ b/tests/integrations/decision_emulation_matrix.json @@ -0,0 +1,72 @@ +{ + "_comment": "Shared source of truth for cross-provider decision LLM-emulation coverage. `scenario` is one rich decision (a deliberately unambiguous support ticket) with 30 questions across all three kinds; jev answers it as the ground truth and every other model must reach the same verdict. Each provider under `providers` lists >=4 current, tool-capable chat models routed as '/'. The Python and TypeScript suites read this file: they SKIP-and-REPORT a model this account cannot reach (auth/model-block/404/rate-limit/connection) but FAIL a 400 (bad request we built) or 500 (emulation returned no decision), and score each model's agreement with jev across the 30 questions. The provider-harness cases generated from this file HARD-ASSERT the anchor verdicts. Emulation runs through provider.Responses (forced emit_decision function tool).", + "_low_confidence_providers": ["vertex", "bedrock", "azure", "parasail", "bedrock_mantle"], + "_agreement_fail_threshold": 0.95, + "_split_question_providers": ["perplexity"], + "_split_question_providers_note": "Capability-limited providers. The suites first try the full 30-question call, then (only for these providers, only on a non-availability failure) retry one question at a time and merge. Perplexity's sonar accepts noul questions but rejects choice and score questions outright (400 'invalid request') - a schema incompatibility, not a size limit - so even the one-at-a-time split cannot complete the set. A residual failure from one of these providers is therefore treated as a documented capability SKIP rather than a defect: the identical schema is accepted by every other provider, and the emulation code is unchanged. This is a test-side policy only.", + "scenario": { + "state": "Customer support ticket #88231. Customer is on the Pro plan and has been a paying subscriber for 3 years. They write: 'This is my FIFTH email in two weeks and NOBODY has replied. You charged my card THREE times for a single order (#A-4021) at $149 each, so $447 total instead of $149. I have screenshots of all three charges. I have asked for a refund every single time and been completely ignored. If the two extra charges are not refunded TODAY I am filing a chargeback with my bank, cancelling my Pro subscription immediately, posting the screenshots on Twitter, and reporting you to the Better Business Bureau. Your competitor Acme offers the same features for less and I am seriously considering switching. This is completely unacceptable and I want a real person to fix this now.'", + "anchors": { + "is_frustrated": 1, + "is_angry": 1, + "mentions_refund": 1, + "threatens_chargeback": 1, + "threatens_cancellation": 1, + "mentions_competitor": 1, + "is_satisfied": 0, + "category": "billing", + "sentiment": "negative", + "customer_tier": "pro" + }, + "questions": { + "is_frustrated": {"kind": "noul", "instructions": "Is the customer frustrated?"}, + "is_angry": {"kind": "noul", "instructions": "Is the customer angry?"}, + "is_satisfied": {"kind": "noul", "instructions": "Is the customer satisfied with the service?"}, + "is_polite": {"kind": "noul", "instructions": "Is the message written in a calm, polite tone?"}, + "mentions_refund": {"kind": "noul", "instructions": "Does the customer ask for a refund?"}, + "threatens_chargeback": {"kind": "noul", "instructions": "Does the customer threaten a bank chargeback?"}, + "threatens_cancellation": {"kind": "noul", "instructions": "Does the customer threaten to cancel their subscription?"}, + "threatens_public_complaint": {"kind": "noul", "instructions": "Does the customer threaten to complain publicly (social media, review sites, or a regulator)?"}, + "mentions_competitor": {"kind": "noul", "instructions": "Does the customer mention a competitor?"}, + "is_billing_related": {"kind": "noul", "instructions": "Is the issue about billing or charges?"}, + "is_repeat_contact": {"kind": "noul", "instructions": "Has the customer contacted support about this before?"}, + "has_attached_evidence": {"kind": "noul", "instructions": "Does the customer say they have evidence such as screenshots?"}, + "mentions_specific_amount": {"kind": "noul", "instructions": "Does the customer cite a specific dollar amount?"}, + "is_long_time_customer": {"kind": "noul", "instructions": "Is this a long-time customer (multiple years)?"}, + "requests_human_response": {"kind": "noul", "instructions": "Does the customer explicitly ask to speak to a real person?"}, + "is_first_time_issue": {"kind": "noul", "instructions": "Is this the first time the customer is raising this issue?"}, + "willing_to_wait": {"kind": "noul", "instructions": "Is the customer willing to wait patiently for a resolution?"}, + "expresses_urgency": {"kind": "noul", "instructions": "Does the customer express that this is urgent?"}, + "mentions_data_loss": {"kind": "noul", "instructions": "Does the customer report lost data?"}, + "reports_product_crash": {"kind": "noul", "instructions": "Does the customer report the product crashing or a software defect?"}, + "category": {"kind": "choice", "instructions": "Pick the ticket category", "criteria": {"billing": "charges, refunds, invoices", "bug": "product defects or crashes", "account": "login or account access", "other": "anything else"}}, + "sentiment": {"kind": "choice", "instructions": "Overall sentiment of the message", "criteria": {"positive": "happy", "neutral": "matter of fact", "negative": "unhappy or hostile"}}, + "primary_emotion": {"kind": "choice", "instructions": "The dominant emotion", "criteria": {"anger": "angry or hostile", "confusion": "confused", "sadness": "sad or disappointed", "joy": "happy"}}, + "customer_tier": {"kind": "choice", "instructions": "Which plan is the customer on?", "criteria": {"free": "free plan", "pro": "pro plan", "enterprise": "enterprise plan"}}, + "recommended_action": {"kind": "choice", "instructions": "The single best next action", "criteria": {"issue_refund": "refund the extra charges", "request_more_info": "ask for more details", "close_ticket": "close with no action", "ignore": "take no action"}}, + "contact_channel": {"kind": "choice", "instructions": "Which channel did the customer use?", "criteria": {"email": "email or support ticket", "phone": "phone call", "chat": "live chat", "social": "social media"}}, + "urgency": {"kind": "score", "instructions": "How urgently does this need a human reply?", "criteria": ["can wait a week", "should be answered this week", "should be answered today", "needs an immediate reply"]}, + "severity": {"kind": "score", "instructions": "How severe is the customer's problem?", "criteria": ["minor", "moderate", "major", "severe"]}, + "churn_risk": {"kind": "score", "instructions": "How likely is this customer to cancel?", "criteria": ["not at all likely", "somewhat likely", "very likely", "almost certain"]}, + "customer_satisfaction": {"kind": "score", "instructions": "How satisfied is the customer right now?", "criteria": ["very unsatisfied", "unsatisfied", "neutral", "satisfied", "very satisfied"]} + } + }, + "providers": { + "openai": ["gpt-5.6", "gpt-5.1", "gpt-5-mini", "gpt-5.6-luna", "gpt-5.6-terra", "gpt-5.4-nano"], + "anthropic": ["claude-opus-5", "claude-sonnet-5", "claude-fable-5-1", "claude-haiku-4-5-20251001"], + "gemini": ["gemini-3.1-pro-preview", "gemini-3.6-flash", "gemini-3.7-flash", "gemini-2.5-pro"], + "vertex": ["gemini-3.7-flash", "gemini-3.6-flash", "claude-opus-4-8", "claude-fable-5-1"], + "bedrock": ["us.anthropic.claude-sonnet-4-5-20250929-v1:0", "global.anthropic.claude-sonnet-4-20250514-v1:0"], + "azure": ["gpt-5.4", "gpt-5.4-mini", "gpt-4.1", "gpt-4o"], + "cohere": ["command-a-03-2025", "command-a-reasoning-08-2025", "command-a-vision-07-2025", "command-a-translate-08-2025"], + "mistral": ["mistral-large-latest", "mistral-medium-latest", "mistral-small-latest", "magistral-medium-latest"], + "groq": ["qwen/qwen3.8-27b"], + "deepseek": ["deepseek-v4-pro", "deepseek-v4-flash", "deepseek-reasoner", "deepseek-chat"], + "cerebras": ["gpt-oss-120b", "llama-3.3-70b", "qwen-3-32b", "llama-4-scout-17b-16e-instruct"], + "xai": ["grok-4.5", "grok-4", "grok-3", "grok-code-fast-1"], + "openrouter": ["openai/gpt-5", "anthropic/claude-opus-4.1", "google/gemini-3.8-flash", "meta-llama/llama-3.3-70b-instruct"], + "perplexity": ["sonar-pro", "sonar-reasoning-pro", "sonar", "sonar-deep-research"], + "parasail": ["parasail-gpt-oss-120b", "parasail-qwen3-235b-a22b", "parasail-deepseek-v3", "parasail-kimi-k2-instruct"], + "bedrock_mantle": ["openai.gpt-oss-120b", "openai.gpt-oss-20b", "anthropic.claude-sonnet-5", "anthropic.claude-opus-4-8"] + } +} diff --git a/tests/integrations/python/config.json b/tests/integrations/python/config.json index 6f00d7a2ddf..4bbe8b6b99b 100644 --- a/tests/integrations/python/config.json +++ b/tests/integrations/python/config.json @@ -13,6 +13,9 @@ }, "providers": { "openai": { + "send_back_raw_request": true, + "send_back_raw_response": true, + "store_raw_request_response": true, "keys": [ { "name": "OpenAI API Key", @@ -27,6 +30,9 @@ } }, "elevenlabs": { + "send_back_raw_request": true, + "send_back_raw_response": true, + "store_raw_request_response": true, "keys": [ { "name": "ElevenLabs API Key", @@ -40,7 +46,27 @@ "default_request_timeout_in_seconds": 300 } }, + "typesafe": { + "send_back_raw_request": true, + "send_back_raw_response": true, + "store_raw_request_response": true, + "keys": [ + { + "name": "Typesafe API Key", + "value": "env.TYPESAFE_API_KEY", + "weight": 1, + "models": ["*"], + "use_for_batch_api": false + } + ], + "network_config": { + "default_request_timeout_in_seconds": 300 + } + }, "xai": { + "send_back_raw_request": true, + "send_back_raw_response": true, + "store_raw_request_response": true, "keys": [ { "name": "Xai API Key", @@ -55,6 +81,9 @@ } }, "huggingface": { + "send_back_raw_request": true, + "send_back_raw_response": true, + "store_raw_request_response": true, "keys": [ { "name": "Hugging Face API Key", @@ -69,7 +98,16 @@ } }, "anthropic": { + "send_back_raw_request": true, + "send_back_raw_response": true, + "store_raw_request_response": true, "keys": [ + { + "name": "Anthropic (harness, no models)", + "value": "env.ANTHROPIC_API_KEY", + "weight": 1, + "models": ["harness-no-model"] + }, { "name": "Anthropic API Key", "value": "env.ANTHROPIC_API_KEY", @@ -86,6 +124,9 @@ } }, "gemini": { + "send_back_raw_request": true, + "send_back_raw_response": true, + "store_raw_request_response": true, "keys": [ { "name": "Gemini API Key", @@ -100,6 +141,9 @@ } }, "vertex": { + "send_back_raw_request": true, + "send_back_raw_response": true, + "store_raw_request_response": true, "keys": [ { "name": "Vertex API Key", @@ -119,7 +163,11 @@ "gemini-3.6-flash", "claude-fable-5-1", "gemini-3-pro-image", - "gemini-3.7-flash" + "gemini-3.7-flash", + "gemini-3.1-flash-lite", + "gemini-3.1-flash-lite-preview", + "gemini-3.1-pro-preview", + "gemini-3-flash-preview" ] }, { @@ -146,7 +194,15 @@ "auth_credentials": "env.VERTEX_CREDENTIALS" }, "weight": 1, - "models": ["gemini-3.6-flash", "gemini-3-pro-image", "gemini-3.7-flash"] + "models": [ + "gemini-3.6-flash", + "gemini-3-pro-image", + "gemini-3.7-flash", + "gemini-3.1-flash-lite", + "gemini-3.1-flash-lite-preview", + "gemini-3.1-pro-preview", + "gemini-3-flash-preview" + ] } ], "network_config": { @@ -158,6 +214,9 @@ } }, "mistral": { + "send_back_raw_request": true, + "send_back_raw_response": true, + "store_raw_request_response": true, "keys": [ { "name": "Mistral API Key", @@ -171,6 +230,9 @@ } }, "cohere": { + "send_back_raw_request": true, + "send_back_raw_response": true, + "store_raw_request_response": true, "keys": [ { "name": "Cohere API Key", @@ -184,6 +246,9 @@ } }, "parasail": { + "send_back_raw_request": true, + "send_back_raw_response": true, + "store_raw_request_response": true, "keys": [ { "name": "Parasail API Key", @@ -197,6 +262,9 @@ } }, "groq": { + "send_back_raw_request": true, + "send_back_raw_response": true, + "store_raw_request_response": true, "keys": [ { "name": "Groq API Key", @@ -210,6 +278,9 @@ } }, "perplexity": { + "send_back_raw_request": true, + "send_back_raw_response": true, + "store_raw_request_response": true, "keys": [ { "name": "Perplexity API Key", @@ -223,6 +294,9 @@ } }, "cerebras": { + "send_back_raw_request": true, + "send_back_raw_response": true, + "store_raw_request_response": true, "keys": [ { "name": "Cerebras API Key", @@ -236,6 +310,9 @@ } }, "openrouter": { + "send_back_raw_request": true, + "send_back_raw_response": true, + "store_raw_request_response": true, "keys": [ { "name": "OpenRouter API Key", @@ -253,6 +330,9 @@ } }, "azure": { + "send_back_raw_request": true, + "send_back_raw_response": true, + "store_raw_request_response": true, "keys": [ { "name": "Azure API Key", @@ -261,22 +341,39 @@ "endpoint": "env.AZURE_ENDPOINT", "deployments": { "gpt-4o": "gpt-4o", + "gpt-4o-low-cap": "gpt-4o-low-cap", + "gpt-4o-batch": "gpt-4o-2", "gpt-4o-mini": "gpt-4o-mini", "gpt-4o-mini-tts": "gpt-4o-mini-tts", - "o1": "o1", - "gpt-4o-batch": "gpt-4o-batch", + "gpt-4o-transcribe": "gpt-4o-transcribe", + "gpt-4.1": "gpt-4.1", "whisper": "whisper", - "text-embedding-3-small": "text-embedding-3-small", - "gpt-image-1": "gpt-image-1", + "text-embedding-ada-002": "text-embedding-ada-002", + "gpt-image-2": "gpt-image-2", + "gpt-realtime": "gpt-realtime", + "gpt-realtime-1.5": "gpt-realtime-1.5", + "gpt-5.2": "gpt-5.2", + "gpt-5.3-chat": "gpt-5.3-chat", + "gpt-5.4": "gpt-5.4", + "gpt-5.4-mini": "gpt-5.4-mini", + "gpt-5.4-nano": "gpt-5.4-nano", + "gpt-5.4-pro": "gpt-5.4-pro", + "gpt-5.5": "gpt-5.5", "gpt-5.6-sol": "gpt-5.6-sol", "gpt-5.6-terra": "gpt-5.6-terra", "gpt-5.6-luna": "gpt-5.6-luna", - "gpt-5.5": "gpt-5.5", - "gpt-5.4": "gpt-5.4", "claude-opus-5": "claude-opus-5", "claude-sonnet-5": "claude-sonnet-5", "claude-opus-4-8": "claude-opus-4-8", - "claude-haiku-4-5": "claude-haiku-4-5" + "claude-opus-4-7": "claude-opus-4-7-1", + "claude-opus-4-6": "claude-opus-4-6", + "claude-opus-4-5": "claude-opus-4-5", + "claude-sonnet-4-6": "claude-sonnet-4-6", + "claude-sonnet-4-5": "claude-sonnet-4-5", + "claude-haiku-4-5": "claude-haiku-4-5", + "DeepSeek-V3.2": "DeepSeek-V3.2", + "DeepSeek-V4-Pro": "DeepSeek-V4-Pro", + "DeepSeek-V4-Flash-0731": "DeepSeek-V4-Flash-0731" } }, "weight": 1, @@ -293,6 +390,9 @@ } }, "bedrock": { + "send_back_raw_request": true, + "send_back_raw_response": true, + "store_raw_request_response": true, "keys": [ { "name": "Bedrock API Key", @@ -311,6 +411,9 @@ } }, "bedrock_mantle": { + "send_back_raw_request": true, + "send_back_raw_response": true, + "store_raw_request_response": true, "keys": [ { "name": "Bedrock Mantle API Key", @@ -329,6 +432,9 @@ } }, "replicate": { + "send_back_raw_request": true, + "send_back_raw_response": true, + "store_raw_request_response": true, "keys": [ { "name": "Replicate API Key", @@ -341,6 +447,9 @@ } }, "runware": { + "send_back_raw_request": true, + "send_back_raw_response": true, + "store_raw_request_response": true, "keys": [ { "name": "Runware API Key", @@ -353,6 +462,9 @@ } }, "runway": { + "send_back_raw_request": true, + "send_back_raw_response": true, + "store_raw_request_response": true, "keys": [ { "name": "Runway API Key", @@ -365,6 +477,9 @@ } }, "nebius": { + "send_back_raw_request": true, + "send_back_raw_response": true, + "store_raw_request_response": true, "keys": [ { "name": "Nebius API Key", @@ -377,6 +492,9 @@ } }, "deepseek": { + "send_back_raw_request": true, + "send_back_raw_response": true, + "store_raw_request_response": true, "keys": [ { "name": "DeepSeek API Key", @@ -391,6 +509,9 @@ } }, "anthropic_pc_auto": { + "send_back_raw_request": true, + "send_back_raw_response": true, + "store_raw_request_response": true, "keys": [ { "name": "Anthropic API Key", @@ -412,6 +533,9 @@ } }, "gemini_pc_auto": { + "send_back_raw_request": true, + "send_back_raw_response": true, + "store_raw_request_response": true, "keys": [ { "name": "Gemini API Key", @@ -433,6 +557,9 @@ } }, "anthropic_pc_points": { + "send_back_raw_request": true, + "send_back_raw_response": true, + "store_raw_request_response": true, "keys": [ { "name": "Anthropic API Key", @@ -464,6 +591,9 @@ } }, "bedrock_pc_auto": { + "send_back_raw_request": true, + "send_back_raw_response": true, + "store_raw_request_response": true, "keys": [ { "name": "Bedrock API Key", @@ -489,6 +619,9 @@ } }, "bedrock_pc_points": { + "send_back_raw_request": true, + "send_back_raw_response": true, + "store_raw_request_response": true, "keys": [ { "name": "Bedrock API Key", @@ -524,6 +657,9 @@ } }, "bedrock_openai_ep": { + "send_back_raw_request": true, + "send_back_raw_response": true, + "store_raw_request_response": true, "keys": [ { "name": "Bedrock API Key", @@ -546,6 +682,9 @@ } }, "openai_dnsdm": { + "send_back_raw_request": true, + "send_back_raw_response": true, + "store_raw_request_response": true, "keys": [ { "name": "OpenAI API Key", @@ -565,6 +704,9 @@ } }, "openai_dnsdm_wait": { + "send_back_raw_request": true, + "send_back_raw_response": true, + "store_raw_request_response": true, "keys": [ { "name": "OpenAI API Key", @@ -585,6 +727,9 @@ } }, "openai_pc_auto": { + "send_back_raw_request": true, + "send_back_raw_response": true, + "store_raw_request_response": true, "keys": [ { "name": "OpenAI API Key", @@ -606,6 +751,9 @@ } }, "openai_pc_points": { + "send_back_raw_request": true, + "send_back_raw_response": true, + "store_raw_request_response": true, "keys": [ { "name": "OpenAI API Key", @@ -637,6 +785,9 @@ } }, "anthropic_pc_edge": { + "send_back_raw_request": true, + "send_back_raw_response": true, + "store_raw_request_response": true, "keys": [ { "name": "Anthropic API Key", @@ -676,6 +827,9 @@ } }, "anthropic_pc_overflow": { + "send_back_raw_request": true, + "send_back_raw_response": true, + "store_raw_request_response": true, "keys": [ { "name": "Anthropic API Key", @@ -723,6 +877,9 @@ } }, "anthropic_pc_absent": { + "send_back_raw_request": true, + "send_back_raw_response": true, + "store_raw_request_response": true, "keys": [ { "name": "Anthropic API Key", @@ -762,6 +919,132 @@ } }, "governance": { + "routing_rules": [ + { + "id": "harness-session-pin-bind", + "priority": 901, + "name": "Harness session pin: bind the session to Anthropic", + "description": "Provider harness only (folder 'session-pin'). A request carrying x-harness-session-pin: bind is routed to Anthropic so its session binds there; the paired request then hits the pinned Vertex rule below. Matches nothing without that header.", + "cel_expression": "\"x-harness-session-pin\" in headers && headers[\"x-harness-session-pin\"] == \"bind\"", + "targets": [ + { + "provider": "anthropic", + "weight": 1 + } + ] + }, + { + "id": "harness-session-pin-vertex", + "priority": 902, + "name": "Harness session pin: Vertex with a pinned key, Anthropic fallback", + "description": "Provider harness only (folder 'session-pin'). Mirrors the production rule behind the 'no supported key found with id ... for provider: anthropic' report: a Vertex target with a pinned key and an Anthropic fallback on the incoming model. A session already bound to Anthropic moves the primary there, and the Vertex key pin must not follow it. Matches nothing without x-harness-session-pin: vertex.", + "cel_expression": "\"x-harness-session-pin\" in headers && headers[\"x-harness-session-pin\"] == \"vertex\"", + "targets": [ + { + "provider": "vertex", + "provider_key_name": "Vertex Global (Claude)", + "weight": 1 + } + ], + "fallbacks": ["anthropic/"] + }, + { + "id": "harness-session-pin-rebind", + "priority": 903, + "name": "Harness session pin: Vertex fails, Anthropic rescues", + "description": "Provider harness only (folder 'session-affinity'). Sends the request to Vertex on a model Vertex rejects so the attempt fails upstream, then falls back to Anthropic on a real model. A served fallback binds the session to Anthropic, which is how the production sessions moved. Matches nothing without x-harness-session-pin: rebind.", + "cel_expression": "\"x-harness-session-pin\" in headers && headers[\"x-harness-session-pin\"] == \"rebind\"", + "targets": [ + { + "provider": "vertex", + "model": "claude-does-not-exist-harness", + "weight": 1 + } + ], + "fallbacks": ["anthropic/claude-sonnet-4-6"] + }, + { + "id": "harness-session-pin-rescue", + "priority": 904, + "name": "Harness session pin: pinned Vertex rescues a failing Anthropic primary", + "description": "Provider harness only (folder 'session-affinity'). Same Vertex target and pin as the 'vertex' rule, but the Anthropic fallback is pinned to a key whose model list excludes every real model. In a session bound to Anthropic on the same model the promoted Anthropic attempt fails at key selection, and the demoted Vertex entry must serve on the key the rule pinned for it. Matches nothing without x-harness-session-pin: rescue.", + "cel_expression": "\"x-harness-session-pin\" in headers && headers[\"x-harness-session-pin\"] == \"rescue\"", + "targets": [ + { + "provider": "vertex", + "provider_key_name": "Vertex Global (Claude)", + "weight": 1 + } + ], + "fallbacks": [ + { + "provider": "anthropic", + "provider_key_name": "Anthropic (harness, no models)" + } + ] + }, + { + "id": "harness-session-pin-vertex-only", + "priority": 905, + "name": "Harness session pin: Vertex only, no fallbacks", + "description": "Provider harness only (folder 'session-affinity'). A one-provider chain: a session bound to Anthropic cannot follow its binding here, so the binding is dropped and the next served request rebinds. Matches nothing without x-harness-session-pin: vertex-only.", + "cel_expression": "\"x-harness-session-pin\" in headers && headers[\"x-harness-session-pin\"] == \"vertex-only\"", + "targets": [ + { + "provider": "vertex", + "provider_key_name": "Vertex Global (Claude)", + "weight": 1 + } + ] + }, + { + "id": "harness-session-pin-anthropic-first", + "priority": 906, + "name": "Harness session pin: Anthropic first, Vertex fallback", + "description": "Provider harness only (folder 'session-affinity'). The mirror image of the 'vertex' rule: Anthropic is the primary and Vertex the fallback on the incoming model. A session bound to Vertex on that model moves Vertex first; a session whose Vertex binding was dropped follows this decision. Matches nothing without x-harness-session-pin: anthropic-first.", + "cel_expression": "\"x-harness-session-pin\" in headers && headers[\"x-harness-session-pin\"] == \"anthropic-first\"", + "targets": [ + { + "provider": "anthropic", + "weight": 1 + } + ], + "fallbacks": ["vertex/"] + }, + { + "id": "harness-session-pin-anthropic-first-other-model", + "priority": 907, + "name": "Harness session pin: Anthropic first, Vertex fallback on another model", + "description": "Provider harness only (folder 'session-affinity'). Anthropic on the incoming model, with Vertex as a fallback on a different model. A session bound to Vertex on the incoming model must not be moved onto that entry: a binding names provider and model together, so the routing decision stands and the session rebinds on Anthropic. Matches nothing without x-harness-session-pin: anthropic-first-other-model.", + "cel_expression": "\"x-harness-session-pin\" in headers && headers[\"x-harness-session-pin\"] == \"anthropic-first-other-model\"", + "targets": [ + { + "provider": "anthropic", + "weight": 1 + } + ], + "fallbacks": [ + { + "provider": "vertex", + "model": "claude-opus-4-8" + } + ] + }, + { + "id": "harness-session-pin-vertex-badkey", + "priority": 908, + "name": "Harness session pin: Vertex pinned to a key that cannot serve the model", + "description": "Provider harness only (folder 'session-affinity'). Vertex on the incoming model, pinned to the Gemini-only key and with no fallback, so the request fails at key selection with 'no supported key found'. A session bound to Vertex on that model follows the binding into the failure, and must forget it. Matches nothing without x-harness-session-pin: vertex-badkey.", + "cel_expression": "\"x-harness-session-pin\" in headers && headers[\"x-harness-session-pin\"] == \"vertex-badkey\"", + "targets": [ + { + "provider": "vertex", + "provider_key_name": "Vertex Global (Gemini 3)", + "weight": 1 + } + ] + } + ], "virtual_keys": [ { "name": "Test Key", @@ -775,6 +1058,12 @@ "key_ids": ["*"], "weight": 1.0 }, + { + "provider": "typesafe", + "allowed_models": ["*"], + "key_ids": ["*"], + "weight": 1.0 + }, { "provider": "elevenlabs", "allowed_models": ["*"], diff --git a/tests/integrations/python/pyproject.toml b/tests/integrations/python/pyproject.toml index 578cde3d1ff..9a09a98c367 100644 --- a/tests/integrations/python/pyproject.toml +++ b/tests/integrations/python/pyproject.toml @@ -51,6 +51,10 @@ dependencies = [ "pytest-xdist>=3.8.0", "pyasn1>=0.6.4", "griffe>=2.3.0", + # typesafe-sdk is newer than exclude-newer above by design (no version + # existed at the cutoff); pinned exact and grafted into uv.lock so no + # other package re-resolves. Its deps are already satisfied by the lock. + "typesafe-sdk==0.7.0", ] [build-system] @@ -146,5 +150,8 @@ exclude_lines = [ # above would otherwise make uv treat it as an editable package, which changes the # root's entry in uv.lock and breaks `uv sync --frozen`. package = false -# Include HTTPX2's security fixes released after the previous cutoff. +# typesafe-sdk (the exception above) is grafted into uv.lock; every other +# package stays at its pre-existing version, so this cutoff keeps the 7-day +# minimum for them. Integration tests run with --frozen so the lock is used +# as-is without re-resolving typesafe past this cutoff. exclude-newer = "2026-09-09T00:00:00Z" diff --git a/tests/integrations/python/tests/test_typesafe.py b/tests/integrations/python/tests/test_typesafe.py new file mode 100644 index 00000000000..749fc5e224c --- /dev/null +++ b/tests/integrations/python/tests/test_typesafe.py @@ -0,0 +1,637 @@ +""" +Typesafe Integration Tests - Official SDK against Bifrost + +🌉 SDK DROP-IN TESTING: +This test suite uses the official TypeSafe Python SDK (typesafe-sdk) pointed at +Bifrost's /typesafe prefix via base_url, so a client written against +api.typesafe.ai must work unchanged through Bifrost. Every call in this file +goes through the SDK - no raw HTTP. + +Covered scenarios: +1. system_one with all three question types (Noul, Choice, Score) +2. Structured (JSON) state and per-answer metadata (confidence, probabilities, legend) +3. Model alias resolution (jev-latest resolves to the versioned model) +4. client.models.list() against Bifrost's synthesized native listing +5. SDK exception parsing of Bifrost's native error body (TypeSafeBadRequestError) +""" + +import json +import math +import os +import time + +import pytest +import requests +from typesafe_sdk import ( + Choice, + Noul, + Score, + TypeSafeBadRequestError, + TypeSafeClient, +) + +from .utils.common import get_bifrost_base_url +from .utils.config_loader import get_config + +STATE = ( + "Customer message: I was double charged last month and nobody replied " + "to my two emails. I want a refund today or I am cancelling." +) + + +@pytest.fixture +def typesafe_client(): + """Official TypeSafe SDK client pointed at Bifrost's /typesafe drop-in. + + Authenticates to Bifrost with the suite's virtual key via the x-bf-vk + header (the cross-provider convention); Bifrost injects the real upstream + key, so the SDK's own api_key never reaches TypeSafe. + """ + config = get_config() + headers = {} + vk = config.get_virtual_key() if config.is_virtual_key_configured() else "" + if vk: + headers["x-bf-vk"] = vk + client = TypeSafeClient( + base_url=f"{get_bifrost_base_url()}/typesafe", + api_key=vk or "dummy-key-bifrost-injects-the-real-one", + headers=headers or None, + ) + yield client + client.close() + + +class TestTypesafeSystemOne: + def test_01_all_question_types(self, typesafe_client): + result = typesafe_client.system_one( + STATE, + { + "is_frustrated": Noul(instructions="Is the customer frustrated?"), + "category": Choice( + instructions="Pick the ticket category", + criteria={"billing": "charges and refunds", "bug": "product defects", "other": "anything else"}, + ), + "urgency": Score( + instructions="Rate how urgently this needs a human reply", + criteria=["can wait a week", "should be answered soon", "needs a reply today"], + ), + }, + model="jev-1.13.0", + ) + + assert result.model == "jev-1.13.0" + assert 0.0 <= result.nouls["is_frustrated"].noul <= 1.0 + assert result.choices["category"].choice in {"billing", "bug", "other"} + assert isinstance(result.scores["urgency"].score, (int, float)) + assert result.usage.input_tokens > 0 + + def test_02_structured_state_and_answer_metadata(self, typesafe_client): + result = typesafe_client.system_one( + {"ticket": {"id": 4211, "body": "The export button crashes the app every time."}, "user_tier": "pro"}, + { + "area": Choice( + instructions="Which product area does the complaint target?", + criteria={"camera": "capture", "stability": "crashes", "support": "service"}, + ), + "priority": Score( + instructions="Bug backlog rank?", + criteria=["backlog", "next sprint", "this sprint", "hotfix now"], + ), + }, + model="jev-1.13.0", + ) + + area = result.choices["area"] + assert area.choice in {"camera", "stability", "support"} + assert area.probabilities is not None and abs(sum(area.probabilities.values()) - 1.0) < 0.05 + priority = result.scores["priority"] + assert priority.legend is not None and len(priority.legend) == 4 + + def test_03_model_alias_resolves(self, typesafe_client): + result = typesafe_client.system_one( + "Reply: Sure, sounds good, see you at 3pm.", + {"is_confirmation": Noul(instructions="Does this reply confirm the meeting?")}, + model="jev-latest", + ) + # Aliases resolve upstream; the response reports the versioned model. + assert result.model.startswith("jev-") + assert result.model != "jev-latest" + + def test_04_structured_criteria_type_matrix(self, typesafe_client): + # The API types criteria descriptions as string | object | array for + # noul keys and score levels, plus null for choice options. One call + # covers every allowed type in every slot; Bifrost must forward all of + # them losslessly instead of rejecting non-string descriptions. + result = typesafe_client.system_one( + STATE, + { + "noul_obj_arr": Noul( + instructions="Is the customer frustrated?", + criteria={ + "true": {"meaning": "clearly upset", "signals": ["threats", "caps"]}, + "false": ["calm", "neutral tone"], + }, + ), + "noul_arr_obj": Noul( + instructions="Does the customer ask for a refund?", + criteria={ + "true": ["asks for money back", "mentions refund"], + "false": {"meaning": "no refund language"}, + }, + ), + "noul_str": Noul( + instructions="Does the customer threaten to cancel?", + criteria={"true": "cancellation is threatened", "false": "no cancellation language"}, + ), + "category": Choice( + instructions="Pick the ticket category", + criteria={ + "billing": {"rubric": "charges and refunds", "examples": ["double charge"]}, + "bug": ["crash", "product defect"], + "support": "service questions", + "other": None, + }, + ), + "urgency": Score( + instructions="Rate how urgently this needs a human reply", + criteria=[ + "can wait a week", + {"level": "should be answered soon"}, + ["needs a reply today", "churn risk"], + ], + ), + }, + model="jev-1.13.0", + ) + + for name in ("noul_obj_arr", "noul_arr_obj", "noul_str"): + assert 0.0 <= result.nouls[name].noul <= 1.0 + assert result.choices["category"].choice in {"billing", "bug", "support", "other"} + urgency = result.scores["urgency"] + assert isinstance(urgency.score, (int, float)) + # The legend echoes each level's description verbatim - structured + # levels come back as objects/arrays, not stringified. + assert urgency.legend is not None and len(urgency.legend) == 3 + assert urgency.legend[0] == "can wait a week" + assert urgency.legend[1] == {"level": "should be answered soon"} + assert urgency.legend[2] == ["needs a reply today", "churn risk"] + + +class TestTypesafeModels: + def test_01_models_list(self, typesafe_client): + listing = typesafe_client.models.list() + names = [m.name for m in listing.models] + assert "jev-1.13.0" in names + assert "jev-latest" in names + assert all("typesafe/" not in name for name in names) + + +class TestTypesafeErrors: + def test_01_bad_request_parses_native_error(self, typesafe_client): + # A choice question with empty criteria is rejected by Bifrost before + # dispatch; the SDK must parse the native {"detail": {...}} body into + # its 400 exception type exactly as it would against api.typesafe.ai. + with pytest.raises(TypeSafeBadRequestError) as excinfo: + typesafe_client.system_one( + STATE, + {"category": Choice(instructions="Pick one", criteria={})}, + model="jev-1.13.0", + ) + assert "criteria" in str(excinfo.value) + + +# LLM fallback / emulation: any tool-capable chat model answers a decision +# request natively (primary) or as a fallback. Exercised through the official +# TypeSafe SDK pointed at an LLM model - the /typesafe drop-in stays the client. +LLM_DECISION_MODEL = "openai/gpt-4o-mini" + + +class TestDecisionLLMEmulation: + def test_01_llm_primary_emulation(self, typesafe_client): + result = typesafe_client.system_one( + STATE, + { + "is_frustrated": Noul(instructions="Is the customer frustrated?"), + "category": Choice( + instructions="Pick the ticket category", + criteria={"billing": "charges and refunds", "bug": "product defects", "other": "anything else"}, + ), + "urgency": Score( + instructions="Rate urgency", + criteria=["low", "medium", "high"], + ), + }, + model=LLM_DECISION_MODEL, + ) + # An LLM emulates the judgment via tool-calling; answers come back in the + # native shape with the LLM-estimated confidence. + assert 0.0 <= result.nouls["is_frustrated"].noul <= 1.0 + assert result.choices["category"].choice in {"billing", "bug", "other"} + assert isinstance(result.scores["urgency"].score, (int, float)) + + def test_02_llm_fallback(self): + # Primary model fails at the provider; the request falls through to the + # LLM fallback which emulates the decision. /v1/decisions is Bifrost's + # native format (no upstream SDK), so this uses the HTTP API directly. + config = get_config() + headers = {"Content-Type": "application/json"} + if config.is_virtual_key_configured(): + headers["x-bf-vk"] = config.get_virtual_key() + response = requests.post( + f"{get_bifrost_base_url()}/v1/decisions", + headers=headers, + json={ + "model": "openai/nonexistent-model-xyz", + "fallbacks": [LLM_DECISION_MODEL], + "state": "Reply: sounds good, see you at 3pm.", + "questions": {"is_confirmation": {"kind": "noul", "instructions": "Does this confirm the meeting?"}}, + }, + timeout=60, + ) + assert response.status_code == 200, response.text + answer = response.json()["answers"]["is_confirmation"] + assert answer["kind"] == "noul" + assert 0.0 <= answer["value"] <= 1.0 + + def test_03_native_vs_emulated_comparison(self, typesafe_client): + # Same decision answered by the native jev judgment model and by an LLM + # emulation; report how the two differ. Both must return well-formed + # answers of the right kinds - the values themselves may differ, which is + # the point of the comparison. + questions = { + "is_frustrated": Noul(instructions="Is the customer frustrated?"), + "category": Choice( + instructions="Pick the ticket category", + criteria={"billing": "charges and refunds", "bug": "product defects", "other": "anything else"}, + ), + "urgency": Score(instructions="Rate urgency", criteria=["low", "medium", "high"]), + } + native = typesafe_client.system_one(STATE, questions, model="jev-1.13.0") + emulated = typesafe_client.system_one(STATE, questions, model=LLM_DECISION_MODEL) + + print("\n=== Decision: native jev vs LLM emulation ===") + print(f"{'question':<14} {'kind':<8} {'jev-1.13.0':<16} {'gpt-4o-mini':<16} match") + for name in questions: + kind = native.answers[name].type + if kind == "noul": + nv, ev = native.nouls[name].noul, emulated.nouls[name].noul + elif kind == "choice": + nv, ev = native.choices[name].choice, emulated.choices[name].choice + else: + nv, ev = native.scores[name].score, emulated.scores[name].score + print(f"{name:<14} {kind:<8} {str(nv):<16} {str(ev):<16} {'=' if nv == ev else 'DIFF'}") + + # Both engines answered every question in the right shape. + for name, q in questions.items(): + assert native.answers[name].type == emulated.answers[name].type + assert native.model.startswith("jev-") + + def test_04_structured_criteria_reach_the_emulating_model(self, typesafe_client): + # The emulation encodes criteria into the forced tool schema. The choice + # option names here are opaque codes; only the criteria descriptions + # (object, string, null, and array shaped) carry meaning, so a correct + # pick on a clearly-billing state proves the rubrics reached the LLM + # rather than being dropped or rejected as non-string. + result = typesafe_client.system_one( + STATE, + { + "approve_review": Noul( + instructions="Should this ticket get a billing review?", + criteria={ + "true": "the customer reports a concrete billing error", + "false": ["no billing problem is described", "general inquiry"], + }, + ), + "pick": Choice( + instructions=( + "Pick the ticket category. The option names are opaque codes; " + "use the criteria descriptions to decide." + ), + criteria={ + "opt_a": {"rubric": "charges and refunds"}, + "opt_b": "product defects", + "opt_c": None, + "opt_d": ["installation help", "how-to setup questions"], + }, + ), + "urgency": Score( + instructions="Rate how urgently this needs a human reply", + criteria=[ + "can wait a week", + {"level": "should be answered soon"}, + ["needs a reply today", "churn risk"], + ], + ), + }, + model=LLM_DECISION_MODEL, + ) + + assert result.nouls["approve_review"].noul >= 0.5 + pick = result.choices["pick"] + assert pick.choice == "opt_a" + # Probabilities are required on native choice and score answers, so the + # emulation must always deliver them. + assert pick.probabilities is not None and abs(sum(pick.probabilities.values()) - 1.0) < 0.05 + urgency = result.scores["urgency"] + assert isinstance(urgency.score, (int, float)) + assert urgency.probabilities is not None and abs(sum(urgency.probabilities.values()) - 1.0) < 0.05 + # The emulated legend must match the native shape: each level's + # description echoed verbatim (string, object, or array). + assert urgency.legend is not None and len(urgency.legend) == 3 + assert urgency.legend[1] == {"level": "should be answered soon"} + assert urgency.legend[2] == ["needs a reply today", "churn risk"] + + +# Cross-provider emulation matrix: every VK-allowed chat provider must answer a +# decision request through an LLM, for at least four current models each. The model +# list is the shared source of truth in +# tests/integrations/decision_emulation_matrix.json, read by both the Python and +# TypeScript suites and by the provider-harness generator. +# +# jev (the native typesafe judgment model) is the ground truth. Each round we take +# jev's decision on a deliberately unambiguous state, then require every reachable +# model to reach the SAME verdict: the identical choice, the same binary noul +# (rounded), and the same score level (rounded). A model that disagrees FAILS - the +# matrix exists to prove emulation matches jev, not merely that it returns a shape. +# +# Only genuine availability - a model this account cannot reach (auth, governance +# block, not-found / retired / deployment gap, rate limit, upstream connection) - +# is SKIPPED. A 400 (a request we built wrong), a 500 (emulation returned no +# decision), or a thrown request error (timeout, reset, connection failure) is +# our problem to see and FAILS. +_MATRIX_PATH = os.path.join(os.path.dirname(__file__), "..", "..", "decision_emulation_matrix.json") + + +def _load_matrix(): + with open(_MATRIX_PATH) as f: + return json.load(f) + + +_MATRIX = _load_matrix() +_MATRIX_PAIRS = [(p, m) for p, models in _MATRIX["providers"].items() for m in models] +# One row per combination (jev included): model_id, status, agreement score, +# client-measured latency, token total, and cost. Drives the final report table. +_MATRIX_ROWS = [] + +# Non-200 codes that mean the model is unreachable on this account, not that our +# request or emulation is wrong. These SKIP; everything else FAILS. A provider that +# rejects a large multi-question tool schema is NOT skipped - the emulation now +# falls back to answering one question at a time (see emulateDecisionViaResponses), +# so Perplexity and similar answer the full set instead of failing. +_AVAILABILITY_CODES = {401, 403, 404, 429, 502, 503, 504} + +# Some providers report an unavailable/renamed/deprecated MODEL as a 400 rather +# than a 404. That is availability, not a request we built wrong, so a 400 whose +# message matches one of these SKIPs; every other 400 (a genuinely malformed +# request) and every 500 (emulation returned no decision) still FAILS. +_AVAILABILITY_MESSAGE_MARKERS = ( + "deprecated", + "no longer available", + "not supported", + "not found", + "does not exist", + "invalid_model", + "model_not_found", + "not allowed for virtual key", +) + + +def _is_availability(status_code, text): + if status_code in _AVAILABILITY_CODES: + return True + if status_code == 400: + low = text.lower() + return any(marker in low for marker in _AVAILABILITY_MESSAGE_MARKERS) + return False + +# One rich decision (30 questions) is the ground-truth scenario; jev answers it and +# every reachable model must agree with jev on at least this fraction of questions. +_SCENARIO = _MATRIX["scenario"] +_MATRIX_STATE = _SCENARIO["state"] +_MATRIX_QUESTIONS = _SCENARIO["questions"] +_AGREEMENT_FAIL_THRESHOLD = _MATRIX.get("_agreement_fail_threshold", 0.7) +# Providers that reject a large multi-question decision tool schema. We do NOT split +# these up front: the batched call is tried first, and only if it fails with a +# non-availability error (Perplexity returns a bare "invalid request" when the +# 30-question schema is too large) do we retry one question at a time and merge. +_SPLIT_PROVIDERS = set(_MATRIX.get("_split_question_providers", [])) +_JEV_MODEL = "typesafe/jev-1.13.0" + + +# The provider-side request timeout is up to 300s and the split path makes up +# to 30 sequential calls, so every matrix fetch carries one shared deadline: +# the whole _fetch_decision (batched call plus any split) must finish inside +# this budget. Mirrors the TypeScript suite's MATRIX_CALL_BUDGET_MS: a blown +# budget FAILS the case with a readable reason - never a skip, no synthetic +# statuses. +_MATRIX_CALL_BUDGET_S = 85.0 + + +class _BudgetExceededError(Exception): + """Raised when the shared per-model budget runs out; fails the case.""" + + +def _remaining(deadline, model_id): + remaining = deadline - time.monotonic() + if remaining <= 0: + raise _BudgetExceededError(f"client-side budget of {_MATRIX_CALL_BUDGET_S:.0f}s exceeded waiting on {model_id}") + return remaining + + +def _post_raw(model_id, questions, deadline): + """One POST to /v1/decisions for the given question set; returns (response, latency_ms).""" + config = get_config() + headers = {"Content-Type": "application/json"} + if config.is_virtual_key_configured(): + headers["x-bf-vk"] = config.get_virtual_key() + start = time.perf_counter() + response = requests.post( + f"{get_bifrost_base_url()}/v1/decisions", + headers=headers, + json={"model": model_id, "state": _MATRIX_STATE, "questions": questions}, + timeout=min(90.0, _remaining(deadline, model_id)), + ) + return response, int((time.perf_counter() - start) * 1000) + + +def _usage_cost(body): + """(total_tokens, cost) from a decision body; cost is '-' when the config + computes none (this profile has no pricing wired for several providers).""" + usage = body.get("usage") or {} + tokens = usage.get("total_tokens") + cost = usage.get("cost") + if isinstance(cost, dict): + cost = cost.get("total") if cost.get("total") is not None else cost + return tokens, cost if cost is not None else "-" + + +def _fetch_decision(model_id): + """Fetch a decision for the full scenario. Tries the batched 30-question call + first; only for split-eligible providers, and only when that batched call fails + with a non-availability error, retries one question at a time and merges. + Returns a normalized dict: status, answers, latency_ms, tokens, cost, text, split.""" + deadline = time.monotonic() + _MATRIX_CALL_BUDGET_S + resp, latency_ms = _post_raw(model_id, _MATRIX_QUESTIONS, deadline) + if resp.status_code == 200: + body = resp.json() + tokens, cost = _usage_cost(body) + return {"status": 200, "answers": body["answers"], "latency_ms": latency_ms, "tokens": tokens, "cost": cost, "text": "", "split": False} + # Batched call failed. Availability -> report as-is (caller decides skip). + if _is_availability(resp.status_code, resp.text): + return {"status": resp.status_code, "answers": None, "latency_ms": latency_ms, "tokens": None, "cost": "-", "text": resp.text, "split": False} + # Non-availability failure: split-eligible providers retry one question at a time. + provider = model_id.split("/", 1)[0] + if provider in _SPLIT_PROVIDERS and len(_MATRIX_QUESTIONS) > 1: + return _fetch_decision_split(model_id, latency_ms, deadline) + return {"status": resp.status_code, "answers": None, "latency_ms": latency_ms, "tokens": None, "cost": "-", "text": resp.text, "split": False} + + +def _fetch_decision_split(model_id, batched_latency_ms, deadline): + """Answer each question in its own call and merge; latency and tokens accumulate + (the failed batched attempt's latency is included, since it was really spent). + The shared deadline bounds the whole sequential walk.""" + merged = {} + total_latency = batched_latency_ms + total_tokens = 0 + for name, q in _MATRIX_QUESTIONS.items(): + resp, latency_ms = _post_raw(model_id, {name: q}, deadline) + total_latency += latency_ms + if resp.status_code != 200: + return {"status": resp.status_code, "answers": None, "latency_ms": total_latency, "tokens": None, "cost": "-", "text": resp.text, "split": True} + body = resp.json() + merged.update(body["answers"]) + tokens, _ = _usage_cost(body) + if isinstance(tokens, int): + total_tokens += tokens + return {"status": 200, "answers": merged, "latency_ms": total_latency, "tokens": total_tokens or None, "cost": "-", "text": "", "split": True} + + +def _half_up(x): + """Round half away from zero toward positive infinity, matching JS Math.round. + + Python's round() rounds half to even (0.5 -> 0, 2.5 -> 2) while the TS suite + uses Math.round (0.5 -> 1, 2.5 -> 3); the two matrix suites must agree on the + same verdict for identical model output, and half-up is the agreed rule. + """ + return math.floor(x + 0.5) + + +def _valid_numeric(kind, x): + """A genuine number for the kind: bools and numeric strings are wrong types, + and a noul outside [0, 1] violates the contract regardless of rounding.""" + if isinstance(x, bool) or not isinstance(x, (int, float)): + return False + if kind == "noul" and not 0.0 <= x <= 1.0: + return False + return True + + +def _matches(kind, ref, got): + """One question: choice matches exactly, noul/score match on the rounded value.""" + if kind == "choice": + return got == ref + if not _valid_numeric(kind, got): + return False + try: + return _half_up(float(ref)) == _half_up(float(got)) + except (TypeError, ValueError): + return False + + +def _agreement(reference, answers): + """(matched, total, mismatched_names) of answers vs the jev reference across + all 30 questions.""" + total = len(_MATRIX_QUESTIONS) + mismatches = [] + for name, q in _MATRIX_QUESTIONS.items(): + ref = (reference.get(name) or {}).get("value") + got = (answers.get(name) or {}).get("value") + if not _matches(q["kind"], ref, got): + mismatches.append(f"{name}(jev={ref},got={got})") + return total - len(mismatches), total, mismatches + + +@pytest.fixture(scope="module") +def jev_reference(): + # jev is the ground truth for the whole matrix; if it cannot answer, the + # comparison is meaningless, so fail loudly rather than skip. + r = _fetch_decision(_JEV_MODEL) + assert r["status"] == 200, f"jev reference decision failed: HTTP {r['status']}: {r['text']}" + answers = r["answers"] + assert len(answers) == len(_MATRIX_QUESTIONS), f"jev answered {len(answers)}/{len(_MATRIX_QUESTIONS)} questions" + total = len(_MATRIX_QUESTIONS) + _MATRIX_ROWS.append({"model_id": _JEV_MODEL, "status": "REF", "agree": total, "total": total, "latency_ms": r["latency_ms"], "tokens": r["tokens"], "cost": r["cost"], "detail": "ground truth"}) + return answers + + +class TestDecisionEmulationMatrix: + @pytest.mark.parametrize("provider,model", _MATRIX_PAIRS, ids=[f"{p}/{m}" for p, m in _MATRIX_PAIRS]) + def test_provider_model_matches_jev(self, provider, model, jev_reference): + model_id = f"{provider}/{model}" + total = len(_MATRIX_QUESTIONS) + try: + r = _fetch_decision(model_id) + except (requests.RequestException, _BudgetExceededError) as exc: + # A thrown request error - a timeout, reset, or connection failure - + # FAILS the case with its reason; it is never a skip. Mirrors the + # TypeScript suite so both matrices report the same verdict for the + # identical condition. + _MATRIX_ROWS.append({"model_id": model_id, "status": "FAIL", "agree": 0, "total": total, "latency_ms": None, "tokens": None, "cost": "-", "detail": f"request error: {exc}"}) + pytest.fail(f"{model_id}: request error: {exc}") + + latency_ms = r["latency_ms"] + if r["status"] != 200: + detail = f"HTTP {r['status']}: {r['text'][:160]}" + if _is_availability(r["status"], r["text"]): + _MATRIX_ROWS.append({"model_id": model_id, "status": "SKIP", "agree": 0, "total": total, "latency_ms": latency_ms, "tokens": None, "cost": "-", "detail": detail}) + pytest.skip(f"{model_id}: {detail}") + # Capability-limited providers (e.g. Perplexity's sonar rejects choice/score + # decision schemas): the split was attempted and could not complete the set, + # so skip rather than fail - the identical schema works on every other provider. + if provider in _SPLIT_PROVIDERS: + _MATRIX_ROWS.append({"model_id": model_id, "status": "SKIP", "agree": 0, "total": total, "latency_ms": latency_ms, "tokens": None, "cost": "-", "detail": "capability-limited - " + detail}) + pytest.skip(f"{model_id}: capability-limited - {detail}") + # A malformed request we built (400) or emulation returning no decision (500): defect. + _MATRIX_ROWS.append({"model_id": model_id, "status": "FAIL", "agree": 0, "total": total, "latency_ms": latency_ms, "tokens": None, "cost": "-", "detail": detail}) + pytest.fail(f"{model_id}: {detail}") + + answers = r["answers"] + tokens, cost = r["tokens"], r["cost"] + # Emulation must answer every question, each with the right kind discriminator + # and a numeric confidence (a malformed response must not pass). + assert len(answers) == total, f"{model_id} answered {len(answers)}/{total} questions: {answers}" + for name, q in _MATRIX_QUESTIONS.items(): + assert answers[name].get("kind") == q["kind"], f"{name} kind = {answers[name].get('kind')}, expected {q['kind']}: {answers}" + assert isinstance(answers[name].get("confidence"), (int, float)), f"{name} confidence not numeric: {answers}" + + matched, total, mismatches = _agreement(jev_reference, answers) + ratio = matched / total + row = {"model_id": model_id, "status": "PASS", "agree": matched, "total": total, "latency_ms": latency_ms, "tokens": tokens, "cost": cost, "detail": f"{matched}/{total} vs jev"} + if ratio < _AGREEMENT_FAIL_THRESHOLD: + row["status"] = "FAIL" + row["detail"] = f"{matched}/{total} < {_AGREEMENT_FAIL_THRESHOLD:.0%}; diffs: " + ", ".join(mismatches[:8]) + _MATRIX_ROWS.append(row) + pytest.fail(f"{model_id} agrees with jev on only {matched}/{total}: {', '.join(mismatches)}") + _MATRIX_ROWS.append(row) + + def test_zzz_report(self): + # Runs after the parametrized cases (alphabetical order within the class) + # to print the overall table: one row per combination (jev included) with + # its agreement score vs jev over the 30-question scenario, latency, and cost. + def fmt(v): + return "-" if v is None else (f"{v:.2f}" if isinstance(v, float) else str(v)) + + header = f"{'combination':<50} {'score':<8} {'latency_ms':<11} {'tokens':<7} {'cost':<8} status" + print(f"\n=== Decision emulation matrix vs jev (1 scenario, {len(_MATRIX_QUESTIONS)} questions) ===") + print(header) + print("-" * len(header)) + # jev pinned first as the reference, then combinations sorted by name. + rows = sorted(_MATRIX_ROWS, key=lambda r: (r["status"] != "REF", r["model_id"])) + for r in rows: + score = f"{r['agree']}/{r['total']}" + print(f"{r['model_id']:<50} {score:<8} {fmt(r['latency_ms']):<11} {fmt(r['tokens']):<7} {str(r['cost']):<8} {r['status']}") + passed = [r for r in _MATRIX_ROWS if r["status"] == "PASS"] + failed = [r for r in _MATRIX_ROWS if r["status"] == "FAIL"] + skipped = [r for r in _MATRIX_ROWS if r["status"] == "SKIP"] + print(f"\nscore = questions agreeing with jev out of {len(_MATRIX_QUESTIONS)}; PASS threshold = {_AGREEMENT_FAIL_THRESHOLD:.0%}") + print(f"PASS={len(passed)} FAIL={len(failed)} SKIP={len(skipped)} TOTAL={len(_MATRIX_PAIRS)} (+jev reference)") diff --git a/tests/integrations/python/uv.lock b/tests/integrations/python/uv.lock index ad54b405a58..fe4e1d46e26 100644 --- a/tests/integrations/python/uv.lock +++ b/tests/integrations/python/uv.lock @@ -782,6 +782,7 @@ dependencies = [ { name = "requests" }, { name = "tokenizers" }, { name = "types-requests" }, + { name = "typesafe-sdk" }, { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/d2/75/4c346f6e2322e545f8452692304bd4eca15a2a0209ab9af6a0d1a7810b67/cohere-5.21.1.tar.gz", hash = "sha256:e5ade4423b928b01ff2038980e1b62b2a5bb412c8ab83e30882753b810a5509f", size = 191272, upload-time = "2026-03-26T15:09:27.857Z" } @@ -5061,6 +5062,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/62/e7/010c87f559e216d83f9dc51e939633fd0d0ead3377340181ab0e223cd3b5/types_requests-2.33.0.20260712-py3-none-any.whl", hash = "sha256:de027e28c171d3da529689cbfa023b0b4eab188c8dfa22fd834eebd2cee6e7bb", size = 21392, upload-time = "2026-07-12T05:14:19.616Z" }, ] +[[package]] +name = "typesafe-sdk" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "httpx2" }, + { name = "pydantic" }, + { name = "pydantic-core" }, + { name = "tenacity" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/28/e2/ac317772d4d5cfabf3cecdeacd83cc838523e240bb4e1cf256837e6005bb/typesafe_sdk-0.7.0.tar.gz", hash = "sha256:930d42fd73cfed6f25bccae488ce0f30f6043eaedd8682ef59734e79ab94a876", size = 22235 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d3/2a/16f4163e6d8827ef80b2ed7a400c80688e1ebf571c10f143f792bdafc654/typesafe_sdk-0.7.0-py3-none-any.whl", hash = "sha256:c6d2257c4b04b8d4d81cff2b55489b4188795fd44fb40b9cef26ff243b2d1080", size = 35352 }, +] + [[package]] name = "typing-extensions" version = "4.16.0" diff --git a/tests/integrations/typescript/config.json b/tests/integrations/typescript/config.json index 29a13b032da..ac779fbac1f 100644 --- a/tests/integrations/typescript/config.json +++ b/tests/integrations/typescript/config.json @@ -15,6 +15,20 @@ "default_request_timeout_in_seconds": 300 } }, + "typesafe": { + "keys": [ + { + "name": "Typesafe API Key", + "value": "env.TYPESAFE_API_KEY", + "weight": 1, + "models": ["*"], + "use_for_batch_api": false + } + ], + "network_config": { + "default_request_timeout_in_seconds": 300 + } + }, "anthropic": { "keys": [ { @@ -205,6 +219,7 @@ "is_active": true, "provider_configs": [ { "provider": "openai", "allowed_models": ["*"], "key_ids": ["*"], "weight": 1.0 }, + { "provider": "typesafe", "allowed_models": ["*"], "key_ids": ["*"], "weight": 1.0 }, { "provider": "anthropic", "allowed_models": ["*"], "key_ids": ["*"], "weight": 1.0 }, { "provider": "gemini", "allowed_models": ["*"], "key_ids": ["*"], "weight": 1.0 }, { "provider": "vertex", "allowed_models": ["*"], "key_ids": ["*"], "weight": 1.0 }, diff --git a/tests/integrations/typescript/package-lock.json b/tests/integrations/typescript/package-lock.json index 561246239c9..d37ce533d68 100644 --- a/tests/integrations/typescript/package-lock.json +++ b/tests/integrations/typescript/package-lock.json @@ -17,6 +17,7 @@ "@langchain/google-genai": "2.1.26", "@langchain/openai": "1.4.4", "@smithy/node-http-handler": "4.7.4", + "@typesafe-ai/sdk": "^0.6.0", "langsmith": "0.6.0", "openai": "6.15.0", "yaml": "2.9.0", @@ -2440,6 +2441,15 @@ "undici-types": "~6.20.0" } }, + "node_modules/@typesafe-ai/sdk": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/@typesafe-ai/sdk/-/sdk-0.6.0.tgz", + "integrity": "sha512-IddX+Q0XM+VagOUZFeP7wZjaO4SHMdvnh2zEBdrZZnXedWI3BNK1lKhMx3ayrkFWvVLbVcUHJy6AVZlY+e6Jaw==", + "license": "MIT", + "engines": { + "node": ">=20" + } + }, "node_modules/@typescript-eslint/eslint-plugin": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.0.0.tgz", diff --git a/tests/integrations/typescript/package.json b/tests/integrations/typescript/package.json index 39c53a59471..dd369e9fb1c 100644 --- a/tests/integrations/typescript/package.json +++ b/tests/integrations/typescript/package.json @@ -21,6 +21,7 @@ "@langchain/google-genai": "2.1.26", "@langchain/openai": "1.4.4", "@smithy/node-http-handler": "4.7.4", + "@typesafe-ai/sdk": "0.6.0", "langsmith": "0.6.0", "openai": "6.15.0", "yaml": "2.9.0", @@ -41,14 +42,14 @@ "ast-v8-to-istanbul": "1.0.0", "brace-expansion@1": "1.1.18", "brace-expansion@2": "2.1.4", + "esbuild": "0.28.1", "js-yaml": "4.3.2", "nanoid": "3.3.18", "obug": "2.1.4", "rollup": "4.60.4", - "esbuild": "0.28.1", "vite": "6.4.3" }, "engines": { "node": ">=22.0.0" } -} +} \ No newline at end of file diff --git a/tests/integrations/typescript/tests/test-typesafe.test.ts b/tests/integrations/typescript/tests/test-typesafe.test.ts new file mode 100644 index 00000000000..d48892e1441 --- /dev/null +++ b/tests/integrations/typescript/tests/test-typesafe.test.ts @@ -0,0 +1,553 @@ +/** + * Typesafe Integration Tests - Official SDK against Bifrost + * + * 🌉 SDK DROP-IN TESTING: + * Uses the official TypeSafe JavaScript SDK (@typesafe-ai/sdk) pointed at + * Bifrost's /typesafe prefix via baseURL, authenticating to Bifrost with the + * suite's virtual key via the x-bf-vk header. Every call goes through the SDK. + */ + +import { readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { BadRequestError, choice, noul, score, TypeSafeClient } from "@typesafe-ai/sdk"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { getVirtualKey, isVirtualKeyConfigured } from "../src/utils/config-loader"; + +const STATE = + "Customer message: I was double charged last month and nobody replied to my two emails. I want a refund today or I am cancelling."; + +let client: TypeSafeClient; + +beforeAll(() => { + const baseUrl = process.env.BIFROST_BASE_URL || "http://localhost:8080"; + const vk = isVirtualKeyConfigured() ? getVirtualKey() : ""; + client = new TypeSafeClient({ + baseURL: `${baseUrl}/typesafe`, + apiKey: vk || "dummy-key-bifrost-injects-the-real-one", + defaultHeaders: vk ? { "x-bf-vk": vk } : undefined, + }); +}); + +describe("Typesafe SDK systemOne", () => { + it("answers all three question types", async () => { + const response = await client.systemOne({ + state: STATE, + model: "jev-1.13.0", + questions: { + is_frustrated: noul("Is the customer frustrated?"), + category: choice("Pick the ticket category", { + billing: "charges and refunds", + bug: "product defects", + other: "anything else", + }), + urgency: score("Rate how urgently this needs a human reply", [ + "can wait a week", + "should be answered soon", + "needs a reply today", + ]), + }, + }); + + expect(response.model).toBe("jev-1.13.0"); + expect(response.answers.is_frustrated.noul).toBeGreaterThanOrEqual(0); + expect(response.answers.is_frustrated.noul).toBeLessThanOrEqual(1); + expect(["billing", "bug", "other"]).toContain(response.answers.category.choice); + expect(typeof response.answers.urgency.score).toBe("number"); + expect(response.usage.input_tokens).toBeGreaterThan(0); + }, 60000); + + it("resolves a model alias to its versioned id", async () => { + const response = await client.systemOne({ + state: "Reply: Sure, sounds good, see you at 3pm.", + model: "jev-latest", + questions: { is_confirmation: noul("Does this reply confirm the meeting?") }, + }); + expect(response.model).toMatch(/^jev-/); + expect(response.model).not.toBe("jev-latest"); + }, 60000); + + it("forwards structured criteria of every allowed type", async () => { + // The API types criteria descriptions as string | object | array for + // noul keys and score levels, plus null for choice options. One call + // covers every allowed type in every slot; Bifrost must forward all of + // them losslessly instead of rejecting non-string descriptions. + const response = await client.systemOne({ + state: STATE, + model: "jev-1.13.0", + questions: { + noul_obj_arr: noul("Is the customer frustrated?", { + true: { meaning: "clearly upset", signals: ["threats", "caps"] }, + false: ["calm", "neutral tone"], + }), + noul_arr_obj: noul("Does the customer ask for a refund?", { + true: ["asks for money back", "mentions refund"], + false: { meaning: "no refund language" }, + }), + noul_str: noul("Does the customer threaten to cancel?", { + true: "cancellation is threatened", + false: "no cancellation language", + }), + category: choice("Pick the ticket category", { + billing: { rubric: "charges and refunds", examples: ["double charge"] }, + bug: ["crash", "product defect"], + support: "service questions", + other: null, + }), + urgency: score("Rate how urgently this needs a human reply", [ + "can wait a week", + { level: "should be answered soon" }, + ["needs a reply today", "churn risk"], + ]), + }, + }); + + for (const name of ["noul_obj_arr", "noul_arr_obj", "noul_str"] as const) { + expect(response.answers[name].noul).toBeGreaterThanOrEqual(0); + expect(response.answers[name].noul).toBeLessThanOrEqual(1); + } + expect(["billing", "bug", "support", "other"]).toContain(response.answers.category.choice); + expect(typeof response.answers.urgency.score).toBe("number"); + // The legend echoes each level's description verbatim - structured + // levels come back as objects/arrays, not stringified. + const legend = response.answers.urgency.legend as Record; + expect(Object.keys(legend ?? {})).toHaveLength(3); + expect(legend[0]).toBe("can wait a week"); + expect(legend[1]).toEqual({ level: "should be answered soon" }); + expect(legend[2]).toEqual(["needs a reply today", "churn risk"]); + }, 60000); +}); + +describe("Typesafe SDK models", () => { + it("lists jev models with bare names", async () => { + // The JS SDK returns the model array directly (unlike the Python SDK's + // wrapper object). + const listing = await client.models.list(); + const names = listing.map((m) => m.name); + expect(names).toContain("jev-1.13.0"); + expect(names).toContain("jev-latest"); + for (const name of names) expect(name).not.toContain("typesafe/"); + }, 30000); +}); + +describe("Typesafe SDK errors", () => { + it("parses Bifrost's native error body into a BadRequestError", async () => { + // A choice question with an empty criteria map is rejected before dispatch; + // the SDK must parse the native {"detail": {...}} body into its 400 type, + // exactly as it would against api.typesafe.ai. + await expect( + client.systemOne({ + state: STATE, + model: "jev-1.13.0", + questions: { category: choice("Pick one", {}) }, + }), + ).rejects.toBeInstanceOf(BadRequestError); + }, 30000); +}); + +// LLM fallback / emulation: any tool-capable chat model answers a decision +// request natively (primary) or as a fallback, exercised through the SDK +// (native endpoint) and the Bifrost-native /v1/decisions API (fallback). +const LLM_DECISION_MODEL = "openai/gpt-4o-mini"; + +describe("Decision LLM emulation", () => { + it("an LLM emulates the decision as the primary model", async () => { + const response = await client.systemOne({ + state: STATE, + model: LLM_DECISION_MODEL, + questions: { + is_frustrated: noul("Is the customer frustrated?"), + category: choice("Pick the ticket category", { + billing: "charges and refunds", + bug: "product defects", + other: "anything else", + }), + urgency: score("Rate urgency", ["low", "medium", "high"]), + }, + }); + expect(response.answers.is_frustrated.noul).toBeGreaterThanOrEqual(0); + expect(response.answers.is_frustrated.noul).toBeLessThanOrEqual(1); + expect(["billing", "bug", "other"]).toContain(response.answers.category.choice); + expect(typeof response.answers.urgency.score).toBe("number"); + }, 60000); + + it("falls through to an LLM fallback when the primary fails", async () => { + // /v1/decisions is Bifrost's native format (no upstream SDK) - HTTP directly. + const baseUrl = process.env.BIFROST_BASE_URL || "http://localhost:8080"; + const headers: Record = { "Content-Type": "application/json" }; + if (isVirtualKeyConfigured()) headers["x-bf-vk"] = getVirtualKey(); + const response = await fetch(`${baseUrl}/v1/decisions`, { + method: "POST", + headers, + body: JSON.stringify({ + model: "openai/nonexistent-model-xyz", + fallbacks: [LLM_DECISION_MODEL], + state: "Reply: sounds good, see you at 3pm.", + questions: { is_confirmation: { kind: "noul", instructions: "Does this confirm the meeting?" } }, + }), + }); + const body = (await response.json()) as any; + expect(response.status, JSON.stringify(body)).toBe(200); + expect(body.answers.is_confirmation.kind).toBe("noul"); + expect(body.answers.is_confirmation.value).toBeGreaterThanOrEqual(0); + expect(body.answers.is_confirmation.value).toBeLessThanOrEqual(1); + }, 60000); + + it("reports the difference between native jev and LLM emulation", async () => { + const questions = { + is_frustrated: noul("Is the customer frustrated?"), + category: choice("Pick the ticket category", { + billing: "charges and refunds", + bug: "product defects", + other: "anything else", + }), + urgency: score("Rate urgency", ["low", "medium", "high"]), + }; + const native = await client.systemOne({ state: STATE, model: "jev-1.13.0", questions }); + const emulated = await client.systemOne({ state: STATE, model: LLM_DECISION_MODEL, questions }); + + const read = (r: any, name: string) => { + const k = r.answers[name].type; + return k === "noul" ? r.answers[name].noul : k === "choice" ? r.answers[name].choice : r.answers[name].score; + }; + console.log("\n=== Decision: native jev vs LLM emulation ==="); + const nativeAnswers = native.answers as Record; + const emulatedAnswers = emulated.answers as Record; + for (const name of Object.keys(questions)) { + const nv = read(native, name); + const ev = read(emulated, name); + console.log(`${name}: jev=${nv} gpt-4o-mini=${ev} ${nv === ev ? "=" : "DIFF"}`); + expect(nativeAnswers[name].type).toBe(emulatedAnswers[name].type); + } + expect(native.model).toMatch(/^jev-/); + expect(emulated.model).not.toBe(native.model); + }, 60000); + + it("forwards structured criteria to the emulating model (opaque option codes)", async () => { + // The emulation encodes criteria into the forced tool schema. The choice + // option names here are opaque codes; only the criteria descriptions + // (object, string, null, and array shaped) carry meaning, so a correct + // pick on a clearly-billing state proves the rubrics reached the LLM + // rather than being dropped or rejected as non-string. + const response = await client.systemOne({ + state: STATE, + model: LLM_DECISION_MODEL, + questions: { + approve_review: noul("Should this ticket get a billing review?", { + true: "the customer reports a concrete billing error", + false: ["no billing problem is described", "general inquiry"], + }), + pick: choice( + "Pick the ticket category. The option names are opaque codes; use the criteria descriptions to decide.", + { + opt_a: { rubric: "charges and refunds" }, + opt_b: "product defects", + opt_c: null, + opt_d: ["installation help", "how-to setup questions"], + }, + ), + urgency: score("Rate how urgently this needs a human reply", [ + "can wait a week", + { level: "should be answered soon" }, + ["needs a reply today", "churn risk"], + ]), + }, + }); + + expect(response.answers.approve_review.noul).toBeGreaterThanOrEqual(0.5); + expect(response.answers.pick.choice).toBe("opt_a"); + // Probabilities are required on native choice and score answers, so the + // emulation must always deliver them. + const pickProbs = Object.values(response.answers.pick.probabilities ?? {}); + expect(Math.abs(pickProbs.reduce((a, b) => a + b, 0) - 1)).toBeLessThan(0.05); + const urgencyProbs = Object.values(response.answers.urgency.probabilities ?? {}); + expect(Math.abs(urgencyProbs.reduce((a, b) => a + b, 0) - 1)).toBeLessThan(0.05); + expect(typeof response.answers.urgency.score).toBe("number"); + // The emulated legend must match the native shape: each level's + // description echoed verbatim (string, object, or array). + const legend = response.answers.urgency.legend as Record; + expect(Object.keys(legend ?? {})).toHaveLength(3); + expect(legend[1]).toEqual({ level: "should be answered soon" }); + expect(legend[2]).toEqual(["needs a reply today", "churn risk"]); + }, 60000); +}); + +// Cross-provider emulation matrix: every VK-allowed chat provider answers a +// decision through an LLM, for >=4 current models each. The scenario (one rich +// support ticket, 30 questions) and the model list are the shared source of truth +// in tests/integrations/decision_emulation_matrix.json (read by the Python suite +// and the harness generator too). +// +// jev (the native typesafe judgment model) is the ground truth: it answers the 30 +// questions and every reachable model must agree with jev on at least the +// threshold fraction (choice exact, noul/score on the rounded value). A model +// below threshold FAILS. Only genuine availability (auth, model-block, not-found, +// rate limit, upstream connection) is SKIPPED; a 400 (a request we built wrong) or +// 500 (emulation returned no decision) FAILS. +const MATRIX_PATH = join(dirname(fileURLToPath(import.meta.url)), "..", "..", "decision_emulation_matrix.json"); +const MATRIX = JSON.parse(readFileSync(MATRIX_PATH, "utf-8")) as { + providers: Record; + scenario: { state: string; questions: Record }; + _agreement_fail_threshold?: number; + _split_question_providers?: string[]; +}; +const MATRIX_PAIRS: Array<{ provider: string; model: string }> = Object.entries(MATRIX.providers).flatMap( + ([provider, models]) => models.map((model) => ({ provider, model })), +); +// Codes meaning the model is unreachable on this account, not that our request is +// wrong. A provider that rejects a large multi-question tool schema is NOT skipped: +// the emulation falls back to one question at a time, so it answers the full set. +const AVAILABILITY_CODES = new Set([401, 403, 404, 429, 502, 503, 504]); +// Some providers report an unavailable/renamed/deprecated MODEL as a 400 rather +// than a 404. That is availability, not a request we built wrong. +const AVAILABILITY_MESSAGE_MARKERS = [ + "deprecated", + "no longer available", + "not supported", + "not found", + "does not exist", + "invalid_model", + "model_not_found", + "not allowed for virtual key", +]; + +function isAvailability(status: number, text: string): boolean { + if (AVAILABILITY_CODES.has(status)) return true; + if (status === 400) { + const low = text.toLowerCase(); + return AVAILABILITY_MESSAGE_MARKERS.some((m) => low.includes(m)); + } + return false; +} +const MATRIX_STATE = MATRIX.scenario.state; +const MATRIX_QUESTIONS = MATRIX.scenario.questions; +const QUESTION_COUNT = Object.keys(MATRIX_QUESTIONS).length; +const AGREEMENT_FAIL_THRESHOLD = MATRIX._agreement_fail_threshold ?? 0.7; +// Providers that reject a large multi-question tool schema. The batched call is +// tried first; only if it fails with a non-availability error do these retry one +// question at a time and merge (Perplexity returns a bare "invalid request" for +// the 30-question schema but answers a single question fine). +const SPLIT_PROVIDERS = new Set(MATRIX._split_question_providers ?? []); +const JEV_MODEL = "typesafe/jev-1.13.0"; + +type MatrixRow = { + modelId: string; + status: string; + agree: number; + total: number; + latencyMs: number | null; + tokens: number | null; + cost: string; + detail: string; +}; +const MATRIX_ROWS: MatrixRow[] = []; + +type DecisionResult = { status: number; answers: any; latencyMs: number; tokens: number | null; cost: string; text: string; split: boolean }; + +// The provider-side request timeout is up to 300s, far above the Vitest case +// timeout, so every matrix call carries a shared client-side deadline: the +// whole fetchDecision (batched call plus any sequential split) must finish +// inside this budget, below the 90s case timeout, so a slow in-flight request +// aborts and the SKIP handling still runs instead of Vitest killing the case. +const MATRIX_CALL_BUDGET_MS = 85_000; + +function isAbortError(e: unknown): boolean { + return e instanceof Error && (e.name === "TimeoutError" || e.name === "AbortError"); +} + +async function postRaw( + modelId: string, + questions: unknown, + deadline: number, +): Promise<{ response: Response; latencyMs: number }> { + const baseUrl = process.env.BIFROST_BASE_URL || "http://localhost:8080"; + const headers: Record = { "Content-Type": "application/json" }; + if (isVirtualKeyConfigured()) headers["x-bf-vk"] = getVirtualKey(); + const start = Date.now(); + const response = await fetch(`${baseUrl}/v1/decisions`, { + method: "POST", + headers, + body: JSON.stringify({ model: modelId, state: MATRIX_STATE, questions }), + signal: AbortSignal.timeout(Math.max(1, deadline - Date.now())), + }); + return { response, latencyMs: Date.now() - start }; +} + +function usageCost(body: any): { tokens: number | null; cost: string } { + const usage = body.usage ?? {}; + let cost = usage.cost; + if (cost && typeof cost === "object") cost = cost.total ?? JSON.stringify(cost); + return { tokens: usage.total_tokens ?? null, cost: cost != null ? String(cost) : "-" }; +} + +async function fetchDecision(modelId: string): Promise { + const deadline = Date.now() + MATRIX_CALL_BUDGET_MS; + try { + const { response, latencyMs } = await postRaw(modelId, MATRIX_QUESTIONS, deadline); + if (response.status === 200) { + const body = (await response.json()) as any; + const { tokens, cost } = usageCost(body); + return { status: 200, answers: body.answers, latencyMs, tokens, cost, text: "", split: false }; + } + const text = await response.text(); + if (isAvailability(response.status, text)) { + return { status: response.status, answers: null, latencyMs, tokens: null, cost: "-", text, split: false }; + } + const provider = modelId.split("/")[0]; + if (SPLIT_PROVIDERS.has(provider) && QUESTION_COUNT > 1) { + return fetchDecisionSplit(modelId, latencyMs, deadline); + } + return { status: response.status, answers: null, latencyMs, tokens: null, cost: "-", text, split: false }; + } catch (e) { + if (isAbortError(e)) throw budgetExceededError(modelId, false); + throw e; + } +} + +// A request still in flight when the client budget runs out FAILS the case - +// no skip and no synthetic status; the budget exists only so the failure is +// this readable error instead of Vitest killing the case at its timeout. +function budgetExceededError(modelId: string, split: boolean): Error { + const err = new Error( + `client-side budget of ${MATRIX_CALL_BUDGET_MS}ms exceeded waiting on ${modelId}${split ? " (split mode)" : ""}`, + ); + err.name = "BudgetExceededError"; + return err; +} + +async function fetchDecisionSplit(modelId: string, batchedLatencyMs: number, deadline: number): Promise { + const merged: Record = {}; + let totalLatency = batchedLatencyMs; + let totalTokens = 0; + try { + for (const [name, q] of Object.entries(MATRIX_QUESTIONS)) { + const { response, latencyMs } = await postRaw(modelId, { [name]: q }, deadline); + totalLatency += latencyMs; + if (response.status !== 200) { + const text = await response.text(); + return { status: response.status, answers: null, latencyMs: totalLatency, tokens: null, cost: "-", text, split: true }; + } + const body = (await response.json()) as any; + Object.assign(merged, body.answers); + const { tokens } = usageCost(body); + if (typeof tokens === "number") totalTokens += tokens; + } + } catch (e) { + if (isAbortError(e)) throw budgetExceededError(modelId, true); + throw e; + } + return { status: 200, answers: merged, latencyMs: totalLatency, tokens: totalTokens || null, cost: "-", text: "", split: true }; +} + +// A genuine number for the kind: booleans and numeric strings are wrong types, +// and a noul outside [0, 1] violates the contract regardless of rounding. +// Mirrors the Python suite's _valid_numeric so both matrix verdicts agree. +function validNumeric(kind: string, x: unknown): x is number { + if (typeof x !== "number" || Number.isNaN(x)) return false; + if (kind === "noul" && (x < 0 || x > 1)) return false; + return true; +} + +function agreement(reference: any, answers: any): { matched: number; total: number; mismatches: string[] } { + const mismatches: string[] = []; + for (const [name, q] of Object.entries(MATRIX_QUESTIONS)) { + const ref = reference[name]?.value; + const got = answers[name]?.value; + // Math.round is half-up, the agreed rounding rule for both matrix + // suites - the Python side's _half_up mirrors it (its round() would + // round half to even and diverge at .5 boundaries). + const ok = + q.kind === "choice" + ? got === ref + : validNumeric(q.kind, got) && Math.round(Number(ref)) === Math.round(got); + if (!ok) mismatches.push(`${name}(jev=${ref},got=${got})`); + } + return { matched: QUESTION_COUNT - mismatches.length, total: QUESTION_COUNT, mismatches }; +} + +describe("Decision emulation matrix", () => { + let jevReference: any; + + beforeAll(async () => { + // jev is the ground truth for the whole matrix; if it cannot answer, the + // comparison is meaningless, so fail loudly rather than skip. + const r = await fetchDecision(JEV_MODEL); + expect(r.status, `jev reference decision failed: ${r.text}`).toBe(200); + jevReference = r.answers; + expect(Object.keys(jevReference).length, "jev did not answer every question").toBe(QUESTION_COUNT); + MATRIX_ROWS.push({ modelId: JEV_MODEL, status: "REF", agree: QUESTION_COUNT, total: QUESTION_COUNT, latencyMs: r.latencyMs, tokens: r.tokens, cost: r.cost, detail: "ground truth" }); + // Above MATRIX_CALL_BUDGET_MS so the abort budget fires with its + // readable BudgetExceededError before Vitest kills the hook. + }, 90000); + + it.each(MATRIX_PAIRS)("$provider/$model matches jev", async ({ provider, model }) => { + const modelId = `${provider}/${model}`; + let r: DecisionResult; + try { + r = await fetchDecision(modelId); + } catch (err) { + // A thrown request error - a blown client-side budget or a transport + // failure - FAILS the case with its reason; it is never a skip. + MATRIX_ROWS.push({ modelId, status: "FAIL", agree: 0, total: QUESTION_COUNT, latencyMs: null, tokens: null, cost: "-", detail: `request error: ${err}` }); + expect.fail(`${modelId}: request error: ${err}`); + return; + } + + const latencyMs = r.latencyMs; + if (r.status !== 200) { + const detail = `HTTP ${r.status}: ${r.text.slice(0, 160)}`; + if (isAvailability(r.status, r.text)) { + MATRIX_ROWS.push({ modelId, status: "SKIP", agree: 0, total: QUESTION_COUNT, latencyMs, tokens: null, cost: "-", detail }); + return; + } + // Capability-limited providers (Perplexity's sonar rejects choice/score decision + // schemas): the split was attempted and could not complete the set, so skip rather + // than fail - the identical schema works on every other provider. + if (SPLIT_PROVIDERS.has(provider)) { + MATRIX_ROWS.push({ modelId, status: "SKIP", agree: 0, total: QUESTION_COUNT, latencyMs, tokens: null, cost: "-", detail: `capability-limited - ${detail}` }); + return; + } + // A malformed request we built (400) or emulation returning no decision (500): defect. + MATRIX_ROWS.push({ modelId, status: "FAIL", agree: 0, total: QUESTION_COUNT, latencyMs, tokens: null, cost: "-", detail }); + expect.fail(`${modelId}: ${detail}`); + } + + const a = r.answers; + const { tokens, cost } = { tokens: r.tokens, cost: r.cost }; + expect(Object.keys(a).length, `${modelId} did not answer every question`).toBe(QUESTION_COUNT); + for (const [name, q] of Object.entries(MATRIX_QUESTIONS)) { + // Right kind discriminator and a numeric confidence - a malformed response + // (missing/undefined confidence, wrong kind) must not pass. + expect(a[name].kind, `${name} kind`).toBe(q.kind); + expect(typeof a[name].confidence, `${name} confidence not numeric`).toBe("number"); + } + const { matched, total, mismatches } = agreement(jevReference, a); + const row: MatrixRow = { modelId, status: "PASS", agree: matched, total, latencyMs, tokens, cost, detail: `${matched}/${total} vs jev` }; + if (matched / total < AGREEMENT_FAIL_THRESHOLD) { + row.status = "FAIL"; + row.detail = `${matched}/${total} < threshold; diffs: ${mismatches.slice(0, 8).join(", ")}`; + MATRIX_ROWS.push(row); + expect.fail(`${modelId} agrees with jev on only ${matched}/${total}: ${mismatches.join(", ")}`); + } + MATRIX_ROWS.push(row); + }, 90000); + + afterAll(() => { + console.log(`\n=== Decision emulation matrix vs jev (1 scenario, ${QUESTION_COUNT} questions) ===`); + const header = `${"combination".padEnd(50)} ${"score".padEnd(8)} ${"latency_ms".padEnd(11)} ${"tokens".padEnd(7)} ${"cost".padEnd(8)} status`; + console.log(header); + console.log("-".repeat(header.length)); + const rows = [...MATRIX_ROWS].sort((x, y) => (x.status === "REF" ? -1 : y.status === "REF" ? 1 : x.modelId.localeCompare(y.modelId))); + for (const r of rows) { + const score = `${r.agree}/${r.total}`; + console.log( + `${r.modelId.padEnd(50)} ${score.padEnd(8)} ${String(r.latencyMs ?? "-").padEnd(11)} ${String(r.tokens ?? "-").padEnd(7)} ${r.cost.padEnd(8)} ${r.status}`, + ); + } + const passed = MATRIX_ROWS.filter((r) => r.status === "PASS"); + const failed = MATRIX_ROWS.filter((r) => r.status === "FAIL"); + const skipped = MATRIX_ROWS.filter((r) => r.status === "SKIP"); + console.log(`\nscore = questions agreeing with jev out of ${QUESTION_COUNT}; PASS threshold = ${Math.round(AGREEMENT_FAIL_THRESHOLD * 100)}%`); + console.log(`PASS=${passed.length} FAIL=${failed.length} SKIP=${skipped.length} TOTAL=${MATRIX_PAIRS.length} (+jev reference)`); + }); +}); diff --git a/transports/bifrost-http/handlers/governance_test.go b/transports/bifrost-http/handlers/governance_test.go index cd55d460cca..2a7bf29e677 100644 --- a/transports/bifrost-http/handlers/governance_test.go +++ b/transports/bifrost-http/handlers/governance_test.go @@ -3331,23 +3331,35 @@ func TestUpdateProviderGovernance_BudgetMutualExclusion(t *testing.T) { func TestValidateRoutingFallbacks(t *testing.T) { + // Inputs are raw JSON so each case also covers RoutingFallback decoding, which is where the + // legacy "provider/model" string is split and an unknown prefix collapses to an empty provider. tests := []struct { name string - fbs []string + fbsJSON string wantErr bool }{ - {name: "nil", fbs: nil, wantErr: false}, - {name: "empty", fbs: []string{}, wantErr: false}, - {name: "provider model", fbs: []string{"openai/gpt-4o"}, wantErr: false}, - {name: "provider slash incoming model", fbs: []string{"azure/"}, wantErr: false}, - {name: "bare known provider name rejected", fbs: []string{"openrouter"}, wantErr: true}, - {name: "bare model rejected", fbs: []string{"gpt-4o"}, wantErr: true}, - {name: "empty element", fbs: []string{"openai/gpt-4o", ""}, wantErr: true}, - {name: "huggingface namespace not a provider prefix", fbs: []string{"meta-llama/Llama-3.1-8B"}, wantErr: true}, + {name: "nil", fbsJSON: `null`, wantErr: false}, + {name: "empty", fbsJSON: `[]`, wantErr: false}, + {name: "provider model", fbsJSON: `["openai/gpt-4o"]`, wantErr: false}, + {name: "provider slash incoming model", fbsJSON: `["azure/"]`, wantErr: false}, + {name: "bare known provider name rejected", fbsJSON: `["openrouter"]`, wantErr: true}, + {name: "bare model rejected", fbsJSON: `["gpt-4o"]`, wantErr: true}, + {name: "empty element", fbsJSON: `["openai/gpt-4o",""]`, wantErr: true}, + {name: "huggingface namespace not a provider prefix", fbsJSON: `["meta-llama/Llama-3.1-8B"]`, wantErr: true}, + {name: "object with pinned key", fbsJSON: `[{"provider":"azure","model":"gpt-4o","key_id":"k1"}]`, wantErr: false}, + {name: "object pinning a key for the incoming model", fbsJSON: `[{"provider":"azure","key_id":"k1"}]`, wantErr: false}, + {name: "key_id without provider rejected", fbsJSON: `[{"key_id":"k1"}]`, wantErr: true}, + {name: "object with unknown provider rejected", fbsJSON: `[{"provider":"opneai","model":"gpt-4o","key_id":"k1"}]`, wantErr: true}, + {name: "object with fork provider accepted", fbsJSON: `[{"provider":"volcengine","model":"doubao-seed-1-6"}]`, wantErr: false}, + {name: "provider_key_name rejected over the API", fbsJSON: `[{"provider":"azure","provider_key_name":"prod"}]`, wantErr: true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - err := validateRoutingFallbacks(tt.fbs) + var fbs []configstoreTables.RoutingFallback + if err := json.Unmarshal([]byte(tt.fbsJSON), &fbs); err != nil { + t.Fatalf("failed to decode fallbacks: %v", err) + } + err := validateRoutingFallbacks(fbs) if tt.wantErr && err == nil { t.Fatal("expected error") } diff --git a/transports/bifrost-http/handlers/inference.go b/transports/bifrost-http/handlers/inference.go index f62f72baf91..a6e647d5b49 100644 --- a/transports/bifrost-http/handlers/inference.go +++ b/transports/bifrost-http/handlers/inference.go @@ -221,6 +221,13 @@ var rerankParamsKnownFields = map[string]bool{ "next_token": true, } +var decisionParamsKnownFields = map[string]bool{ + "model": true, + "state": true, + "questions": true, + "fallbacks": true, +} + var ocrParamsKnownFields = map[string]bool{ "model": true, "id": true, @@ -493,58 +500,52 @@ type VideoGenerationHTTPRequest struct { // UnmarshalJSON unmarshals the responses request input func (r *ResponsesRequestInput) UnmarshalJSON(data []byte) error { - var str string - if err := sonic.Unmarshal(data, &str); err == nil { - r.ResponsesRequestInputStr = &str - r.ResponsesRequestInputArray = nil - return nil + // Peek the first non-whitespace byte to select the decode path without a + // failed-attempt allocation. Strings and null use the string decode path; + // other values use the array path and preserve the original generic error. + for _, b := range data { + switch b { + case ' ', '\t', '\r', '\n': + continue + case '"': + var str string + if err := sonic.Unmarshal(data, &str); err != nil { + return fmt.Errorf("invalid responses request input") + } + r.ResponsesRequestInputStr = &str + r.ResponsesRequestInputArray = nil + return nil + case 'n': + // null decodes to the empty string, matching what the previous + // try-string-first implementation produced. + var str string + if err := sonic.Unmarshal(data, &str); err != nil { + return fmt.Errorf("invalid responses request input") + } + r.ResponsesRequestInputStr = &str + r.ResponsesRequestInputArray = nil + return nil + } + break } var array []schemas.ResponsesMessage - if err := sonic.Unmarshal(data, &array); err == nil { - r.ResponsesRequestInputStr = nil - r.ResponsesRequestInputArray = array - return nil - } - return fmt.Errorf("invalid responses request input") -} - -// UnmarshalJSON implements custom JSON unmarshalling for ResponsesRequest. -// This is needed because ResponsesParameters has a custom UnmarshalJSON method, -// which interferes with sonic's handling of the embedded BifrostParams struct. -func (rr *ResponsesRequest) UnmarshalJSON(data []byte) error { - // First, unmarshal BifrostParams fields directly - type bifrostAlias BifrostParams - var bp bifrostAlias - if err := sonic.Unmarshal(data, &bp); err != nil { - return err - } - rr.BifrostParams = BifrostParams(bp) - - // Unmarshal messages - var inputStruct struct { - Input ResponsesRequestInput `json:"input"` + if err := sonic.Unmarshal(data, &array); err != nil { + return fmt.Errorf("invalid responses request input") } - if err := sonic.Unmarshal(data, &inputStruct); err != nil { - return err - } - rr.Input = inputStruct.Input - - // Unmarshal ResponsesParameters (which has its own custom unmarshaller) - if rr.ResponsesParameters == nil { - rr.ResponsesParameters = &schemas.ResponsesParameters{} - } - if err := sonic.Unmarshal(data, rr.ResponsesParameters); err != nil { - return err - } - + r.ResponsesRequestInputStr = nil + r.ResponsesRequestInputArray = array return nil } // ResponsesRequest is a bifrost responses request +// +// Plain struct decoding handles all fields in one pass. Embed parameters by +// value: Sonic's optdec skips null values reached through an embedded pointer, +// which would retain earlier values when a duplicate parameter is set to null. type ResponsesRequest struct { Input ResponsesRequestInput `json:"input"` BifrostParams - *schemas.ResponsesParameters + schemas.ResponsesParameters } // CompactionHTTPRequest is a bifrost compaction request (subset of responses fields) @@ -573,6 +574,13 @@ type RerankRequest struct { *schemas.RerankParameters } +// DecisionHandlerRequest is a bifrost decision request +type DecisionHandlerRequest struct { + State interface{} `json:"state"` + Questions map[string]schemas.DecisionQuestion `json:"questions"` + BifrostParams +} + // OCRHandlerRequest is a bifrost OCR request type OCRHandlerRequest struct { ID *string `json:"id,omitempty"` @@ -720,6 +728,7 @@ var PathToTypeMapping = map[string]schemas.RequestType{ "/v1/responses": schemas.ResponsesRequest, "/v1/embeddings": schemas.EmbeddingRequest, "/v1/rerank": schemas.RerankRequest, + "/v1/decisions": schemas.DecisionRequest, "/v1/ocr": schemas.OCRRequest, "/v1/audio/speech": schemas.SpeechRequest, "/v1/audio/transcriptions": schemas.TranscriptionRequest, @@ -775,6 +784,7 @@ func (h *CompletionHandler) RegisterRoutes(r *router.Router, middlewares ...sche r.GET("/v1/responses/{response_id}/input_items", lib.ChainMiddlewares(h.responsesInputItems, responsesInputItemsMW...)) r.POST("/v1/embeddings", lib.ChainMiddlewares(h.embeddings, baseMiddlewares...)) r.POST("/v1/rerank", lib.ChainMiddlewares(h.rerank, baseMiddlewares...)) + r.POST("/v1/decisions", lib.ChainMiddlewares(h.evaluation, baseMiddlewares...)) r.POST("/v1/ocr", lib.ChainMiddlewares(h.ocr, baseMiddlewares...)) // ElevenLabs sound-effect models also flow through /v1/audio/speech; the // provider routes them to /v1/sound-generation by model id, keeping SDK and @@ -1109,9 +1119,6 @@ func prepareResponsesRequest(ctx *fasthttp.RequestCtx, config *lib.Config) (*Res if len(req.Input.ResponsesRequestInputArray) == 0 && req.Input.ResponsesRequestInputStr == nil { return nil, nil, fmt.Errorf("input is required for responses") } - if req.ResponsesParameters == nil { - req.ResponsesParameters = &schemas.ResponsesParameters{} - } req.ResponsesParameters.ExtraParams = base.ExtraParams input := req.Input.ResponsesRequestInputArray @@ -1127,7 +1134,7 @@ func prepareResponsesRequest(ctx *fasthttp.RequestCtx, config *lib.Config) (*Res Provider: base.Provider, Model: base.ModelName, Input: input, - Params: req.ResponsesParameters, + Params: &req.ResponsesParameters, Fallbacks: base.Fallbacks, }, nil } @@ -1293,6 +1300,61 @@ func (h *CompletionHandler) rerank(ctx *fasthttp.RequestCtx) { SendJSON(ctx, resp) } +// prepareDecisionRequest prepares a BifrostDecisionRequest from the HTTP request body +func prepareDecisionRequest(ctx *fasthttp.RequestCtx, config *lib.Config) (*DecisionHandlerRequest, *schemas.BifrostDecisionRequest, error) { + req, base, err := prepareRequest[DecisionHandlerRequest](ctx, config, decisionParamsKnownFields) + if err != nil { + return nil, nil, err + } + if req.State == nil { + return nil, nil, fmt.Errorf("state is required for decision") + } + if len(req.Questions) == 0 { + return nil, nil, fmt.Errorf("questions are required for decision") + } + return req, &schemas.BifrostDecisionRequest{ + Provider: base.Provider, + Model: base.ModelName, + State: req.State, + Questions: req.Questions, + Fallbacks: base.Fallbacks, + }, nil +} + +// evaluation handles POST /v1/decisions - Process decision requests +func (h *CompletionHandler) evaluation(ctx *fasthttp.RequestCtx) { + _, bifrostDecisionReq, err := prepareDecisionRequest(ctx, h.config) + if err != nil { + SendError(ctx, fasthttp.StatusBadRequest, err.Error()) + return + } + + // Convert context + bifrostCtx, cancel := lib.ConvertToBifrostContext(ctx, h.config) + defer cancel() + if bifrostCtx == nil { + SendError(ctx, fasthttp.StatusBadRequest, "Failed to convert context") + return + } + + resp, bifrostErr := h.client.DecisionRequest(bifrostCtx, bifrostDecisionReq) + if bifrostErr != nil { + forwardProviderHeadersFromContext(ctx, bifrostCtx) + SendBifrostError(ctx, bifrostErr) + return + } + + if resp != nil { + lib.ApplyBifrostResponseHeaders(ctx, bifrostCtx, resp.ExtraFields) + } + + if streamLargeResponseIfActive(ctx, bifrostCtx) { + return + } + // Send successful response + SendJSON(ctx, resp) +} + // prepareOCRRequest prepares a BifrostOCRRequest from the HTTP request body func prepareOCRRequest(ctx *fasthttp.RequestCtx, config *lib.Config) (*OCRHandlerRequest, *schemas.BifrostOCRRequest, error) { req, base, err := prepareRequest[OCRHandlerRequest](ctx, config, ocrParamsKnownFields) diff --git a/transports/bifrost-http/handlers/integrations.go b/transports/bifrost-http/handlers/integrations.go index 6abcc8af267..023134e0b81 100644 --- a/transports/bifrost-http/handlers/integrations.go +++ b/transports/bifrost-http/handlers/integrations.go @@ -39,6 +39,7 @@ func NewIntegrationHandler(client *bifrost.Bifrost, handlerStore lib.HandlerStor integrations.NewLangChainRouter(client, handlerStore, accessResolver, logger), integrations.NewPydanticAIRouter(client, handlerStore, accessResolver, logger), integrations.NewBedrockRouter(client, handlerStore, accessResolver, logger), + integrations.NewTypesafeRouter(client, handlerStore, accessResolver, logger), // passthrough routers integrations.NewGenAIPassthroughRouter(client, handlerStore, accessResolver, logger), integrations.NewChatGPTPassthroughRouter(client, handlerStore, accessResolver, logger), diff --git a/transports/bifrost-http/handlers/providers_test.go b/transports/bifrost-http/handlers/providers_test.go index d77bba48ab1..7278ec92558 100644 --- a/transports/bifrost-http/handlers/providers_test.go +++ b/transports/bifrost-http/handlers/providers_test.go @@ -1436,6 +1436,63 @@ func accessForProviderPermits(permits ...schemas.ProviderPermit) schemas.Access return grant.NewAccess([]schemas.Permit{permit}, nil, "", nil) } +// accessAllowingAllProviders builds the access a caller permitted every provider carries, the way +// the governance store builds it: the providers its configs do not name are materialised onto the +// permit, so the permit carries its whole grant and every consumer reads one list. +func accessAllowingAllProviders(configured []string, permits ...schemas.ProviderPermit) schemas.Access { + permit := grant.NewPermit(grant.PermitVirtualKey, "vk-test", "Test VK", true, false, + governanceplugin.AppendAllProviderPermits(permits, configured), nil, + grant.WithAllowAllProviders(true)) + return grant.NewAccess([]schemas.Permit{permit}, nil, "", nil) +} + +// A caller permitted every provider is listed every provider, including ones it holds no provider +// permit for. Narrowing to the permits it happens to hold would make the listing refuse what the +// request path admits. +func TestListModels_VKFilterListsProviderAllowedOnlyByAllowAll(t *testing.T) { + SetLogger(&mockLogger{}) + + h := &ProviderHandler{ + inMemoryStore: &lib.Config{ + ClientConfig: &configstore.ClientConfig{}, + Providers: map[schemas.ModelProvider]configstore.ProviderConfig{ + schemas.OpenAI: {Keys: []schemas.Key{{ID: "key-a"}}}, + schemas.Anthropic: {Keys: []schemas.Key{{ID: "key-b"}}}, + }, + }, + modelsManager: &mockModelsManager{ + filtered: map[schemas.ModelProvider][]string{ + schemas.OpenAI: {"gpt-4o"}, + schemas.Anthropic: {"claude-haiku-4-5"}, + }, + }, + } + + query := modelListQuery{ + Limit: 100, + HasVKFilter: true, + Access: accessAllowingAllProviders( + []string{string(schemas.OpenAI), string(schemas.Anthropic)}, + schemas.ProviderPermit{Provider: "openai", AllowedModels: schemas.WhiteList{"*"}}, + ), + } + if !query.Access.IsProviderAllowed(string(schemas.Anthropic)) { + t.Fatal("control failed: allow-all must permit a provider it holds no permit for") + } + + models, total, err := h.listManagementModels(query) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + names := map[string]bool{} + for _, m := range models { + names[m.Name] = true + } + if total != 2 || !names["gpt-4o"] || !names["claude-haiku-4-5"] { + t.Fatalf("expected both providers listed, got total=%d models=%#v", total, models) + } +} + // A blacklisted model is not listed. The listing answers the same question a request does, so a // model the caller would be refused is not advertised to them as available. func TestListModels_VKFilterHidesBlacklistedModel(t *testing.T) { diff --git a/transports/bifrost-http/handlers/requestpayload_test.go b/transports/bifrost-http/handlers/requestpayload_test.go index 174901e57dd..ee429092b61 100644 --- a/transports/bifrost-http/handlers/requestpayload_test.go +++ b/transports/bifrost-http/handlers/requestpayload_test.go @@ -1,9 +1,17 @@ package handlers import ( + "encoding/json" + "fmt" + "maps" + "reflect" "strings" "testing" + "github.com/bytedance/sonic" + "github.com/capsohq/bifrost/core/providers/openai" + providerUtils "github.com/capsohq/bifrost/core/providers/utils" + "github.com/capsohq/bifrost/core/schemas" "github.com/capsohq/bifrost/framework/configstore" "github.com/valyala/fasthttp" ) @@ -47,3 +55,669 @@ func TestPrepareRequestInvalidPayloadDoesNotExposeDecoderDetails(t *testing.T) { t.Fatalf("error exposes decoder internals: %s", msg) } } + +// decodeRequest exercises the production prepareRequest path without extracting extras. +func decodeRequest[T baseRequest](t *testing.T, body []byte) *T { + t.Helper() + ctx := &fasthttp.RequestCtx{} + ctx.Request.SetBody(body) + req, _, err := prepareRequest[T](ctx, nil, nil) + if err != nil { + t.Fatalf("prepareRequest: %v", err) + } + return req +} + +// TestResponsesRequestDecodeAllSections guards the plain-struct decoding of +// ResponsesRequest: Go promotes UnmarshalJSON from embedded types, so +// if ResponsesParameters ever gains a custom UnmarshalJSON, sonic would call +// the promoted method and silently drop Input and BifrostParams. This test +// fails loudly if that happens. (ChatRequest hit exactly this: ChatParameters +// has a custom unmarshaller, hence ChatRequest keeps its explicit one.) +func TestResponsesRequestDecodeAllSections(t *testing.T) { + body := []byte(`{ + "model": "openai/gpt-4o", + "stream": true, + "fallbacks": ["anthropic/claude-sonnet-5"], + "input": [{"role": "user", "content": "hello"}], + "temperature": 0.5, + "max_output_tokens": 128 + }`) + req := decodeRequest[ResponsesRequest](t, body) + + if req.Model != "openai/gpt-4o" { + t.Errorf("BifrostParams.Model lost: got %q", req.Model) + } + if req.Stream == nil || !*req.Stream { + t.Errorf("BifrostParams.Stream lost") + } + if len(req.Fallbacks) != 1 { + t.Errorf("BifrostParams.Fallbacks lost: got %v", req.Fallbacks) + } + if len(req.Input.ResponsesRequestInputArray) != 1 { + t.Fatalf("Input lost: got %d messages", len(req.Input.ResponsesRequestInputArray)) + } + if req.Temperature == nil || *req.Temperature != 0.5 { + t.Errorf("ResponsesParameters.Temperature lost") + } + if req.MaxOutputTokens == nil || *req.MaxOutputTokens != 128 { + t.Errorf("ResponsesParameters.MaxOutputTokens lost") + } +} + +// TestResponsesRequestParamsNonNilWithoutParamFields pins the core request's +// non-nil Params invariant even when the body carries no parameter fields. +func TestResponsesRequestParamsNonNilWithoutParamFields(t *testing.T) { + ctx := &fasthttp.RequestCtx{} + ctx.Request.SetBodyString(`{"model":"openai/gpt-4o","input":"hi"}`) + req, coreReq, err := prepareResponsesRequest(ctx, nil) + if err != nil { + t.Fatal(err) + } + if coreReq.Params == nil { + t.Fatal("Params is nil for a body without parameter fields") + } + if req.Input.ResponsesRequestInputStr == nil || *req.Input.ResponsesRequestInputStr != "hi" { + t.Errorf("string-form input lost") + } +} + +func TestPrepareResponsesRequestNullClearsParameters(t *testing.T) { + for _, tt := range []struct { + field string + value string + }{ + {field: "temperature", value: `0.7`}, + {field: "max_output_tokens", value: `128`}, + {field: "instructions", value: `"previous instructions"`}, + {field: "metadata", value: `{"tag":"previous"}`}, + {field: "include", value: `["file_search_call.results"]`}, + {field: "tools", value: `[{"type":"function","name":"test_fn","parameters":{"type":"object","properties":{}}}]`}, + {field: "tool_choice", value: `"auto"`}, + {field: "reasoning", value: `{"effort":"low"}`}, + {field: "store", value: `true`}, + } { + for _, key := range []string{tt.field, strings.ToUpper(tt.field)} { + t.Run(key, func(t *testing.T) { + ctx := &fasthttp.RequestCtx{} + ctx.Request.SetBodyString(`{"model":"openai/gpt-4o","input":"hi","` + tt.field + `":` + tt.value + `,"` + key + `":null}`) + _, req, err := prepareResponsesRequest(ctx, nil) + if err != nil { + t.Fatal(err) + } + // Check both the internal parameters and the provider's typed + // request, before ExtraParams can mask a stale parameter. + for _, value := range []any{req.Params, openai.ToOpenAIResponsesRequest(nil, req)} { + body, err := providerUtils.MarshalSorted(value) + if err != nil { + t.Fatal(err) + } + var fields map[string]json.RawMessage + if err := json.Unmarshal(body, &fields); err != nil { + t.Fatal(err) + } + if _, ok := fields[tt.field]; ok { + t.Errorf("null must clear %s in %T: %s", tt.field, value, body) + } + } + }) + } + } +} + +func TestPrepareResponsesRequestPreservesRawNull(t *testing.T) { + for _, fields := range []string{ + `"context_management":null`, + `"context_management":[],"context_management":null`, + `"context_management":[],"CONTEXT_MANAGEMENT":null`, + `"c\u006fntext_management":null`, + } { + t.Run(fields, func(t *testing.T) { + ctx := &fasthttp.RequestCtx{} + ctx.Request.SetBodyString(`{"model":"openai/gpt-4o","input":"hi",` + fields + `}`) + _, req, err := prepareResponsesRequest(ctx, nil) + if err != nil { + t.Fatal(err) + } + if got := string(req.Params.ContextManagement); got != "null" { + t.Errorf("context_management = %q, want raw null", got) + } + }) + } +} + +// TestChatRequestDecodeAllSections documents why ChatRequest keeps its custom +// UnmarshalJSON: ChatParameters has one, and dropping ChatRequest's would let +// the promoted method swallow Messages and BifrostParams. +func TestChatRequestDecodeAllSections(t *testing.T) { + body := []byte(`{ + "model": "openai/gpt-4o", + "stream": true, + "messages": [{"role": "user", "content": "hi"}], + "temperature": 0.5 + }`) + req := decodeRequest[ChatRequest](t, body) + + if req.Model != "openai/gpt-4o" { + t.Errorf("BifrostParams.Model lost: got %q", req.Model) + } + if len(req.Messages) != 1 { + t.Errorf("Messages lost: got %d", len(req.Messages)) + } + if req.ChatParameters == nil || req.Temperature == nil || *req.Temperature != 0.5 { + t.Errorf("ChatParameters.Temperature lost") + } +} + +func TestResponsesRequestInputUnmarshalJSONBoundaries(t *testing.T) { + tests := []struct { + name string + data string + want ResponsesRequestInput + }{ + {name: "empty string", data: `""`, want: ResponsesRequestInput{ResponsesRequestInputStr: schemas.Ptr("")}}, + {name: "string with whitespace", data: " \t\r\n\"hello\" \t\r\n", want: ResponsesRequestInput{ResponsesRequestInputStr: schemas.Ptr("hello")}}, + {name: "escaped string", data: `"line\n\"quote\"\u4e16\u754c"`, want: ResponsesRequestInput{ResponsesRequestInputStr: schemas.Ptr("line\n\"quote\"\u4e16\u754c")}}, + {name: "null", data: " \t\r\nnull \t\r\n", want: ResponsesRequestInput{ResponsesRequestInputStr: schemas.Ptr("")}}, + {name: "empty array", data: " \t\r\n[] \t\r\n", want: ResponsesRequestInput{ResponsesRequestInputArray: []schemas.ResponsesMessage{}}}, + { + name: "message array", + data: `[{"role":"user","content":"hello"}]`, + want: ResponsesRequestInput{ResponsesRequestInputArray: []schemas.ResponsesMessage{ + {Role: schemas.Ptr(schemas.ResponsesInputMessageRoleUser), Content: &schemas.ResponsesMessageContent{ContentStr: schemas.Ptr("hello")}}, + }}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Successful input decoding replaces both union fields on reuse. + got := ResponsesRequestInput{ + ResponsesRequestInputStr: schemas.Ptr("previous"), + ResponsesRequestInputArray: []schemas.ResponsesMessage{{ID: schemas.Ptr("old")}}, + } + if err := got.UnmarshalJSON([]byte(tt.data)); err != nil { + t.Fatalf("UnmarshalJSON: %v", err) + } + if !reflect.DeepEqual(got, tt.want) { + t.Fatalf("input = %#v, want %#v", got, tt.want) + } + }) + } + + invalid := []struct { + name string + data string + }{ + {name: "empty", data: ""}, + {name: "whitespace only", data: " \t\r\n"}, + {name: "object", data: `{}`}, + {name: "number", data: `123`}, + {name: "boolean", data: `true`}, + {name: "non JSON whitespace", data: "\vnull"}, + {name: "truncated null", data: `nul`}, + {name: "truncated string", data: `"unterminated`}, + {name: "invalid escape", data: `"\q"`}, + {name: "truncated array", data: `[`}, + {name: "invalid array item", data: `[{"role":"user","content":"new"},42]`}, + {name: "invalid content", data: `[{"role":"user","content":42}]`}, + {name: "trailing comma", data: `[{},]`}, + {name: "trailing value", data: `"text" false`}, + } + for _, tt := range invalid { + t.Run(tt.name, func(t *testing.T) { + before := ResponsesRequestInput{ResponsesRequestInputStr: schemas.Ptr("previous")} + got := before + err := got.UnmarshalJSON([]byte(tt.data)) + const wantError = "invalid responses request input" + if err == nil || err.Error() != wantError { + t.Fatalf("error = %v, want %q", err, wantError) + } + if !reflect.DeepEqual(got, before) { + t.Fatalf("failed decode changed receiver: %#v", got) + } + }) + } +} + +func TestPrepareResponsesRequestInvalidPayload(t *testing.T) { + for _, tt := range []struct { + name string + body string + }{ + {name: "invalid input type", body: `{"model":"openai/gpt-4o","input":42}`}, + {name: "invalid content type", body: `{"model":"openai/gpt-4o","input":[{"role":"user","content":{}}]}`}, + {name: "malformed input", body: `{"model":"openai/gpt-4o","input":[}`}, + {name: "invalid parameter", body: `{"model":"openai/gpt-4o","input":"hello","temperature":"invalid"}`}, + } { + t.Run(tt.name, func(t *testing.T) { + ctx := &fasthttp.RequestCtx{} + ctx.Request.SetBodyString(tt.body) + _, _, err := prepareResponsesRequest(ctx, nil) + if err == nil || err.Error() != "Invalid request payload" { + t.Fatalf("error = %v, want generic invalid payload error", err) + } + }) + } +} + +// TestResponsesRequestNullInputBehavior pins that "input": null keeps decoding +// to an empty string rather than erroring. The peek-first union decoder has no +// case for 'n', so null must be handled explicitly or a client sending it gets +// a 400 where it previously got a synthetic empty user message. +func TestResponsesRequestNullInputBehavior(t *testing.T) { + req := decodeRequest[ResponsesRequest](t, []byte(`{"model":"openai/gpt-4o","input":null}`)) + + if req.Input.ResponsesRequestInputStr == nil { + t.Fatal(`"input": null must decode to an empty string, not leave both union fields nil`) + } + if *req.Input.ResponsesRequestInputStr != "" { + t.Errorf(`"input": null decoded to %q, want ""`, *req.Input.ResponsesRequestInputStr) + } +} + +// TestResponsesMessageContentNullBehavior pins that "content": null decodes to +// an empty string. OpenAI emits null content on assistant turns that carry only +// tool calls, so erroring here would drop valid provider responses. +func TestResponsesMessageContentNullBehavior(t *testing.T) { + var c schemas.ResponsesMessageContent + if err := sonic.Unmarshal([]byte(`null`), &c); err != nil { + t.Fatalf(`"content": null must not error: %v`, err) + } + if c.ContentStr == nil { + t.Fatal(`"content": null must decode to an empty string, not leave both union fields nil`) + } + if *c.ContentStr != "" { + t.Errorf(`"content": null decoded to %q, want ""`, *c.ContentStr) + } +} + +func TestExtractExtraParamsOwnsReturnedData(t *testing.T) { + tests := []struct { + name string + body string + want map[string]any + }{ + { + name: "known fields only", + body: `{"input":[{"content":"hello"}]}`, + want: map[string]any{}, + }, + { + name: "surrounding JSON whitespace", + body: " \t\r\n" + `{"input":[],"extra":"value"}` + " \t\r\n", + want: map[string]any{"extra": "value"}, + }, + { + name: "nested values and escaped keys", + body: `{"input":[],"plain":"value","escaped\u005fkey":"line\nbreak","nested":{"items":["hello",{"text":"world"}]},"nullable":null,"number":1.5,"enabled":true}`, + want: map[string]any{ + "plain": "value", + "escaped_key": "line\nbreak", + "nested": map[string]any{"items": []any{"hello", map[string]any{"text": "world"}}}, + "nullable": nil, + "number": float64(1.5), + "enabled": true, + }, + }, + { + name: "last duplicate key wins", + body: `{"input":[],"extra":"first","ex\u0074ra":{"value":"last"}}`, + want: map[string]any{"extra": map[string]any{"value": "last"}}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + body := []byte(tt.body) + extras, err := extractExtraParams(body, map[string]bool{"input": true}) + if err != nil { + t.Fatalf("extractExtraParams: %v", err) + } + if string(body) != tt.body { + t.Fatal("extractExtraParams modified the request body") + } + // FastHTTP reuses its body buffer after the handler returns. Every + // returned key and nested string must survive that reuse. + for i := range body { + body[i] = 'X' + } + if !reflect.DeepEqual(extras, tt.want) { + t.Fatalf("extras after body reuse = %#v, want %#v", extras, tt.want) + } + }) + } +} + +func TestExtractExtraParamsRejectsMalformedJSON(t *testing.T) { + for _, tt := range []struct { + name string + body string + }{ + {name: "empty body", body: ""}, + {name: "truncated object", body: `{"input":"hello"`}, + {name: "invalid known value", body: `{"input":[}`}, + {name: "invalid extra", body: `{"input":"hello","extra":tru}`}, + {name: "trailing comma", body: `{"input":"hello",}`}, + {name: "trailing value", body: `{"input":"hello"} {}`}, + {name: "trailing non JSON whitespace", body: "{\"input\":\"hello\"}\v"}, + } { + t.Run(tt.name, func(t *testing.T) { + if _, err := extractExtraParams([]byte(tt.body), responsesParamsKnownFields); err == nil { + t.Fatal("expected malformed JSON to fail") + } + }) + } +} + +func TestExtractExtraParamsInvalidValuesMatchDecoder(t *testing.T) { + invalidValues := []struct { + name string + value string + }{ + {name: "invalid escape", value: `"\q"`}, + {name: "invalid unicode escape", value: `"\u12xz"`}, + {name: "short unicode escape", value: `"\u123"`}, + {name: "nested invalid escape", value: `{"items":["\q"]}`}, + {name: "nested invalid key", value: `{"bad\q":true}`}, + {name: "positive overflow", value: `1e1000`}, + {name: "negative overflow", value: `-1e1000`}, + {name: "float64 boundary overflow", value: `1.7976931348623159e308`}, + {name: "integer overflow", value: strings.Repeat("9", 309)}, + {name: "nested overflow", value: `{"items":[1e1000]}`}, + {name: "overwritten nested overflow", value: `{"value":1e1000,"value":1}`}, + } + for _, tt := range invalidValues { + for _, field := range []string{"input", "extra"} { + for _, overwrite := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/%s/overwrite=%t", tt.name, field, overwrite), func(t *testing.T) { + body := `{"valid_extra":"keep","` + field + `":` + tt.value + if overwrite { + body += `,"` + field + `":null` + } + body += `}` + // Sonic's raw-value validation differs between decoder backends. + // Preserve that behavior instead of adding a second, stricter parse. + var raw map[string]json.RawMessage + decodeErr := sonic.Unmarshal([]byte(body), &raw) + extras, err := extractExtraParams([]byte(body), map[string]bool{"input": true}) + if (err == nil) != (decodeErr == nil) { + t.Fatalf("error = %v, decoder error = %v", err, decodeErr) + } + if err != nil { + if extras != nil { + t.Fatalf("returned partial extras on failure: %#v", extras) + } + return + } + want := map[string]any{"valid_extra": "keep"} + if field == "extra" { + var value any + if sonic.Unmarshal(raw[field], &value) == nil { + want[field] = value + } + } + if !reflect.DeepEqual(extras, want) { + t.Fatalf("extras = %#v, want %#v", extras, want) + } + }) + } + } + } + for _, tt := range []struct{ name, body string }{ + {name: "invalid top-level key", body: `{"valid_extra":1,"bad\q":2}`}, + {name: "array root", body: `[]`}, + {name: "string root", body: `"hello"`}, + {name: "number root", body: `123`}, + {name: "boolean root", body: `true`}, + } { + t.Run(tt.name, func(t *testing.T) { + if extras, err := extractExtraParams([]byte(tt.body), nil); err == nil || extras != nil { + t.Fatalf("extras = %#v, error = %v; want nil extras and an error", extras, err) + } + }) + } +} + +func TestExtractExtraParamsValidScalarBoundaries(t *testing.T) { + for _, tt := range []struct { + name string + value string + want any + }{ + {name: "all escapes", value: `"\"\\\/\b\f\n\r\t\u4e16\u754c"`, want: "\"\\/\b\f\n\r\t世界"}, + {name: "surrogate pair", value: `"\ud83d\ude00"`, want: "😀"}, + {name: "unpaired surrogate", value: `"\ud800"`, want: "\ufffd"}, + {name: "numeric string", value: `"1e1000"`, want: "1e1000"}, + {name: "escaped backslash", value: `"\\q"`, want: `\q`}, + {name: "float64 max", value: `1.7976931348623157e308`, want: float64(1.7976931348623157e308)}, + {name: "negative float64 max", value: `-1.7976931348623157e308`, want: float64(-1.7976931348623157e308)}, + {name: "above int64 max", value: `9223372036854775808`, want: float64(9223372036854775808)}, + {name: "underflow", value: `1e-1000`, want: float64(0)}, + } { + t.Run(tt.name, func(t *testing.T) { + body := []byte(`{"input":` + tt.value + `,"extra":` + tt.value + `}`) + extras, err := extractExtraParams(body, map[string]bool{"input": true}) + if err != nil { + t.Fatal(err) + } + if want := map[string]any{"extra": tt.want}; !reflect.DeepEqual(extras, want) { + t.Fatalf("extras = %#v, want %#v", extras, want) + } + }) + } + for _, body := range []string{`{}`, `null`} { + t.Run(body, func(t *testing.T) { + extras, err := extractExtraParams([]byte(body), nil) + if err != nil || len(extras) != 0 { + t.Fatalf("extras = %#v, error = %v; want no extras and no error", extras, err) + } + }) + } +} + +func benchmarkRequestBody(b *testing.B, fields map[string]any, extraCount int) string { + b.Helper() + fields = maps.Clone(fields) + for i := range extraCount { + fields[fmt.Sprintf("custom_%d", i)] = map[string]any{ + "enabled": true, + "values": []any{1, "two", nil}, + } + } + body, err := schemas.MarshalSorted(fields) + if err != nil { + b.Fatal(err) + } + return string(body) +} + +// BenchmarkPrepareSmallRequest exercises the shared prepareRequest path with +// ordinary text payloads and varying numbers of provider-specific parameters. +func BenchmarkPrepareSmallRequest(b *testing.B) { + for _, extraCount := range []int{0, 1, 32} { + b.Run(fmt.Sprintf("Responses/extras=%d", extraCount), func(b *testing.B) { + body := benchmarkRequestBody(b, map[string]any{ + "model": "openai/gpt-4o", "input": "hello", "temperature": 0.7, + }, extraCount) + benchmarkPrepareRequest[ResponsesRequest](b, body, responsesParamsKnownFields, extraCount) + }) + b.Run(fmt.Sprintf("ResponsesArray/extras=%d", extraCount), func(b *testing.B) { + body := benchmarkRequestBody(b, map[string]any{ + "model": "openai/gpt-4o", + "input": []map[string]any{{"role": "user", "content": []map[string]any{ + {"type": "input_text", "text": "hello"}, + }}}, + "temperature": 0.7, + }, extraCount) + benchmarkPrepareRequest[ResponsesRequest](b, body, responsesParamsKnownFields, extraCount) + }) + b.Run(fmt.Sprintf("Chat/extras=%d", extraCount), func(b *testing.B) { + body := benchmarkRequestBody(b, map[string]any{ + "model": "openai/gpt-4o", + "messages": []map[string]any{{"role": "user", "content": "hello"}}, + "temperature": 0.7, + }, extraCount) + benchmarkPrepareRequest[ChatRequest](b, body, chatParamsKnownFields, extraCount) + }) + b.Run(fmt.Sprintf("Embedding/extras=%d", extraCount), func(b *testing.B) { + body := benchmarkRequestBody(b, map[string]any{ + "model": "openai/text-embedding-3-small", "input": "hello", "dimensions": 256, + }, extraCount) + benchmarkPrepareRequest[EmbeddingRequest](b, body, embeddingParamsKnownFields, extraCount) + }) + } +} + +func benchmarkPrepareRequest[T baseRequest](b *testing.B, body string, known map[string]bool, extraCount int) { + b.Helper() + ctx := &fasthttp.RequestCtx{} + ctx.Request.SetBodyString(body) + // Warm the decoder and verify the fixture before timing. + _, base, err := prepareRequest[T](ctx, nil, known) + if err != nil { + b.Fatal(err) + } + if base.Provider != schemas.OpenAI || len(base.ExtraParams) != extraCount { + b.Fatalf("unexpected decoded request: %#v", base) + } + b.SetBytes(int64(len(body))) + b.ReportAllocs() + for b.Loop() { + if _, _, err := prepareRequest[T](ctx, nil, known); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkExtractExtraParams isolates the shared extractor from typed decoding. +func BenchmarkExtractExtraParams(b *testing.B) { + for _, tt := range []struct { + name string + fields map[string]any + extraCount int + }{ + {name: "knownOnly", fields: map[string]any{"model": "openai/gpt-4o", "input": "hello", "temperature": 0.7}}, + {name: "oneExtra", fields: map[string]any{"input": "hello"}, extraCount: 1}, + {name: "manyExtras", fields: map[string]any{"input": "hello"}, extraCount: 32}, + {name: "largeKnown", fields: map[string]any{"input": strings.Repeat("A", 1<<20)}, extraCount: 1}, + {name: "largeExtra", fields: map[string]any{"input": "hello", "large": strings.Repeat("A", 1<<20)}}, + } { + b.Run(tt.name, func(b *testing.B) { + body := []byte(benchmarkRequestBody(b, tt.fields, tt.extraCount)) + wantCount := tt.extraCount + if _, ok := tt.fields["large"]; ok { + wantCount++ + } + extras, err := extractExtraParams(body, responsesParamsKnownFields) + if err != nil { + b.Fatal(err) + } + if len(extras) != wantCount { + b.Fatalf("extra count = %d, want %d", len(extras), wantCount) + } + b.SetBytes(int64(len(body))) + b.ReportAllocs() + for b.Loop() { + if _, err := extractExtraParams(body, responsesParamsKnownFields); err != nil { + b.Fatal(err) + } + } + }) + } +} + +// BenchmarkExtractExtraParamsNestedKnown covers small conversations where +// materializing known message objects can outweigh the cost of raw decoding. +func BenchmarkExtractExtraParamsNestedKnown(b *testing.B) { + for _, count := range []int{1, 8, 32} { + b.Run(fmt.Sprintf("messages=%d", count), func(b *testing.B) { + message := `{"role":"user","content":[{"type":"input_text","text":"hello"}]}` + body := []byte(`{"input":[` + strings.TrimSuffix(strings.Repeat(message+",", count), ",") + `]}`) + b.SetBytes(int64(len(body))) + b.ReportAllocs() + for b.Loop() { + extras, err := extractExtraParams(body, responsesParamsKnownFields) + if err != nil { + b.Fatal(err) + } + if len(extras) != 0 { + b.Fatalf("unexpected extras: %#v", extras) + } + } + }) + } +} + +// buildResponsesBody returns a Responses API JSON body of roughly targetMiB +// mebibytes, shaped like a real multimodal conversation: an input array whose +// bulk is base64-ish image data — the common trigger for large request bodies. +func buildResponsesBody(targetMiB int) string { + const chunkSize = 64 * 1024 + blob := strings.Repeat("A", chunkSize) + messages := targetMiB * 1024 * 1024 / chunkSize + + var b strings.Builder + b.WriteString(`{"model":"openai/gpt-4o","stream":false,"temperature":0.7,"input":[`) + for i := 0; i < messages; i++ { + if i > 0 { + b.WriteByte(',') + } + fmt.Fprintf(&b, + `{"role":"user","content":[{"type":"input_image","image_url":"data:image/png;base64,%s"}]}`, + blob, + ) + } + b.WriteString(`],"unknown_passthrough_field":{"nested":"value"}}`) + return b.String() +} + +// BenchmarkResponsesRequestDecode measures per-request allocation for the +// full prepareRequest path, which includes unmarshal + model resolution + +// extra-param extraction. B/op measures cumulative allocations, not retained +// heap or peak memory. +func BenchmarkResponsesRequestDecode(b *testing.B) { + for _, sizeMiB := range []int{1, 8, 32} { + body := buildResponsesBody(sizeMiB) + + // Build the request once outside the timed loop: SetBodyString copies the + // whole body, which would otherwise dominate the reported B/op. + ctx := &fasthttp.RequestCtx{} + ctx.Request.SetBodyString(body) + + b.Run(fmt.Sprintf("body=%dMiB", sizeMiB), func(b *testing.B) { + b.SetBytes(int64(len(body))) + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, _, err := prepareRequest[ResponsesRequest](ctx, nil, responsesParamsKnownFields) + if err != nil { + b.Fatalf("prepareRequest: %v", err) + } + } + }) + } +} + +// BenchmarkSingleFullBodyParse measures decoding the same bodies into a raw +// field map. It omits typed and nested decoding, model resolution, and extra +// parameter extraction, so allocation ratios do not indicate parse counts. +func BenchmarkSingleFullBodyParse(b *testing.B) { + for _, sizeMiB := range []int{1, 8, 32} { + // Convert once outside the timed loop: []byte(body) copies the whole + // body, which would otherwise be measured on every iteration. + bodyBytes := []byte(buildResponsesBody(sizeMiB)) + + b.Run(fmt.Sprintf("body=%dMiB", sizeMiB), func(b *testing.B) { + b.SetBytes(int64(len(bodyBytes))) + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + var raw map[string]sonic.NoCopyRawMessage + if err := sonic.Unmarshal(bodyBytes, &raw); err != nil { + b.Fatalf("unmarshal: %v", err) + } + } + }) + } +} diff --git a/transports/bifrost-http/handlers/routing.go b/transports/bifrost-http/handlers/routing.go index 9f56210c93d..18b4b1e3f61 100644 --- a/transports/bifrost-http/handlers/routing.go +++ b/transports/bifrost-http/handlers/routing.go @@ -140,32 +140,32 @@ type RoutingTarget struct { // CreateRoutingRuleRequest represents the request body for creating a routing rule type CreateRoutingRuleRequest struct { - Name string `json:"name" validate:"required"` - Description string `json:"description,omitempty"` - Enabled *bool `json:"enabled,omitempty"` // nil = use DB default (true) - ChainRule *bool `json:"chain_rule,omitempty"` // nil = use DB default (false) - CelExpression string `json:"cel_expression"` - Targets []RoutingTarget `json:"targets"` // Required; weights must sum to 1 - Fallbacks []string `json:"fallbacks,omitempty"` - Scope string `json:"scope,omitempty"` // Defaults to "global" if not provided - ScopeID *string `json:"scope_id,omitempty"` - Query map[string]any `json:"query,omitempty"` - Priority int `json:"priority,omitempty"` // Defaults to 0 if not provided + Name string `json:"name" validate:"required"` + Description string `json:"description,omitempty"` + Enabled *bool `json:"enabled,omitempty"` // nil = use DB default (true) + ChainRule *bool `json:"chain_rule,omitempty"` // nil = use DB default (false) + CelExpression string `json:"cel_expression"` + Targets []RoutingTarget `json:"targets"` // Required; weights must sum to 1 + Fallbacks []configstoreTables.RoutingFallback `json:"fallbacks,omitempty"` + Scope string `json:"scope,omitempty"` // Defaults to "global" if not provided + ScopeID *string `json:"scope_id,omitempty"` + Query map[string]any `json:"query,omitempty"` + Priority int `json:"priority,omitempty"` // Defaults to 0 if not provided } // UpdateRoutingRuleRequest represents the request body for updating a routing rule type UpdateRoutingRuleRequest struct { - Name *string `json:"name,omitempty"` - Description *string `json:"description,omitempty"` - Enabled *bool `json:"enabled,omitempty"` - ChainRule *bool `json:"chain_rule,omitempty"` - CelExpression *string `json:"cel_expression,omitempty"` - Targets []RoutingTarget `json:"targets,omitempty"` // If provided, replaces all existing targets; weights must sum to 1 - Fallbacks []string `json:"fallbacks,omitempty"` - Query map[string]any `json:"query,omitempty"` - Priority *int `json:"priority,omitempty"` - Scope *string `json:"scope,omitempty"` - ScopeID *string `json:"scope_id,omitempty"` + Name *string `json:"name,omitempty"` + Description *string `json:"description,omitempty"` + Enabled *bool `json:"enabled,omitempty"` + ChainRule *bool `json:"chain_rule,omitempty"` + CelExpression *string `json:"cel_expression,omitempty"` + Targets []RoutingTarget `json:"targets,omitempty"` // If provided, replaces all existing targets; weights must sum to 1 + Fallbacks []configstoreTables.RoutingFallback `json:"fallbacks,omitempty"` + Query map[string]any `json:"query,omitempty"` + Priority *int `json:"priority,omitempty"` + Scope *string `json:"scope,omitempty"` + ScopeID *string `json:"scope_id,omitempty"` } // validRoutingScopes contains the allowed scope values for routing rules @@ -282,16 +282,17 @@ func validateRoutingTargets(targets []RoutingTarget) error { return nil } -// validateRoutingFallbacks ensures each fallback parses to a non-empty known provider via -// schemas.ParseModelString (e.g. "openai/gpt-4o", or "azure/" to use the incoming model). -func validateRoutingFallbacks(fallbacks []string) error { +// validateRoutingFallbacks ensures each fallback names a known provider, which rejects both an empty string and an unknown provider prefix. +func validateRoutingFallbacks(fallbacks []configstoreTables.RoutingFallback) error { for i, fb := range fallbacks { - if strings.TrimSpace(fb) == "" { - return fmt.Errorf("fallbacks[%d] must not be empty", i) + provider := strings.TrimSpace(string(fb.Provider)) + // The string form already collapses an unknown prefix to "" while decoding; the + // object form names its provider directly, so it is checked against the same set. + if provider == "" || !schemas.IsKnownProvider(provider) { + return fmt.Errorf("fallbacks[%d] %q is invalid: must use a known provider prefix (e.g. \"openai/gpt-4o\" or \"azure/\" for the incoming model)", i, fb.String()) } - provider, _ := schemas.ParseModelString(fb, "") - if provider == "" { - return fmt.Errorf("fallbacks[%d] %q is invalid: must use a known provider prefix (e.g. \"openai/gpt-4o\" or \"azure/\" for the incoming model)", i, fb) + if fb.ProviderKeyName != nil && strings.TrimSpace(*fb.ProviderKeyName) != "" { + return fmt.Errorf("fallbacks[%d] provider_key_name is a config.json-only field; send key_id over the API", i) } } return nil diff --git a/transports/bifrost-http/integrations/anthropic.go b/transports/bifrost-http/integrations/anthropic.go index cc7c624f517..6953d0f480f 100644 --- a/transports/bifrost-http/integrations/anthropic.go +++ b/transports/bifrost-http/integrations/anthropic.go @@ -137,6 +137,9 @@ func createAnthropicMessagesRouteConfig(pathPrefix string, logger schemas.Logger if anthropicReq, ok := req.(*anthropic.AnthropicMessageRequest); ok { bifrostReq := anthropicReq.ToBifrostResponsesRequest(ctx) normalizeBifrostInputContentBlocks(bifrostReq) + // Input is still owned here. Strip once before hooks/fallbacks + // share it, avoiding a conversation-slice copy per GPT attempt. + bifrostReq.ExtractAnthropicBillingHeader() return &schemas.BifrostRequest{ ResponsesRequest: bifrostReq, }, nil diff --git a/transports/bifrost-http/integrations/anthropic_test.go b/transports/bifrost-http/integrations/anthropic_test.go index 8041c727772..79c9fce60ce 100644 --- a/transports/bifrost-http/integrations/anthropic_test.go +++ b/transports/bifrost-http/integrations/anthropic_test.go @@ -14,6 +14,34 @@ import ( "github.com/valyala/fasthttp" ) +func TestAnthropicMessagesBillingHeaderNormalizedBeforeDispatch(t *testing.T) { + ctx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) + header := "x-anthropic-billing-header: cc_version=2.1.270.42c; cc_entrypoint=cli;" + var incoming anthropic.AnthropicMessageRequest + raw := []byte(`{"model":"openai/gpt-4o-mini","max_tokens":64,"system":[{"type":"text","text":"` + header + `"},{"type":"text","text":"Stable instructions"}],"messages":[{"role":"user","content":"Hello"}]}`) + if err := sonic.Unmarshal(raw, &incoming); err != nil { + t.Fatal(err) + } + for _, route := range createAnthropicMessagesRouteConfig("/anthropic", nil) { + request, err := route.RequestConverter(ctx, &incoming) + if err != nil { + t.Fatal(err) + } + blocks := request.ResponsesRequest.Input[0].Content.ContentBlocks + if len(blocks) != 1 || blocks[0].Text == nil || *blocks[0].Text != "Stable instructions" { + t.Fatalf("billing header survived ingress: %+v", blocks) + } + restored := request.ResponsesRequest.WithAnthropicBillingHeader() + if got := *restored.Input[0].Content.ContentBlocks[0].Text; got != header { + t.Fatalf("billing header not retained for Anthropic fallback: %q", got) + } + } + // Normalization only owns the newly converted slices, not the parsed source. + if len(incoming.System.ContentBlocks) != 2 || *incoming.System.ContentBlocks[0].Text != header { + t.Fatal("parsed Anthropic request was mutated") + } +} + // TestAnthropicRawStreamTextCodecRewritesOnlyTextDelta verifies the codec preserves provider-native event structure. func TestAnthropicRawStreamTextCodecRewritesOnlyTextDelta(t *testing.T) { codec := anthropicRawStreamTextCodec{} diff --git a/transports/bifrost-http/integrations/bedrock_test.go b/transports/bifrost-http/integrations/bedrock_test.go index e53d24fc7c3..31620843d5b 100644 --- a/transports/bifrost-http/integrations/bedrock_test.go +++ b/transports/bifrost-http/integrations/bedrock_test.go @@ -531,7 +531,7 @@ func Test_handleStreamingBedrockUnknownErrorResponseFallsBackToEventStreamExcept router := NewGenericRouter(nil, &mockHandlerStore{}, nil, nil, nil, bifrost.NewNoOpLogger()) ctx := &fasthttp.RequestCtx{} - cancelCalled := false + rec := newCancelRecorder() router.handleStreaming(ctx, nil, RouteConfig{ Type: RouteConfigTypeBedrock, StreamConfig: &StreamConfig{ @@ -541,9 +541,7 @@ func Test_handleStreamingBedrockUnknownErrorResponseFallsBackToEventStreamExcept } }, }, - }, stream, func() { - cancelCalled = true - }) + }, stream, rec.cancel) body, err := io.ReadAll(ctx.Response.BodyStream()) require.NoError(t, err) @@ -555,7 +553,13 @@ func Test_handleStreamingBedrockUnknownErrorResponseFallsBackToEventStreamExcept assert.Equal(t, "exception", eventStreamHeaderString(t, msg.Headers, ":message-type")) assert.Equal(t, "InternalServerException", eventStreamHeaderString(t, msg.Headers, ":exception-type")) assert.JSONEq(t, `{"__type":"InternalServerException","message":"An error occurred while processing your request"}`, string(msg.Payload)) - assert.False(t, cancelCalled, "fallback write should not cancel unless the client disconnects") + // handleStreaming cancels the request context on every exit path, a clean end-of-stream + // included, so the client-disconnect watcher goroutine ConvertToBifrostContext started + // cannot outlive the request (see Test_handleStreaming_RetentionCancelsAndLeavesNoGoroutine). + // The cancel therefore no longer distinguishes a successful fallback write from a real + // client disconnect — the frame asserted above is what does: on a disconnect the producer + // returns early and no InternalServerException is ever written. + rec.requireCancelled(t, "handleStreaming did not cancel the request context after the stream ended") } func eventStreamHeaderString(t *testing.T, headers eventstream.Headers, name string) string { diff --git a/transports/bifrost-http/integrations/openai.go b/transports/bifrost-http/integrations/openai.go index ee0dfd91dd5..dbe6e374976 100644 --- a/transports/bifrost-http/integrations/openai.go +++ b/transports/bifrost-http/integrations/openai.go @@ -283,6 +283,32 @@ func openAIResponsesWireConverter(ctx *schemas.BifrostContext, resp *schemas.Bif return openAIWireCostResponse(resp.WithDefaults()), nil } +// openAIChatStreamWireConverter frames raw upstream payloads at the HTTP boundary. +// The provider buffers non-forwarding frames (such as finish and usage) in one +// RawResponse separated by blank lines. SendEvent adds only one data prefix, so +// a bundle must be emitted as preformatted SSE with a prefix on every frame. +// Keep the raw capture itself unchanged for logging and plugins. +func openAIChatStreamWireConverter(_ *schemas.BifrostContext, resp *schemas.BifrostChatResponse) (string, interface{}, error) { + if resp.ExtraFields.Provider == schemas.OpenAI && resp.ExtraFields.RawResponse != nil { + raw := resp.ExtraFields.RawResponse + if frames, ok := raw.(string); ok && strings.Contains(frames, "\n\n") { + var framed strings.Builder + framed.Grow(len(frames) + 8*(strings.Count(frames, "\n\n")+1)) + for frame := range strings.SplitSeq(frames, "\n\n") { + if frame == "" { + continue + } + framed.WriteString("data: ") + framed.WriteString(frame) + framed.WriteString("\n\n") + } + return "", framed.String(), nil + } + return "", raw, nil + } + return "", openAIWireCostResponse(resp), nil +} + // CreateOpenAIRouteConfigs creates route configurations for OpenAI endpoints. func CreateOpenAIRouteConfigs(pathPrefix string, handlerStore lib.HandlerStore) []RouteConfig { var routes []RouteConfig @@ -478,14 +504,7 @@ func CreateOpenAIRouteConfigs(pathPrefix string, handlerStore lib.HandlerStore) }, ResponsesResponseConverter: openAIResponsesWireConverter, StreamConfig: &StreamConfig{ - ChatStreamResponseConverter: func(ctx *schemas.BifrostContext, resp *schemas.BifrostChatResponse) (string, interface{}, error) { - if resp.ExtraFields.Provider == schemas.OpenAI { - if resp.ExtraFields.RawResponse != nil { - return "", resp.ExtraFields.RawResponse, nil - } - } - return "", openAIWireCostResponse(resp), nil - }, + ChatStreamResponseConverter: openAIChatStreamWireConverter, TextStreamResponseConverter: func(ctx *schemas.BifrostContext, resp *schemas.BifrostTextCompletionResponse) (string, interface{}, error) { if resp.ExtraFields.Provider == schemas.OpenAI { if resp.ExtraFields.RawResponse != nil { @@ -621,14 +640,7 @@ func CreateOpenAIRouteConfigs(pathPrefix string, handlerStore lib.HandlerStore) return err }, StreamConfig: &StreamConfig{ - ChatStreamResponseConverter: func(ctx *schemas.BifrostContext, resp *schemas.BifrostChatResponse) (string, interface{}, error) { - if resp.ExtraFields.Provider == schemas.OpenAI { - if resp.ExtraFields.RawResponse != nil { - return "", resp.ExtraFields.RawResponse, nil - } - } - return "", openAIWireCostResponse(resp), nil - }, + ChatStreamResponseConverter: openAIChatStreamWireConverter, ErrorConverter: func(ctx *schemas.BifrostContext, err *schemas.BifrostError) interface{} { return err }, diff --git a/transports/bifrost-http/integrations/openai_stream_framing_test.go b/transports/bifrost-http/integrations/openai_stream_framing_test.go new file mode 100644 index 00000000000..53445a07ea9 --- /dev/null +++ b/transports/bifrost-http/integrations/openai_stream_framing_test.go @@ -0,0 +1,127 @@ +package integrations + +import ( + "context" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + bifrost "github.com/capsohq/bifrost/core" + "github.com/capsohq/bifrost/core/providers/openai" + "github.com/capsohq/bifrost/core/schemas" + "github.com/stretchr/testify/require" + "github.com/valyala/fasthttp" +) + +func TestOpenAIChatStreamWireConverter(t *testing.T) { + role := `{"choices":[{"delta":{"role":"assistant"}}]}` + content := `{"choices":[{"delta":{"content":"hello\\n\\nworld"}}],"extension":true}` + for _, tc := range []struct { + name string + raw interface{} + want interface{} + }{ + {"single frame", content, content}, + {"buffered role and content", role + "\n\n" + content, "data: " + role + "\n\ndata: " + content + "\n\n"}, + {"non-string raw value", []byte(content), []byte(content)}, + } { + t.Run(tc.name, func(t *testing.T) { + resp := &schemas.BifrostChatResponse{} + resp.ExtraFields.Provider = schemas.OpenAI + resp.ExtraFields.RawResponse = tc.raw + event, result, err := openAIChatStreamWireConverter(nil, resp) + require.NoError(t, err) + require.Empty(t, event) + require.Equal(t, tc.want, result) + require.Equal(t, tc.raw, resp.ExtraFields.RawResponse, "raw logging payload must not change") + }) + } + for _, provider := range []schemas.ModelProvider{schemas.OpenAI, schemas.Anthropic} { + resp := &schemas.BifrostChatResponse{} + resp.ExtraFields.Provider = provider + if provider != schemas.OpenAI { + resp.ExtraFields.RawResponse = role + "\n\n" + content + } + _, result, err := openAIChatStreamWireConverter(nil, resp) + require.NoError(t, err) + require.Same(t, resp, result, "normalized responses must retain the existing conversion path") + } +} + +// Exercise the real provider, route converter, and transport writer. Checking +// RawResponse alone misses unframed JSON that a client cannot receive as SSE. +func TestOpenAIChatStreamUsageSSEFraming(t *testing.T) { + for _, finish := range []string{"stop", "tool_calls"} { + for _, route := range CreateOpenAIRouteConfigs("", &mockHandlerStore{}) { + if route.Path != "/v1/chat/completions" && route.Path != "/chat/completions" && route.Path != "/openai/deployments/{deploymentPath:*}" { + continue + } + t.Run(finish+route.Path, func(t *testing.T) { + delta := `{"content":"hello"}` + if finish == "tool_calls" { + delta = `{"tool_calls":[{"index":0,"id":"call_1","type":"function","function":{"name":"get_weather","arguments":"{}"}}]}` + } + frames := []string{ + `{"id":"test","model":"gpt-4o-mini","choices":[{"index":0,"delta":{"role":"assistant"},"finish_reason":null}]}`, + fmt.Sprintf(`{"id":"test","model":"gpt-4o-mini","choices":[{"index":0,"delta":%s,"finish_reason":null}]}`, delta), + fmt.Sprintf(`{"id":"test","model":"gpt-4o-mini","choices":[{"index":0,"delta":{},"finish_reason":%q}]}`, finish), + `{"id":"test","model":"gpt-4o-mini","choices":[],"usage":{"prompt_tokens":5038,"completion_tokens":17,"total_tokens":5055,"prompt_tokens_details":{"cached_tokens":4992}},"unknown_upstream_field":"preserve me"}`, + } + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + for _, frame := range frames { + fmt.Fprintf(w, "data: %s\n\n", frame) + } + fmt.Fprint(w, "data: [DONE]\n\n") + })) + defer upstream.Close() + logger := bifrost.NewNoOpLogger() + provider := openai.NewOpenAIProvider(&schemas.ProviderConfig{ + NetworkConfig: schemas.NetworkConfig{BaseURL: upstream.URL}, + SendBackRawResponse: true, + }, logger) + parent, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + ctx := schemas.NewBifrostContext(parent, schemas.NoDeadline) + stream, err := provider.ChatCompletionStream(ctx, + func(_ *schemas.BifrostContext, response *schemas.BifrostResponse, err *schemas.BifrostError) (*schemas.BifrostResponse, *schemas.BifrostError) { + // The core pipeline normally stamps the routed provider before + // the HTTP integration selects its native passthrough converter. + if response != nil && response.ChatResponse != nil { + response.ChatResponse.ExtraFields.Provider = schemas.OpenAI + } + return response, err + }, nil, schemas.Key{Value: *schemas.NewSecretVar("test-key")}, + &schemas.BifrostChatRequest{ + Provider: schemas.OpenAI, Model: "gpt-4o-mini", + Input: []schemas.ChatMessage{{Role: schemas.ChatMessageRoleUser, Content: &schemas.ChatMessageContent{ContentStr: schemas.Ptr("hello")}}}, + }) + require.Nil(t, err) + router := NewGenericRouter(nil, &mockHandlerStore{}, nil, nil, nil, logger) + httpCtx := &fasthttp.RequestCtx{} + router.handleStreaming(httpCtx, ctx, route, stream, cancel) + body, readErr := io.ReadAll(httpCtx.Response.BodyStream()) + require.NoError(t, readErr) + // SSE dispatches only data fields; bare JSON is an unknown field and + // must not be rescued by a lenient JSON-line parser. + var events []string + for _, event := range strings.Split(string(body), "\n\n") { + var data []string + for _, line := range strings.Split(event, "\n") { + if value, ok := strings.CutPrefix(line, "data:"); ok { + data = append(data, strings.TrimPrefix(value, " ")) + } + } + if len(data) > 0 { + events = append(events, strings.Join(data, "\n")) + } + } + require.Equal(t, append(frames, "[DONE]"), events, "wire stream: %s", body) + }) + } + } +} diff --git a/transports/bifrost-http/integrations/router.go b/transports/bifrost-http/integrations/router.go index 21931f1193b..a56b9f313ab 100644 --- a/transports/bifrost-http/integrations/router.go +++ b/transports/bifrost-http/integrations/router.go @@ -217,6 +217,10 @@ type EmbeddingResponseConverter func(ctx *schemas.BifrostContext, resp *schemas. // It takes a BifrostRerankResponse and returns the format expected by the specific integration. type RerankResponseConverter func(ctx *schemas.BifrostContext, resp *schemas.BifrostRerankResponse) (interface{}, error) +// DecisionResponseConverter is a function that converts BifrostDecisionResponse to integration-specific format. +// It takes a BifrostDecisionResponse and returns the format expected by the specific integration. +type DecisionResponseConverter func(ctx *schemas.BifrostContext, resp *schemas.BifrostDecisionResponse) (interface{}, error) + // OCRResponseConverter is a function that converts BifrostOCRResponse to integration-specific format. // It takes a BifrostOCRResponse and returns the format expected by the specific integration. type OCRResponseConverter func(ctx *schemas.BifrostContext, resp *schemas.BifrostOCRResponse) (interface{}, error) @@ -467,6 +471,7 @@ const ( RouteConfigTypeGenAI RouteConfigType = "genai" RouteConfigTypeBedrock RouteConfigType = "bedrock" RouteConfigTypeCohere RouteConfigType = "cohere" + RouteConfigTypeTypesafe RouteConfigType = "typesafe" ) // RouteConfig defines the configuration for a single route in an integration. @@ -498,6 +503,7 @@ type RouteConfig struct { AsyncResponsesResponseConverter AsyncResponsesResponseConverter // Function to convert AsyncJobResponse to integration format (SHOULD NOT BE NIL) EmbeddingResponseConverter EmbeddingResponseConverter // Function to convert BifrostEmbeddingResponse to integration format (SHOULD NOT BE NIL) RerankResponseConverter RerankResponseConverter // Function to convert BifrostRerankResponse to integration format + DecisionResponseConverter DecisionResponseConverter // Function to convert BifrostDecisionResponse to integration format OCRResponseConverter OCRResponseConverter // Function to convert BifrostOCRResponse to integration format SpeechResponseConverter SpeechResponseConverter // Function to convert BifrostSpeechResponse to integration format (SHOULD NOT BE NIL) TranscriptionResponseConverter TranscriptionResponseConverter // Function to convert BifrostTranscriptionResponse to integration format (SHOULD NOT BE NIL) @@ -1170,6 +1176,29 @@ func (g *GenericRouter) handleNonStreamingRequest(ctx *fasthttp.RequestCtx, conf response = rerankResponse } + case bifrostReq.DecisionRequest != nil: + decisionResponse, bifrostErr := g.client.DecisionRequest(bifrostCtx, bifrostReq.DecisionRequest) + if bifrostErr != nil { + g.sendError(ctx, bifrostCtx, config.ErrorConverter, bifrostErr) + return + } + if config.PostCallback != nil { + if err := config.PostCallback(ctx, req, decisionResponse); err != nil { + g.sendError(ctx, bifrostCtx, config.ErrorConverter, newBifrostError(err, "failed to execute post-request callback")) + return + } + } + if decisionResponse == nil { + g.sendError(ctx, bifrostCtx, config.ErrorConverter, newBifrostError(nil, "Bifrost response is nil after post-request callback")) + return + } + bifrostExtraFields = decisionResponse.ExtraFields + if config.DecisionResponseConverter != nil { + response, err = config.DecisionResponseConverter(bifrostCtx, decisionResponse) + } else { + response = decisionResponse + } + case bifrostReq.OCRRequest != nil: ocrResponse, bifrostErr := g.client.OCRRequest(bifrostCtx, bifrostReq.OCRRequest) if bifrostErr != nil { @@ -2830,9 +2859,15 @@ func (g *GenericRouter) handleStreamingRequest(ctx *fasthttp.RequestCtx, config // // CONTEXT CANCELLATION: // -// The cancel function is called ONLY when client disconnects are detected via write errors. -// Bifrost handles cleanup internally for normal completion and errors, so we only cancel -// upstream streams when write errors indicate the client has disconnected. +// The producer goroutine owns the cancel function and calls it on every exit path: eagerly +// when a write error reveals the client has disconnected (so the upstream stream is torn down +// at once), and otherwise from its deferred cleanup once the stream has finished. +// +// Cancelling on normal completion is not optional. ConvertToBifrostContext starts a +// client-disconnect watcher per request (lib.startClientDisconnectWatcher), and that goroutine +// only stops when this context is cancelled or the client socket dies. Returning without +// cancelling leaks the watcher and the entire request-scoped BifrostContext for as long as the +// client keeps its connection open. func (g *GenericRouter) handleStreaming(ctx *fasthttp.RequestCtx, bifrostCtx *schemas.BifrostContext, config RouteConfig, streamChan chan *schemas.BifrostStreamChunk, cancel context.CancelFunc) { // Signal to tracing middleware that trace completion should be deferred // The streaming callback will complete the trace after the stream ends @@ -2897,6 +2932,15 @@ func (g *GenericRouter) handleStreaming(ctx *fasthttp.RequestCtx, bifrostCtx *sc } defer func() { + // Ends the client-disconnect watcher ConvertToBifrostContext started for this + // request. The watcher's only other exits are the client socket dying or an + // explicit cancel, and its parent (fasthttp's RequestCtx) fires Done only on + // server shutdown, so a stream that completed normally used to leave the + // watcher polling forever and pinning this whole BifrostContext until the + // client's keep-alive connection closed. Registered first so it runs last, + // leaving traceCompleter and the post-hooks below an uncancelled context. + // cancel is idempotent, so the explicit error-path calls still stand. + defer cancel() // Must run before reader.Done(): closing eventCh while the heartbeat goroutine // could still be mid-send on it panics ("send on closed channel"). See // lib.StopSSEHeartbeat's doc for the full ordering rationale. diff --git a/transports/bifrost-http/integrations/router_stream_interception_test.go b/transports/bifrost-http/integrations/router_stream_interception_test.go index adf0673c915..fbe0b55efbb 100644 --- a/transports/bifrost-http/integrations/router_stream_interception_test.go +++ b/transports/bifrost-http/integrations/router_stream_interception_test.go @@ -60,7 +60,7 @@ func newStreamInterceptionError(message string) *schemas.StreamInterceptionError } } -func runHandleStreamingWithInterceptor(t *testing.T, config RouteConfig, interceptErr error) (string, bool) { +func runHandleStreamingWithInterceptor(t *testing.T, config RouteConfig, interceptErr error) (string, *cancelRecorder) { t.Helper() stream := make(chan *schemas.BifrostStreamChunk, 1) @@ -73,14 +73,12 @@ func runHandleStreamingWithInterceptor(t *testing.T, config RouteConfig, interce } router := NewGenericRouter(nil, handlerStore, nil, nil, nil, bifrost.NewNoOpLogger()) ctx := &fasthttp.RequestCtx{} - cancelCalled := false - router.handleStreaming(ctx, nil, config, stream, func() { - cancelCalled = true - }) + rec := newCancelRecorder() + router.handleStreaming(ctx, nil, config, stream, rec.cancel) body, err := io.ReadAll(ctx.Response.BodyStream()) require.NoError(t, err) - return string(body), cancelCalled + return string(body), rec } // A StreamInterceptionError must be emitted through the integration's error @@ -100,7 +98,7 @@ func Test_handleStreamingInterceptionErrorUsesErrorConverter(t *testing.T) { }, } - body, cancelCalled := runHandleStreamingWithInterceptor(t, config, + body, rec := runHandleStreamingWithInterceptor(t, config, wrapInterceptionError(newStreamInterceptionError("Response blocked by content policy"))) assert.Contains(t, body, `data: `) @@ -108,7 +106,7 @@ func Test_handleStreamingInterceptionErrorUsesErrorConverter(t *testing.T) { assert.Contains(t, body, `"type":"guardrail_intervention"`) assert.NotContains(t, body, "failed to intercept chunk", "structured interception errors must not leak the internal plugin wrap") - assert.True(t, cancelCalled, "stream must be terminated after an interception error") + rec.requireCancelled(t, "stream must be terminated after an interception error") } // On Anthropic-style routes the converter returns a complete SSE string with @@ -123,14 +121,14 @@ func Test_handleStreamingInterceptionErrorAnthropicSSEFraming(t *testing.T) { }, } - body, cancelCalled := runHandleStreamingWithInterceptor(t, config, + body, rec := runHandleStreamingWithInterceptor(t, config, wrapInterceptionError(newStreamInterceptionError("Response blocked by content policy"))) assert.True(t, strings.HasPrefix(body, "event: error\ndata: "), "Anthropic streaming errors use the provider's native SSE framing, got: %q", body) assert.Contains(t, body, `"Response blocked by content policy"`) assert.NotContains(t, body, "failed to intercept chunk") - assert.True(t, cancelCalled) + rec.requireCancelled(t, "handleStreaming must cancel the request context before returning") } // Structured interception errors must be sanitized like upstream provider @@ -165,7 +163,7 @@ func Test_handleStreamingInterceptionErrorBedrockEventStream(t *testing.T) { }, } - body, cancelCalled := runHandleStreamingWithInterceptor(t, config, + body, rec := runHandleStreamingWithInterceptor(t, config, wrapInterceptionError(newStreamInterceptionError("Response blocked by content policy"))) require.NotEmpty(t, body) @@ -178,7 +176,7 @@ func Test_handleStreamingInterceptionErrorBedrockEventStream(t *testing.T) { assert.Equal(t, "guardrail_intervention", eventStreamHeaderString(t, msg.Headers, ":exception-type")) assert.JSONEq(t, `{"__type":"guardrail_intervention","message":"Response blocked by content policy"}`, string(msg.Payload)) assert.NotContains(t, string(msg.Payload), "failed to intercept chunk") - assert.True(t, cancelCalled) + rec.requireCancelled(t, "handleStreaming must cancel the request context before returning") } // Plain (non-structured) interceptor errors keep the existing flat behavior. @@ -192,12 +190,12 @@ func Test_handleStreamingPlainInterceptionErrorKeepsFlatFormat(t *testing.T) { }, } - body, cancelCalled := runHandleStreamingWithInterceptor(t, config, + body, rec := runHandleStreamingWithInterceptor(t, config, wrapInterceptionError(fmt.Errorf("plugin exploded"))) assert.Contains(t, body, "event: error\ndata: ") assert.Contains(t, body, `{"error":"failed to intercept chunk with plugin test-plugin: plugin exploded"}`) - assert.True(t, cancelCalled) + rec.requireCancelled(t, "handleStreaming must cancel the request context before returning") } // A route configuration bug must produce a stream error instead of calling a @@ -222,11 +220,11 @@ func Test_handleStreamingMissingSpeechConverterReturnsError(t *testing.T) { router := NewGenericRouter(nil, &mockHandlerStore{}, nil, nil, nil, bifrost.NewNoOpLogger()) ctx := &fasthttp.RequestCtx{} bifrostCtx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) - cancelCalled := false - router.handleStreaming(ctx, bifrostCtx, config, stream, func() { cancelCalled = true }) + rec := newCancelRecorder() + router.handleStreaming(ctx, bifrostCtx, config, stream, rec.cancel) body, err := io.ReadAll(ctx.Response.BodyStream()) require.NoError(t, err) assert.Contains(t, string(body), lib.ClientSafeInternalErrorMessage) - assert.True(t, cancelCalled) + rec.requireCancelled(t, "handleStreaming must cancel the request context before returning") } diff --git a/transports/bifrost-http/integrations/router_test.go b/transports/bifrost-http/integrations/router_test.go index da94213187d..ecd47f57944 100644 --- a/transports/bifrost-http/integrations/router_test.go +++ b/transports/bifrost-http/integrations/router_test.go @@ -10,10 +10,12 @@ import ( "net" "net/http" "strings" + "sync" "testing" "time" "github.com/bytedance/sonic" + bifrost "github.com/capsohq/bifrost/core" "github.com/capsohq/bifrost/core/providers/anthropic" "github.com/capsohq/bifrost/core/providers/openai" "github.com/capsohq/bifrost/core/schemas" @@ -933,3 +935,87 @@ func TestApplyPassthroughCallerAuth_APIKeysStayStripped(t *testing.T) { }) } } + +// Test_handleStreaming_RetentionCancelsAndLeavesNoGoroutine is the regression test for the +// client-disconnect watcher leak found in a production heap dump. +// +// ConvertToBifrostContext starts one lib.startClientDisconnectWatcher goroutine per +// request. That goroutine's only exits are an explicit cancel or the client socket dying; +// its parent is fasthttp's RequestCtx, whose Done fires only on server shutdown. So a +// handler that returns without cancelling leaves the watcher polling the socket every +// 500ms forever, pinning the whole request-scoped BifrostContext with it. +// +// handleStreaming used to cancel ONLY on write errors ("client disconnected"), which meant +// every SUCCESSFUL stream leaked a watcher plus its context until the client's keep-alive +// connection closed. Two production pods showed 596 and 546 watcher goroutines behind just +// 34 and 38 fasthttp connection goroutines -- roughly 15 leaked contexts per open +// connection, holding about 2.0 GB of a 2.66 GB heap that GC could not reclaim because it +// was all genuinely reachable. +// +// A stream that ends normally must cancel, exactly as the client-disconnect and +// passthrough paths already do. +func Test_handleStreaming_RetentionCancelsAndLeavesNoGoroutine(t *testing.T) { + stream := make(chan *schemas.BifrostStreamChunk) + router := NewGenericRouter(nil, &mockHandlerStore{}, nil, nil, nil, bifrost.NewNoOpLogger()) + ctx := &fasthttp.RequestCtx{} + + rec := newCancelRecorder() + router.handleStreaming(ctx, nil, RouteConfig{}, stream, rec.cancel) + + // Drain the response body so the producer's SendEvent calls succeed and it reaches + // its normal end-of-stream path rather than the write-error path (which has always + // cancelled, and would make this test pass for the wrong reason). + readDone := make(chan error, 1) + go func() { + _, err := io.ReadAll(ctx.Response.BodyStream()) + readDone <- err + }() + + stream <- &schemas.BifrostStreamChunk{} + close(stream) // normal completion: no write ever failed + + select { + case err := <-readDone: + require.NoError(t, err) + case <-time.After(5 * time.Second): + t.Fatal("response body stream never closed") + } + + rec.requireCancelled(t, "handleStreaming returned without cancelling the request context on "+ + "normal stream completion: the client-disconnect watcher goroutine and the whole "+ + "BifrostContext it captures leak until the client closes its connection") + + // Deliberately no process-wide goroutine count here. runtime.NumGoroutine() sweeps up + // fasthttp workers and TCP teardown, which flap in a shared test binary, and a flaky + // release gate trains people to rerun until green. The outcome this mechanism protects + // is asserted precisely, by name, in + // lib.TestClientDisconnectWatcher_RetentionNoGoroutineLeak against a real socket. +} + +// cancelRecorder captures the cancel func handed to handleStreaming. +// +// handleStreaming cancels from its producer goroutine's deferred cleanup, which runs after +// the response body stream has closed. A test that samples a plain bool right after +// io.ReadAll therefore both races that goroutine (caught by -race) and can read it before +// the cancel lands. Recording into a channel fixes both. +type cancelRecorder struct { + once sync.Once + done chan struct{} +} + +func newCancelRecorder() *cancelRecorder { + return &cancelRecorder{done: make(chan struct{})} +} + +// cancel is the context.CancelFunc stand-in passed into handleStreaming. +func (c *cancelRecorder) cancel() { c.once.Do(func() { close(c.done) }) } + +// requireCancelled fails the test unless cancel lands within the timeout. +func (c *cancelRecorder) requireCancelled(t *testing.T, msg string) { + t.Helper() + select { + case <-c.done: + case <-time.After(5 * time.Second): + t.Fatal(msg) + } +} diff --git a/transports/bifrost-http/integrations/typesafe.go b/transports/bifrost-http/integrations/typesafe.go new file mode 100644 index 00000000000..acb990085b5 --- /dev/null +++ b/transports/bifrost-http/integrations/typesafe.go @@ -0,0 +1,136 @@ +package integrations + +import ( + "bytes" + "context" + "encoding/json" + "errors" + + "github.com/bytedance/sonic" + bifrost "github.com/capsohq/bifrost/core" + "github.com/capsohq/bifrost/core/providers/typesafe" + "github.com/capsohq/bifrost/core/schemas" + "github.com/capsohq/bifrost/transports/bifrost-http/lib" + "github.com/valyala/fasthttp" +) + +// TypesafeRouter holds route registrations for Typesafe endpoints: the native +// System One evaluation endpoint and model listing. +type TypesafeRouter struct { + *GenericRouter +} + +// NewTypesafeRouter creates a new TypesafeRouter with the given bifrost client. +func NewTypesafeRouter(client *bifrost.Bifrost, handlerStore lib.HandlerStore, accessResolver AccessResolver, logger schemas.Logger) *TypesafeRouter { + return &TypesafeRouter{ + GenericRouter: NewGenericRouter(client, handlerStore, accessResolver, CreateTypesafeRouteConfigs("/typesafe"), nil, logger), + } +} + +// CreateTypesafeRouteConfigs creates route configurations for Typesafe API endpoints. +func CreateTypesafeRouteConfigs(pathPrefix string) []RouteConfig { + var routes []RouteConfig + + // Decision endpoint (v1/systemone) + routes = append(routes, RouteConfig{ + Type: RouteConfigTypeTypesafe, + Path: pathPrefix + "/v1/systemone", + Method: "POST", + GetHTTPRequestType: func(ctx *fasthttp.RequestCtx) schemas.RequestType { + return schemas.DecisionRequest + }, + GetRequestTypeInstance: func(ctx context.Context) interface{} { + return &typesafe.TypesafeDecisionRequest{} + }, + RequestConverter: func(ctx *schemas.BifrostContext, req interface{}) (*schemas.BifrostRequest, error) { + if typesafeReq, ok := req.(*typesafe.TypesafeDecisionRequest); ok { + decisionReq, err := typesafeReq.ToBifrostDecisionRequest(ctx) + if err != nil { + return nil, err + } + return &schemas.BifrostRequest{ + DecisionRequest: decisionReq, + }, nil + } + return nil, errors.New("invalid request type") + }, + DecisionResponseConverter: func(ctx *schemas.BifrostContext, resp *schemas.BifrostDecisionResponse) (interface{}, error) { + native, err := typesafe.ToTypesafeNativeDecisionResponse(resp) + if err != nil { + return nil, err + } + // The raw provider body was kept before post-hooks ran; relay it only while it + // still carries the answers the processed response does, so a post-hook's change + // reaches the client. + if resp.ExtraFields.Provider == schemas.Typesafe && resp.ExtraFields.RawResponse != nil && + typesafeRawAnswersMatch(resp.ExtraFields.RawResponse, native) { + return resp.ExtraFields.RawResponse, nil + } + return native, nil + }, + ErrorConverter: func(ctx *schemas.BifrostContext, err *schemas.BifrostError) interface{} { + return typesafe.ToTypesafeNativeError(err) + }, + }) + + // Models endpoint. Typesafe documents no upstream listing API; the response + // is synthesized from the provider's static catalog in native shape. + routes = append(routes, RouteConfig{ + Type: RouteConfigTypeTypesafe, + Path: pathPrefix + "/v1/models", + Method: "GET", + GetHTTPRequestType: func(ctx *fasthttp.RequestCtx) schemas.RequestType { + return schemas.ListModelsRequest + }, + GetRequestTypeInstance: func(ctx context.Context) interface{} { + return &schemas.BifrostListModelsRequest{} + }, + RequestConverter: func(ctx *schemas.BifrostContext, req interface{}) (*schemas.BifrostRequest, error) { + if listModelsReq, ok := req.(*schemas.BifrostListModelsRequest); ok { + if listModelsReq.Provider == "" { + listModelsReq.Provider = schemas.Typesafe + } + return &schemas.BifrostRequest{ + ListModelsRequest: listModelsReq, + }, nil + } + return nil, errors.New("invalid request type") + }, + ListModelsResponseConverter: func(ctx *schemas.BifrostContext, resp *schemas.BifrostListModelsResponse) (interface{}, error) { + return typesafe.ToTypesafeNativeListModelsResponse(resp), nil + }, + ErrorConverter: func(ctx *schemas.BifrostContext, err *schemas.BifrostError) interface{} { + return typesafe.ToTypesafeNativeError(err) + }, + }) + + return routes +} + +// typesafeRawAnswersMatch reports whether a raw systemone body carries the same answers +// as native, compared in their serialized form. A body that cannot be read as a systemone +// response does not match. +func typesafeRawAnswersMatch(raw interface{}, native *typesafe.TypesafeDecisionResponse) bool { + var body []byte + switch v := raw.(type) { + case json.RawMessage: + body = v + case []byte: + body = v + case string: + body = []byte(v) + default: + marshaled, err := sonic.Marshal(v) + if err != nil { + return false + } + body = marshaled + } + var parsed typesafe.TypesafeDecisionResponse + if err := sonic.Unmarshal(body, &parsed); err != nil { + return false + } + got, err1 := sonic.ConfigStd.Marshal(parsed.Answers) + want, err2 := sonic.ConfigStd.Marshal(native.Answers) + return err1 == nil && err2 == nil && bytes.Equal(got, want) +} diff --git a/transports/bifrost-http/integrations/typesafe_test.go b/transports/bifrost-http/integrations/typesafe_test.go new file mode 100644 index 00000000000..89b95b2ef14 --- /dev/null +++ b/transports/bifrost-http/integrations/typesafe_test.go @@ -0,0 +1,55 @@ +package integrations + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/capsohq/bifrost/core/schemas" +) + +// The raw systemone body is kept before post-hooks run, so it is relayed only while it +// still carries the processed answers; once a post-hook changes an answer the client gets +// the processed response instead. +func TestTypesafeDecisionConverterRelaysRawOnlyWhileAnswersMatch(t *testing.T) { + var convert DecisionResponseConverter + for _, route := range CreateTypesafeRouteConfigs("/typesafe") { + if route.DecisionResponseConverter != nil { + convert = route.DecisionResponseConverter + } + } + if convert == nil { + t.Fatal("no Typesafe decision route") + } + raw := json.RawMessage(`{"model":"jev-1.13.0","answers":{"is_spam":{"type":"noul","noul":0.9}}}`) + response := func(value float64) *schemas.BifrostDecisionResponse { + return &schemas.BifrostDecisionResponse{ + Model: "jev-1.13.0", + Answers: map[string]schemas.DecisionAnswer{"is_spam": {Kind: schemas.DecisionKindNoul, Value: value}}, + ExtraFields: schemas.BifrostResponseExtraFields{ + Provider: schemas.Typesafe, + RawResponse: raw, + }, + } + } + + got, err := convert(nil, response(0.9)) + if err != nil { + t.Fatalf("convert: %v", err) + } + if _, isRaw := got.(json.RawMessage); !isRaw { + t.Fatalf("unchanged answers: got %T, want the raw body", got) + } + + got, err = convert(nil, response(0.1)) + if err != nil { + t.Fatalf("convert: %v", err) + } + if _, isRaw := got.(json.RawMessage); isRaw { + t.Fatal("a post-hook's changed answer was dropped in favour of the raw body") + } + body, _ := json.Marshal(got) + if want := `"noul":0.1`; !json.Valid(body) || !strings.Contains(string(body), want) { + t.Fatalf("converted body %s does not carry %s", body, want) + } +} diff --git a/transports/bifrost-http/integrations/utils.go b/transports/bifrost-http/integrations/utils.go index fb85633ee70..801e30289c4 100644 --- a/transports/bifrost-http/integrations/utils.go +++ b/transports/bifrost-http/integrations/utils.go @@ -368,6 +368,10 @@ func (g *GenericRouter) extractAndParseFallbacks(ctx *schemas.BifrostContext, re if bifrostReq.RerankRequest != nil { bifrostReq.RerankRequest.Fallbacks = parsedFallbacks } + case schemas.DecisionRequest: + if bifrostReq.DecisionRequest != nil { + bifrostReq.DecisionRequest.Fallbacks = parsedFallbacks + } case schemas.SpeechRequest, schemas.SpeechStreamRequest: if bifrostReq.SpeechRequest != nil { bifrostReq.SpeechRequest.Fallbacks = parsedFallbacks diff --git a/transports/bifrost-http/lib/config.go b/transports/bifrost-http/lib/config.go index 7183bfa5126..450d7d143b1 100644 --- a/transports/bifrost-http/lib/config.go +++ b/transports/bifrost-http/lib/config.go @@ -2777,6 +2777,15 @@ func resolveGovernanceKeyReferences(ctx context.Context, config *Config, governa break } } + if !usesNameRefs { + for j := range governanceConfig.RoutingRules[i].ParsedFallbacks { + fallback := &governanceConfig.RoutingRules[i].ParsedFallbacks[j] + if fallback.ProviderKeyName != nil && strings.TrimSpace(*fallback.ProviderKeyName) != "" { + usesNameRefs = true + break + } + } + } if usesNameRefs { break } @@ -2855,6 +2864,31 @@ func resolveGovernanceKeyReferences(ctx context.Context, config *Config, governa target.KeyID = bifrost.Ptr(keyID) target.ProviderKeyName = nil } + + for j := range governanceConfig.RoutingRules[i].ParsedFallbacks { + fallback := &governanceConfig.RoutingRules[i].ParsedFallbacks[j] + keyName := "" + if fallback.ProviderKeyName != nil { + keyName = strings.TrimSpace(*fallback.ProviderKeyName) + } + if keyName == "" { + fallback.ProviderKeyName = nil + continue + } + if strings.TrimSpace(fallback.KeyID) != "" { + return fmt.Errorf("routing rule %q fallback cannot set key_id together with provider_key_name", governanceConfig.RoutingRules[i].ID) + } + if strings.TrimSpace(string(fallback.Provider)) == "" { + return fmt.Errorf("routing rule %q fallback provider_key_name requires provider to be set", governanceConfig.RoutingRules[i].ID) + } + + keyID, err := resolveProviderKeyIDByProviderAndName(string(fallback.Provider), keyName) + if err != nil { + return fmt.Errorf("routing rule %q fallback provider_key_name resolution failed: %w", governanceConfig.RoutingRules[i].ID, err) + } + fallback.KeyID = keyID + fallback.ProviderKeyName = nil + } } for i := range governanceConfig.PricingOverrides { diff --git a/transports/bifrost-http/lib/ctx.go b/transports/bifrost-http/lib/ctx.go index bd7dd7ddde3..060e149c694 100644 --- a/transports/bifrost-http/lib/ctx.go +++ b/transports/bifrost-http/lib/ctx.go @@ -582,6 +582,20 @@ func ConvertToBifrostContext(ctx *fasthttp.RequestCtx, store HandlerStore) (*sch } return true } + // Session affinity: whether this request lets its session decide where it goes. + if keyStr == "x-bf-session-affinity" { + switch strings.ToLower(strings.TrimSpace(string(value))) { + case "on", "true", "1": + bifrostCtx.SetValue(schemas.BifrostContextKeySessionAffinity, true) + case "off", "false", "0": + bifrostCtx.SetValue(schemas.BifrostContextKeySessionAffinity, false) + default: + if logger != nil { + logger.Warn("x-bf-session-affinity is not on or off, ignoring") + } + } + return true + } if labelName, ok := strings.CutPrefix(keyStr, "x-bf-eh-"); ok { // Skip empty header names after prefix removal if labelName == "" { diff --git a/transports/bifrost-http/lib/ctx_test.go b/transports/bifrost-http/lib/ctx_test.go index be18aa52daa..e12d4b45296 100644 --- a/transports/bifrost-http/lib/ctx_test.go +++ b/transports/bifrost-http/lib/ctx_test.go @@ -1,9 +1,11 @@ package lib import ( + "bytes" "context" "errors" "net" + "runtime" "strings" "testing" "time" @@ -1141,3 +1143,132 @@ func TestConvertToBifrostContextCancelsSeededContextWhenClientDisconnects(t *tes t.Fatal("handler never reported an outcome") } } + +// countWatcherGoroutines reports how many goroutines are currently inside +// startClientDisconnectWatcher, by name rather than by counting everything. +// +// runtime.NumGoroutine() is process-global, so a fasthttp worker or a TCP teardown +// finishing at the wrong moment makes a whole-process count flap. That is fatal for a +// release gate: a flaky assertion trains people to rerun until green. Naming the frame +// makes the measurement immune to every goroutine that is not the subject. +func countWatcherGoroutines() int { + buf := make([]byte, 1<<20) + buf = buf[:runtime.Stack(buf, true)] + return bytes.Count(buf, []byte("startClientDisconnectWatcher")) +} + +// waitForWatchers polls until the watcher count drops to want, returning the final count. +// Teardown is not synchronous with cancel, so a poll beats a fixed sleep. +func waitForWatchers(want int, within time.Duration) int { + deadline := time.Now().Add(within) + for time.Now().Before(deadline) { + if n := countWatcherGoroutines(); n <= want { + return n + } + time.Sleep(10 * time.Millisecond) + } + return countWatcherGoroutines() +} + +// TestClientDisconnectWatcher_RetentionNoGoroutineLeak is the retention regression test +// for the leak found in a production heap dump. +// +// ConvertToBifrostContext starts one startClientDisconnectWatcher goroutine per request. +// Its only exits are an explicit cancel or the client socket dying, and its parent is +// fasthttp's RequestCtx, whose Done fires only on server shutdown. A handler that returns +// without cancelling therefore leaves the watcher polling every 500ms forever, pinning the +// entire request-scoped BifrostContext with it. +// +// That is what GenericRouter.handleStreaming used to do on every SUCCESSFUL stream. Two +// production pods showed 596 and 546 watcher goroutines behind just 34 and 38 fasthttp +// connection goroutines, holding roughly 2.0 GB of a 2.66 GB heap that GC could not +// reclaim because all of it was genuinely reachable. +// +// The existing tests here only assert the context's cancellation semantics. None asserts +// the goroutine actually goes away, which is the property that was violated. +func TestClientDisconnectWatcher_RetentionNoGoroutineLeak(t *testing.T) { + if !clientDisconnectPeekSupported { + t.Skip("no socket peeking on this platform, so no watcher goroutine is started") + } + + handlerDone := make(chan struct{}) + baselineCh := make(chan int, 1) + + client := serveOneConnection(t, func(ctx *fasthttp.RequestCtx) { + baselineCh <- countWatcherGoroutines() + bifrostCtx, cancel := ConvertToBifrostContext(ctx, testHandlerStore{}) + if bifrostCtx == nil { + t.Error("expected a context") + } + // Exactly what a correct handler does on the way out. The bug was omitting it. + cancel() + close(handlerDone) + }) + + if _, err := client.Write([]byte(chatCompletionRawRequest)); err != nil { + t.Fatalf("write request: %v", err) + } + var baseline int + select { + case <-handlerDone: + baseline = <-baselineCh + case <-time.After(5 * time.Second): + t.Fatal("handler never ran") + } + + // The client socket stays open, which is the whole point: a leaked watcher would + // keep peeking a healthy keep-alive connection indefinitely. Only the cancel can + // end it, so this fails if the cancel path ever stops reaching the watcher. + if final := waitForWatchers(baseline, 5*time.Second); final > baseline { + t.Errorf("client-disconnect watcher goroutines went %d -> %d and stayed there after "+ + "the request completed; the watcher is outliving its request and pinning the "+ + "request-scoped BifrostContext it captured", baseline, final) + } +} + +// TestClientDisconnectWatcher_RetentionWatcherActuallyStarts stops the test above from passing +// for the wrong reason. If no watcher were ever started, a "no leak" assertion would be +// trivially true, so this pins that one genuinely runs for the life of the request. +func TestClientDisconnectWatcher_RetentionWatcherActuallyStarts(t *testing.T) { + if !clientDisconnectPeekSupported { + t.Skip("no socket peeking on this platform, so no watcher goroutine is started") + } + + type sample struct{ before, during int } + observed := make(chan sample, 1) + release := make(chan struct{}) + + client := serveOneConnection(t, func(ctx *fasthttp.RequestCtx) { + before := countWatcherGoroutines() + _, cancel := ConvertToBifrostContext(ctx, testHandlerStore{}) + defer cancel() + // startClientDisconnectWatcher spawns its goroutine, so sampling immediately can + // run before the scheduler has got to it and fail while the watcher is working + // correctly. Poll until it appears, bounded so a genuinely absent watcher still + // fails rather than hanging. + during := before + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) && during <= before { + time.Sleep(10 * time.Millisecond) + during = countWatcherGoroutines() + } + observed <- sample{before: before, during: during} + <-release + }) + + if _, err := client.Write([]byte(chatCompletionRawRequest)); err != nil { + t.Fatalf("write request: %v", err) + } + + select { + case s := <-observed: + if s.during <= s.before { + t.Errorf("watcher goroutines were %d during the request vs %d just before the "+ + "context was built; none started, which would make the leak test above vacuous", + s.during, s.before) + } + case <-time.After(5 * time.Second): + t.Fatal("handler never reported") + } + close(release) +} diff --git a/transports/bifrost-http/server/server.go b/transports/bifrost-http/server/server.go index 19a98efc38a..292451fb2fc 100644 --- a/transports/bifrost-http/server/server.go +++ b/transports/bifrost-http/server/server.go @@ -323,6 +323,21 @@ func (s *GovernanceInMemoryStore) GetConfiguredProviders() map[schemas.ModelProv return s.Config.Providers } +// GetConfiguredProviderNames builds the name slice under the lock, because provider edits write to +// the map GetConfiguredProviders hands back in place: ranging that map after the lock is released +// is a concurrent iteration and write, which is fatal rather than merely stale. +func (s *GovernanceInMemoryStore) GetConfiguredProviderNames() []string { + providers, err := s.Config.GetAllProviders() + if err != nil { + return nil + } + names := make([]string, 0, len(providers)) + for _, provider := range providers { + names = append(names, string(provider)) + } + return names +} + func (s *GovernanceInMemoryStore) GetMCPClientsAllowedByDefault() map[string]string { return s.Config.GetMCPClientsAllowedByDefault() } diff --git a/transports/bifrost-http/server/server_test.go b/transports/bifrost-http/server/server_test.go index ba5634bfcf1..7030be38f1e 100644 --- a/transports/bifrost-http/server/server_test.go +++ b/transports/bifrost-http/server/server_test.go @@ -1196,3 +1196,46 @@ func TestReloadPromptCacheWithIncompatiblePluginIsNoOp(t *testing.T) { require.NoError(t, server.ReloadPromptCache(context.Background())) } + +// GetConfiguredProviders hands back the live provider map uncopied, which is safe for the callers +// that index it once. The name listing cannot be written that way: provider add, delete and status +// updates all write to that map in place under the write lock, so a caller ranging it after the read +// lock is released hits a concurrent map iteration and write. That is fatal rather than merely +// stale — it takes the process down — so the slice is built under the lock instead. Run with -race. +func TestGetConfiguredProviderNamesIsSafeAgainstConcurrentProviderEdits(t *testing.T) { + config := &lib.Config{ + Providers: map[schemas.ModelProvider]configstore.ProviderConfig{schemas.OpenAI: {}}, + } + store := &GovernanceInMemoryStore{Config: config} + + churn := []schemas.ModelProvider{"churn-a", "churn-b", "churn-c"} + done := make(chan struct{}) + var writers sync.WaitGroup + writers.Add(1) + go func() { + defer writers.Done() + for { + select { + case <-done: + return + default: + } + // The same in-place mutation AddProvider and RemoveProvider make. + for _, provider := range churn { + config.Mu.Lock() + config.Providers[provider] = configstore.ProviderConfig{} + delete(config.Providers, provider) + config.Mu.Unlock() + } + } + }() + + for range 500 { + if names := store.GetConfiguredProviderNames(); len(names) == 0 { + t.Fatal("expected the configured providers to be listed") + } + } + + close(done) + writers.Wait() +} diff --git a/transports/changelog.md b/transports/changelog.md index 7274118f7e3..3f897b2ec8e 100644 --- a/transports/changelog.md +++ b/transports/changelog.md @@ -1,59 +1,35 @@ ## ✨ Features -- **Virtual Key Assignees and Expanded Search** - The virtual key list resolves `assigned_user` for every row via a single batch lookup, and VK search matches team, customer and user names in addition to key fields, so keys can be found by who owns them (#7225, #7226) -- **User Attribution in Prometheus Metrics** - The telemetry plugin's metrics export includes user id and user name labels, so per-user usage and error rates can be graphed and alerted on directly (#7267) -- **Prompt Cache Breakpoints Capability** - A `SupportsPromptCacheBreakpoints` model cap with a name-based fallback for OpenRouter gates `prompt_cache_breakpoint` forwarding, so OpenRouter's non-Claude models stop rejecting requests that carry it (#7261, #7262) -- **MCP Observed Latency** - Native (observed) MCP tool-call latency from `observed_latency_ms` is surfaced in the logs duration column and detail sheet, kept clearly distinct from true execution time and never fabricating a synthetic start timestamp on the timeline (#7271) +- **Pinned Keys on Routing Fallbacks** - Each routing-rule fallback can pin a provider key via `key_id`, or `provider_key_name` in config.json. The UI rule editor lets you pick or clear a key per fallback. Unpinned fallbacks keep the legacy `provider/model` string, so existing rules keep their config hash (#7470, #7379, #7380, #7381) +- **OpenAI Async Tool Execution** - The `async` flag on Responses tools and tool calls, `output_schema` on function tools and `tunnel_id` on MCP tools are now forwarded to OpenAI. `async` is stripped for models without support, and the datasheet `supports_async_tools` field can override this (#7242) +- **GPT-6 Prompt Cache Breakpoints** - Prompt-cache breakpoints now cover the GPT-6 family on OpenAI, Azure, Bedrock and Bedrock Mantle. The datasheet `supports_prompt_cache_breakpoint` field can override this (#7240) +- **GPT-6 Sol and Luna Reasoning Off** - `reasoning.effort: "none"` is forwarded for `gpt-6-sol` and `gpt-6-luna`. Other GPT-6 models keep reasoning on (#7492) ## 🐞 Fixed -- **Resource ID Path Injection** - Caller-supplied resource IDs (batch, file, video, container, response IDs, cached content names) were interpolated into outbound URL paths unvalidated; a crafted ID with `../`, `?`, `#`, percent-encoded bytes or control characters could redirect the request to an unintended upstream endpoint. A central `EscapeResourceID` helper validates and escapes every such ID (#7307, #7310) -- **Bedrock S3 SSRF** - A caller-supplied `s3://` file ID or `s3_bucket` param could control the upstream host, since S3 virtual-hosted URLs are built as `https://{bucket}.{s3host}/{key}`; a bucket like `127.0.0.1:PORT#` opened a SigV4-signed TLS connection to a caller-chosen host. Bucket names are now validated with a DNS-compatible regex (#7315) -- **Caller-Forged Billing Idempotency** - The billing-idempotency key was `(RequestID, AttemptNumber)`, and `RequestID` may come from the caller's `x-request-id` header, so two unrelated requests sharing a deliberately chosen ID collided and the second was silently never charged. An internally minted `BillingNonce` is mixed into the key, making it unforgeable (#7303) -- **Abandoned Non-Streaming Request Hung Forever** - The worker kept a `ctx.Done()` arm on an already-claimed delivery send, a regression from the #6972 delivery fix, so a non-streaming caller could hang indefinitely (#7313) -- **Repeated Empty Thinking Blocks** - Streaming chat chunks emitted empty reasoning/message fields on every content delta, which clients rendered as repeated empty thinking blocks (#7318) -- **Claude Code Thread Continuations Failed Behind Key Rotation** - Claude Code's server-side conversation threads are bound to the upstream account that creates them, and Bifrost's per-request key selection, retries and fallbacks cannot keep a continuation on that account, so `thread: {"type": "continue"}` requests failed with `thread_not_found` (about half the time on a two-key config). The Anthropic integration now declares itself stateless: continuations are refused before the provider call with a 400 whose `details.error_code` is `thread_unsupported_request`, which makes the client resend the turn in full and stop sending the thread field for the rest of the session, and the provider raw-body path strips `thread` from create requests so no orphaned thread state accumulates upstream. Token counting is never refused (#7274) -- **Gemini Flash-Lite Minimal Thinking Promoted to Low** - Normalized `gemini-3.1-flash-lite` requests silently promoted `minimal` thinking to `low` on both Gemini and Vertex; the text model's four supported levels are now registered, preserving the image variant's separate restrictions (#7288) (thanks [@Javtor](https://github.com/Javtor)!) -- **Config.json Virtual Key Limits Broke Under UI Edits** - VK rate limits and budgets created via the config.json standalone-limits flow and then edited through the UI produced duplicate, conflicting ownership records (standalone budgets owned directly by the VK, plus orphaned UUID rate-limit rows created by the UI). Migration `migrate_vk_standalone_limits_to_model_configs` consolidates ownership into VK-scoped model configs while preserving usage counters, and the write paths stop creating the divergent rows (#7291) -- **Bedrock Service Tier Rejections** - Service tier forwarding for Bedrock (Converse and Mantle paths) sent whatever tier was requested; it is now gated on explicit model capability metadata, failing closed when none exists, so models that do not support the requested tier stop rejecting the request (#7266) -- **Anthropic Root-Level Tool Schema Compositions** - Anthropic rejects `oneOf`/`anyOf`/`allOf` at the root of a tool's `input_schema` but accepts them inside properties; root-level compositions are now rewritten into a flat object schema before dispatch, unblocking tools like Codex's `automation_update` (#7265) -- **GenAI Streaming TTS** - Speech stream chunks routed through the `/genai` integration had no converter registered; `ToGeminiSpeechStreamResponse` now serves streaming TTS, and the streaming router returns a clean error instead of panicking when any stream converter is missing (#7248) -- **xAI Usage and Cost** - xAI reports visible and reasoning completion tokens separately; `completion_tokens` now folds reasoning in so `prompt_tokens + completion_tokens = total_tokens` holds, and streaming cost normalization preserves xAI's authoritative `cost_in_usd_ticks` to the terminal usage chunk instead of falling back to catalog pricing (#7245, #7250) -- **OpenAI-Only `search_content_types` Forwarded Everywhere** - `search_content_types` on `web_search` tools is an OpenAI-specific extension; it is now gated behind a per-provider capability check so Bedrock and other OpenAI-compatible backends receive a clean `web_search` tool without the field (#7244) -- **Claude Code `diagnostics` Field Rejected by Non-Native Providers** - Claude Code sends `diagnostics.previous_message_id` on every request; on the typed-sanitizer path Bedrock, Vertex and Azure returned 400 `diagnostics: Extra inputs are not permitted`. The field is stripped for providers that do not support it (#7243) -- **Unreadable Bedrock Error Logs** - AWS returns errors in a flat `{"message": ..., "__type": ...}` shape the shared Anthropic/OpenAI parsers never looked at, so Bedrock errors were logged with no human-readable reason; the root-level message now seeds `BifrostError`, and the logs UI falls back to showing the raw provider error body when no message could be extracted (#7221, #7222) -- **Anthropic `container` Param Dropped** - The string-form `container` param on `/anthropic/v1/messages` was silently dropped, so container reuse provisioned a fresh container every time; it is now carried through the round trip (#5829) (thanks [@AdityaPainuli](https://github.com/AdityaPainuli)!) -- **Responses API Finish Reason Missing From OTEL Spans** - `gen_ai.response.finish_reason` / `finish_reasons` are now emitted on the `llm.call` span for `/v1/responses` requests, so refusals are visible to OTEL consumers, matching the chat path (#7205) (thanks [@bkfl-notai](https://github.com/bkfl-notai)!) -- **Cohere Fallback Response Shape** - The Cohere-compatible route returned the raw Bifrost normalized response when a fallback served the request from a non-Cohere provider, which the Cohere SDK failed to parse; responses are now converted to the Cohere v2 shape (`message`, `finish_reason`) (#7198) -- **Skill Serving Race** - Fixed a race condition in the skill serving handler -- **Log and Dashboard Label Truncation** - Long model and provider labels in the logs table and dashboard charts truncate from the start so the distinctive suffix stays visible, the logs model column is wider, the search field border and icon spacing are cleaned up, and key picker options are keyed by id so duplicate labels stop highlighting together (#7215, #7216, #7217, #7269, #7297) +- **OpenAI Sampling Parameters on Reasoning Models** - `temperature`, `top_logprobs` and `logprobs` are now stripped alongside `top_p` on chat and Responses when the model and effort do not support them. An omitted `reasoning.effort` now counts as `none` only for models that default to no reasoning (#7239) +- **Responses API Wire Shapes** - Structured MCP tool-call errors, object-form `conversation`, array-form MCP `allowed_tools`, `approval_request_id` on MCP approval responses, and `in`/`nin` file search filters now decode and re-encode correctly (#7241) + Go SDK callers: `ResponsesMCPApprovalResponse.ApprovalResponseID` is now `ApprovalRequestID`, the message type is now `mcp_approval_response`, and `ResponsesToolMessage.Error` and `ResponsesParameters.Conversation` are now union types, where they used to be `*string`. +- **OpenRouter Anthropic Cache Breakpoints** - Anthropic models routed through OpenRouter now keep their `cache_control` breakpoints, based on the model capability (#7521) +- **Session Affinity with Pinned Keys** - When session affinity reorders the chain, a routing rule's key pin now moves with its provider, so the pinned key is never looked up under the wrong provider (#7468) +- **Session Affinity Route Matching** - A session's route is now matched on provider and model together. Bindings the request followed into a failure are dropped (#7473) +- **Databricks Gemini System Prompts** - Multiple system and developer messages are merged into one for Gemini models hosted on Databricks, which reject more than one system prompt (#7461) +- **Bedrock Encrypted Reasoning Replay** - Bedrock's "encrypted reasoning was created for a different account or model" error now triggers the strip-and-retry path for unverifiable reasoning +- **Decisions on Bedrock Mantle** - Decision emulation now sends `tool_choice: "auto"` for gpt-oss models on Bedrock Mantle, which reject `"required"`. Leaked parameter tags with surrounding whitespace are now recovered +- **Gemini Transcription Usage** - Usage is reported even when the transcript is empty +- **Routing Rule Enabled State** - Syncing or updating a routing rule that omits `enabled` keeps the stored value, where it used to write NULL +- **Telemetry User Labels Toggle** - `user_labels_enabled` is now saved with the telemetry config (#7490) ## 🗄️ Database Migrations -- **migrate_vk_standalone_limits_to_model_configs** - Consolidates virtual key standalone budgets and rate limits into VK-scoped model configs: standalone budgets are re-pointed to the VK's top-level model config (created if missing, usage preserved), orphaned duplicate UUID rate-limit rows created by the UI are deleted with the model config re-pointed to the canonical config.json row, and `vk.rate_limit_id` is cleared. - - -## 🐙 Closed GitHub Issues - -- [#123](https://github.com/maximhq/bifrost/issues/123) - Files API Support -- [#5707](https://github.com/maximhq/bifrost/issues/5707) - string-form `container` param on /anthropic/v1/messages is silently dropped - container reuse provisions a fresh container -- [#7204](https://github.com/maximhq/bifrost/issues/7204) - Responses API path never sets gen_ai.response.finish_reason in OTEL traces (refusals invisible to observability) -- [#7287](https://github.com/maximhq/bifrost/issues/7287) - Gemini 3.1 Flash-Lite minimal thinking is silently promoted to low -- [#7294](https://github.com/maximhq/bifrost/issues/7294) - Streaming chat chunks emit empty reasoning/message fields on every content delta, causing repeated thinking blocks -- [#7308](https://github.com/maximhq/bifrost/issues/7308) - non-streaming caller hangs forever - worker keeps a ctx.Done() arm on a claimed delivery send (regression from #6972) +- No new database migrations in this release. ## 🍴 Fork changes (txehq/bifrost) -- chore: v2.2.1 is upstream `transports/v2.2.1` (6493abd3d) merged onto the fork's v2.2.0, keeping the fork's Go module paths, custom providers (ModelArk, Volcengine), native Anthropic Messages and xAI passthrough, separate streaming and unary clients, the legacy realtime session endpoints, and release guards. Core, framework and plugins are built from the workspace and are not tagged separately -- note: the abandoned non-streaming request fix (upstream #7313) already shipped in the fork's v2.2.0 -- fix: a caller OAuth bearer forwarded on a passthrough route is stripped before a cleartext non-loopback upstream on the fork's shared native Anthropic Messages client too (Anthropic and xAI), as upstream #7325 does for its own Anthropic and OpenAI passthroughs -- fix: Volcengine and ModelArk file and video IDs are validated and escaped as one path segment before dispatch, as upstream #7307 does for its own providers -- breaking: `POST /v1/realtime/client_secrets`, its `/openai` alias and the deprecated beta `/v1/realtime/sessions` routes now return a Bifrost-minted `ek_bf_...` secret instead of the provider's ephemeral token (upstream #7317). It works only when presented to Bifrost's realtime WebSocket (`/v1/realtime`) or WebRTC (`/v1/realtime/calls`) endpoints, which redeem it for the provider and key it was minted for; a client that sent the secret straight to OpenAI must connect through Bifrost instead -- fix: migration `migrate_vk_standalone_limits_to_model_configs` deletes a VK-scoped model-config rate limit only when it repeats the VK's config.json limit exactly, usage counters and reset times included; any other pair keeps both, still enforced, and is logged for an operator, so the upgrade never loosens a stricter limit -- fix: an Anthropic tool schema whose root `oneOf`/`anyOf`/`allOf` branch is a `$ref` resolves escaped and percent-encoded JSON Pointers per RFC 6901, keeps the constraints written next to the `$ref`, and is refused with a 400 when the reference names nothing, instead of being flattened into a branch that silently drops its properties and requirements -- fix: a Bifrost-minted realtime secret is redeemed only for the provider and key it was minted for, so its provider token can never be sent to another provider's endpoint -- fix: Azure realtime answers 400 for a key without an Azure endpoint (such as an ephemeral token Bifrost did not mint) instead of crashing -- fix: a VK's provider- or model-specific limits no longer take over its key-wide config.json rate limit or budgets; the standalone-limits migration keeps each limit's calendar alignment; a config file that sets a different key-wide rate limit is kept; and the virtual key list sorts by budget spent across budgets the migration moved -- fix: Anthropic tool-schema flattening refuses schemas whose references multiply its work or whose branches constrain undeclared properties with a schema, and resolves references through arrays -- fix: an explicit Bedrock `service_tier` is forwarded when the model catalog has no tier data for the model, as before v2.2.1 -- fix: the telemetry plugin's `user_labels_enabled` setting survives saving +- chore: v2.2.3 is upstream `transports/v2.2.3` (411d62b28) merged onto the fork's v2.2.1, keeping the fork's Go module paths, custom providers (ModelArk, Volcengine), native Anthropic Messages and xAI passthrough, separate streaming and unary clients, the legacy realtime session endpoints and Bifrost-minted realtime secrets, and release guards. Core, framework and plugins are built from the workspace and are not tagged separately +- fix: xAI's native Anthropic Messages route strips Claude Code's `safeguards` and `diagnostics` fields and its per-request `x-anthropic-billing-header` system line before the request leaves for xAI, as upstream #7393, #7243 and #7431 do for every converted non-Anthropic request; the billing line changes on every request and would otherwise defeat xAI's prompt cache. The Anthropic route still forwards them +- fix: a decision answered by emulation (upstream #7361) is billed as the text call it is, from the model's chat or responses pricing row with its cache rates; it previously found no `decisions` row and cost nothing. A decision answer Bifrost rejects, from Typesafe or from emulation, still charges the usage the provider billed +- fix: decision emulation on a custom provider wrapping Bedrock Mantle sends Mantle's `auto` tool choice +- fix: a stream paused for approval that overflows its replay cap delivers only approved content before the overflow error +- note: MCP HTTP and SSE connections honour `HTTP_PROXY`/`HTTPS_PROXY`/`NO_PROXY` again (upstream #7437). Through a proxy, an IP-literal link-local, unspecified or metadata destination is refused before the proxy is contacted, and, unlike upstream, a hostname must resolve on the gateway to addresses that pass the same block, or the connection is refused; the proxy is never handed a name the gateway could not check +- note: the release pipeline gains upstream's `test-memory` allocation gate; its break-glass job uses the `memory-regression-override` environment diff --git a/transports/config.schema.json b/transports/config.schema.json index 7a3f97ad223..bf5d17c8ba5 100644 --- a/transports/config.schema.json +++ b/transports/config.schema.json @@ -463,6 +463,9 @@ "wafer": { "$ref": "#/$defs/provider" }, + "typesafe": { + "$ref": "#/$defs/provider" + }, "vllm": { "$ref": "#/$defs/provider_with_vllm_config" }, @@ -3704,6 +3707,39 @@ }, "additionalProperties": false }, + "routing_fallback": { + "description": "A fallback in a routing rule's chain. Either the 'provider/model' string ('provider/' keeps the incoming model), or an object that additionally pins a provider key.", + "oneOf": [ + { + "type": "string", + "minLength": 1 + }, + { + "type": "object", + "properties": { + "provider": { + "type": "string", + "minLength": 1, + "description": "Fallback provider name (e.g., 'openai', 'azure')" + }, + "model": { + "type": "string", + "minLength": 1, + "description": "Fallback model name. Omit to use the incoming request model" + }, + "provider_key_name": { + "type": "string", + "minLength": 1, + "description": "Optional provider key name. Resolved to internal key_id at config load time. Requires provider." + } + }, + "required": [ + "provider" + ], + "additionalProperties": false + } + ] + }, "routing_target": { "type": "object", "description": "A single weighted routing target within a rule. All fields except weight are optional; omitting provider or model means use the incoming request value. Weights across all targets in a rule must sum to 1.", @@ -3777,7 +3813,7 @@ "type": "array", "description": "Fallback provider chain in order", "items": { - "type": "string" + "$ref": "#/$defs/routing_fallback" } }, "scope": { @@ -8711,6 +8747,7 @@ "responses", "embedding", "rerank", + "decisions", "speech", "transcription", "image_generation", @@ -8778,7 +8815,8 @@ "runware", "fireworks", "sarvam", - "wafer" + "wafer", + "typesafe" ], "description": "Base provider type to extend" }, @@ -8842,6 +8880,9 @@ "rerank": { "type": "boolean" }, + "decisions": { + "type": "boolean" + }, "ocr": { "type": "boolean" }, diff --git a/transports/go.mod b/transports/go.mod index dc4b0aec70a..eb2561fd2a0 100644 --- a/transports/go.mod +++ b/transports/go.mod @@ -15,18 +15,18 @@ require ( github.com/google/uuid v1.6.0 github.com/klauspost/compress v1.20.0 github.com/mark3labs/mcp-go v0.43.2 - github.com/capsohq/bifrost/core v1.9.1 - github.com/capsohq/bifrost/framework v1.7.1 - github.com/capsohq/bifrost/plugins/compat v0.3.1 - github.com/capsohq/bifrost/plugins/governance v1.8.1 - github.com/capsohq/bifrost/plugins/logging v1.8.1 - github.com/capsohq/bifrost/plugins/maxim v1.7.4 - github.com/capsohq/bifrost/plugins/modelcatalogresolver v1.1.4 - github.com/capsohq/bifrost/plugins/otel v1.5.4 - github.com/capsohq/bifrost/plugins/prompts v1.1.4 - github.com/capsohq/bifrost/plugins/routing v1.1.1 - github.com/capsohq/bifrost/plugins/semanticcache v1.6.4 - github.com/capsohq/bifrost/plugins/telemetry v1.8.0 + github.com/capsohq/bifrost/core v1.10.2 + github.com/capsohq/bifrost/framework v1.7.4 + github.com/capsohq/bifrost/plugins/compat v0.3.3 + github.com/capsohq/bifrost/plugins/governance v1.8.3 + github.com/capsohq/bifrost/plugins/logging v1.8.3 + github.com/capsohq/bifrost/plugins/maxim v1.7.6 + github.com/capsohq/bifrost/plugins/modelcatalogresolver v1.1.6 + github.com/capsohq/bifrost/plugins/otel v1.5.6 + github.com/capsohq/bifrost/plugins/prompts v1.1.6 + github.com/capsohq/bifrost/plugins/routing v1.1.3 + github.com/capsohq/bifrost/plugins/semanticcache v1.6.6 + github.com/capsohq/bifrost/plugins/telemetry v1.8.2 github.com/pion/rtcp v1.2.16 github.com/pion/webrtc/v4 v4.2.9 github.com/prometheus/client_golang v1.23.2 @@ -154,7 +154,7 @@ require ( github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.24 // indirect github.com/mattn/go-sqlite3 v1.14.32 // indirect - github.com/capsohq/bifrost/plugins/mocker v1.6.4 // indirect + github.com/capsohq/bifrost/plugins/mocker v1.6.6 // indirect github.com/maximhq/maxim-go v0.2.1 // indirect github.com/molecule-man/go-brrr v1.0.1 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect diff --git a/transports/go.sum b/transports/go.sum index b30c15e8277..b21feade19c 100644 --- a/transports/go.sum +++ b/transports/go.sum @@ -317,32 +317,32 @@ github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsRe github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= github.com/mattn/go-sqlite3 v1.14.32 h1:JD12Ag3oLy1zQA+BNn74xRgaBbdhbNIDYvQUEuuErjs= github.com/mattn/go-sqlite3 v1.14.32/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= -github.com/maximhq/bifrost/core v1.9.1 h1:CF7TTj63htKQbuBL7eAGzou5yoe4XiQ0YMWpAO/atnw= -github.com/maximhq/bifrost/core v1.9.1/go.mod h1:bgTt5CsqB7duKiy/pC0uUMTxIRnvjj0t2OvH7Tlvuz0= -github.com/maximhq/bifrost/framework v1.7.1 h1:pGCdT7ksXmcnCylBpQqSHlQ071QAqgJ1XS7JwI8uV9U= -github.com/maximhq/bifrost/framework v1.7.1/go.mod h1:Acd4YNI+K8SjMPdghUQbjw0plJpqIDOP0s01iQr51Go= -github.com/maximhq/bifrost/plugins/compat v0.3.1 h1:nh5LHkK4k6RlJ/CuV1gC5Cpjo8mjUAFCqI4Mq+xoP4o= -github.com/maximhq/bifrost/plugins/compat v0.3.1/go.mod h1:yupmRHerHdHaki87h+I5F0yysDebn/gFmDd72/VtS1A= -github.com/maximhq/bifrost/plugins/governance v1.8.1 h1:PNOy5o2fo+R2g8bY6tr4LCx8CJv/8JmOTtNg2SIKUy4= -github.com/maximhq/bifrost/plugins/governance v1.8.1/go.mod h1:T+FoyatNH+qqqziXmGNiKqXZw8DB6neKZfu3tXiWojo= -github.com/maximhq/bifrost/plugins/logging v1.8.1 h1:5DwhdoIhFLvjEBq4vVJN3MNSilq5PoKoEAY8mUp7Yag= -github.com/maximhq/bifrost/plugins/logging v1.8.1/go.mod h1:RmPBsRKQE1cK81+8tUW12odG7vQu2OdA9/usotEYfis= -github.com/maximhq/bifrost/plugins/maxim v1.7.4 h1:cKrwIUJvVyJe33OZtOk6WoBH5MSGqekz9zpDVH7iOP8= -github.com/maximhq/bifrost/plugins/maxim v1.7.4/go.mod h1:3epJIrNTVJ1Xyw8D41/Uwax7o9EygAfUSOlYkP5c75c= -github.com/maximhq/bifrost/plugins/mocker v1.6.4 h1:EYiF8Q0gULpgdlnmVECfrbteC1CJgkiHD88IvDNLWPY= -github.com/maximhq/bifrost/plugins/mocker v1.6.4/go.mod h1:Y+oZP7YAloPuznjJRwqWs6hUkNE26moCdl+xHbc25wg= -github.com/maximhq/bifrost/plugins/modelcatalogresolver v1.1.4 h1:+P5qi8lvKXw+f23DAdRSPYYMGtGOvnsI27m34jvc5Qg= -github.com/maximhq/bifrost/plugins/modelcatalogresolver v1.1.4/go.mod h1:FqWPgcNWIEAv+5qeScFjWAnnsJ3yf1ilChnixEsMNxo= -github.com/maximhq/bifrost/plugins/otel v1.5.4 h1:izuYN/W61KS5H43UzNkJvK0t/0p+NNDL2jMCAK6t1z8= -github.com/maximhq/bifrost/plugins/otel v1.5.4/go.mod h1:wPRJf1yIbU1ZDI21FNchu2RQe25N1OK9DkBNPqYsRX0= -github.com/maximhq/bifrost/plugins/prompts v1.1.4 h1:NdNTXE5e6aqFglHuBq/hcxAvCNPQp7u39YlCHEerZhE= -github.com/maximhq/bifrost/plugins/prompts v1.1.4/go.mod h1:HsxFDzn6Oq/XgRtGRS/At+UJKcNQMT5/L5ekjsKMCs8= -github.com/maximhq/bifrost/plugins/routing v1.1.1 h1:qICZdYBftfwUXMsOgDt6H2xsUJ1OdPikRlzDBx6HZPE= -github.com/maximhq/bifrost/plugins/routing v1.1.1/go.mod h1:uRKSPInPUhVKEyms5NDrqitjD6yovMKwAoWMgWTI3BU= -github.com/maximhq/bifrost/plugins/semanticcache v1.6.4 h1:v2GEFn8pA/Bfx3UdfLCC6p8/wyK/3yfQzV+Vzo3FueQ= -github.com/maximhq/bifrost/plugins/semanticcache v1.6.4/go.mod h1:V3pKDRPBDA65ZWC/rxJpRqRQ/JUxN07wRqY0J8lDB3A= -github.com/maximhq/bifrost/plugins/telemetry v1.8.0 h1:ZVO7uuLQ5uMrjNV0zlbsfUe3C7FIIPpjjfjkPsV604w= -github.com/maximhq/bifrost/plugins/telemetry v1.8.0/go.mod h1:68LkCXYusmFLSWr2QJ2HnVo0JyzVTGEHKEMdVJ0QEFc= +github.com/maximhq/bifrost/core v1.10.2 h1:jRvvZN1KJVaX2Pxn9crTrmL3kW2QgDh7d1jsuz6b72k= +github.com/maximhq/bifrost/core v1.10.2/go.mod h1:bgTt5CsqB7duKiy/pC0uUMTxIRnvjj0t2OvH7Tlvuz0= +github.com/maximhq/bifrost/framework v1.7.4 h1:aM4MM3H3dXlPpkvyqRvt337WV8jl3m7DfBehAzs8HNU= +github.com/maximhq/bifrost/framework v1.7.4/go.mod h1:yHmcJhKvvaeoNUJYd5yV/wsNb3jZBwnjw2ny246C55Y= +github.com/maximhq/bifrost/plugins/compat v0.3.3 h1:5bS36HcIDAsRwUbJm9gWToSmlL58BdblhAYLhphM0H8= +github.com/maximhq/bifrost/plugins/compat v0.3.3/go.mod h1:peWRKqQimLJl8QvPEs8Myp/ohXOkzWWx9vqg2hHMnOw= +github.com/maximhq/bifrost/plugins/governance v1.8.3 h1:LNnhrzjsEYGyFyR07nXou9XxIzSAP0RuIn0N/ovdEpk= +github.com/maximhq/bifrost/plugins/governance v1.8.3/go.mod h1:yr3Pmp1wnF/AETjUki7AZx8m3NQme1OPDpxF9ux7E0c= +github.com/maximhq/bifrost/plugins/logging v1.8.3 h1:QSVtgvd/P6JzBmmExEq79F1zhm2UjS1ct/Pz75UgJmY= +github.com/maximhq/bifrost/plugins/logging v1.8.3/go.mod h1:h5S1LybUHVO+5XZCdccARn4Y4YbQwHoYwgPFOkeOIyA= +github.com/maximhq/bifrost/plugins/maxim v1.7.6 h1:lHJcIPzCe8VEwwY9NfhVvcDjxelc2CHDZ8olue1akCA= +github.com/maximhq/bifrost/plugins/maxim v1.7.6/go.mod h1:gh6yPO+Y76SSs1HtnoZfQ71uSCKxuTYpHDsxRbwc7X8= +github.com/maximhq/bifrost/plugins/mocker v1.6.6 h1:oZjsCScTXLMRuGRxiztdpSzR3xO7xniC2phTEs/83Xo= +github.com/maximhq/bifrost/plugins/mocker v1.6.6/go.mod h1:iuR83+IPA4U54vs6fnrpooJH98pNXZCHBT1m3L/Wkz8= +github.com/maximhq/bifrost/plugins/modelcatalogresolver v1.1.6 h1:GBm8HrXj/5CAhtob3XLkGcVn1uhm28fBaT2Aut3zrB8= +github.com/maximhq/bifrost/plugins/modelcatalogresolver v1.1.6/go.mod h1:YEtGxB4jg+kinjnOJIMO3xUMcKfeXjy4RpyCa2uvSz8= +github.com/maximhq/bifrost/plugins/otel v1.5.6 h1:ODPNYB/+ySCtDkfdz+aZEVzftXneeEmpu7MmA+kNchA= +github.com/maximhq/bifrost/plugins/otel v1.5.6/go.mod h1:7bikaux7xcAtyMdWE2uo4VZ00shBC3v5RnmFMX4DhlY= +github.com/maximhq/bifrost/plugins/prompts v1.1.6 h1:RfD3uDQdYxUW/zN1iQsuEzieBFSs/wuby7ZXIv1wdt4= +github.com/maximhq/bifrost/plugins/prompts v1.1.6/go.mod h1:E8sNppcOXLaa5jJypLUcT+3K8w6KlMj7TWjQts21R4Q= +github.com/maximhq/bifrost/plugins/routing v1.1.3 h1:3/Mz5hnk2gt11ZLseVsh2jK1XDvNKec+wX3zAJuf8Cc= +github.com/maximhq/bifrost/plugins/routing v1.1.3/go.mod h1:IDYSQIPHwmkDKVj8CiuQJju96BuNOv5AN8N/MDdapco= +github.com/maximhq/bifrost/plugins/semanticcache v1.6.6 h1:ZLkBaqceXoVUrFWZwgrQkyGIce8zFsDGKTp3mVT1px0= +github.com/maximhq/bifrost/plugins/semanticcache v1.6.6/go.mod h1:+wosrJsdiG85nmYdBKebHRqjPgD3VAhUjUA3pwlcZus= +github.com/maximhq/bifrost/plugins/telemetry v1.8.2 h1:OsRYcbpTm3wvo7A+kw0d9mA8KPKz2yYrguLu4rJH/6o= +github.com/maximhq/bifrost/plugins/telemetry v1.8.2/go.mod h1:/8RQouEnT5qF/jZoog2rlQlSWRYrr/YI7t4G6U9XDd8= github.com/maximhq/maxim-go v0.2.1 h1:hCp8dQ4HsyyNC+y5HCUuY/HFD0sOnGkjL5MdYCHkgEQ= github.com/maximhq/maxim-go v0.2.1/go.mod h1:nwFznXy0Dn4mxXGU4X+BCnE3VP68L+FPEaW0yUgk96o= github.com/molecule-man/go-brrr v1.0.1 h1:cEjgx8hgNw6UGdhQ94SPDbPkKuRbkUcxBO3IzbGpA/o= diff --git a/transports/version b/transports/version index c043eea7767..585940699b5 100644 --- a/transports/version +++ b/transports/version @@ -1 +1 @@ -2.2.1 +2.2.3 diff --git a/ui/app/__error.tsx b/ui/app/__error.tsx index 3a1a9980975..9c2682ed728 100644 --- a/ui/app/__error.tsx +++ b/ui/app/__error.tsx @@ -30,4 +30,4 @@ export function ErrorComponent({ error }: Partial) { ); -} +} \ No newline at end of file diff --git a/ui/app/_fallbacks/enterprise/components/user-groups/sheets/customerDetailSheet.tsx b/ui/app/_fallbacks/enterprise/components/user-groups/sheets/customerDetailSheet.tsx index 888aa9c9d43..9411a70937b 100644 --- a/ui/app/_fallbacks/enterprise/components/user-groups/sheets/customerDetailSheet.tsx +++ b/ui/app/_fallbacks/enterprise/components/user-groups/sheets/customerDetailSheet.tsx @@ -138,12 +138,12 @@ export function CustomerDetailSheet({ customer, open, onOpenChange }: Props) { .sort((a, b) => (b.max_limit || 0) - (a.max_limit || 0)) .map((b) => ( + key={b.id} + current={b.current_usage} + max={b.max_limit} + resetDuration={b.reset_duration} + resetConfig={b.reset_config} + /> ))} ) : ( diff --git a/ui/app/_fallbacks/enterprise/components/virtual-mcps/virtualMcpAccessProfiles.tsx b/ui/app/_fallbacks/enterprise/components/virtual-mcps/virtualMcpAccessProfiles.tsx index 698a86720de..6939bb50e5d 100644 --- a/ui/app/_fallbacks/enterprise/components/virtual-mcps/virtualMcpAccessProfiles.tsx +++ b/ui/app/_fallbacks/enterprise/components/virtual-mcps/virtualMcpAccessProfiles.tsx @@ -2,4 +2,4 @@ // the enterprise build supplies the real "Access Profiles using this Virtual MCP" section. export default function VirtualMcpAccessProfiles(_props: { vmcpId: number; active: boolean }) { return null; -} +} \ No newline at end of file diff --git a/ui/app/main.tsx b/ui/app/main.tsx index 43f82d1955b..5e787cae4e6 100644 --- a/ui/app/main.tsx +++ b/ui/app/main.tsx @@ -90,4 +90,4 @@ createRoot(rootEl).render( , -); +); \ No newline at end of file diff --git a/ui/app/workspace/complexity-router/page.tsx b/ui/app/workspace/complexity-router/page.tsx index 1a131613e40..2c8800279b5 100644 --- a/ui/app/workspace/complexity-router/page.tsx +++ b/ui/app/workspace/complexity-router/page.tsx @@ -797,4 +797,4 @@ export default function ComplexityRouterPage() { ); -} +} \ No newline at end of file diff --git a/ui/app/workspace/complexity-router/views/classifierStatusBadge.tsx b/ui/app/workspace/complexity-router/views/classifierStatusBadge.tsx index 5e98706b3d7..6e077ce981b 100644 --- a/ui/app/workspace/complexity-router/views/classifierStatusBadge.tsx +++ b/ui/app/workspace/complexity-router/views/classifierStatusBadge.tsx @@ -133,7 +133,11 @@ export function ClassifierStatusBadge({ - +

{summary}

{status?.serving_previous && state === "warming" && ( @@ -204,4 +208,4 @@ export function ClassifierStatusBadge({ ); -} +} \ No newline at end of file diff --git a/ui/app/workspace/config/views/clientSettingsView.tsx b/ui/app/workspace/config/views/clientSettingsView.tsx index 782a80934f2..bd8bb75dc02 100644 --- a/ui/app/workspace/config/views/clientSettingsView.tsx +++ b/ui/app/workspace/config/views/clientSettingsView.tsx @@ -589,4 +589,4 @@ export default function ClientSettingsView() { ); -} +} \ No newline at end of file diff --git a/ui/app/workspace/config/views/compatibilityView.tsx b/ui/app/workspace/config/views/compatibilityView.tsx index 551a37d1408..10d7fc8e9d7 100644 --- a/ui/app/workspace/config/views/compatibilityView.tsx +++ b/ui/app/workspace/config/views/compatibilityView.tsx @@ -146,10 +146,10 @@ export default function CompatibilityView() {

- Use Chat Completion APIs for Claude Code, Codex, etc. for Azure Deepseek models. + Use Chat Completion APIs for Claude Code, Codex, etc. for Azure Deepseek models.

diff --git a/ui/app/workspace/custom-pricing/overrides/pricingFields.test.ts b/ui/app/workspace/custom-pricing/overrides/pricingFields.test.ts index de6a7f91c00..711e1e3e822 100644 --- a/ui/app/workspace/custom-pricing/overrides/pricingFields.test.ts +++ b/ui/app/workspace/custom-pricing/overrides/pricingFields.test.ts @@ -121,4 +121,4 @@ describe("pricingFieldError", () => { expect(pricingFieldError("off_peak_cost_multiplier", "-0.5")).toBe("Must be greater than 0 and at most 1"); expect(pricingFieldError("off_peak_cost_multiplier", "1.5")).toBe("Must be greater than 0 and at most 1"); }); -}); +}); \ No newline at end of file diff --git a/ui/app/workspace/dashboard/components/charts/barShape.tsx b/ui/app/workspace/dashboard/components/charts/barShape.tsx index ceebc03db77..78e953dd7b5 100644 --- a/ui/app/workspace/dashboard/components/charts/barShape.tsx +++ b/ui/app/workspace/dashboard/components/charts/barShape.tsx @@ -1,4 +1,4 @@ -import type { ReactNode } from "react"; +import { Children, cloneElement, isValidElement, type ReactElement, type ReactNode } from "react"; import { BarStack, type BarShapeProps, Rectangle, usePlotArea } from "recharts"; // Bars round only the open end (the end that points away from the axis). The @@ -30,10 +30,16 @@ export function rankingBarShape(props: BarShapeProps) { // the stack as one shape instead. // // The cap is sized from the column width Recharts will draw: the plot width -// split across `buckets`, minus the category gap, never wider than `barSize`. +// split across `buckets`, minus a 1px gap, never wider than `barSize`. The +// same width is pushed onto each child : on a numeric X axis a fixed +// barSize wins over barCategoryGap, so dense buckets would otherwise render +// as one fused block with no gap between columns. export function CappedBarStack({ buckets, barSize = 30, children }: { buckets: number; barSize?: number; children: ReactNode }) { const plot = usePlotArea(); - const columnWidth = plot && buckets > 0 ? Math.min(barSize, plot.width / buckets - 2) : barSize; + const columnWidth = plot && buckets > 0 ? Math.max(1, Math.min(barSize, Math.floor(plot.width / buckets) - 1)) : barSize; const r = capRadius(columnWidth); - return {children}; + const sized = Children.map(children, (child) => + isValidElement(child) ? cloneElement(child as ReactElement<{ barSize?: number }>, { barSize: columnWidth }) : child, + ); + return {sized}; } \ No newline at end of file diff --git a/ui/app/workspace/dashboard/page.tsx b/ui/app/workspace/dashboard/page.tsx index d32ff1ff750..5a5e493d759 100644 --- a/ui/app/workspace/dashboard/page.tsx +++ b/ui/app/workspace/dashboard/page.tsx @@ -802,4 +802,4 @@ export default function DashboardPage() {
); -} +} \ No newline at end of file diff --git a/ui/app/workspace/edge-control/devices/page.tsx b/ui/app/workspace/edge-control/devices/page.tsx index 5a2c268626a..a8e3bc34a2c 100644 --- a/ui/app/workspace/edge-control/devices/page.tsx +++ b/ui/app/workspace/edge-control/devices/page.tsx @@ -2,7 +2,10 @@ import DevicesView from "@enterprise/components/edge-control/devicesView"; export default function EdgeDevicesPage() { return ( -
+
); diff --git a/ui/app/workspace/edge-control/inventory/page.tsx b/ui/app/workspace/edge-control/inventory/page.tsx index 831208dc595..7be2afb48c5 100644 --- a/ui/app/workspace/edge-control/inventory/page.tsx +++ b/ui/app/workspace/edge-control/inventory/page.tsx @@ -2,7 +2,10 @@ import InventoryView from "@enterprise/components/edge-control/inventoryView"; export default function EdgeInventoryPage() { return ( -
+
); diff --git a/ui/app/workspace/governance/teams/page.tsx b/ui/app/workspace/governance/teams/page.tsx index 8ca77cae3e3..e5574df5c9e 100644 --- a/ui/app/workspace/governance/teams/page.tsx +++ b/ui/app/workspace/governance/teams/page.tsx @@ -2,7 +2,10 @@ import { TeamsView } from "@enterprise/components/user-groups/teamsView"; export default function GovernanceTeamsPage() { return ( -
+
); diff --git a/ui/app/workspace/governance/views/customerSheet.tsx b/ui/app/workspace/governance/views/customerSheet.tsx index 70bcc3bbc7e..13306e70308 100644 --- a/ui/app/workspace/governance/views/customerSheet.tsx +++ b/ui/app/workspace/governance/views/customerSheet.tsx @@ -370,8 +370,8 @@ export default function CustomerSheet({ open, onOpenChange, customer, onSuccess Align to calendar cycle

- Reset budgets and rate limits at the start of each period (e.g. 1st of month) instead of rolling from creation date. Quarterly budgets always align to fiscal quarter starts. - Applies to durations of a day or longer. + Reset budgets and rate limits at the start of each period (e.g. 1st of month) instead of rolling from creation date. + Quarterly budgets always align to fiscal quarter starts. Applies to durations of a day or longer.

- Reset budgets and rate limits at the start of each period (e.g. 1st of month) instead of rolling from creation date. Quarterly budgets always align to fiscal quarter starts. - Applies to durations of a day or longer. + Reset budgets and rate limits at the start of each period (e.g. 1st of month) instead of rolling from creation date. + Quarterly budgets always align to fiscal quarter starts. Applies to durations of a day or longer.

); -} +} \ No newline at end of file diff --git a/ui/app/workspace/logs/sheets/logDetailView.tsx b/ui/app/workspace/logs/sheets/logDetailView.tsx index d61f7a1c4e6..afca60ebc4e 100644 --- a/ui/app/workspace/logs/sheets/logDetailView.tsx +++ b/ui/app/workspace/logs/sheets/logDetailView.tsx @@ -860,6 +860,15 @@ const messageRoleLabel: Record = { tool: "Tool Result", }; +// Decision logs store the state as the user message and the answers as the +// assistant message; label them by what they actually are. +const decisionRoleLabel = (requestType: string | undefined, role: MessageRole): string | undefined => { + if (requestType !== "decisions") return undefined; + if (role === "user") return "State"; + if (role === "assistant") return "Decision"; + return undefined; +}; + // deriveComplexityRouting returns the complexity tier / classification mechanism / // raw score behind a routing decision. Rows written since the structured columns // exist carry them directly; older rows fall back to parsing the prose routing @@ -976,7 +985,9 @@ function EncryptedReveal({ text, label }: { text: string; label: string }) { function CollapsibleCode({ text, preview = 3, lang, mono = true }: { text: string; preview?: number; lang?: string; mono?: boolean }) { const [open, setOpen] = useState(false); - const lines = text.split("\n"); + // Trailing blank lines would otherwise count as hidden content and render a + // "Show more" that expands to nothing visible. + const lines = text.replace(/\s+$/, "").split("\n"); const shown = open ? lines : lines.slice(0, preview); const hasMore = lines.length > preview; const moreCount = lines.length - preview; @@ -1006,7 +1017,39 @@ function CollapsibleCode({ text, preview = 3, lang, mono = true }: { text: strin ); } -function MessageRow({ role, meta, children, last = false }: { role: MessageRole; meta?: string; children: ReactNode; last?: boolean }) { +// Generated tool identifiers (e.g. Codex-style names with embedded signatures) +// can run to hundreds of characters; truncate the middle and keep the full name +// one hover away. +const TOOL_NAME_MAX = 48; + +function ToolNameLabel({ name }: { name: string }) { + if (name.length <= TOOL_NAME_MAX) return <>{name}; + const truncated = `${name.slice(0, 32)}…${name.slice(-12)}`; + return ( + + + + {truncated} + + + {name} + + ); +} + +function MessageRow({ + role, + meta, + children, + last = false, + label, +}: { + role: MessageRole; + meta?: ReactNode; + children: ReactNode; + last?: boolean; + label?: string; +}) { return (
@@ -1015,7 +1058,7 @@ function MessageRow({ role, meta, children, last = false }: { role: MessageRole;
- {messageRoleLabel[role]} + {label ?? messageRoleLabel[role]} {meta ? {meta} : null}
{children}
@@ -1024,6 +1067,27 @@ function MessageRow({ role, meta, children, last = false }: { role: MessageRole; ); } +// Collapses all but the last two messages of a long input history. The earlier +// turns stay in the DOM order they occurred in; expanding reveals them in place. +function MessageHistoryCollapse({ count, children }: { count: number; children: ReactNode }) { + const [open, setOpen] = useState(false); + return ( + <> + + {open ? children : null} + + ); +} + interface LogDetailViewProps { log: LogEntry | null; resolvedSelectedPromptName?: string; // Current prompt name from prompt-repo when `selected_prompt_id` is set; falls back to stored log name @@ -1628,7 +1692,15 @@ export function LogDetailView({ /> - {(visibleRoles.size < allRoles.length - ? log.input_history?.filter((m) => { - if (!m) return false; - const mainRole = ((m.role as string) || "user") as MessageRole; - const hasReasoning = !!extractChatReasoning(m); - return visibleRoles.has(mainRole) || (hasReasoning && visibleRoles.has("reasoning")); - }) - : log.input_history?.filter(Boolean) - )?.flatMap((message, index) => { - const role = ((message.role as string) || "user") as MessageRole; - const text = extractMessageText(message, activeInputRevealMapping); - const reasoningText = extractChatReasoning(message, activeInputRevealMapping); - const showAll = visibleRoles.size === allRoles.length; - const showMain = showAll || visibleRoles.has(role); - const showReasoning = !!reasoningText && (showAll || visibleRoles.has("reasoning")); - const hasToolCalls = Array.isArray(message.tool_calls) && message.tool_calls.length > 0; - const isOverallLast = - index === (log.input_history?.length ?? 0) - 1 && !log.output_message && !log.error_details?.error.message; - const lineCount = text ? text.split("\n").length : 0; - const approxTokens = text ? Math.max(1, Math.round(text.length / 4)) : 0; - const reasoningTokens = reasoningText ? Math.max(1, Math.round(reasoningText.length / 4)) : 0; - const meta = text - ? role === "system" || role === "tool" - ? `${lineCount} line${lineCount === 1 ? "" : "s"} · ~${approxTokens} tokens` - : `${lineCount} line${lineCount === 1 ? "" : "s"}` - : hasToolCalls - ? `${message.tool_calls!.length} tool call${message.tool_calls!.length === 1 ? "" : "s"}` - : undefined; - const usePlainText = role === "user" || role === "assistant"; - const rows: ReactNode[] = []; - if (showReasoning) { - rows.push( - - - , - ); - } - if (showMain) { - rows.push( - - {text ? ( - usePlainText && isJson(text) ? ( - { - try { - return JSON.stringify(JSON.parse(text), null, 2); - } catch { - return text; - } - })()} - lang="json" - readonly - autoResize - options={{ - collapsibleBlocks: true, - showIndentLines: false, - disableHover: true, - }} - /> - ) : usePlainText ? ( - - ) : ( - - ) - ) : ( - - )} - {text && - Array.isArray(message.content) && - (message.content as ContentBlock[]) - .filter((b) => b.type === "image_url") - .map((b, i) => { - const src = b.image_url?.url; - if (!src) return null; - return Attached image; - })} - {text && - Array.isArray(message.content) && - (message.content as ContentBlock[]) - .filter((b) => b.type === "file" && b.file) - .map((b, i) => ( - { + const historyMessages = + (visibleRoles.size < allRoles.length + ? log.input_history?.filter((m) => { + if (!m) return false; + const mainRole = ((m.role as string) || "user") as MessageRole; + const hasReasoning = !!extractChatReasoning(m); + return visibleRoles.has(mainRole) || (hasReasoning && visibleRoles.has("reasoning")); + }) + : log.input_history?.filter(Boolean)) ?? []; + const messageRows = historyMessages.map((message, index) => { + const role = ((message.role as string) || "user") as MessageRole; + const text = extractMessageText(message, activeInputRevealMapping); + const reasoningText = extractChatReasoning(message, activeInputRevealMapping); + const showAll = visibleRoles.size === allRoles.length; + const showMain = showAll || visibleRoles.has(role); + const showReasoning = !!reasoningText && (showAll || visibleRoles.has("reasoning")); + const hasToolCalls = Array.isArray(message.tool_calls) && message.tool_calls.length > 0; + const isOverallLast = + index === (log.input_history?.length ?? 0) - 1 && !log.output_message && !log.error_details?.error.message; + const lineCount = text ? text.split("\n").length : 0; + const approxTokens = text ? Math.max(1, Math.round(text.length / 4)) : 0; + const reasoningTokens = reasoningText ? Math.max(1, Math.round(reasoningText.length / 4)) : 0; + const meta = text + ? role === "system" || role === "tool" + ? `${lineCount} line${lineCount === 1 ? "" : "s"} · ~${approxTokens} tokens` + : `${lineCount} line${lineCount === 1 ? "" : "s"}` + : hasToolCalls + ? `${message.tool_calls!.length} tool call${message.tool_calls!.length === 1 ? "" : "s"}` + : undefined; + const usePlainText = role === "user" || role === "assistant"; + const rows: ReactNode[] = []; + if (showReasoning) { + rows.push( + + + , + ); + } + if (showMain) { + rows.push( + + {text ? ( + usePlainText && isJson(text) ? ( + { + try { + return JSON.stringify(JSON.parse(text), null, 2); + } catch { + return text; + } + })()} + lang="json" + readonly + autoResize + options={{ + collapsibleBlocks: true, + showIndentLines: false, + disableHover: true, + }} /> - ))} - {hasToolCalls && text ? ( -
- {message - .tool_calls!.map((tc) => tc.function?.name) - .filter(Boolean) - .join(", ") || `${message.tool_calls!.length} tool call${message.tool_calls!.length === 1 ? "" : "s"}`} -
- ) : null} - , - ); - } - return rows; - })} + ) : usePlainText ? ( + + ) : ( + + ) + ) : ( + + )} + {text && + Array.isArray(message.content) && + (message.content as ContentBlock[]) + .filter((b) => b.type === "image_url") + .map((b, i) => { + const src = b.image_url?.url; + if (!src) return null; + return ( + Attached image + ); + })} + {text && + Array.isArray(message.content) && + (message.content as ContentBlock[]) + .filter((b) => b.type === "file" && b.file) + .map((b, i) => ( + + ))} + {hasToolCalls && text ? ( +
+ {message + .tool_calls!.map((tc) => tc.function?.name) + .filter(Boolean) + .join(", ") || `${message.tool_calls!.length} tool call${message.tool_calls!.length === 1 ? "" : "s"}`} +
+ ) : null} + , + ); + } + return rows; + }); + // Show only the last two turns; everything earlier collapses + // behind an expandable history toggle. + const visibleTail = 2; + const splitAt = Math.max(0, messageRows.length - visibleTail); + const earlier = messageRows.slice(0, splitAt); + const tail = messageRows.slice(splitAt); + const earlierCount = earlier.filter((r) => r.length > 0).length; + return ( + <> + {earlierCount > 0 && {earlier}} + {tail} + + ); + })()} {log.output_message && !log.error_details?.error.message && (() => { @@ -3233,7 +3323,7 @@ export function LogDetailView({ ) : null} {showAssistant ? ( - + {showRefusal ? (
@@ -3322,7 +3412,7 @@ export function LogDetailView({ const itemPayload = extractResponsesItemPayload(msg); const lineCount = text ? text.split("\n").length : 0; const approxTokens = text ? Math.max(1, Math.round(text.length / 4)) : 0; - let meta: string | undefined; + let meta: ReactNode | undefined; if (role === "reasoning" && reasoningParts) { const totalLen = reasoningParts.summaries.reduce((acc, s) => acc + s.length, 0) + @@ -3339,24 +3429,33 @@ export function LogDetailView({ ? "encrypted" : undefined; } else { - meta = text - ? role === "system" || role === "tool" - ? msg.name - ? `${msg.name} · ${lineCount} line${lineCount === 1 ? "" : "s"} · ~${approxTokens} tokens` - : `${lineCount} line${lineCount === 1 ? "" : "s"} · ~${approxTokens} tokens` - : `${lineCount} line${lineCount === 1 ? "" : "s"}` - : msg.name - ? msg.name - : msg.type === "function_call_output" && msg.call_id - ? msg.call_id - : Array.isArray(msg.tools) - ? (() => { - const callable = flattenDeclaredTools(msg.tools).length; - return callable !== msg.tools.length - ? `${msg.type} · ${msg.tools.length} declarations · ${callable} callable tools` - : `${msg.type} · ${msg.tools.length} tool${msg.tools.length === 1 ? "" : "s"}`; - })() - : [msg.type, summarizeResponsesToolCall(msg, mapping)].filter(Boolean).join(" · ") || undefined; + meta = text ? ( + role === "system" || role === "tool" ? ( + msg.name ? ( + <> + + {` · ${lineCount} line${lineCount === 1 ? "" : "s"} · ~${approxTokens} tokens`} + + ) : ( + `${lineCount} line${lineCount === 1 ? "" : "s"} · ~${approxTokens} tokens` + ) + ) : ( + `${lineCount} line${lineCount === 1 ? "" : "s"}` + ) + ) : msg.name ? ( + + ) : msg.type === "function_call_output" && msg.call_id ? ( + + ) : Array.isArray(msg.tools) ? ( + (() => { + const callable = flattenDeclaredTools(msg.tools).length; + return callable !== msg.tools.length + ? `${msg.type} · ${msg.tools.length} declarations · ${callable} callable tools` + : `${msg.type} · ${msg.tools.length} tool${msg.tools.length === 1 ? "" : "s"}`; + })() + ) : ( + [msg.type, summarizeResponsesToolCall(msg, mapping)].filter(Boolean).join(" · ") || undefined + ); } const usePlainText = role === "user" || role === "assistant"; return ( @@ -3378,10 +3477,9 @@ export function LogDetailView({
))} {reasoningParts.encrypted ? ( -
-
Encrypted
- -
+ // Ciphertext is noise even at two preview lines; fold it + // entirely until the reader asks for it. + ) : null} {reasoningParts.signatures.length > 0 ? ( { }); it("is storage-disabled when the provider explicitly disables raw storage", () => { - expect( - resolveRawJsonNoticeState({ ...base, providers: [{ name: "openai", store_raw_request_response: false }] }), - ).toBe("storage-disabled"); + expect(resolveRawJsonNoticeState({ ...base, providers: [{ name: "openai", store_raw_request_response: false }] })).toBe( + "storage-disabled", + ); }); it("is unknown when the provider has raw storage enabled", () => { - expect( - resolveRawJsonNoticeState({ ...base, providers: [{ name: "openai", store_raw_request_response: true }] }), - ).toBe("unknown"); + expect(resolveRawJsonNoticeState({ ...base, providers: [{ name: "openai", store_raw_request_response: true }] })).toBe("unknown"); }); it("is unknown when the setting is absent on the provider", () => { @@ -43,9 +41,7 @@ describe("resolveRawJsonNoticeState", () => { }); it("is unknown when this log's provider is not in the list", () => { - expect( - resolveRawJsonNoticeState({ ...base, providers: [{ name: "anthropic", store_raw_request_response: false }] }), - ).toBe("unknown"); + expect(resolveRawJsonNoticeState({ ...base, providers: [{ name: "anthropic", store_raw_request_response: false }] })).toBe("unknown"); }); }); diff --git a/ui/app/workspace/logs/sheets/logDetailsSheet.tsx b/ui/app/workspace/logs/sheets/logDetailsSheet.tsx index 18638a1fb6c..a5dfa651da6 100644 --- a/ui/app/workspace/logs/sheets/logDetailsSheet.tsx +++ b/ui/app/workspace/logs/sheets/logDetailsSheet.tsx @@ -75,7 +75,7 @@ export function LogDetailSheet({ return ( - + {!isFullDataReady ? (
Loading log details @@ -117,4 +117,4 @@ export function LogDetailSheet({ ); -} +} \ No newline at end of file diff --git a/ui/app/workspace/logs/views/logsHeaderView.tsx b/ui/app/workspace/logs/views/logsHeaderView.tsx index 237aa5f67ad..d521efc621a 100644 --- a/ui/app/workspace/logs/views/logsHeaderView.tsx +++ b/ui/app/workspace/logs/views/logsHeaderView.tsx @@ -367,7 +367,7 @@ export function LogsHeaderView({ {/* Full width while the row wraps, so the search field owns its own line instead of squeezing to its 12rem minimum beside the date picker. */} -
+
{ it("reads a rise as positive by default", () => { @@ -231,4 +242,4 @@ describe("tokenSplit", () => { it("gives the whole bar to output when there is no input", () => { expect(tokenSplit(0, 400)).toEqual({ first: 0, second: 100 }); }); -}); +}); \ No newline at end of file diff --git a/ui/app/workspace/logs/views/metricStrip.utils.ts b/ui/app/workspace/logs/views/metricStrip.utils.ts index 5c0e0c2570b..a1aa2432118 100644 --- a/ui/app/workspace/logs/views/metricStrip.utils.ts +++ b/ui/app/workspace/logs/views/metricStrip.utils.ts @@ -235,4 +235,4 @@ export function tokenSplit(first: number, second: number): TokenSplit | null { if (total <= 0) return null; const share = (first / total) * 100; return { first: share, second: 100 - share }; -} +} \ No newline at end of file diff --git a/ui/app/workspace/mcp-registry/library/views/mcpLibraryAddServerSheet.tsx b/ui/app/workspace/mcp-registry/library/views/mcpLibraryAddServerSheet.tsx index ab663d9798e..8376a8ce6b8 100644 --- a/ui/app/workspace/mcp-registry/library/views/mcpLibraryAddServerSheet.tsx +++ b/ui/app/workspace/mcp-registry/library/views/mcpLibraryAddServerSheet.tsx @@ -172,9 +172,9 @@ export function MCPLibraryAddServerSheet({ open, onClose }: MCPLibraryAddServerS

- This listing stays in your organization's library. It is not published to the global Bifrost MCP catalog. It is also a - catalog entry, not a live MCP connection: credentials are never stored on the listing, each person who installs it - supplies their own. + This listing stays in your organization's library. It is not published to the global Bifrost MCP catalog. It is + also a catalog entry, not a live MCP connection: credentials are never stored on the listing, each person who installs + it supplies their own.

diff --git a/ui/app/workspace/mcp-registry/library/views/mcpLibrarySettingsSheet.tsx b/ui/app/workspace/mcp-registry/library/views/mcpLibrarySettingsSheet.tsx index cb6d0b83800..5cae509f645 100644 --- a/ui/app/workspace/mcp-registry/library/views/mcpLibrarySettingsSheet.tsx +++ b/ui/app/workspace/mcp-registry/library/views/mcpLibrarySettingsSheet.tsx @@ -19,8 +19,7 @@ const mcpLibrarySettingsSchema = z.object({ .string() .trim() .refine( - (value) => - value === "" || value.startsWith("http://") || value.startsWith("https://") || value.startsWith("file://"), + (value) => value === "" || value.startsWith("http://") || value.startsWith("https://") || value.startsWith("file://"), "URL must start with http://, https://, or file://", ), // 0 disables background syncing entirely. Force Sync Now still works. @@ -122,8 +121,8 @@ export function MCPLibrarySettingsSheet({ open, onClose }: MCPLibrarySettingsShe

- URL to a custom MCP server catalog. Leave empty to use the default Bifrost catalog. Use a{" "} - file:// URL to load the catalog from local disk in air-gapped deployments. + URL to a custom MCP server catalog. Leave empty to use the default Bifrost catalog. Use a file:// URL to load + the catalog from local disk in air-gapped deployments.

- How often to sync the MCP server catalog from the source URL. Set to 0 to disable background syncing; - Force Sync Now still works. + How often to sync the MCP server catalog from the source URL. Set to 0 to disable background syncing; Force Sync Now still + works.

{ throw redirect({ to: "/workspace/virtual-mcps" }); }, -}); +}); \ No newline at end of file diff --git a/ui/app/workspace/model-limits/views/modelLimitsTable.tsx b/ui/app/workspace/model-limits/views/modelLimitsTable.tsx index adc4109d280..ea37264fb73 100644 --- a/ui/app/workspace/model-limits/views/modelLimitsTable.tsx +++ b/ui/app/workspace/model-limits/views/modelLimitsTable.tsx @@ -423,39 +423,39 @@ export default function ModelLimitsTable({
- {config.scope !== "global" && config.scope_id && config.scope_name ? ( - - - - { - if (!config.scope_id) return; - const target = getModelLimitScope(config.scope ?? "global")?.buildDeepLink?.(config.scope_id); - if (target) navigate(target as never); - }} - > - {config.scope_name} - {getModelLimitScope(config.scope ?? "global")?.buildDeepLink && ( - - )} - - - {config.scope_name} - - - ) : ( - - - )} - {(() => { - const ManagedBy = getModelLimitScope(config.scope ?? "global")?.ManagedByComponent; - return ManagedBy ? : null; - })()} + {config.scope !== "global" && config.scope_id && config.scope_name ? ( + + + + { + if (!config.scope_id) return; + const target = getModelLimitScope(config.scope ?? "global")?.buildDeepLink?.(config.scope_id); + if (target) navigate(target as never); + }} + > + {config.scope_name} + {getModelLimitScope(config.scope ?? "global")?.buildDeepLink && ( + + )} + + + {config.scope_name} + + + ) : ( + - + )} + {(() => { + const ManagedBy = getModelLimitScope(config.scope ?? "global")?.ManagedByComponent; + return ManagedBy ? : null; + })()}
diff --git a/ui/app/workspace/observability/fragments/otelFormFragment.tsx b/ui/app/workspace/observability/fragments/otelFormFragment.tsx index 5ca420779f0..2095b465931 100644 --- a/ui/app/workspace/observability/fragments/otelFormFragment.tsx +++ b/ui/app/workspace/observability/fragments/otelFormFragment.tsx @@ -609,60 +609,60 @@ function OtelProfileSection({ form, control, index, hasOtelAccess, canRemove, op )} />
- ( - - Format - + + + + + + + {traceTypeOptions.map((option) => ( + + {option.label} + + ))} + + + + + )} + /> + ( + + Export Timeout (seconds) - - - + field.onChange(e.target.value === "" ? null : Number(e.target.value))} + /> - - {traceTypeOptions.map((option) => ( - - {option.label} - - ))} - - - - - )} - /> - ( - - Export Timeout (seconds) - - field.onChange(e.target.value === "" ? null : Number(e.target.value))} - /> - - - Maximum time for a single trace export (1-60 seconds). Traces are dropped rather than retried past this limit, so - an unreachable collector cannot slow down request handling. - - - - )} - /> + + Maximum time for a single trace export (1-60 seconds). Traces are dropped rather than retried past this limit, + so an unreachable collector cannot slow down request handling. + + + + )} + />

Overhead breakdown

-

- Export per-component Bifrost overhead latency as a histogram. -

+

Export per-component Bifrost overhead latency as a histogram.

- +
diff --git a/ui/app/workspace/providers/dialogs/databricksMigrationDialog.tsx b/ui/app/workspace/providers/dialogs/databricksMigrationDialog.tsx index fde99193e9e..17a3d47630d 100644 --- a/ui/app/workspace/providers/dialogs/databricksMigrationDialog.tsx +++ b/ui/app/workspace/providers/dialogs/databricksMigrationDialog.tsx @@ -223,24 +223,30 @@ export default function DatabricksMigrationDialog({ show, provider, onDeferred, - {stage === "finished" && result ? (result.ok ? "Migration complete" : "Migration failed") : "Databricks is now a first-party provider"} + {stage === "finished" && result + ? result.ok + ? "Migration complete" + : "Migration failed" + : "Databricks is now a first-party provider"}
{(stage === "intro" || stage === "loading") && ( <>

- {provider.name} is a custom provider pointing at a Databricks workspace. - Bifrost now supports Databricks natively, with personal access tokens, OAuth service principals, Model Serving and AI Gateway - routing. Your existing configuration needs to be migrated to the official provider. + {provider.name} is a custom provider pointing at a Databricks + workspace. Bifrost now supports Databricks natively, with personal access tokens, OAuth service principals, Model + Serving and AI Gateway routing. Your existing configuration needs to be migrated to the official provider.

- Nothing changes until you confirm. You will see exactly what will be copied before the migration runs, and the custom provider is - only removed after the new one is verified. + Nothing changes until you confirm. You will see exactly what will be copied before the migration runs, and the custom + provider is only removed after the new one is verified.

)} - {stage === "preview" &&

Review what will be migrated. Secrets are masked; anything that could not be read must be entered below.

} + {stage === "preview" && ( +

Review what will be migrated. Secrets are masked; anything that could not be read must be entered below.

+ )} {stage === "running" &&

Migrating. Keep this window open until it finishes.

} {stage === "finished" && result &&

{result.message}

}
@@ -273,7 +279,11 @@ export default function DatabricksMigrationDialog({ show, provider, onDeferred, Not now - @@ -286,7 +296,9 @@ export default function DatabricksMigrationDialog({ show, provider, onDeferred, Back Not now - + @@ -485,7 +497,9 @@ function MigrationProgress({ steps }: { steps: MigrationStep[] }) {
{step.label}
{step.detail && ( -
+
{step.detail}
)} @@ -522,4 +536,4 @@ function Stat({ label, value }: { label: string; value: string }) {
{value}
); -} +} \ No newline at end of file diff --git a/ui/app/workspace/providers/dialogs/firstPartyProviderAvailableDialog.tsx b/ui/app/workspace/providers/dialogs/firstPartyProviderAvailableDialog.tsx index 3c1438d99be..c2f8497d0f8 100644 --- a/ui/app/workspace/providers/dialogs/firstPartyProviderAvailableDialog.tsx +++ b/ui/app/workspace/providers/dialogs/firstPartyProviderAvailableDialog.tsx @@ -32,8 +32,8 @@ export default function FirstPartyProviderAvailableDialog({ show, customProvider Official {label} integration available - We noticed you have a custom provider named {customProviderName}. - Bifrost now supports {label} natively. You can delete the custom provider and add the official {label} integration from{" "} + We noticed you have a custom provider named {customProviderName}. Bifrost + now supports {label} natively. You can delete the custom provider and add the official {label} integration from{" "} Add Provider, or keep using your custom provider as is. @@ -44,4 +44,4 @@ export default function FirstPartyProviderAvailableDialog({ show, customProvider ); -} +} \ No newline at end of file diff --git a/ui/app/workspace/providers/fragments/apiKeysFormFragment.tsx b/ui/app/workspace/providers/fragments/apiKeysFormFragment.tsx index c2b6b31e298..feb58cabe46 100644 --- a/ui/app/workspace/providers/fragments/apiKeysFormFragment.tsx +++ b/ui/app/workspace/providers/fragments/apiKeysFormFragment.tsx @@ -370,121 +370,121 @@ export function ApiKeyFormFragment({ control, providerName, baseProviderType, fo )} /> )} - <> - ( - - - - Allowed Models - - - - - - - - -

- Select specific models this key applies to, or choose "Allow All Models" to allow all. Leave empty to deny - all. Aliases must be added by their alias name - listing only the underlying model does not allow the alias - (an alias best-model → gpt-4o requires "best-model" here, not just "gpt-4o"). -

- - - - - } - /> - - - - )} - /> - ( - - - - Blocked Models - - - - - - - - -

- Models this key must never serve. The denylist always wins - if a model appears in both Allowed Models and - here, it is blocked. Select "All Models" to block every model on this key. Aliases are matched by their - alias name - blocking only the underlying model does not block aliases that point to it. -

- - - - - } - /> - - - - )} - /> - ( - - Deployments (Optional) - - Map a request model name to the provider's identifier (deployment name, inference profile ID, etc.). Expand a row for - canonical name, model family, and provider overrides - these drive cost logs and family-based routing. - {isReplicate && ( + <> + ( + + + - {" "} - Replicate deployments are listed only while "Use Deployments Endpoint" is on - otherwise type the owner/name - and press Enter. + Allowed Models + + + + + + + + +

+ Select specific models this key applies to, or choose "Allow All Models" to allow all. Leave empty to deny + all. Aliases must be added by their alias name - listing only the underlying model does not allow the alias + (an alias best-model → gpt-4o requires "best-model" here, not just "gpt-4o"). +

+ + + - )} - - -
- { - form.clearErrors("key.aliases"); - field.onChange(Object.keys(next).length > 0 ? next : {}); - }} - /> -
- - - - )} - /> - + } + /> + + + + )} + /> + ( + + + + Blocked Models + + + + + + + + +

+ Models this key must never serve. The denylist always wins - if a model appears in both Allowed Models and + here, it is blocked. Select "All Models" to block every model on this key. Aliases are matched by their alias + name - blocking only the underlying model does not block aliases that point to it. +

+ + + + + } + /> + + + + )} + /> + ( + + Deployments (Optional) + + Map a request model name to the provider's identifier (deployment name, inference profile ID, etc.). Expand a row for + canonical name, model family, and provider overrides - these drive cost logs and family-based routing. + {isReplicate && ( + <> + {" "} + Replicate deployments are listed only while "Use Deployments Endpoint" is on - otherwise type the owner/name + and press Enter. + + )} + + +
+ { + form.clearErrors("key.aliases"); + field.onChange(Object.keys(next).length > 0 ? next : {}); + }} + /> +
+ + + + )} + /> + {supportsBatchAPI && !isBedrock && !isAzure && !isVertex && } {isAzure && (
@@ -1687,4 +1687,4 @@ export function ApiKeyFormFragment({ control, providerName, baseProviderType, fo )}
); -} +} \ No newline at end of file diff --git a/ui/app/workspace/providers/fragments/governanceFormFragment.tsx b/ui/app/workspace/providers/fragments/governanceFormFragment.tsx index 3e0e0021db3..e3492a9e66a 100644 --- a/ui/app/workspace/providers/fragments/governanceFormFragment.tsx +++ b/ui/app/workspace/providers/fragments/governanceFormFragment.tsx @@ -224,7 +224,8 @@ export function GovernanceFormFragment({ provider }: GovernanceFormFragmentProps Align to calendar cycle

- Reset budgets at the start of each period (e.g. 1st of month) instead of rolling from creation date. Quarterly budgets always align to fiscal quarter starts. + Reset budgets at the start of each period (e.g. 1st of month) instead of rolling from creation date. Quarterly budgets + always align to fiscal quarter starts.

- ); - }) + (fallbacks || []).map((fallback, index) => ( + + )) )}

Fallbacks will be used in the order they are defined

@@ -645,10 +616,143 @@ export function RoutingRuleSheet({ open, onOpenChange, editingRule, onSuccess }: ); } +interface ProviderKeySelectProps { + idPrefix: string; + clearLabel: string; + provider?: string; + keyId?: string; + allKeys: Array<{ key_id: string; name: string; provider: string }>; + onChange: (keyId: string) => void; +} + +/** Renders nothing until a provider is chosen, since keys are scoped to one. */ +function ProviderKeySelect({ idPrefix, clearLabel, provider, keyId, allKeys, onChange }: ProviderKeySelectProps) { + const availableKeys = provider ? allKeys.filter((k) => k.provider === provider).map((k) => ({ id: k.key_id, name: k.name })) : []; + if (!provider || (availableKeys.length === 0 && !keyId)) { + return null; + } + + return ( +
+ +
+ + {keyId && ( + + )} +
+
+ ); +} + +interface FallbackRowProps { + fallback: RoutingFallbackFormData; + index: number; + providerOptions: Array<{ + label: string; + value: string; + icon: React.ReactNode; + }>; + allKeys: Array<{ key_id: string; name: string; provider: string }>; + onUpdate: (index: number, changes: Partial) => void; + onRemove: (index: number) => void; +} + +function FallbackRow({ fallback, index, providerOptions, allKeys, onUpdate, onRemove }: FallbackRowProps) { + const provider = fallback.provider || ""; + + return ( +
+
+
+ onUpdate(index, { provider: value ?? "", model: "", key_id: "" })} + placeholder="Select provider..." + className="h-9" + data-testid={`routing-fallback-${index}-provider-select`} + noPortal + /> +
+
+ onUpdate(index, { model: value })} + placeholder="Incoming (optional)" + isSingleSelect + disabled={!provider} + className="!h-9 !min-h-9 w-full" + /> +
+ +
+ + onUpdate(index, { key_id: value })} + /> +
+ ); +} + interface TargetRowProps { target: RoutingTargetFormData; index: number; - providerOptions: Array<{ label: string; value: string; icon: React.ReactNode }>; + providerOptions: Array<{ + label: string; + value: string; + icon: React.ReactNode; + }>; allKeys: Array<{ key_id: string; name: string; provider: string }>; showRemove: boolean; onUpdate: (index: number, field: keyof RoutingTargetFormData, value: string | number) => void; @@ -656,10 +760,6 @@ interface TargetRowProps { } function TargetRow({ target, index, providerOptions, allKeys, showRemove, onUpdate, onRemove }: TargetRowProps) { - const availableKeys = target.provider - ? allKeys.filter((k) => k.provider === target.provider).map((k) => ({ id: k.key_id, name: k.name })) - : []; - return (
@@ -771,50 +871,14 @@ function TargetRow({ target, index, providerOptions, allKeys, showRemove, onUpda
- {target.provider && (availableKeys.length > 0 || target.key_id) && ( -
- -
- - {target.key_id && ( - - )} -
-
- )} + onUpdate(index, "key_id", value)} + />
); } \ No newline at end of file diff --git a/ui/app/workspace/virtual-keys/hooks/useVirtualKeyUsage.utils.test.ts b/ui/app/workspace/virtual-keys/hooks/useVirtualKeyUsage.utils.test.ts index 786d1ab6b74..233eaa74c20 100644 --- a/ui/app/workspace/virtual-keys/hooks/useVirtualKeyUsage.utils.test.ts +++ b/ui/app/workspace/virtual-keys/hooks/useVirtualKeyUsage.utils.test.ts @@ -45,4 +45,4 @@ describe("resolveDisplayRateLimit", () => { it("shows nothing for an unmanaged key with no rate limit", () => { expect(resolveDisplayRateLimit({ isManagedByProfile: false, profileRateLimit: undefined, vkRateLimit: undefined })).toBeUndefined(); }); -}); +}); \ No newline at end of file diff --git a/ui/app/workspace/virtual-keys/hooks/useVirtualKeyUsage.utils.ts b/ui/app/workspace/virtual-keys/hooks/useVirtualKeyUsage.utils.ts index b87b87e5961..d20dbd19a94 100644 --- a/ui/app/workspace/virtual-keys/hooks/useVirtualKeyUsage.utils.ts +++ b/ui/app/workspace/virtual-keys/hooks/useVirtualKeyUsage.utils.ts @@ -41,4 +41,4 @@ export function resolveDisplayRateLimit({ request_current_usage: profileRateLimit.request_current_usage ?? 0, request_last_reset: profileRateLimit.request_last_reset ?? "", }; -} +} \ No newline at end of file diff --git a/ui/app/workspace/virtual-keys/views/virtualKeyDetailsSheet.tsx b/ui/app/workspace/virtual-keys/views/virtualKeyDetailsSheet.tsx index 4ba166e7cf4..fb5dc6bf87b 100644 --- a/ui/app/workspace/virtual-keys/views/virtualKeyDetailsSheet.tsx +++ b/ui/app/workspace/virtual-keys/views/virtualKeyDetailsSheet.tsx @@ -1,7 +1,7 @@ import { BudgetOverrideDialog } from "@/components/budgetOverrideDialog"; import { BudgetOverrideManagerDialog, type BudgetOverrideSection } from "@/components/budgetOverrideManagerDialog"; import { CopyableId } from "@/components/copyableId"; -import { ModelAccessBadges } from "@/components/modelAccess"; +import { isWildcardList, ModelAccessBadges } from "@/components/modelAccess"; import { SheetNavigationButtons } from "@/components/sheetNavigationButtons"; import { Badge } from "@/components/ui/badge"; import { Label } from "@/components/ui/label"; @@ -235,11 +235,28 @@ export default function VirtualKeyDetailSheet({ {/* Provider Configurations */}
-

Provider Configurations

+
+

Provider Configurations

+ {virtualKey.allow_all_providers && ( + + All providers + + )} +
+ + {/* A key that allows every provider grants ones it holds no entry for, including ones added + later, so the entries below are overrides rather than the whole of what it may reach. */} + {virtualKey.allow_all_providers && ( +

+ Every provider is allowed, including ones added later. Entries below indicate specific provider level configuration. +

+ )}
{!virtualKey.provider_configs || virtualKey.provider_configs.length === 0 ? ( - No providers configured (deny-by-default) + + {virtualKey.allow_all_providers ? "No provider overrides" : "No providers configured (deny-by-default)"} + ) : (
{virtualKey.provider_configs.map((config, index) => ( @@ -279,7 +296,11 @@ export default function VirtualKeyDetailSheet({
Blocked Models
- +
diff --git a/ui/app/workspace/virtual-keys/views/virtualKeySheet.tsx b/ui/app/workspace/virtual-keys/views/virtualKeySheet.tsx index e413d48f5bb..d5af2e8767e 100644 --- a/ui/app/workspace/virtual-keys/views/virtualKeySheet.tsx +++ b/ui/app/workspace/virtual-keys/views/virtualKeySheet.tsx @@ -447,11 +447,11 @@ export default function VirtualKeySheet({ virtualKey, defaultTeamId, onSave, onC })), rate_limit: config.rate_limit ? { - token_max_limit: config.rate_limit.token_max_limit ?? undefined, - token_reset_duration: config.rate_limit.token_reset_duration, - request_max_limit: config.rate_limit.request_max_limit ?? undefined, - request_reset_duration: config.rate_limit.request_reset_duration, - } + token_max_limit: config.rate_limit.token_max_limit ?? undefined, + token_reset_duration: config.rate_limit.token_reset_duration, + request_max_limit: config.rate_limit.request_max_limit ?? undefined, + request_reset_duration: config.rate_limit.request_reset_duration, + } : undefined, model_budgets: config.model_budgets?.map((mb) => ({ model_name: mb.model_name, @@ -463,11 +463,11 @@ export default function VirtualKeySheet({ virtualKey, defaultTeamId, onSave, onC })), rate_limit: mb.rate_limit ? { - token_max_limit: mb.rate_limit.token_max_limit ?? undefined, - token_reset_duration: mb.rate_limit.token_reset_duration, - request_max_limit: mb.rate_limit.request_max_limit ?? undefined, - request_reset_duration: mb.rate_limit.request_reset_duration, - } + token_max_limit: mb.rate_limit.token_max_limit ?? undefined, + token_reset_duration: mb.rate_limit.token_reset_duration, + request_max_limit: mb.rate_limit.request_max_limit ?? undefined, + request_reset_duration: mb.rate_limit.request_reset_duration, + } : undefined, })), })) || [], @@ -486,18 +486,18 @@ export default function VirtualKeySheet({ virtualKey, defaultTeamId, onSave, onC isActive: virtualKey?.is_active ?? true, expiresAt: virtualKey?.expires_at ? (() => { - const d = new Date(virtualKey.expires_at); - return new Date(d.getTime() - d.getTimezoneOffset() * 60000).toISOString().slice(0, 16); - })() + const d = new Date(virtualKey.expires_at); + return new Date(d.getTime() - d.getTimezoneOffset() * 60000).toISOString().slice(0, 16); + })() : null, budgets: virtualKey?.budgets && virtualKey.budgets.length > 0 ? virtualKey.budgets.map((b) => ({ - id: b.id, - max_limit: b.max_limit, - reset_duration: b.reset_duration ?? "1M", - reset_config: b.reset_config, - })) + id: b.id, + max_limit: b.max_limit, + reset_duration: b.reset_duration ?? "1M", + reset_config: b.reset_config, + })) : [], budgetCalendarAligned: virtualKey?.calendar_aligned ?? false, tokenMaxLimit: virtualKey?.rate_limit?.token_max_limit ?? undefined, @@ -1531,9 +1531,9 @@ export default function VirtualKeySheet({ virtualKey, defaultTeamId, onSave, onC fallbackOption={ field.value ? { - value: field.value, - label: field.value === virtualKey?.team_id ? (virtualKey?.team?.name ?? field.value) : field.value, - } + value: field.value, + label: field.value === virtualKey?.team_id ? (virtualKey?.team?.name ?? field.value) : field.value, + } : null } disabled={isTeamLocked} @@ -1561,12 +1561,12 @@ export default function VirtualKeySheet({ virtualKey, defaultTeamId, onSave, onC fallbackOption={ field.value ? { - value: field.value, - label: - field.value === virtualKey?.customer_id - ? (virtualKey?.customer?.name ?? field.value) - : field.value, - } + value: field.value, + label: + field.value === virtualKey?.customer_id + ? (virtualKey?.customer?.name ?? field.value) + : field.value, + } : null } triggerClassName="h-9" @@ -1595,9 +1595,9 @@ export default function VirtualKeySheet({ virtualKey, defaultTeamId, onSave, onC fallbackOption={ field.value ? { - value: field.value, - label: field.value === assignedUserId ? assignedUserLabel : field.value, - } + value: field.value, + label: field.value === assignedUserId ? assignedUserLabel : field.value, + } : null } triggerClassName="h-9" diff --git a/ui/app/workspace/virtual-keys/views/virtualKeySheet.utils.test.ts b/ui/app/workspace/virtual-keys/views/virtualKeySheet.utils.test.ts index 4f7430030b6..8c0c610731a 100644 --- a/ui/app/workspace/virtual-keys/views/virtualKeySheet.utils.test.ts +++ b/ui/app/workspace/virtual-keys/views/virtualKeySheet.utils.test.ts @@ -32,4 +32,4 @@ describe("vmcpAssignmentsDirty", () => { expect(vmcpAssignmentsDirty([1, 2], [1, 2, 3])).toBe(true); expect(vmcpAssignmentsDirty([1, 2], [1])).toBe(true); }); -}); +}); \ No newline at end of file diff --git a/ui/app/workspace/virtual-keys/views/virtualKeySheet.utils.ts b/ui/app/workspace/virtual-keys/views/virtualKeySheet.utils.ts index 9ce80c29d27..f5f29054711 100644 --- a/ui/app/workspace/virtual-keys/views/virtualKeySheet.utils.ts +++ b/ui/app/workspace/virtual-keys/views/virtualKeySheet.utils.ts @@ -18,4 +18,4 @@ export function diffVmcpAssignments(original: number[], staged: number[]): { toA export function vmcpAssignmentsDirty(original: number[], staged: number[]): boolean { const { toAttach, toDetach } = diffVmcpAssignments(original, staged); return toAttach.length > 0 || toDetach.length > 0; -} +} \ No newline at end of file diff --git a/ui/app/workspace/virtual-keys/views/virtualKeysTable.tsx b/ui/app/workspace/virtual-keys/views/virtualKeysTable.tsx index b1edf5b4d0a..876db7fa410 100644 --- a/ui/app/workspace/virtual-keys/views/virtualKeysTable.tsx +++ b/ui/app/workspace/virtual-keys/views/virtualKeysTable.tsx @@ -1066,4 +1066,4 @@ export default function VirtualKeysTable({
); -} +} \ No newline at end of file diff --git a/ui/app/workspace/virtual-keys/views/virtualKeysTable.utils.test.ts b/ui/app/workspace/virtual-keys/views/virtualKeysTable.utils.test.ts index d7f9f5d1491..09a7bc821f0 100644 --- a/ui/app/workspace/virtual-keys/views/virtualKeysTable.utils.test.ts +++ b/ui/app/workspace/virtual-keys/views/virtualKeysTable.utils.test.ts @@ -28,9 +28,9 @@ describe("latestGraceDeadline", () => { }); it("ignores keys without a grace window when picking the latest", () => { - expect( - latestGraceDeadline([{}, { previous_value_expires_at: "2026-08-28T10:00:00.500Z" }, { previous_value_expires_at: null }]), - ).toBe("2026-08-28T10:00:00.500Z"); + expect(latestGraceDeadline([{}, { previous_value_expires_at: "2026-08-28T10:00:00.500Z" }, { previous_value_expires_at: null }])).toBe( + "2026-08-28T10:00:00.500Z", + ); }); }); @@ -82,4 +82,4 @@ describe("csvAssignedToCell", () => { it("labels a resolved user assignment", () => { expect(csvAssignedToCell({ assigned_user: { name: "Ada", email: "ada@acme.com" } })).toBe("User: Ada"); }); -}); +}); \ No newline at end of file diff --git a/ui/app/workspace/virtual-keys/views/virtualKeysTable.utils.ts b/ui/app/workspace/virtual-keys/views/virtualKeysTable.utils.ts index a8909a14001..33f71ccb85f 100644 --- a/ui/app/workspace/virtual-keys/views/virtualKeysTable.utils.ts +++ b/ui/app/workspace/virtual-keys/views/virtualKeysTable.utils.ts @@ -55,4 +55,4 @@ export function csvAssignedToCell(vk: { // Team and customer ride on the row itself, so they are never the unresolved case. if (!vk.team && !vk.customer && vk.assigned_user === undefined) return "Unknown (not resolved)"; return ""; -} +} \ No newline at end of file diff --git a/ui/app/workspace/virtual-mcps/layout.tsx b/ui/app/workspace/virtual-mcps/layout.tsx index 652d6ce9782..72753e5f449 100644 --- a/ui/app/workspace/virtual-mcps/layout.tsx +++ b/ui/app/workspace/virtual-mcps/layout.tsx @@ -13,4 +13,4 @@ function RouteComponent() { export const Route = createFileRoute("/workspace/virtual-mcps")({ component: RouteComponent, -}); +}); \ No newline at end of file diff --git a/ui/app/workspace/virtual-mcps/views/virtualMcpSheet.tsx b/ui/app/workspace/virtual-mcps/views/virtualMcpSheet.tsx index f5da410ca48..cf9e68846bc 100644 --- a/ui/app/workspace/virtual-mcps/views/virtualMcpSheet.tsx +++ b/ui/app/workspace/virtual-mcps/views/virtualMcpSheet.tsx @@ -275,4 +275,4 @@ export default function VirtualMCPSheet({ target, onClose, hasPrev = false, hasN ); -} +} \ No newline at end of file diff --git a/ui/app/workspace/virtual-mcps/views/virtualMcpsTable.tsx b/ui/app/workspace/virtual-mcps/views/virtualMcpsTable.tsx index dd3bb3874d2..f3d6a6e7fa1 100644 --- a/ui/app/workspace/virtual-mcps/views/virtualMcpsTable.tsx +++ b/ui/app/workspace/virtual-mcps/views/virtualMcpsTable.tsx @@ -30,7 +30,20 @@ import { } from "@/lib/store"; import { VirtualMCP } from "@/lib/types/virtualMcps"; import { RbacOperation, RbacResource, useRbac } from "@enterprise/lib"; -import { AlertTriangle, Check, ChevronLeft, ChevronRight, Copy, Loader2, MoreHorizontal, Pencil, Plus, Search, Server, Trash2 } from "lucide-react"; +import { + AlertTriangle, + Check, + ChevronLeft, + ChevronRight, + Copy, + Loader2, + MoreHorizontal, + Pencil, + Plus, + Search, + Server, + Trash2, +} from "lucide-react"; import { useMemo, useState } from "react"; interface VirtualMCPsTableProps { @@ -114,9 +127,14 @@ export default function VirtualMCPsTable({ } if (unreachable === 0 && unreachableServers === 0 && disabledAtSource === 0) return null; const parts: string[] = []; - if (unreachable > 0) parts.push(`${unreachable} ${unreachable === 1 ? "tool comes" : "tools come"} from an unreachable server (shown from the last successful sync)`); - if (unreachableServers > 0) parts.push(`${unreachableServers} source ${unreachableServers === 1 ? "server is" : "servers are"} unreachable`); - if (disabledAtSource > 0) parts.push(`${disabledAtSource} selected ${disabledAtSource === 1 ? "tool is" : "tools are"} disabled at source`); + if (unreachable > 0) + parts.push( + `${unreachable} ${unreachable === 1 ? "tool comes" : "tools come"} from an unreachable server (shown from the last successful sync)`, + ); + if (unreachableServers > 0) + parts.push(`${unreachableServers} source ${unreachableServers === 1 ? "server is" : "servers are"} unreachable`); + if (disabledAtSource > 0) + parts.push(`${disabledAtSource} selected ${disabledAtSource === 1 ? "tool is" : "tools are"} disabled at source`); return parts.join(" · "); }; const [pendingDelete, setPendingDelete] = useState(null); @@ -179,9 +197,7 @@ export default function VirtualMCPsTable({ - - Bundle tools from your MCP servers into a single endpoint, then assign it to virtual keys. - + Bundle tools from your MCP servers into a single endpoint, then assign it to virtual keys.
@@ -242,52 +258,52 @@ export default function VirtualMCPsTable({ virtualMcps.map((row) => { const warning = toolWarning(row); return ( - - {row.name} - - - - {row.tools?.length ?? 0} - - - {clientsLoading ? "…" : clientsError ? "—" : includedToolCount(row)} - {warning && ( - - - - - {warning} - - )} - - - {formatDate(row.created_at)} - - - - e.stopPropagation()}> - toggleEnabled(row, checked)} - aria-label={row.enabled ? "Disable Virtual MCP" : "Enable Virtual MCP"} - data-testid={`virtual-mcp-enabled-switch-${row.id}`} - /> - - e.stopPropagation()} - > - onEdit(row.id)} - onDelete={() => setPendingDelete(row)} - /> - - + + {row.name} + + + + {row.tools?.length ?? 0} + + + {clientsLoading ? "…" : clientsError ? "—" : includedToolCount(row)} + {warning && ( + + + + + {warning} + + )} + + + {formatDate(row.created_at)} + + + + e.stopPropagation()}> + toggleEnabled(row, checked)} + aria-label={row.enabled ? "Disable Virtual MCP" : "Enable Virtual MCP"} + data-testid={`virtual-mcp-enabled-switch-${row.id}`} + /> + + e.stopPropagation()} + > + onEdit(row.id)} + onDelete={() => setPendingDelete(row)} + /> + + ); }) )} @@ -446,4 +462,4 @@ function formatDate(iso: string): string { } catch { return iso; } -} +} \ No newline at end of file diff --git a/ui/app/workspace/webhooks/deliveries/deliveries.page.utils.test.ts b/ui/app/workspace/webhooks/deliveries/deliveries.page.utils.test.ts index 8d84199768f..2fab6db4a4c 100644 --- a/ui/app/workspace/webhooks/deliveries/deliveries.page.utils.test.ts +++ b/ui/app/workspace/webhooks/deliveries/deliveries.page.utils.test.ts @@ -72,4 +72,4 @@ describe("shouldShowEmptyState", () => { it("hides it when the query failed", () => { expect(shouldShowEmptyState({ rowCount: 0, loading: false, error: { status: 500 } })).toBe(false); }); -}); +}); \ No newline at end of file diff --git a/ui/app/workspace/webhooks/deliveries/deliveries.page.utils.ts b/ui/app/workspace/webhooks/deliveries/deliveries.page.utils.ts index c8a94db4447..39367ecf56b 100644 --- a/ui/app/workspace/webhooks/deliveries/deliveries.page.utils.ts +++ b/ui/app/workspace/webhooks/deliveries/deliveries.page.utils.ts @@ -39,4 +39,4 @@ export function initialIdSearchField(filters: WebhookDeliveryFilters): "request_ */ export function shouldShowEmptyState({ rowCount, loading, error }: { rowCount: number; loading: boolean; error?: unknown }): boolean { return rowCount === 0 && !loading && !error; -} +} \ No newline at end of file diff --git a/ui/app/workspace/webhooks/deliveries/views/deliveriesHeaderView.tsx b/ui/app/workspace/webhooks/deliveries/views/deliveriesHeaderView.tsx index 87b37e67bca..3ab636988e9 100644 --- a/ui/app/workspace/webhooks/deliveries/views/deliveriesHeaderView.tsx +++ b/ui/app/workspace/webhooks/deliveries/views/deliveriesHeaderView.tsx @@ -108,7 +108,7 @@ export function DeliveriesHeaderView({ Live -
+
({ name: endpoint?.name ?? "", diff --git a/ui/components/budgetDisplay.test.ts b/ui/components/budgetDisplay.test.ts index cf00ea44e9b..6e1f39add54 100644 --- a/ui/components/budgetDisplay.test.ts +++ b/ui/components/budgetDisplay.test.ts @@ -14,4 +14,4 @@ describe("BudgetDisplay overflow trigger", () => { expect(trigger![1]).toBe("button"); expect(trigger![0]).toMatch(/type="button"/); }); -}); +}); \ No newline at end of file diff --git a/ui/components/budgetDisplay.tsx b/ui/components/budgetDisplay.tsx index 3b47567caa1..a89076bce5e 100644 --- a/ui/components/budgetDisplay.tsx +++ b/ui/components/budgetDisplay.tsx @@ -47,7 +47,7 @@ export function BudgetDisplay({ budgets, calendarAligned, maxVisible }: BudgetDi
- + {formatCurrency(effectiveMaxLimit)} {b.label ? {b.label} : null} {hasOverride ? override : null} diff --git a/ui/components/filters/logsFilterSidebar.tsx b/ui/components/filters/logsFilterSidebar.tsx index b481e8193fa..8969fbd7b80 100644 --- a/ui/components/filters/logsFilterSidebar.tsx +++ b/ui/components/filters/logsFilterSidebar.tsx @@ -9,7 +9,12 @@ import { TruncatedLabel } from "@/components/ui/truncatedLabel"; import { useIsMobile } from "@/hooks/use-mobile"; import { RequestTypeLabels, RequestTypes, RoutingEngineUsedLabels, Statuses } from "@/lib/constants/logs"; import { useGetAvailableFilterDataQuery, useGetProvidersQuery } from "@/lib/store"; -import { COMPLEXITY_MECHANISM_LABELS, COMPLEXITY_MECHANISM_VALUES, COMPLEXITY_TIER_VALUES, LEGACY_COMPLEXITY_TIER_VALUES } from "@/lib/types/complexityRouter"; +import { + COMPLEXITY_MECHANISM_LABELS, + COMPLEXITY_MECHANISM_VALUES, + COMPLEXITY_TIER_VALUES, + LEGACY_COMPLEXITY_TIER_VALUES, +} from "@/lib/types/complexityRouter"; import type { LogFilters } from "@/lib/types/logs"; import { cn } from "@/lib/utils"; import { ChevronDown, LoaderCircle, PanelLeftClose, Plus, RotateCcw, Search } from "lucide-react"; @@ -329,13 +334,14 @@ function SearchableCheckboxList({ onCheckedChange={() => onToggle(item.key)} testId={ testIdPrefix - ? `${testIdPrefix}-checkbox-${normalizeTestIdKey - ? item.key - .toLowerCase() - .replace(/[^a-z0-9]+/g, "-") - .replace(/^-+|-+$/g, "") - : item.key - }` + ? `${testIdPrefix}-checkbox-${ + normalizeTestIdKey + ? item.key + .toLowerCase() + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-+|-+$/g, "") + : item.key + }` : undefined } /> @@ -1480,4 +1486,4 @@ function MetadataFilters({ filters, onFiltersChange, defaultOpen }: FilterCompon )} ); -} +} \ No newline at end of file diff --git a/ui/components/mcp/mcpClientConfigsEditor.tsx b/ui/components/mcp/mcpClientConfigsEditor.tsx index 0f19b4f7364..f60c8c63d61 100644 --- a/ui/components/mcp/mcpClientConfigsEditor.tsx +++ b/ui/components/mcp/mcpClientConfigsEditor.tsx @@ -143,9 +143,8 @@ export function MCPClientConfigsEditor({ The following MCP servers are allowed by default, with all tools enabled, on any virtual key that doesn't configure them - explicitly:{" "} - {defaultMCPClients.map((c) => c.config.name).join(", ")}. Adding an - explicit config for one below overrides that all-tools default. + explicitly: {defaultMCPClients.map((c) => c.config.name).join(", ")}. + Adding an explicit config for one below overrides that all-tools default.
diff --git a/ui/components/modelAccess/modelAccessBadges.tsx b/ui/components/modelAccess/modelAccessBadges.tsx index a462e0a631b..868868828f9 100644 --- a/ui/components/modelAccess/modelAccessBadges.tsx +++ b/ui/components/modelAccess/modelAccessBadges.tsx @@ -15,6 +15,12 @@ interface ModelAccessBadgesProps { * projects carry that as a separate all_models_allowed flag. */ allModels?: boolean; + /** + * For a block list, whether the allow side beside it permits every model. An empty block list + * blocks nothing only when it does: against a restricted allow list, every model outside that + * list is already denied, so "no models blocked" would read as "everything is available". + */ + allowsAllModels?: boolean; /** Rendered when the list is empty; defaults to the mode's standard badge. */ empty?: ReactNode; /** Badge variant for individual entries; defaults per mode. */ @@ -27,7 +33,16 @@ interface ModelAccessBadgesProps { * Read-only rendering of an allow or block side: the "All Models" badge, one * badge per entry (patterns in monospace with an icon), or an empty-state badge. */ -export function ModelAccessBadges({ value, mode, allModels, empty, entryVariant, entryClassName, className }: ModelAccessBadgesProps) { +export function ModelAccessBadges({ + value, + mode, + allModels, + allowsAllModels, + empty, + entryVariant, + entryClassName, + className, +}: ModelAccessBadgesProps) { const entries = (value ?? []).filter((e) => e !== "*"); const isAll = allModels || isWildcardList(value); @@ -51,7 +66,7 @@ export function ModelAccessBadges({ value, mode, allModels, empty, entryVariant, ) : ( - No models blocked + {allowsAllModels === false ? "All models except those allowed" : "No models blocked"} ); } diff --git a/ui/components/notificationCenter.utils.test.ts b/ui/components/notificationCenter.utils.test.ts index efb379974ef..49e47ea285b 100644 --- a/ui/components/notificationCenter.utils.test.ts +++ b/ui/components/notificationCenter.utils.test.ts @@ -35,4 +35,4 @@ describe("shouldHideNotificationTrigger", () => { // A retry in flight behind an open popover must not yank the trigger away. expect(state({ open: true, isLoading: true })).toBe(false); }); -}); +}); \ No newline at end of file diff --git a/ui/components/rateLimitDisplay.test.ts b/ui/components/rateLimitDisplay.test.ts index f7bd52fc9d7..30a5ee6776d 100644 --- a/ui/components/rateLimitDisplay.test.ts +++ b/ui/components/rateLimitDisplay.test.ts @@ -19,4 +19,4 @@ describe("RateLimitDisplay scopeLabel", () => { expect(use).toMatch(/scopeLabel=\{scopeLabel\}/); } }); -}); +}); \ No newline at end of file diff --git a/ui/components/sidebar.tsx b/ui/components/sidebar.tsx index 87540538ab7..663008d32d5 100644 --- a/ui/components/sidebar.tsx +++ b/ui/components/sidebar.tsx @@ -253,20 +253,23 @@ const SidebarItemView = ({ const isHighlighted = !hasSubItems && highlightedUrl === item.url; - const buttonClassName = `group/nav-item relative h-7.5 cursor-pointer rounded-sm border px-3 transition-all duration-200 ${isHighlighted - ? "bg-sidebar-accent text-accent-foreground border-primary/20" - : isActive || isAnySubItemActive - ? "bg-sidebar-accent text-primary border-primary/20" - : item.hasAccess - ? "hover:bg-sidebar-accent hover:text-accent-foreground border-transparent text-slate-500 dark:text-zinc-400" - : "hover:bg-destructive/5 hover:text-muted-foreground text-muted-foreground cursor-not-allowed border-transparent" - } `; + const buttonClassName = `group/nav-item relative h-7.5 cursor-pointer rounded-sm border px-3 transition-all duration-200 ${ + isHighlighted + ? "bg-sidebar-accent text-accent-foreground border-primary/20" + : isActive || isAnySubItemActive + ? "bg-sidebar-accent text-primary border-primary/20" + : item.hasAccess + ? "hover:bg-sidebar-accent hover:text-accent-foreground border-transparent text-slate-500 dark:text-zinc-400" + : "hover:bg-destructive/5 hover:text-muted-foreground text-muted-foreground cursor-not-allowed border-transparent" + } `; const innerContent = (
- + {item.title} {item.tag && ( @@ -367,7 +370,9 @@ const SidebarItemView = ({ {SubItemIcon && ( )} - + {subItem.title} {subItem.tag && ( @@ -413,14 +418,15 @@ const SidebarItemView = ({ const isSubItemActive = subItem.queryParam ? pathname === subItem.url : isRouteMatch(subItem.url); const isSubItemHighlighted = highlightedUrl ? subItemHref.startsWith(highlightedUrl) : false; const SubItemIcon = subItem.icon; - const subItemClassName = `h-7 cursor-pointer rounded-sm px-2 transition-all duration-200 ${isSubItemHighlighted - ? "bg-sidebar-accent text-accent-foreground" - : isSubItemActive - ? "bg-sidebar-accent text-primary font-medium" - : subItem.hasAccess === false - ? "hover:bg-destructive/5 hover:text-muted-foreground text-muted-foreground cursor-not-allowed border-transparent" - : "hover:bg-sidebar-accent hover:text-accent-foreground text-slate-500 dark:text-zinc-400" - }`; + const subItemClassName = `h-7 cursor-pointer rounded-sm px-2 transition-all duration-200 ${ + isSubItemHighlighted + ? "bg-sidebar-accent text-accent-foreground" + : isSubItemActive + ? "bg-sidebar-accent text-primary font-medium" + : subItem.hasAccess === false + ? "hover:bg-destructive/5 hover:text-muted-foreground text-muted-foreground cursor-not-allowed border-transparent" + : "hover:bg-sidebar-accent hover:text-accent-foreground text-slate-500 dark:text-zinc-400" + }`; const subInner = (
{SubItemIcon && ( @@ -974,21 +980,21 @@ export default function AppSidebar() { }, ...(isDbConnected ? [ - { - title: "Prompt Repository", - url: "/workspace/prompt-repo", - icon: FolderGit, - description: "Prompt repository", - hasAccess: hasPromptRepositoryAccess, - }, - { - title: "Skills Repository", - url: "/workspace/skills-repo", - icon: BookOpenText, - description: "Skills repository", - hasAccess: hasSkillsRepositoryAccess, - }, - ] + { + title: "Prompt Repository", + url: "/workspace/prompt-repo", + icon: FolderGit, + description: "Prompt repository", + hasAccess: hasPromptRepositoryAccess, + }, + { + title: "Skills Repository", + url: "/workspace/skills-repo", + icon: BookOpenText, + description: "Skills repository", + hasAccess: hasSkillsRepositoryAccess, + }, + ] : []), { title: "Settings", @@ -1027,14 +1033,14 @@ export default function AppSidebar() { }, ...(IS_ENTERPRISE ? [ - { - title: "Proxy", - url: "/workspace/config/proxy", - icon: Globe, - description: "Proxy configuration", - hasAccess: hasSettingsAccess, - }, - ] + { + title: "Proxy", + url: "/workspace/config/proxy", + icon: Globe, + description: "Proxy configuration", + hasAccess: hasSettingsAccess, + }, + ] : []), { title: "API Keys", @@ -1059,21 +1065,21 @@ export default function AppSidebar() { }, ...(IS_ENTERPRISE ? [ - { - title: "Branding", - url: "/workspace/config/branding", - icon: Palette, - description: "Custom logo and icon", - hasAccess: hasSettingsAccess, - }, - { - title: "License Info", - url: "/workspace/config/license", - icon: BadgeInfo, - description: "Enterprise license information", - hasAccess: hasSettingsAccess, - }, - ] + { + title: "Branding", + url: "/workspace/config/branding", + icon: Palette, + description: "Custom logo and icon", + hasAccess: hasSettingsAccess, + }, + { + title: "License Info", + url: "/workspace/config/license", + icon: BadgeInfo, + description: "Enterprise license information", + hasAccess: hasSettingsAccess, + }, + ] : []), ], }, @@ -1515,7 +1521,7 @@ export default function AppSidebar() {
- + {filteredItems.map((item) => { diff --git a/ui/components/topbar.test.ts b/ui/components/topbar.test.ts index 1c5ab762054..db76beea197 100644 --- a/ui/components/topbar.test.ts +++ b/ui/components/topbar.test.ts @@ -37,4 +37,4 @@ describe("topbar text utilities", () => { expect(used.length).toBeGreaterThan(0); expect(used.filter((name) => !defined.has(name))).toEqual([]); }); -}); +}); \ No newline at end of file diff --git a/ui/components/ui/codeEditor.tsx b/ui/components/ui/codeEditor.tsx index 3f0b6ef42c4..2feb239df10 100644 --- a/ui/components/ui/codeEditor.tsx +++ b/ui/components/ui/codeEditor.tsx @@ -211,9 +211,14 @@ export function CodeEditor(props: CodeEditorProps) { padding: { top: 2, bottom: 2 }, wordWrap: props.wrap ? ("on" as const) : ("off" as const), folding: isFoldingEnabled, - glyphMargin: isFoldingEnabled, - lineNumbersMinChars: props.options?.lineNumbersMinChars ?? 4, - lineDecorationsWidth: isFoldingEnabled ? 18 : 8, + // Folding arrows render in the lineDecorations column (Monaco adds 16px + // for them automatically) - the glyph margin is a separate breakpoint + // column we never use, and enabling it doubles the left gutter. + glyphMargin: false, + // With line numbers off, Monaco still reserves the number column at its + // minimum width - a phantom left gutter on every read-only viewer. + lineNumbersMinChars: props.options?.lineNumbersMinChars ?? ((props.options?.lineNumbers || "off") === "off" ? 0 : 4), + lineDecorationsWidth: isFoldingEnabled ? 2 : 8, showFoldingControls: isFoldingEnabled ? ("always" as const) : ("mouseover" as const), overviewRulerLanes: props.options?.overviewRulerLanes ?? 0, renderLineHighlight: "none" as const, diff --git a/ui/components/ui/quarterStartSelect.tsx b/ui/components/ui/quarterStartSelect.tsx index aca85845371..f484631d931 100644 --- a/ui/components/ui/quarterStartSelect.tsx +++ b/ui/components/ui/quarterStartSelect.tsx @@ -77,4 +77,4 @@ export default function QuarterStartSelect({ "data-testid": testId, value, onCha

); -} +} \ No newline at end of file diff --git a/ui/components/ui/sidebar.tsx b/ui/components/ui/sidebar.tsx index 433b1a4b8d3..a3d6a34e5da 100644 --- a/ui/components/ui/sidebar.tsx +++ b/ui/components/ui/sidebar.tsx @@ -478,7 +478,7 @@ function SidebarMenuAction({ "peer-data-[size=lg]/menu-button:top-2.5", "group-data-[collapsible=icon]:hidden", showOnHover && - "peer-data-[active=true]/menu-button:text-sidebar-accent-foreground group-focus-within/menu-item:opacity-100 group-hover/menu-item:opacity-100 data-[state=open]:opacity-100 md:opacity-0", + "peer-data-[active=true]/menu-button:text-sidebar-accent-foreground group-focus-within/menu-item:opacity-100 group-hover/menu-item:opacity-100 data-[state=open]:opacity-100 md:opacity-0", className, )} {...props} @@ -620,5 +620,5 @@ export { SidebarRail, SidebarSeparator, SidebarTrigger, - useSidebar -}; + useSidebar, +}; \ No newline at end of file diff --git a/ui/hooks/useWebSocket.tsx b/ui/hooks/useWebSocket.tsx index ff3b48d322b..a6d8b13a38d 100644 --- a/ui/hooks/useWebSocket.tsx +++ b/ui/hooks/useWebSocket.tsx @@ -218,4 +218,4 @@ export function useWebSocket() { throw new Error("useWebSocket must be used within a WebSocketProvider"); } return context; -} +} \ No newline at end of file diff --git a/ui/hooks/useWebSocket.utils.test.ts b/ui/hooks/useWebSocket.utils.test.ts index 5c982262de0..541ae85c261 100644 --- a/ui/hooks/useWebSocket.utils.test.ts +++ b/ui/hooks/useWebSocket.utils.test.ts @@ -196,4 +196,4 @@ describe("useWebSocket lifecycle", () => { expect(connect).not.toHaveBeenCalled(); }); }); -}); +}); \ No newline at end of file diff --git a/ui/hooks/useWebSocket.utils.ts b/ui/hooks/useWebSocket.utils.ts index 3179fd7f9c5..4b7130487bc 100644 --- a/ui/hooks/useWebSocket.utils.ts +++ b/ui/hooks/useWebSocket.utils.ts @@ -133,4 +133,4 @@ export function stopWebSocket() { // Already closed. } } -} +} \ No newline at end of file diff --git a/ui/index.html b/ui/index.html index 886e0eeda95..021826ac1ce 100644 --- a/ui/index.html +++ b/ui/index.html @@ -7,11 +7,188 @@ Bifrost - + -
+
+ + + +
- + \ No newline at end of file diff --git a/ui/lib/constants/config.ts b/ui/lib/constants/config.ts index bacfea77e07..0fedd93a6a3 100644 --- a/ui/lib/constants/config.ts +++ b/ui/lib/constants/config.ts @@ -74,6 +74,7 @@ export const ModelPlaceholders = { wafer: "e.g. glm-5.2, kimi-k2.6", databricks: "e.g. databricks-claude-sonnet-4-5, system.ai.claude-sonnet-4-5", "github-copilot": "e.g. gpt-5.5, claude-sonnet-4-6", + typesafe: "e.g. jev-1.13.0, jev-latest, jev-preview", }; export const isKeyRequiredByProvider: Record = { @@ -115,6 +116,7 @@ export const isKeyRequiredByProvider: Record = { wafer: true, databricks: false, "github-copilot": false, + typesafe: true, }; export const DefaultNetworkConfig = { diff --git a/ui/lib/constants/icons.tsx b/ui/lib/constants/icons.tsx index 0def55f4586..6eb3a297898 100644 --- a/ui/lib/constants/icons.tsx +++ b/ui/lib/constants/icons.tsx @@ -1,4 +1,4 @@ -import { Bot, Database, Landmark, Network, Shuffle, Workflow } from "lucide-react"; +import { Bot, Database, Landmark, Link2, Network, Shuffle, Workflow } from "lucide-react"; import { useTheme } from "next-themes"; import { cn } from "../utils"; @@ -857,6 +857,30 @@ export const ProviderIcons = { ); }, + // TypeSafe's geometric logomark, extracted from their brand wordmark SVG. The + // source art uses currentColor, so the icon follows the surrounding text in + // both themes without a theme branch. The wordmark's remaining letterforms are + // dropped: at icon sizes only the mark stays legible. + typesafe: ({ size = "md", className = "" }: IconProps) => { + const resolvedSize = resolveSize(size); + return ( + + TypeSafe + + + ); + }, } as const; // Routing Engine Icons @@ -865,6 +889,7 @@ export const RoutingEngineUsedIcons = { governance: ({ className = "h-5 w-5 text-green-800" }: { className?: string } = {}) => , loadbalancing: ({ className = "h-5 w-5 text-orange-800" }: { className?: string } = {}) => , "model-catalog": ({ className = "h-5 w-5 text-purple-800" }: { className?: string } = {}) => , + "session-affinity": ({ className = "h-5 w-5 text-teal-800" }: { className?: string } = {}) => , core: ({ className = "h-5 w-5 text-sky-800" }: { className?: string } = {}) => , } as const; diff --git a/ui/lib/constants/logs.ts b/ui/lib/constants/logs.ts index c62530a6c09..f4cdb989473 100644 --- a/ui/lib/constants/logs.ts +++ b/ui/lib/constants/logs.ts @@ -38,6 +38,7 @@ export const KnownProvidersNames = [ "wafer", "databricks", "github-copilot", + "typesafe", ] as const; // Local Provider type derived from KNOWN_PROVIDERS constant @@ -92,6 +93,7 @@ export const RequestTypes = [ "responses_input_items", "embedding", "rerank", + "decisions", "speech", "speech_stream", "transcription", @@ -183,6 +185,7 @@ export const ProviderLabels: Record = { wafer: "Wafer", databricks: "Databricks", "github-copilot": "GitHub Copilot", + typesafe: "TypeSafe", } as const; // Helper function to get provider label, supporting custom providers @@ -309,6 +312,7 @@ export const RequestTypeLabels = { embedding: "Embedding", rerank: "Rerank", + decisions: "Decisions", speech: "Speech", speech_stream: "Speech Stream", @@ -398,6 +402,7 @@ export const RequestTypeColors = { embedding: "bg-red-100 text-red-800", rerank: "bg-fuchsia-100 text-fuchsia-800", + decisions: "bg-cyan-100 text-cyan-800", speech: "bg-purple-100 text-purple-800", speech_stream: "bg-pink-100 text-pink-800", @@ -462,6 +467,7 @@ export const RoutingEngineUsedLabels = { governance: "Governance", loadbalancing: "Loadbalancing", "model-catalog": "Model Catalog", + "session-affinity": "Session", core: "Core", } as const; @@ -470,6 +476,7 @@ export const RoutingEngineUsedColors = { governance: "bg-green-100 text-green-800 dark:bg-green-900 dark:text-green-300", loadbalancing: "bg-orange-100 text-orange-800 dark:bg-orange-900 dark:text-orange-300", "model-catalog": "bg-purple-100 text-purple-800 dark:bg-purple-900 dark:text-purple-300", + "session-affinity": "bg-teal-100 text-teal-800 dark:bg-teal-900 dark:text-teal-300", core: "bg-sky-100 text-sky-800 dark:bg-sky-900 dark:text-sky-300", } as const; diff --git a/ui/lib/hooks/useDismissedProviderCollisions.ts b/ui/lib/hooks/useDismissedProviderCollisions.ts index f00a993be4a..e36f3b6f1b8 100644 --- a/ui/lib/hooks/useDismissedProviderCollisions.ts +++ b/ui/lib/hooks/useDismissedProviderCollisions.ts @@ -51,4 +51,4 @@ export function useDismissedProviderCollisions(): { }, []); return { dismissed, dismiss, hydrated }; -} +} \ No newline at end of file diff --git a/ui/lib/schemas/providerForm.copilot.test.ts b/ui/lib/schemas/providerForm.copilot.test.ts index b4f846fcbb2..f9a20b55626 100644 --- a/ui/lib/schemas/providerForm.copilot.test.ts +++ b/ui/lib/schemas/providerForm.copilot.test.ts @@ -111,4 +111,4 @@ describe("ProviderFormSchema github-copilot credentials", () => { expect(parsed.success, parsed.success ? "" : JSON.stringify(parsed.error.issues)).toBe(true); }); }); -}); +}); \ No newline at end of file diff --git a/ui/lib/store/apis/governanceApi.ts b/ui/lib/store/apis/governanceApi.ts index 64d46a6cdf4..8928fcb02f6 100644 --- a/ui/lib/store/apis/governanceApi.ts +++ b/ui/lib/store/apis/governanceApi.ts @@ -983,4 +983,4 @@ export const { useLazyGetGovernanceHealthQuery, useLazyGetModelConfigsQuery, useLazyGetProviderGovernanceQuery, -} = governanceApi; +} = governanceApi; \ No newline at end of file diff --git a/ui/lib/store/apis/logsApi.ts b/ui/lib/store/apis/logsApi.ts index 0e4806fbbc5..bd420908e74 100644 --- a/ui/lib/store/apis/logsApi.ts +++ b/ui/lib/store/apis/logsApi.ts @@ -521,4 +521,4 @@ export const { useCancelRecalculateCostJobMutation, useLazyGetLogByIdQuery, useGetLogByIdQuery, -} = logsApi; +} = logsApi; \ No newline at end of file diff --git a/ui/lib/store/apis/virtualMcpsApi.ts b/ui/lib/store/apis/virtualMcpsApi.ts index f46ecda5842..367f9e35e90 100644 --- a/ui/lib/store/apis/virtualMcpsApi.ts +++ b/ui/lib/store/apis/virtualMcpsApi.ts @@ -76,4 +76,4 @@ export const { useDeleteVirtualMCPMutation, useAttachVirtualMCPVirtualKeyMutation, useDetachVirtualMCPVirtualKeyMutation, -} = virtualMcpsApi; +} = virtualMcpsApi; \ No newline at end of file diff --git a/ui/lib/store/slices/appSlice.test.ts b/ui/lib/store/slices/appSlice.test.ts index 83226ed0128..6e01af63a3b 100644 --- a/ui/lib/store/slices/appSlice.test.ts +++ b/ui/lib/store/slices/appSlice.test.ts @@ -69,7 +69,10 @@ describe("notification state", () => { }); it("drops rows the server has already expired", () => { - let state = reducer(undefined, setNotifications([notification("live", "2026-08-17T10:00:00Z"), expired("stale", "2026-08-17T11:00:00Z")])); + let state = reducer( + undefined, + setNotifications([notification("live", "2026-08-17T10:00:00Z"), expired("stale", "2026-08-17T11:00:00Z")]), + ); expect(state.notifications.map(({ id }) => id)).toEqual(["live"]); state = reducer(state, addNotification(expired("stale-push", "2026-08-17T12:00:00Z"))); @@ -87,4 +90,4 @@ describe("notification state", () => { const state = reducer(undefined, setNotifications([notification("odd", "2026-08-17T10:00:00Z", "not-a-date")])); expect(state.notifications.map(({ id }) => id)).toEqual(["odd"]); }); -}); +}); \ No newline at end of file diff --git a/ui/lib/store/slices/appSlice.ts b/ui/lib/store/slices/appSlice.ts index 72e20428449..82e176a148b 100644 --- a/ui/lib/store/slices/appSlice.ts +++ b/ui/lib/store/slices/appSlice.ts @@ -281,4 +281,4 @@ export const selectUnreadNotificationsCount = createSelector( export const selectNotificationPreferences = (state: { app: AppState }) => state.app.notificationPreferences; export const selectSettings = (state: { app: AppState }) => state.app.settings; export const selectGlobalError = (state: { app: AppState }) => state.app.globalError; -export const selectFeatures = (state: { app: AppState }) => state.app.features; +export const selectFeatures = (state: { app: AppState }) => state.app.features; \ No newline at end of file diff --git a/ui/lib/types/complexityRouter.test.ts b/ui/lib/types/complexityRouter.test.ts index 1de8882f415..6c481a717fd 100644 --- a/ui/lib/types/complexityRouter.test.ts +++ b/ui/lib/types/complexityRouter.test.ts @@ -38,4 +38,4 @@ describe("parseSemanticTimeoutMs", () => { // Partially valid input must not be read as the part that happened to parse. expect(parseSemanticTimeoutMs("1m30x")).toBe(DEFAULT_SEMANTIC_TIMEOUT_MS); }); -}); +}); \ No newline at end of file diff --git a/ui/lib/types/routingRules.ts b/ui/lib/types/routingRules.ts index ed08136ecd3..fb70ce7b374 100644 --- a/ui/lib/types/routingRules.ts +++ b/ui/lib/types/routingRules.ts @@ -5,6 +5,23 @@ import { RuleGroupType } from "react-querybuilder"; +/** A fallback that may pin a provider key. The API accepts and returns the legacy "provider/model" string for unpinned entries. */ +/** Wire object form of a fallback, used only when it pins a provider key. */ +export interface RoutingFallbackObject { + provider?: string; + model?: string; + key_id?: string; +} + +export type RoutingFallbackWire = string | RoutingFallbackObject; + +/** Form state, split so the sheet can drive separate provider and model selects. */ +export interface RoutingFallbackFormData { + provider: string; + model: string; + key_id: string; +} + export interface RoutingTarget { provider?: string; model?: string; @@ -18,7 +35,7 @@ export interface RoutingRule { description: string; cel_expression: string; targets: RoutingTarget[]; - fallbacks?: string[]; + fallbacks?: RoutingFallbackWire[]; scope: "global" | "team" | "customer" | "virtual_key" | "user"; scope_id?: string; priority: number; @@ -34,7 +51,7 @@ export interface CreateRoutingRuleRequest { description?: string; cel_expression?: string; targets: RoutingTarget[]; - fallbacks?: string[]; + fallbacks?: RoutingFallbackWire[]; scope: string; scope_id?: string; priority: number; @@ -77,7 +94,7 @@ export interface RoutingRuleFormData { description: string; cel_expression: string; targets: RoutingTargetFormData[]; - fallbacks: string[]; + fallbacks: RoutingFallbackFormData[]; scope: string; scope_id: string; priority: number; @@ -104,6 +121,12 @@ export const ROUTING_RULE_SCOPES = [ { value: RoutingRuleScope.VirtualKey, label: "Virtual Key" }, ]; +export const DEFAULT_ROUTING_FALLBACK: RoutingFallbackFormData = { + provider: "", + model: "", + key_id: "", +}; + export const DEFAULT_ROUTING_TARGET: RoutingTargetFormData = { provider: "", model: "", diff --git a/ui/lib/types/schemas.databricks.test.ts b/ui/lib/types/schemas.databricks.test.ts index 0614e64d2c2..4606f5c5334 100644 --- a/ui/lib/types/schemas.databricks.test.ts +++ b/ui/lib/types/schemas.databricks.test.ts @@ -116,4 +116,4 @@ describe("modelProviderKeySchema, databricks", () => { }); expect(result.success).toBe(false); }); -}); +}); \ No newline at end of file diff --git a/ui/lib/types/virtualMcps.ts b/ui/lib/types/virtualMcps.ts index 814b6c195c8..97ab1fe7c9f 100644 --- a/ui/lib/types/virtualMcps.ts +++ b/ui/lib/types/virtualMcps.ts @@ -49,4 +49,4 @@ export interface GetVirtualMCPsResponse { total_count: number; limit: number; offset: number; -} +} \ No newline at end of file diff --git a/ui/lib/utils/databricksMigration.test.ts b/ui/lib/utils/databricksMigration.test.ts index 15971827b44..5bdffa8bde2 100644 --- a/ui/lib/utils/databricksMigration.test.ts +++ b/ui/lib/utils/databricksMigration.test.ts @@ -220,10 +220,9 @@ describe("buildDatabricksMigrationPlan", () => { }); it("enables migration once a masked key gets a value, and strips paths from an edited workspace url", () => { - const plan = buildDatabricksMigrationPlan( - custom("my-dbx", { custom_provider_config: { base_provider_type: "openai" } }), - [key("k1", "prod")], - ); + const plan = buildDatabricksMigrationPlan(custom("my-dbx", { custom_provider_config: { base_provider_type: "openai" } }), [ + key("k1", "prod"), + ]); expect(planNeedsInput(plan)).toBe(true); const filled: MigrationPlan = { ...plan, @@ -284,7 +283,8 @@ const fakeApi = (opts: FakeOptions = {}) => { }) as ModelProvider, ), getProviderKeys: (name) => record(["getProviderKeys", name], () => keysByProvider.get(name) ?? []), - createProvider: (body) => record(["createProvider", body.provider], () => ({ name: body.provider, provider_status: "active" }) as ModelProvider), + createProvider: (body) => + record(["createProvider", body.provider], () => ({ name: body.provider, provider_status: "active" }) as ModelProvider), updateProvider: (name) => record(["updateProvider", name], () => ({ name, provider_status: "active" }) as ModelProvider), deleteProvider: (name) => record(["deleteProvider", name], () => undefined), createProviderKey: (provider, k) => @@ -344,7 +344,14 @@ describe("runDatabricksMigration", () => { const steps: string[] = []; const result = await runDatabricksMigration(keyedPlan(), api, (s) => steps.push(s.map((x) => x.status).join(","))); expect(result.ok).toBe(false); - expect(names()).toEqual(["createProvider", "updateProvider", "createProviderKey", "createProviderKey", "deleteProviderKey", "deleteProvider"]); + expect(names()).toEqual([ + "createProvider", + "updateProvider", + "createProviderKey", + "createProviderKey", + "deleteProviderKey", + "deleteProvider", + ]); expect(calls.find((c) => c[0] === "deleteProvider")?.[1]).toBe("databricks"); expect(steps.at(-1)).toContain("failed"); }); @@ -407,4 +414,4 @@ describe("runDatabricksMigration", () => { expect(names()).toContain("deleteProviderKey"); expect(names().filter((n) => n === "deleteProvider")).toHaveLength(1); }); -}); +}); \ No newline at end of file diff --git a/ui/lib/utils/databricksMigration.ts b/ui/lib/utils/databricksMigration.ts index 1b401bd9eb6..fb0e638e5b0 100644 --- a/ui/lib/utils/databricksMigration.ts +++ b/ui/lib/utils/databricksMigration.ts @@ -214,11 +214,13 @@ const buildProviderSettings = (source: ModelProvider, warnings: string[]): Updat const extra_headers = stripAuthorizationHeader(net.extra_headers); if (extra_headers) network_config.extra_headers = extra_headers; if (net.insecure_skip_verify !== undefined) network_config.insecure_skip_verify = net.insecure_skip_verify; - if (net.stream_idle_timeout_in_seconds !== undefined) network_config.stream_idle_timeout_in_seconds = net.stream_idle_timeout_in_seconds; + if (net.stream_idle_timeout_in_seconds !== undefined) + network_config.stream_idle_timeout_in_seconds = net.stream_idle_timeout_in_seconds; if (net.keep_alive_timeout_in_seconds !== undefined) network_config.keep_alive_timeout_in_seconds = net.keep_alive_timeout_in_seconds; if (net.max_conns_per_host !== undefined) network_config.max_conns_per_host = net.max_conns_per_host; if (net.enforce_http2 !== undefined) network_config.enforce_http2 = net.enforce_http2; - if (net.http2_ping_interval_in_seconds !== undefined) network_config.http2_ping_interval_in_seconds = net.http2_ping_interval_in_seconds; + if (net.http2_ping_interval_in_seconds !== undefined) + network_config.http2_ping_interval_in_seconds = net.http2_ping_interval_in_seconds; if (net.beta_header_overrides !== undefined) network_config.beta_header_overrides = net.beta_header_overrides; if (net.allow_private_network !== undefined) network_config.allow_private_network = net.allow_private_network; const caCert = portableSecret(net.ca_cert_pem, "The provider CA certificate", warnings); @@ -786,4 +788,4 @@ export const runDatabricksMigration = async ( } return { ok: false, partial: false, message: errorText(err) }; } -}; +}; \ No newline at end of file diff --git a/ui/lib/utils/duration.test.ts b/ui/lib/utils/duration.test.ts index bc3e9d14216..14657a69737 100644 --- a/ui/lib/utils/duration.test.ts +++ b/ui/lib/utils/duration.test.ts @@ -28,4 +28,4 @@ describe("formatCooldown", () => { test("mixed values are not truncated to whole seconds", () => { expect(formatCooldown(1_500_000_000)).toBe("1500ms"); }); -}); +}); \ No newline at end of file diff --git a/ui/lib/utils/duration.ts b/ui/lib/utils/duration.ts index f59b8c770df..f98dc51d559 100644 --- a/ui/lib/utils/duration.ts +++ b/ui/lib/utils/duration.ts @@ -20,4 +20,4 @@ export const formatCooldown = (value?: number | string): string => { if (value % ns === 0) return `${value / ns}${suffix}`; } return `${value}ns`; -}; +}; \ No newline at end of file diff --git a/ui/lib/utils/providerCollision.test.ts b/ui/lib/utils/providerCollision.test.ts index e1622b69eff..14278afa713 100644 --- a/ui/lib/utils/providerCollision.test.ts +++ b/ui/lib/utils/providerCollision.test.ts @@ -42,4 +42,4 @@ describe("providerCollision", () => { { customName: "GitHub Copilot", knownProvider: "github-copilot" }, ]); }); -}); +}); \ No newline at end of file diff --git a/ui/lib/utils/providerCollision.ts b/ui/lib/utils/providerCollision.ts index 47ed7ab81d2..b87dfc71200 100644 --- a/ui/lib/utils/providerCollision.ts +++ b/ui/lib/utils/providerCollision.ts @@ -39,4 +39,4 @@ export const findCustomProviderCollisions = (providers: ModelProvider[]): Custom } } return collisions; -}; +}; \ No newline at end of file diff --git a/ui/lib/utils/routingRules.test.ts b/ui/lib/utils/routingRules.test.ts new file mode 100644 index 00000000000..2a33ec5e32b --- /dev/null +++ b/ui/lib/utils/routingRules.test.ts @@ -0,0 +1,31 @@ +import { describe, expect, it } from "vitest"; +import { denormalizeFallback, normalizeFallback } from "./routingRules"; + +describe("routing fallback wire format", () => { + it.each([ + ["openai/gpt-4o", { provider: "openai", model: "gpt-4o", key_id: "" }], + ["azure/", { provider: "azure", model: "", key_id: "" }], + ["openai/ft:model/org/v2", { provider: "openai", model: "ft:model/org/v2", key_id: "" }], + ] as const)("round-trips %s", (wire, form) => { + expect(normalizeFallback(wire)).toEqual(form); + expect(denormalizeFallback(form)).toEqual(wire); + }); + + it("keeps the provider delimiter when an unpinned fallback uses the incoming model", () => { + expect(denormalizeFallback({ provider: " azure " })).toBe("azure/"); + }); + + it("round-trips pinned fallback objects with and without a model", () => { + for (const wire of [ + { provider: "openai", model: "gpt-4o", key_id: "key-1" }, + { provider: "azure", key_id: "key-2" }, + ]) { + expect(denormalizeFallback(normalizeFallback(wire))).toEqual(wire); + } + }); + + it("returns to valid legacy syntax when a provider-only fallback pin is cleared", () => { + const form = normalizeFallback({ provider: "azure", key_id: "key-1" }); + expect(denormalizeFallback({ ...form, key_id: "" })).toBe("azure/"); + }); +}); \ No newline at end of file diff --git a/ui/lib/utils/routingRules.ts b/ui/lib/utils/routingRules.ts index 3e78ca42bc6..bf1c2b933da 100644 --- a/ui/lib/utils/routingRules.ts +++ b/ui/lib/utils/routingRules.ts @@ -3,6 +3,8 @@ * Helper functions for CEL validation, formatting, and rule management */ +import { RoutingFallbackFormData, RoutingFallbackWire } from "@/lib/types/routingRules"; + /** * Validates if a CEL expression has basic correct syntax * @param expression - The CEL expression to validate @@ -48,38 +50,37 @@ export function formatFallback(fallback: string): string { } /** - * Parses a fallback string into provider and model - * @param fallback - The fallback string (e.g., "openai/gpt-4o") - * @returns Object with provider and model, or null if invalid - */ -export function parseFallback(fallback: string): { provider: string; model: string } | null { - if (!fallback) return null; - const parts = fallback.split("/"); - if (parts.length !== 2) return null; - return { provider: parts[0], model: parts[1] }; -} - -/** - * Converts fallback array to string format for display/editing - * @param fallbacks - Array of fallback strings - * @returns Comma-separated string + * Normalizes a wire fallback into the provider and model the sheet's two selects need. Unpinned + * fallbacks arrive as the bare "provider/model" string. */ -export function fallbacksToString(fallbacks?: string[]): string { - if (!fallbacks || fallbacks.length === 0) return ""; - return fallbacks.join(", "); +export function normalizeFallback(fallback: RoutingFallbackWire): RoutingFallbackFormData { + if (typeof fallback !== "string") { + return { + provider: fallback.provider ?? "", + model: fallback.model ?? "", + key_id: fallback.key_id ?? "", + }; + } + const separator = fallback.indexOf("/"); + if (separator === -1) { + return { provider: fallback, model: "", key_id: "" }; + } + return { provider: fallback.slice(0, separator), model: fallback.slice(separator + 1), key_id: "" }; } /** - * Converts comma-separated string to fallback array - * @param str - Comma-separated fallback string - * @returns Array of fallback strings + * Renders a fallback back onto the wire, as the bare string unless it pins a key. Sending the object + * form for an unpinned fallback would change the rule's config hash on every save. The trailing + * slash is required when the model is empty: a bare "anthropic" parses to an empty provider. */ -export function stringToFallbacks(str: string): string[] { - if (!str || str.trim().length === 0) return []; - return str - .split(",") - .map((s) => s.trim()) - .filter((s) => s.length > 0); +export function denormalizeFallback(fallback: Partial): RoutingFallbackWire { + const provider = (fallback.provider ?? "").trim(); + const model = (fallback.model ?? "").trim(); + const keyId = (fallback.key_id ?? "").trim(); + if (keyId) { + return model ? { provider, model, key_id: keyId } : { provider, key_id: keyId }; + } + return `${provider}/${model}`; } /** diff --git a/ui/lib/utils/validation.test.ts b/ui/lib/utils/validation.test.ts index 84422a61afe..cc2d2693f8b 100644 --- a/ui/lib/utils/validation.test.ts +++ b/ui/lib/utils/validation.test.ts @@ -44,10 +44,7 @@ describe("hasCopilotApiToken", () => { expect(hasCopilotApiToken({ value: "", ref: "COPILOT_TOKEN", type: "env" })).toBe(true); }); - it.each([undefined, null, "", " ", { value: "", ref: "" }, { value: " ", ref: " " }, {}])( - "treats %p as no token", - (input) => { - expect(hasCopilotApiToken(input as never)).toBe(false); - }, - ); -}); + it.each([undefined, null, "", " ", { value: "", ref: "" }, { value: " ", ref: " " }, {}])("treats %p as no token", (input) => { + expect(hasCopilotApiToken(input as never)).toBe(false); + }); +}); \ No newline at end of file diff --git a/ui/lib/utils/validation.ts b/ui/lib/utils/validation.ts index 8f53142e911..790daa4290f 100644 --- a/ui/lib/utils/validation.ts +++ b/ui/lib/utils/validation.ts @@ -612,4 +612,4 @@ export function hasCopilotApiToken(value: string | { value?: string; ref?: strin if (!value) return false; if (typeof value === "string") return value.trim() !== ""; return !!value.value?.trim() || !!value.ref?.trim(); -} +} \ No newline at end of file