From dc28f7011d978f0cdaef68758b8aad33a7612a5f Mon Sep 17 00:00:00 2001 From: akshaydeo Date: Thu, 29 Oct 2026 09:31:16 -0700 Subject: [PATCH] test fixes --- .github/workflows/pr-tests.yml | 2 +- .github/workflows/release-pipeline.yml | 194 +++++++++++++++--- .github/workflows/run-core-tests.yml | 3 +- .../workflows/scripts/apt-sources-https.sh | 62 ++++++ .../scripts/apt-sources-https.test.sh | 70 +++++++ .../scripts/check-egress-allowlist.sh | 36 +++- .../scripts/install-arm64-runner-deps.sh | 21 +- .../scripts/install-arm64-runner-deps.test.sh | 17 +- .../scripts/install-cross-compilers.sh | 1 + .../scripts/test-api-integrations.sh | 13 +- .github/workflows/scripts/test-warp.sh | 80 ++++++++ .github/workflows/workflow-lint.yml | 3 + framework/warp/topics.go | 70 ++++++- framework/warp/topics_test.go | 95 ++++++++- .../bifrost-v1-warp.postman_collection.json | 6 +- .../e2e/api/runners/build-warp-collection.mjs | 11 +- tests/e2e/api/runners/lib/warp-case.mjs | 3 +- tests/e2e/api/runners/lib/warp-case.test.mjs | 41 +++- 18 files changed, 661 insertions(+), 67 deletions(-) create mode 100755 .github/workflows/scripts/apt-sources-https.sh create mode 100755 .github/workflows/scripts/apt-sources-https.test.sh create mode 100755 .github/workflows/scripts/test-warp.sh diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml index c77765b0603..997ee8931a7 100644 --- a/.github/workflows/pr-tests.yml +++ b/.github/workflows/pr-tests.yml @@ -95,7 +95,7 @@ jobs: api.perplexity.ai:443 api.replicate.com:443 api.x.ai:443 - azure.archive.ubuntu.com:80 + archive.ubuntu.com:443 esm.ubuntu.com:443 bifrost-gateway-resource.openai.azure.com:443 bedrock-mantle.us-east-1.api.aws:443 diff --git a/.github/workflows/release-pipeline.yml b/.github/workflows/release-pipeline.yml index c6590495193..1a6fe21252b 100644 --- a/.github/workflows/release-pipeline.yml +++ b/.github/workflows/release-pipeline.yml @@ -142,7 +142,7 @@ jobs: with: egress-policy: block allowed-endpoints: > - _http._tcp.azure.archive.ubuntu.com:443 + _https._tcp.archive.ubuntu.com:443 _https._tcp.esm.ubuntu.com:443 _https._tcp.motd.ubuntu.com:443 _https._tcp.packages.microsoft.com:443 @@ -150,7 +150,7 @@ jobs: motd.ubuntu.com:443 dl.google.com:443 api.github.com:443 - azure.archive.ubuntu.com:80 + archive.ubuntu.com:443 dl.google.com:443 esm.ubuntu.com:443 github.com:443 @@ -169,6 +169,7 @@ jobs: - name: Install jq run: | + ./.github/workflows/scripts/apt-sources-https.sh sudo apt-get update sudo apt-get install -y jq @@ -240,7 +241,7 @@ jobs: # toolchain downloads. No provider hosts - nothing is executed here. allowed-endpoints: > api.github.com:443 - azure.archive.ubuntu.com:80 + archive.ubuntu.com:443 esm.ubuntu.com:443 fonts.googleapis.com:443 fonts.gstatic.com:443 @@ -371,7 +372,7 @@ jobs: api.replicate.com:443 api.tokenfactory.nebius.com:443 api.x.ai:443 - azure.archive.ubuntu.com:80 + archive.ubuntu.com:443 esm.ubuntu.com:443 bifrost-gateway-resource.openai.azure.com:443 bedrock-mantle.us-east-1.api.aws:443 @@ -428,6 +429,7 @@ jobs: - name: Install jq run: | + ./.github/workflows/scripts/apt-sources-https.sh sudo apt-get update sudo apt-get install -y jq @@ -618,7 +620,7 @@ jobs: api.replicate.com:443 api.tokenfactory.nebius.com:443 api.x.ai:443 - azure.archive.ubuntu.com:80 + archive.ubuntu.com:443 esm.ubuntu.com:443 bifrost-gateway-resource.openai.azure.com:443 bedrock-mantle.us-east-1.api.aws:443 @@ -705,6 +707,7 @@ jobs: # ports the flow uses; stop them so the flow's own configured instances bind. - name: Install proxies and jq run: | + ./.github/workflows/scripts/apt-sources-https.sh sudo apt-get update sudo apt-get install -y jq squid stunnel4 microsocks sudo systemctl disable --now squid 2>/dev/null || true @@ -839,7 +842,7 @@ jobs: api.replicate.com:443 api.tokenfactory.nebius.com:443 api.x.ai:443 - azure.archive.ubuntu.com:80 + archive.ubuntu.com:443 esm.ubuntu.com:443 bifrost-gateway-resource.openai.azure.com:443 bedrock-mantle.us-east-1.api.aws:443 @@ -1197,7 +1200,7 @@ jobs: egress-policy: block allowed-endpoints: > api.github.com:443 - azure.archive.ubuntu.com:80 + archive.ubuntu.com:443 esm.ubuntu.com:443 github.com:443 iojs.org:443 @@ -1215,6 +1218,7 @@ jobs: - name: Install jq run: | + ./.github/workflows/scripts/apt-sources-https.sh sudo apt-get update sudo apt-get install -y jq @@ -1313,7 +1317,7 @@ jobs: api.openai.com:443 api.parasail.io:443 api.tokenfactory.nebius.com:443 - azure.archive.ubuntu.com:80 + archive.ubuntu.com:443 esm.ubuntu.com:443 bedrock-runtime.us-east-1.amazonaws.com:443 bedrock.us-east-1.amazonaws.com:443 @@ -1356,6 +1360,7 @@ jobs: - name: Install jq run: | + ./.github/workflows/scripts/apt-sources-https.sh sudo apt-get update sudo apt-get install -y jq @@ -1657,7 +1662,7 @@ jobs: egress-policy: block allowed-endpoints: > _grpc_config.localhost:443 - _http._tcp.azure.archive.ubuntu.com:443 + _https._tcp.archive.ubuntu.com:443 _https._tcp.esm.ubuntu.com:443 _https._tcp.dl.google.com:443 _https._tcp.motd.ubuntu.com:443 @@ -1679,7 +1684,7 @@ jobs: api.github.com:443 api.openai.com:443 auth.docker.io:443 - azure.archive.ubuntu.com:80 + archive.ubuntu.com:443 codecov.io:443 dl-cdn.alpinelinux.org:443 esm.ubuntu.com:443 @@ -1705,6 +1710,7 @@ jobs: - name: Install jq run: | + ./.github/workflows/scripts/apt-sources-https.sh sudo apt-get update sudo apt-get install -y jq @@ -1950,7 +1956,7 @@ jobs: api.parasail.io:443 api.tokenfactory.nebius.com:443 auth.docker.io:443 - azure.archive.ubuntu.com:80 + archive.ubuntu.com:443 downloads.getmaxim.ai:443 esm.ubuntu.com:443 generativelanguage.googleapis.com:443 @@ -2010,6 +2016,7 @@ jobs: # The SQLite migration tests seed and read the stores through the sqlite3 CLI - name: Install sqlite3 run: | + ./.github/workflows/scripts/apt-sources-https.sh sudo apt-get update sudo apt-get install -y sqlite3 sqlite3 --version @@ -2221,6 +2228,8 @@ jobs: env: CI: "1" SKIP_GATEWAY_BUILD: "1" + # Warp runs in parallel as test-warp. + SKIP_WARP: "1" BIFROST_SETUP_TOKEN: bifrost-e2e-setup-token # Synthetic, non-secret canary for every provider's API redaction tests. BIFROST_SECRET_REDACTION_CANARY: synthetic-secretvar-harness-canary-0123456789 @@ -2277,6 +2286,101 @@ jobs: path: tests/e2e/api/newman-reports/ retention-days: 30 + # Warp's collection is a live, serial LLM evaluation (~150 chat calls, ~25 min), + # so it runs beside test-api-integrations rather than after it. Same gate: Warp + # lives in framework/warp and is served by bifrost-http. + test-warp: + needs: [check-skip, detect-changes, build-gateway] + if: needs.check-skip.outputs.should-skip != 'true' && needs.check-skip.outputs.skip-tests != 'true' && (needs.detect-changes.outputs.core-needs-release == 'true' || needs.detect-changes.outputs.framework-needs-release == 'true' || needs.detect-changes.outputs.bifrost-http-needs-release == 'true') + runs-on: builder-8-32 + timeout-minutes: 45 + permissions: + contents: read + steps: + - name: Harden Runner + uses: step-security/harden-runner@fe104658747b27e96e4f7e80cd0a94068e53901d # v2.16.1 + with: + egress-policy: block + # bifrost-http runs on the host and reaches Postgres and Weaviate through + # their published ports, which DNAT to the pinned bridge IPs in + # .github/workflows/configs/docker-compose.yml (Postgres 172.38.0.11, + # Weaviate 172.38.0.12, which also gossips to itself on 8301). Warp's + # agent and embeddings call OpenAI. + allowed-endpoints: > + 127.0.0.1:5432 + 172.38.0.11:5432 + 172.38.0.12:8080 + 172.38.0.12:8301 + api.github.com:443 + api.openai.com:443 + auth.docker.io:443 + cr.weaviate.io:443 + github.com:443 + iojs.org:443 + nodejs.org:443 + production.cloudflare.docker.com:443 + production.cloudfront.docker.com:443 + proxy.golang.org:443 + registry-1.docker.io:443 + registry.npmjs.org:443 + release-assets.githubusercontent.com:443 + storage.googleapis.com:443 + sum.golang.org:443 + + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Set up Go + uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 + with: + go-version: "1.27.3" + + - name: Set up Node.js + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: "25" + + - name: Install Newman (hash-pinned via lockfile) + run: | + npm ci --no-audit --no-fund + echo "$PWD/node_modules/.bin" >> "$GITHUB_PATH" + working-directory: .github/workflows/tools + + - name: Set up Docker Compose + run: | + docker --version + if ! docker compose version >/dev/null 2>&1; then + echo "Installing Docker Compose..." + ./.github/workflows/scripts/install-docker-compose.sh /usr/local/bin/docker-compose + docker-compose --version + else + echo "Docker Compose plugin is available" + docker compose version + fi + + - name: Download prebuilt bifrost-http binary + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: bifrost-http-binary + path: tmp + + - name: Make bifrost-http executable + run: chmod +x tmp/bifrost-http + + - name: Run Warp test + env: + BIFROST_SETUP_TOKEN: bifrost-e2e-setup-token + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + run: ./.github/workflows/scripts/test-warp.sh + + - name: Upload Warp reports + if: ${{ !cancelled() }} + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: newman-reports-warp + path: tests/e2e/api/newman-reports/warp/ + retention-days: 30 + # Cost accuracy test - validates logged LLM costs against provider usage test-cost-accuracy: needs: [check-skip, detect-changes, build-gateway] @@ -2684,6 +2788,7 @@ jobs: bedrock.us-east-1.amazonaws.com:443 bifrost-gateway-resource.openai.azure.com:443 dl-cdn.alpinelinux.org:443 + _https._tcp.download.docker.com:443 download.docker.com:443 fonts.googleapis.com:443 fonts.gstatic.com:443 @@ -2697,7 +2802,8 @@ jobs: oauth2.googleapis.com:443 openrouter.ai:443 pkg-containers.githubusercontent.com:443 - ports.ubuntu.com:80 + _https._tcp.ports.ubuntu.com:443 + ports.ubuntu.com:443 production.cloudflare.docker.com:443 production.cloudfront.docker.com:443 proxy.golang.org:443 @@ -2707,6 +2813,7 @@ jobs: storage.googleapis.com:443 sum.golang.org:443 telemetry.qdrant.io:443 + tuf-repo-cdn.sigstore.dev:443 www.getbifrost.ai:443 - name: Checkout repository @@ -2819,11 +2926,12 @@ jobs: test-migrations, test-agentgateway, test-api-integrations, + test-warp, test-cost-accuracy, test-docker-image-amd64, test-docker-image-arm64, ] - if: "always() && needs.check-skip.outputs.should-skip != 'true' && needs.detect-changes.result == 'success' && needs.detect-changes.outputs.core-needs-release == 'true' && (needs.test-core.result == 'success' || needs.test-core.result == 'skipped' || (needs.test-core.result == 'failure' && needs.approve-flaky-test-core.result == 'success')) && (needs.test-proxy-e2e.result == 'success' || needs.test-proxy-e2e.result == 'skipped') && (needs.test-memory.result == 'success' || needs.test-memory.result == 'skipped' || (needs.test-memory.result == 'failure' && needs.approve-memory-regression.result == 'success')) && (needs.test-core-integrations.result == 'success' || needs.test-core-integrations.result == 'skipped') && (needs.test-core-unit.result == 'success' || needs.test-core-unit.result == 'skipped') && (needs.test-cli-harness.result == 'success' || needs.test-cli-harness.result == 'skipped') && (needs.test-vllm.result == 'success' || needs.test-vllm.result == 'skipped') && (needs.resolve-cli-versions.result == 'success' || needs.resolve-cli-versions.result == 'skipped') && (needs.test-framework.result == 'success' || needs.test-framework.result == 'skipped') && (needs.test-plugins.result == 'success' || needs.test-plugins.result == 'skipped') && (needs.test-bifrost-http.result == 'success' || needs.test-bifrost-http.result == 'skipped') && (needs.test-live.result == 'success' || needs.test-live.result == 'skipped') && (needs.test-migrations.result == 'success' || needs.test-migrations.result == 'skipped') && (needs.test-agentgateway.result == 'success' || needs.test-agentgateway.result == 'skipped') && (needs.test-api-integrations.result == 'success' || needs.test-api-integrations.result == 'skipped') && (needs.test-cost-accuracy.result == 'success' || needs.test-cost-accuracy.result == 'skipped') && (needs.test-docker-image-amd64.result == 'success' || needs.test-docker-image-amd64.result == 'skipped') && (needs.test-docker-image-arm64.result == 'success' || needs.test-docker-image-arm64.result == 'skipped')" + if: "always() && needs.check-skip.outputs.should-skip != 'true' && needs.detect-changes.result == 'success' && needs.detect-changes.outputs.core-needs-release == 'true' && (needs.test-core.result == 'success' || needs.test-core.result == 'skipped' || (needs.test-core.result == 'failure' && needs.approve-flaky-test-core.result == 'success')) && (needs.test-proxy-e2e.result == 'success' || needs.test-proxy-e2e.result == 'skipped') && (needs.test-memory.result == 'success' || needs.test-memory.result == 'skipped' || (needs.test-memory.result == 'failure' && needs.approve-memory-regression.result == 'success')) && (needs.test-core-integrations.result == 'success' || needs.test-core-integrations.result == 'skipped') && (needs.test-core-unit.result == 'success' || needs.test-core-unit.result == 'skipped') && (needs.test-cli-harness.result == 'success' || needs.test-cli-harness.result == 'skipped') && (needs.test-vllm.result == 'success' || needs.test-vllm.result == 'skipped') && (needs.resolve-cli-versions.result == 'success' || needs.resolve-cli-versions.result == 'skipped') && (needs.test-framework.result == 'success' || needs.test-framework.result == 'skipped') && (needs.test-plugins.result == 'success' || needs.test-plugins.result == 'skipped') && (needs.test-bifrost-http.result == 'success' || needs.test-bifrost-http.result == 'skipped') && (needs.test-live.result == 'success' || needs.test-live.result == 'skipped') && (needs.test-migrations.result == 'success' || needs.test-migrations.result == 'skipped') && (needs.test-agentgateway.result == 'success' || needs.test-agentgateway.result == 'skipped') && (needs.test-api-integrations.result == 'success' || needs.test-api-integrations.result == 'skipped') && (needs.test-warp.result == 'success' || needs.test-warp.result == 'skipped') && (needs.test-cost-accuracy.result == 'success' || needs.test-cost-accuracy.result == 'skipped') && (needs.test-docker-image-amd64.result == 'success' || needs.test-docker-image-amd64.result == 'skipped') && (needs.test-docker-image-arm64.result == 'success' || needs.test-docker-image-arm64.result == 'skipped')" runs-on: builder-8-32 permissions: contents: write @@ -2934,12 +3042,13 @@ jobs: test-migrations, test-agentgateway, test-api-integrations, + test-warp, test-cost-accuracy, test-docker-image-amd64, test-docker-image-arm64, core-release, ] - if: "always() && needs.check-skip.outputs.should-skip != 'true' && needs.detect-changes.result == 'success' && needs.detect-changes.outputs.framework-needs-release == 'true' && (needs.test-core.result == 'success' || needs.test-core.result == 'skipped' || (needs.test-core.result == 'failure' && needs.approve-flaky-test-core.result == 'success')) && (needs.test-proxy-e2e.result == 'success' || needs.test-proxy-e2e.result == 'skipped') && (needs.test-core-integrations.result == 'success' || needs.test-core-integrations.result == 'skipped') && (needs.test-core-unit.result == 'success' || needs.test-core-unit.result == 'skipped') && (needs.test-cli-harness.result == 'success' || needs.test-cli-harness.result == 'skipped') && (needs.resolve-cli-versions.result == 'success' || needs.resolve-cli-versions.result == 'skipped') && (needs.test-framework.result == 'success' || needs.test-framework.result == 'skipped') && (needs.test-plugins.result == 'success' || needs.test-plugins.result == 'skipped') && (needs.test-bifrost-http.result == 'success' || needs.test-bifrost-http.result == 'skipped') && (needs.test-live.result == 'success' || needs.test-live.result == 'skipped') && (needs.test-migrations.result == 'success' || needs.test-migrations.result == 'skipped') && (needs.test-agentgateway.result == 'success' || needs.test-agentgateway.result == 'skipped') && (needs.test-api-integrations.result == 'success' || needs.test-api-integrations.result == 'skipped') && (needs.test-cost-accuracy.result == 'success' || needs.test-cost-accuracy.result == 'skipped') && (needs.test-docker-image-amd64.result == 'success' || needs.test-docker-image-amd64.result == 'skipped') && (needs.test-docker-image-arm64.result == 'success' || needs.test-docker-image-arm64.result == 'skipped') && (needs.detect-changes.outputs.core-needs-release == 'false' || needs.core-release.result == 'success' || needs.core-release.result == 'skipped')" + if: "always() && needs.check-skip.outputs.should-skip != 'true' && needs.detect-changes.result == 'success' && needs.detect-changes.outputs.framework-needs-release == 'true' && (needs.test-core.result == 'success' || needs.test-core.result == 'skipped' || (needs.test-core.result == 'failure' && needs.approve-flaky-test-core.result == 'success')) && (needs.test-proxy-e2e.result == 'success' || needs.test-proxy-e2e.result == 'skipped') && (needs.test-core-integrations.result == 'success' || needs.test-core-integrations.result == 'skipped') && (needs.test-core-unit.result == 'success' || needs.test-core-unit.result == 'skipped') && (needs.test-cli-harness.result == 'success' || needs.test-cli-harness.result == 'skipped') && (needs.resolve-cli-versions.result == 'success' || needs.resolve-cli-versions.result == 'skipped') && (needs.test-framework.result == 'success' || needs.test-framework.result == 'skipped') && (needs.test-plugins.result == 'success' || needs.test-plugins.result == 'skipped') && (needs.test-bifrost-http.result == 'success' || needs.test-bifrost-http.result == 'skipped') && (needs.test-live.result == 'success' || needs.test-live.result == 'skipped') && (needs.test-migrations.result == 'success' || needs.test-migrations.result == 'skipped') && (needs.test-agentgateway.result == 'success' || needs.test-agentgateway.result == 'skipped') && (needs.test-api-integrations.result == 'success' || needs.test-api-integrations.result == 'skipped') && (needs.test-warp.result == 'success' || needs.test-warp.result == 'skipped') && (needs.test-cost-accuracy.result == 'success' || needs.test-cost-accuracy.result == 'skipped') && (needs.test-docker-image-amd64.result == 'success' || needs.test-docker-image-amd64.result == 'skipped') && (needs.test-docker-image-arm64.result == 'success' || needs.test-docker-image-arm64.result == 'skipped') && (needs.detect-changes.outputs.core-needs-release == 'false' || needs.core-release.result == 'success' || needs.core-release.result == 'skipped')" runs-on: builder-8-32 permissions: contents: write @@ -3059,13 +3168,14 @@ jobs: test-migrations, test-agentgateway, test-api-integrations, + test-warp, test-cost-accuracy, test-docker-image-amd64, test-docker-image-arm64, core-release, framework-release, ] - if: "always() && needs.check-skip.outputs.should-skip != 'true' && needs.detect-changes.result == 'success' && needs.detect-changes.outputs.plugins-need-release == 'true' && (needs.test-core.result == 'success' || needs.test-core.result == 'skipped' || (needs.test-core.result == 'failure' && needs.approve-flaky-test-core.result == 'success')) && (needs.test-proxy-e2e.result == 'success' || needs.test-proxy-e2e.result == 'skipped') && (needs.test-core-integrations.result == 'success' || needs.test-core-integrations.result == 'skipped') && (needs.test-core-unit.result == 'success' || needs.test-core-unit.result == 'skipped') && (needs.test-cli-harness.result == 'success' || needs.test-cli-harness.result == 'skipped') && (needs.resolve-cli-versions.result == 'success' || needs.resolve-cli-versions.result == 'skipped') && (needs.test-framework.result == 'success' || needs.test-framework.result == 'skipped') && (needs.test-plugins.result == 'success' || needs.test-plugins.result == 'skipped') && (needs.test-bifrost-http.result == 'success' || needs.test-bifrost-http.result == 'skipped') && (needs.test-live.result == 'success' || needs.test-live.result == 'skipped') && (needs.test-migrations.result == 'success' || needs.test-migrations.result == 'skipped') && (needs.test-agentgateway.result == 'success' || needs.test-agentgateway.result == 'skipped') && (needs.test-api-integrations.result == 'success' || needs.test-api-integrations.result == 'skipped') && (needs.test-cost-accuracy.result == 'success' || needs.test-cost-accuracy.result == 'skipped') && (needs.test-docker-image-amd64.result == 'success' || needs.test-docker-image-amd64.result == 'skipped') && (needs.test-docker-image-arm64.result == 'success' || needs.test-docker-image-arm64.result == 'skipped') && (needs.detect-changes.outputs.core-needs-release == 'false' || needs.core-release.result == 'success' || needs.core-release.result == 'skipped') && (needs.detect-changes.outputs.framework-needs-release == 'false' || needs.framework-release.result == 'success' || needs.framework-release.result == 'skipped')" + if: "always() && needs.check-skip.outputs.should-skip != 'true' && needs.detect-changes.result == 'success' && needs.detect-changes.outputs.plugins-need-release == 'true' && (needs.test-core.result == 'success' || needs.test-core.result == 'skipped' || (needs.test-core.result == 'failure' && needs.approve-flaky-test-core.result == 'success')) && (needs.test-proxy-e2e.result == 'success' || needs.test-proxy-e2e.result == 'skipped') && (needs.test-core-integrations.result == 'success' || needs.test-core-integrations.result == 'skipped') && (needs.test-core-unit.result == 'success' || needs.test-core-unit.result == 'skipped') && (needs.test-cli-harness.result == 'success' || needs.test-cli-harness.result == 'skipped') && (needs.resolve-cli-versions.result == 'success' || needs.resolve-cli-versions.result == 'skipped') && (needs.test-framework.result == 'success' || needs.test-framework.result == 'skipped') && (needs.test-plugins.result == 'success' || needs.test-plugins.result == 'skipped') && (needs.test-bifrost-http.result == 'success' || needs.test-bifrost-http.result == 'skipped') && (needs.test-live.result == 'success' || needs.test-live.result == 'skipped') && (needs.test-migrations.result == 'success' || needs.test-migrations.result == 'skipped') && (needs.test-agentgateway.result == 'success' || needs.test-agentgateway.result == 'skipped') && (needs.test-api-integrations.result == 'success' || needs.test-api-integrations.result == 'skipped') && (needs.test-warp.result == 'success' || needs.test-warp.result == 'skipped') && (needs.test-cost-accuracy.result == 'success' || needs.test-cost-accuracy.result == 'skipped') && (needs.test-docker-image-amd64.result == 'success' || needs.test-docker-image-amd64.result == 'skipped') && (needs.test-docker-image-arm64.result == 'success' || needs.test-docker-image-arm64.result == 'skipped') && (needs.detect-changes.outputs.core-needs-release == 'false' || needs.core-release.result == 'success' || needs.core-release.result == 'skipped') && (needs.detect-changes.outputs.framework-needs-release == 'false' || needs.framework-release.result == 'success' || needs.framework-release.result == 'skipped')" runs-on: builder-8-32 permissions: contents: write @@ -3077,12 +3187,12 @@ jobs: with: egress-policy: block allowed-endpoints: > - _http._tcp.azure.archive.ubuntu.com:443 + _https._tcp.archive.ubuntu.com:443 _https._tcp.esm.ubuntu.com:443 _https._tcp.motd.ubuntu.com:443 _https._tcp.packages.microsoft.com:443 api.github.com:443 - azure.archive.ubuntu.com:80 + archive.ubuntu.com:443 esm.ubuntu.com:443 github.com:443 nodejs.org:443 @@ -3101,6 +3211,7 @@ jobs: - name: Install jq run: | + ./.github/workflows/scripts/apt-sources-https.sh sudo apt-get update sudo apt-get install -y jq @@ -3205,6 +3316,7 @@ jobs: test-migrations, test-agentgateway, test-api-integrations, + test-warp, test-cost-accuracy, test-docker-image-amd64, test-docker-image-arm64, @@ -3212,7 +3324,7 @@ jobs: framework-release, plugins-release, ] - if: "always() && needs.check-skip.outputs.should-skip != 'true' && needs.detect-changes.result == 'success' && needs.detect-changes.outputs.bifrost-http-needs-release == 'true' && (needs.test-core.result == 'success' || needs.test-core.result == 'skipped' || (needs.test-core.result == 'failure' && needs.approve-flaky-test-core.result == 'success')) && (needs.test-proxy-e2e.result == 'success' || needs.test-proxy-e2e.result == 'skipped') && (needs.test-memory.result == 'success' || needs.test-memory.result == 'skipped' || (needs.test-memory.result == 'failure' && needs.approve-memory-regression.result == 'success')) && (needs.test-core-integrations.result == 'success' || needs.test-core-integrations.result == 'skipped') && (needs.test-core-unit.result == 'success' || needs.test-core-unit.result == 'skipped') && (needs.test-cli-harness.result == 'success' || needs.test-cli-harness.result == 'skipped') && (needs.resolve-cli-versions.result == 'success' || needs.resolve-cli-versions.result == 'skipped') && (needs.test-framework.result == 'success' || needs.test-framework.result == 'skipped') && (needs.test-plugins.result == 'success' || needs.test-plugins.result == 'skipped') && (needs.test-bifrost-http.result == 'success' || needs.test-bifrost-http.result == 'skipped') && (needs.test-live.result == 'success' || needs.test-live.result == 'skipped') && (needs.test-migrations.result == 'success' || needs.test-migrations.result == 'skipped') && (needs.test-agentgateway.result == 'success' || needs.test-agentgateway.result == 'skipped') && (needs.test-api-integrations.result == 'success' || needs.test-api-integrations.result == 'skipped') && (needs.test-cost-accuracy.result == 'success' || needs.test-cost-accuracy.result == 'skipped') && (needs.test-docker-image-amd64.result == 'success' || needs.test-docker-image-amd64.result == 'skipped') && (needs.test-docker-image-arm64.result == 'success' || needs.test-docker-image-arm64.result == 'skipped') && (needs.detect-changes.outputs.core-needs-release == 'false' || needs.core-release.result == 'success' || needs.core-release.result == 'skipped') && (needs.detect-changes.outputs.framework-needs-release == 'false' || needs.framework-release.result == 'success' || needs.framework-release.result == 'skipped') && (needs.detect-changes.outputs.plugins-need-release == 'false' || needs.plugins-release.result == 'success' || needs.plugins-release.result == 'skipped')" + if: "always() && needs.check-skip.outputs.should-skip != 'true' && needs.detect-changes.result == 'success' && needs.detect-changes.outputs.bifrost-http-needs-release == 'true' && (needs.test-core.result == 'success' || needs.test-core.result == 'skipped' || (needs.test-core.result == 'failure' && needs.approve-flaky-test-core.result == 'success')) && (needs.test-proxy-e2e.result == 'success' || needs.test-proxy-e2e.result == 'skipped') && (needs.test-memory.result == 'success' || needs.test-memory.result == 'skipped' || (needs.test-memory.result == 'failure' && needs.approve-memory-regression.result == 'success')) && (needs.test-core-integrations.result == 'success' || needs.test-core-integrations.result == 'skipped') && (needs.test-core-unit.result == 'success' || needs.test-core-unit.result == 'skipped') && (needs.test-cli-harness.result == 'success' || needs.test-cli-harness.result == 'skipped') && (needs.resolve-cli-versions.result == 'success' || needs.resolve-cli-versions.result == 'skipped') && (needs.test-framework.result == 'success' || needs.test-framework.result == 'skipped') && (needs.test-plugins.result == 'success' || needs.test-plugins.result == 'skipped') && (needs.test-bifrost-http.result == 'success' || needs.test-bifrost-http.result == 'skipped') && (needs.test-live.result == 'success' || needs.test-live.result == 'skipped') && (needs.test-migrations.result == 'success' || needs.test-migrations.result == 'skipped') && (needs.test-agentgateway.result == 'success' || needs.test-agentgateway.result == 'skipped') && (needs.test-api-integrations.result == 'success' || needs.test-api-integrations.result == 'skipped') && (needs.test-warp.result == 'success' || needs.test-warp.result == 'skipped') && (needs.test-cost-accuracy.result == 'success' || needs.test-cost-accuracy.result == 'skipped') && (needs.test-docker-image-amd64.result == 'success' || needs.test-docker-image-amd64.result == 'skipped') && (needs.test-docker-image-arm64.result == 'success' || needs.test-docker-image-arm64.result == 'skipped') && (needs.detect-changes.outputs.core-needs-release == 'false' || needs.core-release.result == 'success' || needs.core-release.result == 'skipped') && (needs.detect-changes.outputs.framework-needs-release == 'false' || needs.framework-release.result == 'success' || needs.framework-release.result == 'skipped') && (needs.detect-changes.outputs.plugins-need-release == 'false' || needs.plugins-release.result == 'success' || needs.plugins-release.result == 'skipped')" runs-on: builder-8-32 permissions: contents: write @@ -3277,12 +3389,12 @@ jobs: egress-policy: block allowed-endpoints: > 7defe2860d5ee49a1e667e1eeea34b25.r2.cloudflarestorage.com:443 - _http._tcp.azure.archive.ubuntu.com:443 + _https._tcp.archive.ubuntu.com:443 _https._tcp.esm.ubuntu.com:443 _https._tcp.motd.ubuntu.com:443 _https._tcp.packages.microsoft.com:443 api.github.com:443 - azure.archive.ubuntu.com:80 + archive.ubuntu.com:443 esm.ubuntu.com:443 files.pythonhosted.org:443 fonts.googleapis.com:443 @@ -3359,7 +3471,8 @@ jobs: fonts.gstatic.com:443 github.com:443 nodejs.org:443 - ports.ubuntu.com:80 + _https._tcp.ports.ubuntu.com:443 + ports.ubuntu.com:443 proxy.golang.org:443 pypi.org:443 registry.npmjs.org:443 @@ -3480,6 +3593,7 @@ jobs: test-migrations, test-agentgateway, test-api-integrations, + test-warp, test-cost-accuracy, test-docker-image-amd64, test-docker-image-arm64, @@ -3488,7 +3602,7 @@ jobs: plugins-release, bifrost-http-release, ] - if: "always() && needs.check-skip.outputs.should-skip != 'true' && needs.detect-changes.result == 'success' && needs.detect-changes.outputs.docker-needs-release == 'true' && (needs.test-core.result == 'success' || needs.test-core.result == 'skipped' || (needs.test-core.result == 'failure' && needs.approve-flaky-test-core.result == 'success')) && (needs.test-proxy-e2e.result == 'success' || needs.test-proxy-e2e.result == 'skipped') && (needs.test-memory.result == 'success' || needs.test-memory.result == 'skipped' || (needs.test-memory.result == 'failure' && needs.approve-memory-regression.result == 'success')) && (needs.test-core-integrations.result == 'success' || needs.test-core-integrations.result == 'skipped') && (needs.test-core-unit.result == 'success' || needs.test-core-unit.result == 'skipped') && (needs.test-cli-harness.result == 'success' || needs.test-cli-harness.result == 'skipped') && (needs.resolve-cli-versions.result == 'success' || needs.resolve-cli-versions.result == 'skipped') && (needs.test-framework.result == 'success' || needs.test-framework.result == 'skipped') && (needs.test-plugins.result == 'success' || needs.test-plugins.result == 'skipped') && (needs.test-bifrost-http.result == 'success' || needs.test-bifrost-http.result == 'skipped') && (needs.test-live.result == 'success' || needs.test-live.result == 'skipped') && (needs.test-migrations.result == 'success' || needs.test-migrations.result == 'skipped') && (needs.test-agentgateway.result == 'success' || needs.test-agentgateway.result == 'skipped') && (needs.test-api-integrations.result == 'success' || needs.test-api-integrations.result == 'skipped') && (needs.test-cost-accuracy.result == 'success' || needs.test-cost-accuracy.result == 'skipped') && (needs.test-docker-image-amd64.result == 'success' || needs.test-docker-image-amd64.result == 'skipped') && (needs.test-docker-image-arm64.result == 'success' || needs.test-docker-image-arm64.result == 'skipped') && (needs.detect-changes.outputs.core-needs-release == 'false' || needs.core-release.result == 'success' || needs.core-release.result == 'skipped') && (needs.detect-changes.outputs.framework-needs-release == 'false' || needs.framework-release.result == 'success' || needs.framework-release.result == 'skipped') && (needs.detect-changes.outputs.plugins-need-release == 'false' || needs.plugins-release.result == 'success' || needs.plugins-release.result == 'skipped') && (needs.detect-changes.outputs.bifrost-http-needs-release == 'false' || needs.bifrost-http-release.result == 'success' || needs.bifrost-http-release.result == 'skipped')" + if: "always() && needs.check-skip.outputs.should-skip != 'true' && needs.detect-changes.result == 'success' && needs.detect-changes.outputs.docker-needs-release == 'true' && (needs.test-core.result == 'success' || needs.test-core.result == 'skipped' || (needs.test-core.result == 'failure' && needs.approve-flaky-test-core.result == 'success')) && (needs.test-proxy-e2e.result == 'success' || needs.test-proxy-e2e.result == 'skipped') && (needs.test-memory.result == 'success' || needs.test-memory.result == 'skipped' || (needs.test-memory.result == 'failure' && needs.approve-memory-regression.result == 'success')) && (needs.test-core-integrations.result == 'success' || needs.test-core-integrations.result == 'skipped') && (needs.test-core-unit.result == 'success' || needs.test-core-unit.result == 'skipped') && (needs.test-cli-harness.result == 'success' || needs.test-cli-harness.result == 'skipped') && (needs.resolve-cli-versions.result == 'success' || needs.resolve-cli-versions.result == 'skipped') && (needs.test-framework.result == 'success' || needs.test-framework.result == 'skipped') && (needs.test-plugins.result == 'success' || needs.test-plugins.result == 'skipped') && (needs.test-bifrost-http.result == 'success' || needs.test-bifrost-http.result == 'skipped') && (needs.test-live.result == 'success' || needs.test-live.result == 'skipped') && (needs.test-migrations.result == 'success' || needs.test-migrations.result == 'skipped') && (needs.test-agentgateway.result == 'success' || needs.test-agentgateway.result == 'skipped') && (needs.test-api-integrations.result == 'success' || needs.test-api-integrations.result == 'skipped') && (needs.test-warp.result == 'success' || needs.test-warp.result == 'skipped') && (needs.test-cost-accuracy.result == 'success' || needs.test-cost-accuracy.result == 'skipped') && (needs.test-docker-image-amd64.result == 'success' || needs.test-docker-image-amd64.result == 'skipped') && (needs.test-docker-image-arm64.result == 'success' || needs.test-docker-image-arm64.result == 'skipped') && (needs.detect-changes.outputs.core-needs-release == 'false' || needs.core-release.result == 'success' || needs.core-release.result == 'skipped') && (needs.detect-changes.outputs.framework-needs-release == 'false' || needs.framework-release.result == 'success' || needs.framework-release.result == 'skipped') && (needs.detect-changes.outputs.plugins-need-release == 'false' || needs.plugins-release.result == 'success' || needs.plugins-release.result == 'skipped') && (needs.detect-changes.outputs.bifrost-http-needs-release == 'false' || needs.bifrost-http-release.result == 'success' || needs.bifrost-http-release.result == 'skipped')" runs-on: builder-8-32 permissions: contents: write @@ -3603,6 +3717,7 @@ jobs: test-migrations, test-agentgateway, test-api-integrations, + test-warp, test-cost-accuracy, test-docker-image-amd64, test-docker-image-arm64, @@ -3611,7 +3726,7 @@ jobs: plugins-release, bifrost-http-release, ] - if: "always() && needs.check-skip.outputs.should-skip != 'true' && needs.detect-changes.result == 'success' && needs.detect-changes.outputs.docker-needs-release == 'true' && (needs.test-core.result == 'success' || needs.test-core.result == 'skipped' || (needs.test-core.result == 'failure' && needs.approve-flaky-test-core.result == 'success')) && (needs.test-proxy-e2e.result == 'success' || needs.test-proxy-e2e.result == 'skipped') && (needs.test-memory.result == 'success' || needs.test-memory.result == 'skipped' || (needs.test-memory.result == 'failure' && needs.approve-memory-regression.result == 'success')) && (needs.test-core-integrations.result == 'success' || needs.test-core-integrations.result == 'skipped') && (needs.test-core-unit.result == 'success' || needs.test-core-unit.result == 'skipped') && (needs.test-cli-harness.result == 'success' || needs.test-cli-harness.result == 'skipped') && (needs.resolve-cli-versions.result == 'success' || needs.resolve-cli-versions.result == 'skipped') && (needs.test-framework.result == 'success' || needs.test-framework.result == 'skipped') && (needs.test-plugins.result == 'success' || needs.test-plugins.result == 'skipped') && (needs.test-bifrost-http.result == 'success' || needs.test-bifrost-http.result == 'skipped') && (needs.test-live.result == 'success' || needs.test-live.result == 'skipped') && (needs.test-migrations.result == 'success' || needs.test-migrations.result == 'skipped') && (needs.test-agentgateway.result == 'success' || needs.test-agentgateway.result == 'skipped') && (needs.test-api-integrations.result == 'success' || needs.test-api-integrations.result == 'skipped') && (needs.test-cost-accuracy.result == 'success' || needs.test-cost-accuracy.result == 'skipped') && (needs.test-docker-image-amd64.result == 'success' || needs.test-docker-image-amd64.result == 'skipped') && (needs.test-docker-image-arm64.result == 'success' || needs.test-docker-image-arm64.result == 'skipped') && (needs.detect-changes.outputs.core-needs-release == 'false' || needs.core-release.result == 'success' || needs.core-release.result == 'skipped') && (needs.detect-changes.outputs.framework-needs-release == 'false' || needs.framework-release.result == 'success' || needs.framework-release.result == 'skipped') && (needs.detect-changes.outputs.plugins-need-release == 'false' || needs.plugins-release.result == 'success' || needs.plugins-release.result == 'skipped') && (needs.detect-changes.outputs.bifrost-http-needs-release == 'false' || needs.bifrost-http-release.result == 'success' || needs.bifrost-http-release.result == 'skipped')" + if: "always() && needs.check-skip.outputs.should-skip != 'true' && needs.detect-changes.result == 'success' && needs.detect-changes.outputs.docker-needs-release == 'true' && (needs.test-core.result == 'success' || needs.test-core.result == 'skipped' || (needs.test-core.result == 'failure' && needs.approve-flaky-test-core.result == 'success')) && (needs.test-proxy-e2e.result == 'success' || needs.test-proxy-e2e.result == 'skipped') && (needs.test-memory.result == 'success' || needs.test-memory.result == 'skipped' || (needs.test-memory.result == 'failure' && needs.approve-memory-regression.result == 'success')) && (needs.test-core-integrations.result == 'success' || needs.test-core-integrations.result == 'skipped') && (needs.test-core-unit.result == 'success' || needs.test-core-unit.result == 'skipped') && (needs.test-cli-harness.result == 'success' || needs.test-cli-harness.result == 'skipped') && (needs.resolve-cli-versions.result == 'success' || needs.resolve-cli-versions.result == 'skipped') && (needs.test-framework.result == 'success' || needs.test-framework.result == 'skipped') && (needs.test-plugins.result == 'success' || needs.test-plugins.result == 'skipped') && (needs.test-bifrost-http.result == 'success' || needs.test-bifrost-http.result == 'skipped') && (needs.test-live.result == 'success' || needs.test-live.result == 'skipped') && (needs.test-migrations.result == 'success' || needs.test-migrations.result == 'skipped') && (needs.test-agentgateway.result == 'success' || needs.test-agentgateway.result == 'skipped') && (needs.test-api-integrations.result == 'success' || needs.test-api-integrations.result == 'skipped') && (needs.test-warp.result == 'success' || needs.test-warp.result == 'skipped') && (needs.test-cost-accuracy.result == 'success' || needs.test-cost-accuracy.result == 'skipped') && (needs.test-docker-image-amd64.result == 'success' || needs.test-docker-image-amd64.result == 'skipped') && (needs.test-docker-image-arm64.result == 'success' || needs.test-docker-image-arm64.result == 'skipped') && (needs.detect-changes.outputs.core-needs-release == 'false' || needs.core-release.result == 'success' || needs.core-release.result == 'skipped') && (needs.detect-changes.outputs.framework-needs-release == 'false' || needs.framework-release.result == 'success' || needs.framework-release.result == 'skipped') && (needs.detect-changes.outputs.plugins-need-release == 'false' || needs.plugins-release.result == 'success' || needs.plugins-release.result == 'skipped') && (needs.detect-changes.outputs.bifrost-http-needs-release == 'false' || needs.bifrost-http-release.result == 'success' || needs.bifrost-http-release.result == 'skipped')" runs-on: arm-64-2cpu-8gb permissions: contents: write @@ -3628,19 +3743,22 @@ jobs: allowed-endpoints: > auth.docker.io:443 dl-cdn.alpinelinux.org:443 + _https._tcp.download.docker.com:443 download.docker.com:443 fonts.googleapis.com:443 fonts.gstatic.com:443 ghcr.io:443 github.com:443 pkg-containers.githubusercontent.com:443 - ports.ubuntu.com:80 + _https._tcp.ports.ubuntu.com:443 + ports.ubuntu.com:443 production.cloudflare.docker.com:443 production.cloudfront.docker.com:443 proxy.golang.org:443 registry-1.docker.io:443 registry.npmjs.org:443 storage.googleapis.com:443 + tuf-repo-cdn.sigstore.dev:443 - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -3779,6 +3897,7 @@ jobs: test-migrations, test-agentgateway, test-api-integrations, + test-warp, test-cost-accuracy, test-docker-image-amd64, test-docker-image-arm64, @@ -3787,7 +3906,7 @@ jobs: plugins-release, bifrost-http-release, ] - if: "always() && needs.check-skip.outputs.should-skip != 'true' && needs.detect-changes.result == 'success' && needs.detect-changes.outputs.docker-needs-release == 'true' && (needs.test-core.result == 'success' || needs.test-core.result == 'skipped' || (needs.test-core.result == 'failure' && needs.approve-flaky-test-core.result == 'success')) && (needs.test-proxy-e2e.result == 'success' || needs.test-proxy-e2e.result == 'skipped') && (needs.test-memory.result == 'success' || needs.test-memory.result == 'skipped' || (needs.test-memory.result == 'failure' && needs.approve-memory-regression.result == 'success')) && (needs.test-core-integrations.result == 'success' || needs.test-core-integrations.result == 'skipped') && (needs.test-core-unit.result == 'success' || needs.test-core-unit.result == 'skipped') && (needs.test-cli-harness.result == 'success' || needs.test-cli-harness.result == 'skipped') && (needs.resolve-cli-versions.result == 'success' || needs.resolve-cli-versions.result == 'skipped') && (needs.test-framework.result == 'success' || needs.test-framework.result == 'skipped') && (needs.test-plugins.result == 'success' || needs.test-plugins.result == 'skipped') && (needs.test-bifrost-http.result == 'success' || needs.test-bifrost-http.result == 'skipped') && (needs.test-live.result == 'success' || needs.test-live.result == 'skipped') && (needs.test-migrations.result == 'success' || needs.test-migrations.result == 'skipped') && (needs.test-agentgateway.result == 'success' || needs.test-agentgateway.result == 'skipped') && (needs.test-api-integrations.result == 'success' || needs.test-api-integrations.result == 'skipped') && (needs.test-cost-accuracy.result == 'success' || needs.test-cost-accuracy.result == 'skipped') && (needs.test-docker-image-amd64.result == 'success' || needs.test-docker-image-amd64.result == 'skipped') && (needs.test-docker-image-arm64.result == 'success' || needs.test-docker-image-arm64.result == 'skipped') && (needs.detect-changes.outputs.core-needs-release == 'false' || needs.core-release.result == 'success' || needs.core-release.result == 'skipped') && (needs.detect-changes.outputs.framework-needs-release == 'false' || needs.framework-release.result == 'success' || needs.framework-release.result == 'skipped') && (needs.detect-changes.outputs.plugins-need-release == 'false' || needs.plugins-release.result == 'success' || needs.plugins-release.result == 'skipped') && (needs.detect-changes.outputs.bifrost-http-needs-release == 'false' || needs.bifrost-http-release.result == 'success' || needs.bifrost-http-release.result == 'skipped')" + if: "always() && needs.check-skip.outputs.should-skip != 'true' && needs.detect-changes.result == 'success' && needs.detect-changes.outputs.docker-needs-release == 'true' && (needs.test-core.result == 'success' || needs.test-core.result == 'skipped' || (needs.test-core.result == 'failure' && needs.approve-flaky-test-core.result == 'success')) && (needs.test-proxy-e2e.result == 'success' || needs.test-proxy-e2e.result == 'skipped') && (needs.test-memory.result == 'success' || needs.test-memory.result == 'skipped' || (needs.test-memory.result == 'failure' && needs.approve-memory-regression.result == 'success')) && (needs.test-core-integrations.result == 'success' || needs.test-core-integrations.result == 'skipped') && (needs.test-core-unit.result == 'success' || needs.test-core-unit.result == 'skipped') && (needs.test-cli-harness.result == 'success' || needs.test-cli-harness.result == 'skipped') && (needs.resolve-cli-versions.result == 'success' || needs.resolve-cli-versions.result == 'skipped') && (needs.test-framework.result == 'success' || needs.test-framework.result == 'skipped') && (needs.test-plugins.result == 'success' || needs.test-plugins.result == 'skipped') && (needs.test-bifrost-http.result == 'success' || needs.test-bifrost-http.result == 'skipped') && (needs.test-live.result == 'success' || needs.test-live.result == 'skipped') && (needs.test-migrations.result == 'success' || needs.test-migrations.result == 'skipped') && (needs.test-agentgateway.result == 'success' || needs.test-agentgateway.result == 'skipped') && (needs.test-api-integrations.result == 'success' || needs.test-api-integrations.result == 'skipped') && (needs.test-warp.result == 'success' || needs.test-warp.result == 'skipped') && (needs.test-cost-accuracy.result == 'success' || needs.test-cost-accuracy.result == 'skipped') && (needs.test-docker-image-amd64.result == 'success' || needs.test-docker-image-amd64.result == 'skipped') && (needs.test-docker-image-arm64.result == 'success' || needs.test-docker-image-arm64.result == 'skipped') && (needs.detect-changes.outputs.core-needs-release == 'false' || needs.core-release.result == 'success' || needs.core-release.result == 'skipped') && (needs.detect-changes.outputs.framework-needs-release == 'false' || needs.framework-release.result == 'success' || needs.framework-release.result == 'skipped') && (needs.detect-changes.outputs.plugins-need-release == 'false' || needs.plugins-release.result == 'success' || needs.plugins-release.result == 'skipped') && (needs.detect-changes.outputs.bifrost-http-needs-release == 'false' || needs.bifrost-http-release.result == 'success' || needs.bifrost-http-release.result == 'skipped')" runs-on: builder-8-32 permissions: contents: write @@ -3906,6 +4025,7 @@ jobs: test-migrations, test-agentgateway, test-api-integrations, + test-warp, test-cost-accuracy, test-docker-image-amd64, test-docker-image-arm64, @@ -3914,7 +4034,7 @@ jobs: plugins-release, bifrost-http-release, ] - if: "always() && needs.check-skip.outputs.should-skip != 'true' && needs.detect-changes.result == 'success' && needs.detect-changes.outputs.docker-needs-release == 'true' && (needs.test-core.result == 'success' || needs.test-core.result == 'skipped' || (needs.test-core.result == 'failure' && needs.approve-flaky-test-core.result == 'success')) && (needs.test-proxy-e2e.result == 'success' || needs.test-proxy-e2e.result == 'skipped') && (needs.test-memory.result == 'success' || needs.test-memory.result == 'skipped' || (needs.test-memory.result == 'failure' && needs.approve-memory-regression.result == 'success')) && (needs.test-core-integrations.result == 'success' || needs.test-core-integrations.result == 'skipped') && (needs.test-core-unit.result == 'success' || needs.test-core-unit.result == 'skipped') && (needs.test-cli-harness.result == 'success' || needs.test-cli-harness.result == 'skipped') && (needs.resolve-cli-versions.result == 'success' || needs.resolve-cli-versions.result == 'skipped') && (needs.test-framework.result == 'success' || needs.test-framework.result == 'skipped') && (needs.test-plugins.result == 'success' || needs.test-plugins.result == 'skipped') && (needs.test-bifrost-http.result == 'success' || needs.test-bifrost-http.result == 'skipped') && (needs.test-live.result == 'success' || needs.test-live.result == 'skipped') && (needs.test-migrations.result == 'success' || needs.test-migrations.result == 'skipped') && (needs.test-agentgateway.result == 'success' || needs.test-agentgateway.result == 'skipped') && (needs.test-api-integrations.result == 'success' || needs.test-api-integrations.result == 'skipped') && (needs.test-cost-accuracy.result == 'success' || needs.test-cost-accuracy.result == 'skipped') && (needs.test-docker-image-amd64.result == 'success' || needs.test-docker-image-amd64.result == 'skipped') && (needs.test-docker-image-arm64.result == 'success' || needs.test-docker-image-arm64.result == 'skipped') && (needs.detect-changes.outputs.core-needs-release == 'false' || needs.core-release.result == 'success' || needs.core-release.result == 'skipped') && (needs.detect-changes.outputs.framework-needs-release == 'false' || needs.framework-release.result == 'success' || needs.framework-release.result == 'skipped') && (needs.detect-changes.outputs.plugins-need-release == 'false' || needs.plugins-release.result == 'success' || needs.plugins-release.result == 'skipped') && (needs.detect-changes.outputs.bifrost-http-needs-release == 'false' || needs.bifrost-http-release.result == 'success' || needs.bifrost-http-release.result == 'skipped')" + if: "always() && needs.check-skip.outputs.should-skip != 'true' && needs.detect-changes.result == 'success' && needs.detect-changes.outputs.docker-needs-release == 'true' && (needs.test-core.result == 'success' || needs.test-core.result == 'skipped' || (needs.test-core.result == 'failure' && needs.approve-flaky-test-core.result == 'success')) && (needs.test-proxy-e2e.result == 'success' || needs.test-proxy-e2e.result == 'skipped') && (needs.test-memory.result == 'success' || needs.test-memory.result == 'skipped' || (needs.test-memory.result == 'failure' && needs.approve-memory-regression.result == 'success')) && (needs.test-core-integrations.result == 'success' || needs.test-core-integrations.result == 'skipped') && (needs.test-core-unit.result == 'success' || needs.test-core-unit.result == 'skipped') && (needs.test-cli-harness.result == 'success' || needs.test-cli-harness.result == 'skipped') && (needs.resolve-cli-versions.result == 'success' || needs.resolve-cli-versions.result == 'skipped') && (needs.test-framework.result == 'success' || needs.test-framework.result == 'skipped') && (needs.test-plugins.result == 'success' || needs.test-plugins.result == 'skipped') && (needs.test-bifrost-http.result == 'success' || needs.test-bifrost-http.result == 'skipped') && (needs.test-live.result == 'success' || needs.test-live.result == 'skipped') && (needs.test-migrations.result == 'success' || needs.test-migrations.result == 'skipped') && (needs.test-agentgateway.result == 'success' || needs.test-agentgateway.result == 'skipped') && (needs.test-api-integrations.result == 'success' || needs.test-api-integrations.result == 'skipped') && (needs.test-warp.result == 'success' || needs.test-warp.result == 'skipped') && (needs.test-cost-accuracy.result == 'success' || needs.test-cost-accuracy.result == 'skipped') && (needs.test-docker-image-amd64.result == 'success' || needs.test-docker-image-amd64.result == 'skipped') && (needs.test-docker-image-arm64.result == 'success' || needs.test-docker-image-arm64.result == 'skipped') && (needs.detect-changes.outputs.core-needs-release == 'false' || needs.core-release.result == 'success' || needs.core-release.result == 'skipped') && (needs.detect-changes.outputs.framework-needs-release == 'false' || needs.framework-release.result == 'success' || needs.framework-release.result == 'skipped') && (needs.detect-changes.outputs.plugins-need-release == 'false' || needs.plugins-release.result == 'success' || needs.plugins-release.result == 'skipped') && (needs.detect-changes.outputs.bifrost-http-needs-release == 'false' || needs.bifrost-http-release.result == 'success' || needs.bifrost-http-release.result == 'skipped')" runs-on: arm-64-2cpu-8gb permissions: contents: write @@ -3931,13 +4051,15 @@ jobs: allowed-endpoints: > auth.docker.io:443 dl-cdn.alpinelinux.org:443 + _https._tcp.download.docker.com:443 download.docker.com:443 fonts.googleapis.com:443 fonts.gstatic.com:443 ghcr.io:443 github.com:443 pkg-containers.githubusercontent.com:443 - ports.ubuntu.com:80 + _https._tcp.ports.ubuntu.com:443 + ports.ubuntu.com:443 production.cloudflare.docker.com:443 production.cloudfront.docker.com:443 proxy.golang.org:443 @@ -3946,6 +4068,7 @@ jobs: *.quay.io:443 registry.npmjs.org:443 storage.googleapis.com:443 + tuf-repo-cdn.sigstore.dev:443 - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -4084,6 +4207,7 @@ jobs: test-migrations, test-agentgateway, test-api-integrations, + test-warp, test-cost-accuracy, test-docker-image-amd64, test-docker-image-arm64, @@ -4092,7 +4216,7 @@ jobs: plugins-release, bifrost-http-release, ] - if: "always() && needs.check-skip.outputs.should-skip != 'true' && needs.detect-changes.result == 'success' && (needs.test-core.result == 'success' || needs.test-core.result == 'skipped' || (needs.test-core.result == 'failure' && needs.approve-flaky-test-core.result == 'success')) && (needs.test-proxy-e2e.result == 'success' || needs.test-proxy-e2e.result == 'skipped') && (needs.test-core-integrations.result == 'success' || needs.test-core-integrations.result == 'skipped') && (needs.test-core-unit.result == 'success' || needs.test-core-unit.result == 'skipped') && (needs.test-cli-harness.result == 'success' || needs.test-cli-harness.result == 'skipped') && (needs.resolve-cli-versions.result == 'success' || needs.resolve-cli-versions.result == 'skipped') && (needs.test-framework.result == 'success' || needs.test-framework.result == 'skipped') && (needs.test-plugins.result == 'success' || needs.test-plugins.result == 'skipped') && (needs.test-bifrost-http.result == 'success' || needs.test-bifrost-http.result == 'skipped') && (needs.test-live.result == 'success' || needs.test-live.result == 'skipped') && (needs.test-migrations.result == 'success' || needs.test-migrations.result == 'skipped') && (needs.test-agentgateway.result == 'success' || needs.test-agentgateway.result == 'skipped') && (needs.test-api-integrations.result == 'success' || needs.test-api-integrations.result == 'skipped') && (needs.test-cost-accuracy.result == 'success' || needs.test-cost-accuracy.result == 'skipped') && (needs.test-docker-image-amd64.result == 'success' || needs.test-docker-image-amd64.result == 'skipped') && (needs.test-docker-image-arm64.result == 'success' || needs.test-docker-image-arm64.result == 'skipped') && (needs.detect-changes.outputs.core-needs-release == 'false' || needs.core-release.result == 'success' || needs.core-release.result == 'skipped') && (needs.detect-changes.outputs.framework-needs-release == 'false' || needs.framework-release.result == 'success' || needs.framework-release.result == 'skipped') && (needs.detect-changes.outputs.plugins-need-release == 'false' || needs.plugins-release.result == 'success' || needs.plugins-release.result == 'skipped') && (needs.detect-changes.outputs.bifrost-http-needs-release == 'false' || needs.bifrost-http-release.result == 'success' || needs.bifrost-http-release.result == 'skipped')" + if: "always() && needs.check-skip.outputs.should-skip != 'true' && needs.detect-changes.result == 'success' && (needs.test-core.result == 'success' || needs.test-core.result == 'skipped' || (needs.test-core.result == 'failure' && needs.approve-flaky-test-core.result == 'success')) && (needs.test-proxy-e2e.result == 'success' || needs.test-proxy-e2e.result == 'skipped') && (needs.test-core-integrations.result == 'success' || needs.test-core-integrations.result == 'skipped') && (needs.test-core-unit.result == 'success' || needs.test-core-unit.result == 'skipped') && (needs.test-cli-harness.result == 'success' || needs.test-cli-harness.result == 'skipped') && (needs.resolve-cli-versions.result == 'success' || needs.resolve-cli-versions.result == 'skipped') && (needs.test-framework.result == 'success' || needs.test-framework.result == 'skipped') && (needs.test-plugins.result == 'success' || needs.test-plugins.result == 'skipped') && (needs.test-bifrost-http.result == 'success' || needs.test-bifrost-http.result == 'skipped') && (needs.test-live.result == 'success' || needs.test-live.result == 'skipped') && (needs.test-migrations.result == 'success' || needs.test-migrations.result == 'skipped') && (needs.test-agentgateway.result == 'success' || needs.test-agentgateway.result == 'skipped') && (needs.test-api-integrations.result == 'success' || needs.test-api-integrations.result == 'skipped') && (needs.test-warp.result == 'success' || needs.test-warp.result == 'skipped') && (needs.test-cost-accuracy.result == 'success' || needs.test-cost-accuracy.result == 'skipped') && (needs.test-docker-image-amd64.result == 'success' || needs.test-docker-image-amd64.result == 'skipped') && (needs.test-docker-image-arm64.result == 'success' || needs.test-docker-image-arm64.result == 'skipped') && (needs.detect-changes.outputs.core-needs-release == 'false' || needs.core-release.result == 'success' || needs.core-release.result == 'skipped') && (needs.detect-changes.outputs.framework-needs-release == 'false' || needs.framework-release.result == 'success' || needs.framework-release.result == 'skipped') && (needs.detect-changes.outputs.plugins-need-release == 'false' || needs.plugins-release.result == 'success' || needs.plugins-release.result == 'skipped') && (needs.detect-changes.outputs.bifrost-http-needs-release == 'false' || needs.bifrost-http-release.result == 'success' || needs.bifrost-http-release.result == 'skipped')" runs-on: builder-8-32 permissions: contents: write @@ -4170,21 +4294,31 @@ jobs: with: egress-policy: block allowed-endpoints: > - _http._tcp.azure.archive.ubuntu.com:443 + _https._tcp.archive.ubuntu.com:443 _https._tcp.dl.google.com:443 motd.ubuntu.com:443 _https._tcp.esm.ubuntu.com:443 _https._tcp.motd.ubuntu.com:443 _https._tcp.packages.microsoft.com:443 - azure.archive.ubuntu.com:80 + archive.ubuntu.com:443 discord.com:443 discordapp.com:443 dl.google.com:443 + github.com:443 esm.ubuntu.com:443 packages.microsoft.com:443 + # apt-sources-https.sh moves apt to HTTPS before Install jq; nothing else + # from the repo is needed, so check out only that directory. + - name: Checkout CI scripts + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + sparse-checkout: .github/workflows/scripts + persist-credentials: false + - name: Install jq run: | + ./.github/workflows/scripts/apt-sources-https.sh sudo apt-get update sudo apt-get install -y jq diff --git a/.github/workflows/run-core-tests.yml b/.github/workflows/run-core-tests.yml index 8b5dcd80eff..abbf6467ff8 100644 --- a/.github/workflows/run-core-tests.yml +++ b/.github/workflows/run-core-tests.yml @@ -122,7 +122,7 @@ jobs: api.perplexity.ai:443 api.replicate.com:443 api.x.ai:443 - azure.archive.ubuntu.com:80 + archive.ubuntu.com:443 esm.ubuntu.com:443 bifrost-gateway-resource.openai.azure.com:443 bedrock-mantle.us-east-1.api.aws:443 @@ -208,6 +208,7 @@ jobs: - name: Install test dependencies run: | + ./.github/workflows/scripts/apt-sources-https.sh sudo apt-get update sudo apt-get install -y jq diff --git a/.github/workflows/scripts/apt-sources-https.sh b/.github/workflows/scripts/apt-sources-https.sh new file mode 100755 index 00000000000..80a5bde252c --- /dev/null +++ b/.github/workflows/scripts/apt-sources-https.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Points the runner's Ubuntu apt sources at HTTPS. Run it before every +# `apt-get update` in CI. +# +# Runner images ship apt sources on plain HTTP. apt's GPG-signed InRelease and +# per-file SHA256 already stop a tampered package, but HTTP still shows anyone on +# the path what we install, lets them replay a stale (validly signed) InRelease +# to hide security updates, and forces a port-80 entry into every harden-runner +# allowlist. After this runs, apt needs only :443. +# +# http://azure.archive.ubuntu.com -> https://archive.ubuntu.com +# The Azure-local mirror the amd64 images use serves no HTTPS at all (443 +# is closed), so it is swapped for Canonical's primary archive, which +# carries the same pockets, -security included. +# http://[azure.]ports.ubuntu.com -> https://ports.ubuntu.com (arm64) +# http://archive|security.ubuntu.com -> https:// on the same host +# +# Any other http:// source on an ubuntu.com host is an error rather than a +# silent pass: it would reach for :80, which the allowlists no longer carry, and +# fail as an opaque mirror timeout. check-egress-allowlist.sh requires the +# matching :443 hosts and that every apt-using step runs this script first. +# +# APT_ROOT (default /etc/apt) and APT_SUDO (default sudo; empty runs unprivileged) +# exist for apt-sources-https.test.sh. + +APT_ROOT="${APT_ROOT:-/etc/apt}" +APT_SUDO="${APT_SUDO-sudo}" + +files=() +for f in "$APT_ROOT/sources.list" "$APT_ROOT"/sources.list.d/*.list "$APT_ROOT"/sources.list.d/*.sources; do + [[ -f "$f" ]] && files+=("$f") +done +if [[ ${#files[@]} -eq 0 ]]; then + echo "apt-sources-https: no apt sources under $APT_ROOT" + exit 0 +fi + +# Written through a temp copy instead of sed -i so it behaves the same under GNU +# and BSD sed, and `cp` onto the existing file keeps its owner and mode. +tmp="$(mktemp)" +trap 'rm -f "$tmp"' EXIT +for f in "${files[@]}"; do + sed -E \ + -e 's#http://azure\.archive\.ubuntu\.com([/[:space:]]|$)#https://archive.ubuntu.com\1#g' \ + -e 's#http://(azure\.)?ports\.ubuntu\.com([/[:space:]]|$)#https://ports.ubuntu.com\2#g' \ + -e 's#http://(archive|security)\.ubuntu\.com([/[:space:]]|$)#https://\1.ubuntu.com\2#g' \ + "$f" > "$tmp" + if ! cmp -s "$tmp" "$f"; then + ${APT_SUDO:+$APT_SUDO} cp "$tmp" "$f" + echo "apt-sources-https: switched $f to https" + fi +done + +# Comment lines are skipped: stock sources.list carries "http://help.ubuntu.com" +# links in comments that apt never fetches. +if leftover="$(grep -HnE '^[^#]*http://[A-Za-z0-9.-]*ubuntu\.com' "${files[@]}")"; then + echo "::error::apt sources still use http:// on an ubuntu.com host this script does not map to https:" >&2 + echo "$leftover" >&2 + exit 1 +fi diff --git a/.github/workflows/scripts/apt-sources-https.test.sh b/.github/workflows/scripts/apt-sources-https.test.sh new file mode 100755 index 00000000000..d11dd55cab9 --- /dev/null +++ b/.github/workflows/scripts/apt-sources-https.test.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Exercise apt-sources-https.sh against fixture apt trees shaped like the two +# runner images: the arm64 Ubuntu 22.04 base image (one-line sources.list on +# ports.ubuntu.com) and the amd64 Ubuntu 24.04 image (deb822 ubuntu.sources on +# the Azure-local mirror, plus third-party repos). Runs unprivileged. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT + +passed=0 +fail() { cat "$WORK/out" >&2; echo "FAIL: $1" >&2; exit 1; } +ok() { echo "ok - $1"; passed=$((passed + 1)); } +run() { APT_ROOT="$WORK/apt" APT_SUDO="" "$BASH" "$SCRIPT_DIR/apt-sources-https.sh" > "$WORK/out" 2>&1; } +reset() { rm -rf "$WORK/apt" && mkdir -p "$WORK/apt/sources.list.d"; } + +reset +cat > "$WORK/apt/sources.list" <<'EOF' +## See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade +deb http://ports.ubuntu.com/ubuntu-ports jammy main restricted +deb http://azure.ports.ubuntu.com/ubuntu-ports jammy-updates main +deb http://ports.ubuntu.com/ubuntu-ports jammy-security main +EOF +run || fail "arm64 sources errored" +grep -qx "deb https://ports.ubuntu.com/ubuntu-ports jammy main restricted" "$WORK/apt/sources.list" || fail "ports not switched" +grep -qx "deb https://ports.ubuntu.com/ubuntu-ports jammy-updates main" "$WORK/apt/sources.list" || fail "azure.ports not mapped to ports over https" +grep -qx "## See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade" "$WORK/apt/sources.list" || fail "comment rewritten" +ok "arm64 one-line sources move to https://ports.ubuntu.com and comments are left alone" + +reset +cat > "$WORK/apt/sources.list.d/ubuntu.sources" <<'EOF' +Types: deb +URIs: http://azure.archive.ubuntu.com/ubuntu/ +Suites: noble noble-updates noble-backports noble-security +Components: main universe restricted multiverse +Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg +EOF +cat > "$WORK/apt/sources.list.d/microsoft-prod.list" <<'EOF' +deb [arch=amd64,arm64,armhf] https://packages.microsoft.com/ubuntu/24.04/prod noble main +EOF +cp "$WORK/apt/sources.list.d/microsoft-prod.list" "$WORK/ms.before" +run || fail "amd64 sources errored" +grep -qx "URIs: https://archive.ubuntu.com/ubuntu/" "$WORK/apt/sources.list.d/ubuntu.sources" || fail "azure.archive not mapped to https://archive.ubuntu.com" +cmp -s "$WORK/ms.before" "$WORK/apt/sources.list.d/microsoft-prod.list" || fail "third-party repo touched" +ok "amd64 deb822 Azure mirror moves to https://archive.ubuntu.com and other repos are untouched" + +cp "$WORK/apt/sources.list.d/ubuntu.sources" "$WORK/ubuntu.before" +run || fail "second run errored" +cmp -s "$WORK/ubuntu.before" "$WORK/apt/sources.list.d/ubuntu.sources" || fail "second run changed the file" +! grep -q "switched" "$WORK/out" || fail "second run reported a switch" +ok "running twice is a no-op" + +reset +printf 'deb http://security.ubuntu.com/ubuntu noble-security main\n' > "$WORK/apt/sources.list" +run || fail "security sources errored" +grep -qx "deb https://security.ubuntu.com/ubuntu noble-security main" "$WORK/apt/sources.list" || fail "security not switched" +ok "security.ubuntu.com stays on its host over https" + +reset +printf 'deb http://us.archive.ubuntu.com/ubuntu noble main\n' > "$WORK/apt/sources.list" +if run; then fail "accepted an unmapped http ubuntu mirror"; fi +grep -q "us.archive.ubuntu.com" "$WORK/out" || fail "error does not name the leftover source" +ok "an unmapped http:// ubuntu.com mirror fails loudly" + +rm -rf "$WORK/apt" && mkdir -p "$WORK/apt" +run || fail "empty apt root errored" +ok "no apt sources is a no-op" + +echo "$passed passed" diff --git a/.github/workflows/scripts/check-egress-allowlist.sh b/.github/workflows/scripts/check-egress-allowlist.sh index 7b35427ba9b..19e8e56e19a 100755 --- a/.github/workflows/scripts/check-egress-allowlist.sh +++ b/.github/workflows/scripts/check-egress-allowlist.sh @@ -39,18 +39,28 @@ python3 - "${FILES[@]}" <<'PY' import sys import yaml -# Hosts apt-get needs on a GitHub-hosted ubuntu runner. azure.archive.ubuntu.com -# is the Azure-local mirror the runner images point at (plain HTTP), and +# Hosts apt-get needs on a GitHub-hosted ubuntu runner once apt-sources-https.sh +# has moved the sources to HTTPS. The images point at the Azure-local mirror over +# plain HTTP, which serves no HTTPS, so the script swaps it for archive.ubuntu.com; # esm.ubuntu.com serves the Expanded Security Maintenance lists apt refreshes # alongside it. -REQUIRED_APT_HOSTS = ("azure.archive.ubuntu.com:80", "esm.ubuntu.com:443") +REQUIRED_APT_HOSTS = ("archive.ubuntu.com:443", "esm.ubuntu.com:443") # arm64 Ubuntu pulls from the ports mirror (ubuntu-ports), never the archive one; # the arm-64-2cpu-8gb image's apt sources point straight at ports.ubuntu.com. -REQUIRED_ARM64_APT_HOSTS = ("ports.ubuntu.com:80",) +REQUIRED_ARM64_APT_HOSTS = ("ports.ubuntu.com:443",) # install-arm64-runner-deps.sh runs apt-get itself, so a step calling it never # shows "apt-get" here; its `docker` component also adds Docker's apt repo. ARM64_INSTALLER = "install-arm64-runner-deps.sh" -DOCKER_APT_HOST = "download.docker.com:443" +# Installers that run apt-get themselves, and switch the sources to HTTPS first. +APT_INSTALLERS = (ARM64_INSTALLER, "install-cross-compilers.sh") +APT_SOURCES_HTTPS = "apt-sources-https.sh" +# Docker's apt repo, plus apt's SRV probe for it, and the sigstore TUF root that +# the freshly installed dockerd fetches when buildx bootstraps its builder. +DOCKER_HOSTS = ( + "download.docker.com:443", + "_https._tcp.download.docker.com:443", + "tuf-repo-cdn.sigstore.dev:443", +) failures = [] checked = 0 @@ -65,7 +75,7 @@ for path in sys.argv[1:]: steps = job.get("steps") or [] runs = [step.get("run") or "" for step in steps if isinstance(step, dict)] - uses_apt = any("apt-get" in run or ARM64_INSTALLER in run for run in runs) + uses_apt = any("apt-get" in run or any(i in run for i in APT_INSTALLERS) for run in runs) if not uses_apt: continue installs_docker = any( @@ -92,17 +102,27 @@ for path in sys.argv[1:]: checked += 1 allowed = set((with_block.get("allowed-endpoints") or "").split()) + # An inline apt-get update must switch the sources to HTTPS first, in the + # same step: otherwise apt reaches for the plain-HTTP mirror, which no + # allowlist carries any more, and fails as an opaque fetch timeout. + for run in runs: + update = run.find("apt-get update") + if update != -1 and run.find(APT_SOURCES_HTTPS) not in range(0, update): + failures.append((path, job_name, [f"{APT_SOURCES_HTTPS} before apt-get update"])) + plain = sorted(e for e in allowed if e.endswith("ubuntu.com:80")) + if plain: + failures.append((path, job_name, [f"no plain-HTTP apt host ({', '.join(plain)})"])) is_arm = "arm" in str(job.get("runs-on") or "").lower() required = REQUIRED_ARM64_APT_HOSTS if is_arm else REQUIRED_APT_HOSTS if installs_docker: - required += (DOCKER_APT_HOST,) + required += DOCKER_HOSTS missing = [host for host in required if host not in allowed] if missing: failures.append((path, job_name, missing)) for path, job_name, missing in failures: print(f"FAIL {path} :: job '{job_name}' runs apt-get under egress-policy: block " - f"but omits {', '.join(missing)}", file=sys.stderr) + f"but needs {', '.join(missing)}", file=sys.stderr) if failures: print(f"\n{len(failures)} of {checked} apt-using blocked-egress job(s) have an " diff --git a/.github/workflows/scripts/install-arm64-runner-deps.sh b/.github/workflows/scripts/install-arm64-runner-deps.sh index 6c25834556d..811d4de5310 100755 --- a/.github/workflows/scripts/install-arm64-runner-deps.sh +++ b/.github/workflows/scripts/install-arm64-runner-deps.sh @@ -9,8 +9,9 @@ set -euo pipefail # Node binary links against) are all missing. Each component is skipped when it # is already present, so on a full runner image this is a no-op. # -# Runs after harden-runner like every other step. Callers allowlist -# ports.ubuntu.com:80, plus download.docker.com:443 when they ask for `docker`; +# Runs after harden-runner like every other step. apt-sources-https.sh moves the +# Ubuntu sources to HTTPS before the first apt-get update, so callers allowlist +# ports.ubuntu.com:443, plus download.docker.com:443 when they ask for `docker`; # check-egress-allowlist.sh enforces both. Docker's apt repo is trusted only # through a key whose primary fingerprint matches DOCKER_KEY_FINGERPRINT, and # apt then verifies every package against that key. @@ -23,6 +24,8 @@ set -euo pipefail DOCKER_KEY_FINGERPRINT="9DC858229FC7DD38854AE2D88D81803C0EBFCD88" OS_RELEASE="${OS_RELEASE:-/etc/os-release}" +# ${BASH_SOURCE%/*}, not dirname: the installer test runs with a minimal PATH. +APT_SOURCES_HTTPS="${BASH_SOURCE%/*}/apt-sources-https.sh" if [[ $# -eq 0 ]]; then echo "Usage: $0 ..." >&2 @@ -31,6 +34,16 @@ fi has() { command -v "$1" >/dev/null 2>&1; } +# Both install paths call apt-get update; the sources switch to HTTPS once, before the first. +sources_https=0 +apt_update() { + if [[ $sources_https -eq 0 ]]; then + "$BASH" "$APT_SOURCES_HTTPS" + sources_https=1 + fi + sudo apt-get update +} + packages=() want_docker=0 for component in "$@"; do @@ -53,7 +66,7 @@ done if [[ ${#packages[@]} -gt 0 ]]; then echo "Installing from Ubuntu: ${packages[*]}" - sudo apt-get update + apt_update sudo apt-get install -y --no-install-recommends "${packages[@]}" fi @@ -75,7 +88,7 @@ if [[ $want_docker -eq 1 ]]; then sudo install -D -m 0644 "$KEY" /etc/apt/keyrings/docker.asc echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu ${codename} stable" \ | sudo tee /etc/apt/sources.list.d/docker.list >/dev/null - sudo apt-get update + apt_update sudo apt-get install -y --no-install-recommends \ docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin diff --git a/.github/workflows/scripts/install-arm64-runner-deps.test.sh b/.github/workflows/scripts/install-arm64-runner-deps.test.sh index 147c7d68638..899c99b4166 100755 --- a/.github/workflows/scripts/install-arm64-runner-deps.test.sh +++ b/.github/workflows/scripts/install-arm64-runner-deps.test.sh @@ -11,9 +11,13 @@ trap 'rm -rf "$WORK"' EXIT mkdir -p "$WORK/bin" "$WORK/base" # Only these real tools are visible, so the host's own jq/gcc/docker cannot leak in. -for tool in awk bash cat grep mktemp rm seq; do ln -s "$(command -v "$tool")" "$WORK/base/$tool"; done +for tool in awk bash cat cmp cp grep mktemp rm sed seq; do ln -s "$(command -v "$tool")" "$WORK/base/$tool"; done printf 'VERSION_CODENAME=jammy\n' > "$WORK/os-release" +# apt-sources-https.sh rewrites this fixture instead of /etc/apt, unprivileged so +# the sudo stub does not swallow the copy. +mkdir -p "$WORK/apt" +reset_apt() { printf 'deb http://ports.ubuntu.com/ubuntu-ports jammy main\n' > "$WORK/apt/sources.list"; } GOOD_FPR=9DC858229FC7DD38854AE2D88D81803C0EBFCD88 cat > "$WORK/base/sudo" <<'EOF' @@ -61,7 +65,9 @@ cat > "$WORK/present" <<'EOF' #!/usr/bin/env bash exit 0 EOF -chmod +x "$WORK/base/"* "$WORK/docker-full" "$WORK/docker-bare" "$WORK/present" +# Only the stubs: $WORK/base also links to real tools, and GNU chmod follows a +# link into a root-owned binary, so a glob fails for a non-root runner user. +chmod +x "$WORK/base/"{sudo,ldconfig,curl,gpg,dpkg} "$WORK/docker-full" "$WORK/docker-bare" "$WORK/present" passed=0 # run -- ... @@ -77,7 +83,8 @@ run() { done export TEST_LOG="$WORK/log" : > "$TEST_LOG" - PATH="$WORK/bin:$WORK/base" OS_RELEASE="$WORK/os-release" \ + reset_apt + PATH="$WORK/bin:$WORK/base" OS_RELEASE="$WORK/os-release" APT_ROOT="$WORK/apt" APT_SUDO="" \ "$BASH" "$SCRIPT_DIR/install-arm64-runner-deps.sh" "$@" > "$WORK/out" 2>&1 } fail() { cat "$WORK/out" "$TEST_LOG" >&2; echo "FAIL: $1" >&2; exit 1; } @@ -86,12 +93,14 @@ ok() { echo "ok - $1"; passed=$((passed + 1)); } export TEST_HAS_LIBATOMIC=1 TEST_KEY_PRIMARIES="$GOOD_FPR" run "docker jq gh make gcc g++" node docker build jq gh || fail "full image errored" [[ ! -s "$TEST_LOG" ]] || fail "full image still called sudo" +grep -qx "deb http://ports.ubuntu.com/ubuntu-ports jammy main" "$WORK/apt/sources.list" || fail "full image still rewrote apt sources" ok "a full runner image installs nothing" export TEST_HAS_LIBATOMIC=0 run "" node || fail "node errored" grep -qx "sudo apt-get install -y --no-install-recommends libatomic1" "$TEST_LOG" || fail "node did not install exactly libatomic1" -ok "node installs libatomic1 when ldconfig lacks it" +grep -qx "deb https://ports.ubuntu.com/ubuntu-ports jammy main" "$WORK/apt/sources.list" || fail "apt sources not switched to https before apt-get update" +ok "node installs libatomic1 when ldconfig lacks it, after moving apt to https" run "make gcc" build jq gh || fail "build errored" grep -qx "sudo apt-get install -y --no-install-recommends make gcc g++ libc6-dev jq gh" "$TEST_LOG" || fail "build/jq/gh package list wrong" diff --git a/.github/workflows/scripts/install-cross-compilers.sh b/.github/workflows/scripts/install-cross-compilers.sh index d5cac53d398..fa844065865 100755 --- a/.github/workflows/scripts/install-cross-compilers.sh +++ b/.github/workflows/scripts/install-cross-compilers.sh @@ -7,6 +7,7 @@ set -euo pipefail echo "๐Ÿ“ฆ Installing cross-compilation toolchains for Go + CGO..." # Install all required packages +"$(dirname "${BASH_SOURCE[0]}")/apt-sources-https.sh" sudo apt-get update sudo apt-get install -y \ gcc-x86-64-linux-gnu \ diff --git a/.github/workflows/scripts/test-api-integrations.sh b/.github/workflows/scripts/test-api-integrations.sh index ae34e03fb84..8d28a08666c 100755 --- a/.github/workflows/scripts/test-api-integrations.sh +++ b/.github/workflows/scripts/test-api-integrations.sh @@ -4,7 +4,7 @@ set -euo pipefail # API integrations test: compiles bifrost-http, runs it against PostgreSQL using # tests/config.json (with a runtime config_store/logs_store overlay), then runs # the api-management newman collection via tests/e2e/api/runners/run-newman-api-tests.sh. -# The Warp, routing-wiring and model-catalog-wiring collections follow, each on a +# The Warp (unless SKIP_WARP=1), routing-wiring and model-catalog-wiring collections follow, each on a # server of its own. if command -v readlink >/dev/null 2>&1 && readlink -f "$0" >/dev/null 2>&1; then @@ -219,10 +219,13 @@ echo "๐Ÿงช Running api-management newman collection..." # Warp boots its own server on its own database (bifrost_warp_e2e on the same # Postgres), because its answers cover every row in the logs table and the # api-management run above has just written to this server's. It reuses the -# compose stack's Weaviate and the job's OPENAI_API_KEY. -echo "" -echo "๐Ÿงช Running Warp newman collection..." -"$REPO_ROOT/tests/e2e/api/runners/individual/run-newman-warp-tests.sh" --binary "$BIFROST_BINARY" --port "$((PORT + 8))" $REPORT_ARGS +# compose stack's Weaviate and the job's OPENAI_API_KEY. CI sets SKIP_WARP=1 +# and runs it as the parallel test-warp job instead (test-warp.sh). +if [ "${SKIP_WARP:-0}" != "1" ]; then + echo "" + echo "๐Ÿงช Running Warp newman collection..." + "$REPO_ROOT/tests/e2e/api/runners/individual/run-newman-warp-tests.sh" --binary "$BIFROST_BINARY" --port "$((PORT + 8))" $REPORT_ARGS +fi # The wiring collections each boot a clean server (no providers, sqlite stores): # routing-wiring creates the global standard providers (openai, azure, bedrock, diff --git a/.github/workflows/scripts/test-warp.sh b/.github/workflows/scripts/test-warp.sh new file mode 100755 index 00000000000..059e932ddfa --- /dev/null +++ b/.github/workflows/scripts/test-warp.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Warp e2e test: brings up the compose stack's Postgres and Weaviate and runs +# tests/e2e/api/runners/individual/run-newman-warp-tests.sh against the prebuilt +# bifrost-http binary. CI runs this as its own job, beside test-api-integrations +# (which skips Warp under SKIP_WARP=1): the collection is a live, serial LLM +# evaluation of about 25 minutes, so chaining it after the other suites doubled +# that job's wall clock. + +if command -v readlink >/dev/null 2>&1 && readlink -f "$0" >/dev/null 2>&1; then + SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" +else + SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" +fi +REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd -P)" + +COMPOSE_FILE="$REPO_ROOT/.github/workflows/configs/docker-compose.yml" +BIFROST_BINARY="$REPO_ROOT/tmp/bifrost-http" +WARP_RUNNER="$REPO_ROOT/tests/e2e/api/runners/individual/run-newman-warp-tests.sh" +PORT="${PORT:-8088}" + +if [ ! -x "$BIFROST_BINARY" ]; then + echo "โŒ No executable bifrost-http binary at $BIFROST_BINARY" >&2 + exit 1 +fi +if [ -z "${OPENAI_API_KEY:-}" ] && [ -z "${WARP_UPSTREAM_BIFROST:-}" ]; then + echo "โŒ OPENAI_API_KEY (or WARP_UPSTREAM_BIFROST) is required: Warp's agent and embeddings call OpenAI" >&2 + exit 1 +fi + +# The runner builds tests/cmd/seed/warpseed inside the workspace. +cd "$REPO_ROOT" +# shellcheck source=/dev/null +source "$SCRIPT_DIR/setup-go-workspace.sh" + +cleanup() { + local exit_code=$? + echo "๐Ÿงน Stopping Docker services..." + docker compose -f "$COMPOSE_FILE" down 2>/dev/null || true + exit $exit_code +} +trap cleanup EXIT + +# Only what Warp reads: Postgres for its config and logs stores, Weaviate for the +# log index and topics. Redis and Qdrant serve other suites. +echo "๐Ÿณ Starting Postgres and Weaviate..." +docker compose -f "$COMPOSE_FILE" up -d postgres weaviate + +echo "โณ Waiting for Postgres..." +ELAPSED=0 +until [ "$(docker inspect -f '{{.State.Health.Status}}' "$(docker compose -f "$COMPOSE_FILE" ps -q postgres)" 2>/dev/null)" = "healthy" ]; do + if [ $ELAPSED -ge 120 ]; then + echo "โŒ Postgres did not become healthy within 120s" + docker compose -f "$COMPOSE_FILE" ps + exit 1 + fi + sleep 2 + ELAPSED=$((ELAPSED + 2)) +done +echo "โœ… Postgres ready (${ELAPSED}s)" + +echo "โณ Waiting for Weaviate..." +ELAPSED=0 +until curl -fsS -o /dev/null "http://localhost:9000/v1/.well-known/ready"; do + if [ $ELAPSED -ge 120 ]; then + echo "โŒ Weaviate failed readiness check within 120s" + docker compose -f "$COMPOSE_FILE" logs weaviate | tail -50 + exit 1 + fi + sleep 2 + ELAPSED=$((ELAPSED + 2)) +done +echo "โœ… Weaviate ready (${ELAPSED}s)" + +# --json, not --html: the html reporter is not installed, and the JSON report +# keeps every response body, tool-call arguments included, for diagnosing misses. +echo "" +echo "๐Ÿงช Running Warp newman collection..." +"$WARP_RUNNER" --binary "$BIFROST_BINARY" --port "$PORT" --json diff --git a/.github/workflows/workflow-lint.yml b/.github/workflows/workflow-lint.yml index e6485f8f342..74fe0262b94 100644 --- a/.github/workflows/workflow-lint.yml +++ b/.github/workflows/workflow-lint.yml @@ -68,3 +68,6 @@ jobs: - name: Check arm64 runner dependency installer run: bash ./.github/workflows/scripts/install-arm64-runner-deps.test.sh + + - name: Check apt HTTPS source rewriter + run: bash ./.github/workflows/scripts/apt-sources-https.test.sh diff --git a/framework/warp/topics.go b/framework/warp/topics.go index d1cec31df3d..e20b9794f11 100644 --- a/framework/warp/topics.go +++ b/framework/warp/topics.go @@ -853,7 +853,7 @@ func listTopicsTool() Tool { "The window defaults to the last 7 days; filters narrow it to a team, customer, user, virtual key, model, provider, app, status or session, like the other tools. " + "Each topic carries requests, share, distinct sessions, errors, cost, tokens, average latency, a trend block against the period of equal length before the window (has_previous_period, requests_trend, tokens_trend, cost_trend), when it was first seen (new_in_window marks one that appeared inside the window), " + "and example_links that each open one example request. With interval, each topic also carries its series, bucket by bucket. " + - "With group_by theme, the answer is the themes the topics are grouped into - broad subjects such as Billing over the topics Refund Requests and Invoice Questions - each with the same counts added up over its topics, and the topics inside it; the call for 'what broad areas', 'what kinds of things', or a deployment with too many topics to list. " + + "With group_by theme, the answer is the themes the topics are grouped into - broad subjects such as Billing over the topics Refund Requests and Invoice Questions - each with the same counts added up over its topics, and the topics inside it by name, requests, share and link (group_by topic carries a topic's full counts); the call for 'what broad areas', 'what kinds of things', or a deployment with too many topics to list. " + "Requests, shares and sessions count requests and sessions, not people: every model call is one request, so a five-turn chat is five.", schemaJSON: `{ "type": "object", @@ -912,9 +912,12 @@ func listTopicsTool() Tool { narrowed.TopicIDs = []string{id} return logsViewLink(&narrowed) } + // The example ids already travel inside example_links; sending both + // cost a fifth of every row against MaxToolResultBytes. linkTopics := func(topics []Topic) { for index := range topics { topics[index].Link = linkTo(topics[index].ID) + topics[index].SampleLogIDs = nil } } linkTopics(list.Topics) @@ -931,7 +934,14 @@ func listTopicsTool() Tool { "scope": scopeNote(filters, deps.scope), } if query.ByTheme { - out["themes"] = list.Themes + themes := make([]listedTheme, len(list.Themes)) + for index, theme := range list.Themes { + themes[index] = listedTheme{TopicTheme: theme, Topics: make([]listedThemeTopic, len(theme.Topics))} + for inner, topic := range theme.Topics { + themes[index].Topics[inner] = listedThemeTopic{ID: topic.ID, Label: topic.Label, Requests: topic.Requests, Share: topic.Share, New: topic.New, Link: topic.Link} + } + } + out["themes"] = themes out["returned"] = len(list.Themes) out["total_themes"] = list.MatchedThemes out["ungrouped_topics"] = list.Ungrouped @@ -960,11 +970,67 @@ func listTopicsTool() Tool { out["cannot_answer"] = "Not available yet: how many people asked about a topic (sessions are sessions, not people; a query_usage_by user ranking with topic_ids counts the users with a request in it). " + "Asked for that, answer what the topics do show, then say it is not available yet." } + fitTopicResult(out) return out, nil }, } } +// listedTheme is a theme as list_topics answers it: the theme's own counts, +// with the topics inside it named, sized and linked. Every inner topic in full +// multiplied a deployment's topics into the answer and pushed it past +// MaxToolResultBytes; group_by topic carries each topic's full counts. +type listedTheme struct { + TopicTheme + Topics []listedThemeTopic `json:"topics"` +} + +// listedThemeTopic is one topic inside a listedTheme. +type listedThemeTopic struct { + ID string `json:"id"` + Label string `json:"label"` + Requests int64 `json:"requests"` + Share float64 `json:"share"` + New bool `json:"new_in_window,omitempty"` + Link string `json:"link,omitempty"` +} + +// fitTopicResult drops the smallest rows of a list_topics answer until it fits +// MaxToolResultBytes, and says how many it dropped. Over budget, +// boundToolResult discards the whole answer and the model retries at ever +// smaller limits, so a long tail cut here is one call instead of several. +// Ungrouped topics go before themes: the themes are the grouped answer. +func fitTopicResult(out map[string]any) { + dropped := 0 + for { + encoded, err := sonic.MarshalString(out) + if err != nil || len(encoded) <= MaxToolResultBytes { + break + } + if topics, ok := out["topics"].([]Topic); ok && len(topics) > 1 { + out["topics"] = topics[:len(topics)-1] + } else if ungrouped, ok := out["ungrouped_topics"].([]Topic); ok && len(ungrouped) > 0 { + out["ungrouped_topics"] = ungrouped[:len(ungrouped)-1] + } else if themes, ok := out["themes"].([]listedTheme); ok && len(themes) > 1 { + out["themes"] = themes[:len(themes)-1] + } else { + break + } + dropped++ + } + if dropped == 0 { + return + } + if topics, ok := out["topics"].([]Topic); ok { + out["returned"] = len(topics) + } + if themes, ok := out["themes"].([]listedTheme); ok { + out["returned"] = len(themes) + } + out["trimmed"] = fmt.Sprintf("%d of the smallest rows did not fit in one answer and were left out; returned counts what is here. "+ + "Answer from these, and say more exist; narrow the window or filters to see the rest.", dropped) +} + // RegisterTopics binds the clustering, discovery and upkeep jobs to the shared // Sidekiq runner. Only clustering pages vectors out of the vector store; // discovery and upkeep read the log store, so they are registered without one. diff --git a/framework/warp/topics_test.go b/framework/warp/topics_test.go index dd57833988c..cce54188092 100644 --- a/framework/warp/topics_test.go +++ b/framework/warp/topics_test.go @@ -1852,7 +1852,7 @@ func TestWarpListTopicsToolGroupsByTheme(t *testing.T) { out := result.(map[string]any) _, flat := out["topics"] require.False(t, flat, "grouped, the topics come inside their themes") - themes := out["themes"].([]TopicTheme) + themes := out["themes"].([]listedTheme) require.Len(t, themes, 2) require.Equal(t, 2, out["total_themes"]) require.Equal(t, 2, out["returned"]) @@ -1868,9 +1868,14 @@ func TestWarpListTopicsToolGroupsByTheme(t *testing.T) { require.Contains(t, billing.Link, "topic_ids=money") require.Len(t, billing.Topics, 2) require.Equal(t, "Refund requests", billing.Topics[0].Label) - require.Equal(t, "Billing", billing.Topics[0].Theme) - require.Equal(t, "money", billing.Topics[0].ThemeID) require.Contains(t, billing.Topics[0].Link, "topic_ids=refunds") + // Inside a theme a topic is named, sized and linked only: its full counts + // and examples are one group_by topic call away, and repeating them for + // every topic is what pushed the answer past MaxToolResultBytes. + encodedThemes, err := sonic.MarshalString(themes) + require.NoError(t, err) + require.NotContains(t, encodedThemes, "example_links") + require.Contains(t, encodedThemes, `"topics":[{"id":"refunds","label":"Refund requests","requests":6,`) require.EqualValues(t, 5, billing.Topics[1].Requests, "3 of its own and 2 merged in") var inSeries int64 for _, point := range billing.Series { @@ -1925,6 +1930,84 @@ func TestWarpListTopicsToolGroupsByTheme(t *testing.T) { require.Contains(t, out["hint"], "not grouped into themes") } +// The concrete failure this pins: at the Warp e2e deployment's size (31 +// topics in 6 themes, UUID ids, 4 examples each) list_topics with no limit +// serialized past MaxToolResultBytes, so boundToolResult discarded it and told +// the model to lower the limit. It retried at 15, 8, 2, 1 - one question, up to +// seven calls - on the overview, time-range, trend and themes questions alike. +// The defaults have to fit, or "one call answers it" is never true. +func TestWarpListTopicsToolDefaultsFitTheResultBudget(t *testing.T) { + store := topicTestLogStore(t) + ctx := context.Background() + now := time.Date(2026, 9, 5, 12, 0, 0, 0, time.UTC) + labels := []string{ + "Train Travel Time Calculations", "Password Reset Emails", "Shipping Delay Customer Updates", "NDA Confidentiality Clause", + "Resume Bullet Improvement", "Assistant Time Zone Questions", "Subscription Renewal Translations", "Customer Feedback Analysis", + "Lighthouse Keeper Mystery Story", "Refund Policy Explanations", "Invoice Dispute Replies", "SQL Query Optimisation", + "Kubernetes Pod Crash Debugging", "Python Type Error Fixes", "Marketing Email Subject Lines", "Product Description Rewrites", + "Meeting Notes Summaries", "Quarterly Report Drafting", "Recipe Scaling Conversions", "Travel Itinerary Planning", + "Contract Termination Clauses", "GDPR Data Request Replies", "Onboarding Checklist Drafts", "API Rate Limit Errors", + "React Hook Dependency Bugs", "Interview Question Prep", "Cover Letter Drafting", "Budget Spreadsheet Formulas", + "Social Media Captions", "Bug Report Triage", "Release Notes Writing", + } + var topics []logstore.WarpTopic + var themeIDs []string + for index := range 6 { + id := uuid.NewString() + themeIDs = append(themeIDs, id) + topics = append(topics, logstore.WarpTopic{ID: id, Label: fmt.Sprintf("Broad Area Number %d", index), Kind: logstore.WarpTopicKindTheme, EmbeddingSignature: topicTestSignature()}) + } + var rows []logstore.WarpTopicAssignment + for index, label := range labels { + id := uuid.NewString() + examples := []string{uuid.NewString(), uuid.NewString(), uuid.NewString(), uuid.NewString()} + encoded, err := sonic.MarshalString(examples) + require.NoError(t, err) + topics = append(topics, logstore.WarpTopic{ + ID: id, Label: label, ParentID: themeIDs[index%len(themeIDs)], Cohesion: 0.71234, + ExampleLogIDsJSON: encoded, EmbeddingSignature: topicTestSignature(), CreatedAt: now.Add(-20 * 24 * time.Hour), + }) + for _, logID := range examples { + rows = append(rows, logstore.WarpTopicAssignment{LogID: logID, TopicID: id, Timestamp: now.Add(-time.Duration(index+1) * time.Hour), SessionID: uuid.NewString(), Cost: 0.0123, TotalTokens: 1234, Latency: 2345}) + } + rows = append(rows, logstore.WarpTopicAssignment{LogID: uuid.NewString(), TopicID: id, Timestamp: now.Add(-10 * 24 * time.Hour), Cost: 0.0123, TotalTokens: 1234, Latency: 2345}) + } + require.NoError(t, store.UpsertWarpTopics(ctx, topics)) + require.NoError(t, store.UpsertWarpTopicAssignments(ctx, rows)) + + for _, args := range []map[string]any{ + {"filters": map[string]any{"start_time": "-7d", "scope": "all"}}, + {"filters": map[string]any{"start_time": "-24h", "scope": "all"}}, + {"filters": map[string]any{"start_time": "-7d", "scope": "all"}, "group_by": "theme"}, + } { + result, err := runTool(t, "list_topics", topicToolDeps(store, now), args) + require.NoError(t, err) + bound := boundToolResult(result) + require.NotContains(t, bound, "result too large", "args %v: the default answer must reach the model", args) + out := result.(map[string]any) + _, trimmed := out["trimmed"] + require.False(t, trimmed, "args %v: the defaults fit whole, not by cutting rows", args) + if themes, ok := out["themes"].([]listedTheme); ok { + require.Len(t, themes, 6, "every theme, each with its topics") + } else { + require.Len(t, out["topics"], 15, "the default limit") + } + } + + // Past what fits, the smallest rows are cut and the answer says so, rather + // than the whole answer being discarded. + result, err := runTool(t, "list_topics", topicToolDeps(store, now), map[string]any{"filters": map[string]any{"start_time": "-7d", "scope": "all"}, "limit": float64(31)}) + require.NoError(t, err) + out := result.(map[string]any) + require.NotContains(t, boundToolResult(result), "result too large") + topicsListed := out["topics"].([]Topic) + require.Less(t, len(topicsListed), 31) + require.Greater(t, len(topicsListed), 15) + require.Equal(t, len(topicsListed), out["returned"]) + require.Equal(t, 31, out["total_topics"]) + require.Contains(t, out["trimmed"], "did not fit in one answer") +} + func TestWarpListTopicsToolHintsWhenNothingComputed(t *testing.T) { now := time.Date(2026, 9, 5, 12, 0, 0, 0, time.UTC) store := topicTestLogStore(t) @@ -2037,7 +2120,7 @@ func TestWarpListTopicsServesARestrictedCallerThroughAStoreThatScopesItsReads(t require.Len(t, topics, 1) require.Equal(t, "Refund requests", topics[0].Label) require.EqualValues(t, 1, topics[0].Requests) - require.Equal(t, []string{"ours-1"}, topics[0].SampleLogIDs) + require.Equal(t, []string{logDetailLink("ours-1")}, topics[0].SampleLinks) require.Equal(t, int64(1), out["total_requests"]) } @@ -2106,8 +2189,8 @@ func TestWarpListTopicsAnswersARestrictedCallerFromTheirOwnRequests(t *testing.T require.Equal(t, "Refund requests", topics[0].Label) require.EqualValues(t, 2, topics[0].Requests) require.InDelta(t, 2.0/3, topics[0].Share, 1e-9, "of the three requests they may see") - require.Equal(t, []string{"ours-1"}, topics[0].SampleLogIDs, "the example from another team is not offered") - require.Equal(t, []string{logDetailLink("ours-1")}, topics[0].SampleLinks) + require.Equal(t, []string{logDetailLink("ours-1")}, topics[0].SampleLinks, "the example from another team is not offered") + require.Nil(t, topics[0].SampleLogIDs, "the ids travel inside example_links") require.Equal(t, int64(3), out["total_requests"]) require.Equal(t, int64(1), out["unmatched_requests"]) require.Equal(t, "all", out["scope"]) diff --git a/tests/e2e/api/collections/bifrost-v1-warp.postman_collection.json b/tests/e2e/api/collections/bifrost-v1-warp.postman_collection.json index ed892ada30f..91fea1cc586 100644 --- a/tests/e2e/api/collections/bifrost-v1-warp.postman_collection.json +++ b/tests/e2e/api/collections/bifrost-v1-warp.postman_collection.json @@ -28,7 +28,7 @@ }, { "key": "warp_lib", - "value": "function createWarpLib(pm) {\n var STATE_KEY = \"__warp_state\";\n var MAX_RETRIES = 1;\n var MAX_QUESTIONS = 2;\n // Tools that only discover what exists. A reply backed by these alone rests\n // on no data - a refusal should call nothing else.\n var DISCOVERY_TOOLS = [\"describe_filter_space\", \"ask_user\"];\n // Options the driver prefers when Warp asks, most specific first: whole\n // deployment for scope (nobody is identified, so there is no default), then a\n // week for the window, which is what every seeded fact is measured over. The\n // rolling week is tried on its own first: asked \"calendar week or last 7\n // days?\", the seeded facts are the rolling ones whichever option is listed first.\n var DEFAULT_PREFER = [\n \"whole deployment|entire deployment|all traffic|everyone|all teams|everything\",\n \"-7d|7 days|rolling\",\n \"this week|last week\",\n \"-24h|24 hours\",\n ];\n\n function readState() {\n var raw = pm.collectionVariables.get(STATE_KEY);\n if (!raw) return null;\n try {\n return JSON.parse(raw);\n } catch (e) {\n return null;\n }\n }\n\n function saveState(state) {\n pm.collectionVariables.set(STATE_KEY, state ? JSON.stringify(state) : \"\");\n }\n\n function chainKey(name) {\n return \"__warp_chain_\" + name;\n }\n\n function readChain(name) {\n var raw = pm.collectionVariables.get(chainKey(name));\n if (!raw) return null;\n try {\n return JSON.parse(raw);\n } catch (e) {\n return null;\n }\n }\n\n function freshState(c, attempt) {\n var messages = [];\n var conversationId = pm.variables.replaceIn(\"{{$guid}}\");\n if (c.chain && !c.chainStart) {\n var chain = readChain(c.chain);\n if (chain) {\n messages = chain.messages.slice();\n conversationId = chain.conversation_id;\n }\n }\n if (c.history) {\n for (var i = 0; i < c.history.length; i++) {\n var turn = c.history[i];\n var copy = { role: turn.role, content: pm.variables.replaceIn(turn.content) };\n if (turn.question) copy.question = true;\n messages.push(copy);\n }\n }\n messages.push({ role: \"user\", content: pm.variables.replaceIn(c.ask) });\n return {\n name: pm.info.requestName,\n attempt: attempt,\n asked: 0,\n messages: messages,\n conversation_id: conversationId,\n };\n }\n\n // pickAnswer replies to an ask_user question the way a person clicking the\n // dashboard would: the option's hint when it has one, else its label. An\n // option the person's own question already named wins - asked \"what did\n // Platform Engineering spend\", nobody then picks \"Whole deployment\". With no\n // whole-deployment option on a scope question, they type it instead, as the\n // question's free-text escape allows, rather than taking whatever came first.\n function pickAnswer(question, prefer, asked) {\n var options = (question && question.options) || [];\n var said = String(asked || \"\").toLowerCase();\n for (var n = 0; n < options.length; n++) {\n var label = String(options[n].label || \"\").toLowerCase();\n if (label.length >= 3 && said.indexOf(label) >= 0) return options[n].hint || options[n].label;\n }\n var patterns = (prefer || []).concat(DEFAULT_PREFER);\n for (var p = 0; p < patterns.length; p++) {\n var re = new RegExp(patterns[p], \"i\");\n for (var i = 0; i < options.length; i++) {\n var o = options[i];\n if (re.test(o.label || \"\") || re.test(o.hint || \"\")) return o.hint || o.label;\n }\n }\n var scopeQuestion = (question && question.kind === \"scope\") || /whose|scope/i.test((question && question.question) || \"\");\n if (scopeQuestion && (!options.length || question.allow_other)) return \"The whole deployment.\";\n if (options.length) return options[0].hint || options[0].label;\n return \"The whole deployment, over the last 7 days.\";\n }\n\n // Patterns may name run-specific values ({{warp_slowest_id}}), resolved here\n // rather than at build time since they only exist once the seeder has run.\n function pattern(p) {\n return new RegExp(pm.variables.replaceIn(p), \"i\");\n }\n\n function stripCode(text) {\n return String(text || \"\")\n .replace(/```[\\s\\S]*?```/g, \"\")\n .replace(/`[^`]*`/g, \"\");\n }\n\n // evaluate returns the list of expectations a JSON-mode response missed.\n function evaluate(body, expect) {\n var e = expect || {};\n var failures = [];\n if (body.error) {\n failures.push(\"error \" + body.error.code + \": \" + body.error.message);\n return failures;\n }\n var calls = (body.tool_calls || []).filter(function (t) {\n return !t.failed;\n });\n var names = calls.map(function (t) {\n return t.name;\n });\n\n if (e.question === \"required\") {\n if (!body.question) failures.push(\"expected an ask_user question, got an answer\");\n else if (!body.question.options || body.question.options.length < 2)\n failures.push(\"question carried fewer than 2 options\");\n if (body.finish_reason !== \"question\") failures.push(\"finish_reason \" + body.finish_reason + \", want question\");\n return failures;\n }\n if (e.question === \"forbidden\" && body.question) failures.push(\"asked a question: \" + body.question.question);\n\n var finish = e.finish || [\"stop\", \"partial\"];\n if (finish.indexOf(body.finish_reason) < 0) failures.push(\"finish_reason \" + body.finish_reason + \", want one of \" + finish.join(\"/\"));\n\n // A \"required\" question case has already returned above; every case that\n // reaches here, \"forbidden\" included, must have said something.\n var answer = String(body.answer || \"\");\n if (!answer.trim()) failures.push(\"empty answer\");\n\n if (typeof e.maxToolCalls === \"number\" && (body.tool_calls || []).length > e.maxToolCalls)\n failures.push(\"made \" + (body.tool_calls || []).length + \" tool calls, want at most \" + e.maxToolCalls);\n if (e.toolsAny && !e.toolsAny.some(function (t) { return names.indexOf(t) >= 0; }))\n failures.push(\"called none of \" + e.toolsAny.join(\", \") + \" (called: \" + (names.join(\", \") || \"nothing\") + \")\");\n (e.toolsAll || []).forEach(function (t) {\n if (names.indexOf(t) < 0) failures.push(\"did not call \" + t + \" (called: \" + (names.join(\", \") || \"nothing\") + \")\");\n });\n if (e.toolsOnly) {\n // Every attempt, failed ones included: a forbidden tool the model reached\n // for is the behaviour this rules out, whether or not the call worked.\n var attempted = (body.tool_calls || []).map(function (t) { return t.name; });\n var others = attempted.filter(function (n) { return e.toolsOnly.indexOf(n) < 0; });\n if (others.length) failures.push(\"called \" + others.join(\", \") + \", which this question has no use for (allowed: \" + e.toolsOnly.join(\", \") + \")\");\n }\n if (e.noDataTools) {\n var data = names.filter(function (n) { return DISCOVERY_TOOLS.indexOf(n) < 0; });\n if (data.length) failures.push(\"queried data for an out-of-scope question: \" + data.join(\", \"));\n }\n (e.argsMatch || []).forEach(function (m) {\n var re = pattern(m.pattern);\n var hit = calls.some(function (t) {\n return (!m.tool || t.name === m.tool) && re.test(t.arguments || \"\");\n });\n if (!hit) failures.push(\"no \" + (m.tool || \"tool\") + \" call with arguments matching /\" + m.pattern + \"/\");\n });\n // A filter the question never asked for narrows the answer without saying so.\n (e.argsNone || []).forEach(function (m) {\n var re = pattern(m.pattern);\n calls.forEach(function (t) {\n if ((!m.tool || t.name === m.tool) && re.test(t.arguments || \"\"))\n failures.push(t.name + \" was called with arguments matching /\" + m.pattern + \"/: \" + t.arguments);\n });\n });\n\n (e.answerAll || []).forEach(function (p) {\n if (!pattern(p).test(answer)) failures.push(\"answer lacks /\" + p + \"/\");\n });\n if (e.answerAny && !e.answerAny.some(function (p) { return pattern(p).test(answer); }))\n failures.push(\"answer matches none of \" + e.answerAny.map(function (p) { return \"/\" + p + \"/\"; }).join(\" \"));\n (e.answerNone || []).forEach(function (p) {\n if (pattern(p).test(answer)) failures.push(\"answer contains /\" + p + \"/\");\n });\n if (e.answerNoneOutsideCode) {\n var prose = stripCode(answer);\n e.answerNoneOutsideCode.forEach(function (p) {\n if (pattern(p).test(prose)) failures.push(\"answer repeats /\" + p + \"/ outside inline code\");\n });\n }\n if (e.maxAnswerChars && answer.length > e.maxAnswerChars)\n failures.push(\"answer is \" + answer.length + \" chars, want at most \" + e.maxAnswerChars);\n return failures;\n }\n\n function prerequest(c) {\n var state = readState();\n if (!state || state.name !== pm.info.requestName) {\n state = freshState(c, 0);\n saveState(state);\n }\n pm.request.body.update({\n mode: \"raw\",\n raw: JSON.stringify({\n messages: state.messages,\n conversation_id: state.conversation_id,\n stream: false,\n timezone: \"UTC\",\n }),\n options: { raw: { language: \"json\" } },\n });\n }\n\n function summary(body) {\n if (!body) return \"\";\n var tools = (body.tool_calls || []).map(function (t) {\n return t.name + (t.failed ? \"(failed)\" : \"\");\n });\n return \" | tools: [\" + tools.join(\", \") + \"] | finish: \" + body.finish_reason + \" | answer: \" + String(body.answer || \"\").slice(0, 600);\n }\n\n function test(c) {\n var state = readState() || freshState(c, 0);\n var body = null;\n var failures = [];\n if (pm.response.code !== 200) {\n failures.push(\"status \" + pm.response.code + \": \" + pm.response.text().slice(0, 500));\n } else {\n try {\n body = pm.response.json();\n } catch (err) {\n failures.push(\"response is not JSON: \" + pm.response.text().slice(0, 300));\n }\n }\n\n if (body && body.question && (c.expect || {}).question !== \"required\" && (c.expect || {}).question !== \"forbidden\" && state.asked < MAX_QUESTIONS) {\n state.messages.push({ role: \"assistant\", content: body.question.question, question: true });\n state.messages.push({ role: \"user\", content: pickAnswer(body.question, c.prefer, pm.variables.replaceIn(c.ask)) });\n state.asked += 1;\n saveState(state);\n pm.execution.setNextRequest(pm.info.requestName);\n return;\n }\n\n if (body) failures = failures.concat(evaluate(body, c.expect));\n // An answer that was never preceded by the question the case exists to see.\n var minQuestions = (c.expect || {}).minQuestions;\n if (body && typeof minQuestions === \"number\" && state.asked < minQuestions)\n failures.push(\"asked \" + state.asked + \" question(s) before answering, want at least \" + minQuestions);\n\n if (failures.length && state.attempt < MAX_RETRIES) {\n console.log(\"RETRY \" + pm.info.requestName + \": \" + failures.join(\"; \") + summary(body));\n saveState(freshState(c, state.attempt + 1));\n pm.execution.setNextRequest(pm.info.requestName);\n return;\n }\n\n pm.test(c.name, function () {\n if (failures.length) throw new Error(failures.join(\"; \") + summary(body));\n });\n\n // The chain carries on from whatever this case ended with, answered or not,\n // so one miss does not strip the context every later follow-up relies on.\n if (c.chain && body) {\n var ended = state.messages.slice();\n if (body.question) ended.push({ role: \"assistant\", content: body.question.question, question: true });\n else if (body.answer) ended.push({ role: \"assistant\", content: body.answer });\n pm.collectionVariables.set(chainKey(c.chain), JSON.stringify({ messages: ended, conversation_id: state.conversation_id }));\n }\n saveState(null);\n }\n\n return { prerequest: prerequest, test: test, evaluate: evaluate, pickAnswer: pickAnswer, stripCode: stripCode };\n}", + "value": "function createWarpLib(pm) {\n var STATE_KEY = \"__warp_state\";\n var MAX_RETRIES = 1;\n var MAX_QUESTIONS = 2;\n // Tools that only discover what exists. A reply backed by these alone rests\n // on no data - a refusal should call nothing else.\n var DISCOVERY_TOOLS = [\"describe_filter_space\", \"ask_user\"];\n // Options the driver prefers when Warp asks, most specific first: whole\n // deployment for scope (nobody is identified, so there is no default), then a\n // week for the window, which is what every seeded fact is measured over. The\n // rolling week is tried on its own first: asked \"calendar week or last 7\n // days?\", the seeded facts are the rolling ones whichever option is listed first.\n var DEFAULT_PREFER = [\n \"whole deployment|entire deployment|all traffic|everyone|all teams|everything\",\n \"-7d|7 days|rolling\",\n \"this week|last week\",\n \"-24h|24 hours\",\n ];\n\n function readState() {\n var raw = pm.collectionVariables.get(STATE_KEY);\n if (!raw) return null;\n try {\n return JSON.parse(raw);\n } catch (e) {\n return null;\n }\n }\n\n function saveState(state) {\n pm.collectionVariables.set(STATE_KEY, state ? JSON.stringify(state) : \"\");\n }\n\n function chainKey(name) {\n return \"__warp_chain_\" + name;\n }\n\n function readChain(name) {\n var raw = pm.collectionVariables.get(chainKey(name));\n if (!raw) return null;\n try {\n return JSON.parse(raw);\n } catch (e) {\n return null;\n }\n }\n\n function freshState(c, attempt) {\n var messages = [];\n var conversationId = pm.variables.replaceIn(\"{{$guid}}\");\n if (c.chain && !c.chainStart) {\n var chain = readChain(c.chain);\n if (chain) {\n messages = chain.messages.slice();\n conversationId = chain.conversation_id;\n }\n }\n if (c.history) {\n for (var i = 0; i < c.history.length; i++) {\n var turn = c.history[i];\n var copy = { role: turn.role, content: pm.variables.replaceIn(turn.content) };\n if (turn.question) copy.question = true;\n messages.push(copy);\n }\n }\n messages.push({ role: \"user\", content: pm.variables.replaceIn(c.ask) });\n return {\n name: pm.info.requestName,\n attempt: attempt,\n asked: 0,\n messages: messages,\n conversation_id: conversationId,\n };\n }\n\n // pickAnswer replies to an ask_user question the way a person clicking the\n // dashboard would: the option's hint when it has one, else its label. An\n // option the person's own question already named wins - asked \"what did\n // Platform Engineering spend\", nobody then picks \"Whole deployment\". With no\n // whole-deployment option on a scope question, they type it instead, as the\n // question's free-text escape allows, rather than taking whatever came first.\n function pickAnswer(question, prefer, asked) {\n var options = (question && question.options) || [];\n var said = String(asked || \"\").toLowerCase();\n for (var n = 0; n < options.length; n++) {\n var label = String(options[n].label || \"\").toLowerCase();\n if (label.length >= 3 && said.indexOf(label) >= 0) return options[n].hint || options[n].label;\n }\n var patterns = (prefer || []).concat(DEFAULT_PREFER);\n for (var p = 0; p < patterns.length; p++) {\n var re = new RegExp(patterns[p], \"i\");\n for (var i = 0; i < options.length; i++) {\n var o = options[i];\n if (re.test(o.label || \"\") || re.test(o.hint || \"\")) return o.hint || o.label;\n }\n }\n var scopeQuestion = (question && question.kind === \"scope\") || /whose|scope/i.test((question && question.question) || \"\");\n if (scopeQuestion && (!options.length || question.allow_other)) return \"The whole deployment.\";\n if (options.length) return options[0].hint || options[0].label;\n return \"The whole deployment, over the last 7 days.\";\n }\n\n // Patterns may name run-specific values ({{warp_slowest_id}}), resolved here\n // rather than at build time since they only exist once the seeder has run.\n function pattern(p) {\n return new RegExp(pm.variables.replaceIn(p), \"i\");\n }\n\n function stripCode(text) {\n return String(text || \"\")\n .replace(/```[\\s\\S]*?```/g, \"\")\n .replace(/`[^`]*`/g, \"\");\n }\n\n // evaluate returns the list of expectations a JSON-mode response missed.\n function evaluate(body, expect) {\n var e = expect || {};\n var failures = [];\n if (body.error) {\n failures.push(\"error \" + body.error.code + \": \" + body.error.message);\n return failures;\n }\n var calls = (body.tool_calls || []).filter(function (t) {\n return !t.failed;\n });\n var names = calls.map(function (t) {\n return t.name;\n });\n\n if (e.question === \"required\") {\n if (!body.question) failures.push(\"expected an ask_user question, got an answer\");\n else if (!body.question.options || body.question.options.length < 2)\n failures.push(\"question carried fewer than 2 options\");\n if (body.finish_reason !== \"question\") failures.push(\"finish_reason \" + body.finish_reason + \", want question\");\n return failures;\n }\n if (e.question === \"forbidden\" && body.question) failures.push(\"asked a question: \" + body.question.question);\n\n var finish = e.finish || [\"stop\", \"partial\"];\n if (finish.indexOf(body.finish_reason) < 0) failures.push(\"finish_reason \" + body.finish_reason + \", want one of \" + finish.join(\"/\"));\n\n // A \"required\" question case has already returned above; every case that\n // reaches here, \"forbidden\" included, must have said something.\n var answer = String(body.answer || \"\");\n if (!answer.trim()) failures.push(\"empty answer\");\n\n if (typeof e.maxToolCalls === \"number\" && (body.tool_calls || []).length > e.maxToolCalls)\n failures.push(\"made \" + (body.tool_calls || []).length + \" tool calls, want at most \" + e.maxToolCalls);\n if (e.toolsAny && !e.toolsAny.some(function (t) { return names.indexOf(t) >= 0; }))\n failures.push(\"called none of \" + e.toolsAny.join(\", \") + \" (called: \" + (names.join(\", \") || \"nothing\") + \")\");\n (e.toolsAll || []).forEach(function (t) {\n if (names.indexOf(t) < 0) failures.push(\"did not call \" + t + \" (called: \" + (names.join(\", \") || \"nothing\") + \")\");\n });\n if (e.toolsOnly) {\n // Every attempt, failed ones included: a forbidden tool the model reached\n // for is the behaviour this rules out, whether or not the call worked.\n var attempted = (body.tool_calls || []).map(function (t) { return t.name; });\n var others = attempted.filter(function (n) { return e.toolsOnly.indexOf(n) < 0; });\n if (others.length) failures.push(\"called \" + others.join(\", \") + \", which this question has no use for (allowed: \" + e.toolsOnly.join(\", \") + \")\");\n }\n if (e.noDataTools) {\n var data = names.filter(function (n) { return DISCOVERY_TOOLS.indexOf(n) < 0; });\n if (data.length) failures.push(\"queried data for an out-of-scope question: \" + data.join(\", \"));\n }\n (e.argsMatch || []).forEach(function (m) {\n var re = pattern(m.pattern);\n var hit = calls.some(function (t) {\n return (!m.tool || t.name === m.tool) && re.test(t.arguments || \"\");\n });\n if (!hit) failures.push(\"no \" + (m.tool || \"tool\") + \" call with arguments matching /\" + m.pattern + \"/\");\n });\n // A filter the question never asked for narrows the answer without saying so.\n (e.argsNone || []).forEach(function (m) {\n var re = pattern(m.pattern);\n calls.forEach(function (t) {\n if ((!m.tool || t.name === m.tool) && re.test(t.arguments || \"\"))\n failures.push(t.name + \" was called with arguments matching /\" + m.pattern + \"/: \" + t.arguments);\n });\n });\n\n (e.answerAll || []).forEach(function (p) {\n if (!pattern(p).test(answer)) failures.push(\"answer lacks /\" + p + \"/\");\n });\n if (e.answerAny && !e.answerAny.some(function (p) { return pattern(p).test(answer); }))\n failures.push(\"answer matches none of \" + e.answerAny.map(function (p) { return \"/\" + p + \"/\"; }).join(\" \"));\n (e.answerNone || []).forEach(function (p) {\n if (pattern(p).test(answer)) failures.push(\"answer contains /\" + p + \"/\");\n });\n if (e.answerNoneOutsideCode) {\n var prose = stripCode(answer);\n e.answerNoneOutsideCode.forEach(function (p) {\n if (pattern(p).test(prose)) failures.push(\"answer repeats /\" + p + \"/ outside inline code\");\n });\n }\n if (e.maxAnswerChars && answer.length > e.maxAnswerChars)\n failures.push(\"answer is \" + answer.length + \" chars, want at most \" + e.maxAnswerChars);\n return failures;\n }\n\n function prerequest(c) {\n var state = readState();\n if (!state || state.name !== pm.info.requestName) {\n state = freshState(c, 0);\n saveState(state);\n }\n pm.request.body.update({\n mode: \"raw\",\n raw: JSON.stringify({\n messages: state.messages,\n conversation_id: state.conversation_id,\n stream: false,\n timezone: \"UTC\",\n }),\n options: { raw: { language: \"json\" } },\n });\n }\n\n function summary(body) {\n if (!body) return \"\";\n // Arguments included: a repeated call is only diagnosable from what it changed.\n var tools = (body.tool_calls || []).map(function (t) {\n return t.name + (t.failed ? \"(failed)\" : \"\") + (t.arguments ? \" \" + String(t.arguments).slice(0, 200) : \"\");\n });\n return \" | tools: [\" + tools.join(\", \") + \"] | finish: \" + body.finish_reason + \" | answer: \" + String(body.answer || \"\").slice(0, 600);\n }\n\n function test(c) {\n var state = readState() || freshState(c, 0);\n var body = null;\n var failures = [];\n if (pm.response.code !== 200) {\n failures.push(\"status \" + pm.response.code + \": \" + pm.response.text().slice(0, 500));\n } else {\n try {\n body = pm.response.json();\n } catch (err) {\n failures.push(\"response is not JSON: \" + pm.response.text().slice(0, 300));\n }\n }\n\n if (body && body.question && (c.expect || {}).question !== \"required\" && (c.expect || {}).question !== \"forbidden\" && state.asked < MAX_QUESTIONS) {\n state.messages.push({ role: \"assistant\", content: body.question.question, question: true });\n state.messages.push({ role: \"user\", content: pickAnswer(body.question, c.prefer, pm.variables.replaceIn(c.ask)) });\n state.asked += 1;\n saveState(state);\n pm.execution.setNextRequest(pm.info.requestName);\n return;\n }\n\n if (body) failures = failures.concat(evaluate(body, c.expect));\n // An answer that was never preceded by the question the case exists to see.\n var minQuestions = (c.expect || {}).minQuestions;\n if (body && typeof minQuestions === \"number\" && state.asked < minQuestions)\n failures.push(\"asked \" + state.asked + \" question(s) before answering, want at least \" + minQuestions);\n\n if (failures.length && state.attempt < MAX_RETRIES) {\n console.log(\"RETRY \" + pm.info.requestName + \": \" + failures.join(\"; \") + summary(body));\n saveState(freshState(c, state.attempt + 1));\n pm.execution.setNextRequest(pm.info.requestName);\n return;\n }\n\n pm.test(c.name, function () {\n if (failures.length) throw new Error(failures.join(\"; \") + summary(body));\n });\n\n // The chain carries on from whatever this case ended with, answered or not,\n // so one miss does not strip the context every later follow-up relies on.\n if (c.chain && body) {\n var ended = state.messages.slice();\n if (body.question) ended.push({ role: \"assistant\", content: body.question.question, question: true });\n else if (body.answer) ended.push({ role: \"assistant\", content: body.answer });\n pm.collectionVariables.set(chainKey(c.chain), JSON.stringify({ messages: ended, conversation_id: state.conversation_id }));\n }\n saveState(null);\n }\n\n return { prerequest: prerequest, test: test, evaluate: evaluate, pickAnswer: pickAnswer, stripCode: stripCode };\n}", "type": "string" }, { @@ -4980,7 +4980,7 @@ "type": "text/javascript", "exec": [ "var warp = eval('(' + pm.collectionVariables.get('warp_lib') + ')')(pm);", - "warp.prerequest({\"name\":\"Declines to count the people behind a topic\",\"ask\":\"How many different users asked about the largest topic?\",\"expect\":{\"toolsAll\":[\"list_topics\"],\"toolsOnly\":[\"list_topics\"],\"maxToolCalls\":1,\"answerAll\":[\"github\\\\.com/maximhq/bifrost/issues/new\"],\"answerNone\":[\"\\\\b\\\\d+ (different |distinct |unique )?(users|people)\\\\b\"]}});" + "warp.prerequest({\"name\":\"Declines to count the people behind a topic\",\"ask\":\"How many different users asked about the largest topic?\",\"expect\":{\"toolsAll\":[\"list_topics\"],\"toolsOnly\":[\"describe_filter_space\",\"list_topics\",\"query_usage_by\"],\"maxToolCalls\":3,\"answerAll\":[\"github\\\\.com/maximhq/bifrost/issues/new\"],\"answerNone\":[\"\\\\b\\\\d+ (different |distinct |unique )?people\\\\b\"]}});" ] } }, @@ -4990,7 +4990,7 @@ "type": "text/javascript", "exec": [ "var warp = eval('(' + pm.collectionVariables.get('warp_lib') + ')')(pm);", - "warp.test({\"name\":\"Declines to count the people behind a topic\",\"ask\":\"How many different users asked about the largest topic?\",\"expect\":{\"toolsAll\":[\"list_topics\"],\"toolsOnly\":[\"list_topics\"],\"maxToolCalls\":1,\"answerAll\":[\"github\\\\.com/maximhq/bifrost/issues/new\"],\"answerNone\":[\"\\\\b\\\\d+ (different |distinct |unique )?(users|people)\\\\b\"]}});" + "warp.test({\"name\":\"Declines to count the people behind a topic\",\"ask\":\"How many different users asked about the largest topic?\",\"expect\":{\"toolsAll\":[\"list_topics\"],\"toolsOnly\":[\"describe_filter_space\",\"list_topics\",\"query_usage_by\"],\"maxToolCalls\":3,\"answerAll\":[\"github\\\\.com/maximhq/bifrost/issues/new\"],\"answerNone\":[\"\\\\b\\\\d+ (different |distinct |unique )?people\\\\b\"]}});" ] } } diff --git a/tests/e2e/api/runners/build-warp-collection.mjs b/tests/e2e/api/runners/build-warp-collection.mjs index 76fa5070e2e..293cfba19e2 100644 --- a/tests/e2e/api/runners/build-warp-collection.mjs +++ b/tests/e2e/api/runners/build-warp-collection.mjs @@ -374,7 +374,16 @@ const FOLDERS = [ id: "topics-people", name: "Declines to count the people behind a topic", ask: "How many different users asked about the largest topic?", - expect: { ...ONE_LIST_TOPICS, answerAll: [FEATURE_LINK], answerNone: ["\\b\\d+ (different |distinct |unique )?(users|people)\\b"] }, + // The prompt has Warp answer what the data shows first - a query_usage_by + // user ranking with topic_ids counts users with a request in the topic - + // then decline the people count. Users are not people. + expect: { + toolsAll: ["list_topics"], + toolsOnly: ["describe_filter_space", "list_topics", "query_usage_by"], + maxToolCalls: 3, + answerAll: [FEATURE_LINK], + answerNone: ["\\b\\d+ (different |distinct |unique )?people\\b"], + }, }, ], }, diff --git a/tests/e2e/api/runners/lib/warp-case.mjs b/tests/e2e/api/runners/lib/warp-case.mjs index 5201cbde0be..de6bf57c9b5 100644 --- a/tests/e2e/api/runners/lib/warp-case.mjs +++ b/tests/e2e/api/runners/lib/warp-case.mjs @@ -235,8 +235,9 @@ export function createWarpLib(pm) { function summary(body) { if (!body) return ""; + // Arguments included: a repeated call is only diagnosable from what it changed. var tools = (body.tool_calls || []).map(function (t) { - return t.name + (t.failed ? "(failed)" : ""); + return t.name + (t.failed ? "(failed)" : "") + (t.arguments ? " " + String(t.arguments).slice(0, 200) : ""); }); return " | tools: [" + tools.join(", ") + "] | finish: " + body.finish_reason + " | answer: " + String(body.answer || "").slice(0, 600); } diff --git a/tests/e2e/api/runners/lib/warp-case.test.mjs b/tests/e2e/api/runners/lib/warp-case.test.mjs index ffc98e6b444..5098092d29f 100644 --- a/tests/e2e/api/runners/lib/warp-case.test.mjs +++ b/tests/e2e/api/runners/lib/warp-case.test.mjs @@ -147,7 +147,7 @@ test("topic cases allow exactly one list_topics call and no other tool", () => { const { pm } = fakePm(); const lib = createWarpLib(pm); const toolFailure = /tool calls, want at most|no use for|did not call/; - for (const id of ["topics-overview", "topics-past-day", "topics-trend", "topics-people"]) { + for (const id of ["topics-overview", "topics-past-day", "topics-trend"]) { const { expect } = caseFromCollection(id); const tools = (calls) => lib.evaluate(answered("x", calls), expect).filter((f) => toolFailure.test(f)); assert.deepStrictEqual(tools(["list_topics"]), [], `${id}: one list_topics call`); @@ -157,6 +157,26 @@ test("topic cases allow exactly one list_topics call and no other tool", () => { } }); +// Counting people behind a topic is declined, but the prompt and list_topics' +// cannot_answer hint both offer what the data does show first: a query_usage_by +// user ranking with topic_ids counts the users with a request in the topic. +// Users are not people, so "N users" passes and "N people" does not; the +// feature-request link is still required, and the logs are never sliced. +test("topics-people answers the user count from topic_ids, then declines people", () => { + const { pm } = fakePm(); + const lib = createWarpLib(pm); + const { expect } = caseFromCollection("topics-people"); + const link = " Counting distinct people is not available yet: https://github.com/maximhq/bifrost/issues/new"; + const ranking = { name: "query_usage_by", arguments: '{"dimension": "user", "filters": {"topic_ids": ["t-1"]}}' }; + const pass = lib.evaluate(answered("12 users had a request in Refund Requests." + link, ["list_topics", ranking]), expect); + assert.deepStrictEqual(pass, [], "user ranking with topic_ids, then the decline"); + assert.deepStrictEqual(lib.evaluate(answered("Refund Requests is the largest topic." + link, ["list_topics"]), expect), [], "decline alone"); + assert.notDeepStrictEqual(lib.evaluate(answered("12 people asked about it." + link, ["list_topics", ranking]), expect), [], "a people count"); + assert.notDeepStrictEqual(lib.evaluate(answered("12 users had a request in it.", ["list_topics", ranking]), expect), [], "no feature link"); + assert.notDeepStrictEqual(lib.evaluate(answered("x" + link, ["list_topics", "query_logs"]), expect), [], "the logs sliced"); + assert.notDeepStrictEqual(lib.evaluate(answered("x" + link, []), expect), [], "no list_topics call"); +}); + // A survey scoped to a window or a person is answered from one query_logs // sample, never from list_topics, and says it is a sample. test("scoped survey cases are a list_topics call with the scope, never a rebuilt sample", () => { @@ -339,6 +359,25 @@ test("a miss is retried once from scratch, then recorded as a failure", () => { assert.match(recorded.tests[0].message, /answer lacks \/anthropic\//); }); +// A repeated call is only diagnosable from what it changed: three list_topics +// calls read the same unless each one's arguments are in the failure. +test("a failure names each tool call with its arguments", () => { + const { pm, recorded } = fakePm(); + const lib = createWarpLib(pm); + const c = { name: "topics overview", ask: "What do people ask about?", expect: { maxToolCalls: 1 } }; + const calls = [ + { name: "list_topics", arguments: '{"limit": 15}' }, + { name: "list_topics", arguments: '{"limit": 50}' }, + ]; + for (let attempt = 0; attempt < 2; attempt++) { + lib.prerequest(c); + pm.respond(200, answered("Refunds lead.", calls)); + lib.test(c); + } + assert.strictEqual(recorded.tests[0].ok, false); + assert.match(recorded.tests[0].message, /tools: \[list_topics \{"limit": 15\}, list_topics \{"limit": 50\}\]/); +}); + // The no-skip rule: an auth failure, rate limit or 5xx must fail the case // loudly with the body, never pass it quietly. test("a non-200 status fails the case with the response body", () => {