feat(api): oasdiff OpenAI openAPI spec against ours

diff the `/v1/openai/v1` routes against the OpenAI openAPI spec. This will of course only trigger on PRs where the spec is changed.

This will catch errors with new handwritten additions to our openAI compat routes.

Signed-off-by: Charlie Doern <[email protected]>

Author: cdoern

.github/workflows/conformance.yml

@@ -40,6 +40,15 @@ jobs:
           ref: ${{ github.event.pull_request.base.ref }}
           path: 'base'
 
+
+      # Cache openai spec
+      - name: Cache openai openapi spec
+        id: cache-openapi
+        uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809
+        with:
+          path: ~/openai-openapi
+          key: openai-openapi.yml
+
       # Cache oasdiff to avoid checksum failures and speed up builds
       - name: Cache oasdiff
         id: cache-oasdiff
@@ -55,6 +64,12 @@ jobs:
           curl -fsSL https://raw.githubusercontent.com/oasdiff/oasdiff/main/install.sh | sh
           cp /usr/local/bin/oasdiff ~/oasdiff
 
+      - name: Get openai openapi spec
+        if: steps.cache-openapi.outputs.cache-hit != 'true'
+        run: |
+          mkdir -p ~/openai-openapi
+          curl -L https://app.stainless.com/api/spec/documented/openai/openapi.documented.yml -o ~/openai-openapi/openai-openapi.yml
+
       # Setup cached oasdiff
       - name: Setup cached oasdiff
         if: steps.cache-oasdiff.outputs.cache-hit == 'true'
@@ -69,3 +84,27 @@ jobs:
           oasdiff breaking --fail-on ERR base/docs/static/llama-stack-spec.yaml docs/static/llama-stack-spec.yaml --match-path '^/v1/openai/v1' \
           --match-path '^/v1/vector-io' \
           --match-path '^/v1/vector-dbs'
+
+      # Run oasdiff to detect breaking changes in the API specification when compared to the OpenAI openAPI spec
+      - name: Run OpenAPI Breaking Change Diff Against OpenAI API
+        continue-on-error: true
+        shell: bash
+        run: |
+          OPENAI_SPEC=~/openai-openapi/openai-openapi.yml
+          LOCAL_SPEC=docs/static/llama-stack-spec.yaml
+
+          # Extract base OpenAI endpoint paths from the spec file (remove path parameters)
+          BASE_PATHS=$(grep -o '/v1/openai/v1/[^:]*' "$LOCAL_SPEC" | sed 's|/v1/openai/v1/||' | sed 's|/{[^}]*}.*||' | sort -u)
+
+          # Build single regex pattern from extracted base paths
+          PATTERN=$(echo "$BASE_PATHS" | tr '\n' '|' | sed 's/|$//')
+
+          echo "Debug: Extracted base paths:"
+          echo "$BASE_PATHS"
+          echo "Debug: Generated pattern: (^/v1/openai/v1)?/($PATTERN)(/.*)?$"
+
+          oasdiff breaking --fail-on ERR \
+            "$OPENAI_SPEC" \
+            "$LOCAL_SPEC" \
+            --strip-prefix-revision "/v1/openai/v1" \
+            --match-path "(^/v1/openai/v1)?/($PATTERN)(/.*)?$"