From 57e629a107a5e54d9a319ac4fc1e5a69d772610b Mon Sep 17 00:00:00 2001
From: Ambient Code Bot <bot@ambient-code.local>
Date: Thu, 5 Mar 2026 20:09:31 +0000
Subject: [PATCH] fix(leaderboard): use upstream schema for fork PR validation

When a user submits a leaderboard entry from a fork that hasn't been synced
with upstream, the workflow was using the fork's outdated schema for
validation. This caused valid submissions to fail with errors like
"attributes_total: 25 was expected" even though PR #312 had already fixed
the schema to allow 10-25 attributes.

This fix changes the workflow to:
1. Checkout upstream main for validation tools and schema
2. Fetch only the submission file from the PR branch
3. Validate using upstream's agentready installation

This ensures validation always uses the latest schema while still validating
the actual submitted file from the PR.

Fixes the validation failure on PR #332.
See: https://github.com/ambient-code/agentready/pull/312

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
---
 .github/workflows/leaderboard.yml | 45 ++++++++++++++++++++-----------
 1 file changed, 30 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/leaderboard.yml b/.github/workflows/leaderboard.yml
index ffbb9e05..07707232 100644
--- a/.github/workflows/leaderboard.yml
+++ b/.github/workflows/leaderboard.yml
@@ -29,12 +29,26 @@ jobs:
       pull-requests: write      # Post validation results as PR comment
       issues: write             # Required by github-script (PRs are issues in GH API)
     steps:
-      - name: Checkout PR branch
+      # Fix for outdated forks: Always use upstream for validation tools/schema
+      # The PR branch may be from a fork that hasn't synced with upstream,
+      # causing validation to use outdated schema. We checkout upstream main
+      # for the agentready tools, then fetch only the submission file from PR.
+      # See: https://github.com/ambient-code/agentready/pull/312
+      - name: Checkout upstream for validation tools
         uses: actions/checkout@v6
         with:
-          ref: ${{ github.event.pull_request.head.sha }}
+          repository: ${{ github.repository }}
+          ref: main
           fetch-depth: 0
 
+      - name: Fetch PR submission file
+        env:
+          PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+          PR_HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }}
+        run: |
+          # Fetch the PR branch to get the submission file
+          git fetch "https://github.com/${PR_HEAD_REPO}.git" "${PR_HEAD_SHA}:refs/remotes/pr/head"
+
       - name: Set up Python
         uses: actions/setup-python@v6
         with:
@@ -50,11 +64,8 @@ jobs:
       - name: Extract submission details
         id: extract
         run: |
-          # Fetch main branch from upstream for comparison
-          git fetch origin main:refs/remotes/origin/main || true
-
-          # Find changed JSON file
-          CHANGED_FILE=$(git diff --name-only origin/main...HEAD | grep 'submissions/.*-assessment.json' | head -1)
+          # Find changed JSON file between main and PR head
+          CHANGED_FILE=$(git diff --name-only origin/main...pr/head | grep 'submissions/.*-assessment.json' | head -1)
 
           if [ -z "$CHANGED_FILE" ]; then
             echo "No assessment file found in diff"
@@ -63,11 +74,15 @@ jobs:
 
           echo "file=$CHANGED_FILE" >> "$GITHUB_OUTPUT"
 
-          # Parse JSON - all values stored in outputs, not executed
-          REPO_URL=$(jq -r '.repository.url' "$CHANGED_FILE")
-          CLAIMED_SCORE=$(jq -r '.overall_score' "$CHANGED_FILE")
-          REPO_NAME=$(jq -r '.repository.name' "$CHANGED_FILE")
-          RESEARCH_VERSION=$(jq -r '.metadata.research_version // "unknown"' "$CHANGED_FILE")
+          # Extract the submission file from PR branch to a temp location
+          # This ensures we validate the actual submitted file, not main branch
+          git show "pr/head:${CHANGED_FILE}" > /tmp/submission.json
+
+          # Parse JSON from PR's submission file - all values stored in outputs, not executed
+          REPO_URL=$(jq -r '.repository.url' /tmp/submission.json)
+          CLAIMED_SCORE=$(jq -r '.overall_score' /tmp/submission.json)
+          REPO_NAME=$(jq -r '.repository.name' /tmp/submission.json)
+          RESEARCH_VERSION=$(jq -r '.metadata.research_version // "unknown"' /tmp/submission.json)
 
           {
             echo "repo_url=$REPO_URL"
@@ -77,11 +92,11 @@ jobs:
           } >> "$GITHUB_OUTPUT"
 
       - name: Validate JSON schema
-        env:
-          ASSESSMENT_FILE: ${{ steps.extract.outputs.file }}
         run: |
+          # Validate using upstream's schema against the PR's submission file
+          # This ensures outdated forks don't cause false validation failures
           source .venv/bin/activate
-          agentready validate-report "$ASSESSMENT_FILE"
+          agentready validate-report /tmp/submission.json
 
       - name: Verify repository exists and is public
         env: