fix(leaderboard): use upstream schema for fork PR validation

.github/workflows/leaderboard.yml

@@ -29,12 +29,26 @@ jobs:
       pull-requests: write      # Post validation results as PR comment
       issues: write             # Required by github-script (PRs are issues in GH API)
     steps:
-      - name: Checkout PR branch
+      # Fix for outdated forks: Always use upstream for validation tools/schema
+      # The PR branch may be from a fork that hasn't synced with upstream,
+      # causing validation to use outdated schema. We checkout upstream main
+      # for the agentready tools, then fetch only the submission file from PR.
+      # See: https://github.com/ambient-code/agentready/pull/312
+      - name: Checkout upstream for validation tools
         uses: actions/checkout@v6
         with:
-          ref: ${{ github.event.pull_request.head.sha }}
+          repository: ${{ github.repository }}
+          ref: main
           fetch-depth: 0
 
+      - name: Fetch PR submission file
+        env:
+          PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+          PR_HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }}
+        run: |
+          # Fetch the PR branch to get the submission file
+          git fetch "https://github.com/${PR_HEAD_REPO}.git" "${PR_HEAD_SHA}:refs/remotes/pr/head"
+
       - name: Set up Python
         uses: actions/setup-python@v6
         with:
@@ -50,11 +64,8 @@ jobs:
       - name: Extract submission details
         id: extract
         run: |
-          # Fetch main branch from upstream for comparison
-          git fetch origin main:refs/remotes/origin/main || true
-
-          # Find changed JSON file
-          CHANGED_FILE=$(git diff --name-only origin/main...HEAD | grep 'submissions/.*-assessment.json' | head -1)
+          # Find changed JSON file between main and PR head
+          CHANGED_FILE=$(git diff --name-only origin/main...pr/head | grep 'submissions/.*-assessment.json' | head -1)
 
           if [ -z "$CHANGED_FILE" ]; then
             echo "No assessment file found in diff"
@@ -63,11 +74,15 @@ jobs:
 
           echo "file=$CHANGED_FILE" >> "$GITHUB_OUTPUT"
 
-          # Parse JSON - all values stored in outputs, not executed
-          REPO_URL=$(jq -r '.repository.url' "$CHANGED_FILE")
-          CLAIMED_SCORE=$(jq -r '.overall_score' "$CHANGED_FILE")
-          REPO_NAME=$(jq -r '.repository.name' "$CHANGED_FILE")
-          RESEARCH_VERSION=$(jq -r '.metadata.research_version // "unknown"' "$CHANGED_FILE")
+          # Extract the submission file from PR branch to a temp location
+          # This ensures we validate the actual submitted file, not main branch
+          git show "pr/head:${CHANGED_FILE}" > /tmp/submission.json
+
+          # Parse JSON from PR's submission file - all values stored in outputs, not executed
+          REPO_URL=$(jq -r '.repository.url' /tmp/submission.json)
+          CLAIMED_SCORE=$(jq -r '.overall_score' /tmp/submission.json)
+          REPO_NAME=$(jq -r '.repository.name' /tmp/submission.json)
+          RESEARCH_VERSION=$(jq -r '.metadata.research_version // "unknown"' /tmp/submission.json)
 
           {
             echo "repo_url=$REPO_URL"
@@ -77,11 +92,11 @@ jobs:
           } >> "$GITHUB_OUTPUT"
 
       - name: Validate JSON schema
-        env:
-          ASSESSMENT_FILE: ${{ steps.extract.outputs.file }}
         run: |
+          # Validate using upstream's schema against the PR's submission file
+          # This ensures outdated forks don't cause false validation failures
           source .venv/bin/activate
-          agentready validate-report "$ASSESSMENT_FILE"
+          agentready validate-report /tmp/submission.json
 
       - name: Verify repository exists and is public
         env: