Add sccache stats to CI output.

.github/actions/workflow-build/build-workflow.py

@@ -239,7 +239,7 @@ def generate_dispatch_job_image(matrix_job, job_type):
 
 
 def generate_dispatch_job_command(matrix_job, job_type):
-    script_path = "ci/windows" if is_windows(matrix_job) else "ci"
+    script_path = "./ci/windows" if is_windows(matrix_job) else "./ci"
     script_ext = ".ps1" if is_windows(matrix_job) else ".sh"
     script_job_type = job_type
     script_project = matrix_job['project']

.github/actions/workflow-results/action.yml

@@ -24,12 +24,25 @@ runs:
         name: workflow
         path: workflow/
 
-    - name: Download job success artifacts
+    - name: Download job artifacts
       continue-on-error: true # This may fail if no jobs succeed. The checks below will catch this.
       uses: actions/download-artifact@v3
       with:
-        name: dispatch-job-success
-        path: dispatch-job-success/
+        name: jobs
+        path: jobs
+
+    - name: Clean up job artifacts
+      continue-on-error: true
+      shell: bash --noprofile --norc -euo pipefail {0}
+      run: |
+        # Fix artifacts written on windows:
+        sudo apt-get update
+        sudo apt-get install -y dos2unix
+        find jobs -type f -exec dos2unix -q {} \;
+
+        echo "::group::Job artifacts"
+        tree jobs
+        echo "::endgroup::"
 
     - name: Fetch workflow job info
       if: ${{ inputs.github_token != ''}}
@@ -75,7 +88,7 @@ runs:
       continue-on-error: true
       shell: bash --noprofile --norc -euo pipefail {0}
       run: |
-        echo "Generating job summary..."
+        echo "Generating execution summary..."
         python3 "${GITHUB_ACTION_PATH}/prepare-execution-summary.py" workflow/workflow.json results/job_times.json
 
     - name: Prepare final summary

.github/actions/workflow-results/prepare-execution-summary.py

@@ -2,15 +2,16 @@
 
 
 import argparse
+import functools
 import json
 import os
 import re
 import sys
 
 
 def job_succeeded(job):
-    # The job was successful if the artifact file 'dispatch-job-success/dispatch-job-success-<job_id>' exists:
-    return os.path.exists(f'dispatch-job-success/{job["id"]}')
+    # The job was successful if the success file exists:
+    return os.path.exists(f'jobs/{job["id"]}/success')
 
 
 def natural_sort_key(key):
@@ -42,17 +43,21 @@ def extract_jobs(workflow):
     return jobs
 
 
-def create_summary_entry(include_times):
-    summary = {'passed': 0, 'failed': 0}
-
-    if include_times:
-        summary['job_time'] = 0
-        summary['step_time'] = 0
-
-    return summary
+@functools.lru_cache(maxsize=None)
+def get_sccache_stats(job_id):
+    sccache_file = f'jobs/{job_id}/sccache_stats.json'
+    if os.path.exists(sccache_file):
+        with open(sccache_file) as f:
+            return json.load(f)
+    return None
 
 
 def update_summary_entry(entry, job, job_times=None):
+    if 'passed' not in entry:
+        entry['passed'] = 0
+    if 'failed' not in entry:
+        entry['failed'] = 0
+
     if job_succeeded(job):
         entry['passed'] += 1
     else:
@@ -63,15 +68,36 @@ def update_summary_entry(entry, job, job_times=None):
         job_time = time_info["job_seconds"]
         command_time = time_info["command_seconds"]
 
+        if not 'job_time' in entry:
+            entry['job_time'] = 0
+        if not 'step_time' in entry:
+            entry['step_time'] = 0
+
         entry['job_time'] += job_time
         entry['step_time'] += command_time
 
+    sccache_stats = get_sccache_stats(job["id"])
+    if sccache_stats:
+        sccache_stats = sccache_stats['stats']
+        requests = sccache_stats.get('compile_requests', 0)
+        hits = 0
+        if 'cache_hits' in sccache_stats:
+            cache_hits = sccache_stats['cache_hits']
+            if 'counts' in cache_hits:
+                counts = cache_hits['counts']
+                for lang, lang_hits in counts.items():
+                    hits += lang_hits
+        if 'sccache' not in entry:
+            entry['sccache'] = {'requests': requests, 'hits': hits}
+        else:
+            entry['sccache']['requests'] += requests
+            entry['sccache']['hits'] += hits
+
     return entry
 
 
 def build_summary(jobs, job_times=None):
-    summary = create_summary_entry(job_times)
-    summary['projects'] = {}
+    summary = {'projects': {}}
     projects = summary['projects']
 
     for job in jobs:
@@ -81,8 +107,7 @@ def build_summary(jobs, job_times=None):
 
         project = matrix_job["project"]
         if not project in projects:
-            projects[project] = create_summary_entry(job_times)
-            projects[project]['tags'] = {}
+            projects[project] = {'tags': {}}
         tags = projects[project]['tags']
 
         update_summary_entry(projects[project], job, job_times)
@@ -92,16 +117,15 @@ def build_summary(jobs, job_times=None):
                 continue
 
             if not tag in tags:
-                tags[tag] = create_summary_entry(job_times)
-                tags[tag]['values'] = {}
+                tags[tag] = {'values': {}}
             values = tags[tag]['values']
 
             update_summary_entry(tags[tag], job, job_times)
 
             value = str(matrix_job[tag])
 
             if not value in values:
-                values[value] = create_summary_entry(job_times)
+                values[value] = {}
             update_summary_entry(values[value], job, job_times)
 
     # Natural sort the value strings within each tag:
@@ -156,17 +180,24 @@ def get_summary_stats(summary):
     failed = summary['failed']
     total = passed + failed
 
-    percent = int(100 * failed / total) if total > 0 else 0
-    fraction = f"{failed}/{total}"
-    fail_string = f'{percent:>3}% Failed ({fraction})'
+    percent = int(100 * passed / total) if total > 0 else 0
+    pass_string = f'Pass: {percent:>3}%/{total}'
 
-    stats = f'{fail_string:<21}'
+    stats = f'{pass_string:<14}'
 
-    if (summary['job_time']):
+    if 'job_time' in summary and total > 0 and summary['job_time'] > 0:
         job_time = summary['job_time']
         total_job_duration = format_seconds(job_time)
         avg_job_duration = format_seconds(job_time / total)
-        stats += f' | Total Time: {total_job_duration:>7} | Avg Time: {avg_job_duration:>6}'
+        stats += f' | Total Time: {total_job_duration:>7} | Avg Time: {avg_job_duration:>7}'
+
+    if 'sccache' in summary:
+        sccache = summary['sccache']
+        requests = sccache["requests"]
+        hits = sccache["hits"]
+        hit_percent = int(100 * hits / requests) if requests > 0 else 0
+        hit_string = f'Hits: {hit_percent:>3}%/{requests}'
+        stats += f' | {hit_string:<17}'
 
     return stats
 
@@ -193,7 +224,7 @@ def get_project_heading(project, project_summary):
     else:
         flag = '🟩'
 
-    return f'{flag} Project {project}: {get_summary_stats(project_summary)}'
+    return f'{flag} {project}: {get_summary_stats(project_summary)}'
 
 
 def get_tag_line(tag, tag_summary):

.github/actions/workflow-results/verify-job-success.py

@@ -13,13 +13,13 @@ def main():
 
     job_id_map = json.load(args.job_id_map)
 
-    # For each job id, verify that the file 'dispatch-job-success/<job_id>' exists
+    # For each job id, verify that the success artifact exists
     success = True
     for job_id, job_name in job_id_map.items():
-        success_file = f'dispatch-job-success/{job_id}'
+        success_file = f'jobs/{job_id}/success'
         print(f'Verifying job with id "{job_id}": "{job_name}"')
         if not os.path.exists(success_file):
-            print(f'Failed: Artifact "dispatch-job-success/{job_id}" not found')
+            print(f'Failed: Artifact "{success_file}" not found')
             success = False
 
     if not success:

.github/workflows/workflow-dispatch-job.yml

@@ -61,15 +61,16 @@ jobs:
       - name: Checkout repo
         uses: actions/checkout@v3
         with:
-          path: cccl
+          path: ${{github.event.repository.name}}
           persist-credentials: false
-      - name: Move files to coder user home directory
+      - name: Link files to coder user home directory
         run: |
-          cp -R cccl /home/coder/cccl
-          chown -R coder:coder /home/coder/
+          ln -s "$(pwd)/${{github.event.repository.name}}" /home/coder/${{github.event.repository.name}}
+          chown -R coder:coder ${{github.event.repository.name}}
+          chown -R coder:coder /home/coder/${{github.event.repository.name}}
       - name: Add NVCC problem matcher
         run: |
-          echo "::add-matcher::cccl/.github/problem-matchers/problem-matcher.json"
+          echo "::add-matcher::${{github.event.repository.name}}/.github/problem-matchers/problem-matcher.json"
       - name: Get AWS credentials for sccache bucket
         uses: aws-actions/configure-aws-credentials@v2
         with:
@@ -91,7 +92,7 @@ jobs:
           COMMAND: ${{inputs.command}}
         run: |
             set -eo pipefail
-            cd ~/cccl
+            cd ~/${{github.event.repository.name}}
             echo -e "\e[1;34mRunning as 'coder' user in $(pwd):\e[0m"
             echo -e "\e[1;34m${COMMAND}\e[0m"
             eval "${COMMAND}" || exit_code=$?
@@ -109,17 +110,38 @@ jobs:
               echo "   - Continuous Integration (CI) Overview: https://github.com/NVIDIA/cccl/blob/main/ci-overview.md"
               exit $exit_code
             fi
-      - name: Mark job as successful
+      - name: Prepare job artifacts
         id: done
         run: |
-          echo "SUCCESS=true" | tee -a ${GITHUB_OUTPUT}
-          mkdir dispatch-job-success
-          touch dispatch-job-success/${{inputs.id}}
-      - name: Upload dispatch-job-success
+          echo "SUCCESS=true" | tee -a "${GITHUB_OUTPUT}"
+
+          result_dir="jobs/${{inputs.id}}"
+          mkdir -p "$result_dir"
+
+          touch "$result_dir/success"
+
+          # Finds a matching file in the repo directory and copies it to the results directory.
+          find_and_copy() {
+            filename="$1"
+            filepath="$(find ${{github.event.repository.name}} -name "${filename}" -print -quit)"
+            if [[ -z "$filepath" ]]; then
+              echo "${filename} does not exist in repo directory."
+              return 1
+            fi
+            cp -v "$filepath" "$result_dir"
+          }
+
+          find_and_copy "sccache_stats.json" || true # Ignore failures
+
+          echo "job artifacts:"
+          find "$result_dir"
+
+      - name: Upload job artifacts
         uses: actions/upload-artifact@v3
         with:
-          name: dispatch-job-success
-          path: dispatch-job-success/${{inputs.id}}
+          name: jobs
+          path: jobs
+
 
   windows:
     name: ${{inputs.name}}
@@ -147,31 +169,62 @@ jobs:
           role-to-assume: arn:aws:iam::279114543810:role/gha-oidc-NVIDIA
           aws-region: us-east-2
           role-duration-seconds: 43200 # 12 hours
+      - name: Checkout repo
+        uses: actions/checkout@v3
+        with:
+          path: ${{github.event.repository.name}}
+          persist-credentials: false
       - name: Fetch ${{ inputs.image }}
         run: docker pull ${{ inputs.image }}
+      - name: Prepare paths for docker
+        id: paths
+        run: |
+          echo "HOST_REPO=${{ github.workspace }}\${{ github.event.repository.name }}".Replace('\', '/') | Out-File -FilePath $env:GITHUB_OUTPUT -Append
+          echo "MOUNT_REPO=C:/${{ github.event.repository.name }}" | Out-File -FilePath $env:GITHUB_OUTPUT -Append
+          cat $env:GITHUB_OUTPUT
+        shell: powershell
       - name: Run command # Do not change this step's name, it is checked in parse-job-times.py
-        run: >-
-          docker run ${{ inputs.image }} powershell -c "[System.Environment]::SetEnvironmentVariable('AWS_ACCESS_KEY_ID','${{env.AWS_ACCESS_KEY_ID}}')
-                                                        [System.Environment]::SetEnvironmentVariable('AWS_SECRET_ACCESS_KEY','${{env.AWS_SECRET_ACCESS_KEY}}')
-                                                        [System.Environment]::SetEnvironmentVariable('AWS_SESSION_TOKEN','${{env.AWS_SESSION_TOKEN }}')
-                                                        [System.Environment]::SetEnvironmentVariable('SCCACHE_BUCKET','${{env.SCCACHE_BUCKET}}')
-                                                        [System.Environment]::SetEnvironmentVariable('SCCACHE_REGION','${{env.SCCACHE_REGION}}')
-                                                        [System.Environment]::SetEnvironmentVariable('SCCACHE_IDLE_TIMEOUT','${{env.SCCACHE_IDLE_TIMEOUT}}')
-                                                        [System.Environment]::SetEnvironmentVariable('SCCACHE_S3_USE_SSL','${{env.SCCACHE_S3_USE_SSL}}')
-                                                        [System.Environment]::SetEnvironmentVariable('SCCACHE_S3_NO_CREDENTIALS','${{env.SCCACHE_S3_NO_CREDENTIALS}}')
-                                                        git clone https://github.com/NVIDIA/cccl.git;
-                                                        cd cccl;
-                                                        git fetch --all;
-                                                        git checkout ${{github.ref_name}};
-                                                        ${{inputs.command}}"
-      - name: Mark job as successful
+        run: |
+          docker run \
+            --mount type=bind,source="${{steps.paths.outputs.HOST_REPO}}",target="${{steps.paths.outputs.MOUNT_REPO}}" \
+            --workdir "${{steps.paths.outputs.MOUNT_REPO}}" \
+            ${{ inputs.image }} \
+            powershell -c "
+              [System.Environment]::SetEnvironmentVariable('AWS_ACCESS_KEY_ID','${{env.AWS_ACCESS_KEY_ID}}');
+              [System.Environment]::SetEnvironmentVariable('AWS_SECRET_ACCESS_KEY','${{env.AWS_SECRET_ACCESS_KEY}}');
+              [System.Environment]::SetEnvironmentVariable('AWS_SESSION_TOKEN','${{env.AWS_SESSION_TOKEN }}');
+              [System.Environment]::SetEnvironmentVariable('SCCACHE_BUCKET','${{env.SCCACHE_BUCKET}}');
+              [System.Environment]::SetEnvironmentVariable('SCCACHE_REGION','${{env.SCCACHE_REGION}}');
+              [System.Environment]::SetEnvironmentVariable('SCCACHE_IDLE_TIMEOUT','${{env.SCCACHE_IDLE_TIMEOUT}}');
+              [System.Environment]::SetEnvironmentVariable('SCCACHE_S3_USE_SSL','${{env.SCCACHE_S3_USE_SSL}}');
+              [System.Environment]::SetEnvironmentVariable('SCCACHE_S3_NO_CREDENTIALS','${{env.SCCACHE_S3_NO_CREDENTIALS}}');
+              git config --global --add safe.directory C:/cccl;
+              ${{inputs.command}}"
+      - name: Prepare job artifacts
         id: done
         run: |
-          echo "SUCCESS=true" | tee -a ${GITHUB_OUTPUT}
-          mkdir dispatch-job-success
-          touch dispatch-job-success/${{inputs.id}}
-      - name: Upload dispatch-job-success
+          echo "SUCCESS=true" | tee -a "${GITHUB_OUTPUT}"
+
+          result_dir="jobs/${{inputs.id}}"
+          mkdir -p "$result_dir"
+
+          touch "$result_dir/success"
+
+          # Finds a matching file in the repo directory and copies it to the results directory.
+          find_and_copy() {
+            filename="$1"
+            filepath="$(find ${{github.event.repository.name}} -name "${filename}" -print -quit)"
+            if [[ -z "$filepath" ]]; then
+              echo "${filename} does not exist in repo directory."
+              return 1
+            fi
+            cp -v "$filepath" "$result_dir"
+          }
+
+          find_and_copy "sccache_stats.json" || true # Ignore failures
+
+      - name: Upload job artifacts
         uses: actions/upload-artifact@v3
         with:
-          name: dispatch-job-success
-          path: dispatch-job-success/${{inputs.id}}
+          name: jobs
+          path: jobs