|
| 1 | +import csv |
| 2 | +import os |
| 3 | +import subprocess |
| 4 | +from typing import Dict, List |
| 5 | + |
| 6 | +# The nsys metrics to the reports. The value is the list of reports of nsys. |
| 7 | +nsys_metrics_to_reports = { |
| 8 | + # the sum of kernel execution time |
| 9 | + "nsys_gpu_kernel_sum": ["nvtx_kern_sum", "nvtx_sum"], |
| 10 | + # the overhead of kernel launch |
| 11 | + "nsys_launch_overhead": ["nvtx_kern_sum", "nvtx_sum"], |
| 12 | + # the names of kernels |
| 13 | + "nsys_kernel_names": ["nvtx_kern_sum"], |
| 14 | + # the durations of kernels |
| 15 | + "nsys_kernel_durations": ["nvtx_kern_sum"], |
| 16 | + # the duration of nvtx range |
| 17 | + "nsys_nvtx_range_duration": ["nvtx_sum"], |
| 18 | + # the number of kernels |
| 19 | + "nsys_num_of_kernels": ["nvtx_kern_sum"], |
| 20 | +} |
| 21 | + |
| 22 | + |
| 23 | +def read_nsys_report( |
| 24 | + report_path: str, required_metrics: List[str] |
| 25 | +) -> Dict[str, List[float]]: |
| 26 | + assert os.path.exists( |
| 27 | + report_path |
| 28 | + ), f"The nsys report at {report_path} does not exist." |
| 29 | + reports_required = [] |
| 30 | + for metric in required_metrics: |
| 31 | + if metric in nsys_metrics_to_reports: |
| 32 | + reports_required.extend(nsys_metrics_to_reports[metric]) |
| 33 | + reports_required = list(set(reports_required)) |
| 34 | + assert reports_required, "No nsys reports required" |
| 35 | + cmd = f"nsys stats --report {','.join(reports_required)} --force-export=true --format csv --output . --force-overwrite=true {report_path}" |
| 36 | + try: |
| 37 | + subprocess.check_call( |
| 38 | + cmd.split(), stdout=subprocess.DEVNULL, stderr=subprocess.PIPE |
| 39 | + ) |
| 40 | + except subprocess.CalledProcessError as e: |
| 41 | + print(f"Failed to run nsys command: {cmd}\nError: {e}") |
| 42 | + raise e |
| 43 | + # Get the base path and filename without extension |
| 44 | + base_path = os.path.dirname(report_path) |
| 45 | + base_name = os.path.splitext(os.path.basename(report_path))[0] |
| 46 | + |
| 47 | + results = {} |
| 48 | + csv_contents = {} |
| 49 | + |
| 50 | + for report in reports_required: |
| 51 | + csv_path = os.path.join(base_path, f"{base_name}_{report}.csv") |
| 52 | + if not os.path.exists(csv_path): |
| 53 | + raise RuntimeError(f"Expected CSV report not found at {csv_path}") |
| 54 | + |
| 55 | + # Read CSV using DictReader |
| 56 | + with open(csv_path, "r") as f: |
| 57 | + reader = csv.DictReader(f) |
| 58 | + csv_contents[report] = list(reader) |
| 59 | + kernel_duration = [] |
| 60 | + kernel_names = [] |
| 61 | + sum_kernel_duration = 0 |
| 62 | + nvtx_range_duration = 0 |
| 63 | + if "nvtx_kern_sum" in csv_contents: |
| 64 | + # gpu kernel execution time summary |
| 65 | + for row in csv_contents["nvtx_kern_sum"]: |
| 66 | + # use ms as the unit |
| 67 | + kernel_duration.append(float(row["Total Time (ns)"]) / 1_000_000) |
| 68 | + kernel_names.append(row["Kernel Name"]) |
| 69 | + sum_kernel_duration = sum(kernel_duration) |
| 70 | + if "nvtx_sum" in csv_contents: |
| 71 | + # It is supposed to be only one row. The nvtx range is `:tritonbench_range` |
| 72 | + assert len(csv_contents["nvtx_sum"]) == 1 |
| 73 | + # @TODO: nsys has a bug that the unit of nvtx range duration is ms sometimes. |
| 74 | + # waiting for nvidia replys. |
| 75 | + nvtx_range_duration = ( |
| 76 | + float(csv_contents["nvtx_sum"][0]["Total Time (ns)"]) / 1_000_000 |
| 77 | + ) |
| 78 | + |
| 79 | + # Define mapping of metrics to their values. The keys must be in nsys_bench_metrics. |
| 80 | + metrics_map = { |
| 81 | + # Because tritonbench takes the median of numerical values, we need to convert |
| 82 | + # the list of floats to a list of strings. |
| 83 | + "nsys_kernel_durations": [str(duration) for duration in kernel_duration], |
| 84 | + "nsys_kernel_names": kernel_names, |
| 85 | + "nsys_gpu_kernel_sum": sum_kernel_duration, |
| 86 | + "nsys_nvtx_range_duration": nvtx_range_duration, |
| 87 | + "nsys_launch_overhead": nvtx_range_duration - sum_kernel_duration, |
| 88 | + "nsys_num_of_kernels": len(kernel_names), |
| 89 | + } |
| 90 | + # Verify that metrics_map keys match nsys_metrics_to_reports keys |
| 91 | + assert set(metrics_map.keys()) == set(nsys_metrics_to_reports.keys()), ( |
| 92 | + f"Mismatch between metrics_map keys and nsys_metrics_to_reports keys.\n" |
| 93 | + f"metrics_map keys: {set(metrics_map.keys())}\n" |
| 94 | + f"nsys_metrics_to_reports keys: {set(nsys_metrics_to_reports.keys())}" |
| 95 | + ) |
| 96 | + # Add only requested metrics to results |
| 97 | + results.update( |
| 98 | + { |
| 99 | + metric: metrics_map[metric] |
| 100 | + for metric in required_metrics |
| 101 | + if metric in metrics_map |
| 102 | + } |
| 103 | + ) |
| 104 | + |
| 105 | + return results |
0 commit comments