diff --git a/kerneltools-post-process.py b/kerneltools-post-process.py index 4394e27..4ea22ba 100644 --- a/kerneltools-post-process.py +++ b/kerneltools-post-process.py @@ -6,7 +6,7 @@ Runs in the kernel tool's data directory (one per profiler instance). Dispatches to per-subtool handlers based on which output files exist. -Currently handles: turbostat, perf-stat. +Currently handles: turbostat, perf-stat, toplev. Metrics emitted --------------- @@ -23,9 +23,18 @@ turbostat:ipc throughput {cpu: N} perf-stat — per-CPU (from `perf stat -a -A -I N -x , -e cycles,instructions,...`): - perf-stat:ipc throughput {cpu: N} - perf-stat:cache-miss-rate utilization % {cpu: N} - perf-stat:llc-load-miss-rate utilization % {cpu: N} + perf-stat:ipc throughput {cpu: N} + perf-stat:cache-miss-rate utilization % {cpu: N} + perf-stat:backend-stall-rate utilization % {cpu: N} + perf-stat:frontend-stall-rate utilization % {cpu: N} + +toplev — system-wide Top-Down Methodology (from `toplev.py -l3 -I N -x ,`): + toplev:frontend-bound utilization % + toplev:backend-bound utilization % + toplev:memory-bound utilization % + toplev:core-bound utilization % + toplev:bad-speculation utilization % + toplev:retiring utilization % """ from __future__ import annotations @@ -133,6 +142,17 @@ def process_turbostat(log_file: str) -> None: SOURCE_PERF_STAT = "perf-stat" +SOURCE_TOPLEV = "toplev" + +# Mapping from toplev metric path fragments to CDM type names +_TOPLEV_METRIC_MAP = { + "Frontend_Bound": "frontend-bound", + "Backend_Bound.Memory_Bound": "memory-bound", + "Backend_Bound.Core_Bound": "core-bound", + "Backend_Bound": "backend-bound", + "Bad_Speculation": "bad-speculation", + "Retiring": "retiring", +} def process_perf_stat(log_file: str) -> None: @@ -193,12 +213,12 @@ def process_perf_stat(log_file: str) -> None: metrics = CDMMetrics() for (ts_ms, cpu), evts in sorted(intervals.items()): - cycles = evts.get("cycles", 0) - instructions = evts.get("instructions", 0) - cache_miss = evts.get("cache-misses", None) - cache_ref = evts.get("cache-references", None) - llc_miss = evts.get("LLC-load-misses", None) - llc_load = evts.get("LLC-loads", None) + cycles = evts.get("cycles", 0) + instructions = evts.get("instructions", 0) + cache_miss = evts.get("cache-misses", None) + cache_ref = evts.get("cache-references", None) + stall_backend = evts.get("stalled-cycles-backend", None) + stall_frontend = evts.get("stalled-cycles-frontend", None) # Strip "CPU" prefix for the breakout name cpu_num = cpu.replace("CPU", "") if cpu.startswith("CPU") else cpu @@ -223,11 +243,20 @@ def process_perf_stat(log_file: str) -> None: {**sample_base, "value": rate}, ) - if llc_miss is not None and llc_load is not None and llc_load > 0: - rate = llc_miss / llc_load * 100.0 + if stall_backend is not None and cycles > 0: + rate = stall_backend / cycles * 100.0 + metrics.log_sample( + SOURCE_PERF_STAT, + {"source": SOURCE_PERF_STAT, "class": "utilization", "type": "backend-stall-rate"}, + names, + {**sample_base, "value": rate}, + ) + + if stall_frontend is not None and cycles > 0: + rate = stall_frontend / cycles * 100.0 metrics.log_sample( SOURCE_PERF_STAT, - {"source": SOURCE_PERF_STAT, "class": "utilization", "type": "llc-load-miss-rate"}, + {"source": SOURCE_PERF_STAT, "class": "utilization", "type": "frontend-stall-rate"}, names, {**sample_base, "value": rate}, ) @@ -236,6 +265,71 @@ def process_perf_stat(log_file: str) -> None: print("Post-processing for perf-stat complete") +def process_toplev(log_file: str) -> None: + """Parse `toplev.py -l3 -I N -x ,` CSV output and emit CDM metrics. + + toplev CSV columns: + timestamp, cpu, area, metric, value, unit, [description, ...] + + With -x , and no --cpu flag, cpu field is empty (system-wide). + We emit each recognised Top-Down metric as a CDM utilization % sample. + """ + print(f"Post-processing toplev: {log_file}") + + try: + fh, _ = open_read_text_file(log_file) + except FileNotFoundError: + print(f"ERROR: could not open {log_file}") + return + + metrics = CDMMetrics() + found = 0 + + for raw_line in fh: + line = raw_line.strip() + if not line or line.startswith("#"): + continue + parts = line.split(",") + if len(parts) < 5: + continue + try: + ts_s = float(parts[0]) + metric = parts[3].strip() + val_s = parts[4].strip() + except (ValueError, IndexError): + continue + + if val_s in ("", "N/A", "nan"): + continue + try: + value = float(val_s) + except ValueError: + continue + + # Match metric to a known CDM type + cdm_type = None + for key, name in _TOPLEV_METRIC_MAP.items(): + if key in metric: + cdm_type = name + break + if cdm_type is None: + continue + + ts_ms = int(round(ts_s * 1000)) + desc = {"source": SOURCE_TOPLEV, "class": "utilization", "type": cdm_type} + metrics.log_sample(SOURCE_TOPLEV, desc, {}, {"end": ts_ms, "value": value}) + found += 1 + + fh.close() + + if found == 0: + print("WARNING: no toplev metric data found") + return + + metrics.finish_samples() + print(f"Post-processing for toplev complete ({found} data points)") + + def main() -> None: print("kerneltools-post-process") @@ -264,6 +358,16 @@ def main() -> None: elif perf_stat_files: process_perf_stat(perf_stat_files[0]) + toplev_files = [ + f for f in files + if re.match(r"^toplev-stdout\.txt(\.xz)?$", f) + ] + + if len(toplev_files) > 1: + print(f"ERROR: multiple toplev files found: {toplev_files}") + elif toplev_files: + process_toplev(toplev_files[0]) + print("kerneltools post-processing complete") diff --git a/kerneltools-start b/kerneltools-start index 45095a8..81d8c45 100755 --- a/kerneltools-start +++ b/kerneltools-start @@ -18,7 +18,7 @@ echo "mount:" mount echo # defaults -subtools="turbostat" # subtools that will be used, any combination of: turbostat,perf,perf-stat,speed-select-util,trace-cmd,sysfs-trace +subtools="turbostat" # subtools that will be used, any combination of: turbostat,perf,perf-stat,toplev,speed-select-util,trace-cmd,sysfs-trace interval=10 record_opts="" base_freq="" @@ -103,15 +103,33 @@ for subtool in `echo $subtools | sed -e 's/,/ /g'`; do echo "turbostat pid is $turbo_pid" echo "$turbo_pid" >>kerneltools-pids.txt ;; + toplev) + echo "Starting toplev (Top-Down Methodology analysis at ${interval}s intervals)" + # debugfs required for PMU event access + grep -q debugfs /proc/mounts || mount -t debugfs none /sys/kernel/debug 2>/dev/null || true + # toplev.py -l3: three-level Top-Down breakdown (Frontend/Backend-Memory/Backend-Core) + # -I : report every N milliseconds -x ,: CSV output for parsing + # -a: system-wide --no-multiplex: avoid multiplexing errors on busy systems + python3 /usr/local/share/pmu-tools/toplev.py \ + -l3 -I "$((interval * 1000))" -x , -a --no-multiplex \ + > toplev-stdout.txt 2>&1 & + toplev_pid=$! + echo "toplev pid is $toplev_pid" + echo "$toplev_pid" >>kerneltools-pids.txt + ;; perf-stat) echo "Starting perf-stat (per-CPU IPC at ${interval}s intervals)" # debugfs required for some PMU event access grep -q debugfs /proc/mounts || mount -t debugfs none /sys/kernel/debug 2>/dev/null || true # -a: system-wide -A: per-CPU (no aggregation) -I: interval in ms # -x,: CSV output for easier parsing - # Events: cycles, instructions (IPC), cache-misses, LLC-load-misses (cache pressure) + # Events (work on both Intel and AMD via generic hw aliases): + # cycles, instructions → IPC + # cache-misses → L2/L3 cache misses + # stalled-cycles-backend → backend stall % (memory, TLB, exec units) + # stalled-cycles-frontend → frontend stall % (fetch, decode) /usr/bin/perf stat -a -A -I "$((interval * 1000))" -x , \ - -e cycles,instructions,cache-misses,LLC-load-misses \ + -e cycles,instructions,cache-misses,stalled-cycles-backend,stalled-cycles-frontend \ > perf-stat-stdout.txt 2>&1 & perf_stat_pid=$! echo "perf-stat pid is $perf_stat_pid" diff --git a/kerneltools-stop b/kerneltools-stop index 3bd6935..46a4bbc 100755 --- a/kerneltools-stop +++ b/kerneltools-stop @@ -133,6 +133,14 @@ for subtool in `echo $subtools | sed -e 's/,/ /g'`; do echo "Warning: turbostat-stdout.txt was not found" fi ;; + toplev) + if [ -e toplev-stdout.txt ]; then + echo "Compressing toplev output" + ${taskset_cmd} xz --threads=0 toplev-stdout.txt + else + echo "Warning: toplev-stdout.txt was not found" + fi + ;; perf-stat) if [ -e perf-stat-stdout.txt ]; then echo "Compressing perf-stat output" diff --git a/workshop.json b/workshop.json index 11ef9f0..7e3ab17 100644 --- a/workshop.json +++ b/workshop.json @@ -8,7 +8,8 @@ { "name": "default", "requirements": [ - "packages" + "packages", + "pmu_tools" ] } ], @@ -161,6 +162,15 @@ } } }, + { + "name": "pmu_tools", + "type": "manual", + "manual_info": { + "commands": [ + "git clone https://github.com/andikleen/pmu-tools.git /usr/local/share/pmu-tools" + ] + } + }, { "name": "tools_src", "type": "source",