feat: add toplev subtool for Top-Down Methodology analysis

kerneltools-post-process.py

@@ -6,7 +6,7 @@
 
 Runs in the kernel tool's data directory (one per profiler instance).
 Dispatches to per-subtool handlers based on which output files exist.
-Currently handles: turbostat, perf-stat.
+Currently handles: turbostat, perf-stat, toplev.
 
 Metrics emitted
 ---------------
@@ -23,9 +23,18 @@
     turbostat:ipc                   throughput         {cpu: N}
 
 perf-stat — per-CPU (from `perf stat -a -A -I N -x , -e cycles,instructions,...`):
-    perf-stat:ipc                   throughput         {cpu: N}
-    perf-stat:cache-miss-rate       utilization  %     {cpu: N}
-    perf-stat:llc-load-miss-rate    utilization  %     {cpu: N}
+    perf-stat:ipc                       throughput         {cpu: N}
+    perf-stat:cache-miss-rate           utilization  %     {cpu: N}
+    perf-stat:backend-stall-rate        utilization  %     {cpu: N}
+    perf-stat:frontend-stall-rate       utilization  %     {cpu: N}
+
+toplev — system-wide Top-Down Methodology (from `toplev.py -l3 -I N -x ,`):
+    toplev:frontend-bound           utilization  %
+    toplev:backend-bound            utilization  %
+    toplev:memory-bound             utilization  %
+    toplev:core-bound               utilization  %
+    toplev:bad-speculation          utilization  %
+    toplev:retiring                 utilization  %
 """
 
 from __future__ import annotations
@@ -133,6 +142,17 @@ def process_turbostat(log_file: str) -> None:
 
 
 SOURCE_PERF_STAT = "perf-stat"
+SOURCE_TOPLEV = "toplev"
+
+# Mapping from toplev metric path fragments to CDM type names
+_TOPLEV_METRIC_MAP = {
+    "Frontend_Bound":              "frontend-bound",
+    "Backend_Bound.Memory_Bound":  "memory-bound",
+    "Backend_Bound.Core_Bound":    "core-bound",
+    "Backend_Bound":               "backend-bound",
+    "Bad_Speculation":             "bad-speculation",
+    "Retiring":                    "retiring",
+}
 
 
 def process_perf_stat(log_file: str) -> None:
@@ -193,12 +213,12 @@ def process_perf_stat(log_file: str) -> None:
     metrics = CDMMetrics()
 
     for (ts_ms, cpu), evts in sorted(intervals.items()):
-        cycles       = evts.get("cycles", 0)
-        instructions = evts.get("instructions", 0)
-        cache_miss   = evts.get("cache-misses", None)
-        cache_ref    = evts.get("cache-references", None)
-        llc_miss     = evts.get("LLC-load-misses", None)
-        llc_load     = evts.get("LLC-loads", None)
+        cycles          = evts.get("cycles", 0)
+        instructions    = evts.get("instructions", 0)
+        cache_miss      = evts.get("cache-misses", None)
+        cache_ref       = evts.get("cache-references", None)
+        stall_backend   = evts.get("stalled-cycles-backend", None)
+        stall_frontend  = evts.get("stalled-cycles-frontend", None)
 
         # Strip "CPU" prefix for the breakout name
         cpu_num = cpu.replace("CPU", "") if cpu.startswith("CPU") else cpu
@@ -223,11 +243,20 @@ def process_perf_stat(log_file: str) -> None:
                 {**sample_base, "value": rate},
             )
 
-        if llc_miss is not None and llc_load is not None and llc_load > 0:
-            rate = llc_miss / llc_load * 100.0
+        if stall_backend is not None and cycles > 0:
+            rate = stall_backend / cycles * 100.0
+            metrics.log_sample(
+                SOURCE_PERF_STAT,
+                {"source": SOURCE_PERF_STAT, "class": "utilization", "type": "backend-stall-rate"},
+                names,
+                {**sample_base, "value": rate},
+            )
+
+        if stall_frontend is not None and cycles > 0:
+            rate = stall_frontend / cycles * 100.0
             metrics.log_sample(
                 SOURCE_PERF_STAT,
-                {"source": SOURCE_PERF_STAT, "class": "utilization", "type": "llc-load-miss-rate"},
+                {"source": SOURCE_PERF_STAT, "class": "utilization", "type": "frontend-stall-rate"},
                 names,
                 {**sample_base, "value": rate},
             )
@@ -236,6 +265,71 @@ def process_perf_stat(log_file: str) -> None:
     print("Post-processing for perf-stat complete")
 
 
+def process_toplev(log_file: str) -> None:
+    """Parse `toplev.py -l3 -I N -x ,` CSV output and emit CDM metrics.
+
+    toplev CSV columns:
+      timestamp, cpu, area, metric, value, unit, [description, ...]
+
+    With -x , and no --cpu flag, cpu field is empty (system-wide).
+    We emit each recognised Top-Down metric as a CDM utilization % sample.
+    """
+    print(f"Post-processing toplev: {log_file}")
+
+    try:
+        fh, _ = open_read_text_file(log_file)
+    except FileNotFoundError:
+        print(f"ERROR: could not open {log_file}")
+        return
+
+    metrics = CDMMetrics()
+    found = 0
+
+    for raw_line in fh:
+        line = raw_line.strip()
+        if not line or line.startswith("#"):
+            continue
+        parts = line.split(",")
+        if len(parts) < 5:
+            continue
+        try:
+            ts_s   = float(parts[0])
+            metric = parts[3].strip()
+            val_s  = parts[4].strip()
+        except (ValueError, IndexError):
+            continue
+
+        if val_s in ("", "N/A", "nan"):
+            continue
+        try:
+            value = float(val_s)
+        except ValueError:
+            continue
+
+        # Match metric to a known CDM type
+        cdm_type = None
+        for key, name in _TOPLEV_METRIC_MAP.items():
+            if key in metric:
+                cdm_type = name
+                break
+        if cdm_type is None:
+            continue
+
+        ts_ms = int(round(ts_s * 1000))
+        desc = {"source": SOURCE_TOPLEV, "class": "utilization", "type": cdm_type}
+        metrics.log_sample(SOURCE_TOPLEV, desc, {}, {"end": ts_ms, "value": value})
+        found += 1
+
+    fh.close()
+
+    if found == 0:
+        print("WARNING: no toplev metric data found")
+        return
+
+    metrics.finish_samples()
+    print(f"Post-processing for toplev complete ({found} data points)")
+
+
 def main() -> None:
     print("kerneltools-post-process")
 
@@ -264,6 +358,16 @@ def main() -> None:
     elif perf_stat_files:
         process_perf_stat(perf_stat_files[0])
 
+    toplev_files = [
+        f for f in files
+        if re.match(r"^toplev-stdout\.txt(\.xz)?$", f)
+    ]
+
+    if len(toplev_files) > 1:
+        print(f"ERROR: multiple toplev files found: {toplev_files}")
+    elif toplev_files:
+        process_toplev(toplev_files[0])
+
     print("kerneltools post-processing complete")
 
 

kerneltools-start

@@ -18,7 +18,7 @@ echo "mount:"
 mount
 echo
 # defaults
-subtools="turbostat" # subtools that will be used, any combination of: turbostat,perf,perf-stat,speed-select-util,trace-cmd,sysfs-trace <no-spaces>
+subtools="turbostat" # subtools that will be used, any combination of: turbostat,perf,perf-stat,toplev,speed-select-util,trace-cmd,sysfs-trace <no-spaces>
 interval=10
 record_opts=""
 base_freq=""
@@ -103,15 +103,33 @@ for subtool in `echo $subtools | sed -e 's/,/ /g'`; do
             echo "turbostat pid is $turbo_pid"
             echo "$turbo_pid" >>kerneltools-pids.txt
             ;;
+        toplev)
+            echo "Starting toplev (Top-Down Methodology analysis at ${interval}s intervals)"
+            # debugfs required for PMU event access
+            grep -q debugfs /proc/mounts || mount -t debugfs none /sys/kernel/debug 2>/dev/null || true
+            # toplev.py -l3: three-level Top-Down breakdown (Frontend/Backend-Memory/Backend-Core)
+            # -I <ms>: report every N milliseconds  -x ,: CSV output for parsing
+            # -a: system-wide  --no-multiplex: avoid multiplexing errors on busy systems
+            python3 /usr/local/share/pmu-tools/toplev.py \
+                -l3 -I "$((interval * 1000))" -x , -a --no-multiplex \
+                > toplev-stdout.txt 2>&1 &
+            toplev_pid=$!
+            echo "toplev pid is $toplev_pid"
+            echo "$toplev_pid" >>kerneltools-pids.txt
+            ;;
         perf-stat)
             echo "Starting perf-stat (per-CPU IPC at ${interval}s intervals)"
             # debugfs required for some PMU event access
             grep -q debugfs /proc/mounts || mount -t debugfs none /sys/kernel/debug 2>/dev/null || true
             # -a: system-wide  -A: per-CPU (no aggregation)  -I: interval in ms
             # -x,: CSV output for easier parsing
-            # Events: cycles, instructions (IPC), cache-misses, LLC-load-misses (cache pressure)
+            # Events (work on both Intel and AMD via generic hw aliases):
+            #   cycles, instructions            → IPC
+            #   cache-misses                    → L2/L3 cache misses
+            #   stalled-cycles-backend          → backend stall % (memory, TLB, exec units)
+            #   stalled-cycles-frontend         → frontend stall % (fetch, decode)
             /usr/bin/perf stat -a -A -I "$((interval * 1000))" -x , \
-                -e cycles,instructions,cache-misses,LLC-load-misses \
+                -e cycles,instructions,cache-misses,stalled-cycles-backend,stalled-cycles-frontend \
                 > perf-stat-stdout.txt 2>&1 &
             perf_stat_pid=$!
             echo "perf-stat pid is $perf_stat_pid"

kerneltools-stop

@@ -133,6 +133,14 @@ for subtool in `echo $subtools | sed -e 's/,/ /g'`; do
                 echo "Warning: turbostat-stdout.txt was not found"
             fi
             ;;
+        toplev)
+            if [ -e toplev-stdout.txt ]; then
+                echo "Compressing toplev output"
+                ${taskset_cmd} xz --threads=0 toplev-stdout.txt
+            else
+                echo "Warning: toplev-stdout.txt was not found"
+            fi
+            ;;
         perf-stat)
             if [ -e perf-stat-stdout.txt ]; then
                 echo "Compressing perf-stat output"

workshop.json

@@ -8,7 +8,8 @@
         {
             "name": "default",
             "requirements": [
-		"packages"
+		"packages",
+		"pmu_tools"
             ]
         }
     ],
@@ -161,6 +162,15 @@
                 }
             }
         },
+        {
+            "name": "pmu_tools",
+            "type": "manual",
+            "manual_info": {
+                "commands": [
+                    "git clone https://github.com/andikleen/pmu-tools.git /usr/local/share/pmu-tools"
+                ]
+            }
+        },
         {
             "name": "tools_src",
             "type": "source",