From d1a2044573ba2d3f73daa4573be565113aa58b33 Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Mon, 7 Jun 2021 16:20:22 +0100
Subject: [PATCH 01/74] Refactor instruction tracing to support events.

- Rename cpu_log_instr_info to cpu_log_entry.
---
 accel/tcg/log_instr.c    | 456 ++++++++++++++++++++-------------------
 include/exec/log_instr.h |  96 +++++----
 include/qemu/log_instr.h |   2 +-
 3 files changed, 283 insertions(+), 271 deletions(-)

diff --git a/accel/tcg/log_instr.c b/accel/tcg/log_instr.c
index d454eeda7bc..c40f67df639 100644
--- a/accel/tcg/log_instr.c
+++ b/accel/tcg/log_instr.c
@@ -67,14 +67,14 @@
  * active. Each CPU holds a private logging state, that can be controlled
  * individually.
  *
- * TODO(am2419): how do we deal with orderding in case multiple registers are updated?
- * This is critical to recognize which value goes in which register, and also how to
- * tie multiple memory accesses to the respective value/register.
- * We could add an explicit target-specific register ID handle in place of the
- * register name. This could be used also to fetch the register name and would
- * provide an identifier to external parsers.
- * Memory updates are harder to deal with, at least in the current format, perhaps
- * the semantic of the instruction is enough to recover the ordering from a trace.
+ * TODO(am2419): how do we deal with orderding in case multiple registers are
+ * updated? This is critical to recognize which value goes in which register,
+ * and also how to tie multiple memory accesses to the respective
+ * value/register. We could add an explicit target-specific register ID handle
+ * in place of the register name. This could be used also to fetch the register
+ * name and would provide an identifier to external parsers. Memory updates are
+ * harder to deal with, at least in the current format, perhaps the semantic of
+ * the instruction is enough to recover the ordering from a trace.
  */
 
 #ifdef CONFIG_TCG_LOG_INSTR
@@ -100,15 +100,15 @@ extern GArray *debug_regions;
  * Instruction log info associated with each committed log entry.
  * This is stored in the per-cpu log cpustate.
  */
-typedef struct cpu_log_instr_info {
-#define cpu_log_iinfo_startzero asid
+typedef struct cpu_log_entry {
+#define cpu_log_entry_startzero asid
     uint16_t asid;
     int flags;
 /* Entry contains a synchronous exception */
 #define LI_FLAG_INTR_TRAP 1
 /* Entry contains an asynchronous exception */
 #define LI_FLAG_INTR_ASYNC (1 << 1)
-#define LI_FLAG_INTR_MASK 0x3
+#define LI_FLAG_INTR_MASK  0x3
 /* Entry contains a CPU mode-switch and associated code */
 #define LI_FLAG_MODE_SWITCH (1 << 2)
 
@@ -121,7 +121,7 @@ typedef struct cpu_log_instr_info {
     /* Generic instruction opcode buffer */
     int insn_size;
     char insn_bytes[TARGET_MAX_INSN_SIZE];
-#define cpu_log_iinfo_endzero mem
+#define cpu_log_entry_endzero mem
     /*
      * For now we allow multiple accesses to be tied to one instruction.
      * Some architectures may have multiple memory accesses
@@ -136,7 +136,7 @@ typedef struct cpu_log_instr_info {
     GArray *regs;
     /* Extra text-only log */
     GString *txt_buffer;
-} cpu_log_instr_info_t;
+} cpu_log_entry_t;
 
 /*
  * Register update info.
@@ -144,8 +144,8 @@ typedef struct cpu_log_instr_info {
  */
 typedef struct {
     uint16_t flags;
-#define LRI_CAP_REG    1
-#define LRI_HOLDS_CAP  2
+#define LRI_CAP_REG   1
+#define LRI_HOLDS_CAP 2
 
     const char *name;
     union {
@@ -156,7 +156,7 @@ typedef struct {
     };
 } log_reginfo_t;
 
-#define reginfo_is_cap(ri) (ri->flags & LRI_CAP_REG)
+#define reginfo_is_cap(ri)  (ri->flags & LRI_CAP_REG)
 #define reginfo_has_cap(ri) (reginfo_is_cap(ri) && (ri->flags & LRI_HOLDS_CAP))
 
 /*
@@ -165,8 +165,8 @@ typedef struct {
  */
 typedef struct {
     uint8_t flags;
-#define LMI_LD 1
-#define LMI_ST 2
+#define LMI_LD  1
+#define LMI_ST  2
 #define LMI_CAP 4
     MemOp op;
     target_ulong addr;
@@ -187,7 +187,7 @@ struct trace_fmt_hooks {
     void (*emit_header)(CPUArchState *env);
     void (*emit_start)(CPUArchState *env, target_ulong pc);
     void (*emit_stop)(CPUArchState *env, target_ulong pc);
-    void (*emit_entry)(CPUArchState *env, cpu_log_instr_info_t *iinfo);
+    void (*emit_entry)(CPUArchState *env, cpu_log_entry_t *entry);
 };
 typedef struct trace_fmt_hooks trace_fmt_hooks_t;
 
@@ -210,30 +210,30 @@ static trace_fmt_hooks_t trace_formats[];
  */
 typedef struct {
     uint8_t entry_type;
-#define CTE_NO_REG  0   /* No register is changed. */
-#define CTE_GPR     1   /* GPR change (val2) */
-#define CTE_LD_GPR  2   /* Load into GPR (val2) from address (val1) */
-#define CTE_ST_GPR  3   /* Store from GPR (val2) to address (val1) */
-#define CTE_CAP     11  /* Cap change (val2,val3,val4,val5) */
-#define CTE_LD_CAP  12  /* Load Cap (val2,val3,val4,val5) from addr (val1) */
-#define CTE_ST_CAP  13  /* Store Cap (val2,val3,val4,val5) to addr (val1) */
-    uint8_t exception;  /* 0=none, 1=TLB Mod, 2=TLB Load, 3=TLB Store, etc. */
+#define CTE_NO_REG 0   /* No register is changed. */
+#define CTE_GPR    1   /* GPR change (val2) */
+#define CTE_LD_GPR 2   /* Load into GPR (val2) from address (val1) */
+#define CTE_ST_GPR 3   /* Store from GPR (val2) to address (val1) */
+#define CTE_CAP    11  /* Cap change (val2,val3,val4,val5) */
+#define CTE_LD_CAP 12  /* Load Cap (val2,val3,val4,val5) from addr (val1) */
+#define CTE_ST_CAP 13  /* Store Cap (val2,val3,val4,val5) to addr (val1) */
+    uint8_t exception; /* 0=none, 1=TLB Mod, 2=TLB Load, 3=TLB Store, etc. */
 #define CTE_EXCEPTION_NONE 31
-    uint16_t cycles;    /* Currently not used. */
-    uint32_t inst;      /* Encoded instruction. */
-    uint64_t pc;        /* PC value of instruction. */
-    uint64_t val1;      /* val1 is used for memory address. */
-    uint64_t val2;      /* val2, val3, val4, val5 are used for reg content. */
+    uint16_t cycles; /* Currently not used. */
+    uint32_t inst;   /* Encoded instruction. */
+    uint64_t pc;     /* PC value of instruction. */
+    uint64_t val1;   /* val1 is used for memory address. */
+    uint64_t val2;   /* val2, val3, val4, val5 are used for reg content. */
     uint64_t val3;
     uint64_t val4;
     uint64_t val5;
-    uint8_t thread;     /* Hardware thread/CPU (i.e. cpu->cpu_index ) */
-    uint8_t asid;       /* Address Space ID */
+    uint8_t thread; /* Hardware thread/CPU (i.e. cpu->cpu_index ) */
+    uint8_t asid;   /* Address Space ID */
 } __attribute__((packed)) cheri_trace_entry_t;
 
 /* Version 3 Cheri Stream Trace header info */
-#define CTE_QEMU_VERSION    (0x80U + 3)
-#define CTE_QEMU_MAGIC      "CheriTraceV03"
+#define CTE_QEMU_VERSION (0x80U + 3)
+#define CTE_QEMU_MAGIC   "CheriTraceV03"
 
 /* Number of per-cpu ring buffer entries for ring-buffer tracing mode */
 #define MIN_ENTRY_BUFFER_SIZE (1 << 16)
@@ -251,12 +251,12 @@ static inline cpu_log_instr_state_t *get_cpu_log_state(CPUArchState *env)
 /*
  * Fetch the given cpu current instruction info
  */
-static inline cpu_log_instr_info_t *get_cpu_log_instr_info(CPUArchState *env)
+static inline cpu_log_entry_t *get_cpu_log_entry(CPUArchState *env)
 {
     cpu_log_instr_state_t *cpulog = get_cpu_log_state(env);
 
-    return &g_array_index(cpulog->instr_info, cpu_log_instr_info_t,
-        cpulog->ring_head);
+    return &g_array_index(cpulog->instr_info, cpu_log_entry_t,
+                          cpulog->ring_head);
 }
 
 /* Text trace format emitters */
@@ -272,8 +272,8 @@ static inline void emit_text_ldst(log_meminfo_t *minfo, const char *direction)
                "Capability memory access without CHERI support");
 #else
     if (minfo->flags & LMI_CAP) {
-        qemu_log("    Cap Memory %s [" TARGET_FMT_lx "] = v:%d PESBT:"
-                 TARGET_FMT_lx " Cursor:" TARGET_FMT_lx "\n",
+        qemu_log("    Cap Memory %s [" TARGET_FMT_lx
+                 "] = v:%d PESBT:" TARGET_FMT_lx " Cursor:" TARGET_FMT_lx "\n",
                  direction, minfo->addr, minfo->cap.cr_tag,
                  CAP_cc(compress_mem)(&minfo->cap),
                  cap_get_cursor(&minfo->cap));
@@ -289,16 +289,16 @@ static inline void emit_text_ldst(log_meminfo_t *minfo, const char *direction)
                      direction, minfo->addr, minfo->value);
             break;
         case 4:
-            qemu_log("    Memory %s [" TARGET_FMT_lx "] = %08x\n",
-                     direction, minfo->addr, (uint32_t)minfo->value);
+            qemu_log("    Memory %s [" TARGET_FMT_lx "] = %08x\n", direction,
+                     minfo->addr, (uint32_t)minfo->value);
             break;
         case 2:
-            qemu_log("    Memory %s [" TARGET_FMT_lx "] = %04x\n",
-                     direction, minfo->addr, (uint16_t)minfo->value);
+            qemu_log("    Memory %s [" TARGET_FMT_lx "] = %04x\n", direction,
+                     minfo->addr, (uint16_t)minfo->value);
             break;
         case 1:
-            qemu_log("    Memory %s [" TARGET_FMT_lx "] = %02x\n",
-                     direction, minfo->addr, (uint8_t)minfo->value);
+            qemu_log("    Memory %s [" TARGET_FMT_lx "] = %02x\n", direction,
+                     minfo->addr, (uint8_t)minfo->value);
             break;
         }
     }
@@ -311,14 +311,13 @@ static inline void emit_text_reg(log_reginfo_t *rinfo)
 {
 #ifndef TARGET_CHERI
     log_assert(!reginfo_is_cap(rinfo) && "Register marked as capability "
-               "register whitout CHERI support");
+                                         "register whitout CHERI support");
 #else
     if (reginfo_is_cap(rinfo)) {
         if (reginfo_has_cap(rinfo))
             qemu_log("    Write %s|" PRINT_CAP_FMTSTR_L1 "\n"
                      "             |" PRINT_CAP_FMTSTR_L2 "\n",
-                     rinfo->name,
-                     PRINT_CAP_ARGS_L1(&rinfo->cap),
+                     rinfo->name, PRINT_CAP_ARGS_L1(&rinfo->cap),
                      PRINT_CAP_ARGS_L2(&rinfo->cap));
         else
             qemu_log("  %s <- " TARGET_FMT_lx " (setting integer value)\n",
@@ -326,21 +325,20 @@ static inline void emit_text_reg(log_reginfo_t *rinfo)
     } else
 #endif
     {
-        qemu_log("    Write %s = " TARGET_FMT_lx "\n", rinfo->name,
-                 rinfo->gpr);
+        qemu_log("    Write %s = " TARGET_FMT_lx "\n", rinfo->name, rinfo->gpr);
     }
 }
 
 /*
  * Emit textual trace entry to the log.
  */
-static void emit_text_entry(CPUArchState *env, cpu_log_instr_info_t *iinfo)
+static void emit_text_entry(CPUArchState *env, cpu_log_entry_t *entry)
 {
     QemuLogFile *logfile;
     int i;
 
     /* Dump CPU-ID:ASID + address */
-    qemu_log("[%d:%d] ", env_cpu(env)->cpu_index, iinfo->asid);
+    qemu_log("[%d:%d] ", env_cpu(env)->cpu_index, entry->asid);
 
     /*
      * Instruction disassembly, note that we use the instruction info
@@ -349,8 +347,8 @@ static void emit_text_entry(CPUArchState *env, cpu_log_instr_info_t *iinfo)
     rcu_read_lock();
     logfile = qatomic_rcu_read(&qemu_logfile);
     if (logfile) {
-        target_disas_buf(logfile->fd, env_cpu(env), iinfo->insn_bytes,
-                         sizeof(iinfo->insn_bytes), iinfo->pc, 1);
+        target_disas_buf(logfile->fd, env_cpu(env), entry->insn_bytes,
+                         sizeof(entry->insn_bytes), entry->pc, 1);
     }
     rcu_read_unlock();
 
@@ -360,18 +358,19 @@ static void emit_text_entry(CPUArchState *env, cpu_log_instr_info_t *iinfo)
      */
 
     /* Dump mode switching info */
-    if (iinfo->flags & LI_FLAG_MODE_SWITCH)
-        qemu_log("-> Switch to %s mode\n", cpu_get_mode_name(iinfo->next_cpu_mode));
+    if (entry->flags & LI_FLAG_MODE_SWITCH)
+        qemu_log("-> Switch to %s mode\n",
+                 cpu_get_mode_name(entry->next_cpu_mode));
     /* Dump interrupt/exception info */
-    switch (iinfo->flags & LI_FLAG_INTR_MASK) {
+    switch (entry->flags & LI_FLAG_INTR_MASK) {
     case LI_FLAG_INTR_TRAP:
         qemu_log("-> Exception #%u vector 0x" TARGET_FMT_lx
                  " fault-addr 0x" TARGET_FMT_lx "\n",
-                 iinfo->intr_code, iinfo->intr_vector, iinfo->intr_faultaddr);
+                 entry->intr_code, entry->intr_vector, entry->intr_faultaddr);
         break;
     case LI_FLAG_INTR_ASYNC:
         qemu_log("-> Interrupt #%04x vector 0x" TARGET_FMT_lx "\n",
-                 iinfo->intr_code, iinfo->intr_vector);
+                 entry->intr_code, entry->intr_vector);
         break;
     default:
         /* No interrupt */
@@ -379,8 +378,8 @@ static void emit_text_entry(CPUArchState *env, cpu_log_instr_info_t *iinfo)
     }
 
     /* Dump memory access */
-    for (i = 0; i < iinfo->mem->len; i++) {
-        log_meminfo_t *minfo = &g_array_index(iinfo->mem, log_meminfo_t, i);
+    for (i = 0; i < entry->mem->len; i++) {
+        log_meminfo_t *minfo = &g_array_index(entry->mem, log_meminfo_t, i);
         if (minfo->flags & LMI_LD) {
             emit_text_ldst(minfo, "Read");
         } else if (minfo->flags & LMI_ST) {
@@ -389,14 +388,15 @@ static void emit_text_entry(CPUArchState *env, cpu_log_instr_info_t *iinfo)
     }
 
     /* Dump register changes and side-effects */
-    for (i = 0; i < iinfo->regs->len; i++) {
-        log_reginfo_t *rinfo = &g_array_index(iinfo->regs, log_reginfo_t, i);
+    for (i = 0; i < entry->regs->len; i++) {
+        log_reginfo_t *rinfo = &g_array_index(entry->regs, log_reginfo_t, i);
         emit_text_reg(rinfo);
     }
 
     /* Dump extra logged messages, if any */
-    if (iinfo->txt_buffer->len > 0)
-        qemu_log("%s", iinfo->txt_buffer->str);
+    if (entry->txt_buffer->len > 0) {
+        qemu_log("%s", entry->txt_buffer->str);
+    }
 }
 
 /*
@@ -453,37 +453,38 @@ static void emit_cvtrace_header(CPUArchState *env)
  * Emit cvtrace trace entry.
  * Note: this format is very MIPS-specific.
  */
-static void emit_cvtrace_entry(CPUArchState *env, cpu_log_instr_info_t *iinfo)
+static void emit_cvtrace_entry(CPUArchState *env, cpu_log_entry_t *entry)
 {
     FILE *logfile;
     cheri_trace_entry_t entry;
-    static uint16_t cycles = 0; // TODO(am2419): this should be a per-cpu counter.
-    uint32_t *insn = (uint32_t *)&iinfo->insn_bytes[0];
+    /* TODO(am2419): this should be a per-cpu counter. */
+    static uint16_t cycles;
+    uint32_t *insn = (uint32_t *)&entry->insn_bytes[0];
 
     entry.entry_type = CTE_NO_REG;
     entry.thread = (uint8_t)env_cpu(env)->cpu_index;
-    entry.asid = (uint8_t)iinfo->asid;
-    entry.pc = cpu_to_be64(iinfo->pc);
+    entry.asid = (uint8_t)entry->asid;
+    entry.pc = cpu_to_be64(entry->pc);
     entry.cycles = cpu_to_be16(cycles++);
     /*
-     * TODO(am2419): The instruction bytes are alread in target byte-order, however
-     * cheritrace does not currently expect this.
+     * TODO(am2419): The instruction bytes are alread in target byte-order,
+     * however cheritrace does not currently expect this.
      */
     entry.inst = cpu_to_be32(*insn);
-    switch (iinfo->flags & LI_FLAG_INTR_MASK) {
+    switch (entry->flags & LI_FLAG_INTR_MASK) {
     case LI_FLAG_INTR_TRAP:
-        entry.exception = (uint8_t)(iinfo->intr_code & 0xff);
+        entry.exception = (uint8_t)(entry->intr_code & 0xff);
     case LI_FLAG_INTR_ASYNC:
         entry.exception = 0;
     default:
         entry.exception = CTE_EXCEPTION_NONE;
     }
 
-    if (iinfo->regs->len) {
-        log_reginfo_t *rinfo = &g_array_index(iinfo->regs, log_reginfo_t, 0);
+    if (entry->regs->len) {
+        log_reginfo_t *rinfo = &g_array_index(entry->regs, log_reginfo_t, 0);
 #ifndef TARGET_CHERI
         log_assert(!reginfo_is_cap(rinfo) && "Capability register access "
-                   "without CHERI support");
+                                             "without CHERI support");
 #else
         if (reginfo_is_cap(rinfo)) {
             cap_register_t intcap;
@@ -511,11 +512,11 @@ static void emit_cvtrace_entry(CPUArchState *env, cpu_log_instr_info_t *iinfo)
         }
     }
 
-    if (iinfo->mem->len) {
-        log_meminfo_t *minfo = &g_array_index(iinfo->mem, log_meminfo_t, 0);
+    if (entry->mem->len) {
+        log_meminfo_t *minfo = &g_array_index(entry->mem, log_meminfo_t, 0);
 #ifndef TARGET_CHERI
         log_assert((minfo->flags & LMI_CAP) == 0 && "Capability memory access "
-                   "without CHERI support");
+                                                    "without CHERI support");
 #endif
         entry.val1 = cpu_to_be64(minfo->addr);
         // Hack to avoid checking for GPR or CAP
@@ -554,30 +555,30 @@ static inline void emit_stop_event(CPUArchState *env, target_ulong pc)
         trace_format->emit_stop(env, pc);
 }
 
-static inline void emit_entry_event(CPUArchState *env, cpu_log_instr_info_t *iinfo)
+static inline void emit_entry_event(CPUArchState *env, cpu_log_entry_t *entry)
 {
     cpu_log_instr_state_t *cpulog = get_cpu_log_state(env);
 
     if (cpulog->flags & QEMU_LOG_INSTR_FLAG_BUFFERED) {
         cpulog->ring_head = (cpulog->ring_head + 1) % cpulog->instr_info->len;
         if (cpulog->ring_tail == cpulog->ring_head)
-            cpulog->ring_tail = (cpulog->ring_tail + 1) % cpulog->instr_info->len;
-    }
-    else {
-        trace_format->emit_entry(env, iinfo);
+            cpulog->ring_tail =
+                (cpulog->ring_tail + 1) % cpulog->instr_info->len;
+    } else {
+        trace_format->emit_entry(env, entry);
     }
 }
 
 /* Reset instruction info buffer for next instruction */
 static void reset_log_buffer(cpu_log_instr_state_t *cpulog,
-                             cpu_log_instr_info_t *iinfo)
-{
-    memset(&iinfo->cpu_log_iinfo_startzero, 0,
-           ((char *)&iinfo->cpu_log_iinfo_endzero -
-            (char *)&iinfo->cpu_log_iinfo_startzero));
-    g_array_remove_range(iinfo->regs, 0, iinfo->regs->len);
-    g_array_remove_range(iinfo->mem, 0, iinfo->mem->len);
-    g_string_erase(iinfo->txt_buffer, 0, -1);
+                             cpu_log_entry_t *entry)
+{
+    memset(&entry->cpu_log_entry_startzero, 0,
+           ((char *)&entry->cpu_log_entry_endzero -
+            (char *)&entry->cpu_log_entry_startzero));
+    g_array_remove_range(entry->regs, 0, entry->regs->len);
+    g_array_remove_range(entry->mem, 0, entry->mem->len);
+    g_string_erase(entry->txt_buffer, 0, -1);
     cpulog->force_drop = false;
     cpulog->starting = false;
 }
@@ -586,10 +587,10 @@ static void reset_log_buffer(cpu_log_instr_state_t *cpulog,
 static void do_instr_commit(CPUArchState *env)
 {
     cpu_log_instr_state_t *cpulog = get_cpu_log_state(env);
-    cpu_log_instr_info_t *iinfo = get_cpu_log_instr_info(env);
+    cpu_log_entry_t *entry = get_cpu_log_entry(env);
 
     log_assert(cpulog != NULL && "Invalid log state");
-    log_assert(iinfo != NULL && "Invalid log buffer");
+    log_assert(entry != NULL && "Invalid log buffer");
 
     if (cpulog->force_drop)
         return;
@@ -606,22 +607,23 @@ static void do_instr_commit(CPUArchState *env)
         bool match = false;
         for (i = 0; !match && i < debug_regions->len; i++) {
             Range *range = &g_array_index(debug_regions, Range, i);
-            match = range_contains(range, iinfo->pc);
+            match = range_contains(range, entry->pc);
             if (match)
                 break;
 
-            for (j = 0; j < iinfo->mem->len; j++) {
-                log_meminfo_t *minfo = &g_array_index(iinfo->mem, log_meminfo_t, j);
+            for (j = 0; j < entry->mem->len; j++) {
+                log_meminfo_t *minfo =
+                    &g_array_index(entry->mem, log_meminfo_t, j);
                 match = range_contains(range, minfo->addr);
                 if (match)
                     break;
             }
         }
         if (match)
-            emit_entry_event(env, iinfo);
+            emit_entry_event(env, entry);
     } else {
         /* dfilter disabled, always log */
-        emit_entry_event(env, iinfo);
+        emit_entry_event(env, entry);
     }
 }
 
@@ -640,7 +642,7 @@ static void do_cpu_loglevel_switch(CPUState *cpu, run_on_cpu_data data)
 {
     CPUArchState *env = cpu->env_ptr;
     cpu_log_instr_state_t *cpulog = get_cpu_log_state(env);
-    cpu_log_instr_info_t *iinfo = get_cpu_log_instr_info(env);
+    cpu_log_entry_t *entry = get_cpu_log_entry(env);
     qemu_log_instr_loglevel_t prev_level = cpulog->loglevel;
     bool prev_level_active = cpulog->loglevel_active;
     qemu_log_instr_loglevel_t next_level = data.host_int;
@@ -656,18 +658,18 @@ static void do_cpu_loglevel_switch(CPUState *cpu, run_on_cpu_data data)
         break;
     case QEMU_LOG_INSTR_LOGLEVEL_USER:
         /*
-         * Assume iinfo holds the mode switch that caused cpu_loglevel_switch
+         * Assume entry holds the mode switch that caused cpu_loglevel_switch
          * to be called
          */
-        if (iinfo->flags & LI_FLAG_MODE_SWITCH)
-            next_level_active = (iinfo->next_cpu_mode == QEMU_LOG_INSTR_CPU_USER);
+        if (entry->flags & LI_FLAG_MODE_SWITCH)
+            next_level_active =
+                (entry->next_cpu_mode == QEMU_LOG_INSTR_CPU_USER);
         else
             next_level_active = cpu_in_user_mode(env);
         break;
     default:
         log_assert(false && "Invalid cpu instruction log level");
-        warn_report("Invalid cpu %d instruction log level\r",
-                    cpu->cpu_index);
+        warn_report("Invalid cpu %d instruction log level\r", cpu->cpu_index);
     }
 
     /* Update level */
@@ -684,14 +686,14 @@ static void do_cpu_loglevel_switch(CPUState *cpu, run_on_cpu_data data)
     /* Emit start/stop events */
     if (prev_level_active) {
         if (cpulog->starting) {
-            reset_log_buffer(cpulog, iinfo);
+            reset_log_buffer(cpulog, entry);
             return;
         }
         do_instr_commit(env);
         emit_stop_event(env, cpu_get_recent_pc(env));
-        /* Instruction commit may have advanced to the next iinfo buffer slot */
-        iinfo = get_cpu_log_instr_info(env);
-        reset_log_buffer(cpulog, iinfo);
+        /* Instruction commit may have advanced to the next entry buffer slot */
+        entry = get_cpu_log_entry(env);
+        reset_log_buffer(cpulog, entry);
     }
     if (next_level_active) {
         cpulog->starting = true;
@@ -699,10 +701,10 @@ static void do_cpu_loglevel_switch(CPUState *cpu, run_on_cpu_data data)
 }
 
 static void cpu_loglevel_switch(CPUArchState *env,
-    qemu_log_instr_loglevel_t level)
+                                qemu_log_instr_loglevel_t level)
 {
     async_safe_run_on_cpu(env_cpu(env), do_cpu_loglevel_switch,
-        RUN_ON_CPU_HOST_INT(level));
+                          RUN_ON_CPU_HOST_INT(level));
 }
 
 /* Start global logging flag if it was disabled */
@@ -754,9 +756,10 @@ int qemu_log_instr_global_switch(int log_flags)
         level = QEMU_LOG_INSTR_LOGLEVEL_NONE;
     }
 
-    CPU_FOREACH(cpu) {
+    CPU_FOREACH(cpu)
+    {
         async_safe_run_on_cpu(cpu, do_global_loglevel_switch,
-            RUN_ON_CPU_HOST_INT(level));
+                              RUN_ON_CPU_HOST_INT(level));
     }
 
     return log_flags;
@@ -765,26 +768,29 @@ int qemu_log_instr_global_switch(int log_flags)
 /*
  * Initialize instruction info entry from the ring buffer.
  */
-static void qemu_log_instr_info_init(cpu_log_instr_info_t *iinfo)
+static void qemu_log_entry_init(cpu_log_entry_t *entry)
 {
-    if (iinfo->txt_buffer == NULL)
-        iinfo->txt_buffer = g_string_new(NULL);
-    if (iinfo->regs == NULL)
-        iinfo->regs = g_array_new(false, true, sizeof(log_reginfo_t));
-    if (iinfo->mem == NULL)
-        iinfo->mem = g_array_new(false, true, sizeof(log_meminfo_t));
+    if (entry->txt_buffer == NULL) {
+        entry->txt_buffer = g_string_new(NULL);
+    }
+    if (entry->regs == NULL) {
+        entry->regs = g_array_new(false, true, sizeof(log_reginfo_t));
+    }
+    if (entry->mem == NULL) {
+        entry->mem = g_array_new(false, true, sizeof(log_meminfo_t));
+    }
 }
 
 /*
  * Clear an instruction info entry from the ring buffer.
  */
-static void qemu_log_instr_info_destroy(gpointer data)
+static void qemu_log_entry_destroy(gpointer data)
 {
-    cpu_log_instr_info_t *iinfo = data;
+    cpu_log_entry_t *entry = data;
 
-    g_string_free(iinfo->txt_buffer, TRUE);
-    g_array_free(iinfo->regs, TRUE);
-    g_array_free(iinfo->mem, TRUE);
+    g_string_free(entry->txt_buffer, TRUE);
+    g_array_free(entry->regs, TRUE);
+    g_array_free(entry->mem, TRUE);
 }
 
 /*
@@ -797,25 +803,25 @@ static void qemu_log_instr_info_destroy(gpointer data)
 void qemu_log_instr_init(CPUState *cpu)
 {
     cpu_log_instr_state_t *cpulog = &cpu->log_state;
-    GArray *iinfo_ring = g_array_sized_new(FALSE, TRUE,
-        sizeof(cpu_log_instr_info_t), reset_entry_buffer_size);
-    cpu_log_instr_info_t *iinfo;
+    GArray *entry_ring = g_array_sized_new(false, true, sizeof(cpu_log_entry_t),
+                                           reset_entry_buffer_size);
+    cpu_log_entry_t *entry;
     int i;
 
-    g_array_set_size(iinfo_ring, reset_entry_buffer_size);
-    g_array_set_clear_func(iinfo_ring, qemu_log_instr_info_destroy);
+    g_array_set_size(entry_ring, reset_entry_buffer_size);
+    g_array_set_clear_func(entry_ring, qemu_log_entry_destroy);
 
-    for (i = 0; i < iinfo_ring->len; i++) {
-        iinfo = &g_array_index(iinfo_ring, cpu_log_instr_info_t, i);
-        qemu_log_instr_info_init(iinfo);
+    for (i = 0; i < entry_ring->len; i++) {
+        entry = &g_array_index(entry_ring, cpu_log_entry_t, i);
+        qemu_log_entry_init(entry);
     }
 
     cpulog->loglevel = QEMU_LOG_INSTR_LOGLEVEL_NONE;
     cpulog->loglevel_active = false;
-    cpulog->instr_info = iinfo_ring;
+    cpulog->instr_info = entry_ring;
     cpulog->ring_head = 0;
     cpulog->ring_tail = 0;
-    reset_log_buffer(cpulog, iinfo);
+    reset_log_buffer(cpulog, entry);
 
     // Make sure we are using the correct trace format.
     if (trace_format == NULL) {
@@ -830,12 +836,11 @@ void qemu_log_instr_init(CPUState *cpu)
         do_cpu_loglevel_switch(
             cpu, RUN_ON_CPU_HOST_INT(QEMU_LOG_INSTR_LOGLEVEL_USER));
     else if (qemu_loglevel_mask(CPU_LOG_INSTR))
-        do_cpu_loglevel_switch(cpu,
-            RUN_ON_CPU_HOST_INT(QEMU_LOG_INSTR_LOGLEVEL_ALL));
+        do_cpu_loglevel_switch(
+            cpu, RUN_ON_CPU_HOST_INT(QEMU_LOG_INSTR_LOGLEVEL_ALL));
 }
 
-static void
-do_log_buffer_resize(CPUState *cpu, run_on_cpu_data data)
+static void do_log_buffer_resize(CPUState *cpu, run_on_cpu_data data)
 {
     unsigned long new_size = data.host_ulong;
     cpu_log_instr_state_t *cpulog = get_cpu_log_state(cpu->env_ptr);
@@ -850,9 +855,9 @@ do_log_buffer_resize(CPUState *cpu, run_on_cpu_data data)
          * Clear and reinitialize all the entries,
          * a bit overkill but should not be a frequent operation.
          */
-        iinfo = &g_array_index(cpulog->instr_info, cpu_log_instr_info_t, i);
-        qemu_log_instr_info_init(iinfo);
-        reset_log_buffer(cpulog, iinfo);
+        entry = &g_array_index(cpulog->instr_info, cpu_log_entry_t, i);
+        qemu_log_entry_init(entry);
+        reset_log_buffer(cpulog, entry);
     }
 }
 
@@ -861,16 +866,18 @@ void qemu_log_instr_set_buffer_size(unsigned long new_size)
     CPUState *cpu;
 
     if (new_size < MIN_ENTRY_BUFFER_SIZE) {
-        warn_report("New trace entry buffer size is too small < %zu, ignored.\n",
-                    (size_t)MIN_ENTRY_BUFFER_SIZE);
+        warn_report(
+            "New trace entry buffer size is too small < %zu, ignored.",
+            (size_t)MIN_ENTRY_BUFFER_SIZE);
         return;
     }
 
     /* Set this in case this is called from qemu option parsing */
     reset_entry_buffer_size = new_size;
-    CPU_FOREACH(cpu) {
+    CPU_FOREACH(cpu)
+    {
         async_safe_run_on_cpu(cpu, do_log_buffer_resize,
-            RUN_ON_CPU_HOST_ULONG(new_size));
+                              RUN_ON_CPU_HOST_ULONG(new_size));
     }
 }
 
@@ -894,16 +901,16 @@ bool qemu_log_instr_check_enabled(CPUArchState *env)
  * initiated here.
  */
 void qemu_log_instr_mode_switch(CPUArchState *env,
-    qemu_log_instr_cpu_mode_t mode, target_ulong pc)
+                                qemu_log_instr_cpu_mode_t mode, target_ulong pc)
 {
     cpu_log_instr_state_t *cpulog = get_cpu_log_state(env);
-    cpu_log_instr_info_t *iinfo = get_cpu_log_instr_info(env);
+    cpu_log_entry_t *entry = get_cpu_log_entry(env);
 
     log_assert(cpulog != NULL && "Invalid log state");
-    log_assert(iinfo != NULL && "Invalid log info");
+    log_assert(entry != NULL && "Invalid log info");
 
-    iinfo->flags |= LI_FLAG_MODE_SWITCH;
-    iinfo->next_cpu_mode = mode;
+    entry->flags |= LI_FLAG_MODE_SWITCH;
+    entry->next_cpu_mode = mode;
 
     /* If we are not logging in user-only mode, bail */
     if (!qemu_loglevel_mask(CPU_LOG_INSTR) ||
@@ -928,26 +935,27 @@ void qemu_log_instr_drop(CPUArchState *env)
 void qemu_log_instr_commit(CPUArchState *env)
 {
     cpu_log_instr_state_t *cpulog = get_cpu_log_state(env);
-    cpu_log_instr_info_t *iinfo = get_cpu_log_instr_info(env);
+    cpu_log_entry_t *entry = get_cpu_log_entry(env);
 
     log_assert(cpulog != NULL && "Invalid log state");
-    log_assert(iinfo != NULL && "Invalid log info");
+    log_assert(entry != NULL && "Invalid log info");
 
     do_instr_commit(env);
-    /* commit may have advanced to the next iinfo buffer slot */
-    iinfo = get_cpu_log_instr_info(env);
-    reset_log_buffer(cpulog, iinfo);
+    /* commit may have advanced to the next entry buffer slot */
+    entry = get_cpu_log_entry(env);
+    reset_log_buffer(cpulog, entry);
 }
 
-void qemu_log_instr_reg(CPUArchState *env, const char *reg_name, target_ulong value)
+void qemu_log_instr_reg(CPUArchState *env, const char *reg_name,
+                        target_ulong value)
 {
-    cpu_log_instr_info_t *iinfo = get_cpu_log_instr_info(env);
+    cpu_log_entry_t *entry = get_cpu_log_entry(env);
     log_reginfo_t r;
 
     r.flags = 0;
     r.name = reg_name;
     r.gpr = value;
-    g_array_append_val(iinfo->regs, r);
+    g_array_append_val(entry->regs, r);
 }
 
 void helper_qemu_log_instr_reg(CPUArchState *env, const void *reg_name,
@@ -959,15 +967,15 @@ void helper_qemu_log_instr_reg(CPUArchState *env, const void *reg_name,
 
 #ifdef TARGET_CHERI
 void qemu_log_instr_cap(CPUArchState *env, const char *reg_name,
-                         const cap_register_t *cr)
+                        const cap_register_t *cr)
 {
-    cpu_log_instr_info_t *iinfo = get_cpu_log_instr_info(env);
+    cpu_log_entry_t *entry = get_cpu_log_entry(env);
     log_reginfo_t r;
 
     r.flags = LRI_CAP_REG | LRI_HOLDS_CAP;
     r.name = reg_name;
     r.cap = *cr;
-    g_array_append_val(iinfo->regs, r);
+    g_array_append_val(entry->regs, r);
 }
 
 void helper_qemu_log_instr_cap(CPUArchState *env, const void *reg_name,
@@ -978,15 +986,15 @@ void helper_qemu_log_instr_cap(CPUArchState *env, const void *reg_name,
 }
 
 void qemu_log_instr_cap_int(CPUArchState *env, const char *reg_name,
-                             target_ulong value)
+                            target_ulong value)
 {
-    cpu_log_instr_info_t *iinfo = get_cpu_log_instr_info(env);
+    cpu_log_entry_t *entry = get_cpu_log_entry(env);
     log_reginfo_t r;
 
     r.flags = LRI_CAP_REG;
     r.name = reg_name;
     r.gpr = value;
-    g_array_append_val(iinfo->regs, r);
+    g_array_append_val(entry->regs, r);
 }
 #endif
 
@@ -994,14 +1002,14 @@ static inline void qemu_log_instr_mem_int(CPUArchState *env, target_ulong addr,
                                           int flags, TCGMemOpIdx oi,
                                           target_ulong value)
 {
-    cpu_log_instr_info_t *iinfo = get_cpu_log_instr_info(env);
+    cpu_log_entry_t *entry = get_cpu_log_entry(env);
     log_meminfo_t m;
 
     m.flags = flags;
     m.op = get_memop(oi);
     m.addr = addr;
     m.value = value;
-    g_array_append_val(iinfo->mem, m);
+    g_array_append_val(entry->mem, m);
 }
 
 void qemu_log_instr_ld_int(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi,
@@ -1023,18 +1031,18 @@ void qemu_log_instr_st_int(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi,
  * as well. Need to think whether there is value to keep logging what
  * was loaded directly.
  */
-static inline void qemu_log_instr_mem_cap(
-    CPUArchState *env, target_ulong addr, int flags,
-    const cap_register_t *value)
+static inline void qemu_log_instr_mem_cap(CPUArchState *env, target_ulong addr,
+                                          int flags,
+                                          const cap_register_t *value)
 {
-    cpu_log_instr_info_t *iinfo = get_cpu_log_instr_info(env);
+    cpu_log_entry_t *entry = get_cpu_log_entry(env);
     log_meminfo_t m;
 
     m.flags = flags;
     m.op = 0;
     m.addr = addr;
     m.cap = *value;
-    g_array_append_val(iinfo->mem, m);
+    g_array_append_val(entry->mem, m);
 }
 
 void qemu_log_instr_ld_cap(CPUArchState *env, target_ulong addr,
@@ -1052,60 +1060,59 @@ void qemu_log_instr_st_cap(CPUArchState *env, target_ulong addr,
 void qemu_log_instr(CPUArchState *env, target_ulong pc, const char *insn,
                     uint32_t size)
 {
-    cpu_log_instr_info_t *iinfo = get_cpu_log_instr_info(env);
+    cpu_log_entry_t *entry = get_cpu_log_entry(env);
 
-    iinfo->pc = pc;
-    iinfo->insn_size = size;
-    memcpy(iinfo->insn_bytes, insn, size);
+    entry->pc = pc;
+    entry->insn_size = size;
+    memcpy(entry->insn_bytes, insn, size);
 }
 
 void qemu_log_instr_asid(CPUArchState *env, uint16_t asid)
 {
-    cpu_log_instr_info_t *iinfo = get_cpu_log_instr_info(env);
+    cpu_log_entry_t *entry = get_cpu_log_entry(env);
 
-    iinfo->asid = asid;
+    entry->asid = asid;
 }
 
 void qemu_log_instr_exception(CPUArchState *env, uint32_t code,
                               target_ulong vector, target_ulong faultaddr)
 {
-    cpu_log_instr_info_t *iinfo = get_cpu_log_instr_info(env);
+    cpu_log_entry_t *entry = get_cpu_log_entry(env);
 
-    iinfo->flags |= LI_FLAG_INTR_TRAP;
-    iinfo->intr_code = code;
-    iinfo->intr_vector = vector;
-    iinfo->intr_faultaddr = faultaddr;
+    entry->flags |= LI_FLAG_INTR_TRAP;
+    entry->intr_code = code;
+    entry->intr_vector = vector;
+    entry->intr_faultaddr = faultaddr;
 }
 
 void qemu_log_instr_interrupt(CPUArchState *env, uint32_t code,
                               target_ulong vector)
 {
-    cpu_log_instr_info_t *iinfo = get_cpu_log_instr_info(env);
+    cpu_log_entry_t *entry = get_cpu_log_entry(env);
 
-    iinfo->flags |= LI_FLAG_INTR_ASYNC;
-    iinfo->intr_code = code;
-    iinfo->intr_vector = vector;
+    entry->flags |= LI_FLAG_INTR_ASYNC;
+    entry->intr_code = code;
+    entry->intr_vector = vector;
 }
 
 void qemu_log_instr_evt(CPUArchState *env, uint16_t fn, target_ulong arg0,
-                         target_ulong arg1, target_ulong arg2,
-                         target_ulong arg3)
+                        target_ulong arg1, target_ulong arg2, target_ulong arg3)
 {
-    /* iinfo->cv_buffer.entry_type = CVT_EVT; */
-    /* iinfo->cv_buffer.val5 = fn; */
-    /* iinfo->cv_buffer.val1 = arg0; */
-    /* iinfo->cv_buffer.val2 = arg1; */
-    /* iinfo->cv_buffer.val3 = arg2; */
-    /* iinfo->cv_buffer.val4 = arg3; */
+    /* entry->cv_buffer.entry_type = CVT_EVT; */
+    /* entry->cv_buffer.val5 = fn; */
+    /* entry->cv_buffer.val1 = arg0; */
+    /* entry->cv_buffer.val2 = arg1; */
+    /* entry->cv_buffer.val3 = arg2; */
+    /* entry->cv_buffer.val4 = arg3; */
 }
 
 void qemu_log_instr_extra(CPUArchState *env, const char *msg, ...)
 {
-    cpu_log_instr_info_t *iinfo = get_cpu_log_instr_info(env);
+    cpu_log_entry_t *entry = get_cpu_log_entry(env);
     va_list va;
 
     va_start(va, msg);
-    g_string_append_vprintf(iinfo->txt_buffer, msg, va);
+    g_string_append_vprintf(entry->txt_buffer, msg, va);
     va_end(va);
 }
 
@@ -1477,14 +1484,14 @@ void qemu_log_instr_flush(CPUArchState *env)
 {
     cpu_log_instr_state_t *cpulog = get_cpu_log_state(env);
     size_t curr = cpulog->ring_tail;
-    cpu_log_instr_info_t *iinfo;
+    cpu_log_entry_t *entry;
 
     if ((cpulog->flags & QEMU_LOG_INSTR_FLAG_BUFFERED) == 0)
         return;
 
     while (curr != cpulog->ring_head) {
-        iinfo = &g_array_index(cpulog->instr_info, cpu_log_instr_info_t, curr);
-        trace_format->emit_entry(env, iinfo);
+        entry = &g_array_index(cpulog->instr_info, cpu_log_entry_t, curr);
+        trace_format->emit_entry(env, entry);
         curr = (curr + 1) % cpulog->instr_info->len;
     }
     cpulog->ring_tail = cpulog->ring_head;
@@ -1584,7 +1591,8 @@ void helper_qemu_log_instr_allcpu_start()
 {
     CPUState *cpu;
 
-    CPU_FOREACH(cpu) {
+    CPU_FOREACH(cpu)
+    {
         helper_qemu_log_instr_start(cpu->env_ptr, 0);
     }
 }
@@ -1594,7 +1602,8 @@ void helper_qemu_log_instr_allcpu_user_start()
 {
     CPUState *cpu;
 
-    CPU_FOREACH(cpu) {
+    CPU_FOREACH(cpu)
+    {
         helper_qemu_log_instr_user_start(cpu->env_ptr, 0);
     }
 }
@@ -1604,7 +1613,8 @@ void helper_qemu_log_instr_allcpu_stop()
 {
     CPUState *cpu;
 
-    CPU_FOREACH(cpu) {
+    CPU_FOREACH(cpu)
+    {
         helper_qemu_log_instr_stop(cpu->env_ptr, 0);
     }
 }
@@ -1642,37 +1652,31 @@ void helper_qemu_log_instr_store32(CPUArchState *env, target_ulong addr,
         qemu_log_instr_mem_int(env, addr, LMI_ST, oi, (uint64_t)value);
 }
 
-void helper_log_value(CPUArchState *env, const void* ptr, uint64_t value)
+void helper_log_value(CPUArchState *env, const void *ptr, uint64_t value)
 {
     qemu_maybe_log_instr_extra(env, "%s: " TARGET_FMT_plx "\n", ptr, value);
 }
 
 static void emit_nop_start(CPUArchState *env, target_ulong pc) {}
 static void emit_nop_stop(CPUArchState *env, target_ulong pc) {}
-static void emit_nop_entry(CPUArchState *env, cpu_log_instr_info_t *iinfo) {}
+static void emit_nop_entry(CPUArchState *env, cpu_log_entry_t *entry) {}
 
 /*
  * Hooks for each trace format
  */
 static trace_fmt_hooks_t trace_formats[] = {
-    {
-        .emit_header = NULL,
-        .emit_start = emit_text_start,
-        .emit_stop = emit_text_stop,
-        .emit_entry = emit_text_entry
-    },
-    {
-        .emit_header = emit_cvtrace_header,
-        .emit_start = emit_cvtrace_start,
-        .emit_stop = emit_cvtrace_stop,
-        .emit_entry = emit_cvtrace_entry
-    },
-    {
-        .emit_header = NULL,
-        .emit_start = emit_nop_start,
-        .emit_stop = emit_nop_stop,
-        .emit_entry = emit_nop_entry
-    }
+    { .emit_header = NULL,
+      .emit_start = emit_text_start,
+      .emit_stop = emit_text_stop,
+      .emit_entry = emit_text_entry },
+    { .emit_header = emit_cvtrace_header,
+      .emit_start = emit_cvtrace_start,
+      .emit_stop = emit_cvtrace_stop,
+      .emit_entry = emit_cvtrace_entry },
+    { .emit_header = NULL,
+      .emit_start = emit_nop_start,
+      .emit_stop = emit_nop_stop,
+      .emit_entry = emit_nop_entry }
 };
 
 #endif /* CONFIG_TCG_LOG_INSTR */
diff --git a/include/exec/log_instr.h b/include/exec/log_instr.h
index 72dcaace78f..59b5d426cd4 100644
--- a/include/exec/log_instr.h
+++ b/include/exec/log_instr.h
@@ -46,7 +46,8 @@
  * - uint16_t cpu_get_asid(env, pc)
  *   return the hardware address space identifier for a particular address
  * - const char *cpu_get_mode_name(mode)
- *   return the mode name associated with a qemu_log_instr_cpu_mode_t for printing.
+ *   return the mode name associated with a qemu_log_instr_cpu_mode_t for
+ * printing.
  *
  * - Each target should implement their own register update logging helpers that
  *   call into qemu_log_instr_gpr(), qemu_log_instr_cap() and similar interface
@@ -74,10 +75,10 @@
 #define _glue_args(...) , ## __VA_ARGS__
 
 #ifdef CONFIG_TCG_LOG_INSTR
-#define qemu_ctx_logging_enabled(ctx) unlikely(ctx->base.log_instr_enabled)
+#define qemu_ctx_logging_enabled(ctx)   unlikely(ctx->base.log_instr_enabled)
 #define qemu_base_logging_enabled(base) unlikely(base->log_instr_enabled)
 #else
-#define qemu_ctx_logging_enabled(ctx) false
+#define qemu_ctx_logging_enabled(ctx)   false
 #define qemu_base_logging_enabled(base) false
 #endif
 
@@ -85,29 +86,32 @@
  * Helper to simplify checking for either instruction logging or
  * another loglevel enabled
  */
-#define qemu_log_instr_or_mask_enabled(env, mask)       \
-    ((unlikely(qemu_loglevel_mask(mask)) ||             \
-      qemu_log_instr_enabled(env)) ? true : false)
+#define qemu_log_instr_or_mask_enabled(env, mask)                              \
+    ((unlikely(qemu_loglevel_mask(mask)) || qemu_log_instr_enabled(env))       \
+         ? true                                                                \
+         : false)
 
 /*
  * Helper to simplify emitting a message either to instruction
  * logging extra text buffer or when another loglevel is enabled
  */
-#define qemu_log_instr_or_mask_msg(env, mask, msg, ...) do {            \
-        if (qemu_loglevel_mask(mask)) {                                 \
-            qemu_log(msg _glue_args(__VA_ARGS__));                      \
-        } else if (qemu_log_instr_enabled(env)) {                       \
-            qemu_log_instr_extra(env, msg _glue_args(__VA_ARGS__));     \
-        }                                                               \
+#define qemu_log_instr_or_mask_msg(env, mask, msg, ...)                        \
+    do {                                                                       \
+        if (qemu_loglevel_mask(mask)) {                                        \
+            qemu_log(msg _glue_args(__VA_ARGS__));                             \
+        } else if (qemu_log_instr_enabled(env)) {                              \
+            qemu_log_instr_extra(env, msg _glue_args(__VA_ARGS__));            \
+        }                                                                      \
     } while (0)
 
 /*
  * Helper version of qemu_log_instr_extra that checks whether logging is
  * enabled.
  */
-#define qemu_maybe_log_instr_extra(env, msg, ...) do {                  \
-        if (qemu_log_instr_enabled(env))                                \
-            qemu_log_instr_extra(env, msg _glue_args(__VA_ARGS__));     \
+#define qemu_maybe_log_instr_extra(env, msg, ...)                              \
+    do {                                                                       \
+        if (qemu_log_instr_enabled(env))                                       \
+            qemu_log_instr_extra(env, msg _glue_args(__VA_ARGS__));            \
     } while (0)
 
 /*
@@ -115,10 +119,11 @@
  * for the current instruction.
  * Note that this can be ignored by the output trace format.
  */
-#define qemu_log_instr_dbg_reg(env, name, value) do {                   \
-        if (qemu_log_instr_enabled(env))                                \
-            qemu_log_instr_extra(env, "    Write %s = " TARGET_FMT_lx "\n", \
-                                 name, value);                          \
+#define qemu_log_instr_dbg_reg(env, name, value)                               \
+    do {                                                                       \
+        if (qemu_log_instr_enabled(env))                                       \
+            qemu_log_instr_extra(env, "    Write %s = " TARGET_FMT_lx "\n",    \
+                                 name, value);                                 \
     } while (0)
 
 /*
@@ -126,13 +131,14 @@
  * info for the current instruction.
  * Note that this can be ignored by the output trace format.
  */
-#define qemu_log_instr_dbg_cap(env, name, value) do {                   \
-        if (qemu_log_instr_enabled(env))                                \
-            qemu_log_instr_extra(                                       \
-                env, "    Write %s|" PRINT_CAP_FMTSTR_L1 "\n"           \
-                "             |" PRINT_CAP_FMTSTR_L2 "\n",              \
-                name, PRINT_CAP_ARGS_L1(value),                         \
-                PRINT_CAP_ARGS_L2(value));                              \
+#define qemu_log_instr_dbg_cap(env, name, value)                               \
+    do {                                                                       \
+        if (qemu_log_instr_enabled(env))                                       \
+            qemu_log_instr_extra(env,                                          \
+                                 "    Write %s|" PRINT_CAP_FMTSTR_L1 "\n"      \
+                                 "             |" PRINT_CAP_FMTSTR_L2 "\n",    \
+                                 name, PRINT_CAP_ARGS_L1(value),               \
+                                 PRINT_CAP_ARGS_L2(value));                    \
     } while (0)
 
 #ifdef CONFIG_TCG_LOG_INSTR
@@ -188,7 +194,7 @@ void qemu_log_printf_create_globals(void);
 void qemu_log_instr_flush_tcg(bool request_stop);
 
 /* Helper macro to check for instruction logging enabled */
-#define	qemu_log_instr_enabled(env)                                     \
+#define qemu_log_instr_enabled(env)                                            \
     unlikely(qemu_log_instr_check_enabled((env)))
 
 /*
@@ -215,7 +221,8 @@ void qemu_log_instr_stop(CPUArchState *env, target_ulong pc);
  * depending whether the mode parameter is QEMU_LOG_INSTR_CPU_USER or not.
  */
 void qemu_log_instr_mode_switch(CPUArchState *env,
-    qemu_log_instr_cpu_mode_t mode, target_ulong pc);
+                                qemu_log_instr_cpu_mode_t mode,
+                                target_ulong pc);
 
 /*
  * Set the given CPU per-CPU log level.
@@ -309,7 +316,8 @@ void qemu_log_instr_exception(CPUArchState *env, uint32_t code,
 /*
  * Log interrupt event.
  */
-void qemu_log_instr_interrupt(CPUArchState *env, uint32_t code, target_ulong vector);
+void qemu_log_instr_interrupt(CPUArchState *env, uint32_t code,
+                              target_ulong vector);
 
 /*
  * Log magic NOP event, we record a function number and 4 arguments.
@@ -327,22 +335,22 @@ void qemu_log_instr_evt(CPUArchState *env, uint16_t fn, target_ulong arg0,
 void qemu_log_instr_extra(CPUArchState *env, const char *msg, ...);
 
 #else /* ! CONFIG_TCG_LOG_INSTR */
-#define	qemu_log_instr_enabled(cpu) false
-#define	qemu_log_instr_start(env, mode, pc)
-#define	qemu_log_instr_stop(env, mode, pc)
-#define	qemu_log_instr_mode_switch(...)
+#define qemu_log_instr_enabled(cpu) false
+#define qemu_log_instr_start(env, mode, pc)
+#define qemu_log_instr_stop(env, mode, pc)
+#define qemu_log_instr_mode_switch(...)
 #define qemu_log_instr_flush(env)
-#define	qemu_log_instr_reg(...)
-#define	qemu_log_instr_cap(...)
-#define	qemu_log_instr_mem(...)
-#define	qemu_log_instr_instr(...)
-#define	qemu_log_instr_hwtid(...)
-#define	qemu_log_instr_asid(...)
-#define	qemu_log_instr_exception(...)
-#define	qemu_log_instr_interrupt(...)
-#define	qemu_log_instr_env(...)
-#define	qemu_log_instr_extra(...)
-#define	qemu_log_instr_commit(...)
+#define qemu_log_instr_reg(...)
+#define qemu_log_instr_cap(...)
+#define qemu_log_instr_mem(...)
+#define qemu_log_instr_instr(...)
+#define qemu_log_instr_hwtid(...)
+#define qemu_log_instr_asid(...)
+#define qemu_log_instr_exception(...)
+#define qemu_log_instr_interrupt(...)
+#define qemu_log_instr_env(...)
+#define qemu_log_instr_extra(...)
+#define qemu_log_instr_commit(...)
 #define qemu_log_gen_printf(...)
 #define qemu_log_gen_printf_flush(base, flush_early, force_flush)
 #define qemu_log_printf_create_globals(...)
diff --git a/include/qemu/log_instr.h b/include/qemu/log_instr.h
index 1f0afaf5131..ea3c665415b 100644
--- a/include/qemu/log_instr.h
+++ b/include/qemu/log_instr.h
@@ -156,7 +156,7 @@ int qemu_log_instr_global_switch(int log_flags);
 
 /*
  * Update the ring buffer size.
- * Note that this does not guarantee that the existing buffered 
+ * Note that this does not guarantee that the existing buffered
  * entries will be retained.
  */
 void qemu_log_instr_set_buffer_size(unsigned long buffer_size);

From aca996714475f40910753a8fc50ac646f4880dfb Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Mon, 7 Jun 2021 16:52:11 +0100
Subject: [PATCH 02/74] Refactor instruction logging: rename trace_format to
 trace_backend.

---
 accel/tcg/log_instr.c    | 33 +++++++++++++++++----------------
 include/qemu/log_instr.h | 14 +++++++-------
 softmmu/vl.c             |  4 ++--
 3 files changed, 26 insertions(+), 25 deletions(-)

diff --git a/accel/tcg/log_instr.c b/accel/tcg/log_instr.c
index c40f67df639..bc0b4ebee68 100644
--- a/accel/tcg/log_instr.c
+++ b/accel/tcg/log_instr.c
@@ -179,26 +179,26 @@ typedef struct {
 } log_meminfo_t;
 
 /*
- * Callbacks defined by a trace format implementation.
+ * Callbacks defined by a trace backend implementation.
  * These are called to covert instruction tracing events to the corresponding
  * binary or text format.
  */
-struct trace_fmt_hooks {
+struct trace_backend_hooks {
     void (*emit_header)(CPUArchState *env);
     void (*emit_start)(CPUArchState *env, target_ulong pc);
     void (*emit_stop)(CPUArchState *env, target_ulong pc);
     void (*emit_entry)(CPUArchState *env, cpu_log_entry_t *entry);
 };
-typedef struct trace_fmt_hooks trace_fmt_hooks_t;
+typedef struct trace_backend_hooks trace_backend_hooks_t;
 
 /* Global trace format selector. Defaults to text tracing */
-qemu_log_instr_fmt_t qemu_log_instr_format = QLI_FMT_TEXT;
+qemu_log_instr_backend_t qemu_log_instr_backend = QLI_FMT_TEXT;
 
 /* Current format callbacks. */
-static trace_fmt_hooks_t *trace_format = NULL;
+static trace_backend_hooks_t *trace_backend;
 
-/* Existing format callbacks list, indexed by qemu_log_instr_fmt_t */
-static trace_fmt_hooks_t trace_formats[];
+/* Existing format callbacks list, indexed by qemu_log_instr_backend_t */
+static trace_backend_hooks_t trace_backends[];
 
 /*
  * CHERI binary trace format, originally used for MIPS only.
@@ -546,13 +546,13 @@ static void emit_cvtrace_stop(CPUArchState *env, target_ulong pc)
 static inline void emit_start_event(CPUArchState *env, target_ulong pc)
 {
     if ((get_cpu_log_state(env)->flags & QEMU_LOG_INSTR_FLAG_BUFFERED) == 0)
-        trace_format->emit_start(env, pc);
+        trace_backend->emit_start(env, pc);
 }
 
 static inline void emit_stop_event(CPUArchState *env, target_ulong pc)
 {
     if ((get_cpu_log_state(env)->flags & QEMU_LOG_INSTR_FLAG_BUFFERED) == 0)
-        trace_format->emit_stop(env, pc);
+        trace_backend->emit_stop(env, pc);
 }
 
 static inline void emit_entry_event(CPUArchState *env, cpu_log_entry_t *entry)
@@ -565,7 +565,7 @@ static inline void emit_entry_event(CPUArchState *env, cpu_log_entry_t *entry)
             cpulog->ring_tail =
                 (cpulog->ring_tail + 1) % cpulog->instr_info->len;
     } else {
-        trace_format->emit_entry(env, entry);
+        trace_backend->emit_entry(env, entry);
     }
 }
 
@@ -824,11 +824,12 @@ void qemu_log_instr_init(CPUState *cpu)
     reset_log_buffer(cpulog, entry);
 
     // Make sure we are using the correct trace format.
-    if (trace_format == NULL) {
-        trace_format = &trace_formats[qemu_log_instr_format];
+    if (trace_backend == NULL) {
+        trace_backend = &trace_backends[qemu_log_instr_format];
         // Only emit header on first init
-        if (trace_format->emit_header)
-            trace_format->emit_header(cpu->env_ptr);
+        if (trace_backend->emit_header) {
+            trace_backend->emit_header(cpu->env_ptr);
+        }
     }
 
     /* If we are starting with instruction logging enabled, switch it on now */
@@ -1491,7 +1492,7 @@ void qemu_log_instr_flush(CPUArchState *env)
 
     while (curr != cpulog->ring_head) {
         entry = &g_array_index(cpulog->instr_info, cpu_log_entry_t, curr);
-        trace_format->emit_entry(env, entry);
+        trace_backend->emit_entry(env, entry);
         curr = (curr + 1) % cpulog->instr_info->len;
     }
     cpulog->ring_tail = cpulog->ring_head;
@@ -1664,7 +1665,7 @@ static void emit_nop_entry(CPUArchState *env, cpu_log_entry_t *entry) {}
 /*
  * Hooks for each trace format
  */
-static trace_fmt_hooks_t trace_formats[] = {
+static trace_backend_hooks_t trace_backends[] = {
     { .emit_header = NULL,
       .emit_start = emit_text_start,
       .emit_stop = emit_text_stop,
diff --git a/include/qemu/log_instr.h b/include/qemu/log_instr.h
index ea3c665415b..3a4863f4b5a 100644
--- a/include/qemu/log_instr.h
+++ b/include/qemu/log_instr.h
@@ -54,9 +54,9 @@ typedef enum {
     QLI_FMT_TEXT = 0,
     QLI_FMT_CVTRACE = 1,
     QLI_FMT_NOP = 2
-} qemu_log_instr_fmt_t;
+} qemu_log_instr_backend_t;
 
-extern qemu_log_instr_fmt_t qemu_log_instr_format;
+extern qemu_log_instr_backend_t qemu_log_instr_backend;
 
 /*
  * CPU mode. This unifies the logging codes for CPU mode switches.
@@ -88,14 +88,14 @@ typedef enum {
     QEMU_LOG_INSTR_LOGLEVEL_USER = 2,
 } qemu_log_instr_loglevel_t;
 
-static inline void qemu_log_instr_set_format(qemu_log_instr_fmt_t fmt)
+static inline void qemu_log_instr_set_backend(qemu_log_instr_backend_t id)
 {
-    qemu_log_instr_format = fmt;
+    qemu_log_instr_backend = id;
 }
 
-static inline qemu_log_instr_fmt_t qemu_log_instr_get_format()
+static inline qemu_log_instr_backend_t qemu_log_instr_get_backend()
 {
-    return qemu_log_instr_format;
+    return qemu_log_instr_backend;
 }
 
 struct cpu_log_instr_info;
@@ -162,5 +162,5 @@ int qemu_log_instr_global_switch(int log_flags);
 void qemu_log_instr_set_buffer_size(unsigned long buffer_size);
 
 #else /* ! CONFIG_TCG_LOG_INSTR */
-#define qemu_log_instr_set_format(fmt) ((void)0)
+#define qemu_log_instr_set_backend(id) ((void)0)
 #endif /* ! CONFIG_TCG_LOG_INSTR */
diff --git a/softmmu/vl.c b/softmmu/vl.c
index 742179c13cf..633bdcfa5a7 100644
--- a/softmmu/vl.c
+++ b/softmmu/vl.c
@@ -3749,9 +3749,9 @@ void qemu_init(int argc, char **argv, char **envp)
 #if defined(CONFIG_TCG_LOG_INSTR)
             case QEMU_OPTION_cheri_trace_format:
                 if (strcmp(optarg, "text") == 0) {
-                    qemu_log_instr_set_format(QLI_FMT_TEXT);
+                    qemu_log_instr_set_backend(QLI_FMT_TEXT);
                 } else if (strcmp(optarg, "cvtrace") == 0) {
-                    qemu_log_instr_set_format(QLI_FMT_CVTRACE);
+                    qemu_log_instr_set_backend(QLI_FMT_CVTRACE);
                 } else {
                     printf("Invalid choice for cheri-trace-format: '%s'\n", optarg);
                     exit(1);

From de64762d61557c503f3fc2753a5775418e15e296 Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Thu, 15 Jul 2021 14:40:19 +0100
Subject: [PATCH 03/74] Implement an event queue for each instruction entry
 logged.

---
 accel/tcg/log_instr.c    | 225 +++++++++++++++++++--------------------
 include/exec/log_instr.h |  58 +++++++++-
 include/qemu/log_instr.h |   2 +-
 3 files changed, 166 insertions(+), 119 deletions(-)

diff --git a/accel/tcg/log_instr.c b/accel/tcg/log_instr.c
index bc0b4ebee68..fac6e3dc34b 100644
--- a/accel/tcg/log_instr.c
+++ b/accel/tcg/log_instr.c
@@ -105,12 +105,14 @@ typedef struct cpu_log_entry {
     uint16_t asid;
     int flags;
 /* Entry contains a synchronous exception */
-#define LI_FLAG_INTR_TRAP 1
+#define LI_FLAG_INTR_TRAP (1 << 0)
 /* Entry contains an asynchronous exception */
 #define LI_FLAG_INTR_ASYNC (1 << 1)
-#define LI_FLAG_INTR_MASK  0x3
+#define LI_FLAG_INTR_MASK  (LI_FLAG_INTR_TRAP | LI_FLAG_INTR_ASYNC)
 /* Entry contains a CPU mode-switch and associated code */
 #define LI_FLAG_MODE_SWITCH (1 << 2)
+/* Entry contains instruction data or just events */
+#define LI_FLAG_HAS_INSTR_DATA (1 << 3)
 
     qemu_log_instr_cpu_mode_t next_cpu_mode;
     uint32_t intr_code;
@@ -123,17 +125,23 @@ typedef struct cpu_log_entry {
     char insn_bytes[TARGET_MAX_INSN_SIZE];
 #define cpu_log_entry_endzero mem
     /*
+     * Array of log_meminfo_t.
      * For now we allow multiple accesses to be tied to one instruction.
      * Some architectures may have multiple memory accesses
      * in the same instruction (e.g. x86-64 pop r/m64,
      * vector/matrix instructions, load/store pair). It is unclear
      * whether we would treat these as multiple trace "entities".
-     *
-     * Array of log_meminfo_t
      */
     GArray *mem;
-    /* Register modifications. Array of log_reginfo_t */
+    /* Register modifications - array of log_reginfo_t */
     GArray *regs;
+    /*
+     * Events associated with the instruction - array of log_event_t.
+     * Note that events may be present even if the entry does not contain
+     * valid instruction state. This allows the flexibility of only
+     * instrumenting a subset of the instructions/events.
+     */
+    GArray *events;
     /* Extra text-only log */
     GString *txt_buffer;
 } cpu_log_entry_t;
@@ -185,9 +193,8 @@ typedef struct {
  */
 struct trace_backend_hooks {
     void (*emit_header)(CPUArchState *env);
-    void (*emit_start)(CPUArchState *env, target_ulong pc);
-    void (*emit_stop)(CPUArchState *env, target_ulong pc);
-    void (*emit_entry)(CPUArchState *env, cpu_log_entry_t *entry);
+    void (*emit_instr)(CPUArchState *env, cpu_log_entry_t *entry);
+    void (*emit_events)(CPUArchState *env, cpu_log_entry_t *entry);
 };
 typedef struct trace_backend_hooks trace_backend_hooks_t;
 
@@ -330,9 +337,10 @@ static inline void emit_text_reg(log_reginfo_t *rinfo)
 }
 
 /*
- * Emit textual trace entry to the log.
+ * Emit textual representation of the instruction in the given
+ * trace entry to the log.
  */
-static void emit_text_entry(CPUArchState *env, cpu_log_entry_t *entry)
+static void emit_text_instr(CPUArchState *env, cpu_log_entry_t *entry)
 {
     QemuLogFile *logfile;
     int i;
@@ -400,36 +408,36 @@ static void emit_text_entry(CPUArchState *env, cpu_log_entry_t *entry)
 }
 
 /*
- * Emit text tracing start event.
+ * Emit a textual representation of events recorded by the given trace entry.
  */
-static void emit_text_start(CPUArchState *env, target_ulong pc)
+static void emit_text_events(CPUArchState *env, cpu_log_entry_t *entry)
 {
     cpu_log_instr_state_t *cpulog = get_cpu_log_state(env);
+    const log_event_t *event;
+    const char *log_state_op;
+    int i;
 
-    if (cpulog->loglevel == QEMU_LOG_INSTR_LOGLEVEL_USER) {
-        qemu_log("[%u:%u] Requested user-mode only instruction logging "
-                 "@ " TARGET_FMT_lx " \n",
-                 env_cpu(env)->cpu_index, cpu_get_asid(env, pc), pc);
-    } else {
-        qemu_log("[%u:%u] Requested instruction logging @ " TARGET_FMT_lx " \n",
-                 env_cpu(env)->cpu_index, cpu_get_asid(env, pc), pc);
-    }
-}
-
-/*
- * Emit text tracing stop event.
- */
-static void emit_text_stop(CPUArchState *env, target_ulong pc)
-{
-    cpu_log_instr_state_t *cpulog = get_cpu_log_state(env);
+    for (i = 0; i < entry->events->len; i++) {
+        event = &g_array_index(entry->events, const log_event_t, i);
+        if (event->id == LOG_EVENT_STATE) {
+            if (event->state.next_state == LOG_EVENT_STATE_START) {
+                log_state_op = "Requested";
+            } else if (event->state.next_state == LOG_EVENT_STATE_STOP) {
+                log_state_op = "Disabled";
+            }
 
-    if (cpulog->loglevel == QEMU_LOG_INSTR_LOGLEVEL_USER) {
-        qemu_log("[%u:%u] Disabled user-mode only instruction logging "
-                 "@ " TARGET_FMT_lx " \n",
-                 env_cpu(env)->cpu_index, cpu_get_asid(env, pc), pc);
-    } else {
-        qemu_log("[%u:%u] Disabled instruction logging @ " TARGET_FMT_lx " \n",
-                 env_cpu(env)->cpu_index, cpu_get_asid(env, pc), pc);
+            if (cpulog->loglevel == QEMU_LOG_INSTR_LOGLEVEL_USER) {
+                qemu_log("[%u:%u] %s user-mode only instruction logging "
+                         "@ " TARGET_FMT_lx "\n",
+                         env_cpu(env)->cpu_index, cpu_get_asid(env),
+                         log_state_op, event->state.pc);
+            } else {
+                qemu_log("[%u:%u] %s instruction logging @ " TARGET_FMT_lx
+                         "\n",
+                         env_cpu(env)->cpu_index, cpu_get_asid(env),
+                         log_state_op, event->state.pc);
+            }
+        }
     }
 }
 
@@ -456,28 +464,28 @@ static void emit_cvtrace_header(CPUArchState *env)
 static void emit_cvtrace_entry(CPUArchState *env, cpu_log_entry_t *entry)
 {
     FILE *logfile;
-    cheri_trace_entry_t entry;
+    cheri_trace_entry_t ct_entry;
     /* TODO(am2419): this should be a per-cpu counter. */
     static uint16_t cycles;
     uint32_t *insn = (uint32_t *)&entry->insn_bytes[0];
 
-    entry.entry_type = CTE_NO_REG;
-    entry.thread = (uint8_t)env_cpu(env)->cpu_index;
-    entry.asid = (uint8_t)entry->asid;
-    entry.pc = cpu_to_be64(entry->pc);
-    entry.cycles = cpu_to_be16(cycles++);
+    ct_entry.entry_type = CTE_NO_REG;
+    ct_entry.thread = (uint8_t)env_cpu(env)->cpu_index;
+    ct_entry.asid = (uint8_t)entry->asid;
+    ct_entry.pc = cpu_to_be64(entry->pc);
+    ct_entry.cycles = cpu_to_be16(cycles++);
     /*
      * TODO(am2419): The instruction bytes are alread in target byte-order,
      * however cheritrace does not currently expect this.
      */
-    entry.inst = cpu_to_be32(*insn);
+    ct_entry.inst = cpu_to_be32(*insn);
     switch (entry->flags & LI_FLAG_INTR_MASK) {
     case LI_FLAG_INTR_TRAP:
-        entry.exception = (uint8_t)(entry->intr_code & 0xff);
+        ct_entry.exception = (uint8_t)(entry->intr_code & 0xff);
     case LI_FLAG_INTR_ASYNC:
-        entry.exception = 0;
+        ct_entry.exception = 0;
     default:
-        entry.exception = CTE_EXCEPTION_NONE;
+        ct_entry.exception = CTE_EXCEPTION_NONE;
     }
 
     if (entry->regs->len) {
@@ -499,16 +507,16 @@ static void emit_cvtrace_entry(CPUArchState *env, cpu_log_entry_t *entry)
                                  ((uint64_t)COMBINED_PERMS_VALUE(cr) << 1) |
                                  (uint64_t)(cap_is_unsealed(cr) ? 0 : 1));
 
-            entry.entry_type = CTE_CAP;
-            entry.val2 = cpu_to_be64(metadata);
-            entry.val3 = cpu_to_be64(cap_get_cursor(cr));
-            entry.val4 = cpu_to_be64(cap_get_base(cr));
-            entry.val5 = cpu_to_be64(cap_get_length_sat(cr));
+            ct_entry.entry_type = CTE_CAP;
+            ct_entry.val2 = cpu_to_be64(metadata);
+            ct_entry.val3 = cpu_to_be64(cap_get_cursor(cr));
+            ct_entry.val4 = cpu_to_be64(cap_get_base(cr));
+            ct_entry.val5 = cpu_to_be64(cap_get_length_sat(cr));
         } else
 #endif
         {
-            entry.entry_type = CTE_GPR;
-            entry.val2 = cpu_to_be64(rinfo->gpr);
+            ct_entry.entry_type = CTE_GPR;
+            ct_entry.val2 = cpu_to_be64(rinfo->gpr);
         }
     }
 
@@ -518,55 +526,43 @@ static void emit_cvtrace_entry(CPUArchState *env, cpu_log_entry_t *entry)
         log_assert((minfo->flags & LMI_CAP) == 0 && "Capability memory access "
                                                     "without CHERI support");
 #endif
-        entry.val1 = cpu_to_be64(minfo->addr);
+        ct_entry.val1 = cpu_to_be64(minfo->addr);
         // Hack to avoid checking for GPR or CAP
         if (minfo->flags & LMI_LD)
-            entry.entry_type += 1;
+            ct_entry.entry_type += 1;
         else if (minfo->flags & LMI_ST)
-            entry.entry_type += 2;
+            ct_entry.entry_type += 2;
     }
 
     logfile = qemu_log_lock();
-    fwrite(&entry, sizeof(entry), 1, logfile);
+    fwrite(&ct_entry, sizeof(ct_entry), 1, logfile);
     qemu_log_unlock(logfile);
 }
 
-static void emit_cvtrace_start(CPUArchState *env, target_ulong pc)
-{
-    // TODO(am2419) Emit an event for instruction logging start
-}
+/* Core instruction logging implementation */
 
-static void emit_cvtrace_stop(CPUArchState *env, target_ulong pc)
+static inline void emit_start_event(cpu_log_entry_t *entry, target_ulong pc)
 {
-    // TODO(am2419) Emit an event for instruction logging stop
-}
+    log_event_t event;
 
-/* Core instruction logging implementation */
+    event.id = LOG_EVENT_STATE;
+    event.state.next_state = LOG_EVENT_STATE_START;
+    event.state.pc = pc;
+    /* Start events always have incomplete instruction data */
+    entry->flags &= ~LI_FLAG_HAS_INSTR_DATA;
 
-static inline void emit_start_event(CPUArchState *env, target_ulong pc)
-{
-    if ((get_cpu_log_state(env)->flags & QEMU_LOG_INSTR_FLAG_BUFFERED) == 0)
-        trace_backend->emit_start(env, pc);
+    g_array_append_val(entry->events, event);
 }
 
-static inline void emit_stop_event(CPUArchState *env, target_ulong pc)
+static inline void emit_stop_event(cpu_log_entry_t *entry, target_ulong pc)
 {
-    if ((get_cpu_log_state(env)->flags & QEMU_LOG_INSTR_FLAG_BUFFERED) == 0)
-        trace_backend->emit_stop(env, pc);
-}
+    log_event_t event;
 
-static inline void emit_entry_event(CPUArchState *env, cpu_log_entry_t *entry)
-{
-    cpu_log_instr_state_t *cpulog = get_cpu_log_state(env);
+    event.id = LOG_EVENT_STATE;
+    event.state.next_state = LOG_EVENT_STATE_STOP;
+    event.state.pc = pc;
 
-    if (cpulog->flags & QEMU_LOG_INSTR_FLAG_BUFFERED) {
-        cpulog->ring_head = (cpulog->ring_head + 1) % cpulog->instr_info->len;
-        if (cpulog->ring_tail == cpulog->ring_head)
-            cpulog->ring_tail =
-                (cpulog->ring_tail + 1) % cpulog->instr_info->len;
-    } else {
-        trace_backend->emit_entry(env, entry);
-    }
+    g_array_append_val(entry->events, event);
 }
 
 /* Reset instruction info buffer for next instruction */
@@ -578,6 +574,7 @@ static void reset_log_buffer(cpu_log_instr_state_t *cpulog,
             (char *)&entry->cpu_log_entry_startzero));
     g_array_remove_range(entry->regs, 0, entry->regs->len);
     g_array_remove_range(entry->mem, 0, entry->mem->len);
+    g_array_remove_range(entry->events, 0, entry->events->len);
     g_string_erase(entry->txt_buffer, 0, -1);
     cpulog->force_drop = false;
     cpulog->starting = false;
@@ -597,8 +594,7 @@ static void do_instr_commit(CPUArchState *env)
 
     if (cpulog->starting) {
         cpulog->starting = false;
-        emit_start_event(env, cpu_get_recent_pc(env));
-        return;
+        emit_start_event(entry, cpu_get_recent_pc(env));
     }
 
     /* Check for dfilter matches in this instruction */
@@ -622,8 +618,7 @@ static void do_instr_commit(CPUArchState *env)
         if (match)
             emit_entry_event(env, entry);
     } else {
-        /* dfilter disabled, always log */
-        emit_entry_event(env, entry);
+        trace_backend->emit_instr(env, entry);
     }
 }
 
@@ -689,8 +684,8 @@ static void do_cpu_loglevel_switch(CPUState *cpu, run_on_cpu_data data)
             reset_log_buffer(cpulog, entry);
             return;
         }
+        emit_stop_event(entry, cpu_get_recent_pc(env));
         do_instr_commit(env);
-        emit_stop_event(env, cpu_get_recent_pc(env));
         /* Instruction commit may have advanced to the next entry buffer slot */
         entry = get_cpu_log_entry(env);
         reset_log_buffer(cpulog, entry);
@@ -779,6 +774,9 @@ static void qemu_log_entry_init(cpu_log_entry_t *entry)
     if (entry->mem == NULL) {
         entry->mem = g_array_new(false, true, sizeof(log_meminfo_t));
     }
+    if (entry->events == NULL) {
+        entry->events = g_array_new(false, true, sizeof(log_event_t));
+    }
 }
 
 /*
@@ -788,9 +786,10 @@ static void qemu_log_entry_destroy(gpointer data)
 {
     cpu_log_entry_t *entry = data;
 
-    g_string_free(entry->txt_buffer, TRUE);
-    g_array_free(entry->regs, TRUE);
-    g_array_free(entry->mem, TRUE);
+    g_string_free(entry->txt_buffer, true);
+    g_array_free(entry->regs, true);
+    g_array_free(entry->mem, true);
+    g_array_free(entry->events, true);
 }
 
 /*
@@ -825,7 +824,7 @@ void qemu_log_instr_init(CPUState *cpu)
 
     // Make sure we are using the correct trace format.
     if (trace_backend == NULL) {
-        trace_backend = &trace_backends[qemu_log_instr_format];
+        trace_backend = &trace_backends[qemu_log_instr_backend];
         // Only emit header on first init
         if (trace_backend->emit_header) {
             trace_backend->emit_header(cpu->env_ptr);
@@ -1065,6 +1064,7 @@ void qemu_log_instr(CPUArchState *env, target_ulong pc, const char *insn,
 
     entry->pc = pc;
     entry->insn_size = size;
+    entry->flags |= LI_FLAG_HAS_INSTR_DATA;
     memcpy(entry->insn_bytes, insn, size);
 }
 
@@ -1096,15 +1096,11 @@ void qemu_log_instr_interrupt(CPUArchState *env, uint32_t code,
     entry->intr_vector = vector;
 }
 
-void qemu_log_instr_evt(CPUArchState *env, uint16_t fn, target_ulong arg0,
-                        target_ulong arg1, target_ulong arg2, target_ulong arg3)
+void qemu_log_instr_event(CPUArchState *env, log_event_t *evt)
 {
-    /* entry->cv_buffer.entry_type = CVT_EVT; */
-    /* entry->cv_buffer.val5 = fn; */
-    /* entry->cv_buffer.val1 = arg0; */
-    /* entry->cv_buffer.val2 = arg1; */
-    /* entry->cv_buffer.val3 = arg2; */
-    /* entry->cv_buffer.val4 = arg3; */
+    cpu_log_entry_t *entry = get_cpu_log_entry(env);
+
+    g_array_append_val(entry->events, *evt);
 }
 
 void qemu_log_instr_extra(CPUArchState *env, const char *msg, ...)
@@ -1487,12 +1483,20 @@ void qemu_log_instr_flush(CPUArchState *env)
     size_t curr = cpulog->ring_tail;
     cpu_log_entry_t *entry;
 
-    if ((cpulog->flags & QEMU_LOG_INSTR_FLAG_BUFFERED) == 0)
+    /*
+     * If tracing is disabled, force the commit of events in this
+     * trace entry.
+     */
+    if (!qemu_log_instr_check_enabled(env)) {
+        qemu_log_instr_commit(env);
+    }
+    if ((cpulog->flags & QEMU_LOG_INSTR_FLAG_BUFFERED) == 0) {
         return;
+    }
 
     while (curr != cpulog->ring_head) {
         entry = &g_array_index(cpulog->instr_info, cpu_log_entry_t, curr);
-        trace_backend->emit_entry(env, entry);
+        trace_backend->emit_instr(env, entry);
         curr = (curr + 1) % cpulog->instr_info->len;
     }
     cpulog->ring_tail = cpulog->ring_head;
@@ -1658,8 +1662,6 @@ void helper_log_value(CPUArchState *env, const void *ptr, uint64_t value)
     qemu_maybe_log_instr_extra(env, "%s: " TARGET_FMT_plx "\n", ptr, value);
 }
 
-static void emit_nop_start(CPUArchState *env, target_ulong pc) {}
-static void emit_nop_stop(CPUArchState *env, target_ulong pc) {}
 static void emit_nop_entry(CPUArchState *env, cpu_log_entry_t *entry) {}
 
 /*
@@ -1667,17 +1669,14 @@ static void emit_nop_entry(CPUArchState *env, cpu_log_entry_t *entry) {}
  */
 static trace_backend_hooks_t trace_backends[] = {
     { .emit_header = NULL,
-      .emit_start = emit_text_start,
-      .emit_stop = emit_text_stop,
-      .emit_entry = emit_text_entry },
+      .emit_instr = emit_text_instr,
+      .emit_events = emit_text_events },
     { .emit_header = emit_cvtrace_header,
-      .emit_start = emit_cvtrace_start,
-      .emit_stop = emit_cvtrace_stop,
-      .emit_entry = emit_cvtrace_entry },
+      .emit_instr = emit_cvtrace_entry,
+      .emit_events = NULL },
     { .emit_header = NULL,
-      .emit_start = emit_nop_start,
-      .emit_stop = emit_nop_stop,
-      .emit_entry = emit_nop_entry }
+      .emit_instr = emit_nop_entry,
+      .emit_events = emit_nop_entry }
 };
 
 #endif /* CONFIG_TCG_LOG_INSTR */
diff --git a/include/exec/log_instr.h b/include/exec/log_instr.h
index 59b5d426cd4..676bd297467 100644
--- a/include/exec/log_instr.h
+++ b/include/exec/log_instr.h
@@ -187,6 +187,51 @@ void qemu_log_gen_printf_flush(struct DisasContextBase *base, bool flush_early,
                                bool force_flush);
 void qemu_log_printf_create_globals(void);
 
+struct cpu_log_entry;
+
+/*
+ * Trace event identifiers.
+ */
+typedef enum {
+    LOG_EVENT_STATE = 0,
+    LOG_EVENT_CTX_SWITCH = 1,
+} log_event_id_t;
+
+/*
+ * Tracing status changed (e.g. trace start/stop)
+ */
+typedef enum {
+    LOG_EVENT_STATE_START,
+    LOG_EVENT_STATE_STOP
+} log_event_trace_state_t;
+
+typedef struct {
+    log_event_trace_state_t next_state;
+    target_ulong pc;
+} log_event_trace_state_update_t;
+
+/*
+ * Context switch event.
+ */
+typedef struct {
+    target_ulong pid; /* Process ID */
+    target_ulong tid; /* Thread ID */
+    target_ulong cid; /* Compartment ID */
+} log_event_ctx_switch_t;
+
+/*
+ * Trace event.
+ * This records arbitrary higher-level events associated with instruction
+ * entries.
+ */
+typedef struct {
+    log_event_id_t id;
+    union {
+        log_event_trace_state_update_t state;
+        log_event_ctx_switch_t ctx_switch;
+    };
+} log_event_t;
+
 /*
  * Request a flush of the TCG when changing loglevel outside of qemu_log_instr.
  * TODO(am2419): this should be removed from the interface.
@@ -230,9 +275,13 @@ void qemu_log_instr_mode_switch(CPUArchState *env,
 void qemu_log_instr_set_level(CPUArchState *env, qemu_log_instr_loglevel_t lvl);
 
 /*
- * Set the per-CPU log level for all the CPUs.
+ * Log a switch inc CPU modes.
+ * This will also trigger pause and resume of user-only logging activity,
+ * depending whether the mode parameter is QEMU_LOG_INSTR_CPU_USER or not.
  */
-void qemu_log_instr_allcpu_set_level(qemu_log_instr_loglevel_t lvl);
+void qemu_log_instr_mode_switch(CPUArchState *env,
+                                qemu_log_instr_cpu_mode_t mode,
+                                target_ulong pc);
 
 /*
  * Emit all buffered instruction logs.
@@ -324,9 +373,7 @@ void qemu_log_instr_interrupt(CPUArchState *env, uint32_t code,
  * Note that we have 6 bytes left in the cvtrace format, we may need
  * some trickery to reclaim those.
  */
-void qemu_log_instr_evt(CPUArchState *env, uint16_t fn, target_ulong arg0,
-                        target_ulong arg1, target_ulong arg2,
-                        target_ulong arg3);
+void qemu_log_instr_event(CPUArchState *env, log_event_t *evt);
 
 /*
  * Log extra information as a string. Some logging formats may
@@ -350,6 +397,7 @@ void qemu_log_instr_extra(CPUArchState *env, const char *msg, ...);
 #define qemu_log_instr_interrupt(...)
 #define qemu_log_instr_env(...)
 #define qemu_log_instr_extra(...)
+#define qemu_log_instr_event(...)
 #define qemu_log_instr_commit(...)
 #define qemu_log_gen_printf(...)
 #define qemu_log_gen_printf_flush(base, flush_early, force_flush)
diff --git a/include/qemu/log_instr.h b/include/qemu/log_instr.h
index 3a4863f4b5a..f22d163dad5 100644
--- a/include/qemu/log_instr.h
+++ b/include/qemu/log_instr.h
@@ -53,7 +53,7 @@
 typedef enum {
     QLI_FMT_TEXT = 0,
     QLI_FMT_CVTRACE = 1,
-    QLI_FMT_NOP = 2
+    QLI_FMT_NOP = 2,
 } qemu_log_instr_backend_t;
 
 extern qemu_log_instr_backend_t qemu_log_instr_backend;

From 89ece23637a9fadb9d357b4666c04d72494ec034 Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Thu, 15 Jul 2021 14:41:42 +0100
Subject: [PATCH 04/74] Generalize instruction trace filtering.

The -dfilter qemu option has ad-hoc support within TCG instruction tracing,
this generalizes the trace filtering to an arbitrary per-CPU set of functions
that can filter the trace entry based on any parameter.
Note that for now filters are assumed to be stateless.
---
 accel/tcg/log_instr.c    | 154 +++++++++++++++++++++++++++++++++------
 include/exec/log_instr.h |  34 +++++----
 include/qemu/log_instr.h |  10 ++-
 util/log.c               |  26 +++++--
 4 files changed, 182 insertions(+), 42 deletions(-)

diff --git a/accel/tcg/log_instr.c b/accel/tcg/log_instr.c
index fac6e3dc34b..b86ce3a9c14 100644
--- a/accel/tcg/log_instr.c
+++ b/accel/tcg/log_instr.c
@@ -207,6 +207,15 @@ static trace_backend_hooks_t *trace_backend;
 /* Existing format callbacks list, indexed by qemu_log_instr_backend_t */
 static trace_backend_hooks_t trace_backends[];
 
+/* Existing trace filters list, indexed by cpu_log_instr_filter_t */
+static cpu_log_instr_filter_fn_t trace_filters[];
+
+/*
+ * Instruction entry filter function.
+ * Return false if the entry should be dropped, true otherwise.
+ */
+typedef bool (*cpu_log_instr_filter_fn_t)(struct cpu_log_entry *entry);
+
 /*
  * CHERI binary trace format, originally used for MIPS only.
  * The format is limited to one entry per instruction, each
@@ -585,6 +594,8 @@ static void do_instr_commit(CPUArchState *env)
 {
     cpu_log_instr_state_t *cpulog = get_cpu_log_state(env);
     cpu_log_entry_t *entry = get_cpu_log_entry(env);
+    cpu_log_instr_filter_fn_t filter;
+    int i;
 
     log_assert(cpulog != NULL && "Invalid log state");
     log_assert(entry != NULL && "Invalid log buffer");
@@ -592,31 +603,23 @@ static void do_instr_commit(CPUArchState *env)
     if (cpulog->force_drop)
         return;
 
+    for (i = 0; i < cpulog->filters->len; i++) {
+        filter = g_array_index(cpulog->filters, cpu_log_instr_filter_fn_t, i);
+        if (!filter(entry)) {
+            return;
+        }
+    }
+
     if (cpulog->starting) {
         cpulog->starting = false;
         emit_start_event(entry, cpu_get_recent_pc(env));
     }
 
-    /* Check for dfilter matches in this instruction */
-    if (debug_regions) {
-        int i, j;
-        bool match = false;
-        for (i = 0; !match && i < debug_regions->len; i++) {
-            Range *range = &g_array_index(debug_regions, Range, i);
-            match = range_contains(range, entry->pc);
-            if (match)
-                break;
-
-            for (j = 0; j < entry->mem->len; j++) {
-                log_meminfo_t *minfo =
-                    &g_array_index(entry->mem, log_meminfo_t, j);
-                match = range_contains(range, minfo->addr);
-                if (match)
-                    break;
-            }
-        }
-        if (match)
-            emit_entry_event(env, entry);
+    if (cpulog->flags & QEMU_LOG_INSTR_FLAG_BUFFERED) {
+        cpulog->ring_head = (cpulog->ring_head + 1) % cpulog->instr_info->len;
+        if (cpulog->ring_tail == cpulog->ring_head)
+            cpulog->ring_tail =
+                (cpulog->ring_tail + 1) % cpulog->instr_info->len;
     } else {
         trace_backend->emit_instr(env, entry);
     }
@@ -817,6 +820,8 @@ void qemu_log_instr_init(CPUState *cpu)
 
     cpulog->loglevel = QEMU_LOG_INSTR_LOGLEVEL_NONE;
     cpulog->loglevel_active = false;
+    cpulog->filters = g_array_sized_new(
+        false, true, sizeof(cpu_log_instr_filter_fn_t), LOG_INSTR_FILTER_MAX);
     cpulog->instr_info = entry_ring;
     cpulog->ring_head = 0;
     cpulog->ring_tail = 0;
@@ -844,7 +849,7 @@ static void do_log_buffer_resize(CPUState *cpu, run_on_cpu_data data)
 {
     unsigned long new_size = data.host_ulong;
     cpu_log_instr_state_t *cpulog = get_cpu_log_state(cpu->env_ptr);
-    cpu_log_instr_info_t *iinfo;
+    cpu_log_entry_t *entry;
     int i;
 
     g_array_set_size(cpulog->instr_info, new_size);
@@ -1679,4 +1684,111 @@ static trace_backend_hooks_t trace_backends[] = {
       .emit_events = emit_nop_entry }
 };
 
+void qemu_log_instr_add_filter(CPUState *cpu, cpu_log_instr_filter_t filter)
+{
+    cpu_log_instr_state_t *cpulog = &cpu->log_state;
+
+    if (filter >= LOG_INSTR_FILTER_MAX) {
+        warn_report("Instruction trace filter index is invalid");
+        return;
+    }
+    /* XXX Currently we do not check for duplicates */
+    g_array_append_val(cpulog->filters, trace_filters[filter]);
+}
+
+void qemu_log_instr_allcpu_add_filter(cpu_log_instr_filter_t filter)
+{
+    CPUState *cpu;
+
+    CPU_FOREACH(cpu)
+    {
+        qemu_log_instr_add_filter(cpu, filter);
+    }
+}
+
+void qemu_log_instr_remove_filter(CPUState *cpu, cpu_log_instr_filter_t filter)
+{
+    cpu_log_instr_state_t *cpulog = &cpu->log_state;
+    cpu_log_instr_filter_fn_t curr;
+    int i;
+
+    if (filter >= LOG_INSTR_FILTER_MAX) {
+        warn_report("Instruction trace filter index is invalid");
+        return;
+    }
+
+    for (i = 0; i < cpulog->filters->len; i++) {
+        curr = g_array_index(cpulog->filters, cpu_log_instr_filter_fn_t, i);
+        if (curr == trace_filters[filter]) {
+            g_array_remove_index_fast(cpulog->filters, i);
+            break;
+        }
+    }
+}
+
+void qemu_log_instr_allcpu_remove_filter(cpu_log_instr_filter_t filter)
+{
+    CPUState *cpu;
+
+    CPU_FOREACH(cpu)
+    {
+        qemu_log_instr_remove_filter(cpu, filter);
+    }
+}
+
+/*
+ * LOG entry filter reusing the qemu -dfilter infrastructure to
+ * filter instructions that run from or access given address ranges.
+ */
+static bool entry_mem_regions_filter(cpu_log_entry_t *entry)
+{
+    int i, j;
+    bool match = false;
+
+    if (debug_regions == NULL) {
+        return true;
+    }
+
+    /* Check for dfilter matches in this instruction */
+    for (i = 0; !match && i < debug_regions->len; i++) {
+        Range *range = &g_array_index(debug_regions, Range, i);
+        match = range_contains(range, entry->pc);
+        if (match) {
+            break;
+        }
+
+        for (j = 0; j < entry->mem->len; j++) {
+            log_meminfo_t *minfo = &g_array_index(entry->mem, log_meminfo_t, j);
+            match = range_contains(range, minfo->addr);
+            if (match) {
+                break;
+            }
+        }
+    }
+    return match;
+}
+
+/*
+ * Interface to add/remove the -dfilter filter function from the per-cpu filter
+ * list. This is done here as -dfilter is linked in libqemu and does not have
+ * access to target-specific types.
+ * XXX Drop me
+ */
+void qemu_log_instr_mem_filter_update()
+{
+    if (debug_regions) {
+        qemu_log_instr_allcpu_add_filter(LOG_INSTR_FILTER_MEM_RANGE);
+    } else {
+        qemu_log_instr_allcpu_remove_filter(LOG_INSTR_FILTER_MEM_RANGE);
+    }
+}
+
+/*
+ * Trace filters mapping. Note that indices must match the
+ * cpu_log_instr_filter_t enum values.
+ */
+static cpu_log_instr_filter_fn_t trace_filters[] = {
+    entry_mem_regions_filter,
+};
+
 #endif /* CONFIG_TCG_LOG_INSTR */
diff --git a/include/exec/log_instr.h b/include/exec/log_instr.h
index 676bd297467..aea2a3e0b3c 100644
--- a/include/exec/log_instr.h
+++ b/include/exec/log_instr.h
@@ -232,6 +232,19 @@ typedef struct {
     };
 } log_event_t;
 
+/*
+ * Handles for instruction filters/callbacks.
+ * These are used to identify and attach/detach instruction log entry filters,
+ * the rationale for this is that the filters require access to the private
+ * instruction log entry record structure and can only be defined within the
+ * instruction logging core implementation. This solution mimics the trace
+ * backend handling.
+ */
+typedef enum {
+    LOG_INSTR_FILTER_MEM_RANGE = 0,
+    LOG_INSTR_FILTER_MAX
+} cpu_log_instr_filter_t;
+
 /*
  * Request a flush of the TCG when changing loglevel outside of qemu_log_instr.
  * TODO(am2419): this should be removed from the interface.
@@ -250,29 +263,24 @@ void qemu_log_instr_flush_tcg(bool request_stop);
 bool qemu_log_instr_check_enabled(CPUArchState *env);
 
 /*
- * Start instruction tracing. Note that the instruction currently being
- * executed will be replaced by a trace start event.
+ * Register a trace filter for a given CPU.
  */
-void qemu_log_instr_start(CPUArchState *env, target_ulong pc);
+void qemu_log_instr_add_filter(CPUState *env, cpu_log_instr_filter_t filter);
 
 /*
- * Stop instruction tracing.
+ * Register a trace filter for a given CPU.
  */
-void qemu_log_instr_stop(CPUArchState *env, target_ulong pc);
+void qemu_log_instr_allcpu_add_filter(cpu_log_instr_filter_t filter);
 
 /*
- * Log a switch inc CPU modes.
- * This will also trigger pause and resume of user-only logging activity,
- * depending whether the mode parameter is QEMU_LOG_INSTR_CPU_USER or not.
+ * Unregister a trace filter for a given CPU.
  */
-void qemu_log_instr_mode_switch(CPUArchState *env,
-                                qemu_log_instr_cpu_mode_t mode,
-                                target_ulong pc);
+void qemu_log_instr_remove_filter(CPUState *env, cpu_log_instr_filter_t filter);
 
 /*
- * Set the given CPU per-CPU log level.
+ * Unregister a trace filter for a given CPU.
  */
-void qemu_log_instr_set_level(CPUArchState *env, qemu_log_instr_loglevel_t lvl);
+void qemu_log_instr_allcpu_remove_filter(cpu_log_instr_filter_t filter);
 
 /*
  * Log a switch inc CPU modes.
diff --git a/include/qemu/log_instr.h b/include/qemu/log_instr.h
index f22d163dad5..7addb25251b 100644
--- a/include/qemu/log_instr.h
+++ b/include/qemu/log_instr.h
@@ -137,7 +137,8 @@ typedef struct {
     /* Per-CPU flags */
     int flags;
 #define QEMU_LOG_INSTR_FLAG_BUFFERED 1
-
+    /* Trace entry filter functions */
+    GArray *filters;
     /* Ring buffer of log_instr_info */
     GArray *instr_info;
     /* Ring buffer index of the next entry to write */
@@ -161,6 +162,13 @@ int qemu_log_instr_global_switch(int log_flags);
  */
 void qemu_log_instr_set_buffer_size(unsigned long buffer_size);
 
+/*
+ * Trigger update of the instruction entry filter reusing
+ * the -dfilter qemu option to limit tracing to only interesting
+ * memory ranges..
+ */
+void qemu_log_instr_mem_filter_update(void);
+
 #else /* ! CONFIG_TCG_LOG_INSTR */
 #define qemu_log_instr_set_backend(id) ((void)0)
 #endif /* ! CONFIG_TCG_LOG_INSTR */
diff --git a/util/log.c b/util/log.c
index 9705a2570a9..2c8573c520b 100644
--- a/util/log.c
+++ b/util/log.c
@@ -35,6 +35,22 @@ int qemu_loglevel;
 static int log_append = 0;
 GArray *debug_regions;
 
+#ifdef CONFIG_TCG_LOG_INSTR
+__attribute__((weak)) int qemu_log_instr_global_switch(int log_flags);
+__attribute__((weak)) int qemu_log_instr_global_switch(int log_flags)
+{
+    /* Real implementation in accel/tcg/log_instr.c. */
+    return log_flags;
+}
+
+__attribute__((weak)) void qemu_log_instr_mem_filter_update(void);
+__attribute__((weak)) void qemu_log_instr_mem_filter_update()
+{
+    /* Real implementation in accel/tcg/log_instr.c */
+    warn_report("Calling no-op %s\r", __func__);
+}
+#endif
+
 /* Return the number of characters emitted.  */
 int qemu_log(const char *fmt, ...)
 {
@@ -75,13 +91,6 @@ static void qemu_logfile_free(QemuLogFile *logfile)
 
 static bool log_uses_own_buffers;
 
-__attribute__((weak)) int qemu_log_instr_global_switch(int log_flags);
-__attribute__((weak)) int qemu_log_instr_global_switch(int log_flags)
-{
-    /* Real implementation in accel/tcg/log_instr.c. */
-    return log_flags;
-}
-
 /* enable or disable low levels log */
 void qemu_set_log_internal(int log_flags)
 {
@@ -286,6 +295,9 @@ void qemu_set_dfilter_ranges(const char *filter_spec, Error **errp)
     }
 out:
     g_strfreev(ranges);
+#ifdef CONFIG_TCG_LOG_INSTR
+    qemu_log_instr_mem_filter_update();
+#endif
 }
 
 /* fflush() the log file */

From 164538223d2a499e6314a0733d44f3e44643957d Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Thu, 15 Jul 2021 16:51:05 +0100
Subject: [PATCH 05/74] Refactor TCG log instruction backend into multiple
 files.

Each trace format is moved to a separate file for implementation.
---
 accel/tcg/log_instr.c             | 491 ++----------------------------
 accel/tcg/log_instr_cvtrace.c     | 174 +++++++++++
 accel/tcg/log_instr_text.c        | 215 +++++++++++++
 accel/tcg/meson.build             |   2 +
 include/exec/log_instr_internal.h | 179 +++++++++++
 5 files changed, 599 insertions(+), 462 deletions(-)
 create mode 100644 accel/tcg/log_instr_cvtrace.c
 create mode 100644 accel/tcg/log_instr_text.c
 create mode 100644 include/exec/log_instr_internal.h

diff --git a/accel/tcg/log_instr.c b/accel/tcg/log_instr.c
index b86ce3a9c14..0c8308b8e82 100644
--- a/accel/tcg/log_instr.c
+++ b/accel/tcg/log_instr.c
@@ -1,7 +1,7 @@
 /*-
  * SPDX-License-Identifier: BSD-2-Clause
  *
- * Copyright (c) 2020 Alfredo Mazzinghi
+ * Copyright (c) 2020,2021 Alfredo Mazzinghi
  *
  * This software was developed by SRI International and the University of
  * Cambridge Computer Laboratory (Department of Computer Science and
@@ -41,6 +41,7 @@
 #include "exec/log.h"
 #include "exec/helper-proto.h"
 #include "exec/log_instr.h"
+#include "exec/log_instr_internal.h"
 #include "exec/memop.h"
 #include "disas/disas.h"
 #include "exec/translator.h"
@@ -81,12 +82,6 @@
 
 // #define CONFIG_DEBUG_TCG
 
-#ifdef CONFIG_DEBUG_TCG
-#define log_assert(x) assert((x))
-#else
-#define log_assert(x)
-#endif
-
 #ifndef TARGET_MAX_INSN_SIZE
 #error "Target does not define TARGET_MAX_INSN_SIZE in cpu-param.h"
 #endif
@@ -96,460 +91,50 @@
  */
 extern GArray *debug_regions;
 
-/*
- * Instruction log info associated with each committed log entry.
- * This is stored in the per-cpu log cpustate.
- */
-typedef struct cpu_log_entry {
-#define cpu_log_entry_startzero asid
-    uint16_t asid;
-    int flags;
-/* Entry contains a synchronous exception */
-#define LI_FLAG_INTR_TRAP (1 << 0)
-/* Entry contains an asynchronous exception */
-#define LI_FLAG_INTR_ASYNC (1 << 1)
-#define LI_FLAG_INTR_MASK  (LI_FLAG_INTR_TRAP | LI_FLAG_INTR_ASYNC)
-/* Entry contains a CPU mode-switch and associated code */
-#define LI_FLAG_MODE_SWITCH (1 << 2)
-/* Entry contains instruction data or just events */
-#define LI_FLAG_HAS_INSTR_DATA (1 << 3)
-
-    qemu_log_instr_cpu_mode_t next_cpu_mode;
-    uint32_t intr_code;
-    target_ulong intr_vector;
-    target_ulong intr_faultaddr;
-
-    target_ulong pc;
-    /* Generic instruction opcode buffer */
-    int insn_size;
-    char insn_bytes[TARGET_MAX_INSN_SIZE];
-#define cpu_log_entry_endzero mem
-    /*
-     * Array of log_meminfo_t.
-     * For now we allow multiple accesses to be tied to one instruction.
-     * Some architectures may have multiple memory accesses
-     * in the same instruction (e.g. x86-64 pop r/m64,
-     * vector/matrix instructions, load/store pair). It is unclear
-     * whether we would treat these as multiple trace "entities".
-     */
-    GArray *mem;
-    /* Register modifications - array of log_reginfo_t */
-    GArray *regs;
-    /*
-     * Events associated with the instruction - array of log_event_t.
-     * Note that events may be present even if the entry does not contain
-     * valid instruction state. This allows the flexibility of only
-     * instrumenting a subset of the instructions/events.
-     */
-    GArray *events;
-    /* Extra text-only log */
-    GString *txt_buffer;
-} cpu_log_entry_t;
-
-/*
- * Register update info.
- * This records a CPU register update occurred during an instruction.
- */
-typedef struct {
-    uint16_t flags;
-#define LRI_CAP_REG   1
-#define LRI_HOLDS_CAP 2
-
-    const char *name;
-    union {
-        target_ulong gpr;
-#ifdef TARGET_CHERI
-        cap_register_t cap;
-#endif
-    };
-} log_reginfo_t;
-
-#define reginfo_is_cap(ri)  (ri->flags & LRI_CAP_REG)
-#define reginfo_has_cap(ri) (reginfo_is_cap(ri) && (ri->flags & LRI_HOLDS_CAP))
-
-/*
- * Memory access info.
- * This records a memory access occurred during an instruction.
- */
-typedef struct {
-    uint8_t flags;
-#define LMI_LD  1
-#define LMI_ST  2
-#define LMI_CAP 4
-    MemOp op;
-    target_ulong addr;
-    union {
-        uint64_t value;
-#ifdef TARGET_CHERI
-        cap_register_t cap;
-#endif
-    };
-} log_meminfo_t;
-
-/*
- * Callbacks defined by a trace backend implementation.
- * These are called to covert instruction tracing events to the corresponding
- * binary or text format.
- */
-struct trace_backend_hooks {
-    void (*emit_header)(CPUArchState *env);
-    void (*emit_instr)(CPUArchState *env, cpu_log_entry_t *entry);
-    void (*emit_events)(CPUArchState *env, cpu_log_entry_t *entry);
-};
-typedef struct trace_backend_hooks trace_backend_hooks_t;
-
 /* Global trace format selector. Defaults to text tracing */
 qemu_log_instr_backend_t qemu_log_instr_backend = QLI_FMT_TEXT;
 
 /* Current format callbacks. */
 static trace_backend_hooks_t *trace_backend;
 
-/* Existing format callbacks list, indexed by qemu_log_instr_backend_t */
-static trace_backend_hooks_t trace_backends[];
-
-/* Existing trace filters list, indexed by cpu_log_instr_filter_t */
-static cpu_log_instr_filter_fn_t trace_filters[];
+static void emit_nop_entry(CPUArchState *env, cpu_log_entry_t *entry);
 
 /*
- * Instruction entry filter function.
- * Return false if the entry should be dropped, true otherwise.
+ * Existing format callbacks list, indexed by qemu_log_instr_backend_t.
  */
-typedef bool (*cpu_log_instr_filter_fn_t)(struct cpu_log_entry *entry);
+static trace_backend_hooks_t trace_backends[] = {
+    { .emit_header = NULL,
+      .emit_instr = emit_text_instr,
+      .emit_events = emit_text_events },
+    { .emit_header = emit_cvtrace_header,
+      .emit_instr = emit_cvtrace_entry,
+      .emit_events = NULL },
+    { .emit_header = NULL,
+      .emit_instr = emit_nop_entry,
+      .emit_events = emit_nop_entry },
+#ifdef CONFIG_TRACE_PERFETTO
+/* { */
+/*     .emit_header = NULL, */
+/*     .emit_start = emit_perfetto_start, */
+/*     .emit_stop = emit_perfetto_stop, */
+/*     .emit_entry = emit_perfetto_entry */
+/* } */
+#endif
+};
 
-/*
- * CHERI binary trace format, originally used for MIPS only.
- * The format is limited to one entry per instruction, each
- * entry can hold at most one register modification and one
- * memory address.
- * Note that the CHERI format is the legacy MIPS format and
- * assumes big-endian byte order.
- */
-typedef struct {
-    uint8_t entry_type;
-#define CTE_NO_REG 0   /* No register is changed. */
-#define CTE_GPR    1   /* GPR change (val2) */
-#define CTE_LD_GPR 2   /* Load into GPR (val2) from address (val1) */
-#define CTE_ST_GPR 3   /* Store from GPR (val2) to address (val1) */
-#define CTE_CAP    11  /* Cap change (val2,val3,val4,val5) */
-#define CTE_LD_CAP 12  /* Load Cap (val2,val3,val4,val5) from addr (val1) */
-#define CTE_ST_CAP 13  /* Store Cap (val2,val3,val4,val5) to addr (val1) */
-    uint8_t exception; /* 0=none, 1=TLB Mod, 2=TLB Load, 3=TLB Store, etc. */
-#define CTE_EXCEPTION_NONE 31
-    uint16_t cycles; /* Currently not used. */
-    uint32_t inst;   /* Encoded instruction. */
-    uint64_t pc;     /* PC value of instruction. */
-    uint64_t val1;   /* val1 is used for memory address. */
-    uint64_t val2;   /* val2, val3, val4, val5 are used for reg content. */
-    uint64_t val3;
-    uint64_t val4;
-    uint64_t val5;
-    uint8_t thread; /* Hardware thread/CPU (i.e. cpu->cpu_index ) */
-    uint8_t asid;   /* Address Space ID */
-} __attribute__((packed)) cheri_trace_entry_t;
-
-/* Version 3 Cheri Stream Trace header info */
-#define CTE_QEMU_VERSION (0x80U + 3)
-#define CTE_QEMU_MAGIC   "CheriTraceV03"
+/* Existing trace filters list, indexed by cpu_log_instr_filter_t */
+static cpu_log_instr_filter_fn_t trace_filters[];
 
 /* Number of per-cpu ring buffer entries for ring-buffer tracing mode */
 #define MIN_ENTRY_BUFFER_SIZE (1 << 16)
 
 static unsigned long reset_entry_buffer_size = MIN_ENTRY_BUFFER_SIZE;
 
-/*
- * Fetch the log state for a cpu.
- */
-static inline cpu_log_instr_state_t *get_cpu_log_state(CPUArchState *env)
-{
-    return &env_cpu(env)->log_state;
-}
-
-/*
- * Fetch the given cpu current instruction info
- */
-static inline cpu_log_entry_t *get_cpu_log_entry(CPUArchState *env)
-{
-    cpu_log_instr_state_t *cpulog = get_cpu_log_state(env);
-
-    return &g_array_index(cpulog->instr_info, cpu_log_entry_t,
-                          cpulog->ring_head);
-}
-
-/* Text trace format emitters */
-
-/*
- * Emit textual trace representation of memory access
- */
-static inline void emit_text_ldst(log_meminfo_t *minfo, const char *direction)
-{
-
-#ifndef TARGET_CHERI
-    log_assert((minfo->flags & LMI_CAP) == 0 &&
-               "Capability memory access without CHERI support");
-#else
-    if (minfo->flags & LMI_CAP) {
-        qemu_log("    Cap Memory %s [" TARGET_FMT_lx
-                 "] = v:%d PESBT:" TARGET_FMT_lx " Cursor:" TARGET_FMT_lx "\n",
-                 direction, minfo->addr, minfo->cap.cr_tag,
-                 CAP_cc(compress_mem)(&minfo->cap),
-                 cap_get_cursor(&minfo->cap));
-    } else
-#endif
-    {
-        switch (memop_size(minfo->op)) {
-        default:
-            qemu_log("    Unknown memory access width\n");
-            /* fallthrough */
-        case 8:
-            qemu_log("    Memory %s [" TARGET_FMT_lx "] = " TARGET_FMT_plx "\n",
-                     direction, minfo->addr, minfo->value);
-            break;
-        case 4:
-            qemu_log("    Memory %s [" TARGET_FMT_lx "] = %08x\n", direction,
-                     minfo->addr, (uint32_t)minfo->value);
-            break;
-        case 2:
-            qemu_log("    Memory %s [" TARGET_FMT_lx "] = %04x\n", direction,
-                     minfo->addr, (uint16_t)minfo->value);
-            break;
-        case 1:
-            qemu_log("    Memory %s [" TARGET_FMT_lx "] = %02x\n", direction,
-                     minfo->addr, (uint8_t)minfo->value);
-            break;
-        }
-    }
-}
-
-/*
- * Emit textual trace representation of register modification
- */
-static inline void emit_text_reg(log_reginfo_t *rinfo)
-{
-#ifndef TARGET_CHERI
-    log_assert(!reginfo_is_cap(rinfo) && "Register marked as capability "
-                                         "register whitout CHERI support");
-#else
-    if (reginfo_is_cap(rinfo)) {
-        if (reginfo_has_cap(rinfo))
-            qemu_log("    Write %s|" PRINT_CAP_FMTSTR_L1 "\n"
-                     "             |" PRINT_CAP_FMTSTR_L2 "\n",
-                     rinfo->name, PRINT_CAP_ARGS_L1(&rinfo->cap),
-                     PRINT_CAP_ARGS_L2(&rinfo->cap));
-        else
-            qemu_log("  %s <- " TARGET_FMT_lx " (setting integer value)\n",
-                     rinfo->name, rinfo->gpr);
-    } else
-#endif
-    {
-        qemu_log("    Write %s = " TARGET_FMT_lx "\n", rinfo->name, rinfo->gpr);
-    }
-}
-
-/*
- * Emit textual representation of the instruction in the given
- * trace entry to the log.
- */
-static void emit_text_instr(CPUArchState *env, cpu_log_entry_t *entry)
-{
-    QemuLogFile *logfile;
-    int i;
-
-    /* Dump CPU-ID:ASID + address */
-    qemu_log("[%d:%d] ", env_cpu(env)->cpu_index, entry->asid);
-
-    /*
-     * Instruction disassembly, note that we use the instruction info
-     * opcode bytes, without accessing target memory here.
-     */
-    rcu_read_lock();
-    logfile = qatomic_rcu_read(&qemu_logfile);
-    if (logfile) {
-        target_disas_buf(logfile->fd, env_cpu(env), entry->insn_bytes,
-                         sizeof(entry->insn_bytes), entry->pc, 1);
-    }
-    rcu_read_unlock();
-
-    /*
-     * TODO(am2419): what to do with injected instructions?
-     * Is the rvfi_dii_trace state valid at log commit?
-     */
-
-    /* Dump mode switching info */
-    if (entry->flags & LI_FLAG_MODE_SWITCH)
-        qemu_log("-> Switch to %s mode\n",
-                 cpu_get_mode_name(entry->next_cpu_mode));
-    /* Dump interrupt/exception info */
-    switch (entry->flags & LI_FLAG_INTR_MASK) {
-    case LI_FLAG_INTR_TRAP:
-        qemu_log("-> Exception #%u vector 0x" TARGET_FMT_lx
-                 " fault-addr 0x" TARGET_FMT_lx "\n",
-                 entry->intr_code, entry->intr_vector, entry->intr_faultaddr);
-        break;
-    case LI_FLAG_INTR_ASYNC:
-        qemu_log("-> Interrupt #%04x vector 0x" TARGET_FMT_lx "\n",
-                 entry->intr_code, entry->intr_vector);
-        break;
-    default:
-        /* No interrupt */
-        break;
-    }
-
-    /* Dump memory access */
-    for (i = 0; i < entry->mem->len; i++) {
-        log_meminfo_t *minfo = &g_array_index(entry->mem, log_meminfo_t, i);
-        if (minfo->flags & LMI_LD) {
-            emit_text_ldst(minfo, "Read");
-        } else if (minfo->flags & LMI_ST) {
-            emit_text_ldst(minfo, "Write");
-        }
-    }
-
-    /* Dump register changes and side-effects */
-    for (i = 0; i < entry->regs->len; i++) {
-        log_reginfo_t *rinfo = &g_array_index(entry->regs, log_reginfo_t, i);
-        emit_text_reg(rinfo);
-    }
-
-    /* Dump extra logged messages, if any */
-    if (entry->txt_buffer->len > 0) {
-        qemu_log("%s", entry->txt_buffer->str);
-    }
-}
-
-/*
- * Emit a textual representation of events recorded by the given trace entry.
- */
-static void emit_text_events(CPUArchState *env, cpu_log_entry_t *entry)
-{
-    cpu_log_instr_state_t *cpulog = get_cpu_log_state(env);
-    const log_event_t *event;
-    const char *log_state_op;
-    int i;
-
-    for (i = 0; i < entry->events->len; i++) {
-        event = &g_array_index(entry->events, const log_event_t, i);
-        if (event->id == LOG_EVENT_STATE) {
-            if (event->state.next_state == LOG_EVENT_STATE_START) {
-                log_state_op = "Requested";
-            } else if (event->state.next_state == LOG_EVENT_STATE_STOP) {
-                log_state_op = "Disabled";
-            }
-
-            if (cpulog->loglevel == QEMU_LOG_INSTR_LOGLEVEL_USER) {
-                qemu_log("[%u:%u] %s user-mode only instruction logging "
-                         "@ " TARGET_FMT_lx "\n",
-                         env_cpu(env)->cpu_index, cpu_get_asid(env),
-                         log_state_op, event->state.pc);
-            } else {
-                qemu_log("[%u:%u] %s instruction logging @ " TARGET_FMT_lx
-                         "\n",
-                         env_cpu(env)->cpu_index, cpu_get_asid(env),
-                         log_state_op, event->state.pc);
-            }
-        }
-    }
-}
-
-/* CHERI trace V3 format emitters */
-
-/*
- * Emit cvtrace trace trace header. This is a magic byte + string
- */
-static void emit_cvtrace_header(CPUArchState *env)
+static void emit_nop_entry(CPUArchState *env, cpu_log_entry_t *entry)
 {
-    FILE *logfile = qemu_log_lock();
-    char buffer[sizeof(cheri_trace_entry_t)];
-
-    buffer[0] = CTE_QEMU_VERSION;
-    g_strlcpy(buffer + 1, CTE_QEMU_MAGIC, sizeof(buffer) - 2);
-    fwrite(buffer, sizeof(buffer), 1, logfile);
-    qemu_log_unlock(logfile);
+    return;
 }
 
-/*
- * Emit cvtrace trace entry.
- * Note: this format is very MIPS-specific.
- */
-static void emit_cvtrace_entry(CPUArchState *env, cpu_log_entry_t *entry)
-{
-    FILE *logfile;
-    cheri_trace_entry_t ct_entry;
-    /* TODO(am2419): this should be a per-cpu counter. */
-    static uint16_t cycles;
-    uint32_t *insn = (uint32_t *)&entry->insn_bytes[0];
-
-    ct_entry.entry_type = CTE_NO_REG;
-    ct_entry.thread = (uint8_t)env_cpu(env)->cpu_index;
-    ct_entry.asid = (uint8_t)entry->asid;
-    ct_entry.pc = cpu_to_be64(entry->pc);
-    ct_entry.cycles = cpu_to_be16(cycles++);
-    /*
-     * TODO(am2419): The instruction bytes are alread in target byte-order,
-     * however cheritrace does not currently expect this.
-     */
-    ct_entry.inst = cpu_to_be32(*insn);
-    switch (entry->flags & LI_FLAG_INTR_MASK) {
-    case LI_FLAG_INTR_TRAP:
-        ct_entry.exception = (uint8_t)(entry->intr_code & 0xff);
-    case LI_FLAG_INTR_ASYNC:
-        ct_entry.exception = 0;
-    default:
-        ct_entry.exception = CTE_EXCEPTION_NONE;
-    }
-
-    if (entry->regs->len) {
-        log_reginfo_t *rinfo = &g_array_index(entry->regs, log_reginfo_t, 0);
-#ifndef TARGET_CHERI
-        log_assert(!reginfo_is_cap(rinfo) && "Capability register access "
-                                             "without CHERI support");
-#else
-        if (reginfo_is_cap(rinfo)) {
-            cap_register_t intcap;
-            cap_register_t *cr = &rinfo->cap;
-
-            if (!reginfo_has_cap(rinfo)) {
-                // cvtrace expects a null capability in the integer case
-                cr = null_capability(&intcap);
-            }
-            uint64_t metadata = (((uint64_t)cr->cr_tag << 63) |
-                                 ((uint64_t)cap_get_otype_signext(cr) << 32) |
-                                 ((uint64_t)COMBINED_PERMS_VALUE(cr) << 1) |
-                                 (uint64_t)(cap_is_unsealed(cr) ? 0 : 1));
-
-            ct_entry.entry_type = CTE_CAP;
-            ct_entry.val2 = cpu_to_be64(metadata);
-            ct_entry.val3 = cpu_to_be64(cap_get_cursor(cr));
-            ct_entry.val4 = cpu_to_be64(cap_get_base(cr));
-            ct_entry.val5 = cpu_to_be64(cap_get_length_sat(cr));
-        } else
-#endif
-        {
-            ct_entry.entry_type = CTE_GPR;
-            ct_entry.val2 = cpu_to_be64(rinfo->gpr);
-        }
-    }
-
-    if (entry->mem->len) {
-        log_meminfo_t *minfo = &g_array_index(entry->mem, log_meminfo_t, 0);
-#ifndef TARGET_CHERI
-        log_assert((minfo->flags & LMI_CAP) == 0 && "Capability memory access "
-                                                    "without CHERI support");
-#endif
-        ct_entry.val1 = cpu_to_be64(minfo->addr);
-        // Hack to avoid checking for GPR or CAP
-        if (minfo->flags & LMI_LD)
-            ct_entry.entry_type += 1;
-        else if (minfo->flags & LMI_ST)
-            ct_entry.entry_type += 2;
-    }
-
-    logfile = qemu_log_lock();
-    fwrite(&ct_entry, sizeof(ct_entry), 1, logfile);
-    qemu_log_unlock(logfile);
-}
-
-/* Core instruction logging implementation */
-
 static inline void emit_start_event(cpu_log_entry_t *entry, target_ulong pc)
 {
     log_event_t event;
@@ -1524,8 +1109,7 @@ void helper_qemu_log_printf_dump(CPUArchState *env)
         return;
     }
 
-    cpu_log_instr_info_t *iinfo = get_cpu_log_instr_info(env);
-
+    cpu_log_entry_t *entry = get_cpu_log_entry(env);
     while (valid) {
         size_t ndx = ctz64(valid);
         valid ^= (1 << ndx);
@@ -1533,7 +1117,7 @@ void helper_qemu_log_printf_dump(CPUArchState *env)
             cpulog->qemu_log_printf_buf.args +
             (ndx * QEMU_LOG_PRINTF_ARG_MAX);
         const char *fmt = cpulog->qemu_log_printf_buf.fmts[ndx];
-        g_string_append_printf_union_args(iinfo->txt_buffer, fmt, args);
+        g_string_append_printf_union_args(entry->txt_buffer, fmt, args);
     }
 }
 
@@ -1667,23 +1251,6 @@ void helper_log_value(CPUArchState *env, const void *ptr, uint64_t value)
     qemu_maybe_log_instr_extra(env, "%s: " TARGET_FMT_plx "\n", ptr, value);
 }
 
-static void emit_nop_entry(CPUArchState *env, cpu_log_entry_t *entry) {}
-
-/*
- * Hooks for each trace format
- */
-static trace_backend_hooks_t trace_backends[] = {
-    { .emit_header = NULL,
-      .emit_instr = emit_text_instr,
-      .emit_events = emit_text_events },
-    { .emit_header = emit_cvtrace_header,
-      .emit_instr = emit_cvtrace_entry,
-      .emit_events = NULL },
-    { .emit_header = NULL,
-      .emit_instr = emit_nop_entry,
-      .emit_events = emit_nop_entry }
-};
-
 void qemu_log_instr_add_filter(CPUState *cpu, cpu_log_instr_filter_t filter)
 {
     cpu_log_instr_state_t *cpulog = &cpu->log_state;
diff --git a/accel/tcg/log_instr_cvtrace.c b/accel/tcg/log_instr_cvtrace.c
new file mode 100644
index 00000000000..446ac3cd99f
--- /dev/null
+++ b/accel/tcg/log_instr_cvtrace.c
@@ -0,0 +1,174 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2021 Alfredo Mazzinghi
+ *
+ * This software was developed by SRI International and the University of
+ * Cambridge Computer Laboratory (Department of Computer Science and
+ * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the
+ * DARPA SSITH research programme.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * CHERI Trace binary format backend
+ */
+
+#include "qemu/osdep.h"
+#include "qemu/log.h"
+#include "exec/log_instr.h"
+#include "exec/log_instr_internal.h"
+#include "exec/memop.h"
+#include "disas/disas.h"
+
+/*
+ * CHERI binary trace format, originally used for MIPS only.
+ * The format is limited to one entry per instruction, each
+ * entry can hold at most one register modification and one
+ * memory address.
+ * Note that the CHERI format is the legacy MIPS format and
+ * assumes big-endian byte order.
+ */
+typedef struct {
+    uint8_t entry_type;
+#define CTE_NO_REG 0   /* No register is changed. */
+#define CTE_GPR    1   /* GPR change (val2) */
+#define CTE_LD_GPR 2   /* Load into GPR (val2) from address (val1) */
+#define CTE_ST_GPR 3   /* Store from GPR (val2) to address (val1) */
+#define CTE_CAP    11  /* Cap change (val2,val3,val4,val5) */
+#define CTE_LD_CAP 12  /* Load Cap (val2,val3,val4,val5) from addr (val1) */
+#define CTE_ST_CAP 13  /* Store Cap (val2,val3,val4,val5) to addr (val1) */
+    uint8_t exception; /* 0=none, 1=TLB Mod, 2=TLB Load, 3=TLB Store, etc. */
+#define CTE_EXCEPTION_NONE 31
+    uint16_t cycles; /* Currently not used. */
+    uint32_t inst;   /* Encoded instruction. */
+    uint64_t pc;     /* PC value of instruction. */
+    uint64_t val1;   /* val1 is used for memory address. */
+    uint64_t val2;   /* val2, val3, val4, val5 are used for reg content. */
+    uint64_t val3;
+    uint64_t val4;
+    uint64_t val5;
+    uint8_t thread; /* Hardware thread/CPU (i.e. cpu->cpu_index ) */
+    uint8_t asid;   /* Address Space ID */
+} __attribute__((packed)) cheri_trace_entry_t;
+
+/* Version 3 Cheri Stream Trace header info */
+#define CTE_QEMU_VERSION (0x80U + 3)
+#define CTE_QEMU_MAGIC   "CheriTraceV03"
+
+/*
+ * Emit cvtrace trace trace header. This is a magic byte + string
+ */
+void emit_cvtrace_header(CPUArchState *env)
+{
+    FILE *logfile = qemu_log_lock();
+    char buffer[sizeof(cheri_trace_entry_t)];
+
+    buffer[0] = CTE_QEMU_VERSION;
+    g_strlcpy(buffer + 1, CTE_QEMU_MAGIC, sizeof(buffer) - 2);
+    fwrite(buffer, sizeof(buffer), 1, logfile);
+    qemu_log_unlock(logfile);
+}
+
+/*
+ * Emit cvtrace trace entry.
+ * Note: this format is very MIPS-specific.
+ */
+void emit_cvtrace_entry(CPUArchState *env, cpu_log_entry_t *entry)
+{
+    FILE *logfile;
+    cheri_trace_entry_t ct_entry;
+    /* TODO(am2419): this should be a per-cpu counter. */
+    static uint16_t cycles;
+    uint32_t *insn = (uint32_t *)&entry->insn_bytes[0];
+
+    ct_entry.entry_type = CTE_NO_REG;
+    ct_entry.thread = (uint8_t)env_cpu(env)->cpu_index;
+    ct_entry.asid = (uint8_t)entry->asid;
+    ct_entry.pc = cpu_to_be64(entry->pc);
+    ct_entry.cycles = cpu_to_be16(cycles++);
+    /*
+     * TODO(am2419): The instruction bytes are alread in target byte-order,
+     * however cheritrace does not currently expect this.
+     */
+    ct_entry.inst = cpu_to_be32(*insn);
+    switch (entry->flags & LI_FLAG_INTR_MASK) {
+    case LI_FLAG_INTR_TRAP:
+        ct_entry.exception = (uint8_t)(entry->intr_code & 0xff);
+    case LI_FLAG_INTR_ASYNC:
+        ct_entry.exception = 0;
+    default:
+        ct_entry.exception = CTE_EXCEPTION_NONE;
+    }
+
+    if (entry->regs->len) {
+        log_reginfo_t *rinfo = &g_array_index(entry->regs, log_reginfo_t, 0);
+#ifndef TARGET_CHERI
+        log_assert(!reginfo_is_cap(rinfo) && "Capability register access "
+                                             "without CHERI support");
+#else
+        if (reginfo_is_cap(rinfo)) {
+            cap_register_t intcap;
+            cap_register_t *cr = &rinfo->cap;
+
+            if (!reginfo_has_cap(rinfo)) {
+                /* cvtrace expects a null capability in the integer case */
+                cr = null_capability(&intcap);
+            }
+            uint64_t metadata = (((uint64_t)cr->cr_tag << 63) |
+                                 ((uint64_t)cap_get_otype(cr) << 32) |
+                                 ((uint64_t)COMBINED_PERMS_VALUE(cr) << 1) |
+                                 (uint64_t)(cap_is_unsealed(cr) ? 0 : 1));
+
+            ct_entry.entry_type = CTE_CAP;
+            ct_entry.val2 = cpu_to_be64(metadata);
+            ct_entry.val3 = cpu_to_be64(cap_get_cursor(cr));
+            ct_entry.val4 = cpu_to_be64(cap_get_base(cr));
+            ct_entry.val5 = cpu_to_be64(cap_get_length_sat(cr));
+        } else
+#endif
+        {
+            ct_entry.entry_type = CTE_GPR;
+            ct_entry.val2 = cpu_to_be64(rinfo->gpr);
+        }
+    }
+
+    if (entry->mem->len) {
+        log_meminfo_t *minfo = &g_array_index(entry->mem, log_meminfo_t, 0);
+#ifndef TARGET_CHERI
+        log_assert((minfo->flags & LMI_CAP) == 0 && "Capability memory access "
+                                                    "without CHERI support");
+#endif
+        ct_entry.val1 = cpu_to_be64(minfo->addr);
+        /* Hack to avoid checking for GPR or CAP */
+        if (minfo->flags & LMI_LD) {
+            ct_entry.entry_type += 1;
+        } else if (minfo->flags & LMI_ST) {
+            ct_entry.entry_type += 2;
+        }
+    }
+
+    logfile = qemu_log_lock();
+    fwrite(&ct_entry, sizeof(ct_entry), 1, logfile);
+    qemu_log_unlock(logfile);
+}
diff --git a/accel/tcg/log_instr_text.c b/accel/tcg/log_instr_text.c
new file mode 100644
index 00000000000..b1a6a90f57f
--- /dev/null
+++ b/accel/tcg/log_instr_text.c
@@ -0,0 +1,215 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2021 Alfredo Mazzinghi
+ *
+ * This software was developed by SRI International and the University of
+ * Cambridge Computer Laboratory (Department of Computer Science and
+ * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the
+ * DARPA SSITH research programme.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * Text instruction logging backend
+ */
+
+#include "qemu/osdep.h"
+#include "qemu/log.h"
+#include "exec/log_instr.h"
+#include "exec/log_instr_internal.h"
+#include "exec/memop.h"
+#include "disas/disas.h"
+
+/*
+ * Emit textual trace representation of memory access
+ */
+static inline void emit_text_ldst(log_meminfo_t *minfo, const char *direction)
+{
+
+#ifndef TARGET_CHERI
+    log_assert((minfo->flags & LMI_CAP) == 0 &&
+               "Capability memory access without CHERI support");
+#else
+    if (minfo->flags & LMI_CAP) {
+        qemu_log("    Cap Memory %s [" TARGET_FMT_lx
+                 "] = v:%d PESBT:" TARGET_FMT_lx " Cursor:" TARGET_FMT_lx "\n",
+                 direction, minfo->addr, minfo->cap.cr_tag,
+                 CAP_cc(compress_mem)(&minfo->cap),
+                 cap_get_cursor(&minfo->cap));
+    } else
+#endif
+    {
+        switch (memop_size(minfo->op)) {
+        default:
+            qemu_log("    Unknown memory access width\n");
+            /* fallthrough */
+        case 8:
+            qemu_log("    Memory %s [" TARGET_FMT_lx "] = " TARGET_FMT_plx "\n",
+                     direction, minfo->addr, minfo->value);
+            break;
+        case 4:
+            qemu_log("    Memory %s [" TARGET_FMT_lx "] = %08x\n", direction,
+                     minfo->addr, (uint32_t)minfo->value);
+            break;
+        case 2:
+            qemu_log("    Memory %s [" TARGET_FMT_lx "] = %04x\n", direction,
+                     minfo->addr, (uint16_t)minfo->value);
+            break;
+        case 1:
+            qemu_log("    Memory %s [" TARGET_FMT_lx "] = %02x\n", direction,
+                     minfo->addr, (uint8_t)minfo->value);
+            break;
+        }
+    }
+}
+
+/*
+ * Emit textual trace representation of register modification
+ */
+static inline void emit_text_reg(log_reginfo_t *rinfo)
+{
+#ifndef TARGET_CHERI
+    log_assert(!reginfo_is_cap(rinfo) && "Register marked as capability "
+                                         "register whitout CHERI support");
+#else
+    if (reginfo_is_cap(rinfo)) {
+        if (reginfo_has_cap(rinfo))
+            qemu_log("    Write %s|" PRINT_CAP_FMTSTR_L1 "\n"
+                     "             |" PRINT_CAP_FMTSTR_L2 "\n",
+                     rinfo->name, PRINT_CAP_ARGS_L1(&rinfo->cap),
+                     PRINT_CAP_ARGS_L2(&rinfo->cap));
+        else
+            qemu_log("  %s <- " TARGET_FMT_lx " (setting integer value)\n",
+                     rinfo->name, rinfo->gpr);
+    } else
+#endif
+    {
+        qemu_log("    Write %s = " TARGET_FMT_lx "\n", rinfo->name, rinfo->gpr);
+    }
+}
+
+/*
+ * Emit textual representation of the instruction in the given
+ * trace entry to the log.
+ */
+void emit_text_instr(CPUArchState *env, cpu_log_entry_t *entry)
+{
+    QemuLogFile *logfile;
+    int i;
+
+    /* Dump CPU-ID:ASID + address */
+    qemu_log("[%d:%d] ", env_cpu(env)->cpu_index, entry->asid);
+
+    /*
+     * Instruction disassembly, note that we use the instruction info
+     * opcode bytes, without accessing target memory here.
+     */
+    rcu_read_lock();
+    logfile = qatomic_rcu_read(&qemu_logfile);
+    if (logfile) {
+        target_disas_buf(logfile->fd, env_cpu(env), entry->insn_bytes,
+                         sizeof(entry->insn_bytes), entry->pc, 1);
+    }
+    rcu_read_unlock();
+
+    /*
+     * TODO(am2419): what to do with injected instructions?
+     * Is the rvfi_dii_trace state valid at log commit?
+     */
+
+    /* Dump mode switching info */
+    if (entry->flags & LI_FLAG_MODE_SWITCH)
+        qemu_log("-> Switch to %s mode\n",
+                 cpu_get_mode_name(entry->next_cpu_mode));
+    /* Dump interrupt/exception info */
+    switch (entry->flags & LI_FLAG_INTR_MASK) {
+    case LI_FLAG_INTR_TRAP:
+        qemu_log("-> Exception #%u vector 0x" TARGET_FMT_lx
+                 " fault-addr 0x" TARGET_FMT_lx "\n",
+                 entry->intr_code, entry->intr_vector, entry->intr_faultaddr);
+        break;
+    case LI_FLAG_INTR_ASYNC:
+        qemu_log("-> Interrupt #%04x vector 0x" TARGET_FMT_lx "\n",
+                 entry->intr_code, entry->intr_vector);
+        break;
+    default:
+        /* No interrupt */
+        break;
+    }
+
+    /* Dump memory access */
+    for (i = 0; i < entry->mem->len; i++) {
+        log_meminfo_t *minfo = &g_array_index(entry->mem, log_meminfo_t, i);
+        if (minfo->flags & LMI_LD) {
+            emit_text_ldst(minfo, "Read");
+        } else if (minfo->flags & LMI_ST) {
+            emit_text_ldst(minfo, "Write");
+        }
+    }
+
+    /* Dump register changes and side-effects */
+    for (i = 0; i < entry->regs->len; i++) {
+        log_reginfo_t *rinfo = &g_array_index(entry->regs, log_reginfo_t, i);
+        emit_text_reg(rinfo);
+    }
+
+    /* Dump extra logged messages, if any */
+    if (entry->txt_buffer->len > 0) {
+        qemu_log("%s", entry->txt_buffer->str);
+    }
+}
+
+/*
+ * Emit a textual representation of events recorded by the given trace entry.
+ */
+void emit_text_events(CPUArchState *env, cpu_log_entry_t *entry)
+{
+    cpu_log_instr_state_t *cpulog = get_cpu_log_state(env);
+    const log_event_t *event;
+    const char *log_state_op;
+    int i;
+
+    for (i = 0; i < entry->events->len; i++) {
+        event = &g_array_index(entry->events, const log_event_t, i);
+        if (event->id == LOG_EVENT_STATE) {
+            if (event->state.next_state == LOG_EVENT_STATE_START) {
+                log_state_op = "Requested";
+            } else if (event->state.next_state == LOG_EVENT_STATE_STOP) {
+                log_state_op = "Disabled";
+            }
+
+            if (cpulog->loglevel == QEMU_LOG_INSTR_LOGLEVEL_USER) {
+                qemu_log("[%u:%u] %s user-mode only instruction logging "
+                         "@ " TARGET_FMT_lx "\n",
+                         env_cpu(env)->cpu_index, cpu_get_asid(env),
+                         log_state_op, event->state.pc);
+            } else {
+                qemu_log("[%u:%u] %s instruction logging @ " TARGET_FMT_lx
+                         "\n",
+                         env_cpu(env)->cpu_index, cpu_get_asid(env),
+                         log_state_op, event->state.pc);
+            }
+        }
+    }
+}
diff --git a/accel/tcg/meson.build b/accel/tcg/meson.build
index 2b73cd433c0..9c9a2cf9fd1 100644
--- a/accel/tcg/meson.build
+++ b/accel/tcg/meson.build
@@ -14,3 +14,5 @@ specific_ss.add_all(when: 'CONFIG_TCG', if_true: tcg_ss)
 
 specific_ss.add(when: ['CONFIG_SOFTMMU', 'CONFIG_TCG'], if_true: files('tcg-all.c', 'cputlb.c', 'tcg-cpus.c'))
 specific_ss.add(when: ['CONFIG_TCG_LOG_INSTR', 'CONFIG_TCG'], if_true: files('log_instr.c'))
+specific_ss.add(when: ['CONFIG_TCG_LOG_INSTR', 'CONFIG_TCG'], if_true: files('log_instr_text.c'))
+specific_ss.add(when: ['CONFIG_TCG_LOG_INSTR', 'CONFIG_TCG'], if_true: files('log_instr_cvtrace.c'))
diff --git a/include/exec/log_instr_internal.h b/include/exec/log_instr_internal.h
new file mode 100644
index 00000000000..603743f352d
--- /dev/null
+++ b/include/exec/log_instr_internal.h
@@ -0,0 +1,179 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2021 Alfredo Mazzinghi
+ *
+ * This software was developed by SRI International and the University of
+ * Cambridge Computer Laboratory (Department of Computer Science and
+ * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the
+ * DARPA SSITH research programme.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * Internal TCG instruction logging macros and types.
+ * This should only be included by the instruction logging backend
+ * implementation and not from the qemu targets.
+ */
+
+#pragma once
+
+/*
+ * Instruction log info associated with each committed log entry.
+ * This is stored in the per-cpu log cpustate.
+ */
+typedef struct cpu_log_entry {
+#define cpu_log_entry_startzero asid
+    uint16_t asid;
+    int flags;
+/* Entry contains a synchronous exception */
+#define LI_FLAG_INTR_TRAP (1 << 0)
+/* Entry contains an asynchronous exception */
+#define LI_FLAG_INTR_ASYNC (1 << 1)
+#define LI_FLAG_INTR_MASK  (LI_FLAG_INTR_TRAP | LI_FLAG_INTR_ASYNC)
+/* Entry contains a CPU mode-switch and associated code */
+#define LI_FLAG_MODE_SWITCH (1 << 2)
+/* Entry contains instruction data or just events */
+#define LI_FLAG_HAS_INSTR_DATA (1 << 3)
+
+    qemu_log_instr_cpu_mode_t next_cpu_mode;
+    uint32_t intr_code;
+    target_ulong intr_vector;
+    target_ulong intr_faultaddr;
+
+    target_ulong pc;
+    /* Generic instruction opcode buffer */
+    int insn_size;
+    char insn_bytes[TARGET_MAX_INSN_SIZE];
+#define cpu_log_entry_endzero mem
+    /*
+     * Array of log_meminfo_t.
+     * For now we allow multiple accesses to be tied to one instruction.
+     * Some architectures may have multiple memory accesses
+     * in the same instruction (e.g. x86-64 pop r/m64,
+     * vector/matrix instructions, load/store pair). It is unclear
+     * whether we would treat these as multiple trace "entities".
+     */
+    GArray *mem;
+    /* Register modifications - array of log_reginfo_t */
+    GArray *regs;
+    /*
+     * Events associated with the instruction - array of log_event_t.
+     * Note that events may be present even if the entry does not contain
+     * valid instruction state. This allows the flexibility of only
+     * instrumenting a subset of the instructions/events.
+     */
+    GArray *events;
+    /* Extra text-only log */
+    GString *txt_buffer;
+} cpu_log_entry_t;
+
+/*
+ * Register update info.
+ * This records a CPU register update occurred during an instruction.
+ */
+typedef struct {
+    uint16_t flags;
+#define LRI_CAP_REG   1
+#define LRI_HOLDS_CAP 2
+
+    const char *name;
+    union {
+        target_ulong gpr;
+#ifdef TARGET_CHERI
+        cap_register_t cap;
+#endif
+    };
+} log_reginfo_t;
+
+#define reginfo_is_cap(ri)  (ri->flags & LRI_CAP_REG)
+#define reginfo_has_cap(ri) (reginfo_is_cap(ri) && (ri->flags & LRI_HOLDS_CAP))
+
+/*
+ * Memory access info.
+ * This records a memory access occurred during an instruction.
+ */
+typedef struct {
+    uint8_t flags;
+#define LMI_LD  1
+#define LMI_ST  2
+#define LMI_CAP 4
+    MemOp op;
+    target_ulong addr;
+    union {
+        uint64_t value;
+#ifdef TARGET_CHERI
+        cap_register_t cap;
+#endif
+    };
+} log_meminfo_t;
+
+/*
+ * Callbacks defined by a trace backend implementation.
+ * These are called to covert instruction tracing events to the corresponding
+ * binary or text format.
+ */
+struct trace_backend_hooks {
+    void (*emit_header)(CPUArchState *env);
+    void (*emit_instr)(CPUArchState *env, cpu_log_entry_t *entry);
+    void (*emit_events)(CPUArchState *env, cpu_log_entry_t *entry);
+};
+typedef struct trace_backend_hooks trace_backend_hooks_t;
+
+/*
+ * Instruction entry filter function.
+ * Return false if the entry should be dropped, true otherwise.
+ */
+typedef bool (*cpu_log_instr_filter_fn_t)(struct cpu_log_entry *entry);
+
+/* Text backend */
+void emit_text_instr(CPUArchState *env, cpu_log_entry_t *entry);
+void emit_text_events(CPUArchState *env, cpu_log_entry_t *entry);
+/* CVTrace backend */
+void emit_cvtrace_header(CPUArchState *env);
+void emit_cvtrace_entry(CPUArchState *env, cpu_log_entry_t *entry);
+
+#ifdef CONFIG_DEBUG_TCG
+#define log_assert(x) assert((x))
+#else
+#define log_assert(x)
+#endif
+
+/*
+ * Fetch the log state for a cpu.
+ */
+static inline cpu_log_instr_state_t *get_cpu_log_state(CPUArchState *env)
+{
+    return &env_cpu(env)->log_state;
+}
+
+/*
+ * Fetch the given cpu current instruction info
+ */
+static inline cpu_log_entry_t *get_cpu_log_entry(CPUArchState *env)
+{
+    cpu_log_instr_state_t *cpulog = get_cpu_log_state(env);
+
+    return &g_array_index(cpulog->instr_info, cpu_log_entry_t,
+                          cpulog->ring_head);
+}

From a5ab2400e9869e2c5596493786ba4bbdb2368357 Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Mon, 19 Jul 2021 13:47:13 +0100
Subject: [PATCH 06/74] Actually emit events.

---
 accel/tcg/log_instr.c      | 35 ++++++++++++++++++++++++-----------
 accel/tcg/log_instr_text.c |  2 ++
 include/exec/log_instr.h   |  3 ++-
 3 files changed, 28 insertions(+), 12 deletions(-)

diff --git a/accel/tcg/log_instr.c b/accel/tcg/log_instr.c
index 0c8308b8e82..846fb929dde 100644
--- a/accel/tcg/log_instr.c
+++ b/accel/tcg/log_instr.c
@@ -112,14 +112,6 @@ static trace_backend_hooks_t trace_backends[] = {
     { .emit_header = NULL,
       .emit_instr = emit_nop_entry,
       .emit_events = emit_nop_entry },
-#ifdef CONFIG_TRACE_PERFETTO
-/* { */
-/*     .emit_header = NULL, */
-/*     .emit_start = emit_perfetto_start, */
-/*     .emit_stop = emit_perfetto_stop, */
-/*     .emit_entry = emit_perfetto_entry */
-/* } */
-#endif
 };
 
 /* Existing trace filters list, indexed by cpu_log_instr_filter_t */
@@ -175,6 +167,16 @@ static void reset_log_buffer(cpu_log_instr_state_t *cpulog,
 }
 
 /* Common instruction commit implementation */
+static void do_instr_emit(CPUArchState *env, cpu_log_entry_t *entry)
+{
+    if (entry->flags & LI_FLAG_HAS_INSTR_DATA) {
+        trace_backend->emit_instr(env, entry);
+    }
+    if (entry->events->len > 0) {
+        trace_backend->emit_events(env, entry);
+    }
+}
+
 static void do_instr_commit(CPUArchState *env)
 {
     cpu_log_instr_state_t *cpulog = get_cpu_log_state(env);
@@ -206,7 +208,7 @@ static void do_instr_commit(CPUArchState *env)
             cpulog->ring_tail =
                 (cpulog->ring_tail + 1) % cpulog->instr_info->len;
     } else {
-        trace_backend->emit_instr(env, entry);
+        do_instr_emit(env, entry);
     }
 }
 
@@ -1071,7 +1073,14 @@ void qemu_log_instr_flush(CPUArchState *env)
 {
     cpu_log_instr_state_t *cpulog = get_cpu_log_state(env);
     size_t curr = cpulog->ring_tail;
-    cpu_log_entry_t *entry;
+    cpu_log_entry_t *entry = get_cpu_log_entry(env);
+    log_event_t event;
+
+    /* Emit FLUSH event so that it can be picked up by backends */
+    event.id = LOG_EVENT_STATE;
+    event.state.next_state = LOG_EVENT_STATE_FLUSH;
+    event.state.pc = entry->pc;
+    qemu_log_instr_event(env, &event);
 
     /*
      * If tracing is disabled, force the commit of events in this
@@ -1086,7 +1095,7 @@ void qemu_log_instr_flush(CPUArchState *env)
 
     while (curr != cpulog->ring_head) {
         entry = &g_array_index(cpulog->instr_info, cpu_log_entry_t, curr);
-        trace_backend->emit_instr(env, entry);
+        do_instr_emit(env, entry);
         curr = (curr + 1) % cpulog->instr_info->len;
     }
     cpulog->ring_tail = cpulog->ring_head;
@@ -1251,6 +1260,10 @@ void helper_log_value(CPUArchState *env, const void *ptr, uint64_t value)
     qemu_maybe_log_instr_extra(env, "%s: " TARGET_FMT_plx "\n", ptr, value);
 }
 
+/*
+ * Instruction stream filtering
+ */
+
 void qemu_log_instr_add_filter(CPUState *cpu, cpu_log_instr_filter_t filter)
 {
     cpu_log_instr_state_t *cpulog = &cpu->log_state;
diff --git a/accel/tcg/log_instr_text.c b/accel/tcg/log_instr_text.c
index b1a6a90f57f..0a4b182cfbc 100644
--- a/accel/tcg/log_instr_text.c
+++ b/accel/tcg/log_instr_text.c
@@ -197,6 +197,8 @@ void emit_text_events(CPUArchState *env, cpu_log_entry_t *entry)
                 log_state_op = "Requested";
             } else if (event->state.next_state == LOG_EVENT_STATE_STOP) {
                 log_state_op = "Disabled";
+            } else {
+                continue;
             }
 
             if (cpulog->loglevel == QEMU_LOG_INSTR_LOGLEVEL_USER) {
diff --git a/include/exec/log_instr.h b/include/exec/log_instr.h
index aea2a3e0b3c..a8a317c5cc9 100644
--- a/include/exec/log_instr.h
+++ b/include/exec/log_instr.h
@@ -202,7 +202,8 @@ typedef enum {
  */
 typedef enum {
     LOG_EVENT_STATE_START,
-    LOG_EVENT_STATE_STOP
+    LOG_EVENT_STATE_STOP,
+    LOG_EVENT_STATE_FLUSH
 } log_event_trace_state_t;
 
 typedef struct {

From 05fb03adb3235138f59e2e7816475daf55be02b2 Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Sat, 17 Jul 2021 15:59:50 +0100
Subject: [PATCH 07/74] Rename TCG instruction logging backend macros.

---
 accel/tcg/log_instr.c    | 2 +-
 include/qemu/log_instr.h | 6 +++---
 softmmu/vl.c             | 6 +++---
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/accel/tcg/log_instr.c b/accel/tcg/log_instr.c
index 846fb929dde..72de6e73334 100644
--- a/accel/tcg/log_instr.c
+++ b/accel/tcg/log_instr.c
@@ -92,7 +92,7 @@
 extern GArray *debug_regions;
 
 /* Global trace format selector. Defaults to text tracing */
-qemu_log_instr_backend_t qemu_log_instr_backend = QLI_FMT_TEXT;
+qemu_log_instr_backend_t qemu_log_instr_backend = QEMU_LOG_INSTR_BACKEND_TEXT;
 
 /* Current format callbacks. */
 static trace_backend_hooks_t *trace_backend;
diff --git a/include/qemu/log_instr.h b/include/qemu/log_instr.h
index 7addb25251b..0e9eebf136e 100644
--- a/include/qemu/log_instr.h
+++ b/include/qemu/log_instr.h
@@ -51,9 +51,9 @@
  * Instruction logging format
  */
 typedef enum {
-    QLI_FMT_TEXT = 0,
-    QLI_FMT_CVTRACE = 1,
-    QLI_FMT_NOP = 2,
+    QEMU_LOG_INSTR_BACKEND_TEXT = 0,
+    QEMU_LOG_INSTR_BACKEND_CVTRACE = 1,
+    QEMU_LOG_INSTR_BACKEND_NOP = 2,
 } qemu_log_instr_backend_t;
 
 extern qemu_log_instr_backend_t qemu_log_instr_backend;
diff --git a/softmmu/vl.c b/softmmu/vl.c
index 633bdcfa5a7..ae4753600f7 100644
--- a/softmmu/vl.c
+++ b/softmmu/vl.c
@@ -3747,11 +3747,11 @@ void qemu_init(int argc, char **argv, char **envp)
                 }
                 break;
 #if defined(CONFIG_TCG_LOG_INSTR)
-            case QEMU_OPTION_cheri_trace_format:
+            case QEMU_OPTION_cheri_trace_backend:
                 if (strcmp(optarg, "text") == 0) {
-                    qemu_log_instr_set_backend(QLI_FMT_TEXT);
+                    qemu_log_instr_set_backend(QEMU_LOG_INSTR_BACKEND_TEXT);
                 } else if (strcmp(optarg, "cvtrace") == 0) {
-                    qemu_log_instr_set_backend(QLI_FMT_CVTRACE);
+                    qemu_log_instr_set_backend(QEMU_LOG_INSTR_BACKEND_CVTRACE);
                 } else {
                     printf("Invalid choice for cheri-trace-format: '%s'\n", optarg);
                     exit(1);

From ce352017ea03cde02b0461bdb327c73df98d9703 Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Mon, 19 Jul 2021 13:50:10 +0100
Subject: [PATCH 08/74] Add magic NOP to log context switch events.

---
 target/riscv/helper.h                   |  1 +
 target/riscv/insn_trans/trans_rvi.c.inc |  3 ++
 target/riscv/op_helper_log_instr.c      | 37 +++++++++++++++++++++++++
 3 files changed, 41 insertions(+)

diff --git a/target/riscv/helper.h b/target/riscv/helper.h
index d167e4b236c..09c54114a85 100644
--- a/target/riscv/helper.h
+++ b/target/riscv/helper.h
@@ -82,6 +82,7 @@ DEF_HELPER_3(sc_c_cap, tl, env, i32, i32)
 #ifdef CONFIG_TCG_LOG_INSTR
 DEF_HELPER_FLAGS_3(riscv_log_gpr_write, TCG_CALL_NO_RWG, void, env, i32, tl)
 DEF_HELPER_FLAGS_4(riscv_log_instr, TCG_CALL_NO_RWG, void, env, tl, i32, i32)
+DEF_HELPER_FLAGS_2(riscv_log_instr_event, TCG_CALL_NO_RWG, void, env, tl)
 #endif
 
 /* Special functions */
diff --git a/target/riscv/insn_trans/trans_rvi.c.inc b/target/riscv/insn_trans/trans_rvi.c.inc
index 8c00f3dc9a5..e4aa593cee4 100644
--- a/target/riscv/insn_trans/trans_rvi.c.inc
+++ b/target/riscv/insn_trans/trans_rvi.c.inc
@@ -284,6 +284,9 @@ static bool trans_slti(DisasContext *ctx, arg_slti *a)
             gen_helper_qemu_log_instr_stop(cpu_env, tpc);
             ctx->base.is_jmp = DISAS_NORETURN;
             break;
+        case 0x30:
+            gen_helper_riscv_log_instr_event(cpu_env, tpc);
+            break;
         }
         tcg_temp_free(tpc);
 
diff --git a/target/riscv/op_helper_log_instr.c b/target/riscv/op_helper_log_instr.c
index ac933f95b71..f94d00ef001 100644
--- a/target/riscv/op_helper_log_instr.c
+++ b/target/riscv/op_helper_log_instr.c
@@ -33,8 +33,15 @@
 #include "qemu/osdep.h"
 #include "exec/log_instr.h"
 #include "exec/helper-proto.h"
+#include "cheri-lazy-capregs.h"
 #include "cpu.h"
 
+#ifdef TARGET_CHERI
+#define get_gpr_value(env, regnum) get_capreg_cursor(env, regnum)
+#else
+#define get_gpr_value(env, regnum) (env->gpr[regnum])
+#endif
+
 void HELPER(riscv_log_gpr_write)(CPURISCVState *env, uint32_t regnum,
                                  target_ulong value)
 {
@@ -52,3 +59,33 @@ void HELPER(riscv_log_instr)(CPURISCVState *env, target_ulong pc,
         qemu_log_instr(env, pc, (char *)&opcode, opcode_size);
     }
 }
+
+/*
+ * Events are triggered by a magic no-op. The arguments for the event
+ * are passed via the target ABI argument registers.
+ * There is a maximum of 7 arguments supported.
+ * void event_noop(event_type, a1, a2, a3, a4, a5, a6, a7)
+ *
+ * We access GPRs directly from env to work around the argument limit
+ * for TCG helpers.
+ */
+void HELPER(riscv_log_instr_event)(CPURISCVState *env, target_ulong pc)
+{
+    log_event_t event;
+
+    if (!qemu_log_instr_enabled(env)) {
+        return;
+    }
+
+    event.id = get_gpr_value(env, 10);
+    switch (event.id) {
+    case LOG_EVENT_CTX_SWITCH:
+        event.ctx_switch.pid = get_gpr_value(env, 11);
+        event.ctx_switch.tid = get_gpr_value(env, 12);
+        event.ctx_switch.cid = get_gpr_value(env, 13);
+        break;
+    default:
+        return;
+    }
+    qemu_log_instr_event(env, &event);
+}

From d8b64869166773b03a29c94328a5bbcb44c05ab6 Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Sat, 17 Jul 2021 16:01:27 +0100
Subject: [PATCH 09/74] Add stats TCG instruction logging backend.

This backend is intended to capture statistics from the instructions
instead of dumping the whole instruction stream.
Note that this is disabled by default in the configuration as it adds
a dependency on libboost.
---
 accel/tcg/log_instr.c             |  24 ++++--
 accel/tcg/log_instr_stats.c       | 117 ++++++++++++++++++++++++++++++
 accel/tcg/meson.build             |   1 +
 configure                         |   8 ++
 include/exec/log_instr_internal.h |   5 ++
 include/qemu/log_instr.h          |   7 +-
 include/stats/stats.h             |  58 +++++++++++++++
 meson.build                       |  10 ++-
 qemu-options.hx                   |   8 +-
 softmmu/vl.c                      |   4 +
 stats/meson.build                 |  12 +++
 stats/trace_stats.cc              |  96 ++++++++++++++++++++++++
 12 files changed, 338 insertions(+), 12 deletions(-)
 create mode 100644 accel/tcg/log_instr_stats.c
 create mode 100644 include/stats/stats.h
 create mode 100644 stats/meson.build
 create mode 100644 stats/trace_stats.cc

diff --git a/accel/tcg/log_instr.c b/accel/tcg/log_instr.c
index 72de6e73334..ff159e05cf5 100644
--- a/accel/tcg/log_instr.c
+++ b/accel/tcg/log_instr.c
@@ -103,15 +103,24 @@ static void emit_nop_entry(CPUArchState *env, cpu_log_entry_t *entry);
  * Existing format callbacks list, indexed by qemu_log_instr_backend_t.
  */
 static trace_backend_hooks_t trace_backends[] = {
-    { .emit_header = NULL,
+    { .init = NULL,
+      .emit_header = NULL,
       .emit_instr = emit_text_instr,
       .emit_events = emit_text_events },
-    { .emit_header = emit_cvtrace_header,
+    { .init = NULL,
+      .emit_header = emit_cvtrace_header,
       .emit_instr = emit_cvtrace_entry,
       .emit_events = NULL },
-    { .emit_header = NULL,
+    { .init = NULL,
+      .emit_header = NULL,
       .emit_instr = emit_nop_entry,
       .emit_events = emit_nop_entry },
+#ifdef CONFIG_TRACE_STATS
+    { .init = init_stats_backend,
+      .emit_header = NULL,
+      .emit_instr = emit_stats_entry,
+      .emit_events = emit_stats_events },
+#endif
 };
 
 /* Existing trace filters list, indexed by cpu_log_instr_filter_t */
@@ -414,14 +423,19 @@ void qemu_log_instr_init(CPUState *cpu)
     cpulog->ring_tail = 0;
     reset_log_buffer(cpulog, entry);
 
-    // Make sure we are using the correct trace format.
+    /* Make sure we are using the correct trace format. */
     if (trace_backend == NULL) {
         trace_backend = &trace_backends[qemu_log_instr_backend];
-        // Only emit header on first init
+        /* Only emit header on first init */
         if (trace_backend->emit_header) {
+            /* XXX this can probably go away and just use init() */
             trace_backend->emit_header(cpu->env_ptr);
         }
     }
+    /* Initialize backend state on this CPU */
+    if (trace_backend->init) {
+        trace_backend->init(cpu->env_ptr);
+    }
 
     /* If we are starting with instruction logging enabled, switch it on now */
     if (qemu_loglevel_mask(CPU_LOG_INSTR_U))
diff --git a/accel/tcg/log_instr_stats.c b/accel/tcg/log_instr_stats.c
new file mode 100644
index 00000000000..4120ecbeafc
--- /dev/null
+++ b/accel/tcg/log_instr_stats.c
@@ -0,0 +1,117 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2021 Alfredo Mazzinghi
+ *
+ * This software was developed by SRI International and the University of
+ * Cambridge Computer Laboratory (Department of Computer Science and
+ * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the
+ * DARPA SSITH research programme.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * Statistic-gathering instruction logging backend
+ */
+
+#include <stdint.h>
+
+#include "qemu/osdep.h"
+#include "qemu/log.h"
+#include "exec/log_instr.h"
+#include "exec/log_instr_internal.h"
+#include "stats/stats.h"
+
+struct cpu_stats {
+    addr_range_hist_t pc_hist;
+    target_ulong last_pc;
+    target_ulong pc_range_start;
+};
+
+static struct cpu_stats *get_cpu_stats(CPUArchState *env)
+{
+    cpu_log_instr_state_t *cpulog = get_cpu_log_state(env);
+    return (struct cpu_stats *)cpulog->backend_data;
+}
+
+/*
+ * Initialize the backend.
+ */
+void init_stats_backend(CPUArchState *env)
+{
+    cpu_log_instr_state_t *cpulog = get_cpu_log_state(env);
+    struct cpu_stats *stats;
+
+    stats = malloc(sizeof(struct cpu_stats));
+    stats->pc_hist = qemu_stats_addr_range_hist_create();
+    stats->last_pc = 0;
+    stats->pc_range_start = 0;
+    cpulog->backend_data = stats;
+}
+
+/*
+ * Emit a textual representation of events recorded by the given trace entry.
+ */
+void emit_stats_events(CPUArchState *env, cpu_log_entry_t *entry)
+{
+    struct cpu_stats *stats = get_cpu_stats(env);
+    QemuLogFile *logfile;
+    const log_event_t *event;
+    int i;
+
+    log_assert(stats != NULL && "Missing stats backend data");
+
+    for (i = 0; i < entry->events->len; i++) {
+        event = &g_array_index(entry->events, const log_event_t, i);
+        if (event->id == LOG_EVENT_STATE) {
+            if (event->state.next_state == LOG_EVENT_STATE_FLUSH) {
+                rcu_read_lock();
+                logfile = qatomic_rcu_read(&qemu_logfile);
+                if (logfile) {
+                    qemu_stats_addr_range_hist_dump(stats->pc_hist,
+                                                    fileno(logfile->fd), true);
+                }
+                rcu_read_unlock();
+                break;
+            }
+        }
+    }
+}
+
+/*
+ * Emit textual representation of the instruction in the given
+ * trace entry to the log.
+ */
+void emit_stats_entry(CPUArchState *env, cpu_log_entry_t *entry)
+{
+    struct cpu_stats *stats = get_cpu_stats(env);
+
+    log_assert(stats != NULL && "Missing stats backend data");
+
+    if (entry->pc - stats->last_pc > entry->insn_size) {
+        qemu_stats_addr_range_hist_insert(stats->pc_hist, stats->pc_range_start,
+                                          stats->last_pc, 1);
+        stats->pc_range_start = entry->pc;
+    }
+    stats->last_pc = entry->pc;
+}
diff --git a/accel/tcg/meson.build b/accel/tcg/meson.build
index 9c9a2cf9fd1..e0476c72a33 100644
--- a/accel/tcg/meson.build
+++ b/accel/tcg/meson.build
@@ -16,3 +16,4 @@ specific_ss.add(when: ['CONFIG_SOFTMMU', 'CONFIG_TCG'], if_true: files('tcg-all.
 specific_ss.add(when: ['CONFIG_TCG_LOG_INSTR', 'CONFIG_TCG'], if_true: files('log_instr.c'))
 specific_ss.add(when: ['CONFIG_TCG_LOG_INSTR', 'CONFIG_TCG'], if_true: files('log_instr_text.c'))
 specific_ss.add(when: ['CONFIG_TCG_LOG_INSTR', 'CONFIG_TCG'], if_true: files('log_instr_cvtrace.c'))
+specific_ss.add(when: ['CONFIG_TRACE_STATS', 'CONFIG_TCG_LOG_INSTR', 'CONFIG_TCG'], if_true: files('log_instr_stats.c'))
diff --git a/configure b/configure
index 9f89ca22a55..28adfcf7508 100755
--- a/configure
+++ b/configure
@@ -340,6 +340,7 @@ pvrdma=""
 gprof="no"
 debug_tcg="no"
 tcg_log_instr="yes"
+trace_stats="no"
 rvfi_dii="no"
 debug="no"
 sanitizers="no"
@@ -1036,6 +1037,10 @@ for opt do
   ;;
   --disable-tcg-log-instr) tcg_log_instr="no"
   ;;
+  --enable-trace-stats) trace_stats="yes"
+  ;;
+  --disable-trace-stats) trace_stats="no"
+  ;;
   --enable-rvfi-dii) rvfi_dii="yes"
   ;;
   --disable-rvfi-dii) rvfi_dii="no"
@@ -5968,6 +5973,9 @@ fi
 if test "$tcg_log_instr" = "yes" ; then
   echo "CONFIG_TCG_LOG_INSTR=y" >> $config_host_mak
 fi
+if test "$trace_stats" = "yes" ; then
+    echo "CONFIG_TRACE_STATS=y" >> $config_host_mak
+fi
 if test "$rvfi_dii" = "yes" ; then
   echo "CONFIG_RVFI_DII=y" >> $config_host_mak
 fi
diff --git a/include/exec/log_instr_internal.h b/include/exec/log_instr_internal.h
index 603743f352d..d8e3ec96b74 100644
--- a/include/exec/log_instr_internal.h
+++ b/include/exec/log_instr_internal.h
@@ -134,6 +134,7 @@ typedef struct {
  * binary or text format.
  */
 struct trace_backend_hooks {
+    void (*init)(CPUArchState *env);
     void (*emit_header)(CPUArchState *env);
     void (*emit_instr)(CPUArchState *env, cpu_log_entry_t *entry);
     void (*emit_events)(CPUArchState *env, cpu_log_entry_t *entry);
@@ -152,6 +153,10 @@ void emit_text_events(CPUArchState *env, cpu_log_entry_t *entry);
 /* CVTrace backend */
 void emit_cvtrace_header(CPUArchState *env);
 void emit_cvtrace_entry(CPUArchState *env, cpu_log_entry_t *entry);
+/* Stats backen */
+void init_stats_backend(CPUArchState *env);
+void emit_stats_entry(CPUArchState *env, cpu_log_entry_t *entry);
+void emit_stats_events(CPUArchState *env, cpu_log_entry_t *entry);
 
 #ifdef CONFIG_DEBUG_TCG
 #define log_assert(x) assert((x))
diff --git a/include/qemu/log_instr.h b/include/qemu/log_instr.h
index 0e9eebf136e..1b0732a4abd 100644
--- a/include/qemu/log_instr.h
+++ b/include/qemu/log_instr.h
@@ -54,6 +54,9 @@ typedef enum {
     QEMU_LOG_INSTR_BACKEND_TEXT = 0,
     QEMU_LOG_INSTR_BACKEND_CVTRACE = 1,
     QEMU_LOG_INSTR_BACKEND_NOP = 2,
+#ifdef CONFIG_TRACE_STATS
+    QEMU_LOG_INSTR_BACKEND_STATS = 3,
+#endif
 } qemu_log_instr_backend_t;
 
 extern qemu_log_instr_backend_t qemu_log_instr_backend;
@@ -145,8 +148,10 @@ typedef struct {
     size_t ring_head;
     /* Ring buffer index of the first entry to dump */
     size_t ring_tail;
-
+    /* Accumulated debug printf buffer */
     qemu_log_printf_buf_t qemu_log_printf_buf;
+    /* Private trace backend state */
+    void *backend_data;
 } cpu_log_instr_state_t;
 
 /*
diff --git a/include/stats/stats.h b/include/stats/stats.h
new file mode 100644
index 00000000000..be4d12151ce
--- /dev/null
+++ b/include/stats/stats.h
@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2021 Alfredo Mazzinghi
+ * All rights reserved.
+ *
+ * This software was developed by SRI International and the University of
+ * Cambridge Computer Laboratory (Department of Computer Science and
+ * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the
+ * DARPA SSITH research programme.
+ *
+ * @BERI_LICENSE_HEADER_START@
+ *
+ * Licensed to BERI Open Systems C.I.C. (BERI) under one or more contributor
+ * license agreements.  See the NOTICE file distributed with this work for
+ * additional information regarding copyright ownership.  BERI licenses this
+ * file to you under the BERI Hardware-Software License, Version 1.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at:
+ *
+ *   http://www.beri-open-systems.org/legal/license-1-0.txt
+ *
+ * Unless required by applicable law or agreed to in writing, Work distributed
+ * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ * CONDITIONS OF ANY KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ * @BERI_LICENSE_HEADER_END@
+ */
+
+#pragma once
+
+#include <stdint.h>
+#include <stdbool.h>
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+#ifdef CONFIG_TRACE_STATS
+struct addr_range_hist;
+typedef struct addr_range_hist *addr_range_hist_t;
+
+/*
+ * Interface to manipulate an interval histogram representing a count
+ * over address ranges.
+ * The pointer returned is an opaque handle.
+ */
+addr_range_hist_t qemu_stats_addr_range_hist_create(void);
+void qemu_stats_addr_range_hist_destroy(addr_range_hist_t handle);
+void qemu_stats_addr_range_hist_insert(addr_range_hist_t handle,
+    uint64_t start, uint64_t end, uint64_t value);
+void qemu_stats_addr_range_hist_dump(addr_range_hist_t handle, int fd,
+    bool csv_header);
+
+#endif /* CONFIG_TRACE_STATS */
+#ifdef __cplusplus
+}
+#endif
diff --git a/meson.build b/meson.build
index 626e7092fcc..d477a80900d 100644
--- a/meson.build
+++ b/meson.build
@@ -1514,8 +1514,14 @@ util_ss = util_ss.apply(config_all, strict: false)
 libqemuutil = static_library('qemuutil',
                              sources: util_ss.sources() + stub_ss.sources() + genh,
                              dependencies: [util_ss.dependencies(), m, glib, socket, malloc])
-qemuutil = declare_dependency(link_with: libqemuutil,
-                              sources: genh + version_res)
+
+qemuutil_libs = [libqemuutil]
+if 'CONFIG_TRACE_STATS' in config_host
+    subdir('stats')
+    qemuutil_libs += libqemustats
+endif
+
+qemuutil = declare_dependency(link_with: qemuutil_libs,
 
 decodetree = generator(find_program('scripts/decodetree.py'),
                        output: 'decode-@BASENAME@.c.inc',
diff --git a/qemu-options.hx b/qemu-options.hx
index 6d4bd843265..706be14708d 100644
--- a/qemu-options.hx
+++ b/qemu-options.hx
@@ -4116,11 +4116,11 @@ SRST
     Set CHERI instruction trace buffer size to the given number of entries
 ERST
 
-DEF("cheri-trace-format", HAS_ARG, QEMU_OPTION_cheri_trace_format, \
-"-cheri-trace-format [text|cvtrace]     Select CHERI trace mode.\n", QEMU_ARCH_ALL)
+DEF("cheri-trace-backend", HAS_ARG, QEMU_OPTION_cheri_trace_backend, \
+"-cheri-trace-backend [text|cvtrace|stats]     Select CHERI trace mode.\n", QEMU_ARCH_ALL)
 SRST
-``-cheri-trace-format type``
-    Set CHERI trace format to <type> (text or cvtrace)
+``-cheri-trace-backend type``
+    Set CHERI trace backend to <type> (text, cvtrace or stats)
 ERST
 
 DEF("cheri-c2e-on-unrepresentable", 0, QEMU_OPTION_cheri_c2e_on_unrepresentable, \
diff --git a/softmmu/vl.c b/softmmu/vl.c
index ae4753600f7..5e18c10f721 100644
--- a/softmmu/vl.c
+++ b/softmmu/vl.c
@@ -3752,6 +3752,10 @@ void qemu_init(int argc, char **argv, char **envp)
                     qemu_log_instr_set_backend(QEMU_LOG_INSTR_BACKEND_TEXT);
                 } else if (strcmp(optarg, "cvtrace") == 0) {
                     qemu_log_instr_set_backend(QEMU_LOG_INSTR_BACKEND_CVTRACE);
+#ifdef CONFIG_TRACE_STATS
+                } else if (strcmp(optarg, "stats") == 0) {
+                    qemu_log_instr_set_backend(QEMU_LOG_INSTR_BACKEND_STATS);
+#endif
                 } else {
                     printf("Invalid choice for cheri-trace-format: '%s'\n", optarg);
                     exit(1);
diff --git a/stats/meson.build b/stats/meson.build
new file mode 100644
index 00000000000..caab6de10ac
--- /dev/null
+++ b/stats/meson.build
@@ -0,0 +1,12 @@
+
+trace_stats_ss = ss.source_set()
+trace_stats_ss.add(files('trace_stats.cc'))
+
+boost_dep = dependency('boost', modules: ['iostreams'])
+
+trace_stats_ss = trace_stats_ss.apply(config_all, strict: false)
+libqemustats = static_library('qemustats',
+                              sources: trace_stats_ss.sources(),
+                              dependencies: [boost_dep],
+                              cpp_args: ['-DCONFIG_TRACE_STATS'])
+
diff --git a/stats/trace_stats.cc b/stats/trace_stats.cc
new file mode 100644
index 00000000000..14a3b4d702b
--- /dev/null
+++ b/stats/trace_stats.cc
@@ -0,0 +1,96 @@
+/*-
+ * Copyright (c) 2021 Alfredo Mazzinghi
+ * All rights reserved.
+ *
+ * This software was developed by SRI International and the University of
+ * Cambridge Computer Laboratory (Department of Computer Science and
+ * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the
+ * DARPA SSITH research programme.
+ *
+ * @BERI_LICENSE_HEADER_START@
+ *
+ * Licensed to BERI Open Systems C.I.C. (BERI) under one or more contributor
+ * license agreements.  See the NOTICE file distributed with this work for
+ * additional information regarding copyright ownership.  BERI licenses this
+ * file to you under the BERI Hardware-Software License, Version 1.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at:
+ *
+ *   http://www.beri-open-systems.org/legal/license-1-0.txt
+ *
+ * Unless required by applicable law or agreed to in writing, Work distributed
+ * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ * CONDITIONS OF ANY KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ * @BERI_LICENSE_HEADER_END@
+ */
+
+#include "stats/stats.h"
+
+#include <iostream>
+#include <utility>
+#include <boost/iostreams/stream.hpp>
+#include <boost/iostreams/device/file_descriptor.hpp>
+#include <boost/icl/interval_map.hpp>
+
+namespace icl = boost::icl;
+namespace io = boost::iostreams;
+
+using addr_range_imap = icl::interval_map<uint64_t, uint64_t>;
+using addr_range = icl::interval<uint64_t>;
+
+namespace
+{
+
+addr_range::interval_type make_addr_range(uint64_t start, uint64_t end)
+{
+    return addr_range::right_open(start, end);
+}
+
+addr_range_imap &handle2imap(addr_range_hist_t handle)
+{
+    return *reinterpret_cast<addr_range_imap *>(handle);
+}
+
+} // namespace
+
+extern "C" {
+
+addr_range_hist_t qemu_stats_addr_range_hist_create()
+{
+    auto hist = new addr_range_imap();
+    return reinterpret_cast<addr_range_hist_t>(hist);
+}
+
+void qemu_stats_addr_range_hist_destroy(addr_range_hist_t handle)
+{
+    auto hist = &handle2imap(handle);
+    delete hist;
+}
+
+void qemu_stats_addr_range_hist_insert(addr_range_hist_t handle, uint64_t start,
+                                       uint64_t end, uint64_t value)
+{
+    auto &hist = handle2imap(handle);
+    hist += std::make_pair(make_addr_range(start, end), value);
+}
+
+void qemu_stats_addr_range_hist_dump(addr_range_hist_t handle, int fd,
+                                     bool csv_header)
+{
+    printf("XXX-AM: STATS DUMP\n");
+    auto &hist = handle2imap(handle);
+    io::file_descriptor_sink fd_sink(fd, io::never_close_handle);
+    io::stream<io::file_descriptor_sink> ostream(fd_sink);
+
+    if (csv_header)
+        ostream << "CPU, start, end, count" << std::endl;
+    for (const auto &keyval : hist) {
+        auto &range = keyval.first;
+        ostream << std::hex << range.lower() << "," << range.upper() << ","
+                << std::dec << keyval.second << std::endl;
+    }
+}
+
+} /* C */

From edc0a7411f0495b6dcd789e2f5af31777922d016 Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Sat, 27 Feb 2021 17:35:43 +0000
Subject: [PATCH 10/74] Add initialization and setup code for the perfetto
 tracing backend.

- Add cheri-perfetto submodule for tracing.
- Plumb the perfetto tracing backend in the build system
- Add initialization code that registers the tracing categories and starts
  the tracing service at startup
- Add skeleton for the perfetto tracing backend glue code
---
 .gitmodules                           |   4 +
 accel/tcg/log_instr.c                 |  10 ++-
 accel/tcg/log_instr_perfetto.c        |  40 ++++++++++
 accel/tcg/meson.build                 |   1 +
 configure                             |   8 ++
 include/qemu/log_instr.h              |   3 +
 meson.build                           |   8 +-
 stats/meson.build                     |  12 ---
 trace_extra/cheri-perfetto            |   1 +
 trace_extra/meson.build               |  19 +++++
 trace_extra/qemu_perfetto.h           |  46 ++++++++++++
 trace_extra/trace_perfetto.cc         | 103 ++++++++++++++++++++++++++
 {stats => trace_extra}/trace_stats.cc |   1 -
 trace_extra/track.hh                  |  41 ++++++++++
 14 files changed, 278 insertions(+), 19 deletions(-)
 create mode 100644 accel/tcg/log_instr_perfetto.c
 delete mode 100644 stats/meson.build
 create mode 160000 trace_extra/cheri-perfetto
 create mode 100644 trace_extra/meson.build
 create mode 100644 trace_extra/qemu_perfetto.h
 create mode 100644 trace_extra/trace_perfetto.cc
 rename {stats => trace_extra}/trace_stats.cc (98%)
 create mode 100644 trace_extra/track.hh

diff --git a/.gitmodules b/.gitmodules
index 2bdeeacef88..f49c13bb970 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -64,3 +64,7 @@
 [submodule "roms/vbootrom"]
 	path = roms/vbootrom
 	url = https://git.qemu.org/git/vbootrom.git
+[submodule "trace/cheri-perfetto"]
+	path = trace_extra/cheri-perfetto
+	url = https://github.com/qwattash/cheri-perfetto.git
+	branch = cheri-perfetto
diff --git a/accel/tcg/log_instr.c b/accel/tcg/log_instr.c
index ff159e05cf5..55559aa44fd 100644
--- a/accel/tcg/log_instr.c
+++ b/accel/tcg/log_instr.c
@@ -121,6 +121,15 @@ static trace_backend_hooks_t trace_backends[] = {
       .emit_instr = emit_stats_entry,
       .emit_events = emit_stats_events },
 #endif
+#ifdef CONFIG_TRACE_PERFETTO
+/* { */
+/*     .init = init_perfetto_backend, */
+/*     .emit_header = NULL, */
+/*     .emit_start = emit_perfetto_start, */
+/*     .emit_stop = emit_perfetto_stop, */
+/*     .emit_entry = emit_perfetto_entry */
+/* } */
+#endif
 };
 
 /* Existing trace filters list, indexed by cpu_log_instr_filter_t */
@@ -1384,5 +1393,4 @@ void qemu_log_instr_mem_filter_update()
 static cpu_log_instr_filter_fn_t trace_filters[] = {
     entry_mem_regions_filter,
 };
-
 #endif /* CONFIG_TCG_LOG_INSTR */
diff --git a/accel/tcg/log_instr_perfetto.c b/accel/tcg/log_instr_perfetto.c
new file mode 100644
index 00000000000..74f00ea55f9
--- /dev/null
+++ b/accel/tcg/log_instr_perfetto.c
@@ -0,0 +1,40 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2021 Alfredo Mazzinghi
+ *
+ * This software was developed by SRI International and the University of
+ * Cambridge Computer Laboratory (Department of Computer Science and
+ * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the
+ * DARPA SSITH research programme.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * Glue code to bridge TCG instruction logging to the perfetto framework.
+ */
+
+#include "qemu/osdep.h"
+#include "exec/log_instr.h"
+#include "exec/log_instr_internal.h"
+#include "trace_extra/qemu_perfetto.h"
diff --git a/accel/tcg/meson.build b/accel/tcg/meson.build
index e0476c72a33..097a025fd8d 100644
--- a/accel/tcg/meson.build
+++ b/accel/tcg/meson.build
@@ -17,3 +17,4 @@ specific_ss.add(when: ['CONFIG_TCG_LOG_INSTR', 'CONFIG_TCG'], if_true: files('lo
 specific_ss.add(when: ['CONFIG_TCG_LOG_INSTR', 'CONFIG_TCG'], if_true: files('log_instr_text.c'))
 specific_ss.add(when: ['CONFIG_TCG_LOG_INSTR', 'CONFIG_TCG'], if_true: files('log_instr_cvtrace.c'))
 specific_ss.add(when: ['CONFIG_TRACE_STATS', 'CONFIG_TCG_LOG_INSTR', 'CONFIG_TCG'], if_true: files('log_instr_stats.c'))
+specific_ss.add(when: ['CONFIG_TRACE_PERFETTO', 'CONFIG_TCG_LOG_INSTR', 'CONFIG_TCG'], if_true: files('log_instr_perfetto.c'))
diff --git a/configure b/configure
index 28adfcf7508..dc0df44f035 100755
--- a/configure
+++ b/configure
@@ -340,6 +340,7 @@ pvrdma=""
 gprof="no"
 debug_tcg="no"
 tcg_log_instr="yes"
+tcg_log_instr_perfetto="yes"
 trace_stats="no"
 rvfi_dii="no"
 debug="no"
@@ -1037,6 +1038,10 @@ for opt do
   ;;
   --disable-tcg-log-instr) tcg_log_instr="no"
   ;;
+  --enable-perfetto-log-instr) tcg_log_instr_perfetto="yes"
+  ;;
+  --disable-perfetto-log-instr) tcg_log_instr_perfetto="no"
+  ;;
   --enable-trace-stats) trace_stats="yes"
   ;;
   --disable-trace-stats) trace_stats="no"
@@ -5973,6 +5978,9 @@ fi
 if test "$tcg_log_instr" = "yes" ; then
   echo "CONFIG_TCG_LOG_INSTR=y" >> $config_host_mak
 fi
+if test "$tcg_log_instr_perfetto" = "yes" ; then
+    echo "CONFIG_TRACE_PERFETTO=y" >> $config_host_mak
+fi
 if test "$trace_stats" = "yes" ; then
     echo "CONFIG_TRACE_STATS=y" >> $config_host_mak
 fi
diff --git a/include/qemu/log_instr.h b/include/qemu/log_instr.h
index 1b0732a4abd..b2fa01c9426 100644
--- a/include/qemu/log_instr.h
+++ b/include/qemu/log_instr.h
@@ -57,6 +57,9 @@ typedef enum {
 #ifdef CONFIG_TRACE_STATS
     QEMU_LOG_INSTR_BACKEND_STATS = 3,
 #endif
+#ifdef CONFIG_TRACE_PERFETTO
+    QEMU_LOG_INSTR_BACKEND_PERFETTO = 4
+#endif
 } qemu_log_instr_backend_t;
 
 extern qemu_log_instr_backend_t qemu_log_instr_backend;
diff --git a/meson.build b/meson.build
index d477a80900d..abfbe926199 100644
--- a/meson.build
+++ b/meson.build
@@ -1331,7 +1331,7 @@ qapi_gen_depends = [ meson.source_root() / 'scripts/qapi/__init__.py',
 
 tracetool = [
   python, files('scripts/tracetool.py'),
-   '--backend=' + config_host['TRACE_BACKENDS']
+  '--backend=' + config_host['TRACE_BACKENDS']
 ]
 
 qemu_version_cmd = [find_program('scripts/qemu-version.sh'),
@@ -1516,12 +1516,10 @@ libqemuutil = static_library('qemuutil',
                              dependencies: [util_ss.dependencies(), m, glib, socket, malloc])
 
 qemuutil_libs = [libqemuutil]
-if 'CONFIG_TRACE_STATS' in config_host
-    subdir('stats')
-    qemuutil_libs += libqemustats
-endif
+subdir('trace_extra')
 
 qemuutil = declare_dependency(link_with: qemuutil_libs,
+                              sources: genh + version_res)
 
 decodetree = generator(find_program('scripts/decodetree.py'),
                        output: 'decode-@BASENAME@.c.inc',
diff --git a/stats/meson.build b/stats/meson.build
deleted file mode 100644
index caab6de10ac..00000000000
--- a/stats/meson.build
+++ /dev/null
@@ -1,12 +0,0 @@
-
-trace_stats_ss = ss.source_set()
-trace_stats_ss.add(files('trace_stats.cc'))
-
-boost_dep = dependency('boost', modules: ['iostreams'])
-
-trace_stats_ss = trace_stats_ss.apply(config_all, strict: false)
-libqemustats = static_library('qemustats',
-                              sources: trace_stats_ss.sources(),
-                              dependencies: [boost_dep],
-                              cpp_args: ['-DCONFIG_TRACE_STATS'])
-
diff --git a/trace_extra/cheri-perfetto b/trace_extra/cheri-perfetto
new file mode 160000
index 00000000000..6c8dbfabf04
--- /dev/null
+++ b/trace_extra/cheri-perfetto
@@ -0,0 +1 @@
+Subproject commit 6c8dbfabf04f9a2bd701a76ff29f49159933222c
diff --git a/trace_extra/meson.build b/trace_extra/meson.build
new file mode 100644
index 00000000000..2c08ed31d23
--- /dev/null
+++ b/trace_extra/meson.build
@@ -0,0 +1,19 @@
+
+qemutrace_ss = ss.source_set()
+
+qemutrace_ss.add(when: ['CONFIG_TCG', 'CONFIG_TRACE_STATS'],
+                 if_true: files('trace_stats.cc'))
+qemutrace_ss.add(when: ['CONFIG_TCG', 'CONFIG_TRACE_STATS'],
+                 if_true: dependency('boost', modules: ['iostreams']))
+
+qemutrace_ss.add(when: ['CONFIG_TCG', 'CONFIG_TRACE_PERFETTO'],
+                 if_true: files('trace_perfetto.cc', 'cheri-perfetto/sdk/perfetto.cc'))
+
+qemutrace_ss = qemutrace_ss.apply(config_all, strict: false)
+libqemutrace = static_library('qemutrace',
+                              sources: qemutrace_ss.sources(),
+                              dependencies: qemutrace_ss.dependencies(),
+                              include_directories: ['cheri-perfetto/sdk'],
+                              cpp_args: ['-DCONFIG_TRACE_STATS', '-DQEMU_PERFETTO'])
+
+qemuutil_libs += libqemutrace
diff --git a/trace_extra/qemu_perfetto.h b/trace_extra/qemu_perfetto.h
new file mode 100644
index 00000000000..5bb1cf5c63a
--- /dev/null
+++ b/trace_extra/qemu_perfetto.h
@@ -0,0 +1,46 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2021 Alfredo Mazzinghi
+ *
+ * This software was developed by SRI International and the University of
+ * Cambridge Computer Laboratory (Department of Computer Science and
+ * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the
+ * DARPA SSITH research programme.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef TRACE_PERFETTO_H
+#define TRACE_PERFETTO_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+bool perfetto_init(void);
+
+#ifdef __cplusplus
+} /* extern C */
+#endif
+
+#endif /* TRACE_PERFETTO_H */
diff --git a/trace_extra/trace_perfetto.cc b/trace_extra/trace_perfetto.cc
new file mode 100644
index 00000000000..8cca2a8775f
--- /dev/null
+++ b/trace_extra/trace_perfetto.cc
@@ -0,0 +1,103 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2021 Alfredo Mazzinghi
+ *
+ * This software was developed by SRI International and the University of
+ * Cambridge Computer Laboratory (Department of Computer Science and
+ * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the
+ * DARPA SSITH research programme.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <perfetto.h>
+
+#include "qemu/osdep.h"
+#ifndef _WIN32
+#include <pthread.h>
+#include <fcntl.h>
+#endif
+
+#include "trace/event-internal.h"
+#include "qemu_perfetto.h"
+#include "track.hh"
+
+PERFETTO_TRACK_EVENT_STATIC_STORAGE();
+
+namespace {
+
+/* Tracing session pointer */
+std::unique_ptr<perfetto::TracingSession> session;
+
+/*
+ * Initialize perfetto tracing.
+ *
+ * Create the following data sources for qemu events:
+ * - Track events data source
+ * * XXX-AM: should use in process on UNIX only
+ * Should allow flags/options to control backend type, buffer sizes and
+ * other things
+ */
+bool perfetto_start_tracing(void)
+{
+
+    perfetto::TracingInitArgs args;
+    perfetto::TraceConfig cfg;
+
+    args.backends |= perfetto::kInProcessBackend;
+
+    perfetto::Tracing::Initialize(args);
+    perfetto::TrackEvent::Register();
+
+    cfg.add_buffers()->set_size_kb(4096);
+
+    auto *ds_cfg = cfg.add_data_sources()->mutable_config();
+    ds_cfg->set_name("track_event");
+
+    session = perfetto::Tracing::NewTrace();
+
+    int trace_fd = open("test.pftrace", O_RDWR | O_CREAT | O_TRUNC, 0600);
+    session->Setup(cfg, trace_fd);
+    session->StartBlocking();
+
+    return true;
+}
+
+void perfetto_tracing_stop(void)
+{
+    session->StopBlocking();
+}
+
+}
+
+/*
+ * Initialize perfetto tracing.
+ *
+ * Start tracing thread
+ */
+extern "C" bool perfetto_init(void)
+{
+    atexit(perfetto_tracing_stop);
+
+    return perfetto_start_tracing();
+}
diff --git a/stats/trace_stats.cc b/trace_extra/trace_stats.cc
similarity index 98%
rename from stats/trace_stats.cc
rename to trace_extra/trace_stats.cc
index 14a3b4d702b..8aa55c1ce3b 100644
--- a/stats/trace_stats.cc
+++ b/trace_extra/trace_stats.cc
@@ -79,7 +79,6 @@ void qemu_stats_addr_range_hist_insert(addr_range_hist_t handle, uint64_t start,
 void qemu_stats_addr_range_hist_dump(addr_range_hist_t handle, int fd,
                                      bool csv_header)
 {
-    printf("XXX-AM: STATS DUMP\n");
     auto &hist = handle2imap(handle);
     io::file_descriptor_sink fd_sink(fd, io::never_close_handle);
     io::stream<io::file_descriptor_sink> ostream(fd_sink);
diff --git a/trace_extra/track.hh b/trace_extra/track.hh
new file mode 100644
index 00000000000..e99622417df
--- /dev/null
+++ b/trace_extra/track.hh
@@ -0,0 +1,41 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2021 Alfredo Mazzinghi
+ *
+ * This software was developed by SRI International and the University of
+ * Cambridge Computer Laboratory (Department of Computer Science and
+ * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the
+ * DARPA SSITH research programme.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#pragma once
+
+#include <perfetto.h>
+
+PERFETTO_DEFINE_CATEGORIES(
+    perfetto::Category("instructions")
+    .SetDescription("CPU instructions executed"),
+    perfetto::Category("memory")
+    .SetDescription("Tag and memory load/store"));

From c88434e31583a7f166ec89d0eb2ff35c6355094c Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Mon, 9 Aug 2021 09:43:04 +0100
Subject: [PATCH 11/74] Fix stats logging backend output.

---
 accel/tcg/log_instr_stats.c | 10 ++++++++--
 include/stats/stats.h       |  2 +-
 trace_extra/trace_stats.cc  |  9 +++++----
 3 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/accel/tcg/log_instr_stats.c b/accel/tcg/log_instr_stats.c
index 4120ecbeafc..c74e858bcb5 100644
--- a/accel/tcg/log_instr_stats.c
+++ b/accel/tcg/log_instr_stats.c
@@ -77,6 +77,7 @@ void emit_stats_events(CPUArchState *env, cpu_log_entry_t *entry)
     struct cpu_stats *stats = get_cpu_stats(env);
     QemuLogFile *logfile;
     const log_event_t *event;
+    bool header = false;
     int i;
 
     log_assert(stats != NULL && "Missing stats backend data");
@@ -88,8 +89,13 @@ void emit_stats_events(CPUArchState *env, cpu_log_entry_t *entry)
                 rcu_read_lock();
                 logfile = qatomic_rcu_read(&qemu_logfile);
                 if (logfile) {
-                    qemu_stats_addr_range_hist_dump(stats->pc_hist,
-                                                    fileno(logfile->fd), true);
+                    if (ftell(logfile->fd) == 0) {
+                        header = true;
+                    }
+                    qemu_stats_addr_range_hist_dump(
+                        stats->pc_hist, fileno(logfile->fd),
+                        env_cpu(env)->cpu_index, header);
+                    fflush(logfile->fd);
                 }
                 rcu_read_unlock();
                 break;
diff --git a/include/stats/stats.h b/include/stats/stats.h
index be4d12151ce..fe9a596d750 100644
--- a/include/stats/stats.h
+++ b/include/stats/stats.h
@@ -50,7 +50,7 @@ void qemu_stats_addr_range_hist_destroy(addr_range_hist_t handle);
 void qemu_stats_addr_range_hist_insert(addr_range_hist_t handle,
     uint64_t start, uint64_t end, uint64_t value);
 void qemu_stats_addr_range_hist_dump(addr_range_hist_t handle, int fd,
-    bool csv_header);
+    int cpu_id, bool csv_header);
 
 #endif /* CONFIG_TRACE_STATS */
 #ifdef __cplusplus
diff --git a/trace_extra/trace_stats.cc b/trace_extra/trace_stats.cc
index 8aa55c1ce3b..4e8faf67b1f 100644
--- a/trace_extra/trace_stats.cc
+++ b/trace_extra/trace_stats.cc
@@ -77,18 +77,19 @@ void qemu_stats_addr_range_hist_insert(addr_range_hist_t handle, uint64_t start,
 }
 
 void qemu_stats_addr_range_hist_dump(addr_range_hist_t handle, int fd,
-                                     bool csv_header)
+                                     int cpu_id, bool csv_header)
 {
     auto &hist = handle2imap(handle);
     io::file_descriptor_sink fd_sink(fd, io::never_close_handle);
     io::stream<io::file_descriptor_sink> ostream(fd_sink);
 
     if (csv_header)
-        ostream << "CPU, start, end, count" << std::endl;
+        ostream << "CPU,start,end,count" << std::endl;
     for (const auto &keyval : hist) {
         auto &range = keyval.first;
-        ostream << std::hex << range.lower() << "," << range.upper() << ","
-                << std::dec << keyval.second << std::endl;
+        ostream << cpu_id << "," << std::hex << range.lower() << ","
+                << range.upper() << "," << std::dec << keyval.second
+                << std::endl;
     }
 }
 

From b8095818c728d411944c7b9c4aed5be52e956d37 Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Fri, 13 Aug 2021 10:54:38 +0100
Subject: [PATCH 12/74] Improve instruction trace filters support.

- Add command line option to select filters to use.
- Make sure that the filters from command line are applied to the CPUs at
  initialization.
- Add filter to only emit entries containing events.
---
 accel/tcg/log_instr.c                         | 70 +++++++++++++++----
 accel/tcg/log_instr_stats.c                   |  3 +-
 accel/tcg/log_instr_text.c                    |  9 ++-
 include/exec/log_instr.h                      | 13 ----
 .../{stats/stats.h => exec/log_instr_stats.h} | 10 +--
 include/qemu/log_instr.h                      | 29 ++++++++
 qemu-options.hx                               |  7 ++
 softmmu/vl.c                                  |  3 +
 trace_extra/trace_stats.cc                    |  8 ++-
 util/log.c                                    |  8 ++-
 10 files changed, 124 insertions(+), 36 deletions(-)
 rename include/{stats/stats.h => exec/log_instr_stats.h} (88%)

diff --git a/accel/tcg/log_instr.c b/accel/tcg/log_instr.c
index 55559aa44fd..52aaf9c6fb9 100644
--- a/accel/tcg/log_instr.c
+++ b/accel/tcg/log_instr.c
@@ -35,6 +35,7 @@
 #include "qemu/osdep.h"
 #include "qemu/range.h"
 #include "qemu/log.h"
+#include "qapi/error.h"
 #include "cpu-param.h"
 #include "cpu.h"
 #include "exec/exec-all.h"
@@ -135,6 +136,9 @@ static trace_backend_hooks_t trace_backends[] = {
 /* Existing trace filters list, indexed by cpu_log_instr_filter_t */
 static cpu_log_instr_filter_fn_t trace_filters[];
 
+/* Trace filters to activate when a new CPU is seen */
+static GArray *reset_filters;
+
 /* Number of per-cpu ring buffer entries for ring-buffer tracing mode */
 #define MIN_ENTRY_BUFFER_SIZE (1 << 16)
 
@@ -453,6 +457,13 @@ void qemu_log_instr_init(CPUState *cpu)
     else if (qemu_loglevel_mask(CPU_LOG_INSTR))
         do_cpu_loglevel_switch(
             cpu, RUN_ON_CPU_HOST_INT(QEMU_LOG_INSTR_LOGLEVEL_ALL));
+
+    if (reset_filters != NULL) {
+        for (i = 0; i < reset_filters->len; i++) {
+            qemu_log_instr_add_filter(
+                cpu, g_array_index(reset_filters, cpu_log_instr_filter_t, i));
+        }
+    }
 }
 
 static void do_log_buffer_resize(CPUState *cpu, run_on_cpu_data data)
@@ -1290,13 +1301,22 @@ void helper_log_value(CPUArchState *env, const void *ptr, uint64_t value)
 void qemu_log_instr_add_filter(CPUState *cpu, cpu_log_instr_filter_t filter)
 {
     cpu_log_instr_state_t *cpulog = &cpu->log_state;
+    cpu_log_instr_filter_fn_t new_fn, fn;
+    int i;
 
     if (filter >= LOG_INSTR_FILTER_MAX) {
         warn_report("Instruction trace filter index is invalid");
         return;
     }
-    /* XXX Currently we do not check for duplicates */
-    g_array_append_val(cpulog->filters, trace_filters[filter]);
+    new_fn = trace_filters[filter];
+    /* Check for duplicates */
+    for (i = 0; i < cpulog->filters->len; i++) {
+        fn = g_array_index(cpulog->filters, cpu_log_instr_filter_fn_t, i);
+        if (new_fn == fn) {
+            return;
+        }
+    }
+    g_array_append_val(cpulog->filters, new_fn);
 }
 
 void qemu_log_instr_allcpu_add_filter(cpu_log_instr_filter_t filter)
@@ -1339,8 +1359,37 @@ void qemu_log_instr_allcpu_remove_filter(cpu_log_instr_filter_t filter)
     }
 }
 
+void qemu_log_instr_add_startup_filter(cpu_log_instr_filter_t filter)
+{
+    if (reset_filters == NULL) {
+        reset_filters =
+            g_array_new(false, true, sizeof(cpu_log_instr_filter_t));
+    }
+
+    if (first_cpu == NULL) {
+        g_array_append_val(reset_filters, filter);
+    } else {
+        qemu_log_instr_allcpu_add_filter(filter);
+    }
+}
+
+void qemu_log_instr_set_cli_filters(const char *filter_spec, Error **errp)
+{
+    gchar **names = g_strsplit(filter_spec, ",", 0);
+    int i;
+
+    for (i = 0; names[i]; i++) {
+        if (strcmp(names[i], "events") == 0) {
+            qemu_log_instr_add_startup_filter(LOG_FILTER_EVENTS);
+        } else {
+            error_setg(errp, "Invalid trace filter name");
+            break;
+        }
+    }
+}
+
 /*
- * LOG entry filter reusing the qemu -dfilter infrastructure to
+ * Log entry filter reusing the qemu -dfilter infrastructure to
  * filter instructions that run from or access given address ranges.
  */
 static bool entry_mem_regions_filter(cpu_log_entry_t *entry)
@@ -1372,18 +1421,14 @@ static bool entry_mem_regions_filter(cpu_log_entry_t *entry)
 }
 
 /*
- * Interface to add/remove the -dfilter filter function from the per-cpu filter
- * list. This is done here as -dfilter is linked in libqemu and does not have
- * access to target-specific types.
- * XXX Drop me
+ * Log entry filter to retain only entries with events attached.
  */
-void qemu_log_instr_mem_filter_update()
+static bool entry_event_filter(cpu_log_entry_t *entry)
 {
-    if (debug_regions) {
-        qemu_log_instr_allcpu_add_filter(LOG_INSTR_FILTER_MEM_RANGE);
-    } else {
-        qemu_log_instr_allcpu_remove_filter(LOG_INSTR_FILTER_MEM_RANGE);
+    if (entry->events->len > 0) {
+        return true;
     }
+    return false;
 }
 
 /*
@@ -1392,5 +1437,6 @@ void qemu_log_instr_mem_filter_update()
  */
 static cpu_log_instr_filter_fn_t trace_filters[] = {
     entry_mem_regions_filter,
+    entry_event_filter,
 };
 #endif /* CONFIG_TCG_LOG_INSTR */
diff --git a/accel/tcg/log_instr_stats.c b/accel/tcg/log_instr_stats.c
index c74e858bcb5..3fbeac077d0 100644
--- a/accel/tcg/log_instr_stats.c
+++ b/accel/tcg/log_instr_stats.c
@@ -40,7 +40,7 @@
 #include "qemu/log.h"
 #include "exec/log_instr.h"
 #include "exec/log_instr_internal.h"
-#include "stats/stats.h"
+#include "exec/log_instr_stats.h"
 
 struct cpu_stats {
     addr_range_hist_t pc_hist;
@@ -98,6 +98,7 @@ void emit_stats_events(CPUArchState *env, cpu_log_entry_t *entry)
                     fflush(logfile->fd);
                 }
                 rcu_read_unlock();
+                qemu_stats_addr_range_hist_clear(stats->pc_hist);
                 break;
             }
         }
diff --git a/accel/tcg/log_instr_text.c b/accel/tcg/log_instr_text.c
index 0a4b182cfbc..713292eb151 100644
--- a/accel/tcg/log_instr_text.c
+++ b/accel/tcg/log_instr_text.c
@@ -192,7 +192,8 @@ void emit_text_events(CPUArchState *env, cpu_log_entry_t *entry)
 
     for (i = 0; i < entry->events->len; i++) {
         event = &g_array_index(entry->events, const log_event_t, i);
-        if (event->id == LOG_EVENT_STATE) {
+        switch (event->id) {
+        case LOG_EVENT_STATE:
             if (event->state.next_state == LOG_EVENT_STATE_START) {
                 log_state_op = "Requested";
             } else if (event->state.next_state == LOG_EVENT_STATE_STOP) {
@@ -212,6 +213,12 @@ void emit_text_events(CPUArchState *env, cpu_log_entry_t *entry)
                          env_cpu(env)->cpu_index, cpu_get_asid(env),
                          log_state_op, event->state.pc);
             }
+            break;
+        case LOG_EVENT_CTX_SWITCH:
+            qemu_log("Context switch pid=0x" TARGET_FMT_lx
+                     " tid=0x" TARGET_FMT_lx " cid=0x" TARGET_FMT_lx "\n",
+                     event->ctx_switch.pid, event->ctx_switch.tid,
+                     event->ctx_switch.cid);
         }
     }
 }
diff --git a/include/exec/log_instr.h b/include/exec/log_instr.h
index a8a317c5cc9..2c228f9f914 100644
--- a/include/exec/log_instr.h
+++ b/include/exec/log_instr.h
@@ -233,19 +233,6 @@ typedef struct {
     };
 } log_event_t;
 
-/*
- * Handles for instruction filters/callbacks.
- * These are used to identify and attach/detach instruction log entry filters,
- * the rationale for this is that the filters require access to the private
- * instruction log entry record structure and can only be defined within the
- * instruction logging core implementation. This solution mimics the trace
- * backend handling.
- */
-typedef enum {
-    LOG_INSTR_FILTER_MEM_RANGE = 0,
-    LOG_INSTR_FILTER_MAX
-} cpu_log_instr_filter_t;
-
 /*
  * Request a flush of the TCG when changing loglevel outside of qemu_log_instr.
  * TODO(am2419): this should be removed from the interface.
diff --git a/include/stats/stats.h b/include/exec/log_instr_stats.h
similarity index 88%
rename from include/stats/stats.h
rename to include/exec/log_instr_stats.h
index fe9a596d750..c7e130f0315 100644
--- a/include/stats/stats.h
+++ b/include/exec/log_instr_stats.h
@@ -32,8 +32,7 @@
 #include <stdbool.h>
 
 #ifdef __cplusplus
-extern "C"
-{
+extern "C" {
 #endif
 
 #ifdef CONFIG_TRACE_STATS
@@ -47,10 +46,11 @@ typedef struct addr_range_hist *addr_range_hist_t;
  */
 addr_range_hist_t qemu_stats_addr_range_hist_create(void);
 void qemu_stats_addr_range_hist_destroy(addr_range_hist_t handle);
-void qemu_stats_addr_range_hist_insert(addr_range_hist_t handle,
-    uint64_t start, uint64_t end, uint64_t value);
+void qemu_stats_addr_range_hist_insert(addr_range_hist_t handle, uint64_t start,
+                                       uint64_t end, uint64_t value);
 void qemu_stats_addr_range_hist_dump(addr_range_hist_t handle, int fd,
-    int cpu_id, bool csv_header);
+                                     int cpu_id, bool csv_header);
+void qemu_stats_addr_range_hist_clear(addr_range_hist_t handle);
 
 #endif /* CONFIG_TRACE_STATS */
 #ifdef __cplusplus
diff --git a/include/qemu/log_instr.h b/include/qemu/log_instr.h
index b2fa01c9426..bc7427532c9 100644
--- a/include/qemu/log_instr.h
+++ b/include/qemu/log_instr.h
@@ -157,6 +157,20 @@ typedef struct {
     void *backend_data;
 } cpu_log_instr_state_t;
 
+/*
+ * Handles for instruction filters/callbacks.
+ * These are used to identify and attach/detach instruction log entry filters,
+ * the rationale for this is that the filters require access to the private
+ * instruction log entry record structure and can only be defined within the
+ * instruction logging core implementation. This solution mimics the trace
+ * backend handling.
+ */
+typedef enum {
+    LOG_INSTR_FILTER_MEM_RANGE = 0,
+    LOG_FILTER_EVENTS = 1,
+    LOG_INSTR_FILTER_MAX
+} cpu_log_instr_filter_t;
+
 /*
  * Initialize instruction logging for a cpu.
  */
@@ -170,6 +184,21 @@ int qemu_log_instr_global_switch(int log_flags);
  */
 void qemu_log_instr_set_buffer_size(unsigned long buffer_size);
 
+/*
+ * Set filters to activate from string names.
+ * This is safe to call during startup as if no CPU exists, we
+ * will set the filters when the CPU startup occurs.
+ */
+void qemu_log_instr_set_cli_filters(const char *filter_spec, Error **errp);
+
+/*
+ * Add a trace filter during startup. This will be activated on all the CPUs
+ * that are initialized after the call.
+ * After the CPUs have been initialized, this will set the filters on all
+ * existing CPUs.
+ */
+void qemu_log_instr_add_startup_filter(cpu_log_instr_filter_t filter);
+
 /*
  * Trigger update of the instruction entry filter reusing
  * the -dfilter qemu option to limit tracing to only interesting
diff --git a/qemu-options.hx b/qemu-options.hx
index 706be14708d..407e9461059 100644
--- a/qemu-options.hx
+++ b/qemu-options.hx
@@ -4123,6 +4123,13 @@ SRST
     Set CHERI trace backend to <type> (text, cvtrace or stats)
 ERST
 
+DEF("cheri-trace-filters", HAS_ARG, QEMU_OPTION_cheri_trace_filters, \
+"-cheri-trace-filters [event]     Select trace filters to use.\n", QEMU_ARCH_ALL)
+SRST
+``-cheri-trace-filters name[,...]``
+    Set CHERI trace filters to use. Available filters are: events.
+ERST
+
 DEF("cheri-c2e-on-unrepresentable", 0, QEMU_OPTION_cheri_c2e_on_unrepresentable, \
     "-cheri-c2e-on-unrepresentable     Generate C2E exception when a capability becomes unrepresentable\n", QEMU_ARCH_ALL)
 SRST
diff --git a/softmmu/vl.c b/softmmu/vl.c
index 5e18c10f721..998a3cfbbf8 100644
--- a/softmmu/vl.c
+++ b/softmmu/vl.c
@@ -3764,6 +3764,9 @@ void qemu_init(int argc, char **argv, char **envp)
             case QEMU_OPTION_cheri_trace_buffer_size:
                 qemu_log_instr_set_buffer_size(strtoul(optarg, NULL, 0));
                 break;
+            case QEMU_OPTION_cheri_trace_filters:
+                qemu_log_instr_set_cli_filters(optarg, &error_fatal);
+                break;
 #endif /* CONFIG_TCG_LOG_INSTR */
 
 #ifdef TARGET_CHERI
diff --git a/trace_extra/trace_stats.cc b/trace_extra/trace_stats.cc
index 4e8faf67b1f..edb59046b63 100644
--- a/trace_extra/trace_stats.cc
+++ b/trace_extra/trace_stats.cc
@@ -26,7 +26,7 @@
  * @BERI_LICENSE_HEADER_END@
  */
 
-#include "stats/stats.h"
+#include "exec/log_instr_stats.h"
 
 #include <iostream>
 #include <utility>
@@ -93,4 +93,10 @@ void qemu_stats_addr_range_hist_dump(addr_range_hist_t handle, int fd,
     }
 }
 
+void qemu_stats_addr_range_hist_clear(addr_range_hist_t handle)
+{
+    auto &hist = handle2imap(handle);
+    hist.clear();
+}
+
 } /* C */
diff --git a/util/log.c b/util/log.c
index 2c8573c520b..071fdc4f04f 100644
--- a/util/log.c
+++ b/util/log.c
@@ -43,8 +43,10 @@ __attribute__((weak)) int qemu_log_instr_global_switch(int log_flags)
     return log_flags;
 }
 
-__attribute__((weak)) void qemu_log_instr_mem_filter_update(void);
-__attribute__((weak)) void qemu_log_instr_mem_filter_update()
+__attribute__((weak)) void qemu_log_instr_add_startup_filter(
+    cpu_log_instr_filter_t);
+__attribute__((weak))
+void qemu_log_instr_add_startup_filter(cpu_log_instr_filter_t f)
 {
     /* Real implementation in accel/tcg/log_instr.c */
     warn_report("Calling no-op %s\r", __func__);
@@ -296,7 +298,7 @@ void qemu_set_dfilter_ranges(const char *filter_spec, Error **errp)
 out:
     g_strfreev(ranges);
 #ifdef CONFIG_TCG_LOG_INSTR
-    qemu_log_instr_mem_filter_update();
+    qemu_log_instr_add_startup_filter(LOG_INSTR_FILTER_MEM_RANGE);
 #endif
 }
 

From 2ed217d80b4a8bc272288e3d0255d498e7a29680 Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Tue, 7 Sep 2021 19:25:19 +0100
Subject: [PATCH 13/74] Refactor trace stats helper into a class.

---
 accel/tcg/log_instr_stats.c    | 14 +++--
 include/exec/log_instr_stats.h | 18 +++----
 trace_extra/trace_stats.cc     | 97 ++++++++++++++++++++++++----------
 3 files changed, 83 insertions(+), 46 deletions(-)

diff --git a/accel/tcg/log_instr_stats.c b/accel/tcg/log_instr_stats.c
index 3fbeac077d0..4b82bac02ea 100644
--- a/accel/tcg/log_instr_stats.c
+++ b/accel/tcg/log_instr_stats.c
@@ -43,7 +43,7 @@
 #include "exec/log_instr_stats.h"
 
 struct cpu_stats {
-    addr_range_hist_t pc_hist;
+    qemu_cpu_stats_t qstats;
     target_ulong last_pc;
     target_ulong pc_range_start;
 };
@@ -63,7 +63,7 @@ void init_stats_backend(CPUArchState *env)
     struct cpu_stats *stats;
 
     stats = malloc(sizeof(struct cpu_stats));
-    stats->pc_hist = qemu_stats_addr_range_hist_create();
+    stats->qstats = qemu_cpu_stats_create(env_cpu(env)->cpu_index);
     stats->last_pc = 0;
     stats->pc_range_start = 0;
     cpulog->backend_data = stats;
@@ -92,13 +92,11 @@ void emit_stats_events(CPUArchState *env, cpu_log_entry_t *entry)
                     if (ftell(logfile->fd) == 0) {
                         header = true;
                     }
-                    qemu_stats_addr_range_hist_dump(
-                        stats->pc_hist, fileno(logfile->fd),
-                        env_cpu(env)->cpu_index, header);
+                    qemu_cpu_stats_dump(stats->qstats, fileno(logfile->fd),
+                                        header);
                     fflush(logfile->fd);
                 }
                 rcu_read_unlock();
-                qemu_stats_addr_range_hist_clear(stats->pc_hist);
                 break;
             }
         }
@@ -116,8 +114,8 @@ void emit_stats_entry(CPUArchState *env, cpu_log_entry_t *entry)
     log_assert(stats != NULL && "Missing stats backend data");
 
     if (entry->pc - stats->last_pc > entry->insn_size) {
-        qemu_stats_addr_range_hist_insert(stats->pc_hist, stats->pc_range_start,
-                                          stats->last_pc, 1);
+        qemu_cpu_stats_record_bb_hit(stats->qstats, stats->pc_range_start,
+                                     stats->last_pc);
         stats->pc_range_start = entry->pc;
     }
     stats->last_pc = entry->pc;
diff --git a/include/exec/log_instr_stats.h b/include/exec/log_instr_stats.h
index c7e130f0315..d544e856eec 100644
--- a/include/exec/log_instr_stats.h
+++ b/include/exec/log_instr_stats.h
@@ -36,21 +36,21 @@ extern "C" {
 #endif
 
 #ifdef CONFIG_TRACE_STATS
-struct addr_range_hist;
-typedef struct addr_range_hist *addr_range_hist_t;
+struct qemu_cpu_stats;
+typedef struct qemu_cpu_stats *qemu_cpu_stats_t;
 
 /*
  * Interface to manipulate an interval histogram representing a count
  * over address ranges.
  * The pointer returned is an opaque handle.
  */
-addr_range_hist_t qemu_stats_addr_range_hist_create(void);
-void qemu_stats_addr_range_hist_destroy(addr_range_hist_t handle);
-void qemu_stats_addr_range_hist_insert(addr_range_hist_t handle, uint64_t start,
-                                       uint64_t end, uint64_t value);
-void qemu_stats_addr_range_hist_dump(addr_range_hist_t handle, int fd,
-                                     int cpu_id, bool csv_header);
-void qemu_stats_addr_range_hist_clear(addr_range_hist_t handle);
+qemu_cpu_stats_t qemu_cpu_stats_create(int cpu_id);
+void qemu_cpu_stats_destroy(qemu_cpu_stats_t handle);
+void qemu_cpu_stats_dump(qemu_cpu_stats_t handle, int fd, bool csv_header);
+void qemu_cpu_stats_record_bb_hit(qemu_cpu_stats_t handle, uint64_t start,
+                                  uint64_t end);
+void qemu_cpu_stats_record_call(qemu_cpu_stats_t handle, uint64_t addr,
+                                uint64_t link);
 
 #endif /* CONFIG_TRACE_STATS */
 #ifdef __cplusplus
diff --git a/trace_extra/trace_stats.cc b/trace_extra/trace_stats.cc
index edb59046b63..71ea5be788e 100644
--- a/trace_extra/trace_stats.cc
+++ b/trace_extra/trace_stats.cc
@@ -28,6 +28,7 @@
 
 #include "exec/log_instr_stats.h"
 
+#include <map>
 #include <iostream>
 #include <utility>
 #include <boost/iostreams/stream.hpp>
@@ -48,44 +49,32 @@ addr_range::interval_type make_addr_range(uint64_t start, uint64_t end)
     return addr_range::right_open(start, end);
 }
 
-addr_range_imap &handle2imap(addr_range_hist_t handle)
-{
-    return *reinterpret_cast<addr_range_imap *>(handle);
-}
+struct QEMUStats {
+    addr_range_imap call_map;
+    addr_range_imap bb_hit_map;
+    int cpu_id;
 
-} // namespace
-
-extern "C" {
+    QEMUStats(int cpu_id) : cpu_id(cpu_id) {}
 
-addr_range_hist_t qemu_stats_addr_range_hist_create()
-{
-    auto hist = new addr_range_imap();
-    return reinterpret_cast<addr_range_hist_t>(hist);
-}
-
-void qemu_stats_addr_range_hist_destroy(addr_range_hist_t handle)
-{
-    auto hist = &handle2imap(handle);
-    delete hist;
-}
+    void record_bb_hit(uint64_t start, uint64_t end);
+    void record_call(uint64_t addr, uint64_t link);
+    void dump(int fd, bool csv_header);
+    void clear();
+};
 
-void qemu_stats_addr_range_hist_insert(addr_range_hist_t handle, uint64_t start,
-                                       uint64_t end, uint64_t value)
+QEMUStats &handle2stats(qemu_cpu_stats_t handle)
 {
-    auto &hist = handle2imap(handle);
-    hist += std::make_pair(make_addr_range(start, end), value);
+    return *reinterpret_cast<QEMUStats *>(handle);
 }
 
-void qemu_stats_addr_range_hist_dump(addr_range_hist_t handle, int fd,
-                                     int cpu_id, bool csv_header)
+void QEMUStats::dump(int fd, bool csv_header)
 {
-    auto &hist = handle2imap(handle);
     io::file_descriptor_sink fd_sink(fd, io::never_close_handle);
     io::stream<io::file_descriptor_sink> ostream(fd_sink);
 
     if (csv_header)
         ostream << "CPU,start,end,count" << std::endl;
-    for (const auto &keyval : hist) {
+    for (const auto &keyval : bb_hit_map) {
         auto &range = keyval.first;
         ostream << cpu_id << "," << std::hex << range.lower() << ","
                 << range.upper() << "," << std::dec << keyval.second
@@ -93,10 +82,60 @@ void qemu_stats_addr_range_hist_dump(addr_range_hist_t handle, int fd,
     }
 }
 
-void qemu_stats_addr_range_hist_clear(addr_range_hist_t handle)
+void QEMUStats::clear()
+{
+    bb_hit_map.clear();
+    call_map.clear();
+}
+
+void QEMUStats::record_bb_hit(uint64_t start, uint64_t end)
+{
+    bb_hit_map += std::make_pair(make_addr_range(start, end), 1UL);
+}
+
+void QEMUStats::record_call(uint64_t addr, uint64_t link)
+{
+    call_map += std::make_pair(make_addr_range(addr, link), 1UL);
+}
+
+} // namespace
+
+/*
+ * C API wrappers to the stats class.
+ */
+extern "C" {
+
+qemu_cpu_stats_t qemu_cpu_stats_create(int cpu_id)
+{
+    auto *stats = new QEMUStats(cpu_id);
+    return reinterpret_cast<qemu_cpu_stats_t>(stats);
+}
+
+void qemu_cpu_stats_destroy(qemu_cpu_stats_t handle)
+{
+    auto stats = &handle2stats(handle);
+    delete stats;
+}
+
+void qemu_cpu_stats_dump(qemu_cpu_stats_t handle, int fd, bool csv_header)
+{
+    auto stats = handle2stats(handle);
+    stats.dump(fd, csv_header);
+    stats.clear();
+}
+
+void qemu_cpu_stats_record_bb_hit(qemu_cpu_stats_t handle, uint64_t start,
+                                  uint64_t end)
+{
+    auto stats = handle2stats(handle);
+    stats.record_bb_hit(start, end);
+}
+
+void qemu_cpu_stats_record_call(qemu_cpu_stats_t handle, uint64_t addr,
+                                uint64_t link)
 {
-    auto &hist = handle2imap(handle);
-    hist.clear();
+    auto stats = handle2stats(handle);
+    stats.record_call(addr, link);
 }
 
 } /* C */

From cd44395d87670152a22a8bdd72f79190a677d4c4 Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Wed, 8 Sep 2021 12:28:06 +0100
Subject: [PATCH 14/74] Fix broken legacy CVTrace backend.

---
 accel/tcg/log_instr_cvtrace.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/accel/tcg/log_instr_cvtrace.c b/accel/tcg/log_instr_cvtrace.c
index 446ac3cd99f..a299ff2ce62 100644
--- a/accel/tcg/log_instr_cvtrace.c
+++ b/accel/tcg/log_instr_cvtrace.c
@@ -136,7 +136,7 @@ void emit_cvtrace_entry(CPUArchState *env, cpu_log_entry_t *entry)
                 cr = null_capability(&intcap);
             }
             uint64_t metadata = (((uint64_t)cr->cr_tag << 63) |
-                                 ((uint64_t)cap_get_otype(cr) << 32) |
+                                 ((uint64_t)cap_get_otype_unsigned(cr) << 32) |
                                  ((uint64_t)COMBINED_PERMS_VALUE(cr) << 1) |
                                  (uint64_t)(cap_is_unsealed(cr) ? 0 : 1));
 

From a31419ebe520c4bbedda54df2d82f2ec4fb9e001 Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Tue, 7 Sep 2021 19:25:44 +0100
Subject: [PATCH 15/74] Implement a tracing backend based on the perfetto
 tracing framework.

For each CPU we create 2 tracks: one for control events (e.g. tracing
start/stop) and another for log events on that CPU.
Currently we emit instructions and stats events on the CPU track.

- Replace the stats backend with a tracing category in the perfetto backend
- Add new CLI options to control the perfetto backend behaviour:
  1. --cheri-trace-perfetto-categories comma-separated list of categories to
     enable.
  2. --cheri-trace-perfetto-logfile log file to use for the perfetto binary
     trace. We do not use the common qemu logfile to avoid accidental pollution
     and passing the qemu file descriptor is painful due to having to lock the
     logfile via RCU.
- Add hook at qemu shutdown to make sure we flush the log buffers, if the
  tracing backend supports this.
---
 accel/tcg/log_instr.c                         |  68 +++--
 accel/tcg/log_instr_perfetto.c                | 161 +++++++++-
 accel/tcg/log_instr_text.c                    | 135 ++++-----
 accel/tcg/meson.build                         |   1 -
 chardev/char-mux.c                            |  12 +
 configure                                     |   8 -
 hw/misc/sifive_test.c                         |   4 +
 include/exec/log_instr.h                      |  44 ---
 include/exec/log_instr_internal.h             |  49 +--
 include/exec/log_instr_perfetto.h             | 141 +++++++++
 include/qemu/log_instr.h                      |  76 ++++-
 qemu-options.hx                               |  19 +-
 softmmu/vl.c                                  |  17 +-
 trace_extra/meson.build                       |  12 +-
 trace_extra/qemu_perfetto.h                   |   2 +-
 trace_extra/trace_perfetto.cc                 | 285 +++++++++++++++++-
 trace_extra/trace_stats.cc                    | 146 ++++-----
 .../trace_stats.hh                            |  58 ++--
 trace_extra/track.hh                          |  41 ---
 19 files changed, 910 insertions(+), 369 deletions(-)
 create mode 100644 include/exec/log_instr_perfetto.h
 rename include/exec/log_instr_stats.h => trace_extra/trace_stats.hh (60%)
 delete mode 100644 trace_extra/track.hh

diff --git a/accel/tcg/log_instr.c b/accel/tcg/log_instr.c
index 52aaf9c6fb9..1659fe625f6 100644
--- a/accel/tcg/log_instr.c
+++ b/accel/tcg/log_instr.c
@@ -105,31 +105,22 @@ static void emit_nop_entry(CPUArchState *env, cpu_log_entry_t *entry);
  */
 static trace_backend_hooks_t trace_backends[] = {
     { .init = NULL,
+      .sync = NULL,
       .emit_header = NULL,
-      .emit_instr = emit_text_instr,
-      .emit_events = emit_text_events },
+      .emit_instr = emit_text_instr },
     { .init = NULL,
+      .sync = NULL,
       .emit_header = emit_cvtrace_header,
-      .emit_instr = emit_cvtrace_entry,
-      .emit_events = NULL },
+      .emit_instr = emit_cvtrace_entry },
     { .init = NULL,
+      .sync = NULL,
       .emit_header = NULL,
-      .emit_instr = emit_nop_entry,
-      .emit_events = emit_nop_entry },
-#ifdef CONFIG_TRACE_STATS
-    { .init = init_stats_backend,
-      .emit_header = NULL,
-      .emit_instr = emit_stats_entry,
-      .emit_events = emit_stats_events },
-#endif
+      .emit_instr = emit_nop_entry },
 #ifdef CONFIG_TRACE_PERFETTO
-/* { */
-/*     .init = init_perfetto_backend, */
-/*     .emit_header = NULL, */
-/*     .emit_start = emit_perfetto_start, */
-/*     .emit_stop = emit_perfetto_stop, */
-/*     .emit_entry = emit_perfetto_entry */
-/* } */
+    { .init = init_perfetto_backend,
+      .sync = sync_perfetto_backend,
+      .emit_header = NULL,
+      .emit_instr = emit_perfetto_entry }
 #endif
 };
 
@@ -189,15 +180,6 @@ static void reset_log_buffer(cpu_log_instr_state_t *cpulog,
 }
 
 /* Common instruction commit implementation */
-static void do_instr_emit(CPUArchState *env, cpu_log_entry_t *entry)
-{
-    if (entry->flags & LI_FLAG_HAS_INSTR_DATA) {
-        trace_backend->emit_instr(env, entry);
-    }
-    if (entry->events->len > 0) {
-        trace_backend->emit_events(env, entry);
-    }
-}
 
 static void do_instr_commit(CPUArchState *env)
 {
@@ -230,7 +212,7 @@ static void do_instr_commit(CPUArchState *env)
             cpulog->ring_tail =
                 (cpulog->ring_tail + 1) % cpulog->instr_info->len;
     } else {
-        do_instr_emit(env, entry);
+        trace_backend->emit_instr(env, entry);
     }
 }
 
@@ -466,6 +448,32 @@ void qemu_log_instr_init(CPUState *cpu)
     }
 }
 
+static void do_log_backend_sync(CPUState *cpu, run_on_cpu_data _unused)
+{
+    log_assert(trace_backend->sync != NULL &&
+               "Missing sync() callback on trace backend");
+    trace_backend->sync(cpu->env_ptr);
+}
+
+/*
+ * Attempt to syncronize buffers in the tracing backend for each CPU.
+ * NOTE: This is a blocking operation that may delay the exit path.
+ */
+void qemu_log_instr_sync_buffers()
+{
+    CPUState *cpu;
+
+    /* Avoid doing this early if there is no sync function */
+    if (trace_backend->sync == NULL) {
+        return;
+    }
+
+    CPU_FOREACH(cpu)
+    {
+        run_on_cpu(cpu, do_log_backend_sync, RUN_ON_CPU_NULL);
+    }
+}
+
 static void do_log_buffer_resize(CPUState *cpu, run_on_cpu_data data)
 {
     unsigned long new_size = data.host_ulong;
@@ -1129,7 +1137,7 @@ void qemu_log_instr_flush(CPUArchState *env)
 
     while (curr != cpulog->ring_head) {
         entry = &g_array_index(cpulog->instr_info, cpu_log_entry_t, curr);
-        do_instr_emit(env, entry);
+        trace_backend->emit_instr(env, entry);
         curr = (curr + 1) % cpulog->instr_info->len;
     }
     cpulog->ring_tail = cpulog->ring_head;
diff --git a/accel/tcg/log_instr_perfetto.c b/accel/tcg/log_instr_perfetto.c
index 74f00ea55f9..75c350fbc9a 100644
--- a/accel/tcg/log_instr_perfetto.c
+++ b/accel/tcg/log_instr_perfetto.c
@@ -31,10 +31,167 @@
  */
 
 /*
- * Glue code to bridge TCG instruction logging to the perfetto framework.
+ * C Glue code to bridge TCG instruction logging to the C++ perfetto interface.
  */
 
 #include "qemu/osdep.h"
 #include "exec/log_instr.h"
 #include "exec/log_instr_internal.h"
-#include "trace_extra/qemu_perfetto.h"
+#include "exec/log_instr_perfetto.h"
+
+/*
+ * Initialize perfetto tracing.
+ *
+ * Start tracing thread when first called.
+ */
+void init_perfetto_backend(CPUArchState *env)
+{
+    cpu_log_instr_state_t *cpulog = get_cpu_log_state(env);
+    int cpu_id = env_cpu(env)->cpu_index;
+
+    perfetto_init_cpu(cpu_id, &cpulog->backend_data);
+}
+
+/* Syncronize buffers on this CPU. */
+void sync_perfetto_backend(CPUArchState *env)
+{
+    cpu_log_instr_state_t *cpulog = get_cpu_log_state(env);
+
+    perfetto_sync_cpu(cpulog->backend_data);
+}
+
+void emit_perfetto_entry(CPUArchState *env, cpu_log_entry_t *entry)
+{
+    cpu_log_instr_state_t *cpulog = get_cpu_log_state(env);
+
+    perfetto_emit_instr(cpulog->backend_data, entry);
+}
+
+log_event_t *perfetto_log_event(cpu_log_entry_handle handle, int idx)
+{
+    return &g_array_index(handle->events, log_event_t, idx);
+}
+
+/* Helpers to access cpu_log_entry_t fields */
+CPU_LOG_ENTRY_GETTER(uint16_t, asid)
+CPU_LOG_ENTRY_GETTER(int, flags)
+CPU_LOG_ENTRY_GETTER(qemu_log_instr_cpu_mode_t, next_cpu_mode)
+CPU_LOG_ENTRY_GETTER(uint32_t, intr_code)
+CPU_LOG_ENTRY_GETTER(uint64_t, intr_vector)
+CPU_LOG_ENTRY_GETTER(uint64_t, intr_faultaddr)
+CPU_LOG_ENTRY_GETTER(uint64_t, pc)
+CPU_LOG_ENTRY_GETTER_DECL(const char *, insn_bytes)
+{
+    return &handle->insn_bytes[0];
+}
+CPU_LOG_ENTRY_GETTER(int, insn_size)
+CPU_LOG_ENTRY_GETTER_DECL(int, mem)
+{
+    return handle->mem->len;
+}
+CPU_LOG_ENTRY_GETTER_DECL(int, regs)
+{
+    return handle->regs->len;
+}
+CPU_LOG_ENTRY_GETTER_DECL(int, events)
+{
+    return handle->events->len;
+}
+CPU_LOG_ENTRY_GETTER_DECL(const char *, txt_buffer)
+{
+    return handle->txt_buffer->str;
+}
+
+REG_INFO_GETTER(uint16_t, flags)
+REG_INFO_GETTER(const char *, name)
+REG_INFO_GETTER(uint64_t, gpr)
+REG_INFO_GETTER_DECL(cap_register_handle, cap)
+{
+#ifdef TARGET_CHERI
+    log_reginfo_t *r = &g_array_index(handle->regs, log_reginfo_t, idx);
+    return (cap_register_handle)(&r->cap);
+#else
+    return NULL;
+#endif
+}
+
+MEM_INFO_GETTER(uint8_t, flags)
+MEM_INFO_GETTER(int, op)
+MEM_INFO_GETTER(uint64_t, addr)
+MEM_INFO_GETTER(uint64_t, value)
+MEM_INFO_GETTER_DECL(cap_register_handle, cap)
+{
+#ifdef TARGET_CHERI
+    log_meminfo_t *m = &g_array_index(handle->mem, log_meminfo_t, idx);
+    return (cap_register_handle)(&m->cap);
+#else
+    return NULL;
+#endif
+}
+
+CAPREG_GETTER_DECL(bool, tag)
+{
+#ifdef TARGET_CHERI
+    return ((cap_register_t *)handle)->cr_tag;
+#else
+    /*
+     * XXX-AM Error instead? these must be defined for non-CHERI
+     * targets but will never be called.
+     */
+    return 0;
+#endif
+}
+
+CAPREG_GETTER_DECL(bool, sealed)
+{
+#ifdef TARGET_CHERI
+    return !cap_is_unsealed((cap_register_t *)handle);
+#else
+    return 0;
+#endif
+}
+
+CAPREG_GETTER_DECL(uint64_t, base)
+{
+#ifdef TARGET_CHERI
+    return cap_get_base((cap_register_t *)handle);
+#else
+    return 0;
+#endif
+}
+
+CAPREG_GETTER_DECL(uint64_t, cursor)
+{
+#ifdef TARGET_CHERI
+    return cap_get_cursor((cap_register_t *)handle);
+#else
+    return 0;
+#endif
+}
+
+CAPREG_GETTER_DECL(uint64_t, length)
+{
+#ifdef TARGET_CHERI
+    return cap_get_length_full((cap_register_t *)handle);
+#else
+    return 0;
+#endif
+}
+
+CAPREG_GETTER_DECL(uint32_t, perms)
+{
+#ifdef TARGET_CHERI
+    return cap_get_perms((cap_register_t *)handle);
+#else
+    return 0;
+#endif
+}
+
+CAPREG_GETTER_DECL(uint32_t, otype)
+{
+#ifdef TARGET_CHERI
+    return cap_get_otype_unsigned((cap_register_t *)handle);
+#else
+    return 0;
+#endif
+}
diff --git a/accel/tcg/log_instr_text.c b/accel/tcg/log_instr_text.c
index 713292eb151..3ea84c80cf3 100644
--- a/accel/tcg/log_instr_text.c
+++ b/accel/tcg/log_instr_text.c
@@ -115,81 +115,78 @@ static inline void emit_text_reg(log_reginfo_t *rinfo)
  */
 void emit_text_instr(CPUArchState *env, cpu_log_entry_t *entry)
 {
+    cpu_log_instr_state_t *cpulog = get_cpu_log_state(env);
     QemuLogFile *logfile;
+    const log_event_t *event;
+    const char *log_state_op;
     int i;
 
-    /* Dump CPU-ID:ASID + address */
-    qemu_log("[%d:%d] ", env_cpu(env)->cpu_index, entry->asid);
-
-    /*
-     * Instruction disassembly, note that we use the instruction info
-     * opcode bytes, without accessing target memory here.
-     */
-    rcu_read_lock();
-    logfile = qatomic_rcu_read(&qemu_logfile);
-    if (logfile) {
-        target_disas_buf(logfile->fd, env_cpu(env), entry->insn_bytes,
-                         sizeof(entry->insn_bytes), entry->pc, 1);
-    }
-    rcu_read_unlock();
-
-    /*
-     * TODO(am2419): what to do with injected instructions?
-     * Is the rvfi_dii_trace state valid at log commit?
-     */
-
-    /* Dump mode switching info */
-    if (entry->flags & LI_FLAG_MODE_SWITCH)
-        qemu_log("-> Switch to %s mode\n",
-                 cpu_get_mode_name(entry->next_cpu_mode));
-    /* Dump interrupt/exception info */
-    switch (entry->flags & LI_FLAG_INTR_MASK) {
-    case LI_FLAG_INTR_TRAP:
-        qemu_log("-> Exception #%u vector 0x" TARGET_FMT_lx
-                 " fault-addr 0x" TARGET_FMT_lx "\n",
-                 entry->intr_code, entry->intr_vector, entry->intr_faultaddr);
-        break;
-    case LI_FLAG_INTR_ASYNC:
-        qemu_log("-> Interrupt #%04x vector 0x" TARGET_FMT_lx "\n",
-                 entry->intr_code, entry->intr_vector);
-        break;
-    default:
-        /* No interrupt */
-        break;
-    }
+    if (entry->flags & LI_FLAG_HAS_INSTR_DATA) {
+        /* Dump CPU-ID:ASID + address */
+        qemu_log("[%d:%d] ", env_cpu(env)->cpu_index, entry->asid);
+
+        /*
+         * Instruction disassembly, note that we use the instruction info
+         * opcode bytes, without accessing target memory here.
+         */
+        rcu_read_lock();
+        logfile = qatomic_rcu_read(&qemu_logfile);
+        if (logfile) {
+            target_disas_buf(logfile->fd, env_cpu(env), entry->insn_bytes,
+                             sizeof(entry->insn_bytes), entry->pc, 1);
+        }
+        rcu_read_unlock();
+
+        /*
+         * TODO(am2419): what to do with injected instructions?
+         * Is the rvfi_dii_trace state valid at log commit?
+         */
+
+        /* Dump mode switching info */
+        if (entry->flags & LI_FLAG_MODE_SWITCH)
+            qemu_log("-> Switch to %s mode\n",
+                     cpu_get_mode_name(entry->next_cpu_mode));
+        /* Dump interrupt/exception info */
+        switch (entry->flags & LI_FLAG_INTR_MASK) {
+        case LI_FLAG_INTR_TRAP:
+            qemu_log("-> Exception #%u vector 0x" TARGET_FMT_lx
+                     " fault-addr 0x" TARGET_FMT_lx "\n",
+                     entry->intr_code, entry->intr_vector,
+                     entry->intr_faultaddr);
+            break;
+        case LI_FLAG_INTR_ASYNC:
+            qemu_log("-> Interrupt #%04x vector 0x" TARGET_FMT_lx "\n",
+                     entry->intr_code, entry->intr_vector);
+            break;
+        default:
+            /* No interrupt */
+            break;
+        }
 
-    /* Dump memory access */
-    for (i = 0; i < entry->mem->len; i++) {
-        log_meminfo_t *minfo = &g_array_index(entry->mem, log_meminfo_t, i);
-        if (minfo->flags & LMI_LD) {
-            emit_text_ldst(minfo, "Read");
-        } else if (minfo->flags & LMI_ST) {
-            emit_text_ldst(minfo, "Write");
+        /* Dump memory access */
+        for (i = 0; i < entry->mem->len; i++) {
+            log_meminfo_t *minfo = &g_array_index(entry->mem, log_meminfo_t, i);
+            if (minfo->flags & LMI_LD) {
+                emit_text_ldst(minfo, "Read");
+            } else if (minfo->flags & LMI_ST) {
+                emit_text_ldst(minfo, "Write");
+            }
         }
-    }
 
-    /* Dump register changes and side-effects */
-    for (i = 0; i < entry->regs->len; i++) {
-        log_reginfo_t *rinfo = &g_array_index(entry->regs, log_reginfo_t, i);
-        emit_text_reg(rinfo);
+        /* Dump register changes and side-effects */
+        for (i = 0; i < entry->regs->len; i++) {
+            log_reginfo_t *rinfo =
+                &g_array_index(entry->regs, log_reginfo_t, i);
+            emit_text_reg(rinfo);
+        }
     }
 
     /* Dump extra logged messages, if any */
     if (entry->txt_buffer->len > 0) {
         qemu_log("%s", entry->txt_buffer->str);
     }
-}
-
-/*
- * Emit a textual representation of events recorded by the given trace entry.
- */
-void emit_text_events(CPUArchState *env, cpu_log_entry_t *entry)
-{
-    cpu_log_instr_state_t *cpulog = get_cpu_log_state(env);
-    const log_event_t *event;
-    const char *log_state_op;
-    int i;
 
+    /* Emit events recorded by the given trace entry */
     for (i = 0; i < entry->events->len; i++) {
         event = &g_array_index(entry->events, const log_event_t, i);
         switch (event->id) {
@@ -203,20 +200,18 @@ void emit_text_events(CPUArchState *env, cpu_log_entry_t *entry)
             }
 
             if (cpulog->loglevel == QEMU_LOG_INSTR_LOGLEVEL_USER) {
-                qemu_log("[%u:%u] %s user-mode only instruction logging "
-                         "@ " TARGET_FMT_lx "\n",
-                         env_cpu(env)->cpu_index, cpu_get_asid(env),
-                         log_state_op, event->state.pc);
+                qemu_log(
+                    "[%u:%u] %s user-mode only instruction logging @ %lx\n",
+                    env_cpu(env)->cpu_index, cpu_get_asid(env), log_state_op,
+                    event->state.pc);
             } else {
-                qemu_log("[%u:%u] %s instruction logging @ " TARGET_FMT_lx
-                         "\n",
+                qemu_log("[%u:%u] %s instruction logging @ %lx\n",
                          env_cpu(env)->cpu_index, cpu_get_asid(env),
                          log_state_op, event->state.pc);
             }
             break;
         case LOG_EVENT_CTX_SWITCH:
-            qemu_log("Context switch pid=0x" TARGET_FMT_lx
-                     " tid=0x" TARGET_FMT_lx " cid=0x" TARGET_FMT_lx "\n",
+            qemu_log("Context switch pid=0x%lx tid=0x%lx cid=0x%lx\n",
                      event->ctx_switch.pid, event->ctx_switch.tid,
                      event->ctx_switch.cid);
         }
diff --git a/accel/tcg/meson.build b/accel/tcg/meson.build
index 097a025fd8d..8a89e995472 100644
--- a/accel/tcg/meson.build
+++ b/accel/tcg/meson.build
@@ -16,5 +16,4 @@ specific_ss.add(when: ['CONFIG_SOFTMMU', 'CONFIG_TCG'], if_true: files('tcg-all.
 specific_ss.add(when: ['CONFIG_TCG_LOG_INSTR', 'CONFIG_TCG'], if_true: files('log_instr.c'))
 specific_ss.add(when: ['CONFIG_TCG_LOG_INSTR', 'CONFIG_TCG'], if_true: files('log_instr_text.c'))
 specific_ss.add(when: ['CONFIG_TCG_LOG_INSTR', 'CONFIG_TCG'], if_true: files('log_instr_cvtrace.c'))
-specific_ss.add(when: ['CONFIG_TRACE_STATS', 'CONFIG_TCG_LOG_INSTR', 'CONFIG_TCG'], if_true: files('log_instr_stats.c'))
 specific_ss.add(when: ['CONFIG_TRACE_PERFETTO', 'CONFIG_TCG_LOG_INSTR', 'CONFIG_TCG'], if_true: files('log_instr_perfetto.c'))
diff --git a/chardev/char-mux.c b/chardev/char-mux.c
index 6f980bb8364..32d53a44e56 100644
--- a/chardev/char-mux.c
+++ b/chardev/char-mux.c
@@ -26,12 +26,21 @@
 #include "qapi/error.h"
 #include "qemu/module.h"
 #include "qemu/option.h"
+#include "qemu/error-report.h"
+#include "qemu/log_instr.h"
 #include "chardev/char.h"
 #include "sysemu/block-backend.h"
 #include "sysemu/sysemu.h"
 #include "chardev-internal.h"
 
 /* MUX driver for serial I/O splitting */
+#ifdef CONFIG_TCG_LOG_INSTR
+__attribute__((weak)) void qemu_log_instr_sync_buffers()
+{
+    /* Real implementation in accel/tcg/log_instr.c */
+    warn_report("Calling no-op %s\r", __func__);
+}
+#endif
 
 /* Called with chr_write_lock held.  */
 static int mux_chr_write(Chardev *chr, const uint8_t *buf, int len)
@@ -149,6 +158,9 @@ static int mux_proc_byte(Chardev *chr, MuxChardev *d, int ch)
             break;
         case 'x':
             {
+#ifdef CONFIG_TCG_LOG_INSTR
+                 qemu_log_instr_sync_buffers();
+#endif
                  const char *term =  "QEMU: Terminated\n\r";
                  qemu_chr_write_all(chr, (uint8_t *)term, strlen(term));
                  exit(0);
diff --git a/configure b/configure
index dc0df44f035..c2e01faed41 100755
--- a/configure
+++ b/configure
@@ -341,7 +341,6 @@ gprof="no"
 debug_tcg="no"
 tcg_log_instr="yes"
 tcg_log_instr_perfetto="yes"
-trace_stats="no"
 rvfi_dii="no"
 debug="no"
 sanitizers="no"
@@ -1042,10 +1041,6 @@ for opt do
   ;;
   --disable-perfetto-log-instr) tcg_log_instr_perfetto="no"
   ;;
-  --enable-trace-stats) trace_stats="yes"
-  ;;
-  --disable-trace-stats) trace_stats="no"
-  ;;
   --enable-rvfi-dii) rvfi_dii="yes"
   ;;
   --disable-rvfi-dii) rvfi_dii="no"
@@ -5981,9 +5976,6 @@ fi
 if test "$tcg_log_instr_perfetto" = "yes" ; then
     echo "CONFIG_TRACE_PERFETTO=y" >> $config_host_mak
 fi
-if test "$trace_stats" = "yes" ; then
-    echo "CONFIG_TRACE_STATS=y" >> $config_host_mak
-fi
 if test "$rvfi_dii" = "yes" ; then
   echo "CONFIG_RVFI_DII=y" >> $config_host_mak
 fi
diff --git a/hw/misc/sifive_test.c b/hw/misc/sifive_test.c
index 2deb2072cc8..9cce43e7502 100644
--- a/hw/misc/sifive_test.c
+++ b/hw/misc/sifive_test.c
@@ -22,6 +22,7 @@
 #include "hw/sysbus.h"
 #include "qapi/error.h"
 #include "qemu/log.h"
+#include "qemu/log_instr.h"
 #include "qemu/module.h"
 #include "sysemu/runstate.h"
 #include "hw/hw.h"
@@ -42,6 +43,9 @@ static void sifive_test_write(void *opaque, hwaddr addr,
         case FINISHER_FAIL:
             exit(code);
         case FINISHER_PASS:
+#ifdef CONFIG_TCG_LOG_INSTR
+            qemu_log_instr_sync_buffers();
+#endif
             exit(0);
         case FINISHER_RESET:
             qemu_system_reset_request(SHUTDOWN_CAUSE_GUEST_RESET);
diff --git a/include/exec/log_instr.h b/include/exec/log_instr.h
index 2c228f9f914..85cf0ebf138 100644
--- a/include/exec/log_instr.h
+++ b/include/exec/log_instr.h
@@ -189,50 +189,6 @@ void qemu_log_printf_create_globals(void);
 
 struct cpu_log_entry;
 
-/*
- * Trace event identifiers.
- */
-typedef enum {
-    LOG_EVENT_STATE = 0,
-    LOG_EVENT_CTX_SWITCH = 1,
-} log_event_id_t;
-
-/*
- * Tracing status changed (e.g. trace start/stop)
- */
-typedef enum {
-    LOG_EVENT_STATE_START,
-    LOG_EVENT_STATE_STOP,
-    LOG_EVENT_STATE_FLUSH
-} log_event_trace_state_t;
-
-typedef struct {
-    log_event_trace_state_t next_state;
-    target_ulong pc;
-} log_event_trace_state_update_t;
-
-/*
- * Context switch event.
- */
-typedef struct {
-    target_ulong pid; /* Process ID */
-    target_ulong tid; /* Thread ID */
-    target_ulong cid; /* Compartment ID */
-} log_event_ctx_switch_t;
-
-/*
- * Trace event.
- * This records arbitrary higher-level events associated with instruction
- * entries.
- */
-typedef struct {
-    log_event_id_t id;
-    union {
-        log_event_trace_state_update_t state;
-        log_event_ctx_switch_t ctx_switch;
-    };
-} log_event_t;
-
 /*
  * Request a flush of the TCG when changing loglevel outside of qemu_log_instr.
  * TODO(am2419): this should be removed from the interface.
diff --git a/include/exec/log_instr_internal.h b/include/exec/log_instr_internal.h
index d8e3ec96b74..e3e68579790 100644
--- a/include/exec/log_instr_internal.h
+++ b/include/exec/log_instr_internal.h
@@ -38,14 +38,7 @@
 
 #pragma once
 
-/*
- * Instruction log info associated with each committed log entry.
- * This is stored in the per-cpu log cpustate.
- */
-typedef struct cpu_log_entry {
-#define cpu_log_entry_startzero asid
-    uint16_t asid;
-    int flags;
+/* cpu_log_entry flags */
 /* Entry contains a synchronous exception */
 #define LI_FLAG_INTR_TRAP (1 << 0)
 /* Entry contains an asynchronous exception */
@@ -56,6 +49,24 @@ typedef struct cpu_log_entry {
 /* Entry contains instruction data or just events */
 #define LI_FLAG_HAS_INSTR_DATA (1 << 3)
 
+/* register info flags */
+#define LRI_CAP_REG   1
+#define LRI_HOLDS_CAP 2
+
+/* memory op info flags */
+#define LMI_LD  1
+#define LMI_ST  2
+#define LMI_CAP 4
+
+#ifndef __cplusplus
+/*
+ * Instruction log info associated with each committed log entry.
+ * This is stored in the per-cpu log cpustate.
+ */
+typedef struct cpu_log_entry {
+#define cpu_log_entry_startzero asid
+    uint16_t asid;
+    int flags;
     qemu_log_instr_cpu_mode_t next_cpu_mode;
     uint32_t intr_code;
     target_ulong intr_vector;
@@ -92,11 +103,8 @@ typedef struct cpu_log_entry {
  * Register update info.
  * This records a CPU register update occurred during an instruction.
  */
-typedef struct {
+typedef struct log_reginfo {
     uint16_t flags;
-#define LRI_CAP_REG   1
-#define LRI_HOLDS_CAP 2
-
     const char *name;
     union {
         target_ulong gpr;
@@ -113,11 +121,8 @@ typedef struct {
  * Memory access info.
  * This records a memory access occurred during an instruction.
  */
-typedef struct {
+typedef struct log_meminfo {
     uint8_t flags;
-#define LMI_LD  1
-#define LMI_ST  2
-#define LMI_CAP 4
     MemOp op;
     target_ulong addr;
     union {
@@ -135,9 +140,9 @@ typedef struct {
  */
 struct trace_backend_hooks {
     void (*init)(CPUArchState *env);
+    void (*sync)(CPUArchState *env);
     void (*emit_header)(CPUArchState *env);
     void (*emit_instr)(CPUArchState *env, cpu_log_entry_t *entry);
-    void (*emit_events)(CPUArchState *env, cpu_log_entry_t *entry);
 };
 typedef struct trace_backend_hooks trace_backend_hooks_t;
 
@@ -149,14 +154,13 @@ typedef bool (*cpu_log_instr_filter_fn_t)(struct cpu_log_entry *entry);
 
 /* Text backend */
 void emit_text_instr(CPUArchState *env, cpu_log_entry_t *entry);
-void emit_text_events(CPUArchState *env, cpu_log_entry_t *entry);
 /* CVTrace backend */
 void emit_cvtrace_header(CPUArchState *env);
 void emit_cvtrace_entry(CPUArchState *env, cpu_log_entry_t *entry);
-/* Stats backen */
-void init_stats_backend(CPUArchState *env);
-void emit_stats_entry(CPUArchState *env, cpu_log_entry_t *entry);
-void emit_stats_events(CPUArchState *env, cpu_log_entry_t *entry);
+/* Perfetto backend */
+void init_perfetto_backend(CPUArchState *env);
+void sync_perfetto_backend(CPUArchState *env);
+void emit_perfetto_entry(CPUArchState *env, cpu_log_entry_t *entry);
 
 #ifdef CONFIG_DEBUG_TCG
 #define log_assert(x) assert((x))
@@ -182,3 +186,4 @@ static inline cpu_log_entry_t *get_cpu_log_entry(CPUArchState *env)
     return &g_array_index(cpulog->instr_info, cpu_log_entry_t,
                           cpulog->ring_head);
 }
+#endif /* __cplusplus */
diff --git a/include/exec/log_instr_perfetto.h b/include/exec/log_instr_perfetto.h
new file mode 100644
index 00000000000..20a5b11c5ee
--- /dev/null
+++ b/include/exec/log_instr_perfetto.h
@@ -0,0 +1,141 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2021 Alfredo Mazzinghi
+ *
+ * This software was developed by SRI International and the University of
+ * Cambridge Computer Laboratory (Department of Computer Science and
+ * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the
+ * DARPA SSITH research programme.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * Note: This is internal to the C++ perfetto tracing glue code.
+ */
+
+#pragma once
+
+#ifdef __cplusplus
+#include <cstdint>
+#include <perfetto.h>
+
+PERFETTO_DEFINE_CATEGORIES(
+    perfetto::Category("instructions")
+        .SetDescription("CPU instructions executed"),
+    perfetto::Category("memory").SetDescription("Tag and memory load/store"),
+    perfetto::Category("stats").SetDescription("High-level statistics data"),
+    perfetto::Category("ctrl").SetDescription("Tracing control events"));
+#endif
+
+/*
+ * XXX-AM: We can not import directly exec/log_instr_internal.h This is sad as
+ * we need to generate wrappers to fetch cpu_log_entry fields to fill the trace
+ * entry protobufs. For now this is somewhat hacky :(
+ */
+#ifdef __cplusplus
+extern "C" {
+#endif
+/* Opaque handles for cpu_log_entry_t */
+struct cpu_log_entry;
+typedef struct cpu_log_entry *cpu_log_entry_handle;
+struct log_reginfo;
+typedef struct log_reginfo *reg_info_handle;
+struct log_meminfo;
+typedef struct log_meminfo *mem_info_handle;
+typedef void *cap_register_handle;
+
+void perfetto_init_cpu(int cpu_index, void **backend_data);
+void perfetto_sync_cpu(void *backend_data);
+void perfetto_emit_instr(void *backend_data, cpu_log_entry_handle entry_handle);
+
+/*
+ * cpu_log_entry getters implemented by the perfetto backend glue code.
+ * Note: Sadly, the type must be manually be kept in sync with
+ * log_instr_internal.h
+ */
+#define CPU_LOG_ENTRY_GETTER_DECL(type, field)                                 \
+    type perfetto_log_entry_##field(cpu_log_entry_handle handle)
+#define REG_INFO_GETTER_DECL(type, field)                                      \
+    type perfetto_reg_info_##field(cpu_log_entry_handle handle, int idx)
+#define MEM_INFO_GETTER_DECL(type, field)                                      \
+    type perfetto_mem_info_##field(cpu_log_entry_handle handle, int idx)
+#define CAPREG_GETTER_DECL(type, field)                                        \
+    type perfetto_cap_##field(cap_register_handle handle)
+
+#define CPU_LOG_ENTRY_GETTER(type, field)                                      \
+    CPU_LOG_ENTRY_GETTER_DECL(type, field)                                     \
+    {                                                                          \
+        return (type)(handle->field);                                          \
+    }
+#define REG_INFO_GETTER(type, field)                                           \
+    REG_INFO_GETTER_DECL(type, field)                                          \
+    {                                                                          \
+        log_reginfo_t *r = &g_array_index(handle->regs, log_reginfo_t, idx);   \
+        return (type)(r->field);                                               \
+    }
+#define MEM_INFO_GETTER(type, field)                                           \
+    MEM_INFO_GETTER_DECL(type, field)                                          \
+    {                                                                          \
+        log_meminfo_t *m = &g_array_index(handle->mem, log_meminfo_t, idx);    \
+        return (type)(m->field);                                               \
+    }
+
+CPU_LOG_ENTRY_GETTER_DECL(uint16_t, asid);
+CPU_LOG_ENTRY_GETTER_DECL(int, flags);
+CPU_LOG_ENTRY_GETTER_DECL(qemu_log_instr_cpu_mode_t, next_cpu_mode);
+CPU_LOG_ENTRY_GETTER_DECL(uint32_t, intr_code);
+CPU_LOG_ENTRY_GETTER_DECL(uint64_t, intr_vector);
+CPU_LOG_ENTRY_GETTER_DECL(uint64_t, intr_faultaddr);
+CPU_LOG_ENTRY_GETTER_DECL(uint64_t, pc);
+CPU_LOG_ENTRY_GETTER_DECL(int, insn_size);
+CPU_LOG_ENTRY_GETTER_DECL(const char *, insn_bytes);
+CPU_LOG_ENTRY_GETTER_DECL(int, mem);
+CPU_LOG_ENTRY_GETTER_DECL(int, regs);
+CPU_LOG_ENTRY_GETTER_DECL(int, events);
+CPU_LOG_ENTRY_GETTER_DECL(const char *, txt_buffer);
+
+log_event_t *perfetto_log_event(cpu_log_entry_handle handle, int idx);
+
+REG_INFO_GETTER_DECL(uint16_t, flags);
+REG_INFO_GETTER_DECL(const char *, name);
+REG_INFO_GETTER_DECL(uint64_t, gpr);
+REG_INFO_GETTER_DECL(cap_register_handle, cap);
+
+MEM_INFO_GETTER_DECL(uint8_t, flags);
+MEM_INFO_GETTER_DECL(int, op);
+MEM_INFO_GETTER_DECL(uint64_t, addr);
+MEM_INFO_GETTER_DECL(uint64_t, value);
+MEM_INFO_GETTER_DECL(cap_register_handle, cap);
+
+CAPREG_GETTER_DECL(bool, tag);
+CAPREG_GETTER_DECL(bool, sealed);
+CAPREG_GETTER_DECL(uint64_t, base);
+CAPREG_GETTER_DECL(uint64_t, cursor);
+CAPREG_GETTER_DECL(uint64_t, length);
+CAPREG_GETTER_DECL(uint32_t, perms);
+CAPREG_GETTER_DECL(uint32_t, otype);
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/include/qemu/log_instr.h b/include/qemu/log_instr.h
index bc7427532c9..396524e4a67 100644
--- a/include/qemu/log_instr.h
+++ b/include/qemu/log_instr.h
@@ -54,11 +54,8 @@ typedef enum {
     QEMU_LOG_INSTR_BACKEND_TEXT = 0,
     QEMU_LOG_INSTR_BACKEND_CVTRACE = 1,
     QEMU_LOG_INSTR_BACKEND_NOP = 2,
-#ifdef CONFIG_TRACE_STATS
-    QEMU_LOG_INSTR_BACKEND_STATS = 3,
-#endif
 #ifdef CONFIG_TRACE_PERFETTO
-    QEMU_LOG_INSTR_BACKEND_PERFETTO = 4
+    QEMU_LOG_INSTR_BACKEND_PERFETTO = 3
 #endif
 } qemu_log_instr_backend_t;
 
@@ -94,6 +91,65 @@ typedef enum {
     QEMU_LOG_INSTR_LOGLEVEL_USER = 2,
 } qemu_log_instr_loglevel_t;
 
+/*
+ * Trace event identifiers.
+ */
+typedef enum {
+    LOG_EVENT_STATE = 0,
+    LOG_EVENT_CTX_SWITCH = 1,
+} log_event_id_t;
+
+/*
+ * Tracing status changed (e.g. trace start/stop)
+ */
+typedef enum {
+    LOG_EVENT_STATE_START,
+    LOG_EVENT_STATE_STOP,
+    LOG_EVENT_STATE_FLUSH
+} log_event_trace_state_t;
+
+typedef struct {
+    log_event_trace_state_t next_state;
+    uint64_t pc;
+} log_event_trace_state_update_t;
+
+/*
+ * Context switch event.
+ */
+typedef struct {
+    uint64_t pid; /* Process ID */
+    uint64_t tid; /* Thread ID */
+    uint64_t cid; /* Compartment ID */
+} log_event_ctx_switch_t;
+
+/*
+ * Trace event.
+ * This records arbitrary higher-level events associated with instruction
+ * entries.
+ */
+typedef struct {
+    log_event_id_t id;
+    union {
+        log_event_trace_state_update_t state;
+        log_event_ctx_switch_t ctx_switch;
+    };
+} log_event_t;
+
+#ifdef CONFIG_TRACE_PERFETTO
+/* Perfetto backend configuration hooks */
+#ifdef __cplusplus
+extern "C" {
+#endif
+void qemu_log_instr_perfetto_conf_logfile(const char *name);
+int qemu_log_instr_perfetto_conf_categories(const char *category_list);
+#ifdef __cplusplus
+}
+#endif
+#endif /* CONFIG_TRACE_PERFETTO */
+
+#ifndef __cplusplus
+/* No visibility in the perfetto tracing backend */
+
 static inline void qemu_log_instr_set_backend(qemu_log_instr_backend_t id)
 {
     qemu_log_instr_backend = id;
@@ -175,7 +231,16 @@ typedef enum {
  * Initialize instruction logging for a cpu.
  */
 void qemu_log_instr_init(CPUState *env);
-int qemu_log_instr_global_switch(int log_flags);
+
+/*
+ * Hook to make sure all tracing buffers are synced on qemu shutdown.
+ */
+void qemu_log_instr_sync_buffers(void);
+
+/*
+ * Global instruction logging hook from qemu tracing commands.
+ */
+void qemu_log_instr_global_switch(bool request_stop);
 
 /*
  * Update the ring buffer size.
@@ -205,6 +270,7 @@ void qemu_log_instr_add_startup_filter(cpu_log_instr_filter_t filter);
  * memory ranges..
  */
 void qemu_log_instr_mem_filter_update(void);
+#endif /* ! __cplusplus */
 
 #else /* ! CONFIG_TCG_LOG_INSTR */
 #define qemu_log_instr_set_backend(id) ((void)0)
diff --git a/qemu-options.hx b/qemu-options.hx
index 407e9461059..ef396d1481b 100644
--- a/qemu-options.hx
+++ b/qemu-options.hx
@@ -4117,10 +4117,25 @@ SRST
 ERST
 
 DEF("cheri-trace-backend", HAS_ARG, QEMU_OPTION_cheri_trace_backend, \
-"-cheri-trace-backend [text|cvtrace|stats]     Select CHERI trace mode.\n", QEMU_ARCH_ALL)
+"-cheri-trace-backend [text|cvtrace|perfetto]     Select CHERI trace mode.\n", QEMU_ARCH_ALL)
 SRST
 ``-cheri-trace-backend type``
-    Set CHERI trace backend to <type> (text, cvtrace or stats)
+    Set CHERI trace backend to <type> (text, cvtrace or perfetto)
+ERST
+
+DEF("cheri-trace-perfetto-logfile", HAS_ARG, QEMU_OPTION_trace_perfetto_logfile, \
+"-cheri-trace-perfetto-logfile [logfile]     \
+ Set log file for perfetto traces, defaults to qemu_trace.pb.\n", QEMU_ARCH_ALL)
+SRST
+``-cheri-trace-perfetto-logfile [logfile]``
+    Set perfetto trace output file.
+ERST
+
+DEF("cheri-trace-perfetto-categories", HAS_ARG, QEMU_OPTION_trace_perfetto_categories, \
+"-cheri-trace-perfetto-categories category[,...]     Select categories to trace.\n", QEMU_ARCH_ALL)
+SRST
+``-cheri-trace-perfetto-categories category[,...]``
+    Select categories of messages to include in the trace (instructions, stats)
 ERST
 
 DEF("cheri-trace-filters", HAS_ARG, QEMU_OPTION_cheri_trace_filters, \
diff --git a/softmmu/vl.c b/softmmu/vl.c
index 998a3cfbbf8..f9d20853d09 100644
--- a/softmmu/vl.c
+++ b/softmmu/vl.c
@@ -1619,6 +1619,9 @@ void qemu_system_killed(int signal, pid_t pid)
 
 void qemu_system_shutdown_request(ShutdownCause reason)
 {
+#ifdef CONFIG_TCG_LOG_INSTR
+    qemu_log_instr_sync_buffers();
+#endif
     trace_qemu_system_shutdown_request(reason);
     replay_shutdown_request(reason);
     shutdown_requested = reason;
@@ -3752,15 +3755,23 @@ void qemu_init(int argc, char **argv, char **envp)
                     qemu_log_instr_set_backend(QEMU_LOG_INSTR_BACKEND_TEXT);
                 } else if (strcmp(optarg, "cvtrace") == 0) {
                     qemu_log_instr_set_backend(QEMU_LOG_INSTR_BACKEND_CVTRACE);
-#ifdef CONFIG_TRACE_STATS
-                } else if (strcmp(optarg, "stats") == 0) {
-                    qemu_log_instr_set_backend(QEMU_LOG_INSTR_BACKEND_STATS);
+#ifdef CONFIG_TRACE_PERFETTO
+                } else if (strcmp(optarg, "perfetto") == 0) {
+                    qemu_log_instr_set_backend(QEMU_LOG_INSTR_BACKEND_PERFETTO);
 #endif
                 } else {
                     printf("Invalid choice for cheri-trace-format: '%s'\n", optarg);
                     exit(1);
                 }
                 break;
+#ifdef CONFIG_TRACE_PERFETTO
+            case QEMU_OPTION_trace_perfetto_logfile:
+                qemu_log_instr_perfetto_conf_logfile(optarg);
+                break;
+            case QEMU_OPTION_trace_perfetto_categories:
+                qemu_log_instr_perfetto_conf_categories(optarg);
+                break;
+#endif
             case QEMU_OPTION_cheri_trace_buffer_size:
                 qemu_log_instr_set_buffer_size(strtoul(optarg, NULL, 0));
                 break;
diff --git a/trace_extra/meson.build b/trace_extra/meson.build
index 2c08ed31d23..b7a882b960f 100644
--- a/trace_extra/meson.build
+++ b/trace_extra/meson.build
@@ -1,19 +1,19 @@
 
 qemutrace_ss = ss.source_set()
 
-qemutrace_ss.add(when: ['CONFIG_TCG', 'CONFIG_TRACE_STATS'],
-                 if_true: files('trace_stats.cc'))
-qemutrace_ss.add(when: ['CONFIG_TCG', 'CONFIG_TRACE_STATS'],
-                 if_true: dependency('boost', modules: ['iostreams']))
+qemutrace_ss.add(when: ['CONFIG_TCG', 'CONFIG_TRACE_PERFETTO'],
+                 if_true: dependency('boost', modules: ['filesystem']))
 
 qemutrace_ss.add(when: ['CONFIG_TCG', 'CONFIG_TRACE_PERFETTO'],
-                 if_true: files('trace_perfetto.cc', 'cheri-perfetto/sdk/perfetto.cc'))
+                 if_true: files('trace_perfetto.cc', 'trace_stats.cc',
+                                'cheri-perfetto/sdk/perfetto.cc'))
 
 qemutrace_ss = qemutrace_ss.apply(config_all, strict: false)
 libqemutrace = static_library('qemutrace',
                               sources: qemutrace_ss.sources(),
                               dependencies: qemutrace_ss.dependencies(),
                               include_directories: ['cheri-perfetto/sdk'],
-                              cpp_args: ['-DCONFIG_TRACE_STATS', '-DQEMU_PERFETTO'])
+                              cpp_args: ['-DCONFIG_TCG_LOG_INSTR',
+                                         '-DQEMU_PERFETTO'])
 
 qemuutil_libs += libqemutrace
diff --git a/trace_extra/qemu_perfetto.h b/trace_extra/qemu_perfetto.h
index 5bb1cf5c63a..96bd80f5a13 100644
--- a/trace_extra/qemu_perfetto.h
+++ b/trace_extra/qemu_perfetto.h
@@ -37,7 +37,7 @@
 extern "C" {
 #endif
 
-bool perfetto_init(void);
+bool perfetto_init(int cpu_id, void **backend_data);
 
 #ifdef __cplusplus
 } /* extern C */
diff --git a/trace_extra/trace_perfetto.cc b/trace_extra/trace_perfetto.cc
index 8cca2a8775f..5443beff6a5 100644
--- a/trace_extra/trace_perfetto.cc
+++ b/trace_extra/trace_perfetto.cc
@@ -30,25 +30,75 @@
  * SUCH DAMAGE.
  */
 
-#include <perfetto.h>
+/*
+ * QEMU instruction logging bridge to the perfetto framework.
+ */
 
-#include "qemu/osdep.h"
-#ifndef _WIN32
+#include <memory>
+#include <mutex>
+#include <sstream>
+#include <iomanip>
+#include <string>
+#include <vector>
 #include <pthread.h>
 #include <fcntl.h>
-#endif
+#include <perfetto.h>
+#include <boost/filesystem.hpp>
 
-#include "trace/event-internal.h"
-#include "qemu_perfetto.h"
-#include "track.hh"
+#include "qemu/log_instr.h"
+#include "exec/log_instr_internal.h"
+#include "exec/log_instr_perfetto.h"
+#include "trace_extra/trace_stats.hh"
 
 PERFETTO_TRACK_EVENT_STATIC_STORAGE();
 
+namespace fs = boost::filesystem;
+
 namespace {
 
 /* Tracing session pointer */
 std::unique_ptr<perfetto::TracingSession> session;
 
+/* perfetto logfile */
+fs::path logfile("qemu_trace.pb");
+
+/* category strings */
+std::vector<std::string> categories;
+
+static unsigned long
+gen_track_uuid(int cpu_id) {
+    // use perfetto::Uuid?
+    static unsigned long next_track_id = 0x100;
+    return (next_track_id++);
+}
+
+/*
+ * Private per-CPU state.
+ */
+struct perfetto_backend_data {
+    // Per-CPU track. This will be the parent track for all events on this CPU.
+    perfetto::Track cpu_track;
+    // Per-CPU control track. This records tracing control events.
+    perfetto::Track cpu_ctrl_track;
+    // Per-CPU aggregate statistics
+    cheri::qemu_stats stats;
+
+    perfetto_backend_data(int cpu_id) :
+        cpu_track(perfetto::Track::Global(gen_track_uuid(cpu_id))),
+        cpu_ctrl_track(perfetto::Track::Global(gen_track_uuid(cpu_id)))
+    {
+        std::string track_name("CPU " + std::to_string(cpu_id));
+        std::string ctrl_name(track_name + " CTRL");
+
+        auto desc = cpu_track.Serialize();
+        desc.set_name(track_name);
+        perfetto::TrackEvent::SetTrackDescriptor(cpu_track, desc);
+        desc = cpu_ctrl_track.Serialize();
+        desc.set_name(ctrl_name);
+        perfetto::TrackEvent::SetTrackDescriptor(cpu_ctrl_track, desc);
+    }
+};
+
 /*
  * Initialize perfetto tracing.
  *
@@ -63,41 +113,248 @@ bool perfetto_start_tracing(void)
 
     perfetto::TracingInitArgs args;
     perfetto::TraceConfig cfg;
+    perfetto::protos::gen::TrackEventConfig track_cfg;
 
     args.backends |= perfetto::kInProcessBackend;
 
     perfetto::Tracing::Initialize(args);
     perfetto::TrackEvent::Register();
 
-    cfg.add_buffers()->set_size_kb(4096);
+    cfg.add_buffers()->set_size_kb(1 << 16); // 64MiB
+    cfg.set_flush_period_ms(2000);
+    cfg.set_file_write_period_ms(1000);
+    fs::remove(logfile);
+    cfg.set_write_into_file(true);
+    cfg.set_output_path(logfile.string());
 
     auto *ds_cfg = cfg.add_data_sources()->mutable_config();
     ds_cfg->set_name("track_event");
+    track_cfg.add_disabled_categories("*");
+    for (auto &category : categories) {
+        track_cfg.add_enabled_categories(category);
+    }
+    ds_cfg->set_track_event_config_raw(track_cfg.SerializeAsString());
+    auto *producer_cfg = cfg.add_producers();
+    producer_cfg->set_producer_name("qemu-tcg");
+    producer_cfg->set_shm_size_kb(1 << 11); // 2MiB
 
     session = perfetto::Tracing::NewTrace();
 
-    int trace_fd = open("test.pftrace", O_RDWR | O_CREAT | O_TRUNC, 0600);
-    session->Setup(cfg, trace_fd);
+    session->Setup(cfg);
     session->StartBlocking();
 
+    perfetto::ProcessTrack qemu_proc = perfetto::ProcessTrack::Current();
+    perfetto::protos::gen::TrackDescriptor desc = qemu_proc.Serialize();
+    desc.mutable_process()->set_process_name("qemu");
+    perfetto::TrackEvent::SetTrackDescriptor(qemu_proc, desc);
+
     return true;
 }
 
 void perfetto_tracing_stop(void)
 {
+    // NOTE: This is not sufficient for flushing the buffers, we currently
+    // also need to call the buffer sync function for each CPU on the exit path.
+    session->FlushBlocking();
     session->StopBlocking();
 }
 
+perfetto::protos::pbzero::ModeSwitch
+qemu_cpu_mode_to_trace(qemu_log_instr_cpu_mode_t mode)
+{
+    // NOTE: We rely on the fact that the protobuf enum ModeSwitch
+    // uses the same numbering as qemu_log_instr_cpu_mode_t
+    return static_cast<perfetto::protos::pbzero::ModeSwitch>(mode);
+}
+
+void
+trace_cap_register(perfetto::protos::pbzero::Capability *cap,
+                   cap_register_handle chandle)
+{
+    cap->set_valid(perfetto_cap_tag(chandle));
+    cap->set_sealed(perfetto_cap_sealed(chandle));
+    cap->set_base(perfetto_cap_base(chandle));
+    cap->set_length(perfetto_cap_length(chandle));
+    cap->set_cursor(perfetto_cap_cursor(chandle));
+    cap->set_perms(perfetto_cap_perms(chandle));
+    cap->set_otype(perfetto_cap_otype(chandle));
+}
+
+void
+process_extra_events(perfetto_backend_data *data, cpu_log_entry_handle entry)
+{
+    /* Track executed instruction PC to track histograms */
+    uint64_t pc = perfetto_log_entry_pc(entry);
+    int size = perfetto_log_entry_insn_size(entry);
+    data->stats.process_next_pc(pc, size);
+
+    int nevents = perfetto_log_entry_events(entry);
+    for (int i = 0; i < nevents; i++) {
+        log_event_t *evt = perfetto_log_event(entry, i);
+        if (evt->id == LOG_EVENT_STATE) {
+            switch(evt->state.next_state) {
+                case LOG_EVENT_STATE_FLUSH:
+                    data->stats.flush(data->cpu_track);
+                    //perfetto::TrackEvent::Flush();
+                    break;
+                case LOG_EVENT_STATE_START:
+                    TRACE_EVENT_BEGIN("ctrl", "tracing", data->cpu_ctrl_track);
+                    break;
+                case LOG_EVENT_STATE_STOP:
+                    TRACE_EVENT_END("ctrl", data->cpu_ctrl_track);
+                    break;
+            }
+        }
+    }
+}
+
+} /* anonymous */
+
+extern "C" void qemu_log_instr_perfetto_conf_logfile(const char *name)
+{
+    logfile = name;
+}
+
+extern "C" void qemu_log_instr_perfetto_conf_categories(const char *category_str)
+{
+    std::string strspec(category_str);
+    int pos;
+
+    while ((pos = strspec.find(',', 0)) != std::string::npos) {
+        categories.push_back(strspec.substr(0, pos));
+        strspec = strspec.substr(pos + 1);
+    }
+    if (strspec.size())
+        categories.push_back(strspec);
 }
 
 /*
  * Initialize perfetto tracing.
  *
- * Start tracing thread
+ * Start tracing thread when first called.
  */
-extern "C" bool perfetto_init(void)
+extern "C" void perfetto_init_cpu(int cpu_index, void **backend_data)
 {
-    atexit(perfetto_tracing_stop);
+    static std::once_flag init_flag;
+
+    std::call_once(init_flag, [&]() {
+        atexit(perfetto_tracing_stop);
+        perfetto_start_tracing();
+    });
+
+
+    // XXX-AM: The backend data is currently missing a cleanup hook, so it leaks.
+    // This is not a big issue as it should be live until the qemu process terminates.
+    *backend_data = new perfetto_backend_data(cpu_index);
+}
+
+/*
+ * This is required because of how perfetto TraceSession::Flush works,
+ * see comment on this method for more details.
+ * We need to flush the data source directly to see the last packet.
+ */
+extern "C" void perfetto_sync_cpu(void *backend_data)
+{
+    perfetto::TrackEvent::Flush();
+}
+
+extern "C" void perfetto_emit_instr(void *backend_data, cpu_log_entry_handle entry)
+{
+    auto *data = reinterpret_cast<perfetto_backend_data *>(backend_data);
+
+    TRACE_EVENT_INSTANT("instructions", "stream", data->cpu_track,
+        [&](perfetto::EventContext ctx) {
+            auto *qemu_arg = ctx.event()->set_qemu();
+            auto *instr = qemu_arg->set_instr();
+            auto flags =  perfetto_log_entry_flags(entry);
+
+            /*
+             * Populate protobuf from internal qemu structure.
+             * XXX-AM: It would be very nice if we could somehow skip this step
+             * and have the qemu internal representation be protobuf-based, so
+             * that here we only have to embed the qemu-specific packet
+             * into the track_event. Ideally this would save us all this copying around of
+             * the event data but I have no clue about the implications for the stability
+             * of the C/C++ protobuf-generated structures or whether we can embed the
+             * qemu portion of the packet already in the serialized wire-format
+             * (which should be stable by design).
+             * This can probably be done via protozero::Message::AppendScatteredBytes().
+             */
+
+            if (flags & LI_FLAG_HAS_INSTR_DATA) {
+                const char *bytes = perfetto_log_entry_insn_bytes(entry);
+                int size = perfetto_log_entry_insn_size(entry);
+                int nitems;
+                std::stringstream ss;
+
+                // XXX-AM: We can not use a bytes field in the protobuf because perfetto lacks
+                // support. This is slightly sad as this is an high-frequency event.
+                for (int i = 0; i < size; i++) {
+                    ss << std::hex << std::setw(2) << std::setfill('0') <<
+                            (static_cast<unsigned int>(bytes[i]) & 0xff) << " ";
+                }
+                instr->set_opcode(ss.str());
+                instr->set_pc(perfetto_log_entry_pc(entry));
+
+                nitems = perfetto_log_entry_regs(entry);
+                for (int i = 0; i < nitems; i++) {
+                    auto flags = perfetto_reg_info_flags(entry, i);
+                    auto *reginfo = instr->add_reg();
+                    reginfo->set_name(perfetto_reg_info_name(entry, i));
+                    if ((flags & LRI_CAP_REG) && (flags & LRI_HOLDS_CAP)) {
+                        cap_register_handle cap_handle = perfetto_reg_info_cap(entry, i);
+                        auto *capinfo = reginfo->set_cap_val();
+                        trace_cap_register(capinfo, cap_handle);
+                    } else {
+                        reginfo->set_int_val(perfetto_reg_info_gpr(entry, i));
+                    }
+                }
+                nitems = perfetto_log_entry_mem(entry);
+                for (int i = 0; i < nitems; i++) {
+                    auto flags = perfetto_mem_info_flags(entry, i);
+                    auto *meminfo = instr->add_mem();
+                    meminfo->set_addr(perfetto_mem_info_addr(entry, i));
+                    switch (flags) {
+                        case LMI_LD:
+                            meminfo->set_op(perfetto::protos::pbzero::MemInfo::LOAD);
+                            break;
+                        case LMI_LD | LMI_CAP:
+                            meminfo->set_op(perfetto::protos::pbzero::MemInfo::CLOAD);
+                            break;
+                        case LMI_ST:
+                            meminfo->set_op(perfetto::protos::pbzero::MemInfo::STORE);
+                            break;
+                        case LMI_ST | LMI_CAP:
+                            meminfo->set_op(perfetto::protos::pbzero::MemInfo::CSTORE);
+                            break;
+                        default:
+                            // XXX Notify error somehow?
+                            break;
+                    }
+                    if (flags & LMI_CAP) {
+                        auto *capinfo = meminfo->set_cap_val();
+                        cap_register_handle cap_handle = perfetto_reg_info_cap(entry, i);
+                        trace_cap_register(capinfo, cap_handle);
+                    } else {
+                        meminfo->set_int_val(perfetto_mem_info_value(entry, i));
+                    }
+                }
+            }
+            if (flags & LI_FLAG_INTR_MASK) {
+                // interrupt
+                auto *trap = instr->set_trap();
+                if (flags & LI_FLAG_INTR_TRAP)
+                    trap->set_type(perfetto::protos::pbzero::Trap::EXCEPTION);
+                else {
+                    trap->set_type(perfetto::protos::pbzero::Trap::INTERRUPT);
+                }
+                trap->set_trap_number(perfetto_log_entry_intr_code(entry));
+            }
+            if (flags & LI_FLAG_MODE_SWITCH) {
+                auto mode = qemu_cpu_mode_to_trace(perfetto_log_entry_next_cpu_mode(entry));
+                instr->set_mode(mode);
+            }
+        });
 
-    return perfetto_start_tracing();
+    process_extra_events(data, entry);
 }
diff --git a/trace_extra/trace_stats.cc b/trace_extra/trace_stats.cc
index 71ea5be788e..fce9ba1febe 100644
--- a/trace_extra/trace_stats.cc
+++ b/trace_extra/trace_stats.cc
@@ -26,116 +26,72 @@
  * @BERI_LICENSE_HEADER_END@
  */
 
-#include "exec/log_instr_stats.h"
-
-#include <map>
-#include <iostream>
 #include <utility>
-#include <boost/iostreams/stream.hpp>
-#include <boost/iostreams/device/file_descriptor.hpp>
-#include <boost/icl/interval_map.hpp>
+#include <cstdint>
+#include "qemu/log_instr.h"
+#include "exec/log_instr_perfetto.h"
+#include "trace_extra/trace_stats.hh"
 
 namespace icl = boost::icl;
-namespace io = boost::iostreams;
 
-using addr_range_imap = icl::interval_map<uint64_t, uint64_t>;
 using addr_range = icl::interval<uint64_t>;
 
-namespace
+namespace cheri
 {
 
-addr_range::interval_type make_addr_range(uint64_t start, uint64_t end)
+void qemu_stats::flush(perfetto::Track &track)
 {
-    return addr_range::right_open(start, end);
+    /*
+     * XXX-AM: Note: to avoid big packets we should probably write
+     * the histogram samples separate from the hisogram descriptor and
+     * tie them toghether in post-processing.
+     * This would likely require specialized packet processing.
+     */
+    TRACE_EVENT_INSTANT("stats", "bb_hist", track,
+                        [&](perfetto::EventContext ctx) {
+                            auto *qemu_arg = ctx.event()->set_qemu();
+                            auto *hist = qemu_arg->set_histogram();
+                            for (const auto &keyval : bb_hit_map) {
+                                auto &range = keyval.first;
+                                auto *bucket = hist->add_bucket();
+                                bucket->set_start(range.lower());
+                                bucket->set_end(range.upper());
+                                bucket->set_value(keyval.second);
+                            }
+                        });
+
+    TRACE_EVENT_INSTANT("stats", "branch_hist", track,
+                        [&](perfetto::EventContext ctx) {
+                            auto *qemu_arg = ctx.event()->set_qemu();
+                            auto *hist = qemu_arg->set_histogram();
+                            for (const auto &keyval : branch_map) {
+                                auto *bucket = hist->add_bucket();
+                                bucket->set_start(keyval.first);
+                                bucket->set_value(keyval.second);
+                            }
+                        });
+    clear();
 }
 
-struct QEMUStats {
-    addr_range_imap call_map;
-    addr_range_imap bb_hit_map;
-    int cpu_id;
-
-    QEMUStats(int cpu_id) : cpu_id(cpu_id) {}
-
-    void record_bb_hit(uint64_t start, uint64_t end);
-    void record_call(uint64_t addr, uint64_t link);
-    void dump(int fd, bool csv_header);
-    void clear();
-};
-
-QEMUStats &handle2stats(qemu_cpu_stats_t handle)
-{
-    return *reinterpret_cast<QEMUStats *>(handle);
-}
-
-void QEMUStats::dump(int fd, bool csv_header)
-{
-    io::file_descriptor_sink fd_sink(fd, io::never_close_handle);
-    io::stream<io::file_descriptor_sink> ostream(fd_sink);
-
-    if (csv_header)
-        ostream << "CPU,start,end,count" << std::endl;
-    for (const auto &keyval : bb_hit_map) {
-        auto &range = keyval.first;
-        ostream << cpu_id << "," << std::hex << range.lower() << ","
-                << range.upper() << "," << std::dec << keyval.second
-                << std::endl;
-    }
-}
-
-void QEMUStats::clear()
+void qemu_stats::clear()
 {
     bb_hit_map.clear();
-    call_map.clear();
-}
-
-void QEMUStats::record_bb_hit(uint64_t start, uint64_t end)
-{
-    bb_hit_map += std::make_pair(make_addr_range(start, end), 1UL);
-}
-
-void QEMUStats::record_call(uint64_t addr, uint64_t link)
-{
-    call_map += std::make_pair(make_addr_range(addr, link), 1UL);
+    branch_map.clear();
 }
 
-} // namespace
-
-/*
- * C API wrappers to the stats class.
- */
-extern "C" {
-
-qemu_cpu_stats_t qemu_cpu_stats_create(int cpu_id)
+void qemu_stats::process_next_pc(uint64_t pc, int insn_size)
 {
-    auto *stats = new QEMUStats(cpu_id);
-    return reinterpret_cast<qemu_cpu_stats_t>(stats);
-}
-
-void qemu_cpu_stats_destroy(qemu_cpu_stats_t handle)
-{
-    auto stats = &handle2stats(handle);
-    delete stats;
-}
-
-void qemu_cpu_stats_dump(qemu_cpu_stats_t handle, int fd, bool csv_header)
-{
-    auto stats = handle2stats(handle);
-    stats.dump(fd, csv_header);
-    stats.clear();
-}
-
-void qemu_cpu_stats_record_bb_hit(qemu_cpu_stats_t handle, uint64_t start,
-                                  uint64_t end)
-{
-    auto stats = handle2stats(handle);
-    stats.record_bb_hit(start, end);
-}
-
-void qemu_cpu_stats_record_call(qemu_cpu_stats_t handle, uint64_t addr,
-                                uint64_t link)
-{
-    auto stats = handle2stats(handle);
-    stats.record_call(addr, link);
+    if (pc == 0) {
+        pc_range_start = pc;
+        last_pc = pc;
+    } else if (pc - last_pc > insn_size) {
+        // presume we are branching
+        bb_hit_map += std::make_pair(
+            addr_range::right_open(pc_range_start, last_pc), 1UL);
+        pc_range_start = pc;
+        branch_map[pc] += 1;
+    }
+    last_pc = pc;
 }
 
-} /* C */
+} // namespace cheri
diff --git a/include/exec/log_instr_stats.h b/trace_extra/trace_stats.hh
similarity index 60%
rename from include/exec/log_instr_stats.h
rename to trace_extra/trace_stats.hh
index d544e856eec..f7805143e87 100644
--- a/include/exec/log_instr_stats.h
+++ b/trace_extra/trace_stats.hh
@@ -26,33 +26,41 @@
  * @BERI_LICENSE_HEADER_END@
  */
 
-#pragma once
+/*
+ * Internal instruction logging metric aggregation
+ */
 
-#include <stdint.h>
-#include <stdbool.h>
+#pragma once
 
-#ifdef __cplusplus
-extern "C" {
-#endif
+#include <map>
+#include <perfetto.h>
+#include <boost/icl/interval_map.hpp>
 
-#ifdef CONFIG_TRACE_STATS
-struct qemu_cpu_stats;
-typedef struct qemu_cpu_stats *qemu_cpu_stats_t;
+namespace cheri
+{
+class qemu_stats
+{
+    /*
+     * Histogram of basic-block hit during tracing.
+     */
+    boost::icl::interval_map<uint64_t, uint64_t> bb_hit_map;
+    /*
+     * Histogram of branch instruction addresses.
+     * Post-processing must be used to distinguish between function
+     * calls and conditional branches.
+     */
+    std::map<uint64_t, uint64_t> branch_map;
+    /*
+     *  PC-tracking info to detect branches.
+     */
+    uint64_t last_pc;
+    uint64_t pc_range_start;
 
-/*
- * Interface to manipulate an interval histogram representing a count
- * over address ranges.
- * The pointer returned is an opaque handle.
- */
-qemu_cpu_stats_t qemu_cpu_stats_create(int cpu_id);
-void qemu_cpu_stats_destroy(qemu_cpu_stats_t handle);
-void qemu_cpu_stats_dump(qemu_cpu_stats_t handle, int fd, bool csv_header);
-void qemu_cpu_stats_record_bb_hit(qemu_cpu_stats_t handle, uint64_t start,
-                                  uint64_t end);
-void qemu_cpu_stats_record_call(qemu_cpu_stats_t handle, uint64_t addr,
-                                uint64_t link);
+  public:
+    qemu_stats() : last_pc(0), pc_range_start(0) {}
+    void process_next_pc(uint64_t pc, int insn_size);
+    void flush(perfetto::Track &track);
+    void clear();
+};
 
-#endif /* CONFIG_TRACE_STATS */
-#ifdef __cplusplus
-}
-#endif
+} // namespace cheri
diff --git a/trace_extra/track.hh b/trace_extra/track.hh
deleted file mode 100644
index e99622417df..00000000000
--- a/trace_extra/track.hh
+++ /dev/null
@@ -1,41 +0,0 @@
-/*-
- * SPDX-License-Identifier: BSD-2-Clause
- *
- * Copyright (c) 2021 Alfredo Mazzinghi
- *
- * This software was developed by SRI International and the University of
- * Cambridge Computer Laboratory (Department of Computer Science and
- * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the
- * DARPA SSITH research programme.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#pragma once
-
-#include <perfetto.h>
-
-PERFETTO_DEFINE_CATEGORIES(
-    perfetto::Category("instructions")
-    .SetDescription("CPU instructions executed"),
-    perfetto::Category("memory")
-    .SetDescription("Tag and memory load/store"));

From 5b961f1ba89ebcebf7d6d35805d8e483332cf3e9 Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Wed, 22 Sep 2021 15:51:42 +0100
Subject: [PATCH 16/74] Fix flushing in instruction logging perfetto backend.

Flush TrackEvent data source when we receive a FLUSH event in the instruction
logging subsystem. This helps making sure that we do not have lost events when
running multiple tests on qemu.
---
 trace_extra/trace_perfetto.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/trace_extra/trace_perfetto.cc b/trace_extra/trace_perfetto.cc
index 5443beff6a5..3f873311181 100644
--- a/trace_extra/trace_perfetto.cc
+++ b/trace_extra/trace_perfetto.cc
@@ -194,8 +194,9 @@ process_extra_events(perfetto_backend_data *data, cpu_log_entry_handle entry)
         if (evt->id == LOG_EVENT_STATE) {
             switch(evt->state.next_state) {
                 case LOG_EVENT_STATE_FLUSH:
+                    // XXX-AM: Emitting stats and TrackEvent flush should be two separate events TBH.
                     data->stats.flush(data->cpu_track);
-                    //perfetto::TrackEvent::Flush();
+                    perfetto::TrackEvent::Flush();
                     break;
                 case LOG_EVENT_STATE_START:
                     TRACE_EVENT_BEGIN("ctrl", "tracing", data->cpu_ctrl_track);

From 70952baa25de46d789cb4010689baa37a3f90171 Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Thu, 30 Sep 2021 19:01:46 +0100
Subject: [PATCH 17/74] Disable the perfetto tracing backend by default.

Write perfetto configuration status when running configure.
Enable perfetto on Linux CI builds.
---
 Jenkinsfile | 3 +++
 configure   | 2 +-
 meson.build | 1 +
 3 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 6c40e9112a3..d61c587314e 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -133,6 +133,9 @@ selectedConfigs.each { config ->
             if (isDebug) {
                 extraQemuArgs = '--qemu/configure-options=--enable-rvfi-dii --qemu/build-type=Debug --qemu/use-asan'
             }
+            if (os == "linux") {
+               extraQemuArgs += ' --qemu/configure-options=--enable-perfetto-log-instr'
+            }
             def qemuResult = cheribuildProject(target: 'qemu', cpu: 'native', skipArtifacts: true,
                     nodeLabel: null,
                     extraArgs: "--without-sdk --install-prefix=/usr $extraQemuArgs",
diff --git a/configure b/configure
index c2e01faed41..d897d6eaa3f 100755
--- a/configure
+++ b/configure
@@ -340,7 +340,7 @@ pvrdma=""
 gprof="no"
 debug_tcg="no"
 tcg_log_instr="yes"
-tcg_log_instr_perfetto="yes"
+tcg_log_instr_perfetto="no"
 rvfi_dii="no"
 debug="no"
 sanitizers="no"
diff --git a/meson.build b/meson.build
index abfbe926199..8f228da301c 100644
--- a/meson.build
+++ b/meson.build
@@ -2181,6 +2181,7 @@ if config_all.has_key('CONFIG_TCG')
   summary_info += {'TCG debug enabled': config_host.has_key('CONFIG_DEBUG_TCG')}
   summary_info += {'TCG interpreter':   config_host.has_key('CONFIG_TCG_INTERPRETER')}
   summary_info += {'TCG instruction logging': config_host.has_key('CONFIG_TCG_LOG_INSTR')}
+  summary_info += {'TCG instruction logging with google-perfetto': config_host.has_key('CONFIG_TRACE_PERFETTO')}
 endif
 summary_info += {'RVFI-DII':            config_host.has_key('CONFIG_RVFI_DII')}
 summary_info += {'malloc trim support': has_malloc_trim}

From 061f7554dd2e1b059559625d079ec240bdb20cfd Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Mon, 4 Oct 2021 14:23:03 +0100
Subject: [PATCH 18/74] Fix tracking of the current basic-block in perfetto
 trace stats.

Reset state when tracing is started and stopped.
---
 trace_extra/trace_perfetto.cc | 224 ++++++++++++++++++----------------
 trace_extra/trace_stats.cc    |  25 +++-
 trace_extra/trace_stats.hh    |   5 +-
 3 files changed, 147 insertions(+), 107 deletions(-)

diff --git a/trace_extra/trace_perfetto.cc b/trace_extra/trace_perfetto.cc
index 3f873311181..c74b8b4ed12 100644
--- a/trace_extra/trace_perfetto.cc
+++ b/trace_extra/trace_perfetto.cc
@@ -33,7 +33,7 @@
 /*
  * QEMU instruction logging bridge to the perfetto framework.
  */
-
+#include <cassert>
 #include <memory>
 #include <mutex>
 #include <sstream>
@@ -136,7 +136,7 @@ bool perfetto_start_tracing(void)
     ds_cfg->set_track_event_config_raw(track_cfg.SerializeAsString());
     auto *producer_cfg = cfg.add_producers();
     producer_cfg->set_producer_name("qemu-tcg");
-    producer_cfg->set_shm_size_kb(1 << 11); // 2MiB
+    producer_cfg->set_shm_size_kb(1 << 11); // 2MiB XXX need more space
 
     session = perfetto::Tracing::NewTrace();
 
@@ -181,13 +181,8 @@ trace_cap_register(perfetto::protos::pbzero::Capability *cap,
 }
 
 void
-process_extra_events(perfetto_backend_data *data, cpu_log_entry_handle entry)
+process_events(perfetto_backend_data *data, cpu_log_entry_handle entry)
 {
-    /* Track executed instruction PC to track histograms */
-    uint64_t pc = perfetto_log_entry_pc(entry);
-    int size = perfetto_log_entry_insn_size(entry);
-    data->stats.process_next_pc(pc, size);
-
     int nevents = perfetto_log_entry_events(entry);
     for (int i = 0; i < nevents; i++) {
         log_event_t *evt = perfetto_log_event(entry, i);
@@ -195,13 +190,16 @@ process_extra_events(perfetto_backend_data *data, cpu_log_entry_handle entry)
             switch(evt->state.next_state) {
                 case LOG_EVENT_STATE_FLUSH:
                     // XXX-AM: Emitting stats and TrackEvent flush should be two separate events TBH.
+                    TRACE_EVENT_INSTANT("ctrl", "flush", data->cpu_ctrl_track);
                     data->stats.flush(data->cpu_track);
                     perfetto::TrackEvent::Flush();
                     break;
                 case LOG_EVENT_STATE_START:
+                    data->stats.unpause(entry);
                     TRACE_EVENT_BEGIN("ctrl", "tracing", data->cpu_ctrl_track);
                     break;
                 case LOG_EVENT_STATE_STOP:
+                    data->stats.pause(entry);
                     TRACE_EVENT_END("ctrl", data->cpu_ctrl_track);
                     break;
             }
@@ -209,6 +207,115 @@ process_extra_events(perfetto_backend_data *data, cpu_log_entry_handle entry)
     }
 }
 
+void
+process_instr(perfetto_backend_data *data, cpu_log_entry_handle entry)
+{
+    /*
+     * If we have instruction data, we assume that tracing is enabled and run extra
+     * stats gathering to track the executed instruction PC in the stats histograms.
+     * This avoids having to keep a shadow copy of the tracing state in the backend.
+     */
+    data->stats.process_instr(entry);
+
+    /*
+     * XXX-AM: instead of having one big instruction recor, we may have different messages for
+     * optional parts of the instruction message, on the same track/category: e.g. mode swtich,
+     * interrupt information and modified registers?
+     */
+    TRACE_EVENT_INSTANT("instructions", "stream", data->cpu_track,
+        [&](perfetto::EventContext ctx) {
+            auto *qemu_arg = ctx.event()->set_qemu();
+            auto *instr = qemu_arg->set_instr();
+            auto flags =  perfetto_log_entry_flags(entry);
+
+            /*
+             * Populate protobuf from internal qemu structure.
+             * XXX-AM: It would be very nice if we could somehow skip this step
+             * and have the qemu internal representation be protobuf-based, so
+             * that here we only have to embed the qemu-specific packet
+             * into the track_event. Ideally this would save us all this copying around of
+             * the event data but I have no clue about the implications for the stability
+             * of the C/C++ protobuf-generated structures or whether we can embed the
+             * qemu portion of the packet already in the serialized wire-format
+             * (which should be stable by design).
+             * This can probably be done via protozero::Message::AppendScatteredBytes().
+             */
+            assert(flags & LI_FLAG_HAS_INSTR_DATA);
+            const char *bytes = perfetto_log_entry_insn_bytes(entry);
+            int size = perfetto_log_entry_insn_size(entry);
+            int nitems;
+            std::stringstream ss;
+
+            // XXX-AM: We can not use a bytes field in the protobuf because perfetto lacks
+            // support. This is slightly sad as this is an high-frequency event.
+            for (int i = 0; i < size; i++) {
+                ss << std::hex << std::setw(2) << std::setfill('0') <<
+                        (static_cast<unsigned int>(bytes[i]) & 0xff) << " ";
+            }
+            instr->set_opcode(ss.str());
+            instr->set_pc(perfetto_log_entry_pc(entry));
+
+            nitems = perfetto_log_entry_regs(entry);
+            for (int i = 0; i < nitems; i++) {
+                auto flags = perfetto_reg_info_flags(entry, i);
+                auto *reginfo = instr->add_reg();
+                reginfo->set_name(perfetto_reg_info_name(entry, i));
+                if ((flags & LRI_CAP_REG) && (flags & LRI_HOLDS_CAP)) {
+                    cap_register_handle cap_handle = perfetto_reg_info_cap(entry, i);
+                    auto *capinfo = reginfo->set_cap_val();
+                    trace_cap_register(capinfo, cap_handle);
+                } else {
+                    reginfo->set_int_val(perfetto_reg_info_gpr(entry, i));
+                }
+            }
+            nitems = perfetto_log_entry_mem(entry);
+            for (int i = 0; i < nitems; i++) {
+                auto flags = perfetto_mem_info_flags(entry, i);
+                auto *meminfo = instr->add_mem();
+                meminfo->set_addr(perfetto_mem_info_addr(entry, i));
+                switch (flags) {
+                    case LMI_LD:
+                        meminfo->set_op(perfetto::protos::pbzero::MemInfo::LOAD);
+                        break;
+                    case LMI_LD | LMI_CAP:
+                        meminfo->set_op(perfetto::protos::pbzero::MemInfo::CLOAD);
+                        break;
+                    case LMI_ST:
+                        meminfo->set_op(perfetto::protos::pbzero::MemInfo::STORE);
+                        break;
+                    case LMI_ST | LMI_CAP:
+                        meminfo->set_op(perfetto::protos::pbzero::MemInfo::CSTORE);
+                        break;
+                    default:
+                        // XXX Notify error somehow?
+                        break;
+                }
+                if (flags & LMI_CAP) {
+                    auto *capinfo = meminfo->set_cap_val();
+                    cap_register_handle cap_handle = perfetto_reg_info_cap(entry, i);
+                    trace_cap_register(capinfo, cap_handle);
+                } else {
+                    meminfo->set_int_val(perfetto_mem_info_value(entry, i));
+                }
+            }
+
+            if (flags & LI_FLAG_INTR_MASK) {
+                // interrupt
+                auto *trap = instr->set_trap();
+                if (flags & LI_FLAG_INTR_TRAP)
+                    trap->set_type(perfetto::protos::pbzero::Trap::EXCEPTION);
+                else {
+                    trap->set_type(perfetto::protos::pbzero::Trap::INTERRUPT);
+                }
+                trap->set_trap_number(perfetto_log_entry_intr_code(entry));
+            }
+            if (flags & LI_FLAG_MODE_SWITCH) {
+                auto mode = qemu_cpu_mode_to_trace(perfetto_log_entry_next_cpu_mode(entry));
+                instr->set_mode(mode);
+            }
+        });
+}
+
 } /* anonymous */
 
 extern "C" void qemu_log_instr_perfetto_conf_logfile(const char *name)
@@ -262,100 +369,11 @@ extern "C" void perfetto_sync_cpu(void *backend_data)
 extern "C" void perfetto_emit_instr(void *backend_data, cpu_log_entry_handle entry)
 {
     auto *data = reinterpret_cast<perfetto_backend_data *>(backend_data);
+    auto flags = perfetto_log_entry_flags(entry);
 
-    TRACE_EVENT_INSTANT("instructions", "stream", data->cpu_track,
-        [&](perfetto::EventContext ctx) {
-            auto *qemu_arg = ctx.event()->set_qemu();
-            auto *instr = qemu_arg->set_instr();
-            auto flags =  perfetto_log_entry_flags(entry);
-
-            /*
-             * Populate protobuf from internal qemu structure.
-             * XXX-AM: It would be very nice if we could somehow skip this step
-             * and have the qemu internal representation be protobuf-based, so
-             * that here we only have to embed the qemu-specific packet
-             * into the track_event. Ideally this would save us all this copying around of
-             * the event data but I have no clue about the implications for the stability
-             * of the C/C++ protobuf-generated structures or whether we can embed the
-             * qemu portion of the packet already in the serialized wire-format
-             * (which should be stable by design).
-             * This can probably be done via protozero::Message::AppendScatteredBytes().
-             */
-
-            if (flags & LI_FLAG_HAS_INSTR_DATA) {
-                const char *bytes = perfetto_log_entry_insn_bytes(entry);
-                int size = perfetto_log_entry_insn_size(entry);
-                int nitems;
-                std::stringstream ss;
-
-                // XXX-AM: We can not use a bytes field in the protobuf because perfetto lacks
-                // support. This is slightly sad as this is an high-frequency event.
-                for (int i = 0; i < size; i++) {
-                    ss << std::hex << std::setw(2) << std::setfill('0') <<
-                            (static_cast<unsigned int>(bytes[i]) & 0xff) << " ";
-                }
-                instr->set_opcode(ss.str());
-                instr->set_pc(perfetto_log_entry_pc(entry));
-
-                nitems = perfetto_log_entry_regs(entry);
-                for (int i = 0; i < nitems; i++) {
-                    auto flags = perfetto_reg_info_flags(entry, i);
-                    auto *reginfo = instr->add_reg();
-                    reginfo->set_name(perfetto_reg_info_name(entry, i));
-                    if ((flags & LRI_CAP_REG) && (flags & LRI_HOLDS_CAP)) {
-                        cap_register_handle cap_handle = perfetto_reg_info_cap(entry, i);
-                        auto *capinfo = reginfo->set_cap_val();
-                        trace_cap_register(capinfo, cap_handle);
-                    } else {
-                        reginfo->set_int_val(perfetto_reg_info_gpr(entry, i));
-                    }
-                }
-                nitems = perfetto_log_entry_mem(entry);
-                for (int i = 0; i < nitems; i++) {
-                    auto flags = perfetto_mem_info_flags(entry, i);
-                    auto *meminfo = instr->add_mem();
-                    meminfo->set_addr(perfetto_mem_info_addr(entry, i));
-                    switch (flags) {
-                        case LMI_LD:
-                            meminfo->set_op(perfetto::protos::pbzero::MemInfo::LOAD);
-                            break;
-                        case LMI_LD | LMI_CAP:
-                            meminfo->set_op(perfetto::protos::pbzero::MemInfo::CLOAD);
-                            break;
-                        case LMI_ST:
-                            meminfo->set_op(perfetto::protos::pbzero::MemInfo::STORE);
-                            break;
-                        case LMI_ST | LMI_CAP:
-                            meminfo->set_op(perfetto::protos::pbzero::MemInfo::CSTORE);
-                            break;
-                        default:
-                            // XXX Notify error somehow?
-                            break;
-                    }
-                    if (flags & LMI_CAP) {
-                        auto *capinfo = meminfo->set_cap_val();
-                        cap_register_handle cap_handle = perfetto_reg_info_cap(entry, i);
-                        trace_cap_register(capinfo, cap_handle);
-                    } else {
-                        meminfo->set_int_val(perfetto_mem_info_value(entry, i));
-                    }
-                }
-            }
-            if (flags & LI_FLAG_INTR_MASK) {
-                // interrupt
-                auto *trap = instr->set_trap();
-                if (flags & LI_FLAG_INTR_TRAP)
-                    trap->set_type(perfetto::protos::pbzero::Trap::EXCEPTION);
-                else {
-                    trap->set_type(perfetto::protos::pbzero::Trap::INTERRUPT);
-                }
-                trap->set_trap_number(perfetto_log_entry_intr_code(entry));
-            }
-            if (flags & LI_FLAG_MODE_SWITCH) {
-                auto mode = qemu_cpu_mode_to_trace(perfetto_log_entry_next_cpu_mode(entry));
-                instr->set_mode(mode);
-            }
-        });
-
-    process_extra_events(data, entry);
+    /* Process events first to react to START/STOP tracing events if needed */
+    process_events(data, entry);
+    if (flags & LI_FLAG_HAS_INSTR_DATA) {
+        process_instr(data, entry);
+    }
 }
diff --git a/trace_extra/trace_stats.cc b/trace_extra/trace_stats.cc
index fce9ba1febe..01240aebd7f 100644
--- a/trace_extra/trace_stats.cc
+++ b/trace_extra/trace_stats.cc
@@ -29,7 +29,6 @@
 #include <utility>
 #include <cstdint>
 #include "qemu/log_instr.h"
-#include "exec/log_instr_perfetto.h"
 #include "trace_extra/trace_stats.hh"
 
 namespace icl = boost::icl;
@@ -70,21 +69,41 @@ void qemu_stats::flush(perfetto::Track &track)
                                 bucket->set_value(keyval.second);
                             }
                         });
+
     clear();
 }
 
+void qemu_stats::pause(uint64_t pc)
+{
+    /*
+     * XXX should we treat this as the end of the block? otherwise the
+     * current PC slice will be lost.
+     */
+    pc_range_start = last_pc = 0;
+}
+
+void qemu_stats::unpause(uint64_t pc)
+{
+    /* Reset the pc-tracking state */
+    pc_range_start = last_pc = pc;
+}
+
 void qemu_stats::clear()
 {
     bb_hit_map.clear();
     branch_map.clear();
 }
 
-void qemu_stats::process_next_pc(uint64_t pc, int insn_size)
+void qemu_stats::process_instr(perfetto::Track &ctx_track,
+                               cpu_log_entry_handle entry)
 {
+    uint64_t pc = perfetto_log_entry_pc(entry);
+    int size = perfetto_log_entry_insn_size(entry);
+
     if (pc == 0) {
         pc_range_start = pc;
         last_pc = pc;
-    } else if (pc - last_pc > insn_size) {
+    } else if (pc - last_pc > size) {
         // presume we are branching
         bb_hit_map += std::make_pair(
             addr_range::right_open(pc_range_start, last_pc), 1UL);
diff --git a/trace_extra/trace_stats.hh b/trace_extra/trace_stats.hh
index f7805143e87..1fa179907fc 100644
--- a/trace_extra/trace_stats.hh
+++ b/trace_extra/trace_stats.hh
@@ -35,6 +35,7 @@
 #include <map>
 #include <perfetto.h>
 #include <boost/icl/interval_map.hpp>
+#include "exec/log_instr_perfetto.h"
 
 namespace cheri
 {
@@ -58,7 +59,9 @@ class qemu_stats
 
   public:
     qemu_stats() : last_pc(0), pc_range_start(0) {}
-    void process_next_pc(uint64_t pc, int insn_size);
+    void process_instr(perfetto::Track &track, cpu_log_entry_handle entry);
+    void pause(uint64_t pc);
+    void unpause(uint64_t pc);
     void flush(perfetto::Track &track);
     void clear();
 };

From c532165bc45682e7cf2d7e95d92f5049cb9fb18e Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Fri, 8 Oct 2021 23:55:49 +0100
Subject: [PATCH 19/74] Modify meaning of the basic-block hit histogram tracing
 statistic

Record number of instructions hit for each basic-block instead of the
number of hits per basic-block.
---
 trace_extra/trace_stats.cc | 18 ++++++++++--------
 trace_extra/trace_stats.hh |  3 ++-
 2 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/trace_extra/trace_stats.cc b/trace_extra/trace_stats.cc
index 01240aebd7f..d568f0da899 100644
--- a/trace_extra/trace_stats.cc
+++ b/trace_extra/trace_stats.cc
@@ -76,16 +76,19 @@ void qemu_stats::flush(perfetto::Track &track)
 void qemu_stats::pause(uint64_t pc)
 {
     /*
-     * XXX should we treat this as the end of the block? otherwise the
+     * Treat this as the end of the block, otherwise the
      * current PC slice will be lost.
      */
-    pc_range_start = last_pc = 0;
+    bb_hit_map +=
+        std::make_pair(addr_range::right_open(pc_range_start, pc), icount);
+    pc_range_start = last_pc = icount = 0;
 }
 
 void qemu_stats::unpause(uint64_t pc)
 {
     /* Reset the pc-tracking state */
     pc_range_start = last_pc = pc;
+    icount = 0;
 }
 
 void qemu_stats::clear()
@@ -99,16 +102,15 @@ void qemu_stats::process_instr(perfetto::Track &ctx_track,
 {
     uint64_t pc = perfetto_log_entry_pc(entry);
     int size = perfetto_log_entry_insn_size(entry);
-
-    if (pc == 0) {
-        pc_range_start = pc;
-        last_pc = pc;
-    } else if (pc - last_pc > size) {
+    if (pc - last_pc > size) {
         // presume we are branching
         bb_hit_map += std::make_pair(
-            addr_range::right_open(pc_range_start, last_pc), 1UL);
+            addr_range::right_open(pc_range_start, last_pc), icount);
         pc_range_start = pc;
+        icount = 0;
         branch_map[pc] += 1;
+    } else {
+        icount += 1;
     }
     last_pc = pc;
 }
diff --git a/trace_extra/trace_stats.hh b/trace_extra/trace_stats.hh
index 1fa179907fc..0df34480084 100644
--- a/trace_extra/trace_stats.hh
+++ b/trace_extra/trace_stats.hh
@@ -56,9 +56,10 @@ class qemu_stats
      */
     uint64_t last_pc;
     uint64_t pc_range_start;
+    uint64_t icount;
 
   public:
-    qemu_stats() : last_pc(0), pc_range_start(0) {}
+    qemu_stats() : last_pc(0), pc_range_start(0), icount(0) {}
     void process_instr(perfetto::Track &track, cpu_log_entry_handle entry);
     void pause(uint64_t pc);
     void unpause(uint64_t pc);

From 466347fc590284576af131e5e1198b6bee47571d Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Wed, 13 Oct 2021 18:43:10 +0100
Subject: [PATCH 20/74] Fix use of icount in BB histogram

---
 trace_extra/trace_perfetto.cc |  6 +++---
 trace_extra/trace_stats.cc    | 15 +++++++++++----
 trace_extra/trace_stats.hh    |  4 ++--
 3 files changed, 16 insertions(+), 9 deletions(-)

diff --git a/trace_extra/trace_perfetto.cc b/trace_extra/trace_perfetto.cc
index c74b8b4ed12..7ffae28f4b7 100644
--- a/trace_extra/trace_perfetto.cc
+++ b/trace_extra/trace_perfetto.cc
@@ -195,11 +195,11 @@ process_events(perfetto_backend_data *data, cpu_log_entry_handle entry)
                     perfetto::TrackEvent::Flush();
                     break;
                 case LOG_EVENT_STATE_START:
-                    data->stats.unpause(entry);
+                    data->stats_.unpause(data->cpu_track, evt->state.pc);
                     TRACE_EVENT_BEGIN("ctrl", "tracing", data->cpu_ctrl_track);
                     break;
                 case LOG_EVENT_STATE_STOP:
-                    data->stats.pause(entry);
+                    data->stats_.pause(data->cpu_track, evt->state.pc);
                     TRACE_EVENT_END("ctrl", data->cpu_ctrl_track);
                     break;
             }
@@ -218,7 +218,7 @@ process_instr(perfetto_backend_data *data, cpu_log_entry_handle entry)
     data->stats.process_instr(entry);
 
     /*
-     * XXX-AM: instead of having one big instruction recor, we may have different messages for
+     * XXX-AM: instead of having one big instruction record, we may have different messages for
      * optional parts of the instruction message, on the same track/category: e.g. mode swtich,
      * interrupt information and modified registers?
      */
diff --git a/trace_extra/trace_stats.cc b/trace_extra/trace_stats.cc
index d568f0da899..a6024d27e00 100644
--- a/trace_extra/trace_stats.cc
+++ b/trace_extra/trace_stats.cc
@@ -73,7 +73,7 @@ void qemu_stats::flush(perfetto::Track &track)
     clear();
 }
 
-void qemu_stats::pause(uint64_t pc)
+void qemu_stats::pause(perfetto::Track &track, uint64_t pc)
 {
     /*
      * Treat this as the end of the block, otherwise the
@@ -82,13 +82,19 @@ void qemu_stats::pause(uint64_t pc)
     bb_hit_map +=
         std::make_pair(addr_range::right_open(pc_range_start, pc), icount);
     pc_range_start = last_pc = icount = 0;
+    /*
+     * Whenever we pause tracing, we need to sync the stats to trace,
+     * otherwise we may charge the wrong context if tracing
+     * is restarted in another thread.
+     */
+    flush(track);
 }
 
-void qemu_stats::unpause(uint64_t pc)
+void qemu_stats::unpause(perfetto::Track &track, uint64_t pc)
 {
     /* Reset the pc-tracking state */
     pc_range_start = last_pc = pc;
-    icount = 0;
+    icount = 1; // Unpause is called for the first instruction being traced
 }
 
 void qemu_stats::clear()
@@ -107,7 +113,8 @@ void qemu_stats::process_instr(perfetto::Track &ctx_track,
         bb_hit_map += std::make_pair(
             addr_range::right_open(pc_range_start, last_pc), icount);
         pc_range_start = pc;
-        icount = 0;
+        icount =
+            1; // Reset icount triggered on the first instruction of the next BB
         branch_map[pc] += 1;
     } else {
         icount += 1;
diff --git a/trace_extra/trace_stats.hh b/trace_extra/trace_stats.hh
index 0df34480084..a9845121a6d 100644
--- a/trace_extra/trace_stats.hh
+++ b/trace_extra/trace_stats.hh
@@ -61,8 +61,8 @@ class qemu_stats
   public:
     qemu_stats() : last_pc(0), pc_range_start(0), icount(0) {}
     void process_instr(perfetto::Track &track, cpu_log_entry_handle entry);
-    void pause(uint64_t pc);
-    void unpause(uint64_t pc);
+    void pause(perfetto::Track &track, uint64_t pc);
+    void unpause(perfetto::Track &track, uint64_t pc);
     void flush(perfetto::Track &track);
     void clear();
 };

From bae544da36b58aeaaa267fedc42207516d54f886 Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Thu, 14 Oct 2021 01:21:17 +0100
Subject: [PATCH 21/74] Properly propagate the PC for tracing START/STOP
 events.

---
 accel/tcg/log_instr.c             | 134 +++++++++++++++++-------------
 include/exec/log_instr_internal.h |  10 +++
 2 files changed, 84 insertions(+), 60 deletions(-)

diff --git a/accel/tcg/log_instr.c b/accel/tcg/log_instr.c
index 1659fe625f6..794b1178800 100644
--- a/accel/tcg/log_instr.c
+++ b/accel/tcg/log_instr.c
@@ -201,11 +201,6 @@ static void do_instr_commit(CPUArchState *env)
         }
     }
 
-    if (cpulog->starting) {
-        cpulog->starting = false;
-        emit_start_event(entry, cpu_get_recent_pc(env));
-    }
-
     if (cpulog->flags & QEMU_LOG_INSTR_FLAG_BUFFERED) {
         cpulog->ring_head = (cpulog->ring_head + 1) % cpulog->instr_info->len;
         if (cpulog->ring_tail == cpulog->ring_head)
@@ -234,11 +229,15 @@ static void do_cpu_loglevel_switch(CPUState *cpu, run_on_cpu_data data)
     cpu_log_entry_t *entry = get_cpu_log_entry(env);
     qemu_log_instr_loglevel_t prev_level = cpulog->loglevel;
     bool prev_level_active = cpulog->loglevel_active;
-    qemu_log_instr_loglevel_t next_level = data.host_int;
+    /* qemu_log_instr_loglevel_t next_level = data.host_int; */
     bool next_level_active;
+    qemu_log_next_level_arg_t *arg = data.host_ptr;
+    target_ulong pc = (arg->global) ? cpu_get_recent_pc(env) : arg->pc;
+
+    log_assert(qemu_loglevel_mask(CPU_LOG_INSTR));
 
     /* Decide whether we have to pause/resume logging */
-    switch (next_level) {
+    switch (arg->next_level) {
     case QEMU_LOG_INSTR_LOGLEVEL_NONE:
         next_level_active = false;
         break;
@@ -262,23 +261,23 @@ static void do_cpu_loglevel_switch(CPUState *cpu, run_on_cpu_data data)
     }
 
     /* Update level */
-    cpulog->loglevel = next_level;
+    cpulog->loglevel = arg->next_level;
     cpulog->loglevel_active = next_level_active;
 
     /* Check if this was a no-op */
-    if (next_level == prev_level && prev_level_active == next_level_active)
-        return;
-
-    /* Flushing all translations makes things incredibly slow. Instead,
-     * we put whether tracing is currently enabled into cflags */
-
+    if (arg->next_level == prev_level &&
+        prev_level_active == next_level_active) {
+        goto done;
+    }
+    tb_flush(cpu);
     /* Emit start/stop events */
     if (prev_level_active) {
         if (cpulog->starting) {
             reset_log_buffer(cpulog, entry);
-            return;
+            goto done;
         }
-        emit_stop_event(entry, cpu_get_recent_pc(env));
+        /* emit_stop_event(entry, cpu_get_recent_pc(env)); */
+        emit_stop_event(entry, pc);
         do_instr_commit(env);
         /* Instruction commit may have advanced to the next entry buffer slot */
         entry = get_cpu_log_entry(env);
@@ -286,14 +285,27 @@ static void do_cpu_loglevel_switch(CPUState *cpu, run_on_cpu_data data)
     }
     if (next_level_active) {
         cpulog->starting = true;
+        /*
+         * Note: the start event is emitted by the first instruction being
+         * traced
+         */
+        emit_start_event(entry, pc);
     }
+
+done:
+    g_free(arg);
 }
 
-static void cpu_loglevel_switch(CPUArchState *env,
-                                qemu_log_instr_loglevel_t level)
+static void cpu_loglevel_switch(CPUArchState *env, target_ulong pc,
+                                qemu_log_instr_loglevel_t level, bool global)
 {
+    qemu_log_next_level_arg_t *arg = g_new(qemu_log_next_level_arg_t, 1);
+
+    arg->next_level = level;
+    arg->pc = pc;
+    arg->global = global;
     async_safe_run_on_cpu(env_cpu(env), do_cpu_loglevel_switch,
-                          RUN_ON_CPU_HOST_INT(level));
+                          RUN_ON_CPU_HOST_PTR(arg));
 }
 
 /* Start global logging flag if it was disabled */
@@ -310,10 +322,11 @@ static void global_loglevel_enable()
  */
 static void do_global_loglevel_switch(CPUState *cpu, run_on_cpu_data data)
 {
-    qemu_log_instr_loglevel_t level = data.host_int;
+    qemu_log_next_level_arg_t *arg = data.host_ptr;
 
-    if (level != QEMU_LOG_INSTR_LOGLEVEL_NONE)
+    if (arg->next_level != QEMU_LOG_INSTR_LOGLEVEL_NONE) {
         global_loglevel_enable();
+    }
     /*
      * TODO(am2419): To do things cleanly, we should clear the CPU_LOG_INSTR
      * flag when stopping, however to do this we would need to keep track
@@ -321,7 +334,7 @@ static void do_global_loglevel_switch(CPUState *cpu, run_on_cpu_data data)
      * clear the flag on the last CPU.
      * qemu_set_log_internal(qemu_loglevel & (~CPU_LOG_INSTR));
      */
-    do_cpu_loglevel_switch(cpu, RUN_ON_CPU_HOST_INT(level));
+    do_cpu_loglevel_switch(cpu, data);
 }
 
 /*
@@ -334,21 +347,16 @@ static void do_global_loglevel_switch(CPUState *cpu, run_on_cpu_data data)
 int qemu_log_instr_global_switch(int log_flags)
 {
     CPUState *cpu;
-    qemu_log_instr_loglevel_t level;
+    qemu_log_next_level_arg_t *arg = g_new(qemu_log_next_level_arg_t, 1);
 
-    if (log_flags & CPU_LOG_INSTR_U) {
-        level = QEMU_LOG_INSTR_LOGLEVEL_USER;
-        log_flags |= CPU_LOG_INSTR;
-    } else if (log_flags & CPU_LOG_INSTR) {
-        level = QEMU_LOG_INSTR_LOGLEVEL_ALL;
-    } else {
-        level = QEMU_LOG_INSTR_LOGLEVEL_NONE;
-    }
+    arg->next_level = (request_stop) ? QEMU_LOG_INSTR_LOGLEVEL_NONE
+                                     : QEMU_LOG_INSTR_LOGLEVEL_ALL;
+    arg->global = true;
 
     CPU_FOREACH(cpu)
     {
         async_safe_run_on_cpu(cpu, do_global_loglevel_switch,
-                              RUN_ON_CPU_HOST_INT(level));
+                              RUN_ON_CPU_HOST_PTR(arg));
     }
 
     return log_flags;
@@ -398,6 +406,7 @@ void qemu_log_instr_init(CPUState *cpu)
     cpu_log_instr_state_t *cpulog = &cpu->log_state;
     GArray *entry_ring = g_array_sized_new(false, true, sizeof(cpu_log_entry_t),
                                            reset_entry_buffer_size);
+    qemu_log_next_level_arg_t start_level;
     cpu_log_entry_t *entry;
     int i;
 
@@ -433,12 +442,11 @@ void qemu_log_instr_init(CPUState *cpu)
     }
 
     /* If we are starting with instruction logging enabled, switch it on now */
-    if (qemu_loglevel_mask(CPU_LOG_INSTR_U))
-        do_cpu_loglevel_switch(
-            cpu, RUN_ON_CPU_HOST_INT(QEMU_LOG_INSTR_LOGLEVEL_USER));
-    else if (qemu_loglevel_mask(CPU_LOG_INSTR))
-        do_cpu_loglevel_switch(
-            cpu, RUN_ON_CPU_HOST_INT(QEMU_LOG_INSTR_LOGLEVEL_ALL));
+    if (qemu_loglevel_mask(CPU_LOG_INSTR)) {
+        start_level.next_level = QEMU_LOG_INSTR_LOGLEVEL_ALL;
+        start_level.global = true;
+        do_cpu_loglevel_switch(cpu, RUN_ON_CPU_HOST_PTR(&start_level));
+    }
 
     if (reset_filters != NULL) {
         for (i = 0; i < reset_filters->len; i++) {
@@ -548,12 +556,13 @@ void qemu_log_instr_mode_switch(CPUArchState *env,
 
     /* If we are not logging in user-only mode, bail */
     if (!qemu_loglevel_mask(CPU_LOG_INSTR) ||
-        cpulog->loglevel != QEMU_LOG_INSTR_LOGLEVEL_USER)
+        cpulog->loglevel != QEMU_LOG_INSTR_LOGLEVEL_USER) {
         return;
+    }
 
     /* Check if we are switching to an interesting mode */
     if ((mode == QEMU_LOG_INSTR_CPU_USER) != cpulog->loglevel_active) {
-        cpu_loglevel_switch(env, cpulog->loglevel);
+        cpu_loglevel_switch(env, pc, cpulog->loglevel, false);
     }
 }
 
@@ -1193,8 +1202,9 @@ void helper_qemu_log_instr_buffer_flush(CPUArchState *env)
     qemu_log_instr_flush(env);
 }
 
-/* Start logging all instructions on the current CPU */
-void helper_qemu_log_instr_start(CPUArchState *env, target_ulong pc)
+static void do_qemu_log_instr_start(CPUArchState *env, target_ulong pc,
+                                    qemu_log_instr_loglevel_t level,
+                                    bool global)
 {
     cpu_log_instr_state_t *cpulog = get_cpu_log_state(env);
 
@@ -1202,33 +1212,35 @@ void helper_qemu_log_instr_start(CPUArchState *env, target_ulong pc)
     global_loglevel_enable();
 
     /* If we are already started in the correct mode, bail */
-    if (cpulog->loglevel == QEMU_LOG_INSTR_LOGLEVEL_ALL &&
-        cpulog->loglevel_active)
+    if (cpulog->loglevel == level && cpulog->loglevel_active) {
         return;
+    }
 
-    cpu_loglevel_switch(env, QEMU_LOG_INSTR_LOGLEVEL_ALL);
+    cpu_loglevel_switch(env, pc, level, global);
 }
 
-/* Start logging user-only instructions on the current CPU */
-void helper_qemu_log_instr_user_start(CPUArchState *env, target_ulong pc)
+static void do_qemu_log_instr_stop(CPUArchState *env, target_ulong pc,
+                                   bool global)
 {
-    cpu_log_instr_state_t *cpulog = get_cpu_log_state(env);
-
-    log_assert(cpulog != NULL && "Invalid log state");
-    global_loglevel_enable();
+    cpu_loglevel_switch(env, pc, QEMU_LOG_INSTR_LOGLEVEL_NONE, global);
+}
 
-    /* If we are already in the correct mode, bail */
-    if (cpulog->loglevel == QEMU_LOG_INSTR_LOGLEVEL_USER)
-        return;
+/* Start logging all instructions on the current CPU */
+void helper_qemu_log_instr_start(CPUArchState *env, target_ulong pc)
+{
+    do_qemu_log_instr_start(env, pc, QEMU_LOG_INSTR_LOGLEVEL_ALL, false);
+}
 
-    cpu_loglevel_switch(env, QEMU_LOG_INSTR_LOGLEVEL_USER);
+/* Start logging user-only instructions on the current CPU */
+void helper_qemu_log_instr_user_start(CPUArchState *env, target_ulong pc)
+{
+    do_qemu_log_instr_start(env, pc, QEMU_LOG_INSTR_LOGLEVEL_USER, false);
 }
 
 /* Stop logging on the current CPU */
 void helper_qemu_log_instr_stop(CPUArchState *env, target_ulong pc)
 {
-
-    cpu_loglevel_switch(env, QEMU_LOG_INSTR_LOGLEVEL_NONE);
+    do_qemu_log_instr_stop(env, pc, false);
 }
 
 /* Start logging all instructions on all CPUs */
@@ -1238,7 +1250,8 @@ void helper_qemu_log_instr_allcpu_start()
 
     CPU_FOREACH(cpu)
     {
-        helper_qemu_log_instr_start(cpu->env_ptr, 0);
+        do_qemu_log_instr_start(cpu->env_ptr, 0, QEMU_LOG_INSTR_LOGLEVEL_ALL,
+                                true);
     }
 }
 
@@ -1249,7 +1262,8 @@ void helper_qemu_log_instr_allcpu_user_start()
 
     CPU_FOREACH(cpu)
     {
-        helper_qemu_log_instr_user_start(cpu->env_ptr, 0);
+        do_qemu_log_instr_start(cpu->env_ptr, 0, QEMU_LOG_INSTR_LOGLEVEL_USER,
+                                true);
     }
 }
 
@@ -1260,7 +1274,7 @@ void helper_qemu_log_instr_allcpu_stop()
 
     CPU_FOREACH(cpu)
     {
-        helper_qemu_log_instr_stop(cpu->env_ptr, 0);
+        do_qemu_log_instr_stop(cpu->env_ptr, 0, true);
     }
 }
 
diff --git a/include/exec/log_instr_internal.h b/include/exec/log_instr_internal.h
index e3e68579790..1edfb54236a 100644
--- a/include/exec/log_instr_internal.h
+++ b/include/exec/log_instr_internal.h
@@ -59,6 +59,16 @@
 #define LMI_CAP 4
 
 #ifndef __cplusplus
+/*
+ * Temporary argument to the log-level switching callbacks.
+ * This is passed internally to async_run_on_cpu()
+ */
+typedef struct {
+    bool global;
+    qemu_log_instr_loglevel_t next_level;
+    target_ulong pc;
+} qemu_log_next_level_arg_t;
+
 /*
  * Instruction log info associated with each committed log entry.
  * This is stored in the per-cpu log cpustate.

From 0ad2c2eb7bf8659cdc67c71dfd6e32846eab2511 Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Thu, 7 Oct 2021 16:45:43 +0100
Subject: [PATCH 22/74] Implement guest-driven context tracking for the
 perfetto log backend.

The guest OS can use reserved event NOPs to inform the perfetto backend
about context changes (e.g. a context switch or a compartment switch).
The perfetto backend allocates a separate track to each context and
emits events to the active track in each CPU.
Note that the notion of context switch is entirely given by the guest OS
and it is responsible for correctly updating the state on the backend.
The probe effect in the code due to NOPs instructions should be negligible.
---
 accel/tcg/log_instr.c                |   4 +-
 accel/tcg/log_instr_text.c           |   8 +-
 include/exec/log_instr_perfetto.h    |  17 ++-
 include/qemu/log_instr.h             |  25 ++--
 target/riscv/op_helper_log_instr.c   |  10 +-
 trace_extra/guest_context_tracker.cc | 176 +++++++++++++++++++++++++++
 trace_extra/guest_context_tracker.hh |  72 +++++++++++
 trace_extra/meson.build              |   1 +
 trace_extra/trace_perfetto.cc        |  86 +++++++------
 trace_extra/trace_stats.cc           |   1 +
 10 files changed, 342 insertions(+), 58 deletions(-)
 create mode 100644 trace_extra/guest_context_tracker.cc
 create mode 100644 trace_extra/guest_context_tracker.hh

diff --git a/accel/tcg/log_instr.c b/accel/tcg/log_instr.c
index 794b1178800..51b7c2724e1 100644
--- a/accel/tcg/log_instr.c
+++ b/accel/tcg/log_instr.c
@@ -229,10 +229,9 @@ static void do_cpu_loglevel_switch(CPUState *cpu, run_on_cpu_data data)
     cpu_log_entry_t *entry = get_cpu_log_entry(env);
     qemu_log_instr_loglevel_t prev_level = cpulog->loglevel;
     bool prev_level_active = cpulog->loglevel_active;
-    /* qemu_log_instr_loglevel_t next_level = data.host_int; */
-    bool next_level_active;
     qemu_log_next_level_arg_t *arg = data.host_ptr;
     target_ulong pc = (arg->global) ? cpu_get_recent_pc(env) : arg->pc;
+    bool next_level_active;
 
     log_assert(qemu_loglevel_mask(CPU_LOG_INSTR));
 
@@ -276,7 +275,6 @@ static void do_cpu_loglevel_switch(CPUState *cpu, run_on_cpu_data data)
             reset_log_buffer(cpulog, entry);
             goto done;
         }
-        /* emit_stop_event(entry, cpu_get_recent_pc(env)); */
         emit_stop_event(entry, pc);
         do_instr_commit(env);
         /* Instruction commit may have advanced to the next entry buffer slot */
diff --git a/accel/tcg/log_instr_text.c b/accel/tcg/log_instr_text.c
index 3ea84c80cf3..0cd8f054ff6 100644
--- a/accel/tcg/log_instr_text.c
+++ b/accel/tcg/log_instr_text.c
@@ -210,10 +210,12 @@ void emit_text_instr(CPUArchState *env, cpu_log_entry_t *entry)
                          log_state_op, event->state.pc);
             }
             break;
-        case LOG_EVENT_CTX_SWITCH:
+        case LOG_EVENT_CTX_UPDATE:
             qemu_log("Context switch pid=0x%lx tid=0x%lx cid=0x%lx\n",
-                     event->ctx_switch.pid, event->ctx_switch.tid,
-                     event->ctx_switch.cid);
+                     event->ctx_update.pid, event->ctx_update.tid,
+                     event->ctx_update.cid);
+        default:
+            assert(0 && "unknown event ID");
         }
     }
 }
diff --git a/include/exec/log_instr_perfetto.h b/include/exec/log_instr_perfetto.h
index 20a5b11c5ee..97210548ca5 100644
--- a/include/exec/log_instr_perfetto.h
+++ b/include/exec/log_instr_perfetto.h
@@ -40,12 +40,27 @@
 #include <cstdint>
 #include <perfetto.h>
 
+/*
+ * For each CPU we have the following tracks:
+ * CPU N: track for generic CPU events, contains the following categories:
+ *   - instructions: events related to the instruction stream
+ *   - sched: scheduling events on the CPU (context switches)
+ *   - trap: Contains interrupt and trap events with help from OS-driven events
+ *   - ctrl: Tracing control events (e.g. when tracing is started and stopped)
+ * Process/Thread/Compartment tracks:
+ *   these are dynamic tracks that are created in response to OS-driven events,
+ *   we rely on the OS to pass the pid/tid/cid triple when relevant.
+ *   Each of these tracks is used to accumulate per-thread and per-compartment
+ *   information.
+ */
 PERFETTO_DEFINE_CATEGORIES(
     perfetto::Category("instructions")
         .SetDescription("CPU instructions executed"),
     perfetto::Category("memory").SetDescription("Tag and memory load/store"),
     perfetto::Category("stats").SetDescription("High-level statistics data"),
-    perfetto::Category("ctrl").SetDescription("Tracing control events"));
+    perfetto::Category("ctrl").SetDescription("Tracing control events"),
+    perfetto::Category("trap").SetDescription("CPU trap events"),
+    perfetto::Category("sched").SetDescription("Scheduling events"));
 #endif
 
 /*
diff --git a/include/qemu/log_instr.h b/include/qemu/log_instr.h
index 396524e4a67..a7a617faaed 100644
--- a/include/qemu/log_instr.h
+++ b/include/qemu/log_instr.h
@@ -94,10 +94,7 @@ typedef enum {
 /*
  * Trace event identifiers.
  */
-typedef enum {
-    LOG_EVENT_STATE = 0,
-    LOG_EVENT_CTX_SWITCH = 1,
-} log_event_id_t;
+typedef enum { LOG_EVENT_STATE = 0, LOG_EVENT_CTX_UPDATE = 1 } log_event_id_t;
 
 /*
  * Tracing status changed (e.g. trace start/stop)
@@ -113,14 +110,22 @@ typedef struct {
     uint64_t pc;
 } log_event_trace_state_update_t;
 
+typedef enum {
+    /*
+     * Switch context from the current to the new (proc, thread, compartment) ID
+     */
+    LOG_EVENT_CTX_OP_SWITCH = 1
+} log_event_ctx_update_op_t;
+
 /*
- * Context switch event.
+ * Context update event.
  */
 typedef struct {
-    uint64_t pid; /* Process ID */
-    uint64_t tid; /* Thread ID */
-    uint64_t cid; /* Compartment ID */
-} log_event_ctx_switch_t;
+    log_event_ctx_update_op_t op; /* What changed */
+    uint64_t pid;                 /* Process ID */
+    uint64_t tid;                 /* Thread ID */
+    uint64_t cid;                 /* Compartment ID */
+} log_event_ctx_update_t;
 
 /*
  * Trace event.
@@ -131,7 +136,7 @@ typedef struct {
     log_event_id_t id;
     union {
         log_event_trace_state_update_t state;
-        log_event_ctx_switch_t ctx_switch;
+        log_event_ctx_update_t ctx_update;
     };
 } log_event_t;
 
diff --git a/target/riscv/op_helper_log_instr.c b/target/riscv/op_helper_log_instr.c
index f94d00ef001..f5442493eb3 100644
--- a/target/riscv/op_helper_log_instr.c
+++ b/target/riscv/op_helper_log_instr.c
@@ -79,12 +79,14 @@ void HELPER(riscv_log_instr_event)(CPURISCVState *env, target_ulong pc)
 
     event.id = get_gpr_value(env, 10);
     switch (event.id) {
-    case LOG_EVENT_CTX_SWITCH:
-        event.ctx_switch.pid = get_gpr_value(env, 11);
-        event.ctx_switch.tid = get_gpr_value(env, 12);
-        event.ctx_switch.cid = get_gpr_value(env, 13);
+    case LOG_EVENT_CTX_UPDATE:
+        event.ctx_update.op = get_gpr_value(env, 11);
+        event.ctx_update.pid = get_gpr_value(env, 12);
+        event.ctx_update.tid = get_gpr_value(env, 13);
+        event.ctx_update.cid = get_gpr_value(env, 14);
         break;
     default:
+        warn_report("Unsupported event ID for TCG instr logging %d", event.id);
         return;
     }
     qemu_log_instr_event(env, &event);
diff --git a/trace_extra/guest_context_tracker.cc b/trace_extra/guest_context_tracker.cc
new file mode 100644
index 00000000000..664f521f6c9
--- /dev/null
+++ b/trace_extra/guest_context_tracker.cc
@@ -0,0 +1,176 @@
+/*-
+ * Copyright (c) 2021 Alfredo Mazzinghi
+ * All rights reserved.
+ *
+ * This software was developed by SRI International and the University of
+ * Cambridge Computer Laboratory (Department of Computer Science and
+ * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the
+ * DARPA SSITH research programme.
+ *
+ * @BERI_LICENSE_HEADER_START@
+ *
+ * Licensed to BERI Open Systems C.I.C. (BERI) under one or more contributor
+ * license agreements.  See the NOTICE file distributed with this work for
+ * additional information regarding copyright ownership.  BERI licenses this
+ * file to you under the BERI Hardware-Software License, Version 1.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at:
+ *
+ *   http://www.beri-open-systems.org/legal/license-1-0.txt
+ *
+ * Unless required by applicable law or agreed to in writing, Work distributed
+ * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ * CONDITIONS OF ANY KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ * @BERI_LICENSE_HEADER_END@
+ */
+
+#include <atomic>
+#include <mutex>
+#include <string>
+#include "trace_extra/guest_context_tracker.hh"
+
+namespace {
+
+struct qemu_context_track : public perfetto::Track
+{
+    const uint64_t pid;
+    const uint64_t tid;
+    const uint64_t cid;
+
+    using qemu_ctx_id = std::tuple<uint64_t, uint64_t, uint64_t>;
+
+    static std::shared_ptr<qemu_context_track>
+    make_context_track(uint64_t pid, uint64_t tid, uint64_t cid)
+    {
+        return std::make_shared<qemu_context_track>(pid, tid, cid);
+    }
+
+    qemu_ctx_id
+    get_id() const
+    {
+        return std::make_tuple(pid, tid, cid);
+    }
+
+    perfetto::protos::gen::TrackDescriptor
+    Serialize() const
+    {
+        auto desc = Track::Serialize();
+        auto qemu_desc = desc.mutable_qemu_context();
+        qemu_desc->set_pid(pid);
+        qemu_desc->set_tid(tid);
+        qemu_desc->set_cid(cid);
+        return desc;
+    }
+
+    qemu_context_track(uint64_t pid, uint64_t tid, uint64_t cid)
+            : perfetto::Track(cheri::gen_track_uuid(), perfetto::Track()), pid(pid), tid(tid), cid(cid)
+    {}
+
+    ~qemu_context_track()
+    {
+        perfetto::TrackEvent::EraseTrackDescriptor(*this);
+    }
+};
+
+/*
+ * Dynamically created tracks.
+ * Here we do not commit to a hierarchical model of process/thread, as context
+ * events are guest-driven. Instead we defer any hierarchical interpretation of
+ * the context identifier to post-processing.
+ */
+std::mutex tracks_lock;
+std::map<qemu_context_track::qemu_ctx_id, std::shared_ptr<qemu_context_track>> tracks;
+} /* anonymous */
+
+namespace std {
+/*
+ * Make the qemu track ID hashable
+ */
+template<> struct hash<qemu_context_track::qemu_ctx_id>
+{
+    std::size_t operator()(const qemu_context_track::qemu_ctx_id& key) const noexcept
+    {
+        return (std::hash<uint64_t>{}(std::get<0>(key)) ^
+                std::hash<uint64_t>{}(std::get<1>(key)) ^
+                std::hash<uint64_t>{}(std::get<2>(key)));
+
+    }
+};
+} /* std */
+
+namespace cheri
+{
+
+/* Helper to generate unique IDs for dynamic tracks */
+unsigned long
+gen_track_uuid()
+{
+    // use perfetto::Uuid?
+    static std::atomic<unsigned long> next_track_id(0xdead);
+    return (next_track_id++);
+}
+
+guest_context_tracker::guest_context_tracker(int cpu_id) :
+        cpu_track_(perfetto::Track::Global(gen_track_uuid()))
+{
+    std::string track_name("CPU " + std::to_string(cpu_id));
+    auto desc = cpu_track_.Serialize();
+    desc.set_name(track_name);
+    perfetto::TrackEvent::SetTrackDescriptor(cpu_track_, desc);
+}
+
+void
+guest_context_tracker::context_update(const log_event_ctx_update_t *evt)
+{
+    // Currently this is the only one existing
+    assert(evt->op == LOG_EVENT_CTX_OP_SWITCH && "Invalid ctx update op");
+    /*
+     * Find or create the track associated with the new context and set it as the current
+     * context track on the CPU. This will be used to log per-context events.
+     * XXX-AM: Currently it is assumed that contexts are unique, this is clearly not the case
+     * with PID/TID reuse and we should do something to detect this. This is mostly relevant
+     * only for long-lived traces.
+     */
+
+    /* Fetch the tracks for the new context */
+    std::lock_guard<std::mutex> tracks_guard(tracks_lock);
+    qemu_context_track::qemu_ctx_id key = std::make_tuple(evt->pid, evt->tid, evt->cid);
+    auto ctx_track_iter = tracks.find(key);
+    std::shared_ptr<qemu_context_track> track;
+    if (ctx_track_iter == tracks.end()) {
+        /* New context */
+        track = qemu_context_track::make_context_track(evt->pid, evt->tid, evt->cid);
+        auto desc = track->Serialize();
+        desc.set_name("CTX " + std::to_string(evt->pid) + ":" +
+                      std::to_string(evt->tid) + ":" +
+                      std::to_string(evt->cid));
+        perfetto::TrackEvent::SetTrackDescriptor(*track, desc);
+        tracks.emplace(key, track);
+    } else {
+        /* Existing context */
+        track = ctx_track_iter->second;
+    }
+    ctx_track_ = track;
+    /* XXX-AM: for debugging, remove */
+    TRACE_EVENT_INSTANT("sched", "switch", cpu_track_,
+                        "pid", evt->pid, "tid", evt->tid, "cid", evt->cid);
+}
+
+perfetto::Track&
+guest_context_tracker::get_cpu_track()
+{
+    return cpu_track_;
+}
+
+perfetto::Track&
+guest_context_tracker::get_ctx_track()
+{
+    if (ctx_track_)
+        return *ctx_track_;
+    else
+        return cpu_track_;
+}
+
+} /* cheri */
diff --git a/trace_extra/guest_context_tracker.hh b/trace_extra/guest_context_tracker.hh
new file mode 100644
index 00000000000..bbca48c7234
--- /dev/null
+++ b/trace_extra/guest_context_tracker.hh
@@ -0,0 +1,72 @@
+/*-
+ * Copyright (c) 2021 Alfredo Mazzinghi
+ * All rights reserved.
+ *
+ * This software was developed by SRI International and the University of
+ * Cambridge Computer Laboratory (Department of Computer Science and
+ * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the
+ * DARPA SSITH research programme.
+ *
+ * @BERI_LICENSE_HEADER_START@
+ *
+ * Licensed to BERI Open Systems C.I.C. (BERI) under one or more contributor
+ * license agreements.  See the NOTICE file distributed with this work for
+ * additional information regarding copyright ownership.  BERI licenses this
+ * file to you under the BERI Hardware-Software License, Version 1.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at:
+ *
+ *   http://www.beri-open-systems.org/legal/license-1-0.txt
+ *
+ * Unless required by applicable law or agreed to in writing, Work distributed
+ * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ * CONDITIONS OF ANY KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ * @BERI_LICENSE_HEADER_END@
+ */
+
+#pragma once
+
+#include <memory>
+#include <perfetto.h>
+#include "qemu/log_instr.h"
+#include "exec/log_instr_perfetto.h"
+
+namespace cheri
+{
+/*
+ * Track current guest context associated to a CPU.
+ * This allows to charge events to the correct context by recording
+ * what is currently active on a CPU.
+ * There is one guest_context_tracker per virtual CPU.
+ * We manage the following tracks for each CPU:
+ * 1. A global CPU N track, for events charged to a CPU from an unknown context
+ *    or where the context is not relevant.
+ * 2. Each new process in the system gets allocated a new perfetto::ProcessTrack
+ *    TODO: Currently there is no way to notify when the PID is discarded.
+ * 3. Each new thread in the system gets allocated a new perfetto::ThreadTrack and
+ *    the corresponding process as parent.
+ * 4. Each new compartment ID gets allocated a new perfetto::CheriCompartmentTrack.
+ *    Note that compartment tracks are not associated to a single thread, instead
+ *    multiple threads may record events there.
+ */
+class guest_context_tracker
+{
+    perfetto::Track cpu_track_;
+    std::shared_ptr<perfetto::Track> ctx_track_;
+  public:
+    guest_context_tracker(int cpu_id);
+    void context_update(const log_event_ctx_update_t *evt);
+    perfetto::Track& get_cpu_track();
+    perfetto::Track& get_ctx_track();
+};
+
+/*
+ * Generate a unique track ID.
+ * The ID is guaranteed to be unique within a qemu run, but not with
+ * respect to other data sources.
+ */
+unsigned long gen_track_uuid();
+
+} /* cheri */
diff --git a/trace_extra/meson.build b/trace_extra/meson.build
index b7a882b960f..d7c796d1cad 100644
--- a/trace_extra/meson.build
+++ b/trace_extra/meson.build
@@ -6,6 +6,7 @@ qemutrace_ss.add(when: ['CONFIG_TCG', 'CONFIG_TRACE_PERFETTO'],
 
 qemutrace_ss.add(when: ['CONFIG_TCG', 'CONFIG_TRACE_PERFETTO'],
                  if_true: files('trace_perfetto.cc', 'trace_stats.cc',
+                                'guest_context_tracker.cc',
                                 'cheri-perfetto/sdk/perfetto.cc'))
 
 qemutrace_ss = qemutrace_ss.apply(config_all, strict: false)
diff --git a/trace_extra/trace_perfetto.cc b/trace_extra/trace_perfetto.cc
index 7ffae28f4b7..86349c8a142 100644
--- a/trace_extra/trace_perfetto.cc
+++ b/trace_extra/trace_perfetto.cc
@@ -39,6 +39,7 @@
 #include <sstream>
 #include <iomanip>
 #include <string>
+#include <map>
 #include <vector>
 #include <pthread.h>
 #include <fcntl.h>
@@ -49,6 +50,7 @@
 #include "exec/log_instr_internal.h"
 #include "exec/log_instr_perfetto.h"
 #include "trace_extra/trace_stats.hh"
+#include "trace_extra/guest_context_tracker.hh"
 
 PERFETTO_TRACK_EVENT_STATIC_STORAGE();
 
@@ -65,37 +67,30 @@ fs::path logfile("qemu_trace.pb");
 /* category strings */
 std::vector<std::string> categories;
 
-static unsigned long
-gen_track_uuid(int cpu_id) {
-    // use perfetto::Uuid?
-    static unsigned long next_track_id = 0x100;
-    return (next_track_id++);
-}
+/* Global scheduling event track */
+std::unique_ptr<perfetto::Track> sched_track;
 
 /*
  * Private per-CPU state.
  */
-struct perfetto_backend_data {
-    // Per-CPU track. This will be the parent track for all events on this CPU.
-    perfetto::Track cpu_track;
+struct perfetto_backend_data
+{
     // Per-CPU control track. This records tracing control events.
-    perfetto::Track cpu_ctrl_track;
+    perfetto::Track ctrl_track_;
     // Per-CPU aggregate statistics
-    cheri::qemu_stats stats;
+    cheri::qemu_stats stats_;
+    cheri::guest_context_tracker ctx_tracker_;
 
     perfetto_backend_data(int cpu_id) :
-        cpu_track(perfetto::Track::Global(gen_track_uuid(cpu_id))),
-        cpu_ctrl_track(perfetto::Track::Global(gen_track_uuid(cpu_id)))
+        ctrl_track_(perfetto::Track::Global(cheri::gen_track_uuid())),
+        ctx_tracker_(cpu_id)
+
     {
-        std::string track_name("CPU " + std::to_string(cpu_id));
-        std::string ctrl_name(track_name + " CTRL");
+        std::string track_name("CPU " + std::to_string(cpu_id) + " ctrl");
 
-        auto desc = cpu_track.Serialize();
+        auto desc = ctrl_track_.Serialize();
         desc.set_name(track_name);
-        perfetto::TrackEvent::SetTrackDescriptor(cpu_track, desc);
-        desc = cpu_ctrl_track.Serialize();
-        desc.set_name(ctrl_name);
-        perfetto::TrackEvent::SetTrackDescriptor(cpu_ctrl_track, desc);
+        perfetto::TrackEvent::SetTrackDescriptor(ctrl_track_, desc);
     }
 };
 
@@ -120,7 +115,7 @@ bool perfetto_start_tracing(void)
     perfetto::Tracing::Initialize(args);
     perfetto::TrackEvent::Register();
 
-    cfg.add_buffers()->set_size_kb(1 << 16); // 64MiB
+    cfg.add_buffers()->set_size_kb(1 << 17); // 128MiB
     cfg.set_flush_period_ms(2000);
     cfg.set_file_write_period_ms(1000);
     fs::remove(logfile);
@@ -136,7 +131,7 @@ bool perfetto_start_tracing(void)
     ds_cfg->set_track_event_config_raw(track_cfg.SerializeAsString());
     auto *producer_cfg = cfg.add_producers();
     producer_cfg->set_producer_name("qemu-tcg");
-    producer_cfg->set_shm_size_kb(1 << 11); // 2MiB XXX need more space
+    producer_cfg->set_shm_size_kb(1 << 15); // 32MiB
 
     session = perfetto::Tracing::NewTrace();
 
@@ -144,10 +139,14 @@ bool perfetto_start_tracing(void)
     session->StartBlocking();
 
     perfetto::ProcessTrack qemu_proc = perfetto::ProcessTrack::Current();
-    perfetto::protos::gen::TrackDescriptor desc = qemu_proc.Serialize();
+    auto desc = qemu_proc.Serialize();
     desc.mutable_process()->set_process_name("qemu");
     perfetto::TrackEvent::SetTrackDescriptor(qemu_proc, desc);
 
+    sched_track.reset(new perfetto::Track(cheri::gen_track_uuid()));
+    desc = sched_track->Serialize();
+    desc.set_name("Scheduler ctrl");
+    perfetto::TrackEvent::SetTrackDescriptor(*sched_track, desc);
     return true;
 }
 
@@ -184,45 +183,58 @@ void
 process_events(perfetto_backend_data *data, cpu_log_entry_handle entry)
 {
     int nevents = perfetto_log_entry_events(entry);
+    bool have_startstop_event = false;
+
+    /*
+     * Note: LOG_EVENT_STATE events are emitted even when tracing is disabled.
+     * The rest of the events should only be emitted when tracing is active.
+     */
     for (int i = 0; i < nevents; i++) {
         log_event_t *evt = perfetto_log_event(entry, i);
         if (evt->id == LOG_EVENT_STATE) {
-            switch(evt->state.next_state) {
+            switch (evt->state.next_state) {
                 case LOG_EVENT_STATE_FLUSH:
-                    // XXX-AM: Emitting stats and TrackEvent flush should be two separate events TBH.
-                    TRACE_EVENT_INSTANT("ctrl", "flush", data->cpu_ctrl_track);
-                    data->stats.flush(data->cpu_track);
+                    TRACE_EVENT_INSTANT("ctrl", "flush", data->ctrl_track_);
                     perfetto::TrackEvent::Flush();
                     break;
                 case LOG_EVENT_STATE_START:
-                    data->stats_.unpause(data->cpu_track, evt->state.pc);
-                    TRACE_EVENT_BEGIN("ctrl", "tracing", data->cpu_ctrl_track);
+                    data->stats_.unpause(data->ctx_tracker_.get_ctx_track(), evt->state.pc);
+                    TRACE_EVENT_BEGIN("ctrl", "tracing", data->ctrl_track_);
+                    have_startstop_event = true;
                     break;
                 case LOG_EVENT_STATE_STOP:
-                    data->stats_.pause(data->cpu_track, evt->state.pc);
-                    TRACE_EVENT_END("ctrl", data->cpu_ctrl_track);
+                    data->stats_.pause(data->ctx_tracker_.get_ctx_track(), evt->state.pc);
+                    TRACE_EVENT_END("ctrl", data->ctrl_track_);
+                    have_startstop_event = true;
                     break;
+                default:
+                    assert(false && "Invalid state event");
             }
+        } else if (evt->id == LOG_EVENT_CTX_UPDATE) {
+            // Dump the stats for the current context to the right context track
+            data->stats_.flush(data->ctx_tracker_.get_ctx_track());
+            data->ctx_tracker_.context_update(&evt->ctx_update);
         }
     }
-}
 
-void
-process_instr(perfetto_backend_data *data, cpu_log_entry_handle entry)
-{
     /*
      * If we have instruction data, we assume that tracing is enabled and run extra
      * stats gathering to track the executed instruction PC in the stats histograms.
      * This avoids having to keep a shadow copy of the tracing state in the backend.
      */
-    data->stats.process_instr(entry);
+    if (!have_startstop_event && (perfetto_log_entry_flags(entry) & LI_FLAG_HAS_INSTR_DATA) != 0)
+        data->stats_.process_instr(data->ctx_tracker_.get_ctx_track(), entry);
+}
 
+void
+process_instr(perfetto_backend_data *data, cpu_log_entry_handle entry)
+{
     /*
      * XXX-AM: instead of having one big instruction record, we may have different messages for
      * optional parts of the instruction message, on the same track/category: e.g. mode swtich,
      * interrupt information and modified registers?
      */
-    TRACE_EVENT_INSTANT("instructions", "stream", data->cpu_track,
+    TRACE_EVENT_INSTANT("instructions", "stream", data->ctx_tracker_.get_ctx_track(),
         [&](perfetto::EventContext ctx) {
             auto *qemu_arg = ctx.event()->set_qemu();
             auto *instr = qemu_arg->set_instr();
diff --git a/trace_extra/trace_stats.cc b/trace_extra/trace_stats.cc
index a6024d27e00..df8403371da 100644
--- a/trace_extra/trace_stats.cc
+++ b/trace_extra/trace_stats.cc
@@ -94,6 +94,7 @@ void qemu_stats::unpause(perfetto::Track &track, uint64_t pc)
 {
     /* Reset the pc-tracking state */
     pc_range_start = last_pc = pc;
+    /* unpause will be called on the first instruction to be traced? */
     icount = 1; // Unpause is called for the first instruction being traced
 }
 

From c3dafaa2ec42c9cd6b19e667b7b8ef8410ba70d8 Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Wed, 20 Oct 2021 15:54:25 +0100
Subject: [PATCH 23/74] Add TODO for better tracing context switches in
 perfetto backend.

---
 trace_extra/guest_context_tracker.hh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/trace_extra/guest_context_tracker.hh b/trace_extra/guest_context_tracker.hh
index bbca48c7234..825d603d564 100644
--- a/trace_extra/guest_context_tracker.hh
+++ b/trace_extra/guest_context_tracker.hh
@@ -50,6 +50,7 @@ namespace cheri
  * 4. Each new compartment ID gets allocated a new perfetto::CheriCompartmentTrack.
  *    Note that compartment tracks are not associated to a single thread, instead
  *    multiple threads may record events there.
+ * 5. TODO: The context shoud include information on the EL and address space
  */
 class guest_context_tracker
 {

From 75620022fb4f2a5bebeff49d353cd3397326aded Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Sat, 30 Oct 2021 02:54:55 +0100
Subject: [PATCH 24/74] Fix instruction logging backend context tracing.

- Add extra LOG_EVENT_CTX_OP_SETUP variant of the OP_SWITCH context update
event.
  This is propagated even when tracing is off and can be used to setup the
  context information before tracing starts.
- Fix statistics trace state reset when tracing is paused and when the
  active context changes.
- Add a subclass for QEMU context tracks. This follows the pattern of perfetto
  ThreadTrack and ProcessTrack.
- Add events to RISC-V target.
---
 accel/tcg/log_instr.c                |   7 +
 include/qemu/log_instr.h             |  18 ++-
 target/riscv/cpu.h                   |  24 ++++
 target/riscv/cpu_helper.c            |  18 +--
 target/riscv/op_helper_log_instr.c   |   8 +-
 trace_extra/guest_context_tracker.cc | 204 ++++++++++++++++-----------
 trace_extra/guest_context_tracker.hh |  47 ++++--
 trace_extra/trace_perfetto.cc        |  27 ++--
 trace_extra/trace_stats.cc           |  54 ++++---
 trace_extra/trace_stats.hh           |  15 +-
 10 files changed, 268 insertions(+), 154 deletions(-)

diff --git a/accel/tcg/log_instr.c b/accel/tcg/log_instr.c
index 51b7c2724e1..90b0cdf561b 100644
--- a/accel/tcg/log_instr.c
+++ b/accel/tcg/log_instr.c
@@ -149,6 +149,13 @@ static inline void emit_start_event(cpu_log_entry_t *entry, target_ulong pc)
     event.state.pc = pc;
     /* Start events always have incomplete instruction data */
     entry->flags &= ~LI_FLAG_HAS_INSTR_DATA;
+    /*
+     * Also update PC to the one given in the start event.
+     * This ensures that the pc field is always correct, even on the first
+     * incomplete entry of the trace, where the start trigger occurs.
+     * XXX-AM: Does this mean that we can do away with the state.pc field?
+     */
+    entry->pc = pc;
 
     g_array_append_val(entry->events, event);
 }
diff --git a/include/qemu/log_instr.h b/include/qemu/log_instr.h
index a7a617faaed..ae7f264d0a3 100644
--- a/include/qemu/log_instr.h
+++ b/include/qemu/log_instr.h
@@ -114,17 +114,25 @@ typedef enum {
     /*
      * Switch context from the current to the new (proc, thread, compartment) ID
      */
-    LOG_EVENT_CTX_OP_SWITCH = 1
+    LOG_EVENT_CTX_OP_SWITCH = 1,
+    /*
+     * Same as LOG_EVENT_CTX_OP_SWITCH but should bypass tracing activation
+     * status, Meaning that these events will reach the backend even when
+     * tracing is off. This is useful setup the correct context identifier
+     * before switching on tracing.
+     */
+    LOG_EVENT_CTX_OP_SETUP = 2
 } log_event_ctx_update_op_t;
 
 /*
  * Context update event.
  */
 typedef struct {
-    log_event_ctx_update_op_t op; /* What changed */
-    uint64_t pid;                 /* Process ID */
-    uint64_t tid;                 /* Thread ID */
-    uint64_t cid;                 /* Compartment ID */
+    log_event_ctx_update_op_t op;   /* What changed */
+    uint64_t pid;                   /* Process ID */
+    uint64_t tid;                   /* Thread ID */
+    uint64_t cid;                   /* Compartment ID */
+    qemu_log_instr_cpu_mode_t mode; /* CPU mode */
 } log_event_ctx_update_t;
 
 /*
diff --git a/target/riscv/cpu.h b/target/riscv/cpu.h
index 7e43fdadf3f..f42a5e77766 100644
--- a/target/riscv/cpu.h
+++ b/target/riscv/cpu.h
@@ -737,6 +737,30 @@ static inline const char *cpu_get_mode_name(qemu_log_instr_cpu_mode_t mode)
         return riscv_cpu_mode_names[mode];
     return "<invalid>";
 }
+
+/*
+ * Currently this is not required by the instruction logging backend, it
+ * is used internally in the target to get the correct code for the mode.
+ */
+static inline qemu_log_instr_cpu_mode_t cpu_priv_to_mode(target_ulong priv)
+{
+    qemu_log_instr_cpu_mode_t mode;
+
+    switch (priv) {
+    case PRV_M:
+        mode = RISCV_LOG_INSTR_CPU_M;
+        break;
+    case PRV_S:
+        mode = RISCV_LOG_INSTR_CPU_S;
+        break;
+    case PRV_H:
+        mode = RISCV_LOG_INSTR_CPU_H;
+        break;
+    default:
+        mode = RISCV_LOG_INSTR_CPU_U;
+    }
+    return mode;
+}
 #endif
 
 int riscv_csrrw(CPURISCVState *env, int csrno, target_ulong *ret_value,
diff --git a/target/riscv/cpu_helper.c b/target/riscv/cpu_helper.c
index 4b2518543b1..881c1086853 100644
--- a/target/riscv/cpu_helper.c
+++ b/target/riscv/cpu_helper.c
@@ -339,22 +339,8 @@ void riscv_cpu_set_mode(CPURISCVState *env, target_ulong newpriv)
     env->load_res = -1;
 
 #ifdef CONFIG_TCG_LOG_INSTR
-    qemu_log_instr_cpu_mode_t mode;
-
-    switch (newpriv) {
-    case PRV_M:
-        mode = RISCV_LOG_INSTR_CPU_M;
-        break;
-    case PRV_S:
-        mode = RISCV_LOG_INSTR_CPU_S;
-        break;
-    case PRV_H:
-        mode = RISCV_LOG_INSTR_CPU_H;
-        break;
-    default:
-        mode = RISCV_LOG_INSTR_CPU_U;
-    }
-    qemu_log_instr_mode_switch(env, mode, cpu_get_recent_pc(env));
+    qemu_log_instr_mode_switch(env, cpu_priv_to_mode(newpriv),
+        cpu_get_recent_pc(env));
 #endif
 }
 
diff --git a/target/riscv/op_helper_log_instr.c b/target/riscv/op_helper_log_instr.c
index f5442493eb3..7e751abfdb4 100644
--- a/target/riscv/op_helper_log_instr.c
+++ b/target/riscv/op_helper_log_instr.c
@@ -73,17 +73,21 @@ void HELPER(riscv_log_instr_event)(CPURISCVState *env, target_ulong pc)
 {
     log_event_t event;
 
-    if (!qemu_log_instr_enabled(env)) {
+    event.id = get_gpr_value(env, 10);
+    if (!qemu_log_instr_enabled(env) &&
+        (event.id != LOG_EVENT_CTX_UPDATE ||
+         event.ctx_update.op != LOG_EVENT_CTX_OP_SETUP)) {
         return;
     }
 
-    event.id = get_gpr_value(env, 10);
     switch (event.id) {
     case LOG_EVENT_CTX_UPDATE:
+    case LOG_EVENT_CTX_OP_SETUP:
         event.ctx_update.op = get_gpr_value(env, 11);
         event.ctx_update.pid = get_gpr_value(env, 12);
         event.ctx_update.tid = get_gpr_value(env, 13);
         event.ctx_update.cid = get_gpr_value(env, 14);
+        event.ctx_update.mode = cpu_priv_to_mode(env->priv);
         break;
     default:
         warn_report("Unsupported event ID for TCG instr logging %d", event.id);
diff --git a/trace_extra/guest_context_tracker.cc b/trace_extra/guest_context_tracker.cc
index 664f521f6c9..ebaa0309593 100644
--- a/trace_extra/guest_context_tracker.cc
+++ b/trace_extra/guest_context_tracker.cc
@@ -31,49 +31,8 @@
 #include <string>
 #include "trace_extra/guest_context_tracker.hh"
 
-namespace {
-
-struct qemu_context_track : public perfetto::Track
+namespace
 {
-    const uint64_t pid;
-    const uint64_t tid;
-    const uint64_t cid;
-
-    using qemu_ctx_id = std::tuple<uint64_t, uint64_t, uint64_t>;
-
-    static std::shared_ptr<qemu_context_track>
-    make_context_track(uint64_t pid, uint64_t tid, uint64_t cid)
-    {
-        return std::make_shared<qemu_context_track>(pid, tid, cid);
-    }
-
-    qemu_ctx_id
-    get_id() const
-    {
-        return std::make_tuple(pid, tid, cid);
-    }
-
-    perfetto::protos::gen::TrackDescriptor
-    Serialize() const
-    {
-        auto desc = Track::Serialize();
-        auto qemu_desc = desc.mutable_qemu_context();
-        qemu_desc->set_pid(pid);
-        qemu_desc->set_tid(tid);
-        qemu_desc->set_cid(cid);
-        return desc;
-    }
-
-    qemu_context_track(uint64_t pid, uint64_t tid, uint64_t cid)
-            : perfetto::Track(cheri::gen_track_uuid(), perfetto::Track()), pid(pid), tid(tid), cid(cid)
-    {}
-
-    ~qemu_context_track()
-    {
-        perfetto::TrackEvent::EraseTrackDescriptor(*this);
-    }
-};
-
 /*
  * Dynamically created tracks.
  * Here we do not commit to a hierarchical model of process/thread, as context
@@ -81,39 +40,84 @@ struct qemu_context_track : public perfetto::Track
  * the context identifier to post-processing.
  */
 std::mutex tracks_lock;
-std::map<qemu_context_track::qemu_ctx_id, std::shared_ptr<qemu_context_track>> tracks;
-} /* anonymous */
+std::map<cheri::qemu_context_track::qemu_ctx_id,
+         std::shared_ptr<cheri::qemu_context_track>>
+    tracks;
+} // namespace
 
-namespace std {
+namespace std
+{
 /*
  * Make the qemu track ID hashable
  */
-template<> struct hash<qemu_context_track::qemu_ctx_id>
-{
-    std::size_t operator()(const qemu_context_track::qemu_ctx_id& key) const noexcept
+template <> struct hash<cheri::qemu_context_track::qemu_ctx_id> {
+    std::size_t
+    operator()(const cheri::qemu_context_track::qemu_ctx_id &key) const noexcept
     {
         return (std::hash<uint64_t>{}(std::get<0>(key)) ^
                 std::hash<uint64_t>{}(std::get<1>(key)) ^
-                std::hash<uint64_t>{}(std::get<2>(key)));
-
+                std::hash<uint64_t>{}(std::get<2>(key)) ^
+                std::hash<uint64_t>{}(std::get<3>(key)));
     }
 };
-} /* std */
+} // namespace std
 
 namespace cheri
 {
 
 /* Helper to generate unique IDs for dynamic tracks */
-unsigned long
-gen_track_uuid()
+unsigned long gen_track_uuid()
 {
     // use perfetto::Uuid?
     static std::atomic<unsigned long> next_track_id(0xdead);
     return (next_track_id++);
 }
 
-guest_context_tracker::guest_context_tracker(int cpu_id) :
-        cpu_track_(perfetto::Track::Global(gen_track_uuid()))
+perfetto::protos::pbzero::ModeSwitch
+qemu_cpu_mode_to_trace(qemu_log_instr_cpu_mode_t mode)
+{
+    // NOTE: We rely on the fact that the protobuf enum ModeSwitch
+    // uses the same numbering as qemu_log_instr_cpu_mode_t
+    return static_cast<perfetto::protos::pbzero::ModeSwitch>(mode);
+}
+
+/* static */
+std::shared_ptr<qemu_context_track>
+qemu_context_track::make_context_track(qemu_ctx_id key)
+{
+    return std::make_shared<qemu_context_track>(key);
+}
+
+qemu_context_track::qemu_ctx_id qemu_context_track::get_id() const
+{
+    return std::make_tuple(pid, tid, cid, mode);
+}
+
+perfetto::protos::gen::TrackDescriptor qemu_context_track::Serialize() const
+{
+    auto desc = Track::Serialize();
+    auto qemu_desc = desc.mutable_qemu_context();
+    qemu_desc->set_pid(pid);
+    qemu_desc->set_tid(tid);
+    qemu_desc->set_cid(cid);
+    // qemu_desc->set_mode(mode); // TODO not yet in cheri-perfetto
+    return desc;
+}
+
+qemu_context_track::qemu_context_track(qemu_ctx_id key)
+    : perfetto::Track(cheri::gen_track_uuid(), perfetto::Track()),
+      pid(std::get<0>(key)), tid(std::get<1>(key)), cid(std::get<2>(key)),
+      mode(std::get<3>(key))
+{
+}
+
+qemu_context_track::~qemu_context_track()
+{
+    perfetto::TrackEvent::EraseTrackDescriptor(*this);
+}
+
+guest_context_tracker::guest_context_tracker(int cpu_id)
+    : cpu_track_(perfetto::Track::Global(gen_track_uuid()))
 {
     std::string track_name("CPU " + std::to_string(cpu_id));
     auto desc = cpu_track_.Serialize();
@@ -121,51 +125,81 @@ guest_context_tracker::guest_context_tracker(int cpu_id) :
     perfetto::TrackEvent::SetTrackDescriptor(cpu_track_, desc);
 }
 
-void
-guest_context_tracker::context_update(const log_event_ctx_update_t *evt)
+void guest_context_tracker::mode_update(
+    perfetto::protos::pbzero::ModeSwitch new_mode)
+{
+    if (ctx_track_ == nullptr)
+        return;
+
+    auto key = ctx_track_->get_id();
+    std::get<3>(key) = new_mode;
+    /* Check if we have a track for this mode */
+    std::shared_ptr<qemu_context_track> track;
+    {
+        std::lock_guard<std::mutex> tracks_guard(tracks_lock);
+        auto found_track_iter = tracks.find(key);
+        if (found_track_iter == tracks.end()) {
+            track = qemu_context_track::make_context_track(key);
+            auto desc = track->Serialize();
+            desc.set_name("CTX " + std::to_string(track->pid) + ":" +
+                          std::to_string(track->tid) + ":" +
+                          std::to_string(track->cid) + ":" +
+                          std::to_string(track->mode));
+            perfetto::TrackEvent::SetTrackDescriptor(*track, desc);
+            tracks.emplace(key, track);
+        } else {
+            /* Existing context */
+            track = found_track_iter->second;
+        }
+    }
+    ctx_track_ = track;
+}
+
+void guest_context_tracker::context_update(const log_event_ctx_update_t *evt)
 {
     // Currently this is the only one existing
     assert(evt->op == LOG_EVENT_CTX_OP_SWITCH && "Invalid ctx update op");
     /*
-     * Find or create the track associated with the new context and set it as the current
-     * context track on the CPU. This will be used to log per-context events.
-     * XXX-AM: Currently it is assumed that contexts are unique, this is clearly not the case
-     * with PID/TID reuse and we should do something to detect this. This is mostly relevant
-     * only for long-lived traces.
+     * Find or create the track associated with the new context and set it as
+     * the current context track on the CPU. This will be used to log
+     * per-context events.
+     * XXX-AM: Currently it is assumed that contexts are unique, this is clearly
+     * not the case with PID/TID reuse and we should do something to detect
+     * this. This seems mostly relevant only for long-lived traces.
      */
 
+    auto mode = qemu_cpu_mode_to_trace(evt->mode);
     /* Fetch the tracks for the new context */
-    std::lock_guard<std::mutex> tracks_guard(tracks_lock);
-    qemu_context_track::qemu_ctx_id key = std::make_tuple(evt->pid, evt->tid, evt->cid);
-    auto ctx_track_iter = tracks.find(key);
+    qemu_context_track::qemu_ctx_id key =
+        std::make_tuple(evt->pid, evt->tid, evt->cid, mode);
     std::shared_ptr<qemu_context_track> track;
-    if (ctx_track_iter == tracks.end()) {
-        /* New context */
-        track = qemu_context_track::make_context_track(evt->pid, evt->tid, evt->cid);
-        auto desc = track->Serialize();
-        desc.set_name("CTX " + std::to_string(evt->pid) + ":" +
-                      std::to_string(evt->tid) + ":" +
-                      std::to_string(evt->cid));
-        perfetto::TrackEvent::SetTrackDescriptor(*track, desc);
-        tracks.emplace(key, track);
-    } else {
-        /* Existing context */
-        track = ctx_track_iter->second;
+    {
+        std::lock_guard<std::mutex> tracks_guard(tracks_lock);
+        auto ctx_track_iter = tracks.find(key);
+        if (ctx_track_iter == tracks.end()) {
+            /* New context */
+            track = qemu_context_track::make_context_track(key);
+            auto desc = track->Serialize();
+            desc.set_name("CTX " + std::to_string(evt->pid) + ":" +
+                          std::to_string(evt->tid) + ":" +
+                          std::to_string(evt->cid) + ":" +
+                          std::to_string(mode));
+            perfetto::TrackEvent::SetTrackDescriptor(*track, desc);
+            tracks.emplace(key, track);
+        } else {
+            /* Existing context */
+            track = ctx_track_iter->second;
+        }
     }
     ctx_track_ = track;
-    /* XXX-AM: for debugging, remove */
-    TRACE_EVENT_INSTANT("sched", "switch", cpu_track_,
-                        "pid", evt->pid, "tid", evt->tid, "cid", evt->cid);
 }
 
-perfetto::Track&
-guest_context_tracker::get_cpu_track()
+perfetto::Track &guest_context_tracker::get_cpu_track()
 {
     return cpu_track_;
 }
 
-perfetto::Track&
-guest_context_tracker::get_ctx_track()
+perfetto::Track &guest_context_tracker::get_ctx_track()
 {
     if (ctx_track_)
         return *ctx_track_;
@@ -173,4 +207,4 @@ guest_context_tracker::get_ctx_track()
         return cpu_track_;
 }
 
-} /* cheri */
+} // namespace cheri
diff --git a/trace_extra/guest_context_tracker.hh b/trace_extra/guest_context_tracker.hh
index 825d603d564..57629e4112a 100644
--- a/trace_extra/guest_context_tracker.hh
+++ b/trace_extra/guest_context_tracker.hh
@@ -35,6 +35,29 @@
 
 namespace cheri
 {
+
+struct qemu_context_track : public perfetto::Track {
+
+    using cpu_mode_type = perfetto::protos::pbzero::ModeSwitch;
+    /* (pid, tid, cid, mode) */
+    using qemu_ctx_id = std::tuple<uint64_t, uint64_t, uint64_t, cpu_mode_type>;
+
+    const uint64_t pid;
+    const uint64_t tid;
+    const uint64_t cid;
+    const cpu_mode_type mode;
+
+    static std::shared_ptr<qemu_context_track>
+    make_context_track(qemu_ctx_id key);
+
+    qemu_ctx_id get_id() const;
+
+    perfetto::protos::gen::TrackDescriptor Serialize() const;
+
+    qemu_context_track(qemu_ctx_id key);
+    ~qemu_context_track();
+};
+
 /*
  * Track current guest context associated to a CPU.
  * This allows to charge events to the correct context by recording
@@ -45,22 +68,25 @@ namespace cheri
  *    or where the context is not relevant.
  * 2. Each new process in the system gets allocated a new perfetto::ProcessTrack
  *    TODO: Currently there is no way to notify when the PID is discarded.
- * 3. Each new thread in the system gets allocated a new perfetto::ThreadTrack and
- *    the corresponding process as parent.
- * 4. Each new compartment ID gets allocated a new perfetto::CheriCompartmentTrack.
- *    Note that compartment tracks are not associated to a single thread, instead
- *    multiple threads may record events there.
+ * 3. Each new thread in the system gets allocated a new perfetto::ThreadTrack
+ * and the corresponding process as parent.
+ * 4. Each new compartment ID gets allocated a new
+ * perfetto::CheriCompartmentTrack. Note that compartment tracks are not
+ * associated to a single thread, instead multiple threads may record events
+ * there.
  * 5. TODO: The context shoud include information on the EL and address space
  */
 class guest_context_tracker
 {
     perfetto::Track cpu_track_;
-    std::shared_ptr<perfetto::Track> ctx_track_;
+    std::shared_ptr<qemu_context_track> ctx_track_;
+
   public:
     guest_context_tracker(int cpu_id);
     void context_update(const log_event_ctx_update_t *evt);
-    perfetto::Track& get_cpu_track();
-    perfetto::Track& get_ctx_track();
+    void mode_update(perfetto::protos::pbzero::ModeSwitch new_mode);
+    perfetto::Track &get_cpu_track();
+    perfetto::Track &get_ctx_track();
 };
 
 /*
@@ -70,4 +96,7 @@ class guest_context_tracker
  */
 unsigned long gen_track_uuid();
 
-} /* cheri */
+perfetto::protos::pbzero::ModeSwitch
+qemu_cpu_mode_to_trace(qemu_log_instr_cpu_mode_t mode);
+
+} // namespace cheri
diff --git a/trace_extra/trace_perfetto.cc b/trace_extra/trace_perfetto.cc
index 86349c8a142..fec12fe7b0b 100644
--- a/trace_extra/trace_perfetto.cc
+++ b/trace_extra/trace_perfetto.cc
@@ -80,6 +80,12 @@ struct perfetto_backend_data
     // Per-CPU aggregate statistics
     cheri::qemu_stats stats_;
     cheri::guest_context_tracker ctx_tracker_;
+    /*
+     * Cached copy of the logging activation status.
+     * It is easier to keep a cached copy here than exposing the loglevel_active
+     * flag to the C++ interface.
+     */
+    bool loglevel_active_;
 
     perfetto_backend_data(int cpu_id) :
         ctrl_track_(perfetto::Track::Global(cheri::gen_track_uuid())),
@@ -158,14 +164,6 @@ void perfetto_tracing_stop(void)
     session->StopBlocking();
 }
 
-perfetto::protos::pbzero::ModeSwitch
-qemu_cpu_mode_to_trace(qemu_log_instr_cpu_mode_t mode)
-{
-    // NOTE: We rely on the fact that the protobuf enum ModeSwitch
-    // uses the same numbering as qemu_log_instr_cpu_mode_t
-    return static_cast<perfetto::protos::pbzero::ModeSwitch>(mode);
-}
-
 void
 trace_cap_register(perfetto::protos::pbzero::Capability *cap,
                    cap_register_handle chandle)
@@ -212,10 +210,19 @@ process_events(perfetto_backend_data *data, cpu_log_entry_handle entry)
             }
         } else if (evt->id == LOG_EVENT_CTX_UPDATE) {
             // Dump the stats for the current context to the right context track
-            data->stats_.flush(data->ctx_tracker_.get_ctx_track());
+            data->stats_.pause(data->ctx_tracker_.get_ctx_track(), perfetto_log_entry_pc(entry));
             data->ctx_tracker_.context_update(&evt->ctx_update);
+            data->stats_.unpause(data->ctx_tracker_.get_ctx_track(), perfetto_log_entry_pc(entry));
         }
     }
+    if (perfetto_log_entry_flags(entry) & LI_FLAG_MODE_SWITCH) {
+        auto mode = cheri::qemu_cpu_mode_to_trace(perfetto_log_entry_next_cpu_mode(entry));
+        // XXX-AM: consider making the mode switch an event, possibly with its own separate pc signaling
+        // the PC of the next instruction we are landing to?
+        data->stats_.pause(data->ctx_tracker_.get_ctx_track(), perfetto_log_entry_pc(entry));
+        data->ctx_tracker_.mode_update(mode);
+        data->stats_.unpause(data->ctx_tracker_.get_ctx_track(), perfetto_log_entry_pc(entry));
+    }
 
     /*
      * If we have instruction data, we assume that tracing is enabled and run extra
@@ -322,7 +329,7 @@ process_instr(perfetto_backend_data *data, cpu_log_entry_handle entry)
                 trap->set_trap_number(perfetto_log_entry_intr_code(entry));
             }
             if (flags & LI_FLAG_MODE_SWITCH) {
-                auto mode = qemu_cpu_mode_to_trace(perfetto_log_entry_next_cpu_mode(entry));
+                auto mode = cheri::qemu_cpu_mode_to_trace(perfetto_log_entry_next_cpu_mode(entry));
                 instr->set_mode(mode);
             }
         });
diff --git a/trace_extra/trace_stats.cc b/trace_extra/trace_stats.cc
index df8403371da..f3c2be0a0dc 100644
--- a/trace_extra/trace_stats.cc
+++ b/trace_extra/trace_stats.cc
@@ -50,7 +50,7 @@ void qemu_stats::flush(perfetto::Track &track)
                         [&](perfetto::EventContext ctx) {
                             auto *qemu_arg = ctx.event()->set_qemu();
                             auto *hist = qemu_arg->set_histogram();
-                            for (const auto &keyval : bb_hit_map) {
+                            for (const auto &keyval : bb_hit_map_) {
                                 auto &range = keyval.first;
                                 auto *bucket = hist->add_bucket();
                                 bucket->set_start(range.lower());
@@ -63,7 +63,7 @@ void qemu_stats::flush(perfetto::Track &track)
                         [&](perfetto::EventContext ctx) {
                             auto *qemu_arg = ctx.event()->set_qemu();
                             auto *hist = qemu_arg->set_histogram();
-                            for (const auto &keyval : branch_map) {
+                            for (const auto &keyval : branch_map_) {
                                 auto *bucket = hist->add_bucket();
                                 bucket->set_start(keyval.first);
                                 bucket->set_value(keyval.second);
@@ -75,13 +75,16 @@ void qemu_stats::flush(perfetto::Track &track)
 
 void qemu_stats::pause(perfetto::Track &track, uint64_t pc)
 {
-    /*
-     * Treat this as the end of the block, otherwise the
-     * current PC slice will be lost.
-     */
-    bb_hit_map +=
-        std::make_pair(addr_range::right_open(pc_range_start, pc), icount);
-    pc_range_start = last_pc = icount = 0;
+    if (paused_)
+        return;
+    paused_ = true;
+    assert(pc_range_start_ != 0 && "PC bb start was not updated");
+    assert(pc != 0 && "pause PC can not be zero");
+    /* Treat this as the end of the block, otherwise the current PC slice will
+     * be lost. */
+    bb_hit_map_ +=
+        std::make_pair(addr_range::right_open(pc_range_start_, pc), icount_);
+    pc_range_start_ = last_pc_ = next_instr_pc_ = icount_ = 0;
     /*
      * Whenever we pause tracing, we need to sync the stats to trace,
      * otherwise we may charge the wrong context if tracing
@@ -92,16 +95,20 @@ void qemu_stats::pause(perfetto::Track &track, uint64_t pc)
 
 void qemu_stats::unpause(perfetto::Track &track, uint64_t pc)
 {
+    if (!paused_)
+        return;
+    paused_ = false;
     /* Reset the pc-tracking state */
-    pc_range_start = last_pc = pc;
+    pc_range_start_ = last_pc_ = pc;
+    next_instr_pc_ = -1;
     /* unpause will be called on the first instruction to be traced? */
-    icount = 1; // Unpause is called for the first instruction being traced
+    icount_ = 1; // Unpause is called for the first instruction being traced
 }
 
 void qemu_stats::clear()
 {
-    bb_hit_map.clear();
-    branch_map.clear();
+    bb_hit_map_.clear();
+    branch_map_.clear();
 }
 
 void qemu_stats::process_instr(perfetto::Track &ctx_track,
@@ -109,18 +116,23 @@ void qemu_stats::process_instr(perfetto::Track &ctx_track,
 {
     uint64_t pc = perfetto_log_entry_pc(entry);
     int size = perfetto_log_entry_insn_size(entry);
-    if (pc - last_pc > size) {
+    // When unpausing: accept anything as a next instruction when next_instr_pc
+    // == -1
+    if (pc != next_instr_pc_ && next_instr_pc_ != -1) {
         // presume we are branching
-        bb_hit_map += std::make_pair(
-            addr_range::right_open(pc_range_start, last_pc), icount);
-        pc_range_start = pc;
-        icount =
+        assert(pc_range_start_ != 0 && "PC bb start was not updated");
+        assert(last_pc_ != 0 && "process last_pc can not be zero");
+        bb_hit_map_ += std::make_pair(
+            addr_range::right_open(pc_range_start_, last_pc_), icount_);
+        pc_range_start_ = pc;
+        icount_ =
             1; // Reset icount triggered on the first instruction of the next BB
-        branch_map[pc] += 1;
+        branch_map_[pc] += 1;
     } else {
-        icount += 1;
+        icount_ += 1;
     }
-    last_pc = pc;
+    next_instr_pc_ = pc + size;
+    last_pc_ = pc;
 }
 
 } // namespace cheri
diff --git a/trace_extra/trace_stats.hh b/trace_extra/trace_stats.hh
index a9845121a6d..77755a895fc 100644
--- a/trace_extra/trace_stats.hh
+++ b/trace_extra/trace_stats.hh
@@ -44,22 +44,25 @@ class qemu_stats
     /*
      * Histogram of basic-block hit during tracing.
      */
-    boost::icl::interval_map<uint64_t, uint64_t> bb_hit_map;
+    boost::icl::interval_map<uint64_t, uint64_t> bb_hit_map_;
     /*
      * Histogram of branch instruction addresses.
      * Post-processing must be used to distinguish between function
      * calls and conditional branches.
      */
-    std::map<uint64_t, uint64_t> branch_map;
+    std::map<uint64_t, uint64_t> branch_map_;
     /*
      *  PC-tracking info to detect branches.
      */
-    uint64_t last_pc;
-    uint64_t pc_range_start;
-    uint64_t icount;
+    uint64_t last_pc_;
+    uint64_t next_instr_pc_;
+    uint64_t pc_range_start_;
+    uint64_t icount_;
+    /* Protect against double pause() calls */
+    bool paused_;
 
   public:
-    qemu_stats() : last_pc(0), pc_range_start(0), icount(0) {}
+    qemu_stats() : last_pc_(0), pc_range_start_(0), icount_(0), paused_(true) {}
     void process_instr(perfetto::Track &track, cpu_log_entry_handle entry);
     void pause(perfetto::Track &track, uint64_t pc);
     void unpause(perfetto::Track &track, uint64_t pc);

From 472de436dcd1b63c11bc401e73a51a9d9c2d850c Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Tue, 2 Nov 2021 11:26:52 +0000
Subject: [PATCH 25/74] Fix inline function prototype to silence warnings.

---
 include/qemu/log_instr.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/include/qemu/log_instr.h b/include/qemu/log_instr.h
index ae7f264d0a3..34f47152377 100644
--- a/include/qemu/log_instr.h
+++ b/include/qemu/log_instr.h
@@ -168,7 +168,7 @@ static inline void qemu_log_instr_set_backend(qemu_log_instr_backend_t id)
     qemu_log_instr_backend = id;
 }
 
-static inline qemu_log_instr_backend_t qemu_log_instr_get_backend()
+static inline qemu_log_instr_backend_t qemu_log_instr_get_backend(void)
 {
     return qemu_log_instr_backend;
 }

From 9d0bd2c0b70339b6c28f71cbabcd5044f1416485 Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Tue, 2 Nov 2021 13:46:24 +0000
Subject: [PATCH 26/74] Update user logging to match the use of
 QEMU_LOG_INSTR_U.

---
 accel/tcg/log_instr.c      | 26 +++++++++++++++++++-------
 accel/tcg/log_instr_text.c |  9 +++++----
 include/qemu/log_instr.h   |  2 +-
 util/log.c                 |  1 +
 4 files changed, 26 insertions(+), 12 deletions(-)

diff --git a/accel/tcg/log_instr.c b/accel/tcg/log_instr.c
index 90b0cdf561b..dbe0f12ead6 100644
--- a/accel/tcg/log_instr.c
+++ b/accel/tcg/log_instr.c
@@ -354,8 +354,14 @@ int qemu_log_instr_global_switch(int log_flags)
     CPUState *cpu;
     qemu_log_next_level_arg_t *arg = g_new(qemu_log_next_level_arg_t, 1);
 
-    arg->next_level = (request_stop) ? QEMU_LOG_INSTR_LOGLEVEL_NONE
-                                     : QEMU_LOG_INSTR_LOGLEVEL_ALL;
+    if (log_flags & CPU_LOG_INSTR_U) {
+        arg->next_level = QEMU_LOG_INSTR_LOGLEVEL_USER;
+        log_flags |= CPU_LOG_INSTR;
+    } else if (log_flags & CPU_LOG_INSTR) {
+        arg->next_level = QEMU_LOG_INSTR_LOGLEVEL_ALL;
+    } else {
+        arg->next_level = QEMU_LOG_INSTR_LOGLEVEL_NONE;
+    }
     arg->global = true;
 
     CPU_FOREACH(cpu)
@@ -447,8 +453,14 @@ void qemu_log_instr_init(CPUState *cpu)
     }
 
     /* If we are starting with instruction logging enabled, switch it on now */
-    if (qemu_loglevel_mask(CPU_LOG_INSTR)) {
-        start_level.next_level = QEMU_LOG_INSTR_LOGLEVEL_ALL;
+    if (qemu_loglevel_mask(CPU_LOG_INSTR | CPU_LOG_INSTR_U)) {
+        if (qemu_loglevel_mask(CPU_LOG_INSTR_U)) {
+            assert(qemu_loglevel_mask(CPU_LOG_INSTR) &&
+                   "CPU_LOG_INSTR_U implies CPU_LOG_INSTR broken");
+            start_level.next_level = QEMU_LOG_INSTR_LOGLEVEL_USER;
+        } else {
+            start_level.next_level = QEMU_LOG_INSTR_LOGLEVEL_ALL;
+        }
         start_level.global = true;
         do_cpu_loglevel_switch(cpu, RUN_ON_CPU_HOST_PTR(&start_level));
     }
@@ -1249,7 +1261,7 @@ void helper_qemu_log_instr_stop(CPUArchState *env, target_ulong pc)
 }
 
 /* Start logging all instructions on all CPUs */
-void helper_qemu_log_instr_allcpu_start()
+void helper_qemu_log_instr_allcpu_start(void)
 {
     CPUState *cpu;
 
@@ -1261,7 +1273,7 @@ void helper_qemu_log_instr_allcpu_start()
 }
 
 /* Start logging user-only instructions on all CPUs */
-void helper_qemu_log_instr_allcpu_user_start()
+void helper_qemu_log_instr_allcpu_user_start(void)
 {
     CPUState *cpu;
 
@@ -1273,7 +1285,7 @@ void helper_qemu_log_instr_allcpu_user_start()
 }
 
 /* Stop logging instructions on all CPUs */
-void helper_qemu_log_instr_allcpu_stop()
+void helper_qemu_log_instr_allcpu_stop(void)
 {
     CPUState *cpu;
 
diff --git a/accel/tcg/log_instr_text.c b/accel/tcg/log_instr_text.c
index 0cd8f054ff6..c601b3fcf64 100644
--- a/accel/tcg/log_instr_text.c
+++ b/accel/tcg/log_instr_text.c
@@ -202,12 +202,13 @@ void emit_text_instr(CPUArchState *env, cpu_log_entry_t *entry)
             if (cpulog->loglevel == QEMU_LOG_INSTR_LOGLEVEL_USER) {
                 qemu_log(
                     "[%u:%u] %s user-mode only instruction logging @ %lx\n",
-                    env_cpu(env)->cpu_index, cpu_get_asid(env), log_state_op,
-                    event->state.pc);
+                    env_cpu(env)->cpu_index, cpu_get_asid(env, event->state.pc),
+                    log_state_op, event->state.pc);
             } else {
                 qemu_log("[%u:%u] %s instruction logging @ %lx\n",
-                         env_cpu(env)->cpu_index, cpu_get_asid(env),
-                         log_state_op, event->state.pc);
+                         env_cpu(env)->cpu_index,
+                         cpu_get_asid(env, event->state.pc), log_state_op,
+                         event->state.pc);
             }
             break;
         case LOG_EVENT_CTX_UPDATE:
diff --git a/include/qemu/log_instr.h b/include/qemu/log_instr.h
index 34f47152377..e46026d132b 100644
--- a/include/qemu/log_instr.h
+++ b/include/qemu/log_instr.h
@@ -253,7 +253,7 @@ void qemu_log_instr_sync_buffers(void);
 /*
  * Global instruction logging hook from qemu tracing commands.
  */
-void qemu_log_instr_global_switch(bool request_stop);
+int qemu_log_instr_global_switch(int log_flags);
 
 /*
  * Update the ring buffer size.
diff --git a/util/log.c b/util/log.c
index 071fdc4f04f..101c9f2f97e 100644
--- a/util/log.c
+++ b/util/log.c
@@ -40,6 +40,7 @@ __attribute__((weak)) int qemu_log_instr_global_switch(int log_flags);
 __attribute__((weak)) int qemu_log_instr_global_switch(int log_flags)
 {
     /* Real implementation in accel/tcg/log_instr.c. */
+    warn_report("Calling no-op %s\r", __func__);
     return log_flags;
 }
 

From f7616d68db7470ccc1f075b84809c9e7ca103197 Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Wed, 3 Nov 2021 15:56:47 +0000
Subject: [PATCH 27/74] Handle unlikely case for qemu perfetto stats recording.

Avoid creating histogram buckets starting at 0 when pause is called
but no instructions have been executed yet.
---
 trace_extra/trace_stats.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/trace_extra/trace_stats.cc b/trace_extra/trace_stats.cc
index f3c2be0a0dc..100127f084f 100644
--- a/trace_extra/trace_stats.cc
+++ b/trace_extra/trace_stats.cc
@@ -78,7 +78,9 @@ void qemu_stats::pause(perfetto::Track &track, uint64_t pc)
     if (paused_)
         return;
     paused_ = true;
-    assert(pc_range_start_ != 0 && "PC bb start was not updated");
+    /* Nothing to flush */
+    if (pc_range_start_ == 0)
+        return;
     assert(pc != 0 && "pause PC can not be zero");
     /* Treat this as the end of the block, otherwise the current PC slice will
      * be lost. */

From 4db533778a54d68db90c89adb06fe7cc2e58ddf1 Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Mon, 8 Nov 2021 15:48:09 +0000
Subject: [PATCH 28/74] Refactor the way instruction logging statistics are
 handled.

 - Use a separate statistics object for each context detected.
 - Rate-limit the dumping of the stats histogram to trace so that we
   do not uselessy generate huge traces.
---
 trace_extra/guest_context_tracker.cc | 24 +++++++++++++++----
 trace_extra/guest_context_tracker.hh | 35 +++++++++++++++++++++++-----
 trace_extra/trace_perfetto.cc        | 26 +++++++++++++--------
 trace_extra/trace_stats.cc           | 17 ++++++++++----
 trace_extra/trace_stats.hh           |  9 ++++++-
 5 files changed, 85 insertions(+), 26 deletions(-)

diff --git a/trace_extra/guest_context_tracker.cc b/trace_extra/guest_context_tracker.cc
index ebaa0309593..372e6591568 100644
--- a/trace_extra/guest_context_tracker.cc
+++ b/trace_extra/guest_context_tracker.cc
@@ -34,10 +34,8 @@
 namespace
 {
 /*
- * Dynamically created tracks.
- * Here we do not commit to a hierarchical model of process/thread, as context
- * events are guest-driven. Instead we defer any hierarchical interpretation of
- * the context identifier to post-processing.
+ * Dynamically created tracks and track data.
+ * The tracks lock protects both tracks and ctx_data.
  */
 std::mutex tracks_lock;
 std::map<cheri::qemu_context_track::qemu_ctx_id,
@@ -194,6 +192,16 @@ void guest_context_tracker::context_update(const log_event_ctx_update_t *evt)
     ctx_track_ = track;
 }
 
+void guest_context_tracker::flush_all_ctx_data()
+{
+    /* Flush per-CPU stats */
+    cpu_ctx_data_.stats.flush(cpu_track_);
+    for (auto &id_and_track : tracks) {
+        qemu_context_data &ctx_data = id_and_track.second->ctx_data;
+        ctx_data.stats.flush(*id_and_track.second);
+    }
+}
+
 perfetto::Track &guest_context_tracker::get_cpu_track()
 {
     return cpu_track_;
@@ -207,4 +215,12 @@ perfetto::Track &guest_context_tracker::get_ctx_track()
         return cpu_track_;
 }
 
+qemu_context_data &guest_context_tracker::get_ctx_data()
+{
+    if (ctx_track_)
+        return ctx_track_->ctx_data;
+    else
+        return cpu_ctx_data_;
+}
+
 } // namespace cheri
diff --git a/trace_extra/guest_context_tracker.hh b/trace_extra/guest_context_tracker.hh
index 57629e4112a..8019ffc173a 100644
--- a/trace_extra/guest_context_tracker.hh
+++ b/trace_extra/guest_context_tracker.hh
@@ -29,13 +29,24 @@
 #pragma once
 
 #include <memory>
+#include <iterator>
+#include <utility>
 #include <perfetto.h>
 #include "qemu/log_instr.h"
 #include "exec/log_instr_perfetto.h"
+#include "trace_extra/trace_stats.hh"
 
 namespace cheri
 {
 
+/*
+ * Per-context data.
+ * This is allocated and switched together with the context track.
+ */
+struct qemu_context_data {
+    qemu_stats stats;
+};
+
 struct qemu_context_track : public perfetto::Track {
 
     using cpu_mode_type = perfetto::protos::pbzero::ModeSwitch;
@@ -46,6 +57,7 @@ struct qemu_context_track : public perfetto::Track {
     const uint64_t tid;
     const uint64_t cid;
     const cpu_mode_type mode;
+    qemu_context_data ctx_data;
 
     static std::shared_ptr<qemu_context_track>
     make_context_track(qemu_ctx_id key);
@@ -64,29 +76,40 @@ struct qemu_context_track : public perfetto::Track {
  * what is currently active on a CPU.
  * There is one guest_context_tracker per virtual CPU.
  * We manage the following tracks for each CPU:
- * 1. A global CPU N track, for events charged to a CPU from an unknown context
+ * - A global CPU N track, for events charged to a CPU from an unknown context
  *    or where the context is not relevant.
- * 2. Each new process in the system gets allocated a new perfetto::ProcessTrack
+ * - Each new process in the system gets allocated a new perfetto::ProcessTrack
  *    TODO: Currently there is no way to notify when the PID is discarded.
- * 3. Each new thread in the system gets allocated a new perfetto::ThreadTrack
+ * - Each new thread in the system gets allocated a new perfetto::ThreadTrack
  * and the corresponding process as parent.
- * 4. Each new compartment ID gets allocated a new
+ * - TODO Each new compartment ID gets allocated a new
  * perfetto::CheriCompartmentTrack. Note that compartment tracks are not
  * associated to a single thread, instead multiple threads may record events
- * there.
- * 5. TODO: The context shoud include information on the EL and address space
+ * there. Each new context can have associated data that is charged to that
+ * context and only emitted to the track periodically. This is to avoid
+ * generating too much data if it can be aggregated. Other options could be to
+ * avoid the track subsystem and use a custom perfetto data source with
+ * respective changes in the trace processor.
+ * TODO:
+ *  - The context shoud include information on address space
  */
 class guest_context_tracker
 {
+    /* CPU track for events not assigned (or assignable) to a single context */
     perfetto::Track cpu_track_;
+    /* context data for the CPU track */
+    qemu_context_data cpu_ctx_data_;
+    /* Currently active context track */
     std::shared_ptr<qemu_context_track> ctx_track_;
 
   public:
     guest_context_tracker(int cpu_id);
     void context_update(const log_event_ctx_update_t *evt);
     void mode_update(perfetto::protos::pbzero::ModeSwitch new_mode);
+    void flush_all_ctx_data();
     perfetto::Track &get_cpu_track();
     perfetto::Track &get_ctx_track();
+    qemu_context_data &get_ctx_data();
 };
 
 /*
diff --git a/trace_extra/trace_perfetto.cc b/trace_extra/trace_perfetto.cc
index fec12fe7b0b..22284da8730 100644
--- a/trace_extra/trace_perfetto.cc
+++ b/trace_extra/trace_perfetto.cc
@@ -77,8 +77,7 @@ struct perfetto_backend_data
 {
     // Per-CPU control track. This records tracing control events.
     perfetto::Track ctrl_track_;
-    // Per-CPU aggregate statistics
-    cheri::qemu_stats stats_;
+    // Tracker that resolves the current context scheduled on a CPU
     cheri::guest_context_tracker ctx_tracker_;
     /*
      * Cached copy of the logging activation status.
@@ -182,6 +181,8 @@ process_events(perfetto_backend_data *data, cpu_log_entry_handle entry)
 {
     int nevents = perfetto_log_entry_events(entry);
     bool have_startstop_event = false;
+    auto *curr_ctx_data = &data->ctx_tracker_.get_ctx_data();
+    auto *curr_ctx_track = &data->ctx_tracker_.get_ctx_track();
 
     /*
      * Note: LOG_EVENT_STATE events are emitted even when tracing is disabled.
@@ -193,15 +194,16 @@ process_events(perfetto_backend_data *data, cpu_log_entry_handle entry)
             switch (evt->state.next_state) {
                 case LOG_EVENT_STATE_FLUSH:
                     TRACE_EVENT_INSTANT("ctrl", "flush", data->ctrl_track_);
+                    data->ctx_tracker_.flush_all_ctx_data();
                     perfetto::TrackEvent::Flush();
                     break;
                 case LOG_EVENT_STATE_START:
-                    data->stats_.unpause(data->ctx_tracker_.get_ctx_track(), evt->state.pc);
+                    curr_ctx_data->stats.unpause(*curr_ctx_track, evt->state.pc);
                     TRACE_EVENT_BEGIN("ctrl", "tracing", data->ctrl_track_);
                     have_startstop_event = true;
                     break;
                 case LOG_EVENT_STATE_STOP:
-                    data->stats_.pause(data->ctx_tracker_.get_ctx_track(), evt->state.pc);
+                    curr_ctx_data->stats.pause(*curr_ctx_track, evt->state.pc);
                     TRACE_EVENT_END("ctrl", data->ctrl_track_);
                     have_startstop_event = true;
                     break;
@@ -209,19 +211,23 @@ process_events(perfetto_backend_data *data, cpu_log_entry_handle entry)
                     assert(false && "Invalid state event");
             }
         } else if (evt->id == LOG_EVENT_CTX_UPDATE) {
-            // Dump the stats for the current context to the right context track
-            data->stats_.pause(data->ctx_tracker_.get_ctx_track(), perfetto_log_entry_pc(entry));
+            // Swap current context.
+            curr_ctx_data->stats.pause(*curr_ctx_track, perfetto_log_entry_pc(entry));
             data->ctx_tracker_.context_update(&evt->ctx_update);
-            data->stats_.unpause(data->ctx_tracker_.get_ctx_track(), perfetto_log_entry_pc(entry));
+            curr_ctx_data = &data->ctx_tracker_.get_ctx_data();
+            curr_ctx_track = &data->ctx_tracker_.get_ctx_track();
+            curr_ctx_data->stats.unpause(*curr_ctx_track, perfetto_log_entry_pc(entry));
         }
     }
     if (perfetto_log_entry_flags(entry) & LI_FLAG_MODE_SWITCH) {
         auto mode = cheri::qemu_cpu_mode_to_trace(perfetto_log_entry_next_cpu_mode(entry));
         // XXX-AM: consider making the mode switch an event, possibly with its own separate pc signaling
         // the PC of the next instruction we are landing to?
-        data->stats_.pause(data->ctx_tracker_.get_ctx_track(), perfetto_log_entry_pc(entry));
+        curr_ctx_data->stats.pause(*curr_ctx_track, perfetto_log_entry_pc(entry));
         data->ctx_tracker_.mode_update(mode);
-        data->stats_.unpause(data->ctx_tracker_.get_ctx_track(), perfetto_log_entry_pc(entry));
+        curr_ctx_data = &data->ctx_tracker_.get_ctx_data();
+        curr_ctx_track = &data->ctx_tracker_.get_ctx_track();
+        curr_ctx_data->stats.unpause(*curr_ctx_track, perfetto_log_entry_pc(entry));
     }
 
     /*
@@ -230,7 +236,7 @@ process_events(perfetto_backend_data *data, cpu_log_entry_handle entry)
      * This avoids having to keep a shadow copy of the tracing state in the backend.
      */
     if (!have_startstop_event && (perfetto_log_entry_flags(entry) & LI_FLAG_HAS_INSTR_DATA) != 0)
-        data->stats_.process_instr(data->ctx_tracker_.get_ctx_track(), entry);
+        curr_ctx_data->stats.process_instr(*curr_ctx_track, entry);
 }
 
 void
diff --git a/trace_extra/trace_stats.cc b/trace_extra/trace_stats.cc
index 100127f084f..57083669dc3 100644
--- a/trace_extra/trace_stats.cc
+++ b/trace_extra/trace_stats.cc
@@ -31,6 +31,7 @@
 #include "qemu/log_instr.h"
 #include "trace_extra/trace_stats.hh"
 
+using namespace std::chrono_literals;
 namespace icl = boost::icl;
 
 using addr_range = icl::interval<uint64_t>;
@@ -88,11 +89,17 @@ void qemu_stats::pause(perfetto::Track &track, uint64_t pc)
         std::make_pair(addr_range::right_open(pc_range_start_, pc), icount_);
     pc_range_start_ = last_pc_ = next_instr_pc_ = icount_ = 0;
     /*
-     * Whenever we pause tracing, we need to sync the stats to trace,
-     * otherwise we may charge the wrong context if tracing
-     * is restarted in another thread.
+     * Whenever we pause tracing, we may sync the stats to trace.
+     * This is to avoid having to dump packets too large into the trace, however
+     * we could get around this by just dumping the full stats only when a flush
+     * is triggered.
      */
-    flush(track);
+    auto now = std::chrono::steady_clock::now();
+    auto interval = now - last_flush_time_;
+    if (interval > 1s) {
+        last_flush_time_ = now;
+        flush(track);
+    }
 }
 
 void qemu_stats::unpause(perfetto::Track &track, uint64_t pc)
@@ -120,7 +127,7 @@ void qemu_stats::process_instr(perfetto::Track &ctx_track,
     int size = perfetto_log_entry_insn_size(entry);
     // When unpausing: accept anything as a next instruction when next_instr_pc
     // == -1
-    if (pc != next_instr_pc_ && next_instr_pc_ != -1) {
+    if (pc != next_instr_pc_ && next_instr_pc_ != -1UL) {
         // presume we are branching
         assert(pc_range_start_ != 0 && "PC bb start was not updated");
         assert(last_pc_ != 0 && "process last_pc can not be zero");
diff --git a/trace_extra/trace_stats.hh b/trace_extra/trace_stats.hh
index 77755a895fc..f8a60746221 100644
--- a/trace_extra/trace_stats.hh
+++ b/trace_extra/trace_stats.hh
@@ -32,6 +32,7 @@
 
 #pragma once
 
+#include <chrono>
 #include <map>
 #include <perfetto.h>
 #include <boost/icl/interval_map.hpp>
@@ -60,9 +61,15 @@ class qemu_stats
     uint64_t icount_;
     /* Protect against double pause() calls */
     bool paused_;
+    /* Time of last flush */
+    std::chrono::time_point<std::chrono::steady_clock> last_flush_time_;
 
   public:
-    qemu_stats() : last_pc_(0), pc_range_start_(0), icount_(0), paused_(true) {}
+    qemu_stats()
+        : last_pc_(0), pc_range_start_(0), icount_(0), paused_(true),
+          last_flush_time_(std::chrono::steady_clock::now())
+    {
+    }
     void process_instr(perfetto::Track &track, cpu_log_entry_handle entry);
     void pause(perfetto::Track &track, uint64_t pc);
     void unpause(perfetto::Track &track, uint64_t pc);

From ac02845082ab61971e81b0d13915855da2f3d133 Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Thu, 11 Nov 2021 17:13:22 +0000
Subject: [PATCH 29/74] Fix support for the instruction logging
 EVENT_CTX_UPDATE.

Unbreak handling of LOG_INSTR_EVENT_CTX_OP_SETUP operation so that it
is correctly propagated to the backend.
---
 include/qemu/log_instr.h             |  2 +-
 target/riscv/op_helper_log_instr.c   | 11 +++++------
 trace_extra/guest_context_tracker.cc |  6 ++++--
 trace_extra/trace_perfetto.cc        | 13 ++++++++-----
 4 files changed, 18 insertions(+), 14 deletions(-)

diff --git a/include/qemu/log_instr.h b/include/qemu/log_instr.h
index e46026d132b..ecb0e2e5511 100644
--- a/include/qemu/log_instr.h
+++ b/include/qemu/log_instr.h
@@ -121,7 +121,7 @@ typedef enum {
      * tracing is off. This is useful setup the correct context identifier
      * before switching on tracing.
      */
-    LOG_EVENT_CTX_OP_SETUP = 2
+    LOG_EVENT_CTX_OP_SETUP = 2,
 } log_event_ctx_update_op_t;
 
 /*
diff --git a/target/riscv/op_helper_log_instr.c b/target/riscv/op_helper_log_instr.c
index 7e751abfdb4..693f6cf1f18 100644
--- a/target/riscv/op_helper_log_instr.c
+++ b/target/riscv/op_helper_log_instr.c
@@ -74,15 +74,9 @@ void HELPER(riscv_log_instr_event)(CPURISCVState *env, target_ulong pc)
     log_event_t event;
 
     event.id = get_gpr_value(env, 10);
-    if (!qemu_log_instr_enabled(env) &&
-        (event.id != LOG_EVENT_CTX_UPDATE ||
-         event.ctx_update.op != LOG_EVENT_CTX_OP_SETUP)) {
-        return;
-    }
 
     switch (event.id) {
     case LOG_EVENT_CTX_UPDATE:
-    case LOG_EVENT_CTX_OP_SETUP:
         event.ctx_update.op = get_gpr_value(env, 11);
         event.ctx_update.pid = get_gpr_value(env, 12);
         event.ctx_update.tid = get_gpr_value(env, 13);
@@ -93,5 +87,10 @@ void HELPER(riscv_log_instr_event)(CPURISCVState *env, target_ulong pc)
         warn_report("Unsupported event ID for TCG instr logging %d", event.id);
         return;
     }
+    if (!qemu_log_instr_enabled(env) &&
+        (event.id != LOG_EVENT_CTX_UPDATE ||
+         event.ctx_update.op != LOG_EVENT_CTX_OP_SETUP)) {
+        return;
+    }
     qemu_log_instr_event(env, &event);
 }
diff --git a/trace_extra/guest_context_tracker.cc b/trace_extra/guest_context_tracker.cc
index 372e6591568..4dfb0565f1c 100644
--- a/trace_extra/guest_context_tracker.cc
+++ b/trace_extra/guest_context_tracker.cc
@@ -155,8 +155,10 @@ void guest_context_tracker::mode_update(
 
 void guest_context_tracker::context_update(const log_event_ctx_update_t *evt)
 {
-    // Currently this is the only one existing
-    assert(evt->op == LOG_EVENT_CTX_OP_SWITCH && "Invalid ctx update op");
+    // Currently these are the only ones existing
+    assert((evt->op == LOG_EVENT_CTX_OP_SWITCH ||
+            evt->op == LOG_EVENT_CTX_OP_SETUP) &&
+           "Invalid ctx update op");
     /*
      * Find or create the track associated with the new context and set it as
      * the current context track on the CPU. This will be used to log
diff --git a/trace_extra/trace_perfetto.cc b/trace_extra/trace_perfetto.cc
index 22284da8730..444c637085a 100644
--- a/trace_extra/trace_perfetto.cc
+++ b/trace_extra/trace_perfetto.cc
@@ -212,11 +212,14 @@ process_events(perfetto_backend_data *data, cpu_log_entry_handle entry)
             }
         } else if (evt->id == LOG_EVENT_CTX_UPDATE) {
             // Swap current context.
-            curr_ctx_data->stats.pause(*curr_ctx_track, perfetto_log_entry_pc(entry));
-            data->ctx_tracker_.context_update(&evt->ctx_update);
-            curr_ctx_data = &data->ctx_tracker_.get_ctx_data();
-            curr_ctx_track = &data->ctx_tracker_.get_ctx_track();
-            curr_ctx_data->stats.unpause(*curr_ctx_track, perfetto_log_entry_pc(entry));
+            if (evt->ctx_update.op == LOG_EVENT_CTX_OP_SETUP ||
+                evt->ctx_update.op == LOG_EVENT_CTX_OP_SWITCH) {
+                curr_ctx_data->stats.pause(*curr_ctx_track, perfetto_log_entry_pc(entry));
+                data->ctx_tracker_.context_update(&evt->ctx_update);
+                curr_ctx_data = &data->ctx_tracker_.get_ctx_data();
+                curr_ctx_track = &data->ctx_tracker_.get_ctx_track();
+                curr_ctx_data->stats.unpause(*curr_ctx_track, perfetto_log_entry_pc(entry));
+            }
         }
     }
     if (perfetto_log_entry_flags(entry) & LI_FLAG_MODE_SWITCH) {

From 2e9ba94196d750fa541d669b8099e22e45934107 Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Thu, 18 Nov 2021 13:07:35 +0000
Subject: [PATCH 30/74] Fix handling of context update events in text
 instruction logging backend.

---
 accel/tcg/log_instr_text.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/accel/tcg/log_instr_text.c b/accel/tcg/log_instr_text.c
index c601b3fcf64..2d14006efce 100644
--- a/accel/tcg/log_instr_text.c
+++ b/accel/tcg/log_instr_text.c
@@ -215,6 +215,7 @@ void emit_text_instr(CPUArchState *env, cpu_log_entry_t *entry)
             qemu_log("Context switch pid=0x%lx tid=0x%lx cid=0x%lx\n",
                      event->ctx_update.pid, event->ctx_update.tid,
                      event->ctx_update.cid);
+            break;
         default:
             assert(0 && "unknown event ID");
         }

From 505b4098c361a4e84368cf3a83a7947e02913dc4 Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Fri, 19 Nov 2021 21:29:24 +0000
Subject: [PATCH 31/74] Minor format and comment fix in instruction logging
 implemenation.

---
 include/exec/log_instr_perfetto.h | 2 +-
 include/qemu/log_instr.h          | 7 +++++++
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/include/exec/log_instr_perfetto.h b/include/exec/log_instr_perfetto.h
index 97210548ca5..07b01885ab1 100644
--- a/include/exec/log_instr_perfetto.h
+++ b/include/exec/log_instr_perfetto.h
@@ -44,7 +44,7 @@
  * For each CPU we have the following tracks:
  * CPU N: track for generic CPU events, contains the following categories:
  *   - instructions: events related to the instruction stream
- *   - sched: scheduling events on the CPU (context switches)
+ *   - sched: scheduling events on the CPU or on context-track
  *   - trap: Contains interrupt and trap events with help from OS-driven events
  *   - ctrl: Tracing control events (e.g. when tracing is started and stopped)
  * Process/Thread/Compartment tracks:
diff --git a/include/qemu/log_instr.h b/include/qemu/log_instr.h
index ecb0e2e5511..66ef247c6b0 100644
--- a/include/qemu/log_instr.h
+++ b/include/qemu/log_instr.h
@@ -79,6 +79,13 @@ typedef enum {
     QEMU_LOG_INSTR_CPU_MODE_MAX
 } qemu_log_instr_cpu_mode_t;
 
+/*
+ * XXX-AM: Consider using protobufs for log events so that they
+ * are already in a serialized format when they reach the
+ * perfetto backend.
+ * How this would affect other formats?
+ */
+
 /*
  * Instruction logging per-CPU log level
  */

From 92dd4280a3b41084797d06c44765edf17bda2d69 Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Fri, 19 Nov 2021 23:44:11 +0000
Subject: [PATCH 32/74] Fix building of ARM target with refactored instruction
 trace backends.

---
 accel/tcg/log_instr_cvtrace.c  | 1 +
 accel/tcg/log_instr_perfetto.c | 2 ++
 accel/tcg/log_instr_text.c     | 1 +
 3 files changed, 4 insertions(+)

diff --git a/accel/tcg/log_instr_cvtrace.c b/accel/tcg/log_instr_cvtrace.c
index a299ff2ce62..133349884a4 100644
--- a/accel/tcg/log_instr_cvtrace.c
+++ b/accel/tcg/log_instr_cvtrace.c
@@ -36,6 +36,7 @@
 
 #include "qemu/osdep.h"
 #include "qemu/log.h"
+#include "cpu.h"
 #include "exec/log_instr.h"
 #include "exec/log_instr_internal.h"
 #include "exec/memop.h"
diff --git a/accel/tcg/log_instr_perfetto.c b/accel/tcg/log_instr_perfetto.c
index 75c350fbc9a..cc736905846 100644
--- a/accel/tcg/log_instr_perfetto.c
+++ b/accel/tcg/log_instr_perfetto.c
@@ -35,6 +35,8 @@
  */
 
 #include "qemu/osdep.h"
+#include "qemu/log_instr.h"
+#include "cpu.h"
 #include "exec/log_instr.h"
 #include "exec/log_instr_internal.h"
 #include "exec/log_instr_perfetto.h"
diff --git a/accel/tcg/log_instr_text.c b/accel/tcg/log_instr_text.c
index 2d14006efce..9de88139163 100644
--- a/accel/tcg/log_instr_text.c
+++ b/accel/tcg/log_instr_text.c
@@ -36,6 +36,7 @@
 
 #include "qemu/osdep.h"
 #include "qemu/log.h"
+#include "cpu.h"
 #include "exec/log_instr.h"
 #include "exec/log_instr_internal.h"
 #include "exec/memop.h"

From 5634234a406d9c47f5a4e7742f72336df03a01c1 Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Fri, 19 Nov 2021 23:46:23 +0000
Subject: [PATCH 33/74] Properly handle perfetto logging backend dependency
 resolution.

---
 trace_extra/meson.build | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/trace_extra/meson.build b/trace_extra/meson.build
index d7c796d1cad..5c9ca9427c8 100644
--- a/trace_extra/meson.build
+++ b/trace_extra/meson.build
@@ -1,10 +1,11 @@
 
 qemutrace_ss = ss.source_set()
 
-qemutrace_ss.add(when: ['CONFIG_TCG', 'CONFIG_TRACE_PERFETTO'],
-                 if_true: dependency('boost', modules: ['filesystem']))
+if config_all.has_key('CONFIG_TCG_LOG_INSTR') and config_all.has_key('CONFIG_TRACE_PERFETTO')
+   qemutrace_ss.add(dependency('boost', modules: ['filesystem']))
+endif
 
-qemutrace_ss.add(when: ['CONFIG_TCG', 'CONFIG_TRACE_PERFETTO'],
+qemutrace_ss.add(when: ['CONFIG_TCG', 'CONFIG_TCG_LOG_INSTR', 'CONFIG_TRACE_PERFETTO'],
                  if_true: files('trace_perfetto.cc', 'trace_stats.cc',
                                 'guest_context_tracker.cc',
                                 'cheri-perfetto/sdk/perfetto.cc'))

From 15df8103be94623762ec94fd1dee185bc04dc582 Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Sat, 20 Nov 2021 00:35:28 +0000
Subject: [PATCH 34/74] Add missing dependency for perfetto tracing backend.

---
 trace_extra/meson.build | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/trace_extra/meson.build b/trace_extra/meson.build
index 5c9ca9427c8..66424f3bcfa 100644
--- a/trace_extra/meson.build
+++ b/trace_extra/meson.build
@@ -2,7 +2,7 @@
 qemutrace_ss = ss.source_set()
 
 if config_all.has_key('CONFIG_TCG_LOG_INSTR') and config_all.has_key('CONFIG_TRACE_PERFETTO')
-   qemutrace_ss.add(dependency('boost', modules: ['filesystem']))
+   qemutrace_ss.add(dependency('boost', modules: ['filesystem', 'system']))
 endif
 
 qemutrace_ss.add(when: ['CONFIG_TCG', 'CONFIG_TCG_LOG_INSTR', 'CONFIG_TRACE_PERFETTO'],

From 69473301d322d90818812b66477e889079957aab Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Sat, 20 Nov 2021 01:00:48 +0000
Subject: [PATCH 35/74] Pass instruction logging code through clang-format.

---
 target/riscv/op_helper_log_instr.c |   5 +-
 trace_extra/trace_perfetto.cc      | 174 ++++++++++++++++-------------
 2 files changed, 98 insertions(+), 81 deletions(-)

diff --git a/target/riscv/op_helper_log_instr.c b/target/riscv/op_helper_log_instr.c
index 693f6cf1f18..56a4c6b36e9 100644
--- a/target/riscv/op_helper_log_instr.c
+++ b/target/riscv/op_helper_log_instr.c
@@ -46,7 +46,10 @@ void HELPER(riscv_log_gpr_write)(CPURISCVState *env, uint32_t regnum,
                                  target_ulong value)
 {
     if (qemu_log_instr_enabled(env)) {
-        // TODO(am2419): should be using qemu_log_isntr_cap_int() when TARGET_CHERI?
+        /*
+         * TODO(am2419): should be using qemu_log_isntr_cap_int() when
+         * TARGET_CHERI?
+         */
         qemu_log_instr_reg(env, riscv_int_regnames[regnum], value);
     }
 }
diff --git a/trace_extra/trace_perfetto.cc b/trace_extra/trace_perfetto.cc
index 444c637085a..8ecd0bc19b0 100644
--- a/trace_extra/trace_perfetto.cc
+++ b/trace_extra/trace_perfetto.cc
@@ -56,7 +56,8 @@ PERFETTO_TRACK_EVENT_STATIC_STORAGE();
 
 namespace fs = boost::filesystem;
 
-namespace {
+namespace
+{
 
 /* Tracing session pointer */
 std::unique_ptr<perfetto::TracingSession> session;
@@ -73,8 +74,7 @@ std::unique_ptr<perfetto::Track> sched_track;
 /*
  * Private per-CPU state.
  */
-struct perfetto_backend_data
-{
+struct perfetto_backend_data {
     // Per-CPU control track. This records tracing control events.
     perfetto::Track ctrl_track_;
     // Tracker that resolves the current context scheduled on a CPU
@@ -86,9 +86,9 @@ struct perfetto_backend_data
      */
     bool loglevel_active_;
 
-    perfetto_backend_data(int cpu_id) :
-        ctrl_track_(perfetto::Track::Global(cheri::gen_track_uuid())),
-        ctx_tracker_(cpu_id)
+    perfetto_backend_data(int cpu_id)
+        : ctrl_track_(perfetto::Track::Global(cheri::gen_track_uuid())),
+          ctx_tracker_(cpu_id)
 
     {
         std::string track_name("CPU " + std::to_string(cpu_id) + " ctrl");
@@ -163,9 +163,8 @@ void perfetto_tracing_stop(void)
     session->StopBlocking();
 }
 
-void
-trace_cap_register(perfetto::protos::pbzero::Capability *cap,
-                   cap_register_handle chandle)
+void trace_cap_register(perfetto::protos::pbzero::Capability *cap,
+                        cap_register_handle chandle)
 {
     cap->set_valid(perfetto_cap_tag(chandle));
     cap->set_sealed(perfetto_cap_sealed(chandle));
@@ -176,8 +175,7 @@ trace_cap_register(perfetto::protos::pbzero::Capability *cap,
     cap->set_otype(perfetto_cap_otype(chandle));
 }
 
-void
-process_events(perfetto_backend_data *data, cpu_log_entry_handle entry)
+void process_events(perfetto_backend_data *data, cpu_log_entry_handle entry)
 {
     int nevents = perfetto_log_entry_events(entry);
     bool have_startstop_event = false;
@@ -192,81 +190,91 @@ process_events(perfetto_backend_data *data, cpu_log_entry_handle entry)
         log_event_t *evt = perfetto_log_event(entry, i);
         if (evt->id == LOG_EVENT_STATE) {
             switch (evt->state.next_state) {
-                case LOG_EVENT_STATE_FLUSH:
-                    TRACE_EVENT_INSTANT("ctrl", "flush", data->ctrl_track_);
-                    data->ctx_tracker_.flush_all_ctx_data();
-                    perfetto::TrackEvent::Flush();
-                    break;
-                case LOG_EVENT_STATE_START:
-                    curr_ctx_data->stats.unpause(*curr_ctx_track, evt->state.pc);
-                    TRACE_EVENT_BEGIN("ctrl", "tracing", data->ctrl_track_);
-                    have_startstop_event = true;
-                    break;
-                case LOG_EVENT_STATE_STOP:
-                    curr_ctx_data->stats.pause(*curr_ctx_track, evt->state.pc);
-                    TRACE_EVENT_END("ctrl", data->ctrl_track_);
-                    have_startstop_event = true;
-                    break;
-                default:
-                    assert(false && "Invalid state event");
+            case LOG_EVENT_STATE_FLUSH:
+                TRACE_EVENT_INSTANT("ctrl", "flush", data->ctrl_track_);
+                data->ctx_tracker_.flush_all_ctx_data();
+                perfetto::TrackEvent::Flush();
+                break;
+            case LOG_EVENT_STATE_START:
+                curr_ctx_data->stats.unpause(*curr_ctx_track, evt->state.pc);
+                TRACE_EVENT_BEGIN("ctrl", "tracing", data->ctrl_track_);
+                have_startstop_event = true;
+                break;
+            case LOG_EVENT_STATE_STOP:
+                curr_ctx_data->stats.pause(*curr_ctx_track, evt->state.pc);
+                TRACE_EVENT_END("ctrl", data->ctrl_track_);
+                have_startstop_event = true;
+                break;
+            default:
+                assert(false && "Invalid state event");
             }
         } else if (evt->id == LOG_EVENT_CTX_UPDATE) {
             // Swap current context.
             if (evt->ctx_update.op == LOG_EVENT_CTX_OP_SETUP ||
                 evt->ctx_update.op == LOG_EVENT_CTX_OP_SWITCH) {
-                curr_ctx_data->stats.pause(*curr_ctx_track, perfetto_log_entry_pc(entry));
+                curr_ctx_data->stats.pause(*curr_ctx_track,
+                                           perfetto_log_entry_pc(entry));
                 data->ctx_tracker_.context_update(&evt->ctx_update);
                 curr_ctx_data = &data->ctx_tracker_.get_ctx_data();
                 curr_ctx_track = &data->ctx_tracker_.get_ctx_track();
-                curr_ctx_data->stats.unpause(*curr_ctx_track, perfetto_log_entry_pc(entry));
+                curr_ctx_data->stats.unpause(*curr_ctx_track,
+                                             perfetto_log_entry_pc(entry));
             }
         }
     }
     if (perfetto_log_entry_flags(entry) & LI_FLAG_MODE_SWITCH) {
-        auto mode = cheri::qemu_cpu_mode_to_trace(perfetto_log_entry_next_cpu_mode(entry));
-        // XXX-AM: consider making the mode switch an event, possibly with its own separate pc signaling
-        // the PC of the next instruction we are landing to?
-        curr_ctx_data->stats.pause(*curr_ctx_track, perfetto_log_entry_pc(entry));
+        auto mode = cheri::qemu_cpu_mode_to_trace(
+            perfetto_log_entry_next_cpu_mode(entry));
+        // XXX-AM: consider making the mode switch an event, possibly with its
+        // own separate pc signaling the PC of the next instruction we are
+        // landing to?
+        curr_ctx_data->stats.pause(*curr_ctx_track,
+                                   perfetto_log_entry_pc(entry));
         data->ctx_tracker_.mode_update(mode);
         curr_ctx_data = &data->ctx_tracker_.get_ctx_data();
         curr_ctx_track = &data->ctx_tracker_.get_ctx_track();
-        curr_ctx_data->stats.unpause(*curr_ctx_track, perfetto_log_entry_pc(entry));
+        curr_ctx_data->stats.unpause(*curr_ctx_track,
+                                     perfetto_log_entry_pc(entry));
     }
 
     /*
-     * If we have instruction data, we assume that tracing is enabled and run extra
-     * stats gathering to track the executed instruction PC in the stats histograms.
-     * This avoids having to keep a shadow copy of the tracing state in the backend.
+     * If we have instruction data, we assume that tracing is enabled and run
+     * extra stats gathering to track the executed instruction PC in the stats
+     * histograms. This avoids having to keep a shadow copy of the tracing state
+     * in the backend.
      */
-    if (!have_startstop_event && (perfetto_log_entry_flags(entry) & LI_FLAG_HAS_INSTR_DATA) != 0)
+    if (!have_startstop_event &&
+        (perfetto_log_entry_flags(entry) & LI_FLAG_HAS_INSTR_DATA) != 0)
         curr_ctx_data->stats.process_instr(*curr_ctx_track, entry);
 }
 
-void
-process_instr(perfetto_backend_data *data, cpu_log_entry_handle entry)
+void process_instr(perfetto_backend_data *data, cpu_log_entry_handle entry)
 {
     /*
-     * XXX-AM: instead of having one big instruction record, we may have different messages for
-     * optional parts of the instruction message, on the same track/category: e.g. mode swtich,
-     * interrupt information and modified registers?
+     * XXX-AM: instead of having one big instruction record, we may have
+     * different messages for optional parts of the instruction message, on the
+     * same track/category: e.g. mode swtich, interrupt information and modified
+     * registers?
      */
-    TRACE_EVENT_INSTANT("instructions", "stream", data->ctx_tracker_.get_ctx_track(),
+    TRACE_EVENT_INSTANT(
+        "instructions", "stream", data->ctx_tracker_.get_ctx_track(),
         [&](perfetto::EventContext ctx) {
             auto *qemu_arg = ctx.event()->set_qemu();
             auto *instr = qemu_arg->set_instr();
-            auto flags =  perfetto_log_entry_flags(entry);
+            auto flags = perfetto_log_entry_flags(entry);
 
             /*
              * Populate protobuf from internal qemu structure.
              * XXX-AM: It would be very nice if we could somehow skip this step
              * and have the qemu internal representation be protobuf-based, so
              * that here we only have to embed the qemu-specific packet
-             * into the track_event. Ideally this would save us all this copying around of
-             * the event data but I have no clue about the implications for the stability
-             * of the C/C++ protobuf-generated structures or whether we can embed the
-             * qemu portion of the packet already in the serialized wire-format
-             * (which should be stable by design).
-             * This can probably be done via protozero::Message::AppendScatteredBytes().
+             * into the track_event. Ideally this would save us all this copying
+             * around of the event data but I have no clue about the
+             * implications for the stability of the C/C++ protobuf-generated
+             * structures or whether we can embed the qemu portion of the packet
+             * already in the serialized wire-format (which should be stable by
+             * design). This can probably be done via
+             * protozero::Message::AppendScatteredBytes().
              */
             assert(flags & LI_FLAG_HAS_INSTR_DATA);
             const char *bytes = perfetto_log_entry_insn_bytes(entry);
@@ -274,11 +282,12 @@ process_instr(perfetto_backend_data *data, cpu_log_entry_handle entry)
             int nitems;
             std::stringstream ss;
 
-            // XXX-AM: We can not use a bytes field in the protobuf because perfetto lacks
-            // support. This is slightly sad as this is an high-frequency event.
+            // XXX-AM: We can not use a bytes field in the protobuf because
+            // perfetto lacks support. This is slightly sad as this is an
+            // high-frequency event.
             for (int i = 0; i < size; i++) {
-                ss << std::hex << std::setw(2) << std::setfill('0') <<
-                        (static_cast<unsigned int>(bytes[i]) & 0xff) << " ";
+                ss << std::hex << std::setw(2) << std::setfill('0')
+                   << (static_cast<unsigned int>(bytes[i]) & 0xff) << " ";
             }
             instr->set_opcode(ss.str());
             instr->set_pc(perfetto_log_entry_pc(entry));
@@ -289,7 +298,8 @@ process_instr(perfetto_backend_data *data, cpu_log_entry_handle entry)
                 auto *reginfo = instr->add_reg();
                 reginfo->set_name(perfetto_reg_info_name(entry, i));
                 if ((flags & LRI_CAP_REG) && (flags & LRI_HOLDS_CAP)) {
-                    cap_register_handle cap_handle = perfetto_reg_info_cap(entry, i);
+                    cap_register_handle cap_handle =
+                        perfetto_reg_info_cap(entry, i);
                     auto *capinfo = reginfo->set_cap_val();
                     trace_cap_register(capinfo, cap_handle);
                 } else {
@@ -302,25 +312,26 @@ process_instr(perfetto_backend_data *data, cpu_log_entry_handle entry)
                 auto *meminfo = instr->add_mem();
                 meminfo->set_addr(perfetto_mem_info_addr(entry, i));
                 switch (flags) {
-                    case LMI_LD:
-                        meminfo->set_op(perfetto::protos::pbzero::MemInfo::LOAD);
-                        break;
-                    case LMI_LD | LMI_CAP:
-                        meminfo->set_op(perfetto::protos::pbzero::MemInfo::CLOAD);
-                        break;
-                    case LMI_ST:
-                        meminfo->set_op(perfetto::protos::pbzero::MemInfo::STORE);
-                        break;
-                    case LMI_ST | LMI_CAP:
-                        meminfo->set_op(perfetto::protos::pbzero::MemInfo::CSTORE);
-                        break;
-                    default:
-                        // XXX Notify error somehow?
-                        break;
+                case LMI_LD:
+                    meminfo->set_op(perfetto::protos::pbzero::MemInfo::LOAD);
+                    break;
+                case LMI_LD | LMI_CAP:
+                    meminfo->set_op(perfetto::protos::pbzero::MemInfo::CLOAD);
+                    break;
+                case LMI_ST:
+                    meminfo->set_op(perfetto::protos::pbzero::MemInfo::STORE);
+                    break;
+                case LMI_ST | LMI_CAP:
+                    meminfo->set_op(perfetto::protos::pbzero::MemInfo::CSTORE);
+                    break;
+                default:
+                    // XXX Notify error somehow?
+                    break;
                 }
                 if (flags & LMI_CAP) {
                     auto *capinfo = meminfo->set_cap_val();
-                    cap_register_handle cap_handle = perfetto_reg_info_cap(entry, i);
+                    cap_register_handle cap_handle =
+                        perfetto_reg_info_cap(entry, i);
                     trace_cap_register(capinfo, cap_handle);
                 } else {
                     meminfo->set_int_val(perfetto_mem_info_value(entry, i));
@@ -338,20 +349,22 @@ process_instr(perfetto_backend_data *data, cpu_log_entry_handle entry)
                 trap->set_trap_number(perfetto_log_entry_intr_code(entry));
             }
             if (flags & LI_FLAG_MODE_SWITCH) {
-                auto mode = cheri::qemu_cpu_mode_to_trace(perfetto_log_entry_next_cpu_mode(entry));
+                auto mode = cheri::qemu_cpu_mode_to_trace(
+                    perfetto_log_entry_next_cpu_mode(entry));
                 instr->set_mode(mode);
             }
         });
 }
 
-} /* anonymous */
+} // namespace
 
 extern "C" void qemu_log_instr_perfetto_conf_logfile(const char *name)
 {
     logfile = name;
 }
 
-extern "C" void qemu_log_instr_perfetto_conf_categories(const char *category_str)
+extern "C" void
+qemu_log_instr_perfetto_conf_categories(const char *category_str)
 {
     std::string strspec(category_str);
     int pos;
@@ -378,9 +391,9 @@ extern "C" void perfetto_init_cpu(int cpu_index, void **backend_data)
         perfetto_start_tracing();
     });
 
-
-    // XXX-AM: The backend data is currently missing a cleanup hook, so it leaks.
-    // This is not a big issue as it should be live until the qemu process terminates.
+    // XXX-AM: The backend data is currently missing a cleanup hook, so it
+    // leaks. This is not a big issue as it should be live until the qemu
+    // process terminates.
     *backend_data = new perfetto_backend_data(cpu_index);
 }
 
@@ -394,7 +407,8 @@ extern "C" void perfetto_sync_cpu(void *backend_data)
     perfetto::TrackEvent::Flush();
 }
 
-extern "C" void perfetto_emit_instr(void *backend_data, cpu_log_entry_handle entry)
+extern "C" void perfetto_emit_instr(void *backend_data,
+                                    cpu_log_entry_handle entry)
 {
     auto *data = reinterpret_cast<perfetto_backend_data *>(backend_data);
     auto flags = perfetto_log_entry_flags(entry);

From 97b722784590cac7b6bdb209fd466e475fcbfa2d Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Tue, 30 Nov 2021 17:04:57 +0000
Subject: [PATCH 36/74] Update perfetto instruction logging to match
 cheri-perfetto histogram messages.

---
 trace_extra/guest_context_tracker.cc | 30 ++++------------
 trace_extra/trace_stats.cc           | 11 +++---
 trace_extra/trace_stats.hh           | 10 ++++--
 trace_extra/tuple_index.hh           | 51 ++++++++++++++++++++++++++++
 4 files changed, 72 insertions(+), 30 deletions(-)
 create mode 100644 trace_extra/tuple_index.hh

diff --git a/trace_extra/guest_context_tracker.cc b/trace_extra/guest_context_tracker.cc
index 4dfb0565f1c..6773d2afcfa 100644
--- a/trace_extra/guest_context_tracker.cc
+++ b/trace_extra/guest_context_tracker.cc
@@ -29,39 +29,21 @@
 #include <atomic>
 #include <mutex>
 #include <string>
+#include <unordered_map>
 #include "trace_extra/guest_context_tracker.hh"
 
-namespace
+namespace cheri
 {
+
 /*
  * Dynamically created tracks and track data.
  * The tracks lock protects both tracks and ctx_data.
  */
 std::mutex tracks_lock;
-std::map<cheri::qemu_context_track::qemu_ctx_id,
-         std::shared_ptr<cheri::qemu_context_track>>
+std::unordered_map<qemu_context_track::qemu_ctx_id,
+                   std::shared_ptr<qemu_context_track>,
+                   cheri::tuple_hasher<qemu_context_track::qemu_ctx_id>>
     tracks;
-} // namespace
-
-namespace std
-{
-/*
- * Make the qemu track ID hashable
- */
-template <> struct hash<cheri::qemu_context_track::qemu_ctx_id> {
-    std::size_t
-    operator()(const cheri::qemu_context_track::qemu_ctx_id &key) const noexcept
-    {
-        return (std::hash<uint64_t>{}(std::get<0>(key)) ^
-                std::hash<uint64_t>{}(std::get<1>(key)) ^
-                std::hash<uint64_t>{}(std::get<2>(key)) ^
-                std::hash<uint64_t>{}(std::get<3>(key)));
-    }
-};
-} // namespace std
-
-namespace cheri
-{
 
 /* Helper to generate unique IDs for dynamic tracks */
 unsigned long gen_track_uuid()
diff --git a/trace_extra/trace_stats.cc b/trace_extra/trace_stats.cc
index 57083669dc3..a32bb13a8b4 100644
--- a/trace_extra/trace_stats.cc
+++ b/trace_extra/trace_stats.cc
@@ -53,7 +53,7 @@ void qemu_stats::flush(perfetto::Track &track)
                             auto *hist = qemu_arg->set_histogram();
                             for (const auto &keyval : bb_hit_map_) {
                                 auto &range = keyval.first;
-                                auto *bucket = hist->add_bucket();
+                                auto *bucket = hist->add_bb_bucket();
                                 bucket->set_start(range.lower());
                                 bucket->set_end(range.upper());
                                 bucket->set_value(keyval.second);
@@ -65,8 +65,10 @@ void qemu_stats::flush(perfetto::Track &track)
                             auto *qemu_arg = ctx.event()->set_qemu();
                             auto *hist = qemu_arg->set_histogram();
                             for (const auto &keyval : branch_map_) {
-                                auto *bucket = hist->add_bucket();
-                                bucket->set_start(keyval.first);
+                                auto *bucket = hist->add_branch_bucket();
+                                branch_map_id id = keyval.first;
+                                bucket->set_source(std::get<0>(id));
+                                bucket->set_target(std::get<1>(id));
                                 bucket->set_value(keyval.second);
                             }
                         });
@@ -136,7 +138,8 @@ void qemu_stats::process_instr(perfetto::Track &ctx_track,
         pc_range_start_ = pc;
         icount_ =
             1; // Reset icount triggered on the first instruction of the next BB
-        branch_map_[pc] += 1;
+        branch_map_id branch_id = std::make_tuple(last_pc_, pc);
+        branch_map_[branch_id] += 1;
     } else {
         icount_ += 1;
     }
diff --git a/trace_extra/trace_stats.hh b/trace_extra/trace_stats.hh
index f8a60746221..7d94f508942 100644
--- a/trace_extra/trace_stats.hh
+++ b/trace_extra/trace_stats.hh
@@ -33,13 +33,18 @@
 #pragma once
 
 #include <chrono>
-#include <map>
+#include <unordered_map>
 #include <perfetto.h>
 #include <boost/icl/interval_map.hpp>
 #include "exec/log_instr_perfetto.h"
+#include "tuple_index.hh"
 
 namespace cheri
 {
+
+/* tuple (source-addr, target-addr) */
+using branch_map_id = std::tuple<uint64_t, uint64_t>;
+
 class qemu_stats
 {
     /*
@@ -51,7 +56,8 @@ class qemu_stats
      * Post-processing must be used to distinguish between function
      * calls and conditional branches.
      */
-    std::map<uint64_t, uint64_t> branch_map_;
+    std::unordered_map<branch_map_id, uint64_t, tuple_hasher<branch_map_id>>
+        branch_map_;
     /*
      *  PC-tracking info to detect branches.
      */
diff --git a/trace_extra/tuple_index.hh b/trace_extra/tuple_index.hh
new file mode 100644
index 00000000000..f74530fd4ab
--- /dev/null
+++ b/trace_extra/tuple_index.hh
@@ -0,0 +1,51 @@
+/*-
+ * Copyright (c) 2021 Alfredo Mazzinghi
+ * All rights reserved.
+ *
+ * This software was developed by SRI International and the University of
+ * Cambridge Computer Laboratory (Department of Computer Science and
+ * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the
+ * DARPA SSITH research programme.
+ *
+ * @BERI_LICENSE_HEADER_START@
+ *
+ * Licensed to BERI Open Systems C.I.C. (BERI) under one or more contributor
+ * license agreements.  See the NOTICE file distributed with this work for
+ * additional information regarding copyright ownership.  BERI licenses this
+ * file to you under the BERI Hardware-Software License, Version 1.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at:
+ *
+ *   http://www.beri-open-systems.org/legal/license-1-0.txt
+ *
+ * Unless required by applicable law or agreed to in writing, Work distributed
+ * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ * CONDITIONS OF ANY KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ * @BERI_LICENSE_HEADER_END@
+ */
+
+#pragma once
+
+#include <cstddef>
+#include <type_traits>
+#include <utility>
+#include <boost/functional/hash.hpp>
+
+namespace cheri
+{
+template<typename T> struct is_tuple_impl: std::false_type {};
+template<typename ...T> struct is_tuple_impl<std::tuple<T...>>: std::true_type {};
+template<typename T> struct is_tuple : is_tuple_impl<std::decay_t<T>> {};
+
+template<typename T, typename Enable=void> struct tuple_hasher;
+
+template<typename T>
+struct tuple_hasher<T, typename std::enable_if<is_tuple<T>::value>::type>
+{
+    std::size_t operator()(const T& tuple) const {
+        return boost::hash_value(tuple);
+    }
+};
+}

From 4f3800693a73c6190edb9ba4f9fa872d9a128142 Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Tue, 30 Nov 2021 17:06:02 +0000
Subject: [PATCH 37/74] Comply with CHERI-architecture reserved RISC-V opcodes
 for tracing.

---
 target/riscv/insn_trans/trans_rvi.c.inc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/target/riscv/insn_trans/trans_rvi.c.inc b/target/riscv/insn_trans/trans_rvi.c.inc
index e4aa593cee4..eb44d48d1ea 100644
--- a/target/riscv/insn_trans/trans_rvi.c.inc
+++ b/target/riscv/insn_trans/trans_rvi.c.inc
@@ -284,7 +284,7 @@ static bool trans_slti(DisasContext *ctx, arg_slti *a)
             gen_helper_qemu_log_instr_stop(cpu_env, tpc);
             ctx->base.is_jmp = DISAS_NORETURN;
             break;
-        case 0x30:
+        case 0x2f:
             gen_helper_riscv_log_instr_event(cpu_env, tpc);
             break;
         }

From d11345a007a9f9922700a6acc66f1088d8d51942 Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Thu, 9 Dec 2021 14:40:52 +0000
Subject: [PATCH 38/74] Introduce support for guest-defined trace markers.

---
 accel/tcg/log_instr_text.c         |  3 +
 include/exec/log_instr.h           | 29 +++++++---
 include/exec/log_instr_perfetto.h  |  4 +-
 include/qemu/log_instr.h           |  7 ++-
 target/riscv/op_helper_log_instr.c |  8 ++-
 trace_extra/trace_perfetto.cc      | 93 +++++++++++++++++-------------
 6 files changed, 93 insertions(+), 51 deletions(-)

diff --git a/accel/tcg/log_instr_text.c b/accel/tcg/log_instr_text.c
index 9de88139163..39cd2e55905 100644
--- a/accel/tcg/log_instr_text.c
+++ b/accel/tcg/log_instr_text.c
@@ -217,6 +217,9 @@ void emit_text_instr(CPUArchState *env, cpu_log_entry_t *entry)
                      event->ctx_update.pid, event->ctx_update.tid,
                      event->ctx_update.cid);
             break;
+        case LOG_EVENT_MARKER:
+            qemu_log("Guest trace marker %lx\n", event->marker);
+            break;
         default:
             assert(0 && "unknown event ID");
         }
diff --git a/include/exec/log_instr.h b/include/exec/log_instr.h
index 85cf0ebf138..b67f13f3900 100644
--- a/include/exec/log_instr.h
+++ b/include/exec/log_instr.h
@@ -190,21 +190,36 @@ void qemu_log_printf_create_globals(void);
 struct cpu_log_entry;
 
 /*
- * Request a flush of the TCG when changing loglevel outside of qemu_log_instr.
- * TODO(am2419): this should be removed from the interface.
+ * Check whether instruction tracing is enabled.
+ * Note: changes to the instruction logging flags result in a call to
+ * tb_flush so we can do the logging checks at translate time as well.
  */
-void qemu_log_instr_flush_tcg(bool request_stop);
+bool qemu_log_instr_check_enabled(CPUArchState *env);
 
 /* Helper macro to check for instruction logging enabled */
 #define qemu_log_instr_enabled(env)                                            \
     unlikely(qemu_log_instr_check_enabled((env)))
 
 /*
- * Check whether instruction tracing is enabled.
- * Note: changes to the instruction logging flags result in a call to
- * tb_flush so we can do the logging checks at translate time as well.
+ * Helper macro to check whether an event should be appended to the trace or
+ * not. This takes into account the event data to allow some events to be
+ * appended to the current log buffer entry even when tracing is disabled. These
+ * events will only become visible in the trace when tracing is started, in the
+ * first log entry. This is intended to be used only for OS-driven events that
+ * are relatively low-frequency w.r.t. the number of instructions executed.
  */
-bool qemu_log_instr_check_enabled(CPUArchState *env);
+static inline bool qemu_log_instr_event_enabled(CPUArchState *env,
+                                                log_event_t *event)
+{
+    if (event->id == LOG_EVENT_MARKER) {
+        return true;
+    }
+    if (event->id == LOG_EVENT_CTX_UPDATE &&
+        event->ctx_update.op == LOG_EVENT_CTX_OP_SETUP) {
+        return true;
+    }
+    return qemu_log_instr_check_enabled(env);
+}
 
 /*
  * Register a trace filter for a given CPU.
diff --git a/include/exec/log_instr_perfetto.h b/include/exec/log_instr_perfetto.h
index 07b01885ab1..122acd93f6d 100644
--- a/include/exec/log_instr_perfetto.h
+++ b/include/exec/log_instr_perfetto.h
@@ -60,7 +60,9 @@ PERFETTO_DEFINE_CATEGORIES(
     perfetto::Category("stats").SetDescription("High-level statistics data"),
     perfetto::Category("ctrl").SetDescription("Tracing control events"),
     perfetto::Category("trap").SetDescription("CPU trap events"),
-    perfetto::Category("sched").SetDescription("Scheduling events"));
+    perfetto::Category("sched").SetDescription("Scheduling events"),
+    perfetto::Category("marker").SetDescription(
+        "Guest trace timestamp markers"));
 #endif
 
 /*
diff --git a/include/qemu/log_instr.h b/include/qemu/log_instr.h
index 66ef247c6b0..6b8d6c48957 100644
--- a/include/qemu/log_instr.h
+++ b/include/qemu/log_instr.h
@@ -101,7 +101,11 @@ typedef enum {
 /*
  * Trace event identifiers.
  */
-typedef enum { LOG_EVENT_STATE = 0, LOG_EVENT_CTX_UPDATE = 1 } log_event_id_t;
+typedef enum {
+    LOG_EVENT_STATE = 0,
+    LOG_EVENT_CTX_UPDATE = 1,
+    LOG_EVENT_MARKER = 2
+} log_event_id_t;
 
 /*
  * Tracing status changed (e.g. trace start/stop)
@@ -152,6 +156,7 @@ typedef struct {
     union {
         log_event_trace_state_update_t state;
         log_event_ctx_update_t ctx_update;
+        uint64_t marker;
     };
 } log_event_t;
 
diff --git a/target/riscv/op_helper_log_instr.c b/target/riscv/op_helper_log_instr.c
index 56a4c6b36e9..1880d1b24f8 100644
--- a/target/riscv/op_helper_log_instr.c
+++ b/target/riscv/op_helper_log_instr.c
@@ -86,13 +86,15 @@ void HELPER(riscv_log_instr_event)(CPURISCVState *env, target_ulong pc)
         event.ctx_update.cid = get_gpr_value(env, 14);
         event.ctx_update.mode = cpu_priv_to_mode(env->priv);
         break;
+    case LOG_EVENT_MARKER:
+        event.marker = get_gpr_value(env, 11);
+        break;
     default:
         warn_report("Unsupported event ID for TCG instr logging %d", event.id);
         return;
     }
-    if (!qemu_log_instr_enabled(env) &&
-        (event.id != LOG_EVENT_CTX_UPDATE ||
-         event.ctx_update.op != LOG_EVENT_CTX_OP_SETUP)) {
+
+    if (!qemu_log_instr_event_enabled(env, &event)) {
         return;
     }
     qemu_log_instr_event(env, &event);
diff --git a/trace_extra/trace_perfetto.cc b/trace_extra/trace_perfetto.cc
index 8ecd0bc19b0..bf4908414d5 100644
--- a/trace_extra/trace_perfetto.cc
+++ b/trace_extra/trace_perfetto.cc
@@ -179,8 +179,8 @@ void process_events(perfetto_backend_data *data, cpu_log_entry_handle entry)
 {
     int nevents = perfetto_log_entry_events(entry);
     bool have_startstop_event = false;
-    auto *curr_ctx_data = &data->ctx_tracker_.get_ctx_data();
-    auto *curr_ctx_track = &data->ctx_tracker_.get_ctx_track();
+    auto *ctx_data = &data->ctx_tracker_.get_ctx_data();
+    auto *ctx_track = &data->ctx_tracker_.get_ctx_track();
 
     /*
      * Note: LOG_EVENT_STATE events are emitted even when tracing is disabled.
@@ -188,38 +188,56 @@ void process_events(perfetto_backend_data *data, cpu_log_entry_handle entry)
      */
     for (int i = 0; i < nevents; i++) {
         log_event_t *evt = perfetto_log_event(entry, i);
-        if (evt->id == LOG_EVENT_STATE) {
-            switch (evt->state.next_state) {
-            case LOG_EVENT_STATE_FLUSH:
-                TRACE_EVENT_INSTANT("ctrl", "flush", data->ctrl_track_);
-                data->ctx_tracker_.flush_all_ctx_data();
-                perfetto::TrackEvent::Flush();
+        switch (evt->id) {
+            case LOG_EVENT_STATE:
+                {
+                    switch (evt->state.next_state) {
+                        case LOG_EVENT_STATE_FLUSH:
+                            TRACE_EVENT_INSTANT("ctrl", "flush", data->ctrl_track_);
+                            data->ctx_tracker_.flush_all_ctx_data();
+                            perfetto::TrackEvent::Flush();
+                            break;
+                        case LOG_EVENT_STATE_START:
+                            ctx_data->stats.unpause(*ctx_track, evt->state.pc);
+                            TRACE_EVENT_BEGIN("ctrl", "tracing", data->ctrl_track_);
+                            have_startstop_event = true;
+                            break;
+                        case LOG_EVENT_STATE_STOP:
+                            ctx_data->stats.pause(*ctx_track, evt->state.pc);
+                            TRACE_EVENT_END("ctrl", data->ctrl_track_);
+                            have_startstop_event = true;
+                            break;
+                        default:
+                            assert(false && "Invalid state event");
+                    }
+                }
                 break;
-            case LOG_EVENT_STATE_START:
-                curr_ctx_data->stats.unpause(*curr_ctx_track, evt->state.pc);
-                TRACE_EVENT_BEGIN("ctrl", "tracing", data->ctrl_track_);
-                have_startstop_event = true;
+            case LOG_EVENT_CTX_UPDATE:
+                {
+                    /* Swap current context. */
+                    if (evt->ctx_update.op == LOG_EVENT_CTX_OP_SETUP ||
+                        evt->ctx_update.op == LOG_EVENT_CTX_OP_SWITCH) {
+                        ctx_data->stats.pause(*ctx_track, perfetto_log_entry_pc(entry));
+                        data->ctx_tracker_.context_update(&evt->ctx_update);
+                        /* Reload data and track as context_update will have changed them */
+                        ctx_data = &data->ctx_tracker_.get_ctx_data();
+                        ctx_track = &data->ctx_tracker_.get_ctx_track();
+                        ctx_data->stats.unpause(*ctx_track, perfetto_log_entry_pc(entry));
+                    }
+                }
                 break;
-            case LOG_EVENT_STATE_STOP:
-                curr_ctx_data->stats.pause(*curr_ctx_track, evt->state.pc);
-                TRACE_EVENT_END("ctrl", data->ctrl_track_);
-                have_startstop_event = true;
+            case LOG_EVENT_MARKER:
+                {
+                    auto cpu_track = data->ctx_tracker_.get_cpu_track();
+                    TRACE_EVENT_INSTANT("marker", "guest", cpu_track,
+                        [&](perfetto::EventContext ctx) {
+                            auto *qemu_arg = ctx.event()->set_qemu();
+                            qemu_arg->set_marker(evt->marker);
+                        });
+                }
                 break;
             default:
-                assert(false && "Invalid state event");
-            }
-        } else if (evt->id == LOG_EVENT_CTX_UPDATE) {
-            // Swap current context.
-            if (evt->ctx_update.op == LOG_EVENT_CTX_OP_SETUP ||
-                evt->ctx_update.op == LOG_EVENT_CTX_OP_SWITCH) {
-                curr_ctx_data->stats.pause(*curr_ctx_track,
-                                           perfetto_log_entry_pc(entry));
-                data->ctx_tracker_.context_update(&evt->ctx_update);
-                curr_ctx_data = &data->ctx_tracker_.get_ctx_data();
-                curr_ctx_track = &data->ctx_tracker_.get_ctx_track();
-                curr_ctx_data->stats.unpause(*curr_ctx_track,
-                                             perfetto_log_entry_pc(entry));
-            }
+                assert(false && "Invalid event identifier");
         }
     }
     if (perfetto_log_entry_flags(entry) & LI_FLAG_MODE_SWITCH) {
@@ -228,13 +246,11 @@ void process_events(perfetto_backend_data *data, cpu_log_entry_handle entry)
         // XXX-AM: consider making the mode switch an event, possibly with its
         // own separate pc signaling the PC of the next instruction we are
         // landing to?
-        curr_ctx_data->stats.pause(*curr_ctx_track,
-                                   perfetto_log_entry_pc(entry));
+        ctx_data->stats.pause(*ctx_track, perfetto_log_entry_pc(entry));
         data->ctx_tracker_.mode_update(mode);
-        curr_ctx_data = &data->ctx_tracker_.get_ctx_data();
-        curr_ctx_track = &data->ctx_tracker_.get_ctx_track();
-        curr_ctx_data->stats.unpause(*curr_ctx_track,
-                                     perfetto_log_entry_pc(entry));
+        ctx_data = &data->ctx_tracker_.get_ctx_data();
+        ctx_track = &data->ctx_tracker_.get_ctx_track();
+        ctx_data->stats.unpause(*ctx_track, perfetto_log_entry_pc(entry));
     }
 
     /*
@@ -245,7 +261,7 @@ void process_events(perfetto_backend_data *data, cpu_log_entry_handle entry)
      */
     if (!have_startstop_event &&
         (perfetto_log_entry_flags(entry) & LI_FLAG_HAS_INSTR_DATA) != 0)
-        curr_ctx_data->stats.process_instr(*curr_ctx_track, entry);
+        ctx_data->stats.process_instr(*ctx_track, entry);
 }
 
 void process_instr(perfetto_backend_data *data, cpu_log_entry_handle entry)
@@ -256,8 +272,7 @@ void process_instr(perfetto_backend_data *data, cpu_log_entry_handle entry)
      * same track/category: e.g. mode swtich, interrupt information and modified
      * registers?
      */
-    TRACE_EVENT_INSTANT(
-        "instructions", "stream", data->ctx_tracker_.get_ctx_track(),
+    TRACE_EVENT_INSTANT("instructions", "stream", data->ctx_tracker_.get_ctx_track(),
         [&](perfetto::EventContext ctx) {
             auto *qemu_arg = ctx.event()->set_qemu();
             auto *instr = qemu_arg->set_instr();

From ae92b6918ae40d68ab211fdaaec15aa11cc2d62e Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Sat, 18 Dec 2021 17:45:29 +0000
Subject: [PATCH 39/74] Add instruction logging backend that emits a protobuf
 stream.

This is inspired to the perfetto log backend and has two main use
cases:
 - Provide a portable way to parse full instruction traces, without
 requiring all the perfetto backend dependencies.
 - Detect performance and disk-space issues with the perfetto backend
 serialization mechanism.
Note that this backend only logs the instruction stream and does not
support all the extra data framing, selection and aggregation that
the perfetto backend performs.
---
 accel/tcg/log_instr.c                         |   8 +-
 accel/tcg/log_instr_protobuf.c                | 319 ++++++++++++++++++
 accel/tcg/meson.build                         |   5 +
 configure                                     |   8 +
 disas.c                                       |  38 +++
 include/disas/disas.h                         |   3 +
 include/exec/log_instr_internal.h             |   7 +
 include/qemu/log_instr.h                      |  15 +-
 meson.build                                   |   1 +
 softmmu/vl.c                                  |   4 +
 trace_extra/meson.build                       |   2 +
 trace_extra/proto/meson.build                 |  11 +
 .../proto/protobuf_backend_entry.proto        |  95 ++++++
 13 files changed, 509 insertions(+), 7 deletions(-)
 create mode 100644 accel/tcg/log_instr_protobuf.c
 create mode 100644 trace_extra/proto/meson.build
 create mode 100644 trace_extra/proto/protobuf_backend_entry.proto

diff --git a/accel/tcg/log_instr.c b/accel/tcg/log_instr.c
index dbe0f12ead6..dbb07e341dd 100644
--- a/accel/tcg/log_instr.c
+++ b/accel/tcg/log_instr.c
@@ -120,7 +120,13 @@ static trace_backend_hooks_t trace_backends[] = {
     { .init = init_perfetto_backend,
       .sync = sync_perfetto_backend,
       .emit_header = NULL,
-      .emit_instr = emit_perfetto_entry }
+      .emit_instr = emit_perfetto_entry },
+#endif
+#ifdef CONFIG_TRACE_PROTOBUF
+    { .init = init_protobuf_backend,
+      .sync = sync_protobuf_backend,
+      .emit_header = NULL,
+      .emit_instr = emit_protobuf_entry },
 #endif
 };
 
diff --git a/accel/tcg/log_instr_protobuf.c b/accel/tcg/log_instr_protobuf.c
new file mode 100644
index 00000000000..dd9490014f7
--- /dev/null
+++ b/accel/tcg/log_instr_protobuf.c
@@ -0,0 +1,319 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2021 Alfredo Mazzinghi
+ *
+ * This software was developed by SRI International and the University of
+ * Cambridge Computer Laboratory (Department of Computer Science and
+ * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the
+ * DARPA SSITH research programme.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * Similar functionality as the text backend but with a more stable format.
+ */
+
+#include <assert.h>
+
+#include "qemu/osdep.h"
+#include "qemu/log.h"
+#include "cpu.h"
+#include "exec/log_instr.h"
+#include "exec/log_instr_internal.h"
+#include "exec/memop.h"
+#include "disas/disas.h"
+
+#include "protobuf_backend_entry.pb-c.h"
+
+static FILE *protobuf_logfile;
+
+#ifdef TARGET_CHERI
+static QEMULogInstrCapability *emit_protobuf_capability(cap_register_t *cap)
+{
+    QEMULogInstrCapability *pb_cap = g_malloc(sizeof(QEMULogInstrCapability));
+
+    qemulog_instr_capability__init(pb_cap);
+    pb_cap->valid = cap->cr_tag;
+    pb_cap->sealed = cap_is_sealed_with_type(cap);
+    pb_cap->perms = COMBINED_PERMS_VALUE(cap);
+    /* pb_cap->flags = cap_get_flags(cap); */
+    pb_cap->otype = cap_get_otype_unsigned(cap);
+    pb_cap->cap_base = cap_get_base(cap);
+    pb_cap->cap_cursor = cap_get_cursor(cap);
+    pb_cap->cap_length = cap_get_length_sat(cap);
+    return pb_cap;
+}
+#endif
+
+static void emit_protobuf_meminfo(log_meminfo_t *minfo,
+                                  QEMULogInstrMem **pb_mem)
+{
+    QEMULogInstrMem *mem = g_malloc(sizeof(QEMULogInstrMem));
+
+    qemulog_instr_mem__init(mem);
+    mem->is_load = (minfo->flags & LMI_LD) != 0;
+    mem->size = memop_size(minfo->op);
+    mem->addr = minfo->addr;
+#ifndef TARGET_CHERI
+    assert((minfo->flags & LMI_CAP) == 0 &&
+           "Capability memory access without CHERI support");
+#else
+    if (minfo->flags & LMI_CAP) {
+        mem->value_case = QEMULOG_INSTR_MEM__VALUE_CAP_VALUE;
+        mem->cap_value = emit_protobuf_capability(&minfo->cap);
+    } else
+#endif
+    {
+        mem->value_case = QEMULOG_INSTR_MEM__VALUE_INT_VALUE;
+        mem->int_value = minfo->value;
+    }
+    *pb_mem = mem;
+}
+
+static void release_protobuf_meminfo(QEMULogInstrMem *pb_mem)
+{
+    if (pb_mem->value_case == QEMULOG_INSTR_MEM__VALUE_CAP_VALUE) {
+        g_free(pb_mem->cap_value);
+    }
+    g_free(pb_mem);
+}
+
+static void emit_protobuf_reginfo(log_reginfo_t *rinfo,
+                                  QEMULogInstrReg **pb_reg)
+{
+    QEMULogInstrReg *reg = g_malloc(sizeof(QEMULogInstrReg));
+
+    qemulog_instr_reg__init(reg);
+    /* Safe to de-const cast as the pointer is only used during packing */
+    reg->name = (char *)rinfo->name;
+
+#ifndef TARGET_CHERI
+    assert(!reginfo_is_cap(rinfo) && "Register marked as capability "
+                                     "register whitout CHERI support");
+#else
+    if (reginfo_is_cap(rinfo)) {
+        if (reginfo_has_cap(rinfo)) {
+            reg->value_case = QEMULOG_INSTR_REG__VALUE_CAP_VALUE;
+            reg->cap_value = emit_protobuf_capability(&rinfo->cap);
+        } else {
+            reg->value_case = QEMULOG_INSTR_REG__VALUE_INT_VALUE;
+            reg->int_value = rinfo->gpr;
+        }
+    } else
+#endif
+    {
+        reg->value_case = QEMULOG_INSTR_REG__VALUE_INT_VALUE;
+        reg->int_value = rinfo->gpr;
+    }
+    *pb_reg = reg;
+}
+
+static void release_protobuf_reginfo(QEMULogInstrReg *pb_reg)
+{
+    if (pb_reg->value_case == QEMULOG_INSTR_REG__VALUE_CAP_VALUE) {
+        g_free(pb_reg->cap_value);
+    }
+    g_free(pb_reg);
+}
+
+static void emit_protobuf_event(log_event_t *evtinfo, QEMULogInstrEvt **pb_evt)
+{
+    QEMULogInstrEvt *evt = g_malloc(sizeof(QEMULogInstrEvt));
+
+    qemulog_instr_evt__init(evt);
+    switch (evtinfo->id) {
+    case LOG_EVENT_STATE:
+        evt->event_case = QEMULOG_INSTR_EVT__EVENT_STATE;
+        evt->state = g_malloc(sizeof(QEMULogInstrEvtTraceState));
+        qemulog_instr_evt_trace_state__init(evt->state);
+        evt->state->next_state = (LogEventTraceState)evtinfo->state.next_state;
+        evt->state->pc = evtinfo->state.pc;
+        break;
+    case LOG_EVENT_CTX_UPDATE:
+        evt->event_case = QEMULOG_INSTR_EVT__EVENT_CTX_UPDATE;
+        evt->ctx_update = g_malloc(sizeof(QEMULogInstrEvtCtxUpdate));
+        qemulog_instr_evt_ctx_update__init(evt->ctx_update);
+        evt->ctx_update->pid = evtinfo->ctx_update.pid;
+        evt->ctx_update->tid = evtinfo->ctx_update.tid;
+        evt->ctx_update->cid = evtinfo->ctx_update.cid;
+        /* Safe to de-const cast as the pointer is only used during packing */
+        evt->ctx_update->mode =
+            (char *)cpu_get_mode_name(evtinfo->ctx_update.mode);
+        break;
+    case LOG_EVENT_MARKER:
+        evt->event_case = QEMULOG_INSTR_EVT__EVENT_MARKER;
+        evt->marker = evtinfo->marker;
+        break;
+    default:
+        assert(0 && "unknown event ID");
+    }
+    *pb_evt = evt;
+}
+
+static void release_protobuf_event(QEMULogInstrEvt *pb_evt)
+{
+    if (pb_evt->event_case == QEMULOG_INSTR_EVT__EVENT_STATE) {
+        g_free(pb_evt->state);
+    }
+    if (pb_evt->event_case == QEMULOG_INSTR_EVT__EVENT_CTX_UPDATE) {
+        g_free(pb_evt->ctx_update);
+    }
+    g_free(pb_evt);
+}
+
+void init_protobuf_backend(CPUArchState *env)
+{
+    /*
+     * TODO should handle a separate log file to avoid pollution of the main log
+     * file with binary data from protobufs. Since this seems to be a common
+     * pattern, maybe generalize it.
+     *
+     * XXX we may want to store a cache of protobuf structures in the backend
+     * data so that we avoid all the g_malloc() and g_free().
+     */
+
+    if (protobuf_logfile == NULL) {
+        protobuf_logfile = fopen("qemu-protobuf.pb", "w+b");
+    }
+}
+
+void sync_protobuf_backend(CPUArchState *env)
+{
+    if (protobuf_logfile != NULL) {
+        fflush(protobuf_logfile);
+    }
+}
+
+void emit_protobuf_entry(CPUArchState *env, cpu_log_entry_t *entry)
+{
+    QEMULogInstrEntry pb_entry = QEMULOG_INSTR_ENTRY__INIT;
+    QEMULogInstrExc pb_exc = QEMULOG_INSTR_EXC__INIT;
+    QEMULogInstrMem **pb_mem;
+    QEMULogInstrReg **pb_reg;
+    QEMULogInstrEvt **pb_evt;
+    uint8_t *buf;
+    size_t len;
+    uint32_t preamble = 0;
+    int i;
+
+    pb_entry.pc = entry->pc;
+
+    if (entry->flags & LI_FLAG_HAS_INSTR_DATA) {
+        pb_entry.insn_case = QEMULOG_INSTR_ENTRY__INSN_DISAS;
+        pb_entry.disas = disas_one_strbuf(env_cpu(env), entry->insn_bytes,
+                                          sizeof(entry->insn_bytes), entry->pc);
+        pb_entry.cpu = env_cpu(env)->cpu_index;
+        pb_entry.asid = entry->asid;
+        if (entry->flags & LI_FLAG_MODE_SWITCH) {
+            /*
+             * Safe to de-const cast as the pointer is only used during packing
+             */
+            pb_entry.mode = (char *)cpu_get_mode_name(entry->next_cpu_mode);
+        }
+        if (entry->flags & LI_FLAG_INTR_MASK) {
+            pb_entry.exception = &pb_exc;
+            pb_exc.vector = entry->intr_vector;
+            pb_exc.code = entry->intr_code;
+            if (entry->flags & LI_FLAG_INTR_TRAP) {
+                pb_exc.type = QEMULOG_INSTR_EXC_TYPE__TRAP;
+                pb_exc.faultaddr = entry->intr_faultaddr;
+            } else if (entry->flags & LI_FLAG_INTR_ASYNC) {
+                pb_exc.type = QEMULOG_INSTR_EXC_TYPE__INTR;
+            } else {
+                assert(0 && "invalid exception flags");
+            }
+        }
+        if (entry->mem->len > 0) {
+            pb_mem = g_malloc(entry->mem->len * sizeof(QEMULogInstrMem *));
+            pb_entry.n_mem = entry->mem->len;
+            pb_entry.mem = pb_mem;
+            for (i = 0; i < entry->mem->len; i++, pb_mem++) {
+                log_meminfo_t *minfo =
+                    &g_array_index(entry->mem, log_meminfo_t, i);
+                emit_protobuf_meminfo(minfo, pb_mem);
+            }
+        }
+        if (entry->regs->len > 0) {
+            pb_reg = g_malloc(entry->regs->len * sizeof(QEMULogInstrReg *));
+            pb_entry.n_reg = entry->regs->len;
+            pb_entry.reg = pb_reg;
+            for (i = 0; i < entry->regs->len; i++, pb_reg++) {
+                log_reginfo_t *rinfo =
+                    &g_array_index(entry->regs, log_reginfo_t, i);
+                emit_protobuf_reginfo(rinfo, pb_reg);
+            }
+        }
+    }
+
+    if (entry->events->len > 0) {
+        pb_evt = NULL;
+        pb_evt = g_malloc(entry->events->len * sizeof(QEMULogInstrEvt *));
+        pb_entry.n_evt = entry->events->len;
+        pb_entry.evt = pb_evt;
+        for (i = 0; i < entry->events->len; i++, pb_evt++) {
+            log_event_t *event = &g_array_index(entry->events, log_event_t, i);
+            emit_protobuf_event(event, pb_evt);
+        }
+    }
+
+    len = qemulog_instr_entry__get_packed_size(&pb_entry);
+    preamble = len;
+    assert(preamble == len && "Message preamble overflow");
+
+    buf = g_malloc(len + sizeof(preamble));
+    *((uint32_t *)buf) = preamble;
+    qemulog_instr_entry__pack(&pb_entry, buf + sizeof(preamble));
+
+    /*
+     * Technically we should not use the qemu logfile as it is not open in
+     * binary mode
+     */
+    qemu_flockfile(protobuf_logfile);
+    fwrite(buf, len + sizeof(preamble), 1, protobuf_logfile);
+    qemu_funlockfile(protobuf_logfile);
+
+    if (entry->flags & LI_FLAG_HAS_INSTR_DATA) {
+        g_free(pb_entry.disas);
+
+        if (entry->mem->len > 0) {
+            for (i = 0; i < entry->mem->len; i++) {
+                release_protobuf_meminfo(pb_entry.mem[i]);
+            }
+            g_free(pb_entry.mem);
+        }
+        if (entry->regs->len > 0) {
+            for (i = 0; i < entry->regs->len; i++) {
+                release_protobuf_reginfo(pb_entry.reg[i]);
+            }
+            g_free(pb_entry.reg);
+        }
+    }
+    if (entry->events->len > 0) {
+        for (i = 0; i < entry->events->len; i++) {
+            release_protobuf_event(pb_entry.evt[i]);
+        }
+        g_free(pb_entry.evt);
+    }
+    g_free(buf);
+}
diff --git a/accel/tcg/meson.build b/accel/tcg/meson.build
index 8a89e995472..23a4b0a30e0 100644
--- a/accel/tcg/meson.build
+++ b/accel/tcg/meson.build
@@ -17,3 +17,8 @@ specific_ss.add(when: ['CONFIG_TCG_LOG_INSTR', 'CONFIG_TCG'], if_true: files('lo
 specific_ss.add(when: ['CONFIG_TCG_LOG_INSTR', 'CONFIG_TCG'], if_true: files('log_instr_text.c'))
 specific_ss.add(when: ['CONFIG_TCG_LOG_INSTR', 'CONFIG_TCG'], if_true: files('log_instr_cvtrace.c'))
 specific_ss.add(when: ['CONFIG_TRACE_PERFETTO', 'CONFIG_TCG_LOG_INSTR', 'CONFIG_TCG'], if_true: files('log_instr_perfetto.c'))
+specific_ss.add(when: ['CONFIG_TRACE_PROTOBUF', 'CONFIG_TCG_LOG_INSTR', 'CONFIG_TCG'], if_true: files('log_instr_protobuf.c'))
+
+if config_all.has_key('CONFIG_TCG_LOG_INSTR') and config_all.has_key('CONFIG_TRACE_PROTOBUF')
+   specific_ss.add(dependency('libprotobuf-c'))
+endif
diff --git a/configure b/configure
index d897d6eaa3f..61d5879b5b2 100755
--- a/configure
+++ b/configure
@@ -341,6 +341,7 @@ gprof="no"
 debug_tcg="no"
 tcg_log_instr="yes"
 tcg_log_instr_perfetto="no"
+tcg_log_instr_protobuf="no"
 rvfi_dii="no"
 debug="no"
 sanitizers="no"
@@ -1041,6 +1042,10 @@ for opt do
   ;;
   --disable-perfetto-log-instr) tcg_log_instr_perfetto="no"
   ;;
+  --enable-protobuf-log-instr) tcg_log_instr_protobuf="yes"
+  ;;
+  --disable-protobuf-log-instr) tcg_log_instr_protobuf="no"
+  ;;
   --enable-rvfi-dii) rvfi_dii="yes"
   ;;
   --disable-rvfi-dii) rvfi_dii="no"
@@ -5976,6 +5981,9 @@ fi
 if test "$tcg_log_instr_perfetto" = "yes" ; then
     echo "CONFIG_TRACE_PERFETTO=y" >> $config_host_mak
 fi
+if test "$tcg_log_instr_protobuf" = "yes" ; then
+    echo "CONFIG_TRACE_PROTOBUF=y" >> $config_host_mak
+fi
 if test "$rvfi_dii" = "yes" ; then
   echo "CONFIG_RVFI_DII=y" >> $config_host_mak
 fi
diff --git a/disas.c b/disas.c
index a0de01cc39a..df82a451146 100644
--- a/disas.c
+++ b/disas.c
@@ -355,6 +355,44 @@ char *plugin_disas(CPUState *cpu, uint64_t addr, size_t size)
     return g_string_free(ds, false);
 }
 
+/*
+ * Disassemble one instruction to a string buffer.
+ */
+char *disas_one_strbuf(CPUState *cpu, void *code, unsigned long size,
+                       target_ulong vma)
+{
+    CPUClass *cc = CPU_GET_CLASS(cpu);
+    CPUDebug s;
+    GString *ds = g_string_new(NULL);
+
+    initialize_debug_target(&s, cpu);
+    s.info.fprintf_func = plugin_printf;
+    s.info.stream = (FILE *)ds;  /* abuse this slot */
+    s.info.buffer_vma = vma;
+    s.info.buffer_length = size;
+
+#ifdef TARGET_WORDS_BIGENDIAN
+    s.info.endian = BFD_ENDIAN_BIG;
+#else
+    s.info.endian = BFD_ENDIAN_LITTLE;
+#endif
+
+    if (cc->disas_set_info) {
+        cc->disas_set_info(cpu, &s.info);
+    }
+
+    if (s.info.cap_arch >= 0 && cap_disas_target(&s.info, vma, size)) {
+        ; /* done */
+    } else if (s.info.print_insn) {
+        s.info.print_insn(vma, &s.info);
+    } else {
+        ; /* cannot disassemble -- return empty string */
+    }
+
+    /* Return the buffer, freeing the GString container.  */
+    return g_string_free(ds, false);
+}
+
 /* Disassemble this for me please... (debugging). */
 void disas(FILE *out, const void *code, unsigned long size)
 {
diff --git a/include/disas/disas.h b/include/disas/disas.h
index ca3a801913f..3d43b62c4b1 100644
--- a/include/disas/disas.h
+++ b/include/disas/disas.h
@@ -18,6 +18,9 @@ void monitor_disas(Monitor *mon, CPUState *cpu,
 
 char *plugin_disas(CPUState *cpu, uint64_t addr, size_t size);
 
+char *disas_one_strbuf(CPUState *cpu, void *code, unsigned long size,
+                       target_ulong vma);
+
 /* Look up symbol for debugging purpose.  Returns "" if unknown. */
 const char *lookup_symbol(target_ulong orig_addr);
 #endif
diff --git a/include/exec/log_instr_internal.h b/include/exec/log_instr_internal.h
index 1edfb54236a..875cfb9225d 100644
--- a/include/exec/log_instr_internal.h
+++ b/include/exec/log_instr_internal.h
@@ -167,10 +167,17 @@ void emit_text_instr(CPUArchState *env, cpu_log_entry_t *entry);
 /* CVTrace backend */
 void emit_cvtrace_header(CPUArchState *env);
 void emit_cvtrace_entry(CPUArchState *env, cpu_log_entry_t *entry);
+#ifdef CONFIG_TRACE_PERFETTO
 /* Perfetto backend */
 void init_perfetto_backend(CPUArchState *env);
 void sync_perfetto_backend(CPUArchState *env);
 void emit_perfetto_entry(CPUArchState *env, cpu_log_entry_t *entry);
+#endif
+#ifdef CONFIG_TRACE_PROTOBUF
+void init_protobuf_backend(CPUArchState *env);
+void sync_protobuf_backend(CPUArchState *env);
+void emit_protobuf_entry(CPUArchState *env, cpu_log_entry_t *entry);
+#endif
 
 #ifdef CONFIG_DEBUG_TCG
 #define log_assert(x) assert((x))
diff --git a/include/qemu/log_instr.h b/include/qemu/log_instr.h
index 6b8d6c48957..8cbffd4ca37 100644
--- a/include/qemu/log_instr.h
+++ b/include/qemu/log_instr.h
@@ -55,7 +55,10 @@ typedef enum {
     QEMU_LOG_INSTR_BACKEND_CVTRACE = 1,
     QEMU_LOG_INSTR_BACKEND_NOP = 2,
 #ifdef CONFIG_TRACE_PERFETTO
-    QEMU_LOG_INSTR_BACKEND_PERFETTO = 3
+    QEMU_LOG_INSTR_BACKEND_PERFETTO = 3,
+#endif
+#ifdef CONFIG_TRACE_PROTOBUF
+    QEMU_LOG_INSTR_BACKEND_PROTOBUF = 4,
 #endif
 } qemu_log_instr_backend_t;
 
@@ -111,9 +114,9 @@ typedef enum {
  * Tracing status changed (e.g. trace start/stop)
  */
 typedef enum {
-    LOG_EVENT_STATE_START,
-    LOG_EVENT_STATE_STOP,
-    LOG_EVENT_STATE_FLUSH
+    LOG_EVENT_STATE_START = 0,
+    LOG_EVENT_STATE_STOP = 1,
+    LOG_EVENT_STATE_FLUSH = 2
 } log_event_trace_state_t;
 
 typedef struct {
@@ -125,14 +128,14 @@ typedef enum {
     /*
      * Switch context from the current to the new (proc, thread, compartment) ID
      */
-    LOG_EVENT_CTX_OP_SWITCH = 1,
+    LOG_EVENT_CTX_OP_SWITCH = 0,
     /*
      * Same as LOG_EVENT_CTX_OP_SWITCH but should bypass tracing activation
      * status, Meaning that these events will reach the backend even when
      * tracing is off. This is useful setup the correct context identifier
      * before switching on tracing.
      */
-    LOG_EVENT_CTX_OP_SETUP = 2,
+    LOG_EVENT_CTX_OP_SETUP = 1,
 } log_event_ctx_update_op_t;
 
 /*
diff --git a/meson.build b/meson.build
index 8f228da301c..68d81e5659f 100644
--- a/meson.build
+++ b/meson.build
@@ -2182,6 +2182,7 @@ if config_all.has_key('CONFIG_TCG')
   summary_info += {'TCG interpreter':   config_host.has_key('CONFIG_TCG_INTERPRETER')}
   summary_info += {'TCG instruction logging': config_host.has_key('CONFIG_TCG_LOG_INSTR')}
   summary_info += {'TCG instruction logging with google-perfetto': config_host.has_key('CONFIG_TRACE_PERFETTO')}
+  summary_info += {'TCG instruction logging with protobuf backend': config_host.has_key('CONFIG_TRACE_PROTOBUF')}
 endif
 summary_info += {'RVFI-DII':            config_host.has_key('CONFIG_RVFI_DII')}
 summary_info += {'malloc trim support': has_malloc_trim}
diff --git a/softmmu/vl.c b/softmmu/vl.c
index f9d20853d09..fc4e44a3ca5 100644
--- a/softmmu/vl.c
+++ b/softmmu/vl.c
@@ -3758,6 +3758,10 @@ void qemu_init(int argc, char **argv, char **envp)
 #ifdef CONFIG_TRACE_PERFETTO
                 } else if (strcmp(optarg, "perfetto") == 0) {
                     qemu_log_instr_set_backend(QEMU_LOG_INSTR_BACKEND_PERFETTO);
+#endif
+#ifdef CONFIG_TRACE_PROTOBUF
+                } else if (strcmp(optarg, "protobuf") == 0) {
+                    qemu_log_instr_set_backend(QEMU_LOG_INSTR_BACKEND_PROTOBUF);
 #endif
                 } else {
                     printf("Invalid choice for cheri-trace-format: '%s'\n", optarg);
diff --git a/trace_extra/meson.build b/trace_extra/meson.build
index 66424f3bcfa..08b0a48b03b 100644
--- a/trace_extra/meson.build
+++ b/trace_extra/meson.build
@@ -19,3 +19,5 @@ libqemutrace = static_library('qemutrace',
                                          '-DQEMU_PERFETTO'])
 
 qemuutil_libs += libqemutrace
+
+subdir('proto')
diff --git a/trace_extra/proto/meson.build b/trace_extra/proto/meson.build
new file mode 100644
index 00000000000..c5e5834820a
--- /dev/null
+++ b/trace_extra/proto/meson.build
@@ -0,0 +1,11 @@
+
+if config_all.has_key('CONFIG_TCG_LOG_INSTR') and config_all.has_key('CONFIG_TRACE_PROTOBUF')
+   proc_protoc = find_program('protoc')
+   protobuf_backend_files = files('protobuf_backend_entry.proto')
+   gen_protos = custom_target('protobuf_backend_protos',
+                              input: protobuf_backend_files,
+                              output: ['protobuf_backend_entry.pb-c.c', 'protobuf_backend_entry.pb-c.h'],
+                              command: [proc_protoc, '--proto_path=@CURRENT_SOURCE_DIR@', '--c_out=@OUTDIR@', '@INPUT@'],
+                              depend_files: protobuf_backend_files)
+   specific_ss.add(gen_protos)
+endif
diff --git a/trace_extra/proto/protobuf_backend_entry.proto b/trace_extra/proto/protobuf_backend_entry.proto
new file mode 100644
index 00000000000..d90a53f3289
--- /dev/null
+++ b/trace_extra/proto/protobuf_backend_entry.proto
@@ -0,0 +1,95 @@
+
+syntax = "proto3";
+
+// package qemu.protos;
+
+enum QEMULogInstrExcType {
+  TRAP = 0;
+  INTR = 1;
+}
+
+/* Values must be kept in sync with log_event_trace_state_t */
+enum LogEventTraceState {
+  START = 0;
+  STOP = 1;
+  FLUSH = 2;
+}
+
+/* Values must be kept in sync with log_event_ctx_update_op_t */
+enum LogEventCtxUpdateOp {
+  LOG_EVENT_CTX_OP_SWITCH = 0;
+  LOG_EVENT_CTX_OP_SETUP = 1;
+}
+
+message QEMULogInstrCapability {
+  bool valid = 1;
+  bool sealed = 2;
+  uint32 perms = 3;
+  uint32 otype = 4;
+  uint64 cap_base = 5;
+  uint64 cap_cursor = 6;
+  uint64 cap_length = 7;
+}
+
+message QEMULogInstrMem {
+  bool is_load = 1;
+  int32 size = 2;
+  fixed64 addr = 3;
+  oneof value {
+    QEMULogInstrCapability cap_value = 4;
+    double fp_value = 5;
+    fixed64 int_value = 6;
+  }
+}
+
+message QEMULogInstrReg {
+  string name = 1;
+  oneof value {
+    fixed64 int_value = 2;
+    double fp_value = 3;
+    QEMULogInstrCapability cap_value = 4;
+  }
+}
+
+message QEMULogInstrEvtTraceState {
+  LogEventTraceState next_state = 1;
+  fixed64 pc = 2;
+}
+
+message QEMULogInstrEvtCtxUpdate {
+  LogEventCtxUpdateOp op = 1;
+  uint64 pid = 2;
+  uint64 tid = 3;
+  uint64 cid = 4;
+  string mode = 5;
+}
+
+message QEMULogInstrEvt {
+  oneof event {
+    QEMULogInstrEvtTraceState state = 1;
+    QEMULogInstrEvtCtxUpdate ctx_update = 2;
+    uint64 marker = 3;
+  }
+}
+
+message QEMULogInstrExc {
+  QEMULogInstrExcType type = 1;
+  fixed64 vector = 2;
+  int32 code = 3;
+  fixed64 faultaddr = 4;
+}
+
+message QEMULogInstrEntry {
+  fixed64 pc = 1;
+  oneof insn {
+    string disas = 2;
+    bytes opcode = 3;
+  }
+  int32 cpu = 4;
+  int32 asid = 5;
+  string mode = 6;
+  QEMULogInstrExc exception = 7;
+  repeated QEMULogInstrMem mem = 8;
+  repeated QEMULogInstrReg reg = 9;
+  repeated QEMULogInstrEvt evt = 10;
+}

From 5d94acb4fcf87d33501449db68e6cb079b3066fe Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Wed, 22 Dec 2021 18:43:33 +0000
Subject: [PATCH 40/74] Add experimental instruction logging backend that
 generates JSON data.

This should have similar efficiency to the text instruction format
but be more stable and easier to parse. The protobuf backend should
be preferred for parsing.
---
 accel/tcg/log_instr.c             |  49 +++++-
 accel/tcg/log_instr_json.c        | 248 ++++++++++++++++++++++++++++++
 accel/tcg/meson.build             |   5 +
 configure                         |   8 +
 include/exec/log_instr_internal.h |   5 +
 include/qemu/log_instr.h          |  24 +++
 meson.build                       |   1 +
 qemu-options.hx                   |   7 +
 softmmu/vl.c                      |   9 ++
 9 files changed, 348 insertions(+), 8 deletions(-)
 create mode 100644 accel/tcg/log_instr_json.c

diff --git a/accel/tcg/log_instr.c b/accel/tcg/log_instr.c
index dbb07e341dd..811af245e4d 100644
--- a/accel/tcg/log_instr.c
+++ b/accel/tcg/log_instr.c
@@ -128,6 +128,12 @@ static trace_backend_hooks_t trace_backends[] = {
       .emit_header = NULL,
       .emit_instr = emit_protobuf_entry },
 #endif
+#ifdef CONFIG_TRACE_JSON
+    { .init = init_json_backend,
+      .sync = sync_json_backend,
+      .emit_header = NULL,
+      .emit_instr = emit_json_entry },
+#endif
 };
 
 /* Existing trace filters list, indexed by cpu_log_instr_filter_t */
@@ -141,11 +147,36 @@ static GArray *reset_filters;
 
 static unsigned long reset_entry_buffer_size = MIN_ENTRY_BUFFER_SIZE;
 
+static bool trace_debug;
+
 static void emit_nop_entry(CPUArchState *env, cpu_log_entry_t *entry)
 {
     return;
 }
 
+static void dump_debug_stats(CPUState *cpu)
+{
+    cpu_log_instr_state_t *cpulog = get_cpu_log_state(cpu->env_ptr);
+    qemu_log_instr_stats_t *stats = &cpulog->stats;
+
+    if (!trace_debug) {
+        return;
+    }
+
+    fprintf(stderr, "TCG Instruction tracing statistics: CPU #%d\n",
+            cpu->cpu_index);
+    fprintf(stderr, "entries emitted: %lu\n", stats->entries_emitted);
+    fprintf(stderr, "trace slices: %lu\n", stats->trace_start);
+    if (stats->trace_start != stats->trace_stop) {
+        fprintf(stderr, "Unbalanced trace stop: %lu\n", stats->trace_stop);
+    }
+}
+
+void qemu_log_instr_enable_trace_debug()
+{
+    trace_debug = true;
+}
+
 static inline void emit_start_event(cpu_log_entry_t *entry, target_ulong pc)
 {
     log_event_t event;
@@ -221,6 +252,7 @@ static void do_instr_commit(CPUArchState *env)
                 (cpulog->ring_tail + 1) % cpulog->instr_info->len;
     } else {
         trace_backend->emit_instr(env, entry);
+        QEMU_LOG_INSTR_INC_STAT(cpulog, entries_emitted);
     }
 }
 
@@ -289,6 +321,7 @@ static void do_cpu_loglevel_switch(CPUState *cpu, run_on_cpu_data data)
             goto done;
         }
         emit_stop_event(entry, pc);
+        QEMU_LOG_INSTR_INC_STAT(cpulog, trace_stop);
         do_instr_commit(env);
         /* Instruction commit may have advanced to the next entry buffer slot */
         entry = get_cpu_log_entry(env);
@@ -301,6 +334,7 @@ static void do_cpu_loglevel_switch(CPUState *cpu, run_on_cpu_data data)
          * traced
          */
         emit_start_event(entry, pc);
+        QEMU_LOG_INSTR_INC_STAT(cpulog, trace_start);
     }
 
 done:
@@ -477,13 +511,16 @@ void qemu_log_instr_init(CPUState *cpu)
                 cpu, g_array_index(reset_filters, cpu_log_instr_filter_t, i));
         }
     }
+
+    memset(&cpulog->stats, 0, sizeof(cpulog->stats));
 }
 
 static void do_log_backend_sync(CPUState *cpu, run_on_cpu_data _unused)
 {
-    log_assert(trace_backend->sync != NULL &&
-               "Missing sync() callback on trace backend");
-    trace_backend->sync(cpu->env_ptr);
+    if (trace_backend->sync != NULL) {
+        trace_backend->sync(cpu->env_ptr);
+    }
+    dump_debug_stats(cpu);
 }
 
 /*
@@ -494,11 +531,6 @@ void qemu_log_instr_sync_buffers()
 {
     CPUState *cpu;
 
-    /* Avoid doing this early if there is no sync function */
-    if (trace_backend->sync == NULL) {
-        return;
-    }
-
     CPU_FOREACH(cpu)
     {
         run_on_cpu(cpu, do_log_backend_sync, RUN_ON_CPU_NULL);
@@ -1170,6 +1202,7 @@ void qemu_log_instr_flush(CPUArchState *env)
     while (curr != cpulog->ring_head) {
         entry = &g_array_index(cpulog->instr_info, cpu_log_entry_t, curr);
         trace_backend->emit_instr(env, entry);
+        QEMU_LOG_INSTR_INC_STAT(cpulog, entries_emitted);
         curr = (curr + 1) % cpulog->instr_info->len;
     }
     cpulog->ring_tail = cpulog->ring_head;
diff --git a/accel/tcg/log_instr_json.c b/accel/tcg/log_instr_json.c
new file mode 100644
index 00000000000..3587c3063a0
--- /dev/null
+++ b/accel/tcg/log_instr_json.c
@@ -0,0 +1,248 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2021 Alfredo Mazzinghi
+ *
+ * This software was developed by SRI International and the University of
+ * Cambridge Computer Laboratory (Department of Computer Science and
+ * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the
+ * DARPA SSITH research programme.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * Similar functionality as the text backend but with a more stable format.
+ */
+
+#include <assert.h>
+#include <cjson/cJSON.h>
+
+#include "qemu/osdep.h"
+#include "qemu/log.h"
+#include "cpu.h"
+#include "exec/log_instr.h"
+#include "exec/log_instr_internal.h"
+#include "exec/memop.h"
+#include "disas/disas.h"
+
+static inline cJSON *emit_json_hex(uint64_t value)
+{
+    char buf[sizeof(value) * 2 + 1];
+
+    snprintf((char *)&buf, sizeof(buf), "%lx", value);
+    return cJSON_CreateString((char *)&buf);
+}
+
+#ifdef TARGET_CHERI
+static inline cJSON *emit_json_capability(cap_register_t *cap)
+{
+    cJSON *js_cap = cJSON_CreateObject();
+    cJSON *valid = cJSON_CreateBool(cap->cr_tag);
+    cJSON *sealed = cJSON_CreateBool(cap_is_sealed_with_type(cap));
+    cJSON *perms = cJSON_CreateNumber(COMBINED_PERMS_VALUE(cap));
+    cJSON *flags = cJSON_CreateNumber(cap_get_flags(cap));
+    cJSON *base = emit_json_hex(cap_get_base(cap));
+    cJSON *length = emit_json_hex(cap_get_length_sat(cap));
+    cJSON *offset = emit_json_hex(cap_get_offset(cap));
+    cJSON *otype = emit_json_hex(cap_get_otype_unsigned(cap));
+
+    cJSON_AddItemToObject(js_cap, "v", valid);
+    cJSON_AddItemToObject(js_cap, "s", sealed);
+    cJSON_AddItemToObject(js_cap, "p", perms);
+    cJSON_AddItemToObject(js_cap, "f", flags);
+    cJSON_AddItemToObject(js_cap, "b", base);
+    cJSON_AddItemToObject(js_cap, "l", length);
+    cJSON_AddItemToObject(js_cap, "o", offset);
+    cJSON_AddItemToObject(js_cap, "t", otype);
+    return js_cap;
+}
+#endif
+
+static inline void emit_json_ldst(log_meminfo_t *minfo, cJSON *list)
+{
+    cJSON *mem = cJSON_CreateObject();
+    cJSON *is_load = cJSON_CreateBool((minfo->flags & LMI_LD) != 0);
+    cJSON *size = cJSON_CreateNumber(memop_size(minfo->op));
+    cJSON *addr = emit_json_hex(minfo->addr);
+    cJSON *value;
+
+#ifndef TARGET_CHERI
+    assert((minfo->flags & LMI_CAP) == 0 &&
+           "Capability memory access without CHERI support");
+#else
+    if (minfo->flags & LMI_CAP) {
+        value = emit_json_capability(&minfo->cap);
+    } else
+#endif
+    {
+        value = emit_json_hex(minfo->value);
+    }
+
+    cJSON_AddItemToObject(mem, "is_load", is_load);
+    cJSON_AddItemToObject(mem, "size", size);
+    cJSON_AddItemToObject(mem, "addr", addr);
+    cJSON_AddItemToObject(mem, "value", value);
+    cJSON_AddItemToArray(list, mem);
+}
+
+static inline void emit_json_reg(log_reginfo_t *rinfo, cJSON *list)
+{
+    cJSON *reg = cJSON_CreateObject();
+    cJSON *name = cJSON_CreateString(rinfo->name);
+    cJSON *value;
+
+    cJSON_AddItemToObject(reg, "name", name);
+
+#ifndef TARGET_CHERI
+    assert(!reginfo_is_cap(rinfo) && "Register marked as capability "
+                                     "register whitout CHERI support");
+#else
+    if (reginfo_is_cap(rinfo)) {
+        if (reginfo_has_cap(rinfo)) {
+            value = emit_json_capability(&rinfo->cap);
+        } else {
+            value = cJSON_CreateNumber(rinfo->gpr);
+        }
+    } else
+#endif
+    {
+        value = cJSON_CreateNumber(rinfo->gpr);
+    }
+
+    cJSON_AddItemToObject(reg, "value", value);
+    cJSON_AddItemToArray(list, reg);
+}
+
+void sync_json_backend(CPUArchState *env)
+{
+    /*
+     * If we are shutting down, close the json entry list manually
+     * and add a dummy entry. This is an hack but it's easier than
+     * deleting the preceding comma.
+     */
+    qemu_log("{}]");
+}
+
+void init_json_backend(CPUArchState *env)
+{
+    /* Initialize the json logfile */
+    static bool initialized;
+
+    if (!initialized) {
+        initialized = true;
+        qemu_log("[");
+    }
+}
+
+void emit_json_entry(CPUArchState *env, cpu_log_entry_t *entry)
+{
+    const log_event_t *event;
+    int i;
+
+    /*
+     * XXX would be nice to use libxo but it's an annoying requirement for Linux
+     * targets.
+     */
+    cJSON *js_entry = cJSON_CreateObject();
+
+    if (entry->flags & LI_FLAG_HAS_INSTR_DATA) {
+        cJSON *pc = emit_json_hex(entry->pc);
+        cJSON *cpu = cJSON_CreateNumber(env_cpu(env)->cpu_index);
+        cJSON *asid = cJSON_CreateNumber(entry->asid);
+        cJSON *insn = cJSON_CreateString(
+            disas_one_strbuf(env_cpu(env), entry->insn_bytes,
+                             sizeof(entry->insn_bytes), entry->pc));
+        cJSON_AddItemToObject(js_entry, "pc", pc);
+        cJSON_AddItemToObject(js_entry, "cpu", cpu);
+        cJSON_AddItemToObject(js_entry, "asid", asid);
+        cJSON_AddItemToObject(js_entry, "insn", insn);
+
+        if (entry->flags & LI_FLAG_MODE_SWITCH) {
+            cJSON *mode =
+                cJSON_CreateString(cpu_get_mode_name(entry->next_cpu_mode));
+            cJSON_AddItemToObject(js_entry, "mode-switch", mode);
+        }
+
+        if (entry->flags & LI_FLAG_INTR_MASK) {
+            cJSON *exc = cJSON_CreateObject();
+            cJSON *vec = cJSON_CreateNumber(entry->intr_vector);
+            cJSON *code = cJSON_CreateNumber(entry->intr_code);
+            cJSON *faultaddr = NULL;
+            cJSON *type;
+            if (entry->flags & LI_FLAG_INTR_TRAP) {
+                type = cJSON_CreateString("trap");
+                faultaddr = emit_json_hex(entry->intr_faultaddr);
+
+            } else if (entry->flags & LI_FLAG_INTR_ASYNC) {
+                type = cJSON_CreateString("intr");
+            } else {
+                assert(0 && "invalid exception flags");
+            }
+            cJSON_AddItemToObject(exc, "type", type);
+            cJSON_AddItemToObject(exc, "vec", vec);
+            cJSON_AddItemToObject(exc, "code", code);
+            if (faultaddr) {
+                cJSON_AddItemToObject(exc, "faultaddr", faultaddr);
+            }
+            cJSON_AddItemToObject(js_entry, "exc", exc);
+        }
+
+        if (entry->mem->len > 0) {
+            cJSON *mem = cJSON_CreateArray();
+            for (i = 0; i < entry->mem->len; i++) {
+                log_meminfo_t *minfo =
+                    &g_array_index(entry->mem, log_meminfo_t, i);
+                emit_json_ldst(minfo, mem);
+            }
+            cJSON_AddItemToObject(js_entry, "mem", mem);
+        }
+
+        if (entry->regs->len > 0) {
+            cJSON *reg = cJSON_CreateArray();
+            for (i = 0; i < entry->regs->len; i++) {
+                log_reginfo_t *rinfo =
+                    &g_array_index(entry->regs, log_reginfo_t, i);
+                emit_json_reg(rinfo, reg);
+            }
+            cJSON_AddItemToObject(js_entry, "reg", reg);
+        }
+    }
+
+    if (entry->events->len > 0) {
+        for (i = 0; i < entry->events->len; i++) {
+            event = &g_array_index(entry->events, const log_event_t, i);
+            switch (event->id) {
+            case LOG_EVENT_STATE:
+                break;
+            case LOG_EVENT_CTX_UPDATE:
+                break;
+            case LOG_EVENT_MARKER:
+                break;
+            default:
+                assert(0 && "unknown event ID");
+            }
+        }
+    }
+    qemu_log("%s,", cJSON_PrintUnformatted(js_entry));
+    cJSON_Delete(js_entry);
+}
diff --git a/accel/tcg/meson.build b/accel/tcg/meson.build
index 23a4b0a30e0..711490f8171 100644
--- a/accel/tcg/meson.build
+++ b/accel/tcg/meson.build
@@ -18,7 +18,12 @@ specific_ss.add(when: ['CONFIG_TCG_LOG_INSTR', 'CONFIG_TCG'], if_true: files('lo
 specific_ss.add(when: ['CONFIG_TCG_LOG_INSTR', 'CONFIG_TCG'], if_true: files('log_instr_cvtrace.c'))
 specific_ss.add(when: ['CONFIG_TRACE_PERFETTO', 'CONFIG_TCG_LOG_INSTR', 'CONFIG_TCG'], if_true: files('log_instr_perfetto.c'))
 specific_ss.add(when: ['CONFIG_TRACE_PROTOBUF', 'CONFIG_TCG_LOG_INSTR', 'CONFIG_TCG'], if_true: files('log_instr_protobuf.c'))
+specific_ss.add(when: ['CONFIG_TRACE_JSON', 'CONFIG_TCG_LOG_INSTR', 'CONFIG_TCG'], if_true: files('log_instr_json.c'))
 
 if config_all.has_key('CONFIG_TCG_LOG_INSTR') and config_all.has_key('CONFIG_TRACE_PROTOBUF')
    specific_ss.add(dependency('libprotobuf-c'))
 endif
+
+if config_all.has_key('CONFIG_TCG_LOG_INSTR') and config_all.has_key('CONFIG_TRACE_JSON')
+   specific_ss.add(dependency('libcjson'))
+endif
diff --git a/configure b/configure
index 61d5879b5b2..0b1f9fda713 100755
--- a/configure
+++ b/configure
@@ -342,6 +342,7 @@ debug_tcg="no"
 tcg_log_instr="yes"
 tcg_log_instr_perfetto="no"
 tcg_log_instr_protobuf="no"
+tcg_log_instr_json="no"
 rvfi_dii="no"
 debug="no"
 sanitizers="no"
@@ -1046,6 +1047,10 @@ for opt do
   ;;
   --disable-protobuf-log-instr) tcg_log_instr_protobuf="no"
   ;;
+  --enable-json-log-instr) tcg_log_instr_json="yes"
+  ;;
+  --disable-json-log-instr) tcg_log_instr_json="no"
+  ;;
   --enable-rvfi-dii) rvfi_dii="yes"
   ;;
   --disable-rvfi-dii) rvfi_dii="no"
@@ -5984,6 +5989,9 @@ fi
 if test "$tcg_log_instr_protobuf" = "yes" ; then
     echo "CONFIG_TRACE_PROTOBUF=y" >> $config_host_mak
 fi
+if test "$tcg_log_instr_json" = "yes" ; then
+    echo "CONFIG_TRACE_JSON=y" >> $config_host_mak
+fi
 if test "$rvfi_dii" = "yes" ; then
   echo "CONFIG_RVFI_DII=y" >> $config_host_mak
 fi
diff --git a/include/exec/log_instr_internal.h b/include/exec/log_instr_internal.h
index 875cfb9225d..3123ea8c239 100644
--- a/include/exec/log_instr_internal.h
+++ b/include/exec/log_instr_internal.h
@@ -178,6 +178,11 @@ void init_protobuf_backend(CPUArchState *env);
 void sync_protobuf_backend(CPUArchState *env);
 void emit_protobuf_entry(CPUArchState *env, cpu_log_entry_t *entry);
 #endif
+#ifdef CONFIG_TRACE_JSON
+void init_json_backend(CPUArchState *env);
+void sync_json_backend(CPUArchState *env);
+void emit_json_entry(CPUArchState *env, cpu_log_entry_t *entry);
+#endif
 
 #ifdef CONFIG_DEBUG_TCG
 #define log_assert(x) assert((x))
diff --git a/include/qemu/log_instr.h b/include/qemu/log_instr.h
index 8cbffd4ca37..1f45378c643 100644
--- a/include/qemu/log_instr.h
+++ b/include/qemu/log_instr.h
@@ -60,6 +60,9 @@ typedef enum {
 #ifdef CONFIG_TRACE_PROTOBUF
     QEMU_LOG_INSTR_BACKEND_PROTOBUF = 4,
 #endif
+#ifdef CONFIG_TRACE_JSON
+    QEMU_LOG_INSTR_BACKEND_JSON = 5,
+#endif
 } qemu_log_instr_backend_t;
 
 extern qemu_log_instr_backend_t qemu_log_instr_backend;
@@ -212,6 +215,20 @@ typedef struct {
     uint64_t valid_entries; /* bitmap of which entries are valid */
 } qemu_log_printf_buf_t;
 
+/*
+ * Per-CPU tracing statistics.
+ */
+typedef struct qemu_log_instr_stats {
+    uint64_t entries_emitted;
+    uint64_t trace_start;
+    uint64_t trace_stop;
+} qemu_log_instr_stats_t;
+
+#define QEMU_LOG_INSTR_INC_STAT(cpu_state, stat) \
+    do {                                         \
+        cpu_state->stats.stat++;                 \
+    } while (0)
+
 /*
  * Per-cpu logging state.
  */
@@ -239,6 +256,8 @@ typedef struct {
     qemu_log_printf_buf_t qemu_log_printf_buf;
     /* Private trace backend state */
     void *backend_data;
+    /* Statistics for debugging */
+    qemu_log_instr_stats_t stats;
 } cpu_log_instr_state_t;
 
 /*
@@ -298,6 +317,11 @@ void qemu_log_instr_add_startup_filter(cpu_log_instr_filter_t filter);
  * memory ranges..
  */
 void qemu_log_instr_mem_filter_update(void);
+
+/*
+ * Enable debug statistics recording.
+ */
+void qemu_log_instr_enable_trace_debug(void);
 #endif /* ! __cplusplus */
 
 #else /* ! CONFIG_TCG_LOG_INSTR */
diff --git a/meson.build b/meson.build
index 68d81e5659f..e4db27e6323 100644
--- a/meson.build
+++ b/meson.build
@@ -2183,6 +2183,7 @@ if config_all.has_key('CONFIG_TCG')
   summary_info += {'TCG instruction logging': config_host.has_key('CONFIG_TCG_LOG_INSTR')}
   summary_info += {'TCG instruction logging with google-perfetto': config_host.has_key('CONFIG_TRACE_PERFETTO')}
   summary_info += {'TCG instruction logging with protobuf backend': config_host.has_key('CONFIG_TRACE_PROTOBUF')}
+  summary_info += {'TCG instruction logging with json backend': config_host.has_key('CONFIG_TRACE_JSON')}
 endif
 summary_info += {'RVFI-DII':            config_host.has_key('CONFIG_RVFI_DII')}
 summary_info += {'malloc trim support': has_malloc_trim}
diff --git a/qemu-options.hx b/qemu-options.hx
index ef396d1481b..6cf4b3b0362 100644
--- a/qemu-options.hx
+++ b/qemu-options.hx
@@ -4145,6 +4145,13 @@ SRST
     Set CHERI trace filters to use. Available filters are: events.
 ERST
 
+DEF("cheri-trace-debug", 0, QEMU_OPTION_cheri_trace_debug, \
+"-cheri-trace-debug     Enable debug stats.\n", QEMU_ARCH_ALL)
+SRST
+``-cheri-trace-debug ``
+    Enable CHERI instruction tracing debugging statistics.
+ERST
+
 DEF("cheri-c2e-on-unrepresentable", 0, QEMU_OPTION_cheri_c2e_on_unrepresentable, \
     "-cheri-c2e-on-unrepresentable     Generate C2E exception when a capability becomes unrepresentable\n", QEMU_ARCH_ALL)
 SRST
diff --git a/softmmu/vl.c b/softmmu/vl.c
index fc4e44a3ca5..0656be9533d 100644
--- a/softmmu/vl.c
+++ b/softmmu/vl.c
@@ -3755,6 +3755,8 @@ void qemu_init(int argc, char **argv, char **envp)
                     qemu_log_instr_set_backend(QEMU_LOG_INSTR_BACKEND_TEXT);
                 } else if (strcmp(optarg, "cvtrace") == 0) {
                     qemu_log_instr_set_backend(QEMU_LOG_INSTR_BACKEND_CVTRACE);
+                } else if (strcmp(optarg, "nop") == 0) {
+                    qemu_log_instr_set_backend(QEMU_LOG_INSTR_BACKEND_NOP);
 #ifdef CONFIG_TRACE_PERFETTO
                 } else if (strcmp(optarg, "perfetto") == 0) {
                     qemu_log_instr_set_backend(QEMU_LOG_INSTR_BACKEND_PERFETTO);
@@ -3762,6 +3764,10 @@ void qemu_init(int argc, char **argv, char **envp)
 #ifdef CONFIG_TRACE_PROTOBUF
                 } else if (strcmp(optarg, "protobuf") == 0) {
                     qemu_log_instr_set_backend(QEMU_LOG_INSTR_BACKEND_PROTOBUF);
+#endif
+#ifdef CONFIG_TRACE_JSON
+                } else if (strcmp(optarg, "json") == 0) {
+                    qemu_log_instr_set_backend(QEMU_LOG_INSTR_BACKEND_JSON);
 #endif
                 } else {
                     printf("Invalid choice for cheri-trace-format: '%s'\n", optarg);
@@ -3782,6 +3788,9 @@ void qemu_init(int argc, char **argv, char **envp)
             case QEMU_OPTION_cheri_trace_filters:
                 qemu_log_instr_set_cli_filters(optarg, &error_fatal);
                 break;
+            case QEMU_OPTION_cheri_trace_debug:
+                qemu_log_instr_enable_trace_debug();
+                break;
 #endif /* CONFIG_TCG_LOG_INSTR */
 
 #ifdef TARGET_CHERI

From 3f712cd14475597469bad316b9812ee632cfb3ed Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Thu, 23 Dec 2021 11:34:17 +0000
Subject: [PATCH 41/74] Fix build without the cheri-perfetto submodule.

Only look for cheri-perfetto files if perfetto instruction logging is enabled.
---
 trace_extra/meson.build | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/trace_extra/meson.build b/trace_extra/meson.build
index 08b0a48b03b..c7802420a42 100644
--- a/trace_extra/meson.build
+++ b/trace_extra/meson.build
@@ -3,12 +3,10 @@ qemutrace_ss = ss.source_set()
 
 if config_all.has_key('CONFIG_TCG_LOG_INSTR') and config_all.has_key('CONFIG_TRACE_PERFETTO')
    qemutrace_ss.add(dependency('boost', modules: ['filesystem', 'system']))
-endif
-
-qemutrace_ss.add(when: ['CONFIG_TCG', 'CONFIG_TCG_LOG_INSTR', 'CONFIG_TRACE_PERFETTO'],
-                 if_true: files('trace_perfetto.cc', 'trace_stats.cc',
+   qemutrace_ss.add(files('trace_perfetto.cc', 'trace_stats.cc',
                                 'guest_context_tracker.cc',
                                 'cheri-perfetto/sdk/perfetto.cc'))
+endif
 
 qemutrace_ss = qemutrace_ss.apply(config_all, strict: false)
 libqemutrace = static_library('qemutrace',

From 2f3921588564518e391035719bff6df144e20eba Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Wed, 19 Jan 2022 21:58:52 +0000
Subject: [PATCH 42/74] Update CHERI perfetto interface.

 - Use the same protobuf descriptor for instructions as the one used
 by the protobuf backend.
 - Use the CHERI context track implementation to embed context
 identifiers in the trace instead of using the Track name.
---
 trace_extra/guest_context_tracker.cc |  16 +--
 trace_extra/guest_context_tracker.hh |   6 +-
 trace_extra/trace_perfetto.cc        | 142 +++++++++++++--------------
 trace_extra/tuple_index.hh           |  22 +++--
 4 files changed, 93 insertions(+), 93 deletions(-)

diff --git a/trace_extra/guest_context_tracker.cc b/trace_extra/guest_context_tracker.cc
index 6773d2afcfa..41176ec29ca 100644
--- a/trace_extra/guest_context_tracker.cc
+++ b/trace_extra/guest_context_tracker.cc
@@ -53,12 +53,12 @@ unsigned long gen_track_uuid()
     return (next_track_id++);
 }
 
-perfetto::protos::pbzero::ModeSwitch
+perfetto::protos::pbzero::QEMULogEntryModeSwitch
 qemu_cpu_mode_to_trace(qemu_log_instr_cpu_mode_t mode)
 {
     // NOTE: We rely on the fact that the protobuf enum ModeSwitch
     // uses the same numbering as qemu_log_instr_cpu_mode_t
-    return static_cast<perfetto::protos::pbzero::ModeSwitch>(mode);
+    return static_cast<perfetto::protos::pbzero::QEMULogEntryModeSwitch>(mode);
 }
 
 /* static */
@@ -76,11 +76,11 @@ qemu_context_track::qemu_ctx_id qemu_context_track::get_id() const
 perfetto::protos::gen::TrackDescriptor qemu_context_track::Serialize() const
 {
     auto desc = Track::Serialize();
-    auto qemu_desc = desc.mutable_qemu_context();
-    qemu_desc->set_pid(pid);
-    qemu_desc->set_tid(tid);
-    qemu_desc->set_cid(cid);
-    // qemu_desc->set_mode(mode); // TODO not yet in cheri-perfetto
+    auto cheri_desc = desc.mutable_cheri_context();
+    cheri_desc->set_pid(pid);
+    cheri_desc->set_tid(tid);
+    cheri_desc->set_cid(cid);
+    cheri_desc->set_el(mode);
     return desc;
 }
 
@@ -106,7 +106,7 @@ guest_context_tracker::guest_context_tracker(int cpu_id)
 }
 
 void guest_context_tracker::mode_update(
-    perfetto::protos::pbzero::ModeSwitch new_mode)
+    perfetto::protos::pbzero::QEMULogEntryModeSwitch new_mode)
 {
     if (ctx_track_ == nullptr)
         return;
diff --git a/trace_extra/guest_context_tracker.hh b/trace_extra/guest_context_tracker.hh
index 8019ffc173a..67804a7f6c5 100644
--- a/trace_extra/guest_context_tracker.hh
+++ b/trace_extra/guest_context_tracker.hh
@@ -49,7 +49,7 @@ struct qemu_context_data {
 
 struct qemu_context_track : public perfetto::Track {
 
-    using cpu_mode_type = perfetto::protos::pbzero::ModeSwitch;
+    using cpu_mode_type = perfetto::protos::pbzero::QEMULogEntryModeSwitch;
     /* (pid, tid, cid, mode) */
     using qemu_ctx_id = std::tuple<uint64_t, uint64_t, uint64_t, cpu_mode_type>;
 
@@ -105,7 +105,7 @@ class guest_context_tracker
   public:
     guest_context_tracker(int cpu_id);
     void context_update(const log_event_ctx_update_t *evt);
-    void mode_update(perfetto::protos::pbzero::ModeSwitch new_mode);
+    void mode_update(perfetto::protos::pbzero::QEMULogEntryModeSwitch new_mode);
     void flush_all_ctx_data();
     perfetto::Track &get_cpu_track();
     perfetto::Track &get_ctx_track();
@@ -119,7 +119,7 @@ class guest_context_tracker
  */
 unsigned long gen_track_uuid();
 
-perfetto::protos::pbzero::ModeSwitch
+perfetto::protos::pbzero::QEMULogEntryModeSwitch
 qemu_cpu_mode_to_trace(qemu_log_instr_cpu_mode_t mode);
 
 } // namespace cheri
diff --git a/trace_extra/trace_perfetto.cc b/trace_extra/trace_perfetto.cc
index bf4908414d5..d9110d14161 100644
--- a/trace_extra/trace_perfetto.cc
+++ b/trace_extra/trace_perfetto.cc
@@ -46,6 +46,7 @@
 #include <perfetto.h>
 #include <boost/filesystem.hpp>
 
+// #include "qemu/cpu-defs.h"
 #include "qemu/log_instr.h"
 #include "exec/log_instr_internal.h"
 #include "exec/log_instr_perfetto.h"
@@ -163,14 +164,14 @@ void perfetto_tracing_stop(void)
     session->StopBlocking();
 }
 
-void trace_cap_register(perfetto::protos::pbzero::Capability *cap,
+void trace_cap_register(perfetto::protos::pbzero::QEMULogEntryCapability *cap,
                         cap_register_handle chandle)
 {
     cap->set_valid(perfetto_cap_tag(chandle));
     cap->set_sealed(perfetto_cap_sealed(chandle));
-    cap->set_base(perfetto_cap_base(chandle));
-    cap->set_length(perfetto_cap_length(chandle));
-    cap->set_cursor(perfetto_cap_cursor(chandle));
+    cap->set_cap_base(perfetto_cap_base(chandle));
+    cap->set_cap_length(perfetto_cap_length(chandle));
+    cap->set_cap_cursor(perfetto_cap_cursor(chandle));
     cap->set_perms(perfetto_cap_perms(chandle));
     cap->set_otype(perfetto_cap_otype(chandle));
 }
@@ -189,55 +190,51 @@ void process_events(perfetto_backend_data *data, cpu_log_entry_handle entry)
     for (int i = 0; i < nevents; i++) {
         log_event_t *evt = perfetto_log_event(entry, i);
         switch (evt->id) {
-            case LOG_EVENT_STATE:
-                {
-                    switch (evt->state.next_state) {
-                        case LOG_EVENT_STATE_FLUSH:
-                            TRACE_EVENT_INSTANT("ctrl", "flush", data->ctrl_track_);
-                            data->ctx_tracker_.flush_all_ctx_data();
-                            perfetto::TrackEvent::Flush();
-                            break;
-                        case LOG_EVENT_STATE_START:
-                            ctx_data->stats.unpause(*ctx_track, evt->state.pc);
-                            TRACE_EVENT_BEGIN("ctrl", "tracing", data->ctrl_track_);
-                            have_startstop_event = true;
-                            break;
-                        case LOG_EVENT_STATE_STOP:
-                            ctx_data->stats.pause(*ctx_track, evt->state.pc);
-                            TRACE_EVENT_END("ctrl", data->ctrl_track_);
-                            have_startstop_event = true;
-                            break;
-                        default:
-                            assert(false && "Invalid state event");
-                    }
-                }
+        case LOG_EVENT_STATE: {
+            switch (evt->state.next_state) {
+            case LOG_EVENT_STATE_FLUSH:
+                TRACE_EVENT_INSTANT("ctrl", "flush", data->ctrl_track_);
+                data->ctx_tracker_.flush_all_ctx_data();
+                perfetto::TrackEvent::Flush();
                 break;
-            case LOG_EVENT_CTX_UPDATE:
-                {
-                    /* Swap current context. */
-                    if (evt->ctx_update.op == LOG_EVENT_CTX_OP_SETUP ||
-                        evt->ctx_update.op == LOG_EVENT_CTX_OP_SWITCH) {
-                        ctx_data->stats.pause(*ctx_track, perfetto_log_entry_pc(entry));
-                        data->ctx_tracker_.context_update(&evt->ctx_update);
-                        /* Reload data and track as context_update will have changed them */
-                        ctx_data = &data->ctx_tracker_.get_ctx_data();
-                        ctx_track = &data->ctx_tracker_.get_ctx_track();
-                        ctx_data->stats.unpause(*ctx_track, perfetto_log_entry_pc(entry));
-                    }
-                }
+            case LOG_EVENT_STATE_START:
+                ctx_data->stats.unpause(*ctx_track, evt->state.pc);
+                TRACE_EVENT_BEGIN("ctrl", "tracing", data->ctrl_track_);
+                have_startstop_event = true;
                 break;
-            case LOG_EVENT_MARKER:
-                {
-                    auto cpu_track = data->ctx_tracker_.get_cpu_track();
-                    TRACE_EVENT_INSTANT("marker", "guest", cpu_track,
-                        [&](perfetto::EventContext ctx) {
-                            auto *qemu_arg = ctx.event()->set_qemu();
-                            qemu_arg->set_marker(evt->marker);
-                        });
-                }
+            case LOG_EVENT_STATE_STOP:
+                ctx_data->stats.pause(*ctx_track, evt->state.pc);
+                TRACE_EVENT_END("ctrl", data->ctrl_track_);
+                have_startstop_event = true;
                 break;
             default:
-                assert(false && "Invalid event identifier");
+                assert(false && "Invalid state event");
+            }
+        } break;
+        case LOG_EVENT_CTX_UPDATE: {
+            /* Swap current context. */
+            if (evt->ctx_update.op == LOG_EVENT_CTX_OP_SETUP ||
+                evt->ctx_update.op == LOG_EVENT_CTX_OP_SWITCH) {
+                ctx_data->stats.pause(*ctx_track, perfetto_log_entry_pc(entry));
+                data->ctx_tracker_.context_update(&evt->ctx_update);
+                /* Reload data and track as context_update will have changed
+                 * them */
+                ctx_data = &data->ctx_tracker_.get_ctx_data();
+                ctx_track = &data->ctx_tracker_.get_ctx_track();
+                ctx_data->stats.unpause(*ctx_track,
+                                        perfetto_log_entry_pc(entry));
+            }
+        } break;
+        case LOG_EVENT_MARKER: {
+            auto cpu_track = data->ctx_tracker_.get_cpu_track();
+            TRACE_EVENT_INSTANT("marker", "guest", cpu_track,
+                                [&](perfetto::EventContext ctx) {
+                                    auto *qemu_arg = ctx.event()->set_qemu();
+                                    qemu_arg->set_marker(evt->marker);
+                                });
+        } break;
+        default:
+            assert(false && "Invalid event identifier");
         }
     }
     if (perfetto_log_entry_flags(entry) & LI_FLAG_MODE_SWITCH) {
@@ -272,7 +269,8 @@ void process_instr(perfetto_backend_data *data, cpu_log_entry_handle entry)
      * same track/category: e.g. mode swtich, interrupt information and modified
      * registers?
      */
-    TRACE_EVENT_INSTANT("instructions", "stream", data->ctx_tracker_.get_ctx_track(),
+    TRACE_EVENT_INSTANT(
+        "instructions", "stream", data->ctx_tracker_.get_ctx_track(),
         [&](perfetto::EventContext ctx) {
             auto *qemu_arg = ctx.event()->set_qemu();
             auto *instr = qemu_arg->set_instr();
@@ -295,16 +293,8 @@ void process_instr(perfetto_backend_data *data, cpu_log_entry_handle entry)
             const char *bytes = perfetto_log_entry_insn_bytes(entry);
             int size = perfetto_log_entry_insn_size(entry);
             int nitems;
-            std::stringstream ss;
-
-            // XXX-AM: We can not use a bytes field in the protobuf because
-            // perfetto lacks support. This is slightly sad as this is an
-            // high-frequency event.
-            for (int i = 0; i < size; i++) {
-                ss << std::hex << std::setw(2) << std::setfill('0')
-                   << (static_cast<unsigned int>(bytes[i]) & 0xff) << " ";
-            }
-            instr->set_opcode(ss.str());
+
+            instr->set_opcode((const uint8_t *)bytes, size);
             instr->set_pc(perfetto_log_entry_pc(entry));
 
             nitems = perfetto_log_entry_regs(entry);
@@ -315,10 +305,10 @@ void process_instr(perfetto_backend_data *data, cpu_log_entry_handle entry)
                 if ((flags & LRI_CAP_REG) && (flags & LRI_HOLDS_CAP)) {
                     cap_register_handle cap_handle =
                         perfetto_reg_info_cap(entry, i);
-                    auto *capinfo = reginfo->set_cap_val();
+                    auto *capinfo = reginfo->set_cap_value();
                     trace_cap_register(capinfo, cap_handle);
                 } else {
-                    reginfo->set_int_val(perfetto_reg_info_gpr(entry, i));
+                    reginfo->set_int_value(perfetto_reg_info_gpr(entry, i));
                 }
             }
             nitems = perfetto_log_entry_mem(entry);
@@ -328,45 +318,51 @@ void process_instr(perfetto_backend_data *data, cpu_log_entry_handle entry)
                 meminfo->set_addr(perfetto_mem_info_addr(entry, i));
                 switch (flags) {
                 case LMI_LD:
-                    meminfo->set_op(perfetto::protos::pbzero::MemInfo::LOAD);
+                    meminfo->set_op(
+                        perfetto::protos::pbzero::QEMULogEntryMem::LOAD);
                     break;
                 case LMI_LD | LMI_CAP:
-                    meminfo->set_op(perfetto::protos::pbzero::MemInfo::CLOAD);
+                    meminfo->set_op(
+                        perfetto::protos::pbzero::QEMULogEntryMem::CLOAD);
                     break;
                 case LMI_ST:
-                    meminfo->set_op(perfetto::protos::pbzero::MemInfo::STORE);
+                    meminfo->set_op(
+                        perfetto::protos::pbzero::QEMULogEntryMem::STORE);
                     break;
                 case LMI_ST | LMI_CAP:
-                    meminfo->set_op(perfetto::protos::pbzero::MemInfo::CSTORE);
+                    meminfo->set_op(
+                        perfetto::protos::pbzero::QEMULogEntryMem::CSTORE);
                     break;
                 default:
                     // XXX Notify error somehow?
                     break;
                 }
                 if (flags & LMI_CAP) {
-                    auto *capinfo = meminfo->set_cap_val();
+                    auto *capinfo = meminfo->set_cap_value();
                     cap_register_handle cap_handle =
                         perfetto_reg_info_cap(entry, i);
                     trace_cap_register(capinfo, cap_handle);
                 } else {
-                    meminfo->set_int_val(perfetto_mem_info_value(entry, i));
+                    meminfo->set_int_value(perfetto_mem_info_value(entry, i));
                 }
             }
 
             if (flags & LI_FLAG_INTR_MASK) {
                 // interrupt
-                auto *trap = instr->set_trap();
+                auto *exc = instr->set_exception();
                 if (flags & LI_FLAG_INTR_TRAP)
-                    trap->set_type(perfetto::protos::pbzero::Trap::EXCEPTION);
+                    exc->set_type(
+                        perfetto::protos::pbzero::QEMULogEntryExcType::TRAP);
                 else {
-                    trap->set_type(perfetto::protos::pbzero::Trap::INTERRUPT);
+                    exc->set_type(
+                        perfetto::protos::pbzero::QEMULogEntryExcType::INTR);
                 }
-                trap->set_trap_number(perfetto_log_entry_intr_code(entry));
+                exc->set_code(perfetto_log_entry_intr_code(entry));
             }
             if (flags & LI_FLAG_MODE_SWITCH) {
                 auto mode = cheri::qemu_cpu_mode_to_trace(
                     perfetto_log_entry_next_cpu_mode(entry));
-                instr->set_mode(mode);
+                instr->set_mode_code(mode);
             }
         });
 }
diff --git a/trace_extra/tuple_index.hh b/trace_extra/tuple_index.hh
index f74530fd4ab..07e5b40bfb2 100644
--- a/trace_extra/tuple_index.hh
+++ b/trace_extra/tuple_index.hh
@@ -35,17 +35,21 @@
 
 namespace cheri
 {
-template<typename T> struct is_tuple_impl: std::false_type {};
-template<typename ...T> struct is_tuple_impl<std::tuple<T...>>: std::true_type {};
-template<typename T> struct is_tuple : is_tuple_impl<std::decay_t<T>> {};
+template <typename T> struct is_tuple_impl : std::false_type {
+};
+template <typename... T>
+struct is_tuple_impl<std::tuple<T...>> : std::true_type {
+};
+template <typename T> struct is_tuple : is_tuple_impl<std::decay_t<T>> {
+};
 
-template<typename T, typename Enable=void> struct tuple_hasher;
+template <typename T, typename Enable = void> struct tuple_hasher;
 
-template<typename T>
-struct tuple_hasher<T, typename std::enable_if<is_tuple<T>::value>::type>
-{
-    std::size_t operator()(const T& tuple) const {
+template <typename T>
+struct tuple_hasher<T, typename std::enable_if<is_tuple<T>::value>::type> {
+    std::size_t operator()(const T &tuple) const
+    {
         return boost::hash_value(tuple);
     }
 };
-}
+} // namespace cheri

From 81e840dcae2c67cef35ab866e98d599dda8cf91f Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Tue, 1 Feb 2022 21:53:28 +0000
Subject: [PATCH 43/74] Use perfetto protobuf definition for the protobuf
 backend.

---
 accel/tcg/log_instr_protobuf.c                | 140 ++++++++++++------
 trace_extra/meson.build                       |  19 ++-
 trace_extra/proto/meson.build                 |  11 --
 .../proto/protobuf_backend_entry.proto        |  95 ------------
 4 files changed, 111 insertions(+), 154 deletions(-)
 delete mode 100644 trace_extra/proto/meson.build
 delete mode 100644 trace_extra/proto/protobuf_backend_entry.proto

diff --git a/accel/tcg/log_instr_protobuf.c b/accel/tcg/log_instr_protobuf.c
index dd9490014f7..0c3ac493671 100644
--- a/accel/tcg/log_instr_protobuf.c
+++ b/accel/tcg/log_instr_protobuf.c
@@ -44,16 +44,56 @@
 #include "exec/memop.h"
 #include "disas/disas.h"
 
-#include "protobuf_backend_entry.pb-c.h"
+#include "qemu_log_entry.pb-c.h"
 
 static FILE *protobuf_logfile;
 
+/*
+ * Helpers to simplify access to namespaced protobuf values
+ * For consistency, redefine types to mirror the generated
+ * code from protobuf-c but strip the package prefix.
+ */
+#define GEN_TYPE_NS(x)     Perfetto__Protos__##x
+#define GEN_TYPE_PREFIX(x) GEN_TYPE_NS(QEMULogEntry##x)
+typedef GEN_TYPE_PREFIX(Capability) QEMULogEntryCapability;
+typedef GEN_TYPE_PREFIX(Mem) QEMULogEntryMem;
+typedef GEN_TYPE_PREFIX(Reg) QEMULogEntryReg;
+typedef GEN_TYPE_PREFIX(Evt) QEMULogEntryEvt;
+typedef GEN_TYPE_PREFIX(Exc) QEMULogEntryExc;
+typedef GEN_TYPE_PREFIX() QEMULogEntry;
+typedef GEN_TYPE_PREFIX(EvtTraceState) QEMULogEntryEvtTraceState;
+typedef GEN_TYPE_PREFIX(EvtCtxUpdate) QEMULogEntryEvtCtxUpdate;
+typedef GEN_TYPE_NS(QEMULogEventTraceState) QEMULogEventTraceState;
+
+#define GEN_FN_NS(x)                        perfetto__protos__##x
+#define GEN_FN_PREFIX(x)                    GEN_FN_NS(qemulog_entry_##x)
+#define qemulog_entry_capability__init      GEN_FN_PREFIX(capability__init)
+#define qemulog_entry_mem__init             GEN_FN_PREFIX(mem__init)
+#define qemulog_entry_reg__init             GEN_FN_PREFIX(reg__init)
+#define qemulog_entry_evt__init             GEN_FN_PREFIX(evt__init)
+#define qemulog_entry_evt_trace_state__init GEN_FN_PREFIX(evt_trace_state__init)
+#define qemulog_entry_evt_ctx_update__init  GEN_FN_PREFIX(evt_ctx_update__init)
+#define qemulog_entry_evt_regdump__init     GEN_FN_PREFIX(evt_regdump__init)
+#define qemulog_entry__get_packed_size      GEN_FN_PREFIX(_get_packed_size)
+#define qemulog_entry__pack                 GEN_FN_PREFIX(_pack)
+
+#define GEN_CONST_NS(x)         PERFETTO__PROTOS__##x
+#define GEN_CONST_PREFIX(x)     GEN_CONST_NS(QEMULOG_ENTRY_##x)
+#define ENUM_ENTRY_MEM_VALUE(v) GEN_CONST_PREFIX(MEM__VALUE_##v)
+#define ENUM_ENTRY_REG_VALUE(v) GEN_CONST_PREFIX(REG__VALUE_##v)
+#define ENUM_ENTRY_EVT_CASE(v)  GEN_CONST_PREFIX(EVT__EVENT_##v)
+#define ENUM_ENTRY_INSN_CASE(v) GEN_CONST_PREFIX(_INSN_##v)
+#define ENUM_ENTRY_MODE_CASE(v) GEN_CONST_PREFIX(_MODE_MODE_##v)
+#define ENUM_ENTRY_EXC_CASE(v)  GEN_CONST_PREFIX(EXC_TYPE__##v)
+#define QEMULOG_ENTRY__INIT     GEN_CONST_NS(QEMULOG_ENTRY__INIT)
+#define QEMULOG_ENTRY_EXC__INIT GEN_CONST_PREFIX(EXC__INIT)
+
 #ifdef TARGET_CHERI
-static QEMULogInstrCapability *emit_protobuf_capability(cap_register_t *cap)
+static QEMULogEntryCapability *emit_protobuf_capability(cap_register_t *cap)
 {
-    QEMULogInstrCapability *pb_cap = g_malloc(sizeof(QEMULogInstrCapability));
+    QEMULogEntryCapability *pb_cap = g_malloc(sizeof(QEMULogEntryCapability));
 
-    qemulog_instr_capability__init(pb_cap);
+    qemulog_entry_capability__init(pb_cap);
     pb_cap->valid = cap->cr_tag;
     pb_cap->sealed = cap_is_sealed_with_type(cap);
     pb_cap->perms = COMBINED_PERMS_VALUE(cap);
@@ -67,11 +107,11 @@ static QEMULogInstrCapability *emit_protobuf_capability(cap_register_t *cap)
 #endif
 
 static void emit_protobuf_meminfo(log_meminfo_t *minfo,
-                                  QEMULogInstrMem **pb_mem)
+                                  QEMULogEntryMem **pb_mem)
 {
-    QEMULogInstrMem *mem = g_malloc(sizeof(QEMULogInstrMem));
+    QEMULogEntryMem *mem = g_malloc(sizeof(QEMULogEntryMem));
 
-    qemulog_instr_mem__init(mem);
+    qemulog_entry_mem__init(mem);
     mem->is_load = (minfo->flags & LMI_LD) != 0;
     mem->size = memop_size(minfo->op);
     mem->addr = minfo->addr;
@@ -80,31 +120,31 @@ static void emit_protobuf_meminfo(log_meminfo_t *minfo,
            "Capability memory access without CHERI support");
 #else
     if (minfo->flags & LMI_CAP) {
-        mem->value_case = QEMULOG_INSTR_MEM__VALUE_CAP_VALUE;
+        mem->value_case = ENUM_ENTRY_MEM_VALUE(CAP_VALUE);
         mem->cap_value = emit_protobuf_capability(&minfo->cap);
     } else
 #endif
     {
-        mem->value_case = QEMULOG_INSTR_MEM__VALUE_INT_VALUE;
+        mem->value_case = ENUM_ENTRY_MEM_VALUE(INT_VALUE);
         mem->int_value = minfo->value;
     }
     *pb_mem = mem;
 }
 
-static void release_protobuf_meminfo(QEMULogInstrMem *pb_mem)
+static void release_protobuf_meminfo(QEMULogEntryMem *pb_mem)
 {
-    if (pb_mem->value_case == QEMULOG_INSTR_MEM__VALUE_CAP_VALUE) {
+    if (pb_mem->value_case == ENUM_ENTRY_MEM_VALUE(CAP_VALUE)) {
         g_free(pb_mem->cap_value);
     }
     g_free(pb_mem);
 }
 
 static void emit_protobuf_reginfo(log_reginfo_t *rinfo,
-                                  QEMULogInstrReg **pb_reg)
+                                  QEMULogEntryReg **pb_reg)
 {
-    QEMULogInstrReg *reg = g_malloc(sizeof(QEMULogInstrReg));
+    QEMULogEntryReg *reg = g_malloc(sizeof(QEMULogEntryReg));
 
-    qemulog_instr_reg__init(reg);
+    qemulog_entry_reg__init(reg);
     /* Safe to de-const cast as the pointer is only used during packing */
     reg->name = (char *)rinfo->name;
 
@@ -114,46 +154,47 @@ static void emit_protobuf_reginfo(log_reginfo_t *rinfo,
 #else
     if (reginfo_is_cap(rinfo)) {
         if (reginfo_has_cap(rinfo)) {
-            reg->value_case = QEMULOG_INSTR_REG__VALUE_CAP_VALUE;
+            reg->value_case = ENUM_ENTRY_REG_VALUE(CAP_VALUE);
             reg->cap_value = emit_protobuf_capability(&rinfo->cap);
         } else {
-            reg->value_case = QEMULOG_INSTR_REG__VALUE_INT_VALUE;
+            reg->value_case = ENUM_ENTRY_REG_VALUE(INT_VALUE);
             reg->int_value = rinfo->gpr;
         }
     } else
 #endif
     {
-        reg->value_case = QEMULOG_INSTR_REG__VALUE_INT_VALUE;
+        reg->value_case = ENUM_ENTRY_REG_VALUE(INT_VALUE);
         reg->int_value = rinfo->gpr;
     }
     *pb_reg = reg;
 }
 
-static void release_protobuf_reginfo(QEMULogInstrReg *pb_reg)
+static void release_protobuf_reginfo(QEMULogEntryReg *pb_reg)
 {
-    if (pb_reg->value_case == QEMULOG_INSTR_REG__VALUE_CAP_VALUE) {
+    if (pb_reg->value_case == ENUM_ENTRY_REG_VALUE(CAP_VALUE)) {
         g_free(pb_reg->cap_value);
     }
     g_free(pb_reg);
 }
 
-static void emit_protobuf_event(log_event_t *evtinfo, QEMULogInstrEvt **pb_evt)
+static void emit_protobuf_event(log_event_t *evtinfo, QEMULogEntryEvt **pb_evt)
 {
-    QEMULogInstrEvt *evt = g_malloc(sizeof(QEMULogInstrEvt));
+    QEMULogEntryEvt *evt = g_malloc(sizeof(QEMULogEntryEvt));
 
-    qemulog_instr_evt__init(evt);
+    qemulog_entry_evt__init(evt);
     switch (evtinfo->id) {
     case LOG_EVENT_STATE:
-        evt->event_case = QEMULOG_INSTR_EVT__EVENT_STATE;
-        evt->state = g_malloc(sizeof(QEMULogInstrEvtTraceState));
-        qemulog_instr_evt_trace_state__init(evt->state);
-        evt->state->next_state = (LogEventTraceState)evtinfo->state.next_state;
+        evt->event_case = ENUM_ENTRY_EVT_CASE(STATE);
+        evt->state = g_malloc(sizeof(QEMULogEntryEvtTraceState));
+        qemulog_entry_evt_trace_state__init(evt->state);
+        evt->state->next_state =
+            (QEMULogEventTraceState)evtinfo->state.next_state;
         evt->state->pc = evtinfo->state.pc;
         break;
     case LOG_EVENT_CTX_UPDATE:
-        evt->event_case = QEMULOG_INSTR_EVT__EVENT_CTX_UPDATE;
-        evt->ctx_update = g_malloc(sizeof(QEMULogInstrEvtCtxUpdate));
-        qemulog_instr_evt_ctx_update__init(evt->ctx_update);
+        evt->event_case = ENUM_ENTRY_EVT_CASE(CTX_UPDATE);
+        evt->ctx_update = g_malloc(sizeof(QEMULogEntryEvtCtxUpdate));
+        qemulog_entry_evt_ctx_update__init(evt->ctx_update);
         evt->ctx_update->pid = evtinfo->ctx_update.pid;
         evt->ctx_update->tid = evtinfo->ctx_update.tid;
         evt->ctx_update->cid = evtinfo->ctx_update.cid;
@@ -162,7 +203,7 @@ static void emit_protobuf_event(log_event_t *evtinfo, QEMULogInstrEvt **pb_evt)
             (char *)cpu_get_mode_name(evtinfo->ctx_update.mode);
         break;
     case LOG_EVENT_MARKER:
-        evt->event_case = QEMULOG_INSTR_EVT__EVENT_MARKER;
+        evt->event_case = ENUM_ENTRY_EVT_CASE(MARKER);
         evt->marker = evtinfo->marker;
         break;
     default:
@@ -171,14 +212,17 @@ static void emit_protobuf_event(log_event_t *evtinfo, QEMULogInstrEvt **pb_evt)
     *pb_evt = evt;
 }
 
-static void release_protobuf_event(QEMULogInstrEvt *pb_evt)
+static void release_protobuf_event(QEMULogEntryEvt *pb_evt)
 {
-    if (pb_evt->event_case == QEMULOG_INSTR_EVT__EVENT_STATE) {
+    if (pb_evt->event_case == ENUM_ENTRY_EVT_CASE(STATE)) {
         g_free(pb_evt->state);
     }
-    if (pb_evt->event_case == QEMULOG_INSTR_EVT__EVENT_CTX_UPDATE) {
+    if (pb_evt->event_case == ENUM_ENTRY_EVT_CASE(CTX_UPDATE)) {
         g_free(pb_evt->ctx_update);
     }
+    /* if (pb_evt->event_case == ENUM_ENTRY_EVT_CASE(REGDUMP)) { */
+    /*     g_free(pb_evt->regdump); */
+    /* } */
     g_free(pb_evt);
 }
 
@@ -207,11 +251,11 @@ void sync_protobuf_backend(CPUArchState *env)
 
 void emit_protobuf_entry(CPUArchState *env, cpu_log_entry_t *entry)
 {
-    QEMULogInstrEntry pb_entry = QEMULOG_INSTR_ENTRY__INIT;
-    QEMULogInstrExc pb_exc = QEMULOG_INSTR_EXC__INIT;
-    QEMULogInstrMem **pb_mem;
-    QEMULogInstrReg **pb_reg;
-    QEMULogInstrEvt **pb_evt;
+    QEMULogEntry pb_entry = QEMULOG_ENTRY__INIT;
+    QEMULogEntryExc pb_exc = QEMULOG_ENTRY_EXC__INIT;
+    QEMULogEntryMem **pb_mem;
+    QEMULogEntryReg **pb_reg;
+    QEMULogEntryEvt **pb_evt;
     uint8_t *buf;
     size_t len;
     uint32_t preamble = 0;
@@ -220,7 +264,7 @@ void emit_protobuf_entry(CPUArchState *env, cpu_log_entry_t *entry)
     pb_entry.pc = entry->pc;
 
     if (entry->flags & LI_FLAG_HAS_INSTR_DATA) {
-        pb_entry.insn_case = QEMULOG_INSTR_ENTRY__INSN_DISAS;
+        pb_entry.insn_case = ENUM_ENTRY_INSN_CASE(DISAS);
         pb_entry.disas = disas_one_strbuf(env_cpu(env), entry->insn_bytes,
                                           sizeof(entry->insn_bytes), entry->pc);
         pb_entry.cpu = env_cpu(env)->cpu_index;
@@ -229,23 +273,25 @@ void emit_protobuf_entry(CPUArchState *env, cpu_log_entry_t *entry)
             /*
              * Safe to de-const cast as the pointer is only used during packing
              */
-            pb_entry.mode = (char *)cpu_get_mode_name(entry->next_cpu_mode);
+            pb_entry.mode_case = ENUM_ENTRY_MODE_CASE(NAME);
+            pb_entry.mode_name =
+                (char *)cpu_get_mode_name(entry->next_cpu_mode);
         }
         if (entry->flags & LI_FLAG_INTR_MASK) {
             pb_entry.exception = &pb_exc;
             pb_exc.vector = entry->intr_vector;
             pb_exc.code = entry->intr_code;
             if (entry->flags & LI_FLAG_INTR_TRAP) {
-                pb_exc.type = QEMULOG_INSTR_EXC_TYPE__TRAP;
+                pb_exc.type = ENUM_ENTRY_EXC_CASE(TRAP);
                 pb_exc.faultaddr = entry->intr_faultaddr;
             } else if (entry->flags & LI_FLAG_INTR_ASYNC) {
-                pb_exc.type = QEMULOG_INSTR_EXC_TYPE__INTR;
+                pb_exc.type = ENUM_ENTRY_EXC_CASE(INTR);
             } else {
                 assert(0 && "invalid exception flags");
             }
         }
         if (entry->mem->len > 0) {
-            pb_mem = g_malloc(entry->mem->len * sizeof(QEMULogInstrMem *));
+            pb_mem = g_malloc(entry->mem->len * sizeof(QEMULogEntryMem *));
             pb_entry.n_mem = entry->mem->len;
             pb_entry.mem = pb_mem;
             for (i = 0; i < entry->mem->len; i++, pb_mem++) {
@@ -255,7 +301,7 @@ void emit_protobuf_entry(CPUArchState *env, cpu_log_entry_t *entry)
             }
         }
         if (entry->regs->len > 0) {
-            pb_reg = g_malloc(entry->regs->len * sizeof(QEMULogInstrReg *));
+            pb_reg = g_malloc(entry->regs->len * sizeof(QEMULogEntryReg *));
             pb_entry.n_reg = entry->regs->len;
             pb_entry.reg = pb_reg;
             for (i = 0; i < entry->regs->len; i++, pb_reg++) {
@@ -268,7 +314,7 @@ void emit_protobuf_entry(CPUArchState *env, cpu_log_entry_t *entry)
 
     if (entry->events->len > 0) {
         pb_evt = NULL;
-        pb_evt = g_malloc(entry->events->len * sizeof(QEMULogInstrEvt *));
+        pb_evt = g_malloc(entry->events->len * sizeof(QEMULogEntryEvt *));
         pb_entry.n_evt = entry->events->len;
         pb_entry.evt = pb_evt;
         for (i = 0; i < entry->events->len; i++, pb_evt++) {
@@ -277,13 +323,13 @@ void emit_protobuf_entry(CPUArchState *env, cpu_log_entry_t *entry)
         }
     }
 
-    len = qemulog_instr_entry__get_packed_size(&pb_entry);
+    len = qemulog_entry__get_packed_size(&pb_entry);
     preamble = len;
     assert(preamble == len && "Message preamble overflow");
 
     buf = g_malloc(len + sizeof(preamble));
     *((uint32_t *)buf) = preamble;
-    qemulog_instr_entry__pack(&pb_entry, buf + sizeof(preamble));
+    qemulog_entry__pack(&pb_entry, buf + sizeof(preamble));
 
     /*
      * Technically we should not use the qemu logfile as it is not open in
diff --git a/trace_extra/meson.build b/trace_extra/meson.build
index c7802420a42..e63051a7496 100644
--- a/trace_extra/meson.build
+++ b/trace_extra/meson.build
@@ -1,3 +1,4 @@
+fs = import('fs')
 
 qemutrace_ss = ss.source_set()
 
@@ -18,4 +19,20 @@ libqemutrace = static_library('qemutrace',
 
 qemuutil_libs += libqemutrace
 
-subdir('proto')
+if config_all.has_key('CONFIG_TCG_LOG_INSTR') and config_all.has_key('CONFIG_TRACE_PROTOBUF')
+   proc_protoc = find_program('protoc')
+   proto_path = meson.current_source_dir() / 'cheri-perfetto/protos/perfetto/trace/track_event'
+   protobuf_backend_files = files(proto_path / 'qemu_log_entry.proto')
+   output_files = []
+   foreach f : protobuf_backend_files
+      fpath = '@0@'.format(f)
+      output_files += fs.replace_suffix(fs.name(fpath), '.pb-c.c')
+      output_files += fs.replace_suffix(fs.name(fpath), '.pb-c.h')
+   endforeach
+   gen_protos = custom_target('protobuf_backend_protos',
+                              input: protobuf_backend_files,
+                              output: output_files,
+                              command: [proc_protoc, '--proto_path=@0@'.format(proto_path), '--c_out=@OUTDIR@', '@INPUT@'],
+                              depend_files: protobuf_backend_files)
+   specific_ss.add(gen_protos)
+endif
diff --git a/trace_extra/proto/meson.build b/trace_extra/proto/meson.build
deleted file mode 100644
index c5e5834820a..00000000000
--- a/trace_extra/proto/meson.build
+++ /dev/null
@@ -1,11 +0,0 @@
-
-if config_all.has_key('CONFIG_TCG_LOG_INSTR') and config_all.has_key('CONFIG_TRACE_PROTOBUF')
-   proc_protoc = find_program('protoc')
-   protobuf_backend_files = files('protobuf_backend_entry.proto')
-   gen_protos = custom_target('protobuf_backend_protos',
-                              input: protobuf_backend_files,
-                              output: ['protobuf_backend_entry.pb-c.c', 'protobuf_backend_entry.pb-c.h'],
-                              command: [proc_protoc, '--proto_path=@CURRENT_SOURCE_DIR@', '--c_out=@OUTDIR@', '@INPUT@'],
-                              depend_files: protobuf_backend_files)
-   specific_ss.add(gen_protos)
-endif
diff --git a/trace_extra/proto/protobuf_backend_entry.proto b/trace_extra/proto/protobuf_backend_entry.proto
deleted file mode 100644
index d90a53f3289..00000000000
--- a/trace_extra/proto/protobuf_backend_entry.proto
+++ /dev/null
@@ -1,95 +0,0 @@
-
-syntax = "proto3";
-
-// package qemu.protos;
-
-enum QEMULogInstrExcType {
-  TRAP = 0;
-  INTR = 1;
-}
-
-/* Values must be kept in sync with log_event_trace_state_t */
-enum LogEventTraceState {
-  START = 0;
-  STOP = 1;
-  FLUSH = 2;
-}
-
-/* Values must be kept in sync with log_event_ctx_update_op_t */
-enum LogEventCtxUpdateOp {
-  LOG_EVENT_CTX_OP_SWITCH = 0;
-  LOG_EVENT_CTX_OP_SETUP = 1;
-}
-
-message QEMULogInstrCapability {
-  bool valid = 1;
-  bool sealed = 2;
-  uint32 perms = 3;
-  uint32 otype = 4;
-  uint64 cap_base = 5;
-  uint64 cap_cursor = 6;
-  uint64 cap_length = 7;
-}
-
-message QEMULogInstrMem {
-  bool is_load = 1;
-  int32 size = 2;
-  fixed64 addr = 3;
-  oneof value {
-    QEMULogInstrCapability cap_value = 4;
-    double fp_value = 5;
-    fixed64 int_value = 6;
-  }
-}
-
-message QEMULogInstrReg {
-  string name = 1;
-  oneof value {
-    fixed64 int_value = 2;
-    double fp_value = 3;
-    QEMULogInstrCapability cap_value = 4;
-  }
-}
-
-message QEMULogInstrEvtTraceState {
-  LogEventTraceState next_state = 1;
-  fixed64 pc = 2;
-}
-
-message QEMULogInstrEvtCtxUpdate {
-  LogEventCtxUpdateOp op = 1;
-  uint64 pid = 2;
-  uint64 tid = 3;
-  uint64 cid = 4;
-  string mode = 5;
-}
-
-message QEMULogInstrEvt {
-  oneof event {
-    QEMULogInstrEvtTraceState state = 1;
-    QEMULogInstrEvtCtxUpdate ctx_update = 2;
-    uint64 marker = 3;
-  }
-}
-
-message QEMULogInstrExc {
-  QEMULogInstrExcType type = 1;
-  fixed64 vector = 2;
-  int32 code = 3;
-  fixed64 faultaddr = 4;
-}
-
-message QEMULogInstrEntry {
-  fixed64 pc = 1;
-  oneof insn {
-    string disas = 2;
-    bytes opcode = 3;
-  }
-  int32 cpu = 4;
-  int32 asid = 5;
-  string mode = 6;
-  QEMULogInstrExc exception = 7;
-  repeated QEMULogInstrMem mem = 8;
-  repeated QEMULogInstrReg reg = 9;
-  repeated QEMULogInstrEvt evt = 10;
-}

From c0124037a01c10ff9a61a43aedb60b436300be70 Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Tue, 1 Feb 2022 21:58:21 +0000
Subject: [PATCH 44/74] Implement log event to dump register contents to trace.

Introduce target-dependent hooks to produce register dumps in the
instruction logging subsystem.
Add interface functions for target to emit registers in the event,
this keeps log_reginfo_t private.
---
 accel/tcg/log_instr.c              | 82 +++++++++++++++++++++++++++
 accel/tcg/log_instr_json.c         | 46 +++++++++++++--
 accel/tcg/log_instr_protobuf.c     |  2 +
 accel/tcg/log_instr_text.c         |  9 ++-
 include/exec/log_instr.h           | 33 +++++++++--
 include/qemu/log_instr.h           | 27 ++++++++-
 target/arm/helper.c                | 10 +++-
 target/i386/helper.c               |  7 +++
 target/mips/op_helper_log_instr.c  |  5 ++
 target/riscv/helper.h              |  1 +
 target/riscv/op_helper_log_instr.c | 90 ++++++++++++++++++++++++------
 trace_extra/trace_perfetto.cc      |  5 +-
 12 files changed, 283 insertions(+), 34 deletions(-)

diff --git a/accel/tcg/log_instr.c b/accel/tcg/log_instr.c
index 811af245e4d..519e4ed399b 100644
--- a/accel/tcg/log_instr.c
+++ b/accel/tcg/log_instr.c
@@ -177,6 +177,17 @@ void qemu_log_instr_enable_trace_debug()
     trace_debug = true;
 }
 
+static void emit_regdump_event(CPUArchState *env, cpu_log_entry_t *entry)
+{
+    log_event_t event;
+
+    event.id = LOG_EVENT_REGDUMP;
+    if (cpu_log_instr_event_regdump(env, &event)) {
+        return;
+    }
+    g_array_append_val(entry->events, event);
+}
+
 static inline void emit_start_event(cpu_log_entry_t *entry, target_ulong pc)
 {
     log_event_t event;
@@ -212,11 +223,26 @@ static inline void emit_stop_event(cpu_log_entry_t *entry, target_ulong pc)
 static void reset_log_buffer(cpu_log_instr_state_t *cpulog,
                              cpu_log_entry_t *entry)
 {
+    log_event_t *evt;
+    int i;
+
     memset(&entry->cpu_log_entry_startzero, 0,
            ((char *)&entry->cpu_log_entry_endzero -
             (char *)&entry->cpu_log_entry_startzero));
     g_array_remove_range(entry->regs, 0, entry->regs->len);
     g_array_remove_range(entry->mem, 0, entry->mem->len);
+    /*
+     * Need to free any dynamic allocation in the event structures to
+     * avoid leaking memory. This is called before the log entry is
+     * reused, so the memory might be reclaimed much later than the allocation
+     * time.
+     */
+    for (i = 0; i < entry->events->len; i++) {
+        evt = &g_array_index(entry->events, log_event_t, i);
+        if (evt->id == LOG_EVENT_REGDUMP) {
+            g_array_free(evt->reg_dump.gpr, true);
+        }
+    }
     g_array_remove_range(entry->events, 0, entry->events->len);
     g_string_erase(entry->txt_buffer, 0, -1);
     cpulog->force_drop = false;
@@ -334,6 +360,7 @@ static void do_cpu_loglevel_switch(CPUState *cpu, run_on_cpu_data data)
          * traced
          */
         emit_start_event(entry, pc);
+        emit_regdump_event(env, entry);
         QEMU_LOG_INSTR_INC_STAT(cpulog, trace_start);
     }
 
@@ -438,10 +465,19 @@ static void qemu_log_entry_init(cpu_log_entry_t *entry)
 static void qemu_log_entry_destroy(gpointer data)
 {
     cpu_log_entry_t *entry = data;
+    log_event_t *evt;
+    int i;
 
     g_string_free(entry->txt_buffer, true);
     g_array_free(entry->regs, true);
     g_array_free(entry->mem, true);
+    for (i = 0; i < entry->events->len; i++) {
+        evt = &g_array_index(entry->events, log_event_t, i);
+        if (evt->id == LOG_EVENT_REGDUMP) {
+            /* Need to free the register dump array */
+            g_array_free(evt->reg_dump.gpr, true);
+        }
+    }
     g_array_free(entry->events, true);
 }
 
@@ -798,9 +834,55 @@ void qemu_log_instr_event(CPUArchState *env, log_event_t *evt)
 {
     cpu_log_entry_t *entry = get_cpu_log_entry(env);
 
+    /* Note: transfer ownership of dynamically allocated data in evt */
     g_array_append_val(entry->events, *evt);
 }
 
+void qemu_log_instr_event_create_regdump(log_event_t *evt, int nregs)
+{
+    evt->reg_dump.gpr = g_array_sized_new(false, false, sizeof(log_reginfo_t),
+                                          nregs * sizeof(log_reginfo_t));
+}
+
+void qemu_log_instr_event_dump_reg(log_event_t *evt, const char *reg_name,
+                                   target_ulong value)
+{
+    log_reginfo_t r;
+
+    r.flags = 0;
+    r.name = reg_name;
+    r.gpr = value;
+    /*
+     * Assume that the reg_dump array has been initialized,
+     * should put an assertion in here.
+     */
+    g_array_append_val(evt->reg_dump.gpr, r);
+}
+
+#ifdef TARGET_CHERI
+void qemu_log_instr_event_dump_cap(log_event_t *evt, const char *reg_name,
+                                   const cap_register_t *value)
+{
+    log_reginfo_t r;
+
+    r.flags = LRI_CAP_REG | LRI_HOLDS_CAP;
+    r.name = reg_name;
+    r.cap = *value;
+    g_array_append_val(evt->reg_dump.gpr, r);
+}
+
+void qemu_log_instr_event_dump_cap_int(log_event_t *evt, const char *reg_name,
+                                       target_ulong value)
+{
+    log_reginfo_t r;
+
+    r.flags = LRI_CAP_REG;
+    r.name = reg_name;
+    r.gpr = value;
+    g_array_append_val(evt->reg_dump.gpr, r);
+}
+#endif
+
 void qemu_log_instr_extra(CPUArchState *env, const char *msg, ...)
 {
     cpu_log_entry_t *entry = get_cpu_log_entry(env);
diff --git a/accel/tcg/log_instr_json.c b/accel/tcg/log_instr_json.c
index 3587c3063a0..7fc0774b147 100644
--- a/accel/tcg/log_instr_json.c
+++ b/accel/tcg/log_instr_json.c
@@ -45,7 +45,7 @@
 #include "exec/memop.h"
 #include "disas/disas.h"
 
-static inline cJSON *emit_json_hex(uint64_t value)
+static cJSON *emit_json_hex(uint64_t value)
 {
     char buf[sizeof(value) * 2 + 1];
 
@@ -54,7 +54,7 @@ static inline cJSON *emit_json_hex(uint64_t value)
 }
 
 #ifdef TARGET_CHERI
-static inline cJSON *emit_json_capability(cap_register_t *cap)
+static cJSON *emit_json_capability(cap_register_t *cap)
 {
     cJSON *js_cap = cJSON_CreateObject();
     cJSON *valid = cJSON_CreateBool(cap->cr_tag);
@@ -78,7 +78,7 @@ static inline cJSON *emit_json_capability(cap_register_t *cap)
 }
 #endif
 
-static inline void emit_json_ldst(log_meminfo_t *minfo, cJSON *list)
+static void emit_json_ldst(log_meminfo_t *minfo, cJSON *list)
 {
     cJSON *mem = cJSON_CreateObject();
     cJSON *is_load = cJSON_CreateBool((minfo->flags & LMI_LD) != 0);
@@ -105,7 +105,7 @@ static inline void emit_json_ldst(log_meminfo_t *minfo, cJSON *list)
     cJSON_AddItemToArray(list, mem);
 }
 
-static inline void emit_json_reg(log_reginfo_t *rinfo, cJSON *list)
+static void emit_json_reg(log_reginfo_t *rinfo, cJSON *list)
 {
     cJSON *reg = cJSON_CreateObject();
     cJSON *name = cJSON_CreateString(rinfo->name);
@@ -133,6 +133,36 @@ static inline void emit_json_reg(log_reginfo_t *rinfo, cJSON *list)
     cJSON_AddItemToArray(list, reg);
 }
 
+static void emit_json_evt_state(const log_event_trace_state_update_t *evt,
+                                cJSON *list)
+{
+    cJSON *update = cJSON_CreateObject();
+    cJSON *evt_name = cJSON_CreateString("state");
+    cJSON *next_state = cJSON_CreateNumber(evt->next_state);
+    cJSON *pc = cJSON_CreateNumber(evt->pc);
+
+    cJSON_AddItemToObject(update, "id", evt_name);
+    cJSON_AddItemToObject(update, "next", next_state);
+    cJSON_AddItemToObject(update, "pc", pc);
+    cJSON_AddItemToArray(list, update);
+}
+
+static void emit_json_evt_regdump(const log_event_regdump_t *evt, cJSON *list)
+{
+    cJSON *regdump = cJSON_CreateObject();
+    cJSON *evt_name = cJSON_CreateString("rdump");
+    cJSON *gpr = cJSON_CreateArray();
+    int i;
+
+    for (i = 0; i < evt->gpr->len; i++) {
+        log_reginfo_t *rinfo = &g_array_index(evt->gpr, log_reginfo_t, i);
+        emit_json_reg(rinfo, gpr);
+    }
+    cJSON_AddItemToObject(regdump, "id", evt_name);
+    cJSON_AddItemToObject(regdump, "gpr", gpr);
+    cJSON_AddItemToObject(list, "rdump", regdump);
+}
+
 void sync_json_backend(CPUArchState *env)
 {
     /*
@@ -229,19 +259,27 @@ void emit_json_entry(CPUArchState *env, cpu_log_entry_t *entry)
     }
 
     if (entry->events->len > 0) {
+        cJSON *js_evt = cJSON_CreateArray();
         for (i = 0; i < entry->events->len; i++) {
             event = &g_array_index(entry->events, const log_event_t, i);
             switch (event->id) {
             case LOG_EVENT_STATE:
+                emit_json_evt_state(&event->state, js_evt);
                 break;
             case LOG_EVENT_CTX_UPDATE:
+                /* TODO Unimpl */
                 break;
             case LOG_EVENT_MARKER:
+                /* TODO Unimpl */
+                break;
+            case LOG_EVENT_REGDUMP:
+                emit_json_evt_regdump(&event->reg_dump, js_evt);
                 break;
             default:
                 assert(0 && "unknown event ID");
             }
         }
+        cJSON_AddItemToObject(js_entry, "evt", js_evt);
     }
     qemu_log("%s,", cJSON_PrintUnformatted(js_entry));
     cJSON_Delete(js_entry);
diff --git a/accel/tcg/log_instr_protobuf.c b/accel/tcg/log_instr_protobuf.c
index 0c3ac493671..10bb0c2588c 100644
--- a/accel/tcg/log_instr_protobuf.c
+++ b/accel/tcg/log_instr_protobuf.c
@@ -206,6 +206,8 @@ static void emit_protobuf_event(log_event_t *evtinfo, QEMULogEntryEvt **pb_evt)
         evt->event_case = ENUM_ENTRY_EVT_CASE(MARKER);
         evt->marker = evtinfo->marker;
         break;
+    case LOG_EVENT_REGDUMP:
+        break;
     default:
         assert(0 && "unknown event ID");
     }
diff --git a/accel/tcg/log_instr_text.c b/accel/tcg/log_instr_text.c
index 39cd2e55905..522327dd947 100644
--- a/accel/tcg/log_instr_text.c
+++ b/accel/tcg/log_instr_text.c
@@ -120,7 +120,7 @@ void emit_text_instr(CPUArchState *env, cpu_log_entry_t *entry)
     QemuLogFile *logfile;
     const log_event_t *event;
     const char *log_state_op;
-    int i;
+    int i, j;
 
     if (entry->flags & LI_FLAG_HAS_INSTR_DATA) {
         /* Dump CPU-ID:ASID + address */
@@ -220,6 +220,13 @@ void emit_text_instr(CPUArchState *env, cpu_log_entry_t *entry)
         case LOG_EVENT_MARKER:
             qemu_log("Guest trace marker %lx\n", event->marker);
             break;
+        case LOG_EVENT_REGDUMP:
+            qemu_log("Register dump\n");
+            for (j = 0; j < event->reg_dump.gpr->len; j++) {
+                log_reginfo_t *r =
+                    &g_array_index(event->reg_dump.gpr, log_reginfo_t, j);
+                emit_text_reg(r);
+            }
         default:
             assert(0 && "unknown event ID");
         }
diff --git a/include/exec/log_instr.h b/include/exec/log_instr.h
index b67f13f3900..fbe59a812a4 100644
--- a/include/exec/log_instr.h
+++ b/include/exec/log_instr.h
@@ -48,6 +48,9 @@
  * - const char *cpu_get_mode_name(mode)
  *   return the mode name associated with a qemu_log_instr_cpu_mode_t for
  * printing.
+ * - bool cpu_log_instr_event_regdump(env, event)
+ *   Builds a register dump for the target, returns true if the register dump
+ *   was not produced and the event should be cancelled.
  *
  * - Each target should implement their own register update logging helpers that
  *   call into qemu_log_instr_gpr(), qemu_log_instr_cap() and similar interface
@@ -211,9 +214,6 @@ bool qemu_log_instr_check_enabled(CPUArchState *env);
 static inline bool qemu_log_instr_event_enabled(CPUArchState *env,
                                                 log_event_t *event)
 {
-    if (event->id == LOG_EVENT_MARKER) {
-        return true;
-    }
     if (event->id == LOG_EVENT_CTX_UPDATE &&
         event->ctx_update.op == LOG_EVENT_CTX_OP_SETUP) {
         return true;
@@ -336,12 +336,32 @@ void qemu_log_instr_interrupt(CPUArchState *env, uint32_t code,
                               target_ulong vector);
 
 /*
- * Log magic NOP event, we record a function number and 4 arguments.
- * Note that we have 6 bytes left in the cvtrace format, we may need
- * some trickery to reclaim those.
+ * Log extra events.
+ * XXX we can avoid a copy if we make this return a newly allocated
+ * log_event_t instead.
  */
 void qemu_log_instr_event(CPUArchState *env, log_event_t *evt);
 
+/*
+ * Each target must define this function to implement
+ * register dump events.
+ */
+bool cpu_log_instr_event_regdump(CPUArchState *env, log_event_t *evt);
+
+/*
+ * Interface to fill register dump log_event_t entries.
+ * This mirrors the qemu_log_instr_reg/cap/cap_int functions.
+ */
+void qemu_log_instr_event_create_regdump(log_event_t *evt, int nregs);
+void qemu_log_instr_event_dump_reg(log_event_t *evt, const char *reg_name,
+                                   target_ulong value);
+#ifdef TARGET_CHERI
+void qemu_log_instr_event_dump_cap(log_event_t *evt, const char *reg_name,
+                                   const cap_register_t *value);
+void qemu_log_instr_event_dump_cap_int(log_event_t *evt, const char *reg_name,
+                                       target_ulong value);
+#endif
+
 /*
  * Log extra information as a string. Some logging formats may
  * ignore this.
@@ -356,6 +376,7 @@ void qemu_log_instr_extra(CPUArchState *env, const char *msg, ...);
 #define qemu_log_instr_flush(env)
 #define qemu_log_instr_reg(...)
 #define qemu_log_instr_cap(...)
+#define qemu_log_instr_cap_int(...)
 #define qemu_log_instr_mem(...)
 #define qemu_log_instr_instr(...)
 #define qemu_log_instr_hwtid(...)
diff --git a/include/qemu/log_instr.h b/include/qemu/log_instr.h
index 1f45378c643..8c180d97720 100644
--- a/include/qemu/log_instr.h
+++ b/include/qemu/log_instr.h
@@ -31,6 +31,7 @@
  */
 
 #pragma once
+#include <glib.h>
 
 /*
  * CPU-independant instruction logging configuration helpers.
@@ -110,7 +111,8 @@ typedef enum {
 typedef enum {
     LOG_EVENT_STATE = 0,
     LOG_EVENT_CTX_UPDATE = 1,
-    LOG_EVENT_MARKER = 2
+    LOG_EVENT_MARKER = 2,
+    LOG_EVENT_REGDUMP = 3,
 } log_event_id_t;
 
 /*
@@ -152,16 +154,39 @@ typedef struct {
     qemu_log_instr_cpu_mode_t mode; /* CPU mode */
 } log_event_ctx_update_t;
 
+/*
+ * Register dump event.
+ * This is used to emit guest register dumps. The main use case is to
+ * recover lost state due to pause/unpause of the trace.
+ */
+typedef struct {
+    /*
+     * Register info - array of log_reginfo_t
+     * Use the qemu_log_instr_event_rdump_* interface to add entries.
+     */
+    GArray *gpr;
+} log_event_regdump_t;
+
 /*
  * Trace event.
  * This records arbitrary higher-level events associated with instruction
  * entries.
+ * Currently, in order to reduce complexity, the event definitions are
+ * target-agnostic, and we assume that all targets interested in emitting
+ * the events will need to implement some logic to fill the data structures
+ * taking care of any casting necessary.
+ * This is at the cost of wasting some space for architectures that do not
+ * have 64bit words. An additional burden is put on the target to perform
+ * some dynamic allocation for some events. This is sub-optimal but the
+ * alternative requires extra time to implement and this is good enough
+ * for now.
  */
 typedef struct {
     log_event_id_t id;
     union {
         log_event_trace_state_update_t state;
         log_event_ctx_update_t ctx_update;
+        log_event_regdump_t reg_dump;
         uint64_t marker;
     };
 } log_event_t;
diff --git a/target/arm/helper.c b/target/arm/helper.c
index ee72a1a3c18..c9fe942ee74 100644
--- a/target/arm/helper.c
+++ b/target/arm/helper.c
@@ -13780,10 +13780,10 @@ void aarch64_sve_change_el(CPUARMState *env, int old_el,
 }
 #endif
 
-#ifdef TARGET_CHERI
-
 #ifdef CONFIG_TCG_LOG_INSTR
 
+#ifdef TARGET_CHERI
+
 void HELPER(arm_log_instr)(CPUARMState *env, target_ulong pc, uint32_t opcode)
 {
     if (qemu_log_instr_enabled(env)) {
@@ -13793,4 +13793,10 @@ void HELPER(arm_log_instr)(CPUARMState *env, target_ulong pc, uint32_t opcode)
 }
 
 #endif
+
+bool cpu_log_instr_event_regdump(CPUARMState *env, log_event_t *event)
+{
+    return true;
+}
+
 #endif
diff --git a/target/i386/helper.c b/target/i386/helper.c
index 034f46bcc21..86f56a549f0 100644
--- a/target/i386/helper.c
+++ b/target/i386/helper.c
@@ -1198,3 +1198,10 @@ void x86_stq_phys(CPUState *cs, hwaddr addr, uint64_t val)
     address_space_stq(as, addr, val, attrs, NULL);
 }
 #endif
+
+#ifdef CONFIG_TCG_LOG_INSTR
+bool cpu_log_instr_event_regdump(CPUX86State *env, log_event_t *event)
+{
+    return true;
+}
+#endif
diff --git a/target/mips/op_helper_log_instr.c b/target/mips/op_helper_log_instr.c
index fcc8d0421ce..f6fdf1f0e5f 100644
--- a/target/mips/op_helper_log_instr.c
+++ b/target/mips/op_helper_log_instr.c
@@ -100,6 +100,11 @@ void helper_mips_log_instr_drop(CPUMIPSState *env)
     qemu_log_instr_drop(env);
 }
 
+bool cpu_log_instr_event_regdump(CPUMIPSState *env, log_event_t *event)
+{
+    return true;
+}
+
 /*
  * Note: do not check for logging enabled first, as this triggers
  * user logging start.
diff --git a/target/riscv/helper.h b/target/riscv/helper.h
index 09c54114a85..c030dee4b5d 100644
--- a/target/riscv/helper.h
+++ b/target/riscv/helper.h
@@ -83,6 +83,7 @@ DEF_HELPER_3(sc_c_cap, tl, env, i32, i32)
 DEF_HELPER_FLAGS_3(riscv_log_gpr_write, TCG_CALL_NO_RWG, void, env, i32, tl)
 DEF_HELPER_FLAGS_4(riscv_log_instr, TCG_CALL_NO_RWG, void, env, tl, i32, i32)
 DEF_HELPER_FLAGS_2(riscv_log_instr_event, TCG_CALL_NO_RWG, void, env, tl)
+DEF_HELPER_FLAGS_1(riscv_log_instr_regdump, TCG_CALL_NO_RWG, void, env)
 #endif
 
 /* Special functions */
diff --git a/target/riscv/op_helper_log_instr.c b/target/riscv/op_helper_log_instr.c
index 1880d1b24f8..a0554eeefc2 100644
--- a/target/riscv/op_helper_log_instr.c
+++ b/target/riscv/op_helper_log_instr.c
@@ -33,8 +33,11 @@
 #include "qemu/osdep.h"
 #include "exec/log_instr.h"
 #include "exec/helper-proto.h"
-#include "cheri-lazy-capregs.h"
 #include "cpu.h"
+#ifdef TARGET_CHERI
+#include "cheri-helper-utils.h"
+#include "cheri-lazy-capregs.h"
+#endif
 
 #ifdef TARGET_CHERI
 #define get_gpr_value(env, regnum) get_capreg_cursor(env, regnum)
@@ -78,24 +81,75 @@ void HELPER(riscv_log_instr_event)(CPURISCVState *env, target_ulong pc)
 
     event.id = get_gpr_value(env, 10);
 
-    switch (event.id) {
-    case LOG_EVENT_CTX_UPDATE:
-        event.ctx_update.op = get_gpr_value(env, 11);
-        event.ctx_update.pid = get_gpr_value(env, 12);
-        event.ctx_update.tid = get_gpr_value(env, 13);
-        event.ctx_update.cid = get_gpr_value(env, 14);
-        event.ctx_update.mode = cpu_priv_to_mode(env->priv);
-        break;
-    case LOG_EVENT_MARKER:
-        event.marker = get_gpr_value(env, 11);
-        break;
-    default:
-        warn_report("Unsupported event ID for TCG instr logging %d", event.id);
-        return;
+    if (qemu_log_instr_enabled(env)) {
+        switch (event.id) {
+        case LOG_EVENT_CTX_UPDATE:
+            event.ctx_update.op = get_gpr_value(env, 11);
+            event.ctx_update.pid = get_gpr_value(env, 12);
+            event.ctx_update.tid = get_gpr_value(env, 13);
+            event.ctx_update.cid = get_gpr_value(env, 14);
+            event.ctx_update.mode = cpu_priv_to_mode(env->priv);
+            break;
+        case LOG_EVENT_MARKER:
+            event.marker = get_gpr_value(env, 11);
+            break;
+        case LOG_EVENT_REGDUMP:
+            cpu_log_instr_event_regdump(env, &event);
+            break;
+        default:
+            warn_report("Unsupported event ID for TCG instr logging %d",
+                        event.id);
+            return;
+        }
+
+        qemu_log_instr_event(env, &event);
     }
+}
 
-    if (!qemu_log_instr_event_enabled(env, &event)) {
-        return;
+/*
+ * Target-dependent hook to emit register dump events.
+ * This is automatically called by the tracing subsystem
+ * when tracing is resumed.
+ */
+bool cpu_log_instr_event_regdump(CPURISCVState *env, log_event_t *event)
+{
+    int i;
+    int nregs;
+
+    event->id = LOG_EVENT_REGDUMP;
+#ifdef TARGET_CHERI
+    nregs = NUM_LAZY_CAP_REGS;
+#else
+    nregs = 32;
+#endif
+    qemu_log_instr_event_create_regdump(event, nregs);
+    for (i = 0; i < nregs; i++) {
+#ifdef TARGET_CHERI
+        const cap_register_t *value = get_readonly_capreg(env, i);
+        if (value->cr_tag)
+            qemu_log_instr_event_dump_cap(event, riscv_int_regnames[i], value);
+        else
+            qemu_log_instr_event_dump_cap_int(event, riscv_int_regnames[i],
+                                              cap_get_cursor(value));
+#else
+        target_ulong value = env->gpr[i];
+        qemu_log_instr_event_dump_reg(event, riscv_int_regnames[i], value);
+#endif
+    }
+
+    return false;
+}
+
+/*
+ * Emit a register dump event.
+ * Only dump general purpose registers for now. No fpu, vector or CPU registers.
+ */
+void HELPER(riscv_log_instr_regdump)(CPURISCVState *env)
+{
+    log_event_t event;
+
+    if (qemu_log_instr_enabled(env)) {
+        cpu_log_instr_event_regdump(env, &event);
+        qemu_log_instr_event(env, &event);
     }
-    qemu_log_instr_event(env, &event);
 }
diff --git a/trace_extra/trace_perfetto.cc b/trace_extra/trace_perfetto.cc
index d9110d14161..0aa99af972b 100644
--- a/trace_extra/trace_perfetto.cc
+++ b/trace_extra/trace_perfetto.cc
@@ -233,6 +233,8 @@ void process_events(perfetto_backend_data *data, cpu_log_entry_handle entry)
                                     qemu_arg->set_marker(evt->marker);
                                 });
         } break;
+        case LOG_EVENT_REGDUMP:
+            break;
         default:
             assert(false && "Invalid event identifier");
         }
@@ -334,8 +336,7 @@ void process_instr(perfetto_backend_data *data, cpu_log_entry_handle entry)
                         perfetto::protos::pbzero::QEMULogEntryMem::CSTORE);
                     break;
                 default:
-                    // XXX Notify error somehow?
-                    break;
+                    assert(false && "Invalid meminfo flag");
                 }
                 if (flags & LMI_CAP) {
                     auto *capinfo = meminfo->set_cap_value();

From 03f34ab1f75c0823d623fbf598a83cf1af179cc6 Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Wed, 2 Feb 2022 18:32:52 +0000
Subject: [PATCH 45/74] Fix allocation of temporary argument to
 do_cpu_loglevel_switch.

The function expects an argument structure allocated by g_new.
---
 accel/tcg/log_instr.c | 36 ++++++++++++++++++++----------------
 1 file changed, 20 insertions(+), 16 deletions(-)

diff --git a/accel/tcg/log_instr.c b/accel/tcg/log_instr.c
index 519e4ed399b..54a8a4f6c28 100644
--- a/accel/tcg/log_instr.c
+++ b/accel/tcg/log_instr.c
@@ -419,20 +419,23 @@ static void do_global_loglevel_switch(CPUState *cpu, run_on_cpu_data data)
 int qemu_log_instr_global_switch(int log_flags)
 {
     CPUState *cpu;
-    qemu_log_next_level_arg_t *arg = g_new(qemu_log_next_level_arg_t, 1);
-
-    if (log_flags & CPU_LOG_INSTR_U) {
-        arg->next_level = QEMU_LOG_INSTR_LOGLEVEL_USER;
-        log_flags |= CPU_LOG_INSTR;
-    } else if (log_flags & CPU_LOG_INSTR) {
-        arg->next_level = QEMU_LOG_INSTR_LOGLEVEL_ALL;
-    } else {
-        arg->next_level = QEMU_LOG_INSTR_LOGLEVEL_NONE;
-    }
-    arg->global = true;
+    qemu_log_next_level_arg_t *arg;
 
     CPU_FOREACH(cpu)
     {
+        arg = g_new(qemu_log_next_level_arg_t, 1);
+
+        if (log_flags & CPU_LOG_INSTR_U) {
+            arg->next_level = QEMU_LOG_INSTR_LOGLEVEL_USER;
+            log_flags |= CPU_LOG_INSTR;
+        } else if (log_flags & CPU_LOG_INSTR) {
+            arg->next_level = QEMU_LOG_INSTR_LOGLEVEL_ALL;
+        } else {
+            arg->next_level = QEMU_LOG_INSTR_LOGLEVEL_NONE;
+        }
+        arg->global = true;
+
+        /* arg ownership transferred to do_global_loglevel_switch */
         async_safe_run_on_cpu(cpu, do_global_loglevel_switch,
                               RUN_ON_CPU_HOST_PTR(arg));
     }
@@ -493,7 +496,7 @@ void qemu_log_instr_init(CPUState *cpu)
     cpu_log_instr_state_t *cpulog = &cpu->log_state;
     GArray *entry_ring = g_array_sized_new(false, true, sizeof(cpu_log_entry_t),
                                            reset_entry_buffer_size);
-    qemu_log_next_level_arg_t start_level;
+    qemu_log_next_level_arg_t *start_level;
     cpu_log_entry_t *entry;
     int i;
 
@@ -530,15 +533,16 @@ void qemu_log_instr_init(CPUState *cpu)
 
     /* If we are starting with instruction logging enabled, switch it on now */
     if (qemu_loglevel_mask(CPU_LOG_INSTR | CPU_LOG_INSTR_U)) {
+        start_level = g_new(qemu_log_next_level_arg_t, 1);
         if (qemu_loglevel_mask(CPU_LOG_INSTR_U)) {
             assert(qemu_loglevel_mask(CPU_LOG_INSTR) &&
                    "CPU_LOG_INSTR_U implies CPU_LOG_INSTR broken");
-            start_level.next_level = QEMU_LOG_INSTR_LOGLEVEL_USER;
+            start_level->next_level = QEMU_LOG_INSTR_LOGLEVEL_USER;
         } else {
-            start_level.next_level = QEMU_LOG_INSTR_LOGLEVEL_ALL;
+            start_level->next_level = QEMU_LOG_INSTR_LOGLEVEL_ALL;
         }
-        start_level.global = true;
-        do_cpu_loglevel_switch(cpu, RUN_ON_CPU_HOST_PTR(&start_level));
+        start_level->global = true;
+        do_cpu_loglevel_switch(cpu, RUN_ON_CPU_HOST_PTR(start_level));
     }
 
     if (reset_filters != NULL) {

From 21334c12090be8b611c5aa067e8fa0780893206a Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Fri, 18 Feb 2022 15:10:30 +0000
Subject: [PATCH 46/74] Remove log event variant that bypasses log enable.

After testing this was deemed unnecessary.
---
 include/exec/log_instr.h | 18 ------------------
 include/qemu/log_instr.h |  9 +--------
 2 files changed, 1 insertion(+), 26 deletions(-)

diff --git a/include/exec/log_instr.h b/include/exec/log_instr.h
index fbe59a812a4..6ccecec5d81 100644
--- a/include/exec/log_instr.h
+++ b/include/exec/log_instr.h
@@ -203,24 +203,6 @@ bool qemu_log_instr_check_enabled(CPUArchState *env);
 #define qemu_log_instr_enabled(env)                                            \
     unlikely(qemu_log_instr_check_enabled((env)))
 
-/*
- * Helper macro to check whether an event should be appended to the trace or
- * not. This takes into account the event data to allow some events to be
- * appended to the current log buffer entry even when tracing is disabled. These
- * events will only become visible in the trace when tracing is started, in the
- * first log entry. This is intended to be used only for OS-driven events that
- * are relatively low-frequency w.r.t. the number of instructions executed.
- */
-static inline bool qemu_log_instr_event_enabled(CPUArchState *env,
-                                                log_event_t *event)
-{
-    if (event->id == LOG_EVENT_CTX_UPDATE &&
-        event->ctx_update.op == LOG_EVENT_CTX_OP_SETUP) {
-        return true;
-    }
-    return qemu_log_instr_check_enabled(env);
-}
-
 /*
  * Register a trace filter for a given CPU.
  */
diff --git a/include/qemu/log_instr.h b/include/qemu/log_instr.h
index 8c180d97720..5b457182119 100644
--- a/include/qemu/log_instr.h
+++ b/include/qemu/log_instr.h
@@ -133,14 +133,7 @@ typedef enum {
     /*
      * Switch context from the current to the new (proc, thread, compartment) ID
      */
-    LOG_EVENT_CTX_OP_SWITCH = 0,
-    /*
-     * Same as LOG_EVENT_CTX_OP_SWITCH but should bypass tracing activation
-     * status, Meaning that these events will reach the backend even when
-     * tracing is off. This is useful setup the correct context identifier
-     * before switching on tracing.
-     */
-    LOG_EVENT_CTX_OP_SETUP = 1,
+    LOG_EVENT_CTX_OP_SWITCH = 0
 } log_event_ctx_update_op_t;
 
 /*

From 47348d2ca2156d4fff2291bccbd9efa1761ce579 Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Fri, 25 Feb 2022 10:31:14 +0000
Subject: [PATCH 47/74] Implement register dumps for protobuf instruction
 tracing backend.

---
 accel/tcg/log_instr_protobuf.c | 68 +++++++++++++++++++++++-----------
 trace_extra/cheri-perfetto     |  2 +-
 2 files changed, 47 insertions(+), 23 deletions(-)

diff --git a/accel/tcg/log_instr_protobuf.c b/accel/tcg/log_instr_protobuf.c
index 10bb0c2588c..31743277d7f 100644
--- a/accel/tcg/log_instr_protobuf.c
+++ b/accel/tcg/log_instr_protobuf.c
@@ -63,6 +63,7 @@ typedef GEN_TYPE_PREFIX(Exc) QEMULogEntryExc;
 typedef GEN_TYPE_PREFIX() QEMULogEntry;
 typedef GEN_TYPE_PREFIX(EvtTraceState) QEMULogEntryEvtTraceState;
 typedef GEN_TYPE_PREFIX(EvtCtxUpdate) QEMULogEntryEvtCtxUpdate;
+typedef GEN_TYPE_PREFIX(EvtRegdump) QEMULogEntryEvtRegdump;
 typedef GEN_TYPE_NS(QEMULogEventTraceState) QEMULogEventTraceState;
 
 #define GEN_FN_NS(x)                        perfetto__protos__##x
@@ -179,52 +180,75 @@ static void release_protobuf_reginfo(QEMULogEntryReg *pb_reg)
 
 static void emit_protobuf_event(log_event_t *evtinfo, QEMULogEntryEvt **pb_evt)
 {
-    QEMULogEntryEvt *evt = g_malloc(sizeof(QEMULogEntryEvt));
+    QEMULogEntryEvt *pb_evt_entry = g_malloc(sizeof(QEMULogEntryEvt));
+    int i;
 
-    qemulog_entry_evt__init(evt);
+    qemulog_entry_evt__init(pb_evt_entry);
     switch (evtinfo->id) {
     case LOG_EVENT_STATE:
-        evt->event_case = ENUM_ENTRY_EVT_CASE(STATE);
-        evt->state = g_malloc(sizeof(QEMULogEntryEvtTraceState));
-        qemulog_entry_evt_trace_state__init(evt->state);
-        evt->state->next_state =
+        pb_evt_entry->event_case = ENUM_ENTRY_EVT_CASE(STATE);
+        pb_evt_entry->state = g_malloc(sizeof(QEMULogEntryEvtTraceState));
+        qemulog_entry_evt_trace_state__init(pb_evt_entry->state);
+        pb_evt_entry->state->next_state =
             (QEMULogEventTraceState)evtinfo->state.next_state;
-        evt->state->pc = evtinfo->state.pc;
+        pb_evt_entry->state->pc = evtinfo->state.pc;
         break;
     case LOG_EVENT_CTX_UPDATE:
-        evt->event_case = ENUM_ENTRY_EVT_CASE(CTX_UPDATE);
-        evt->ctx_update = g_malloc(sizeof(QEMULogEntryEvtCtxUpdate));
-        qemulog_entry_evt_ctx_update__init(evt->ctx_update);
-        evt->ctx_update->pid = evtinfo->ctx_update.pid;
-        evt->ctx_update->tid = evtinfo->ctx_update.tid;
-        evt->ctx_update->cid = evtinfo->ctx_update.cid;
+        pb_evt_entry->event_case = ENUM_ENTRY_EVT_CASE(CTX_UPDATE);
+        pb_evt_entry->ctx_update = g_malloc(sizeof(QEMULogEntryEvtCtxUpdate));
+        qemulog_entry_evt_ctx_update__init(pb_evt_entry->ctx_update);
+        pb_evt_entry->ctx_update->pid = evtinfo->ctx_update.pid;
+        pb_evt_entry->ctx_update->tid = evtinfo->ctx_update.tid;
+        pb_evt_entry->ctx_update->cid = evtinfo->ctx_update.cid;
         /* Safe to de-const cast as the pointer is only used during packing */
-        evt->ctx_update->mode =
+        pb_evt_entry->ctx_update->mode =
             (char *)cpu_get_mode_name(evtinfo->ctx_update.mode);
         break;
     case LOG_EVENT_MARKER:
-        evt->event_case = ENUM_ENTRY_EVT_CASE(MARKER);
-        evt->marker = evtinfo->marker;
-        break;
-    case LOG_EVENT_REGDUMP:
+        pb_evt_entry->event_case = ENUM_ENTRY_EVT_CASE(MARKER);
+        pb_evt_entry->marker = evtinfo->marker;
         break;
+    case LOG_EVENT_REGDUMP: {
+        log_event_regdump_t *regdump = &evtinfo->reg_dump;
+        QEMULogEntryEvtRegdump *pb_regdump =
+            g_malloc(sizeof(QEMULogEntryEvtRegdump));
+        QEMULogEntryReg **pb_regs;
+        qemulog_entry_evt_regdump__init(pb_regdump);
+
+        pb_evt_entry->event_case = ENUM_ENTRY_EVT_CASE(REGDUMP);
+        pb_evt_entry->regdump = pb_regdump;
+        pb_regdump->n_regs = regdump->gpr->len;
+        pb_regs = g_malloc(regdump->gpr->len * sizeof(QEMULogEntryReg *));
+        pb_regdump->regs = pb_regs;
+        for (i = 0; i < regdump->gpr->len; i++, pb_regs++) {
+            log_reginfo_t *rinfo =
+                &g_array_index(regdump->gpr, log_reginfo_t, i);
+            emit_protobuf_reginfo(rinfo, pb_regs);
+        }
+    } break;
     default:
         assert(0 && "unknown event ID");
     }
-    *pb_evt = evt;
+    *pb_evt = pb_evt_entry;
 }
 
 static void release_protobuf_event(QEMULogEntryEvt *pb_evt)
 {
+    int i;
+
     if (pb_evt->event_case == ENUM_ENTRY_EVT_CASE(STATE)) {
         g_free(pb_evt->state);
     }
     if (pb_evt->event_case == ENUM_ENTRY_EVT_CASE(CTX_UPDATE)) {
         g_free(pb_evt->ctx_update);
     }
-    /* if (pb_evt->event_case == ENUM_ENTRY_EVT_CASE(REGDUMP)) { */
-    /*     g_free(pb_evt->regdump); */
-    /* } */
+    if (pb_evt->event_case == ENUM_ENTRY_EVT_CASE(REGDUMP)) {
+        for (i = 0; i < pb_evt->regdump->n_regs; i++) {
+            release_protobuf_reginfo(pb_evt->regdump->regs[i]);
+        }
+        g_free(pb_evt->regdump->regs);
+        g_free(pb_evt->regdump);
+    }
     g_free(pb_evt);
 }
 
diff --git a/trace_extra/cheri-perfetto b/trace_extra/cheri-perfetto
index 6c8dbfabf04..98682fd0e43 160000
--- a/trace_extra/cheri-perfetto
+++ b/trace_extra/cheri-perfetto
@@ -1 +1 @@
-Subproject commit 6c8dbfabf04f9a2bd701a76ff29f49159933222c
+Subproject commit 98682fd0e432c44ac54e637922f22e13786325da

From 49abdf281d29b3c0f8e4ebf8aebb72f64a9b287e Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Fri, 25 Feb 2022 10:31:33 +0000
Subject: [PATCH 48/74] Retire instruction logging event type for context
 updates.

Remove the event variant that bypasses log enable. This is useless as
switching on/off tracing around the event is feasible and the event is
rare enough that it is not a performance issue.
---
 trace_extra/guest_context_tracker.cc | 4 +---
 trace_extra/trace_perfetto.cc        | 3 +--
 2 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/trace_extra/guest_context_tracker.cc b/trace_extra/guest_context_tracker.cc
index 41176ec29ca..bb6313a64e6 100644
--- a/trace_extra/guest_context_tracker.cc
+++ b/trace_extra/guest_context_tracker.cc
@@ -138,9 +138,7 @@ void guest_context_tracker::mode_update(
 void guest_context_tracker::context_update(const log_event_ctx_update_t *evt)
 {
     // Currently these are the only ones existing
-    assert((evt->op == LOG_EVENT_CTX_OP_SWITCH ||
-            evt->op == LOG_EVENT_CTX_OP_SETUP) &&
-           "Invalid ctx update op");
+    assert((evt->op == LOG_EVENT_CTX_OP_SWITCH) && "Invalid ctx update op");
     /*
      * Find or create the track associated with the new context and set it as
      * the current context track on the CPU. This will be used to log
diff --git a/trace_extra/trace_perfetto.cc b/trace_extra/trace_perfetto.cc
index 0aa99af972b..35d83f0b6cf 100644
--- a/trace_extra/trace_perfetto.cc
+++ b/trace_extra/trace_perfetto.cc
@@ -213,8 +213,7 @@ void process_events(perfetto_backend_data *data, cpu_log_entry_handle entry)
         } break;
         case LOG_EVENT_CTX_UPDATE: {
             /* Swap current context. */
-            if (evt->ctx_update.op == LOG_EVENT_CTX_OP_SETUP ||
-                evt->ctx_update.op == LOG_EVENT_CTX_OP_SWITCH) {
+            if (evt->ctx_update.op == LOG_EVENT_CTX_OP_SWITCH) {
                 ctx_data->stats.pause(*ctx_track, perfetto_log_entry_pc(entry));
                 data->ctx_tracker_.context_update(&evt->ctx_update);
                 /* Reload data and track as context_update will have changed

From 7dcf6841263407ff1564078604a89be21b03faff Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Tue, 26 Apr 2022 20:41:27 +0100
Subject: [PATCH 49/74] Implement RISC-V tracing counter NOPs.

Add NOP that guests can use to create and increment/decrement
counters.
The counters are emitted into the perfetto trace stream as normal
perfetto counter tracks.
---
 accel/tcg/log_instr_json.c           |   3 +
 accel/tcg/log_instr_protobuf.c       |   2 +
 accel/tcg/log_instr_text.c           |   9 ++
 include/exec/log_instr_perfetto.h    |   3 +-
 include/qemu/log_instr.h             |  24 +++-
 target/riscv/helper.h                |   1 -
 target/riscv/op_helper_log_instr.c   |  51 +++++---
 trace_extra/cheri-perfetto           |   2 +-
 trace_extra/guest_context_tracker.cc | 136 ++++++++++-----------
 trace_extra/guest_context_tracker.hh |  99 +++++++++++----
 trace_extra/meson.build              |   4 +-
 trace_extra/trace_counters.cc        | 163 +++++++++++++++++++++++++
 trace_extra/trace_counters.hh        | 112 +++++++++++++++++
 trace_extra/trace_perfetto.cc        | 176 ++++++++++++++++++---------
 14 files changed, 603 insertions(+), 182 deletions(-)
 create mode 100644 trace_extra/trace_counters.cc
 create mode 100644 trace_extra/trace_counters.hh

diff --git a/accel/tcg/log_instr_json.c b/accel/tcg/log_instr_json.c
index 7fc0774b147..80dec851c4b 100644
--- a/accel/tcg/log_instr_json.c
+++ b/accel/tcg/log_instr_json.c
@@ -275,6 +275,9 @@ void emit_json_entry(CPUArchState *env, cpu_log_entry_t *entry)
             case LOG_EVENT_REGDUMP:
                 emit_json_evt_regdump(&event->reg_dump, js_evt);
                 break;
+            case LOG_EVENT_COUNTER:
+                /* TODO Unimpl */
+                break;
             default:
                 assert(0 && "unknown event ID");
             }
diff --git a/accel/tcg/log_instr_protobuf.c b/accel/tcg/log_instr_protobuf.c
index 31743277d7f..1e630a15412 100644
--- a/accel/tcg/log_instr_protobuf.c
+++ b/accel/tcg/log_instr_protobuf.c
@@ -226,6 +226,8 @@ static void emit_protobuf_event(log_event_t *evtinfo, QEMULogEntryEvt **pb_evt)
             emit_protobuf_reginfo(rinfo, pb_regs);
         }
     } break;
+    case LOG_EVENT_COUNTER:
+        break;
     default:
         assert(0 && "unknown event ID");
     }
diff --git a/accel/tcg/log_instr_text.c b/accel/tcg/log_instr_text.c
index 522327dd947..743bbb17add 100644
--- a/accel/tcg/log_instr_text.c
+++ b/accel/tcg/log_instr_text.c
@@ -121,6 +121,7 @@ void emit_text_instr(CPUArchState *env, cpu_log_entry_t *entry)
     const log_event_t *event;
     const char *log_state_op;
     int i, j;
+    bool incremental;
 
     if (entry->flags & LI_FLAG_HAS_INSTR_DATA) {
         /* Dump CPU-ID:ASID + address */
@@ -227,6 +228,14 @@ void emit_text_instr(CPUArchState *env, cpu_log_entry_t *entry)
                     &g_array_index(event->reg_dump.gpr, log_reginfo_t, j);
                 emit_text_reg(r);
             }
+            break;
+        case LOG_EVENT_COUNTER:
+            incremental = log_event_counter_incremental(event->counter.flags);
+            qemu_log("Counter %s %s[%d]: %lx\n", (incremental) ? "INC" : "ABS",
+                     event->counter.name,
+                     log_event_counter_slot(event->counter.flags),
+                     event->counter.value);
+            break;
         default:
             assert(0 && "unknown event ID");
         }
diff --git a/include/exec/log_instr_perfetto.h b/include/exec/log_instr_perfetto.h
index 122acd93f6d..ee5a41dc26a 100644
--- a/include/exec/log_instr_perfetto.h
+++ b/include/exec/log_instr_perfetto.h
@@ -62,7 +62,8 @@ PERFETTO_DEFINE_CATEGORIES(
     perfetto::Category("trap").SetDescription("CPU trap events"),
     perfetto::Category("sched").SetDescription("Scheduling events"),
     perfetto::Category("marker").SetDescription(
-        "Guest trace timestamp markers"));
+        "Guest trace timestamp markers"),
+    perfetto::Category("counter").SetDescription("Guest-driven counters"));
 #endif
 
 /*
diff --git a/include/qemu/log_instr.h b/include/qemu/log_instr.h
index 5b457182119..a7f19cf5b09 100644
--- a/include/qemu/log_instr.h
+++ b/include/qemu/log_instr.h
@@ -47,6 +47,8 @@
 #define QEMU_LOG_PRINTF_BUF_DEPTH 32
 /* Early flush if buffer gets this full. */
 #define QEMU_LOG_PRINTF_FLUSH_BARRIER 32
+/* Max name size for qemu counters and other events */
+#define QEMU_LOG_EVENT_MAX_NAMELEN 64
 
 /*
  * Instruction logging format
@@ -113,6 +115,7 @@ typedef enum {
     LOG_EVENT_CTX_UPDATE = 1,
     LOG_EVENT_MARKER = 2,
     LOG_EVENT_REGDUMP = 3,
+    LOG_EVENT_COUNTER = 4,
 } log_event_id_t;
 
 /*
@@ -160,6 +163,20 @@ typedef struct {
     GArray *gpr;
 } log_event_regdump_t;
 
+/*
+ * Counter event.
+ * This represents an arbitrary named counter. Generally driven from the
+ * guest OS via NOPs, but can also be generated internally.
+ */
+typedef struct {
+    char name[QEMU_LOG_EVENT_MAX_NAMELEN];
+    int64_t value;
+    uint64_t flags;
+} log_event_counter_t;
+
+#define log_event_counter_slot(flags)        (flags & 0xffff)
+#define log_event_counter_incremental(flags) ((flags >> 32) & 0x01)
+
 /*
  * Trace event.
  * This records arbitrary higher-level events associated with instruction
@@ -181,6 +198,7 @@ typedef struct {
         log_event_ctx_update_t ctx_update;
         log_event_regdump_t reg_dump;
         uint64_t marker;
+        log_event_counter_t counter;
     };
 } log_event_t;
 
@@ -242,9 +260,9 @@ typedef struct qemu_log_instr_stats {
     uint64_t trace_stop;
 } qemu_log_instr_stats_t;
 
-#define QEMU_LOG_INSTR_INC_STAT(cpu_state, stat) \
-    do {                                         \
-        cpu_state->stats.stat++;                 \
+#define QEMU_LOG_INSTR_INC_STAT(cpu_state, stat)                               \
+    do {                                                                       \
+        cpu_state->stats.stat++;                                               \
     } while (0)
 
 /*
diff --git a/target/riscv/helper.h b/target/riscv/helper.h
index c030dee4b5d..09c54114a85 100644
--- a/target/riscv/helper.h
+++ b/target/riscv/helper.h
@@ -83,7 +83,6 @@ DEF_HELPER_3(sc_c_cap, tl, env, i32, i32)
 DEF_HELPER_FLAGS_3(riscv_log_gpr_write, TCG_CALL_NO_RWG, void, env, i32, tl)
 DEF_HELPER_FLAGS_4(riscv_log_instr, TCG_CALL_NO_RWG, void, env, tl, i32, i32)
 DEF_HELPER_FLAGS_2(riscv_log_instr_event, TCG_CALL_NO_RWG, void, env, tl)
-DEF_HELPER_FLAGS_1(riscv_log_instr_regdump, TCG_CALL_NO_RWG, void, env)
 #endif
 
 /* Special functions */
diff --git a/target/riscv/op_helper_log_instr.c b/target/riscv/op_helper_log_instr.c
index a0554eeefc2..5ef6dfe40d7 100644
--- a/target/riscv/op_helper_log_instr.c
+++ b/target/riscv/op_helper_log_instr.c
@@ -45,6 +45,29 @@
 #define get_gpr_value(env, regnum) (env->gpr[regnum])
 #endif
 
+/*
+ * Read a null-terminated guest string without requiring a strlen() in
+ * the guest. This is probably slower but does not perturb guest call-graph.
+ */
+static int read_guest_cstring(CPURISCVState *env, target_ulong vaddr,
+                              char *buffer, size_t maxlen)
+{
+    int ret = 0;
+
+    buffer[maxlen - 1] = '\0';
+    while (maxlen > 1) {
+        ret = cpu_memory_rw_debug(env_cpu(env), vaddr, buffer, 1, false);
+        if (ret != 0 || *buffer == '\0') {
+            break;
+        }
+        maxlen--;
+        vaddr++;
+        buffer++;
+    }
+
+    return ret;
+}
+
 void HELPER(riscv_log_gpr_write)(CPURISCVState *env, uint32_t regnum,
                                  target_ulong value)
 {
@@ -78,6 +101,8 @@ void HELPER(riscv_log_instr)(CPURISCVState *env, target_ulong pc,
 void HELPER(riscv_log_instr_event)(CPURISCVState *env, target_ulong pc)
 {
     log_event_t event;
+    target_ulong vaddr;
+    int ret;
 
     event.id = get_gpr_value(env, 10);
 
@@ -96,6 +121,18 @@ void HELPER(riscv_log_instr_event)(CPURISCVState *env, target_ulong pc)
         case LOG_EVENT_REGDUMP:
             cpu_log_instr_event_regdump(env, &event);
             break;
+        case LOG_EVENT_COUNTER:
+            vaddr = get_gpr_value(env, 11);
+            ret = read_guest_cstring(env, vaddr, (char *)event.counter.name,
+                                     sizeof(event.counter.name));
+            if (ret != 0) {
+                warn_report("LOG EVENT HELPER: Could not read guest string at"
+                            " vaddr 0x" TARGET_FMT_lx, vaddr);
+                return;
+            }
+            event.counter.value = (int64_t)get_gpr_value(env, 12);
+            event.counter.flags = get_gpr_value(env, 13);
+            break;
         default:
             warn_report("Unsupported event ID for TCG instr logging %d",
                         event.id);
@@ -139,17 +176,3 @@ bool cpu_log_instr_event_regdump(CPURISCVState *env, log_event_t *event)
 
     return false;
 }
-
-/*
- * Emit a register dump event.
- * Only dump general purpose registers for now. No fpu, vector or CPU registers.
- */
-void HELPER(riscv_log_instr_regdump)(CPURISCVState *env)
-{
-    log_event_t event;
-
-    if (qemu_log_instr_enabled(env)) {
-        cpu_log_instr_event_regdump(env, &event);
-        qemu_log_instr_event(env, &event);
-    }
-}
diff --git a/trace_extra/cheri-perfetto b/trace_extra/cheri-perfetto
index 98682fd0e43..2bbdf5a6ccd 160000
--- a/trace_extra/cheri-perfetto
+++ b/trace_extra/cheri-perfetto
@@ -1 +1 @@
-Subproject commit 98682fd0e432c44ac54e637922f22e13786325da
+Subproject commit 2bbdf5a6ccddbee6df68dbd16c7c6e136bddd6d6
diff --git a/trace_extra/guest_context_tracker.cc b/trace_extra/guest_context_tracker.cc
index bb6313a64e6..cb590acd1b6 100644
--- a/trace_extra/guest_context_tracker.cc
+++ b/trace_extra/guest_context_tracker.cc
@@ -37,13 +37,12 @@ namespace cheri
 
 /*
  * Dynamically created tracks and track data.
- * The tracks lock protects both tracks and ctx_data.
+ * The track_state_lock must be held for updating the track_state map.
  */
-std::mutex tracks_lock;
-std::unordered_map<qemu_context_track::qemu_ctx_id,
-                   std::shared_ptr<qemu_context_track>,
-                   cheri::tuple_hasher<qemu_context_track::qemu_ctx_id>>
-    tracks;
+std::mutex track_state_lock;
+std::unordered_map<qemu_ctx_id, std::shared_ptr<qemu_context_state>,
+                   tuple_hasher<qemu_ctx_id>>
+    track_state_map;
 
 /* Helper to generate unique IDs for dynamic tracks */
 unsigned long gen_track_uuid()
@@ -61,16 +60,24 @@ qemu_cpu_mode_to_trace(qemu_log_instr_cpu_mode_t mode)
     return static_cast<perfetto::protos::pbzero::QEMULogEntryModeSwitch>(mode);
 }
 
-/* static */
-std::shared_ptr<qemu_context_track>
-qemu_context_track::make_context_track(qemu_ctx_id key)
+qemu_cpu_track::qemu_cpu_track(int id)
+    : perfetto::Track(gen_track_uuid(), perfetto::Track()), cpu_id(id)
 {
-    return std::make_shared<qemu_context_track>(key);
 }
 
-qemu_context_track::qemu_ctx_id qemu_context_track::get_id() const
+perfetto::protos::gen::TrackDescriptor qemu_cpu_track::Serialize() const
 {
-    return std::make_tuple(pid, tid, cid, mode);
+    auto desc = Track::Serialize();
+    std::string name("CPU " + std::to_string(cpu_id));
+    desc.set_name(name);
+    return desc;
+}
+
+void qemu_cpu_track::Serialize(
+    perfetto::protos::pbzero::TrackDescriptor *desc) const
+{
+    auto bytes = Serialize().SerializeAsString();
+    desc->AppendRawProtoBytes(bytes.data(), bytes.size());
 }
 
 perfetto::protos::gen::TrackDescriptor qemu_context_track::Serialize() const
@@ -81,58 +88,54 @@ perfetto::protos::gen::TrackDescriptor qemu_context_track::Serialize() const
     cheri_desc->set_tid(tid);
     cheri_desc->set_cid(cid);
     cheri_desc->set_el(mode);
+    desc.set_name("CTX " + std::to_string(pid) + ":" + std::to_string(tid) +
+                  ":" + std::to_string(cid) + ":" + std::to_string(mode));
     return desc;
 }
 
+void qemu_context_track::Serialize(
+    perfetto::protos::pbzero::TrackDescriptor *desc) const
+{
+    auto bytes = Serialize().SerializeAsString();
+    desc->AppendRawProtoBytes(bytes.data(), bytes.size());
+}
+
 qemu_context_track::qemu_context_track(qemu_ctx_id key)
-    : perfetto::Track(cheri::gen_track_uuid(), perfetto::Track()),
+    : perfetto::Track(gen_track_uuid(), perfetto::Track()),
       pid(std::get<0>(key)), tid(std::get<1>(key)), cid(std::get<2>(key)),
       mode(std::get<3>(key))
 {
 }
 
-qemu_context_track::~qemu_context_track()
+qemu_ctx_id qemu_context_track::get_id() const
 {
-    perfetto::TrackEvent::EraseTrackDescriptor(*this);
+    return std::make_tuple(pid, tid, cid, mode);
 }
 
-guest_context_tracker::guest_context_tracker(int cpu_id)
-    : cpu_track_(perfetto::Track::Global(gen_track_uuid()))
-{
-    std::string track_name("CPU " + std::to_string(cpu_id));
-    auto desc = cpu_track_.Serialize();
-    desc.set_name(track_name);
-    perfetto::TrackEvent::SetTrackDescriptor(cpu_track_, desc);
-}
+guest_context_tracker::guest_context_tracker(int cpu_id) : cpu_state(cpu_id) {}
 
 void guest_context_tracker::mode_update(
     perfetto::protos::pbzero::QEMULogEntryModeSwitch new_mode)
 {
-    if (ctx_track_ == nullptr)
+    if (ctx_state == nullptr)
         return;
 
-    auto key = ctx_track_->get_id();
+    auto key = ctx_state->track.get_id();
     std::get<3>(key) = new_mode;
     /* Check if we have a track for this mode */
-    std::shared_ptr<qemu_context_track> track;
+    std::shared_ptr<qemu_context_state> state;
     {
-        std::lock_guard<std::mutex> tracks_guard(tracks_lock);
-        auto found_track_iter = tracks.find(key);
-        if (found_track_iter == tracks.end()) {
-            track = qemu_context_track::make_context_track(key);
-            auto desc = track->Serialize();
-            desc.set_name("CTX " + std::to_string(track->pid) + ":" +
-                          std::to_string(track->tid) + ":" +
-                          std::to_string(track->cid) + ":" +
-                          std::to_string(track->mode));
-            perfetto::TrackEvent::SetTrackDescriptor(*track, desc);
-            tracks.emplace(key, track);
+        std::lock_guard<std::mutex> track_state_guard(track_state_lock);
+        auto it = track_state_map.find(key);
+        if (it == track_state_map.end()) {
+            state = std::make_shared<qemu_context_state>(key);
+            track_state_map.emplace(key, state);
         } else {
             /* Existing context */
-            track = found_track_iter->second;
+            state = it->second;
         }
     }
-    ctx_track_ = track;
+    ctx_state = state;
 }
 
 void guest_context_tracker::context_update(const log_event_ctx_update_t *evt)
@@ -150,59 +153,44 @@ void guest_context_tracker::context_update(const log_event_ctx_update_t *evt)
 
     auto mode = qemu_cpu_mode_to_trace(evt->mode);
     /* Fetch the tracks for the new context */
-    qemu_context_track::qemu_ctx_id key =
-        std::make_tuple(evt->pid, evt->tid, evt->cid, mode);
-    std::shared_ptr<qemu_context_track> track;
+    qemu_ctx_id key = std::make_tuple(evt->pid, evt->tid, evt->cid, mode);
+    std::shared_ptr<qemu_context_state> state;
     {
-        std::lock_guard<std::mutex> tracks_guard(tracks_lock);
-        auto ctx_track_iter = tracks.find(key);
-        if (ctx_track_iter == tracks.end()) {
-            /* New context */
-            track = qemu_context_track::make_context_track(key);
-            auto desc = track->Serialize();
-            desc.set_name("CTX " + std::to_string(evt->pid) + ":" +
-                          std::to_string(evt->tid) + ":" +
-                          std::to_string(evt->cid) + ":" +
-                          std::to_string(mode));
-            perfetto::TrackEvent::SetTrackDescriptor(*track, desc);
-            tracks.emplace(key, track);
+        std::lock_guard<std::mutex> track_state_guard(track_state_lock);
+        auto it = track_state_map.find(key);
+        if (it == track_state_map.end()) {
+            state = std::make_shared<qemu_context_state>(key);
+            track_state_map.emplace(key, state);
         } else {
             /* Existing context */
-            track = ctx_track_iter->second;
+            state = it->second;
         }
     }
-    ctx_track_ = track;
+    ctx_state = state;
 }
 
 void guest_context_tracker::flush_all_ctx_data()
 {
     /* Flush per-CPU stats */
-    cpu_ctx_data_.stats.flush(cpu_track_);
-    for (auto &id_and_track : tracks) {
-        qemu_context_data &ctx_data = id_and_track.second->ctx_data;
-        ctx_data.stats.flush(*id_and_track.second);
+    cpu_state.stats.flush(cpu_state.track);
+    std::lock_guard<std::mutex> track_state_guard(track_state_lock);
+    for (auto id_and_state : track_state_map) {
+        qemu_stats &stats = id_and_state.second->stats;
+        stats.flush(id_and_state.second->track);
     }
 }
 
-perfetto::Track &guest_context_tracker::get_cpu_track()
+qemu_fallback_state *guest_context_tracker::get_cpu_state()
 {
-    return cpu_track_;
-}
-
-perfetto::Track &guest_context_tracker::get_ctx_track()
-{
-    if (ctx_track_)
-        return *ctx_track_;
-    else
-        return cpu_track_;
+    return &cpu_state;
 }
 
-qemu_context_data &guest_context_tracker::get_ctx_data()
+qemu_tracker_state *guest_context_tracker::get_ctx_state()
 {
-    if (ctx_track_)
-        return ctx_track_->ctx_data;
+    if (ctx_state)
+        return ctx_state.get();
     else
-        return cpu_ctx_data_;
+        return &cpu_state;
 }
 
 } // namespace cheri
diff --git a/trace_extra/guest_context_tracker.hh b/trace_extra/guest_context_tracker.hh
index 67804a7f6c5..9169693da2c 100644
--- a/trace_extra/guest_context_tracker.hh
+++ b/trace_extra/guest_context_tracker.hh
@@ -29,6 +29,7 @@
 #pragma once
 
 #include <memory>
+#include <mutex>
 #include <iterator>
 #include <utility>
 #include <perfetto.h>
@@ -39,35 +40,84 @@
 namespace cheri
 {
 
+using cpu_mode_type = perfetto::protos::pbzero::QEMULogEntryModeSwitch;
+/* (pid, tid, cid, mode) */
+using qemu_ctx_id = std::tuple<uint64_t, uint64_t, uint64_t, cpu_mode_type>;
+
 /*
- * Per-context data.
- * This is allocated and switched together with the context track.
+ * Custom track representing a QEMU guest context.
  */
-struct qemu_context_data {
-    qemu_stats stats;
-};
-
 struct qemu_context_track : public perfetto::Track {
-
-    using cpu_mode_type = perfetto::protos::pbzero::QEMULogEntryModeSwitch;
-    /* (pid, tid, cid, mode) */
-    using qemu_ctx_id = std::tuple<uint64_t, uint64_t, uint64_t, cpu_mode_type>;
-
     const uint64_t pid;
     const uint64_t tid;
     const uint64_t cid;
     const cpu_mode_type mode;
-    qemu_context_data ctx_data;
-
-    static std::shared_ptr<qemu_context_track>
-    make_context_track(qemu_ctx_id key);
 
     qemu_ctx_id get_id() const;
+    perfetto::protos::gen::TrackDescriptor Serialize() const;
+    void Serialize(perfetto::protos::pbzero::TrackDescriptor *desc) const;
+    qemu_context_track(qemu_ctx_id key);
+};
+
+/*
+ * Custom track representing a QEMU vcpu.
+ */
+struct qemu_cpu_track : public perfetto::Track {
+    const int cpu_id;
 
     perfetto::protos::gen::TrackDescriptor Serialize() const;
+    void Serialize(perfetto::protos::pbzero::TrackDescriptor *desc) const;
+    qemu_cpu_track(int id);
+};
 
-    qemu_context_track(qemu_ctx_id key);
-    ~qemu_context_track();
+/*
+ * Common CPU and context data.
+ */
+struct qemu_tracker_state {
+    qemu_stats stats;
+
+    virtual perfetto::Track *get_track() = 0;
+    virtual ~qemu_tracker_state() = default;
+};
+
+struct qemu_context_state : public qemu_tracker_state {
+    qemu_context_track track;
+
+    qemu_context_state(qemu_ctx_id id) : track(id)
+    {
+        auto desc = track.Serialize();
+        perfetto::TrackEvent::SetTrackDescriptor(track, desc);
+    }
+
+    ~qemu_context_state()
+    {
+        perfetto::TrackEvent::EraseTrackDescriptor(track);
+    }
+
+    perfetto::Track *get_track()
+    {
+        return &track;
+    }
+};
+
+struct qemu_fallback_state : public qemu_tracker_state {
+    qemu_cpu_track track;
+
+    qemu_fallback_state(int id) : track(id)
+    {
+        auto desc = track.Serialize();
+        perfetto::TrackEvent::SetTrackDescriptor(track, desc);
+    }
+
+    ~qemu_fallback_state()
+    {
+        perfetto::TrackEvent::EraseTrackDescriptor(track);
+    }
+
+    perfetto::Track *get_track()
+    {
+        return &track;
+    }
 };
 
 /*
@@ -95,21 +145,18 @@ struct qemu_context_track : public perfetto::Track {
  */
 class guest_context_tracker
 {
-    /* CPU track for events not assigned (or assignable) to a single context */
-    perfetto::Track cpu_track_;
-    /* context data for the CPU track */
-    qemu_context_data cpu_ctx_data_;
-    /* Currently active context track */
-    std::shared_ptr<qemu_context_track> ctx_track_;
+    /* CPU tracks and fallback data */
+    qemu_fallback_state cpu_state;
+    /* Currently active context data */
+    std::shared_ptr<qemu_context_state> ctx_state;
 
   public:
     guest_context_tracker(int cpu_id);
     void context_update(const log_event_ctx_update_t *evt);
     void mode_update(perfetto::protos::pbzero::QEMULogEntryModeSwitch new_mode);
     void flush_all_ctx_data();
-    perfetto::Track &get_cpu_track();
-    perfetto::Track &get_ctx_track();
-    qemu_context_data &get_ctx_data();
+    qemu_fallback_state *get_cpu_state();
+    qemu_tracker_state *get_ctx_state();
 };
 
 /*
diff --git a/trace_extra/meson.build b/trace_extra/meson.build
index e63051a7496..b391ddca81b 100644
--- a/trace_extra/meson.build
+++ b/trace_extra/meson.build
@@ -5,8 +5,8 @@ qemutrace_ss = ss.source_set()
 if config_all.has_key('CONFIG_TCG_LOG_INSTR') and config_all.has_key('CONFIG_TRACE_PERFETTO')
    qemutrace_ss.add(dependency('boost', modules: ['filesystem', 'system']))
    qemutrace_ss.add(files('trace_perfetto.cc', 'trace_stats.cc',
-                                'guest_context_tracker.cc',
-                                'cheri-perfetto/sdk/perfetto.cc'))
+                          'guest_context_tracker.cc', 'trace_counters.cc',
+                          'cheri-perfetto/sdk/perfetto.cc'))
 endif
 
 qemutrace_ss = qemutrace_ss.apply(config_all, strict: false)
diff --git a/trace_extra/trace_counters.cc b/trace_extra/trace_counters.cc
new file mode 100644
index 00000000000..4e094ed7e03
--- /dev/null
+++ b/trace_extra/trace_counters.cc
@@ -0,0 +1,163 @@
+/*-
+ * Copyright (c) 2022 Alfredo Mazzinghi
+ * All rights reserved.
+ *
+ * This software was developed by SRI International and the University of
+ * Cambridge Computer Laboratory (Department of Computer Science and
+ * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the
+ * DARPA SSITH research programme.
+ *
+ * @BERI_LICENSE_HEADER_START@
+ *
+ * Licensed to BERI Open Systems C.I.C. (BERI) under one or more contributor
+ * license agreements.  See the NOTICE file distributed with this work for
+ * additional information regarding copyright ownership.  BERI licenses this
+ * file to you under the BERI Hardware-Software License, Version 1.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at:
+ *
+ *   http://www.beri-open-systems.org/legal/license-1-0.txt
+ *
+ * Unless required by applicable law or agreed to in writing, Work distributed
+ * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ * CONDITIONS OF ANY KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ * @BERI_LICENSE_HEADER_END@
+ */
+
+#include <cstdio>
+#include "trace_extra/trace_counters.hh"
+#include "trace_extra/guest_context_tracker.hh"
+
+namespace cheri
+{
+
+namespace
+{
+/*
+ * Global counter state.
+ */
+qemu_counters_state global_counters;
+
+} // namespace
+
+qemu_counter_track::qemu_counter_track(uint64_t uuid_, qemu_counter_id id)
+        : perfetto::Track(uuid_, perfetto::Track()), name(std::get<0>(id)),
+          slot(std::get<1>(id))
+{
+}
+
+perfetto::protos::gen::TrackDescriptor qemu_counter_track::Serialize() const
+{
+    auto desc = Track::Serialize();
+    /* TODO process custom trace descriptor in trace processor */
+    desc.set_name(name + ":" + std::to_string(slot));
+    auto *counter_desc = desc.mutable_counter();
+    /* TODO fill counter descriptor with optional data */
+    return desc;
+}
+
+void qemu_counter_track::Serialize(
+    perfetto::protos::pbzero::TrackDescriptor *desc) const
+{
+    auto bytes = Serialize().SerializeAsString();
+    desc->AppendRawProtoBytes(bytes.data(), bytes.size());
+}
+
+qemu_counter::qemu_counter(qemu_counter_id id, int64_t reset)
+        : track(gen_track_uuid(), id), value(reset)
+{
+    auto desc = track.Serialize();
+    perfetto::TrackEvent::SetTrackDescriptor(track, desc);
+}
+
+qemu_counter::~qemu_counter()
+{
+    perfetto::TrackEvent::EraseTrackDescriptor(track);
+}
+
+void qemu_counter::emit(int64_t sample) const
+{
+    PERFETTO_INTERNAL_TRACK_EVENT(
+        "counter", /*name=*/nullptr,
+        perfetto::protos::pbzero::TrackEvent::TYPE_COUNTER, track,
+        [&](perfetto::EventContext ctx) {
+            auto *event = ctx.event();
+            event->set_counter_value(value);
+        });
+}
+
+bool qemu_counters_state::try_set(qemu_counter_id id, int64_t value)
+{
+    std::shared_lock<std::shared_timed_mutex> lock(mutex);
+    auto it = counters.find(id);
+    if (it != counters.end()) {
+        it->second->value = value;
+        it->second->emit(value);
+        return false;
+    }
+    return true;
+}
+
+boost::optional<int64_t>
+qemu_counters_state::try_fetch_add(qemu_counter_id id, int64_t value)
+{
+    std::shared_lock<std::shared_timed_mutex> lock(mutex);
+    auto it = counters.find(id);
+    if (it != counters.end()) {
+        value = it->second->value.fetch_add(value);
+        it->second->emit(value);
+        return value;
+    }
+    return boost::none;
+}
+
+void qemu_counters_state::set(qemu_counter_id id, int64_t value)
+{
+    if (try_set(id, value)) {
+        /* Upgrade lock for insert */
+        std::unique_lock<std::shared_timed_mutex> lock(mutex);
+        auto it = counters.find(id);
+        if (it == counters.end()) {
+            auto counter = std::make_unique<qemu_counter>(id, value);
+            auto emplaced = counters.emplace(id, std::move(counter));
+            it = emplaced.first;
+        } else {
+            it->second->value = value;
+        }
+        it->second->emit(value);
+    }
+}
+
+int64_t qemu_counters_state::inc(qemu_counter_id id, int64_t value)
+{
+    auto result = try_fetch_add(id, value);
+    if (!result) {
+        /* Upgrade lock for insert */
+        std::unique_lock<std::shared_timed_mutex> lock(mutex);
+        auto it = counters.find(id);
+        if (it == counters.end()) {
+            auto counter = std::make_unique<qemu_counter>(id, value);
+            auto emplaced = counters.emplace(id, std::move(counter));
+            it = emplaced.first;
+        } else {
+            value = it->second->value.fetch_add(value);
+        }
+        it->second->emit(value);
+        return value;
+    }
+    return result.value();
+}
+
+void global_counter_inc_emit(qemu_counter_id id, int64_t value)
+{
+    global_counters.inc(id, value);
+}
+
+void global_counter_set_emit(qemu_counter_id id, int64_t value)
+{
+    global_counters.set(id, value);
+}
+
+} // namespace cheri
diff --git a/trace_extra/trace_counters.hh b/trace_extra/trace_counters.hh
new file mode 100644
index 00000000000..ab3bd6c4537
--- /dev/null
+++ b/trace_extra/trace_counters.hh
@@ -0,0 +1,112 @@
+/*-
+ * Copyright (c) 2022 Alfredo Mazzinghi
+ * All rights reserved.
+ *
+ * This software was developed by SRI International and the University of
+ * Cambridge Computer Laboratory (Department of Computer Science and
+ * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the
+ * DARPA SSITH research programme.
+ *
+ * @BERI_LICENSE_HEADER_START@
+ *
+ * Licensed to BERI Open Systems C.I.C. (BERI) under one or more contributor
+ * license agreements.  See the NOTICE file distributed with this work for
+ * additional information regarding copyright ownership.  BERI licenses this
+ * file to you under the BERI Hardware-Software License, Version 1.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at:
+ *
+ *   http://www.beri-open-systems.org/legal/license-1-0.txt
+ *
+ * Unless required by applicable law or agreed to in writing, Work distributed
+ * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ * CONDITIONS OF ANY KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ * @BERI_LICENSE_HEADER_END@
+ */
+
+#pragma once
+
+#include <atomic>
+#include <shared_mutex>
+#include <memory>
+#include <unordered_map>
+#include <utility>
+#include <string>
+#include <perfetto.h>
+#include <boost/optional.hpp>
+#include <boost/utility/string_view.hpp>
+
+#include "trace_extra/tuple_index.hh"
+
+namespace cheri
+{
+
+/* (counter_name, counter_slot) */
+using qemu_counter_id = std::tuple<std::string, unsigned int>;
+
+/*
+ * Custom track representing a dynamically allocated counter.
+ * This is very similar to perfetto::CounterTrack but sadly it can not
+ * be subclassed cleanly, does not support dynamic naming and does not yet
+ * support incremental counters.
+ */
+class qemu_counter_track : public perfetto::Track
+{
+    const std::string name;
+    const unsigned int slot;
+
+  public:
+    void Serialize(perfetto::protos::pbzero::TrackDescriptor *) const;
+    perfetto::protos::gen::TrackDescriptor Serialize() const;
+    qemu_counter_track(uint64_t uuid_, qemu_counter_id id);
+};
+
+struct qemu_counter {
+    qemu_counter_track track;
+    std::atomic<int64_t> value;
+
+    void emit(int64_t sample) const;
+    qemu_counter(qemu_counter_id id, int64_t reset);
+    qemu_counter(const qemu_counter &other) = delete;
+    ~qemu_counter();
+};
+
+/*
+ * Manage blocks of named counters.
+ * This can be reused at multiple levels: there is a global set of counters
+ * for events that are common to the system, a per-cpu set of counters that
+ * emit on the CPU tracks and a per-context set of counters that are context
+ * switched by the context manager and emit on the per-context counter tracks.
+ */
+class qemu_counters_state
+{
+    /* The lock only protects find, insert, remove. Value updates are done
+     * with atomics, so we do not need to get an exclusive lock to update
+     * counter values.
+     */
+    std::shared_timed_mutex mutex;
+    std::unordered_map<qemu_counter_id, std::unique_ptr<qemu_counter>,
+                       tuple_hasher<qemu_counter_id>>
+        counters;
+
+    bool try_set(qemu_counter_id id, int64_t value);
+    boost::optional<int64_t> try_fetch_add(qemu_counter_id id, int64_t value);
+
+  public:
+    qemu_counters_state() {}
+
+    /* Set counter value or create new counter, emit the counter value */
+    void set(qemu_counter_id id, int64_t value);
+    /* Increment counter value or create new counter, emit the counter value */
+    int64_t inc(qemu_counter_id id, int64_t value);
+};
+
+/* Increment counter and emit value to trace, for incremental counter handling
+ */
+void global_counter_inc_emit(qemu_counter_id id, int64_t value);
+/* Set counter value and emit, for absolute counter handling */
+void global_counter_set_emit(qemu_counter_id id, int64_t value);
+
+} // namespace cheri
diff --git a/trace_extra/trace_perfetto.cc b/trace_extra/trace_perfetto.cc
index 35d83f0b6cf..503f4cc7816 100644
--- a/trace_extra/trace_perfetto.cc
+++ b/trace_extra/trace_perfetto.cc
@@ -51,6 +51,7 @@
 #include "exec/log_instr_internal.h"
 #include "exec/log_instr_perfetto.h"
 #include "trace_extra/trace_stats.hh"
+#include "trace_extra/trace_counters.hh"
 #include "trace_extra/guest_context_tracker.hh"
 
 PERFETTO_TRACK_EVENT_STATIC_STORAGE();
@@ -59,6 +60,7 @@ namespace fs = boost::filesystem;
 
 namespace
 {
+using namespace cheri;
 
 /* Tracing session pointer */
 std::unique_ptr<perfetto::TracingSession> session;
@@ -77,26 +79,26 @@ std::unique_ptr<perfetto::Track> sched_track;
  */
 struct perfetto_backend_data {
     // Per-CPU control track. This records tracing control events.
-    perfetto::Track ctrl_track_;
+    perfetto::Track ctrl_track;
     // Tracker that resolves the current context scheduled on a CPU
-    cheri::guest_context_tracker ctx_tracker_;
+    cheri::guest_context_tracker ctx_tracker;
     /*
      * Cached copy of the logging activation status.
      * It is easier to keep a cached copy here than exposing the loglevel_active
      * flag to the C++ interface.
      */
-    bool loglevel_active_;
+    bool loglevel_active;
 
     perfetto_backend_data(int cpu_id)
-        : ctrl_track_(perfetto::Track::Global(cheri::gen_track_uuid())),
-          ctx_tracker_(cpu_id)
+        : ctrl_track(perfetto::Track::Global(cheri::gen_track_uuid())),
+          ctx_tracker(cpu_id)
 
     {
         std::string track_name("CPU " + std::to_string(cpu_id) + " ctrl");
 
-        auto desc = ctrl_track_.Serialize();
+        auto desc = ctrl_track.Serialize();
         desc.set_name(track_name);
-        perfetto::TrackEvent::SetTrackDescriptor(ctrl_track_, desc);
+        perfetto::TrackEvent::SetTrackDescriptor(ctrl_track, desc);
     }
 };
 
@@ -144,6 +146,11 @@ bool perfetto_start_tracing(void)
     session->Setup(cfg);
     session->StartBlocking();
 
+    /*
+     * XXX there something wrong with initialization as we still emit a QEMU
+     * process and thread tracks. Would be nice to avoid that.
+     */
+
     perfetto::ProcessTrack qemu_proc = perfetto::ProcessTrack::Current();
     auto desc = qemu_proc.Serialize();
     desc.mutable_process()->set_process_name("qemu");
@@ -176,12 +183,88 @@ void trace_cap_register(perfetto::protos::pbzero::QEMULogEntryCapability *cap,
     cap->set_otype(perfetto_cap_otype(chandle));
 }
 
+/*
+ * Handle tracing control event. Returns true if the event changes the trace
+ * enable state (i.e. start or stop).
+ */
+bool process_state_event(perfetto_backend_data *data, log_event_t *evt)
+{
+    bool has_startstop_event = false;
+    auto *state = data->ctx_tracker.get_ctx_state();
+
+    switch (evt->state.next_state) {
+    case LOG_EVENT_STATE_FLUSH:
+        TRACE_EVENT_INSTANT("ctrl", "flush", data->ctrl_track);
+        data->ctx_tracker.flush_all_ctx_data();
+        perfetto::TrackEvent::Flush();
+        break;
+    case LOG_EVENT_STATE_START:
+        state->stats.unpause(*state->get_track(), evt->state.pc);
+        TRACE_EVENT_BEGIN("ctrl", "tracing", data->ctrl_track);
+        has_startstop_event = true;
+        break;
+    case LOG_EVENT_STATE_STOP:
+        state->stats.pause(*state->get_track(), evt->state.pc);
+        TRACE_EVENT_END("ctrl", data->ctrl_track);
+        has_startstop_event = true;
+        break;
+    default:
+        assert(false && "Invalid state event");
+    }
+    return has_startstop_event;
+}
+
+/*
+ * Handle context switch events
+ */
+void process_context_event(perfetto_backend_data *data,
+                           cpu_log_entry_handle entry, log_event_t *evt)
+{
+    auto *state = data->ctx_tracker.get_ctx_state();
+
+    /* Swap current context. */
+    if (evt->ctx_update.op == LOG_EVENT_CTX_OP_SWITCH) {
+        state->stats.pause(*state->get_track(), perfetto_log_entry_pc(entry));
+        data->ctx_tracker.context_update(&evt->ctx_update);
+        /* Reload data and track as context_update will have changed them */
+        state = data->ctx_tracker.get_ctx_state();
+        state->stats.unpause(*state->get_track(), perfetto_log_entry_pc(entry));
+    }
+}
+
+/*
+ * Handle trace markers.
+ * These are always emitted on the CPU tracks.
+ */
+void process_marker_event(perfetto_backend_data *data, log_event_t *evt)
+{
+    auto *cpu_state = data->ctx_tracker.get_cpu_state();
+
+    TRACE_EVENT_INSTANT("marker", "guest", *cpu_state->get_track(),
+                        [&](perfetto::EventContext ctx) {
+                            auto *qemu_arg = ctx.event()->set_qemu();
+                            qemu_arg->set_marker(evt->marker);
+                        });
+}
+
+/*
+ * Handle counters.
+ */
+void process_counter_event(perfetto_backend_data *data, log_event_t *evt)
+{
+    qemu_counter_id cnt_id = std::make_tuple(
+        evt->counter.name, log_event_counter_slot(evt->counter.flags));
+    if (log_event_counter_incremental(evt->counter.flags)) {
+        global_counter_inc_emit(cnt_id, evt->counter.value);
+    } else {
+        global_counter_set_emit(cnt_id, evt->counter.value);
+    }
+}
+
 void process_events(perfetto_backend_data *data, cpu_log_entry_handle entry)
 {
     int nevents = perfetto_log_entry_events(entry);
-    bool have_startstop_event = false;
-    auto *ctx_data = &data->ctx_tracker_.get_ctx_data();
-    auto *ctx_track = &data->ctx_tracker_.get_ctx_track();
+    bool has_startstop_event = false;
 
     /*
      * Note: LOG_EVENT_STATE events are emitted even when tracing is disabled.
@@ -190,65 +273,37 @@ void process_events(perfetto_backend_data *data, cpu_log_entry_handle entry)
     for (int i = 0; i < nevents; i++) {
         log_event_t *evt = perfetto_log_event(entry, i);
         switch (evt->id) {
-        case LOG_EVENT_STATE: {
-            switch (evt->state.next_state) {
-            case LOG_EVENT_STATE_FLUSH:
-                TRACE_EVENT_INSTANT("ctrl", "flush", data->ctrl_track_);
-                data->ctx_tracker_.flush_all_ctx_data();
-                perfetto::TrackEvent::Flush();
-                break;
-            case LOG_EVENT_STATE_START:
-                ctx_data->stats.unpause(*ctx_track, evt->state.pc);
-                TRACE_EVENT_BEGIN("ctrl", "tracing", data->ctrl_track_);
-                have_startstop_event = true;
-                break;
-            case LOG_EVENT_STATE_STOP:
-                ctx_data->stats.pause(*ctx_track, evt->state.pc);
-                TRACE_EVENT_END("ctrl", data->ctrl_track_);
-                have_startstop_event = true;
-                break;
-            default:
-                assert(false && "Invalid state event");
-            }
-        } break;
-        case LOG_EVENT_CTX_UPDATE: {
-            /* Swap current context. */
-            if (evt->ctx_update.op == LOG_EVENT_CTX_OP_SWITCH) {
-                ctx_data->stats.pause(*ctx_track, perfetto_log_entry_pc(entry));
-                data->ctx_tracker_.context_update(&evt->ctx_update);
-                /* Reload data and track as context_update will have changed
-                 * them */
-                ctx_data = &data->ctx_tracker_.get_ctx_data();
-                ctx_track = &data->ctx_tracker_.get_ctx_track();
-                ctx_data->stats.unpause(*ctx_track,
-                                        perfetto_log_entry_pc(entry));
-            }
-        } break;
-        case LOG_EVENT_MARKER: {
-            auto cpu_track = data->ctx_tracker_.get_cpu_track();
-            TRACE_EVENT_INSTANT("marker", "guest", cpu_track,
-                                [&](perfetto::EventContext ctx) {
-                                    auto *qemu_arg = ctx.event()->set_qemu();
-                                    qemu_arg->set_marker(evt->marker);
-                                });
-        } break;
+        case LOG_EVENT_STATE:
+            has_startstop_event = process_state_event(data, evt);
+            break;
+        case LOG_EVENT_CTX_UPDATE:
+            process_context_event(data, entry, evt);
+            break;
+        case LOG_EVENT_MARKER:
+            process_marker_event(data, evt);
+            break;
         case LOG_EVENT_REGDUMP:
             break;
+        case LOG_EVENT_COUNTER:
+            process_counter_event(data, evt);
+            break;
         default:
             assert(false && "Invalid event identifier");
         }
     }
+    auto *state = data->ctx_tracker.get_ctx_state();
+
     if (perfetto_log_entry_flags(entry) & LI_FLAG_MODE_SWITCH) {
         auto mode = cheri::qemu_cpu_mode_to_trace(
             perfetto_log_entry_next_cpu_mode(entry));
         // XXX-AM: consider making the mode switch an event, possibly with its
         // own separate pc signaling the PC of the next instruction we are
         // landing to?
-        ctx_data->stats.pause(*ctx_track, perfetto_log_entry_pc(entry));
-        data->ctx_tracker_.mode_update(mode);
-        ctx_data = &data->ctx_tracker_.get_ctx_data();
-        ctx_track = &data->ctx_tracker_.get_ctx_track();
-        ctx_data->stats.unpause(*ctx_track, perfetto_log_entry_pc(entry));
+        state->stats.pause(*state->get_track(), perfetto_log_entry_pc(entry));
+        data->ctx_tracker.mode_update(mode);
+        /* reload possibly updated state and track */
+        state = data->ctx_tracker.get_ctx_state();
+        state->stats.unpause(*state->get_track(), perfetto_log_entry_pc(entry));
     }
 
     /*
@@ -257,9 +312,9 @@ void process_events(perfetto_backend_data *data, cpu_log_entry_handle entry)
      * histograms. This avoids having to keep a shadow copy of the tracing state
      * in the backend.
      */
-    if (!have_startstop_event &&
+    if (!has_startstop_event &&
         (perfetto_log_entry_flags(entry) & LI_FLAG_HAS_INSTR_DATA) != 0)
-        ctx_data->stats.process_instr(*ctx_track, entry);
+        state->stats.process_instr(*state->get_track(), entry);
 }
 
 void process_instr(perfetto_backend_data *data, cpu_log_entry_handle entry)
@@ -270,8 +325,9 @@ void process_instr(perfetto_backend_data *data, cpu_log_entry_handle entry)
      * same track/category: e.g. mode swtich, interrupt information and modified
      * registers?
      */
+    auto *state = data->ctx_tracker.get_ctx_state();
     TRACE_EVENT_INSTANT(
-        "instructions", "stream", data->ctx_tracker_.get_ctx_track(),
+        "instructions", "stream", *state->get_track(),
         [&](perfetto::EventContext ctx) {
             auto *qemu_arg = ctx.event()->set_qemu();
             auto *instr = qemu_arg->set_instr();

From 692ea29a3104c643fcf21ac8547ebd2d3d2935bf Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Tue, 12 Jul 2022 13:14:59 +0100
Subject: [PATCH 50/74] Add per-context counters.

These are still not hooked to the guest NOPs.
---
 trace_extra/guest_context_tracker.hh |  7 +++++--
 trace_extra/trace_counters.cc        | 22 +++++++++++++---------
 trace_extra/trace_counters.hh        | 12 ++++++++----
 trace_extra/trace_perfetto.cc        |  3 ++-
 4 files changed, 28 insertions(+), 16 deletions(-)

diff --git a/trace_extra/guest_context_tracker.hh b/trace_extra/guest_context_tracker.hh
index 9169693da2c..92b5d6ae0e3 100644
--- a/trace_extra/guest_context_tracker.hh
+++ b/trace_extra/guest_context_tracker.hh
@@ -36,6 +36,7 @@
 #include "qemu/log_instr.h"
 #include "exec/log_instr_perfetto.h"
 #include "trace_extra/trace_stats.hh"
+#include "trace_extra/trace_counters.hh"
 
 namespace cheri
 {
@@ -82,8 +83,9 @@ struct qemu_tracker_state {
 
 struct qemu_context_state : public qemu_tracker_state {
     qemu_context_track track;
+    qemu_counters_state counters;
 
-    qemu_context_state(qemu_ctx_id id) : track(id)
+    qemu_context_state(qemu_ctx_id id) : track(id), counters(track)
     {
         auto desc = track.Serialize();
         perfetto::TrackEvent::SetTrackDescriptor(track, desc);
@@ -102,8 +104,9 @@ struct qemu_context_state : public qemu_tracker_state {
 
 struct qemu_fallback_state : public qemu_tracker_state {
     qemu_cpu_track track;
+    qemu_counters_state counters;
 
-    qemu_fallback_state(int id) : track(id)
+    qemu_fallback_state(int id) : track(id), counters(track)
     {
         auto desc = track.Serialize();
         perfetto::TrackEvent::SetTrackDescriptor(track, desc);
diff --git a/trace_extra/trace_counters.cc b/trace_extra/trace_counters.cc
index 4e094ed7e03..9d830375373 100644
--- a/trace_extra/trace_counters.cc
+++ b/trace_extra/trace_counters.cc
@@ -42,9 +42,10 @@ qemu_counters_state global_counters;
 
 } // namespace
 
-qemu_counter_track::qemu_counter_track(uint64_t uuid_, qemu_counter_id id)
-        : perfetto::Track(uuid_, perfetto::Track()), name(std::get<0>(id)),
-          slot(std::get<1>(id))
+qemu_counter_track::qemu_counter_track(uint64_t uuid_, qemu_counter_id id,
+                                       perfetto::Track parent)
+    : perfetto::Track(uuid_, parent), name(std::get<0>(id)),
+      slot(std::get<1>(id))
 {
 }
 
@@ -65,8 +66,9 @@ void qemu_counter_track::Serialize(
     desc->AppendRawProtoBytes(bytes.data(), bytes.size());
 }
 
-qemu_counter::qemu_counter(qemu_counter_id id, int64_t reset)
-        : track(gen_track_uuid(), id), value(reset)
+qemu_counter::qemu_counter(qemu_counter_id id, int64_t reset,
+                           perfetto::Track parent)
+    : track(gen_track_uuid(), id, parent), value(reset)
 {
     auto desc = track.Serialize();
     perfetto::TrackEvent::SetTrackDescriptor(track, desc);
@@ -100,8 +102,8 @@ bool qemu_counters_state::try_set(qemu_counter_id id, int64_t value)
     return true;
 }
 
-boost::optional<int64_t>
-qemu_counters_state::try_fetch_add(qemu_counter_id id, int64_t value)
+boost::optional<int64_t> qemu_counters_state::try_fetch_add(qemu_counter_id id,
+                                                            int64_t value)
 {
     std::shared_lock<std::shared_timed_mutex> lock(mutex);
     auto it = counters.find(id);
@@ -120,7 +122,8 @@ void qemu_counters_state::set(qemu_counter_id id, int64_t value)
         std::unique_lock<std::shared_timed_mutex> lock(mutex);
         auto it = counters.find(id);
         if (it == counters.end()) {
-            auto counter = std::make_unique<qemu_counter>(id, value);
+            auto counter =
+                std::make_unique<qemu_counter>(id, value, parent_track);
             auto emplaced = counters.emplace(id, std::move(counter));
             it = emplaced.first;
         } else {
@@ -138,7 +141,8 @@ int64_t qemu_counters_state::inc(qemu_counter_id id, int64_t value)
         std::unique_lock<std::shared_timed_mutex> lock(mutex);
         auto it = counters.find(id);
         if (it == counters.end()) {
-            auto counter = std::make_unique<qemu_counter>(id, value);
+            auto counter =
+                std::make_unique<qemu_counter>(id, value, parent_track);
             auto emplaced = counters.emplace(id, std::move(counter));
             it = emplaced.first;
         } else {
diff --git a/trace_extra/trace_counters.hh b/trace_extra/trace_counters.hh
index ab3bd6c4537..50d9a27ac9a 100644
--- a/trace_extra/trace_counters.hh
+++ b/trace_extra/trace_counters.hh
@@ -60,7 +60,8 @@ class qemu_counter_track : public perfetto::Track
   public:
     void Serialize(perfetto::protos::pbzero::TrackDescriptor *) const;
     perfetto::protos::gen::TrackDescriptor Serialize() const;
-    qemu_counter_track(uint64_t uuid_, qemu_counter_id id);
+    qemu_counter_track(uint64_t uuid_, qemu_counter_id id,
+                       perfetto::Track parent);
 };
 
 struct qemu_counter {
@@ -68,7 +69,7 @@ struct qemu_counter {
     std::atomic<int64_t> value;
 
     void emit(int64_t sample) const;
-    qemu_counter(qemu_counter_id id, int64_t reset);
+    qemu_counter(qemu_counter_id id, int64_t reset, perfetto::Track parent);
     qemu_counter(const qemu_counter &other) = delete;
     ~qemu_counter();
 };
@@ -82,10 +83,12 @@ struct qemu_counter {
  */
 class qemu_counters_state
 {
-    /* The lock only protects find, insert, remove. Value updates are done
+    /*
+     * The lock only protects find, insert, remove. Value updates are done
      * with atomics, so we do not need to get an exclusive lock to update
      * counter values.
      */
+    perfetto::Track parent_track;
     std::shared_timed_mutex mutex;
     std::unordered_map<qemu_counter_id, std::unique_ptr<qemu_counter>,
                        tuple_hasher<qemu_counter_id>>
@@ -95,7 +98,8 @@ class qemu_counters_state
     boost::optional<int64_t> try_fetch_add(qemu_counter_id id, int64_t value);
 
   public:
-    qemu_counters_state() {}
+    qemu_counters_state() : parent_track(perfetto::Track()) {}
+    qemu_counters_state(perfetto::Track track) : parent_track(track) {}
 
     /* Set counter value or create new counter, emit the counter value */
     void set(qemu_counter_id id, int64_t value);
diff --git a/trace_extra/trace_perfetto.cc b/trace_extra/trace_perfetto.cc
index 503f4cc7816..49c293f65ab 100644
--- a/trace_extra/trace_perfetto.cc
+++ b/trace_extra/trace_perfetto.cc
@@ -319,13 +319,14 @@ void process_events(perfetto_backend_data *data, cpu_log_entry_handle entry)
 
 void process_instr(perfetto_backend_data *data, cpu_log_entry_handle entry)
 {
+    auto *state = data->ctx_tracker.get_ctx_state();
+
     /*
      * XXX-AM: instead of having one big instruction record, we may have
      * different messages for optional parts of the instruction message, on the
      * same track/category: e.g. mode swtich, interrupt information and modified
      * registers?
      */
-    auto *state = data->ctx_tracker.get_ctx_state();
     TRACE_EVENT_INSTANT(
         "instructions", "stream", *state->get_track(),
         [&](perfetto::EventContext ctx) {

From ec20ea40278c62d60e29eb7c946b41fdc06060de Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Tue, 12 Jul 2022 13:03:59 +0100
Subject: [PATCH 51/74] Point cheri-perfetto submodule to the correct
 repository.

---
 .gitmodules                | 2 ++
 trace_extra/cheri-perfetto | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/.gitmodules b/.gitmodules
index f49c13bb970..cc00260ba11 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -68,3 +68,5 @@
 	path = trace_extra/cheri-perfetto
 	url = https://github.com/qwattash/cheri-perfetto.git
 	branch = cheri-perfetto
+[submodule "cheri-perfetto"]
+	url = https://github.com/CTSRD-CHERI/cheri-perfetto.git
diff --git a/trace_extra/cheri-perfetto b/trace_extra/cheri-perfetto
index 2bbdf5a6ccd..3415bc5a15a 160000
--- a/trace_extra/cheri-perfetto
+++ b/trace_extra/cheri-perfetto
@@ -1 +1 @@
-Subproject commit 2bbdf5a6ccddbee6df68dbd16c7c6e136bddd6d6
+Subproject commit 3415bc5a15a1231a005739340be643e533bcf754

From 1cdf7bce397011c2d4dc6d51539067b9d6e3061e Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Tue, 12 Jul 2022 13:15:55 +0100
Subject: [PATCH 52/74] Introduce interval tracks support in the perfetto
 tracing backend.

Replace the stats implementation with a new one based on the
cheri-perfetto interval tracks.
The stats implementation tracks basic block icount and hits, along
with branches. Now there are multiple categories for each different
statistic instead of a grouped one.
---
 include/exec/log_instr_perfetto.h    |   6 +-
 trace_extra/guest_context_tracker.cc | 130 ++++++++++++++++++++---
 trace_extra/guest_context_tracker.hh | 125 ++++++++++++++++++----
 trace_extra/meson.build              |   3 +-
 trace_extra/trace_counters.cc        |   1 -
 trace_extra/trace_perfetto.cc        |  38 +++----
 trace_extra/trace_stats.cc           | 150 ---------------------------
 trace_extra/trace_stats.hh           |  86 ---------------
 8 files changed, 243 insertions(+), 296 deletions(-)
 delete mode 100644 trace_extra/trace_stats.cc
 delete mode 100644 trace_extra/trace_stats.hh

diff --git a/include/exec/log_instr_perfetto.h b/include/exec/log_instr_perfetto.h
index ee5a41dc26a..4cfef8cca6e 100644
--- a/include/exec/log_instr_perfetto.h
+++ b/include/exec/log_instr_perfetto.h
@@ -57,13 +57,15 @@ PERFETTO_DEFINE_CATEGORIES(
     perfetto::Category("instructions")
         .SetDescription("CPU instructions executed"),
     perfetto::Category("memory").SetDescription("Tag and memory load/store"),
-    perfetto::Category("stats").SetDescription("High-level statistics data"),
     perfetto::Category("ctrl").SetDescription("Tracing control events"),
     perfetto::Category("trap").SetDescription("CPU trap events"),
     perfetto::Category("sched").SetDescription("Scheduling events"),
     perfetto::Category("marker").SetDescription(
         "Guest trace timestamp markers"),
-    perfetto::Category("counter").SetDescription("Guest-driven counters"));
+    perfetto::Category("counter").SetDescription("Guest-driven counters"),
+    perfetto::Category("bb_hit").SetDescription("Basic Block hit counts"),
+    perfetto::Category("bb_icount").SetDescription("Basic Block instr counts"),
+    perfetto::Category("br_hit").SetDescription("Branch hit count"));
 #endif
 
 /*
diff --git a/trace_extra/guest_context_tracker.cc b/trace_extra/guest_context_tracker.cc
index cb590acd1b6..ce7019f2bc7 100644
--- a/trace_extra/guest_context_tracker.cc
+++ b/trace_extra/guest_context_tracker.cc
@@ -27,11 +27,18 @@
  */
 
 #include <atomic>
+#include <limits>
 #include <mutex>
+#include <random>
 #include <string>
+#include <utility>
 #include <unordered_map>
 #include "trace_extra/guest_context_tracker.hh"
 
+namespace icl = boost::icl;
+
+using addr_range = icl::interval<uint64_t>;
+
 namespace cheri
 {
 
@@ -47,9 +54,26 @@ std::unordered_map<qemu_ctx_id, std::shared_ptr<qemu_context_state>,
 /* Helper to generate unique IDs for dynamic tracks */
 unsigned long gen_track_uuid()
 {
-    // use perfetto::Uuid?
-    static std::atomic<unsigned long> next_track_id(0xdead);
-    return (next_track_id++);
+    static std::mutex rnd_mutex;
+    static std::random_device rand_dev;
+    static std::mt19937_64 rand_gen(rand_dev());
+    static std::uniform_int_distribution<unsigned long> track_uuid_gen(
+        0ul, std::numeric_limits<unsigned long>::max());
+
+    {
+        const std::lock_guard<std::mutex> lock(rnd_mutex);
+        return track_uuid_gen(rand_gen);
+    }
+}
+
+std::string ctx_track_name(qemu_ctx_id key)
+{
+    uint64_t pid = std::get<0>(key);
+    uint64_t tid = std::get<1>(key);
+    uint64_t cid = std::get<2>(key);
+    int mode = std::get<3>(key);
+    return "CTX " + std::to_string(pid) + ":" + std::to_string(tid) + ":" +
+           std::to_string(cid) + ":" + std::to_string(mode);
 }
 
 perfetto::protos::pbzero::QEMULogEntryModeSwitch
@@ -88,8 +112,7 @@ perfetto::protos::gen::TrackDescriptor qemu_context_track::Serialize() const
     cheri_desc->set_tid(tid);
     cheri_desc->set_cid(cid);
     cheri_desc->set_el(mode);
-    desc.set_name("CTX " + std::to_string(pid) + ":" + std::to_string(tid) +
-                  ":" + std::to_string(cid) + ":" + std::to_string(mode));
+    desc.set_name(name);
     return desc;
 }
 
@@ -103,7 +126,7 @@ void qemu_context_track::Serialize(
 qemu_context_track::qemu_context_track(qemu_ctx_id key)
     : perfetto::Track(gen_track_uuid(), perfetto::Track()),
       pid(std::get<0>(key)), tid(std::get<1>(key)), cid(std::get<2>(key)),
-      mode(std::get<3>(key))
+      mode(std::get<3>(key)), name(ctx_track_name(key))
 {
 }
 
@@ -112,9 +135,75 @@ qemu_ctx_id qemu_context_track::get_id() const
     return std::make_tuple(pid, tid, cid, mode);
 }
 
+void qemu_bb_tracker::track_next(cpu_log_entry_handle entry, uint64_t pc)
+{
+    if (pc == 0) {
+        pc = perfetto_log_entry_pc(entry);
+    }
+    int size = perfetto_log_entry_insn_size(entry);
+    /* When resuming after reset accept anything as a next instruction */
+    if (bb_start == 0) {
+        bb_start = pc;
+        bb_icount = 0;
+    } else if (pc != next_pc && next_pc != 0) {
+        /* Presume we are branching */
+        assert(bb_start != 0 && "PC bb start was not updated");
+        assert(last_pc != 0 && "last_pc can not be zero");
+        /* Update basic block info */
+        bb_map += std::make_pair(addr_range::right_open(bb_start, last_pc),
+                                 bb_value(1ul, bb_icount));
+        /* Update branch info */
+        branch_id bid = std::make_pair(last_pc, pc);
+        branch_map[bid] += 1;
+        /* Reset */
+        bb_start = pc;
+        bb_icount = 0;
+    }
+    bb_icount += 1;
+    next_pc = pc + size;
+    last_pc = pc;
+}
+
+void qemu_bb_tracker::reset(uint64_t pc)
+{
+    if (bb_start == 0) {
+        /* Nothing to flush */
+        return;
+    }
+    /* With pc=0 flush discarding any currently active bb */
+    if (pc != 0) {
+        bb_map += std::make_pair(addr_range::right_open(bb_start, pc),
+                                 bb_value(1ul, bb_icount));
+    }
+    bb_start = bb_icount = next_pc = last_pc = 0;
+
+    /* XXX-AM: Somewhat arbitrary tuning constant, depend on the SHM size */
+    if (boost::icl::interval_count(bb_map) > 5000 || branch_map.size() > 5000)
+        flush();
+}
+
+void qemu_bb_tracker::flush()
+{
+    for (const auto &keyval : bb_map) {
+        auto &interval = keyval.first;
+        TRACE_INTERVAL("bb_hit", track, interval.lower(), interval.upper(),
+                       keyval.second.hit);
+        TRACE_INTERVAL("bb_icount", track, interval.lower(), interval.upper(),
+                       keyval.second.icount);
+    }
+    for (const auto &keyval : branch_map) {
+        auto &bid = keyval.first;
+        TRACE_INTERVAL("br_hit", track, std::get<0>(bid), std::get<1>(bid),
+                       keyval.second);
+    }
+    bb_map.clear();
+    branch_map.clear();
+}
+
 guest_context_tracker::guest_context_tracker(int cpu_id) : cpu_state(cpu_id) {}
 
 void guest_context_tracker::mode_update(
+    cpu_log_entry_handle entry,
     perfetto::protos::pbzero::QEMULogEntryModeSwitch new_mode)
 {
     if (ctx_state == nullptr)
@@ -135,10 +224,15 @@ void guest_context_tracker::mode_update(
             state = it->second;
         }
     }
+    if (ctx_state && ctx_state != state) {
+        /* We are changing context */
+        ctx_state->bb_tracker->reset(perfetto_log_entry_pc(entry));
+    }
     ctx_state = state;
 }
 
-void guest_context_tracker::context_update(const log_event_ctx_update_t *evt)
+void guest_context_tracker::context_update(cpu_log_entry_handle entry,
+                                           const log_event_ctx_update_t *evt)
 {
     // Currently these are the only ones existing
     assert((evt->op == LOG_EVENT_CTX_OP_SWITCH) && "Invalid ctx update op");
@@ -166,17 +260,29 @@ void guest_context_tracker::context_update(const log_event_ctx_update_t *evt)
             state = it->second;
         }
     }
+    if (ctx_state && ctx_state != state) {
+        /* We are changing context */
+        ctx_state->bb_tracker->reset(perfetto_log_entry_pc(entry));
+    }
     ctx_state = state;
 }
 
-void guest_context_tracker::flush_all_ctx_data()
+void guest_context_tracker::flush_all_ctx_data(uint64_t pc)
 {
-    /* Flush per-CPU stats */
-    cpu_state.stats.flush(cpu_state.track);
+    /* Flush per-CPU and current context stats */
+    cpu_state.bb_tracker->reset(pc);
+    cpu_state.bb_tracker->flush();
+    if (ctx_state) {
+        ctx_state->bb_tracker->reset(pc);
+        ctx_state->bb_tracker->flush();
+    }
     std::lock_guard<std::mutex> track_state_guard(track_state_lock);
     for (auto id_and_state : track_state_map) {
-        qemu_stats &stats = id_and_state.second->stats;
-        stats.flush(id_and_state.second->track);
+        qemu_bb_tracker *bb_tracker = id_and_state.second->bb_tracker.get();
+        if (bb_tracker != ctx_state->bb_tracker.get()) {
+            bb_tracker->reset(0);
+            bb_tracker->flush();
+        }
     }
 }
 
diff --git a/trace_extra/guest_context_tracker.hh b/trace_extra/guest_context_tracker.hh
index 92b5d6ae0e3..665afd84091 100644
--- a/trace_extra/guest_context_tracker.hh
+++ b/trace_extra/guest_context_tracker.hh
@@ -31,11 +31,16 @@
 #include <memory>
 #include <mutex>
 #include <iterator>
+#include <string>
+#include <unordered_map>
 #include <utility>
 #include <perfetto.h>
 #include "qemu/log_instr.h"
 #include "exec/log_instr_perfetto.h"
-#include "trace_extra/trace_stats.hh"
+
+#include <boost/icl/interval_map.hpp>
+
+#include "tuple_index.hh"
 #include "trace_extra/trace_counters.hh"
 
 namespace cheri
@@ -45,6 +50,16 @@ using cpu_mode_type = perfetto::protos::pbzero::QEMULogEntryModeSwitch;
 /* (pid, tid, cid, mode) */
 using qemu_ctx_id = std::tuple<uint64_t, uint64_t, uint64_t, cpu_mode_type>;
 
+/*
+ * Generate a unique track ID.
+ * The ID is guaranteed to be unique within a qemu run, but not with
+ * respect to other data sources.
+ */
+unsigned long gen_track_uuid();
+
+perfetto::protos::pbzero::QEMULogEntryModeSwitch
+qemu_cpu_mode_to_trace(qemu_log_instr_cpu_mode_t mode);
+
 /*
  * Custom track representing a QEMU guest context.
  */
@@ -53,6 +68,7 @@ struct qemu_context_track : public perfetto::Track {
     const uint64_t tid;
     const uint64_t cid;
     const cpu_mode_type mode;
+    const std::string name;
 
     qemu_ctx_id get_id() const;
     perfetto::protos::gen::TrackDescriptor Serialize() const;
@@ -71,11 +87,85 @@ struct qemu_cpu_track : public perfetto::Track {
     qemu_cpu_track(int id);
 };
 
+/*
+ * Effective basic block execution tracker.
+ * This tracks the execution of ranges of code not interrupted by branches.
+ * Note that these are 'effective', in the sense that a measured range may
+ * contain branch instructions but the branch is never taken.
+ * Interrupts will cause spurious block detection.
+ */
+class qemu_bb_tracker
+{
+    using branch_id = std::tuple<uint64_t, uint64_t>;
+
+    struct bb_value {
+        uint64_t hit;
+        uint64_t icount;
+
+        bb_value() : hit(0), icount(0) {}
+        bb_value(uint64_t h, uint64_t i) : hit(h), icount(i) {}
+        bb_value(const bb_value &o) = default;
+
+        bb_value operator+(const bb_value &o) const
+        {
+            return bb_value(hit + o.hit, icount + o.icount);
+        }
+
+        bb_value &operator+=(const bb_value &o)
+        {
+            hit += o.hit;
+            icount += o.icount;
+            return *this;
+        }
+
+        bool operator==(const bb_value &o) const
+        {
+            return hit == o.hit && icount == o.icount;
+        }
+    };
+    /*
+     * Histogram of basic-block ranges with associated # hits and instruction
+     * count
+     */
+    boost::icl::interval_map<uint64_t, bb_value> bb_map;
+    /*
+     * Branch map (src, target) -> # hits
+     */
+    std::unordered_map<branch_id, uint64_t, tuple_hasher<branch_id>> branch_map;
+
+    /*
+     * Interval track on which we emit data.
+     */
+    perfetto::IntervalTrack track;
+    uint64_t last_pc;
+    uint64_t next_pc;
+    uint64_t bb_start;
+    uint64_t bb_icount;
+
+  public:
+    qemu_bb_tracker(perfetto::Track &context_track, const char *name)
+        : track(gen_track_uuid(), name, context_track), last_pc(0), next_pc(0),
+          bb_start(0)
+    {
+        auto desc = track.Serialize();
+        perfetto::TrackEvent::SetTrackDescriptor(track, desc);
+    }
+    virtual ~qemu_bb_tracker()
+    {
+        perfetto::TrackEvent::EraseTrackDescriptor(track);
+    }
+
+    void track_next(cpu_log_entry_handle entry, uint64_t pc = 0);
+    void reset(uint64_t pc);
+    void flush();
+};
+
 /*
  * Common CPU and context data.
  */
 struct qemu_tracker_state {
-    qemu_stats stats;
+    std::unique_ptr<qemu_bb_tracker> bb_tracker;
+    std::unique_ptr<qemu_counters_state> counters;
 
     virtual perfetto::Track *get_track() = 0;
     virtual ~qemu_tracker_state() = default;
@@ -83,12 +173,15 @@ struct qemu_tracker_state {
 
 struct qemu_context_state : public qemu_tracker_state {
     qemu_context_track track;
-    qemu_counters_state counters;
 
-    qemu_context_state(qemu_ctx_id id) : track(id), counters(track)
+    qemu_context_state(qemu_ctx_id id) : track(id)
     {
         auto desc = track.Serialize();
         perfetto::TrackEvent::SetTrackDescriptor(track, desc);
+
+        bb_tracker =
+            std::make_unique<qemu_bb_tracker>(track, track.name.c_str());
+        counters = std::make_unique<qemu_counters_state>(track);
     }
 
     ~qemu_context_state()
@@ -104,12 +197,14 @@ struct qemu_context_state : public qemu_tracker_state {
 
 struct qemu_fallback_state : public qemu_tracker_state {
     qemu_cpu_track track;
-    qemu_counters_state counters;
 
-    qemu_fallback_state(int id) : track(id), counters(track)
+    qemu_fallback_state(int id) : track(id)
     {
         auto desc = track.Serialize();
         perfetto::TrackEvent::SetTrackDescriptor(track, desc);
+
+        bb_tracker = std::make_unique<qemu_bb_tracker>(track, "CPU track");
+        counters = std::make_unique<qemu_counters_state>(track);
     }
 
     ~qemu_fallback_state()
@@ -155,21 +250,13 @@ class guest_context_tracker
 
   public:
     guest_context_tracker(int cpu_id);
-    void context_update(const log_event_ctx_update_t *evt);
-    void mode_update(perfetto::protos::pbzero::QEMULogEntryModeSwitch new_mode);
-    void flush_all_ctx_data();
+    void context_update(cpu_log_entry_handle,
+                        const log_event_ctx_update_t *evt);
+    void mode_update(cpu_log_entry_handle entry,
+                     perfetto::protos::pbzero::QEMULogEntryModeSwitch new_mode);
+    void flush_all_ctx_data(uint64_t pc);
     qemu_fallback_state *get_cpu_state();
     qemu_tracker_state *get_ctx_state();
 };
 
-/*
- * Generate a unique track ID.
- * The ID is guaranteed to be unique within a qemu run, but not with
- * respect to other data sources.
- */
-unsigned long gen_track_uuid();
-
-perfetto::protos::pbzero::QEMULogEntryModeSwitch
-qemu_cpu_mode_to_trace(qemu_log_instr_cpu_mode_t mode);
-
 } // namespace cheri
diff --git a/trace_extra/meson.build b/trace_extra/meson.build
index b391ddca81b..be65adeb277 100644
--- a/trace_extra/meson.build
+++ b/trace_extra/meson.build
@@ -4,8 +4,7 @@ qemutrace_ss = ss.source_set()
 
 if config_all.has_key('CONFIG_TCG_LOG_INSTR') and config_all.has_key('CONFIG_TRACE_PERFETTO')
    qemutrace_ss.add(dependency('boost', modules: ['filesystem', 'system']))
-   qemutrace_ss.add(files('trace_perfetto.cc', 'trace_stats.cc',
-                          'guest_context_tracker.cc', 'trace_counters.cc',
+   qemutrace_ss.add(files('trace_perfetto.cc', 'guest_context_tracker.cc', 'trace_counters.cc',
                           'cheri-perfetto/sdk/perfetto.cc'))
 endif
 
diff --git a/trace_extra/trace_counters.cc b/trace_extra/trace_counters.cc
index 9d830375373..4bcd00e5c47 100644
--- a/trace_extra/trace_counters.cc
+++ b/trace_extra/trace_counters.cc
@@ -26,7 +26,6 @@
  * @BERI_LICENSE_HEADER_END@
  */
 
-#include <cstdio>
 #include "trace_extra/trace_counters.hh"
 #include "trace_extra/guest_context_tracker.hh"
 
diff --git a/trace_extra/trace_perfetto.cc b/trace_extra/trace_perfetto.cc
index 49c293f65ab..a2fc00b1443 100644
--- a/trace_extra/trace_perfetto.cc
+++ b/trace_extra/trace_perfetto.cc
@@ -50,7 +50,6 @@
 #include "qemu/log_instr.h"
 #include "exec/log_instr_internal.h"
 #include "exec/log_instr_perfetto.h"
-#include "trace_extra/trace_stats.hh"
 #include "trace_extra/trace_counters.hh"
 #include "trace_extra/guest_context_tracker.hh"
 
@@ -123,7 +122,7 @@ bool perfetto_start_tracing(void)
     perfetto::Tracing::Initialize(args);
     perfetto::TrackEvent::Register();
 
-    cfg.add_buffers()->set_size_kb(1 << 17); // 128MiB
+    cfg.add_buffers()->set_size_kb(1 << 19); // 512MiB
     cfg.set_flush_period_ms(2000);
     cfg.set_file_write_period_ms(1000);
     fs::remove(logfile);
@@ -139,7 +138,7 @@ bool perfetto_start_tracing(void)
     ds_cfg->set_track_event_config_raw(track_cfg.SerializeAsString());
     auto *producer_cfg = cfg.add_producers();
     producer_cfg->set_producer_name("qemu-tcg");
-    producer_cfg->set_shm_size_kb(1 << 15); // 32MiB
+    producer_cfg->set_shm_size_kb(1 << 18); // 256MiB
 
     session = perfetto::Tracing::NewTrace();
 
@@ -187,7 +186,8 @@ void trace_cap_register(perfetto::protos::pbzero::QEMULogEntryCapability *cap,
  * Handle tracing control event. Returns true if the event changes the trace
  * enable state (i.e. start or stop).
  */
-bool process_state_event(perfetto_backend_data *data, log_event_t *evt)
+bool process_state_event(perfetto_backend_data *data,
+                         cpu_log_entry_handle entry, log_event_t *evt)
 {
     bool has_startstop_event = false;
     auto *state = data->ctx_tracker.get_ctx_state();
@@ -195,16 +195,16 @@ bool process_state_event(perfetto_backend_data *data, log_event_t *evt)
     switch (evt->state.next_state) {
     case LOG_EVENT_STATE_FLUSH:
         TRACE_EVENT_INSTANT("ctrl", "flush", data->ctrl_track);
-        data->ctx_tracker.flush_all_ctx_data();
+        data->ctx_tracker.flush_all_ctx_data(evt->state.pc);
         perfetto::TrackEvent::Flush();
         break;
     case LOG_EVENT_STATE_START:
-        state->stats.unpause(*state->get_track(), evt->state.pc);
+        state->bb_tracker->track_next(entry, evt->state.pc);
         TRACE_EVENT_BEGIN("ctrl", "tracing", data->ctrl_track);
         has_startstop_event = true;
         break;
     case LOG_EVENT_STATE_STOP:
-        state->stats.pause(*state->get_track(), evt->state.pc);
+        state->bb_tracker->reset(evt->state.pc);
         TRACE_EVENT_END("ctrl", data->ctrl_track);
         has_startstop_event = true;
         break;
@@ -224,11 +224,7 @@ void process_context_event(perfetto_backend_data *data,
 
     /* Swap current context. */
     if (evt->ctx_update.op == LOG_EVENT_CTX_OP_SWITCH) {
-        state->stats.pause(*state->get_track(), perfetto_log_entry_pc(entry));
-        data->ctx_tracker.context_update(&evt->ctx_update);
-        /* Reload data and track as context_update will have changed them */
-        state = data->ctx_tracker.get_ctx_state();
-        state->stats.unpause(*state->get_track(), perfetto_log_entry_pc(entry));
+        data->ctx_tracker.context_update(entry, &evt->ctx_update);
     }
 }
 
@@ -274,7 +270,7 @@ void process_events(perfetto_backend_data *data, cpu_log_entry_handle entry)
         log_event_t *evt = perfetto_log_event(entry, i);
         switch (evt->id) {
         case LOG_EVENT_STATE:
-            has_startstop_event = process_state_event(data, evt);
+            has_startstop_event = process_state_event(data, entry, evt);
             break;
         case LOG_EVENT_CTX_UPDATE:
             process_context_event(data, entry, evt);
@@ -291,19 +287,11 @@ void process_events(perfetto_backend_data *data, cpu_log_entry_handle entry)
             assert(false && "Invalid event identifier");
         }
     }
-    auto *state = data->ctx_tracker.get_ctx_state();
 
     if (perfetto_log_entry_flags(entry) & LI_FLAG_MODE_SWITCH) {
         auto mode = cheri::qemu_cpu_mode_to_trace(
             perfetto_log_entry_next_cpu_mode(entry));
-        // XXX-AM: consider making the mode switch an event, possibly with its
-        // own separate pc signaling the PC of the next instruction we are
-        // landing to?
-        state->stats.pause(*state->get_track(), perfetto_log_entry_pc(entry));
-        data->ctx_tracker.mode_update(mode);
-        /* reload possibly updated state and track */
-        state = data->ctx_tracker.get_ctx_state();
-        state->stats.unpause(*state->get_track(), perfetto_log_entry_pc(entry));
+        data->ctx_tracker.mode_update(entry, mode);
     }
 
     /*
@@ -313,8 +301,10 @@ void process_events(perfetto_backend_data *data, cpu_log_entry_handle entry)
      * in the backend.
      */
     if (!has_startstop_event &&
-        (perfetto_log_entry_flags(entry) & LI_FLAG_HAS_INSTR_DATA) != 0)
-        state->stats.process_instr(*state->get_track(), entry);
+        (perfetto_log_entry_flags(entry) & LI_FLAG_HAS_INSTR_DATA) != 0) {
+        auto *state = data->ctx_tracker.get_ctx_state();
+        state->bb_tracker->track_next(entry);
+    }
 }
 
 void process_instr(perfetto_backend_data *data, cpu_log_entry_handle entry)
diff --git a/trace_extra/trace_stats.cc b/trace_extra/trace_stats.cc
deleted file mode 100644
index a32bb13a8b4..00000000000
--- a/trace_extra/trace_stats.cc
+++ /dev/null
@@ -1,150 +0,0 @@
-/*-
- * Copyright (c) 2021 Alfredo Mazzinghi
- * All rights reserved.
- *
- * This software was developed by SRI International and the University of
- * Cambridge Computer Laboratory (Department of Computer Science and
- * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the
- * DARPA SSITH research programme.
- *
- * @BERI_LICENSE_HEADER_START@
- *
- * Licensed to BERI Open Systems C.I.C. (BERI) under one or more contributor
- * license agreements.  See the NOTICE file distributed with this work for
- * additional information regarding copyright ownership.  BERI licenses this
- * file to you under the BERI Hardware-Software License, Version 1.0 (the
- * "License"); you may not use this file except in compliance with the
- * License.  You may obtain a copy of the License at:
- *
- *   http://www.beri-open-systems.org/legal/license-1-0.txt
- *
- * Unless required by applicable law or agreed to in writing, Work distributed
- * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
- * CONDITIONS OF ANY KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations under the License.
- *
- * @BERI_LICENSE_HEADER_END@
- */
-
-#include <utility>
-#include <cstdint>
-#include "qemu/log_instr.h"
-#include "trace_extra/trace_stats.hh"
-
-using namespace std::chrono_literals;
-namespace icl = boost::icl;
-
-using addr_range = icl::interval<uint64_t>;
-
-namespace cheri
-{
-
-void qemu_stats::flush(perfetto::Track &track)
-{
-    /*
-     * XXX-AM: Note: to avoid big packets we should probably write
-     * the histogram samples separate from the hisogram descriptor and
-     * tie them toghether in post-processing.
-     * This would likely require specialized packet processing.
-     */
-    TRACE_EVENT_INSTANT("stats", "bb_hist", track,
-                        [&](perfetto::EventContext ctx) {
-                            auto *qemu_arg = ctx.event()->set_qemu();
-                            auto *hist = qemu_arg->set_histogram();
-                            for (const auto &keyval : bb_hit_map_) {
-                                auto &range = keyval.first;
-                                auto *bucket = hist->add_bb_bucket();
-                                bucket->set_start(range.lower());
-                                bucket->set_end(range.upper());
-                                bucket->set_value(keyval.second);
-                            }
-                        });
-
-    TRACE_EVENT_INSTANT("stats", "branch_hist", track,
-                        [&](perfetto::EventContext ctx) {
-                            auto *qemu_arg = ctx.event()->set_qemu();
-                            auto *hist = qemu_arg->set_histogram();
-                            for (const auto &keyval : branch_map_) {
-                                auto *bucket = hist->add_branch_bucket();
-                                branch_map_id id = keyval.first;
-                                bucket->set_source(std::get<0>(id));
-                                bucket->set_target(std::get<1>(id));
-                                bucket->set_value(keyval.second);
-                            }
-                        });
-
-    clear();
-}
-
-void qemu_stats::pause(perfetto::Track &track, uint64_t pc)
-{
-    if (paused_)
-        return;
-    paused_ = true;
-    /* Nothing to flush */
-    if (pc_range_start_ == 0)
-        return;
-    assert(pc != 0 && "pause PC can not be zero");
-    /* Treat this as the end of the block, otherwise the current PC slice will
-     * be lost. */
-    bb_hit_map_ +=
-        std::make_pair(addr_range::right_open(pc_range_start_, pc), icount_);
-    pc_range_start_ = last_pc_ = next_instr_pc_ = icount_ = 0;
-    /*
-     * Whenever we pause tracing, we may sync the stats to trace.
-     * This is to avoid having to dump packets too large into the trace, however
-     * we could get around this by just dumping the full stats only when a flush
-     * is triggered.
-     */
-    auto now = std::chrono::steady_clock::now();
-    auto interval = now - last_flush_time_;
-    if (interval > 1s) {
-        last_flush_time_ = now;
-        flush(track);
-    }
-}
-
-void qemu_stats::unpause(perfetto::Track &track, uint64_t pc)
-{
-    if (!paused_)
-        return;
-    paused_ = false;
-    /* Reset the pc-tracking state */
-    pc_range_start_ = last_pc_ = pc;
-    next_instr_pc_ = -1;
-    /* unpause will be called on the first instruction to be traced? */
-    icount_ = 1; // Unpause is called for the first instruction being traced
-}
-
-void qemu_stats::clear()
-{
-    bb_hit_map_.clear();
-    branch_map_.clear();
-}
-
-void qemu_stats::process_instr(perfetto::Track &ctx_track,
-                               cpu_log_entry_handle entry)
-{
-    uint64_t pc = perfetto_log_entry_pc(entry);
-    int size = perfetto_log_entry_insn_size(entry);
-    // When unpausing: accept anything as a next instruction when next_instr_pc
-    // == -1
-    if (pc != next_instr_pc_ && next_instr_pc_ != -1UL) {
-        // presume we are branching
-        assert(pc_range_start_ != 0 && "PC bb start was not updated");
-        assert(last_pc_ != 0 && "process last_pc can not be zero");
-        bb_hit_map_ += std::make_pair(
-            addr_range::right_open(pc_range_start_, last_pc_), icount_);
-        pc_range_start_ = pc;
-        icount_ =
-            1; // Reset icount triggered on the first instruction of the next BB
-        branch_map_id branch_id = std::make_tuple(last_pc_, pc);
-        branch_map_[branch_id] += 1;
-    } else {
-        icount_ += 1;
-    }
-    next_instr_pc_ = pc + size;
-    last_pc_ = pc;
-}
-
-} // namespace cheri
diff --git a/trace_extra/trace_stats.hh b/trace_extra/trace_stats.hh
deleted file mode 100644
index 7d94f508942..00000000000
--- a/trace_extra/trace_stats.hh
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Copyright (c) 2021 Alfredo Mazzinghi
- * All rights reserved.
- *
- * This software was developed by SRI International and the University of
- * Cambridge Computer Laboratory (Department of Computer Science and
- * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the
- * DARPA SSITH research programme.
- *
- * @BERI_LICENSE_HEADER_START@
- *
- * Licensed to BERI Open Systems C.I.C. (BERI) under one or more contributor
- * license agreements.  See the NOTICE file distributed with this work for
- * additional information regarding copyright ownership.  BERI licenses this
- * file to you under the BERI Hardware-Software License, Version 1.0 (the
- * "License"); you may not use this file except in compliance with the
- * License.  You may obtain a copy of the License at:
- *
- *   http://www.beri-open-systems.org/legal/license-1-0.txt
- *
- * Unless required by applicable law or agreed to in writing, Work distributed
- * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
- * CONDITIONS OF ANY KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations under the License.
- *
- * @BERI_LICENSE_HEADER_END@
- */
-
-/*
- * Internal instruction logging metric aggregation
- */
-
-#pragma once
-
-#include <chrono>
-#include <unordered_map>
-#include <perfetto.h>
-#include <boost/icl/interval_map.hpp>
-#include "exec/log_instr_perfetto.h"
-#include "tuple_index.hh"
-
-namespace cheri
-{
-
-/* tuple (source-addr, target-addr) */
-using branch_map_id = std::tuple<uint64_t, uint64_t>;
-
-class qemu_stats
-{
-    /*
-     * Histogram of basic-block hit during tracing.
-     */
-    boost::icl::interval_map<uint64_t, uint64_t> bb_hit_map_;
-    /*
-     * Histogram of branch instruction addresses.
-     * Post-processing must be used to distinguish between function
-     * calls and conditional branches.
-     */
-    std::unordered_map<branch_map_id, uint64_t, tuple_hasher<branch_map_id>>
-        branch_map_;
-    /*
-     *  PC-tracking info to detect branches.
-     */
-    uint64_t last_pc_;
-    uint64_t next_instr_pc_;
-    uint64_t pc_range_start_;
-    uint64_t icount_;
-    /* Protect against double pause() calls */
-    bool paused_;
-    /* Time of last flush */
-    std::chrono::time_point<std::chrono::steady_clock> last_flush_time_;
-
-  public:
-    qemu_stats()
-        : last_pc_(0), pc_range_start_(0), icount_(0), paused_(true),
-          last_flush_time_(std::chrono::steady_clock::now())
-    {
-    }
-    void process_instr(perfetto::Track &track, cpu_log_entry_handle entry);
-    void pause(perfetto::Track &track, uint64_t pc);
-    void unpause(perfetto::Track &track, uint64_t pc);
-    void flush(perfetto::Track &track);
-    void clear();
-};
-
-} // namespace cheri

From 719e787665ed62340d931b68fd1193afa3d4b720 Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Wed, 13 Jul 2022 16:20:35 +0100
Subject: [PATCH 53/74] Fix instruction opcode dumping to the perfetto message
 format.

This works around the inability to use bytes fields in perfetto
track event messages.
---
 trace_extra/cheri-perfetto    |  2 +-
 trace_extra/trace_perfetto.cc | 15 +++++++++++++++
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/trace_extra/cheri-perfetto b/trace_extra/cheri-perfetto
index 3415bc5a15a..9df9ea6d147 160000
--- a/trace_extra/cheri-perfetto
+++ b/trace_extra/cheri-perfetto
@@ -1 +1 @@
-Subproject commit 3415bc5a15a1231a005739340be643e533bcf754
+Subproject commit 9df9ea6d1474e1589240a853a4173dcf094fefed
diff --git a/trace_extra/trace_perfetto.cc b/trace_extra/trace_perfetto.cc
index a2fc00b1443..08738e65e03 100644
--- a/trace_extra/trace_perfetto.cc
+++ b/trace_extra/trace_perfetto.cc
@@ -342,7 +342,22 @@ void process_instr(perfetto_backend_data *data, cpu_log_entry_handle entry)
             int size = perfetto_log_entry_insn_size(entry);
             int nitems;
 
+            /* Due to perfetto limitations, use the opcode message for now */
+#ifdef NOTYET
             instr->set_opcode((const uint8_t *)bytes, size);
+#else
+            if (size <= sizeof(uint64_t)) {
+                uint64_t value;
+                memcpy(&value, bytes, size);
+                auto *opcode = instr->set_opcode_obj();
+                opcode->set_value(value);
+                opcode->set_size(size);
+            } else {
+                // Throw an error or something
+                assert(false && "Not reached");
+            }
+#endif
+
             instr->set_pc(perfetto_log_entry_pc(entry));
 
             nitems = perfetto_log_entry_regs(entry);

From 2d86dafcef7b340ebbf38d63943002b29a48ea86 Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Wed, 13 Jul 2022 18:16:08 +0100
Subject: [PATCH 54/74] Fix cheri-perfetto new remote URL.

---
 .gitmodules | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/.gitmodules b/.gitmodules
index cc00260ba11..0dd2472378d 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -66,7 +66,5 @@
 	url = https://git.qemu.org/git/vbootrom.git
 [submodule "trace/cheri-perfetto"]
 	path = trace_extra/cheri-perfetto
-	url = https://github.com/qwattash/cheri-perfetto.git
-	branch = cheri-perfetto
-[submodule "cheri-perfetto"]
 	url = https://github.com/CTSRD-CHERI/cheri-perfetto.git
+	branch = cheri-perfetto

From 5e5a6da55ecf6c5b15e9840dbf3ae3f45368fdd0 Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Thu, 14 Jul 2022 17:13:01 +0100
Subject: [PATCH 55/74] Add option to select output file for the protobuf
 tracing backend.

---
 accel/tcg/log_instr_protobuf.c | 7 ++++++-
 include/qemu/log_instr.h       | 4 ++++
 qemu-options.hx                | 8 ++++++++
 softmmu/vl.c                   | 5 +++++
 4 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/accel/tcg/log_instr_protobuf.c b/accel/tcg/log_instr_protobuf.c
index 1e630a15412..fbb8ff4c727 100644
--- a/accel/tcg/log_instr_protobuf.c
+++ b/accel/tcg/log_instr_protobuf.c
@@ -266,7 +266,7 @@ void init_protobuf_backend(CPUArchState *env)
      */
 
     if (protobuf_logfile == NULL) {
-        protobuf_logfile = fopen("qemu-protobuf.pb", "w+b");
+        protobuf_logfile = fopen("qemu-trace.pb", "w+b");
     }
 }
 
@@ -391,3 +391,8 @@ void emit_protobuf_entry(CPUArchState *env, cpu_log_entry_t *entry)
     }
     g_free(buf);
 }
+
+void qemu_log_instr_protobuf_conf_logfile(const char *name)
+{
+    protobuf_logfile = fopen(name, "w+b");
+}
diff --git a/include/qemu/log_instr.h b/include/qemu/log_instr.h
index a7f19cf5b09..e1e128265d1 100644
--- a/include/qemu/log_instr.h
+++ b/include/qemu/log_instr.h
@@ -214,6 +214,10 @@ int qemu_log_instr_perfetto_conf_categories(const char *category_list);
 #endif
 #endif /* CONFIG_TRACE_PERFETTO */
 
+#ifdef CONFIG_TRACE_PROTOBUF
+void qemu_log_instr_protobuf_conf_logfile(const char *name);
+#endif /* CONFIG_TRACE_PROTOBUF  */
+
 #ifndef __cplusplus
 /* No visibility in the perfetto tracing backend */
 
diff --git a/qemu-options.hx b/qemu-options.hx
index 6cf4b3b0362..4be3cd7a29a 100644
--- a/qemu-options.hx
+++ b/qemu-options.hx
@@ -4138,6 +4138,14 @@ SRST
     Select categories of messages to include in the trace (instructions, stats)
 ERST
 
+DEF("cheri-trace-protobuf-logfile", HAS_ARG, QEMU_OPTION_trace_protobuf_logfile, \
+"-cheri-trace-protobuf-logfile [logfile]     \
+ Set log file for protobuf traces, defaults to qemu_trace.pb.\n", QEMU_ARCH_ALL)
+SRST
+``-cheri-trace-protobuf-logfile [logfile]``
+    Set protobuf trace output file.
+ERST
+
 DEF("cheri-trace-filters", HAS_ARG, QEMU_OPTION_cheri_trace_filters, \
 "-cheri-trace-filters [event]     Select trace filters to use.\n", QEMU_ARCH_ALL)
 SRST
diff --git a/softmmu/vl.c b/softmmu/vl.c
index 0656be9533d..0246b47db37 100644
--- a/softmmu/vl.c
+++ b/softmmu/vl.c
@@ -3781,6 +3781,11 @@ void qemu_init(int argc, char **argv, char **envp)
             case QEMU_OPTION_trace_perfetto_categories:
                 qemu_log_instr_perfetto_conf_categories(optarg);
                 break;
+#endif
+#ifdef CONFIG_TRACE_PERFETTO
+            case QEMU_OPTION_trace_protobuf_logfile:
+                qemu_log_instr_protobuf_conf_logfile(optarg);
+                break;
 #endif
             case QEMU_OPTION_cheri_trace_buffer_size:
                 qemu_log_instr_set_buffer_size(strtoul(optarg, NULL, 0));

From 674fc6eacbff364141b284b25ba9b31f3da85150 Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Thu, 28 Jul 2022 11:06:06 +0100
Subject: [PATCH 56/74] Unbreak -icount mode for RISC-V.

---
 target/riscv/op_helper_cheri.c | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/target/riscv/op_helper_cheri.c b/target/riscv/op_helper_cheri.c
index cb115200668..8588b5415e6 100644
--- a/target/riscv/op_helper_cheri.c
+++ b/target/riscv/op_helper_cheri.c
@@ -156,7 +156,11 @@ void HELPER(cspecialrw)(CPUArchState *env, uint32_t cd, uint32_t cs,
 {
     uintptr_t _host_return_address = GETPC();
     // Ensure that env->PCC.cursor is correct:
-    cpu_restore_state(env_cpu(env), _host_return_address, false);
+    /*
+     * This breaks -icount because it triggers early deallocation of the TB.
+     * It seems that it is only required for logging anyway.
+     */
+    /* cpu_restore_state(env_cpu(env), _host_return_address, false); */
 
     assert(index <= 31 && "Bug in translator?");
     enum SCRAccessMode mode = scr_info[index].access;

From 7576074e29af36865ba4d40427fdf7e0eeb706aa Mon Sep 17 00:00:00 2001
From: Mingle Chen <mc2262@cl.cam.ac.uk>
Date: Mon, 18 Jul 2022 18:08:14 +0100
Subject: [PATCH 57/74] Add an experimental interceptor in perfetto to trace
 memory load and store

---
 .gitmodules                       |  3 ++
 accel/tcg/log_instr_perfetto.c    | 11 ++++-
 include/exec/log_instr_perfetto.h |  3 ++
 include/qemu/log_instr.h          |  2 +
 qemu-options.hx                   | 15 ++++++
 softmmu/vl.c                      |  7 ++-
 trace_extra/dynamorio             |  1 +
 trace_extra/memory_interceptor.cc | 57 ++++++++++++++++++++++
 trace_extra/memory_interceptor.hh | 38 +++++++++++++++
 trace_extra/meson.build           |  2 +-
 trace_extra/trace_perfetto.cc     | 79 +++++++++++++++++++++++++++++--
 11 files changed, 212 insertions(+), 6 deletions(-)
 create mode 160000 trace_extra/dynamorio
 create mode 100644 trace_extra/memory_interceptor.cc
 create mode 100644 trace_extra/memory_interceptor.hh

diff --git a/.gitmodules b/.gitmodules
index 0dd2472378d..0251c513c04 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -68,3 +68,6 @@
 	path = trace_extra/cheri-perfetto
 	url = https://github.com/CTSRD-CHERI/cheri-perfetto.git
 	branch = cheri-perfetto
+[submodule "trace_extra/dynamorio"]
+	path = trace_extra/dynamorio
+	url = https://github.com/DynamoRIO/dynamorio.git
diff --git a/accel/tcg/log_instr_perfetto.c b/accel/tcg/log_instr_perfetto.c
index cc736905846..282d05e695e 100644
--- a/accel/tcg/log_instr_perfetto.c
+++ b/accel/tcg/log_instr_perfetto.c
@@ -40,7 +40,7 @@
 #include "exec/log_instr.h"
 #include "exec/log_instr_internal.h"
 #include "exec/log_instr_perfetto.h"
-
+#include "target/cheri-common/cheri_defs.h"
 /*
  * Initialize perfetto tracing.
  *
@@ -197,3 +197,12 @@ CAPREG_GETTER_DECL(uint32_t, otype)
     return 0;
 #endif
 }
+
+TARGET_CAP_DECL(uint32_t, size)
+{
+#ifdef TARGET_CHERI
+    return CHERI_CAP_SIZE;
+#else
+    return 0;
+#endif
+}
diff --git a/include/exec/log_instr_perfetto.h b/include/exec/log_instr_perfetto.h
index 4cfef8cca6e..17227fdcc54 100644
--- a/include/exec/log_instr_perfetto.h
+++ b/include/exec/log_instr_perfetto.h
@@ -102,6 +102,7 @@ void perfetto_emit_instr(void *backend_data, cpu_log_entry_handle entry_handle);
     type perfetto_mem_info_##field(cpu_log_entry_handle handle, int idx)
 #define CAPREG_GETTER_DECL(type, field)                                        \
     type perfetto_cap_##field(cap_register_handle handle)
+#define TARGET_CAP_DECL(type, field) type perfetto_target_cap_##field()
 
 #define CPU_LOG_ENTRY_GETTER(type, field)                                      \
     CPU_LOG_ENTRY_GETTER_DECL(type, field)                                     \
@@ -156,6 +157,8 @@ CAPREG_GETTER_DECL(uint64_t, length);
 CAPREG_GETTER_DECL(uint32_t, perms);
 CAPREG_GETTER_DECL(uint32_t, otype);
 
+TARGET_CAP_DECL(uint32_t, size);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/include/qemu/log_instr.h b/include/qemu/log_instr.h
index e1e128265d1..0e8dbbc951e 100644
--- a/include/qemu/log_instr.h
+++ b/include/qemu/log_instr.h
@@ -209,6 +209,8 @@ extern "C" {
 #endif
 void qemu_log_instr_perfetto_conf_logfile(const char *name);
 int qemu_log_instr_perfetto_conf_categories(const char *category_list);
+void qemu_log_instr_perfetto_enable_interceptor(void);
+void qemu_log_instr_perfetto_interceptor_logfile(const char *name);
 #ifdef __cplusplus
 }
 #endif
diff --git a/qemu-options.hx b/qemu-options.hx
index 4be3cd7a29a..aa218127290 100644
--- a/qemu-options.hx
+++ b/qemu-options.hx
@@ -4138,6 +4138,21 @@ SRST
     Select categories of messages to include in the trace (instructions, stats)
 ERST
 
+DEF("cheri-trace-perfetto-interceptor-logfile", HAS_ARG, QEMU_OPTION_trace_perfetto_interceptor_logfile, \
+"-cheri-trace-perfetto-categories category[,...]     \
+ Set log file for perfetto interceptor, defaults to mem_access.trace.\n", QEMU_ARCH_ALL)
+SRST
+``-cheri-trace-perfetto-interceptor-logfile [logfile]``
+    Set perfetto interceptor trace output file.
+ERST
+
+DEF("cheri-trace-perfetto-enable-interceptor", 0, QEMU_OPTION_trace_perfetto_enable_interceptor, \
+"-cheri-trace-perfetto-enable-interceptor     Enable perfetto interceptor.\n", QEMU_ARCH_ALL)
+SRST
+``-cheri-trace-perfetto-enable-interceptor ``
+    Enable perfetto interceptor, skips normal perfetto tracing service.
+ERST
+
 DEF("cheri-trace-protobuf-logfile", HAS_ARG, QEMU_OPTION_trace_protobuf_logfile, \
 "-cheri-trace-protobuf-logfile [logfile]     \
  Set log file for protobuf traces, defaults to qemu_trace.pb.\n", QEMU_ARCH_ALL)
diff --git a/softmmu/vl.c b/softmmu/vl.c
index 0246b47db37..97dd4376fde 100644
--- a/softmmu/vl.c
+++ b/softmmu/vl.c
@@ -3781,8 +3781,13 @@ void qemu_init(int argc, char **argv, char **envp)
             case QEMU_OPTION_trace_perfetto_categories:
                 qemu_log_instr_perfetto_conf_categories(optarg);
                 break;
+            case QEMU_OPTION_trace_perfetto_enable_interceptor:
+                qemu_log_instr_perfetto_enable_interceptor();
+                break;
+            case QEMU_OPTION_trace_perfetto_interceptor_logfile:
+                qemu_log_instr_perfetto_interceptor_logfile(optarg);
 #endif
-#ifdef CONFIG_TRACE_PERFETTO
+#ifdef CONFIG_TRACE_PROTOBUF
             case QEMU_OPTION_trace_protobuf_logfile:
                 qemu_log_instr_protobuf_conf_logfile(optarg);
                 break;
diff --git a/trace_extra/dynamorio b/trace_extra/dynamorio
new file mode 160000
index 00000000000..acb883e8d64
--- /dev/null
+++ b/trace_extra/dynamorio
@@ -0,0 +1 @@
+Subproject commit acb883e8d64d0174f76a2eacf06cdf62b76732af
diff --git a/trace_extra/memory_interceptor.cc b/trace_extra/memory_interceptor.cc
new file mode 100644
index 00000000000..ad696b03c8a
--- /dev/null
+++ b/trace_extra/memory_interceptor.cc
@@ -0,0 +1,57 @@
+//
+// Created by mc2262 on 15/07/22.
+//
+
+#include "trace_extra/memory_interceptor.hh"
+
+DynamorioTraceInterceptor::ThreadLocalState::ThreadLocalState(
+    ThreadLocalStateArgs &)
+{
+}
+
+ofstream DynamorioTraceInterceptor::mem_logfile;
+
+void DynamorioTraceInterceptor::OnTracePacket(InterceptorContext context)
+{
+    perfetto::protos::pbzero::TracePacket::Decoder packet(
+        context.packet_data.data, context.packet_data.size);
+    if (packet.has_track_event()) {
+        perfetto::protos::pbzero::TrackEvent::Decoder track_event(
+            packet.track_event());
+        if (track_event.has_qemu()) {
+            perfetto::protos::pbzero::QEMUEventInfo::Decoder qemu(
+                track_event.qemu());
+            if (qemu.has_instr()) {
+                perfetto::protos::pbzero::QEMULogEntry::Decoder instr(
+                    qemu.instr());
+                if (instr.has_mem()) {
+                    for (auto iter = instr.mem(); iter; iter++) {
+                        perfetto::protos::pbzero::QEMULogEntryMem::Decoder mem(
+                            *iter);
+                        // create a trace_entry_t compatible with drcachesim
+                        trace_entry_t trace;
+                        trace.addr = reinterpret_cast<addr_t>(mem.addr());
+                        trace.size = mem.size();
+                        switch (mem.op()) {
+                        case perfetto::protos::pbzero::
+                            QEMULogEntryMem_MemOp_LOAD:
+                        case perfetto::protos::pbzero::
+                            QEMULogEntryMem_MemOp_CLOAD:
+                            trace.type = TRACE_TYPE_READ;
+                            break;
+                        case perfetto::protos::pbzero::
+                            QEMULogEntryMem_MemOp_STORE:
+                        case perfetto::protos::pbzero::
+                            QEMULogEntryMem_MemOp_CSTORE:
+                            trace.type = TRACE_TYPE_WRITE;
+                            break;
+                        default:
+                            break;
+                        }
+                        mem_logfile.write((char *)&trace, sizeof(trace));
+                    }
+                }
+            }
+        }
+    }
+}
diff --git a/trace_extra/memory_interceptor.hh b/trace_extra/memory_interceptor.hh
new file mode 100644
index 00000000000..fdf5d097036
--- /dev/null
+++ b/trace_extra/memory_interceptor.hh
@@ -0,0 +1,38 @@
+//
+// Created by mc2262 on 15/07/22.
+//
+
+#pragma once
+#include <perfetto.h>
+#include <iostream>
+#include <fstream>
+#include "dynamorio/clients/drcachesim/common/trace_entry.h"
+
+using namespace std;
+
+class DynamorioTraceInterceptor
+    : public perfetto::Interceptor<DynamorioTraceInterceptor>
+{
+
+  public:
+    static ofstream mem_logfile;
+    ~DynamorioTraceInterceptor() override = default;
+
+    struct ThreadLocalState
+        : public perfetto::InterceptorBase::ThreadLocalState {
+        ThreadLocalState(ThreadLocalStateArgs &);
+
+        ~ThreadLocalState() override = default;
+    };
+
+    // This function is called for each intercepted trace packet. |context|
+    // contains information about the trace packet as well as other state
+    // tracked by the interceptor (e.g., see ThreadLocalState).
+    //
+    // Intercepted trace data is provided in the form of serialized protobuf
+    // bytes, accessed through the |context.packet_data| field.
+    //
+    // Warning: this function can be called on any thread at any time. See
+    // below for how to safely access shared interceptor data from here.
+    static void OnTracePacket(InterceptorContext context);
+};
diff --git a/trace_extra/meson.build b/trace_extra/meson.build
index be65adeb277..d6a6c9ad740 100644
--- a/trace_extra/meson.build
+++ b/trace_extra/meson.build
@@ -4,7 +4,7 @@ qemutrace_ss = ss.source_set()
 
 if config_all.has_key('CONFIG_TCG_LOG_INSTR') and config_all.has_key('CONFIG_TRACE_PERFETTO')
    qemutrace_ss.add(dependency('boost', modules: ['filesystem', 'system']))
-   qemutrace_ss.add(files('trace_perfetto.cc', 'guest_context_tracker.cc', 'trace_counters.cc',
+   qemutrace_ss.add(files('trace_perfetto.cc', 'guest_context_tracker.cc', 'trace_counters.cc', 'memory_interceptor.cc',
                           'cheri-perfetto/sdk/perfetto.cc'))
 endif
 
diff --git a/trace_extra/trace_perfetto.cc b/trace_extra/trace_perfetto.cc
index 08738e65e03..0ccff6544f9 100644
--- a/trace_extra/trace_perfetto.cc
+++ b/trace_extra/trace_perfetto.cc
@@ -53,6 +53,8 @@
 #include "trace_extra/trace_counters.hh"
 #include "trace_extra/guest_context_tracker.hh"
 
+#include "trace_extra/memory_interceptor.hh"
+
 PERFETTO_TRACK_EVENT_STATIC_STORAGE();
 
 namespace fs = boost::filesystem;
@@ -67,12 +69,21 @@ std::unique_ptr<perfetto::TracingSession> session;
 /* perfetto logfile */
 fs::path logfile("qemu_trace.pb");
 
+/* perfetto interceptor trace file */
+string mem_logfile_name = "mem_access.trace";
+
+/* enable perfetto interceptor */
+bool enable_interceptor = false;
+
 /* category strings */
 std::vector<std::string> categories;
 
 /* Global scheduling event track */
 std::unique_ptr<perfetto::Track> sched_track;
 
+/* MO_SIZE from MemOp in /include/exec/memop.h */
+constexpr int MO_SIZE = 3;
+
 /*
  * Private per-CPU state.
  */
@@ -100,7 +111,6 @@ struct perfetto_backend_data {
         perfetto::TrackEvent::SetTrackDescriptor(ctrl_track, desc);
     }
 };
-
 /*
  * Initialize perfetto tracing.
  *
@@ -122,13 +132,47 @@ bool perfetto_start_tracing(void)
     perfetto::Tracing::Initialize(args);
     perfetto::TrackEvent::Register();
 
+    if (enable_interceptor) {
+        DynamorioTraceInterceptor::mem_logfile.open(mem_logfile_name,
+                                                    ios::binary);
+        // drcachesim needs a header in the file, so we create it here
+        trace_entry_t header{ .type = TRACE_TYPE_HEADER,
+                              .size = 0,
+                              .addr = TRACE_ENTRY_VERSION };
+        DynamorioTraceInterceptor::mem_logfile.write((char *)&header,
+                                                     sizeof(header));
+
+        // dub thread and pid as we only have one process one thread for now
+        trace_entry_t thread{ .type = TRACE_TYPE_THREAD, .size = 4, .addr = 1 };
+        trace_entry_t pid{ .type = TRACE_TYPE_PID, .size = 4, .addr = 1 };
+
+        // dub timestamp and cpuid
+        trace_entry_t timestamp{ .type = TRACE_TYPE_MARKER,
+                                 .size = TRACE_MARKER_TYPE_TIMESTAMP,
+                                 .addr = 0 };
+        trace_entry_t cpuid{ .type = TRACE_TYPE_MARKER,
+                             .size = TRACE_MARKER_TYPE_CPU_ID,
+                             .addr = 0 };
+
+        DynamorioTraceInterceptor::mem_logfile.write((char *)&thread,
+                                                     sizeof(thread));
+        DynamorioTraceInterceptor::mem_logfile.write((char *)&pid, sizeof(pid));
+        DynamorioTraceInterceptor::mem_logfile.write((char *)&timestamp,
+                                                     sizeof(timestamp));
+        DynamorioTraceInterceptor::mem_logfile.write((char *)&cpuid,
+                                                     sizeof(cpuid));
+
+        perfetto::InterceptorDescriptor interceptor_desc;
+        interceptor_desc.set_name("console");
+        DynamorioTraceInterceptor::Register(interceptor_desc);
+    }
+
     cfg.add_buffers()->set_size_kb(1 << 19); // 512MiB
     cfg.set_flush_period_ms(2000);
     cfg.set_file_write_period_ms(1000);
     fs::remove(logfile);
     cfg.set_write_into_file(true);
     cfg.set_output_path(logfile.string());
-
     auto *ds_cfg = cfg.add_data_sources()->mutable_config();
     ds_cfg->set_name("track_event");
     track_cfg.add_disabled_categories("*");
@@ -136,6 +180,11 @@ bool perfetto_start_tracing(void)
         track_cfg.add_enabled_categories(category);
     }
     ds_cfg->set_track_event_config_raw(track_cfg.SerializeAsString());
+    if (enable_interceptor) {
+        // redirect to interceptor
+        ds_cfg->mutable_interceptor_config()->set_name("console");
+    }
+
     auto *producer_cfg = cfg.add_producers();
     producer_cfg->set_producer_name("qemu-tcg");
     producer_cfg->set_shm_size_kb(1 << 18); // 256MiB
@@ -168,6 +217,16 @@ void perfetto_tracing_stop(void)
     // also need to call the buffer sync function for each CPU on the exit path.
     session->FlushBlocking();
     session->StopBlocking();
+    if (enable_interceptor) {
+        // add footer to tracing file
+        trace_entry_t footer;
+        footer.type = TRACE_TYPE_FOOTER;
+        footer.size = 0;
+        footer.addr = 0;
+        DynamorioTraceInterceptor::mem_logfile.write((char *)&footer,
+                                                     sizeof(footer));
+        DynamorioTraceInterceptor::mem_logfile.close();
+    }
 }
 
 void trace_cap_register(perfetto::protos::pbzero::QEMULogEntryCapability *cap,
@@ -342,7 +401,7 @@ void process_instr(perfetto_backend_data *data, cpu_log_entry_handle entry)
             int size = perfetto_log_entry_insn_size(entry);
             int nitems;
 
-            /* Due to perfetto limitations, use the opcode message for now */
+        /* Due to perfetto limitations, use the opcode message for now */
 #ifdef NOTYET
             instr->set_opcode((const uint8_t *)bytes, size);
 #else
@@ -379,6 +438,7 @@ void process_instr(perfetto_backend_data *data, cpu_log_entry_handle entry)
                 auto flags = perfetto_mem_info_flags(entry, i);
                 auto *meminfo = instr->add_mem();
                 meminfo->set_addr(perfetto_mem_info_addr(entry, i));
+
                 switch (flags) {
                 case LMI_LD:
                     meminfo->set_op(
@@ -404,8 +464,12 @@ void process_instr(perfetto_backend_data *data, cpu_log_entry_handle entry)
                     cap_register_handle cap_handle =
                         perfetto_reg_info_cap(entry, i);
                     trace_cap_register(capinfo, cap_handle);
+                    // set cap size
+                    meminfo->set_size(perfetto_target_cap_size());
                 } else {
                     meminfo->set_int_value(perfetto_mem_info_value(entry, i));
+                    meminfo->set_size(
+                        1 << (perfetto_mem_info_op(entry, i) & MO_SIZE));
                 }
             }
 
@@ -450,6 +514,15 @@ qemu_log_instr_perfetto_conf_categories(const char *category_str)
         categories.push_back(strspec);
 }
 
+extern "C" void qemu_log_instr_perfetto_interceptor_logfile(const char *name)
+{
+    mem_logfile_name = name;
+}
+
+extern "C" void qemu_log_instr_perfetto_enable_interceptor()
+{
+    enable_interceptor = true;
+}
 /*
  * Initialize perfetto tracing.
  *

From 7fb7bfa42610cf14c4e38749628093211c6e1adb Mon Sep 17 00:00:00 2001
From: Mingle Chen <mc2262@cl.cam.ac.uk>
Date: Fri, 29 Jul 2022 18:01:57 +0100
Subject: [PATCH 58/74] add gzip compression for interceptor output file

---
 trace_extra/memory_interceptor.cc | 49 +++++++++++++++++++++++++++----
 trace_extra/memory_interceptor.hh | 40 +++++++++++++++++++++----
 trace_extra/meson.build           |  2 +-
 trace_extra/trace_perfetto.cc     | 13 ++++----
 4 files changed, 88 insertions(+), 16 deletions(-)

diff --git a/trace_extra/memory_interceptor.cc b/trace_extra/memory_interceptor.cc
index ad696b03c8a..f948d87d136 100644
--- a/trace_extra/memory_interceptor.cc
+++ b/trace_extra/memory_interceptor.cc
@@ -1,6 +1,34 @@
-//
-// Created by mc2262 on 15/07/22.
-//
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2022 Mingle Chen
+ *
+ * This software was developed by SRI International and the University of
+ * Cambridge Computer Laboratory (Department of Computer Science and
+ * Technology) under Defense Advanced Research Projects Agency (DARPA)
+ * Contract No. HR001122C0110 ("ETC").
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
 
 #include "trace_extra/memory_interceptor.hh"
 
@@ -9,7 +37,7 @@ DynamorioTraceInterceptor::ThreadLocalState::ThreadLocalState(
 {
 }
 
-ofstream DynamorioTraceInterceptor::mem_logfile;
+io::filtering_ostream DynamorioTraceInterceptor::mem_logfile;
 
 void DynamorioTraceInterceptor::OnTracePacket(InterceptorContext context)
 {
@@ -21,14 +49,25 @@ void DynamorioTraceInterceptor::OnTracePacket(InterceptorContext context)
         if (track_event.has_qemu()) {
             perfetto::protos::pbzero::QEMUEventInfo::Decoder qemu(
                 track_event.qemu());
+            // create trace_entry_t compatible with drcachesim
             if (qemu.has_instr()) {
                 perfetto::protos::pbzero::QEMULogEntry::Decoder instr(
                     qemu.instr());
+                if (instr.has_pc() && instr.has_opcode_obj()) {
+                    perfetto::protos::pbzero::Opcode::Decoder opcode(
+                        instr.opcode_obj());
+                    if (opcode.has_size()) {
+                        trace_entry_t entry;
+                        entry.type = TRACE_TYPE_INSTR;
+                        entry.addr = reinterpret_cast<addr_t>(instr.pc());
+                        entry.size = opcode.size();
+                        mem_logfile.write((char *)(&entry), sizeof(entry));
+                    }
+                }
                 if (instr.has_mem()) {
                     for (auto iter = instr.mem(); iter; iter++) {
                         perfetto::protos::pbzero::QEMULogEntryMem::Decoder mem(
                             *iter);
-                        // create a trace_entry_t compatible with drcachesim
                         trace_entry_t trace;
                         trace.addr = reinterpret_cast<addr_t>(mem.addr());
                         trace.size = mem.size();
diff --git a/trace_extra/memory_interceptor.hh b/trace_extra/memory_interceptor.hh
index fdf5d097036..b2f985d8e95 100644
--- a/trace_extra/memory_interceptor.hh
+++ b/trace_extra/memory_interceptor.hh
@@ -1,21 +1,51 @@
-//
-// Created by mc2262 on 15/07/22.
-//
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2022 Mingle Chen
+ *
+ * This software was developed by SRI International and the University of
+ * Cambridge Computer Laboratory (Department of Computer Science and
+ * Technology) under Defense Advanced Research Projects Agency (DARPA)
+ * Contract No. HR001122C0110 ("ETC").
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
 
 #pragma once
 #include <perfetto.h>
 #include <iostream>
-#include <fstream>
 #include "dynamorio/clients/drcachesim/common/trace_entry.h"
+#include <boost/iostreams/device/file_descriptor.hpp>
+#include <boost/iostreams/filtering_stream.hpp>
 
 using namespace std;
+namespace io = boost::iostreams;
 
 class DynamorioTraceInterceptor
     : public perfetto::Interceptor<DynamorioTraceInterceptor>
 {
 
   public:
-    static ofstream mem_logfile;
+    static io::filtering_ostream mem_logfile;
     ~DynamorioTraceInterceptor() override = default;
 
     struct ThreadLocalState
diff --git a/trace_extra/meson.build b/trace_extra/meson.build
index d6a6c9ad740..8f8dbbfb6b4 100644
--- a/trace_extra/meson.build
+++ b/trace_extra/meson.build
@@ -3,7 +3,7 @@ fs = import('fs')
 qemutrace_ss = ss.source_set()
 
 if config_all.has_key('CONFIG_TCG_LOG_INSTR') and config_all.has_key('CONFIG_TRACE_PERFETTO')
-   qemutrace_ss.add(dependency('boost', modules: ['filesystem', 'system']))
+   qemutrace_ss.add(dependency('boost', modules: ['filesystem', 'system', 'iostreams']))
    qemutrace_ss.add(files('trace_perfetto.cc', 'guest_context_tracker.cc', 'trace_counters.cc', 'memory_interceptor.cc',
                           'cheri-perfetto/sdk/perfetto.cc'))
 endif
diff --git a/trace_extra/trace_perfetto.cc b/trace_extra/trace_perfetto.cc
index 0ccff6544f9..9b8d0304cd7 100644
--- a/trace_extra/trace_perfetto.cc
+++ b/trace_extra/trace_perfetto.cc
@@ -45,7 +45,8 @@
 #include <fcntl.h>
 #include <perfetto.h>
 #include <boost/filesystem.hpp>
-
+#include <boost/iostreams/filter/gzip.hpp>
+#include <fstream>
 // #include "qemu/cpu-defs.h"
 #include "qemu/log_instr.h"
 #include "exec/log_instr_internal.h"
@@ -70,7 +71,7 @@ std::unique_ptr<perfetto::TracingSession> session;
 fs::path logfile("qemu_trace.pb");
 
 /* perfetto interceptor trace file */
-string mem_logfile_name = "mem_access.trace";
+string mem_logfile_name = "mem_access.trace.gz";
 
 /* enable perfetto interceptor */
 bool enable_interceptor = false;
@@ -133,8 +134,10 @@ bool perfetto_start_tracing(void)
     perfetto::TrackEvent::Register();
 
     if (enable_interceptor) {
-        DynamorioTraceInterceptor::mem_logfile.open(mem_logfile_name,
-                                                    ios::binary);
+        DynamorioTraceInterceptor::mem_logfile.push(io::gzip_compressor());
+        DynamorioTraceInterceptor::mem_logfile.push(
+            io::file_descriptor_sink(mem_logfile_name));
+
         // drcachesim needs a header in the file, so we create it here
         trace_entry_t header{ .type = TRACE_TYPE_HEADER,
                               .size = 0,
@@ -225,7 +228,7 @@ void perfetto_tracing_stop(void)
         footer.addr = 0;
         DynamorioTraceInterceptor::mem_logfile.write((char *)&footer,
                                                      sizeof(footer));
-        DynamorioTraceInterceptor::mem_logfile.close();
+        io::close(DynamorioTraceInterceptor::mem_logfile);
     }
 }
 

From 746fca32ac45fdb1e69a2fc4ee5f676d535bd9b6 Mon Sep 17 00:00:00 2001
From: Mingle Chen <mc2262@cl.cam.ac.uk>
Date: Wed, 10 Aug 2022 17:36:00 +0100
Subject: [PATCH 59/74] Point dynamrio submodule to CHERI fork

---
 .gitmodules | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitmodules b/.gitmodules
index 0251c513c04..09791be7a69 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -70,4 +70,4 @@
 	branch = cheri-perfetto
 [submodule "trace_extra/dynamorio"]
 	path = trace_extra/dynamorio
-	url = https://github.com/DynamoRIO/dynamorio.git
+	url = https://github.com/CTSRD-CHERI/dynamorio.git

From 1ea363c7da907d12beb258415b6070afb38abdb6 Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Thu, 28 Jul 2022 12:33:04 +0100
Subject: [PATCH 60/74] Drop the instruction tracing backend emit_header hook.

This was essentially unused, and can be replicated during initialization.
---
 accel/tcg/log_instr.c             | 14 ++------------
 accel/tcg/log_instr_cvtrace.c     |  8 +++++++-
 include/exec/log_instr_internal.h |  1 -
 3 files changed, 9 insertions(+), 14 deletions(-)

diff --git a/accel/tcg/log_instr.c b/accel/tcg/log_instr.c
index 54a8a4f6c28..2eed0ac6a60 100644
--- a/accel/tcg/log_instr.c
+++ b/accel/tcg/log_instr.c
@@ -106,32 +106,27 @@ static void emit_nop_entry(CPUArchState *env, cpu_log_entry_t *entry);
 static trace_backend_hooks_t trace_backends[] = {
     { .init = NULL,
       .sync = NULL,
-      .emit_header = NULL,
       .emit_instr = emit_text_instr },
-    { .init = NULL,
+    { .init = emit_cvtrace_header,
       .sync = NULL,
-      .emit_header = emit_cvtrace_header,
       .emit_instr = emit_cvtrace_entry },
     { .init = NULL,
       .sync = NULL,
-      .emit_header = NULL,
       .emit_instr = emit_nop_entry },
 #ifdef CONFIG_TRACE_PERFETTO
     { .init = init_perfetto_backend,
       .sync = sync_perfetto_backend,
-      .emit_header = NULL,
+      .emit_debug = emit_perfetto_debug,
       .emit_instr = emit_perfetto_entry },
 #endif
 #ifdef CONFIG_TRACE_PROTOBUF
     { .init = init_protobuf_backend,
       .sync = sync_protobuf_backend,
-      .emit_header = NULL,
       .emit_instr = emit_protobuf_entry },
 #endif
 #ifdef CONFIG_TRACE_JSON
     { .init = init_json_backend,
       .sync = sync_json_backend,
-      .emit_header = NULL,
       .emit_instr = emit_json_entry },
 #endif
 };
@@ -520,11 +515,6 @@ void qemu_log_instr_init(CPUState *cpu)
     /* Make sure we are using the correct trace format. */
     if (trace_backend == NULL) {
         trace_backend = &trace_backends[qemu_log_instr_backend];
-        /* Only emit header on first init */
-        if (trace_backend->emit_header) {
-            /* XXX this can probably go away and just use init() */
-            trace_backend->emit_header(cpu->env_ptr);
-        }
     }
     /* Initialize backend state on this CPU */
     if (trace_backend->init) {
diff --git a/accel/tcg/log_instr_cvtrace.c b/accel/tcg/log_instr_cvtrace.c
index 133349884a4..fb10f9b1d3f 100644
--- a/accel/tcg/log_instr_cvtrace.c
+++ b/accel/tcg/log_instr_cvtrace.c
@@ -82,9 +82,15 @@ typedef struct {
  */
 void emit_cvtrace_header(CPUArchState *env)
 {
-    FILE *logfile = qemu_log_lock();
+    static bool initialized;
+    FILE *logfile;
     char buffer[sizeof(cheri_trace_entry_t)];
 
+    if (initialized) {
+        return;
+    }
+    initialized = true;
+    logfile = qemu_log_lock();
     buffer[0] = CTE_QEMU_VERSION;
     g_strlcpy(buffer + 1, CTE_QEMU_MAGIC, sizeof(buffer) - 2);
     fwrite(buffer, sizeof(buffer), 1, logfile);
diff --git a/include/exec/log_instr_internal.h b/include/exec/log_instr_internal.h
index 3123ea8c239..a90d47c5ee2 100644
--- a/include/exec/log_instr_internal.h
+++ b/include/exec/log_instr_internal.h
@@ -151,7 +151,6 @@ typedef struct log_meminfo {
 struct trace_backend_hooks {
     void (*init)(CPUArchState *env);
     void (*sync)(CPUArchState *env);
-    void (*emit_header)(CPUArchState *env);
     void (*emit_instr)(CPUArchState *env, cpu_log_entry_t *entry);
 };
 typedef struct trace_backend_hooks trace_backend_hooks_t;

From 8c80e973afc51dc3616b2ec31093f79b17c4a838 Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Sat, 30 Jul 2022 14:43:31 +0100
Subject: [PATCH 61/74] Improve perfetto trace session configuration.

---
 trace_extra/trace_perfetto.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/trace_extra/trace_perfetto.cc b/trace_extra/trace_perfetto.cc
index 9b8d0304cd7..4095fa12266 100644
--- a/trace_extra/trace_perfetto.cc
+++ b/trace_extra/trace_perfetto.cc
@@ -171,7 +171,7 @@ bool perfetto_start_tracing(void)
     }
 
     cfg.add_buffers()->set_size_kb(1 << 19); // 512MiB
-    cfg.set_flush_period_ms(2000);
+    cfg.set_flush_period_ms(5000);
     cfg.set_file_write_period_ms(1000);
     fs::remove(logfile);
     cfg.set_write_into_file(true);
@@ -190,7 +190,7 @@ bool perfetto_start_tracing(void)
 
     auto *producer_cfg = cfg.add_producers();
     producer_cfg->set_producer_name("qemu-tcg");
-    producer_cfg->set_shm_size_kb(1 << 18); // 256MiB
+    producer_cfg->set_shm_size_kb(1 << 15); // 32MiB (the current max allowed)
 
     session = perfetto::Tracing::NewTrace();
 

From 385939df666885daf0e76c62330f8d8d2729666a Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Sat, 30 Jul 2022 14:51:13 +0100
Subject: [PATCH 62/74] Introduce support for logging qemu debugging counters
 with perfetto.

Add two counters to measure instruction count when tracing is enabled/disabled.
---
 accel/tcg/cpu-exec.c                 | 20 ++++++++++++++++++++
 accel/tcg/log_instr.c                | 20 +++++++++++---------
 accel/tcg/log_instr_perfetto.c       |  7 +++++++
 include/exec/log_instr.h             |  6 ++++++
 include/exec/log_instr_internal.h    |  2 ++
 include/exec/log_instr_perfetto.h    |  4 +++-
 include/qemu/log_instr.h             | 13 +++++++++++++
 trace_extra/guest_context_tracker.cc |  8 ++++++++
 trace_extra/guest_context_tracker.hh | 11 +++++++++++
 trace_extra/trace_perfetto.cc        | 11 ++++++++++-
 10 files changed, 91 insertions(+), 11 deletions(-)

diff --git a/accel/tcg/cpu-exec.c b/accel/tcg/cpu-exec.c
index b834006e8ab..d1700f5230d 100644
--- a/accel/tcg/cpu-exec.c
+++ b/accel/tcg/cpu-exec.c
@@ -32,6 +32,7 @@
 #include "exec/tb-hash.h"
 #include "exec/tb-lookup.h"
 #include "exec/log.h"
+#include "exec/log_instr.h"
 #include "qemu/main-loop.h"
 #if defined(TARGET_I386) && !defined(CONFIG_USER_ONLY)
 #include "hw/i386/apic.h"
@@ -701,8 +702,27 @@ static inline void cpu_loop_exec_tb(CPUState *cpu, TranslationBlock *tb,
     /* Instruction counter expired.  */
     assert(icount_enabled());
 #ifndef CONFIG_USER_ONLY
+#if defined(CONFIG_TCG_LOG_INSTR) && defined(CONFIG_TRACE_PERFETTO)
+    {
+        int64_t executed = cpu->icount_budget -
+            (cpu_neg(cpu)->icount_decr.u16.low + cpu->icount_extra);
+        /*
+         * Assume that it is not possible to exit a tb chain with
+         * where tracing occurred but the first or last tb don't have
+         * the tracing flag set.
+         */
+        if (tb->cflags & CF_LOG_INSTR) {
+            qemu_log_instr_counter(cpu, QEMU_LOG_INSTR_DBG_INSN_TRACING_ICOUNT,
+                                   executed);
+        } else {
+            qemu_log_instr_counter(cpu, QEMU_LOG_INSTR_DBG_INSN_ICOUNT,
+                                   executed);
+        }
+    }
+#endif
     /* Ensure global icount has gone forward */
     icount_update(cpu);
+
     /* Refill decrementer and continue execution.  */
     insns_left = MIN(0xffff, cpu->icount_budget);
     cpu_neg(cpu)->icount_decr.u16.low = insns_left;
diff --git a/accel/tcg/log_instr.c b/accel/tcg/log_instr.c
index 2eed0ac6a60..02715c3fe6e 100644
--- a/accel/tcg/log_instr.c
+++ b/accel/tcg/log_instr.c
@@ -104,15 +104,11 @@ static void emit_nop_entry(CPUArchState *env, cpu_log_entry_t *entry);
  * Existing format callbacks list, indexed by qemu_log_instr_backend_t.
  */
 static trace_backend_hooks_t trace_backends[] = {
-    { .init = NULL,
-      .sync = NULL,
-      .emit_instr = emit_text_instr },
+    { .init = NULL, .sync = NULL, .emit_instr = emit_text_instr },
     { .init = emit_cvtrace_header,
       .sync = NULL,
       .emit_instr = emit_cvtrace_entry },
-    { .init = NULL,
-      .sync = NULL,
-      .emit_instr = emit_nop_entry },
+    { .init = NULL, .sync = NULL, .emit_instr = emit_nop_entry },
 #ifdef CONFIG_TRACE_PERFETTO
     { .init = init_perfetto_backend,
       .sync = sync_perfetto_backend,
@@ -593,9 +589,8 @@ void qemu_log_instr_set_buffer_size(unsigned long new_size)
     CPUState *cpu;
 
     if (new_size < MIN_ENTRY_BUFFER_SIZE) {
-        warn_report(
-            "New trace entry buffer size is too small < %zu, ignored.",
-            (size_t)MIN_ENTRY_BUFFER_SIZE);
+        warn_report("New trace entry buffer size is too small < %zu, ignored.",
+                    (size_t)MIN_ENTRY_BUFFER_SIZE);
         return;
     }
 
@@ -1284,6 +1279,13 @@ void qemu_log_instr_flush(CPUArchState *env)
     cpulog->ring_tail = cpulog->ring_head;
 }
 
+void qemu_log_instr_counter(CPUState *cpu, QEMUDebugCounter name, long value)
+{
+    if (trace_backend->emit_debug) {
+        trace_backend->emit_debug(cpu->env_ptr, name, value);
+    }
+}
+
 /* Instruction logging helpers */
 
 /* Dump out all the accumalated printf's */
diff --git a/accel/tcg/log_instr_perfetto.c b/accel/tcg/log_instr_perfetto.c
index 282d05e695e..984fbbc89b0 100644
--- a/accel/tcg/log_instr_perfetto.c
+++ b/accel/tcg/log_instr_perfetto.c
@@ -62,6 +62,13 @@ void sync_perfetto_backend(CPUArchState *env)
     perfetto_sync_cpu(cpulog->backend_data);
 }
 
+void emit_perfetto_debug(CPUArchState *env, QEMUDebugCounter name, long value)
+{
+    cpu_log_instr_state_t *cpulog = get_cpu_log_state(env);
+
+    perfetto_emit_debug(cpulog->backend_data, name, value);
+}
+
 void emit_perfetto_entry(CPUArchState *env, cpu_log_entry_t *entry)
 {
     cpu_log_instr_state_t *cpulog = get_cpu_log_state(env);
diff --git a/include/exec/log_instr.h b/include/exec/log_instr.h
index 6ccecec5d81..9e606c8c48b 100644
--- a/include/exec/log_instr.h
+++ b/include/exec/log_instr.h
@@ -350,6 +350,12 @@ void qemu_log_instr_event_dump_cap_int(log_event_t *evt, const char *reg_name,
  */
 void qemu_log_instr_extra(CPUArchState *env, const char *msg, ...);
 
+/*
+ * Tracepoint for qemu internals.
+ * This should not be used for guest events.
+ */
+void qemu_log_instr_counter(CPUState *cpu, QEMUDebugCounter name, long value);
+
 #else /* ! CONFIG_TCG_LOG_INSTR */
 #define qemu_log_instr_enabled(cpu) false
 #define qemu_log_instr_start(env, mode, pc)
diff --git a/include/exec/log_instr_internal.h b/include/exec/log_instr_internal.h
index a90d47c5ee2..09f08478307 100644
--- a/include/exec/log_instr_internal.h
+++ b/include/exec/log_instr_internal.h
@@ -151,6 +151,7 @@ typedef struct log_meminfo {
 struct trace_backend_hooks {
     void (*init)(CPUArchState *env);
     void (*sync)(CPUArchState *env);
+    void (*emit_debug)(CPUArchState *env, QEMUDebugCounter name, long value);
     void (*emit_instr)(CPUArchState *env, cpu_log_entry_t *entry);
 };
 typedef struct trace_backend_hooks trace_backend_hooks_t;
@@ -170,6 +171,7 @@ void emit_cvtrace_entry(CPUArchState *env, cpu_log_entry_t *entry);
 /* Perfetto backend */
 void init_perfetto_backend(CPUArchState *env);
 void sync_perfetto_backend(CPUArchState *env);
+void emit_perfetto_debug(CPUArchState *env, QEMUDebugCounter name, long value);
 void emit_perfetto_entry(CPUArchState *env, cpu_log_entry_t *entry);
 #endif
 #ifdef CONFIG_TRACE_PROTOBUF
diff --git a/include/exec/log_instr_perfetto.h b/include/exec/log_instr_perfetto.h
index 17227fdcc54..b82c20d225a 100644
--- a/include/exec/log_instr_perfetto.h
+++ b/include/exec/log_instr_perfetto.h
@@ -65,7 +65,8 @@ PERFETTO_DEFINE_CATEGORIES(
     perfetto::Category("counter").SetDescription("Guest-driven counters"),
     perfetto::Category("bb_hit").SetDescription("Basic Block hit counts"),
     perfetto::Category("bb_icount").SetDescription("Basic Block instr counts"),
-    perfetto::Category("br_hit").SetDescription("Branch hit count"));
+    perfetto::Category("br_hit").SetDescription("Branch hit count"),
+    perfetto::Category("qemu-debug").SetDescription("QEMU debug counters"));
 #endif
 
 /*
@@ -87,6 +88,7 @@ typedef void *cap_register_handle;
 
 void perfetto_init_cpu(int cpu_index, void **backend_data);
 void perfetto_sync_cpu(void *backend_data);
+void perfetto_emit_debug(void *backend_data, QEMUDebugCounter name, long value);
 void perfetto_emit_instr(void *backend_data, cpu_log_entry_handle entry_handle);
 
 /*
diff --git a/include/qemu/log_instr.h b/include/qemu/log_instr.h
index 0e8dbbc951e..32c74b7c820 100644
--- a/include/qemu/log_instr.h
+++ b/include/qemu/log_instr.h
@@ -202,6 +202,19 @@ typedef struct {
     };
 } log_event_t;
 
+/*
+ * Internal QEMU debug counters.
+ * These are always traced, regardless of whether tracing is enabled or not.
+ * This enumeration defines the name of the debug counters for the backends
+ * to use.
+ */
+typedef enum {
+    QEMU_LOG_INSTR_DBG_INSN_ICOUNT = 0,
+    QEMU_LOG_INSTR_DBG_INSN_TRACING_ICOUNT = 1,
+    /* Must be last */
+    QEMU_LOG_INSTR_DBG_MAX
+} QEMUDebugCounter;
+
 #ifdef CONFIG_TRACE_PERFETTO
 /* Perfetto backend configuration hooks */
 #ifdef __cplusplus
diff --git a/trace_extra/guest_context_tracker.cc b/trace_extra/guest_context_tracker.cc
index ce7019f2bc7..584d382f836 100644
--- a/trace_extra/guest_context_tracker.cc
+++ b/trace_extra/guest_context_tracker.cc
@@ -51,6 +51,14 @@ std::unordered_map<qemu_ctx_id, std::shared_ptr<qemu_context_state>,
                    tuple_hasher<qemu_ctx_id>>
     track_state_map;
 
+/*
+ * Constant mapping of QEMUDebugCounter IDs to counter track names
+ */
+const std::array<const std::string, QEMU_LOG_INSTR_DBG_MAX> debug_counter_names{
+    "tb_icount",
+    "tb_icount_tracing",
+};
+
 /* Helper to generate unique IDs for dynamic tracks */
 unsigned long gen_track_uuid()
 {
diff --git a/trace_extra/guest_context_tracker.hh b/trace_extra/guest_context_tracker.hh
index 665afd84091..357d36c7417 100644
--- a/trace_extra/guest_context_tracker.hh
+++ b/trace_extra/guest_context_tracker.hh
@@ -28,6 +28,7 @@
 
 #pragma once
 
+#include <array>
 #include <memory>
 #include <mutex>
 #include <iterator>
@@ -60,6 +61,9 @@ unsigned long gen_track_uuid();
 perfetto::protos::pbzero::QEMULogEntryModeSwitch
 qemu_cpu_mode_to_trace(qemu_log_instr_cpu_mode_t mode);
 
+extern const std::array<const std::string, QEMU_LOG_INSTR_DBG_MAX>
+    debug_counter_names;
+
 /*
  * Custom track representing a QEMU guest context.
  */
@@ -212,6 +216,13 @@ struct qemu_fallback_state : public qemu_tracker_state {
         perfetto::TrackEvent::EraseTrackDescriptor(track);
     }
 
+    perfetto::CounterTrack
+    get_debug_counter_track(QEMUDebugCounter counter_index)
+    {
+        return perfetto::CounterTrack(
+            debug_counter_names[counter_index].c_str(), track);
+    }
+
     perfetto::Track *get_track()
     {
         return &track;
diff --git a/trace_extra/trace_perfetto.cc b/trace_extra/trace_perfetto.cc
index 4095fa12266..af1e91352f5 100644
--- a/trace_extra/trace_perfetto.cc
+++ b/trace_extra/trace_perfetto.cc
@@ -282,7 +282,6 @@ bool process_state_event(perfetto_backend_data *data,
 void process_context_event(perfetto_backend_data *data,
                            cpu_log_entry_handle entry, log_event_t *evt)
 {
-    auto *state = data->ctx_tracker.get_ctx_state();
 
     /* Swap current context. */
     if (evt->ctx_update.op == LOG_EVENT_CTX_OP_SWITCH) {
@@ -556,6 +555,16 @@ extern "C" void perfetto_sync_cpu(void *backend_data)
     perfetto::TrackEvent::Flush();
 }
 
+extern "C" void perfetto_emit_debug(void *backend_data, QEMUDebugCounter index,
+                                    long value)
+{
+    auto *data = reinterpret_cast<perfetto_backend_data *>(backend_data);
+    qemu_fallback_state *cpu_state = data->ctx_tracker.get_cpu_state();
+
+    TRACE_COUNTER("qemu-debug", cpu_state->get_debug_counter_track(index),
+                  value);
+}
+
 extern "C" void perfetto_emit_instr(void *backend_data,
                                     cpu_log_entry_handle entry)
 {

From 6a44e1d7abc89898f14628ca2cd10d6bf5fd2d08 Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Sat, 30 Jul 2022 15:46:13 +0100
Subject: [PATCH 63/74] Disable branch and basic block tracking when
 unnecessary.

Only maintain the branch and basic block hit histograms when
the corresponding trace categories are enabled.
---
 trace_extra/guest_context_tracker.cc | 57 +++++++++++++++++++---------
 1 file changed, 39 insertions(+), 18 deletions(-)

diff --git a/trace_extra/guest_context_tracker.cc b/trace_extra/guest_context_tracker.cc
index 584d382f836..24ee0386af6 100644
--- a/trace_extra/guest_context_tracker.cc
+++ b/trace_extra/guest_context_tracker.cc
@@ -145,6 +145,11 @@ qemu_ctx_id qemu_context_track::get_id() const
 
 void qemu_bb_tracker::track_next(cpu_log_entry_handle entry, uint64_t pc)
 {
+    if (!TRACE_EVENT_CATEGORY_ENABLED("bb_hit") &&
+        !TRACE_EVENT_CATEGORY_ENABLED("br_hit")) {
+        return;
+    }
+
     if (pc == 0) {
         pc = perfetto_log_entry_pc(entry);
     }
@@ -174,6 +179,11 @@ void qemu_bb_tracker::track_next(cpu_log_entry_handle entry, uint64_t pc)
 
 void qemu_bb_tracker::reset(uint64_t pc)
 {
+    if (!TRACE_EVENT_CATEGORY_ENABLED("bb_hit") &&
+        !TRACE_EVENT_CATEGORY_ENABLED("br_hit")) {
+        return;
+    }
+
     if (bb_start == 0) {
         /* Nothing to flush */
         return;
@@ -232,9 +242,13 @@ void guest_context_tracker::mode_update(
             state = it->second;
         }
     }
-    if (ctx_state && ctx_state != state) {
-        /* We are changing context */
-        ctx_state->bb_tracker->reset(perfetto_log_entry_pc(entry));
+
+    if (TRACE_EVENT_CATEGORY_ENABLED("bb_hit") ||
+        TRACE_EVENT_CATEGORY_ENABLED("br_hit")) {
+        if (ctx_state && ctx_state != state) {
+            /* We are changing context */
+            ctx_state->bb_tracker->reset(perfetto_log_entry_pc(entry));
+        }
     }
     ctx_state = state;
 }
@@ -268,9 +282,13 @@ void guest_context_tracker::context_update(cpu_log_entry_handle entry,
             state = it->second;
         }
     }
-    if (ctx_state && ctx_state != state) {
-        /* We are changing context */
-        ctx_state->bb_tracker->reset(perfetto_log_entry_pc(entry));
+
+    if (TRACE_EVENT_CATEGORY_ENABLED("bb_hit") ||
+        TRACE_EVENT_CATEGORY_ENABLED("br_hit")) {
+        if (ctx_state && ctx_state != state) {
+            /* We are changing context */
+            ctx_state->bb_tracker->reset(perfetto_log_entry_pc(entry));
+        }
     }
     ctx_state = state;
 }
@@ -278,18 +296,21 @@ void guest_context_tracker::context_update(cpu_log_entry_handle entry,
 void guest_context_tracker::flush_all_ctx_data(uint64_t pc)
 {
     /* Flush per-CPU and current context stats */
-    cpu_state.bb_tracker->reset(pc);
-    cpu_state.bb_tracker->flush();
-    if (ctx_state) {
-        ctx_state->bb_tracker->reset(pc);
-        ctx_state->bb_tracker->flush();
-    }
-    std::lock_guard<std::mutex> track_state_guard(track_state_lock);
-    for (auto id_and_state : track_state_map) {
-        qemu_bb_tracker *bb_tracker = id_and_state.second->bb_tracker.get();
-        if (bb_tracker != ctx_state->bb_tracker.get()) {
-            bb_tracker->reset(0);
-            bb_tracker->flush();
+    if (TRACE_EVENT_CATEGORY_ENABLED("bb_hit") ||
+        TRACE_EVENT_CATEGORY_ENABLED("br_hit")) {
+        cpu_state.bb_tracker->reset(pc);
+        cpu_state.bb_tracker->flush();
+        if (ctx_state) {
+            ctx_state->bb_tracker->reset(pc);
+            ctx_state->bb_tracker->flush();
+        }
+        std::lock_guard<std::mutex> track_state_guard(track_state_lock);
+        for (auto id_and_state : track_state_map) {
+            qemu_bb_tracker *bb_tracker = id_and_state.second->bb_tracker.get();
+            if (bb_tracker != ctx_state->bb_tracker.get()) {
+                bb_tracker->reset(0);
+                bb_tracker->flush();
+            }
         }
     }
 }

From 9660d3b26be5428e3f66bf7c5fd4619b9838eefb Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Thu, 18 Aug 2022 14:25:40 +0100
Subject: [PATCH 64/74] Disable tb_flush when switching log level.

This should not be needed because we have flags marking TBs whenever
they contain instruction tracing instrumentation.
---
 accel/tcg/log_instr.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/accel/tcg/log_instr.c b/accel/tcg/log_instr.c
index 02715c3fe6e..501adfc7da4 100644
--- a/accel/tcg/log_instr.c
+++ b/accel/tcg/log_instr.c
@@ -330,7 +330,7 @@ static void do_cpu_loglevel_switch(CPUState *cpu, run_on_cpu_data data)
         prev_level_active == next_level_active) {
         goto done;
     }
-    tb_flush(cpu);
+    /* tb_flush(cpu); */
     /* Emit start/stop events */
     if (prev_level_active) {
         if (cpulog->starting) {

From 717464cf11436d7197711c03d7518c735ca5967b Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Wed, 1 Feb 2023 17:37:51 +0000
Subject: [PATCH 65/74] Fix io_uring_prep_poll_remove call.

This is broken by recent liburing versions.
---
 util/fdmon-io_uring.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/util/fdmon-io_uring.c b/util/fdmon-io_uring.c
index 1461dfa4074..cb3e057b2f5 100644
--- a/util/fdmon-io_uring.c
+++ b/util/fdmon-io_uring.c
@@ -179,7 +179,7 @@ static void add_poll_remove_sqe(AioContext *ctx, AioHandler *node)
 {
     struct io_uring_sqe *sqe = get_sqe(ctx);
 
-    io_uring_prep_poll_remove(sqe, node);
+    io_uring_prep_poll_remove(sqe, (uintptr_t)node);
 }
 
 /* Add a timeout that self-cancels when another cqe becomes ready */

From 82b638fed125b73e4ba2303328b3ed752612ef1e Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Fri, 31 Mar 2023 16:50:47 +0100
Subject: [PATCH 66/74] Always use a fixed number of entries in the trace
 backends list.

---
 accel/tcg/log_instr.c | 32 ++++++++++++++++++++++----------
 1 file changed, 22 insertions(+), 10 deletions(-)

diff --git a/accel/tcg/log_instr.c b/accel/tcg/log_instr.c
index 501adfc7da4..e8d53270fe1 100644
--- a/accel/tcg/log_instr.c
+++ b/accel/tcg/log_instr.c
@@ -104,27 +104,39 @@ static void emit_nop_entry(CPUArchState *env, cpu_log_entry_t *entry);
  * Existing format callbacks list, indexed by qemu_log_instr_backend_t.
  */
 static trace_backend_hooks_t trace_backends[] = {
-    { .init = NULL, .sync = NULL, .emit_instr = emit_text_instr },
-    { .init = emit_cvtrace_header,
+    {
+      .init = init_text_backend,
+      .sync = sync_text_backend,
+      .pre_commit = text_pre_commit_instr,
+      .emit_instr = emit_text_instr
+    }, {
+      .emit_instr = emit_nop_entry
+    }, {
+#ifdef CONFIG_TRACE_CVTRACE
+      .init = emit_cvtrace_header,
       .sync = NULL,
-      .emit_instr = emit_cvtrace_entry },
-    { .init = NULL, .sync = NULL, .emit_instr = emit_nop_entry },
+      .emit_instr = emit_cvtrace_entry
+#endif
+    }, {
 #ifdef CONFIG_TRACE_PERFETTO
-    { .init = init_perfetto_backend,
+      .init = init_perfetto_backend,
       .sync = sync_perfetto_backend,
       .emit_debug = emit_perfetto_debug,
-      .emit_instr = emit_perfetto_entry },
+      .emit_instr = emit_perfetto_entry
 #endif
+    }, {
 #ifdef CONFIG_TRACE_PROTOBUF
-    { .init = init_protobuf_backend,
+      .init = init_protobuf_backend,
       .sync = sync_protobuf_backend,
-      .emit_instr = emit_protobuf_entry },
+      .emit_instr = emit_protobuf_entry
 #endif
+    }, {
 #ifdef CONFIG_TRACE_JSON
-    { .init = init_json_backend,
+      .init = init_json_backend,
       .sync = sync_json_backend,
-      .emit_instr = emit_json_entry },
+      .emit_instr = emit_json_entry
 #endif
+    },
 };
 
 /* Existing trace filters list, indexed by cpu_log_instr_filter_t */

From 6a9bbaa493a0b18656df5b4f82496fc2fc19ac81 Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Fri, 31 Mar 2023 16:59:48 +0100
Subject: [PATCH 67/74] Update disasm helpers to emit single instructions as
 strings.

Add disas_one_strbuf() to disassemble one instruction to an existing
string buffer.
Add disas_one_str() to disassemble one instruction to a dynamically
allocated string.
---
 disas.c               | 62 +++++++++++++++++++++++++++++++++++++------
 include/disas/disas.h |  7 +++--
 2 files changed, 59 insertions(+), 10 deletions(-)

diff --git a/disas.c b/disas.c
index df82a451146..a54b3bd7e35 100644
--- a/disas.c
+++ b/disas.c
@@ -355,19 +355,16 @@ char *plugin_disas(CPUState *cpu, uint64_t addr, size_t size)
     return g_string_free(ds, false);
 }
 
-/*
- * Disassemble one instruction to a string buffer.
- */
-char *disas_one_strbuf(CPUState *cpu, void *code, unsigned long size,
-                       target_ulong vma)
+static inline void do_disas_one(CPUState *cpu, void *code, unsigned long size, target_ulong vma, fprintf_function dump_fn, void *dump_arg)
 {
     CPUClass *cc = CPU_GET_CLASS(cpu);
     CPUDebug s;
-    GString *ds = g_string_new(NULL);
 
     initialize_debug_target(&s, cpu);
-    s.info.fprintf_func = plugin_printf;
-    s.info.stream = (FILE *)ds;  /* abuse this slot */
+    s.info.read_memory_func = host_read_memory;
+    s.info.fprintf_func = dump_fn;
+    s.info.stream = (FILE *)dump_arg;  /* abuse this slot */
+    s.info.buffer = code;
     s.info.buffer_vma = vma;
     s.info.buffer_length = size;
 
@@ -388,11 +385,60 @@ char *disas_one_strbuf(CPUState *cpu, void *code, unsigned long size,
     } else {
         ; /* cannot disassemble -- return empty string */
     }
+}
+
+/*
+ * Disassemble one instruction to a buffer.
+ * The string returned must be freed by the caller with g_free().
+ */
+char *disas_one_str(CPUState *cpu, void *code, unsigned long size,
+                    target_ulong vma)
+{
+    GString *ds = g_string_new(NULL);
+
+    do_disas_one(cpu, code, size, vma, plugin_printf, ds);
 
     /* Return the buffer, freeing the GString container.  */
     return g_string_free(ds, false);
 }
 
+struct disas_one_args {
+    char *buf;
+    int maxlen;
+};
+
+static int strbuf_printf(FILE *stream, const char *fmt, ...)
+{
+    struct disas_one_args *args = (struct disas_one_args *)stream;
+    va_list va;
+    int n;
+
+    va_start(va, fmt);
+    n = vsnprintf(args->buf, args->maxlen, fmt, va);
+    va_end(va);
+
+    /* Update args for append behaviour */
+    if (n >= args->maxlen) {
+        n = args->maxlen;
+        args->maxlen = 0;
+    } else {
+        args->buf += n;
+        args->maxlen -= n;
+    }
+
+    return n;
+}
+
+/*
+ * Disassemble one instruction to an existing, fixed size, string buffer.
+ */
+void disas_one_strbuf(CPUState *cpu, void *code, unsigned long size,
+                       target_ulong vma, char *buf, int bufsize)
+{
+    struct disas_one_args args = {buf, bufsize};
+    do_disas_one(cpu, code, size, vma, strbuf_printf, &args);
+}
+
 /* Disassemble this for me please... (debugging). */
 void disas(FILE *out, const void *code, unsigned long size)
 {
diff --git a/include/disas/disas.h b/include/disas/disas.h
index 3d43b62c4b1..c22c611e485 100644
--- a/include/disas/disas.h
+++ b/include/disas/disas.h
@@ -18,8 +18,11 @@ void monitor_disas(Monitor *mon, CPUState *cpu,
 
 char *plugin_disas(CPUState *cpu, uint64_t addr, size_t size);
 
-char *disas_one_strbuf(CPUState *cpu, void *code, unsigned long size,
-                       target_ulong vma);
+char *disas_one_str(CPUState *cpu, void *code, unsigned long size,
+                    target_ulong vma);
+
+void disas_one_strbuf(CPUState *cpu, void *code, unsigned long size,
+                      target_ulong vma, char *buf, int bufsize);
 
 /* Look up symbol for debugging purpose.  Returns "" if unknown. */
 const char *lookup_symbol(target_ulong orig_addr);

From e952789a1ecfa4a8428293275164324e35f52c5d Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Fri, 31 Mar 2023 16:52:15 +0100
Subject: [PATCH 68/74] Use disas_one_str in protobuf backend.

---
 accel/tcg/log_instr_protobuf.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/accel/tcg/log_instr_protobuf.c b/accel/tcg/log_instr_protobuf.c
index fbb8ff4c727..2ee9b106591 100644
--- a/accel/tcg/log_instr_protobuf.c
+++ b/accel/tcg/log_instr_protobuf.c
@@ -254,7 +254,7 @@ static void release_protobuf_event(QEMULogEntryEvt *pb_evt)
     g_free(pb_evt);
 }
 
-void init_protobuf_backend(CPUArchState *env)
+void init_protobuf_backend(CPUArchState *env, const char *trace_logfile)
 {
     /*
      * TODO should handle a separate log file to avoid pollution of the main log
@@ -266,7 +266,7 @@ void init_protobuf_backend(CPUArchState *env)
      */
 
     if (protobuf_logfile == NULL) {
-        protobuf_logfile = fopen("qemu-trace.pb", "w+b");
+        protobuf_logfile = fopen(trace_logfile, "w+b");
     }
 }
 
@@ -293,8 +293,8 @@ void emit_protobuf_entry(CPUArchState *env, cpu_log_entry_t *entry)
 
     if (entry->flags & LI_FLAG_HAS_INSTR_DATA) {
         pb_entry.insn_case = ENUM_ENTRY_INSN_CASE(DISAS);
-        pb_entry.disas = disas_one_strbuf(env_cpu(env), entry->insn_bytes,
-                                          sizeof(entry->insn_bytes), entry->pc);
+        pb_entry.disas = disas_one_str(env_cpu(env), entry->insn_bytes,
+                                       sizeof(entry->insn_bytes), entry->pc);
         pb_entry.cpu = env_cpu(env)->cpu_index;
         pb_entry.asid = entry->asid;
         if (entry->flags & LI_FLAG_MODE_SWITCH) {

From b1ba45d1c01aad980ad6aa7e7b6b3414e39bcf37 Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Fri, 31 Mar 2023 19:17:47 +0100
Subject: [PATCH 69/74] Add explicit file parameter to non-text tracing
 backends.

- Conditionally compile cvtrace
- This is done because some of these backends emit binary data that
  is logically incompatible with the standard qemu output file.
  The downside is the we need to handle our own locking if we
  want to support multiple CPUs.
---
 accel/tcg/log_instr.c             | 33 ++++++++++++++-----------------
 accel/tcg/log_instr_json.c        |  2 +-
 accel/tcg/log_instr_perfetto.c    | 11 ++---------
 accel/tcg/log_instr_protobuf.c    | 11 +----------
 include/exec/log_instr_internal.h | 31 ++++++++++++++++++++++-------
 include/exec/log_instr_perfetto.h |  2 +-
 include/qemu/log_instr.h          | 12 +++++------
 qemu-options.hx                   | 10 +++++-----
 softmmu/vl.c                      | 13 +++++-------
 trace_extra/trace_perfetto.cc     |  9 +++------
 10 files changed, 62 insertions(+), 72 deletions(-)

diff --git a/accel/tcg/log_instr.c b/accel/tcg/log_instr.c
index e8d53270fe1..8e66b3b7295 100644
--- a/accel/tcg/log_instr.c
+++ b/accel/tcg/log_instr.c
@@ -69,14 +69,6 @@
  * active. Each CPU holds a private logging state, that can be controlled
  * individually.
  *
- * TODO(am2419): how do we deal with orderding in case multiple registers are
- * updated? This is critical to recognize which value goes in which register,
- * and also how to tie multiple memory accesses to the respective
- * value/register. We could add an explicit target-specific register ID handle
- * in place of the register name. This could be used also to fetch the register
- * name and would provide an identifier to external parsers. Memory updates are
- * harder to deal with, at least in the current format, perhaps the semantic of
- * the instruction is enough to recover the ordering from a trace.
  */
 
 #ifdef CONFIG_TCG_LOG_INSTR
@@ -95,6 +87,8 @@ extern GArray *debug_regions;
 /* Global trace format selector. Defaults to text tracing */
 qemu_log_instr_backend_t qemu_log_instr_backend = QEMU_LOG_INSTR_BACKEND_TEXT;
 
+static const char *trace_logfile_name = "qemu_trace.txt";
+
 /* Current format callbacks. */
 static trace_backend_hooks_t *trace_backend;
 
@@ -105,9 +99,6 @@ static void emit_nop_entry(CPUArchState *env, cpu_log_entry_t *entry);
  */
 static trace_backend_hooks_t trace_backends[] = {
     {
-      .init = init_text_backend,
-      .sync = sync_text_backend,
-      .pre_commit = text_pre_commit_instr,
       .emit_instr = emit_text_instr
     }, {
       .emit_instr = emit_nop_entry
@@ -145,12 +136,7 @@ static cpu_log_instr_filter_fn_t trace_filters[];
 /* Trace filters to activate when a new CPU is seen */
 static GArray *reset_filters;
 
-/* Number of per-cpu ring buffer entries for ring-buffer tracing mode */
-#define MIN_ENTRY_BUFFER_SIZE (1 << 16)
-
-static unsigned long reset_entry_buffer_size = MIN_ENTRY_BUFFER_SIZE;
-
-static bool trace_debug;
+bool trace_debug;
 
 static void emit_nop_entry(CPUArchState *env, cpu_log_entry_t *entry)
 {
@@ -487,6 +473,16 @@ static void qemu_log_entry_destroy(gpointer data)
     g_array_free(entry->events, true);
 }
 
+/*
+ * CLI parameters hook to configure the logfile for instruction traces.
+ * We stop reusing the qemu logfile because we don't want to RCU-lock it
+ * and we want to run our own aio context with it.
+ */
+void qemu_log_instr_conf_logfile(const char *name)
+{
+    trace_logfile_name = name;
+}
+
 /*
  * This must be called upon cpu creation.
  * Initializes the per-CPU logging state and data structures.
@@ -524,9 +520,10 @@ void qemu_log_instr_init(CPUState *cpu)
     if (trace_backend == NULL) {
         trace_backend = &trace_backends[qemu_log_instr_backend];
     }
+
     /* Initialize backend state on this CPU */
     if (trace_backend->init) {
-        trace_backend->init(cpu->env_ptr);
+        trace_backend->init(cpu->env_ptr, trace_logfile_name);
     }
 
     /* If we are starting with instruction logging enabled, switch it on now */
diff --git a/accel/tcg/log_instr_json.c b/accel/tcg/log_instr_json.c
index 80dec851c4b..4a9474bd0a2 100644
--- a/accel/tcg/log_instr_json.c
+++ b/accel/tcg/log_instr_json.c
@@ -173,7 +173,7 @@ void sync_json_backend(CPUArchState *env)
     qemu_log("{}]");
 }
 
-void init_json_backend(CPUArchState *env)
+void init_json_backend(CPUArchState *env, const char *logfile)
 {
     /* Initialize the json logfile */
     static bool initialized;
diff --git a/accel/tcg/log_instr_perfetto.c b/accel/tcg/log_instr_perfetto.c
index 984fbbc89b0..71d62b426f4 100644
--- a/accel/tcg/log_instr_perfetto.c
+++ b/accel/tcg/log_instr_perfetto.c
@@ -46,12 +46,12 @@
  *
  * Start tracing thread when first called.
  */
-void init_perfetto_backend(CPUArchState *env)
+void init_perfetto_backend(CPUArchState *env, const char *logfile)
 {
     cpu_log_instr_state_t *cpulog = get_cpu_log_state(env);
     int cpu_id = env_cpu(env)->cpu_index;
 
-    perfetto_init_cpu(cpu_id, &cpulog->backend_data);
+    perfetto_init_cpu(cpu_id, logfile, &cpulog->backend_data);
 }
 
 /* Syncronize buffers on this CPU. */
@@ -62,13 +62,6 @@ void sync_perfetto_backend(CPUArchState *env)
     perfetto_sync_cpu(cpulog->backend_data);
 }
 
-void emit_perfetto_debug(CPUArchState *env, QEMUDebugCounter name, long value)
-{
-    cpu_log_instr_state_t *cpulog = get_cpu_log_state(env);
-
-    perfetto_emit_debug(cpulog->backend_data, name, value);
-}
-
 void emit_perfetto_entry(CPUArchState *env, cpu_log_entry_t *entry)
 {
     cpu_log_instr_state_t *cpulog = get_cpu_log_state(env);
diff --git a/accel/tcg/log_instr_protobuf.c b/accel/tcg/log_instr_protobuf.c
index 2ee9b106591..9fe883ff9df 100644
--- a/accel/tcg/log_instr_protobuf.c
+++ b/accel/tcg/log_instr_protobuf.c
@@ -1,7 +1,7 @@
 /*
  * SPDX-License-Identifier: BSD-2-Clause
  *
- * Copyright (c) 2021 Alfredo Mazzinghi
+ * Copyright (c) 2021, 2023 Alfredo Mazzinghi
  *
  * This software was developed by SRI International and the University of
  * Cambridge Computer Laboratory (Department of Computer Science and
@@ -257,10 +257,6 @@ static void release_protobuf_event(QEMULogEntryEvt *pb_evt)
 void init_protobuf_backend(CPUArchState *env, const char *trace_logfile)
 {
     /*
-     * TODO should handle a separate log file to avoid pollution of the main log
-     * file with binary data from protobufs. Since this seems to be a common
-     * pattern, maybe generalize it.
-     *
      * XXX we may want to store a cache of protobuf structures in the backend
      * data so that we avoid all the g_malloc() and g_free().
      */
@@ -391,8 +387,3 @@ void emit_protobuf_entry(CPUArchState *env, cpu_log_entry_t *entry)
     }
     g_free(buf);
 }
-
-void qemu_log_instr_protobuf_conf_logfile(const char *name)
-{
-    protobuf_logfile = fopen(name, "w+b");
-}
diff --git a/include/exec/log_instr_internal.h b/include/exec/log_instr_internal.h
index 09f08478307..4b5d7c003eb 100644
--- a/include/exec/log_instr_internal.h
+++ b/include/exec/log_instr_internal.h
@@ -149,9 +149,9 @@ typedef struct log_meminfo {
  * binary or text format.
  */
 struct trace_backend_hooks {
-    void (*init)(CPUArchState *env);
+    void (*init)(CPUArchState *env, const char *logfile);
     void (*sync)(CPUArchState *env);
-    void (*emit_debug)(CPUArchState *env, QEMUDebugCounter name, long value);
+    void (*pre_commit)(CPUArchState *env, cpu_log_entry_t *entry);
     void (*emit_instr)(CPUArchState *env, cpu_log_entry_t *entry);
 };
 typedef struct trace_backend_hooks trace_backend_hooks_t;
@@ -162,25 +162,33 @@ typedef struct trace_backend_hooks trace_backend_hooks_t;
  */
 typedef bool (*cpu_log_instr_filter_fn_t)(struct cpu_log_entry *entry);
 
+/*
+ * Trace debug mode active. This should be constant during execution,
+ * it is only set via CLI flags before CPU execution.
+ */
+extern bool trace_debug;
+
 /* Text backend */
+void init_text_backend(CPUArchState *env, const char *logfile);
+void sync_text_backend(CPUArchState *env);
+void text_pre_commit_instr(CPUArchState *env, cpu_log_entry_t *entry);
 void emit_text_instr(CPUArchState *env, cpu_log_entry_t *entry);
 /* CVTrace backend */
 void emit_cvtrace_header(CPUArchState *env);
 void emit_cvtrace_entry(CPUArchState *env, cpu_log_entry_t *entry);
 #ifdef CONFIG_TRACE_PERFETTO
 /* Perfetto backend */
-void init_perfetto_backend(CPUArchState *env);
+void init_perfetto_backend(CPUArchState *env, const char *logfile);
 void sync_perfetto_backend(CPUArchState *env);
-void emit_perfetto_debug(CPUArchState *env, QEMUDebugCounter name, long value);
 void emit_perfetto_entry(CPUArchState *env, cpu_log_entry_t *entry);
 #endif
 #ifdef CONFIG_TRACE_PROTOBUF
-void init_protobuf_backend(CPUArchState *env);
+void init_protobuf_backend(CPUArchState *env, const char *logfile);
 void sync_protobuf_backend(CPUArchState *env);
 void emit_protobuf_entry(CPUArchState *env, cpu_log_entry_t *entry);
 #endif
 #ifdef CONFIG_TRACE_JSON
-void init_json_backend(CPUArchState *env);
+void init_json_backend(CPUArchState *env, const char *logfile);
 void sync_json_backend(CPUArchState *env);
 void emit_json_entry(CPUArchState *env, cpu_log_entry_t *entry);
 #endif
@@ -199,8 +207,17 @@ static inline cpu_log_instr_state_t *get_cpu_log_state(CPUArchState *env)
     return &env_cpu(env)->log_state;
 }
 
+/* Minmum number of per-cpu ring buffer entries */
+#define MIN_ENTRY_BUFFER_SIZE (1 << 16)
+#define DEFAULT_ENTRY_BUFFER_SIZE MIN_ENTRY_BUFFER_SIZE
+
+/*
 /*
- * Fetch the given cpu current instruction info
+ * Fetch the log entry we are currently populating.
+ *
+ * This should only be called on the CPU thread, where instructions are
+ * recorded. The access to ring_head is not synchronized as it is only possible
+ * to have concurrent reads while populating the entry.
  */
 static inline cpu_log_entry_t *get_cpu_log_entry(CPUArchState *env)
 {
diff --git a/include/exec/log_instr_perfetto.h b/include/exec/log_instr_perfetto.h
index b82c20d225a..c72dd812b4a 100644
--- a/include/exec/log_instr_perfetto.h
+++ b/include/exec/log_instr_perfetto.h
@@ -86,7 +86,7 @@ struct log_meminfo;
 typedef struct log_meminfo *mem_info_handle;
 typedef void *cap_register_handle;
 
-void perfetto_init_cpu(int cpu_index, void **backend_data);
+void perfetto_init_cpu(int cpu_index, const char *logfile, void **backend_data);
 void perfetto_sync_cpu(void *backend_data);
 void perfetto_emit_debug(void *backend_data, QEMUDebugCounter name, long value);
 void perfetto_emit_instr(void *backend_data, cpu_log_entry_handle entry_handle);
diff --git a/include/qemu/log_instr.h b/include/qemu/log_instr.h
index 32c74b7c820..6069a54f881 100644
--- a/include/qemu/log_instr.h
+++ b/include/qemu/log_instr.h
@@ -55,8 +55,10 @@
  */
 typedef enum {
     QEMU_LOG_INSTR_BACKEND_TEXT = 0,
-    QEMU_LOG_INSTR_BACKEND_CVTRACE = 1,
-    QEMU_LOG_INSTR_BACKEND_NOP = 2,
+    QEMU_LOG_INSTR_BACKEND_NOP = 1,
+#ifdef CONFIG_TRACE_CVTRACE
+    QEMU_LOG_INSTR_BACKEND_CVTRACE = 2,
+#endif
 #ifdef CONFIG_TRACE_PERFETTO
     QEMU_LOG_INSTR_BACKEND_PERFETTO = 3,
 #endif
@@ -214,13 +216,13 @@ typedef enum {
     /* Must be last */
     QEMU_LOG_INSTR_DBG_MAX
 } QEMUDebugCounter;
+void qemu_log_instr_conf_logfile(const char *name);
 
 #ifdef CONFIG_TRACE_PERFETTO
 /* Perfetto backend configuration hooks */
 #ifdef __cplusplus
 extern "C" {
 #endif
-void qemu_log_instr_perfetto_conf_logfile(const char *name);
 int qemu_log_instr_perfetto_conf_categories(const char *category_list);
 void qemu_log_instr_perfetto_enable_interceptor(void);
 void qemu_log_instr_perfetto_interceptor_logfile(const char *name);
@@ -229,10 +231,6 @@ void qemu_log_instr_perfetto_interceptor_logfile(const char *name);
 #endif
 #endif /* CONFIG_TRACE_PERFETTO */
 
-#ifdef CONFIG_TRACE_PROTOBUF
-void qemu_log_instr_protobuf_conf_logfile(const char *name);
-#endif /* CONFIG_TRACE_PROTOBUF  */
-
 #ifndef __cplusplus
 /* No visibility in the perfetto tracing backend */
 
diff --git a/qemu-options.hx b/qemu-options.hx
index aa218127290..db9f3baa2fd 100644
--- a/qemu-options.hx
+++ b/qemu-options.hx
@@ -4123,12 +4123,12 @@ SRST
     Set CHERI trace backend to <type> (text, cvtrace or perfetto)
 ERST
 
-DEF("cheri-trace-perfetto-logfile", HAS_ARG, QEMU_OPTION_trace_perfetto_logfile, \
-"-cheri-trace-perfetto-logfile [logfile]     \
- Set log file for perfetto traces, defaults to qemu_trace.pb.\n", QEMU_ARCH_ALL)
+DEF("cheri-trace-logfile", HAS_ARG, QEMU_OPTION_trace_logfile, \
+"-cheri-trace-logfile [logfile]     \
+ Set log file for TCG instruction traces, defaults to qemu_trace.txt.\n", QEMU_ARCH_ALL)
 SRST
-``-cheri-trace-perfetto-logfile [logfile]``
-    Set perfetto trace output file.
+``-cheri-trace-logfile [logfile]``
+    Set trace output file.
 ERST
 
 DEF("cheri-trace-perfetto-categories", HAS_ARG, QEMU_OPTION_trace_perfetto_categories, \
diff --git a/softmmu/vl.c b/softmmu/vl.c
index 97dd4376fde..ba7413d9236 100644
--- a/softmmu/vl.c
+++ b/softmmu/vl.c
@@ -3753,8 +3753,10 @@ void qemu_init(int argc, char **argv, char **envp)
             case QEMU_OPTION_cheri_trace_backend:
                 if (strcmp(optarg, "text") == 0) {
                     qemu_log_instr_set_backend(QEMU_LOG_INSTR_BACKEND_TEXT);
+#ifdef CONFIG_TRACE_CVTRACE
                 } else if (strcmp(optarg, "cvtrace") == 0) {
                     qemu_log_instr_set_backend(QEMU_LOG_INSTR_BACKEND_CVTRACE);
+#endif
                 } else if (strcmp(optarg, "nop") == 0) {
                     qemu_log_instr_set_backend(QEMU_LOG_INSTR_BACKEND_NOP);
 #ifdef CONFIG_TRACE_PERFETTO
@@ -3774,10 +3776,10 @@ void qemu_init(int argc, char **argv, char **envp)
                     exit(1);
                 }
                 break;
-#ifdef CONFIG_TRACE_PERFETTO
-            case QEMU_OPTION_trace_perfetto_logfile:
-                qemu_log_instr_perfetto_conf_logfile(optarg);
+            case QEMU_OPTION_trace_logfile:
+                qemu_log_instr_conf_logfile(optarg);
                 break;
+#ifdef CONFIG_TRACE_PERFETTO
             case QEMU_OPTION_trace_perfetto_categories:
                 qemu_log_instr_perfetto_conf_categories(optarg);
                 break;
@@ -3786,11 +3788,6 @@ void qemu_init(int argc, char **argv, char **envp)
                 break;
             case QEMU_OPTION_trace_perfetto_interceptor_logfile:
                 qemu_log_instr_perfetto_interceptor_logfile(optarg);
-#endif
-#ifdef CONFIG_TRACE_PROTOBUF
-            case QEMU_OPTION_trace_protobuf_logfile:
-                qemu_log_instr_protobuf_conf_logfile(optarg);
-                break;
 #endif
             case QEMU_OPTION_cheri_trace_buffer_size:
                 qemu_log_instr_set_buffer_size(strtoul(optarg, NULL, 0));
diff --git a/trace_extra/trace_perfetto.cc b/trace_extra/trace_perfetto.cc
index af1e91352f5..c677f4cddf9 100644
--- a/trace_extra/trace_perfetto.cc
+++ b/trace_extra/trace_perfetto.cc
@@ -497,11 +497,6 @@ void process_instr(perfetto_backend_data *data, cpu_log_entry_handle entry)
 
 } // namespace
 
-extern "C" void qemu_log_instr_perfetto_conf_logfile(const char *name)
-{
-    logfile = name;
-}
-
 extern "C" void
 qemu_log_instr_perfetto_conf_categories(const char *category_str)
 {
@@ -530,11 +525,13 @@ extern "C" void qemu_log_instr_perfetto_enable_interceptor()
  *
  * Start tracing thread when first called.
  */
-extern "C" void perfetto_init_cpu(int cpu_index, void **backend_data)
+extern "C" void perfetto_init_cpu(int cpu_index, const char *logfile_path,
+                                  void **backend_data)
 {
     static std::once_flag init_flag;
 
     std::call_once(init_flag, [&]() {
+        logfile = logfile_path;
         atexit(perfetto_tracing_stop);
         perfetto_start_tracing();
     });

From a2ac5900fa7a9b6b6c2b9459b7d08c3a58f265fd Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Fri, 31 Mar 2023 19:27:53 +0100
Subject: [PATCH 70/74] Cleanup signatures in instruction logging
 infrastructure.

---
 accel/tcg/log_instr.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/accel/tcg/log_instr.c b/accel/tcg/log_instr.c
index 8e66b3b7295..9e5815d31ea 100644
--- a/accel/tcg/log_instr.c
+++ b/accel/tcg/log_instr.c
@@ -161,7 +161,7 @@ static void dump_debug_stats(CPUState *cpu)
     }
 }
 
-void qemu_log_instr_enable_trace_debug()
+void qemu_log_instr_enable_trace_debug(void)
 {
     trace_debug = true;
 }
@@ -370,7 +370,7 @@ static void cpu_loglevel_switch(CPUArchState *env, target_ulong pc,
 }
 
 /* Start global logging flag if it was disabled */
-static void global_loglevel_enable()
+static void global_loglevel_enable(void)
 {
     if (!qemu_loglevel_mask(CPU_LOG_INSTR))
         qemu_set_log_internal(qemu_loglevel | CPU_LOG_INSTR);
@@ -562,7 +562,7 @@ static void do_log_backend_sync(CPUState *cpu, run_on_cpu_data _unused)
  * Attempt to syncronize buffers in the tracing backend for each CPU.
  * NOTE: This is a blocking operation that may delay the exit path.
  */
-void qemu_log_instr_sync_buffers()
+void qemu_log_instr_sync_buffers(void)
 {
     CPUState *cpu;
 

From 659bcc5b2f8b944a2e13affab7b9acee53aa0ddb Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Fri, 31 Mar 2023 19:30:46 +0100
Subject: [PATCH 71/74] Add the current log level transition in the log state
 transition event.

This should help to inform log backends for management purposes.
---
 accel/tcg/log_instr.c    | 11 +++++++----
 include/qemu/log_instr.h |  1 +
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/accel/tcg/log_instr.c b/accel/tcg/log_instr.c
index 9e5815d31ea..52dcf9c34df 100644
--- a/accel/tcg/log_instr.c
+++ b/accel/tcg/log_instr.c
@@ -177,12 +177,13 @@ static void emit_regdump_event(CPUArchState *env, cpu_log_entry_t *entry)
     g_array_append_val(entry->events, event);
 }
 
-static inline void emit_start_event(cpu_log_entry_t *entry, target_ulong pc)
+static inline void emit_start_event(cpu_log_instr_state_t *cpulog, cpu_log_entry_t *entry, target_ulong pc)
 {
     log_event_t event;
 
     event.id = LOG_EVENT_STATE;
     event.state.next_state = LOG_EVENT_STATE_START;
+    event.state.level = cpulog->loglevel;
     event.state.pc = pc;
     /* Start events always have incomplete instruction data */
     entry->flags &= ~LI_FLAG_HAS_INSTR_DATA;
@@ -197,12 +198,13 @@ static inline void emit_start_event(cpu_log_entry_t *entry, target_ulong pc)
     g_array_append_val(entry->events, event);
 }
 
-static inline void emit_stop_event(cpu_log_entry_t *entry, target_ulong pc)
+static inline void emit_stop_event(cpu_log_instr_state_t *cpulog, cpu_log_entry_t *entry, target_ulong pc)
 {
     log_event_t event;
 
     event.id = LOG_EVENT_STATE;
     event.state.next_state = LOG_EVENT_STATE_STOP;
+    event.state.level = cpulog->loglevel;
     event.state.pc = pc;
 
     g_array_append_val(entry->events, event);
@@ -335,7 +337,7 @@ static void do_cpu_loglevel_switch(CPUState *cpu, run_on_cpu_data data)
             reset_log_buffer(cpulog, entry);
             goto done;
         }
-        emit_stop_event(entry, pc);
+        emit_stop_event(cpulog, entry, pc);
         QEMU_LOG_INSTR_INC_STAT(cpulog, trace_stop);
         do_instr_commit(env);
         /* Instruction commit may have advanced to the next entry buffer slot */
@@ -348,7 +350,7 @@ static void do_cpu_loglevel_switch(CPUState *cpu, run_on_cpu_data data)
          * Note: the start event is emitted by the first instruction being
          * traced
          */
-        emit_start_event(entry, pc);
+        emit_start_event(cpulog, entry, pc);
         emit_regdump_event(env, entry);
         QEMU_LOG_INSTR_INC_STAT(cpulog, trace_start);
     }
@@ -1265,6 +1267,7 @@ void qemu_log_instr_flush(CPUArchState *env)
     /* Emit FLUSH event so that it can be picked up by backends */
     event.id = LOG_EVENT_STATE;
     event.state.next_state = LOG_EVENT_STATE_FLUSH;
+    event.state.level = cpulog->loglevel;
     event.state.pc = entry->pc;
     qemu_log_instr_event(env, &event);
 
diff --git a/include/qemu/log_instr.h b/include/qemu/log_instr.h
index 6069a54f881..2f7777d7a79 100644
--- a/include/qemu/log_instr.h
+++ b/include/qemu/log_instr.h
@@ -131,6 +131,7 @@ typedef enum {
 
 typedef struct {
     log_event_trace_state_t next_state;
+    qemu_log_instr_loglevel_t level;
     uint64_t pc;
 } log_event_trace_state_update_t;
 

From 9ae03c63ca261e559e7e5ad1085b78507e6d79fc Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Fri, 31 Mar 2023 19:49:12 +0100
Subject: [PATCH 72/74] Rename per-CPU instruction logging buffer to
 entry_buffer.

This is better than instr_info as it reflects its use.
---
 accel/tcg/log_instr.c             | 16 ++++++++--------
 include/exec/log_instr_internal.h |  3 ++-
 include/qemu/log_instr.h          |  4 ++--
 3 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/accel/tcg/log_instr.c b/accel/tcg/log_instr.c
index 52dcf9c34df..a459808c11e 100644
--- a/accel/tcg/log_instr.c
+++ b/accel/tcg/log_instr.c
@@ -263,10 +263,10 @@ static void do_instr_commit(CPUArchState *env)
     }
 
     if (cpulog->flags & QEMU_LOG_INSTR_FLAG_BUFFERED) {
-        cpulog->ring_head = (cpulog->ring_head + 1) % cpulog->instr_info->len;
+        cpulog->ring_head = (cpulog->ring_head + 1) % cpulog->entry_buffer->len;
         if (cpulog->ring_tail == cpulog->ring_head)
             cpulog->ring_tail =
-                (cpulog->ring_tail + 1) % cpulog->instr_info->len;
+                (cpulog->ring_tail + 1) % cpulog->entry_buffer->len;
     } else {
         trace_backend->emit_instr(env, entry);
         QEMU_LOG_INSTR_INC_STAT(cpulog, entries_emitted);
@@ -513,7 +513,7 @@ void qemu_log_instr_init(CPUState *cpu)
     cpulog->loglevel_active = false;
     cpulog->filters = g_array_sized_new(
         false, true, sizeof(cpu_log_instr_filter_fn_t), LOG_INSTR_FILTER_MAX);
-    cpulog->instr_info = entry_ring;
+    cpulog->entry_buffer = entry_ring;
     cpulog->ring_head = 0;
     cpulog->ring_tail = 0;
     reset_log_buffer(cpulog, entry);
@@ -581,15 +581,15 @@ static void do_log_buffer_resize(CPUState *cpu, run_on_cpu_data data)
     cpu_log_entry_t *entry;
     int i;
 
-    g_array_set_size(cpulog->instr_info, new_size);
+    g_array_set_size(cpulog->entry_buffer, new_size);
     cpulog->ring_head = 0;
     cpulog->ring_tail = 0;
-    for (i = 0; i < cpulog->instr_info->len; i++) {
+    for (i = 0; i < cpulog->entry_buffer->len; i++) {
         /*
          * Clear and reinitialize all the entries,
          * a bit overkill but should not be a frequent operation.
          */
-        entry = &g_array_index(cpulog->instr_info, cpu_log_entry_t, i);
+        entry = &g_array_index(cpulog->entry_buffer, cpu_log_entry_t, i);
         qemu_log_entry_init(entry);
         reset_log_buffer(cpulog, entry);
     }
@@ -1283,10 +1283,10 @@ void qemu_log_instr_flush(CPUArchState *env)
     }
 
     while (curr != cpulog->ring_head) {
-        entry = &g_array_index(cpulog->instr_info, cpu_log_entry_t, curr);
+        entry = &g_array_index(cpulog->entry_buffer, cpu_log_entry_t, curr);
         trace_backend->emit_instr(env, entry);
         QEMU_LOG_INSTR_INC_STAT(cpulog, entries_emitted);
-        curr = (curr + 1) % cpulog->instr_info->len;
+        curr = (curr + 1) % cpulog->entry_buffer->len;
     }
     cpulog->ring_tail = cpulog->ring_head;
 }
diff --git a/include/exec/log_instr_internal.h b/include/exec/log_instr_internal.h
index 4b5d7c003eb..8736d715a04 100644
--- a/include/exec/log_instr_internal.h
+++ b/include/exec/log_instr_internal.h
@@ -223,7 +223,8 @@ static inline cpu_log_entry_t *get_cpu_log_entry(CPUArchState *env)
 {
     cpu_log_instr_state_t *cpulog = get_cpu_log_state(env);
 
-    return &g_array_index(cpulog->instr_info, cpu_log_entry_t,
+    return &g_array_index(cpulog->entry_buffer, cpu_log_entry_t,
                           cpulog->ring_head);
 }
+
 #endif /* __cplusplus */
diff --git a/include/qemu/log_instr.h b/include/qemu/log_instr.h
index 2f7777d7a79..2631be2f0e3 100644
--- a/include/qemu/log_instr.h
+++ b/include/qemu/log_instr.h
@@ -245,8 +245,6 @@ static inline qemu_log_instr_backend_t qemu_log_instr_get_backend(void)
     return qemu_log_instr_backend;
 }
 
-struct cpu_log_instr_info;
-
 typedef union {
     char charv;
     short shortv;
@@ -312,6 +310,8 @@ typedef struct {
     void *backend_data;
     /* Statistics for debugging */
     qemu_log_instr_stats_t stats;
+    /* Ring buffer of cpu_log_entry */
+    GArray *entry_buffer;
 } cpu_log_instr_state_t;
 
 /*

From 1d91718e47095348f62fb615bc2d63fcfd5b6d4d Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Fri, 31 Mar 2023 20:05:31 +0100
Subject: [PATCH 73/74] Remove perfetto debug counters.

---
 accel/tcg/cpu-exec.c              | 18 ------------------
 accel/tcg/log_instr.c             |  8 --------
 include/exec/log_instr.h          |  6 ------
 include/exec/log_instr_perfetto.h |  1 -
 include/qemu/log_instr.h          | 12 ------------
 trace_extra/trace_perfetto.cc     | 10 ----------
 6 files changed, 55 deletions(-)

diff --git a/accel/tcg/cpu-exec.c b/accel/tcg/cpu-exec.c
index d1700f5230d..518cc8cedb3 100644
--- a/accel/tcg/cpu-exec.c
+++ b/accel/tcg/cpu-exec.c
@@ -702,24 +702,6 @@ static inline void cpu_loop_exec_tb(CPUState *cpu, TranslationBlock *tb,
     /* Instruction counter expired.  */
     assert(icount_enabled());
 #ifndef CONFIG_USER_ONLY
-#if defined(CONFIG_TCG_LOG_INSTR) && defined(CONFIG_TRACE_PERFETTO)
-    {
-        int64_t executed = cpu->icount_budget -
-            (cpu_neg(cpu)->icount_decr.u16.low + cpu->icount_extra);
-        /*
-         * Assume that it is not possible to exit a tb chain with
-         * where tracing occurred but the first or last tb don't have
-         * the tracing flag set.
-         */
-        if (tb->cflags & CF_LOG_INSTR) {
-            qemu_log_instr_counter(cpu, QEMU_LOG_INSTR_DBG_INSN_TRACING_ICOUNT,
-                                   executed);
-        } else {
-            qemu_log_instr_counter(cpu, QEMU_LOG_INSTR_DBG_INSN_ICOUNT,
-                                   executed);
-        }
-    }
-#endif
     /* Ensure global icount has gone forward */
     icount_update(cpu);
 
diff --git a/accel/tcg/log_instr.c b/accel/tcg/log_instr.c
index a459808c11e..ea879f401ab 100644
--- a/accel/tcg/log_instr.c
+++ b/accel/tcg/log_instr.c
@@ -112,7 +112,6 @@ static trace_backend_hooks_t trace_backends[] = {
 #ifdef CONFIG_TRACE_PERFETTO
       .init = init_perfetto_backend,
       .sync = sync_perfetto_backend,
-      .emit_debug = emit_perfetto_debug,
       .emit_instr = emit_perfetto_entry
 #endif
     }, {
@@ -1291,13 +1290,6 @@ void qemu_log_instr_flush(CPUArchState *env)
     cpulog->ring_tail = cpulog->ring_head;
 }
 
-void qemu_log_instr_counter(CPUState *cpu, QEMUDebugCounter name, long value)
-{
-    if (trace_backend->emit_debug) {
-        trace_backend->emit_debug(cpu->env_ptr, name, value);
-    }
-}
-
 /* Instruction logging helpers */
 
 /* Dump out all the accumalated printf's */
diff --git a/include/exec/log_instr.h b/include/exec/log_instr.h
index 9e606c8c48b..6ccecec5d81 100644
--- a/include/exec/log_instr.h
+++ b/include/exec/log_instr.h
@@ -350,12 +350,6 @@ void qemu_log_instr_event_dump_cap_int(log_event_t *evt, const char *reg_name,
  */
 void qemu_log_instr_extra(CPUArchState *env, const char *msg, ...);
 
-/*
- * Tracepoint for qemu internals.
- * This should not be used for guest events.
- */
-void qemu_log_instr_counter(CPUState *cpu, QEMUDebugCounter name, long value);
-
 #else /* ! CONFIG_TCG_LOG_INSTR */
 #define qemu_log_instr_enabled(cpu) false
 #define qemu_log_instr_start(env, mode, pc)
diff --git a/include/exec/log_instr_perfetto.h b/include/exec/log_instr_perfetto.h
index c72dd812b4a..c594cdabcb1 100644
--- a/include/exec/log_instr_perfetto.h
+++ b/include/exec/log_instr_perfetto.h
@@ -88,7 +88,6 @@ typedef void *cap_register_handle;
 
 void perfetto_init_cpu(int cpu_index, const char *logfile, void **backend_data);
 void perfetto_sync_cpu(void *backend_data);
-void perfetto_emit_debug(void *backend_data, QEMUDebugCounter name, long value);
 void perfetto_emit_instr(void *backend_data, cpu_log_entry_handle entry_handle);
 
 /*
diff --git a/include/qemu/log_instr.h b/include/qemu/log_instr.h
index 2631be2f0e3..8911e945243 100644
--- a/include/qemu/log_instr.h
+++ b/include/qemu/log_instr.h
@@ -205,18 +205,6 @@ typedef struct {
     };
 } log_event_t;
 
-/*
- * Internal QEMU debug counters.
- * These are always traced, regardless of whether tracing is enabled or not.
- * This enumeration defines the name of the debug counters for the backends
- * to use.
- */
-typedef enum {
-    QEMU_LOG_INSTR_DBG_INSN_ICOUNT = 0,
-    QEMU_LOG_INSTR_DBG_INSN_TRACING_ICOUNT = 1,
-    /* Must be last */
-    QEMU_LOG_INSTR_DBG_MAX
-} QEMUDebugCounter;
 void qemu_log_instr_conf_logfile(const char *name);
 
 #ifdef CONFIG_TRACE_PERFETTO
diff --git a/trace_extra/trace_perfetto.cc b/trace_extra/trace_perfetto.cc
index c677f4cddf9..3caf0fcab49 100644
--- a/trace_extra/trace_perfetto.cc
+++ b/trace_extra/trace_perfetto.cc
@@ -552,16 +552,6 @@ extern "C" void perfetto_sync_cpu(void *backend_data)
     perfetto::TrackEvent::Flush();
 }
 
-extern "C" void perfetto_emit_debug(void *backend_data, QEMUDebugCounter index,
-                                    long value)
-{
-    auto *data = reinterpret_cast<perfetto_backend_data *>(backend_data);
-    qemu_fallback_state *cpu_state = data->ctx_tracker.get_cpu_state();
-
-    TRACE_COUNTER("qemu-debug", cpu_state->get_debug_counter_track(index),
-                  value);
-}
-
 extern "C" void perfetto_emit_instr(void *backend_data,
                                     cpu_log_entry_handle entry)
 {

From d83469d317436e793e8344fd8409bff59746862f Mon Sep 17 00:00:00 2001
From: Alfredo Mazzinghi <am2419@cl.cam.ac.uk>
Date: Fri, 31 Mar 2023 20:11:05 +0100
Subject: [PATCH 74/74] Restore entry buffer size var.

---
 accel/tcg/log_instr.c | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/accel/tcg/log_instr.c b/accel/tcg/log_instr.c
index ea879f401ab..621d6e52c0c 100644
--- a/accel/tcg/log_instr.c
+++ b/accel/tcg/log_instr.c
@@ -89,6 +89,14 @@ qemu_log_instr_backend_t qemu_log_instr_backend = QEMU_LOG_INSTR_BACKEND_TEXT;
 
 static const char *trace_logfile_name = "qemu_trace.txt";
 
+/*
+ * The number of entries in the trace buffer at startup.
+ *
+ * This is set by hooks in the command line argument parsing before
+ * CPUs are initialized.
+ */
+static unsigned long reset_entry_buffer_size = DEFAULT_ENTRY_BUFFER_SIZE;
+
 /* Current format callbacks. */
 static trace_backend_hooks_t *trace_backend;