Basic multi-threaded capability tag implementation

accel/tcg/cpu-exec.c

@@ -40,6 +40,7 @@
 #include "exec/cpu-all.h"
 #include "sysemu/cpu-timers.h"
 #include "sysemu/replay.h"
+#include "cheri_tagmem.h"
 
 /* -icount align implementation. */
 
@@ -278,6 +279,7 @@ void cpu_exec_step_atomic(CPUState *cpu)
         cpu_tb_exec(cpu, tb, &tb_exit);
         cc->cpu_exec_exit(cpu);
     } else {
+        cheri_tag_locks_exception_thrown(cpu);
         /*
          * The mmap_lock is dropped by tb_gen_code if it runs out of
          * memory.
@@ -746,6 +748,7 @@ int cpu_exec(CPUState *cpu)
 
     /* prepare setjmp context for exception handling */
     if (sigsetjmp(cpu->jmp_env, 0) != 0) {
+
 #if defined(__clang__) || !QEMU_GNUC_PREREQ(4, 6)
         /* Some compilers wrongly smash all local variables after
          * siglongjmp. There were bug reports for gcc 4.5.0 and clang.
@@ -761,6 +764,9 @@ int cpu_exec(CPUState *cpu)
 #ifndef CONFIG_SOFTMMU
         tcg_debug_assert(!have_mmap_lock());
 #endif
+
+        cheri_tag_locks_exception_thrown(cpu);
+
         if (qemu_mutex_iothread_locked()) {
             qemu_mutex_unlock_iothread();
         }

accel/tcg/tcg-runtime.h

@@ -153,12 +153,30 @@ DEF_HELPER_3(ddc_check_bounds_store, void, env, tl, tl)
 #endif
 /* Same but relative to PCC */
 DEF_HELPER_3(pcc_check_bounds, void, env, tl, tl)
-/* Clear tags due to a store. Only call this after the store succeeded. */
+
+/* TODO: Which of the following tag invalidates can be TCG_CALL_NO_RWG or
+ * TCG_CALL_NO_WG? tags are not globals, so it should be all of them?
+ */
+
+/* Clear tags due to a store. Only calll this after the store succeeded. */
 DEF_HELPER_3(cheri_invalidate_tags, void, env, cap_checked_ptr, memop_idx)
+
 /* Clear tags due to a store, last argument is whether the store succeeded. */
 DEF_HELPER_4(cheri_invalidate_tags_condition, void, env, cap_checked_ptr,
              memop_idx, i32)
 
+DEF_HELPER_3(cheri_invalidate_lock_tags_start, void, env, cap_checked_ptr,
+             memop_idx)
+DEF_HELPER_3(cheri_invalidate_lock_tags_start_or_dummy, void, env,
+             cap_checked_ptr, memop_idx)
+DEF_HELPER_3(cheri_invalidate_lock_tags_end, void, env, cap_checked_ptr,
+             memop_idx)
+DEF_HELPER_4(cheri_invalidate_lock_tags_end_condition, void, env,
+             cap_checked_ptr, memop_idx, i32)
+
+DEF_HELPER_2(cheri_invalidate_lock_tags_assert_exist, void, env,
+             cap_checked_ptr)
+
 #endif
 
 DEF_HELPER_FLAGS_3(gvec_mov, TCG_CALL_NO_RWG, void, ptr, ptr, i32)

default-configs/targets/mips64cheri128-softmmu.mak

@@ -6,5 +6,3 @@ INCLUDE_WORKAROUND=mips64-softmmu.mak
 # Same as mips64-softmmu.mak but with the extra mips64-cheri-c128.xml
 TARGET_XML_FILES=gdb-xml/mips64-cpu.xml gdb-xml/mips64-cp0.xml gdb-xml/mips64-fpu.xml gdb-xml/mips64-sys.xml gdb-xml/mips64-cheri-c128.xml
 TARGET_CHERI=y
-# Capability loads/stores+tagged memory don't work with MTTCG
-TARGET_SUPPORTS_MTTCG=n

default-configs/targets/riscv32cheri-softmmu.mak

@@ -6,5 +6,3 @@ INCLUDE_WORKAROUND=riscv32-softmmu.mak
 # Same as riscv32-softmmu.mak but with the extra riscv-32bit-cheri.xml
 TARGET_XML_FILES= gdb-xml/riscv-32bit-cpu.xml gdb-xml/riscv-32bit-fpu.xml gdb-xml/riscv-64bit-fpu.xml gdb-xml/riscv-32bit-csr.xml gdb-xml/riscv-32bit-virtual.xml gdb-xml/riscv-32bit-cheri.xml
 TARGET_CHERI=y
-# Capability loads/stores+tagged memory don't work with MTTCG
-TARGET_SUPPORTS_MTTCG=n

default-configs/targets/riscv64cheri-softmmu.mak

@@ -6,5 +6,3 @@ INCLUDE_WORKAROUND=riscv64-softmmu.mak
 # Same as riscv64-softmmu.mak but with the extra riscv-64bit-cheri.xml
 TARGET_XML_FILES= gdb-xml/riscv-64bit-cpu.xml gdb-xml/riscv-32bit-fpu.xml gdb-xml/riscv-64bit-fpu.xml gdb-xml/riscv-64bit-csr.xml gdb-xml/riscv-64bit-virtual.xml gdb-xml/riscv-64bit-cheri.xml
 TARGET_CHERI=y
-# Capability loads/stores+tagged memory don't work with MTTCG
-TARGET_SUPPORTS_MTTCG=n

include/hw/core/cpu.h

@@ -20,6 +20,7 @@
 #ifndef QEMU_CPU_H
 #define QEMU_CPU_H
 
+#include "target/cheri-common/cheri_tagmem_ex_locks.h"
 #include "hw/qdev-core.h"
 #include "disas/dis-asm.h"
 #include "exec/hwaddr.h"
@@ -477,6 +478,9 @@ struct CPUState {
 #ifdef CONFIG_TCG_LOG_INSTR
     cpu_log_instr_state_t log_state;
 #endif
+
+    /* TARGET_CHERI is poison in this context. This structure is small. */
+    cheri_exception_locks_t cheri_exception_locks;
 };
 
 typedef QTAILQ_HEAD(CPUTailQ, CPUState) CPUTailQ;

include/qemu/bitops.h

@@ -594,6 +594,20 @@ static inline uint64_t byte_unpack_64(uint8_t x)
     return (unpacked & 0x0101010101010101);
 }
 
+/**
+ * byte_unpack_64:
+ * @x Any 8-bit value
+ * @return A 64-bit value where each byte contains a single bit from x swapped.
+ */
+static inline uint64_t byte_unpack_swap_64(uint8_t x)
+{
+    uint64_t unpacked = x & 0xFF;
+    unpacked = (unpacked << 32) | (unpacked >> 4);
+    unpacked = (unpacked << 16) | (unpacked >> 2);
+    unpacked = (unpacked << 8) | (unpacked >> 1);
+    return (unpacked & 0x0101010101010101);
+}
+
 /**
  * byte_pack_64:
  * @x Any 64-bit value
@@ -608,6 +622,20 @@ static inline uint8_t byte_pack_64(uint64_t x)
     return (x & 0xFF);
 }
 
+/**
+ * byte_pack_64:
+ * @x Any 64-bit value
+ * @return A byte that contains the lowest bit of each byte in x reversed
+ */
+static inline uint8_t byte_pack_swap_64(uint64_t x)
+{
+    x &= 0x0101010101010101;
+    x = (x >> 8) | (x << 1);
+    x = (x >> 16) | (x << 2);
+    x = (x >> 32) | (x << 4);
+    return (x & 0xFF);
+}
+
 /**
  * byte_unpack_32:
  * @x Any 8-bit value (top 4 bits ignored)
@@ -621,6 +649,19 @@ static inline uint32_t byte_unpack_32(uint8_t x)
     return (unpacked & 0x01010101);
 }
 
+/**
+ * byte_unpack_32:
+ * @x Any 8-bit value (top 4 bits ignored)
+ * @return A 32-bit value where each byte contains a single bit from x reserved.
+ */
+static inline uint32_t byte_unpack_swap_32(uint8_t x)
+{
+    uint32_t unpacked = x & 0xF;
+    unpacked = (unpacked << 16) | (unpacked >> 2);
+    unpacked = (unpacked << 8) | (unpacked >> 1);
+    return (unpacked & 0x01010101);
+}
+
 /**
  * byte_pack_32:
  * @x Any 32-bit value
@@ -634,4 +675,16 @@ static inline uint8_t byte_pack_32(uint32_t x)
     return (x & 0xF);
 }
 
+/**
+ * byte_pack_32:
+ * @x Any 32-bit value
+ * @return A byte that contains the lowest bit of each byte in x reserved
+ */
+static inline uint8_t byte_pack_swap_32(uint32_t x)
+{
+    x &= 0x01010101;
+    x = (x >> 8) | (x << 1);
+    x = (x >> 16) | (x << 2);
+    return (x & 0xF);
+}
 #endif

include/tcg/tcg-op.h

@@ -29,6 +29,16 @@
 #include "exec/helper-proto.h"
 #include "exec/helper-gen.h"
 
+/* CHERI associates a lock with every capability sized word in order to
+ * provide tag memory. As this cost is already being paid, extra atomics
+ * are not needed for writes, although reads that do not require a tag may
+ * very well still need them. */
+#ifdef TARGET_CHERI
+#define ALL_WRITES_ATOMIC 1
+#else
+#define ALL_WRITES_ATOMIC 0
+#endif
+
 /* Basic output routines.  Not for general consumption.  */
 
 void tcg_gen_op1(TCGOpcode, TCGArg);
@@ -927,10 +937,19 @@ void tcg_gen_qemu_st_i64_with_checked_addr(TCGv_i64 ret,
 // as to whether a tag clear should take place.
 void tcg_gen_qemu_st_i64_with_checked_addr_cond_invalidate(
     TCGv_i64 val, TCGv_cap_checked_ptr addr, TCGArg idx, MemOp memop,
-    bool invalidate);
-// To be used in conjunction with
-// tcg_gen_qemu_st_i64_with_checked_addr_cond_invalidate to manually clear a
-// tag
+    bool invalidate, bool take_lock);
+
+/* To be used in conjunction with
+ * tcg_gen_qemu_st_i64_with_checked_addr_cond_invalidate to manually clear a
+ * tag. Call start before the operation, and end after. Pass the return of
+ * start as the @oi argument to end.
+ */
+
+TCGv_i32 handle_conditional_invalidate_start(TCGv_cap_checked_ptr checked_addr,
+                                             MemOp memop, TCGArg mmu_idx);
+void handle_conditional_invalidate_end(TCGv_cap_checked_ptr checked_addr,
+                                       TCGv_i32 oi, TCGv_i32 store_happens);
+
 void handle_conditional_invalidate(TCGv_cap_checked_ptr checked_addr,
                                    MemOp memop, TCGArg mmu_idx,
                                    TCGv_i32 store_happens);