diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index dfe2b573d..3a8b7fe03 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -41,8 +41,6 @@ jobs: tags: ${{ github.repository_owner }}/machine-emulator:devel.build push: false load: true - cache-from: type=gha,scope=debian - cache-to: type=gha,mode=max,scope=debian build-args: | DEBUG=${{ (startsWith(github.ref, 'refs/tags/v') && 'no' || 'yes') }} GIT_COMMIT=${GITHUB_SHA} @@ -60,8 +58,6 @@ jobs: tags: cartesi/machine-emulator:amd64_deb push: false load: true - cache-from: type=gha,scope=debian - cache-to: type=gha,mode=max,scope=debian build-args: | DEBUG=${{ (startsWith(github.ref, 'refs/tags/v') && 'no' || 'yes') }} GIT_COMMIT=${GITHUB_SHA} @@ -82,8 +78,6 @@ jobs: tags: cartesi/machine-emulator:arm64_deb push: false load: true - cache-from: type=gha,scope=debian - cache-to: type=gha,mode=max,scope=debian build-args: | DEBUG=${{ (startsWith(github.ref, 'refs/tags/v') && 'no' || 'yes') }} GIT_COMMIT=${GITHUB_SHA} @@ -146,8 +140,6 @@ jobs: tags: ${{ github.repository_owner }}/machine-emulator:builder push: false load: true - cache-from: type=gha,scope=debian - cache-to: type=gha,mode=max,scope=debian build-args: | DEBUG=${{ (startsWith(github.ref, 'refs/tags/v') && 'no' || 'yes') }} GIT_COMMIT=${GITHUB_SHA} @@ -164,8 +156,6 @@ jobs: tags: ${{ github.repository_owner }}/machine-emulator:devel push: false load: true - cache-from: type=gha,scope=debian - cache-to: type=gha,mode=max,scope=debian build-args: | DEBUG=${{ (startsWith(github.ref, 'refs/tags/v') && 'no' || 'yes') }} GIT_COMMIT=${GITHUB_SHA} @@ -321,8 +311,6 @@ jobs: tags: ${{ github.repository_owner }}/machine-emulator:builder push: false load: true - cache-from: type=gha,scope=debian - cache-to: type=gha,mode=max,scope=debian build-args: | DEBUG=${{ (startsWith(github.ref, 'refs/tags/v') && 'no' || 'yes') }} GIT_COMMIT=${GITHUB_SHA} @@ -339,8 +327,6 @@ jobs: tags: ${{ github.repository_owner }}/machine-emulator:devel push: false load: true - cache-from: type=gha,scope=debian - cache-to: type=gha,mode=max,scope=debian build-args: | DEBUG=${{ (startsWith(github.ref, 'refs/tags/v') && 'no' || 'yes') }} GIT_COMMIT=${GITHUB_SHA} @@ -478,8 +464,6 @@ jobs: tags: ${{ github.repository_owner }}/machine-emulator:devel push: false load: true - cache-from: type=gha,scope=debian - cache-to: type=gha,mode=max,scope=debian build-args: | DEBUG=${{ (startsWith(github.ref, 'refs/tags/v') && 'no' || 'yes') }} GIT_COMMIT=${GITHUB_SHA} @@ -541,8 +525,6 @@ jobs: tags: ${{ github.repository_owner }}/machine-emulator:builder push: false load: true - cache-from: type=gha,scope=debian-coverage - cache-to: type=gha,mode=max,scope=debian-coverage build-args: | GIT_COMMIT=${GITHUB_SHA} DEBUG=yes @@ -623,8 +605,6 @@ jobs: tags: ${{ github.repository_owner }}/machine-emulator:builder push: false load: true - cache-from: type=gha,scope=debian-sanitize - cache-to: type=gha,mode=max,scope=debian-sanitize build-args: | DEBUG=yes GIT_COMMIT=${GITHUB_SHA} @@ -703,8 +683,6 @@ jobs: platforms: linux/amd64,linux/arm64 tags: ${{ steps.docker_image_tags.outputs.tags }} push: true - cache-from: type=gha,scope=debian - cache-to: type=gha,mode=max,scope=debian build-args: | DEBUG=${{ (startsWith(github.ref, 'refs/tags/v') && 'no' || 'yes') }} GIT_COMMIT=${GITHUB_SHA} diff --git a/.gitignore b/.gitignore index 80177f4d5..ef549f836 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ *.a *.lib *.wasm +*.TODO build pkg diff --git a/src/Makefile b/src/Makefile index e183c8b51..c4b2a54fa 100644 --- a/src/Makefile +++ b/src/Makefile @@ -151,7 +151,8 @@ HASH_LIBS= #DEFS+= -DMT_ALL_DIRTY -WARNS=-Wall -Wextra -Wpedantic +C_WARNS=-Wall -Wextra -Wpedantic +CXX_WARNS=$(C_WARNS) -Wno-missing-field-initializers # Place our include directories before the system's INCS+= \ @@ -305,8 +306,8 @@ else DEFS+=-DNO_THREADS endif -CXXFLAGS+=$(OPTFLAGS) -std=gnu++20 -fvisibility=hidden -MMD $(PICCFLAGS) $(CC_MARCH) $(INCS) $(GCFLAGS) $(UBFLAGS) $(DEFS) $(WARNS) -CFLAGS+=$(OPTFLAGS) -std=gnu99 -fvisibility=hidden -MMD $(PICCFLAGS) $(CC_MARCH) $(INCS) $(GCFLAGS) $(UBFLAGS) $(DEFS) $(WARNS) +CXXFLAGS+=$(OPTFLAGS) -std=gnu++20 -fvisibility=hidden -MMD $(PICCFLAGS) $(CC_MARCH) $(INCS) $(GCFLAGS) $(UBFLAGS) $(DEFS) $(CXX_WARNS) +CFLAGS+=$(OPTFLAGS) -std=gnu99 -fvisibility=hidden -MMD $(PICCFLAGS) $(CC_MARCH) $(INCS) $(GCFLAGS) $(UBFLAGS) $(DEFS) $(C_WARNS) LDFLAGS+=$(UBFLAGS) ifeq ($(coverage),yes) @@ -357,14 +358,12 @@ LIBCARTESI_OBJS:= \ machine.o \ memory-address-range.o \ os.o \ + os-mmap.o \ plic-address-range.o \ pristine-merkle-tree.o \ replay-step-state-access-interop.o \ send-cmio-response.o \ sha3.o \ - shadow-state-address-range.o \ - shadow-tlb-address-range.o \ - shadow-uarch-state-address-range.o \ uarch-pristine-hash.o \ uarch-pristine-ram.o \ uarch-pristine-state-hash.o \ diff --git a/src/address-range-constants.h b/src/address-range-constants.h index 1c4fdb65f..77f07e279 100644 --- a/src/address-range-constants.h +++ b/src/address-range-constants.h @@ -25,14 +25,16 @@ namespace cartesi { /// \brief Fixed address ranges. enum AR_ranges : uint64_t { - AR_SHADOW_STATE_START = EXPAND_UINT64_C(AR_SHADOW_STATE_START_DEF), ///< Start of shadow state range - AR_SHADOW_STATE_LENGTH = EXPAND_UINT64_C(AR_SHADOW_STATE_LENGTH_DEF), ///< Length of shadow state range - AR_PMAS_START = EXPAND_UINT64_C(AR_PMAS_START_DEF), ///< Start of PMAS list range - AR_PMAS_LENGTH = EXPAND_UINT64_C(AR_PMAS_LENGTH_DEF), ///< Length of PMAS list range - AR_DTB_START = EXPAND_UINT64_C(AR_DTB_START_DEF), ///< Start of DTB range - AR_DTB_LENGTH = EXPAND_UINT64_C(AR_DTB_LENGTH_DEF), ///< Length of DTB range - AR_SHADOW_TLB_START = EXPAND_UINT64_C(AR_SHADOW_TLB_START_DEF), ///< Start of shadow TLB range - AR_SHADOW_TLB_LENGTH = EXPAND_UINT64_C(AR_SHADOW_TLB_LENGTH_DEF), ///< Length of shadow TLB range + AR_SHADOW_STATE_START = EXPAND_UINT64_C(AR_SHADOW_STATE_START_DEF), ///< Start of shadow state range + AR_SHADOW_STATE_LENGTH = EXPAND_UINT64_C(AR_SHADOW_STATE_LENGTH_DEF), ///< Length of shadow state range + AR_SHADOW_REGISTERS_START = EXPAND_UINT64_C(AR_SHADOW_REGISTERS_START_DEF), ///< Start of shadow registers range + AR_SHADOW_REGISTERS_LENGTH = EXPAND_UINT64_C(AR_SHADOW_REGISTERS_LENGTH_DEF), ///< Length of shadow registers range + AR_SHADOW_TLB_START = EXPAND_UINT64_C(AR_SHADOW_TLB_START_DEF), ///< Start of shadow TLB range + AR_SHADOW_TLB_LENGTH = EXPAND_UINT64_C(AR_SHADOW_TLB_LENGTH_DEF), ///< Length of shadow TLB range + AR_PMAS_START = EXPAND_UINT64_C(AR_PMAS_START_DEF), ///< Start of PMAS list range + AR_PMAS_LENGTH = EXPAND_UINT64_C(AR_PMAS_LENGTH_DEF), ///< Length of PMAS list range + AR_DTB_START = EXPAND_UINT64_C(AR_DTB_START_DEF), ///< Start of DTB range + AR_DTB_LENGTH = EXPAND_UINT64_C(AR_DTB_LENGTH_DEF), ///< Length of DTB range AR_SHADOW_UARCH_STATE_START = EXPAND_UINT64_C(AR_SHADOW_UARCH_STATE_START_DEF), ///< Start of uarch shadow state range AR_SHADOW_UARCH_STATE_LENGTH = @@ -61,6 +63,10 @@ enum AR_ranges : uint64_t { AR_RAM_START = EXPAND_UINT64_C(AR_RAM_START_DEF), ///< Start of RAM range }; +static_assert(AR_SHADOW_STATE_LENGTH >= AR_SHADOW_REGISTERS_LENGTH + AR_SHADOW_TLB_LENGTH); +static_assert(AR_SHADOW_TLB_START == AR_SHADOW_REGISTERS_START + AR_SHADOW_REGISTERS_LENGTH); +static_assert(AR_SHADOW_STATE_START == AR_SHADOW_REGISTERS_START); + /// \brief PMA constants. enum AR_constants : uint64_t { AR_PAGE_SIZE_LOG2 = EXPAND_UINT64_C(AR_PAGE_SIZE_LOG2_DEF), ///< Log2 of physical memory page size. diff --git a/src/address-range-defines.h b/src/address-range-defines.h index 4f49e9141..dd89cc9a9 100644 --- a/src/address-range-defines.h +++ b/src/address-range-defines.h @@ -19,11 +19,13 @@ // NOLINTBEGIN(cppcoreguidelines-macro-usage,cppcoreguidelines-macro-to-enum,modernize-macro-to-enum) #define AR_SHADOW_STATE_START_DEF 0x0 ///< Shadow start address -#define AR_SHADOW_STATE_LENGTH_DEF 0x1000 ///< Shadow length in bytes +#define AR_SHADOW_STATE_LENGTH_DEF 0x8000 ///< Shadow length in bytes +#define AR_SHADOW_REGISTERS_START_DEF 0x0 ///< Shadow registers start address +#define AR_SHADOW_REGISTERS_LENGTH_DEF 0x1000 ///< Shadow registers length in bytes +#define AR_SHADOW_TLB_START_DEF 0x1000 ///< Shadow TLB start address +#define AR_SHADOW_TLB_LENGTH_DEF 0x6000 ///< Shadow TLB length in bytes #define AR_PMAS_START_DEF 0x10000 ///< PMA Array start address #define AR_PMAS_LENGTH_DEF 0x1000 ///< PMA Array length in bytes -#define AR_SHADOW_TLB_START_DEF 0x20000 ///< TLB start address -#define AR_SHADOW_TLB_LENGTH_DEF 0x6000 ///< TLB length in bytes #define AR_SHADOW_UARCH_STATE_START_DEF 0x400000 ///< microarchitecture shadow state start address #define AR_SHADOW_UARCH_STATE_LENGTH_DEF 0x1000 ///< microarchitecture shadow state length #define AR_UARCH_RAM_START_DEF 0x600000 ///< microarchitecture RAM start address diff --git a/src/address-range.h b/src/address-range.h index be681ccfb..c179d387d 100644 --- a/src/address-range.h +++ b/src/address-range.h @@ -220,6 +220,20 @@ class address_range { return m_flags.IW; } + /// \brief Tests if range can be replaced + /// \returns True if and only if range can be replaced + bool is_replaceable() const noexcept { + switch (m_flags.DID) { + case PMA_ISTART_DID::memory: + case PMA_ISTART_DID::flash_drive: + case PMA_ISTART_DID::cmio_rx_buffer: + case PMA_ISTART_DID::cmio_tx_buffer: + return true; + default: + return false; + } + } + /// \brief Returns driver ID associated to range /// \returns Teh driver ID PMA_ISTART_DID get_driver_id() const noexcept { @@ -327,6 +341,18 @@ class address_range { return do_is_page_marked_dirty(offset); } + /// \brief Returns true if the mapped memory is read-only on the host + /// \returns True if the memory is read-only in the host + bool is_host_read_only() const noexcept { + return do_is_host_read_only(); + } + + /// \brief Returns true if pages marked as dirty once are uncleanable + /// \returns True if and only if dirty pages are uncleanable + bool is_page_uncleanable() const noexcept { + return do_is_page_uncleanable(); + } + private: // Default implementation of peek() always fails virtual bool do_peek(const machine & /*m*/, uint64_t /*offset*/, uint64_t /*length*/, @@ -371,6 +397,14 @@ class address_range { virtual bool do_is_page_marked_dirty(uint64_t /*offset*/) const noexcept { return true; } + + virtual bool do_is_host_read_only() const noexcept { + return false; + } + + virtual bool do_is_page_uncleanable() const noexcept { + return false; + } }; template diff --git a/src/cartesi-machine.lua b/src/cartesi-machine.lua index a339a0985..9ac3e285e 100755 --- a/src/cartesi-machine.lua +++ b/src/cartesi-machine.lua @@ -107,6 +107,9 @@ where options are: data_filename: dht_filename: shared + create + truncate + read_only mount: user: @@ -137,6 +140,18 @@ where options are: target modifications to flash drive modify the memory and hash tree files. by default, image files are not modified and changes are lost. + create (optional) + create the backing storage file, shared must also be true. + + truncate (optional) + truncate the memory length to match memory lengths different from the backing storage, + in case of shared flash drive, then it also truncates the underlying backing file. + by default, when a length is present it must also match the backing storage length. + + read_only (optional) + mark flash drive as read-only, disallowing write attempts from the host or the guest. + by default, flash drives are not read-only, thus writable. + mount (optional) whether the flash drive should be mounted automatically in init. by default, the drive is mounted if there is an image file backing it, @@ -171,11 +186,12 @@ where options are: --ram=:[,:[,...]...] --dtb=:[,:[,...]...] - --tlb=:[,:[,...]...] + --processor=:[,:[,...]...] --cmio-rx-buffer=:[,:[,...]...] --cmio-tx-buffer=:[,:[,...]...] --pmas=:[,:[,...]...] --uarch-ram=:[,:[,...]...] + --uarch-processor=:[,:[,...]...] configures file storage for other memory ranges in the machine : is one of @@ -471,6 +487,31 @@ where options are: --load= load machine previously stored in . + --init-shared= + initialize a machine sharing snapshot created in . + writable address ranges are shared, thus runtime modifications persist in the snapshot. + writable address ranges use reflinks on copy-on-write filesystems for efficient copying. + read-only address ranges use hard links to avoid unnecessary copying. + address ranges sparsity is preserved to minimize host storage usage. + + MUST BE USED WITH --no-rollback + + --load-shared= + load an existing machine sharing snapshot from , + writable address ranges are shared, thus runtime modifications persists in the snapshot. + address ranges sparsity is preserved to minimize host storage usage. + + MUST BE USED WITH --no-rollback + + --clone-shared=, + clones a snapshot from source directory to destination directory and load a machine sharing it. + writable address ranges are shared, thus runtime modifications persist in the snapshot. + writable address ranges use reflinks on copy-on-write filesystems for efficient copying. + read-only address ranges use hard links to avoid unnecessary copying. + address ranges sparsity is preserved to minimize host storage usage. + + MUST BE USED WITH --no-rollback + --initial-hash print initial state hash before running machine. @@ -614,6 +655,9 @@ local flash_data_filename = { root = images_path .. "rootfs.ext2" } local flash_dht_filename = {} local flash_label_order = { "root" } local flash_shared = {} +local flash_create = {} +local flash_truncate = {} +local flash_read_only = {} local flash_mount = {} local flash_user = {} local flash_start = {} @@ -1012,13 +1056,6 @@ local options = { return true end, }, - { - "^(%-%-tlb%=(.+))$", - function(all, opts) - tlb.backing_store = parse_backing_store(opts, "tlb", all, tlb.backing_store) - return true - end, - }, { "^(%-%-pmas%=(.+))$", function(all, opts) @@ -1146,6 +1183,9 @@ local options = { data_filename = true, dht_filename = true, shared = true, + create = true, + truncate = true, + read_only = true, mount = true, user = true, length = true, @@ -1155,6 +1195,9 @@ local options = { if f.data_filename == true then f.data_filename = "" end if f.dht_filename == true then f.dht_filename = "" end assert(not f.shared or f.shared == true, "invalid flash drive shared value in " .. all) + assert(not f.create or f.create == true, "invalid flash drive create value in " .. all) + assert(not f.truncate or f.truncate == true, "invalid flash drive read_only value in " .. all) + assert(not f.read_only or f.read_only == true, "invalid flash drive truncate value in " .. all) if f.mount == nil then -- mount only if there is a file backing if f.data_filename and f.data_filename ~= "" then @@ -1179,8 +1222,14 @@ local options = { flash_start[d] = f.start or flash_start[d] flash_length[d] = f.length or flash_length[d] flash_shared[d] = f.shared or flash_shared[d] + flash_create[d] = f.create or flash_create[d] + flash_truncate[d] = f.truncate or flash_truncate[d] + flash_read_only[d] = f.read_only or flash_read_only[d] flash_mount[d] = f.mount or flash_mount[d] flash_user[d] = f.user or flash_user[d] + if d == "root" and f.read_only then -- Mount root filesystem as read-only + dtb.bootargs = dtb.bootargs:gsub("rw", "ro") + end return true end, }, @@ -1335,6 +1384,9 @@ local options = { flash_start.root = nil flash_length.root = nil flash_shared.root = nil + flash_create.root = nil + flash_truncate.root = nil + flash_read_only.root = nil table.remove(flash_label_order, 1) dtb.bootargs = dtb.bootargs:gsub(" root=$", "") return true @@ -1719,7 +1771,7 @@ end local function dump_value_proofs(machine, desired_proofs, config) if #desired_proofs > 0 then - assert(config.processor.iunrep == 0, "proofs are meaningless in unreproducible mode") + assert(config.processor.registers.iunrep == 0, "proofs are meaningless in unreproducible mode") end for _, desired in ipairs(desired_proofs) do local proof = machine:get_proof(desired.address, desired.log2_size) @@ -1800,17 +1852,20 @@ else -- Build machine config local config = { processor = { - iunrep = unreproducible and 1 or 0, + registers = { + iunrep = unreproducible and 1 or 0, + htif = { + iconsole = (htif_console_getchar and cartesi.HTIF_CONSOLE_CMD_GETCHAR_MASK or 0) + | cartesi.HTIF_CONSOLE_CMD_PUTCHAR_MASK, + iyield = (htif_yield_automatic and cartesi.HTIF_YIELD_CMD_AUTOMATIC_MASK or 0) + | (htif_yield_manual and cartesi.HTIF_YIELD_CMD_MANUAL_MASK or 0), + }, + }, }, ram = ram, dtb = dtb, flash_drive = {}, tlb = tlb, - htif = { - console_getchar = htif_console_getchar, - yield_automatic = htif_yield_automatic, - yield_manual = htif_yield_manual, - }, virtio = virtio, cmio = cmio, pmas = pmas, @@ -1843,7 +1898,10 @@ echo " data_filename = flash_data_filename[label], dht_filename = flash_dht_filename[label], shared = flash_shared[label], + create = flash_create[label], + truncate = flash_truncate[label], }, + read_only = flash_read_only[label], start = flash_start[label], length = flash_length[label] or -1, } @@ -1990,15 +2048,20 @@ local cmio_yield_command = { } local function check_cmio_htif_config(htif) - assert(not htif.console_getchar, "console getchar must be disabled for cmio") - assert(htif.yield_manual, "yield manual must be enabled for cmio") - assert(htif.yield_automatic, "yield automatic must be enabled for cmio") + assert((htif.iconsole & cartesi.HTIF_CONSOLE_CMD_GETCHAR_MASK) == 0, "console getchar must be disabled for cmio") + assert( + htif.iyield == (cartesi.HTIF_YIELD_CMD_MANUAL_MASK | cartesi.HTIF_YIELD_CMD_AUTOMATIC_MASK), + "yield manual must be enabled for cmio" + ) end local function get_and_print_yield(machine, htif) local cmd, reason, data = machine:receive_cmio_request() if cmd == cartesi.CMIO_YIELD_COMMAND_AUTOMATIC and reason == cartesi.CMIO_YIELD_AUTOMATIC_REASON_PROGRESS then - stderr("Progress: %6.2f" .. (htif.console_getchar and "\n" or "\r"), string.unpack("I4", data) / 10) + stderr( + "Progress: %6.2f" .. ((htif.iconsole & cartesi.HTIF_CONSOLE_CMD_GETCHAR_MASK) ~= 0 and "\n" or "\r"), + string.unpack("I4", data) / 10 + ) return cmd, reason, data end local cmd_str = cmio_yield_command[cmd] or "Unknown" @@ -2068,7 +2131,7 @@ local function save_cmio_inspect_state_report(inspect, data) end local function store_machine(machine, config, dir) - assert(config.processor.iunrep == 0, "hashes are meaningless in unreproducible mode") + assert(config.processor.registers.iunrep == 0, "hashes are meaningless in unreproducible mode") stderr("Storing machine: please wait\n") local values = {} if dir:find("%%h") then values.h = util.hexhash(machine:get_root_hash()) end @@ -2097,13 +2160,13 @@ if gdb_address then assert(address and port, "invalid address for GDB") gdb_stub:listen_and_wait_gdb(address, tonumber(port)) end -if config.processor.iunrep ~= 0 then stderr("Running in unreproducible mode!\n") end +if config.processor.registers.iunrep ~= 0 then stderr("Running in unreproducible mode!\n") end if cmio_advance or cmio_inspect then - check_cmio_htif_config(config.htif) + check_cmio_htif_config(config.processor.registers.htif) assert(remote_address or not perform_rollbacks, "cmio requires --remote-address for snapshot/commit/rollback") end if initial_hash then - assert(config.processor.iunrep == 0, "hashes are meaningless in unreproducible mode") + assert(config.processor.registers.iunrep == 0, "hashes are meaningless in unreproducible mode") print_root_hash(machine, stderr_unsilenceable) end dump_value_proofs(machine, initial_proof, config) @@ -2190,7 +2253,7 @@ while math.ult(machine:read_reg("mcycle"), max_mcycle) do break -- deal with yield manual elseif machine:read_reg("iflags_Y") ~= 0 then - local _, reason, data = get_and_print_yield(machine, config.htif) + local _, reason, data = get_and_print_yield(machine, config.processor.registers.htif) -- there was an exception if reason == cartesi.CMIO_YIELD_MANUAL_REASON_TX_EXCEPTION then stderr("cmio exception with payload: %q\n", data) @@ -2254,7 +2317,7 @@ while math.ult(machine:read_reg("mcycle"), max_mcycle) do end -- deal with yield automatic elseif machine:read_reg("iflags_X") ~= 0 then - local _, reason, data = get_and_print_yield(machine, config.htif) + local _, reason, data = get_and_print_yield(machine, config.processor.registers.htif) -- we have fed an advance state input if cmio_advance and cmio_advance.next_input_index > cmio_advance.input_index_begin then if reason == cartesi.CMIO_YIELD_AUTOMATIC_REASON_TX_OUTPUT then @@ -2310,7 +2373,7 @@ if max_uarch_cycle > 0 then end if gdb_stub then gdb_stub:close() end if log_step_uarch then - assert(config.processor.iunrep == 0, "micro step proof is meaningless in unreproducible mode") + assert(config.processor.registers.iunrep == 0, "micro step proof is meaningless in unreproducible mode") stderr("Gathering micro step log: please wait\n") util.dump_log(machine:log_step_uarch(cartesi.ACCESS_LOG_TYPE_ANNOTATIONS), io.stderr) end @@ -2320,7 +2383,7 @@ if log_reset_uarch then end if dump_memory_ranges then dump_pmas(machine) end if final_hash then - assert(config.processor.iunrep == 0, "hashes are meaningless in unreproducible mode") + assert(config.processor.registers.iunrep == 0, "hashes are meaningless in unreproducible mode") print_root_hash(machine, stderr_unsilenceable) end dump_value_proofs(machine, final_proof, config) diff --git a/src/clua-cartesi.cpp b/src/clua-cartesi.cpp index 1326f0806..5a196b45c 100644 --- a/src/clua-cartesi.cpp +++ b/src/clua-cartesi.cpp @@ -30,6 +30,7 @@ extern "C" { #include "base64.h" #include "clua-i-machine.h" #include "clua.h" +#include "htif-constants.h" #include "keccak-256-hasher.h" #include "machine-c-api.h" #include "machine-c-version.h" @@ -225,6 +226,12 @@ CM_API int luaopen_cartesi(lua_State *L) { clua_setintegerfield(L, CM_AR_CMIO_TX_BUFFER_START, "AR_CMIO_TX_BUFFER_START", -1); clua_setintegerfield(L, CM_AR_CMIO_TX_BUFFER_LOG2_SIZE, "AR_CMIO_TX_BUFFER_LOG2_SIZE", -1); clua_setintegerfield(L, CM_AR_RAM_START, "AR_RAM_START", -1); + // HTIF masks + clua_setintegerfield(L, HTIF_HALT_CMD_HALT_MASK, "HTIF_HALT_CMD_HALT_MASK", -1); + clua_setintegerfield(L, HTIF_CONSOLE_CMD_GETCHAR_MASK, "HTIF_CONSOLE_CMD_GETCHAR_MASK", -1); + clua_setintegerfield(L, HTIF_CONSOLE_CMD_PUTCHAR_MASK, "HTIF_CONSOLE_CMD_PUTCHAR_MASK", -1); + clua_setintegerfield(L, HTIF_YIELD_CMD_MANUAL_MASK, "HTIF_YIELD_CMD_MANUAL_MASK", -1); + clua_setintegerfield(L, HTIF_YIELD_CMD_AUTOMATIC_MASK, "HTIF_YIELD_CMD_AUTOMATIC_MASK", -1); // Set other constants used by internal tests clua_setintegerfield(L, UARCH_STATE_START_ADDRESS, "UARCH_STATE_START_ADDRESS", -1); clua_setintegerfield(L, UARCH_STATE_LOG2_SIZE, "UARCH_STATE_LOG2_SIZE", -1); diff --git a/src/dtb.cpp b/src/dtb.cpp index 7024fd3ad..077eb113f 100644 --- a/src/dtb.cpp +++ b/src/dtb.cpp @@ -98,7 +98,7 @@ void dtb_init(const machine_config &c, unsigned char *dtb_start, uint64_t dtb_le fdt.prop_u32("reg", 0); fdt.prop_string("status", "okay"); fdt.prop_string("compatible", "riscv"); - fdt.prop_string("riscv,isa", misa_to_isa_string(c.processor.misa)); + fdt.prop_string("riscv,isa", misa_to_isa_string(c.processor.registers.misa)); fdt.prop_string("mmu-type", "riscv,sv39"); fdt.prop_u32("clock-frequency", RTC_CLOCK_FREQ); { // interrupt-controller @@ -207,13 +207,13 @@ void dtb_init(const machine_config &c, unsigned char *dtb_start, uint64_t dtb_le fdt.end_node(); // yield - if (c.htif.yield_manual || c.htif.yield_automatic) { + if (c.processor.registers.htif.iyield != 0) { fdt.begin_node("yield"); fdt.prop_string("compatible", "ctsi-yield"); - if (c.htif.yield_manual) { + if ((c.processor.registers.htif.iyield & HTIF_YIELD_CMD_MANUAL_MASK) != 0) { fdt.prop_empty("manual"); } - if (c.htif.yield_automatic) { + if ((c.processor.registers.htif.iyield & HTIF_YIELD_CMD_AUTOMATIC_MASK) != 0) { fdt.prop_empty("automatic"); } fdt.end_node(); diff --git a/src/hot-tlb.h b/src/hot-tlb.h new file mode 100644 index 000000000..471d461c3 --- /dev/null +++ b/src/hot-tlb.h @@ -0,0 +1,53 @@ +// Copyright Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: LGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify it under +// the terms of the GNU Lesser General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) any +// later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +// PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License along +// with this program (see COPYING). If not, see . +// + +#ifndef HOT_TLB_H +#define HOT_TLB_H + +/// \file +/// \brief TLB definitions +/// \details The Translation Lookaside Buffer is a small cache used to speed up translation between address spaces. + +#include +#include +#include + +#include "host-addr.h" +#include "shadow-tlb.h" + +namespace cartesi { + +/// \brief Hot TLB slot. +/// \details +/// Given a target virtual address vaddr within a page matching vaddr_page in TLB slot, the corresponding host address +/// haddr = vaddr + vh_offset. +struct hot_tlb_slot final { + uint64_t vaddr_page{TLB_INVALID_PAGE}; ///< Target virtual address of start of page + host_addr vh_offset{0}; ///< Offset from target virtual address in the same page to host address +}; + +using hot_tlb_set = std::array; +using hot_tlb_state = std::array; + +static_assert(sizeof(uint64_t) >= sizeof(uintptr_t), "TLB expects host pointer fit in 64 bits"); + +// We need to ensure TLB state sizes are fixed across different platforms +static_assert(sizeof(hot_tlb_state) == 3 * TLB_SET_SIZE * 2 * sizeof(uint64_t), "unexpected hot TLB state size"); +static_assert(alignof(hot_tlb_state) == sizeof(uint64_t), "unexpected hot TLB state alignment"); + +} // namespace cartesi + +#endif diff --git a/src/htif-constants.h b/src/htif-constants.h index 1a57a13f6..744f69bba 100644 --- a/src/htif-constants.h +++ b/src/htif-constants.h @@ -102,6 +102,15 @@ enum HTIF_commands : uint64_t { HTIF_YIELD_CMD_AUTOMATIC = HTIF_YIELD_CMD_AUTOMATIC_DEF, }; +/// \brief HTIF commands masks +enum HTIF_commands_masks : uint64_t { + HTIF_HALT_CMD_HALT_MASK = UINT64_C(1) << HTIF_HALT_CMD_HALT, + HTIF_CONSOLE_CMD_GETCHAR_MASK = UINT64_C(1) << HTIF_CONSOLE_CMD_GETCHAR, + HTIF_CONSOLE_CMD_PUTCHAR_MASK = UINT64_C(1) << HTIF_CONSOLE_CMD_PUTCHAR, + HTIF_YIELD_CMD_MANUAL_MASK = UINT64_C(1) << HTIF_YIELD_CMD_MANUAL, + HTIF_YIELD_CMD_AUTOMATIC_MASK = UINT64_C(1) << HTIF_YIELD_CMD_AUTOMATIC, +}; + /// \brief HTIF yield reasons enum HTIF_yield_reason : uint64_t { HTIF_YIELD_AUTOMATIC_REASON_PROGRESS = HTIF_YIELD_AUTOMATIC_REASON_PROGRESS_DEF, diff --git a/src/i-prefer-shadow-state.h b/src/i-prefer-shadow-state.h index 4d4f83213..dd75b9f56 100644 --- a/src/i-prefer-shadow-state.h +++ b/src/i-prefer-shadow-state.h @@ -25,8 +25,8 @@ #include #include "meta.h" -#include "shadow-state.h" -#include "tlb.h" +#include "shadow-registers.h" +#include "shadow-tlb.h" namespace cartesi { @@ -48,12 +48,12 @@ class i_prefer_shadow_state { // CRTP } public: - uint64_t read_shadow_state(shadow_state_what what) const { - return derived().do_read_shadow_state(what); + uint64_t read_shadow_register(shadow_registers_what what) const { + return derived().do_read_shadow_register(what); } - void write_shadow_state(shadow_state_what what, uint64_t val) const { - derived().do_write_shadow_state(what, val); + void write_shadow_register(shadow_registers_what what, uint64_t val) const { + derived().do_write_shadow_register(what, val); } }; diff --git a/src/i-prefer-shadow-uarch-state.h b/src/i-prefer-shadow-uarch-state.h index f540cff75..af9319cf1 100644 --- a/src/i-prefer-shadow-uarch-state.h +++ b/src/i-prefer-shadow-uarch-state.h @@ -25,8 +25,8 @@ #include #include "meta.h" +#include "shadow-tlb.h" #include "shadow-uarch-state.h" -#include "tlb.h" namespace cartesi { diff --git a/src/i-state-access.h b/src/i-state-access.h index c9cbb1a0d..e216b3d4e 100644 --- a/src/i-state-access.h +++ b/src/i-state-access.h @@ -30,7 +30,7 @@ #include "i-prefer-shadow-state.h" #include "meta.h" #include "poor-type-name.h" -#include "tlb.h" +#include "shadow-tlb.h" namespace cartesi { @@ -51,7 +51,7 @@ using i_state_access_fast_addr_t = typename i_state_access_fast_addr(this); } - uint64_t prefer_read_shadow_state(shadow_state_what what) const { - const auto val = derived().read_shadow_state(what); - [[maybe_unused]] const auto *const what_name = shadow_state_get_what_name(what); - dsa_printf("%s::read_shadow_state(%s) = %" PRIu64 "(0x%" PRIx64 ")\n", get_name(), what_name, val, val); + uint64_t prefer_read_shadow_register(shadow_registers_what what) const { + const auto val = derived().read_shadow_register(what); + [[maybe_unused]] const auto *const what_name = shadow_registers_get_what_name(what); + dsa_printf("%s::read_shadow_register(%s) = %" PRIu64 "(0x%" PRIx64 ")\n", get_name(), what_name, val, val); return val; } - void prefer_write_shadow_state(shadow_state_what what, uint64_t val) const { - derived().write_shadow_state(what, val); - [[maybe_unused]] const auto *const what_name = shadow_state_get_what_name(what); - dsa_printf("%s::write_shadow_state(%s, %" PRIu64 "(0x%" PRIx64 "))\n", get_name(), what_name, val, val); + void prefer_write_shadow_register(shadow_registers_what what, uint64_t val) const { + derived().write_shadow_register(what, val); + [[maybe_unused]] const auto *const what_name = shadow_registers_get_what_name(what); + dsa_printf("%s::write_shadow_register(%s, %" PRIu64 "(0x%" PRIx64 "))\n", get_name(), what_name, val, val); } public: @@ -146,7 +146,7 @@ class i_state_access { // CRTP dsa_printf("%s::read_x(%d) = %" PRIu64 "(0x%" PRIx64 ")\n", get_name(), i, val, val); return val; } else { - return prefer_read_shadow_state(shadow_state_get_what(shadow_state_what::x0, i)); + return prefer_read_shadow_register(shadow_registers_get_what(shadow_registers_what::x0, i)); } } @@ -160,7 +160,7 @@ class i_state_access { // CRTP derived().do_write_x(i, val); dsa_printf("%s::write_x(%d, %" PRIu64 "(0x%" PRIx64 "))\n", get_name(), i, val, val); } else { - prefer_write_shadow_state(shadow_state_get_what(shadow_state_what::x0, i), val); + prefer_write_shadow_register(shadow_registers_get_what(shadow_registers_what::x0, i), val); } } @@ -173,7 +173,7 @@ class i_state_access { // CRTP dsa_printf("%s::read_f(%d) = %" PRIu64 "(0x%" PRIx64 ")\n", get_name(), i, val, val); return val; } else { - return prefer_read_shadow_state(shadow_state_get_what(shadow_state_what::f0, i)); + return prefer_read_shadow_register(shadow_registers_get_what(shadow_registers_what::f0, i)); } } @@ -185,7 +185,7 @@ class i_state_access { // CRTP derived().do_write_f(i, val); dsa_printf("%s::write_f(%d, %" PRIu64 "(%" PRIx64 "))\n", get_name(), i, val, val); } else { - prefer_write_shadow_state(shadow_state_get_what(shadow_state_what::f0, i), val); + prefer_write_shadow_register(shadow_registers_get_what(shadow_registers_what::f0, i), val); } } @@ -377,15 +377,15 @@ class i_state_access { // CRTP return val; } - /// \brief Reads TLB's vp_offset + /// \brief Reads TLB's vf_offset /// \tparam USE TLB set /// \param slot_index Slot index /// \returns Value in slot. template - fast_addr read_tlb_vp_offset(uint64_t slot_index) const { + fast_addr read_tlb_vf_offset(uint64_t slot_index) const { [[maybe_unused]] const auto fast_addr_name = std::is_same_v ? "phys_addr" : "fast_addr"; - const auto val = derived().template do_read_tlb_vp_offset(slot_index); - dsa_printf("%s::read_tlb_vp_offset<%" PRIu64 ">(%" PRIu64 ") = %s{0x%" PRIx64 "}\n", get_name(), SET, + const auto val = derived().template do_read_tlb_vf_offset(slot_index); + dsa_printf("%s::read_tlb_vf_offset<%" PRIu64 ">(%" PRIu64 ") = %s{0x%" PRIx64 "}\n", get_name(), SET, slot_index, fast_addr_name, static_cast(val)); return val; } @@ -406,16 +406,16 @@ class i_state_access { // CRTP /// \tparam USE TLB set /// \param slot_index Slot index /// \param vaddr_page Value to write - /// \param vp_offset Value to write + /// \param vf_offset Value to write /// \param pma_index Value to write /// \detail Writes to the TLB must be modify all fields atomically to prevent an inconsistent state. /// This simplifies all state access implementations. template - void write_tlb(uint64_t slot_index, uint64_t vaddr_page, fast_addr vp_offset, uint64_t pma_index) const { - derived().template do_write_tlb(slot_index, vaddr_page, vp_offset, pma_index); + void write_tlb(uint64_t slot_index, uint64_t vaddr_page, fast_addr vf_offset, uint64_t pma_index) const { + derived().template do_write_tlb(slot_index, vaddr_page, vf_offset, pma_index); [[maybe_unused]] const auto fast_addr_name = std::is_same_v ? "phys_addr" : "fast_addr"; dsa_printf("%s::write_tlb<%" PRIu64 ">(%" PRIu64 ", 0x%" PRIx64 ", %s{0x%" PRIx64 "}, %" PRIu64 ")\n", - get_name(), SET, slot_index, vaddr_page, fast_addr_name, static_cast(vp_offset), pma_index); + get_name(), SET, slot_index, vaddr_page, fast_addr_name, static_cast(vf_offset), pma_index); } /// \brief Marks a page as dirty diff --git a/src/i-uarch-state-access.h b/src/i-uarch-state-access.h index 3e290c768..d697eb153 100644 --- a/src/i-uarch-state-access.h +++ b/src/i-uarch-state-access.h @@ -24,7 +24,7 @@ #include "assert-printf.h" #include "i-prefer-shadow-uarch-state.h" #include "meta.h" -#include "tlb.h" +#include "shadow-tlb.h" // NOLINTBEGIN(cppcoreguidelines-macro-usage) #define DEFINE_USA_READ(REG) \ diff --git a/src/interpret.cpp b/src/interpret.cpp index 11c5a6f31..d3ebc3c44 100644 --- a/src/interpret.cpp +++ b/src/interpret.cpp @@ -102,6 +102,7 @@ #include "compiler-defines.h" #include "device-state-access.h" #include "find-pma.h" +#include "hot-tlb.h" #include "i-accept-counters.h" #include "i-interactive-state-access.h" #include "i-state-access.h" @@ -109,7 +110,6 @@ #include "riscv-constants.h" #include "rtc.h" #include "soft-float.h" -#include "tlb.h" #include "translate-virtual-address.h" #include "uint128.h" @@ -876,16 +876,16 @@ static void flush_tlb_slot(const STATE_ACCESS a, uint64_t slot_index) { const auto old_vaddr_page = a.template read_tlb_vaddr_page(slot_index); if (old_vaddr_page != TLB_INVALID_PAGE) { auto old_pma_index = a.template read_tlb_pma_index(slot_index); - const auto old_faddr_page = old_vaddr_page + a.template read_tlb_vp_offset(slot_index); + const auto old_faddr_page = old_vaddr_page + a.template read_tlb_vf_offset(slot_index); a.mark_dirty_page(old_faddr_page, old_pma_index); } } // We do not leave garbage behind in empty slots // (It would make state access classes trickier to implement) const auto vaddr_page = TLB_INVALID_PAGE; - const auto vp_offset = i_state_access_fast_addr_t{}; + const auto vf_offset = i_state_access_fast_addr_t{}; const auto pma_index = TLB_INVALID_PMA_INDEX; - a.template write_tlb(slot_index, vaddr_page, vp_offset, pma_index); + a.template write_tlb(slot_index, vaddr_page, vf_offset, pma_index); } /// \brief Flushes out an entire TLB set @@ -930,18 +930,18 @@ static void flush_tlb_vaddr(const STATE_ACCESS a, uint64_t /* vaddr */) { /// \param vaddr Virtual address of new mapping /// \param paddr Corresponding physical address /// \param paddr Index of PMA where paddr falls -/// \param vp_offset Receives the new vp_offset that will be stored in the slot +/// \param vf_offset Receives the new vf_offset that will be stored in the slot /// \returns The implementation-defined fast address corresponding to paddr template static i_state_access_fast_addr_t replace_tlb_entry(const STATE_ACCESS a, uint64_t vaddr, uint64_t paddr, - uint64_t pma_index, i_state_access_fast_addr_t &vp_offset) { + uint64_t pma_index, i_state_access_fast_addr_t &vf_offset) { [[maybe_unused]] auto note = a.make_scoped_note("replace_tlb_entry"); const auto slot_index = tlb_slot_index(vaddr); flush_tlb_slot(a, slot_index); const auto vaddr_page = tlb_addr_page(vaddr); const auto faddr = a.get_faddr(paddr, pma_index); - vp_offset = faddr - vaddr; - a.template write_tlb(slot_index, vaddr_page, vp_offset, pma_index); + vf_offset = faddr - vaddr; + a.template write_tlb(slot_index, vaddr_page, vf_offset, pma_index); return faddr; } @@ -955,8 +955,8 @@ static i_state_access_fast_addr_t replace_tlb_entry(const STATE_AC /// \returns The implementation-defined fast address corresponding to paddr template static FORCE_INLINE auto replace_tlb_entry(const STATE_ACCESS a, uint64_t vaddr, uint64_t paddr, uint64_t pma_index) { - i_state_access_fast_addr_t vp_offset{0}; - return replace_tlb_entry(a, vaddr, paddr, pma_index, vp_offset); + i_state_access_fast_addr_t vf_offset{0}; + return replace_tlb_entry(a, vaddr, paddr, pma_index, vf_offset); } /// \brief Read an aligned word from virtual memory (slow path that goes through virtual address translation). @@ -1046,8 +1046,8 @@ static FORCE_INLINE bool read_virtual_memory(const STATE_ACCESS a, uint64_t &pc, return status; } const auto pma_index = a.template read_tlb_pma_index(slot_index); - const auto vp_offset = a.template read_tlb_vp_offset(slot_index); - const auto faddr = vaddr + vp_offset; + const auto vf_offset = a.template read_tlb_vf_offset(slot_index); + const auto faddr = vaddr + vf_offset; a.template read_memory_word(faddr, pma_index, pval); DUMP_STATS_INCR(a, "tlb.rhit"); return true; @@ -1128,8 +1128,8 @@ static FORCE_INLINE execute_status write_virtual_memory(const STATE_ACCESS a, ui return status; } const auto pma_index = a.template read_tlb_pma_index(slot_index); - const auto vp_offset = a.template read_tlb_vp_offset(slot_index); - const auto faddr = vaddr + vp_offset; + const auto vf_offset = a.template read_tlb_vf_offset(slot_index); + const auto faddr = vaddr + vf_offset; a.template write_memory_word(faddr, pma_index, static_cast(val64)); DUMP_STATS_INCR(a, "tlb.whit"); return execute_status::success; @@ -5378,13 +5378,13 @@ enum class fetch_status : int { /// \param a Machine state accessor object. /// \param pc Virtual address for the current instruction being executed. /// \param vaddr Virtual address to be fetched. -/// \param vp_offset Receives vp_offset in the TLB slot +/// \param vf_offset Receives vf_offset in the TLB slot /// \param pma_index Receives the index of PMA where vaddr falls /// \return Returns fetch_status::success if load succeeded, fetch_status::exception if it caused an exception. // In that case, raise the exception. template static FORCE_INLINE fetch_status fetch_translate_pc_slow(const STATE_ACCESS a, uint64_t &pc, uint64_t vaddr, - i_state_access_fast_addr_t &vp_offset, uint64_t &pma_index) { + i_state_access_fast_addr_t &vf_offset, uint64_t &pma_index) { uint64_t paddr{}; // Walk page table and obtain the physical address if (unlikely(!translate_virtual_address(a, &paddr, vaddr, PTE_XWR_X_SHIFT))) { @@ -5399,7 +5399,7 @@ static FORCE_INLINE fetch_status fetch_translate_pc_slow(const STATE_ACCESS a, u pc = raise_exception(a, pc, MCAUSE_INSN_ACCESS_FAULT, vaddr); return fetch_status::exception; } - replace_tlb_entry(a, vaddr, paddr, pma_index, vp_offset); + replace_tlb_entry(a, vaddr, paddr, pma_index, vf_offset); return fetch_status::success; } @@ -5408,22 +5408,22 @@ static FORCE_INLINE fetch_status fetch_translate_pc_slow(const STATE_ACCESS a, u /// \param a Machine state accessor object. /// \param pc Virtual address for the current instruction being executed. /// \param vaddr Virtual address to be fetched. -/// \param vp_offset Receives vp_offset in the TLB slot +/// \param vf_offset Receives vf_offset in the TLB slot /// \param pma_index Receives the index of PMA where vaddr falls /// \return Returns fetch_status::success if load succeeded, fetch_status::exception if it caused an exception. // In that case, raise the exception. template static FORCE_INLINE fetch_status fetch_translate_pc(const STATE_ACCESS a, uint64_t &pc, uint64_t vaddr, - i_state_access_fast_addr_t &vp_offset, uint64_t &pma_index) { + i_state_access_fast_addr_t &vf_offset, uint64_t &pma_index) { // Try to perform the address translation via TLB first const uint64_t slot_index = tlb_slot_index(vaddr); const uint64_t slot_vaddr_page = a.template read_tlb_vaddr_page(slot_index); if (unlikely(!tlb_is_hit(slot_vaddr_page, vaddr))) { DUMP_STATS_INCR(a, "tlb.cmiss"); // Outline the slow path into a function call to minimize host CPU code cache pressure - return fetch_translate_pc_slow(a, pc, vaddr, vp_offset, pma_index); + return fetch_translate_pc_slow(a, pc, vaddr, vf_offset, pma_index); } - vp_offset = a.template read_tlb_vp_offset(slot_index); + vf_offset = a.template read_tlb_vf_offset(slot_index); pma_index = a.template read_tlb_pma_index(slot_index); DUMP_STATS_INCR(a, "tlb.chit"); return fetch_status::success; @@ -5435,32 +5435,32 @@ static FORCE_INLINE fetch_status fetch_translate_pc(const STATE_ACCESS a, uint64 /// \param pc Virtual address for the current instruction being executed. /// \param insn Receives the instruction. /// \param last_vaddr_page Receives and updates vaddr_page for cache. -/// \param last_vp_offset Receives and updates vp_offset for cache. +/// \param last_vf_offset Receives and updates vf_offset for cache. /// \param last_pma_index Receives and updates pma_index for cache. /// \return Returns fetch_status::success if load succeeded, fetch_status::exception if it caused an exception. // In that case, raise the exception. template static FORCE_INLINE fetch_status fetch_insn(const STATE_ACCESS a, uint64_t &pc, uint32_t &insn, - uint64_t &last_vaddr_page, i_state_access_fast_addr_t &last_vp_offset, uint64_t &last_pma_index) { + uint64_t &last_vaddr_page, i_state_access_fast_addr_t &last_vf_offset, uint64_t &last_pma_index) { [[maybe_unused]] auto note = a.make_scoped_note("fetch_insn"); i_state_access_fast_addr_t faddr{0}; const uint64_t pc_vaddr_page = tlb_addr_page(pc); // If pc is in the same page as the last pc fetch, // we can just reuse last fetch translation, skipping TLB or slow address translation altogether. if (likely(pc_vaddr_page == last_vaddr_page)) { - faddr = pc + last_vp_offset; + faddr = pc + last_vf_offset; } else { // Not in the same page as last the fetch, we need to perform address translation - i_state_access_fast_addr_t pc_vp_offset{}; + i_state_access_fast_addr_t pc_vf_offset{}; uint64_t pc_pma_index{}; - if (unlikely(fetch_translate_pc(a, pc, pc, pc_vp_offset, pc_pma_index) == fetch_status::exception)) { + if (unlikely(fetch_translate_pc(a, pc, pc, pc_vf_offset, pc_pma_index) == fetch_status::exception)) { return fetch_status::exception; } // Update fetch address translation cache last_vaddr_page = pc_vaddr_page; - last_vp_offset = pc_vp_offset; + last_vf_offset = pc_vf_offset; last_pma_index = pc_pma_index; - faddr = pc + pc_vp_offset; + faddr = pc + pc_vf_offset; } // The following code assumes pc is always 2-byte aligned, this is guaranteed by RISC-V spec. // If pc is pointing to the very last 2 bytes of a page, it's crossing a page boundary. @@ -5473,15 +5473,15 @@ static FORCE_INLINE fetch_status fetch_insn(const STATE_ACCESS a, uint64_t &pc, if (unlikely(insn_is_uncompressed(insn))) { // We have to perform a new address translation to read the next 2 bytes since we changed pages. const uint64_t pc2 = pc + 2; - i_state_access_fast_addr_t pc2_vp_offset{}; + i_state_access_fast_addr_t pc2_vf_offset{}; uint64_t pc2_pma_index{}; - if (unlikely(fetch_translate_pc(a, pc, pc2, pc2_vp_offset, pc2_pma_index) == fetch_status::exception)) { + if (unlikely(fetch_translate_pc(a, pc, pc2, pc2_vf_offset, pc2_pma_index) == fetch_status::exception)) { return fetch_status::exception; } last_vaddr_page = tlb_addr_page(pc2); - last_vp_offset = pc2_vp_offset; + last_vf_offset = pc2_vf_offset; last_pma_index = pc2_pma_index; - faddr = pc2 + last_vp_offset; + faddr = pc2 + last_vf_offset; a.template read_memory_word(faddr, last_pma_index, &insn16); insn |= insn16 << 16; } @@ -5524,7 +5524,7 @@ static NO_INLINE execute_status interpret_loop(const STATE_ACCESS a, uint64_t mc // Initialize fetch address translation cache invalidated uint64_t fetch_vaddr_page = TLB_INVALID_PAGE; uint64_t fetch_pma_index = TLB_INVALID_PMA_INDEX; - i_state_access_fast_addr_t fetch_vp_offset{}; + i_state_access_fast_addr_t fetch_vf_offset{}; // The outer loop continues until there is an interruption that should be handled // externally, or mcycle reaches mcycle_end @@ -5563,7 +5563,7 @@ static NO_INLINE execute_status interpret_loop(const STATE_ACCESS a, uint64_t mc uint32_t insn = 0; // Try to fetch the next instruction - if (likely(fetch_insn(a, pc, insn, fetch_vaddr_page, fetch_vp_offset, fetch_pma_index) == + if (likely(fetch_insn(a, pc, insn, fetch_vaddr_page, fetch_vf_offset, fetch_pma_index) == fetch_status::success)) { // clang-format off // NOLINTBEGIN diff --git a/src/json-util.cpp b/src/json-util.cpp index 719578307..2349bd54c 100644 --- a/src/json-util.cpp +++ b/src/json-util.cpp @@ -1159,7 +1159,7 @@ template void ju_get_opt_field(const nlohmann::json &j, const std:: not_default_constructible &value, const std::string &path); template -void ju_get_opt_field(const nlohmann::json &j, const K &key, processor_config &value, const std::string &path) { +void ju_get_opt_field(const nlohmann::json &j, const K &key, registers_state &value, const std::string &path) { if (!contains(j, key, path)) { return; } @@ -1259,10 +1259,28 @@ void ju_get_opt_field(const nlohmann::json &j, const K &key, processor_config &v ju_get_opt_field(jconfig, "senvcfg"s, value.senvcfg, new_path); ju_get_opt_field(jconfig, "ilrsc"s, value.ilrsc, new_path); ju_get_opt_field(jconfig, "iprv"s, value.iprv, new_path); - ju_get_opt_field(jconfig, "iflags_X"s, value.iflags_X, new_path); - ju_get_opt_field(jconfig, "iflags_Y"s, value.iflags_Y, new_path); - ju_get_opt_field(jconfig, "iflags_H"s, value.iflags_H, new_path); + ju_get_opt_field(jconfig, "iflags"s, value.iflags, new_path); ju_get_opt_field(jconfig, "iunrep"s, value.iunrep, new_path); + ju_get_opt_field(jconfig, "clint"s, value.clint, new_path); + ju_get_opt_field(jconfig, "plic"s, value.plic, new_path); + ju_get_opt_field(jconfig, "htif"s, value.htif, new_path); +} + +template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, registers_state &value, + const std::string &path); + +template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, registers_state &value, + const std::string &path); + +template +void ju_get_opt_field(const nlohmann::json &j, const K &key, processor_config &value, const std::string &path) { + if (!contains(j, key, path)) { + return; + } + const auto &jconfig = j[key]; + const auto new_path = path + to_string(key) + "/"; + ju_get_opt_field(jconfig, "registers"s, value.registers, new_path); + ju_get_opt_field(jconfig, "backing_store"s, value.backing_store, new_path); } template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, processor_config &value, @@ -1334,6 +1352,7 @@ void ju_get_opt_field(const nlohmann::json &j, const K &key, hash_tree_config &v const auto &jconfig = j[key]; const auto new_path = path + to_string(key) + "/"; ju_get_opt_field(jconfig, "shared"s, value.shared, new_path); + ju_get_opt_field(jconfig, "create"s, value.create, new_path); ju_get_opt_field(jconfig, "truncate"s, value.truncate, new_path); ju_get_opt_field(jconfig, "hasher"s, value.hasher, new_path); ju_get_opt_field(jconfig, "sht_filename"s, value.sht_filename, new_path); @@ -1355,6 +1374,7 @@ void ju_get_opt_field(const nlohmann::json &j, const K &key, backing_store_confi const auto &jconfig = j[key]; const auto new_path = path + to_string(key) + "/"; ju_get_opt_field(jconfig, "shared"s, value.shared, new_path); + ju_get_opt_field(jconfig, "create"s, value.create, new_path); ju_get_opt_field(jconfig, "truncate"s, value.truncate, new_path); ju_get_opt_field(jconfig, "data_filename"s, value.data_filename, new_path); ju_get_opt_field(jconfig, "dht_filename"s, value.dht_filename, new_path); @@ -1481,7 +1501,25 @@ template void ju_get_opt_field(const nlohmann::json &j, const std:: const std::string &path); template -void ju_get_opt_field(const nlohmann::json &j, const K &key, clint_config &value, const std::string &path) { +void ju_get_opt_field(const nlohmann::json &j, const K &key, iflags_state &value, const std::string &path) { + if (!contains(j, key, path)) { + return; + } + const auto &jconfig = j[key]; + const auto new_path = path + to_string(key) + "/"; + ju_get_opt_field(jconfig, "X"s, value.X, new_path); + ju_get_opt_field(jconfig, "Y"s, value.Y, new_path); + ju_get_opt_field(jconfig, "H"s, value.H, new_path); +} + +template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, iflags_state &value, + const std::string &path); + +template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, iflags_state &value, + const std::string &path); + +template +void ju_get_opt_field(const nlohmann::json &j, const K &key, clint_state &value, const std::string &path) { if (!contains(j, key, path)) { return; } @@ -1490,14 +1528,14 @@ void ju_get_opt_field(const nlohmann::json &j, const K &key, clint_config &value ju_get_opt_field(jconfig, "mtimecmp"s, value.mtimecmp, new_path); } -template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, clint_config &value, +template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, clint_state &value, const std::string &path); -template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, clint_config &value, +template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, clint_state &value, const std::string &path); template -void ju_get_opt_field(const nlohmann::json &j, const K &key, plic_config &value, const std::string &path) { +void ju_get_opt_field(const nlohmann::json &j, const K &key, plic_state &value, const std::string &path) { if (!contains(j, key, path)) { return; } @@ -1507,14 +1545,14 @@ void ju_get_opt_field(const nlohmann::json &j, const K &key, plic_config &value, ju_get_opt_field(jconfig, "girqsrvd"s, value.girqsrvd, new_path); } -template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, plic_config &value, +template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, plic_state &value, const std::string &path); -template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, plic_config &value, +template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, plic_state &value, const std::string &path); template -void ju_get_opt_field(const nlohmann::json &j, const K &key, htif_config &value, const std::string &path) { +void ju_get_opt_field(const nlohmann::json &j, const K &key, htif_state &value, const std::string &path) { if (!contains(j, key, path)) { return; } @@ -1522,15 +1560,15 @@ void ju_get_opt_field(const nlohmann::json &j, const K &key, htif_config &value, const auto new_path = path + to_string(key) + "/"; ju_get_opt_field(jconfig, "fromhost"s, value.fromhost, new_path); ju_get_opt_field(jconfig, "tohost"s, value.tohost, new_path); - ju_get_opt_field(jconfig, "console_getchar"s, value.console_getchar, new_path); - ju_get_opt_field(jconfig, "yield_manual"s, value.yield_manual, new_path); - ju_get_opt_field(jconfig, "yield_automatic"s, value.yield_automatic, new_path); + ju_get_opt_field(jconfig, "ihalt"s, value.ihalt, new_path); + ju_get_opt_field(jconfig, "iconsole"s, value.iconsole, new_path); + ju_get_opt_field(jconfig, "iyield"s, value.iyield, new_path); } -template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, htif_config &value, +template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, htif_state &value, const std::string &path); -template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, htif_config &value, +template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, htif_state &value, const std::string &path); template @@ -1551,7 +1589,7 @@ template void ju_get_opt_field(const nlohmann::json &j, const std:: const std::string &path); template -void ju_get_opt_field(const nlohmann::json &j, const K &key, uarch_processor_config &value, const std::string &path) { +void ju_get_opt_field(const nlohmann::json &j, const K &key, uarch_registers_state &value, const std::string &path) { if (!contains(j, key, path)) { return; } @@ -1594,6 +1632,23 @@ void ju_get_opt_field(const nlohmann::json &j, const K &key, uarch_processor_con ju_get_opt_field(jconfig, "halt_flag"s, value.halt_flag, new_path); } +template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, uarch_registers_state &value, + const std::string &path); + +template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, + uarch_registers_state &value, const std::string &path); + +template +void ju_get_opt_field(const nlohmann::json &j, const K &key, uarch_processor_config &value, const std::string &path) { + if (!contains(j, key, path)) { + return; + } + const auto &jconfig = j[key]; + const auto new_path = path + to_string(key) + "/"; + ju_get_opt_field(jconfig, "registers"s, value.registers, new_path); + ju_get_opt_field(jconfig, "backing_store"s, value.backing_store, new_path); +} + template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, uarch_processor_config &value, const std::string &path); @@ -1628,10 +1683,6 @@ void ju_get_opt_field(const nlohmann::json &j, const K &key, machine_config &val ju_get_opt_field(config, "ram"s, value.ram, new_path); ju_get_opt_field(config, "dtb"s, value.dtb, new_path); ju_get_opt_field(config, "flash_drive"s, value.flash_drive, new_path); - ju_get_opt_field(config, "tlb"s, value.tlb, new_path); - ju_get_opt_field(config, "clint"s, value.clint, new_path); - ju_get_opt_field(config, "plic"s, value.plic, new_path); - ju_get_opt_field(config, "htif"s, value.htif, new_path); ju_get_opt_field(config, "virtio"s, value.virtio, new_path); ju_get_opt_field(config, "cmio"s, value.cmio, new_path); ju_get_opt_field(config, "pmas"s, value.pmas, new_path); @@ -1782,7 +1833,7 @@ void to_json(nlohmann::json &j, const access_log &log) { } void to_json(nlohmann::json &j, const backing_store_config &config) { - j = nlohmann::json{{"shared", config.shared}, {"truncate", config.truncate}, + j = nlohmann::json{{"shared", config.shared}, {"create", config.create}, {"truncate", config.truncate}, {"data_filename", config.data_filename}, {"dht_filename", config.dht_filename}}; } @@ -1796,12 +1847,12 @@ void to_json(nlohmann::json &j, const memory_range_config &config) { } void to_json(nlohmann::json &j, const hash_tree_config &config) { - j = nlohmann::json{{"hasher", config.hasher}, {"shared", config.shared}, {"truncate", config.truncate}, - {"sht_filename", config.sht_filename}, {"phtc_filename", config.phtc_filename}, + j = nlohmann::json{{"hasher", config.hasher}, {"shared", config.shared}, {"create", config.create}, + {"truncate", config.truncate}, {"sht_filename", config.sht_filename}, {"phtc_filename", config.phtc_filename}, {"phtc_size", config.phtc_size}}; } -void to_json(nlohmann::json &j, const processor_config &config) { +void to_json(nlohmann::json &j, const registers_state &config) { j = nlohmann::json{{"x0", config.x[0]}, {"x1", config.x[1]}, {"x2", config.x[2]}, {"x3", config.x[3]}, {"x4", config.x[4]}, {"x5", config.x[5]}, {"x6", config.x[6]}, {"x7", config.x[7]}, {"x8", config.x[8]}, {"x9", config.x[9]}, {"x10", config.x[10]}, {"x11", config.x[11]}, {"x12", config.x[12]}, {"x13", config.x[13]}, @@ -1824,8 +1875,12 @@ void to_json(nlohmann::json &j, const processor_config &config) { {"medeleg", config.medeleg}, {"mideleg", config.mideleg}, {"mcounteren", config.mcounteren}, {"menvcfg", config.menvcfg}, {"stvec", config.stvec}, {"sscratch", config.sscratch}, {"sepc", config.sepc}, {"scause", config.scause}, {"stval", config.stval}, {"satp", config.satp}, {"scounteren", config.scounteren}, - {"senvcfg", config.senvcfg}, {"ilrsc", config.ilrsc}, {"iprv", config.iprv}, {"iflags_X", config.iflags_X}, - {"iflags_Y", config.iflags_Y}, {"iflags_H", config.iflags_H}, {"iunrep", config.iunrep}}; + {"senvcfg", config.senvcfg}, {"ilrsc", config.ilrsc}, {"iprv", config.iprv}, {"iflags", config.iflags}, + {"iunrep", config.iunrep}, {"clint", config.clint}, {"plic", config.plic}, {"htif", config.htif}}; +} + +void to_json(nlohmann::json &j, const processor_config &config) { + j = nlohmann::json{{"registers", config.registers}, {"backing_store", config.backing_store}}; } void to_json(nlohmann::json &j, const flash_drive_configs &fs) { @@ -1887,26 +1942,34 @@ void to_json(nlohmann::json &j, const dtb_config &config) { }; } -void to_json(nlohmann::json &j, const clint_config &config) { +void to_json(nlohmann::json &j, const iflags_state &config) { + j = nlohmann::json{ + {"X", config.X}, + {"Y", config.Y}, + {"H", config.H}, + }; +} + +void to_json(nlohmann::json &j, const clint_state &config) { j = nlohmann::json{ {"mtimecmp", config.mtimecmp}, }; } -void to_json(nlohmann::json &j, const plic_config &config) { +void to_json(nlohmann::json &j, const plic_state &config) { j = nlohmann::json{ {"girqpend", config.girqpend}, {"girqsrvd", config.girqsrvd}, }; } -void to_json(nlohmann::json &j, const htif_config &config) { +void to_json(nlohmann::json &j, const htif_state &config) { j = nlohmann::json{ {"fromhost", config.fromhost}, {"tohost", config.tohost}, - {"console_getchar", config.console_getchar}, - {"yield_manual", config.yield_manual}, - {"yield_automatic", config.yield_automatic}, + {"ihalt", config.ihalt}, + {"iconsole", config.iconsole}, + {"iyield", config.iyield}, }; } @@ -1917,7 +1980,7 @@ void to_json(nlohmann::json &j, const cmio_config &config) { }; } -void to_json(nlohmann::json &j, const uarch_processor_config &config) { +void to_json(nlohmann::json &j, const uarch_registers_state &config) { j = nlohmann::json{ {"x0", config.x[0]}, {"x1", config.x[1]}, @@ -1957,6 +2020,10 @@ void to_json(nlohmann::json &j, const uarch_processor_config &config) { }; } +void to_json(nlohmann::json &j, const uarch_processor_config &config) { + j = nlohmann::json{{"registers", config.registers}, {"backing_store", config.backing_store}}; +} + void to_json(nlohmann::json &j, const uarch_config &config) { j = nlohmann::json{ {"processor", config.processor}, @@ -1970,10 +2037,6 @@ void to_json(nlohmann::json &j, const machine_config &config) { {"ram", config.ram}, {"dtb", config.dtb}, {"flash_drive", config.flash_drive}, - {"tlb", config.tlb}, - {"clint", config.clint}, - {"plic", config.plic}, - {"htif", config.htif}, {"virtio", config.virtio}, {"cmio", config.cmio}, {"pmas", config.pmas}, diff --git a/src/json-util.h b/src/json-util.h index 2823e0cb7..4f6e9d186 100644 --- a/src/json-util.h +++ b/src/json-util.h @@ -307,6 +307,16 @@ template void ju_get_opt_field(const nlohmann::json &j, const K &key, not_default_constructible &optional, const std::string &path = "params/"); +/// \brief Attempts to load a registers_state object from a field in a JSON object +/// \tparam K Key type (explicit extern declarations for uint64_t and std::string are provided) +/// \param j JSON object to load from +/// \param key Key to load value from +/// \param value Object to store value +/// \param path Path to j +template +void ju_get_opt_field(const nlohmann::json &j, const K &key, registers_state &value, + const std::string &path = "params/"); + /// \brief Attempts to load a processor_config object from a field in a JSON object /// \tparam K Key type (explicit extern declarations for uint64_t and std::string are provided) /// \param j JSON object to load from @@ -395,32 +405,41 @@ template void ju_get_opt_field(const nlohmann::json &j, const K &key, virtio_configs &value, const std::string &path = "params/"); -/// \brief Attempts to load a clint_config object from a field in a JSON object +/// \brief Attempts to load a iflags_state object from a field in a JSON object +/// \tparam K Key type (explicit extern declarations for uint64_t and std::string are provided) +/// \param j JSON object to load from +/// \param key Key to load value from +/// \param value Object to store value +/// \param path Path to j +template +void ju_get_opt_field(const nlohmann::json &j, const K &key, iflags_state &value, const std::string &path = "params/"); + +/// \brief Attempts to load a clint_state object from a field in a JSON object /// \tparam K Key type (explicit extern declarations for uint64_t and std::string are provided) /// \param j JSON object to load from /// \param key Key to load value from /// \param value Object to store value /// \param path Path to j template -void ju_get_opt_field(const nlohmann::json &j, const K &key, clint_config &value, const std::string &path = "params/"); +void ju_get_opt_field(const nlohmann::json &j, const K &key, clint_state &value, const std::string &path = "params/"); -/// \brief Attempts to load a plic_config object from a field in a JSON object +/// \brief Attempts to load a plic_state object from a field in a JSON object /// \tparam K Key type (explicit extern declarations for uint64_t and std::string are provided) /// \param j JSON object to load from /// \param key Key to load value from /// \param value Object to store value /// \param path Path to j template -void ju_get_opt_field(const nlohmann::json &j, const K &key, plic_config &value, const std::string &path = "params/"); +void ju_get_opt_field(const nlohmann::json &j, const K &key, plic_state &value, const std::string &path = "params/"); -/// \brief Attempts to load an htif_config object from a field in a JSON object +/// \brief Attempts to load an htif_state object from a field in a JSON object /// \tparam K Key type (explicit extern declarations for uint64_t and std::string are provided) /// \param j JSON object to load from /// \param key Key to load value from /// \param value Object to store value /// \param path Path to j template -void ju_get_opt_field(const nlohmann::json &j, const K &key, htif_config &value, const std::string &path = "params/"); +void ju_get_opt_field(const nlohmann::json &j, const K &key, htif_state &value, const std::string &path = "params/"); /// \brief Attempts to load a cmio_config object from a field in a JSON object /// \tparam K Key type (explicit extern declarations for uint64_t and std::string are provided) @@ -441,6 +460,16 @@ template void ju_get_opt_field(const nlohmann::json &j, const K &key, std::optional &optional, const std::string &path = "params/"); +/// \brief Attempts to load an uarch_registers_state object from a field in a JSON object +/// \tparam K Key type (explicit extern declarations for uint64_t and std::string are provided) +/// \param j JSON object to load from +/// \param key Key to load value from +/// \param value Object to store value +/// \param path Path to j +template +void ju_get_opt_field(const nlohmann::json &j, const K &key, uarch_registers_state &value, + const std::string &path = "params/"); + /// \brief Attempts to load an uarch_processor_config object from a field in a JSON object /// \tparam K Key type (explicit extern declarations for uint64_t and std::string are provided) /// \param j JSON object to load from @@ -613,16 +642,19 @@ void to_json(nlohmann::json &j, const access_log &log); void to_json(nlohmann::json &j, const backing_store_config &config); void to_json(nlohmann::json &j, const backing_store_config_only &config); void to_json(nlohmann::json &j, const memory_range_config &config); +void to_json(nlohmann::json &j, const registers_state &config); void to_json(nlohmann::json &j, const processor_config &config); void to_json(nlohmann::json &j, const flash_drive_configs &fs); void to_json(nlohmann::json &j, const virtio_device_config &config); void to_json(nlohmann::json &j, const virtio_configs &vs); void to_json(nlohmann::json &j, const ram_config &config); void to_json(nlohmann::json &j, const dtb_config &config); -void to_json(nlohmann::json &j, const clint_config &config); -void to_json(nlohmann::json &j, const plic_config &config); -void to_json(nlohmann::json &j, const htif_config &config); +void to_json(nlohmann::json &j, const iflags_state &config); +void to_json(nlohmann::json &j, const clint_state &config); +void to_json(nlohmann::json &j, const plic_state &config); +void to_json(nlohmann::json &j, const htif_state &config); void to_json(nlohmann::json &j, const cmio_config &config); +void to_json(nlohmann::json &j, const uarch_registers_state &config); void to_json(nlohmann::json &j, const uarch_processor_config &config); void to_json(nlohmann::json &j, const uarch_config &config); void to_json(nlohmann::json &j, const hash_tree_config &config); @@ -721,6 +753,10 @@ extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &k not_default_constructible &value, const std::string &base = "params/"); extern template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, not_default_constructible &value, const std::string &base = "params/"); +extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, registers_state &value, + const std::string &base = "params/"); +extern template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, registers_state &value, + const std::string &base = "params/"); extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, processor_config &value, const std::string &base = "params/"); extern template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, processor_config &value, @@ -757,17 +793,21 @@ extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &k const std::string &base = "params/"); extern template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, virtio_configs &value, const std::string &base = "params/"); -extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, clint_config &value, +extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, iflags_state &value, + const std::string &base = "params/"); +extern template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, iflags_state &value, const std::string &base = "params/"); -extern template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, clint_config &value, +extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, clint_state &value, const std::string &base = "params/"); -extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, plic_config &value, +extern template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, clint_state &value, const std::string &base = "params/"); -extern template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, plic_config &value, +extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, plic_state &value, const std::string &base = "params/"); -extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, htif_config &value, +extern template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, plic_state &value, const std::string &base = "params/"); -extern template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, htif_config &value, +extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, htif_state &value, + const std::string &base = "params/"); +extern template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, htif_state &value, const std::string &base = "params/"); extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, cmio_config &value, const std::string &base = "params/"); @@ -777,6 +817,10 @@ extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &k const std::string &base = "params/"); extern template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, std::optional &value, const std::string &base = "params/"); +extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, uarch_registers_state &value, + const std::string &base = "params/"); +extern template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, uarch_registers_state &value, + const std::string &base = "params/"); extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, uarch_processor_config &value, const std::string &base = "params/"); extern template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, uarch_processor_config &value, diff --git a/src/jsonrpc-discover.json b/src/jsonrpc-discover.json index 6cb89098c..b8619a03e 100644 --- a/src/jsonrpc-discover.json +++ b/src/jsonrpc-discover.json @@ -1000,8 +1000,8 @@ } } }, - "ProcessorConfig": { - "title": "ProcessorConfig", + "RegistersConfig": { + "title": "RegistersConfig", "type": "object", "properties": { "x0": { @@ -1341,6 +1341,9 @@ "length": { "$ref": "#/components/schemas/UnsignedInteger" }, + "read_only": { + "type": "boolean" + }, "backing_store": { "$ref": "#/components/schemas/BackingStoreConfig" } @@ -1359,6 +1362,9 @@ "shared": { "type": "boolean" }, + "create": { + "type": "boolean" + }, "truncate": { "type": "boolean" } @@ -1430,15 +1436,6 @@ "$ref": "#/components/schemas/MemoryRangeConfig" } }, - "TLBConfig": { - "title": "TLBConfig", - "type": "object", - "properties": { - "backing_store": { - "$ref": "#/components/schemas/BackingStoreConfig" - } - } - }, "CLINTConfig": { "title": "CLINTConfig", "type": "object", @@ -1481,8 +1478,8 @@ } } }, - "UarchProcessorConfig": { - "title": "UarchProcessorConfig", + "UarchRegistersConfig": { + "title": "UarchRegistersConfig", "type": "object", "properties": { "x0": { @@ -1608,8 +1605,8 @@ "title": "UarchConfig", "type": "object", "properties": { - "processor": { - "$ref": "#/components/schemas/UarchProcessorConfig" + "registers": { + "$ref": "#/components/schemas/UarchRegistersConfig" }, "ram": { "$ref": "#/components/schemas/UarchRAMConfig" @@ -1647,6 +1644,12 @@ "shared": { "type": "boolean" }, + "create": { + "type": "boolean" + }, + "truncate": { + "type": "boolean" + }, "sht_filename": { "type": "string" }, @@ -1669,8 +1672,8 @@ "title": "MachineConfig", "type": "object", "properties": { - "processor": { - "$ref": "#/components/schemas/ProcessorConfig" + "registers": { + "$ref": "#/components/schemas/RegistersConfig" }, "ram": { "$ref": "#/components/schemas/RAMConfig" @@ -1681,9 +1684,6 @@ "flash_drive": { "$ref": "#/components/schemas/FlashDriveConfigs" }, - "tlb": { - "$ref": "#/components/schemas/TLBConfig" - }, "clint": { "$ref": "#/components/schemas/CLINTConfig" }, diff --git a/src/machine-c-api.h b/src/machine-c-api.h index 9f486f255..90ba25073 100644 --- a/src/machine-c-api.h +++ b/src/machine-c-api.h @@ -130,7 +130,7 @@ typedef enum cm_cmio_yield_reason { /// \brief Machine x, f, and control and status registers. typedef enum cm_reg { - // Processor x registers + // Machine x registers CM_REG_X0, CM_REG_X1, CM_REG_X2, @@ -163,7 +163,7 @@ typedef enum cm_reg { CM_REG_X29, CM_REG_X30, CM_REG_X31, - // Processor f registers + // Machine f registers CM_REG_F0, CM_REG_F1, CM_REG_F2, @@ -196,7 +196,7 @@ typedef enum cm_reg { CM_REG_F29, CM_REG_F30, CM_REG_F31, - // Processor CSRs + // Machine CSRs CM_REG_PC, CM_REG_FCSR, CM_REG_MVENDORID, diff --git a/src/machine-config.cpp b/src/machine-config.cpp index ea11b6f98..c44d0c999 100644 --- a/src/machine-config.cpp +++ b/src/machine-config.cpp @@ -31,7 +31,7 @@ #include "json-util.h" #include "pmas-constants.h" -static constexpr uint32_t archive_version = 5; +static constexpr uint32_t archive_version = 6; namespace cartesi { @@ -62,28 +62,34 @@ std::string machine_config::get_config_filename(const std::string &dir) { return dir + "/config.json"; } -static void adjust_backing_store(uint64_t start, uint64_t length, const std::string &dir, backing_store_config &c) { +void machine_config::adjust_backing_store_config(uint64_t start, uint64_t length, const std::string &dir, + backing_store_config &c) { + c.shared = false; + c.create = false; + c.truncate = false; c.data_filename = machine_config::get_data_filename(dir, start, length); c.dht_filename = machine_config::get_dht_filename(dir, start, length); } -static void adjust_hash_tree(const std::string &dir, hash_tree_config &c) { +void machine_config::adjust_hash_tree_config(const std::string &dir, hash_tree_config &c) { c.sht_filename = machine_config::get_sht_filename(dir); c.phtc_filename = machine_config::get_phtc_filename(dir); } -static void adjust_backing_store(machine_config &c, const std::string &dir) { - adjust_backing_store(AR_RAM_START, c.ram.length, dir, c.ram.backing_store); - adjust_backing_store(AR_DTB_START, AR_DTB_LENGTH, dir, c.dtb.backing_store); - for (auto &f : c.flash_drive) { - adjust_backing_store(f.start, f.length, dir, f.backing_store); +void machine_config::adjust_backing_stores(const std::string &dir) { + adjust_backing_store_config(AR_RAM_START, ram.length, dir, ram.backing_store); + adjust_backing_store_config(AR_DTB_START, AR_DTB_LENGTH, dir, dtb.backing_store); + for (auto &f : flash_drive) { + adjust_backing_store_config(f.start, f.length, dir, f.backing_store); } - adjust_backing_store(AR_SHADOW_TLB_START, AR_SHADOW_TLB_LENGTH, dir, c.tlb.backing_store); - adjust_backing_store(AR_CMIO_RX_BUFFER_START, AR_CMIO_RX_BUFFER_LENGTH, dir, c.cmio.rx_buffer.backing_store); - adjust_backing_store(AR_CMIO_TX_BUFFER_START, AR_CMIO_TX_BUFFER_LENGTH, dir, c.cmio.tx_buffer.backing_store); - adjust_backing_store(AR_PMAS_START, AR_PMAS_LENGTH, dir, c.pmas.backing_store); - adjust_backing_store(AR_UARCH_RAM_START, AR_UARCH_RAM_LENGTH, dir, c.uarch.ram.backing_store); - adjust_hash_tree(dir, c.hash_tree); + adjust_backing_store_config(AR_SHADOW_STATE_START, AR_SHADOW_STATE_LENGTH, dir, processor.backing_store); + adjust_backing_store_config(AR_CMIO_RX_BUFFER_START, AR_CMIO_RX_BUFFER_LENGTH, dir, cmio.rx_buffer.backing_store); + adjust_backing_store_config(AR_CMIO_TX_BUFFER_START, AR_CMIO_TX_BUFFER_LENGTH, dir, cmio.tx_buffer.backing_store); + adjust_backing_store_config(AR_PMAS_START, AR_PMAS_LENGTH, dir, pmas.backing_store); + adjust_backing_store_config(AR_SHADOW_UARCH_STATE_START, AR_SHADOW_UARCH_STATE_LENGTH, dir, + uarch.processor.backing_store); + adjust_backing_store_config(AR_UARCH_RAM_START, AR_UARCH_RAM_LENGTH, dir, uarch.ram.backing_store); + adjust_hash_tree_config(dir, hash_tree); } machine_config machine_config::load(const std::string &dir) { @@ -107,7 +113,7 @@ machine_config machine_config::load(const std::string &dir) { std::to_string(jv.get()) + ")"); } ju_get_field(j, std::string("config"), c, ""); - adjust_backing_store(c, dir); + c.adjust_backing_stores(dir); } catch (std::exception &e) { throw std::runtime_error{e.what()}; } diff --git a/src/machine-config.h b/src/machine-config.h index e8b058ca5..687171ca6 100644 --- a/src/machine-config.h +++ b/src/machine-config.h @@ -24,6 +24,8 @@ #include #include "riscv-constants.h" +#include "shadow-registers.h" +#include "shadow-uarch-state.h" namespace cartesi { @@ -34,57 +36,21 @@ enum machine_config_constants { VIRTIO_HOSTFWD_MAX = 16, ///< Maximum number of virtio net user host forward ports }; -/// \brief Processor state config -struct processor_config final { - std::array x{REG_X0, REG_X1, REG_X2, REG_X3, REG_X4, REG_X5, REG_X6, REG_X7, REG_X8, REG_X9, - REG_X10, REG_X11, REG_X12, REG_X13, REG_X14, REG_X15, REG_X16, REG_X17, REG_X18, REG_X19, REG_X20, REG_X21, - REG_X22, REG_X23, REG_X24, REG_X25, REG_X26, REG_X27, REG_X28, REG_X29, REG_X30, - REG_X31}; ///< Value of general-purpose registers - std::array f{}; ///< Value of floating-point registers - uint64_t pc{PC_INIT}; ///< Value of pc - uint64_t fcsr{FCSR_INIT}; ///< Value of fcsr CSR - uint64_t mvendorid{MVENDORID_INIT}; ///< Value of mvendorid CSR - uint64_t marchid{MARCHID_INIT}; ///< Value of marchid CSR - uint64_t mimpid{MIMPID_INIT}; ///< Value of mimpid CSR - uint64_t mcycle{MCYCLE_INIT}; ///< Value of mcycle CSR - uint64_t icycleinstret{ICYCLEINSTRET_INIT}; ///< Value of icycleinstret CSR - uint64_t mstatus{MSTATUS_INIT}; ///< Value of mstatus CSR - uint64_t mtvec{MTVEC_INIT}; ///< Value of mtvec CSR - uint64_t mscratch{MSCRATCH_INIT}; ///< Value of mscratch CSR - uint64_t mepc{MEPC_INIT}; ///< Value of mepc CSR - uint64_t mcause{MCAUSE_INIT}; ///< Value of mcause CSR - uint64_t mtval{MTVAL_INIT}; ///< Value of mtval CSR - uint64_t misa{MISA_INIT}; ///< Value of misa CSR - uint64_t mie{MIE_INIT}; ///< Value of mie CSR - uint64_t mip{MIP_INIT}; ///< Value of mip CSR - uint64_t medeleg{MEDELEG_INIT}; ///< Value of medeleg CSR - uint64_t mideleg{MIDELEG_INIT}; ///< Value of mideleg CSR - uint64_t mcounteren{MCOUNTEREN_INIT}; ///< Value of mcounteren CSR - uint64_t menvcfg{MENVCFG_INIT}; ///< Value of menvcfg CSR - uint64_t stvec{STVEC_INIT}; ///< Value of stvec CSR - uint64_t sscratch{SSCRATCH_INIT}; ///< Value of sscratch CSR - uint64_t sepc{SEPC_INIT}; ///< Value of sepc CSR - uint64_t scause{SCAUSE_INIT}; ///< Value of scause CSR - uint64_t stval{STVAL_INIT}; ///< Value of stval CSR - uint64_t satp{SATP_INIT}; ///< Value of satp CSR - uint64_t scounteren{SCOUNTEREN_INIT}; ///< Value of scounteren CSR - uint64_t senvcfg{SENVCFG_INIT}; ///< Value of senvcfg CSR - uint64_t ilrsc{ILRSC_INIT}; ///< Value of ilrsc CSR - uint64_t iprv{IPRV_INIT}; ///< Value of iprv CSR - uint64_t iflags_X{IFLAGS_X_INIT}; ///< Value of iflags_X CSR - uint64_t iflags_Y{IFLAGS_Y_INIT}; ///< Value of iflags_Y CSR - uint64_t iflags_H{IFLAGS_H_INIT}; ///< Value of iflags_H CSR - uint64_t iunrep{IUNREP_INIT}; ///< Value of iunrep CSR -}; - /// \brief Backing store config struct backing_store_config final { bool shared{false}; ///< Should changes be reflected in backing store? + bool create{false}; ///< Should backing store be created? bool truncate{false}; ///< Should backing store be truncated to correct size? std::string data_filename; ///< Backing store for associated memory address range std::string dht_filename; ///< Backing store for corresponding dense hash-tree }; +/// \brief Processor state config +struct processor_config final { + registers_state registers; + backing_store_config backing_store; +}; + /// \brief Config with only backing store config field struct backing_store_config_only final { backing_store_config backing_store; @@ -92,8 +58,8 @@ struct backing_store_config_only final { /// \brief RAM state config struct ram_config final { - uint64_t length{0}; ///< RAM length - backing_store_config backing_store; ///< Backing store + uint64_t length{0}; ///< RAM length + backing_store_config backing_store{.truncate = true}; ///< Backing store }; /// \brief DTB state config @@ -117,29 +83,6 @@ struct memory_range_config final { /// \brief List of flash drives using flash_drive_configs = std::vector; -/// \brief TLB device state config -using tlb_config = backing_store_config_only; - -/// \brief CLINT device state config -struct clint_config final { - uint64_t mtimecmp{MTIMECMP_INIT}; ///< Value of mtimecmp CSR -}; - -/// \brief PLIC device state config -struct plic_config final { - uint64_t girqpend{GIRQPEND_INIT}; ///< Value of girqpend CSR - uint64_t girqsrvd{GIRQSRVD_INIT}; ///< Value of girqsrvd CSR -}; - -/// \brief HTIF device state config -struct htif_config final { - uint64_t fromhost{FROMHOST_INIT}; ///< Value of fromhost CSR - uint64_t tohost{TOHOST_INIT}; ///< Value of tohost CSR - bool console_getchar{false}; ///< Make console getchar available? - bool yield_manual{true}; ///< Make yield manual available? - bool yield_automatic{true}; ///< Make yield automatic available? -}; - /// \brief VirtIO console device state config struct virtio_console_config final {}; @@ -193,24 +136,23 @@ using pmas_config = backing_store_config_only; /// \brief Uarch RAM config using uarch_ram_config = backing_store_config_only; -/// \brief Uarch processor config +/// \brief Uarch processor state config struct uarch_processor_config final { - std::array x{}; ///< Value of general-purpose registers - uint64_t pc{UARCH_PC_INIT}; ///< Value of pc - uint64_t cycle{UARCH_CYCLE_INIT}; ///< Value of ucycle counter - uint64_t halt_flag{}; + uarch_registers_state registers; ///< Uarch registers + backing_store_config backing_store; }; /// \brief Uarch config struct uarch_config final { - uarch_processor_config processor{}; ///< Uarch processor - uarch_ram_config ram{}; ///< Uarch RAM + uarch_processor_config processor{}; ///< Uarch processor + uarch_ram_config ram{.backing_store = {.truncate = true}}; ///< Uarch RAM }; /// \brief Hash tree config struct hash_tree_config final { std::string hasher{"keccak"}; ///< What hashing function to use? bool shared{false}; ///< Should changes be reflected in backing store? + bool create{false}; ///< Should backing store be created to correct size? bool truncate{false}; ///< Should backing store be truncated to correct size? std::string sht_filename; ///< Backing storage for sparse hash-tree std::string phtc_filename; ///< Backing storage for page hash-tree cache @@ -223,10 +165,6 @@ struct machine_config final { ram_config ram{}; ///< RAM config dtb_config dtb{}; ///< Device Tree config flash_drive_configs flash_drive; ///< Flash drives config - tlb_config tlb{}; ///< Translation Look-aside Buffer config - clint_config clint{}; ///< Core-Local Interruptor config - plic_config plic{}; ///< Platform-Level Interrupt Controller config - htif_config htif{}; ///< Host-Target config InterFace config virtio_configs virtio; ///< VirtIO devices config cmio_config cmio{}; ///< Cartesi Machine IO config pmas_config pmas{}; ///< Physical Memory Attributes config @@ -248,6 +186,13 @@ struct machine_config final { /// \brief Get the name where global page hash-tree cache will be stored in a directory static std::string get_phtc_filename(const std::string &dir); + static void adjust_backing_store_config(uint64_t start, uint64_t length, const std::string &dir, + backing_store_config &c); + + static void adjust_hash_tree_config(const std::string &dir, hash_tree_config &c); + + void adjust_backing_stores(const std::string &dir); + /// \brief Loads a machine config from a directory /// \param dir Directory from whence "config" will be loaded /// \returns The config loaded diff --git a/src/machine-reg.h b/src/machine-reg.h index d8865c4a9..277b69bac 100644 --- a/src/machine-reg.h +++ b/src/machine-reg.h @@ -17,8 +17,7 @@ #ifndef MACHINE_REG_H #define MACHINE_REG_H -#include "shadow-state-address-range.h" -#include "shadow-uarch-state-address-range.h" +#include "shadow-uarch-state.h" /// \file /// \brief Cartesi machine registers @@ -27,113 +26,112 @@ namespace cartesi { /// \brief List of machine registers enum class machine_reg : uint64_t { - // Processor x registers - x0 = static_cast(shadow_state_what::x0), - x1 = static_cast(shadow_state_what::x1), - x2 = static_cast(shadow_state_what::x2), - x3 = static_cast(shadow_state_what::x3), - x4 = static_cast(shadow_state_what::x4), - x5 = static_cast(shadow_state_what::x5), - x6 = static_cast(shadow_state_what::x6), - x7 = static_cast(shadow_state_what::x7), - x8 = static_cast(shadow_state_what::x8), - x9 = static_cast(shadow_state_what::x9), - x10 = static_cast(shadow_state_what::x10), - x11 = static_cast(shadow_state_what::x11), - x12 = static_cast(shadow_state_what::x12), - x13 = static_cast(shadow_state_what::x13), - x14 = static_cast(shadow_state_what::x14), - x15 = static_cast(shadow_state_what::x15), - x16 = static_cast(shadow_state_what::x16), - x17 = static_cast(shadow_state_what::x17), - x18 = static_cast(shadow_state_what::x18), - x19 = static_cast(shadow_state_what::x19), - x20 = static_cast(shadow_state_what::x20), - x21 = static_cast(shadow_state_what::x21), - x22 = static_cast(shadow_state_what::x22), - x23 = static_cast(shadow_state_what::x23), - x24 = static_cast(shadow_state_what::x24), - x25 = static_cast(shadow_state_what::x25), - x26 = static_cast(shadow_state_what::x26), - x27 = static_cast(shadow_state_what::x27), - x28 = static_cast(shadow_state_what::x28), - x29 = static_cast(shadow_state_what::x29), - x30 = static_cast(shadow_state_what::x30), - x31 = static_cast(shadow_state_what::x31), - f0 = static_cast(shadow_state_what::f0), - f1 = static_cast(shadow_state_what::f1), - f2 = static_cast(shadow_state_what::f2), - f3 = static_cast(shadow_state_what::f3), - f4 = static_cast(shadow_state_what::f4), - f5 = static_cast(shadow_state_what::f5), - f6 = static_cast(shadow_state_what::f6), - f7 = static_cast(shadow_state_what::f7), - f8 = static_cast(shadow_state_what::f8), - f9 = static_cast(shadow_state_what::f9), - f10 = static_cast(shadow_state_what::f10), - f11 = static_cast(shadow_state_what::f11), - f12 = static_cast(shadow_state_what::f12), - f13 = static_cast(shadow_state_what::f13), - f14 = static_cast(shadow_state_what::f14), - f15 = static_cast(shadow_state_what::f15), - f16 = static_cast(shadow_state_what::f16), - f17 = static_cast(shadow_state_what::f17), - f18 = static_cast(shadow_state_what::f18), - f19 = static_cast(shadow_state_what::f19), - f20 = static_cast(shadow_state_what::f20), - f21 = static_cast(shadow_state_what::f21), - f22 = static_cast(shadow_state_what::f22), - f23 = static_cast(shadow_state_what::f23), - f24 = static_cast(shadow_state_what::f24), - f25 = static_cast(shadow_state_what::f25), - f26 = static_cast(shadow_state_what::f26), - f27 = static_cast(shadow_state_what::f27), - f28 = static_cast(shadow_state_what::f28), - f29 = static_cast(shadow_state_what::f29), - f30 = static_cast(shadow_state_what::f30), - f31 = static_cast(shadow_state_what::f31), - pc = static_cast(shadow_state_what::pc), - fcsr = static_cast(shadow_state_what::fcsr), - mvendorid = static_cast(shadow_state_what::mvendorid), - marchid = static_cast(shadow_state_what::marchid), - mimpid = static_cast(shadow_state_what::mimpid), - mcycle = static_cast(shadow_state_what::mcycle), - icycleinstret = static_cast(shadow_state_what::icycleinstret), - mstatus = static_cast(shadow_state_what::mstatus), - mtvec = static_cast(shadow_state_what::mtvec), - mscratch = static_cast(shadow_state_what::mscratch), - mepc = static_cast(shadow_state_what::mepc), - mcause = static_cast(shadow_state_what::mcause), - mtval = static_cast(shadow_state_what::mtval), - misa = static_cast(shadow_state_what::misa), - mie = static_cast(shadow_state_what::mie), - mip = static_cast(shadow_state_what::mip), - medeleg = static_cast(shadow_state_what::medeleg), - mideleg = static_cast(shadow_state_what::mideleg), - mcounteren = static_cast(shadow_state_what::mcounteren), - menvcfg = static_cast(shadow_state_what::menvcfg), - stvec = static_cast(shadow_state_what::stvec), - sscratch = static_cast(shadow_state_what::sscratch), - sepc = static_cast(shadow_state_what::sepc), - scause = static_cast(shadow_state_what::scause), - stval = static_cast(shadow_state_what::stval), - satp = static_cast(shadow_state_what::satp), - scounteren = static_cast(shadow_state_what::scounteren), - senvcfg = static_cast(shadow_state_what::senvcfg), - ilrsc = static_cast(shadow_state_what::ilrsc), - iprv = static_cast(shadow_state_what::iprv), - iflags_X = static_cast(shadow_state_what::iflags_X), - iflags_Y = static_cast(shadow_state_what::iflags_Y), - iflags_H = static_cast(shadow_state_what::iflags_H), - iunrep = static_cast(shadow_state_what::iunrep), - clint_mtimecmp = static_cast(shadow_state_what::clint_mtimecmp), - plic_girqpend = static_cast(shadow_state_what::plic_girqpend), - plic_girqsrvd = static_cast(shadow_state_what::plic_girqsrvd), - htif_tohost = static_cast(shadow_state_what::htif_tohost), - htif_fromhost = static_cast(shadow_state_what::htif_fromhost), - htif_ihalt = static_cast(shadow_state_what::htif_ihalt), - htif_iconsole = static_cast(shadow_state_what::htif_iconsole), - htif_iyield = static_cast(shadow_state_what::htif_iyield), + x0 = static_cast(shadow_registers_what::x0), + x1 = static_cast(shadow_registers_what::x1), + x2 = static_cast(shadow_registers_what::x2), + x3 = static_cast(shadow_registers_what::x3), + x4 = static_cast(shadow_registers_what::x4), + x5 = static_cast(shadow_registers_what::x5), + x6 = static_cast(shadow_registers_what::x6), + x7 = static_cast(shadow_registers_what::x7), + x8 = static_cast(shadow_registers_what::x8), + x9 = static_cast(shadow_registers_what::x9), + x10 = static_cast(shadow_registers_what::x10), + x11 = static_cast(shadow_registers_what::x11), + x12 = static_cast(shadow_registers_what::x12), + x13 = static_cast(shadow_registers_what::x13), + x14 = static_cast(shadow_registers_what::x14), + x15 = static_cast(shadow_registers_what::x15), + x16 = static_cast(shadow_registers_what::x16), + x17 = static_cast(shadow_registers_what::x17), + x18 = static_cast(shadow_registers_what::x18), + x19 = static_cast(shadow_registers_what::x19), + x20 = static_cast(shadow_registers_what::x20), + x21 = static_cast(shadow_registers_what::x21), + x22 = static_cast(shadow_registers_what::x22), + x23 = static_cast(shadow_registers_what::x23), + x24 = static_cast(shadow_registers_what::x24), + x25 = static_cast(shadow_registers_what::x25), + x26 = static_cast(shadow_registers_what::x26), + x27 = static_cast(shadow_registers_what::x27), + x28 = static_cast(shadow_registers_what::x28), + x29 = static_cast(shadow_registers_what::x29), + x30 = static_cast(shadow_registers_what::x30), + x31 = static_cast(shadow_registers_what::x31), + f0 = static_cast(shadow_registers_what::f0), + f1 = static_cast(shadow_registers_what::f1), + f2 = static_cast(shadow_registers_what::f2), + f3 = static_cast(shadow_registers_what::f3), + f4 = static_cast(shadow_registers_what::f4), + f5 = static_cast(shadow_registers_what::f5), + f6 = static_cast(shadow_registers_what::f6), + f7 = static_cast(shadow_registers_what::f7), + f8 = static_cast(shadow_registers_what::f8), + f9 = static_cast(shadow_registers_what::f9), + f10 = static_cast(shadow_registers_what::f10), + f11 = static_cast(shadow_registers_what::f11), + f12 = static_cast(shadow_registers_what::f12), + f13 = static_cast(shadow_registers_what::f13), + f14 = static_cast(shadow_registers_what::f14), + f15 = static_cast(shadow_registers_what::f15), + f16 = static_cast(shadow_registers_what::f16), + f17 = static_cast(shadow_registers_what::f17), + f18 = static_cast(shadow_registers_what::f18), + f19 = static_cast(shadow_registers_what::f19), + f20 = static_cast(shadow_registers_what::f20), + f21 = static_cast(shadow_registers_what::f21), + f22 = static_cast(shadow_registers_what::f22), + f23 = static_cast(shadow_registers_what::f23), + f24 = static_cast(shadow_registers_what::f24), + f25 = static_cast(shadow_registers_what::f25), + f26 = static_cast(shadow_registers_what::f26), + f27 = static_cast(shadow_registers_what::f27), + f28 = static_cast(shadow_registers_what::f28), + f29 = static_cast(shadow_registers_what::f29), + f30 = static_cast(shadow_registers_what::f30), + f31 = static_cast(shadow_registers_what::f31), + pc = static_cast(shadow_registers_what::pc), + fcsr = static_cast(shadow_registers_what::fcsr), + mvendorid = static_cast(shadow_registers_what::mvendorid), + marchid = static_cast(shadow_registers_what::marchid), + mimpid = static_cast(shadow_registers_what::mimpid), + mcycle = static_cast(shadow_registers_what::mcycle), + icycleinstret = static_cast(shadow_registers_what::icycleinstret), + mstatus = static_cast(shadow_registers_what::mstatus), + mtvec = static_cast(shadow_registers_what::mtvec), + mscratch = static_cast(shadow_registers_what::mscratch), + mepc = static_cast(shadow_registers_what::mepc), + mcause = static_cast(shadow_registers_what::mcause), + mtval = static_cast(shadow_registers_what::mtval), + misa = static_cast(shadow_registers_what::misa), + mie = static_cast(shadow_registers_what::mie), + mip = static_cast(shadow_registers_what::mip), + medeleg = static_cast(shadow_registers_what::medeleg), + mideleg = static_cast(shadow_registers_what::mideleg), + mcounteren = static_cast(shadow_registers_what::mcounteren), + menvcfg = static_cast(shadow_registers_what::menvcfg), + stvec = static_cast(shadow_registers_what::stvec), + sscratch = static_cast(shadow_registers_what::sscratch), + sepc = static_cast(shadow_registers_what::sepc), + scause = static_cast(shadow_registers_what::scause), + stval = static_cast(shadow_registers_what::stval), + satp = static_cast(shadow_registers_what::satp), + scounteren = static_cast(shadow_registers_what::scounteren), + senvcfg = static_cast(shadow_registers_what::senvcfg), + ilrsc = static_cast(shadow_registers_what::ilrsc), + iprv = static_cast(shadow_registers_what::iprv), + iflags_X = static_cast(shadow_registers_what::iflags_X), + iflags_Y = static_cast(shadow_registers_what::iflags_Y), + iflags_H = static_cast(shadow_registers_what::iflags_H), + iunrep = static_cast(shadow_registers_what::iunrep), + clint_mtimecmp = static_cast(shadow_registers_what::clint_mtimecmp), + plic_girqpend = static_cast(shadow_registers_what::plic_girqpend), + plic_girqsrvd = static_cast(shadow_registers_what::plic_girqsrvd), + htif_tohost = static_cast(shadow_registers_what::htif_tohost), + htif_fromhost = static_cast(shadow_registers_what::htif_fromhost), + htif_ihalt = static_cast(shadow_registers_what::htif_ihalt), + htif_iconsole = static_cast(shadow_registers_what::htif_iconsole), + htif_iyield = static_cast(shadow_registers_what::htif_iyield), first_ = x0, last_ = htif_iyield, @@ -200,7 +198,7 @@ static constexpr machine_reg machine_reg_enum(machine_reg reg, int i) { return static_cast(static_cast(reg) + (i * sizeof(uint64_t))); } -static constexpr machine_reg machine_reg_enum(shadow_state_what reg) { +static constexpr machine_reg machine_reg_enum(shadow_registers_what reg) { return static_cast(reg); } @@ -211,7 +209,7 @@ static constexpr machine_reg machine_reg_enum(shadow_uarch_state_what reg) { static constexpr const char *machine_reg_get_name(machine_reg reg) { const auto ureg = static_cast(reg); if (ureg >= static_cast(machine_reg::first_) && ureg <= static_cast(machine_reg::last_)) { - return shadow_state_get_what_name(static_cast(reg)); + return shadow_registers_get_what_name(static_cast(reg)); } if (ureg >= static_cast(machine_reg::uarch_first_) && ureg <= static_cast(machine_reg::uarch_last_)) { diff --git a/src/machine-state.h b/src/machine-state.h deleted file mode 100644 index 8fcfa1c6e..000000000 --- a/src/machine-state.h +++ /dev/null @@ -1,122 +0,0 @@ -// Copyright Cartesi and individual authors (see AUTHORS) -// SPDX-License-Identifier: LGPL-3.0-or-later -// -// This program is free software: you can redistribute it and/or modify it under -// the terms of the GNU Lesser General Public License as published by the Free -// Software Foundation, either version 3 of the License, or (at your option) any -// later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT ANY -// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A -// PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License along -// with this program (see COPYING). If not, see . -// - -#ifndef MACHINE_STATE_H -#define MACHINE_STATE_H - -/// \file -/// \brief Cartesi machine state structure definition. - -#include -#include -#include - -#include "address-range.h" -#include "riscv-constants.h" -#include "tlb.h" - -namespace cartesi { - -/// \brief Machine state. -/// \details The machine_state structure contains the entire -/// state of a Cartesi machine. -struct machine_state { - machine_state() = default; - ~machine_state() = default; - - /// \brief No copy or move constructor or assignment - machine_state(const machine_state &other) = delete; - machine_state(machine_state &&other) = delete; - machine_state &operator=(const machine_state &other) = delete; - machine_state &operator=(machine_state &&other) = delete; - - // The following state fields are very hot, - // and are carefully ordered to have better data locality in the interpreter loop. - // The X registers are the very first to optimize access of registers in the interpreter. - std::array x{}; ///< Register file - uint64_t mcycle{}; ///< CSR mcycle. - uint64_t pc{}; ///< Program counter. - uint64_t fcsr{}; ///< CSR fcsr. - std::array f{}; ///< Floating-point register file. - - uint64_t iprv{}; ///< Privilege level (Cartesi-specific). - - uint64_t mstatus{}; ///< CSR mstatus. - uint64_t mtvec{}; ///< CSR mtvec. - uint64_t mscratch{}; ///< CSR mscratch. - uint64_t mepc{}; ///< CSR mepc. - uint64_t mcause{}; ///< CSR mcause. - uint64_t mtval{}; ///< CSR mtval. - uint64_t misa{}; ///< CSR misa. - - uint64_t mie{}; ///< CSR mie. - uint64_t mip{}; ///< CSR mip. - uint64_t medeleg{}; ///< CSR medeleg. - uint64_t mideleg{}; ///< CSR mideleg. - uint64_t mcounteren{}; ///< CSR mcounteren. - uint64_t menvcfg{}; ///< CSR menvcfg. - - uint64_t stvec{}; ///< CSR stvec. - uint64_t sscratch{}; ///< CSR sscratch. - uint64_t sepc{}; ///< CSR sepc. - uint64_t scause{}; ///< CSR scause. - uint64_t stval{}; ///< CSR stval. - uint64_t satp{}; ///< CSR satp. - uint64_t scounteren{}; ///< CSR scounteren. - uint64_t senvcfg{}; ///< CSR senvcfg. - - // Cartesi-specific state - uint64_t ilrsc{}; ///< For LR/SC instructions (Cartesi-specific). - uint64_t icycleinstret{}; ///< CSR icycleinstret (Cartesi-specific). - struct { - uint64_t X{}; ///< CPU has yielded with automatic reset (Cartesi-specific). - uint64_t Y{}; ///< CPU has yielded with manual reset (Cartesi-specific). - uint64_t H{}; ///< CPU has been permanently halted (Cartesi-specific). - } iflags; - uint64_t iunrep{}; ///< Unreproducible mode (Cartesi-specific). - - /// \brief CLINT state - struct { - uint64_t mtimecmp{}; ///< CSR mtimecmp. - } clint; - - /// \brief PLIC state - struct { - uint64_t girqpend{}; ///< CSR girqpend (global interrupts pending). - uint64_t girqsrvd{}; ///< CSR girqsrvd (global interrupts served). - } plic; - - /// \brief TLB state - tlb_state tlb{}; - - /// \brief HTIF state - struct { - uint64_t tohost{}; ///< CSR tohost. - uint64_t fromhost{}; ///< CSR fromhost. - uint64_t ihalt{}; ///< CSR ihalt. - uint64_t iconsole{}; ///< CSR iconsole. - uint64_t iyield{}; ///< CSR iyield. - } htif; - - /// Soft yield - bool soft_yield{}; - - std::vector pmas; ///< Indices of address ranges that interpret can find -}; - -} // namespace cartesi - -#endif diff --git a/src/machine.cpp b/src/machine.cpp index c537231d7..0e00e2991 100644 --- a/src/machine.cpp +++ b/src/machine.cpp @@ -36,6 +36,7 @@ #include #include "access-log.h" +#include "address-range-constants.h" #include "address-range-description.h" #include "address-range.h" #include "clint-address-range.h" @@ -43,6 +44,7 @@ #include "device-state-access.h" #include "dtb.h" #include "host-addr.h" +#include "hot-tlb.h" #include "htif-address-range.h" #include "htif-constants.h" #include "i-device-state-access.h" @@ -55,6 +57,7 @@ #include "memory-address-range.h" #include "plic-address-range.h" #include "pmas.h" +#include "processor-state.h" #include "record-send-cmio-state-access.h" #include "record-step-state-access.h" #include "replay-send-cmio-state-access.h" @@ -62,12 +65,8 @@ #include "riscv-constants.h" #include "rtc.h" #include "send-cmio-response.h" -#include "shadow-state-address-range.h" -#include "shadow-tlb-address-range.h" -#include "shadow-uarch-state-address-range.h" #include "state-access.h" #include "strict-aliasing.h" -#include "tlb.h" #include "translate-virtual-address.h" #include "uarch-constants.h" #include "uarch-interpret.h" @@ -93,30 +92,9 @@ using namespace std::string_literals; static const auto throw_invalid_argument = [](const char *err) { throw std::invalid_argument{err}; }; -/// \brief Creates a memory address range. -/// \param d Description of address range for use in error messages. -/// \param start Target physical address where range starts. -/// \param length Length of range, in bytes. -/// \param f Flags for address range. -/// \param backing_store Backing store configuration for range. -/// \returns New address range with flags already set. -/// \details If \p backing_store.data_filename is non-empty and file is large enough to back entire address range, -/// return a memory-mapped range, otherwise use calloc. -static inline auto make_memory_address_range(const std::string &d, uint64_t start, uint64_t length, pmas_flags flags, - const backing_store_config &backing_store) { - if (backing_store.data_filename.empty() && backing_store.shared) { - throw std::invalid_argument{"shared address range requires non-empty memory filename when initializing " + d}; - } - if (backing_store.data_filename.empty() || - length > static_cast(os_get_file_length(backing_store.data_filename.c_str()))) { - return make_callocd_memory_address_range(d, start, length, flags, backing_store.data_filename); - } - return make_mmapd_memory_address_range(d, start, length, flags, backing_store.data_filename, backing_store.shared); -} - void machine::check_address_range(const address_range &ar, register_where where) { if (!where.interpret && !where.merkle) { - throw std::runtime_error{"address range "s + ar.get_description() + " must be registered somwhere"s}; + throw std::runtime_error{"address range "s + ar.get_description() + " must be registered somewhere"s}; } const auto start = ar.get_start(); const auto length = ar.get_length(); @@ -137,49 +115,64 @@ void machine::check_address_range(const address_range &ar, register_where where) } } -static bool is_protected(PMA_ISTART_DID DID) { - switch (DID) { - case PMA_ISTART_DID::memory: - case PMA_ISTART_DID::flash_drive: - case PMA_ISTART_DID::cmio_rx_buffer: - case PMA_ISTART_DID::cmio_tx_buffer: - return false; - default: - return true; - } -} - void machine::replace_memory_range(const memory_range_config &config) { for (auto &existing : m_ars) { - //??D Need to add check for new read-only property here if (existing->get_start() == config.start && existing->get_length() == config.length) { - if (!existing->is_memory() || is_protected(existing->get_driver_id())) { + if (!existing->is_replaceable()) { throw std::invalid_argument{"attempt to replace a protected range "s + existing->get_description()}; } + if (existing->is_host_read_only()) { + throw std::invalid_argument{"attempt to replace a read-only range "s + existing->get_description()}; + } // Replace range, preserving original flags. // This will automatically start with all pages dirty. - existing = make_moved_unique(make_memory_address_range(existing->get_description(), existing->get_start(), - existing->get_length(), existing->get_flags(), config.backing_store)); + existing = make_moved_unique(memory_address_range{existing->get_description(), existing->get_start(), + existing->get_length(), existing->get_flags(), config.backing_store, + memory_address_range_flags{.read_only = config.read_only}}); return; } } throw std::invalid_argument{"attempt to replace inexistent memory range"}; } -void machine::init_uarch(const uarch_config &c) { - using reg = machine_reg; - write_reg(reg::uarch_pc, c.processor.pc); - write_reg(reg::uarch_cycle, c.processor.cycle); - write_reg(reg::uarch_halt_flag, c.processor.halt_flag); - // General purpose registers - for (int i = 1; i < UARCH_X_REG_COUNT; i++) { - write_reg(machine_reg_enum(reg::uarch_x0, i), c.processor.x[i]); - } - // Register shadow state - m_us.shadow_state = ®ister_address_range(make_shadow_uarch_state_address_range(AR_SHADOW_UARCH_STATE_START, - AR_SHADOW_UARCH_STATE_LENGTH, throw_invalid_argument), +void machine::init_uarch_processor(const uarch_processor_config &c) { + // Register shadow uarch state + static constexpr pmas_flags shadow_uarch_state_flags{ + .M = true, + .IO = false, + .R = false, + .W = false, + .X = false, + .IR = false, + .IW = false, + .DID = PMA_ISTART_DID::shadow_uarch_state, + }; + auto &us = register_address_range(memory_address_range{"shadow uarch state"s, AR_SHADOW_UARCH_STATE_START, + AR_SHADOW_UARCH_STATE_LENGTH, shadow_uarch_state_flags, c.backing_store, + memory_address_range_flags{.page_uncleanable = true}}, register_where{.merkle = true, .interpret = false}); - // Register RAM + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + m_us = reinterpret_cast(us.get_host_memory()); + + // Mark pages that are permanently dirty in the shadow + us.mark_dirty_pages(AR_SHADOW_UARCH_STATE_START, AR_SHADOW_UARCH_STATE_LENGTH); + + // Initialize shadow uarch state + if (c.backing_store.data_filename.empty() || c.backing_store.create) { + *m_us = uarch_processor_state{}; + + // Initialize uarch registers + write_reg(reg::uarch_pc, c.registers.pc); + write_reg(reg::uarch_cycle, c.registers.cycle); + write_reg(reg::uarch_halt_flag, c.registers.halt_flag); + for (int i = 1; i < UARCH_X_REG_COUNT; i++) { + write_reg(machine_reg_enum(reg::uarch_x0, i), c.registers.x[i]); + } + } +} + +void machine::init_uarch_ram(const uarch_ram_config &c) { + // Register uarch RAM if (uarch_pristine_ram_len > AR_UARCH_RAM_LENGTH) { throw std::runtime_error("embedded uarch RAM image does not fit in uarch memory"); } @@ -194,20 +187,53 @@ void machine::init_uarch(const uarch_config &c) { .DID = PMA_ISTART_DID::memory, }; constexpr auto ram_description = "uarch RAM"; - if (c.ram.backing_store.data_filename.empty()) { - m_us.ram = ®ister_address_range( - make_callocd_memory_address_range(ram_description, AR_UARCH_RAM_START, UARCH_RAM_LENGTH, uram_flags), - register_where{.merkle = true, .interpret = false}); - memcpy(m_us.ram->get_host_memory(), uarch_pristine_ram, uarch_pristine_ram_len); - } else { - m_us.ram = ®ister_address_range(make_memory_address_range(ram_description, AR_UARCH_RAM_START, - UARCH_RAM_LENGTH, uram_flags, c.ram.backing_store), - register_where{.merkle = true, .interpret = false}); + auto &uram = register_address_range( + memory_address_range{ram_description, AR_UARCH_RAM_START, UARCH_RAM_LENGTH, uram_flags, c.backing_store}, + register_where{.merkle = true, .interpret = false}); + + // Initialize uarch RAM + if (c.backing_store.data_filename.empty() || c.backing_store.create) { + memcpy(uram.get_host_memory(), uarch_pristine_ram, uarch_pristine_ram_len); } } void machine::init_processor(processor_config &p, const machine_runtime_config &r) { + // Register shadow state + static constexpr pmas_flags shadow_state_flags{ + .M = true, + .IO = false, + .R = false, + .W = false, + .X = false, + .IR = false, + .IW = false, + .DID = PMA_ISTART_DID::shadow_state, + }; + auto &shadow = register_address_range( + memory_address_range{"shadow state"s, AR_SHADOW_STATE_START, AR_SHADOW_STATE_LENGTH, shadow_state_flags, + p.backing_store, memory_address_range_flags{.page_uncleanable = true}, sizeof(processor_state)}, + register_where{.merkle = true, .interpret = false}); + + // Mark pages that are permanently dirty in the shadow + shadow.mark_dirty_pages(AR_SHADOW_REGISTERS_START, AR_SHADOW_REGISTERS_LENGTH); + shadow.mark_dirty_pages(AR_SHADOW_TLB_START, AR_SHADOW_TLB_LENGTH); + + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + m_s = reinterpret_cast(shadow.get_host_memory()); + // Initialize shadow state + if (p.backing_store.data_filename.empty() || p.backing_store.create) { + // Initialize registers and TLB to default values + *m_s = processor_state{}; + + // Initialize custom registers + init_registers(p.registers, r); + + // The machine penumbra state will be initialized later after all PMAs are registers + } +} + +void machine::init_registers(registers_state &p, const machine_runtime_config &r) { if (p.marchid == UINT64_C(-1)) { p.marchid = MARCHID_INIT; } @@ -270,10 +296,24 @@ void machine::init_processor(processor_config &p, const machine_runtime_config & write_reg(reg::senvcfg, p.senvcfg); write_reg(reg::ilrsc, p.ilrsc); write_reg(reg::iprv, p.iprv); - write_reg(reg::iflags_X, p.iflags_X); - write_reg(reg::iflags_Y, p.iflags_Y); - write_reg(reg::iflags_H, p.iflags_H); + write_reg(reg::iflags_X, p.iflags.X); + write_reg(reg::iflags_Y, p.iflags.Y); + write_reg(reg::iflags_H, p.iflags.H); write_reg(reg::iunrep, p.iunrep); + + // Copy HTIF state to from config to machine + write_reg(reg::htif_tohost, p.htif.tohost); + write_reg(reg::htif_fromhost, p.htif.fromhost); + write_reg(reg::htif_ihalt, p.htif.ihalt); + write_reg(reg::htif_iconsole, p.htif.iconsole); + write_reg(reg::htif_iyield, p.htif.iyield); + + // Copy CLINT state to from config to machine + write_reg(reg::clint_mtimecmp, p.clint.mtimecmp); + + // Copy PLIC state from config to machine + write_reg(reg::plic_girqpend, p.plic.girqpend); + write_reg(reg::plic_girqsrvd, p.plic.girqsrvd); } void machine::init_ram_ar(const ram_config &ram) { @@ -291,7 +331,7 @@ void machine::init_ram_ar(const ram_config &ram) { if (ram.length == 0) { throw std::invalid_argument("RAM length cannot be zero"); } - register_address_range(make_memory_address_range("RAM"s, AR_RAM_START, ram.length, ram_flags, ram.backing_store), + register_address_range(memory_address_range{"RAM"s, AR_RAM_START, ram.length, ram_flags, ram.backing_store}, register_where{.merkle = true, .interpret = true}); } @@ -299,17 +339,6 @@ void machine::init_flash_drive_ars(flash_drive_configs &flash_drive) { if (flash_drive.size() > FLASH_DRIVE_MAX) { throw std::invalid_argument{"too many flash drives"}; } - // Flags for flash drives - static const pmas_flags flash_flags{ - .M = true, - .IO = false, - .R = true, - .W = true, - .X = false, - .IR = true, - .IW = true, - .DID = PMA_ISTART_DID::flash_drive, - }; // Register all flash drives int i = 0; // NOLINT(misc-const-correctness) for (auto &f : flash_drive) { @@ -322,7 +351,7 @@ void machine::init_flash_drive_ars(flash_drive_configs &flash_drive) { const auto &image_filename = f.backing_store.data_filename; if (f.length == UINT64_C(-1)) { if (image_filename.empty()) { - throw std::system_error{errno, std::generic_category(), + throw std::runtime_error{ "unable to auto-detect length of "s.append(flash_description).append(" with empty image file")}; } auto fp = make_unique_fopen(image_filename.c_str(), "rb"); @@ -341,8 +370,19 @@ void machine::init_flash_drive_ars(flash_drive_configs &flash_drive) { } f.length = length; } - register_address_range( - make_memory_address_range(flash_description, f.start, f.length, flash_flags, f.backing_store), + // Flags for flash drives + const pmas_flags flash_flags{ + .M = true, + .IO = false, + .R = true, + .W = !f.read_only, + .X = false, + .IR = true, + .IW = !f.read_only, + .DID = PMA_ISTART_DID::flash_drive, + }; + register_address_range(memory_address_range{flash_description, f.start, f.length, flash_flags, f.backing_store, + memory_address_range_flags{.read_only = f.read_only}}, register_where{.merkle = true, .interpret = true}); i++; } @@ -417,22 +457,10 @@ void machine::init_virtio_ars(const virtio_configs &virtio, uint64_t iunrep) { } } -void machine::init_htif_ar(const htif_config &h) { +void machine::init_htif_ar() { // Register HTIF device register_address_range(make_htif_address_range(throw_invalid_argument), register_where{.merkle = false, .interpret = true}); - // Copy HTIF state to from config to machine - write_reg(reg::htif_tohost, h.tohost); - write_reg(reg::htif_fromhost, h.fromhost); - // Only command in halt device is command 0 and it is always available - const uint64_t htif_ihalt = static_cast(true) << HTIF_HALT_CMD_HALT; - write_reg(reg::htif_ihalt, htif_ihalt); - const uint64_t htif_iconsole = static_cast(h.console_getchar) << HTIF_CONSOLE_CMD_GETCHAR | - static_cast(true) << HTIF_CONSOLE_CMD_PUTCHAR; - write_reg(reg::htif_iconsole, htif_iconsole); - const uint64_t htif_iyield = static_cast(h.yield_manual) << HTIF_YIELD_CMD_MANUAL | - static_cast(h.yield_automatic) << HTIF_YIELD_CMD_AUTOMATIC; - write_reg(reg::htif_iyield, htif_iyield); } void machine::init_cmio_ars(const cmio_config &c) { @@ -456,11 +484,11 @@ void machine::init_cmio_ars(const cmio_config &c) { .IW = true, .DID = PMA_ISTART_DID::cmio_rx_buffer, }; - register_address_range(make_memory_address_range("CMIO tx buffer memory range"s, AR_CMIO_TX_BUFFER_START, - AR_CMIO_TX_BUFFER_LENGTH, tx_flags, c.tx_buffer.backing_store), + register_address_range(memory_address_range{"CMIO tx buffer memory range"s, AR_CMIO_TX_BUFFER_START, + AR_CMIO_TX_BUFFER_LENGTH, tx_flags, c.tx_buffer.backing_store}, register_where{.merkle = true, .interpret = true}); - register_address_range(make_memory_address_range("CMIO rx buffer memory range"s, AR_CMIO_RX_BUFFER_START, - AR_CMIO_RX_BUFFER_LENGTH, rx_flags, c.rx_buffer.backing_store), + register_address_range(memory_address_range{"CMIO rx buffer memory range"s, AR_CMIO_RX_BUFFER_START, + AR_CMIO_RX_BUFFER_LENGTH, rx_flags, c.rx_buffer.backing_store}, register_where{.merkle = true, .interpret = true}); } @@ -483,56 +511,71 @@ void machine::init_ars_descriptions() { std::ranges::sort(m_ards, [](auto &a, auto &b) { return a.start < b.start; }); } -void machine::init_clint_ar(const clint_config &c) { +void machine::init_clint_ar() { // Register CLINT device register_address_range(make_clint_address_range(throw_invalid_argument), register_where{.merkle = false, .interpret = true}); - // Copy CLINT state to from config to machine - write_reg(reg::clint_mtimecmp, c.mtimecmp); } -void machine::init_plic_ar(const plic_config &p) { +void machine::init_plic_ar() { // Register PLIC device register_address_range(make_plic_address_range(throw_invalid_argument), register_where{.merkle = false, .interpret = true}); - // Copy PLIC state from config to machine - write_reg(reg::plic_girqpend, p.girqpend); - write_reg(reg::plic_girqsrvd, p.girqsrvd); } void machine::init_pmas_contents(const pmas_config &config, memory_address_range &pmas) const { static_assert(sizeof(pmas_state) == PMA_MAX * 2 * sizeof(uint64_t), "inconsistent PMAs state length"); static_assert(AR_PMAS_LENGTH >= sizeof(pmas_state), "PMAs address range too short"); - if (config.backing_store.data_filename.empty()) { - if (m_s.pmas.size() >= PMA_MAX - 1) { // Leave room for a sentinel empty address range after all others + if (config.backing_store.data_filename.empty() || config.backing_store.create) { + if (m_pmas.size() >= PMA_MAX - 1) { // Leave room for a sentinel empty address range after all others throw std::runtime_error{"too many address ranges"}; } // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) auto &dest = *reinterpret_cast(pmas.get_host_memory()); - std::ranges::transform(m_s.pmas, dest.begin(), [this](auto i) { + std::ranges::transform(m_pmas, dest.begin(), [this](auto i) { return pmas_entry{.istart = m_ars[i]->get_istart(), .ilength = m_ars[i]->get_ilength()}; }); } } -void machine::init_tlb_contents(const tlb_config &config) { - if (const auto &image_filename = config.backing_store.data_filename; !image_filename.empty()) { - auto shadow_tlb_ptr = make_unique_mmap(image_filename.c_str(), 1, false /* not shared */); - auto &shadow_tlb = *shadow_tlb_ptr; - for (auto set_index : {TLB_CODE, TLB_READ, TLB_WRITE}) { - for (uint64_t slot_index = 0; slot_index < TLB_SET_SIZE; ++slot_index) { - const auto vaddr_page = shadow_tlb[set_index][slot_index].vaddr_page; - const auto vp_offset = shadow_tlb[set_index][slot_index].vp_offset; - const auto pma_index = shadow_tlb[set_index][slot_index].pma_index; - check_shadow_tlb(set_index, slot_index, vaddr_page, vp_offset, pma_index, "stored TLB is corrupt: "s); - write_shadow_tlb(set_index, slot_index, vaddr_page, vp_offset, pma_index); +void machine::init_hot_tlb_contents() { + for (auto set_index : {TLB_CODE, TLB_READ, TLB_WRITE}) { + for (uint64_t slot_index = 0; slot_index < TLB_SET_SIZE; ++slot_index) { + const auto &shadow_slot = m_s->shadow_tlb[set_index][slot_index]; + const auto vaddr_page = shadow_slot.vaddr_page; + const auto vp_offset = shadow_slot.vp_offset; + const auto pma_index = shadow_slot.pma_index; + const auto zero_padding_ = shadow_slot.zero_padding_; + host_addr vh_offset{}; + if (zero_padding_ != 0) { + throw std::domain_error{"stored TLB is corrupt: inconsistent padding"}; } - } - } else { - for (auto set_index : {TLB_CODE, TLB_READ, TLB_WRITE}) { - for (uint64_t slot_index = 0; slot_index < TLB_SET_SIZE; ++slot_index) { - write_tlb(set_index, slot_index, TLB_INVALID_PAGE, host_addr{}, TLB_INVALID_PMA_INDEX); + if (vaddr_page != TLB_INVALID_PAGE) { + if (pma_index >= m_pmas.size()) { + throw std::domain_error{"stored TLB is corrupt: pma_index is out of range"s}; + } + const auto &ar = read_pma(pma_index); + if (!ar.is_memory()) { + throw std::invalid_argument{"stored TLB is corrupt: pma_index does not point to memory range"s}; + } + if ((vaddr_page & PAGE_OFFSET_MASK) != 0) { + throw std::invalid_argument{"stored TLB is corrupt: vaddr_page is not aligned"s}; + } + const auto paddr_page = vaddr_page + vp_offset; + if ((paddr_page & PAGE_OFFSET_MASK) != 0) { + throw std::invalid_argument{"stored TLB is corrupt: vp_offset is not aligned"s}; + } + const auto pmas_end = ar.get_start() + (ar.get_length() - AR_PAGE_SIZE); + if (paddr_page < ar.get_start() || paddr_page > pmas_end) { + throw std::invalid_argument{"stored TLB is corrupt: vp_offset is inconsistent with pma_index"s}; + } + vh_offset = get_host_addr(paddr_page, pma_index) - vaddr_page; + } else if (pma_index != TLB_INVALID_PMA_INDEX || vp_offset != 0) { + throw std::domain_error{"stored TLB is corrupt: inconsistent empty slot"}; } + auto &hot_slot = m_s->hot_tlb[set_index][slot_index]; + hot_slot.vaddr_page = vaddr_page; + hot_slot.vh_offset = vh_offset; } } } @@ -550,11 +593,11 @@ static inline auto make_dtb_address_range(const dtb_config &config) { .IW = true, .DID = PMA_ISTART_DID::memory, }; - return make_memory_address_range("DTB"s, AR_DTB_START, AR_DTB_LENGTH, dtb_flags, config.backing_store); + return memory_address_range{"DTB"s, AR_DTB_START, AR_DTB_LENGTH, dtb_flags, config.backing_store}; } void machine::init_dtb_contents(const machine_config &config, memory_address_range &dtb) { - if (config.dtb.backing_store.data_filename.empty()) { + if (config.dtb.backing_store.data_filename.empty() || config.dtb.backing_store.create) { dtb_init(config, dtb.get_host_memory(), dtb.get_length()); } } @@ -570,54 +613,51 @@ static inline auto make_pmas_address_range(const pmas_config &config) { .IW = false, .DID = PMA_ISTART_DID::memory, }; - return make_memory_address_range("PMAs", AR_PMAS_START, AR_PMAS_LENGTH, m_pmas_flags, config.backing_store); + return memory_address_range{"PMAs", AR_PMAS_START, AR_PMAS_LENGTH, m_pmas_flags, config.backing_store}; } // ??D It is best to leave the std::move() on r because it may one day be necessary! // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) -machine::machine(machine_config c, machine_runtime_config r) : m_c{std::move(c)}, m_r{std::move(r)} { - init_uarch(m_c.uarch); +machine::machine(machine_config c, machine_runtime_config r) : m_c{std::move(c)}, m_r{r}, m_soft_yield(r.soft_yield) { + // We need to initialize the machine processor first thing to allocate its shadow state init_processor(m_c.processor, m_r); - m_s.soft_yield = m_r.soft_yield; + // We need to initialize the uarch processor state as the second thing to allocate its shadow state + init_uarch_processor(m_c.uarch.processor); + init_uarch_ram(m_c.uarch.ram); init_ram_ar(m_c.ram); // Will populate when initialization of PMAs is done auto &dtb = register_address_range(make_dtb_address_range(m_c.dtb), register_where{.merkle = true, .interpret = true}); init_flash_drive_ars(m_c.flash_drive); init_cmio_ars(m_c.cmio); - init_htif_ar(m_c.htif); - init_clint_ar(m_c.clint); - init_plic_ar(m_c.plic); - // Will populate when initialization of PMAs is done - register_address_range(make_shadow_tlb_address_range(throw_invalid_argument), - register_where{.merkle = true, .interpret = false}); - register_address_range(make_shadow_state_address_range(throw_invalid_argument), - register_where{.merkle = true, .interpret = false}); + init_htif_ar(); + init_clint_ar(); + init_plic_ar(); // Will populate when initialization of PMAs is done auto &pmas = register_address_range(make_pmas_address_range(m_c.pmas), register_where{.merkle = true, .interpret = true}); - init_virtio_ars(m_c.virtio, m_c.processor.iunrep); + init_virtio_ars(m_c.virtio, m_c.processor.registers.iunrep); // Populate PMAs contents. // This must be done after all PMA entries are already registered, so we encode them into the shadow init_pmas_contents(m_c.pmas, pmas); - // Initialize TLB contents. + // Initialize hot TLB contents. // This must be done after all PMA entries are already registered, so we can lookup page addresses - init_tlb_contents(m_c.tlb); + init_hot_tlb_contents(); // Initialize DTB contents. // This must be done after all PMA entries are already registered, so we can lookup flash drive parameters init_dtb_contents(m_c, dtb); init_merkle_ars(); init_ars_descriptions(); - init_tty(m_c.htif, m_r.htif, m_c.processor.iunrep); + init_tty(m_c.processor.registers.htif, m_r.htif, m_c.processor.registers.iunrep); // Disable SIGPIPE handler, because this signal can be raised and terminate the emulator process // when calling write() on closed file descriptors. // This can happen with the stdout console file descriptors or network file descriptors. os_disable_sigpipe(); } -void machine::init_tty(const htif_config &h, const htif_runtime_config &r, uint64_t iunrep) const { +void machine::init_tty(const htif_state &h, const htif_runtime_config &r, uint64_t iunrep) const { // Initialize TTY if console input is enabled - if (h.console_getchar || has_virtio_console()) { + if ((h.iconsole & HTIF_CONSOLE_CMD_GETCHAR_MASK) != 0 || has_virtio_console()) { if (iunrep == 0) { throw std::invalid_argument{"TTY stdin is only supported in unreproducible machines"}; } @@ -686,7 +726,7 @@ bool machine::has_virtio_console() const { } bool machine::has_htif_console() const { - return static_cast(read_reg(reg::htif_iconsole) & (1 << HTIF_CONSOLE_CMD_GETCHAR)); + return static_cast(read_reg(reg::htif_iconsole) & HTIF_CONSOLE_CMD_GETCHAR); } /// \brief Returns copy of initialization config. @@ -702,101 +742,10 @@ const machine_runtime_config &machine::get_runtime_config() const { /// \brief Changes the machine runtime config. void machine::set_runtime_config(machine_runtime_config r) { m_r = std::move(r); // NOLINT(hicpp-move-const-arg,performance-move-const-arg) - m_s.soft_yield = m_r.soft_yield; + m_soft_yield = m_r.soft_yield; os_silence_putchar(m_r.htif.no_console_putchar); } -static void clear_backing_store_filenames(backing_store_config &config) { - config.data_filename.clear(); - config.dht_filename.clear(); -} - -machine_config machine::get_serialization_config() const { - if (read_reg(reg::iunrep) != 0) { - throw std::runtime_error{"cannot serialize configuration of unreproducible machines"}; - } - // Initialize with copy of original config - machine_config c = m_c; - // Copy current processor state to config - for (int i = 1; i < X_REG_COUNT; ++i) { - c.processor.x[i] = read_reg(machine_reg_enum(reg::x0, i)); - } - for (int i = 0; i < F_REG_COUNT; ++i) { - c.processor.f[i] = read_reg(machine_reg_enum(reg::f0, i)); - } - c.processor.pc = read_reg(reg::pc); - c.processor.fcsr = read_reg(reg::fcsr); - c.processor.mvendorid = read_reg(reg::mvendorid); - c.processor.marchid = read_reg(reg::marchid); - c.processor.mimpid = read_reg(reg::mimpid); - c.processor.mcycle = read_reg(reg::mcycle); - c.processor.icycleinstret = read_reg(reg::icycleinstret); - c.processor.mstatus = read_reg(reg::mstatus); - c.processor.mtvec = read_reg(reg::mtvec); - c.processor.mscratch = read_reg(reg::mscratch); - c.processor.mepc = read_reg(reg::mepc); - c.processor.mcause = read_reg(reg::mcause); - c.processor.mtval = read_reg(reg::mtval); - c.processor.misa = read_reg(reg::misa); - c.processor.mie = read_reg(reg::mie); - c.processor.mip = read_reg(reg::mip); - c.processor.medeleg = read_reg(reg::medeleg); - c.processor.mideleg = read_reg(reg::mideleg); - c.processor.mcounteren = read_reg(reg::mcounteren); - c.processor.menvcfg = read_reg(reg::menvcfg); - c.processor.stvec = read_reg(reg::stvec); - c.processor.sscratch = read_reg(reg::sscratch); - c.processor.sepc = read_reg(reg::sepc); - c.processor.scause = read_reg(reg::scause); - c.processor.stval = read_reg(reg::stval); - c.processor.satp = read_reg(reg::satp); - c.processor.scounteren = read_reg(reg::scounteren); - c.processor.senvcfg = read_reg(reg::senvcfg); - c.processor.ilrsc = read_reg(reg::ilrsc); - c.processor.iprv = read_reg(reg::iprv); - c.processor.iflags_X = read_reg(reg::iflags_X); - c.processor.iflags_Y = read_reg(reg::iflags_Y); - c.processor.iflags_H = read_reg(reg::iflags_H); - c.processor.iunrep = read_reg(reg::iunrep); - // Copy current CLINT state to config - c.clint.mtimecmp = read_reg(reg::clint_mtimecmp); - // Copy current PLIC state to config - c.plic.girqpend = read_reg(reg::plic_girqpend); - c.plic.girqsrvd = read_reg(reg::plic_girqsrvd); - // Copy current HTIF state to config - c.htif.tohost = read_reg(reg::htif_tohost); - c.htif.fromhost = read_reg(reg::htif_fromhost); - // c.htif.halt = read_reg(reg::htif_ihalt); // hard-coded to true - c.htif.console_getchar = static_cast(read_reg(reg::htif_iconsole) & (1 << HTIF_CONSOLE_CMD_GETCHAR)); - c.htif.yield_manual = static_cast(read_reg(reg::htif_iyield) & (1 << HTIF_YIELD_CMD_MANUAL)); - c.htif.yield_automatic = static_cast(read_reg(reg::htif_iyield) & (1 << HTIF_YIELD_CMD_AUTOMATIC)); - // Ensure we don't mess with DTB by writing the original bootargs - // over the potentially modified memory region we serialize - c.dtb.bootargs.clear(); - // Copy current uarch state to config - c.uarch.processor.cycle = read_reg(reg::uarch_cycle); - c.uarch.processor.halt_flag = read_reg(reg::uarch_halt_flag); - c.uarch.processor.pc = read_reg(reg::uarch_pc); - for (int i = 1; i < UARCH_X_REG_COUNT; i++) { - c.uarch.processor.x[i] = read_reg(machine_reg_enum(reg::uarch_x0, i)); - } - // Remove backing filenames from serialization - // (they will be ignored by save and load for security reasons) - clear_backing_store_filenames(c.ram.backing_store); - clear_backing_store_filenames(c.dtb.backing_store); - for (auto &f : c.flash_drive) { - clear_backing_store_filenames(f.backing_store); - } - clear_backing_store_filenames(c.tlb.backing_store); - clear_backing_store_filenames(c.cmio.rx_buffer.backing_store); - clear_backing_store_filenames(c.cmio.tx_buffer.backing_store); - clear_backing_store_filenames(c.pmas.backing_store); - clear_backing_store_filenames(c.uarch.ram.backing_store); - c.hash_tree.sht_filename.clear(); - c.hash_tree.phtc_filename.clear(); - return c; -} - uint64_t machine::get_paddr(host_addr haddr, uint64_t pma_index) const { return static_cast(haddr + get_hp_offset(pma_index)); } @@ -818,63 +767,30 @@ void machine::mark_dirty_page(host_addr haddr, uint64_t pma_index) { uint64_t machine::read_shadow_tlb(TLB_set_index set_index, uint64_t slot_index, shadow_tlb_what reg) const { switch (reg) { case shadow_tlb_what::vaddr_page: - return m_s.tlb.hot[set_index][slot_index].vaddr_page; - case shadow_tlb_what::vp_offset: { - const auto vaddr_page = m_s.tlb.hot[set_index][slot_index].vaddr_page; - if (vaddr_page != TLB_INVALID_PAGE) { - const auto vh_offset = m_s.tlb.hot[set_index][slot_index].vh_offset; - const auto haddr_page = vaddr_page + vh_offset; - const auto pma_index = m_s.tlb.cold[set_index][slot_index].pma_index; - return get_paddr(haddr_page, pma_index) - vaddr_page; - } - return 0; - } + return m_s->shadow_tlb[set_index][slot_index].vaddr_page; + case shadow_tlb_what::vp_offset: + return m_s->shadow_tlb[set_index][slot_index].vp_offset; case shadow_tlb_what::pma_index: - return m_s.tlb.cold[set_index][slot_index].pma_index; + return m_s->shadow_tlb[set_index][slot_index].pma_index; case shadow_tlb_what::zero_padding_: - return 0; + return m_s->shadow_tlb[set_index][slot_index].zero_padding_; default: throw std::domain_error{"unknown shadow TLB register"}; } } -void machine::check_shadow_tlb(TLB_set_index set_index, uint64_t slot_index, uint64_t vaddr_page, uint64_t vp_offset, - uint64_t pma_index, const std::string &prefix) const { - if (set_index > TLB_LAST_) { - throw std::domain_error{prefix + "TLB set index is out of range"s}; - } - if (slot_index >= TLB_SET_SIZE) { - throw std::domain_error{prefix + "TLB slot index is out of range"s}; - } - if (vaddr_page != TLB_INVALID_PAGE) { - if (pma_index >= m_s.pmas.size()) { - throw std::domain_error{prefix + "pma_index is out of range"s}; - } - const auto &ar = read_pma(pma_index); - if (!ar.is_memory()) { - throw std::invalid_argument{prefix + "pma_index does not point to memory range"s}; - } - if ((vaddr_page & PAGE_OFFSET_MASK) != 0) { - throw std::invalid_argument{prefix + "vaddr_page is not aligned"s}; - } - const auto paddr_page = vaddr_page + vp_offset; - if ((paddr_page & PAGE_OFFSET_MASK) != 0) { - throw std::invalid_argument{prefix + "vp_offset is not aligned"s}; - } - const auto pmas_end = ar.get_start() + (ar.get_length() - AR_PAGE_SIZE); - if (paddr_page < ar.get_start() || paddr_page > pmas_end) { - throw std::invalid_argument{prefix + "vp_offset is inconsistent with pma_index"s}; - } - } else if (pma_index != TLB_INVALID_PMA_INDEX || vp_offset != 0) { - throw std::domain_error{prefix + "inconsistent empty TLB slot"}; - } -} - void machine::write_tlb(TLB_set_index set_index, uint64_t slot_index, uint64_t vaddr_page, host_addr vh_offset, uint64_t pma_index) { - m_s.tlb.hot[set_index][slot_index].vaddr_page = vaddr_page; - m_s.tlb.hot[set_index][slot_index].vh_offset = vh_offset; - m_s.tlb.cold[set_index][slot_index].pma_index = pma_index; + uint64_t vp_offset = 0; + if (vaddr_page != TLB_INVALID_PAGE) { + vp_offset = get_paddr(vh_offset, pma_index); + } + m_s->hot_tlb[set_index][slot_index].vaddr_page = vaddr_page; + m_s->hot_tlb[set_index][slot_index].vh_offset = vh_offset; + m_s->shadow_tlb[set_index][slot_index].vaddr_page = vaddr_page; + m_s->shadow_tlb[set_index][slot_index].vp_offset = vp_offset; + m_s->shadow_tlb[set_index][slot_index].pma_index = pma_index; + m_s->shadow_tlb[set_index][slot_index].zero_padding_ = 0; } void machine::write_shadow_tlb(TLB_set_index set_index, uint64_t slot_index, uint64_t vaddr_page, uint64_t vp_offset, @@ -937,18 +853,16 @@ void machine::store_address_range(const address_range &ar, const std::string &di } void machine::store_address_ranges(const machine_config &c, const std::string &directory) const { - if (read_reg(reg::iunrep) != 0) { - throw std::runtime_error{"cannot store address ranges of unreproducible machines"}; - } store_address_range(find_address_range(AR_DTB_START), directory); store_address_range(find_address_range(AR_RAM_START), directory); - store_address_range(find_address_range(AR_SHADOW_TLB_START), directory); // Could iterate over PMAs checking for those with a drive DID but this is easier for (const auto &f : c.flash_drive) { store_address_range(find_address_range(f.start), directory); } + store_address_range(find_address_range(AR_SHADOW_STATE_START), directory); store_address_range(find_address_range(AR_CMIO_RX_BUFFER_START), directory); store_address_range(find_address_range(AR_CMIO_TX_BUFFER_START), directory); + store_address_range(find_address_range(AR_SHADOW_UARCH_STATE_START), directory); store_address_range(find_address_range(AR_UARCH_RAM_START), directory); store_address_range(find_address_range(AR_PMAS_START), directory); } @@ -962,6 +876,9 @@ static void store_hash(const machine::hash_type &h, const std::string &dir) { } void machine::store(const std::string &dir) const { + if (read_reg(reg::iunrep) != 0) { + throw std::runtime_error{"cannot store address ranges of unreproducible machines"}; + } if (os_mkdir(dir.c_str(), 0700) != 0) { throw std::system_error{errno, std::generic_category(), "error creating directory '"s + dir + "'"s}; } @@ -973,7 +890,8 @@ void machine::store(const std::string &dir) const { m_t.get_root_hash(h); store_hash(h, dir); } - auto c = get_serialization_config(); + machine_config c = m_c; + c.adjust_backing_stores("."); c.store(dir); store_address_ranges(c, dir); } @@ -1037,7 +955,7 @@ void machine::dump_stats() { machine::~machine() { // Cleanup TTY if console input was enabled - if (m_c.htif.console_getchar || has_virtio_console()) { + if ((m_c.processor.registers.htif.iconsole & HTIF_CONSOLE_CMD_GETCHAR_MASK) != 0 || has_virtio_console()) { os_close_tty(); } dump_insn_hist(); @@ -1045,140 +963,139 @@ machine::~machine() { } uint64_t machine::read_reg(reg r) const { - using reg = machine_reg; switch (r) { case reg::x0: - return m_s.x[0]; + return m_s->registers.x[0]; case reg::x1: - return m_s.x[1]; + return m_s->registers.x[1]; case reg::x2: - return m_s.x[2]; + return m_s->registers.x[2]; case reg::x3: - return m_s.x[3]; + return m_s->registers.x[3]; case reg::x4: - return m_s.x[4]; + return m_s->registers.x[4]; case reg::x5: - return m_s.x[5]; + return m_s->registers.x[5]; case reg::x6: - return m_s.x[6]; + return m_s->registers.x[6]; case reg::x7: - return m_s.x[7]; + return m_s->registers.x[7]; case reg::x8: - return m_s.x[8]; + return m_s->registers.x[8]; case reg::x9: - return m_s.x[9]; + return m_s->registers.x[9]; case reg::x10: - return m_s.x[10]; + return m_s->registers.x[10]; case reg::x11: - return m_s.x[11]; + return m_s->registers.x[11]; case reg::x12: - return m_s.x[12]; + return m_s->registers.x[12]; case reg::x13: - return m_s.x[13]; + return m_s->registers.x[13]; case reg::x14: - return m_s.x[14]; + return m_s->registers.x[14]; case reg::x15: - return m_s.x[15]; + return m_s->registers.x[15]; case reg::x16: - return m_s.x[16]; + return m_s->registers.x[16]; case reg::x17: - return m_s.x[17]; + return m_s->registers.x[17]; case reg::x18: - return m_s.x[18]; + return m_s->registers.x[18]; case reg::x19: - return m_s.x[19]; + return m_s->registers.x[19]; case reg::x20: - return m_s.x[20]; + return m_s->registers.x[20]; case reg::x21: - return m_s.x[21]; + return m_s->registers.x[21]; case reg::x22: - return m_s.x[22]; + return m_s->registers.x[22]; case reg::x23: - return m_s.x[23]; + return m_s->registers.x[23]; case reg::x24: - return m_s.x[24]; + return m_s->registers.x[24]; case reg::x25: - return m_s.x[25]; + return m_s->registers.x[25]; case reg::x26: - return m_s.x[26]; + return m_s->registers.x[26]; case reg::x27: - return m_s.x[27]; + return m_s->registers.x[27]; case reg::x28: - return m_s.x[28]; + return m_s->registers.x[28]; case reg::x29: - return m_s.x[29]; + return m_s->registers.x[29]; case reg::x30: - return m_s.x[30]; + return m_s->registers.x[30]; case reg::x31: - return m_s.x[31]; + return m_s->registers.x[31]; case reg::f0: - return m_s.f[0]; + return m_s->registers.f[0]; case reg::f1: - return m_s.f[1]; + return m_s->registers.f[1]; case reg::f2: - return m_s.f[2]; + return m_s->registers.f[2]; case reg::f3: - return m_s.f[3]; + return m_s->registers.f[3]; case reg::f4: - return m_s.f[4]; + return m_s->registers.f[4]; case reg::f5: - return m_s.f[5]; + return m_s->registers.f[5]; case reg::f6: - return m_s.f[6]; + return m_s->registers.f[6]; case reg::f7: - return m_s.f[7]; + return m_s->registers.f[7]; case reg::f8: - return m_s.f[8]; + return m_s->registers.f[8]; case reg::f9: - return m_s.f[9]; + return m_s->registers.f[9]; case reg::f10: - return m_s.f[10]; + return m_s->registers.f[10]; case reg::f11: - return m_s.f[11]; + return m_s->registers.f[11]; case reg::f12: - return m_s.f[12]; + return m_s->registers.f[12]; case reg::f13: - return m_s.f[13]; + return m_s->registers.f[13]; case reg::f14: - return m_s.f[14]; + return m_s->registers.f[14]; case reg::f15: - return m_s.f[15]; + return m_s->registers.f[15]; case reg::f16: - return m_s.f[16]; + return m_s->registers.f[16]; case reg::f17: - return m_s.f[17]; + return m_s->registers.f[17]; case reg::f18: - return m_s.f[18]; + return m_s->registers.f[18]; case reg::f19: - return m_s.f[19]; + return m_s->registers.f[19]; case reg::f20: - return m_s.f[20]; + return m_s->registers.f[20]; case reg::f21: - return m_s.f[21]; + return m_s->registers.f[21]; case reg::f22: - return m_s.f[22]; + return m_s->registers.f[22]; case reg::f23: - return m_s.f[23]; + return m_s->registers.f[23]; case reg::f24: - return m_s.f[24]; + return m_s->registers.f[24]; case reg::f25: - return m_s.f[25]; + return m_s->registers.f[25]; case reg::f26: - return m_s.f[26]; + return m_s->registers.f[26]; case reg::f27: - return m_s.f[27]; + return m_s->registers.f[27]; case reg::f28: - return m_s.f[28]; + return m_s->registers.f[28]; case reg::f29: - return m_s.f[29]; + return m_s->registers.f[29]; case reg::f30: - return m_s.f[30]; + return m_s->registers.f[30]; case reg::f31: - return m_s.f[31]; + return m_s->registers.f[31]; case reg::pc: - return m_s.pc; + return m_s->registers.pc; case reg::fcsr: - return m_s.fcsr; + return m_s->registers.fcsr; case reg::mvendorid: return MVENDORID_INIT; case reg::marchid: @@ -1186,165 +1103,165 @@ uint64_t machine::read_reg(reg r) const { case reg::mimpid: return MIMPID_INIT; case reg::mcycle: - return m_s.mcycle; + return m_s->registers.mcycle; case reg::icycleinstret: - return m_s.icycleinstret; + return m_s->registers.icycleinstret; case reg::mstatus: - return m_s.mstatus; + return m_s->registers.mstatus; case reg::mtvec: - return m_s.mtvec; + return m_s->registers.mtvec; case reg::mscratch: - return m_s.mscratch; + return m_s->registers.mscratch; case reg::mepc: - return m_s.mepc; + return m_s->registers.mepc; case reg::mcause: - return m_s.mcause; + return m_s->registers.mcause; case reg::mtval: - return m_s.mtval; + return m_s->registers.mtval; case reg::misa: - return m_s.misa; + return m_s->registers.misa; case reg::mie: - return m_s.mie; + return m_s->registers.mie; case reg::mip: - return m_s.mip; + return m_s->registers.mip; case reg::medeleg: - return m_s.medeleg; + return m_s->registers.medeleg; case reg::mideleg: - return m_s.mideleg; + return m_s->registers.mideleg; case reg::mcounteren: - return m_s.mcounteren; + return m_s->registers.mcounteren; case reg::menvcfg: - return m_s.menvcfg; + return m_s->registers.menvcfg; case reg::stvec: - return m_s.stvec; + return m_s->registers.stvec; case reg::sscratch: - return m_s.sscratch; + return m_s->registers.sscratch; case reg::sepc: - return m_s.sepc; + return m_s->registers.sepc; case reg::scause: - return m_s.scause; + return m_s->registers.scause; case reg::stval: - return m_s.stval; + return m_s->registers.stval; case reg::satp: - return m_s.satp; + return m_s->registers.satp; case reg::scounteren: - return m_s.scounteren; + return m_s->registers.scounteren; case reg::senvcfg: - return m_s.senvcfg; + return m_s->registers.senvcfg; case reg::ilrsc: - return m_s.ilrsc; + return m_s->registers.ilrsc; case reg::iprv: - return m_s.iprv; + return m_s->registers.iprv; case reg::iflags_X: - return m_s.iflags.X; + return m_s->registers.iflags.X; case reg::iflags_Y: - return m_s.iflags.Y; + return m_s->registers.iflags.Y; case reg::iflags_H: - return m_s.iflags.H; + return m_s->registers.iflags.H; case reg::iunrep: - return m_s.iunrep; + return m_s->registers.iunrep; case reg::clint_mtimecmp: - return m_s.clint.mtimecmp; + return m_s->registers.clint.mtimecmp; case reg::plic_girqpend: - return m_s.plic.girqpend; + return m_s->registers.plic.girqpend; case reg::plic_girqsrvd: - return m_s.plic.girqsrvd; + return m_s->registers.plic.girqsrvd; case reg::htif_tohost: - return m_s.htif.tohost; + return m_s->registers.htif.tohost; case reg::htif_fromhost: - return m_s.htif.fromhost; + return m_s->registers.htif.fromhost; case reg::htif_ihalt: - return m_s.htif.ihalt; + return m_s->registers.htif.ihalt; case reg::htif_iconsole: - return m_s.htif.iconsole; + return m_s->registers.htif.iconsole; case reg::htif_iyield: - return m_s.htif.iyield; + return m_s->registers.htif.iyield; case reg::uarch_x0: - return m_us.x[0]; + return m_us->registers.x[0]; case reg::uarch_x1: - return m_us.x[1]; + return m_us->registers.x[1]; case reg::uarch_x2: - return m_us.x[2]; + return m_us->registers.x[2]; case reg::uarch_x3: - return m_us.x[3]; + return m_us->registers.x[3]; case reg::uarch_x4: - return m_us.x[4]; + return m_us->registers.x[4]; case reg::uarch_x5: - return m_us.x[5]; + return m_us->registers.x[5]; case reg::uarch_x6: - return m_us.x[6]; + return m_us->registers.x[6]; case reg::uarch_x7: - return m_us.x[7]; + return m_us->registers.x[7]; case reg::uarch_x8: - return m_us.x[8]; + return m_us->registers.x[8]; case reg::uarch_x9: - return m_us.x[9]; + return m_us->registers.x[9]; case reg::uarch_x10: - return m_us.x[10]; + return m_us->registers.x[10]; case reg::uarch_x11: - return m_us.x[11]; + return m_us->registers.x[11]; case reg::uarch_x12: - return m_us.x[12]; + return m_us->registers.x[12]; case reg::uarch_x13: - return m_us.x[13]; + return m_us->registers.x[13]; case reg::uarch_x14: - return m_us.x[14]; + return m_us->registers.x[14]; case reg::uarch_x15: - return m_us.x[15]; + return m_us->registers.x[15]; case reg::uarch_x16: - return m_us.x[16]; + return m_us->registers.x[16]; case reg::uarch_x17: - return m_us.x[17]; + return m_us->registers.x[17]; case reg::uarch_x18: - return m_us.x[18]; + return m_us->registers.x[18]; case reg::uarch_x19: - return m_us.x[19]; + return m_us->registers.x[19]; case reg::uarch_x20: - return m_us.x[20]; + return m_us->registers.x[20]; case reg::uarch_x21: - return m_us.x[21]; + return m_us->registers.x[21]; case reg::uarch_x22: - return m_us.x[22]; + return m_us->registers.x[22]; case reg::uarch_x23: - return m_us.x[23]; + return m_us->registers.x[23]; case reg::uarch_x24: - return m_us.x[24]; + return m_us->registers.x[24]; case reg::uarch_x25: - return m_us.x[25]; + return m_us->registers.x[25]; case reg::uarch_x26: - return m_us.x[26]; + return m_us->registers.x[26]; case reg::uarch_x27: - return m_us.x[27]; + return m_us->registers.x[27]; case reg::uarch_x28: - return m_us.x[28]; + return m_us->registers.x[28]; case reg::uarch_x29: - return m_us.x[29]; + return m_us->registers.x[29]; case reg::uarch_x30: - return m_us.x[30]; + return m_us->registers.x[30]; case reg::uarch_x31: - return m_us.x[31]; + return m_us->registers.x[31]; case reg::uarch_pc: - return m_us.pc; + return m_us->registers.pc; case reg::uarch_cycle: - return m_us.cycle; + return m_us->registers.cycle; case reg::uarch_halt_flag: - return m_us.halt_flag; + return m_us->registers.halt_flag; case reg::htif_tohost_dev: - return HTIF_DEV_FIELD(m_s.htif.tohost); + return HTIF_DEV_FIELD(m_s->registers.htif.tohost); case reg::htif_tohost_cmd: - return HTIF_CMD_FIELD(m_s.htif.tohost); + return HTIF_CMD_FIELD(m_s->registers.htif.tohost); case reg::htif_tohost_reason: - return HTIF_REASON_FIELD(m_s.htif.tohost); + return HTIF_REASON_FIELD(m_s->registers.htif.tohost); case reg::htif_tohost_data: - return HTIF_DATA_FIELD(m_s.htif.tohost); + return HTIF_DATA_FIELD(m_s->registers.htif.tohost); case reg::htif_fromhost_dev: - return HTIF_DEV_FIELD(m_s.htif.fromhost); + return HTIF_DEV_FIELD(m_s->registers.htif.fromhost); case reg::htif_fromhost_cmd: - return HTIF_CMD_FIELD(m_s.htif.fromhost); + return HTIF_CMD_FIELD(m_s->registers.htif.fromhost); case reg::htif_fromhost_reason: - return HTIF_REASON_FIELD(m_s.htif.fromhost); + return HTIF_REASON_FIELD(m_s->registers.htif.fromhost); case reg::htif_fromhost_data: - return HTIF_DATA_FIELD(m_s.htif.fromhost); + return HTIF_DATA_FIELD(m_s->registers.htif.fromhost); default: throw std::invalid_argument{"unknown register"}; return 0; // never reached @@ -1356,199 +1273,199 @@ void machine::write_reg(reg w, uint64_t value) { case reg::x0: throw std::invalid_argument{"register is read-only"}; case reg::x1: - m_s.x[1] = value; + m_s->registers.x[1] = value; break; case reg::x2: - m_s.x[2] = value; + m_s->registers.x[2] = value; break; case reg::x3: - m_s.x[3] = value; + m_s->registers.x[3] = value; break; case reg::x4: - m_s.x[4] = value; + m_s->registers.x[4] = value; break; case reg::x5: - m_s.x[5] = value; + m_s->registers.x[5] = value; break; case reg::x6: - m_s.x[6] = value; + m_s->registers.x[6] = value; break; case reg::x7: - m_s.x[7] = value; + m_s->registers.x[7] = value; break; case reg::x8: - m_s.x[8] = value; + m_s->registers.x[8] = value; break; case reg::x9: - m_s.x[9] = value; + m_s->registers.x[9] = value; break; case reg::x10: - m_s.x[10] = value; + m_s->registers.x[10] = value; break; case reg::x11: - m_s.x[11] = value; + m_s->registers.x[11] = value; break; case reg::x12: - m_s.x[12] = value; + m_s->registers.x[12] = value; break; case reg::x13: - m_s.x[13] = value; + m_s->registers.x[13] = value; break; case reg::x14: - m_s.x[14] = value; + m_s->registers.x[14] = value; break; case reg::x15: - m_s.x[15] = value; + m_s->registers.x[15] = value; break; case reg::x16: - m_s.x[16] = value; + m_s->registers.x[16] = value; break; case reg::x17: - m_s.x[17] = value; + m_s->registers.x[17] = value; break; case reg::x18: - m_s.x[18] = value; + m_s->registers.x[18] = value; break; case reg::x19: - m_s.x[19] = value; + m_s->registers.x[19] = value; break; case reg::x20: - m_s.x[20] = value; + m_s->registers.x[20] = value; break; case reg::x21: - m_s.x[21] = value; + m_s->registers.x[21] = value; break; case reg::x22: - m_s.x[22] = value; + m_s->registers.x[22] = value; break; case reg::x23: - m_s.x[23] = value; + m_s->registers.x[23] = value; break; case reg::x24: - m_s.x[24] = value; + m_s->registers.x[24] = value; break; case reg::x25: - m_s.x[25] = value; + m_s->registers.x[25] = value; break; case reg::x26: - m_s.x[26] = value; + m_s->registers.x[26] = value; break; case reg::x27: - m_s.x[27] = value; + m_s->registers.x[27] = value; break; case reg::x28: - m_s.x[28] = value; + m_s->registers.x[28] = value; break; case reg::x29: - m_s.x[29] = value; + m_s->registers.x[29] = value; break; case reg::x30: - m_s.x[30] = value; + m_s->registers.x[30] = value; break; case reg::x31: - m_s.x[31] = value; + m_s->registers.x[31] = value; break; case reg::f0: - m_s.f[0] = value; + m_s->registers.f[0] = value; break; case reg::f1: - m_s.f[1] = value; + m_s->registers.f[1] = value; break; case reg::f2: - m_s.f[2] = value; + m_s->registers.f[2] = value; break; case reg::f3: - m_s.f[3] = value; + m_s->registers.f[3] = value; break; case reg::f4: - m_s.f[4] = value; + m_s->registers.f[4] = value; break; case reg::f5: - m_s.f[5] = value; + m_s->registers.f[5] = value; break; case reg::f6: - m_s.f[6] = value; + m_s->registers.f[6] = value; break; case reg::f7: - m_s.f[7] = value; + m_s->registers.f[7] = value; break; case reg::f8: - m_s.f[8] = value; + m_s->registers.f[8] = value; break; case reg::f9: - m_s.f[9] = value; + m_s->registers.f[9] = value; break; case reg::f10: - m_s.f[10] = value; + m_s->registers.f[10] = value; break; case reg::f11: - m_s.f[11] = value; + m_s->registers.f[11] = value; break; case reg::f12: - m_s.f[12] = value; + m_s->registers.f[12] = value; break; case reg::f13: - m_s.f[13] = value; + m_s->registers.f[13] = value; break; case reg::f14: - m_s.f[14] = value; + m_s->registers.f[14] = value; break; case reg::f15: - m_s.f[15] = value; + m_s->registers.f[15] = value; break; case reg::f16: - m_s.f[16] = value; + m_s->registers.f[16] = value; break; case reg::f17: - m_s.f[17] = value; + m_s->registers.f[17] = value; break; case reg::f18: - m_s.f[18] = value; + m_s->registers.f[18] = value; break; case reg::f19: - m_s.f[19] = value; + m_s->registers.f[19] = value; break; case reg::f20: - m_s.f[20] = value; + m_s->registers.f[20] = value; break; case reg::f21: - m_s.f[21] = value; + m_s->registers.f[21] = value; break; case reg::f22: - m_s.f[22] = value; + m_s->registers.f[22] = value; break; case reg::f23: - m_s.f[23] = value; + m_s->registers.f[23] = value; break; case reg::f24: - m_s.f[24] = value; + m_s->registers.f[24] = value; break; case reg::f25: - m_s.f[25] = value; + m_s->registers.f[25] = value; break; case reg::f26: - m_s.f[26] = value; + m_s->registers.f[26] = value; break; case reg::f27: - m_s.f[27] = value; + m_s->registers.f[27] = value; break; case reg::f28: - m_s.f[28] = value; + m_s->registers.f[28] = value; break; case reg::f29: - m_s.f[29] = value; + m_s->registers.f[29] = value; break; case reg::f30: - m_s.f[30] = value; + m_s->registers.f[30] = value; break; case reg::f31: - m_s.f[31] = value; + m_s->registers.f[31] = value; break; case reg::pc: - m_s.pc = value; + m_s->registers.pc = value; break; case reg::fcsr: - m_s.fcsr = value; + m_s->registers.fcsr = value; break; case reg::mvendorid: throw std::invalid_argument{"register is read-only"}; @@ -1557,243 +1474,243 @@ void machine::write_reg(reg w, uint64_t value) { case reg::mimpid: throw std::invalid_argument{"register is read-only"}; case reg::mcycle: - m_s.mcycle = value; + m_s->registers.mcycle = value; break; case reg::icycleinstret: - m_s.icycleinstret = value; + m_s->registers.icycleinstret = value; break; case reg::mstatus: - m_s.mstatus = value; + m_s->registers.mstatus = value; break; case reg::mtvec: - m_s.mtvec = value; + m_s->registers.mtvec = value; break; case reg::mscratch: - m_s.mscratch = value; + m_s->registers.mscratch = value; break; case reg::mepc: - m_s.mepc = value; + m_s->registers.mepc = value; break; case reg::mcause: - m_s.mcause = value; + m_s->registers.mcause = value; break; case reg::mtval: - m_s.mtval = value; + m_s->registers.mtval = value; break; case reg::misa: - m_s.misa = value; + m_s->registers.misa = value; break; case reg::mie: - m_s.mie = value; + m_s->registers.mie = value; break; case reg::mip: - m_s.mip = value; + m_s->registers.mip = value; break; case reg::medeleg: - m_s.medeleg = value; + m_s->registers.medeleg = value; break; case reg::mideleg: - m_s.mideleg = value; + m_s->registers.mideleg = value; break; case reg::mcounteren: - m_s.mcounteren = value; + m_s->registers.mcounteren = value; break; case reg::menvcfg: - m_s.menvcfg = value; + m_s->registers.menvcfg = value; break; case reg::stvec: - m_s.stvec = value; + m_s->registers.stvec = value; break; case reg::sscratch: - m_s.sscratch = value; + m_s->registers.sscratch = value; break; case reg::sepc: - m_s.sepc = value; + m_s->registers.sepc = value; break; case reg::scause: - m_s.scause = value; + m_s->registers.scause = value; break; case reg::stval: - m_s.stval = value; + m_s->registers.stval = value; break; case reg::satp: - m_s.satp = value; + m_s->registers.satp = value; break; case reg::scounteren: - m_s.scounteren = value; + m_s->registers.scounteren = value; break; case reg::senvcfg: - m_s.senvcfg = value; + m_s->registers.senvcfg = value; break; case reg::ilrsc: - m_s.ilrsc = value; + m_s->registers.ilrsc = value; break; case reg::iprv: - m_s.iprv = value; + m_s->registers.iprv = value; break; case reg::iflags_X: - m_s.iflags.X = value; + m_s->registers.iflags.X = value; break; case reg::iflags_Y: - m_s.iflags.Y = value; + m_s->registers.iflags.Y = value; break; case reg::iflags_H: - m_s.iflags.H = value; + m_s->registers.iflags.H = value; break; case reg::iunrep: - m_s.iunrep = value; + m_s->registers.iunrep = value; break; case reg::clint_mtimecmp: - m_s.clint.mtimecmp = value; + m_s->registers.clint.mtimecmp = value; break; case reg::plic_girqpend: - m_s.plic.girqpend = value; + m_s->registers.plic.girqpend = value; break; case reg::plic_girqsrvd: - m_s.plic.girqsrvd = value; + m_s->registers.plic.girqsrvd = value; break; case reg::htif_tohost: - m_s.htif.tohost = value; + m_s->registers.htif.tohost = value; break; case reg::htif_fromhost: - m_s.htif.fromhost = value; + m_s->registers.htif.fromhost = value; break; case reg::htif_ihalt: - m_s.htif.ihalt = value; + m_s->registers.htif.ihalt = value; break; case reg::htif_iconsole: - m_s.htif.iconsole = value; + m_s->registers.htif.iconsole = value; break; case reg::htif_iyield: - m_s.htif.iyield = value; + m_s->registers.htif.iyield = value; break; case reg::uarch_x0: throw std::invalid_argument{"register is read-only"}; case reg::uarch_x1: - m_us.x[1] = value; + m_us->registers.x[1] = value; break; case reg::uarch_x2: - m_us.x[2] = value; + m_us->registers.x[2] = value; break; case reg::uarch_x3: - m_us.x[3] = value; + m_us->registers.x[3] = value; break; case reg::uarch_x4: - m_us.x[4] = value; + m_us->registers.x[4] = value; break; case reg::uarch_x5: - m_us.x[5] = value; + m_us->registers.x[5] = value; break; case reg::uarch_x6: - m_us.x[6] = value; + m_us->registers.x[6] = value; break; case reg::uarch_x7: - m_us.x[7] = value; + m_us->registers.x[7] = value; break; case reg::uarch_x8: - m_us.x[8] = value; + m_us->registers.x[8] = value; break; case reg::uarch_x9: - m_us.x[9] = value; + m_us->registers.x[9] = value; break; case reg::uarch_x10: - m_us.x[10] = value; + m_us->registers.x[10] = value; break; case reg::uarch_x11: - m_us.x[11] = value; + m_us->registers.x[11] = value; break; case reg::uarch_x12: - m_us.x[12] = value; + m_us->registers.x[12] = value; break; case reg::uarch_x13: - m_us.x[13] = value; + m_us->registers.x[13] = value; break; case reg::uarch_x14: - m_us.x[14] = value; + m_us->registers.x[14] = value; break; case reg::uarch_x15: - m_us.x[15] = value; + m_us->registers.x[15] = value; break; case reg::uarch_x16: - m_us.x[16] = value; + m_us->registers.x[16] = value; break; case reg::uarch_x17: - m_us.x[17] = value; + m_us->registers.x[17] = value; break; case reg::uarch_x18: - m_us.x[18] = value; + m_us->registers.x[18] = value; break; case reg::uarch_x19: - m_us.x[19] = value; + m_us->registers.x[19] = value; break; case reg::uarch_x20: - m_us.x[20] = value; + m_us->registers.x[20] = value; break; case reg::uarch_x21: - m_us.x[21] = value; + m_us->registers.x[21] = value; break; case reg::uarch_x22: - m_us.x[22] = value; + m_us->registers.x[22] = value; break; case reg::uarch_x23: - m_us.x[23] = value; + m_us->registers.x[23] = value; break; case reg::uarch_x24: - m_us.x[24] = value; + m_us->registers.x[24] = value; break; case reg::uarch_x25: - m_us.x[25] = value; + m_us->registers.x[25] = value; break; case reg::uarch_x26: - m_us.x[26] = value; + m_us->registers.x[26] = value; break; case reg::uarch_x27: - m_us.x[27] = value; + m_us->registers.x[27] = value; break; case reg::uarch_x28: - m_us.x[28] = value; + m_us->registers.x[28] = value; break; case reg::uarch_x29: - m_us.x[29] = value; + m_us->registers.x[29] = value; break; case reg::uarch_x30: - m_us.x[30] = value; + m_us->registers.x[30] = value; break; case reg::uarch_x31: - m_us.x[31] = value; + m_us->registers.x[31] = value; break; case reg::uarch_pc: - m_us.pc = value; + m_us->registers.pc = value; break; case reg::uarch_cycle: - m_us.cycle = value; + m_us->registers.cycle = value; break; case reg::uarch_halt_flag: - m_us.halt_flag = value; + m_us->registers.halt_flag = value; break; case reg::htif_tohost_dev: - m_s.htif.tohost = HTIF_REPLACE_DEV(m_s.htif.tohost, value); + m_s->registers.htif.tohost = HTIF_REPLACE_DEV(m_s->registers.htif.tohost, value); break; case reg::htif_tohost_cmd: - m_s.htif.tohost = HTIF_REPLACE_CMD(m_s.htif.tohost, value); + m_s->registers.htif.tohost = HTIF_REPLACE_CMD(m_s->registers.htif.tohost, value); break; case reg::htif_tohost_reason: - m_s.htif.tohost = HTIF_REPLACE_REASON(m_s.htif.tohost, value); + m_s->registers.htif.tohost = HTIF_REPLACE_REASON(m_s->registers.htif.tohost, value); break; case reg::htif_tohost_data: - m_s.htif.tohost = HTIF_REPLACE_DATA(m_s.htif.tohost, value); + m_s->registers.htif.tohost = HTIF_REPLACE_DATA(m_s->registers.htif.tohost, value); break; case reg::htif_fromhost_dev: - m_s.htif.fromhost = HTIF_REPLACE_DEV(m_s.htif.fromhost, value); + m_s->registers.htif.fromhost = HTIF_REPLACE_DEV(m_s->registers.htif.fromhost, value); break; case reg::htif_fromhost_cmd: - m_s.htif.fromhost = HTIF_REPLACE_CMD(m_s.htif.fromhost, value); + m_s->registers.htif.fromhost = HTIF_REPLACE_CMD(m_s->registers.htif.fromhost, value); break; case reg::htif_fromhost_reason: - m_s.htif.fromhost = HTIF_REPLACE_REASON(m_s.htif.fromhost, value); + m_s->registers.htif.fromhost = HTIF_REPLACE_REASON(m_s->registers.htif.fromhost, value); break; case reg::htif_fromhost_data: - m_s.htif.fromhost = HTIF_REPLACE_DATA(m_s.htif.fromhost, value); + m_s->registers.htif.fromhost = HTIF_REPLACE_DATA(m_s->registers.htif.fromhost, value); break; default: throw std::invalid_argument{"unknown register"}; @@ -1813,19 +1730,18 @@ uint64_t machine::get_reg_address(reg r) { } void machine::mark_write_tlb_dirty_pages() const { - auto &hot_set = m_s.tlb.hot[TLB_WRITE]; - auto &cold_set = m_s.tlb.cold[TLB_WRITE]; + auto &hot_set = m_s->hot_tlb[TLB_WRITE]; + auto &shadow_set = m_s->shadow_tlb[TLB_WRITE]; for (uint64_t slot_index = 0; slot_index < TLB_SET_SIZE; ++slot_index) { const auto &hot_slot = hot_set[slot_index]; if (hot_slot.vaddr_page != TLB_INVALID_PAGE) { - auto haddr_page = hot_slot.vaddr_page + hot_slot.vh_offset; - const auto &cold_slot = cold_set[slot_index]; + const auto &shadow_slot = shadow_set[slot_index]; // NOLINTNEXTLINE(cppcoreguidelines-pro-type-const-cast) - auto &ar = const_cast(read_pma(cold_slot.pma_index)); + auto &ar = const_cast(read_pma(shadow_slot.pma_index)); if (!ar.is_memory()) { throw std::runtime_error{"could not mark dirty page for a TLB entry: TLB is corrupt"}; } - auto paddr_page = get_paddr(haddr_page, cold_slot.pma_index); + auto paddr_page = hot_slot.vaddr_page + shadow_slot.vp_offset; if (!ar.contains_absolute(paddr_page, AR_PAGE_SIZE)) { throw std::runtime_error{"could not mark dirty page for a TLB entry: TLB is corrupt"}; } @@ -1849,7 +1765,7 @@ bool machine::verify_dirty_page_maps() const { for (const auto &ar : m_ars) { for (uint64_t offset = 0; offset < ar->get_length(); offset += AR_PAGE_SIZE) { const uint64_t page_address = ar->get_start() + offset; - if (ar->is_memory()) { + if (ar->is_memory() && !ar->is_page_uncleanable()) { const unsigned char *page_data = nullptr; ar->peek(*this, offset, AR_PAGE_SIZE, &page_data, scratch.get()); hash_type stored; @@ -2022,14 +1938,14 @@ const char *machine::get_what_name(uint64_t paddr) { return "uarch.ram"; } // If in shadow, return refined name + if (paddr >= AR_SHADOW_REGISTERS_START && paddr - AR_SHADOW_REGISTERS_START < AR_SHADOW_REGISTERS_LENGTH) { + return shadow_registers_get_what_name(shadow_registers_get_what(paddr)); + } if (paddr >= AR_SHADOW_TLB_START && paddr - AR_SHADOW_TLB_START < AR_SHADOW_TLB_LENGTH) { [[maybe_unused]] TLB_set_index set_index{}; [[maybe_unused]] uint64_t slot_index{}; return shadow_tlb_get_what_name(shadow_tlb_get_what(paddr, set_index, slot_index)); } - if (paddr >= AR_SHADOW_STATE_START && paddr - AR_SHADOW_STATE_START < AR_SHADOW_STATE_LENGTH) { - return shadow_state_get_what_name(shadow_state_get_what(paddr)); - } if (paddr >= AR_PMAS_START && paddr - AR_PMAS_START < AR_PMAS_LENGTH) { return pmas_get_what_name(pmas_get_what(paddr)); } @@ -2117,6 +2033,12 @@ static inline void foreach_aligned_chunk(uint64_t start, uint64_t length, uint64 } } +static constexpr bool range_overlaps(uint64_t a_start, uint64_t a_length, uint64_t b_start, uint64_t b_length) { + const uint64_t a_end = a_start + a_length - 1; + const uint64_t b_end = b_start + b_length - 1; + return (a_start <= b_end && a_end >= b_start); +} + void machine::read_memory(uint64_t paddr, unsigned char *data, uint64_t length) const { if (length == 0) { return; @@ -2190,8 +2112,8 @@ void machine::write_memory(uint64_t paddr, const unsigned char *data, uint64_t l if (!ar.is_memory()) { throw std::invalid_argument{"address range to write is not entirely in single memory range"}; } - if (is_protected(ar.get_driver_id())) { - throw std::invalid_argument{"attempt to write to protected memory range"}; + if (ar.is_host_read_only()) { + throw std::invalid_argument{"address range is read-only in the host"}; } foreach_aligned_chunk(paddr, length, AR_PAGE_SIZE, [&ar, paddr, data](auto chunk_start, auto chunk_length) { const auto *src = data + (chunk_start - paddr); @@ -2203,6 +2125,10 @@ void machine::write_memory(uint64_t paddr, const unsigned char *data, uint64_t l ar.mark_dirty_page(offset); } }); + if (range_overlaps(paddr, length, AR_SHADOW_TLB_START, AR_SHADOW_TLB_LENGTH)) { + // Must reinitialize hot TLB again to reflect changes in the shadow TLB + init_hot_tlb_contents(); + } } void machine::fill_memory(uint64_t paddr, uint8_t val, uint64_t length) { @@ -2216,8 +2142,13 @@ void machine::fill_memory(uint64_t paddr, uint8_t val, uint64_t length) { if (!ar.is_memory()) { throw std::invalid_argument{"address range to fill is not entirely in memory PMA"}; } - if (is_protected(ar.get_driver_id())) { - throw std::invalid_argument{"attempt fill to protected memory range"}; + if (ar.is_host_read_only()) { + throw std::invalid_argument{"address range is read-only in the host"}; + } + if (range_overlaps(paddr, length, AR_SHADOW_TLB_START, AR_SHADOW_TLB_LENGTH)) { + // Filling shadow TLB memory doesn't make much sense because it would leave + // corrupt its state in most situations. + throw std::invalid_argument{"attempted fill to shadow TLB memory range"}; } // The case of filling a range with zeros is special and optimized for uarch reset foreach_aligned_chunk(paddr, length, AR_PAGE_SIZE, [&ar, val](auto chunk_start, auto chunk_length) { @@ -2320,25 +2251,6 @@ void machine::write_word(uint64_t paddr, uint64_t val) { if ((paddr & (sizeof(uint64_t) - 1)) != 0) { throw std::domain_error{"attempted misaligned write to word"}; } - // If in shadow, forward to write_reg - if (paddr >= AR_SHADOW_STATE_START && paddr - AR_SHADOW_STATE_START < AR_SHADOW_STATE_LENGTH) { - auto reg = shadow_state_get_what(paddr); - if (reg == shadow_state_what::unknown_) { - throw std::runtime_error("unhandled write to shadow state"); - } - write_reg(machine_reg_enum(reg), val); - return; - } - // If in uarch shadow, forward to write_reg - if (paddr >= AR_SHADOW_UARCH_STATE_START && paddr - AR_SHADOW_UARCH_STATE_START < AR_SHADOW_UARCH_STATE_LENGTH) { - auto reg = shadow_uarch_state_get_what(paddr); - if (reg == shadow_uarch_state_what::unknown_) { - throw std::runtime_error("unhandled write to shadow uarch state"); - } - write_reg(machine_reg_enum(reg), val); - return; - } - // Otherwise, try the slow path auto &ar = find_address_range(paddr, sizeof(uint64_t)); if (!ar.is_memory() || ar.get_host_memory() == nullptr) { std::ostringstream err; @@ -2346,12 +2258,17 @@ void machine::write_word(uint64_t paddr, uint64_t val) { << std::dec << paddr << ")"; throw std::runtime_error{err.str()}; } - if (!ar.is_writeable()) { + if (ar.is_host_read_only()) { std::ostringstream err; err << "attempted memory write to read-only " << ar.get_description() << " at address 0x" << std::hex << paddr << "(" << std::dec << paddr << ")"; throw std::runtime_error{err.str()}; } + if (range_overlaps(paddr, 8, AR_SHADOW_TLB_START, AR_SHADOW_TLB_LENGTH)) { + // Writing a single word to shadow TLB memory doesn't make much sense, + // it would leave corrupt its state in most situations. + throw std::invalid_argument{"attempted memory word write to shadow TLB memory range"}; + } const auto offset = paddr - ar.get_start(); aliased_aligned_write(ar.get_host_memory() + offset, val); ar.mark_dirty_page(offset); @@ -2397,7 +2314,6 @@ void machine::verify_send_cmio_response(uint16_t reason, const unsigned char *da } void machine::reset_uarch() { - using reg = machine_reg; write_reg(reg::uarch_halt_flag, UARCH_HALT_FLAG_INIT); write_reg(reg::uarch_pc, UARCH_PC_INIT); write_reg(reg::uarch_cycle, UARCH_CYCLE_INIT); @@ -2406,12 +2322,10 @@ void machine::reset_uarch() { write_reg(machine_reg_enum(reg::uarch_x0, i), UARCH_X_INIT); } // Load embedded pristine RAM image - const auto uram_length = m_us.ram->get_length(); - const auto uram_start = m_us.ram->get_start(); + write_memory(AR_UARCH_RAM_START, uarch_pristine_ram, uarch_pristine_ram_len); // Reset RAM to initial state - write_memory(uram_start, uarch_pristine_ram, uarch_pristine_ram_len); - if (uram_length > uarch_pristine_ram_len) { - fill_memory(uram_start + uarch_pristine_ram_len, 0, uram_length - uarch_pristine_ram_len); + if (AR_UARCH_RAM_LENGTH > uarch_pristine_ram_len) { + fill_memory(AR_UARCH_RAM_START + uarch_pristine_ram_len, 0, AR_UARCH_RAM_LENGTH - uarch_pristine_ram_len); } } @@ -2531,7 +2445,7 @@ interpreter_break_reason machine::log_step(uint64_t mcycle_count, const std::str interpreter_break_reason machine::verify_step(const hash_type &root_hash_before, const std::string &filename, uint64_t mcycle_count, const hash_type &root_hash_after) { auto data_length = os_get_file_length(filename.c_str(), "step log file"); - auto data = make_unique_mmap(filename.c_str(), data_length, false /* not shared */); + auto data = make_unique_mmap(data_length, os_mmap_flags{}, filename, data_length); replay_step_state_access::context context; replay_step_state_access a(context, data.get(), data_length, root_hash_before); uint64_t mcycle_end{}; @@ -2556,7 +2470,7 @@ interpreter_break_reason machine::run(uint64_t mcycle_end) { std::pair machine::poll_external_interrupts(uint64_t mcycle, uint64_t mcycle_max) { const auto status = execute_status::success; // Only poll external interrupts if we are in unreproducible mode - if (unlikely(m_s.iunrep)) { + if (unlikely(m_s->registers.iunrep)) { // Convert the relative interval of cycles we can wait to the interval of host time we can wait uint64_t timeout_us = (mcycle_max - mcycle) / RTC_CYCLES_PER_US; int64_t start_us = 0; diff --git a/src/machine.h b/src/machine.h index 723409b8d..826726d87 100644 --- a/src/machine.h +++ b/src/machine.h @@ -38,12 +38,12 @@ #include "machine-merkle-tree.h" #include "machine-reg.h" #include "machine-runtime-config.h" -#include "machine-state.h" #include "os.h" #include "pmas-constants.h" +#include "processor-state.h" #include "shadow-tlb.h" #include "uarch-interpret.h" -#include "uarch-state.h" +#include "uarch-processor-state.h" #include "virtio-address-range.h" namespace cartesi { @@ -60,8 +60,8 @@ constexpr skip_merkle_tree_update_t skip_merkle_tree_update; /// \brief Cartesi Machine implementation class machine final { private: - mutable machine_state m_s; ///< Big machine state - mutable uarch_state m_us; ///< Microarchitecture state + mutable processor_state *m_s{nullptr}; ///< Big machine processor state + mutable uarch_processor_state *m_us{nullptr}; ///< Microarchitecture state mutable std::vector> m_ars; ///< All address ranges mutable machine_merkle_tree m_t; ///< Merkle tree of state machine_config m_c; ///< Copy of initialization config @@ -70,6 +70,8 @@ class machine final { std::vector m_virtio_ars; ///< VirtIO address ranges address_range_descriptions m_ards; ///< Address range descriptions listed by get_address_ranges() std::unordered_map m_counters; ///< Counters used for statistics collection + std::vector m_pmas; ///< Indices of address ranges that interpret can find + bool m_soft_yield{false}; ///< Whether runtime soft yields are enabled ///< Where to register an address range struct register_where { @@ -97,7 +99,7 @@ class machine final { /// This means the an index into the container that owns all address ranges will always refers to same address range /// after subsequent calls to register_address_range() and calls to replace_address_range() as well. /// \details Besides the container that stores the address ranges, the machine maintains subsets of address ranges. - /// The "merkle" address range container lists the indices of the address ranges taht will be considered by + /// The "merkle" address range container lists the indices of the address ranges that will be considered by /// the Merkle tree during the computation of the state hash. /// The "interpret" address range container lists the indices of the address ranges that will be visible from within /// the interpreter. @@ -113,7 +115,7 @@ class machine final { const auto index = m_ars.size(); // Get index new address range will occupy m_ars.push_back(std::move(ptr)); // Move ptr to list of address ranges if (where.interpret) { // Register with interpreter - m_s.pmas.push_back(index); + m_pmas.push_back(index); } if (where.merkle) { // Register with Merkle tree m_merkle_ars.push_back(index); @@ -138,15 +140,24 @@ class machine final { /// \param pma_index Index of the memory PMA for the desired offset host_addr get_hp_offset(uint64_t pma_index) const; - /// \brief Initializes microarchitecture - /// \param c Microarchitecture configuration - void init_uarch(const uarch_config &c); + /// \brief Initializes microarchitecture processor + /// \param c Microarchitecture processor configuration + void init_uarch_processor(const uarch_processor_config &c); - /// \brief Initializes registers + /// \brief Initializes microarchitecture RAM + /// \param c Microarchitecture RAM configuration + void init_uarch_ram(const uarch_ram_config &c); + + /// \brief Initializes processor /// \param p Processor configuration /// \param r Machine runtime configuration void init_processor(processor_config &p, const machine_runtime_config &r); + /// \brief Initializes registers + /// \param p Registers configuration + /// \param r Machine runtime configuration + void init_registers(registers_state &p, const machine_runtime_config &r); + /// \brief Initializes RAM address range /// \param ram RAM configuration void init_ram_ar(const ram_config &ram); @@ -162,28 +173,28 @@ class machine final { /// \brief Initializes HTIF device address range /// \param h HTIF configuration - void init_htif_ar(const htif_config &h); + void init_htif_ar(); /// \brief Initializes TTY if needed /// \param h HTIF configuration /// \param r HTIF runtime configuration /// \param iunrep Initial value of iunrep CSR - void init_tty(const htif_config &h, const htif_runtime_config &r, uint64_t iunrep) const; + void init_tty(const htif_state &h, const htif_runtime_config &r, uint64_t iunrep) const; /// \brief Initializes CLINT device address range /// \param c CLINT configuration - void init_clint_ar(const clint_config &c); + void init_clint_ar(); /// \brief Initializes PLIC device address range /// \param p PLIC configuration - void init_plic_ar(const plic_config &p); + void init_plic_ar(); /// \brief Initializes CMIO address ranges /// \param c CMIO configuration void init_cmio_ars(const cmio_config &c); /// \brief Initializes the address ranges involced in the Merkle tree - /// \detail This can only be called after all address ranges have been registerd + /// \detail This can only be called after all address ranges have been registered void init_merkle_ars(); /// \brief Initializes the address range descriptions returned by get_address_ranges() @@ -195,10 +206,9 @@ class machine final { /// \detail This can only be called after all PMAs have been added void init_pmas_contents(const pmas_config &config, memory_address_range &pmas) const; - /// \brief Initializes contents of machine TLB, from image in disk or with default values - /// \param config TLB config + /// \brief Initialize hot TLB contents /// \detail This can only be called after all PMAs have been added - void init_tlb_contents(const tlb_config &config); + void init_hot_tlb_contents(); /// \brief Initializes contents of machine DTB, if image was not available /// \param config Machine configuration @@ -307,23 +317,23 @@ class machine final { static machine_config get_default_config(); /// \brief Returns machine state for direct access. - machine_state &get_state() { - return m_s; + processor_state &get_state() { + return *m_s; } /// \brief Returns machine state for direct read-only access. - const machine_state &get_state() const { - return m_s; + const processor_state &get_state() const { + return *m_s; } /// \brief Returns uarch state for direct access. - uarch_state &get_uarch_state() { - return m_us; + uarch_processor_state &get_uarch_state() { + return *m_us; } /// \brief Returns uarch state for direct read-only access. - const uarch_state &get_uarch_state() const { - return m_us; + const uarch_processor_state &get_uarch_state() const { + return *m_us; } /// \brief Returns a list of descriptions for all PMA entries registered in the machine, sorted by start @@ -450,7 +460,7 @@ class machine final { /// \param paddr Word address (aligned to 64-bit boundary). /// \details The word can be in a writeable area of the address space. /// This includes the shadow state and the shadow uarch state. - /// (But does NOT include memory-mapped devices, the shadow tlb, shadow PMAs, or unnocupied memory regions.) + /// (But does NOT include memory-mapped devices, shadow PMAs, or unnocupied memory regions.) void write_word(uint64_t paddr, uint64_t val); /// \brief Reads a chunk of data, by its target physical address and length. @@ -497,12 +507,12 @@ class machine final { /// \param index Index of desired address range /// \returns Desired address range, or an empty sentinel if index is out of bounds const address_range &read_pma(uint64_t index) const noexcept { - if (index >= m_s.pmas.size()) { + if (index >= m_pmas.size()) { static constexpr auto sentinel = make_empty_address_range("sentinel"); return sentinel; } // NOLINTNEXTLINE(bugprone-narrowing-conversions) - return *m_ars[static_cast(m_s.pmas[static_cast(index)])]; + return *m_ars[static_cast(m_pmas[static_cast(index)])]; } /// \brief Returns the address range associated to the PMA at a given index @@ -547,10 +557,6 @@ class machine final { /// \returns true if they are, false if there is an error. bool verify_dirty_page_maps() const; - /// \brief Copies the current state into a configuration for serialization - /// \returns The configuration - machine_config get_serialization_config() const; - /// \brief Returns copy of initialization config. const machine_config &get_initial_config() const; @@ -615,15 +621,6 @@ class machine final { void write_shadow_tlb(TLB_set_index set_index, uint64_t slot_index, uint64_t vaddr_page, uint64_t vp_offset, uint64_t pma_index); - /// \brief Check consistency of TLB slot - /// \param set_index TLB_CODE, TLB_READ, or TLB_WRITE - /// \param slot_index Index of slot to update - /// \param vaddr_page Virtual address of page to map - /// \param vp_offset Offset from target virtual addresses to target physical addresses within page - /// \param pma_index Index of PMA where address falls - void check_shadow_tlb(TLB_set_index set_index, uint64_t slot_index, uint64_t vaddr_page, uint64_t vp_offset, - uint64_t pma_index, const std::string &prefix = "") const; - /// \brief Reads a TLB register /// \param set_index TLB_CODE, TLB_READ, or TLB_WRITE /// \param slot_index Index of slot to read @@ -678,6 +675,11 @@ class machine final { const auto &get_counters() { return m_counters; } + + /// \brief Returns whether runtime soft yields are enabled + bool is_soft_yield() const { + return m_soft_yield; + } }; } // namespace cartesi diff --git a/src/memory-address-range.cpp b/src/memory-address-range.cpp index da61d5d12..a8173341e 100644 --- a/src/memory-address-range.cpp +++ b/src/memory-address-range.cpp @@ -28,10 +28,20 @@ class base_error : public std::invalid_argument { }; memory_address_range::memory_address_range(const std::string &description, uint64_t start, uint64_t length, - const pmas_flags &flags, const std::string &image_filename, const mmapd &m) try : + const pmas_flags &flags, const backing_store_config &backing_store, const memory_address_range_flags &ar_flags, + uint64_t host_length) try : address_range(description.c_str(), start, length, flags, [](const char *err) { throw base_error{err}; }), - m_ptr{make_unique_mmap(image_filename.c_str(), length, m.shared)}, - m_host_memory{std::get(m_ptr).get()} { + m_ptr{make_unique_mmap(host_length != 0 ? host_length : length, + os_mmap_flags{ + .read_only = ar_flags.read_only && !backing_store.create, + .shared = backing_store.shared, + .create = backing_store.create, + .truncate = backing_store.truncate, + .lock = !backing_store.data_filename.empty(), + }, + backing_store.data_filename, length)}, + m_host_memory{m_ptr.get()}, + m_ar_flags{ar_flags} { if (!is_memory()) { throw std::invalid_argument{"memory range must be flagged memory when initializing "s + description}; } @@ -44,47 +54,4 @@ memory_address_range::memory_address_range(const std::string &description, uint6 throw std::invalid_argument{"unknown exception when initializing "s + description}; } -memory_address_range::memory_address_range(const std::string &description, uint64_t start, uint64_t length, - const pmas_flags &flags, const std::string &image_filename, const callocd & /*c*/) try : - address_range(description.c_str(), start, length, flags, [](const char *err) { throw base_error{err}; }), - m_ptr{make_unique_calloc(length)}, - m_host_memory{std::get(m_ptr).get()} { - if (!is_memory()) { - throw std::invalid_argument{"memory range must be flagged memory when initializing "s + description}; - } - m_dirty_page_map.resize((length / (8 * AR_PAGE_SIZE)) + 1, 0xff); - // Try to load image file, if any - if (!image_filename.empty()) { - auto fp = make_unique_fopen(image_filename.c_str(), "rb", std::nothrow_t{}); - if (!fp) { - throw std::system_error{errno, std::generic_category(), "error opening image file '"s + image_filename}; - } - // Get file size - if (fseek(fp.get(), 0, SEEK_END) != 0) { - throw std::system_error{errno, std::generic_category(), - "error obtaining length of image file '"s + image_filename}; - } - const auto file_length = static_cast(ftello(fp.get())); - if (fseek(fp.get(), 0, SEEK_SET) != 0) { - throw std::system_error{errno, std::generic_category(), - "error obtaining length of image file '"s + image_filename}; - } - // Check against PMA range size - if (file_length > length) { - throw std::runtime_error{"image file '"s + image_filename + "' is too large for range"s}; - } - // Read to host memory - const auto read_length = static_cast(fread(m_host_memory, 1, file_length, fp.get())); - if (read_length != file_length) { - throw std::runtime_error{"error reading from image file '"s + image_filename}; - } - } -} catch (base_error &b) { - throw; // already contains the description -} catch (std::exception &e) { - throw std::invalid_argument{e.what() + " when initializing "s + description}; -} catch (...) { - throw std::invalid_argument{"unknown exception when initializing "s + description}; -} - } // namespace cartesi diff --git a/src/memory-address-range.h b/src/memory-address-range.h index e7b828147..1aeaf3de2 100644 --- a/src/memory-address-range.h +++ b/src/memory-address-range.h @@ -21,6 +21,8 @@ #include #include "address-range.h" +#include "machine-config.h" +#include "os-mmap.h" #include "unique-c-ptr.h" namespace cartesi { @@ -31,50 +33,32 @@ class machine; /// \file /// \brief An address range occupied by memory -class memory_address_range : public address_range { - - using callocd_ptr = unique_calloc_ptr; - using mmapd_ptr = unique_mmap_ptr; - - std::variant - m_ptr; +struct memory_address_range_flags final { + bool read_only{false}; ///< Whether the memory is read-only on the host + bool page_uncleanable{false}; //< Whether the memory page dirty state can be cleaned +}; +class memory_address_range : public address_range { + unique_mmap_ptr m_ptr; ///< Pointer to mapped memory unsigned char *m_host_memory; ///< Start of associated memory region in host. + memory_address_range_flags m_ar_flags; ///< Memory address range specific flags. std::vector m_dirty_page_map; ///< Map of dirty pages. public: using ptr_type = std::unique_ptr; - /// \brief Mmap'd range data (shared or not). - struct mmapd { - bool shared; - }; - /// \brief Constructor for mmap'd ranges. /// \param description Description of address range for use in error messages /// \param start Start of address range /// \param length Length of address range /// \param flags Range flags /// \param image_filename Path to backing file. - /// \param m Mmap'd range data (shared or not). - memory_address_range(const std::string &description, uint64_t start, uint64_t length, const pmas_flags &flags, - const std::string &image_filename, const mmapd &m); - - /// \brief Calloc'd range data (just a tag). - struct callocd {}; - - /// \brief Constructor for calloc'd ranges. - /// \param description Description of address range for use in error messages - /// \param start Start of address range - /// \param length Length of address range - /// \param flags Range flags - /// \param image_filename Path to backing file. - /// \param c Calloc'd range data (just a tag). + /// \param host_length Length of host memory to be mapped. + /// This value can exceed the specified length, effectively creating an additional region + /// of memory that is not associated with the backing files. memory_address_range(const std::string &description, uint64_t start, uint64_t length, const pmas_flags &flags, - const std::string &image_filename, const callocd & /*c*/); + const backing_store_config &backing_store = {}, const memory_address_range_flags &ar_flags = {}, + uint64_t host_length = 0); memory_address_range(const memory_address_range &) = delete; memory_address_range &operator=(const memory_address_range &) = delete; @@ -100,6 +84,10 @@ class memory_address_range : public address_range { } void do_mark_clean_page(uint64_t offset) noexcept override { + if (m_ar_flags.page_uncleanable) { + // Dirty pages on this address range are permanently dirty and cannot be cleaned. + return; + } auto page_index = offset >> AR_constants::AR_PAGE_SIZE_LOG2; auto map_index = page_index >> 3; assert(map_index < m_dirty_page_map.size()); @@ -107,6 +95,10 @@ class memory_address_range : public address_range { } void do_mark_pages_clean() noexcept override { + if (m_ar_flags.page_uncleanable) { + // Dirty pages on this address range are permanently dirty and cannot be cleaned. + return; + } std::fill(m_dirty_page_map.begin(), m_dirty_page_map.end(), 0); } @@ -117,6 +109,14 @@ class memory_address_range : public address_range { return (m_dirty_page_map[map_index] & (1 << (page_index & 7))) != 0; } + bool do_is_host_read_only() const noexcept override { + return m_ar_flags.read_only; + } + + bool is_page_uncleanable() const noexcept { + return m_ar_flags.read_only; + } + bool do_peek(const machine & /*m*/, uint64_t offset, uint64_t length, const unsigned char **data, unsigned char * /*scratch*/) const noexcept override { if (contains_relative(offset, length)) { @@ -128,16 +128,6 @@ class memory_address_range : public address_range { } }; -static inline auto make_callocd_memory_address_range(const std::string &description, uint64_t start, uint64_t length, - pmas_flags flags, const std::string &image_filename = {}) { - return memory_address_range{description, start, length, flags, image_filename, memory_address_range::callocd{}}; -} - -static inline auto make_mmapd_memory_address_range(const std::string &description, uint64_t start, uint64_t length, - pmas_flags flags, const std::string &image_filename, bool shared) { - return memory_address_range{description, start, length, flags, image_filename, memory_address_range::mmapd{shared}}; -} - } // namespace cartesi #endif diff --git a/src/mock-address-range.h b/src/mock-address-range.h index ee73d9c0c..dd7b18714 100644 --- a/src/mock-address-range.h +++ b/src/mock-address-range.h @@ -26,13 +26,11 @@ #include "htif-address-range.h" #include "plic-address-range.h" #include "pmas.h" -#include "shadow-state-address-range.h" -#include "shadow-tlb-address-range.h" namespace cartesi { -using mock_address_range = std::variant; +using mock_address_range = + std::variant; using mock_address_ranges = std::array; @@ -61,10 +59,6 @@ static inline mock_address_range make_mock_address_range(uint64_t istart, uint64 return check_mock_address_range(make_empty_address_range("empty"), start, ilength, flags, abrt); } switch (flags.DID) { - case PMA_ISTART_DID::shadow_state: - return check_mock_address_range(make_shadow_state_address_range(abrt), start, ilength, flags, abrt); - case PMA_ISTART_DID::shadow_TLB: - return check_mock_address_range(make_shadow_tlb_address_range(abrt), start, ilength, flags, abrt); case PMA_ISTART_DID::CLINT: return check_mock_address_range(make_clint_address_range(abrt), start, ilength, flags, abrt); case PMA_ISTART_DID::PLIC: @@ -91,10 +85,6 @@ address_range &get_mock_address_range(mock_address_range &mock, ABRT abrt) { return std::get<3>(mock); case 4: return std::get<4>(mock); - case 5: - return std::get<5>(mock); - case 6: - return std::get<6>(mock); default: { static auto unhandled = make_empty_address_range("unhandled mock address range"); abrt("unhandled mock address range"); @@ -102,7 +92,7 @@ address_range &get_mock_address_range(mock_address_range &mock, ABRT abrt) { return unhandled; } } - static_assert(std::variant_size_v == 7); + static_assert(std::variant_size_v == 5); } } // namespace cartesi diff --git a/src/os-features.h b/src/os-features.h index 62b631ff0..7bdc84ce4 100644 --- a/src/os-features.h +++ b/src/os-features.h @@ -72,4 +72,8 @@ #define HAVE_FORK #endif +#if !defined(NO_FLOCK) && (defined(__linux__) || defined(__unix__) || defined(__APPLE__)) && !defined(__wasi__) +#define HAVE_FLOCK +#endif + #endif diff --git a/src/os-mmap.cpp b/src/os-mmap.cpp new file mode 100644 index 000000000..f5c7bf158 --- /dev/null +++ b/src/os-mmap.cpp @@ -0,0 +1,647 @@ +// Copyright Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: LGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify it under +// the terms of the GNU Lesser General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) any +// later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +// PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License along +// with this program (see COPYING). If not, see . +// + +#ifdef _WIN32 +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#ifndef _CRT_SECURE_NO_WARNINGS +#define _CRT_SECURE_NO_WARNINGS +#endif +#include +#endif // _WIN32 + +#include +#include +#include + +#include "os-features.h" +#include "os-mmap.h" +#include "scope-exit.h" + +//------------------------------------------------------------------------------ + +#if defined(HAVE_MMAP) +#include // open +#include // mmap/munmap +#include // fstat +#include // write/read/close +#else +#include // fopen/fclose/fread/fwrite/fflush +#include // calloc/free +#endif + +#if defined(HAVE_FLOCK) +#include // flock +#endif + +namespace cartesi { + +using namespace std::string_literals; + +constexpr uint64_t DEFAULT_MMAP_PAGE_SIZE = 4096; + +uint64_t os_get_mmap_page_size() { +#ifdef HAVE_MMAP + const auto page_size = sysconf(_SC_PAGESIZE); + if (page_size < 0) { + throw std::system_error{errno, std::generic_category(), "unable to retrieve system page size"s}; + } + return static_cast(page_size); + +#elif defined(_WIN32) + SYSTEM_INFO si; + GetSystemInfo(&si); + return static_cast(si.dwPageSize); + +#else + return DEFAULT_MMAP_PAGE_SIZE; + +#endif // HAVE_MMAP +} + +os_mmapd os_mmap(uint64_t length, const os_mmap_flags &flags, const std::string &backing_filename, + uint64_t backing_length) { + // Check some preconditions + if (backing_filename.empty() && (flags.lock || flags.shared || flags.create)) { + throw std::invalid_argument{"backing filename must be specified"s}; + } + if (flags.create && !flags.shared) { + throw std::invalid_argument{"created backing files must be shared"s}; + } + if (flags.read_only && flags.shared && (flags.create || flags.truncate)) { + throw std::invalid_argument{"cannot create or truncate backing files that are shared read-only"s}; + } + if (length < backing_length) { + throw std::invalid_argument{"length must be greater or equal than max backing length"s}; + } + const bool shared_write = flags.shared && !flags.read_only; + + // Ensure mapped pages are 4096-byte aligned for compatibility with + // routines using efficient SIMD operations on memory pages. + const uint64_t page_size = os_get_mmap_page_size(); + if (page_size < DEFAULT_MMAP_PAGE_SIZE) { + throw std::runtime_error{"system memory page size is less than "s + std::to_string(DEFAULT_MMAP_PAGE_SIZE)}; + } + +#ifdef HAVE_MMAP + int backing_fd = -1; + uint64_t backing_file_length = 0; + + // Auto close backing file on failure + auto backing_closer = scope_fail([&] { + if (backing_fd >= 0) { + // Close backing file + close(backing_fd); + // Remove backing file in case it was created + if (flags.create) { + unlink(backing_filename.c_str()); + } + } + }); + + // Handle backing file if specified + if (!backing_filename.empty()) { + // Determine backing file open flags + int oflags = (shared_write ? O_RDWR : O_RDONLY); + oflags |= O_CLOEXEC; // to remove file locks on fork + exec + mode_t omode = 0; + if (flags.create) { + oflags |= O_CREAT | O_EXCL; + omode = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH; + } + + // Open backing file + backing_fd = open(backing_filename.c_str(), oflags, omode); + if (backing_fd < 0) { + throw std::system_error{errno, std::generic_category(), + "unable to open backing file '"s + backing_filename + "'"s}; + } + +#ifdef HAVE_FLOCK + // Lock backing file immediately after opening it + if (flags.lock) { + const int flockop = (shared_write ? LOCK_EX : LOCK_SH) | LOCK_NB; + if (flock(backing_fd, flockop) < 0) { + throw std::system_error{errno, std::generic_category(), + "unable to lock backing file '"s + backing_filename + "'"s}; + } + } +#endif // HAVE_FLOCK + + // Get backing file size + struct stat statbuf{}; + if (fstat(backing_fd, &statbuf) < 0) { + throw std::system_error{errno, std::generic_category(), + "unable to obtain length of backing file '"s + backing_filename + "'"s}; + } + backing_file_length = static_cast(statbuf.st_size); + + // When backing file length mismatch the desired backing length we may need to truncate + if (backing_file_length != backing_length) { + if (!(flags.truncate || flags.create)) { // Can we truncate? + throw std::runtime_error{"backing file '"s + backing_filename + "' length ("s + + std::to_string(backing_file_length) + ") does not match desired backing length ("s + + std::to_string(backing_length) + ")"s}; + } + if (flags.shared) { // Truncate backing file to match the desired length + if (ftruncate(backing_fd, static_cast(backing_length)) < 0) { + throw std::runtime_error{"unable to truncate backing file '"s + backing_filename + "' length ("s + + std::to_string(backing_file_length) + ") to desired backing length ("s + + std::to_string(backing_length) + ")"s}; + } + backing_file_length = backing_length; + } else { + // Backing file length may be less than desired backing length, + // but this is fine, as we are not sharing the file. + // The gap between the file length and the memory length will be allocated as anonymous memory. + } + } + } + + // Determine map memory flags + int mflags = flags.shared ? MAP_SHARED : MAP_PRIVATE; + if (backing_fd < 0) { // The mapping is not backed by any file + mflags |= MAP_ANONYMOUS; + } + + // Determine map protection flags + int mprot = PROT_READ; + if (!flags.read_only) { + mprot |= PROT_WRITE; + } + + // Unmap memory in case memory mapping fails + void *host_memory = nullptr; + auto memory_closer = scope_fail([&] { + if (host_memory != nullptr) { + munmap(host_memory, length); + } + }); + + // Map memory + if (backing_fd >= 0 && length != backing_file_length) { + // Here we need to split the mapping into two parts: + // 1. Map the entire memory range with anonymous memory first + // 2. Replace memory map related to backing file portion + // This will leave a contiguous memory region where the first portion is backed by a file, + // and the second portion is backed by anonymous memory. + + // Map entire memory range, this is required to use the subsequent MAP_FIXED safely. + host_memory = mmap(nullptr, length, mprot, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (host_memory == MAP_FAILED) { + throw std::system_error{errno, std::generic_category(), "unable to map memory"s}; + } + + // Replace memory map related to backing file portion + void *backing_host_memory = mmap(host_memory, backing_file_length, mprot, mflags | MAP_FIXED, backing_fd, 0); + if (backing_host_memory != host_memory) { + throw std::system_error{errno, std::generic_category(), + "unable to map backing file '"s + backing_filename + "' to memory"s}; + } + + // POSIX ensures that any partial page at the end of a file is zero-filled, + // and modifications beyond the file's end are never written to disk. + // However on Linux, writing beyond such a partial page updates the page cache but not the file, + // and subsequent mappings may see the cached modifications even after the file is closed and unmapped, + // so we have to zero fill the last partial page just to be safe. + // See more about this on BUGS section in mmap() Linux man pages. + const uint64_t partial_page_size = backing_file_length % page_size; + const uint64_t partial_page_remaining = std::min(page_size - partial_page_size, length - backing_file_length); + if (partial_page_remaining > 0) { + if (flags.read_only) { + // We can't write on read-only mappings. + // This kind of mapping is unlikely to happen, so there is no need to support it. + throw std::system_error{errno, std::generic_category(), + "possible non zero partial page when mapping backing file '"s + backing_filename + "' to memory"s}; + } + std::memset(std::bit_cast(backing_host_memory) + backing_file_length, 0, partial_page_remaining); + } + } else { // Can perform a single mmap() + host_memory = mmap(nullptr, length, mprot, mflags, backing_fd, 0); + if (host_memory == MAP_FAILED) { + if (backing_fd < 0) { + throw std::system_error{errno, std::generic_category(), "unable to map memory"s}; + } + throw std::system_error{errno, std::generic_category(), + "unable to map backing file '"s + backing_filename + "' to memory"s}; + } + } + + return os_mmapd{.host_memory = host_memory, + .length = length, + .backing_sync_length = shared_write ? backing_length : 0, + .backing_fd = backing_fd}; + +#elif defined(_WIN32) + HANDLE backing_fh = INVALID_HANDLE_VALUE; + HANDLE backing_mapping = nullptr; + HANDLE memory_mapping = nullptr; + void *backing_host_memory = nullptr; + void *host_memory = nullptr; + uint64_t backing_file_length = 0; + + // Auto cleanup on failure + auto failure_cleaner = scope_fail([&] { + if (backing_host_memory != nullptr) { + std::ignore = UnmapViewOfFile(backing_host_memory); + } + if (host_memory != nullptr) { + std::ignore = UnmapViewOfFile(host_memory); + } + if (backing_mapping != nullptr) { + std::ignore = CloseHandle(backing_mapping); + } + if (memory_mapping != nullptr) { + std::ignore = CloseHandle(memory_mapping); + } + if (backing_fh != INVALID_HANDLE_VALUE) { + if (flags.lock) { + OVERLAPPED overlapped{}; + std::ignore = UnlockFileEx(backing_fh, 0, MAXDWORD, MAXDWORD, &overlapped); + } + // Close backing file + std::ignore = CloseHandle(backing_fh); + // Remove backing file in case it was created + if (flags.create) { + std::ignore = DeleteFileA(backing_filename.c_str()); + } + } + }); + + // Handle backing file if specified + if (!backing_filename.empty()) { + // Determine backing file open flags + const DWORD open_access = GENERIC_READ | (!flags.read_only ? GENERIC_WRITE : 0); + const DWORD share_mode = FILE_SHARE_READ | (!flags.read_only ? FILE_SHARE_WRITE : 0); + const DWORD creation = flags.create ? CREATE_NEW : OPEN_EXISTING; + + // Open backing file + backing_fh = CreateFileA(backing_filename.c_str(), open_access, share_mode, nullptr, creation, + FILE_ATTRIBUTE_NORMAL, nullptr); + if (backing_fh == INVALID_HANDLE_VALUE) { + throw std::system_error{static_cast(GetLastError()), std::system_category(), + "unable to open backing file '"s + backing_filename + "'"s}; + } + + // Lock backing file immediately after opening it + if (flags.lock) { + DWORD lock_flags = LOCKFILE_FAIL_IMMEDIATELY; // Non-blocking lock + if (shared_write) { + lock_flags |= LOCKFILE_EXCLUSIVE_LOCK; // Exclusive lock for writing + } + OVERLAPPED overlapped{}; + if (!LockFileEx(backing_fh, lock_flags, 0, MAXDWORD, MAXDWORD, &overlapped)) { + throw std::system_error{static_cast(GetLastError()), std::system_category(), + "unable to lock backing file '" + backing_filename + "'"}; + } + } + + // Get backing file size + LARGE_INTEGER size{}; + if (!GetFileSizeEx(backing_fh, &size)) { + throw std::system_error{static_cast(GetLastError()), std::system_category(), + "unable to obtain length of backing file '"s + backing_filename + "'"s}; + } + backing_file_length = static_cast(size.QuadPart); + + // When backing file length mismatch the desired backing length we may need to truncate + if (backing_file_length != backing_length) { + if (!(flags.truncate || flags.create)) { // Can we truncate? + throw std::runtime_error{"backing file '"s + backing_filename + "' length ("s + + std::to_string(backing_file_length) + ") does not match desired backing length ("s + + std::to_string(backing_length) + ")"s}; + } + if (flags.shared) { // Truncate backing file to match the desired length + // Set file pointer to the new size position + LARGE_INTEGER new_size{.QuadPart = static_cast(backing_length)}; + if (!SetFilePointerEx(backing_fh, new_size, nullptr, FILE_BEGIN)) { + throw std::system_error{static_cast(GetLastError()), std::system_category(), + "unable to seek backing file '"s + backing_filename + "'"s}; + } + + // Set the end of file at the current position + if (!SetEndOfFile(backing_fh)) { + throw std::system_error{static_cast(GetLastError()), std::system_category(), + "unable to truncate backing file '"s + backing_filename + "'"s}; + } + + // Initialize truncated bytes to zeros + if (backing_length > backing_file_length) { + // Set file pointer to truncate position + LARGE_INTEGER truncate_pos{.QuadPart = static_cast(backing_file_length)}; + if (!SetFilePointerEx(backing_fh, truncate_pos, nullptr, FILE_BEGIN)) { + throw std::system_error{static_cast(GetLastError()), std::system_category(), + "unable to seek backing file '"s + backing_filename + "'"s}; + } + + // Write zeros in chunks + static uint8_t zero_buffer[65536]{}; + uint64_t remaining = backing_length - backing_file_length; + while (remaining > 0) { + const auto write_size = static_cast(std::min(remaining, sizeof(zero_buffer))); + DWORD bytes_written = 0; + if (!WriteFile(backing_fh, zero_buffer, write_size, &bytes_written, nullptr) || + bytes_written != write_size) { + throw std::system_error{static_cast(GetLastError()), std::system_category(), + "unable to write backing file '"s + backing_filename + "'"s}; + } + remaining -= bytes_written; + } + } + + // Flush file changes + if (!FlushFileBuffers(backing_fh)) { + throw std::system_error{static_cast(GetLastError()), std::system_category(), + "unable to flush backing file '"s + backing_filename + "'"s}; + } + + backing_file_length = backing_length; + } else { + // Backing file length may be less than desired backing length, + // but this is fine, as we are not sharing the file. + // The gap between the file length and the memory length will be allocated as anonymous memory. + } + } + } + + HANDLE memory_handle = INVALID_HANDLE_VALUE; + if (backing_file_length > 0 && backing_file_length == length) { + // When backing file length matches memory length, we can map it directly + memory_handle = backing_fh; + } else if (backing_file_length > 0) { + // Unfortunately Windows does not easily support creating a contiguous memory region with + // mixed file-backed and anonymous memory mappings. Instead, we create an auxiliary + // mapping for the backing file and handle data transfers explicitly: + // 1) Copy data from backing file to host memory during mapping + // 2) Copy modified data back to the backing file during unmapping in case of shared write access + + // Determine backing file mapping flags + const DWORD backing_access = shared_write ? FILE_MAP_WRITE : FILE_MAP_READ; + const DWORD backing_protect = shared_write ? PAGE_READWRITE : PAGE_READONLY; + + // Create backing file mapping + backing_mapping = + CreateFileMappingA(backing_fh, nullptr, backing_protect, static_cast(backing_file_length >> 32), + static_cast(backing_file_length & 0xffffffff), nullptr); + if (backing_mapping == nullptr) { + throw std::system_error{static_cast(GetLastError()), std::system_category(), + "unable to create memory mapping of backing file '"s + backing_filename + "'"s}; + } + + // Map backing file + backing_host_memory = MapViewOfFile(backing_mapping, backing_access, 0, 0, backing_file_length); + if (backing_host_memory == nullptr) { + throw std::system_error{static_cast(GetLastError()), std::system_category(), + "unable to map memory of backing file '"s + backing_filename + "'"s}; + } + } + + // Determine memory mapping flags + const DWORD protect = flags.read_only ? PAGE_READONLY : PAGE_READWRITE; + DWORD access = FILE_MAP_READ; + if (!flags.read_only) { + access |= FILE_MAP_WRITE; + if (!flags.shared) { + access |= FILE_MAP_COPY; + } + } + + // Create memory mapping + memory_mapping = CreateFileMappingA(memory_handle, nullptr, protect, static_cast(length >> 32), + static_cast(length & 0xffffffff), nullptr); + if (memory_mapping == nullptr) { + throw std::system_error{static_cast(GetLastError()), std::system_category(), + "unable to create memory mapping"s}; + } + + // Map memory + host_memory = MapViewOfFile(memory_mapping, access, 0, 0, length); + if (host_memory == nullptr) { + throw std::system_error{static_cast(GetLastError()), std::system_category(), "unable to map memory"s}; + } + + // Copy contents from the backing file if needed + if (backing_host_memory != nullptr && backing_file_length > 0) { + std::memcpy(host_memory, backing_host_memory, backing_file_length); + } + + return os_mmapd{.host_memory = host_memory, + .length = length, + .backing_sync_length = shared_write ? backing_length : 0, + .memory_mapping = static_cast(memory_mapping), + .backing_host_memory = backing_host_memory, + .backing_mapping = backing_mapping, + .backing_fh = static_cast(backing_fh), + .backing_lock = flags.lock}; + +#else // Fallback implementation using standard C APIs + // Over-allocate to ensure we can align the pointer and store the original pointer. + // Use calloc() instead of aligned_alloc() to initialize memory to zeros. + std::size_t space = length + DEFAULT_MMAP_PAGE_SIZE - 1; + void *unaligned_host_memory = std::calloc(1, space); // NOLINT(cppcoreguidelines-no-malloc,hicpp-no-malloc) + if (unaligned_host_memory == nullptr) { + throw std::bad_alloc{}; + } + + // Automatically deallocate memory on failure + auto memory_closer = scope_fail([&] { + std::free(unaligned_host_memory); // NOLINT(cppcoreguidelines-no-malloc,hicpp-no-malloc) + }); + + // Align allocated memory + void *host_memory = unaligned_host_memory; + if (std::align(DEFAULT_MMAP_PAGE_SIZE, length, host_memory, space) == nullptr) { + throw std::runtime_error{"unable to align allocated memory"}; + } + + FILE *backing_fp = nullptr; + + // Automatically close backing file on failure + auto backing_closer = scope_fail([&] { + if (backing_fp != nullptr) { + // Close backing file + std::ignore = std::fclose(backing_fp); + // Remove backing file in case it was created + if (flags.create) { + std::ignore = std::remove(backing_filename.c_str()); + } + } + }); + + // Handle backing file if specified + if (!backing_filename.empty()) { + // Determine backing file open flags + const char *mode{}; + if (flags.create) { + // Check if file already exists first + auto *fp = std::fopen(backing_filename.c_str(), "rb"); + if (fp != nullptr) { + std::ignore = std::fclose(fp); + throw std::system_error{errno, std::generic_category(), + "unable to create backing file '"s + backing_filename + "': file already exists"s}; + } + mode = "w+b"; + } else if (shared_write) { + mode = "r+b"; + } else { + mode = "rb"; + } + + // Open backing file + backing_fp = std::fopen(backing_filename.c_str(), mode); + if (backing_fp == nullptr) { + throw std::system_error{errno, std::generic_category(), + "unable to open backing file '"s + backing_filename + "'"s}; + } + + // Get backing file size + if (std::fseek(backing_fp, 0, SEEK_END) < 0) { + throw std::system_error{errno, std::generic_category(), + "unable to obtain length of backing file '"s + backing_filename + "'"s}; + } + const auto file_size = std::ftell(backing_fp); + if (file_size < 0) { + throw std::system_error{errno, std::generic_category(), + "unable to obtain length of backing file '"s + backing_filename + "'"s}; + } + const auto backing_file_length = static_cast(file_size); + + // Copy backing file contents + if (std::fseek(backing_fp, 0, SEEK_SET) < 0) { + throw std::system_error{errno, std::generic_category(), + "unable to seek to beginning of backing file '"s + backing_filename + "'"s}; + } + const auto copy_length = std::min(backing_file_length, backing_length); + if (static_cast(std::fread(host_memory, 1, copy_length, backing_fp)) != copy_length) { + throw std::system_error{errno, std::generic_category(), + "unable to read from backing file '"s + backing_filename + "'"s}; + } + + // When backing file length mismatch the desired backing length we may need to truncate + if (backing_file_length != backing_length) { + if (!(flags.truncate || flags.create)) { // Can we truncate? + throw std::runtime_error{"backing file '"s + backing_filename + "' length ("s + + std::to_string(backing_file_length) + ") does not match desired backing length ("s + + std::to_string(backing_length) + ")"s}; + } + if (flags.shared) { // Truncate backing file to match the desired length + // In order to truncate the file using standard C API, + // we need recreate from scratch and copy over the existing content. + if (std::fclose(backing_fp) != 0) { + throw std::system_error{errno, std::generic_category(), + "unable to truncate backing file '"s + backing_filename + "': fclose() failed"s}; + } + backing_fp = std::fopen(backing_filename.c_str(), "w+b"); + if (backing_fp == nullptr) { + throw std::system_error{errno, std::generic_category(), + "unable to truncate backing file '"s + backing_filename + "': fopen() failed"s}; + } + if (static_cast(std::fwrite(host_memory, 1, backing_length, backing_fp)) != backing_length) { + throw std::system_error{errno, std::generic_category(), + "unable to truncate backing file '"s + backing_filename + "': fwrite() failed"s}; + } + if (std::fflush(backing_fp) != 0) { + throw std::system_error{errno, std::generic_category(), + "unable to truncate backing file '"s + backing_filename + "': fflush() failed"s}; + } + } else { + // Backing file length may be less than desired backing length, + // but this is fine, as we are not sharing the file. + // The gap between the file length and the memory length will be allocated as anonymous memory. + } + } + + // Rewind the backing file to the beginning + if (std::fseek(backing_fp, 0, SEEK_SET) < 0) { + throw std::system_error{errno, std::generic_category(), + "unable to rewind backing file '"s + backing_filename + "'"s}; + } + } + + return os_mmapd{.host_memory = host_memory, + .length = length, + .backing_sync_length = shared_write ? backing_length : 0, + .backing_fp = backing_fp, + .unaligned_host_memory = unaligned_host_memory}; +#endif // HAVE_MMAP +} + +void os_munmap(const os_mmapd &mmapd) noexcept { +#ifdef HAVE_MMAP + if (mmapd.host_memory != nullptr && mmapd.length > 0) { + if (mmapd.backing_sync_length > 0) { + // Request the kernel to flush the mapped file to disk asynchronously. + // This may reduce the risk of file corruption in the event of unexpected host power loss. + std::ignore = msync(mmapd.host_memory, mmapd.backing_sync_length, MS_ASYNC); + } + std::ignore = munmap(mmapd.host_memory, mmapd.length); + } + if (mmapd.backing_fd != -1) { + // Closing a file will also release file locks + std::ignore = close(mmapd.backing_fd); + } + +#elif defined(_WIN32) // Windows implementation + // Flush changes to disk if necessary + if (mmapd.backing_sync_length > 0 && mmapd.host_memory != nullptr) { + if (mmapd.backing_host_memory != nullptr) { // Copy contents to the backing file + std::memcpy(mmapd.backing_host_memory, mmapd.host_memory, mmapd.backing_sync_length); + std::ignore = FlushViewOfFile(mmapd.backing_host_memory, mmapd.backing_sync_length); + } else { + std::ignore = FlushViewOfFile(mmapd.host_memory, mmapd.backing_sync_length); + } + if (mmapd.backing_fh != nullptr && mmapd.backing_fh != INVALID_HANDLE_VALUE) { + std::ignore = FlushFileBuffers(mmapd.backing_fh); + } + } + if (mmapd.host_memory != nullptr) { + std::ignore = UnmapViewOfFile(mmapd.host_memory); + } + if (mmapd.backing_host_memory != nullptr) { + std::ignore = UnmapViewOfFile(mmapd.backing_host_memory); + } + if (mmapd.memory_mapping != nullptr) { + std::ignore = CloseHandle(static_cast(mmapd.memory_mapping)); + } + if (mmapd.backing_mapping != nullptr) { + std::ignore = CloseHandle(static_cast(mmapd.backing_mapping)); + } + if (mmapd.backing_fh != nullptr && mmapd.backing_fh != INVALID_HANDLE_VALUE) { + if (mmapd.backing_lock) { + OVERLAPPED overlapped{}; + std::ignore = UnlockFileEx(static_cast(mmapd.backing_fh), 0, MAXDWORD, MAXDWORD, &overlapped); + } + std::ignore = CloseHandle(static_cast(mmapd.backing_fh)); + } + +#else // Fallback implementation using standard C APIs + if (mmapd.backing_sync_length > 0 && mmapd.host_memory != nullptr && mmapd.backing_fp != nullptr) { + // Write back changes to backing file + std::ignore = std::fwrite(mmapd.host_memory, 1, mmapd.backing_sync_length, mmapd.backing_fp); + std::ignore = std::fflush(mmapd.backing_fp); + } + if (mmapd.unaligned_host_memory != nullptr) { + std::free(mmapd.unaligned_host_memory); // NOLINT(cppcoreguidelines-no-malloc,hicpp-no-malloc) + } + if (mmapd.backing_fp != nullptr) { + std::ignore = std::fclose(mmapd.backing_fp); + } + +#endif // HAVE_MMAP +} + +} // namespace cartesi diff --git a/src/os-mmap.h b/src/os-mmap.h new file mode 100644 index 000000000..0f9f4542d --- /dev/null +++ b/src/os-mmap.h @@ -0,0 +1,123 @@ +// Copyright Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: LGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify it under +// the terms of the GNU Lesser General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) any +// later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +// PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License along +// with this program (see COPYING). If not, see . +// + +#ifndef OS_MMAP_H +#define OS_MMAP_H + +/// \file +/// \brief Operating system memory mapping operations. +/// \details \{ +/// This header file defines the interface for memory mapping operations for the emulator. +/// It provides a cross-platform abstraction for memory-mapped regions, supporting both anonymous +/// and file-backed mappings. The implementation ensures compatibility with various operating +/// systems, including Linux, Windows, and fallback environments without native `mmap` support. +/// +/// Key Features: +/// - Memory mapping with optional file backing. +/// - Support for read-only, shared, and exclusive access modes. +/// - Automatic alignment to system page size for efficient memory access. +/// - Cross-platform compatibility with platform-specific optimizations. +/// +/// This module is designed to handle complex memory mapping scenarios, such as partial file +/// mappings, file truncation, and synchronization of file-backed memory regions. +/// \} + +#include +#include +#include + +#include "os-features.h" + +#ifndef HAVE_MMAP +#include +#endif + +namespace cartesi { + +/// \brief Flags for memory mapping operations. +struct os_mmap_flags { + bool read_only{false}; ///< Mark mapped memory as read-only + bool shared{false}; ///< Share mapped memory with the backing file + bool create{false}; ///< Create the backing file (requires shared to be true) + bool truncate{false}; ///< Truncate backing file to match the specified backing file length + bool lock{false}; ///< Lock the backing file for exclusive writing when shared, otherwise for shared reading +}; + +/// \brief Structure representing a memory-mapped region. +struct os_mmapd { + void *host_memory{nullptr}; ///< Pointer to the mapped memory region + uint64_t length{0}; ///< The total size of the mapped memory + uint64_t backing_sync_length{0}; ///< Length of file-backed portion for which memory sync is needed +#ifdef HAVE_MMAP + int backing_fd{-1}; ///< File descriptor of the backing file +#elif defined(_WIN32) + void *memory_mapping{nullptr}; ///< Handle of the memory mapping + void *backing_host_memory{nullptr}; ///< Pointer to the backing file mapped memory region + void *backing_mapping{nullptr}; ///< Handle of the backing file mapping + void *backing_fh{nullptr}; ///< Handle of the backing file + bool backing_lock{false}; ///< Whether the backing file was locked +#else + FILE *backing_fp{nullptr}; ///< Pointer of the backing file + void *unaligned_host_memory{nullptr}; ///< Pointer to the memory that we can deallocate with std::free() +#endif +}; + +/// \brief Retrieves the system's memory page size. +/// \details Typically 4KB, but may vary (e.g., 8KB on Solaris, 16KB on macOS arm64). +uint64_t os_get_mmap_page_size(); + +/// \brief Maps a memory region, optionally backed by a file. +/// \param length Total memory length to map. +/// \param flags Flags for the mapping. +/// \param backing_filename Path to the file to back the memory mapping (if empty, creates anonymous mapping). +/// \param backing_length The expected or intended size of the backing file in bytes (must be <= length). +/// \returns Structure containing the memory mapping information. +/// \details The memory is guaranteed to be aligned to 4096-byte boundaries. +/// Memory above backing file length are zereod, and modifications to that region are not written out to the file. +os_mmapd os_mmap(uint64_t length, const os_mmap_flags &flags = {}, const std::string &backing_filename = {}, + uint64_t backing_length = 0); + +/// \brief Unmaps a previously mapped memory region. +void os_munmap(const os_mmapd &mmapd) noexcept; + +namespace detail { + +struct mmap_deleter { + os_mmapd m_mmapd; + explicit mmap_deleter(os_mmapd mmapd) : m_mmapd(mmapd) {}; + template + void operator()(T * /*ptr*/) const noexcept { + os_munmap(m_mmapd); + } +}; + +} // namespace detail + +template +using unique_mmap_ptr = std::unique_ptr; + +template +static inline auto make_unique_mmap(size_t nmemb, const os_mmap_flags &flags = {}, + const std::string &backing_filename = {}, uint64_t backing_length = 0) { + const size_t size = nmemb * sizeof(T); + const os_mmapd mmapd = os_mmap(size, flags, backing_filename, backing_length); // os_map throws on error + T *ptr = static_cast(mmapd.host_memory); + return unique_mmap_ptr(ptr, detail::mmap_deleter{mmapd}); +} + +} // namespace cartesi + +#endif diff --git a/src/os.cpp b/src/os.cpp index 819d13dd9..78f7574dd 100644 --- a/src/os.cpp +++ b/src/os.cpp @@ -43,7 +43,7 @@ #include #endif -#if defined(HAVE_TTY) || defined(HAVE_MMAP) || defined(HAVE_TERMIOS) || defined(_WIN32) +#if defined(HAVE_TTY) || defined(HAVE_TERMIOS) || defined(_WIN32) #include // open #endif @@ -54,11 +54,7 @@ #endif #endif -#ifdef HAVE_MMAP -#include // mmap/munmap -#endif - -#if defined(HAVE_MMAP) || defined(HAVE_MKDIR) || defined(_WIN32) +#if defined(HAVE_MKDIR) || defined(_WIN32) #include // fstat/mkdir #endif @@ -89,7 +85,7 @@ #else // not _WIN32 -#if defined(HAVE_TTY) || defined(HAVE_MMAP) || defined(HAVE_TERMIOS) || defined(HAVE_USLEEP) +#if defined(HAVE_TTY) || defined(HAVE_TERMIOS) || defined(HAVE_USLEEP) #include // write/read/close/usleep/fork #endif @@ -557,148 +553,6 @@ int os_mkdir(const char *path, [[maybe_unused]] int mode) { #endif // HAVE_MKDIR } -void *os_map_file(const char *path, uint64_t length, bool shared) { - if ((path == nullptr) || *path == '\0') { - throw std::runtime_error{"image file path must be specified"s}; - } - -#ifdef HAVE_MMAP - const int oflag = shared ? O_RDWR : O_RDONLY; - - // Try to open image file - const int backing_file = open(path, oflag); - if (backing_file < 0) { - throw std::system_error{errno, std::generic_category(), "could not open image file '"s + path + "'"s}; - } - - // Try to get file size - struct stat statbuf{}; - if (fstat(backing_file, &statbuf) < 0) { - close(backing_file); - throw std::system_error{errno, std::generic_category(), - "unable to obtain length of image file '"s + path + "'"s}; - } - - // Check that it matches range length - if (static_cast(statbuf.st_size) != length) { - close(backing_file); - throw std::invalid_argument{"image file '"s + path + "' size ("s + - std::to_string(static_cast(statbuf.st_size)) + ") does not match range length ("s + - std::to_string(length) + ")"s}; - } - - // Try to map image file to host memory - const int mflag = shared ? MAP_SHARED : MAP_PRIVATE; - void *host_memory = mmap(nullptr, length, PROT_READ | PROT_WRITE, mflag, backing_file, 0); - if (host_memory == MAP_FAILED) { // NOLINT(cppcoreguidelines-pro-type-cstyle-cast,performance-no-int-to-ptr) - close(backing_file); - throw std::system_error{errno, std::generic_category(), "could not map image file '"s + path + "' to memory"s}; - } - - // We can close the file after mapping it, because the OS will retain a reference of the file on its own - close(backing_file); - return host_memory; - -#elif defined(_WIN32) - const int oflag = (shared ? _O_RDWR : _O_RDONLY) | _O_BINARY; - - // Try to open image file - const int backing_file = _open(path, oflag); - if (backing_file < 0) { - throw std::system_error{errno, std::generic_category(), "could not open image file '"s + path + "'"s}; - } - - // Try to get file size - struct __stat64 statbuf{}; - if (_fstat64(backing_file, &statbuf) < 0) { - _close(backing_file); - throw std::system_error{errno, std::generic_category(), - "unable to obtain length of image file '"s + path + "'"s}; - } - - // Check that it matches range length - if (static_cast(statbuf.st_size) != length) { - _close(backing_file); - throw std::invalid_argument{"image file '"s + path + "' size ("s + - std::to_string(static_cast(statbuf.st_size)) + ") does not match range length ("s + - std::to_string(length) + ")"s}; - } - - // Try to map image file to host memory - DWORD flProtect = shared ? PAGE_READWRITE : PAGE_READONLY; - // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) - HANDLE hFile = reinterpret_cast(_get_osfhandle(backing_file)); - HANDLE hFileMappingObject = CreateFileMapping(hFile, NULL, flProtect, length >> 32, length & 0xffffffff, NULL); - if (!hFileMappingObject) { - _close(backing_file); - throw std::system_error{errno, std::generic_category(), "could not map image file '"s + path + "' to memory"s}; - } - - DWORD dwDesiredAccess = shared ? FILE_MAP_WRITE : FILE_MAP_COPY; - void *host_memory = MapViewOfFile(hFileMappingObject, dwDesiredAccess, 0, 0, length); - if (!host_memory) { - _close(backing_file); - throw std::system_error{errno, std::generic_category(), "could not map image file '"s + path + "' to memory"s}; - } - - // We can close the file after mapping it, because the OS will retain a reference of the file on its own - _close(backing_file); - return host_memory; - -#else - if (shared) { - throw std::runtime_error{"shared image file mapping is unsupported"s}; - } - - auto fp = unique_fopen(path, "rb", std::nothrow_t{}); - if (!fp) { - throw std::system_error{errno, std::generic_category(), "error opening image file '"s + path + "'"s}; - } - // Get file size - if (fseek(fp.get(), 0, SEEK_END)) { - throw std::system_error{errno, std::generic_category(), - "error obtaining length of image file '"s + path + "'"s}; - } - auto file_length = ftell(fp.get()); - if (fseek(fp.get(), 0, SEEK_SET)) { - throw std::system_error{errno, std::generic_category(), - "error obtaining length of image file '"s + path + "'"s}; - } - // Check against PMA range size - if (static_cast(file_length) > length) { - throw std::runtime_error{"image file '"s + path + "' of "s + " is too large for range"s}; - } - - // use calloc to improve performance - // NOLINTNEXTLINE(cppcoreguidelines-no-malloc,hicpp-no-malloc) - auto host_memory = static_cast(std::calloc(1, length)); - if (!host_memory) { - throw std::runtime_error{"error allocating memory"s}; - } - - // Read to host memory - std::ignore = fread(host_memory, 1, length, fp.get()); - if (ferror(fp.get())) { - throw std::system_error{errno, std::generic_category(), "error reading from image file '"s + path + "'"s}; - } - return host_memory; - -#endif // HAVE_MMAP -} - -void os_unmap_file(void *host_memory, [[maybe_unused]] uint64_t length) { -#ifdef HAVE_MMAP - munmap(host_memory, length); - -#elif defined(_WIN32) - UnmapViewOfFile(host_memory); - -#else - std::free(host_memory); - -#endif // HAVE_MMAP -} - int64_t os_now_us() { std::chrono::time_point start{}; static bool started = false; diff --git a/src/os.h b/src/os.h index 5593c0d1b..2c4bbd0eb 100644 --- a/src/os.h +++ b/src/os.h @@ -93,12 +93,6 @@ void os_silence_putchar(bool yes); /// \brief Creates a new directory int os_mkdir(const char *path, int mode); -/// \brief Maps a file to memory -void *os_map_file(const char *path, uint64_t length, bool shared); - -/// \brief Unmaps a file from memory -void os_unmap_file(void *host_memory, uint64_t length); - /// \brief Get time elapsed since its first call with microsecond precision int64_t os_now_us(); diff --git a/src/pmas-constants.h b/src/pmas-constants.h index ff87b0031..4744f56ad 100644 --- a/src/pmas-constants.h +++ b/src/pmas-constants.h @@ -59,7 +59,6 @@ enum class PMA_ISTART_DID : uint8_t { empty = PMA_EMPTY_DID_DEF, ///< DID for empty range memory = PMA_MEMORY_DID_DEF, ///< DID for memory shadow_state = PMA_SHADOW_STATE_DID_DEF, ///< DID for shadow state - shadow_TLB = PMA_SHADOW_TLB_DID_DEF, ///< DID for shadow TLB flash_drive = PMA_FLASH_DRIVE_DID_DEF, ///< DID for flash drives CLINT = PMA_CLINT_DID_DEF, ///< DID for CLINT device PLIC = PMA_PLIC_DID_DEF, ///< DID for PLIC device diff --git a/src/pmas-defines.h b/src/pmas-defines.h index 38008e7c7..42561cd27 100644 --- a/src/pmas-defines.h +++ b/src/pmas-defines.h @@ -20,18 +20,17 @@ #define PMA_MAX_DEF 32 ///< Maximum number of PMAs -#define PMA_EMPTY_DID_DEF 0 ///< Device ID for empty range -#define PMA_MEMORY_DID_DEF 1 ///< Device ID for memory -#define PMA_SHADOW_STATE_DID_DEF 2 ///< Device ID for shadow state device -#define PMA_FLASH_DRIVE_DID_DEF 3 ///< Device ID for flash drive device -#define PMA_CLINT_DID_DEF 4 ///< Device ID for CLINT device -#define PMA_HTIF_DID_DEF 5 ///< Device ID for HTIF device -#define PMA_SHADOW_TLB_DID_DEF 6 ///< Device ID for shadow TLB device -#define PMA_CMIO_RX_BUFFER_DID_DEF 7 ///< Device ID for cmio RX buffer -#define PMA_CMIO_TX_BUFFER_DID_DEF 8 ///< Device ID for cmio TX buffer -#define PMA_PLIC_DID_DEF 10 ///< Device ID for PLIC device -#define PMA_VIRTIO_DID_DEF 11 ///< Device ID for VirtIO devices -#define PMA_SHADOW_UARCH_STATE_DID_DEF 12 ///< Device ID for uarch shadow state device +#define PMA_EMPTY_DID_DEF 0 ///< Device ID for empty range +#define PMA_MEMORY_DID_DEF 1 ///< Device ID for memory +#define PMA_SHADOW_STATE_DID_DEF 2 ///< Device ID for shadow state device +#define PMA_FLASH_DRIVE_DID_DEF 3 ///< Device ID for flash drive device +#define PMA_CLINT_DID_DEF 4 ///< Device ID for CLINT device +#define PMA_HTIF_DID_DEF 5 ///< Device ID for HTIF device +#define PMA_PLIC_DID_DEF 6 ///< Device ID for PLIC device +#define PMA_CMIO_RX_BUFFER_DID_DEF 7 ///< Device ID for cmio RX buffer +#define PMA_CMIO_TX_BUFFER_DID_DEF 8 ///< Device ID for cmio TX buffer +#define PMA_SHADOW_UARCH_STATE_DID_DEF 9 ///< Device ID for uarch shadow state device +#define PMA_VIRTIO_DID_DEF 10 ///< Device ID for VirtIO devices // helper for using UINT64_C with defines #ifndef EXPAND_UINT64_C diff --git a/src/pmas.h b/src/pmas.h index 819981e40..c8ef4f1d2 100644 --- a/src/pmas.h +++ b/src/pmas.h @@ -38,8 +38,6 @@ static constexpr const char *pmas_get_DID_name(PMA_ISTART_DID did) { return "DID.memory"; case PMA_ISTART_DID::shadow_state: return "DID.shadow_state"; - case PMA_ISTART_DID::shadow_TLB: - return "DID.shadow_TLB"; case PMA_ISTART_DID::flash_drive: return "DID.flash_drive"; case PMA_ISTART_DID::CLINT: @@ -55,7 +53,7 @@ static constexpr const char *pmas_get_DID_name(PMA_ISTART_DID did) { case PMA_ISTART_DID::cmio_tx_buffer: return "DID.cmio_tx_buffer"; case PMA_ISTART_DID::shadow_uarch_state: - return "DID.shadow_uarch"; + return "DID.shadow_uarch_state"; } return "DID.unknown"; } diff --git a/src/processor-state.h b/src/processor-state.h new file mode 100644 index 000000000..3a4c420c3 --- /dev/null +++ b/src/processor-state.h @@ -0,0 +1,77 @@ +// Copyright Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: LGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify it under +// the terms of the GNU Lesser General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) any +// later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +// PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License along +// with this program (see COPYING). If not, see . +// + +#ifndef PROCESSOR_STATE_H +#define PROCESSOR_STATE_H + +/// \file +/// \brief Cartesi machine processor state structure definition. + +#include +#include +#include +#include + +#include "address-range.h" +#include "hot-tlb.h" +#include "riscv-constants.h" +#include "shadow-registers.h" +#include "shadow-tlb.h" + +namespace cartesi { + +/// brief Shadow state. +/// \details It's stored in the processor backing file. +struct shadow_state final { + registers_state registers; ///< Registers state + uint64_t registers_padding_[406]{}; ///< Padding to align next field to a page boundary + shadow_tlb_state shadow_tlb; ///< Shadow TLB state + uint64_t shadow_tlb_padding[512]{}; ///< Padding to align next field to a page boundary +}; + +/// \brief Penumbra state. +/// \details It's not stored in the processor backing file, it's only visible in host resident memory during runtime. +struct penumbra_state final { + hot_tlb_state hot_tlb; ///< Hot TLB state +}; + +/// \brief Machine processor state. +/// \details Contains the registers and TLB state of a machine. +struct processor_state final { + registers_state registers; ///< Registers state + uint64_t registers_padding_[406]{}; ///< Padding to align next field to a page boundary + shadow_tlb_state shadow_tlb; ///< Shadow TLB state + uint64_t shadow_tlb_padding[512]{}; ///< Padding to align next field to a page boundary + hot_tlb_state hot_tlb; ///< Hot TLB state +}; + +static_assert(offsetof(processor_state, shadow_tlb) % AR_PAGE_SIZE == 0, + "shadow tlb state must be aligned to a page boundary"); +static_assert(offsetof(processor_state, hot_tlb) % AR_PAGE_SIZE == 0, + "hot tlb state must be aligned to a page boundary"); + +static_assert(sizeof(processor_state) % AR_PAGE_SIZE == 0, "processor state size must be multiple of a page size"); +static_assert(sizeof(shadow_state) == AR_SHADOW_STATE_LENGTH, "unexpected shadow state size"); + +// The size of the shadow state should align with the largest page size used by the supported operating systems. +// For instance, macOS on arm64 currently utilizes a page size of 16KB. +// Aligning the shadow state size with the page size ensures that memory allocations made with os_mmap() +// result in the backing shadow file occupying whole pages, avoiding partial page mappings. +static_assert(sizeof(shadow_state) % 16384 == 0, "shadow state size must be multiple of a 16KB"); + +} // namespace cartesi + +#endif diff --git a/src/record-send-cmio-state-access.h b/src/record-send-cmio-state-access.h index 9b365ccf2..9026b61c4 100644 --- a/src/record-send-cmio-state-access.h +++ b/src/record-send-cmio-state-access.h @@ -32,10 +32,10 @@ #include "i-hasher.h" #include "i-state-access.h" #include "machine-merkle-tree.h" -#include "machine-state.h" #include "machine.h" #include "meta.h" -#include "shadow-state.h" +#include "processor-state.h" +#include "shadow-registers.h" namespace cartesi { @@ -185,18 +185,18 @@ class record_send_cmio_state_access : friend i_state_access; void do_write_iflags_Y(uint64_t val) const { - log_before_write_write_and_update(machine_reg_address(machine_reg::iflags_Y), m_m.get_state().iflags.Y, val, - "iflags.Y"); + log_before_write_write_and_update(machine_reg_address(machine_reg::iflags_Y), + m_m.get_state().registers.iflags.Y, val, "iflags.Y"); } uint64_t do_read_iflags_Y() const { log_read(machine_reg_address(machine_reg::iflags_Y), "iflags.Y"); - return m_m.get_state().iflags.Y; + return m_m.get_state().registers.iflags.Y; } void do_write_htif_fromhost(uint64_t val) const { log_before_write_write_and_update(machine_reg_address(machine_reg::htif_fromhost), - m_m.get_state().htif.fromhost, val, "htif.fromhost"); + m_m.get_state().registers.htif.fromhost, val, "htif.fromhost"); } void do_write_memory_with_padding(uint64_t paddr, const unsigned char *data, uint64_t data_length, diff --git a/src/record-step-state-access.h b/src/record-step-state-access.h index e1cc987a5..861a2fd19 100644 --- a/src/record-step-state-access.h +++ b/src/record-step-state-access.h @@ -201,11 +201,11 @@ class record_step_state_access : // ----- friend i_prefer_shadow_state; - uint64_t do_read_shadow_state(shadow_state_what what) const { + uint64_t do_read_shadow_register(shadow_registers_what what) const { return log_read_reg(machine_reg_enum(what)); } - void do_write_shadow_state(shadow_state_what what, uint64_t val) const { + void do_write_shadow_register(shadow_registers_what what, uint64_t val) const { log_write_reg(machine_reg_enum(what), val); } @@ -264,7 +264,7 @@ class record_step_state_access : //??D This is still a bit too complicated for my taste template - host_addr do_read_tlb_vp_offset(uint64_t slot_index) const { + host_addr do_read_tlb_vf_offset(uint64_t slot_index) const { // During initialization, replay_step_state_access translates all vp_offset to corresponding vh_offset // At deinitialization, it translates them back // To do that, it needs the corresponding paddr_page = vaddr_page + vp_offset, and page data itself @@ -275,12 +275,11 @@ class record_step_state_access : // We still need to touch the page data // Writes to the TLB slot are atomic, so we know the values in a slot are ALWAYS internally consistent. // This means we can safely use all other fields to find paddr_page. - const auto vaddr_page = m_m.get_state().tlb.hot[SET][slot_index].vaddr_page; - const auto vh_offset = m_m.get_state().tlb.hot[SET][slot_index].vh_offset; + const auto vaddr_page = m_m.get_state().hot_tlb[SET][slot_index].vaddr_page; + const auto vh_offset = m_m.get_state().hot_tlb[SET][slot_index].vh_offset; if (vaddr_page != TLB_INVALID_PAGE) { - const auto pma_index = m_m.get_state().tlb.cold[SET][slot_index].pma_index; - const auto haddr_page = vaddr_page + vh_offset; - auto paddr_page = m_m.get_paddr(haddr_page, pma_index); + const auto vp_offset = m_m.get_state().shadow_tlb[SET][slot_index].vp_offset; + const auto paddr_page = vaddr_page + vp_offset; touch_page(paddr_page); } return vh_offset; @@ -299,12 +298,10 @@ class record_step_state_access : // We still need to touch the page data if (vaddr_page != TLB_INVALID_PAGE) { const auto haddr_page = vaddr_page + vh_offset; - auto paddr_page = m_m.get_paddr(haddr_page, pma_index); + const auto paddr_page = m_m.get_paddr(haddr_page, pma_index); touch_page(paddr_page); } - m_m.get_state().tlb.hot[SET][slot_index].vaddr_page = vaddr_page; - m_m.get_state().tlb.hot[SET][slot_index].vh_offset = vh_offset; - m_m.get_state().tlb.cold[SET][slot_index].pma_index = pma_index; + m_m.write_tlb(SET, slot_index, vaddr_page, vh_offset, pma_index); } fast_addr do_get_faddr(uint64_t paddr, uint64_t pma_index) const { diff --git a/src/replay-send-cmio-state-access.h b/src/replay-send-cmio-state-access.h index f22449b05..42688f143 100644 --- a/src/replay-send-cmio-state-access.h +++ b/src/replay-send-cmio-state-access.h @@ -34,7 +34,7 @@ #include "machine-merkle-tree.h" #include "meta.h" #include "riscv-constants.h" -#include "shadow-state.h" +#include "shadow-registers.h" #include "unique-c-ptr.h" namespace cartesi { diff --git a/src/replay-step-state-access.h b/src/replay-step-state-access.h index 853cccbb0..f7986d8a2 100644 --- a/src/replay-step-state-access.h +++ b/src/replay-step-state-access.h @@ -31,7 +31,7 @@ #include "pmas.h" #include "replay-step-state-access-interop.h" #include "riscv-constants.h" -#include "shadow-state.h" +#include "shadow-registers.h" #include "shadow-tlb.h" #include "shadow-uarch-state.h" #include "strict-aliasing.h" @@ -231,7 +231,7 @@ class replay_step_state_access : auto *vp_offset_log = try_find_page(vp_offset_field_addr & ~PAGE_OFFSET_MASK); const auto vaddr_page_field_addr = shadow_tlb_get_abs_addr(SET, slot_index, shadow_tlb_what::vaddr_page); auto *vaddr_page_log = try_find_page(vaddr_page_field_addr & ~PAGE_OFFSET_MASK); - // If vp_offset was accessed during record, both it and vaddr_apge will appear in the log + // If vp_offset was accessed during record, both it and vaddr_page will appear in the log // (record_step_state_access makes sure of it) // Otherwise, we do not need to translate if (vp_offset_log == nullptr || vaddr_page_log == nullptr) { @@ -268,7 +268,7 @@ class replay_step_state_access : auto *vp_offset_log = try_find_page(vp_offset_field_addr & ~PAGE_OFFSET_MASK); const auto vaddr_page_field_addr = shadow_tlb_get_abs_addr(SET, slot_index, shadow_tlb_what::vaddr_page); auto *vaddr_page_log = try_find_page(vaddr_page_field_addr & ~PAGE_OFFSET_MASK); - // If vp_offset was accessed during record, both it and vaddr_apge will appear in the log + // If vp_offset was accessed during record, both it and vaddr_page will appear in the log // (record_step_state_access makes sure of it) // Otherwise, we do not need to translate if (vp_offset_log == nullptr || vaddr_page_log == nullptr) { @@ -372,7 +372,7 @@ class replay_step_state_access : } //??D we should probably optimize access to the shadow so it doesn't perform a translation every time - // We can do this by caching the vh_offset trasnslation of the processor shadow page. This is easy if + // We can do this by caching the vh_offset translation of the registers shadow page. This is easy if // static_assert(sizeof(shadow_state) <= AR_PAGE_SIZE, "shadow state must fit in single page"); uint64_t check_read_reg(machine_reg reg) const { const auto haddr = do_get_faddr(machine_reg_address(reg)); @@ -380,7 +380,7 @@ class replay_step_state_access : } //??D we should probably optimize access to the shadow so it doesn't perform a translation every time - // We can do this by caching the vh_offset trasnslation of the processor shadow page. This is easy if + // We can do this by caching the vh_offset translation of the registers shadow page. This is easy if // static_assert(sizeof(shadow_state) <= AR_PAGE_SIZE, "shadow state must fit in single page"); void check_write_reg(machine_reg reg, uint64_t val) const { const auto haddr = do_get_faddr(machine_reg_address(reg)); @@ -402,11 +402,11 @@ class replay_step_state_access : // ----- friend i_prefer_shadow_state; - uint64_t do_read_shadow_state(shadow_state_what what) const { + uint64_t do_read_shadow_register(shadow_registers_what what) const { return check_read_reg(machine_reg_enum(what)); } - void do_write_shadow_state(shadow_state_what what, uint64_t val) const { + void do_write_shadow_register(shadow_registers_what what, uint64_t val) const { check_write_reg(machine_reg_enum(what), val); } @@ -481,7 +481,7 @@ class replay_step_state_access : } template - host_addr do_read_tlb_vp_offset(uint64_t slot_index) const { + host_addr do_read_tlb_vf_offset(uint64_t slot_index) const { return check_read_tlb(SET, slot_index, shadow_tlb_what::vp_offset); } diff --git a/src/riscv-constants.h b/src/riscv-constants.h index 91853ba30..1c7644585 100644 --- a/src/riscv-constants.h +++ b/src/riscv-constants.h @@ -20,6 +20,7 @@ #include #include "address-range-constants.h" +#include "htif-constants.h" #include "machine-c-version.h" #include "pmas-constants.h" @@ -441,32 +442,35 @@ enum CARTESI_init : uint64_t { MTVAL_INIT = UINT64_C(0), ///< Initial value for mtval MISA_INIT = (MISA_MXL_VALUE << MISA_MXL_SHIFT) | MISA_EXT_S_MASK | MISA_EXT_U_MASK | MISA_EXT_I_MASK | MISA_EXT_M_MASK | MISA_EXT_A_MASK | MISA_EXT_F_MASK | MISA_EXT_D_MASK | - MISA_EXT_C_MASK, ///< Initial value for misa - MIE_INIT = UINT64_C(0), ///< Initial value for mie - MIP_INIT = UINT64_C(0), ///< Initial value for mip - MEDELEG_INIT = UINT64_C(0), ///< Initial value for medeleg - MIDELEG_INIT = UINT64_C(0), ///< Initial value for mideleg - MCOUNTEREN_INIT = UINT64_C(0), ///< Initial value for mcounteren - STVEC_INIT = UINT64_C(0), ///< Initial value for stvec - SSCRATCH_INIT = UINT64_C(0), ///< Initial value for sscratch - SEPC_INIT = UINT64_C(0), ///< Initial value for sepc - SCAUSE_INIT = UINT64_C(0), ///< Initial value for scause - STVAL_INIT = UINT64_C(0), ///< Initial value for stval - SATP_INIT = UINT64_C(0), ///< Initial value for satp - SCOUNTEREN_INIT = UINT64_C(0), ///< Initial value for scounteren - ILRSC_INIT = UINT64_C(-1), ///< Initial value for ilrsc - IPRV_INIT = PRV_M, ///< Initial value for iprv - IFLAGS_X_INIT = UINT64_C(0), ///< Initial value for iflags_X - IFLAGS_Y_INIT = UINT64_C(0), ///< Initial value for iflags_Y - IFLAGS_H_INIT = UINT64_C(0), ///< Initial value for iflags_H - IUNREP_INIT = UINT64_C(0), ///< Initial value for iunrep - MTIMECMP_INIT = UINT64_C(0), ///< Initial value for mtimecmp - GIRQPEND_INIT = UINT64_C(0), ///< Initial value for girqpend - GIRQSRVD_INIT = UINT64_C(0), ///< Initial value for girqsrvd - FROMHOST_INIT = UINT64_C(0), ///< Initial value for fromhost - TOHOST_INIT = UINT64_C(0), ///< Initial value for tohost - MENVCFG_INIT = UINT64_C(0), ///< Initial value for menvcfg - SENVCFG_INIT = UINT64_C(0), ///< Initial value for senvcfg + MISA_EXT_C_MASK, ///< Initial value for misa + MIE_INIT = UINT64_C(0), ///< Initial value for mie + MIP_INIT = UINT64_C(0), ///< Initial value for mip + MEDELEG_INIT = UINT64_C(0), ///< Initial value for medeleg + MIDELEG_INIT = UINT64_C(0), ///< Initial value for mideleg + MCOUNTEREN_INIT = UINT64_C(0), ///< Initial value for mcounteren + STVEC_INIT = UINT64_C(0), ///< Initial value for stvec + SSCRATCH_INIT = UINT64_C(0), ///< Initial value for sscratch + SEPC_INIT = UINT64_C(0), ///< Initial value for sepc + SCAUSE_INIT = UINT64_C(0), ///< Initial value for scause + STVAL_INIT = UINT64_C(0), ///< Initial value for stval + SATP_INIT = UINT64_C(0), ///< Initial value for satp + SCOUNTEREN_INIT = UINT64_C(0), ///< Initial value for scounteren + ILRSC_INIT = UINT64_C(-1), ///< Initial value for ilrsc + IPRV_INIT = PRV_M, ///< Initial value for iprv + IFLAGS_X_INIT = UINT64_C(0), ///< Initial value for iflags_X + IFLAGS_Y_INIT = UINT64_C(0), ///< Initial value for iflags_Y + IFLAGS_H_INIT = UINT64_C(0), ///< Initial value for iflags_H + IUNREP_INIT = UINT64_C(0), ///< Initial value for iunrep + MTIMECMP_INIT = UINT64_C(0), ///< Initial value for mtimecmp + GIRQPEND_INIT = UINT64_C(0), ///< Initial value for girqpend + GIRQSRVD_INIT = UINT64_C(0), ///< Initial value for girqsrvd + FROMHOST_INIT = UINT64_C(0), ///< Initial value for fromhost + TOHOST_INIT = UINT64_C(0), ///< Initial value for tohost + MENVCFG_INIT = UINT64_C(0), ///< Initial value for menvcfg + SENVCFG_INIT = UINT64_C(0), ///< Initial value for senvcfg + IHALT_INIT = HTIF_HALT_CMD_HALT_MASK, ///< Initial value for ihalt + ICONSOLE_INIT = HTIF_CONSOLE_CMD_PUTCHAR_MASK, ///< Initial value for iconsole + IYIELD_INIT = HTIF_YIELD_CMD_MANUAL_MASK | HTIF_YIELD_CMD_AUTOMATIC_MASK, ///< Initial value for iyield UARCH_HALT_FLAG_INIT = UINT64_C(0), ///< Initial value for microarchitecture halt flag UARCH_X_INIT = UINT64_C(0), ///< Initial value for microarchitecture general purpose register x UARCH_PC_INIT = AR_UARCH_RAM_START, ///< Initial value for microarchitecture pc diff --git a/src/scope-exit.h b/src/scope-exit.h new file mode 100644 index 000000000..6d2fc196c --- /dev/null +++ b/src/scope-exit.h @@ -0,0 +1,102 @@ +// Copyright Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: LGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify it under +// the terms of the GNU Lesser General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) any +// later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +// PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License along +// with this program (see COPYING). If not, see . +// + +#ifndef SCOPE_EXIT_H +#define SCOPE_EXIT_H + +//??E An official scoped_exit utility is supposed to become part of C++ in the future, +// when that happens, we could potentially replace this implementation. + +#include +#include + +namespace cartesi { + +template +class scope_exit { +public: + explicit scope_exit(F &&f) noexcept : m_func(std::move(f)) {} + + /// \brief Calls the exit function when the scope is exited when active, then destroys the scope_exit. + ~scope_exit() { + exit(); + } + + scope_exit(const scope_exit &) = delete; + scope_exit &operator=(const scope_exit &) = delete; + scope_exit(scope_exit &&) = delete; + scope_exit &operator=(scope_exit &&) = delete; + + /// \brief Invokes the exit function if active and then becomes inactive. + void exit() { + if (m_active) { + m_active = false; + m_func(); + } + } + + /// \brief Makes the scope_exit inactive. + void release() noexcept { + m_active = false; + } + +private: + F m_func; + bool m_active{true}; +}; + +template +scope_exit make_scope_exit(F &&f) { + return scope_exit(std::forward(f)); +} + +template +class scope_fail { +public: + explicit scope_fail(F &&f) noexcept : m_func(std::move(f)), m_uncaught_on_entry(std::uncaught_exceptions()) {} + + /// \brief Calls the fail function when the scope is exited after an exception is thrown while active, + /// then destroys the scope_fail. + ~scope_fail() noexcept { + if (m_active && std::uncaught_exceptions() > m_uncaught_on_entry) { + m_func(); + } + } + + scope_fail(const scope_fail &) = delete; + scope_fail &operator=(const scope_fail &) = delete; + scope_fail(scope_fail &&) = delete; + scope_fail &operator=(scope_fail &&) = delete; + + /// \brief Makes the scope_fail inactive + void release() noexcept { + m_active = false; + } + +private: + F m_func; + int m_uncaught_on_entry; + bool m_active{true}; +}; + +template +scope_fail make_scope_fail(F &&f) { + return scope_fail(std::forward(f)); +} + +}; // namespace cartesi + +#endif diff --git a/src/shadow-peek.h b/src/shadow-peek.h deleted file mode 100644 index 8d094bc80..000000000 --- a/src/shadow-peek.h +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright Cartesi and individual authors (see AUTHORS) -// SPDX-License-Identifier: LGPL-3.0-or-later -// -// This program is free software: you can redistribute it and/or modify it under -// the terms of the GNU Lesser General Public License as published by the Free -// Software Foundation, either version 3 of the License, or (at your option) any -// later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT ANY -// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A -// PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License along -// with this program (see COPYING). If not, see . -// - -#ifndef SHADOW_PEEK_H -#define SHADOW_PEEK_H - -#include - -namespace cartesi { - -// Forward declarations -class machine; -class address_range; - -/// \file -/// \brief Peeks part of a shadow -template -static bool shadow_peek(READ_REG_F read_reg, const address_range &ar, const machine &m, uint64_t offset, - uint64_t length, const unsigned char **data, unsigned char *scratch) { - // Must be inside range - if (!ar.contains_relative(offset, length)) { - *data = nullptr; - return false; - } - // Initial, potentially partial register read - const uint64_t offset_aligned = offset & ~(sizeof(uint64_t) - 1); - if (offset > offset_aligned) { - const auto val = read_reg(m, ar.get_start() + offset_aligned); - const auto partial = offset - offset_aligned; - memcpy(scratch, &val, partial); - length -= partial; - offset += partial; - } - // Now we are aligned, do all complete registers - const uint64_t paddr_start = ar.get_start() + offset; - const uint64_t length_aligned = length & ~(sizeof(uint64_t) - 1); - const uint64_t paddr_end = paddr_start + length_aligned; - for (uint64_t paddr = paddr_start; paddr < paddr_end; paddr += sizeof(uint64_t)) { - const auto val = read_reg(m, paddr); - aliased_aligned_write(scratch + (paddr - paddr_start), val); - } - // Final, potentially partial register read - if (length > length_aligned) { - const auto partial = length - length_aligned; - const auto val = read_reg(m, paddr_end); - memcpy(scratch + length_aligned, &val, partial); - } - *data = scratch; - return true; -} - -} // namespace cartesi - -#endif diff --git a/src/shadow-registers.h b/src/shadow-registers.h new file mode 100644 index 000000000..807c7dcf3 --- /dev/null +++ b/src/shadow-registers.h @@ -0,0 +1,470 @@ +// Copyright Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: LGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify it under +// the terms of the GNU Lesser General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) any +// later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +// PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License along +// with this program (see COPYING). If not, see . +// + +#ifndef SHADOW_REGISTERS_STATE_H +#define SHADOW_REGISTERS_STATE_H + +#include +#include + +#include "address-range.h" +#include "compiler-defines.h" +#include "pmas-constants.h" +#include "riscv-constants.h" + +/// \file +/// \brief Shadow state definitions. + +namespace cartesi { + +/// \brief Internal flags state (Cartesi specific). +struct iflags_state final { + uint64_t X{IFLAGS_X_INIT}; ///< CPU has yielded with automatic reset. + uint64_t Y{IFLAGS_Y_INIT}; ///< CPU has yielded with manual reset. + uint64_t H{IFLAGS_H_INIT}; ///< CPU has been permanently halted. +}; + +/// \brief CLINT (Core-Local Interruptor) state +struct clint_state final { + uint64_t mtimecmp{MTIMECMP_INIT}; ///< CSR mtimecmp. +}; + +/// \brief PLIC (Platform-Level Interrupt Controller) state +struct plic_state final { + uint64_t girqpend{GIRQPEND_INIT}; ///< CSR girqpend (global interrupts pending). + uint64_t girqsrvd{GIRQSRVD_INIT}; ///< CSR girqsrvd (global interrupts served). +}; + +/// HTIF (Host-Target config InterFace) state +struct htif_state final { + uint64_t tohost{TOHOST_INIT}; ///< CSR tohost. + uint64_t fromhost{FROMHOST_INIT}; ///< CSR fromhost. + uint64_t ihalt{IHALT_INIT}; ///< CSR ihalt (Cartesi-specific). + uint64_t iconsole{ICONSOLE_INIT}; ///< CSR iconsole (Cartesi-specific). + uint64_t iyield{IYIELD_INIT}; ///< CSR iyield (Cartesi-specific). +}; + +/// \brief Machine registers state +struct registers_state final { + // The X registers are the very first to optimize access of registers in the interpreter. + uint64_t x[X_REG_COUNT]{REG_X0, REG_X1, REG_X2, REG_X3, REG_X4, REG_X5, REG_X6, REG_X7, REG_X8, REG_X9, REG_X10, + REG_X11, REG_X12, REG_X13, REG_X14, REG_X15, REG_X16, REG_X17, REG_X18, REG_X19, REG_X20, REG_X21, REG_X22, + REG_X23, REG_X24, REG_X25, REG_X26, REG_X27, REG_X28, REG_X29, REG_X30, REG_X31}; ///< Register file. + // The following registers are carefully ordered to have better data locality in the interpreter loop. + uint64_t mcycle{MCYCLE_INIT}; ///< CSR mcycle. + uint64_t pc{PC_INIT}; ///< Program counter. + uint64_t fcsr{FCSR_INIT}; ///< CSR fcsr. + uint64_t f[F_REG_COUNT]{}; ///< Floating-point register file. + uint64_t iprv{IPRV_INIT}; ///< Privilege level (Cartesi-specific). + + // RISC-V machine CSRs + uint64_t mstatus{MSTATUS_INIT}; ///< CSR mstatus. + uint64_t mtvec{MTVEC_INIT}; ///< CSR mtvec. + uint64_t mscratch{MSCRATCH_INIT}; ///< CSR mscratch. + uint64_t mepc{MEPC_INIT}; ///< CSR mepc. + uint64_t mcause{MCAUSE_INIT}; ///< CSR mcause. + uint64_t mtval{MTVAL_INIT}; ///< CSR mtval. + uint64_t misa{MISA_INIT}; ///< CSR misa. + uint64_t mie{MIE_INIT}; ///< CSR mie. + uint64_t mip{MIP_INIT}; ///< CSR mip. + uint64_t medeleg{MEDELEG_INIT}; ///< CSR medeleg. + uint64_t mideleg{MIDELEG_INIT}; ///< CSR mideleg. + uint64_t mcounteren{MCOUNTEREN_INIT}; ///< CSR mcounteren. + uint64_t menvcfg{MENVCFG_INIT}; ///< CSR menvcfg. + uint64_t mvendorid{MVENDORID_INIT}; ///< CSR mvendorid. + uint64_t marchid{MARCHID_INIT}; ///< CSR marchid. + uint64_t mimpid{MIMPID_INIT}; ///< CSR mimpid. + + // RISC-V supervisor CSRs + uint64_t stvec{STVEC_INIT}; ///< CSR stvec. + uint64_t sscratch{SSCRATCH_INIT}; ///< CSR sscratch. + uint64_t sepc{SEPC_INIT}; ///< CSR sepc. + uint64_t scause{SCAUSE_INIT}; ///< CSR scause. + uint64_t stval{STVAL_INIT}; ///< CSR stval. + uint64_t satp{SATP_INIT}; ///< CSR satp. + uint64_t scounteren{SCOUNTEREN_INIT}; ///< CSR scounteren. + uint64_t senvcfg{SENVCFG_INIT}; ///< CSR senvcfg. + + // Cartesi-specific state + uint64_t ilrsc{ILRSC_INIT}; ///< For LR/SC instructions. + uint64_t icycleinstret{ICYCLEINSTRET_INIT}; ///< Difference between mcycle and minstret. + uint64_t iunrep{IUNREP_INIT}; ///< Unreproducible mode. + + iflags_state iflags; ///< Internal flags (Cartesi specific). + clint_state clint; ///< CLINT registers. + plic_state plic; ///< PLIC registers. + htif_state htif; ///< HTIF registers. +}; + +/// \brief Shadow memory layout +using shadow_registers_state = registers_state; + +// We need strong guarantees that shadow_state has fixed size and alignment across platforms. +static_assert(sizeof(shadow_registers_state) == 106 * sizeof(uint64_t), "unexpected registers state size"); +static_assert(alignof(shadow_registers_state) == sizeof(uint64_t), "unexpected registers state alignment"); + +enum class shadow_registers_what : uint64_t { + x0 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, x[0]), + x1 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, x[1]), + x2 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, x[2]), + x3 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, x[3]), + x4 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, x[4]), + x5 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, x[5]), + x6 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, x[6]), + x7 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, x[7]), + x8 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, x[8]), + x9 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, x[9]), + x10 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, x[10]), + x11 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, x[11]), + x12 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, x[12]), + x13 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, x[13]), + x14 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, x[14]), + x15 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, x[15]), + x16 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, x[16]), + x17 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, x[17]), + x18 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, x[18]), + x19 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, x[19]), + x20 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, x[20]), + x21 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, x[21]), + x22 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, x[22]), + x23 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, x[23]), + x24 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, x[24]), + x25 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, x[25]), + x26 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, x[26]), + x27 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, x[27]), + x28 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, x[28]), + x29 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, x[29]), + x30 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, x[30]), + x31 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, x[31]), + f0 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, f[0]), + f1 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, f[1]), + f2 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, f[2]), + f3 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, f[3]), + f4 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, f[4]), + f5 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, f[5]), + f6 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, f[6]), + f7 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, f[7]), + f8 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, f[8]), + f9 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, f[9]), + f10 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, f[10]), + f11 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, f[11]), + f12 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, f[12]), + f13 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, f[13]), + f14 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, f[14]), + f15 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, f[15]), + f16 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, f[16]), + f17 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, f[17]), + f18 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, f[18]), + f19 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, f[19]), + f20 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, f[20]), + f21 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, f[21]), + f22 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, f[22]), + f23 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, f[23]), + f24 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, f[24]), + f25 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, f[25]), + f26 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, f[26]), + f27 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, f[27]), + f28 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, f[28]), + f29 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, f[29]), + f30 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, f[30]), + f31 = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, f[31]), + pc = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, pc), + fcsr = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, fcsr), + mvendorid = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, mvendorid), + marchid = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, marchid), + mimpid = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, mimpid), + mcycle = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, mcycle), + icycleinstret = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, icycleinstret), + mstatus = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, mstatus), + mtvec = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, mtvec), + mscratch = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, mscratch), + mepc = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, mepc), + mcause = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, mcause), + mtval = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, mtval), + misa = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, misa), + mie = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, mie), + mip = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, mip), + medeleg = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, medeleg), + mideleg = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, mideleg), + mcounteren = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, mcounteren), + menvcfg = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, menvcfg), + stvec = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, stvec), + sscratch = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, sscratch), + sepc = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, sepc), + scause = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, scause), + stval = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, stval), + satp = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, satp), + scounteren = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, scounteren), + senvcfg = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, senvcfg), + ilrsc = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, ilrsc), + iprv = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, iprv), + iflags_X = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, iflags.X), + iflags_Y = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, iflags.Y), + iflags_H = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, iflags.H), + iunrep = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, iunrep), + clint_mtimecmp = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, clint.mtimecmp), + plic_girqpend = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, plic.girqpend), + plic_girqsrvd = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, plic.girqsrvd), + htif_tohost = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, htif.tohost), + htif_fromhost = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, htif.fromhost), + htif_ihalt = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, htif.ihalt), + htif_iconsole = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, htif.iconsole), + htif_iyield = AR_SHADOW_REGISTERS_START + offsetof(shadow_registers_state, htif.iyield), + unknown_ = UINT64_C(1) << 63, // Outside of RISC-V address space +}; + +static constexpr shadow_registers_what shadow_registers_get_what(uint64_t paddr) { + if (paddr < AR_SHADOW_REGISTERS_START || paddr - AR_SHADOW_REGISTERS_START >= sizeof(shadow_registers_state) || + (paddr & (sizeof(uint64_t) - 1)) != 0) { + return shadow_registers_what::unknown_; + } + // NOLINTNEXTLINE(clang-analyzer-optin.core.EnumCastOutOfRange) + return static_cast(paddr); +} + +static constexpr shadow_registers_what shadow_registers_get_what(shadow_registers_what what, int i) { + return static_cast(static_cast(what) + (i * sizeof(uint64_t))); +} + +static constexpr const char *shadow_registers_get_what_name(shadow_registers_what what) { + const auto paddr = static_cast(what); + if (paddr < AR_SHADOW_REGISTERS_START || paddr - AR_SHADOW_REGISTERS_START >= sizeof(shadow_registers_state) || + (paddr & (sizeof(uint64_t) - 1)) != 0) { + return "state.unknown_"; + } + using reg = shadow_registers_what; + switch (what) { + case reg::x0: + return "x0"; + case reg::x1: + return "x1"; + case reg::x2: + return "x2"; + case reg::x3: + return "x3"; + case reg::x4: + return "x4"; + case reg::x5: + return "x5"; + case reg::x6: + return "x6"; + case reg::x7: + return "x7"; + case reg::x8: + return "x8"; + case reg::x9: + return "x9"; + case reg::x10: + return "x10"; + case reg::x11: + return "x11"; + case reg::x12: + return "x12"; + case reg::x13: + return "x13"; + case reg::x14: + return "x14"; + case reg::x15: + return "x15"; + case reg::x16: + return "x16"; + case reg::x17: + return "x17"; + case reg::x18: + return "x18"; + case reg::x19: + return "x19"; + case reg::x20: + return "x20"; + case reg::x21: + return "x21"; + case reg::x22: + return "x22"; + case reg::x23: + return "x23"; + case reg::x24: + return "x24"; + case reg::x25: + return "x25"; + case reg::x26: + return "x26"; + case reg::x27: + return "x27"; + case reg::x28: + return "x28"; + case reg::x29: + return "x29"; + case reg::x30: + return "x30"; + case reg::x31: + return "x31"; + case reg::f0: + return "f0"; + case reg::f1: + return "f1"; + case reg::f2: + return "f2"; + case reg::f3: + return "f3"; + case reg::f4: + return "f4"; + case reg::f5: + return "f5"; + case reg::f6: + return "f6"; + case reg::f7: + return "f7"; + case reg::f8: + return "f8"; + case reg::f9: + return "f9"; + case reg::f10: + return "f10"; + case reg::f11: + return "f11"; + case reg::f12: + return "f12"; + case reg::f13: + return "f13"; + case reg::f14: + return "f14"; + case reg::f15: + return "f15"; + case reg::f16: + return "f16"; + case reg::f17: + return "f17"; + case reg::f18: + return "f18"; + case reg::f19: + return "f19"; + case reg::f20: + return "f20"; + case reg::f21: + return "f21"; + case reg::f22: + return "f22"; + case reg::f23: + return "f23"; + case reg::f24: + return "f24"; + case reg::f25: + return "f25"; + case reg::f26: + return "f26"; + case reg::f27: + return "f27"; + case reg::f28: + return "f28"; + case reg::f29: + return "f29"; + case reg::f30: + return "f30"; + case reg::f31: + return "f31"; + case reg::pc: + return "pc"; + case reg::fcsr: + return "fcsr"; + case reg::mvendorid: + return "mvendorid"; + case reg::marchid: + return "marchid"; + case reg::mimpid: + return "mimpid"; + case reg::mcycle: + return "mcycle"; + case reg::icycleinstret: + return "icycleinstret"; + case reg::mstatus: + return "mstatus"; + case reg::mtvec: + return "mtvec"; + case reg::mscratch: + return "mscratch"; + case reg::mepc: + return "mepc"; + case reg::mcause: + return "mcause"; + case reg::mtval: + return "mtval"; + case reg::misa: + return "misa"; + case reg::mie: + return "mie"; + case reg::mip: + return "mip"; + case reg::medeleg: + return "medeleg"; + case reg::mideleg: + return "mideleg"; + case reg::mcounteren: + return "mcounteren"; + case reg::menvcfg: + return "menvcfg"; + case reg::stvec: + return "stvec"; + case reg::sscratch: + return "sscratch"; + case reg::sepc: + return "sepc"; + case reg::scause: + return "scause"; + case reg::stval: + return "stval"; + case reg::satp: + return "satp"; + case reg::scounteren: + return "scounteren"; + case reg::senvcfg: + return "senvcfg"; + case reg::ilrsc: + return "ilrsc"; + case reg::iprv: + return "iprv"; + case reg::iflags_X: + return "iflags.X"; + case reg::iflags_Y: + return "iflags.Y"; + case reg::iflags_H: + return "iflags.H"; + case reg::iunrep: + return "iunrep"; + case reg::clint_mtimecmp: + return "clint.mtimecmp"; + case reg::plic_girqpend: + return "plic.girqpend"; + case reg::plic_girqsrvd: + return "plic.girqsrvd"; + case reg::htif_tohost: + return "htif.tohost"; + case reg::htif_fromhost: + return "htif.fromhost"; + case reg::htif_ihalt: + return "htif.ihalt"; + case reg::htif_iconsole: + return "htif.iconsole"; + case reg::htif_iyield: + return "htif.iyield"; + case reg::unknown_: + return "state.unknown_"; + } + return "state.unknown_"; +} + +} // namespace cartesi + +#endif diff --git a/src/shadow-state-address-range.cpp b/src/shadow-state-address-range.cpp deleted file mode 100644 index d2944946c..000000000 --- a/src/shadow-state-address-range.cpp +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright Cartesi and individual authors (see AUTHORS) -// SPDX-License-Identifier: LGPL-3.0-or-later -// -// This program is free software: you can redistribute it and/or modify it under -// the terms of the GNU Lesser General Public License as published by the Free -// Software Foundation, either version 3 of the License, or (at your option) any -// later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT ANY -// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A -// PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License along -// with this program (see COPYING). If not, see . -// - -#include "shadow-state-address-range.h" - -#include - -#include "machine-reg.h" -#include "machine.h" -#include "shadow-peek.h" - -namespace cartesi { - -bool shadow_state_address_range::do_peek(const machine &m, uint64_t offset, uint64_t length, const unsigned char **data, - unsigned char *scratch) const noexcept { - // If past useful range - if (offset >= sizeof(shadow_state)) { - *data = nullptr; - return contains_relative(offset, length); - } - // Otherwise, copy and return register data - return shadow_peek( - [](const machine &m, uint64_t paddr) { - auto reg = shadow_state_get_what(paddr); - uint64_t val = 0; - if (reg != shadow_state_what::unknown_) { - val = m.read_reg(machine_reg_enum(reg)); - } - return val; - }, - *this, m, offset, length, data, scratch); -} - -} // namespace cartesi diff --git a/src/shadow-state-address-range.h b/src/shadow-state-address-range.h deleted file mode 100644 index d11e7e996..000000000 --- a/src/shadow-state-address-range.h +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright Cartesi and individual authors (see AUTHORS) -// SPDX-License-Identifier: LGPL-3.0-or-later -// -// This program is free software: you can redistribute it and/or modify it under -// the terms of the GNU Lesser General Public License as published by the Free -// Software Foundation, either version 3 of the License, or (at your option) any -// later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT ANY -// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A -// PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License along -// with this program (see COPYING). If not, see . -// - -#ifndef SHADOW_STATE_ADDRESS_RANGE_H -#define SHADOW_STATE_ADDRESS_RANGE_H - -#include -#include - -#include "address-range-constants.h" -#include "address-range.h" -#include "shadow-state.h" - -/// \file -/// \brief Shadow state address range. - -namespace cartesi { - -class shadow_state_address_range final : public address_range { - - static constexpr pmas_flags m_shadow_state_flags{ - .M = false, - .IO = false, - .R = false, - .W = false, - .X = false, - .IR = false, - .IW = false, - .DID = PMA_ISTART_DID::shadow_state, - }; - -public: - template - explicit shadow_state_address_range(ABRT abrt) : - address_range("shadow state", AR_SHADOW_STATE_START, AR_SHADOW_STATE_LENGTH, m_shadow_state_flags, abrt) { - ; - } - - shadow_state_address_range(const shadow_state_address_range &) = default; - shadow_state_address_range &operator=(const shadow_state_address_range &) = default; - shadow_state_address_range(shadow_state_address_range &&) noexcept = default; - shadow_state_address_range &operator=(shadow_state_address_range &&) noexcept = default; - ~shadow_state_address_range() override = default; - -private: -#ifndef MICROARCHITECTURE - bool do_peek(const machine &m, uint64_t offset, uint64_t length, const unsigned char **data, - unsigned char *scratch) const noexcept override; -#endif -}; - -template -static inline shadow_state_address_range make_shadow_state_address_range(ABRT abrt) { - return shadow_state_address_range{abrt}; -} - -} // namespace cartesi - -#endif diff --git a/src/shadow-state.h b/src/shadow-state.h deleted file mode 100644 index 57ebad79c..000000000 --- a/src/shadow-state.h +++ /dev/null @@ -1,432 +0,0 @@ -// Copyright Cartesi and individual authors (see AUTHORS) -// SPDX-License-Identifier: LGPL-3.0-or-later -// -// This program is free software: you can redistribute it and/or modify it under -// the terms of the GNU Lesser General Public License as published by the Free -// Software Foundation, either version 3 of the License, or (at your option) any -// later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT ANY -// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A -// PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License along -// with this program (see COPYING). If not, see . -// - -#ifndef SHADOW_STATE_H -#define SHADOW_STATE_H - -#include -#include - -#include "address-range.h" -#include "compiler-defines.h" -#include "pmas-constants.h" -#include "riscv-constants.h" - -/// \file -/// \brief Shadow state definitions. - -namespace cartesi { - -/// \brief Shadow memory layout -struct PACKED shadow_state { - uint64_t x[X_REG_COUNT]; ///< Register file. - uint64_t f[F_REG_COUNT]; ///< Floating-point register file. - uint64_t pc; - uint64_t fcsr; - uint64_t mvendorid; - uint64_t marchid; - uint64_t mimpid; - uint64_t mcycle; - uint64_t icycleinstret; - uint64_t mstatus; - uint64_t mtvec; - uint64_t mscratch; - uint64_t mepc; - uint64_t mcause; - uint64_t mtval; - uint64_t misa; - uint64_t mie; - uint64_t mip; - uint64_t medeleg; - uint64_t mideleg; - uint64_t mcounteren; - uint64_t menvcfg; - uint64_t stvec; - uint64_t sscratch; - uint64_t sepc; - uint64_t scause; - uint64_t stval; - uint64_t satp; - uint64_t scounteren; - uint64_t senvcfg; - uint64_t ilrsc; - uint64_t iprv; - uint64_t iflags_X; - uint64_t iflags_Y; - uint64_t iflags_H; - uint64_t iunrep; - uint64_t clint_mtimecmp; - uint64_t plic_girqpend; - uint64_t plic_girqsrvd; - uint64_t htif_tohost; - uint64_t htif_fromhost; - uint64_t htif_ihalt; - uint64_t htif_iconsole; - uint64_t htif_iyield; -}; - -enum class shadow_state_what : uint64_t { - x0 = AR_SHADOW_STATE_START + offsetof(shadow_state, x[0]), - x1 = AR_SHADOW_STATE_START + offsetof(shadow_state, x[1]), - x2 = AR_SHADOW_STATE_START + offsetof(shadow_state, x[2]), - x3 = AR_SHADOW_STATE_START + offsetof(shadow_state, x[3]), - x4 = AR_SHADOW_STATE_START + offsetof(shadow_state, x[4]), - x5 = AR_SHADOW_STATE_START + offsetof(shadow_state, x[5]), - x6 = AR_SHADOW_STATE_START + offsetof(shadow_state, x[6]), - x7 = AR_SHADOW_STATE_START + offsetof(shadow_state, x[7]), - x8 = AR_SHADOW_STATE_START + offsetof(shadow_state, x[8]), - x9 = AR_SHADOW_STATE_START + offsetof(shadow_state, x[9]), - x10 = AR_SHADOW_STATE_START + offsetof(shadow_state, x[10]), - x11 = AR_SHADOW_STATE_START + offsetof(shadow_state, x[11]), - x12 = AR_SHADOW_STATE_START + offsetof(shadow_state, x[12]), - x13 = AR_SHADOW_STATE_START + offsetof(shadow_state, x[13]), - x14 = AR_SHADOW_STATE_START + offsetof(shadow_state, x[14]), - x15 = AR_SHADOW_STATE_START + offsetof(shadow_state, x[15]), - x16 = AR_SHADOW_STATE_START + offsetof(shadow_state, x[16]), - x17 = AR_SHADOW_STATE_START + offsetof(shadow_state, x[17]), - x18 = AR_SHADOW_STATE_START + offsetof(shadow_state, x[18]), - x19 = AR_SHADOW_STATE_START + offsetof(shadow_state, x[19]), - x20 = AR_SHADOW_STATE_START + offsetof(shadow_state, x[20]), - x21 = AR_SHADOW_STATE_START + offsetof(shadow_state, x[21]), - x22 = AR_SHADOW_STATE_START + offsetof(shadow_state, x[22]), - x23 = AR_SHADOW_STATE_START + offsetof(shadow_state, x[23]), - x24 = AR_SHADOW_STATE_START + offsetof(shadow_state, x[24]), - x25 = AR_SHADOW_STATE_START + offsetof(shadow_state, x[25]), - x26 = AR_SHADOW_STATE_START + offsetof(shadow_state, x[26]), - x27 = AR_SHADOW_STATE_START + offsetof(shadow_state, x[27]), - x28 = AR_SHADOW_STATE_START + offsetof(shadow_state, x[28]), - x29 = AR_SHADOW_STATE_START + offsetof(shadow_state, x[29]), - x30 = AR_SHADOW_STATE_START + offsetof(shadow_state, x[30]), - x31 = AR_SHADOW_STATE_START + offsetof(shadow_state, x[31]), - f0 = AR_SHADOW_STATE_START + offsetof(shadow_state, f[0]), - f1 = AR_SHADOW_STATE_START + offsetof(shadow_state, f[1]), - f2 = AR_SHADOW_STATE_START + offsetof(shadow_state, f[2]), - f3 = AR_SHADOW_STATE_START + offsetof(shadow_state, f[3]), - f4 = AR_SHADOW_STATE_START + offsetof(shadow_state, f[4]), - f5 = AR_SHADOW_STATE_START + offsetof(shadow_state, f[5]), - f6 = AR_SHADOW_STATE_START + offsetof(shadow_state, f[6]), - f7 = AR_SHADOW_STATE_START + offsetof(shadow_state, f[7]), - f8 = AR_SHADOW_STATE_START + offsetof(shadow_state, f[8]), - f9 = AR_SHADOW_STATE_START + offsetof(shadow_state, f[9]), - f10 = AR_SHADOW_STATE_START + offsetof(shadow_state, f[10]), - f11 = AR_SHADOW_STATE_START + offsetof(shadow_state, f[11]), - f12 = AR_SHADOW_STATE_START + offsetof(shadow_state, f[12]), - f13 = AR_SHADOW_STATE_START + offsetof(shadow_state, f[13]), - f14 = AR_SHADOW_STATE_START + offsetof(shadow_state, f[14]), - f15 = AR_SHADOW_STATE_START + offsetof(shadow_state, f[15]), - f16 = AR_SHADOW_STATE_START + offsetof(shadow_state, f[16]), - f17 = AR_SHADOW_STATE_START + offsetof(shadow_state, f[17]), - f18 = AR_SHADOW_STATE_START + offsetof(shadow_state, f[18]), - f19 = AR_SHADOW_STATE_START + offsetof(shadow_state, f[19]), - f20 = AR_SHADOW_STATE_START + offsetof(shadow_state, f[20]), - f21 = AR_SHADOW_STATE_START + offsetof(shadow_state, f[21]), - f22 = AR_SHADOW_STATE_START + offsetof(shadow_state, f[22]), - f23 = AR_SHADOW_STATE_START + offsetof(shadow_state, f[23]), - f24 = AR_SHADOW_STATE_START + offsetof(shadow_state, f[24]), - f25 = AR_SHADOW_STATE_START + offsetof(shadow_state, f[25]), - f26 = AR_SHADOW_STATE_START + offsetof(shadow_state, f[26]), - f27 = AR_SHADOW_STATE_START + offsetof(shadow_state, f[27]), - f28 = AR_SHADOW_STATE_START + offsetof(shadow_state, f[28]), - f29 = AR_SHADOW_STATE_START + offsetof(shadow_state, f[29]), - f30 = AR_SHADOW_STATE_START + offsetof(shadow_state, f[30]), - f31 = AR_SHADOW_STATE_START + offsetof(shadow_state, f[31]), - pc = AR_SHADOW_STATE_START + offsetof(shadow_state, pc), - fcsr = AR_SHADOW_STATE_START + offsetof(shadow_state, fcsr), - mvendorid = AR_SHADOW_STATE_START + offsetof(shadow_state, mvendorid), - marchid = AR_SHADOW_STATE_START + offsetof(shadow_state, marchid), - mimpid = AR_SHADOW_STATE_START + offsetof(shadow_state, mimpid), - mcycle = AR_SHADOW_STATE_START + offsetof(shadow_state, mcycle), - icycleinstret = AR_SHADOW_STATE_START + offsetof(shadow_state, icycleinstret), - mstatus = AR_SHADOW_STATE_START + offsetof(shadow_state, mstatus), - mtvec = AR_SHADOW_STATE_START + offsetof(shadow_state, mtvec), - mscratch = AR_SHADOW_STATE_START + offsetof(shadow_state, mscratch), - mepc = AR_SHADOW_STATE_START + offsetof(shadow_state, mepc), - mcause = AR_SHADOW_STATE_START + offsetof(shadow_state, mcause), - mtval = AR_SHADOW_STATE_START + offsetof(shadow_state, mtval), - misa = AR_SHADOW_STATE_START + offsetof(shadow_state, misa), - mie = AR_SHADOW_STATE_START + offsetof(shadow_state, mie), - mip = AR_SHADOW_STATE_START + offsetof(shadow_state, mip), - medeleg = AR_SHADOW_STATE_START + offsetof(shadow_state, medeleg), - mideleg = AR_SHADOW_STATE_START + offsetof(shadow_state, mideleg), - mcounteren = AR_SHADOW_STATE_START + offsetof(shadow_state, mcounteren), - menvcfg = AR_SHADOW_STATE_START + offsetof(shadow_state, menvcfg), - stvec = AR_SHADOW_STATE_START + offsetof(shadow_state, stvec), - sscratch = AR_SHADOW_STATE_START + offsetof(shadow_state, sscratch), - sepc = AR_SHADOW_STATE_START + offsetof(shadow_state, sepc), - scause = AR_SHADOW_STATE_START + offsetof(shadow_state, scause), - stval = AR_SHADOW_STATE_START + offsetof(shadow_state, stval), - satp = AR_SHADOW_STATE_START + offsetof(shadow_state, satp), - scounteren = AR_SHADOW_STATE_START + offsetof(shadow_state, scounteren), - senvcfg = AR_SHADOW_STATE_START + offsetof(shadow_state, senvcfg), - ilrsc = AR_SHADOW_STATE_START + offsetof(shadow_state, ilrsc), - iprv = AR_SHADOW_STATE_START + offsetof(shadow_state, iprv), - iflags_X = AR_SHADOW_STATE_START + offsetof(shadow_state, iflags_X), - iflags_Y = AR_SHADOW_STATE_START + offsetof(shadow_state, iflags_Y), - iflags_H = AR_SHADOW_STATE_START + offsetof(shadow_state, iflags_H), - iunrep = AR_SHADOW_STATE_START + offsetof(shadow_state, iunrep), - clint_mtimecmp = AR_SHADOW_STATE_START + offsetof(shadow_state, clint_mtimecmp), - plic_girqpend = AR_SHADOW_STATE_START + offsetof(shadow_state, plic_girqpend), - plic_girqsrvd = AR_SHADOW_STATE_START + offsetof(shadow_state, plic_girqsrvd), - htif_tohost = AR_SHADOW_STATE_START + offsetof(shadow_state, htif_tohost), - htif_fromhost = AR_SHADOW_STATE_START + offsetof(shadow_state, htif_fromhost), - htif_ihalt = AR_SHADOW_STATE_START + offsetof(shadow_state, htif_ihalt), - htif_iconsole = AR_SHADOW_STATE_START + offsetof(shadow_state, htif_iconsole), - htif_iyield = AR_SHADOW_STATE_START + offsetof(shadow_state, htif_iyield), - unknown_ = UINT64_C(1) << 63, // Outside of RISC-V address space -}; - -static constexpr shadow_state_what shadow_state_get_what(uint64_t paddr) { - if (paddr < AR_SHADOW_STATE_START || paddr - AR_SHADOW_STATE_START >= sizeof(shadow_state) || - (paddr & (sizeof(uint64_t) - 1)) != 0) { - return shadow_state_what::unknown_; - } - // NOLINTNEXTLINE(clang-analyzer-optin.core.EnumCastOutOfRange) - return static_cast(paddr); -} - -static constexpr shadow_state_what shadow_state_get_what(shadow_state_what what, int i) { - return static_cast(static_cast(what) + (i * sizeof(uint64_t))); -} - -static constexpr const char *shadow_state_get_what_name(shadow_state_what what) { - const auto paddr = static_cast(what); - if (paddr < AR_SHADOW_STATE_START || paddr - AR_SHADOW_STATE_START >= sizeof(shadow_state) || - (paddr & (sizeof(uint64_t) - 1)) != 0) { - return "state.unknown_"; - } - using reg = shadow_state_what; - switch (what) { - case reg::x0: - return "x0"; - case reg::x1: - return "x1"; - case reg::x2: - return "x2"; - case reg::x3: - return "x3"; - case reg::x4: - return "x4"; - case reg::x5: - return "x5"; - case reg::x6: - return "x6"; - case reg::x7: - return "x7"; - case reg::x8: - return "x8"; - case reg::x9: - return "x9"; - case reg::x10: - return "x10"; - case reg::x11: - return "x11"; - case reg::x12: - return "x12"; - case reg::x13: - return "x13"; - case reg::x14: - return "x14"; - case reg::x15: - return "x15"; - case reg::x16: - return "x16"; - case reg::x17: - return "x17"; - case reg::x18: - return "x18"; - case reg::x19: - return "x19"; - case reg::x20: - return "x20"; - case reg::x21: - return "x21"; - case reg::x22: - return "x22"; - case reg::x23: - return "x23"; - case reg::x24: - return "x24"; - case reg::x25: - return "x25"; - case reg::x26: - return "x26"; - case reg::x27: - return "x27"; - case reg::x28: - return "x28"; - case reg::x29: - return "x29"; - case reg::x30: - return "x30"; - case reg::x31: - return "x31"; - case reg::f0: - return "f0"; - case reg::f1: - return "f1"; - case reg::f2: - return "f2"; - case reg::f3: - return "f3"; - case reg::f4: - return "f4"; - case reg::f5: - return "f5"; - case reg::f6: - return "f6"; - case reg::f7: - return "f7"; - case reg::f8: - return "f8"; - case reg::f9: - return "f9"; - case reg::f10: - return "f10"; - case reg::f11: - return "f11"; - case reg::f12: - return "f12"; - case reg::f13: - return "f13"; - case reg::f14: - return "f14"; - case reg::f15: - return "f15"; - case reg::f16: - return "f16"; - case reg::f17: - return "f17"; - case reg::f18: - return "f18"; - case reg::f19: - return "f19"; - case reg::f20: - return "f20"; - case reg::f21: - return "f21"; - case reg::f22: - return "f22"; - case reg::f23: - return "f23"; - case reg::f24: - return "f24"; - case reg::f25: - return "f25"; - case reg::f26: - return "f26"; - case reg::f27: - return "f27"; - case reg::f28: - return "f28"; - case reg::f29: - return "f29"; - case reg::f30: - return "f30"; - case reg::f31: - return "f31"; - case reg::pc: - return "pc"; - case reg::fcsr: - return "fcsr"; - case reg::mvendorid: - return "mvendorid"; - case reg::marchid: - return "marchid"; - case reg::mimpid: - return "mimpid"; - case reg::mcycle: - return "mcycle"; - case reg::icycleinstret: - return "icycleinstret"; - case reg::mstatus: - return "mstatus"; - case reg::mtvec: - return "mtvec"; - case reg::mscratch: - return "mscratch"; - case reg::mepc: - return "mepc"; - case reg::mcause: - return "mcause"; - case reg::mtval: - return "mtval"; - case reg::misa: - return "misa"; - case reg::mie: - return "mie"; - case reg::mip: - return "mip"; - case reg::medeleg: - return "medeleg"; - case reg::mideleg: - return "mideleg"; - case reg::mcounteren: - return "mcounteren"; - case reg::menvcfg: - return "menvcfg"; - case reg::stvec: - return "stvec"; - case reg::sscratch: - return "sscratch"; - case reg::sepc: - return "sepc"; - case reg::scause: - return "scause"; - case reg::stval: - return "stval"; - case reg::satp: - return "satp"; - case reg::scounteren: - return "scounteren"; - case reg::senvcfg: - return "senvcfg"; - case reg::ilrsc: - return "ilrsc"; - case reg::iprv: - return "iprv"; - case reg::iflags_X: - return "iflags.X"; - case reg::iflags_Y: - return "iflags.Y"; - case reg::iflags_H: - return "iflags.H"; - case reg::iunrep: - return "iunrep"; - case reg::clint_mtimecmp: - return "clint.mtimecmp"; - case reg::plic_girqpend: - return "plic.girqpend"; - case reg::plic_girqsrvd: - return "plic.girqsrvd"; - case reg::htif_tohost: - return "htif.tohost"; - case reg::htif_fromhost: - return "htif.fromhost"; - case reg::htif_ihalt: - return "htif.ihalt"; - case reg::htif_iconsole: - return "htif.iconsole"; - case reg::htif_iyield: - return "htif.iyield"; - case reg::unknown_: - return "state.unknown_"; - } - return "state.unknown_"; -} - -} // namespace cartesi - -#endif diff --git a/src/shadow-tlb-address-range.cpp b/src/shadow-tlb-address-range.cpp deleted file mode 100644 index d9f52fc96..000000000 --- a/src/shadow-tlb-address-range.cpp +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright Cartesi and individual authors (see AUTHORS) -// SPDX-License-Identifier: LGPL-3.0-or-later -// -// This program is free software: you can redistribute it and/or modify it under -// the terms of the GNU Lesser General Public License as published by the Free -// Software Foundation, either version 3 of the License, or (at your option) any -// later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT ANY -// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A -// PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License along -// with this program (see COPYING). If not, see . -// - -#include "shadow-tlb-address-range.h" - -#include - -#include "machine.h" -#include "shadow-peek.h" - -namespace cartesi { - -bool shadow_tlb_address_range::do_peek(const machine &m, uint64_t offset, uint64_t length, const unsigned char **data, - unsigned char *scratch) const noexcept { - // If past useful range - if (offset >= sizeof(shadow_tlb_state)) { - *data = nullptr; - return contains_relative(offset, length); - } - // Otherwise, copy and return register data - return shadow_peek( - [](const machine &m, uint64_t paddr) { - TLB_set_index set_index{}; - uint64_t slot_index{}; - auto reg = shadow_tlb_get_what(paddr, set_index, slot_index); - uint64_t val = 0; - if (reg != shadow_tlb_what::unknown_) { - val = m.read_shadow_tlb(set_index, slot_index, reg); - } - return val; - }, - *this, m, offset, length, data, scratch); -} - -} // namespace cartesi diff --git a/src/shadow-tlb-address-range.h b/src/shadow-tlb-address-range.h deleted file mode 100644 index e82ba76c2..000000000 --- a/src/shadow-tlb-address-range.h +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright Cartesi and individual authors (see AUTHORS) -// SPDX-License-Identifier: LGPL-3.0-or-later -// -// This program is free software: you can redistribute it and/or modify it under -// the terms of the GNU Lesser General Public License as published by the Free -// Software Foundation, either version 3 of the License, or (at your option) any -// later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT ANY -// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A -// PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License along -// with this program (see COPYING). If not, see . -// - -#ifndef SHADOW_TLB_ADDRESS_RANGE_H -#define SHADOW_TLB_ADDRESS_RANGE_H - -#include -#include - -#include "address-range-constants.h" -#include "address-range.h" -#include "shadow-state.h" - -/// \file -/// \brief Shadow state address range. - -namespace cartesi { - -class shadow_tlb_address_range final : public address_range { - - static constexpr pmas_flags m_shadow_tlb_flags{ - .M = false, - .IO = false, - .R = false, - .W = false, - .X = false, - .IR = false, - .IW = false, - .DID = PMA_ISTART_DID::shadow_TLB, - }; - -public: - template - explicit shadow_tlb_address_range(ABRT abrt) : - address_range("shadow TLB", AR_SHADOW_TLB_START, AR_SHADOW_TLB_LENGTH, m_shadow_tlb_flags, abrt) { - ; - } - - shadow_tlb_address_range(const shadow_tlb_address_range &) = default; - shadow_tlb_address_range &operator=(const shadow_tlb_address_range &) = default; - shadow_tlb_address_range(shadow_tlb_address_range &&) noexcept = default; - shadow_tlb_address_range &operator=(shadow_tlb_address_range &&) noexcept = default; - ~shadow_tlb_address_range() override = default; - -private: -#ifndef MICROARCHITECTURE - bool do_peek(const machine &m, uint64_t offset, uint64_t length, const unsigned char **data, - unsigned char *scratch) const noexcept override; -#endif -}; - -template -static inline shadow_tlb_address_range make_shadow_tlb_address_range(ABRT abrt) { - return shadow_tlb_address_range{abrt}; -} - -} // namespace cartesi - -#endif diff --git a/src/shadow-tlb.h b/src/shadow-tlb.h index f6e9bfeee..d8589d11b 100644 --- a/src/shadow-tlb.h +++ b/src/shadow-tlb.h @@ -26,10 +26,45 @@ #include #include "compiler-defines.h" -#include "tlb.h" +#include "pmas-constants.h" +#include "riscv-constants.h" namespace cartesi { +/// \brief Index of TLB set +enum TLB_set_index : uint64_t { TLB_CODE, TLB_READ, TLB_WRITE, TLB_LAST_ = TLB_WRITE, TLB_NUM_SETS_ = TLB_WRITE + 1 }; + +/// \brief TLB constants. +enum TLB_constants : uint64_t { + TLB_SET_SIZE = 256, + TLB_INVALID_PAGE = UINT64_C(-1), + TLB_INVALID_PMA_INDEX = UINT64_C(-1), +}; + +/// \brief Gets a TLB slot index for a page. +/// \param vaddr Target virtual address. +/// \returns TLB slot index. +constexpr uint64_t tlb_slot_index(uint64_t vaddr) { + return (vaddr >> LOG2_PAGE_SIZE) & (TLB_SET_SIZE - 1); +} + +/// \brief Checks for a TLB hit. +/// \tparam T Type of access needed (uint8_t, uint16_t, uint32_t, uint64_t). +/// \param slot_vaddr_page vaddr_page in TLB slot +/// \param vaddr Target virtual address being looked up. +/// \returns True on hit, false otherwise. +template +constexpr bool tlb_is_hit(uint64_t slot_vaddr_page, uint64_t vaddr) { + // Make sure misaligned accesses are always considered a miss + // Otherwise, we could report a hit for a word that goes past the end of the page. + // Aligned accesses smaller than a page size cannot straddle two pages. + return slot_vaddr_page == (vaddr & ~(PAGE_OFFSET_MASK & ~(sizeof(T) - 1))); +} + +constexpr uint64_t tlb_addr_page(uint64_t addr) { + return addr & ~PAGE_OFFSET_MASK; +} + /// \brief Shadow TLB slot /// \details /// Given a target virtual address vaddr within a page matching vaddr_page in TLB slot, the corresponding @@ -39,12 +74,11 @@ namespace cartesi { /// We can only do /aligned/ atomic writes. /// Therefore, TLB slot cannot be misaligned. /// To complete the power-of-two size, we include a zero_padding_ entry. -struct PACKED shadow_tlb_slot { - uint64_t vaddr_page; ///< Target virtual address of start of page - uint64_t vp_offset; ///< Offset from target virtual address to target physical address within page - uint64_t pma_index; ///< Index of PMA where physical page falls - /// and host addresses - uint64_t zero_padding_; ///< Padding to make sure the sizeof(shadow_tlb_slot) is a power of 2 +struct shadow_tlb_slot final { + uint64_t vaddr_page{TLB_INVALID_PAGE}; ///< Target virtual address of start of page + uint64_t vp_offset{0}; ///< Offset from target virtual address to target physical address within page + uint64_t pma_index{TLB_INVALID_PMA_INDEX}; ///< Index of PMA where physical page falls and host addresses + uint64_t zero_padding_{0}; ///< Padding to make sure the sizeof(shadow_tlb_slot) is a power of 2 }; constexpr uint64_t SHADOW_TLB_SLOT_SIZE = sizeof(shadow_tlb_slot); diff --git a/src/shadow-uarch-state-address-range.cpp b/src/shadow-uarch-state-address-range.cpp deleted file mode 100644 index 61dfdf649..000000000 --- a/src/shadow-uarch-state-address-range.cpp +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright Cartesi and individual authors (see AUTHORS) -// SPDX-License-Identifier: LGPL-3.0-or-later -// -// This program is free software: you can redistribute it and/or modify it under -// the terms of the GNU Lesser General Public License as published by the Free -// Software Foundation, either version 3 of the License, or (at your option) any -// later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT ANY -// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A -// PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License along -// with this program (see COPYING). If not, see . -// - -#include "shadow-uarch-state-address-range.h" - -#include - -#include "machine-reg.h" -#include "machine.h" -#include "pmas-constants.h" -#include "shadow-peek.h" -#include "shadow-uarch-state.h" - -namespace cartesi { - -/// \brief Shadow uarch state device peek callback. See ::pmas_peek. -bool shadow_uarch_state_address_range::do_peek(const machine &m, uint64_t offset, uint64_t length, - const unsigned char **data, unsigned char *scratch) const noexcept { - // If past useful range - if (offset >= sizeof(shadow_uarch_state)) { - *data = nullptr; - return contains_relative(offset, length); - } - // Otherwise, copy and return register data - return shadow_peek( - [](const machine &m, uint64_t paddr) { - const auto reg = shadow_uarch_state_get_what(paddr); - uint64_t val = 0; - if (reg != shadow_uarch_state_what::unknown_) { - val = m.read_reg(machine_reg_enum(reg)); - } - return val; - }, - *this, m, offset, length, data, scratch); -} - -} // namespace cartesi diff --git a/src/shadow-uarch-state-address-range.h b/src/shadow-uarch-state-address-range.h deleted file mode 100644 index 025219248..000000000 --- a/src/shadow-uarch-state-address-range.h +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright Cartesi and individual authors (see AUTHORS) -// SPDX-License-Identifier: LGPL-3.0-or-later -// -// This program is free software: you can redistribute it and/or modify it under -// the terms of the GNU Lesser General Public License as published by the Free -// Software Foundation, either version 3 of the License, or (at your option) any -// later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT ANY -// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A -// PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License along -// with this program (see COPYING). If not, see . -// - -#ifndef SHADOW_UARCH_STATE_ADDRESS_RANGE_H -#define SHADOW_UARCH_STATE_ADDRESS_RANGE_H - -#include -#include - -#include "address-range.h" -#include "shadow-uarch-state.h" - -/// \file -/// \brief Shadow uarch state address range. - -namespace cartesi { - -class shadow_uarch_state_address_range final : public address_range { - - static constexpr pmas_flags m_shadow_uarch_state_flags{ - .M = false, - .IO = false, - .R = false, - .W = false, - .X = false, - .IR = false, - .IW = false, - .DID = PMA_ISTART_DID::shadow_uarch_state, - }; - -public: - template - shadow_uarch_state_address_range(uint64_t start, uint64_t length, ABRT abrt) : - address_range("shadow uarch state", start, length, m_shadow_uarch_state_flags, abrt) { - ; - } - - shadow_uarch_state_address_range(const shadow_uarch_state_address_range &) = delete; - shadow_uarch_state_address_range &operator=(const shadow_uarch_state_address_range &) = delete; - shadow_uarch_state_address_range &operator=(shadow_uarch_state_address_range &&) noexcept = delete; - - shadow_uarch_state_address_range(shadow_uarch_state_address_range &&) noexcept = default; - ~shadow_uarch_state_address_range() override = default; - -private: -#ifndef MICROARCHITECTURE - bool do_peek(const machine &m, uint64_t offset, uint64_t length, const unsigned char **data, - unsigned char *scratch) const noexcept override; -#endif -}; - -template -static inline shadow_uarch_state_address_range make_shadow_uarch_state_address_range(uint64_t start, uint64_t length, - ABRT abrt) { - return shadow_uarch_state_address_range{start, length, abrt}; -} - -} // namespace cartesi - -#endif diff --git a/src/shadow-uarch-state.h b/src/shadow-uarch-state.h index ded6e0201..868100379 100644 --- a/src/shadow-uarch-state.h +++ b/src/shadow-uarch-state.h @@ -28,14 +28,21 @@ namespace cartesi { -/// \brief Shadow uarch memory layout -struct PACKED shadow_uarch_state { - uint64_t halt_flag; - uint64_t cycle; - uint64_t pc; - uint64_t x[UARCH_X_REG_COUNT]; +/// \brief Uarch registers state +struct uarch_registers_state final { + uint64_t halt_flag{UARCH_HALT_FLAG_INIT}; + uint64_t cycle{UARCH_CYCLE_INIT}; + uint64_t pc{UARCH_PC_INIT}; + uint64_t x[UARCH_X_REG_COUNT]{}; }; +/// \brief Shadow uarch memory layout +using shadow_uarch_state = uarch_registers_state; + +// We need strong guarantees that shadow_uarch_state has fixed size and alignment across platforms. +static_assert(sizeof(shadow_uarch_state) == 35 * sizeof(uint64_t), "unexpected uarch registers state size"); +static_assert(alignof(shadow_uarch_state) == sizeof(uint64_t), "unexpected uarch registers state alignment"); + enum class shadow_uarch_state_what : uint64_t { uarch_halt_flag = AR_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, halt_flag), uarch_cycle = AR_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, cycle), diff --git a/src/state-access.h b/src/state-access.h index 6f079853a..31eae76db 100644 --- a/src/state-access.h +++ b/src/state-access.h @@ -33,10 +33,10 @@ #include "i-interactive-state-access.h" #include "i-state-access.h" #include "interpret.h" -#include "machine-state.h" #include "machine.h" #include "os.h" #include "pmas-constants.h" +#include "processor-state.h" #include "riscv-constants.h" #include "shadow-tlb.h" #include "strict-aliasing.h" @@ -77,44 +77,44 @@ class state_access : friend i_state_access; uint64_t do_read_x(int i) const { - return m_m.get_state().x[i]; + return m_m.get_state().registers.x[i]; } void do_write_x(int i, uint64_t val) const { assert(i != 0); - m_m.get_state().x[i] = val; + m_m.get_state().registers.x[i] = val; } uint64_t do_read_f(int i) const { - return m_m.get_state().f[i]; + return m_m.get_state().registers.f[i]; } void do_write_f(int i, uint64_t val) const { - m_m.get_state().f[i] = val; + m_m.get_state().registers.f[i] = val; } uint64_t do_read_pc() const { - return m_m.get_state().pc; + return m_m.get_state().registers.pc; } void do_write_pc(uint64_t val) const { - m_m.get_state().pc = val; + m_m.get_state().registers.pc = val; } uint64_t do_read_fcsr() const { - return m_m.get_state().fcsr; + return m_m.get_state().registers.fcsr; } void do_write_fcsr(uint64_t val) const { - m_m.get_state().fcsr = val; + m_m.get_state().registers.fcsr = val; } uint64_t do_read_icycleinstret() const { - return m_m.get_state().icycleinstret; + return m_m.get_state().registers.icycleinstret; } void do_write_icycleinstret(uint64_t val) const { - m_m.get_state().icycleinstret = val; + m_m.get_state().registers.icycleinstret = val; } uint64_t do_read_mvendorid() const { // NOLINT(readability-convert-member-functions-to-static) @@ -130,279 +130,279 @@ class state_access : } uint64_t do_read_mcycle() const { - return m_m.get_state().mcycle; + return m_m.get_state().registers.mcycle; } void do_write_mcycle(uint64_t val) const { - m_m.get_state().mcycle = val; + m_m.get_state().registers.mcycle = val; } uint64_t do_read_mstatus() const { - return m_m.get_state().mstatus; + return m_m.get_state().registers.mstatus; } void do_write_mstatus(uint64_t val) const { - m_m.get_state().mstatus = val; + m_m.get_state().registers.mstatus = val; } uint64_t do_read_menvcfg() const { - return m_m.get_state().menvcfg; + return m_m.get_state().registers.menvcfg; } void do_write_menvcfg(uint64_t val) const { - m_m.get_state().menvcfg = val; + m_m.get_state().registers.menvcfg = val; } uint64_t do_read_mtvec() const { - return m_m.get_state().mtvec; + return m_m.get_state().registers.mtvec; } void do_write_mtvec(uint64_t val) const { - m_m.get_state().mtvec = val; + m_m.get_state().registers.mtvec = val; } uint64_t do_read_mscratch() const { - return m_m.get_state().mscratch; + return m_m.get_state().registers.mscratch; } void do_write_mscratch(uint64_t val) const { - m_m.get_state().mscratch = val; + m_m.get_state().registers.mscratch = val; } uint64_t do_read_mepc() const { - return m_m.get_state().mepc; + return m_m.get_state().registers.mepc; } void do_write_mepc(uint64_t val) const { - m_m.get_state().mepc = val; + m_m.get_state().registers.mepc = val; } uint64_t do_read_mcause() const { - return m_m.get_state().mcause; + return m_m.get_state().registers.mcause; } void do_write_mcause(uint64_t val) const { - m_m.get_state().mcause = val; + m_m.get_state().registers.mcause = val; } uint64_t do_read_mtval() const { - return m_m.get_state().mtval; + return m_m.get_state().registers.mtval; } void do_write_mtval(uint64_t val) const { - m_m.get_state().mtval = val; + m_m.get_state().registers.mtval = val; } uint64_t do_read_misa() const { - return m_m.get_state().misa; + return m_m.get_state().registers.misa; } void do_write_misa(uint64_t val) const { - m_m.get_state().misa = val; + m_m.get_state().registers.misa = val; } uint64_t do_read_mie() const { - return m_m.get_state().mie; + return m_m.get_state().registers.mie; } void do_write_mie(uint64_t val) const { - m_m.get_state().mie = val; + m_m.get_state().registers.mie = val; } uint64_t do_read_mip() const { - return m_m.get_state().mip; + return m_m.get_state().registers.mip; } void do_write_mip(uint64_t val) const { - m_m.get_state().mip = val; + m_m.get_state().registers.mip = val; } uint64_t do_read_medeleg() const { - return m_m.get_state().medeleg; + return m_m.get_state().registers.medeleg; } void do_write_medeleg(uint64_t val) const { - m_m.get_state().medeleg = val; + m_m.get_state().registers.medeleg = val; } uint64_t do_read_mideleg() const { - return m_m.get_state().mideleg; + return m_m.get_state().registers.mideleg; } void do_write_mideleg(uint64_t val) const { - m_m.get_state().mideleg = val; + m_m.get_state().registers.mideleg = val; } uint64_t do_read_mcounteren() const { - return m_m.get_state().mcounteren; + return m_m.get_state().registers.mcounteren; } void do_write_mcounteren(uint64_t val) const { - m_m.get_state().mcounteren = val; + m_m.get_state().registers.mcounteren = val; } uint64_t do_read_senvcfg() const { - return m_m.get_state().senvcfg; + return m_m.get_state().registers.senvcfg; } void do_write_senvcfg(uint64_t val) const { - m_m.get_state().senvcfg = val; + m_m.get_state().registers.senvcfg = val; } uint64_t do_read_stvec() const { - return m_m.get_state().stvec; + return m_m.get_state().registers.stvec; } void do_write_stvec(uint64_t val) const { - m_m.get_state().stvec = val; + m_m.get_state().registers.stvec = val; } uint64_t do_read_sscratch() const { - return m_m.get_state().sscratch; + return m_m.get_state().registers.sscratch; } void do_write_sscratch(uint64_t val) const { - m_m.get_state().sscratch = val; + m_m.get_state().registers.sscratch = val; } uint64_t do_read_sepc() const { - return m_m.get_state().sepc; + return m_m.get_state().registers.sepc; } void do_write_sepc(uint64_t val) const { - m_m.get_state().sepc = val; + m_m.get_state().registers.sepc = val; } uint64_t do_read_scause() const { - return m_m.get_state().scause; + return m_m.get_state().registers.scause; } void do_write_scause(uint64_t val) const { - m_m.get_state().scause = val; + m_m.get_state().registers.scause = val; } uint64_t do_read_stval() const { - return m_m.get_state().stval; + return m_m.get_state().registers.stval; } void do_write_stval(uint64_t val) const { - m_m.get_state().stval = val; + m_m.get_state().registers.stval = val; } uint64_t do_read_satp() const { - return m_m.get_state().satp; + return m_m.get_state().registers.satp; } void do_write_satp(uint64_t val) const { - m_m.get_state().satp = val; + m_m.get_state().registers.satp = val; } uint64_t do_read_scounteren() const { - return m_m.get_state().scounteren; + return m_m.get_state().registers.scounteren; } void do_write_scounteren(uint64_t val) const { - m_m.get_state().scounteren = val; + m_m.get_state().registers.scounteren = val; } uint64_t do_read_ilrsc() const { - return m_m.get_state().ilrsc; + return m_m.get_state().registers.ilrsc; } void do_write_ilrsc(uint64_t val) const { - m_m.get_state().ilrsc = val; + m_m.get_state().registers.ilrsc = val; } uint64_t do_read_iprv() const { - return m_m.get_state().iprv; + return m_m.get_state().registers.iprv; } void do_write_iprv(uint64_t val) const { - m_m.get_state().iprv = val; + m_m.get_state().registers.iprv = val; } uint64_t do_read_iflags_X() const { - return m_m.get_state().iflags.X; + return m_m.get_state().registers.iflags.X; } void do_write_iflags_X(uint64_t val) const { - m_m.get_state().iflags.X = val; + m_m.get_state().registers.iflags.X = val; } uint64_t do_read_iflags_Y() const { - return m_m.get_state().iflags.Y; + return m_m.get_state().registers.iflags.Y; } void do_write_iflags_Y(uint64_t val) const { - m_m.get_state().iflags.Y = val; + m_m.get_state().registers.iflags.Y = val; } uint64_t do_read_iflags_H() const { - return m_m.get_state().iflags.H; + return m_m.get_state().registers.iflags.H; } void do_write_iflags_H(uint64_t val) const { - m_m.get_state().iflags.H = val; + m_m.get_state().registers.iflags.H = val; } uint64_t do_read_iunrep() const { - return m_m.get_state().iunrep; + return m_m.get_state().registers.iunrep; } void do_write_iunrep(uint64_t val) const { - m_m.get_state().iunrep = val; + m_m.get_state().registers.iunrep = val; } uint64_t do_read_clint_mtimecmp() const { - return m_m.get_state().clint.mtimecmp; + return m_m.get_state().registers.clint.mtimecmp; } void do_write_clint_mtimecmp(uint64_t val) const { - m_m.get_state().clint.mtimecmp = val; + m_m.get_state().registers.clint.mtimecmp = val; } uint64_t do_read_plic_girqpend() const { - return m_m.get_state().plic.girqpend; + return m_m.get_state().registers.plic.girqpend; } void do_write_plic_girqpend(uint64_t val) const { - m_m.get_state().plic.girqpend = val; + m_m.get_state().registers.plic.girqpend = val; } uint64_t do_read_plic_girqsrvd() const { - return m_m.get_state().plic.girqsrvd; + return m_m.get_state().registers.plic.girqsrvd; } void do_write_plic_girqsrvd(uint64_t val) const { - m_m.get_state().plic.girqsrvd = val; + m_m.get_state().registers.plic.girqsrvd = val; } uint64_t do_read_htif_fromhost() const { - return m_m.get_state().htif.fromhost; + return m_m.get_state().registers.htif.fromhost; } void do_write_htif_fromhost(uint64_t val) const { - m_m.get_state().htif.fromhost = val; + m_m.get_state().registers.htif.fromhost = val; } uint64_t do_read_htif_tohost() const { - return m_m.get_state().htif.tohost; + return m_m.get_state().registers.htif.tohost; } void do_write_htif_tohost(uint64_t val) const { - m_m.get_state().htif.tohost = val; + m_m.get_state().registers.htif.tohost = val; } uint64_t do_read_htif_ihalt() const { - return m_m.get_state().htif.ihalt; + return m_m.get_state().registers.htif.ihalt; } uint64_t do_read_htif_iconsole() const { - return m_m.get_state().htif.iconsole; + return m_m.get_state().registers.htif.iconsole; } uint64_t do_read_htif_iyield() const { - return m_m.get_state().htif.iyield; + return m_m.get_state().registers.htif.iyield; } bool do_read_memory(uint64_t paddr, unsigned char *data, uint64_t length) const { @@ -458,17 +458,17 @@ class state_access : template uint64_t do_read_tlb_vaddr_page(uint64_t slot_index) const { - return m_m.get_state().tlb.hot[SET][slot_index].vaddr_page; + return m_m.get_state().hot_tlb[SET][slot_index].vaddr_page; } template - host_addr do_read_tlb_vp_offset(uint64_t slot_index) const { - return m_m.get_state().tlb.hot[SET][slot_index].vh_offset; + host_addr do_read_tlb_vf_offset(uint64_t slot_index) const { + return m_m.get_state().hot_tlb[SET][slot_index].vh_offset; } template uint64_t do_read_tlb_pma_index(uint64_t slot_index) const { - return m_m.get_state().tlb.cold[SET][slot_index].pma_index; + return m_m.get_state().shadow_tlb[SET][slot_index].pma_index; } template @@ -504,7 +504,7 @@ class state_access : } bool do_get_soft_yield() const { - return m_m.get_state().soft_yield; + return m_m.is_soft_yield(); } // NOLINTNEXTLINE(readability-convert-member-functions-to-static) diff --git a/src/tlb.h b/src/tlb.h deleted file mode 100644 index f78905b1d..000000000 --- a/src/tlb.h +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright Cartesi and individual authors (see AUTHORS) -// SPDX-License-Identifier: LGPL-3.0-or-later -// -// This program is free software: you can redistribute it and/or modify it under -// the terms of the GNU Lesser General Public License as published by the Free -// Software Foundation, either version 3 of the License, or (at your option) any -// later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT ANY -// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A -// PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License along -// with this program (see COPYING). If not, see . -// - -#ifndef TLB_H -#define TLB_H - -/// \file -/// \brief TLB definitions -/// \details The Translation Lookaside Buffer is a small cache used to speed up translation between address spaces. - -#include -#include -#include - -#include "host-addr.h" -#include "pmas-constants.h" -#include "riscv-constants.h" - -namespace cartesi { - -/// \brief Index of TLB set -enum TLB_set_index : uint64_t { TLB_CODE, TLB_READ, TLB_WRITE, TLB_LAST_ = TLB_WRITE }; - -/// \brief TLB constants. -enum TLB_constants : uint64_t { - TLB_SET_SIZE = 256, - TLB_INVALID_PAGE = UINT64_C(-1), - TLB_INVALID_PMA_INDEX = UINT64_C(-1) -}; - -/// \brief TLB hot slot. -/// \details -/// Given a target virtual address vaddr within a page matching vaddr_page in TLB slot, the corresponding host address -/// haddr = vaddr + vh_offset. -struct tlb_hot_slot final { - uint64_t vaddr_page; ///< Target virtual address of start of page - host_addr vh_offset; ///< Offset from target virtual address in the same page to host address -}; - -using tlb_hot_set = std::array; - -/// \brief TLB cold slot. -/// \details -/// The pma_index helps translate between target physical addresses and host addresses when needed. -struct tlb_cold_slot final { - uint64_t pma_index; ///< Index of PMA where physical address falls -}; - -using tlb_cold_set = std::array; - -/// \brief TLB state. -struct tlb_state { - std::array hot; - std::array cold; -}; - -static_assert(sizeof(uint64_t) >= sizeof(uintptr_t), "TLB expects host pointer fit in 64 bits"); -//??D why? -static_assert((sizeof(tlb_hot_slot) & (sizeof(tlb_hot_slot) - 1)) == 0, "TLB slot size must be a power of 2"); -static_assert((sizeof(tlb_cold_slot) & (sizeof(tlb_cold_slot) - 1)) == 0, "TLB slot size must be a power of 2"); - -/// \brief Gets a TLB slot index for a page. -/// \param vaddr Target virtual address. -/// \returns TLB slot index. -constexpr uint64_t tlb_slot_index(uint64_t vaddr) { - return (vaddr >> LOG2_PAGE_SIZE) & (TLB_SET_SIZE - 1); -} - -/// \brief Checks for a TLB hit. -/// \tparam T Type of access needed (uint8_t, uint16_t, uint32_t, uint64_t). -/// \param slot_vaddr_page vaddr_page in TLB slot -/// \param vaddr Target virtual address being looked up. -/// \returns True on hit, false otherwise. -template -constexpr bool tlb_is_hit(uint64_t slot_vaddr_page, uint64_t vaddr) { - // Make sure misaligned accesses are always considered a miss - // Otherwise, we could report a hit for a word that goes past the end of the page. - // Aligned accesses smaller than a page size cannot straddle two pages. - return slot_vaddr_page == (vaddr & ~(PAGE_OFFSET_MASK & ~(sizeof(T) - 1))); -} - -constexpr uint64_t tlb_addr_page(uint64_t addr) { - return addr & ~PAGE_OFFSET_MASK; -} - -} // namespace cartesi - -#endif diff --git a/src/shadow-tlb-factory.h b/src/uarch-processor-state.h similarity index 59% rename from src/shadow-tlb-factory.h rename to src/uarch-processor-state.h index 635b63d3e..f23374a37 100644 --- a/src/shadow-tlb-factory.h +++ b/src/uarch-processor-state.h @@ -14,21 +14,28 @@ // with this program (see COPYING). If not, see . // -#ifndef SHADOW_TLB_FACTORY_H -#define SHADOW_TLB_FACTORY_H +#ifndef UARCH_PROCESSOR_STATE_H +#define UARCH_PROCESSOR_STATE_H /// \file -/// \brief TLB device factory. +/// \brief Cartesi microarchitecture machine processor state structure definition. +#include #include +#include + +#include "memory-address-range.h" +#include "riscv-constants.h" +#include "shadow-uarch-state.h" namespace cartesi { -/// \brief Creates a PMA entry for the TLB device -/// \param start Start address for memory range. -/// \param length Length of memory range. -/// \returns Corresponding PMA entry -pmas_entry make_shadow_tlb_pmas_entry(uint64_t start, uint64_t length); +struct uarch_processor_state final { + uarch_registers_state registers; ///< Uarch registers + uint64_t registers_padding_[477]{}; ///< Padding to align next field to a page boundary +}; + +static_assert(sizeof(uarch_processor_state) % 4096 == 0, "machine state size must be multiple of a page size"); } // namespace cartesi diff --git a/src/uarch-record-state-access.h b/src/uarch-record-state-access.h index 498c4681f..d56c07e4b 100644 --- a/src/uarch-record-state-access.h +++ b/src/uarch-record-state-access.h @@ -41,7 +41,7 @@ #include "uarch-constants.h" #include "uarch-pristine-state-hash.h" #include "uarch-pristine.h" -#include "uarch-state.h" +#include "uarch-processor-state.h" namespace cartesi { diff --git a/src/uarch-state-access.h b/src/uarch-state-access.h index fc1b7ef45..5df2c3f55 100644 --- a/src/uarch-state-access.h +++ b/src/uarch-state-access.h @@ -45,9 +45,9 @@ class uarch_state_access : /// \param um Reference to uarch state. /// \param m Reference to machine state. explicit uarch_state_access(machine &m) : m_m(m) { - const auto &uram = m_m.get_uarch_state().ram; - const auto haddr = cast_ptr_to_host_addr(uram->get_host_memory()); - const auto paddr = uram->get_start(); + const auto &uram = m_m.find_address_range(AR_UARCH_RAM_START, AR_UARCH_RAM_LENGTH); + const auto haddr = cast_ptr_to_host_addr(uram.get_host_memory()); + const auto paddr = AR_UARCH_RAM_START; // initialize translation cache from paddr in uarch RAM to host address m_uram_ph_offset = haddr - paddr; } @@ -59,36 +59,36 @@ class uarch_state_access : friend i_uarch_state_access; uint64_t do_read_uarch_x(int i) const { - return m_m.get_uarch_state().x[i]; + return m_m.get_uarch_state().registers.x[i]; } void do_write_uarch_x(int i, uint64_t val) const { assert(i != 0); - m_m.get_uarch_state().x[i] = val; + m_m.get_uarch_state().registers.x[i] = val; } uint64_t do_read_uarch_pc() const { - return m_m.get_uarch_state().pc; + return m_m.get_uarch_state().registers.pc; } void do_write_uarch_pc(uint64_t val) const { - m_m.get_uarch_state().pc = val; + m_m.get_uarch_state().registers.pc = val; } uint64_t do_read_uarch_cycle() const { - return m_m.get_uarch_state().cycle; + return m_m.get_uarch_state().registers.cycle; } void do_write_uarch_cycle(uint64_t val) const { - m_m.get_uarch_state().cycle = val; + m_m.get_uarch_state().registers.cycle = val; } uint64_t do_read_uarch_halt_flag() const { - return m_m.get_uarch_state().halt_flag; + return m_m.get_uarch_state().registers.halt_flag; } void do_write_uarch_halt_flag(uint64_t v) const { - m_m.get_uarch_state().halt_flag = v; + m_m.get_uarch_state().registers.halt_flag = v; } uint64_t do_read_word(uint64_t paddr) const { diff --git a/src/uarch-state.h b/src/uarch-state.h deleted file mode 100644 index 51c0db901..000000000 --- a/src/uarch-state.h +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright Cartesi and individual authors (see AUTHORS) -// SPDX-License-Identifier: LGPL-3.0-or-later -// -// This program is free software: you can redistribute it and/or modify it under -// the terms of the GNU Lesser General Public License as published by the Free -// Software Foundation, either version 3 of the License, or (at your option) any -// later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT ANY -// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A -// PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License along -// with this program (see COPYING). If not, see . -// - -#ifndef UARCH_STATE_H -#define UARCH_STATE_H - -/// \file -/// \brief Cartesi microarchitecture machine state structure definition. - -#include -#include -#include - -#include "memory-address-range.h" -#include "riscv-constants.h" - -namespace cartesi { - -struct uarch_state { - uarch_state() = default; - ~uarch_state() = default; - - /// \brief No copy or move constructor or assignment - uarch_state(const uarch_state &other) = delete; - uarch_state(uarch_state &&other) = delete; - uarch_state &operator=(const uarch_state &other) = delete; - uarch_state &operator=(uarch_state &&other) = delete; - - uint64_t pc{}; ///< Program counter. - std::array x{}; ///< Register file. - uint64_t cycle{}; ///< Cycles counter - uint64_t halt_flag{}; - address_range *shadow_state{}; ///< Shadow uarch state - memory_address_range *ram{}; ///< Memory range for uarch RAM -}; - -} // namespace cartesi - -#endif diff --git a/src/unique-c-ptr.h b/src/unique-c-ptr.h index 965ab7523..4feb149d7 100644 --- a/src/unique-c-ptr.h +++ b/src/unique-c-ptr.h @@ -48,15 +48,6 @@ struct fclose_deleter { } }; -struct mmap_deleter { - size_t m_size; - explicit mmap_deleter(size_t size) : m_size(size) {}; - template - void operator()(T *ptr) const { - os_unmap_file(ptr, m_size); - } -}; - } // namespace detail template @@ -64,9 +55,6 @@ using unique_calloc_ptr = std::unique_ptr; using unique_file_ptr = std::unique_ptr; -template -using unique_mmap_ptr = std::unique_ptr; - template static inline auto make_unique_calloc(size_t nmemb) { // NOLINTNEXTLINE(cppcoreguidelines-no-malloc,hicpp-no-malloc) @@ -96,13 +84,6 @@ static inline auto make_unique_fopen(const char *pathname, const char *mode, con return unique_file_ptr{fopen(pathname, mode)}; } -template -static inline auto make_unique_mmap(const char *pathname, size_t nmemb, bool shared) { - const size_t size = nmemb * sizeof(T); - T *ptr = static_cast(os_map_file(pathname, size, shared)); // os_map_file throws on error - return unique_mmap_ptr(ptr, detail::mmap_deleter{size}); -} - template static auto inline make_moved_unique(T &&t) requires std::is_rvalue_reference_v diff --git a/tests/lua/htif-console.lua b/tests/lua/htif-console.lua index 17274893c..ee8a60ce0 100755 --- a/tests/lua/htif-console.lua +++ b/tests/lua/htif-console.lua @@ -21,7 +21,9 @@ local test_util = require("cartesi.tests.util") local config_base = { processor = { - iunrep = 1, + registers = { + iunrep = 1, + }, }, ram = { backing_store = { diff --git a/tests/lua/htif-yield.lua b/tests/lua/htif-yield.lua index e62272b5d..6c62d3610 100755 --- a/tests/lua/htif-yield.lua +++ b/tests/lua/htif-yield.lua @@ -203,9 +203,13 @@ local function test(machine_config, yield_automatic_enable, yield_manual_enable) yield_automatic_enable and "on" or "off", yield_manual_enable and "on" or "off" ) - machine_config.htif = { - yield_automatic = yield_automatic_enable, - yield_manual = yield_manual_enable, + machine_config.processor = { + registers = { + htif = { + iyield = (yield_automatic_enable and cartesi.HTIF_YIELD_CMD_AUTOMATIC_MASK or 0) + | (yield_manual_enable and cartesi.HTIF_YIELD_CMD_MANUAL_MASK or 0), + }, + }, } local machine = cartesi.machine(machine_config) for _, v in ipairs(yields) do diff --git a/tests/lua/log-with-mtime-transition.lua b/tests/lua/log-with-mtime-transition.lua index 4d3298952..f8f2461d0 100755 --- a/tests/lua/log-with-mtime-transition.lua +++ b/tests/lua/log-with-mtime-transition.lua @@ -4,7 +4,9 @@ local cartesi = require("cartesi") local config = { processor = { - mcycle = 99, + registers = { + mcycle = 99, + }, }, ram = { length = 1 << 12, diff --git a/tests/lua/machine-bind.lua b/tests/lua/machine-bind.lua index 3e416fab3..88bd8305a 100755 --- a/tests/lua/machine-bind.lua +++ b/tests/lua/machine-bind.lua @@ -147,60 +147,12 @@ end local SHADOW_BASE = 0x0 -local cpu_reg_addr = { - pc = 512, - fcsr = 520, - mvendorid = 528, - marchid = 536, - mimpid = 544, - mcycle = 552, - icycleinstret = 560, - mstatus = 568, - mtvec = 576, - mscratch = 584, - mepc = 592, - mcause = 600, - mtval = 608, - misa = 616, - mie = 624, - mip = 632, - medeleg = 640, - mideleg = 648, - mcounteren = 656, - menvcfg = 664, - stvec = 672, - sscratch = 680, - sepc = 688, - scause = 696, - stval = 704, - satp = 712, - scounteren = 720, - senvcfg = 728, - ilrsc = 736, - iprv = 744, - iflags_X = 752, - iflags_Y = 760, - iflags_H = 768, - iunrep = 776, - clint_mtimecmp = 784, - plic_girqpend = 792, - plic_girqsrvd = 800, - htif_tohost = 808, - htif_fromhost = 816, - htif_ihalt = 824, - htif_iconsole = 832, - htif_iyield = 840, -} -for i = 0, 31 do - cpu_reg_addr["x" .. i] = i * 8 -end - local function get_uarch_cpu_reg_test_values() - local processor = {} + local registers = {} for i = 0, 31 do - processor["x" .. i] = 0x10000 + (i * 8) + registers["x" .. i] = 0x10000 + (i * 8) end - return processor + return registers end local function get_cpu_reg_test_values() @@ -269,13 +221,16 @@ local function build_machine_config(config_options) -- Create new machine local initial_reg_values = get_cpu_reg_test_values() local config = { - processor = config_options.processor or initial_reg_values, + processor = config_options.processor or { + registers = initial_reg_values, + }, ram = { length = 1 << 20 }, - htif = config_options.htif or nil, cmio = config_options.cmio or nil, uarch = config_options.uarch or { - processor = get_uarch_cpu_reg_test_values(), + processor = { + registers = get_uarch_cpu_reg_test_values(), + }, ram = { length = 0x1000, backing_store = { @@ -327,8 +282,7 @@ do_test("machine should have default config shadow register values", function(ma initial_reg_values.mimpid = nil -- Check initialization and shadow reads for k, v in pairs(initial_reg_values) do - local r = machine:read_word(cpu_reg_addr[k]) - assert(v == r) + assert(v == machine:read_word(machine:get_reg_address(k))) end end) @@ -351,9 +305,9 @@ end) print("\n\ntesting get_reg_address function binding") do_test("should return address value for registers", function(machine) -- Check register address - for k, v in pairs(cpu_reg_addr) do - local u = machine:get_reg_address(k) - assert(u == v, "invalid return for " .. v) + local initial_reg_values = get_cpu_reg_test_values() + for k, v in pairs(initial_reg_values) do + assert(machine:read_reg(k) == machine:read_word(machine:get_reg_address(k)), "invalid return for " .. v) end end) @@ -419,10 +373,6 @@ local function test_config(config) "ram", "dtb", "flash_drive", - "tlb", - "clint", - "plic", - "htif", "virtio", "cmio", "pmas", @@ -430,13 +380,13 @@ local function test_config(config) "hash_tree", } for _, field in ipairs(config_fields) do - assertfield(config, field, "table") + assertfield(config, field, "table", "config", "needed") end for i = 0, 31 do - assertfield(config, "processor.x" .. i, "number") + assertfield(config.processor, "registers.x" .. i, "number", "processor", "needed") end for i = 0, 31 do - assertfield(config, "processor.f" .. i, "number") + assertfield(config.processor, "registers.f" .. i, "number", "processor", "needed") end local csrs = { "pc", @@ -469,35 +419,36 @@ local function test_config(config) "senvcfg", "ilrsc", "iprv", - "iflags_X", - "iflags_Y", - "iflags_H", + "iflags.X", + "iflags.Y", + "iflags.H", "iunrep", + "htif.ihalt", + "htif.iconsole", + "htif.iyield", + "htif.tohost", + "htif.fromhost", + "clint.mtimecmp", + "plic.girqpend", + "plic.girqsrvd", } for _, csr in ipairs(csrs) do - assertfield(config, "processor." .. csr, "number") + assertfield(config.processor, "registers." .. csr, "number", "processor", "needed") end - assertfield(config, "htif.console_getchar", "boolean") - assertfield(config, "htif.yield_manual", "boolean") - assertfield(config, "htif.yield_automatic", "boolean") - assertfield(config, "htif.tohost", "number") - assertfield(config, "htif.fromhost", "number") - assertfield(config, "clint.mtimecmp", "number") - assertfield(config, "plic.girqpend", "number") - assertfield(config, "plic.girqsrvd", "number") assertfield(config, "ram.length", "number") test_backing_store_config(config, "ram.backing_store") assertfield(config, "dtb.bootargs", "string") assertfield(config, "dtb.init", "string") assertfield(config, "dtb.entrypoint", "string") test_backing_store_config(config, "dtb.backing_store") - test_backing_store_config(config, "tlb.backing_store") + test_backing_store_config(config, "processor.backing_store") test_backing_store_config(config, "pmas.backing_store") for i, f in ipairs(config.flash_drive or {}) do test_flash_drive_config(f, "config.flash_drive[" .. i .. "]") end test_backing_store_config(config, "cmio.rx_buffer.backing_store") test_backing_store_config(config, "cmio.tx_buffer.backing_store") + test_backing_store_config(config, "uarch.processor.backing_store") test_backing_store_config(config, "uarch.ram.backing_store") assertfield(config, "hash_tree.shared", "boolean") assertfield(config, "hash_tree.truncate", "boolean") @@ -533,16 +484,19 @@ do_test("should have expected values", function(machine) -- Check initial config local initial_config = machine:get_initial_config() test_config(initial_config) - assert(initial_config.processor.pc == 0x200, "wrong pc reg initial config value") - assert(initial_config.processor.ilrsc == 0x2e0, "wrong ilrsc reg initial config value") - assert(initial_config.processor.mstatus == 0x230, "wrong mstatus reg initial config value") - assert(initial_config.clint.mtimecmp == 0, "wrong clint mtimecmp initial config value") - assert(initial_config.plic.girqpend == 0, "wrong plic girqpend initial config value") - assert(initial_config.plic.girqsrvd == 0, "wrong plic girqsrvd initial config value") - assert(initial_config.htif.fromhost == 0, "wrong htif fromhost initial config value") - assert(initial_config.htif.tohost == 0, "wrong htif tohost initial config value") - assert(initial_config.htif.yield_automatic == true, "wrong htif yield automatic initial config value") - assert(initial_config.htif.yield_manual == true, "wrong htif yield manual initial config value") + assert(initial_config.processor.registers.pc == 0x200, "wrong pc reg initial config value") + assert(initial_config.processor.registers.ilrsc == 0x2e0, "wrong ilrsc reg initial config value") + assert(initial_config.processor.registers.mstatus == 0x230, "wrong mstatus reg initial config value") + assert(initial_config.processor.registers.clint.mtimecmp == 0, "wrong clint mtimecmp initial config value") + assert(initial_config.processor.registers.plic.girqpend == 0, "wrong plic girqpend initial config value") + assert(initial_config.processor.registers.plic.girqsrvd == 0, "wrong plic girqsrvd initial config value") + assert(initial_config.processor.registers.htif.fromhost == 0, "wrong htif fromhost initial config value") + assert(initial_config.processor.registers.htif.tohost == 0, "wrong htif tohost initial config value") + assert( + initial_config.processor.registers.htif.iyield + == cartesi.HTIF_YIELD_CMD_MANUAL_MASK | cartesi.HTIF_YIELD_CMD_AUTOMATIC_MASK, + "wrong htif yield automatic initial config value" + ) end) print("\n\n test read_reg") @@ -555,7 +509,7 @@ do_test("should return expected values", function(machine) initial_reg_values.htif_fromhost = 0x0 initial_reg_values.htif_ihalt = 0x0 initial_reg_values.htif_iconsole = 0x0 - initial_reg_values.htif_iyield = 3 + initial_reg_values.htif_iyield = cartesi.HTIF_YIELD_CMD_MANUAL_MASK | cartesi.HTIF_YIELD_CMD_AUTOMATIC_MASK -- Check register read local to_ignore = { @@ -569,7 +523,7 @@ do_test("should return expected values", function(machine) htif_ihalt = true, htif_iconsole = true, } - for k in pairs(cpu_reg_addr) do + for k in pairs(initial_reg_values) do if not to_ignore[k] then assert(machine:read_reg(k) == initial_reg_values[k], "wrong " .. k .. " value") end @@ -864,7 +818,7 @@ test_util.make_do_test(build_machine, machine_type, { assert(ram_image == expected_ram_image) end) -test_util.make_do_test(build_machine, machine_type, { processor = { mcycle = 1 }, uarch = {} })( +test_util.make_do_test(build_machine, machine_type, { processor = { registers = { mcycle = 1 } }, uarch = {} })( "It should use the embedded uarch-ram.bin when the uarch config is not provided", function(machine) assert(machine:read_reg("mcycle") == 1) @@ -909,13 +863,15 @@ test_util.make_do_test(build_machine, machine_type, { uarch = {} })( local test_reset_uarch_config = { processor = { - halt_flag = 1, - cycle = 1, - pc = 0, + registers = { + halt_flag = 1, + cycle = 1, + pc = 0, + }, }, } for i = 0, 31 do - test_reset_uarch_config.processor["x" .. i] = 0x10000 + (i * 8) + test_reset_uarch_config.processor.registers["x" .. i] = 0x10000 + (i * 8) end local function test_reset_uarch(machine, with_log, with_annotations) @@ -924,7 +880,7 @@ local function test_reset_uarch(machine, with_log, with_annotations) assert(machine:read_reg("uarch_cycle") == 1) assert(machine:read_reg("uarch_pc") == 0) for i = 1, 31 do - assert(machine:read_reg("uarch_x" .. i) == test_reset_uarch_config.processor["x" .. i]) + assert(machine:read_reg("uarch_x" .. i) == test_reset_uarch_config.processor.registers["x" .. i]) end -- modify uarch ram local gibberish = "mydataol12345678" @@ -1195,7 +1151,7 @@ do_test("uarch ecall putchar should print char to console", function() } local uarch_ram_path = test_util.create_test_uarch_program(program) local machine = cartesi.machine { - processor = initial_reg_values, + registers = initial_reg_values, ram = {length = 1 << 20}, uarch = { ram = { backing_store = { data_filename = uarch_ram_path } } @@ -1336,14 +1292,14 @@ do_test("Dump of log produced by send_cmio_response should match", function(mach local data = "0123456789" local reason = 7 local log = machine:log_send_cmio_response(reason, data, cartesi.ACCESS_LOG_TYPE_ANNOTATIONS) - -- luacheck: push no max line length - local expected_dump = "begin send_cmio_response\n" - .. " 1: read iflags.Y@0x2f8(760): 0x1(1)\n" - .. ' 2: write cmio rx buffer@0x60000000(1610612736): hash:"290decd9"(2^5 bytes) -> hash:"555b1f6d"(2^5 bytes)\n' - .. " 3: write htif.fromhost@0x330(816): 0x0(0) -> 0x70000000a(30064771082)\n" - .. " 4: write iflags.Y@0x2f8(760): 0x1(1) -> 0x0(0)\n" - .. "end send_cmio_response\n" - -- luacheck: pop + local expected_dump = [[ +begin send_cmio_response + 1: read iflags.Y@0x300(768): 0x1(1) + 2: write cmio rx buffer@0x60000000(1610612736): hash:"290decd9"(2^5 bytes) -> hash:"555b1f6d"(2^5 bytes) + 3: write htif.fromhost@0x330(816): 0x0(0) -> 0x70000000a(30064771082) + 4: write iflags.Y@0x300(768): 0x1(1) -> 0x0(0) +end send_cmio_response +]] local temp_file = test_util.new_temp_file() util.dump_log(log, temp_file) local actual_dump = temp_file:read_all() diff --git a/tests/lua/machine-test.lua b/tests/lua/machine-test.lua index a6e287a88..90bb4fe64 100755 --- a/tests/lua/machine-test.lua +++ b/tests/lua/machine-test.lua @@ -154,9 +154,9 @@ do_test("machine halt and yield flags and config matches", function(machine) -- Get machine default config and test for known fields local initial_config = machine:get_initial_config() -- test_util.print_table(initial_config) - assert(initial_config["processor"]["marchid"] == cartesi.MARCHID, "marchid value does not match") - assert(initial_config["processor"]["pc"] == cartesi.AR_RAM_START, "pc value does not match") - assert(initial_config["ram"]["length"] == 1048576, "ram length value does not match") + assert(initial_config.processor.registers.marchid == cartesi.MARCHID, "marchid value does not match") + assert(initial_config.processor.registers.pc == cartesi.AR_RAM_START, "pc value does not match") + assert(initial_config.ram.length == 1048576, "ram length value does not match") -- Check machine is not halted assert(machine:read_reg("iflags_H") == 0, "machine shouldn't be halted") -- Check machine is not yielded @@ -340,7 +340,6 @@ end) print("\n\n check replace flash drives") test_util.make_do_test(build_machine, machine_type, { - processor = {}, ram = { length = 1 << 20 }, flash_drive = { { diff --git a/tests/lua/run-rv64i-arch-test.lua b/tests/lua/run-rv64i-arch-test.lua index bce7540c6..bbed08695 100755 --- a/tests/lua/run-rv64i-arch-test.lua +++ b/tests/lua/run-rv64i-arch-test.lua @@ -49,7 +49,6 @@ local config = { }, }, }, - processor = {}, ram = { length = 0x1000 }, } diff --git a/tests/lua/test-jsonrpc-fork.lua b/tests/lua/test-jsonrpc-fork.lua index 97ffcb7b1..8c7ea6146 100755 --- a/tests/lua/test-jsonrpc-fork.lua +++ b/tests/lua/test-jsonrpc-fork.lua @@ -117,7 +117,7 @@ local function fork_tree(address, x, depth) if depth == 0 then local config = machine:get_default_config() x = {} - for k, v in pairs(config.processor) do + for k, v in pairs(config.processor.registers) do if k:sub(1, 1) == "x" then x[tonumber(k:sub(2))] = v end diff --git a/tests/misc/test-machine-c-api.cpp b/tests/misc/test-machine-c-api.cpp index 03508d487..9ee8a51cd 100644 --- a/tests/misc/test-machine-c-api.cpp +++ b/tests/misc/test-machine-c-api.cpp @@ -163,7 +163,9 @@ class machine_flash_simple_fixture : public incomplete_machine_fixture { public: machine_flash_simple_fixture() { _machine_config["flash_drive"] = {{{"start", 0x80000000000000}, {"length", 0x3c00000}, {"read_only", false}, - {"backing_store", {{"shared", false}, {"truncate", false}, {"data_filename", ""}, {"dht_filename", ""}}}}}; + {"backing_store", + {{"shared", false}, {"create", false}, {"truncate", false}, {"data_filename", ""}, + {"dht_filename", ""}}}}}; } machine_flash_simple_fixture(const machine_flash_simple_fixture &other) = delete; @@ -313,7 +315,7 @@ BOOST_FIXTURE_TEST_CASE_NOLINT(store_machine_config_version_test, store_file_fix auto j = nlohmann::json::parse(ifs); BOOST_REQUIRE(j.contains("archive_version")); BOOST_REQUIRE(j["archive_version"].is_number_integer()); - BOOST_CHECK_EQUAL(j["archive_version"].get(), 5); + BOOST_CHECK_EQUAL(j["archive_version"].get(), 6); } BOOST_FIXTURE_TEST_CASE_NOLINT(store_null_machine_test, ordinary_machine_fixture) { @@ -535,14 +537,14 @@ BOOST_FIXTURE_TEST_CASE_NOLINT(read_write_virtual_memory_null_data_test, ordinar BOOST_FIXTURE_TEST_CASE_NOLINT(write_memory_invalid_address_range_test, ordinary_machine_fixture) { uint64_t write_value = 0x1234; - uint64_t address = 0x100; + uint64_t address = 0x40008000; // HTIF std::array write_data{}; memcpy(write_data.data(), &write_value, write_data.size()); cm_error error_code = cm_write_memory(_machine, address, write_data.data(), write_data.size()); BOOST_CHECK_EQUAL(error_code, CM_ERROR_INVALID_ARGUMENT); std::string result = cm_get_last_error_message(); - std::string origin("address range to write is not entirely in single memory range"); + std::string origin("attempted write to device memory range"); BOOST_CHECK_EQUAL(origin, result); } @@ -628,14 +630,14 @@ BOOST_FIXTURE_TEST_CASE_NOLINT(read_write_memory_massive_test, ordinary_machine_ BOOST_FIXTURE_TEST_CASE_NOLINT(write_virtual_memory_invalid_address_range_test, ordinary_machine_fixture) { uint64_t write_value = 0x1234; - uint64_t address = 0x100; + uint64_t address = 0x40008000; // HTIF std::array write_data{}; memcpy(write_data.data(), &write_value, write_data.size()); cm_error error_code = cm_write_virtual_memory(_machine, address, write_data.data(), write_data.size()); BOOST_CHECK_EQUAL(error_code, CM_ERROR_INVALID_ARGUMENT); std::string result = cm_get_last_error_message(); - std::string origin("address range to write is not entirely in single memory range"); + std::string origin("attempted write to device memory range"); BOOST_CHECK_EQUAL(origin, result); } @@ -1017,7 +1019,7 @@ BOOST_FIXTURE_TEST_CASE_NOLINT(read_write_f_basic_test, ordinary_machine_fixture uint64_t f2_addr{}; BOOST_CHECK_EQUAL(cm_get_reg_address(_machine, CM_REG_F2, &f2_addr), CM_ERROR_OK); - BOOST_CHECK_EQUAL(static_cast(0x110), f2_addr); + BOOST_CHECK_EQUAL(static_cast(0x128), f2_addr); } BOOST_FIXTURE_TEST_CASE_NOLINT(read_write_uarch_x_basic_test, ordinary_machine_fixture) { @@ -1068,7 +1070,7 @@ BOOST_FIXTURE_TEST_CASE_NOLINT(read_write_reg_basic_test, ordinary_machine_fixtu uint64_t pc_addr{}; BOOST_CHECK_EQUAL(cm_get_reg_address(_machine, CM_REG_PC, &pc_addr), CM_ERROR_OK); - BOOST_CHECK_EQUAL(static_cast(0x200), pc_addr); + BOOST_CHECK_EQUAL(static_cast(0x108), pc_addr); } BOOST_AUTO_TEST_CASE_NOLINT(verify_merkle_tree_null_machine_test) { diff --git a/uarch/compute-uarch-pristine-hash.cpp b/uarch/compute-uarch-pristine-hash.cpp index 87ed4f92e..d3d0edca9 100644 --- a/uarch/compute-uarch-pristine-hash.cpp +++ b/uarch/compute-uarch-pristine-hash.cpp @@ -22,7 +22,7 @@ #include #include #include -#include +#include #include /// \file diff --git a/uarch/machine-uarch-bridge-state-access.h b/uarch/machine-uarch-bridge-state-access.h index 65230497e..40adec521 100644 --- a/uarch/machine-uarch-bridge-state-access.h +++ b/uarch/machine-uarch-bridge-state-access.h @@ -88,11 +88,11 @@ class machine_uarch_bridge_state_access : // ----- friend i_prefer_shadow_state; - uint64_t do_read_shadow_state(shadow_state_what what) const { + uint64_t do_read_shadow_register(shadow_registers_what what) const { return bridge_read_reg(machine_reg_enum(what)); } - void do_write_shadow_state(shadow_state_what what, uint64_t val) const { + void do_write_shadow_register(shadow_registers_what what, uint64_t val) const { bridge_write_reg(machine_reg_enum(what), val); } @@ -151,7 +151,7 @@ class machine_uarch_bridge_state_access : } template - uint64_t do_read_tlb_vp_offset(uint64_t slot_index) const { + uint64_t do_read_tlb_vf_offset(uint64_t slot_index) const { return bridge_read_shadow_tlb(SET, slot_index, shadow_tlb_what::vp_offset); }