From 6c9f9cc1dc7231507c7442e37ee44238c3d8f4a3 Mon Sep 17 00:00:00 2001 From: Diego Nehab <1635557+diegonehab@users.noreply.github.com> Date: Mon, 13 Jan 2025 13:14:24 +0000 Subject: [PATCH 01/82] refactor: unify mock_pma_entry implementations --- src/mock-pma-entry.h | 165 +++++++++++++++++++++++++++++ src/pma.cpp | 12 --- src/pma.h | 18 ---- src/replay-step-state-access.h | 141 ++---------------------- uarch/uarch-machine-state-access.h | 162 +++------------------------- 5 files changed, 187 insertions(+), 311 deletions(-) create mode 100644 src/mock-pma-entry.h diff --git a/src/mock-pma-entry.h b/src/mock-pma-entry.h new file mode 100644 index 000000000..883bd6725 --- /dev/null +++ b/src/mock-pma-entry.h @@ -0,0 +1,165 @@ +// Copyright Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: LGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify it under +// the terms of the GNU Lesser General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) any +// later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +// PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License along +// with this program (see COPYING). If not, see . +// + +#ifndef MOCK_PMA_ENTRY_H +#define MOCK_PMA_ENTRY_H + +#include "clint.h" +#include "htif.h" +#include "plic.h" +#include "pma-constants.h" +#include "shadow-state.h" +#include "shadow-tlb.h" + +namespace cartesi { + +class mock_pma_entry { +public: + struct flags { + bool M; + bool IO; + bool E; + bool R; + bool W; + bool X; + bool IR; + bool IW; + PMA_ISTART_DID DID; + }; + +private: + uint64_t m_pma_index; + uint64_t m_start; + uint64_t m_length; + flags m_flags; + const pma_driver *m_driver{nullptr}; + + static constexpr flags split_flags(uint64_t istart) { + flags f{}; + f.M = ((istart & PMA_ISTART_M_MASK) >> PMA_ISTART_M_SHIFT) != 0; + f.IO = ((istart & PMA_ISTART_IO_MASK) >> PMA_ISTART_IO_SHIFT) != 0; + f.E = ((istart & PMA_ISTART_E_MASK) >> PMA_ISTART_E_SHIFT) != 0; + f.R = ((istart & PMA_ISTART_R_MASK) >> PMA_ISTART_R_SHIFT) != 0; + f.W = ((istart & PMA_ISTART_W_MASK) >> PMA_ISTART_W_SHIFT) != 0; + f.X = ((istart & PMA_ISTART_X_MASK) >> PMA_ISTART_X_SHIFT) != 0; + f.IR = ((istart & PMA_ISTART_IR_MASK) >> PMA_ISTART_IR_SHIFT) != 0; + f.IW = ((istart & PMA_ISTART_IW_MASK) >> PMA_ISTART_IW_SHIFT) != 0; + f.DID = static_cast((istart & PMA_ISTART_DID_MASK) >> PMA_ISTART_DID_SHIFT); + return f; + } + +public: + template + mock_pma_entry(uint64_t pma_index, uint64_t istart, uint64_t ilength, ERR_F errf) : + m_pma_index{pma_index}, + m_start{istart & PMA_ISTART_START_MASK}, + m_length{ilength}, + m_flags{split_flags(istart)} { + if (m_flags.IO) { + switch (m_flags.DID) { + case PMA_ISTART_DID::shadow_state: + m_driver = &shadow_state_driver; + break; + case PMA_ISTART_DID::shadow_TLB: + m_driver = &shadow_tlb_driver; + break; + case PMA_ISTART_DID::CLINT: + m_driver = &clint_driver; + break; + case PMA_ISTART_DID::PLIC: + m_driver = &plic_driver; + break; + case PMA_ISTART_DID::HTIF: + m_driver = &htif_driver; + break; + default: + errf("unsupported device in mock_pma_entry"); + break; + } + } + } + + uint64_t get_index() const { + return m_pma_index; + } + + flags get_flags() const { + return m_flags; + } + + uint64_t get_start() const { + return m_start; + } + + uint64_t get_length() const { + return m_length; + } + + bool get_istart_M() const { + return m_flags.M; + } + + bool get_istart_IO() const { + return m_flags.IO; + } + + bool get_istart_E() const { + return m_flags.E; + } + + bool get_istart_R() const { + return m_flags.R; + } + + bool get_istart_W() const { + return m_flags.W; + } + + bool get_istart_X() const { + return m_flags.X; + } + + bool get_istart_IR() const { + return m_flags.IR; + } + + const auto *get_driver() const { + return m_driver; + } + + const auto &get_device_noexcept() const { + return *this; + } + + static void *get_context() { + return nullptr; + } + + // NOLINTNEXTLINE(readability-convert-member-functions-to-static) + void mark_dirty_page(uint64_t address_in_range) { + (void) address_in_range; + // Dummy implementation. + } +}; + +template +static inline mock_pma_entry make_mock_pma_entry(uint64_t index, uint64_t istart, uint64_t ilength, ERR_F errf) { + return mock_pma_entry{index, istart, ilength, errf}; +} + +} // namespace cartesi + +#endif diff --git a/src/pma.cpp b/src/pma.cpp index 45d29a5e5..9a112b4dd 100644 --- a/src/pma.cpp +++ b/src/pma.cpp @@ -73,11 +73,6 @@ pma_memory::pma_memory(const std::string &description, uint64_t length, const ca } } -pma_memory::pma_memory(const std::string & /*description*/, uint64_t length, const mockd & /*m*/) : - m_length{length}, - m_host_memory{nullptr}, - m_mmapped{false} {} - pma_memory::pma_memory(const std::string &description, uint64_t length, const std::string &path, const callocd &c) : pma_memory{description, length, c} { // Try to load image file, if any @@ -259,13 +254,6 @@ pma_entry make_callocd_memory_pma_entry(const std::string &description, uint64_t memory_peek}; } -pma_entry make_mockd_memory_pma_entry(const std::string &description, uint64_t start, uint64_t length) { - if (length == 0) { - throw std::invalid_argument{description + " length cannot be zero"s}; - } - return pma_entry{description, start, length, pma_memory{description, length, pma_memory::mockd{}}, pma_peek_error}; -} - pma_entry make_device_pma_entry(const std::string &description, uint64_t start, uint64_t length, pma_peek peek, const pma_driver *driver, void *context) { if (length == 0) { diff --git a/src/pma.h b/src/pma.h index 2224cfdcf..7c2ae90aa 100644 --- a/src/pma.h +++ b/src/pma.h @@ -123,9 +123,6 @@ class pma_memory final { /// \brief Calloc'd range data (just a tag). struct callocd {}; - /// \brief Mock'd range data (just a tag). - struct mockd {}; - /// \brief Constructor for calloc'd ranges. /// \param description Informative description of PMA entry for use in error messages /// \param length Length of range. @@ -139,12 +136,6 @@ class pma_memory final { /// \param c Calloc'd range data (just a tag). pma_memory(const std::string &description, uint64_t length, const callocd &c); - /// \brief Constructor for mock ranges. - /// \param description Informative description of PMA entry for use in error messages - /// \param length Length of range. - /// \param m Mock'd range data (just a tag). - pma_memory(const std::string &description, uint64_t length, const mockd &m); - /// \brief No copy constructor pma_memory(const pma_memory &other) = delete; @@ -551,15 +542,6 @@ pma_entry make_callocd_memory_pma_entry(const std::string &description, uint64_t pma_entry make_mmapd_memory_pma_entry(const std::string &description, uint64_t start, uint64_t length, const std::string &path, bool shared); -/// \brief Creates a PMA entry for a new mock memory region (no allocation). -/// \param description Informative description of PMA entry for use in error messages -/// \param start Start of physical memory range in the target address -/// space on which to map the memory region. -/// \param length Length of physical memory range in the -/// target address space on which to map the memory region. -/// \returns Corresponding PMA entry -pma_entry make_mockd_memory_pma_entry(const std::string &description, uint64_t start, uint64_t length); - /// \brief Creates a PMA entry for a new memory-mapped IO device. /// \param description Informative description of PMA entry for use in error messages /// \param start Start of physical memory range in the target address diff --git a/src/replay-step-state-access.h b/src/replay-step-state-access.h index b8d61d3d2..6f1162223 100644 --- a/src/replay-step-state-access.h +++ b/src/replay-step-state-access.h @@ -20,12 +20,10 @@ #include #include -#include "clint.h" #include "compiler-defines.h" #include "device-state-access.h" -#include "htif.h" #include "i-state-access.h" -#include "plic.h" +#include "mock-pma-entry.h" #include "pma-constants.h" #include "replay-step-state-access-interop.h" #include "riscv-constants.h" @@ -42,136 +40,6 @@ namespace cartesi { // \file this code is designed to be compiled for a free-standing environment. // Environment-specific functions have the prefix "interop_" and are declared in "replay-step-state-access-interop.h" -//??D can we merge this calss with uarch_pma_entry in uarch/uarch-machine-state-access.h ? -//??D maybe implement base class in pma.h and subclass here if there is some minor difference? -class mock_pma_entry final { -public: - struct flags { - bool M; - bool IO; - bool E; - bool R; - bool W; - bool X; - bool IR; - bool IW; - PMA_ISTART_DID DID; - }; - -private: - uint64_t m_pma_index; - uint64_t m_start; - uint64_t m_length; - flags m_flags; - const pma_driver *m_driver{nullptr}; - - static constexpr flags split_flags(uint64_t istart) { - flags f{}; - f.M = ((istart & PMA_ISTART_M_MASK) >> PMA_ISTART_M_SHIFT) != 0; - f.IO = ((istart & PMA_ISTART_IO_MASK) >> PMA_ISTART_IO_SHIFT) != 0; - f.E = ((istart & PMA_ISTART_E_MASK) >> PMA_ISTART_E_SHIFT) != 0; - f.R = ((istart & PMA_ISTART_R_MASK) >> PMA_ISTART_R_SHIFT) != 0; - f.W = ((istart & PMA_ISTART_W_MASK) >> PMA_ISTART_W_SHIFT) != 0; - f.X = ((istart & PMA_ISTART_X_MASK) >> PMA_ISTART_X_SHIFT) != 0; - f.IR = ((istart & PMA_ISTART_IR_MASK) >> PMA_ISTART_IR_SHIFT) != 0; - f.IW = ((istart & PMA_ISTART_IW_MASK) >> PMA_ISTART_IW_SHIFT) != 0; - f.DID = static_cast((istart & PMA_ISTART_DID_MASK) >> PMA_ISTART_DID_SHIFT); - return f; - } - -public: - mock_pma_entry(uint64_t pma_index, uint64_t istart, uint64_t ilength) : - m_pma_index{pma_index}, - m_start{istart & PMA_ISTART_START_MASK}, - m_length{ilength}, - m_flags{split_flags(istart)} { - if (m_flags.IO) { - switch (m_flags.DID) { - case PMA_ISTART_DID::shadow_state: - m_driver = &shadow_state_driver; - break; - case PMA_ISTART_DID::shadow_TLB: - m_driver = &shadow_tlb_driver; - break; - case PMA_ISTART_DID::CLINT: - m_driver = &clint_driver; - break; - case PMA_ISTART_DID::PLIC: - m_driver = &plic_driver; - break; - case PMA_ISTART_DID::HTIF: - m_driver = &htif_driver; - break; - default: - interop_throw_runtime_error("unsupported device in build_mock_pma_entry"); - break; - } - } - } - - uint64_t get_index() const { - return m_pma_index; - } - - flags get_flags() const { - return m_flags; - } - - uint64_t get_start() const { - return m_start; - } - - uint64_t get_length() const { - return m_length; - } - - bool get_istart_M() const { - return m_flags.M; - } - - bool get_istart_IO() const { - return m_flags.IO; - } - - bool get_istart_E() const { - return m_flags.E; - } - - bool get_istart_R() const { - return m_flags.R; - } - - bool get_istart_W() const { - return m_flags.W; - } - - bool get_istart_X() const { - return m_flags.X; - } - - bool get_istart_IR() const { - return m_flags.IR; - } - - const auto *get_driver() const { - return m_driver; - } - - const auto &get_device_noexcept() const { - return *this; - } - - static void *get_context() { - return nullptr; - } - - // NOLINTNEXTLINE(readability-convert-member-functions-to-static) - void mark_dirty_page(uint64_t address_in_range) { - (void) address_in_range; - // Dummy implementation. - } -}; - // \brief checks if a buffer is large enough to hold a data block of N elements of size S starting at a given offset // \param max The maximum offset allowed // \param current The current offset @@ -896,8 +764,11 @@ class replay_step_state_access : public i_state_access(index); + if (!m_pmas[i]) { + m_pmas[i] = + make_mock_pma_entry(index, istart, ilength, [](const char *err) { interop_throw_runtime_error(err); }); } // NOLINTNEXTLINE(bugprone-unchecked-optional-access) return m_context.pmas[index].value(); diff --git a/uarch/uarch-machine-state-access.h b/uarch/uarch-machine-state-access.h index aa9346eb9..d43c64d86 100644 --- a/uarch/uarch-machine-state-access.h +++ b/uarch/uarch-machine-state-access.h @@ -19,10 +19,9 @@ #include "uarch-runtime.h" // must be included first, because of assert -#include "clint.h" -#include "plic.h" +#define MOCK_THROW_RUNTIME_ERROR(err) abort() +#include "mock-pma-entry.h" #include "device-state-access.h" -#include "htif.h" #include "i-state-access.h" #include "pma-constants.h" #include "riscv-constants.h" @@ -50,142 +49,9 @@ static void raw_write_memory(uint64_t paddr, T val) { *p = val; } -//??D can we merge this class with the mock_pma_entry in src/replay-step-state-access.h ? -//??D maybe implement base class in pma.h and subclass here if there is some minor difference? -class uarch_pma_entry final { -public: - struct flags { - bool M{}; - bool IO{}; - bool E{}; - bool R{}; - bool W{}; - bool X{}; - bool IR{}; - bool IW{}; - PMA_ISTART_DID DID{PMA_ISTART_DID::memory}; - }; - -private: - - uint64_t m_pma_index; - uint64_t m_start; - uint64_t m_length; - flags m_flags; - const pma_driver *m_driver{nullptr}; - - static constexpr flags split_flags(uint64_t istart) { - flags f{}; - f.M = (((istart & PMA_ISTART_M_MASK) >> PMA_ISTART_M_SHIFT) != 0); - f.IO = (((istart & PMA_ISTART_IO_MASK) >> PMA_ISTART_IO_SHIFT) != 0); - f.E = (((istart & PMA_ISTART_E_MASK) >> PMA_ISTART_E_SHIFT) != 0); - f.R = (((istart & PMA_ISTART_R_MASK) >> PMA_ISTART_R_SHIFT) != 0); - f.W = (((istart & PMA_ISTART_W_MASK) >> PMA_ISTART_W_SHIFT) != 0); - f.X = (((istart & PMA_ISTART_X_MASK) >> PMA_ISTART_X_SHIFT) != 0); - f.IR = (((istart & PMA_ISTART_IR_MASK) >> PMA_ISTART_IR_SHIFT) != 0); - f.IW = (((istart & PMA_ISTART_IW_MASK) >> PMA_ISTART_IW_SHIFT) != 0); - f.DID = static_cast((istart & PMA_ISTART_DID_MASK) >> PMA_ISTART_DID_SHIFT); - return f; - } - -public: - - uarch_pma_entry(uint64_t pma_index, uint64_t istart, uint64_t ilength) : - m_pma_index{pma_index}, - m_start{istart & PMA_ISTART_START_MASK}, - m_length{ilength}, - m_flags{split_flags(istart)} { - if (m_flags.IO) { - switch (m_flags.DID) { - case PMA_ISTART_DID::shadow_state: - m_driver = &shadow_state_driver; - break; - case PMA_ISTART_DID::shadow_TLB: - m_driver = &shadow_tlb_driver; - break; - case PMA_ISTART_DID::CLINT: - m_driver = &clint_driver; - break; - case PMA_ISTART_DID::PLIC: - m_driver = &plic_driver; - break; - case PMA_ISTART_DID::HTIF: - m_driver = &htif_driver; - break; - default: - // Other unsupported device in uarch (eg. VirtIO) - abort(); - break; - } - } - } - - uint64_t get_index() const { - return m_pma_index; - } - - flags get_flags() const { - return m_flags; - } - - uint64_t get_start() const { - return m_start; - } - - uint64_t get_length() const { - return m_length; - } - - bool get_istart_M() const { - return m_flags.M; - } - - bool get_istart_IO() const { - return m_flags.IO; - } - - bool get_istart_E() const { - return m_flags.E; - } - - bool get_istart_R() const { - return m_flags.R; - } - - bool get_istart_W() const { - return m_flags.W; - } - - bool get_istart_X() const { - return m_flags.X; - } - - bool get_istart_IR() const { - return m_flags.IR; - } - - const pma_driver *get_driver() const { - return m_driver; - } - - const auto &get_device_noexcept() const { - return *this; - } - - static void *get_context() { - return nullptr; - } - - void mark_dirty_page(uint64_t /*address_in_range*/) { - // Dummy implementation here. - // This runs in microarchitecture. - // The Host pages affected by writes will be marked dirty by uarch_bridge. - } -}; - // Provides access to the state of the big emulator from microcode -class uarch_machine_state_access : public i_state_access { - std::array, PMA_MAX> &m_pmas; //NOLINT(cppcoreguidelines-avoid-const-or-ref-data-members) +class uarch_machine_state_access : public i_state_access { + std::array, PMA_MAX> m_pmas; public: explicit uarch_machine_state_access(std::array, PMA_MAX>& pmas) : m_pmas(pmas) {} @@ -196,7 +62,7 @@ class uarch_machine_state_access : public i_state_access; + friend i_state_access; // NOLINTBEGIN(readability-convert-member-functions-to-static) @@ -569,17 +435,21 @@ class uarch_machine_state_access : public i_state_access(index); + if (!m_pmas[i]) { + m_pmas[i] = make_mock_pma_entry(index, istart, ilength, [](const char * /*err*/) { + abort(); + }); } // NOLINTNEXTLINE(bugprone-unchecked-optional-access) return m_pmas[index].value(); } - unsigned char *do_get_host_memory(uarch_pma_entry &/*pma*/) { + unsigned char *do_get_host_memory(mock_pma_entry &/*pma*/) { return nullptr; } @@ -639,14 +509,14 @@ class uarch_machine_state_access : public i_state_access - unsigned char *do_replace_tlb_entry(uint64_t vaddr, uint64_t paddr, uarch_pma_entry &pma) { + unsigned char *do_replace_tlb_entry(uint64_t vaddr, uint64_t paddr, mock_pma_entry &pma) { uint64_t eidx = tlb_get_entry_index(vaddr); volatile tlb_cold_entry &tlbce = do_get_tlb_entry_cold(eidx); volatile tlb_hot_entry &tlbhe = do_get_tlb_hot_entry(eidx); // Mark page that was on TLB as dirty so we know to update the Merkle tree if constexpr (ETYPE == TLB_WRITE) { if (tlbhe.vaddr_page != TLB_INVALID_PAGE) { - uarch_pma_entry &pma = do_read_pma_entry(tlbce.pma_index); + mock_pma_entry &pma = do_read_pma_entry(tlbce.pma_index); pma.mark_dirty_page(tlbce.paddr_page - pma.get_start()); } } @@ -675,7 +545,7 @@ class uarch_machine_state_access : public i_state_access(eidx); - uarch_pma_entry &pma = do_read_pma_entry(tlbce.pma_index); + mock_pma_entry &pma = do_read_pma_entry(tlbce.pma_index); pma.mark_dirty_page(tlbce.paddr_page - pma.get_start()); } else { tlbhe.vaddr_page = TLB_INVALID_PAGE; From 1153cc63b6899829a68c40cfd8d455a7ca9b050f Mon Sep 17 00:00:00 2001 From: Diego Nehab <1635557+diegonehab@users.noreply.github.com> Date: Mon, 20 Jan 2025 14:39:28 +0000 Subject: [PATCH 02/82] refactor: remove htif depencency on os.h --- src/Makefile | 2 +- src/device-state-access.h | 8 + src/htif.cpp | 6 +- src/i-device-state-access.h | 14 + src/i-state-access.h | 12 + src/record-step-state-access.h | 8 + src/replay-step-state-access.h | 8 + src/state-access.h | 11 + uarch/uarch-machine-state-access.h | 41 +- uarch/uarch-printf.c | 1834 ++++++++++++++-------------- uarch/uarch-printf.h | 225 ++-- uarch/uarch-runtime.cpp | 20 - uarch/uarch-runtime.h | 2 +- 13 files changed, 1113 insertions(+), 1078 deletions(-) diff --git a/src/Makefile b/src/Makefile index eff7a414c..2fcc1a4c4 100644 --- a/src/Makefile +++ b/src/Makefile @@ -282,7 +282,7 @@ CLANG_TIDY=clang-tidy CLANG_TIDY_TARGETS=$(patsubst %.cpp,%.clang-tidy,$(patsubst %.c,%.clang-tidy,$(LINTER_SOURCES))) CLANG_FORMAT=clang-format -CLANG_FORMAT_UARCH_FILES:=$(wildcard ../uarch/*.cpp) +CLANG_FORMAT_UARCH_FILES:=$(wildcard ../uarch/*.cpp, ../uarch/*.h) CLANG_FORMAT_UARCH_FILES:=$(filter-out %uarch-printf%,$(strip $(CLANG_FORMAT_UARCH_FILES))) CLANG_FORMAT_FILES:=$(wildcard *.cpp) $(wildcard *.c) $(wildcard *.h) $(wildcard *.hpp) $(CLANG_FORMAT_UARCH_FILES) CLANG_FORMAT_IGNORE_FILES:=interpret-jump-table.h diff --git a/src/device-state-access.h b/src/device-state-access.h index 97dcad701..812d2c1cb 100644 --- a/src/device-state-access.h +++ b/src/device-state-access.h @@ -146,6 +146,14 @@ class device_state_access : public i_device_state_access { bool do_write_memory(uint64_t paddr, const unsigned char *data, uint64_t length) override { return m_a.write_memory(paddr, data, length); } + + void do_putchar(uint8_t c) override { + return m_a.putchar(c); + } + + int do_getchar() override { + return m_a.getchar(); + } }; } // namespace cartesi diff --git a/src/htif.cpp b/src/htif.cpp index fc5b00b39..9ffcec2ef 100644 --- a/src/htif.cpp +++ b/src/htif.cpp @@ -19,7 +19,6 @@ #include "htif.h" #include "i-device-state-access.h" #include "interpret.h" -#include "os.h" #include "pma-driver.h" namespace cartesi { @@ -93,15 +92,14 @@ static execute_status htif_console(i_device_state_access *a, uint64_t cmd, uint6 if (cmd < 64 && (((a->read_htif_iconsole() >> cmd) & 1) != 0)) { if (cmd == HTIF_CONSOLE_CMD_PUTCHAR) { const uint8_t ch = data & 0xff; - os_putchar(ch); + a->putchar(ch); a->write_htif_fromhost(HTIF_BUILD(HTIF_DEV_CONSOLE, cmd, 0, 0)); } else if (cmd == HTIF_CONSOLE_CMD_GETCHAR) { // In blockchain, this command will never be enabled as there is no way to input the same character // to every participant in a dispute: where would c come from? So if the code reached here in the // blockchain, there must be some serious bug // In interactive mode, we just get the next character from the console and send it back in the ack - os_poll_tty(0); - const int c = os_getchar() + 1; + const int c = a->getchar() + 1; a->write_htif_fromhost(HTIF_BUILD(HTIF_DEV_CONSOLE, cmd, 0, static_cast(c))); } } diff --git a/src/i-device-state-access.h b/src/i-device-state-access.h index efdfaa06e..784e2c684 100644 --- a/src/i-device-state-access.h +++ b/src/i-device-state-access.h @@ -193,6 +193,18 @@ class i_device_state_access { return do_write_memory(paddr, data, length); } + /// \brief Writes a character to the console + /// \param c Character to output + void putchar(uint8_t c) { + do_putchar(c); + } + + /// \brief Reads a character from the console + /// \returns Character read if any, -1 otherwise + int getchar() { + return do_getchar(); + } + private: virtual void do_set_mip(uint64_t mask) = 0; virtual void do_reset_mip(uint64_t mask) = 0; @@ -216,6 +228,8 @@ class i_device_state_access { virtual uint64_t do_read_htif_iyield() = 0; virtual bool do_read_memory(uint64_t paddr, unsigned char *data, uint64_t length) = 0; virtual bool do_write_memory(uint64_t paddr, const unsigned char *data, uint64_t length) = 0; + virtual void do_putchar(uint8_t c) = 0; + virtual int do_getchar() = 0; }; } // namespace cartesi diff --git a/src/i-state-access.h b/src/i-state-access.h index 694d728ad..d255899ee 100644 --- a/src/i-state-access.h +++ b/src/i-state-access.h @@ -740,6 +740,18 @@ class i_state_access { // CRTP return derived().do_write_memory_with_padding(paddr, data, data_length, write_length_log2_size); } + /// \brief Writes a character to the console + /// \param c Character to output + void putchar(uint8_t c) { + return derived().do_putchar(c); + } + + /// \brief Reads a character from the console + /// \returns Character read if any, -1 otherwise + int getchar() { + return derived().do_getchar(); + } + #ifdef DUMP_COUNTERS auto &get_statistics() { return derived().do_get_statistics(); diff --git a/src/record-step-state-access.h b/src/record-step-state-access.h index b1b443d78..a54cc2555 100644 --- a/src/record-step-state-access.h +++ b/src/record-step-state-access.h @@ -756,6 +756,14 @@ class record_step_state_access : public i_state_access { } } + // NOLINTNEXTLINE(readability-convert-member-functions-to-static) + void do_putchar(uint8_t c) { + os_putchar(c); + } + + // NOLINTNEXTLINE(readability-convert-member-functions-to-static) + int do_getchar() { + os_poll_tty(0); + return os_getchar(); + } + #ifdef DUMP_COUNTERS machine_statistics &do_get_statistics() { return m_m.get_state().stats; diff --git a/uarch/uarch-machine-state-access.h b/uarch/uarch-machine-state-access.h index d43c64d86..6d21dae88 100644 --- a/uarch/uarch-machine-state-access.h +++ b/uarch/uarch-machine-state-access.h @@ -20,17 +20,17 @@ #include "uarch-runtime.h" // must be included first, because of assert #define MOCK_THROW_RUNTIME_ERROR(err) abort() -#include "mock-pma-entry.h" +#include "compiler-defines.h" #include "device-state-access.h" #include "i-state-access.h" +#include "machine-reg.h" +#include "mock-pma-entry.h" #include "pma-constants.h" #include "riscv-constants.h" -#include "machine-reg.h" #include "shadow-pmas.h" +#include "strict-aliasing.h" #include "uarch-constants.h" #include "uarch-defines.h" -#include "strict-aliasing.h" -#include "compiler-defines.h" #include namespace cartesi { @@ -66,10 +66,9 @@ class uarch_machine_state_access : public i_state_access - void do_read_memory_word(uint64_t paddr, const unsigned char */*hpage*/, uint64_t /*hoffset*/, T *pval) { + void do_read_memory_word(uint64_t paddr, const unsigned char * /*hpage*/, uint64_t /*hoffset*/, T *pval) { *pval = raw_read_memory(paddr); } - bool do_read_memory(uint64_t /*paddr*/, unsigned char */*data*/, uint64_t /*length*/) { + bool do_read_memory(uint64_t /*paddr*/, unsigned char * /*data*/, uint64_t /*length*/) { // This is not implemented yet because it's not being used abort(); return false; } - bool do_write_memory(uint64_t /*paddr*/, const unsigned char */*data*/, uint64_t /*length*/) { + bool do_write_memory(uint64_t /*paddr*/, const unsigned char * /*data*/, uint64_t /*length*/) { // This is not implemented yet because it's not being used abort(); return false; } template - void do_write_memory_word(uint64_t paddr, const unsigned char */*hpage*/, uint64_t /*hoffset*/, T val) { + void do_write_memory_word(uint64_t paddr, const unsigned char * /*hpage*/, uint64_t /*hoffset*/, T val) { raw_write_memory(paddr, val); } @@ -441,20 +440,18 @@ class uarch_machine_state_access : public i_state_access(index); if (!m_pmas[i]) { - m_pmas[i] = make_mock_pma_entry(index, istart, ilength, [](const char * /*err*/) { - abort(); - }); + m_pmas[i] = make_mock_pma_entry(index, istart, ilength, [](const char * /*err*/) { abort(); }); } // NOLINTNEXTLINE(bugprone-unchecked-optional-access) return m_pmas[index].value(); } - unsigned char *do_get_host_memory(mock_pma_entry &/*pma*/) { + unsigned char *do_get_host_memory(mock_pma_entry & /*pma*/) { return nullptr; } template - volatile tlb_hot_entry& do_get_tlb_hot_entry(uint64_t eidx) { + volatile tlb_hot_entry &do_get_tlb_hot_entry(uint64_t eidx) { // Volatile is used, so the compiler does not optimize out, or do of order writes. // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast,performance-no-int-to-ptr) volatile tlb_hot_entry *tlbe = reinterpret_cast(tlb_get_entry_hot_abs_addr(eidx)); @@ -462,7 +459,7 @@ class uarch_machine_state_access : public i_state_access - volatile tlb_cold_entry& do_get_tlb_entry_cold(uint64_t eidx) { + volatile tlb_cold_entry &do_get_tlb_entry_cold(uint64_t eidx) { // Volatile is used, so the compiler does not optimize out, or do of order writes. // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast,performance-no-int-to-ptr) volatile tlb_cold_entry *tlbe = reinterpret_cast(tlb_get_entry_cold_abs_addr(eidx)); @@ -534,7 +531,7 @@ class uarch_machine_state_access : public i_state_access(paddr_page); + return cast_addr_to_ptr(paddr_page); } template @@ -573,6 +570,14 @@ class uarch_machine_state_access : public i_state_access -#include - -#include "uarch-printf.h" - - -// define this globally (e.g. gcc -DPRINTF_INCLUDE_CONFIG_H ...) to include the -// printf_config.h header file -// default: undefined -#ifdef PRINTF_INCLUDE_CONFIG_H -#include "printf_config.h" -#endif - - -// 'ntoa' conversion buffer size, this must be big enough to hold one converted -// numeric number including padded zeros (dynamically created on stack) -// default: 32 byte -#ifndef PRINTF_NTOA_BUFFER_SIZE -#define PRINTF_NTOA_BUFFER_SIZE 32U -#endif - -// 'ftoa' conversion buffer size, this must be big enough to hold one converted -// float number including padded zeros (dynamically created on stack) -// default: 32 byte -#ifndef PRINTF_FTOA_BUFFER_SIZE -#define PRINTF_FTOA_BUFFER_SIZE 32U -#endif - -// support for the floating point type (%f) -// default: activated -#ifndef PRINTF_DISABLE_SUPPORT_FLOAT -#define PRINTF_SUPPORT_FLOAT -#endif - -// support for exponential floating point notation (%e/%g) -// default: activated -#ifndef PRINTF_DISABLE_SUPPORT_EXPONENTIAL -#define PRINTF_SUPPORT_EXPONENTIAL -#endif - -// define the default floating point precision -// default: 6 digits -#ifndef PRINTF_DEFAULT_FLOAT_PRECISION -#define PRINTF_DEFAULT_FLOAT_PRECISION 6U -#endif - -// define the largest float suitable to print with %f -// default: 1e9 -#ifndef PRINTF_MAX_FLOAT -#define PRINTF_MAX_FLOAT 1e9 -#endif - -// support for the long long types (%llu or %p) -// default: activated -#ifndef PRINTF_DISABLE_SUPPORT_LONG_LONG -#define PRINTF_SUPPORT_LONG_LONG -#endif - -// support for the ptrdiff_t type (%t) -// ptrdiff_t is normally defined in as long or long long type -// default: activated -#ifndef PRINTF_DISABLE_SUPPORT_PTRDIFF_T -#define PRINTF_SUPPORT_PTRDIFF_T -#endif - -/////////////////////////////////////////////////////////////////////////////// - -// internal flag definitions -#define FLAGS_ZEROPAD (1U << 0U) -#define FLAGS_LEFT (1U << 1U) -#define FLAGS_PLUS (1U << 2U) -#define FLAGS_SPACE (1U << 3U) -#define FLAGS_HASH (1U << 4U) -#define FLAGS_UPPERCASE (1U << 5U) -#define FLAGS_CHAR (1U << 6U) -#define FLAGS_SHORT (1U << 7U) -#define FLAGS_LONG (1U << 8U) -#define FLAGS_LONG_LONG (1U << 9U) -#define FLAGS_PRECISION (1U << 10U) -#define FLAGS_ADAPT_EXP (1U << 11U) - - -// import float.h for DBL_MAX -#if defined(PRINTF_SUPPORT_FLOAT) -#include -#endif - - -// output function type -typedef void (*out_fct_type)(char character, void* buffer, size_t idx, size_t maxlen); - - -// wrapper (used as buffer) for output function type -typedef struct { - void (*fct)(char character, void* arg); - void* arg; -} out_fct_wrap_type; - - -// internal buffer output -static inline void _out_buffer(char character, void* buffer, size_t idx, size_t maxlen) -{ - if (idx < maxlen) { - ((char*)buffer)[idx] = character; - } -} - - -// internal null output -static inline void _out_null(char character, void* buffer, size_t idx, size_t maxlen) -{ - (void)character; (void)buffer; (void)idx; (void)maxlen; -} - - -// internal _putchar wrapper -static inline void _out_char(char character, void* buffer, size_t idx, size_t maxlen) -{ - (void)buffer; (void)idx; (void)maxlen; - if (character) { - _putchar(character); - } -} - - -// internal output function wrapper -static inline void _out_fct(char character, void* buffer, size_t idx, size_t maxlen) -{ - (void)idx; (void)maxlen; - if (character) { - // buffer is the output fct pointer - ((out_fct_wrap_type*)buffer)->fct(character, ((out_fct_wrap_type*)buffer)->arg); - } -} - - -// internal secure strlen -// \return The length of the string (excluding the terminating 0) limited by 'maxsize' -static inline unsigned int _strnlen_s(const char* str, size_t maxsize) -{ - const char* s; - for (s = str; *s && maxsize--; ++s); - return (unsigned int)(s - str); -} - - -// internal test if char is a digit (0-9) -// \return true if char is a digit -static inline bool _is_digit(char ch) -{ - return (ch >= '0') && (ch <= '9'); -} - - -// internal ASCII string to unsigned int conversion -static unsigned int _atoi(const char** str) -{ - unsigned int i = 0U; - while (_is_digit(**str)) { - i = i * 10U + (unsigned int)(*((*str)++) - '0'); - } - return i; -} - - -// output the specified string in reverse, taking care of any zero-padding -static size_t _out_rev(out_fct_type out, char* buffer, size_t idx, size_t maxlen, const char* buf, size_t len, unsigned int width, unsigned int flags) -{ - const size_t start_idx = idx; - - // pad spaces up to given width - if (!(flags & FLAGS_LEFT) && !(flags & FLAGS_ZEROPAD)) { - for (size_t i = len; i < width; i++) { - out(' ', buffer, idx++, maxlen); - } - } - - // reverse string - while (len) { - out(buf[--len], buffer, idx++, maxlen); - } - - // append pad spaces up to given width - if (flags & FLAGS_LEFT) { - while (idx - start_idx < width) { - out(' ', buffer, idx++, maxlen); - } - } - - return idx; -} - - -// internal itoa format -static size_t _ntoa_format(out_fct_type out, char* buffer, size_t idx, size_t maxlen, char* buf, size_t len, bool negative, unsigned int base, unsigned int prec, unsigned int width, unsigned int flags) -{ - // pad leading zeros - if (!(flags & FLAGS_LEFT)) { - if (width && (flags & FLAGS_ZEROPAD) && (negative || (flags & (FLAGS_PLUS | FLAGS_SPACE)))) { - width--; - } - while ((len < prec) && (len < PRINTF_NTOA_BUFFER_SIZE)) { - buf[len++] = '0'; - } - while ((flags & FLAGS_ZEROPAD) && (len < width) && (len < PRINTF_NTOA_BUFFER_SIZE)) { - buf[len++] = '0'; - } - } - - // handle hash - if (flags & FLAGS_HASH) { - if (!(flags & FLAGS_PRECISION) && len && ((len == prec) || (len == width))) { - len--; - if (len && (base == 16U)) { - len--; - } - } - if ((base == 16U) && !(flags & FLAGS_UPPERCASE) && (len < PRINTF_NTOA_BUFFER_SIZE)) { - buf[len++] = 'x'; - } - else if ((base == 16U) && (flags & FLAGS_UPPERCASE) && (len < PRINTF_NTOA_BUFFER_SIZE)) { - buf[len++] = 'X'; - } - else if ((base == 2U) && (len < PRINTF_NTOA_BUFFER_SIZE)) { - buf[len++] = 'b'; - } - if (len < PRINTF_NTOA_BUFFER_SIZE) { - buf[len++] = '0'; - } - } - - if (len < PRINTF_NTOA_BUFFER_SIZE) { - if (negative) { - buf[len++] = '-'; - } - else if (flags & FLAGS_PLUS) { - buf[len++] = '+'; // ignore the space if the '+' exists - } - else if (flags & FLAGS_SPACE) { - buf[len++] = ' '; - } - } - - return _out_rev(out, buffer, idx, maxlen, buf, len, width, flags); -} - - -// internal itoa for 'long' type -static size_t _ntoa_long(out_fct_type out, char* buffer, size_t idx, size_t maxlen, unsigned long value, bool negative, unsigned long base, unsigned int prec, unsigned int width, unsigned int flags) -{ - char buf[PRINTF_NTOA_BUFFER_SIZE]; - size_t len = 0U; - - // no hash for 0 values - if (!value) { - flags &= ~FLAGS_HASH; - } - - // write if precision != 0 and value is != 0 - if (!(flags & FLAGS_PRECISION) || value) { - do { - const char digit = (char)(value % base); - buf[len++] = digit < 10 ? '0' + digit : (flags & FLAGS_UPPERCASE ? 'A' : 'a') + digit - 10; - value /= base; - } while (value && (len < PRINTF_NTOA_BUFFER_SIZE)); - } - - return _ntoa_format(out, buffer, idx, maxlen, buf, len, negative, (unsigned int)base, prec, width, flags); -} - - -// internal itoa for 'long long' type -#if defined(PRINTF_SUPPORT_LONG_LONG) -static size_t _ntoa_long_long(out_fct_type out, char* buffer, size_t idx, size_t maxlen, unsigned long long value, bool negative, unsigned long long base, unsigned int prec, unsigned int width, unsigned int flags) -{ - char buf[PRINTF_NTOA_BUFFER_SIZE]; - size_t len = 0U; - - // no hash for 0 values - if (!value) { - flags &= ~FLAGS_HASH; - } - - // write if precision != 0 and value is != 0 - if (!(flags & FLAGS_PRECISION) || value) { - do { - const char digit = (char)(value % base); - buf[len++] = digit < 10 ? '0' + digit : (flags & FLAGS_UPPERCASE ? 'A' : 'a') + digit - 10; - value /= base; - } while (value && (len < PRINTF_NTOA_BUFFER_SIZE)); - } - - return _ntoa_format(out, buffer, idx, maxlen, buf, len, negative, (unsigned int)base, prec, width, flags); -} -#endif // PRINTF_SUPPORT_LONG_LONG - - -#if defined(PRINTF_SUPPORT_FLOAT) - -#if defined(PRINTF_SUPPORT_EXPONENTIAL) -// forward declaration so that _ftoa can switch to exp notation for values > PRINTF_MAX_FLOAT -static size_t _etoa(out_fct_type out, char* buffer, size_t idx, size_t maxlen, double value, unsigned int prec, unsigned int width, unsigned int flags); -#endif - - -// internal ftoa for fixed decimal floating point -static size_t _ftoa(out_fct_type out, char* buffer, size_t idx, size_t maxlen, double value, unsigned int prec, unsigned int width, unsigned int flags) -{ - char buf[PRINTF_FTOA_BUFFER_SIZE]; - size_t len = 0U; - double diff = 0.0; - - // powers of 10 - static const double pow10[] = { 1, 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000 }; - - // test for special values - if (value != value) - return _out_rev(out, buffer, idx, maxlen, "nan", 3, width, flags); - if (value < -DBL_MAX) - return _out_rev(out, buffer, idx, maxlen, "fni-", 4, width, flags); - if (value > DBL_MAX) - return _out_rev(out, buffer, idx, maxlen, (flags & FLAGS_PLUS) ? "fni+" : "fni", (flags & FLAGS_PLUS) ? 4U : 3U, width, flags); - - // test for very large values - // standard printf behavior is to print EVERY whole number digit -- which could be 100s of characters overflowing your buffers == bad - if ((value > PRINTF_MAX_FLOAT) || (value < -PRINTF_MAX_FLOAT)) { -#if defined(PRINTF_SUPPORT_EXPONENTIAL) - return _etoa(out, buffer, idx, maxlen, value, prec, width, flags); -#else - return 0U; -#endif - } - - // test for negative - bool negative = false; - if (value < 0) { - negative = true; - value = 0 - value; - } - - // set default precision, if not set explicitly - if (!(flags & FLAGS_PRECISION)) { - prec = PRINTF_DEFAULT_FLOAT_PRECISION; - } - // limit precision to 9, cause a prec >= 10 can lead to overflow errors - while ((len < PRINTF_FTOA_BUFFER_SIZE) && (prec > 9U)) { - buf[len++] = '0'; - prec--; - } - - int whole = (int)value; - double tmp = (value - whole) * pow10[prec]; - unsigned long frac = (unsigned long)tmp; - diff = tmp - frac; - - if (diff > 0.5) { - ++frac; - // handle rollover, e.g. case 0.99 with prec 1 is 1.0 - if (frac >= pow10[prec]) { - frac = 0; - ++whole; - } - } - else if (diff < 0.5) { - } - else if ((frac == 0U) || (frac & 1U)) { - // if halfway, round up if odd OR if last digit is 0 - ++frac; - } - - if (prec == 0U) { - diff = value - (double)whole; - if ((!(diff < 0.5) || (diff > 0.5)) && (whole & 1)) { - // exactly 0.5 and ODD, then round up - // 1.5 -> 2, but 2.5 -> 2 - ++whole; - } - } - else { - unsigned int count = prec; - // now do fractional part, as an unsigned number - while (len < PRINTF_FTOA_BUFFER_SIZE) { - --count; - buf[len++] = (char)(48U + (frac % 10U)); - if (!(frac /= 10U)) { - break; - } - } - // add extra 0s - while ((len < PRINTF_FTOA_BUFFER_SIZE) && (count-- > 0U)) { - buf[len++] = '0'; - } - if (len < PRINTF_FTOA_BUFFER_SIZE) { - // add decimal - buf[len++] = '.'; - } - } - - // do whole part, number is reversed - while (len < PRINTF_FTOA_BUFFER_SIZE) { - buf[len++] = (char)(48 + (whole % 10)); - if (!(whole /= 10)) { - break; - } - } - - // pad leading zeros - if (!(flags & FLAGS_LEFT) && (flags & FLAGS_ZEROPAD)) { - if (width && (negative || (flags & (FLAGS_PLUS | FLAGS_SPACE)))) { - width--; - } - while ((len < width) && (len < PRINTF_FTOA_BUFFER_SIZE)) { - buf[len++] = '0'; - } - } - - if (len < PRINTF_FTOA_BUFFER_SIZE) { - if (negative) { - buf[len++] = '-'; - } - else if (flags & FLAGS_PLUS) { - buf[len++] = '+'; // ignore the space if the '+' exists - } - else if (flags & FLAGS_SPACE) { - buf[len++] = ' '; - } - } - - return _out_rev(out, buffer, idx, maxlen, buf, len, width, flags); -} - - -#if defined(PRINTF_SUPPORT_EXPONENTIAL) -// internal ftoa variant for exponential floating-point type, contributed by Martijn Jasperse -static size_t _etoa(out_fct_type out, char* buffer, size_t idx, size_t maxlen, double value, unsigned int prec, unsigned int width, unsigned int flags) -{ - // check for NaN and special values - if ((value != value) || (value > DBL_MAX) || (value < -DBL_MAX)) { - return _ftoa(out, buffer, idx, maxlen, value, prec, width, flags); - } - - // determine the sign - const bool negative = value < 0; - if (negative) { - value = -value; - } - - // default precision - if (!(flags & FLAGS_PRECISION)) { - prec = PRINTF_DEFAULT_FLOAT_PRECISION; - } - - // determine the decimal exponent - // based on the algorithm by David Gay (https://www.ampl.com/netlib/fp/dtoa.c) - union { - uint64_t U; - double F; - } conv; - - conv.F = value; - int exp2 = (int)((conv.U >> 52U) & 0x07FFU) - 1023; // effectively log2 - conv.U = (conv.U & ((1ULL << 52U) - 1U)) | (1023ULL << 52U); // drop the exponent so conv.F is now in [1,2) - // now approximate log10 from the log2 integer part and an expansion of ln around 1.5 - int expval = (int)(0.1760912590558 + exp2 * 0.301029995663981 + (conv.F - 1.5) * 0.289529654602168); - // now we want to compute 10^expval but we want to be sure it won't overflow - exp2 = (int)(expval * 3.321928094887362 + 0.5); - const double z = expval * 2.302585092994046 - exp2 * 0.6931471805599453; - const double z2 = z * z; - conv.U = (uint64_t)(exp2 + 1023) << 52U; - // compute exp(z) using continued fractions, see https://en.wikipedia.org/wiki/Exponential_function#Continued_fractions_for_ex - conv.F *= 1 + 2 * z / (2 - z + (z2 / (6 + (z2 / (10 + z2 / 14))))); - // correct for rounding errors - if (value < conv.F) { - expval--; - conv.F /= 10; - } - - // the exponent format is "%+03d" and largest value is "307", so set aside 4-5 characters - unsigned int minwidth = ((expval < 100) && (expval > -100)) ? 4U : 5U; - - // in "%g" mode, "prec" is the number of *significant figures* not decimals - if (flags & FLAGS_ADAPT_EXP) { - // do we want to fall-back to "%f" mode? - if ((value >= 1e-4) && (value < 1e6)) { - if ((int)prec > expval) { - prec = (unsigned)((int)prec - expval - 1); - } - else { - prec = 0; - } - flags |= FLAGS_PRECISION; // make sure _ftoa respects precision - // no characters in exponent - minwidth = 0U; - expval = 0; - } - else { - // we use one sigfig for the whole part - if ((prec > 0) && (flags & FLAGS_PRECISION)) { - --prec; - } - } - } - - // will everything fit? - unsigned int fwidth = width; - if (width > minwidth) { - // we didn't fall-back so subtract the characters required for the exponent - fwidth -= minwidth; - } else { - // not enough characters, so go back to default sizing - fwidth = 0U; - } - if ((flags & FLAGS_LEFT) && minwidth) { - // if we're padding on the right, DON'T pad the floating part - fwidth = 0U; - } - - // rescale the float value - if (expval) { - value /= conv.F; - } - - // output the floating part - const size_t start_idx = idx; - idx = _ftoa(out, buffer, idx, maxlen, negative ? -value : value, prec, fwidth, flags & ~FLAGS_ADAPT_EXP); - - // output the exponent part - if (minwidth) { - // output the exponential symbol - out((flags & FLAGS_UPPERCASE) ? 'E' : 'e', buffer, idx++, maxlen); - // output the exponent value - idx = _ntoa_long(out, buffer, idx, maxlen, (expval < 0) ? -expval : expval, expval < 0, 10, 0, minwidth-1, FLAGS_ZEROPAD | FLAGS_PLUS); - // might need to right-pad spaces - if (flags & FLAGS_LEFT) { - while (idx - start_idx < width) out(' ', buffer, idx++, maxlen); - } - } - return idx; -} -#endif // PRINTF_SUPPORT_EXPONENTIAL -#endif // PRINTF_SUPPORT_FLOAT - - -// internal vsnprintf -static int _vsnprintf(out_fct_type out, char* buffer, const size_t maxlen, const char* format, va_list va) -{ - unsigned int flags, width, precision, n; - size_t idx = 0U; - - if (!buffer) { - // use null output function - out = _out_null; - } - - while (*format) - { - // format specifier? %[flags][width][.precision][length] - if (*format != '%') { - // no - out(*format, buffer, idx++, maxlen); - format++; - continue; - } - else { - // yes, evaluate it - format++; - } - - // evaluate flags - flags = 0U; - do { - switch (*format) { - case '0': flags |= FLAGS_ZEROPAD; format++; n = 1U; break; - case '-': flags |= FLAGS_LEFT; format++; n = 1U; break; - case '+': flags |= FLAGS_PLUS; format++; n = 1U; break; - case ' ': flags |= FLAGS_SPACE; format++; n = 1U; break; - case '#': flags |= FLAGS_HASH; format++; n = 1U; break; - default : n = 0U; break; - } - } while (n); - - // evaluate width field - width = 0U; - if (_is_digit(*format)) { - width = _atoi(&format); - } - else if (*format == '*') { - const int w = va_arg(va, int); - if (w < 0) { - flags |= FLAGS_LEFT; // reverse padding - width = (unsigned int)-w; - } - else { - width = (unsigned int)w; - } - format++; - } - - // evaluate precision field - precision = 0U; - if (*format == '.') { - flags |= FLAGS_PRECISION; - format++; - if (_is_digit(*format)) { - precision = _atoi(&format); - } - else if (*format == '*') { - const int prec = (int)va_arg(va, int); - precision = prec > 0 ? (unsigned int)prec : 0U; - format++; - } - } - - // evaluate length field - switch (*format) { - case 'l' : - flags |= FLAGS_LONG; - format++; - if (*format == 'l') { - flags |= FLAGS_LONG_LONG; - format++; - } - break; - case 'h' : - flags |= FLAGS_SHORT; - format++; - if (*format == 'h') { - flags |= FLAGS_CHAR; - format++; - } - break; -#if defined(PRINTF_SUPPORT_PTRDIFF_T) - case 't' : - flags |= (sizeof(ptrdiff_t) == sizeof(long) ? FLAGS_LONG : FLAGS_LONG_LONG); - format++; - break; -#endif - case 'j' : - flags |= (sizeof(intmax_t) == sizeof(long) ? FLAGS_LONG : FLAGS_LONG_LONG); - format++; - break; - case 'z' : - flags |= (sizeof(size_t) == sizeof(long) ? FLAGS_LONG : FLAGS_LONG_LONG); - format++; - break; - default : - break; - } - - // evaluate specifier - switch (*format) { - case 'd' : - case 'i' : - case 'u' : - case 'x' : - case 'X' : - case 'o' : - case 'b' : { - // set the base - unsigned int base; - if (*format == 'x' || *format == 'X') { - base = 16U; - } - else if (*format == 'o') { - base = 8U; - } - else if (*format == 'b') { - base = 2U; - } - else { - base = 10U; - flags &= ~FLAGS_HASH; // no hash for dec format - } - // uppercase - if (*format == 'X') { - flags |= FLAGS_UPPERCASE; - } - - // no plus or space flag for u, x, X, o, b - if ((*format != 'i') && (*format != 'd')) { - flags &= ~(FLAGS_PLUS | FLAGS_SPACE); - } - - // ignore '0' flag when precision is given - if (flags & FLAGS_PRECISION) { - flags &= ~FLAGS_ZEROPAD; - } - - // convert the integer - if ((*format == 'i') || (*format == 'd')) { - // signed - if (flags & FLAGS_LONG_LONG) { -#if defined(PRINTF_SUPPORT_LONG_LONG) - const long long value = va_arg(va, long long); - idx = _ntoa_long_long(out, buffer, idx, maxlen, (unsigned long long)(value > 0 ? value : 0 - value), value < 0, base, precision, width, flags); -#endif - } - else if (flags & FLAGS_LONG) { - const long value = va_arg(va, long); - idx = _ntoa_long(out, buffer, idx, maxlen, (unsigned long)(value > 0 ? value : 0 - value), value < 0, base, precision, width, flags); - } - else { - const int value = (flags & FLAGS_CHAR) ? (char)va_arg(va, int) : (flags & FLAGS_SHORT) ? (short int)va_arg(va, int) : va_arg(va, int); - idx = _ntoa_long(out, buffer, idx, maxlen, (unsigned int)(value > 0 ? value : 0 - value), value < 0, base, precision, width, flags); - } - } - else { - // unsigned - if (flags & FLAGS_LONG_LONG) { -#if defined(PRINTF_SUPPORT_LONG_LONG) - idx = _ntoa_long_long(out, buffer, idx, maxlen, va_arg(va, unsigned long long), false, base, precision, width, flags); -#endif - } - else if (flags & FLAGS_LONG) { - idx = _ntoa_long(out, buffer, idx, maxlen, va_arg(va, unsigned long), false, base, precision, width, flags); - } - else { - const unsigned int value = (flags & FLAGS_CHAR) ? (unsigned char)va_arg(va, unsigned int) : (flags & FLAGS_SHORT) ? (unsigned short int)va_arg(va, unsigned int) : va_arg(va, unsigned int); - idx = _ntoa_long(out, buffer, idx, maxlen, value, false, base, precision, width, flags); - } - } - format++; - break; - } -#if defined(PRINTF_SUPPORT_FLOAT) - case 'f' : - case 'F' : - if (*format == 'F') flags |= FLAGS_UPPERCASE; - idx = _ftoa(out, buffer, idx, maxlen, va_arg(va, double), precision, width, flags); - format++; - break; -#if defined(PRINTF_SUPPORT_EXPONENTIAL) - case 'e': - case 'E': - case 'g': - case 'G': - if ((*format == 'g')||(*format == 'G')) flags |= FLAGS_ADAPT_EXP; - if ((*format == 'E')||(*format == 'G')) flags |= FLAGS_UPPERCASE; - idx = _etoa(out, buffer, idx, maxlen, va_arg(va, double), precision, width, flags); - format++; - break; -#endif // PRINTF_SUPPORT_EXPONENTIAL -#endif // PRINTF_SUPPORT_FLOAT - case 'c' : { - unsigned int l = 1U; - // pre padding - if (!(flags & FLAGS_LEFT)) { - while (l++ < width) { - out(' ', buffer, idx++, maxlen); - } - } - // char output - out((char)va_arg(va, int), buffer, idx++, maxlen); - // post padding - if (flags & FLAGS_LEFT) { - while (l++ < width) { - out(' ', buffer, idx++, maxlen); - } - } - format++; - break; - } - - case 's' : { - const char* p = va_arg(va, char*); - unsigned int l = _strnlen_s(p, precision ? precision : (size_t)-1); - // pre padding - if (flags & FLAGS_PRECISION) { - l = (l < precision ? l : precision); - } - if (!(flags & FLAGS_LEFT)) { - while (l++ < width) { - out(' ', buffer, idx++, maxlen); - } - } - // string output - while ((*p != 0) && (!(flags & FLAGS_PRECISION) || precision--)) { - out(*(p++), buffer, idx++, maxlen); - } - // post padding - if (flags & FLAGS_LEFT) { - while (l++ < width) { - out(' ', buffer, idx++, maxlen); - } - } - format++; - break; - } - - case 'p' : { - width = sizeof(void*) * 2U; - flags |= FLAGS_ZEROPAD | FLAGS_UPPERCASE; -#if defined(PRINTF_SUPPORT_LONG_LONG) - const bool is_ll = sizeof(uintptr_t) == sizeof(long long); - if (is_ll) { - idx = _ntoa_long_long(out, buffer, idx, maxlen, (uintptr_t)va_arg(va, void*), false, 16U, precision, width, flags); - } - else { -#endif - idx = _ntoa_long(out, buffer, idx, maxlen, (unsigned long)((uintptr_t)va_arg(va, void*)), false, 16U, precision, width, flags); -#if defined(PRINTF_SUPPORT_LONG_LONG) - } -#endif - format++; - break; - } - - case '%' : - out('%', buffer, idx++, maxlen); - format++; - break; - - default : - out(*format, buffer, idx++, maxlen); - format++; - break; - } - } - - // termination - out((char)0, buffer, idx < maxlen ? idx : maxlen - 1U, maxlen); - - // return written chars without terminating \0 - return (int)idx; -} - - -/////////////////////////////////////////////////////////////////////////////// - -int printf_(const char* format, ...) -{ - va_list va; - va_start(va, format); - char buffer[1]; - const int ret = _vsnprintf(_out_char, buffer, (size_t)-1, format, va); - va_end(va); - return ret; -} - - -int sprintf_(char* buffer, const char* format, ...) -{ - va_list va; - va_start(va, format); - const int ret = _vsnprintf(_out_buffer, buffer, (size_t)-1, format, va); - va_end(va); - return ret; -} - - -int snprintf_(char* buffer, size_t count, const char* format, ...) -{ - va_list va; - va_start(va, format); - const int ret = _vsnprintf(_out_buffer, buffer, count, format, va); - va_end(va); - return ret; -} - - -int vprintf_(const char* format, va_list va) -{ - char buffer[1]; - return _vsnprintf(_out_char, buffer, (size_t)-1, format, va); -} - - -int vsnprintf_(char* buffer, size_t count, const char* format, va_list va) -{ - return _vsnprintf(_out_buffer, buffer, count, format, va); -} - - -int fctprintf(void (*out)(char character, void* arg), void* arg, const char* format, ...) -{ - va_list va; - va_start(va, format); - const out_fct_wrap_type out_fct_wrap = { out, arg }; - const int ret = _vsnprintf(_out_fct, (char*)(uintptr_t)&out_fct_wrap, (size_t)-1, format, va); - va_end(va); - return ret; -} +/////////////////////////////////////////////////////////////////////////////// +// \author (c) Marco Paland (info@paland.com) +// 2014-2019, PALANDesign Hannover, Germany +// +// \license The MIT License (MIT) +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. +// +// \brief Tiny printf, sprintf and (v)snprintf implementation, optimized for speed on +// embedded systems with a very limited resources. These routines are thread +// safe and reentrant! +// Use this instead of the bloated standard/newlib printf cause these use +// malloc for printf (and may not be thread safe). +// +/////////////////////////////////////////////////////////////////////////////// + +#define PRINTF_DISABLE_SUPPORT_FLOAT +#define PRINTF_DISABLE_SUPPORT_EXPONENTIAL + +#include +#include + +#include "uarch-printf.h" + + +// define this globally (e.g. gcc -DPRINTF_INCLUDE_CONFIG_H ...) to include the +// printf_config.h header file +// default: undefined +#ifdef PRINTF_INCLUDE_CONFIG_H +#include "printf_config.h" +#endif + + +// 'ntoa' conversion buffer size, this must be big enough to hold one converted +// numeric number including padded zeros (dynamically created on stack) +// default: 32 byte +#ifndef PRINTF_NTOA_BUFFER_SIZE +#define PRINTF_NTOA_BUFFER_SIZE 32U +#endif + +// 'ftoa' conversion buffer size, this must be big enough to hold one converted +// float number including padded zeros (dynamically created on stack) +// default: 32 byte +#ifndef PRINTF_FTOA_BUFFER_SIZE +#define PRINTF_FTOA_BUFFER_SIZE 32U +#endif + +// support for the floating point type (%f) +// default: activated +#ifndef PRINTF_DISABLE_SUPPORT_FLOAT +#define PRINTF_SUPPORT_FLOAT +#endif + +// support for exponential floating point notation (%e/%g) +// default: activated +#ifndef PRINTF_DISABLE_SUPPORT_EXPONENTIAL +#define PRINTF_SUPPORT_EXPONENTIAL +#endif + +// define the default floating point precision +// default: 6 digits +#ifndef PRINTF_DEFAULT_FLOAT_PRECISION +#define PRINTF_DEFAULT_FLOAT_PRECISION 6U +#endif + +// define the largest float suitable to print with %f +// default: 1e9 +#ifndef PRINTF_MAX_FLOAT +#define PRINTF_MAX_FLOAT 1e9 +#endif + +// support for the long long types (%llu or %p) +// default: activated +#ifndef PRINTF_DISABLE_SUPPORT_LONG_LONG +#define PRINTF_SUPPORT_LONG_LONG +#endif + +// support for the ptrdiff_t type (%t) +// ptrdiff_t is normally defined in as long or long long type +// default: activated +#ifndef PRINTF_DISABLE_SUPPORT_PTRDIFF_T +#define PRINTF_SUPPORT_PTRDIFF_T +#endif + +/////////////////////////////////////////////////////////////////////////////// + +// internal flag definitions +#define FLAGS_ZEROPAD (1U << 0U) +#define FLAGS_LEFT (1U << 1U) +#define FLAGS_PLUS (1U << 2U) +#define FLAGS_SPACE (1U << 3U) +#define FLAGS_HASH (1U << 4U) +#define FLAGS_UPPERCASE (1U << 5U) +#define FLAGS_CHAR (1U << 6U) +#define FLAGS_SHORT (1U << 7U) +#define FLAGS_LONG (1U << 8U) +#define FLAGS_LONG_LONG (1U << 9U) +#define FLAGS_PRECISION (1U << 10U) +#define FLAGS_ADAPT_EXP (1U << 11U) + + +// import float.h for DBL_MAX +#if defined(PRINTF_SUPPORT_FLOAT) +#include +#endif + + +// output function type +typedef void (*out_fct_type)(char character, void* buffer, size_t idx, size_t maxlen); + + +// wrapper (used as buffer) for output function type +typedef struct { + void (*fct)(char character, void* arg); + void* arg; +} out_fct_wrap_type; + + +// internal buffer output +static inline void _out_buffer(char character, void* buffer, size_t idx, size_t maxlen) +{ + if (idx < maxlen) { + ((char*)buffer)[idx] = character; + } +} + + +// internal null output +static inline void _out_null(char character, void* buffer, size_t idx, size_t maxlen) +{ + (void)character; (void)buffer; (void)idx; (void)maxlen; +} + + +// internal _putchar wrapper +static inline void _out_char(char character, void* buffer, size_t idx, size_t maxlen) +{ + (void)buffer; (void)idx; (void)maxlen; + if (character) { + _putchar(character); + } +} + + +// internal output function wrapper +static inline void _out_fct(char character, void* buffer, size_t idx, size_t maxlen) +{ + (void)idx; (void)maxlen; + if (character) { + // buffer is the output fct pointer + ((out_fct_wrap_type*)buffer)->fct(character, ((out_fct_wrap_type*)buffer)->arg); + } +} + + +// internal secure strlen +// \return The length of the string (excluding the terminating 0) limited by 'maxsize' +static inline unsigned int _strnlen_s(const char* str, size_t maxsize) +{ + const char* s; + for (s = str; *s && maxsize--; ++s); + return (unsigned int)(s - str); +} + + +// internal test if char is a digit (0-9) +// \return true if char is a digit +static inline bool _is_digit(char ch) +{ + return (ch >= '0') && (ch <= '9'); +} + + +// internal ASCII string to unsigned int conversion +static unsigned int _atoi(const char** str) +{ + unsigned int i = 0U; + while (_is_digit(**str)) { + i = i * 10U + (unsigned int)(*((*str)++) - '0'); + } + return i; +} + + +// output the specified string in reverse, taking care of any zero-padding +static size_t _out_rev(out_fct_type out, char* buffer, size_t idx, size_t maxlen, const char* buf, size_t len, unsigned int width, unsigned int flags) +{ + const size_t start_idx = idx; + + // pad spaces up to given width + if (!(flags & FLAGS_LEFT) && !(flags & FLAGS_ZEROPAD)) { + for (size_t i = len; i < width; i++) { + out(' ', buffer, idx++, maxlen); + } + } + + // reverse string + while (len) { + out(buf[--len], buffer, idx++, maxlen); + } + + // append pad spaces up to given width + if (flags & FLAGS_LEFT) { + while (idx - start_idx < width) { + out(' ', buffer, idx++, maxlen); + } + } + + return idx; +} + + +// internal itoa format +static size_t _ntoa_format(out_fct_type out, char* buffer, size_t idx, size_t maxlen, char* buf, size_t len, bool negative, unsigned int base, unsigned int prec, unsigned int width, unsigned int flags) +{ + // pad leading zeros + if (!(flags & FLAGS_LEFT)) { + if (width && (flags & FLAGS_ZEROPAD) && (negative || (flags & (FLAGS_PLUS | FLAGS_SPACE)))) { + width--; + } + while ((len < prec) && (len < PRINTF_NTOA_BUFFER_SIZE)) { + buf[len++] = '0'; + } + while ((flags & FLAGS_ZEROPAD) && (len < width) && (len < PRINTF_NTOA_BUFFER_SIZE)) { + buf[len++] = '0'; + } + } + + // handle hash + if (flags & FLAGS_HASH) { + if (!(flags & FLAGS_PRECISION) && len && ((len == prec) || (len == width))) { + len--; + if (len && (base == 16U)) { + len--; + } + } + if ((base == 16U) && !(flags & FLAGS_UPPERCASE) && (len < PRINTF_NTOA_BUFFER_SIZE)) { + buf[len++] = 'x'; + } + else if ((base == 16U) && (flags & FLAGS_UPPERCASE) && (len < PRINTF_NTOA_BUFFER_SIZE)) { + buf[len++] = 'X'; + } + else if ((base == 2U) && (len < PRINTF_NTOA_BUFFER_SIZE)) { + buf[len++] = 'b'; + } + if (len < PRINTF_NTOA_BUFFER_SIZE) { + buf[len++] = '0'; + } + } + + if (len < PRINTF_NTOA_BUFFER_SIZE) { + if (negative) { + buf[len++] = '-'; + } + else if (flags & FLAGS_PLUS) { + buf[len++] = '+'; // ignore the space if the '+' exists + } + else if (flags & FLAGS_SPACE) { + buf[len++] = ' '; + } + } + + return _out_rev(out, buffer, idx, maxlen, buf, len, width, flags); +} + + +// internal itoa for 'long' type +static size_t _ntoa_long(out_fct_type out, char* buffer, size_t idx, size_t maxlen, unsigned long value, bool negative, unsigned long base, unsigned int prec, unsigned int width, unsigned int flags) +{ + char buf[PRINTF_NTOA_BUFFER_SIZE]; + size_t len = 0U; + + // no hash for 0 values + if (!value) { + flags &= ~FLAGS_HASH; + } + + // write if precision != 0 and value is != 0 + if (!(flags & FLAGS_PRECISION) || value) { + do { + const char digit = (char)(value % base); + buf[len++] = digit < 10 ? '0' + digit : (flags & FLAGS_UPPERCASE ? 'A' : 'a') + digit - 10; + value /= base; + } while (value && (len < PRINTF_NTOA_BUFFER_SIZE)); + } + + return _ntoa_format(out, buffer, idx, maxlen, buf, len, negative, (unsigned int)base, prec, width, flags); +} + + +// internal itoa for 'long long' type +#if defined(PRINTF_SUPPORT_LONG_LONG) +static size_t _ntoa_long_long(out_fct_type out, char* buffer, size_t idx, size_t maxlen, unsigned long long value, bool negative, unsigned long long base, unsigned int prec, unsigned int width, unsigned int flags) +{ + char buf[PRINTF_NTOA_BUFFER_SIZE]; + size_t len = 0U; + + // no hash for 0 values + if (!value) { + flags &= ~FLAGS_HASH; + } + + // write if precision != 0 and value is != 0 + if (!(flags & FLAGS_PRECISION) || value) { + do { + const char digit = (char)(value % base); + buf[len++] = digit < 10 ? '0' + digit : (flags & FLAGS_UPPERCASE ? 'A' : 'a') + digit - 10; + value /= base; + } while (value && (len < PRINTF_NTOA_BUFFER_SIZE)); + } + + return _ntoa_format(out, buffer, idx, maxlen, buf, len, negative, (unsigned int)base, prec, width, flags); +} +#endif // PRINTF_SUPPORT_LONG_LONG + + +#if defined(PRINTF_SUPPORT_FLOAT) + +#if defined(PRINTF_SUPPORT_EXPONENTIAL) +// forward declaration so that _ftoa can switch to exp notation for values > PRINTF_MAX_FLOAT +static size_t _etoa(out_fct_type out, char* buffer, size_t idx, size_t maxlen, double value, unsigned int prec, unsigned int width, unsigned int flags); +#endif + + +// internal ftoa for fixed decimal floating point +static size_t _ftoa(out_fct_type out, char* buffer, size_t idx, size_t maxlen, double value, unsigned int prec, unsigned int width, unsigned int flags) +{ + char buf[PRINTF_FTOA_BUFFER_SIZE]; + size_t len = 0U; + double diff = 0.0; + + // powers of 10 + static const double pow10[] = { 1, 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000 }; + + // test for special values + if (value != value) + return _out_rev(out, buffer, idx, maxlen, "nan", 3, width, flags); + if (value < -DBL_MAX) + return _out_rev(out, buffer, idx, maxlen, "fni-", 4, width, flags); + if (value > DBL_MAX) + return _out_rev(out, buffer, idx, maxlen, (flags & FLAGS_PLUS) ? "fni+" : "fni", (flags & FLAGS_PLUS) ? 4U : 3U, width, flags); + + // test for very large values + // standard printf behavior is to print EVERY whole number digit -- which could be 100s of characters overflowing your buffers == bad + if ((value > PRINTF_MAX_FLOAT) || (value < -PRINTF_MAX_FLOAT)) { +#if defined(PRINTF_SUPPORT_EXPONENTIAL) + return _etoa(out, buffer, idx, maxlen, value, prec, width, flags); +#else + return 0U; +#endif + } + + // test for negative + bool negative = false; + if (value < 0) { + negative = true; + value = 0 - value; + } + + // set default precision, if not set explicitly + if (!(flags & FLAGS_PRECISION)) { + prec = PRINTF_DEFAULT_FLOAT_PRECISION; + } + // limit precision to 9, cause a prec >= 10 can lead to overflow errors + while ((len < PRINTF_FTOA_BUFFER_SIZE) && (prec > 9U)) { + buf[len++] = '0'; + prec--; + } + + int whole = (int)value; + double tmp = (value - whole) * pow10[prec]; + unsigned long frac = (unsigned long)tmp; + diff = tmp - frac; + + if (diff > 0.5) { + ++frac; + // handle rollover, e.g. case 0.99 with prec 1 is 1.0 + if (frac >= pow10[prec]) { + frac = 0; + ++whole; + } + } + else if (diff < 0.5) { + } + else if ((frac == 0U) || (frac & 1U)) { + // if halfway, round up if odd OR if last digit is 0 + ++frac; + } + + if (prec == 0U) { + diff = value - (double)whole; + if ((!(diff < 0.5) || (diff > 0.5)) && (whole & 1)) { + // exactly 0.5 and ODD, then round up + // 1.5 -> 2, but 2.5 -> 2 + ++whole; + } + } + else { + unsigned int count = prec; + // now do fractional part, as an unsigned number + while (len < PRINTF_FTOA_BUFFER_SIZE) { + --count; + buf[len++] = (char)(48U + (frac % 10U)); + if (!(frac /= 10U)) { + break; + } + } + // add extra 0s + while ((len < PRINTF_FTOA_BUFFER_SIZE) && (count-- > 0U)) { + buf[len++] = '0'; + } + if (len < PRINTF_FTOA_BUFFER_SIZE) { + // add decimal + buf[len++] = '.'; + } + } + + // do whole part, number is reversed + while (len < PRINTF_FTOA_BUFFER_SIZE) { + buf[len++] = (char)(48 + (whole % 10)); + if (!(whole /= 10)) { + break; + } + } + + // pad leading zeros + if (!(flags & FLAGS_LEFT) && (flags & FLAGS_ZEROPAD)) { + if (width && (negative || (flags & (FLAGS_PLUS | FLAGS_SPACE)))) { + width--; + } + while ((len < width) && (len < PRINTF_FTOA_BUFFER_SIZE)) { + buf[len++] = '0'; + } + } + + if (len < PRINTF_FTOA_BUFFER_SIZE) { + if (negative) { + buf[len++] = '-'; + } + else if (flags & FLAGS_PLUS) { + buf[len++] = '+'; // ignore the space if the '+' exists + } + else if (flags & FLAGS_SPACE) { + buf[len++] = ' '; + } + } + + return _out_rev(out, buffer, idx, maxlen, buf, len, width, flags); +} + + +#if defined(PRINTF_SUPPORT_EXPONENTIAL) +// internal ftoa variant for exponential floating-point type, contributed by Martijn Jasperse +static size_t _etoa(out_fct_type out, char* buffer, size_t idx, size_t maxlen, double value, unsigned int prec, unsigned int width, unsigned int flags) +{ + // check for NaN and special values + if ((value != value) || (value > DBL_MAX) || (value < -DBL_MAX)) { + return _ftoa(out, buffer, idx, maxlen, value, prec, width, flags); + } + + // determine the sign + const bool negative = value < 0; + if (negative) { + value = -value; + } + + // default precision + if (!(flags & FLAGS_PRECISION)) { + prec = PRINTF_DEFAULT_FLOAT_PRECISION; + } + + // determine the decimal exponent + // based on the algorithm by David Gay (https://www.ampl.com/netlib/fp/dtoa.c) + union { + uint64_t U; + double F; + } conv; + + conv.F = value; + int exp2 = (int)((conv.U >> 52U) & 0x07FFU) - 1023; // effectively log2 + conv.U = (conv.U & ((1ULL << 52U) - 1U)) | (1023ULL << 52U); // drop the exponent so conv.F is now in [1,2) + // now approximate log10 from the log2 integer part and an expansion of ln around 1.5 + int expval = (int)(0.1760912590558 + exp2 * 0.301029995663981 + (conv.F - 1.5) * 0.289529654602168); + // now we want to compute 10^expval but we want to be sure it won't overflow + exp2 = (int)(expval * 3.321928094887362 + 0.5); + const double z = expval * 2.302585092994046 - exp2 * 0.6931471805599453; + const double z2 = z * z; + conv.U = (uint64_t)(exp2 + 1023) << 52U; + // compute exp(z) using continued fractions, see https://en.wikipedia.org/wiki/Exponential_function#Continued_fractions_for_ex + conv.F *= 1 + 2 * z / (2 - z + (z2 / (6 + (z2 / (10 + z2 / 14))))); + // correct for rounding errors + if (value < conv.F) { + expval--; + conv.F /= 10; + } + + // the exponent format is "%+03d" and largest value is "307", so set aside 4-5 characters + unsigned int minwidth = ((expval < 100) && (expval > -100)) ? 4U : 5U; + + // in "%g" mode, "prec" is the number of *significant figures* not decimals + if (flags & FLAGS_ADAPT_EXP) { + // do we want to fall-back to "%f" mode? + if ((value >= 1e-4) && (value < 1e6)) { + if ((int)prec > expval) { + prec = (unsigned)((int)prec - expval - 1); + } + else { + prec = 0; + } + flags |= FLAGS_PRECISION; // make sure _ftoa respects precision + // no characters in exponent + minwidth = 0U; + expval = 0; + } + else { + // we use one sigfig for the whole part + if ((prec > 0) && (flags & FLAGS_PRECISION)) { + --prec; + } + } + } + + // will everything fit? + unsigned int fwidth = width; + if (width > minwidth) { + // we didn't fall-back so subtract the characters required for the exponent + fwidth -= minwidth; + } else { + // not enough characters, so go back to default sizing + fwidth = 0U; + } + if ((flags & FLAGS_LEFT) && minwidth) { + // if we're padding on the right, DON'T pad the floating part + fwidth = 0U; + } + + // rescale the float value + if (expval) { + value /= conv.F; + } + + // output the floating part + const size_t start_idx = idx; + idx = _ftoa(out, buffer, idx, maxlen, negative ? -value : value, prec, fwidth, flags & ~FLAGS_ADAPT_EXP); + + // output the exponent part + if (minwidth) { + // output the exponential symbol + out((flags & FLAGS_UPPERCASE) ? 'E' : 'e', buffer, idx++, maxlen); + // output the exponent value + idx = _ntoa_long(out, buffer, idx, maxlen, (expval < 0) ? -expval : expval, expval < 0, 10, 0, minwidth-1, FLAGS_ZEROPAD | FLAGS_PLUS); + // might need to right-pad spaces + if (flags & FLAGS_LEFT) { + while (idx - start_idx < width) out(' ', buffer, idx++, maxlen); + } + } + return idx; +} +#endif // PRINTF_SUPPORT_EXPONENTIAL +#endif // PRINTF_SUPPORT_FLOAT + + +// internal vsnprintf +static int _vsnprintf(out_fct_type out, char* buffer, const size_t maxlen, const char* format, va_list va) +{ + unsigned int flags, width, precision, n; + size_t idx = 0U; + + if (!buffer) { + // use null output function + out = _out_null; + } + + while (*format) + { + // format specifier? %[flags][width][.precision][length] + if (*format != '%') { + // no + out(*format, buffer, idx++, maxlen); + format++; + continue; + } + else { + // yes, evaluate it + format++; + } + + // evaluate flags + flags = 0U; + do { + switch (*format) { + case '0': flags |= FLAGS_ZEROPAD; format++; n = 1U; break; + case '-': flags |= FLAGS_LEFT; format++; n = 1U; break; + case '+': flags |= FLAGS_PLUS; format++; n = 1U; break; + case ' ': flags |= FLAGS_SPACE; format++; n = 1U; break; + case '#': flags |= FLAGS_HASH; format++; n = 1U; break; + default : n = 0U; break; + } + } while (n); + + // evaluate width field + width = 0U; + if (_is_digit(*format)) { + width = _atoi(&format); + } + else if (*format == '*') { + const int w = va_arg(va, int); + if (w < 0) { + flags |= FLAGS_LEFT; // reverse padding + width = (unsigned int)-w; + } + else { + width = (unsigned int)w; + } + format++; + } + + // evaluate precision field + precision = 0U; + if (*format == '.') { + flags |= FLAGS_PRECISION; + format++; + if (_is_digit(*format)) { + precision = _atoi(&format); + } + else if (*format == '*') { + const int prec = (int)va_arg(va, int); + precision = prec > 0 ? (unsigned int)prec : 0U; + format++; + } + } + + // evaluate length field + switch (*format) { + case 'l' : + flags |= FLAGS_LONG; + format++; + if (*format == 'l') { + flags |= FLAGS_LONG_LONG; + format++; + } + break; + case 'h' : + flags |= FLAGS_SHORT; + format++; + if (*format == 'h') { + flags |= FLAGS_CHAR; + format++; + } + break; +#if defined(PRINTF_SUPPORT_PTRDIFF_T) + case 't' : + flags |= (sizeof(ptrdiff_t) == sizeof(long) ? FLAGS_LONG : FLAGS_LONG_LONG); + format++; + break; +#endif + case 'j' : + flags |= (sizeof(intmax_t) == sizeof(long) ? FLAGS_LONG : FLAGS_LONG_LONG); + format++; + break; + case 'z' : + flags |= (sizeof(size_t) == sizeof(long) ? FLAGS_LONG : FLAGS_LONG_LONG); + format++; + break; + default : + break; + } + + // evaluate specifier + switch (*format) { + case 'd' : + case 'i' : + case 'u' : + case 'x' : + case 'X' : + case 'o' : + case 'b' : { + // set the base + unsigned int base; + if (*format == 'x' || *format == 'X') { + base = 16U; + } + else if (*format == 'o') { + base = 8U; + } + else if (*format == 'b') { + base = 2U; + } + else { + base = 10U; + flags &= ~FLAGS_HASH; // no hash for dec format + } + // uppercase + if (*format == 'X') { + flags |= FLAGS_UPPERCASE; + } + + // no plus or space flag for u, x, X, o, b + if ((*format != 'i') && (*format != 'd')) { + flags &= ~(FLAGS_PLUS | FLAGS_SPACE); + } + + // ignore '0' flag when precision is given + if (flags & FLAGS_PRECISION) { + flags &= ~FLAGS_ZEROPAD; + } + + // convert the integer + if ((*format == 'i') || (*format == 'd')) { + // signed + if (flags & FLAGS_LONG_LONG) { +#if defined(PRINTF_SUPPORT_LONG_LONG) + const long long value = va_arg(va, long long); + idx = _ntoa_long_long(out, buffer, idx, maxlen, (unsigned long long)(value > 0 ? value : 0 - value), value < 0, base, precision, width, flags); +#endif + } + else if (flags & FLAGS_LONG) { + const long value = va_arg(va, long); + idx = _ntoa_long(out, buffer, idx, maxlen, (unsigned long)(value > 0 ? value : 0 - value), value < 0, base, precision, width, flags); + } + else { + const int value = (flags & FLAGS_CHAR) ? (char)va_arg(va, int) : (flags & FLAGS_SHORT) ? (short int)va_arg(va, int) : va_arg(va, int); + idx = _ntoa_long(out, buffer, idx, maxlen, (unsigned int)(value > 0 ? value : 0 - value), value < 0, base, precision, width, flags); + } + } + else { + // unsigned + if (flags & FLAGS_LONG_LONG) { +#if defined(PRINTF_SUPPORT_LONG_LONG) + idx = _ntoa_long_long(out, buffer, idx, maxlen, va_arg(va, unsigned long long), false, base, precision, width, flags); +#endif + } + else if (flags & FLAGS_LONG) { + idx = _ntoa_long(out, buffer, idx, maxlen, va_arg(va, unsigned long), false, base, precision, width, flags); + } + else { + const unsigned int value = (flags & FLAGS_CHAR) ? (unsigned char)va_arg(va, unsigned int) : (flags & FLAGS_SHORT) ? (unsigned short int)va_arg(va, unsigned int) : va_arg(va, unsigned int); + idx = _ntoa_long(out, buffer, idx, maxlen, value, false, base, precision, width, flags); + } + } + format++; + break; + } +#if defined(PRINTF_SUPPORT_FLOAT) + case 'f' : + case 'F' : + if (*format == 'F') flags |= FLAGS_UPPERCASE; + idx = _ftoa(out, buffer, idx, maxlen, va_arg(va, double), precision, width, flags); + format++; + break; +#if defined(PRINTF_SUPPORT_EXPONENTIAL) + case 'e': + case 'E': + case 'g': + case 'G': + if ((*format == 'g')||(*format == 'G')) flags |= FLAGS_ADAPT_EXP; + if ((*format == 'E')||(*format == 'G')) flags |= FLAGS_UPPERCASE; + idx = _etoa(out, buffer, idx, maxlen, va_arg(va, double), precision, width, flags); + format++; + break; +#endif // PRINTF_SUPPORT_EXPONENTIAL +#endif // PRINTF_SUPPORT_FLOAT + case 'c' : { + unsigned int l = 1U; + // pre padding + if (!(flags & FLAGS_LEFT)) { + while (l++ < width) { + out(' ', buffer, idx++, maxlen); + } + } + // char output + out((char)va_arg(va, int), buffer, idx++, maxlen); + // post padding + if (flags & FLAGS_LEFT) { + while (l++ < width) { + out(' ', buffer, idx++, maxlen); + } + } + format++; + break; + } + + case 's' : { + const char* p = va_arg(va, char*); + unsigned int l = _strnlen_s(p, precision ? precision : (size_t)-1); + // pre padding + if (flags & FLAGS_PRECISION) { + l = (l < precision ? l : precision); + } + if (!(flags & FLAGS_LEFT)) { + while (l++ < width) { + out(' ', buffer, idx++, maxlen); + } + } + // string output + while ((*p != 0) && (!(flags & FLAGS_PRECISION) || precision--)) { + out(*(p++), buffer, idx++, maxlen); + } + // post padding + if (flags & FLAGS_LEFT) { + while (l++ < width) { + out(' ', buffer, idx++, maxlen); + } + } + format++; + break; + } + + case 'p' : { + width = sizeof(void*) * 2U; + flags |= FLAGS_ZEROPAD | FLAGS_UPPERCASE; +#if defined(PRINTF_SUPPORT_LONG_LONG) + const bool is_ll = sizeof(uintptr_t) == sizeof(long long); + if (is_ll) { + idx = _ntoa_long_long(out, buffer, idx, maxlen, (uintptr_t)va_arg(va, void*), false, 16U, precision, width, flags); + } + else { +#endif + idx = _ntoa_long(out, buffer, idx, maxlen, (unsigned long)((uintptr_t)va_arg(va, void*)), false, 16U, precision, width, flags); +#if defined(PRINTF_SUPPORT_LONG_LONG) + } +#endif + format++; + break; + } + + case '%' : + out('%', buffer, idx++, maxlen); + format++; + break; + + default : + out(*format, buffer, idx++, maxlen); + format++; + break; + } + } + + // termination + out((char)0, buffer, idx < maxlen ? idx : maxlen - 1U, maxlen); + + // return written chars without terminating \0 + return (int)idx; +} + + +/////////////////////////////////////////////////////////////////////////////// + +int printf_(const char* format, ...) +{ + va_list va; + va_start(va, format); + char buffer[1]; + const int ret = _vsnprintf(_out_char, buffer, (size_t)-1, format, va); + va_end(va); + return ret; +} + + +int sprintf_(char* buffer, const char* format, ...) +{ + va_list va; + va_start(va, format); + const int ret = _vsnprintf(_out_buffer, buffer, (size_t)-1, format, va); + va_end(va); + return ret; +} + + +int snprintf_(char* buffer, size_t count, const char* format, ...) +{ + va_list va; + va_start(va, format); + const int ret = _vsnprintf(_out_buffer, buffer, count, format, va); + va_end(va); + return ret; +} + + +int vprintf_(const char* format, va_list va) +{ + char buffer[1]; + return _vsnprintf(_out_char, buffer, (size_t)-1, format, va); +} + + +int vsnprintf_(char* buffer, size_t count, const char* format, va_list va) +{ + return _vsnprintf(_out_buffer, buffer, count, format, va); +} + + +int fctprintf(void (*out)(char character, void* arg), void* arg, const char* format, ...) +{ + va_list va; + va_start(va, format); + const out_fct_wrap_type out_fct_wrap = { out, arg }; + const int ret = _vsnprintf(_out_fct, (char*)(uintptr_t)&out_fct_wrap, (size_t)-1, format, va); + va_end(va); + return ret; +} diff --git a/uarch/uarch-printf.h b/uarch/uarch-printf.h index 6104ccfb7..43341e9ef 100644 --- a/uarch/uarch-printf.h +++ b/uarch/uarch-printf.h @@ -1,117 +1,108 @@ -/////////////////////////////////////////////////////////////////////////////// -// \author (c) Marco Paland (info@paland.com) -// 2014-2019, PALANDesign Hannover, Germany -// -// \license The MIT License (MIT) -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. -// -// \brief Tiny printf, sprintf and snprintf implementation, optimized for speed on -// embedded systems with a very limited resources. -// Use this instead of bloated standard/newlib printf. -// These routines are thread safe and reentrant. -// -/////////////////////////////////////////////////////////////////////////////// - -#ifndef _PRINTF_H_ -#define _PRINTF_H_ - -#include -#include - - -#ifdef __cplusplus -extern "C" { -#endif - - -/** - * Output a character to a custom device like UART, used by the printf() function - * This function is declared here only. You have to write your custom implementation somewhere - * \param character Character to output - */ -void _putchar(char character); - - -/** - * Tiny printf implementation - * You have to implement _putchar if you use printf() - * To avoid conflicts with the regular printf() API it is overridden by macro defines - * and internal underscore-appended functions like printf_() are used - * \param format A string that specifies the format of the output - * \return The number of characters that are written into the array, not counting the terminating null character - */ -#define printf printf_ -int printf_(const char* format, ...); - - -/** - * Tiny sprintf implementation - * Due to security reasons (buffer overflow) YOU SHOULD CONSIDER USING (V)SNPRINTF INSTEAD! - * \param buffer A pointer to the buffer where to store the formatted string. MUST be big enough to store the output! - * \param format A string that specifies the format of the output - * \return The number of characters that are WRITTEN into the buffer, not counting the terminating null character - */ -#define sprintf sprintf_ -int sprintf_(char* buffer, const char* format, ...); - - -/** - * Tiny snprintf/vsnprintf implementation - * \param buffer A pointer to the buffer where to store the formatted string - * \param count The maximum number of characters to store in the buffer, including a terminating null character - * \param format A string that specifies the format of the output - * \param va A value identifying a variable arguments list - * \return The number of characters that COULD have been written into the buffer, not counting the terminating - * null character. A value equal or larger than count indicates truncation. Only when the returned value - * is non-negative and less than count, the string has been completely written. - */ -#define snprintf snprintf_ -#define vsnprintf vsnprintf_ -int snprintf_(char* buffer, size_t count, const char* format, ...); -int vsnprintf_(char* buffer, size_t count, const char* format, va_list va); - - -/** - * Tiny vprintf implementation - * \param format A string that specifies the format of the output - * \param va A value identifying a variable arguments list - * \return The number of characters that are WRITTEN into the buffer, not counting the terminating null character - */ -#define vprintf vprintf_ -int vprintf_(const char* format, va_list va); - - -/** - * printf with output function - * You may use this as dynamic alternative to printf() with its fixed _putchar() output - * \param out An output function which takes one character and an argument pointer - * \param arg An argument pointer for user data passed to output function - * \param format A string that specifies the format of the output - * \return The number of characters that are sent to the output function, not counting the terminating null character - */ -int fctprintf(void (*out)(char character, void* arg), void* arg, const char* format, ...); - - -#ifdef __cplusplus -} -#endif - - -#endif // _PRINTF_H_ +/////////////////////////////////////////////////////////////////////////////// +// \author (c) Marco Paland (info@paland.com) +// 2014-2019, PALANDesign Hannover, Germany +// +// \license The MIT License (MIT) +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. +// +// \brief Tiny printf, sprintf and snprintf implementation, optimized for speed on +// embedded systems with a very limited resources. +// Use this instead of bloated standard/newlib printf. +// These routines are thread safe and reentrant. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef _PRINTF_H_ +#define _PRINTF_H_ + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Output a character to a custom device like UART, used by the printf() function + * This function is declared here only. You have to write your custom implementation somewhere + * \param character Character to output + */ +void _putchar(char character); + +/** + * Tiny printf implementation + * You have to implement _putchar if you use printf() + * To avoid conflicts with the regular printf() API it is overridden by macro defines + * and internal underscore-appended functions like printf_() are used + * \param format A string that specifies the format of the output + * \return The number of characters that are written into the array, not counting the terminating null character + */ +#define printf printf_ +int printf_(const char *format, ...); + +/** + * Tiny sprintf implementation + * Due to security reasons (buffer overflow) YOU SHOULD CONSIDER USING (V)SNPRINTF INSTEAD! + * \param buffer A pointer to the buffer where to store the formatted string. MUST be big enough to store the output! + * \param format A string that specifies the format of the output + * \return The number of characters that are WRITTEN into the buffer, not counting the terminating null character + */ +#define sprintf sprintf_ +int sprintf_(char *buffer, const char *format, ...); + +/** + * Tiny snprintf/vsnprintf implementation + * \param buffer A pointer to the buffer where to store the formatted string + * \param count The maximum number of characters to store in the buffer, including a terminating null character + * \param format A string that specifies the format of the output + * \param va A value identifying a variable arguments list + * \return The number of characters that COULD have been written into the buffer, not counting the terminating + * null character. A value equal or larger than count indicates truncation. Only when the returned value + * is non-negative and less than count, the string has been completely written. + */ +#define snprintf snprintf_ +#define vsnprintf vsnprintf_ +int snprintf_(char *buffer, size_t count, const char *format, ...); +int vsnprintf_(char *buffer, size_t count, const char *format, va_list va); + +/** + * Tiny vprintf implementation + * \param format A string that specifies the format of the output + * \param va A value identifying a variable arguments list + * \return The number of characters that are WRITTEN into the buffer, not counting the terminating null character + */ +#define vprintf vprintf_ +int vprintf_(const char *format, va_list va); + +/** + * printf with output function + * You may use this as dynamic alternative to printf() with its fixed _putchar() output + * \param out An output function which takes one character and an argument pointer + * \param arg An argument pointer for user data passed to output function + * \param format A string that specifies the format of the output + * \return The number of characters that are sent to the output function, not counting the terminating null character + */ +int fctprintf(void (*out)(char character, void *arg), void *arg, const char *format, ...); + +#ifdef __cplusplus +} +#endif + +#endif // _PRINTF_H_ diff --git a/uarch/uarch-runtime.cpp b/uarch/uarch-runtime.cpp index bc52e5d62..bb8758875 100644 --- a/uarch/uarch-runtime.cpp +++ b/uarch/uarch-runtime.cpp @@ -103,23 +103,3 @@ extern "C" NO_RETURN void abort() { // execution will never reach this point __builtin_trap(); } - -namespace cartesi { - -void os_open_tty() {} - -void os_close_tty() {} - -bool os_poll_tty(uint64_t /*timeout_us*/) { - return false; -} - -int os_getchar() { - return -1; -} - -void os_putchar(uint8_t ch) { - _putchar(ch); -} - -} // namespace cartesi diff --git a/uarch/uarch-runtime.h b/uarch/uarch-runtime.h index cbf0f2464..96e804e13 100644 --- a/uarch/uarch-runtime.h +++ b/uarch/uarch-runtime.h @@ -17,8 +17,8 @@ #ifndef UARCH_RUNTIME_H #define UARCH_RUNTIME_H -#include "uarch-printf.h" #include "compiler-defines.h" +#include "uarch-printf.h" #include #include From cf6a83ca02daddf71655134095ac26ef5dca4318 Mon Sep 17 00:00:00 2001 From: Diego Nehab <1635557+diegonehab@users.noreply.github.com> Date: Thu, 9 Jan 2025 16:59:42 +0000 Subject: [PATCH 03/82] refactor: move TLB management to interpret.cpp --- src/Makefile | 11 +- src/find-pma-entry.h | 17 +- src/host-addr.h | 98 ++++ src/i-state-access.h | 264 ++++++----- src/interpret.cpp | 258 ++++++++--- src/machine-state.h | 2 +- src/machine.cpp | 204 +++++--- src/machine.h | 31 ++ src/pma.h | 10 + src/record-send-cmio-state-access.h | 21 +- src/record-step-state-access.h | 254 +++++----- src/replay-send-cmio-state-access.h | 22 +- src/replay-step-state-access.h | 690 ++++++++++++++-------------- src/shadow-pmas.h | 10 + src/shadow-state.h | 1 + src/shadow-tlb-factory.cpp | 56 +-- src/shadow-tlb.h | 122 ++--- src/state-access.h | 249 ++++------ src/strict-aliasing.h | 94 ++-- src/tlb.h | 98 ++++ src/translate-virtual-address.h | 24 +- src/uarch-bridge.h | 28 +- src/uarch-solidity-compat.h | 23 +- src/uarch-state-access.h | 4 + src/uarch-step.cpp | 28 +- 25 files changed, 1480 insertions(+), 1139 deletions(-) create mode 100644 src/host-addr.h create mode 100644 src/tlb.h diff --git a/src/Makefile b/src/Makefile index 2fcc1a4c4..f7780c2fb 100644 --- a/src/Makefile +++ b/src/Makefile @@ -374,20 +374,21 @@ LIBCARTESI_OBJS:= \ base64.o \ interpret.o \ virtual-machine.o \ - uarch-machine.o \ - uarch-step.o \ - uarch-reset-state.o \ sha3.o \ machine-merkle-tree.o \ pristine-merkle-tree.o \ - uarch-interpret.o \ machine-c-api.o \ uarch-pristine-ram.o \ uarch-pristine-state-hash.o \ uarch-pristine-hash.o \ - send-cmio-response.o \ replay-step-state-access-interop.o + #send-cmio-response.o \ + #uarch-machine.o \ + #uarch-interpret.o \ + #uarch-step.o \ + #uarch-reset-state.o \ + CARTESI_CLUA_OBJS:= \ clua.o \ clua-i-virtual-machine.o diff --git a/src/find-pma-entry.h b/src/find-pma-entry.h index a9a41f919..1e2b223b9 100644 --- a/src/find-pma-entry.h +++ b/src/find-pma-entry.h @@ -27,10 +27,11 @@ namespace cartesi { /// \tparam STATE_ACCESS Class of machine state accessor object. /// \param a Machine state accessor object. /// \param paddr Target physical address of word. +/// \param index Receives index where PMA entry was found. /// \returns PMA entry where word falls, or empty sentinel. template -static FORCE_INLINE auto &find_pma_entry(STATE_ACCESS a, uint64_t paddr) { - uint64_t index = 0; +auto &find_pma_entry(STATE_ACCESS &a, uint64_t paddr, uint64_t &index) { + index = 0; while (true) { auto &pma = a.read_pma_entry(index); const auto length = pma.get_length(); @@ -53,6 +54,18 @@ static FORCE_INLINE auto &find_pma_entry(STATE_ACCESS a, uint64_t paddr) { } } +/// \brief Returns PMAs entry where a word falls. +/// \tparam T uint8_t, uint16_t, uint32_t, or uint64_t. +/// \tparam STATE_ACCESS Class of machine state accessor object. +/// \param a Machine state accessor object. +/// \param paddr Target physical address of word. +/// \returns PMA entry where word falls, or empty sentinel. +template +FORCE_INLINE auto &find_pma_entry(STATE_ACCESS &a, uint64_t paddr) { + uint64_t index = 0; + return find_pma_entry(a, paddr, index); +} + } // namespace cartesi #endif // FIND_PMA_ENTRY_H diff --git a/src/host-addr.h b/src/host-addr.h new file mode 100644 index 000000000..b6a33f6d1 --- /dev/null +++ b/src/host-addr.h @@ -0,0 +1,98 @@ +// Copyright Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: LGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify it under +// the terms of the GNU Lesser General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) any +// later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +// PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License along +// with this program (see COPYING). If not, see . +// + +#ifndef HOST_ADDR_H +#define HOST_ADDR_H + +namespace cartesi { + +// This is simply an uint64_t as a separate type, not automatically convertible to uint64_t +// It prevents any attempt of passing a host_addr where one is not expected +enum class host_addr : uint64_t {}; + +// Comparison operaator +static constexpr bool operator<(host_addr a, host_addr b) { + return static_cast(a) < static_cast(b); +} + +// Addition between host_addr +static constexpr host_addr operator+(host_addr a, host_addr b) { + return host_addr{static_cast(a) + static_cast(b)}; +} + +// Subtraction between host_addr +static constexpr host_addr operator-(host_addr a, host_addr b) { + return host_addr{static_cast(a) - static_cast(b)}; +} + +// Addition between host_addr and uint64_t +static constexpr host_addr operator+(uint64_t a, host_addr b) { + return host_addr{a + static_cast(b)}; +} + +// Addition between host_addr and uint64_t +static constexpr host_addr operator+(host_addr a, uint64_t b) { + return host_addr{static_cast(a) + b}; +} + +// Subtraction between host_addr and uint64_t +static constexpr host_addr operator-(uint64_t a, host_addr b) { + return host_addr{a - static_cast(b)}; +} + +// Subtraction between host_addr and uint64_t +static constexpr host_addr operator-(host_addr a, uint64_t b) { + return host_addr{static_cast(a) - b}; +} + +/// \brief Converts pointer to host_addr. +/// \tparam PTR Pointer type to perform the cast. +/// \param ptr The pointer to retrieve its unsigned integer representation. +/// \returns Corresponding host_addr. +/// \details Use cast_host_addr_to_ptr The address returned by this function, +/// can later be converted to a pointer using cast_addr_to_ptr, +/// and must be read/written using aliased_aligned_read/aliased_aligned_write, +/// otherwise strict aliasing rules may be violated. +static inline host_addr cast_ptr_to_host_addr(const void *ptr) { + static_assert(sizeof(void *) == sizeof(uintptr_t)); + static_assert(sizeof(host_addr) >= sizeof(uintptr_t)); + // Note that bellow we cast the pointer to void* first, + // according to the C spec this is required is to ensure the same presentation, before casting to uintptr_t + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast,bugprone-casting-through-void) + return host_addr{reinterpret_cast(ptr)}; +} + +/// \brief Casts a pointer to an unsigned integer. +/// \details The pointer returned by this function +/// must only be read/written using aliased_aligned_read/aliased_aligned_write, +/// otherwise strict aliasing rules may be violated. +/// \tparam T Unsigned integer type to cast to. +/// \tparam PTR Pointer type to perform the cast. +/// \param addr The address of the pointer represented by an unsigned integer. +/// \returns A pointer. +static inline void *cast_host_addr_to_ptr(host_addr host_addr) { + // Enforcement on type arguments + static_assert(sizeof(void *) == sizeof(uintptr_t)); + static_assert(sizeof(host_addr) >= sizeof(uintptr_t)); + // Note that bellow we cast the address to void* first, + // according to the C spec this is required is to ensure the same presentation, before casting to PTR + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast,bugprone-casting-through-void,performance-no-int-to-ptr) + return reinterpret_cast(static_cast(host_addr)); +} + +} // namespace cartesi + +#endif diff --git a/src/i-state-access.h b/src/i-state-access.h index d255899ee..2b2fad014 100644 --- a/src/i-state-access.h +++ b/src/i-state-access.h @@ -26,13 +26,25 @@ #include "compiler-defines.h" #include "meta.h" -#include "shadow-tlb.h" +#include "tlb.h" namespace cartesi { // Forward declarations enum class bracket_type; +// Type trait that should return the pma_entry type for a state access class +template +struct i_state_access_pma_entry {}; +template +using i_state_access_pma_entry_t = typename i_state_access_pma_entry::type; + +// Type trait that should return the fast_addr type for a state access class +template +struct i_state_access_fast_addr {}; +template +using i_state_access_fast_addr_t = typename i_state_access_fast_addr::type; + /// \class i_state_access /// \brief Interface for machine state access. /// \details \{ @@ -40,8 +52,8 @@ enum class bracket_type; /// The "run" function does not need a log, and must be as fast as possible. /// Both functions share the exact same implementation of what it means to advance the machine state by one cycle. /// In this common implementation, all state accesses go through a class that implements the i_state_access interface. -/// When looging is needed, a logged_state_access class is used. -/// When no logging is needed, a state_access class is used. +/// When logging is needed, a record state access class is used. +/// When no logging is needed, a direct state access class is used. /// /// In a typical design, i_state_access would be pure virtual. /// For speed, we avoid virtual methods and instead use templates. @@ -54,7 +66,7 @@ enum class bracket_type; /// Methods are provided to read and write each state component. /// \} /// \tparam DERIVED Derived class implementing the interface. (An example of CRTP.) -template +template class i_state_access { // CRTP i_state_access() = default; friend DERIVED; @@ -70,6 +82,10 @@ class i_state_access { // CRTP } public: + using pma_entry = i_state_access_pma_entry_t; + using fast_addr = i_state_access_fast_addr_t; + + //??D We should probably remove this from the interface /// \brief Returns machine state for direct access. auto &get_naked_state() { return derived().do_get_naked_state(); @@ -99,8 +115,8 @@ class i_state_access { // CRTP /// \brief Writes register to general-purpose register. /// \param reg Register index. /// \param val New register value. - /// \details Writes to register zero *break* the machine. There is an assertion to catch this, but NDEBUG will let - /// the value pass through. + /// \details Writes to register zero *break* the machine. + /// There is an assertion to catch this, but NDEBUG will let the value pass through. void write_x(int reg, uint64_t val) { return derived().do_write_x(reg, val); } @@ -591,21 +607,9 @@ class i_state_access { // CRTP return derived().do_read_htif_iyield(); } - /// \brief Poll for external interrupts. - /// \param mcycle Current machine mcycle. - /// \param mcycle_max Maximum mcycle to wait for interrupts. - /// \returns A pair, the first value is the new machine mcycle advanced by the relative elapsed time while - /// polling, the second value is a boolean that is true when the poll is stopped due do an external interrupt - /// request. - /// \details When mcycle_max is greater than mcycle, this function will sleep until an external interrupt - /// is triggered or mcycle_max relative elapsed time is reached. - std::pair poll_external_interrupts(uint64_t mcycle, uint64_t mcycle_max) { - return derived().do_poll_external_interrupts(mcycle, mcycle_max); - } - /// \brief Reads PMA entry at a given index. /// \param index Index of PMA - PMA_ENTRY_TYPE &read_pma_entry(uint64_t index) { + pma_entry &read_pma_entry(uint64_t index) { return derived().do_read_pma_entry(index); } @@ -631,98 +635,124 @@ class i_state_access { // CRTP return derived().do_write_memory(paddr, data, length); } - /// \brief Reads a word from memory. - /// \tparam T Type of word to read. - /// \param paddr Target physical address. - /// \param hpage Pointer to page start in host memory. - /// \param hoffset Offset in page (must be aligned to sizeof(T)). - /// \param pval Pointer to word receiving value. - template - void read_memory_word(uint64_t paddr, const unsigned char *hpage, uint64_t hoffset, T *pval) { - static_assert(std::is_integral_v && sizeof(T) <= sizeof(uint64_t), "unsupported type"); - return derived().template do_read_memory_word(paddr, hpage, hoffset, pval); - } - - /// \brief Writes a word to memory. - /// \tparam T Type of word to write. - /// \param paddr Target physical address. - /// \param hpage Pointer to page start in host memory. - /// \param hoffset Offset in page (must be aligned to sizeof(T)). - /// \param val Value to be written. - template - void write_memory_word(uint64_t paddr, unsigned char *hpage, uint64_t hoffset, T val) { - static_assert(std::is_integral_v && sizeof(T) <= sizeof(uint64_t), "unsupported type"); - return derived().template do_write_memory_word(paddr, hpage, hoffset, val); - } - - auto get_host_memory(PMA_ENTRY_TYPE &pma) { - return derived().do_get_host_memory(pma); - } - - /// \brief Try to translate a virtual address to a host pointer through the TLB. - /// \tparam ETYPE TLB entry type. - /// \tparam T Type of word that would be read with the pointer. - /// \param vaddr Target virtual address. - /// \param phptr Pointer to host pointer receiving value. - /// \returns True if successful (TLB hit), false otherwise. - template - bool translate_vaddr_via_tlb(uint64_t vaddr, unsigned char **phptr) { - return derived().template do_translate_vaddr_via_tlb(vaddr, phptr); + /// \brief Write a data buffer to memory padded with 0 + /// \param paddr Destination physical address. + /// \param data Pointer to source data buffer. + /// \param data_length Length of data buffer. + /// \param write_length_log2_size Log2 size of the total write length. + void write_memory_with_padding(uint64_t paddr, const unsigned char *data, uint64_t data_length, + int write_length_log2_size) { + return derived().do_write_memory_with_padding(paddr, data, data_length, write_length_log2_size); } - /// \brief Try to read a word from memory through the TLB. - /// \tparam ETYPE TLB entry type. - /// \tparam T Type of word to read. - /// \param vaddr Target virtual address. + /// \brief Reads a word from memory. + /// \tparam T Type of word to read, potentially unaligned. + /// \tparam A Type to which \p paddr and \p haddr are known to be aligned. + /// \param faddr Implementation-defined fast address. /// \param pval Pointer to word receiving value. - /// \returns True if successful (TLB hit), false otherwise. - template - bool read_memory_word_via_tlb(uint64_t vaddr, T *pval) { - static_assert(std::is_integral_v && sizeof(T) <= sizeof(uint64_t), "unsupported type"); - return derived().template do_read_memory_word_via_tlb(vaddr, pval); + /// \warning T must not cross page boundary starting from \p faddr + /// \warning T may or may not cross a Merkle tree word boundary starting from \p faddr! + template + void read_memory_word(fast_addr faddr, uint64_t pma_index, T *pval) { + static_assert(std::is_integral::value && sizeof(T) <= sizeof(uint64_t), "unsupported type"); + return derived().template do_read_memory_word(faddr, pma_index, pval); } - /// \brief Try to write a word to memory through the TLB. - /// \tparam ETYPE TLB entry type. + /// \brief Writes a word to memory. /// \tparam T Type of word to write. - /// \param vaddr Target virtual address. + /// \tparam A Type to which \p paddr and \p haddr are known to be aligned. + /// \param faddr Implementation-defined fast address. /// \param val Value to be written. - /// \returns True if successful (TLB hit), false otherwise. - template - bool write_memory_word_via_tlb(uint64_t vaddr, T val) { - static_assert(std::is_integral_v && sizeof(T) <= sizeof(uint64_t), "unsupported type"); - return derived().template do_write_memory_word_via_tlb(vaddr, val); + /// \details \p haddr is ONLY valid when there is a host machine. + /// It should never be referenced outside of this context. + /// \warning T must not cross page boundary starting from \p faddr + /// \warning T may or may not cross a Merkle tree word boundary starting from \p faddr! + template + void write_memory_word(fast_addr faddr, uint64_t pma_index, T val) { + static_assert(std::is_integral::value && sizeof(T) <= sizeof(uint64_t), "unsupported type"); + return derived().template do_write_memory_word(faddr, pma_index, val); + } + + /// \brief Reads TLB's vaddr_page + /// \tparam USE TLB set + /// \param slot_index Slot index + /// \returns Value in slot. + template + uint64_t read_tlb_vaddr_page(uint64_t slot_index) { + return derived().template do_read_tlb_vaddr_page(slot_index); + } + + /// \brief Reads TLB's vp_offset + /// \tparam USE TLB set + /// \param slot_index Slot index + /// \returns Value in slot. + template + fast_addr read_tlb_vp_offset(uint64_t slot_index) { + return derived().template do_read_tlb_vp_offset(slot_index); + } + + /// \brief Reads TLB's pma_index + /// \tparam USE TLB set + /// \param slot_index Slot index + /// \returns Value in slot. + template + uint64_t read_tlb_pma_index(uint64_t slot_index) { + return derived().template do_read_tlb_pma_index(slot_index); + } + + /// \brief Writes to a TLB slot + /// \tparam USE TLB set + /// \param slot_index Slot index + /// \param vaddr_page Value to write + /// \param vp_offset Value to write + /// \param pma_index Value to write + /// \detail Writes to the TLB must be modify all fields atomically to prevent an inconsistent state. + /// This simplifies all state access implementations. + template + void write_tlb(uint64_t slot_index, uint64_t vaddr_page, fast_addr vp_offset, uint64_t pma_index) { + return derived().template do_write_tlb(slot_index, vaddr_page, vp_offset, pma_index); + } + + /// \brief Converts a target physical address to the implementation-defined fast address + /// \param paddr Target physical address to convert + /// \param pma_index Index of PMA where address falls + /// \returns Correspnding implementation-defined fast address + fast_addr get_faddr(uint64_t paddr, uint64_t pma_index) const { + return derived().do_get_faddr(paddr, pma_index); + } + + /// \brief Marks a page as dirty + /// \param faddr Implementation-defined fast address. + /// \param pma_index Index of PMA where page falls + /// \details When there is a host machine, the Merkle tree only updates the hashes for pages that + /// have been modified. Pages can only be written to if they appear in the write TLB. Therefore, + /// the Merkle tree only considers the pages that are currently in the write TLB and those that + /// have been marked dirty. When a page leaves the write TLB, it is marked dirty. + /// If the state belongs to a host machine, then this call MUST be forwarded to machine::mark_dirty_page(); + void mark_dirty_page(fast_addr faddr, uint64_t pma_index) { + return derived().do_mark_dirty_page(faddr, pma_index); } - /// \brief Replaces an entry in the TLB. - /// \tparam ETYPE TLB entry type to replace. - /// \param vaddr Target virtual address. - /// \param paddr Target physical address. - /// \param pma PMA entry for the physical address. - /// \returns Pointer to page start in host memory. - template - unsigned char *replace_tlb_entry(uint64_t vaddr, uint64_t paddr, PMA_ENTRY_TYPE &pma) { - return derived().template do_replace_tlb_entry(vaddr, paddr, pma); - } - - /// \brief Invalidates all TLB entries of a type. - /// \tparam ETYPE TLB entry type to flush. - template - void flush_tlb_type() { - return derived().template do_flush_tlb_type(); + /// \brief Writes a character to the console + /// \param c Character to output + void putchar(uint8_t c) { + return derived().do_putchar(c); } - /// \brief Invalidates all TLB entries of all types. - NO_INLINE void flush_all_tlb() { - derived().template flush_tlb_type(); - derived().template flush_tlb_type(); - derived().template flush_tlb_type(); - } + // --- + // These methods ONLY need to be implemented when state belongs to non-reproducible host machine + // --- - /// \brief Invalidates TLB entries for a specific virtual address. - /// \param vaddr Target virtual address. - NO_INLINE void flush_tlb_vaddr(uint64_t vaddr) { - return derived().do_flush_tlb_vaddr(vaddr); + /// \brief Poll for external interrupts. + /// \param mcycle Current machine mcycle. + /// \param mcycle_max Maximum mcycle to wait for interrupts. + /// \returns A pair, the first value is the new machine mcycle advanced by the relative elapsed time while + /// polling, the second value is a boolean that is true when the poll is stopped due do an external interrupt + /// request. + /// \details When mcycle_max is greater than mcycle, this function will sleep until an external interrupt + /// is triggered or mcycle_max relative elapsed time is reached. + std::pair poll_external_interrupts(uint64_t mcycle, uint64_t mcycle_max) { + return derived().do_poll_external_interrupts(mcycle, mcycle_max); } /// \brief Returns true if soft yield HINT instruction is enabled at runtime @@ -730,22 +760,6 @@ class i_state_access { // CRTP return derived().do_get_soft_yield(); } - /// \brief Write a data buffer to memory padded with 0 - /// \param paddr Destination physical address. - /// \param data Pointer to source data buffer. - /// \param data_length Length of data buffer. - /// \param write_length_log2_size Log2 size of the total write length. - void write_memory_with_padding(uint64_t paddr, const unsigned char *data, uint64_t data_length, - int write_length_log2_size) { - return derived().do_write_memory_with_padding(paddr, data, data_length, write_length_log2_size); - } - - /// \brief Writes a character to the console - /// \param c Character to output - void putchar(uint8_t c) { - return derived().do_putchar(c); - } - /// \brief Reads a character from the console /// \returns Character read if any, -1 otherwise int getchar() { @@ -753,10 +767,30 @@ class i_state_access { // CRTP } #ifdef DUMP_COUNTERS + //??D we should probably remove this from the interface auto &get_statistics() { return derived().do_get_statistics(); } #endif + +protected: + + /// \brief Default implementation when state does not belong to non-reproducible host machine + bool do_get_soft_yield() { // NOLINT(readability-convert-member-functions-to-static) + return false; + } + + /// \brief Default implementation when state does not belong to non-reproducible host machine + std::pair do_poll_external_interrupts(uint64_t mcycle, + uint64_t /* mcycle_max */) { // NOLINT(readability-convert-member-functions-to-static) + return {mcycle, false}; + } + + /// \brief Default implementation when state does not belong to non-reproducible host machine + int do_getchar() { // NOLINT(readability-convert-member-functions-to-static) + return -1; + } + }; /// \brief SFINAE test implementation of the i_state_access interface diff --git a/src/interpret.cpp b/src/interpret.cpp index 231f8bb3b..d986d4f7e 100644 --- a/src/interpret.cpp +++ b/src/interpret.cpp @@ -21,8 +21,10 @@ #include #ifdef MICROARCHITECTURE +#ifdef OKUARCH #include "uarch-machine-state-access.h" #include "uarch-runtime.h" +#endif #else #include "record-step-state-access.h" #include "replay-step-state-access.h" @@ -31,9 +33,10 @@ #endif // MICROARCHITECTURE #include "compiler-defines.h" +#include "device-state-access.h" #include "i-state-access.h" #include "machine-statistics.h" -#include "shadow-tlb.h" +#include "tlb.h" /// \file /// \brief Interpreter implementation. @@ -334,7 +337,7 @@ static FORCE_INLINE void set_prv(STATE_ACCESS a, int new_prv) { INC_COUNTER(a.get_statistics(), priv_level[new_prv]); a.write_iprv(new_prv); // Invalidate all TLB entries - a.flush_all_tlb(); + flush_all_tlb(a); INC_COUNTER(a.get_statistics(), tlb_flush_all); INC_COUNTER(a.get_statistics(), tlb_flush_set_prv); //??D new privileged 1.11 draft says invalidation should @@ -823,6 +826,104 @@ static FORCE_INLINE int32_t insn_get_C_SWSP_imm(uint32_t insn) { return static_cast(((insn >> (9 - 2)) & 0x3c) | ((insn >> (7 - 6)) & 0xc0)); } +/// \brief Flushes out a TLB slot +/// \tparam USE TLB set +/// \tparam STATE_ACCESS Class of machine state accessor object. +/// \param a Machine state accessor object. +/// \param slot_index Slot index +template +static void flush_tlb_slot(STATE_ACCESS &a, uint64_t slot_index) { + // Make sure a valid page leaving the write TLB is marked as dirty + // We must do this BEFORE we modify the TLB entries themselves + // (Otherwise, we could stop uarch before it marks the page dirty but after + // the entry is no longer in the TLB, which would cause the Merkle tree to + // miss a dirty page.) + if constexpr (USE == TLB_WRITE) { + const auto old_vaddr_page = a.template read_tlb_vaddr_page(slot_index); + if (old_vaddr_page != TLB_INVALID_PAGE) { + auto old_pma_index = a.template read_tlb_pma_index(slot_index); + const auto old_faddr_page = old_vaddr_page + a.template read_tlb_vp_offset(slot_index); + a.mark_dirty_page(old_faddr_page, old_pma_index); + } + } + // We do not leave garbage behind in empty slots + // (It would make state access classes trickier to implement) + const auto vaddr_page = TLB_INVALID_PAGE; + const auto vp_offset = i_state_access_fast_addr_t{}; + const auto pma_index = TLB_INVALID_PMA_INDEX; + a.template write_tlb(slot_index, vaddr_page, vp_offset, pma_index); +} + +/// \brief Flushes out an entire TLB set +/// \tparam USE TLB set +/// \tparam STATE_ACCESS Class of machine state accessor object. +/// \param a Machine state accessor object. +template +static void flush_tlb_set(STATE_ACCESS &a) { + for (uint64_t slot_index = 0; slot_index < PMA_TLB_SIZE; ++slot_index) { + flush_tlb_slot(a, slot_index); + } +} + +/// \brief Flushes out the entire TLB +/// \tparam STATE_ACCESS Class of machine state accessor object. +/// \param a Machine state accessor object. +template +static void flush_all_tlb(STATE_ACCESS &a) { + flush_tlb_set(a); + flush_tlb_set(a); + flush_tlb_set(a); +} + +/// \brief Flushes out a single virtual mapping in the TLB +/// \tparam STATE_ACCESS Class of machine state accessor object. +/// \param a Machine state accessor object. +/// \param vaddr Virtual address to remove form mapping +template +static void flush_tlb_vaddr(STATE_ACCESS &a, uint64_t /* vaddr */) { + // We can't flush just one TLB entry for that specific virtual address, + // because megapages/gigapages may be in use while this TLB implementation ignores it, + // so we have to flush all addresses. + flush_tlb_set(a); + flush_tlb_set(a); + flush_tlb_set(a); +} + +/// \brief Replaces a virtual mapping in a TLB set +/// \tparam USE TLB set +/// \tparam STATE_ACCESS Class of machine state accessor object. +/// \param a Machine state accessor object. +/// \param vaddr Virtual address of new mapping +/// \param paddr Corresponding physical address +/// \param paddr Index of PMA where paddr falls +/// \param vp_offset Receives the new vp_offset that will be stored in the slot +/// \returns The implementation-defined fast address corresponding to paddr +template +static i_state_access_fast_addr_t replace_tlb_entry(STATE_ACCESS &a, uint64_t vaddr, uint64_t paddr, + uint64_t pma_index, i_state_access_fast_addr_t &vp_offset) { + const auto slot_index = tlb_slot_index(vaddr); + flush_tlb_slot(a, slot_index); + const auto vaddr_page = tlb_addr_page(vaddr); + const auto faddr = a.get_faddr(paddr, pma_index); + vp_offset = faddr - vaddr; + a.template write_tlb(slot_index, vaddr_page, vp_offset, pma_index); + return faddr; +} + +/// \brief Replaces a virtual mapping in a TLB set +/// \tparam USE TLB set +/// \tparam STATE_ACCESS Class of machine state accessor object. +/// \param a Machine state accessor object. +/// \param vaddr Virtual address of new mapping +/// \param paddr Corresponding physical address +/// \param paddr Index of PMA where paddr falls +/// \returns The implementation-defined fast address corresponding to paddr +template +static FORCE_INLINE auto replace_tlb_entry(STATE_ACCESS &a, uint64_t vaddr, uint64_t paddr, uint64_t pma_index) { + i_state_access_fast_addr_t vp_offset{0}; + return replace_tlb_entry(a, vaddr, paddr, pma_index, vp_offset); +} + /// \brief Read an aligned word from virtual memory (slow path that goes through virtual address translation). /// \tparam T uint8_t, uint16_t, uint32_t, or uint64_t. /// \tparam STATE_ACCESS Class of machine state accessor object. @@ -855,12 +956,12 @@ static NO_INLINE std::pair read_virtual_memory_slow(STATE_ACCESS vaddr); return {false, pc}; } - auto &pma = find_pma_entry(a, paddr); + uint64_t pma_index = 0; + const auto &pma = find_pma_entry(a, paddr, pma_index); if (likely(pma.get_istart_R())) { if (likely(pma.get_istart_M())) { - unsigned char *hpage = a.template replace_tlb_entry(vaddr, paddr, pma); - const uint64_t hoffset = vaddr & PAGE_OFFSET_MASK; - a.read_memory_word(paddr, hpage, hoffset, pval); + const auto faddr = replace_tlb_entry(a, vaddr, paddr, pma_index); + a.template read_memory_word(faddr, pma_index, pval); return {true, pc}; } if (likely(pma.get_istart_IO())) { @@ -893,7 +994,9 @@ static NO_INLINE std::pair read_virtual_memory_slow(STATE_ACCESS template static FORCE_INLINE bool read_virtual_memory(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint64_t vaddr, T *pval) { // Try hitting the TLB - if (unlikely(!(a.template read_memory_word_via_tlb(vaddr, pval)))) { + const auto slot_index = tlb_slot_index(vaddr); + const auto slot_vaddr_page = a.template read_tlb_vaddr_page(slot_index); + if (unlikely(!tlb_is_hit(slot_vaddr_page, vaddr))) { // Outline the slow path into a function call to minimize host CPU code cache pressure INC_COUNTER(a.get_statistics(), tlb_rmiss); T val = 0; // Don't pass pval reference directly so the compiler can store it in a register @@ -903,6 +1006,10 @@ static FORCE_INLINE bool read_virtual_memory(STATE_ACCESS a, uint64_t &pc, uint6 pc = new_pc; return status; } + const auto pma_index = a.template read_tlb_pma_index(slot_index); + const auto vp_offset = a.template read_tlb_vp_offset(slot_index); + const auto faddr = vaddr + vp_offset; + a.template read_memory_word(faddr, pma_index, pval); INC_COUNTER(a.get_statistics(), tlb_rhit); return true; } @@ -936,12 +1043,12 @@ static NO_INLINE std::pair write_virtual_memory_slow(S pc = raise_exception(a, pc, MCAUSE_STORE_AMO_PAGE_FAULT, vaddr); return {execute_status::failure, pc}; } - auto &pma = find_pma_entry(a, paddr); + uint64_t pma_index = 0; + auto &pma = find_pma_entry(a, paddr, pma_index); if (likely(pma.get_istart_W())) { if (likely(pma.get_istart_M())) { - unsigned char *hpage = a.template replace_tlb_entry(vaddr, paddr, pma); - const uint64_t hoffset = vaddr & PAGE_OFFSET_MASK; - a.write_memory_word(paddr, hpage, hoffset, static_cast(val64)); + const auto faddr = replace_tlb_entry(a, vaddr, paddr, pma_index); + a.write_memory_word(faddr, pma_index, static_cast(val64)); return {execute_status::success, pc}; } if (likely(pma.get_istart_IO())) { @@ -971,13 +1078,19 @@ template static FORCE_INLINE execute_status write_virtual_memory(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint64_t vaddr, uint64_t val64) { // Try hitting the TLB - if (unlikely((!a.template write_memory_word_via_tlb(vaddr, static_cast(val64))))) { - INC_COUNTER(a.get_statistics(), tlb_wmiss); + const uint64_t slot_index = tlb_slot_index(vaddr); + const uint64_t slot_vaddr_page = a.template read_tlb_vaddr_page(slot_index); + if (unlikely(!tlb_is_hit(slot_vaddr_page, vaddr))) { // Outline the slow path into a function call to minimize host CPU code cache pressure + INC_COUNTER(a.get_statistics(), tlb_wmiss); auto [status, new_pc] = write_virtual_memory_slow(a, pc, mcycle, vaddr, val64); pc = new_pc; return status; } + const auto pma_index = a.template read_tlb_pma_index(slot_index); + const auto vp_offset = a.template read_tlb_vp_offset(slot_index); + const auto faddr = vaddr + vp_offset; + a.template write_memory_word(faddr, pma_index, static_cast(val64)); INC_COUNTER(a.get_statistics(), tlb_whit); return execute_status::success; } @@ -2041,7 +2154,7 @@ static NO_INLINE execute_status write_csr_satp(STATE_ACCESS a, uint64_t val) { // because software is required to execute SFENCE.VMA when recycling an ASID. const uint64_t mod = old_satp ^ stap; if (mod & (SATP_ASID_MASK | SATP_MODE_MASK)) { - a.flush_all_tlb(); + flush_all_tlb(a); INC_COUNTER(a.get_statistics(), tlb_flush_all); INC_COUNTER(a.get_statistics(), tlb_flush_satp); return execute_status::success_and_flush_fetch; @@ -2103,11 +2216,11 @@ static NO_INLINE execute_status write_csr_mstatus(STATE_ACCESS a, uint64_t val) // Flush TLBs when needed if (flush_tlb_read) { - a.template flush_tlb_type(); + flush_tlb_set(a); INC_COUNTER(a.get_statistics(), tlb_flush_read); } if (flush_tlb_write) { - a.template flush_tlb_type(); + flush_tlb_set(a); INC_COUNTER(a.get_statistics(), tlb_flush_write); } if (flush_tlb_read || flush_tlb_write) { @@ -3358,7 +3471,7 @@ static FORCE_INLINE execute_status execute_SFENCE_VMA(STATE_ACCESS a, uint64_t & const uint32_t rs1 = insn_get_rs1(insn); [[maybe_unused]] const uint32_t rs2 = insn_get_rs2(insn); if (rs1 == 0) { - a.flush_all_tlb(); + flush_all_tlb(a); #ifdef DUMP_COUNTERS INC_COUNTER(a.get_statistics(), tlb_flush_all); if (rs2 == 0) { @@ -3373,7 +3486,7 @@ static FORCE_INLINE execute_status execute_SFENCE_VMA(STATE_ACCESS a, uint64_t & #endif } else { const uint64_t vaddr = a.read_x(rs1); - a.flush_tlb_vaddr(vaddr); + flush_tlb_vaddr(a, vaddr); #ifdef DUMP_COUNTERS INC_COUNTER(a.get_statistics(), tlb_flush_vaddr); if (rs2 == 0) { @@ -5212,12 +5325,13 @@ enum class fetch_status : int { /// \param a Machine state accessor object. /// \param pc Virtual address for the current instruction being executed. /// \param vaddr Virtual address to be fetched. -/// \param phptr Receives host pointer. +/// \param vp_offset Receives vp_offset in the TLB slot +/// \param pma_index Receives the index of PMA where vaddr falls /// \return Returns fetch_status::success if load succeeded, fetch_status::exception if it caused an exception. // In that case, raise the exception. template static FORCE_INLINE fetch_status fetch_translate_pc_slow(STATE_ACCESS a, uint64_t &pc, uint64_t vaddr, - unsigned char **phptr) { + i_state_access_fast_addr_t &vp_offset, uint64_t &pma_index) { uint64_t paddr{}; // Walk page table and obtain the physical address if (unlikely(!translate_virtual_address(a, &paddr, vaddr, PTE_XWR_X_SHIFT))) { @@ -5225,16 +5339,14 @@ static FORCE_INLINE fetch_status fetch_translate_pc_slow(STATE_ACCESS a, uint64_ return fetch_status::exception; } // Walk memory map to find the range that contains the physical address - auto &pma = find_pma_entry(a, paddr); + const auto &pma = find_pma_entry(a, paddr, pma_index); // We only execute directly from RAM (as in "random access memory") // If the range is not memory or not executable, this as a PMA violation if (unlikely(!pma.get_istart_M() || !pma.get_istart_X())) { pc = raise_exception(a, pc, MCAUSE_INSN_ACCESS_FAULT, vaddr); return fetch_status::exception; } - unsigned char *hpage = a.template replace_tlb_entry(vaddr, paddr, pma); - const uint64_t hoffset = vaddr & PAGE_OFFSET_MASK; - *phptr = hpage + hoffset; + replace_tlb_entry(a, vaddr, paddr, pma_index, vp_offset); return fetch_status::success; } @@ -5243,18 +5355,23 @@ static FORCE_INLINE fetch_status fetch_translate_pc_slow(STATE_ACCESS a, uint64_ /// \param a Machine state accessor object. /// \param pc Virtual address for the current instruction being executed. /// \param vaddr Virtual address to be fetched. -/// \param phptr Receives the host pointer. +/// \param vp_offset Receives vp_offset in the TLB slot +/// \param pma_index Receives the index of PMA where vaddr falls /// \return Returns fetch_status::success if load succeeded, fetch_status::exception if it caused an exception. // In that case, raise the exception. template static FORCE_INLINE fetch_status fetch_translate_pc(STATE_ACCESS a, uint64_t &pc, uint64_t vaddr, - unsigned char **phptr) { + i_state_access_fast_addr_t &vp_offset, uint64_t &pma_index) { // Try to perform the address translation via TLB first - if (unlikely(!(a.template translate_vaddr_via_tlb(vaddr, phptr)))) { + const uint64_t slot_index = tlb_slot_index(vaddr); + const uint64_t slot_vaddr_page = a.template read_tlb_vaddr_page(slot_index); + if (unlikely(!tlb_is_hit(slot_vaddr_page, vaddr))) { INC_COUNTER(a.get_statistics(), tlb_cmiss); // Outline the slow path into a function call to minimize host CPU code cache pressure - return fetch_translate_pc_slow(a, pc, vaddr, phptr); + return fetch_translate_pc_slow(a, pc, vaddr, vp_offset, pma_index); } + vp_offset = a.template read_tlb_vp_offset(slot_index); + pma_index = a.template read_tlb_pma_index(slot_index); INC_COUNTER(a.get_statistics(), tlb_chit); return fetch_status::success; } @@ -5264,66 +5381,57 @@ static FORCE_INLINE fetch_status fetch_translate_pc(STATE_ACCESS a, uint64_t &pc /// \param a Machine state accessor object. /// \param pc Virtual address for the current instruction being executed. /// \param insn Receives the instruction. -/// \param fetch_vaddr_page Fetch virtual address translation page cache. -/// \param fetch_vh_offset Fetch virtual address host pointer offset cache. +/// \param last_vaddr_page Receives and updates vaddr_page for cache. +/// \param last_vp_offset Receives and updates vp_offset for cache. +/// \param last_pma_index Receives and updates pma_index for cache. /// \return Returns fetch_status::success if load succeeded, fetch_status::exception if it caused an exception. // In that case, raise the exception. template -static FORCE_INLINE fetch_status fetch_insn(STATE_ACCESS a, uint64_t &pc, uint32_t &insn, uint64_t &fetch_vaddr_page, - uint64_t &fetch_vh_offset) { - // Efficiently checks if current pc is in the same page as last pc fetch - // and it's not crossing a page boundary. - if (likely((pc ^ fetch_vaddr_page) < (PMA_PAGE_SIZE - 2))) { - // Fetch pc is in the same page as the last pc fetch and it's not crossing a page boundary, - // we can just reuse last fetch translation, skipping TLB or slow address translation altogether. - const unsigned char *hptr = cast_addr_to_ptr(pc + fetch_vh_offset); - - // Here we are sure that reading 4 bytes won't cross a page boundary. - // However pc may not be 4 byte aligned, at best it can only be 2-byte aligned, - // therefore we must perform a misaligned 4 byte read on a 2 byte aligned pointer. - // In case pc holds a compressed instruction, insn will store 2 additional bytes, - // but this is fine because later the instruction decoder will discard them. - insn = aliased_unaligned_read(hptr); - return fetch_status::success; - } - // Fetch pc is either not the same as last cache or crossing a page boundary. - - // Perform address translation - unsigned char *hptr = nullptr; - if (unlikely(fetch_translate_pc(a, pc, pc, &hptr) == fetch_status::exception)) { - return fetch_status::exception; +static FORCE_INLINE fetch_status fetch_insn(STATE_ACCESS a, uint64_t &pc, uint32_t &insn, uint64_t &last_vaddr_page, + i_state_access_fast_addr_t &last_vp_offset, uint64_t &last_pma_index) { + i_state_access_fast_addr_t faddr{0}; + const uint64_t pc_vaddr_page = tlb_addr_page(pc); + // If pc is in the same page as the last pc fetch, + // we can just reuse last fetch translation, skipping TLB or slow address translation altogether. + if (likely(pc_vaddr_page == last_vaddr_page)) { + faddr = pc + last_vp_offset; + } else { + // Not in the same page as last the fetch, we need to perform address translation + if (unlikely(fetch_translate_pc(a, pc, pc, last_vp_offset, last_pma_index) == fetch_status::exception)) { + return fetch_status::exception; + } + // Update fetch address translation cache + last_vaddr_page = pc_vaddr_page; + faddr = pc + last_vp_offset; } - // Update fetch address translation cache - fetch_vaddr_page = pc & ~PAGE_OFFSET_MASK; - fetch_vh_offset = cast_ptr_to_addr(hptr) - pc; - // The following code assumes pc is always 2-byte aligned, this is guaranteed by RISC-V spec. // If pc is pointing to the very last 2 bytes of a page, it's crossing a page boundary. if (unlikely(((~pc & PAGE_OFFSET_MASK) >> 1) == 0)) { // Here we are crossing page boundary, this is unlikely (1 in 2048 possible cases) - insn = aliased_aligned_read(hptr); + uint16_t insn16 = 0; + a.template read_memory_word(faddr, last_pma_index, &insn16); + insn = insn16; // If not a compressed instruction, we must read 2 additional bytes from the next page. if (unlikely(insn_is_uncompressed(insn))) { // We have to perform a new address translation to read the next 2 bytes since we changed pages. - const uint64_t vaddr = pc + 2; - if (unlikely(fetch_translate_pc(a, pc, vaddr, &hptr) == fetch_status::exception)) { + const uint64_t pc2 = pc + 2; + if (unlikely(fetch_translate_pc(a, pc, pc2, last_vp_offset, last_pma_index) == fetch_status::exception)) { return fetch_status::exception; } - // Update fetch translation cache - fetch_vaddr_page = vaddr; - fetch_vh_offset = cast_ptr_to_addr(hptr) - vaddr; - // Produce the final 4-byte instruction - insn |= aliased_aligned_read(hptr) << 16; + last_vaddr_page = tlb_addr_page(pc2); + faddr = pc2 + last_vp_offset; + a.template read_memory_word(faddr, last_pma_index, &insn16); + insn |= insn16 << 16; } return fetch_status::success; } // Here we are sure that reading 4 bytes won't cross a page boundary. - // However pc may not be 4 byte aligned, at best it can only be 2-byte aligned, - // therefore we must perform a misaligned 4 byte read on a 2 byte aligned pointer. + // However pc may not be 4-byte aligned, at worst it could be only 2-byte aligned, + // therefore we must perform a misaligned 4-byte read on a 2-byte aligned pointer. // In case pc holds a compressed instruction, insn will store 2 additional bytes, // but this is fine because later the instruction decoder will discard them. - insn = aliased_unaligned_read(hptr); + a.template read_memory_word(faddr, last_pma_index, &insn); return fetch_status::success; } @@ -5352,8 +5460,9 @@ static NO_INLINE execute_status interpret_loop(STATE_ACCESS a, uint64_t mcycle_e uint64_t pc = a.read_pc(); // Initialize fetch address translation cache invalidated - uint64_t fetch_vaddr_page = ~pc; - uint64_t fetch_vh_offset = 0; + uint64_t fetch_vaddr_page = TLB_INVALID_PAGE; + uint64_t fetch_pma_index = TLB_INVALID_PMA_INDEX; + i_state_access_fast_addr_t fetch_vp_offset{}; // The outer loop continues until there is an interruption that should be handled // externally, or mcycle reaches mcycle_end @@ -5390,7 +5499,8 @@ static NO_INLINE execute_status interpret_loop(STATE_ACCESS a, uint64_t mcycle_e uint32_t insn = 0; // Try to fetch the next instruction - if (likely(fetch_insn(a, pc, insn, fetch_vaddr_page, fetch_vh_offset) == fetch_status::success)) { + if (likely(fetch_insn(a, pc, insn, fetch_vaddr_page, fetch_vp_offset, fetch_pma_index) == + fetch_status::success)) { // clang-format off // NOLINTBEGIN execute_status status; // explicit uninitialized as an optimization @@ -5856,7 +5966,7 @@ static NO_INLINE execute_status interpret_loop(STATE_ACCESS a, uint64_t mcycle_e // due to MRET/SRET instructions (execute_status::success_and_serve_interrupts) // As a simplification (and optimization), the next line will also invalidate in more cases, // but this it's fine. - fetch_vaddr_page = ~pc; + fetch_vaddr_page = TLB_INVALID_PAGE; // All status above execute_status::success_and_serve_interrupts will require breaking the loop if (unlikely(status >= execute_status::success_and_serve_interrupts)) { // Increment the cycle counter mcycle @@ -5932,14 +6042,16 @@ interpreter_break_reason interpret(STATE_ACCESS a, uint64_t mcycle_end) { } if (status == execute_status::success_and_yield) { return interpreter_break_reason::yielded_softly; - } // Reached mcycle_end + } // Reached mcycle_end assert(a.read_mcycle() == mcycle_end); // LCOV_EXCL_LINE return interpreter_break_reason::reached_target_mcycle; } #ifdef MICROARCHITECTURE +#ifdef OKUARCH // Explicit instantiation for uarch_machine_state_access template interpreter_break_reason interpret(uarch_machine_state_access a, uint64_t mcycle_end); +#endif #else // Explicit instantiation for state_access template interpreter_break_reason interpret(state_access a, uint64_t mcycle_end); diff --git a/src/machine-state.h b/src/machine-state.h index e13edea6d..0b85a386e 100644 --- a/src/machine-state.h +++ b/src/machine-state.h @@ -106,7 +106,7 @@ struct machine_state { } plic; /// \brief TLB state - shadow_tlb_state tlb{}; + tlb_state tlb{}; /// \brief HTIF state struct { diff --git a/src/machine.cpp b/src/machine.cpp index 46ff2acd8..f18acd299 100644 --- a/src/machine.cpp +++ b/src/machine.cpp @@ -65,14 +65,17 @@ #include "shadow-uarch-state.h" #include "state-access.h" #include "strict-aliasing.h" +#include "tlb.h" #include "translate-virtual-address.h" -#include "uarch-interpret.h" #include "uarch-pristine-state-hash.h" +#ifdef OKUARCH +#include "uarch-interpret.h" #include "uarch-record-state-access.h" #include "uarch-replay-state-access.h" #include "uarch-reset-state.h" #include "uarch-state-access.h" #include "uarch-step.h" +#endif #include "unique-c-ptr.h" #include "virtio-console.h" #include "virtio-device.h" @@ -223,48 +226,70 @@ void machine::replace_memory_range(const memory_range_config &range) { throw std::invalid_argument{"attempt to replace inexistent memory range"}; } -template -static void load_tlb_entry(machine &m, uint64_t eidx, unsigned char *hmem) { - tlb_hot_entry &tlbhe = m.get_state().tlb.hot[ETYPE][eidx]; - tlb_cold_entry &tlbce = m.get_state().tlb.cold[ETYPE][eidx]; - auto vaddr_page = aliased_aligned_read(hmem + tlb_get_vaddr_page_rel_addr(eidx)); - auto paddr_page = aliased_aligned_read(hmem + tlb_get_paddr_page_rel_addr(eidx)); - auto pma_index = aliased_aligned_read(hmem + tlb_get_pma_index_rel_addr(eidx)); - if (vaddr_page != TLB_INVALID_PAGE) { - if ((vaddr_page & ~PAGE_OFFSET_MASK) != vaddr_page) { - throw std::invalid_argument{"misaligned virtual page address in TLB entry"}; +void machine::init_tlb() { + for (auto use : {TLB_CODE, TLB_READ, TLB_WRITE}) { + auto &hot_set = m_s.tlb.hot[use]; + auto &cold_set = m_s.tlb.cold[use]; + for (uint64_t slot_index = 0; slot_index < TLB_SET_SIZE; ++slot_index) { + auto &hot_slot = hot_set[slot_index]; + hot_slot.vaddr_page = TLB_INVALID_PAGE; + hot_slot.vh_offset = host_addr{}; + auto &cold_slot = cold_set[slot_index]; + cold_slot.pma_index = TLB_INVALID_PMA_INDEX; } - if ((paddr_page & ~PAGE_OFFSET_MASK) != paddr_page) { - throw std::invalid_argument{"misaligned physical page address in TLB entry"}; - } - const pma_entry &pma = m.find_pma_entry(paddr_page); - // Checks if the PMA still valid - if (pma.get_length() == 0 || !pma.get_istart_M() || pma_index >= m.get_state().pmas.size() || - &pma != &m.get_state().pmas[pma_index]) { - throw std::invalid_argument{"invalid PMA for TLB entry"}; + } +} + +void machine::init_tlb(const shadow_tlb_state &shadow_tlb) { + const char *err_prefix = "stored TLB is corrupted: "; + for (auto use : {TLB_CODE, TLB_READ, TLB_WRITE}) { + auto &hot_set = m_s.tlb.hot[use]; + auto &cold_set = m_s.tlb.cold[use]; + auto &shadow_set = shadow_tlb[use]; + for (uint64_t slot_index = 0; slot_index < TLB_SET_SIZE; ++slot_index) { + // copy from shadow to live TLB + auto &shadow_slot = shadow_set[slot_index]; + auto &cold_slot = cold_set[slot_index]; + const auto pma_index = cold_slot.pma_index = shadow_slot.pma_index; + auto &hot_slot = hot_set[slot_index]; + const auto vaddr_page = hot_slot.vaddr_page = shadow_slot.vaddr_page; + // check consistency of slot, if valid, and compute live host address offset + if (shadow_slot.vaddr_page != TLB_INVALID_PAGE) { + // virtual page address must be page-aligned + if (shadow_slot.vaddr_page & PAGE_OFFSET_MASK) { + throw std::invalid_argument{err_prefix + "invalid vaddr_page in TLB slot"s}; + } + // PMA index cannot be outside pmas array + if (pma_index >= m_s.pmas.size()) { + throw std::invalid_argument{err_prefix + "invalid pma_index in active TLB slot"s}; + } + // PMA at index must be a non-empty memory PMA + const auto &pma = m_s.pmas[pma_index]; + if (pma.get_length() == 0 || !pma.get_istart_M()) { + throw std::invalid_argument{err_prefix + "invalid pma_index in active TLB slot"s}; + } + auto paddr_page = shadow_slot.vaddr_page + shadow_slot.vp_offset; + // translated physical page address must be page-aligned + if (paddr_page & PAGE_OFFSET_MASK) { + throw std::invalid_argument{err_prefix + "invalid vp_offset in active TLB slot"s}; + } + // translated physical page address must fall entirely into designated PMA + if (paddr_page < pma.get_start() || paddr_page - pma.get_start() > pma.get_length() - PMA_PAGE_SIZE) { + throw std::invalid_argument{err_prefix + "invalid vp_offset in active TLB slot"s}; + } + hot_slot.vh_offset = get_host_addr(paddr_page, pma_index) - vaddr_page; + } else { + if (shadow_slot.vp_offset != 0) { + throw std::invalid_argument{err_prefix + "invalid vp_offset in empty TLB slot"s}; + } + if (pma_index != TLB_INVALID_PMA_INDEX) { + throw std::invalid_argument{err_prefix + "invalid pma_index in empty TLB slot"s}; + } + // We never leave garbage behind, even in invalid slots + hot_slot.vh_offset = host_addr{}; + } } - const unsigned char *hpage = pma.get_memory().get_host_memory() + (paddr_page - pma.get_start()); - // Valid TLB entry - tlbhe.vaddr_page = vaddr_page; - tlbhe.vh_offset = cast_ptr_to_addr(hpage) - vaddr_page; - tlbce.paddr_page = paddr_page; - tlbce.pma_index = pma_index; - } else { // Empty or invalidated TLB entry - tlbhe.vaddr_page = vaddr_page; - tlbhe.vh_offset = 0; - tlbce.paddr_page = paddr_page; - tlbce.pma_index = pma_index; - } -} - -template -static void init_tlb_entry(machine &m, uint64_t eidx) { - tlb_hot_entry &tlbhe = m.get_state().tlb.hot[ETYPE][eidx]; - tlb_cold_entry &tlbce = m.get_state().tlb.cold[ETYPE][eidx]; - tlbhe.vaddr_page = TLB_INVALID_PAGE; - tlbhe.vh_offset = 0; - tlbce.paddr_page = TLB_INVALID_PAGE; - tlbce.pma_index = TLB_INVALID_PMA; + } } machine::machine(const machine_config &c, const machine_runtime_config &r) : m_c{c}, m_uarch{c.uarch}, m_r{r} { @@ -511,24 +536,16 @@ machine::machine(const machine_config &c, const machine_runtime_config &r) : m_c // Last, add sentinel PMA m_merkle_pmas.push_back(&m_s.empty_pma); - // Initialize TLB device - // this must be done after all PMA entries are already registered, so we can lookup page addresses + // Initialize TLB device. + // This must be done after all PMA entries are already registered, so we can lookup page addresses if (!m_c.tlb.image_filename.empty()) { - // Create a temporary PMA entry just to load TLB contents from an image file - pma_entry tlb_image_pma = make_mmapd_memory_pma_entry("shadow TLB device"s, PMA_SHADOW_TLB_START, - PMA_SHADOW_TLB_LENGTH, m_c.tlb.image_filename, false); - unsigned char *hmem = tlb_image_pma.get_memory().get_host_memory(); - for (uint64_t i = 0; i < PMA_TLB_SIZE; ++i) { - load_tlb_entry(*this, i, hmem); - load_tlb_entry(*this, i, hmem); - load_tlb_entry(*this, i, hmem); - } + unsigned char *buf = os_map_file(m_c.tlb.image_filename.c_str(), PMA_SHADOW_TLB_LENGTH, false); + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + const auto &shadow_tlb = *reinterpret_cast(buf); + init_tlb(shadow_tlb); + os_unmap_file(buf, PMA_SHADOW_TLB_LENGTH); } else { - for (uint64_t i = 0; i < PMA_TLB_SIZE; ++i) { - init_tlb_entry(*this, i); - init_tlb_entry(*this, i); - init_tlb_entry(*this, i); - } + init_tlb(); } // Initialize TTY if console input is enabled @@ -769,6 +786,33 @@ pma_entry &machine::find_pma_entry(const CONTAINER &pmas, uint64_t paddr, uint64 return const_cast(std::as_const(*this).find_pma_entry(pmas, paddr, length)); } +uint64_t machine::get_paddr(host_addr haddr, uint64_t pma_index) const { + return static_cast(haddr + get_hp_offset(pma_index)); +} + +host_addr machine::get_host_addr(uint64_t paddr, uint64_t pma_index) const { + return host_addr{paddr} - get_hp_offset(pma_index); +} + +void machine::mark_dirty_page(host_addr haddr, uint64_t pma_index) { + auto paddr = get_paddr(haddr, pma_index); + auto &pma = m_s.pmas[static_cast(pma_index)]; + pma.mark_dirty_page(paddr - pma.get_start()); +} + +host_addr machine::get_hp_offset(uint64_t pma_index) const { + if (pma_index >= m_s.pmas.size()) { + throw std::domain_error{"PMA is out of range"}; + } + const auto &pma = m_s.pmas[static_cast(pma_index)]; + if (!pma.get_istart_M()) { + throw std::domain_error{"PMA is not memory"}; + } + auto haddr = cast_ptr_to_host_addr(pma.get_memory().get_host_memory()); + auto paddr = pma.get_start(); + return paddr - haddr; +} + template const pma_entry &machine::find_pma_entry(const CONTAINER &pmas, uint64_t paddr, uint64_t length) const { for (const auto &p : pmas) { @@ -1663,18 +1707,22 @@ uint64_t machine::get_reg_address(reg r) { } void machine::mark_write_tlb_dirty_pages() const { - for (uint64_t i = 0; i < PMA_TLB_SIZE; ++i) { - const tlb_hot_entry &tlbhe = m_s.tlb.hot[TLB_WRITE][i]; - if (tlbhe.vaddr_page != TLB_INVALID_PAGE) { - const tlb_cold_entry &tlbce = m_s.tlb.cold[TLB_WRITE][i]; - if (tlbce.pma_index >= m_s.pmas.size()) { + auto &hot_set = m_s.tlb.hot[TLB_WRITE]; + auto &cold_set = m_s.tlb.cold[TLB_WRITE]; + for (uint64_t slot_index = 0; slot_index < PMA_TLB_SIZE; ++slot_index) { + const auto &hot_slot = hot_set[slot_index]; + if (hot_slot.vaddr_page != TLB_INVALID_PAGE) { + auto haddr_page = hot_slot.vaddr_page + hot_slot.vh_offset; + const auto &cold_slot = cold_set[slot_index]; + if (cold_slot.pma_index >= m_s.pmas.size()) { throw std::runtime_error{"could not mark dirty page for a TLB entry: TLB is corrupt"}; } - pma_entry &pma = m_s.pmas[tlbce.pma_index]; - if (!pma.contains(tlbce.paddr_page, PMA_PAGE_SIZE)) { + auto paddr_page = get_paddr(haddr_page, cold_slot.pma_index); + pma_entry &pma = m_s.pmas[cold_slot.pma_index]; + if (!pma.contains(paddr_page, PMA_PAGE_SIZE)) { throw std::runtime_error{"could not mark dirty page for a TLB entry: TLB is corrupt"}; } - pma.mark_dirty_page(tlbce.paddr_page - pma.get_start()); + pma.mark_dirty_page(paddr_page - pma.get_start()); } } } @@ -2137,11 +2185,14 @@ void machine::verify_send_cmio_response(uint16_t reason, const unsigned char *da } void machine::reset_uarch() { +#if OKUARCH uarch_state_access a(m_uarch.get_state(), get_state()); uarch_reset_state(a); +#endif } access_log machine::log_reset_uarch(const access_log::type &log_type) { +#if OKUARCH hash_type root_hash_before; get_root_hash(root_hash_before); // Call uarch_reset_state with a uarch_record_state_access object @@ -2155,10 +2206,13 @@ access_log machine::log_reset_uarch(const access_log::type &log_type) { get_root_hash(root_hash_after); verify_reset_uarch(root_hash_before, *a.get_log(), root_hash_after); return std::move(*a.get_log()); +#endif + return access_log{log_type}; } void machine::verify_reset_uarch(const hash_type &root_hash_before, const access_log &log, const hash_type &root_hash_after) { +#if OKUARCH // There must be at least one access in log if (log.get_accesses().empty()) { throw std::invalid_argument{"too few accesses in log"}; @@ -2173,12 +2227,19 @@ void machine::verify_reset_uarch(const hash_type &root_hash_before, const access if (obtained_root_hash != root_hash_after) { throw std::invalid_argument{"mismatch in root hash after replay"}; } +#endif + (void) root_hash_before; + (void) log; + (void) root_hash_after; } // Declaration of explicit instantiation in module uarch-step.cpp +#if OKUARCH extern template UArchStepStatus uarch_step(uarch_record_state_access &a); +#endif access_log machine::log_step_uarch(const access_log::type &log_type) { +#if OKUARCH if (m_uarch.get_state().ram.get_istart_E()) { throw std::runtime_error("microarchitecture RAM is not present"); } @@ -2194,13 +2255,18 @@ access_log machine::log_step_uarch(const access_log::type &log_type) { get_root_hash(root_hash_after); verify_step_uarch(root_hash_before, *a.get_log(), root_hash_after); return std::move(*a.get_log()); +#endif + return access_log{log_type}; } // Declaration of explicit instantiation in module uarch-step.cpp +#if OKUARCH extern template UArchStepStatus uarch_step(uarch_replay_state_access &a); +#endif void machine::verify_step_uarch(const hash_type &root_hash_before, const access_log &log, const hash_type &root_hash_after) { +#if OKUARCH // There must be at least one access in log if (log.get_accesses().empty()) { throw std::invalid_argument{"too few accesses in log"}; @@ -2215,6 +2281,10 @@ void machine::verify_step_uarch(const hash_type &root_hash_before, const access_ if (obtained_root_hash != root_hash_after) { throw std::invalid_argument{"mismatch in root hash after replay"}; } +#endif + (void) root_hash_before; + (void) log; + (void) root_hash_after; } machine_config machine::get_default_config() { @@ -2223,6 +2293,7 @@ machine_config machine::get_default_config() { // NOLINTNEXTLINE(readability-convert-member-functions-to-static) uarch_interpreter_break_reason machine::run_uarch(uint64_t uarch_cycle_end) { +#if OKUARCH if (read_reg(reg::iunrep) != 0) { throw std::runtime_error("microarchitecture cannot be used with unreproducible machines"); } @@ -2231,6 +2302,9 @@ uarch_interpreter_break_reason machine::run_uarch(uint64_t uarch_cycle_end) { } uarch_state_access a(m_uarch.get_state(), get_state()); return uarch_interpret(a, uarch_cycle_end); +#endif + (void) uarch_cycle_end; + return uarch_interpreter_break_reason::uarch_halted; } interpreter_break_reason machine::log_step(uint64_t mcycle_count, const std::string &filename) { diff --git a/src/machine.h b/src/machine.h index c7bb5f705..07ea6e104 100644 --- a/src/machine.h +++ b/src/machine.h @@ -28,6 +28,7 @@ #include #include "access-log.h" +#include "host-addr.h" #include "i-device-state-access.h" #include "interpret.h" #include "machine-config.h" @@ -125,6 +126,17 @@ class machine final { template const pma_entry &find_pma_entry(const CONTAINER &pmas, uint64_t paddr, uint64_t length) const; + /// \brief Returns offset that converts between machine host addresses and target physical addresses + /// \param pma_index Index of the memory PMA for the desired offset + host_addr get_hp_offset(uint64_t pma_index) const; + + /// \brief Initializes machine TLB from scratch + void init_tlb(); + + /// \brief Initializes machine TLB from shadow tlb + /// \param shadow_tlb Shadow TLB loaded from disk + void init_tlb(const shadow_tlb_state &shadow_tlb); + public: /// \brief Type of hash using hash_type = machine_merkle_tree::hash_type; @@ -430,6 +442,25 @@ class machine final { /// \param length Length of response data. void send_cmio_response(uint16_t reason, const unsigned char *data, uint64_t length); + /// \brief Converts from machine host address to target physical address + /// \param haddr Machine host address to convert + /// \param pma_index Index of PMA where address falls + /// \returns Corresponding target physical address + /// \details This method also converts from vh_offset to vp_offset + uint64_t get_paddr(host_addr haddr, uint64_t pma_index) const; + + /// \brief Converts from target physical address to machine host address + /// \param paddr Target physical address to convert + /// \param pma_index Index of PMA where address falls + /// \returns Corresponding machine host address + /// \details This method also converts from vp_offset to vh_offset + host_addr get_host_addr(uint64_t paddr, uint64_t pma_index) const; + + /// \brief Marks a page as dirty + /// \param haddr Machine host address within page + /// \param pma_index Index of PMA where address falls + void mark_dirty_page(host_addr haddr, uint64_t pma_index); + /// \brief Sends cmio response and returns an access log /// \param reason Reason for sending response. /// \param data Response data. diff --git a/src/pma.h b/src/pma.h index 7c2ae90aa..69d53d035 100644 --- a/src/pma.h +++ b/src/pma.h @@ -342,6 +342,11 @@ class pma_entry final { return *std::get_if(&m_data); } + /// \returns data specific to IO ranges (cannot throw exceptions). + const pma_memory &get_memory_noexcept() const { + return *std::get_if(&m_data); + } + /// \returns data specific to IO ranges const pma_device &get_device() const { return std::get(m_data); @@ -352,6 +357,11 @@ class pma_entry final { return std::get(m_data); } + /// \returns data specific to IO ranges (cannot throw exceptions). + const pma_device &get_device_noexcept() const { + return *std::get_if(&m_data); + } + /// \returns data specific to IO ranges (cannot throw exceptions). pma_device &get_device_noexcept() { return *std::get_if(&m_data); diff --git a/src/record-send-cmio-state-access.h b/src/record-send-cmio-state-access.h index c7f66cd76..3c595e53a 100644 --- a/src/record-send-cmio-state-access.h +++ b/src/record-send-cmio-state-access.h @@ -27,6 +27,7 @@ #include #include "access-log.h" +#include "host-addr.h" #include "i-hasher.h" #include "i-state-access.h" #include "machine-merkle-tree.h" @@ -38,10 +39,23 @@ namespace cartesi { +class record_send_cmio_state_access; + +// Type trait that should return the pma_entry type for a state access class +template <> +struct i_state_access_pma_entry { + using type = pma_entry; +}; +// Type trait that should return the fast_addr type for a state access class +template <> +struct i_state_access_fast_addr { + using type = host_addr; +}; + /// \class record_send_cmio_state_access /// \details This records all state accesses that happen during the execution of /// a machine::send_cmio_response() function call -class record_send_cmio_state_access : public i_state_access { +class record_send_cmio_state_access : public i_state_access { using hasher_type = machine_merkle_tree::hasher_type; using hash_type = machine_merkle_tree::hash_type; // NOLINTBEGIN(cppcoreguidelines-avoid-const-or-ref-data-members) @@ -63,6 +77,8 @@ class record_send_cmio_state_access : public i_state_access; + /// \brief Logs a read access of a uint64_t word from the machine state. /// \param paligned Physical address in the machine state, aligned to a 64-bit word. /// \param text Textual description of the access. @@ -168,9 +184,6 @@ class record_send_cmio_state_access : public i_state_access; - void do_push_bracket(bracket_type &type, const char *text) { m_log.push_bracket(type, text); } diff --git a/src/record-step-state-access.h b/src/record-step-state-access.h index a54cc2555..8c94c439f 100644 --- a/src/record-step-state-access.h +++ b/src/record-step-state-access.h @@ -23,6 +23,7 @@ #include "device-state-access.h" #include "i-state-access.h" #include "shadow-pmas.h" +#include "shadow-tlb.h" #include "unique-c-ptr.h" #include #include @@ -30,13 +31,25 @@ namespace cartesi { +class record_step_state_access; + +// Type trait that should return the pma_entry type for a state access class +template <> +struct i_state_access_pma_entry { + using type = pma_entry; +}; +// Type trait that should return the fast_addr type for a state access class +template <> +struct i_state_access_fast_addr { + using type = host_addr; +}; + /// \class record_step_state_access /// \brief Records machine state access into a step log file -class record_step_state_access : public i_state_access { -public: +class record_step_state_access : public i_state_access { constexpr static int TREE_LOG2_ROOT_SIZE = machine_merkle_tree::get_log2_root_size(); constexpr static int TREE_LOG2_PAGE_SIZE = machine_merkle_tree::get_log2_page_size(); - constexpr static uint64_t TREE_PAGE_SIZE = UINT64_C(1) << TREE_LOG2_PAGE_SIZE; + constexpr static uint64_t TREE_PAGE_SIZE = UINT64_C(1) << LOG2_PAGE_SIZE; using address_type = machine_merkle_tree::address_type; using page_data_type = std::array; @@ -45,6 +58,8 @@ class record_step_state_access : public i_state_access; using page_indices_type = std::vector; +public: + struct context { /// \brief Constructor of record step state access context /// \param filename where to save the log @@ -113,12 +128,14 @@ class record_step_state_access : public i_state_access; + using pma_entry_type = pma_entry; + using fast_addr_type = host_addr; + friend i_state_access; /// \brief Mark a page as touched and save its contents /// \param address address of the page void touch_page(address_type address) const { - auto page = address & ~(TREE_PAGE_SIZE - 1); + auto page = address & ~PAGE_OFFSET_MASK; if (m_context.touched_pages.find(page) != m_context.touched_pages.end()) { return; // already saved } @@ -590,180 +607,115 @@ class record_step_state_access : public i_state_access do_poll_external_interrupts(uint64_t mcycle, uint64_t mcycle_max) { - (void) mcycle_max; - return {mcycle, false}; - } - - template - void do_read_memory_word(uint64_t paddr, const unsigned char *hpage, uint64_t hoffset, T *pval) const { - (void) paddr; - touch_page(paddr); - *pval = cartesi::aliased_aligned_read(hpage + hoffset); - } - - template - void do_write_memory_word(uint64_t paddr, unsigned char *hpage, uint64_t hoffset, T val) { - (void) paddr; - touch_page(paddr); - aliased_aligned_write(hpage + hoffset, val); - } - // NOLINTNEXTLINE(readability-convert-member-functions-to-static) bool do_read_memory(uint64_t paddr, const unsigned char *data, uint64_t length) const { (void) paddr; (void) data; (void) length; - throw std::runtime_error("Unexpected call to do_read_memory"); + throw std::runtime_error("unexpected call to record_step_state_access::read_memory"); } // NOLINTNEXTLINE(readability-convert-member-functions-to-static) bool do_write_memory(uint64_t paddr, const unsigned char *data, uint64_t length) { - (void) paddr; (void) data; (void) length; - throw std::runtime_error("Unexpected call to do_write_memory"); - } - - static unsigned char *do_get_host_memory(pma_entry &pma) { - return pma.get_memory_noexcept().get_host_memory(); + throw std::runtime_error("unexpected call to record_step_state_access::write_memory"); } pma_entry &do_read_pma_entry(uint64_t index) { assert(index < PMA_MAX); - touch_page(shadow_pmas_get_pma_abs_addr(index)); - return m_m.get_state().pmas[index]; - } - - template - bool do_translate_vaddr_via_tlb(uint64_t vaddr, unsigned char **phptr) { - const uint64_t eidx = tlb_get_entry_index(vaddr); - touch_page(tlb_get_entry_hot_abs_addr(eidx)); - touch_page(tlb_get_entry_cold_abs_addr(eidx)); - const tlb_hot_entry &tlbhe = m_m.get_state().tlb.hot[ETYPE][eidx]; - if (unlikely(!tlb_is_hit(tlbhe.vaddr_page, vaddr))) { - return false; + // replay_step_state_access reconstructs a mock_pma_entry from the + // corresponding istart and ilength fields in the shadow pmas + // so we mark the page where they live here + touch_page(shadow_pmas_get_pma_istart_abs_addr(index)); + touch_page(shadow_pmas_get_pma_ilength_abs_addr(index)); + // NOLINTNEXTLINE(bugprone-narrowing-conversions) + return m_m.get_state().pmas[static_cast(index)]; + } + + template + void do_read_memory_word(host_addr haddr, uint64_t pma_index, T *pval) { + touch_page(m_m.get_paddr(haddr, pma_index)); + *pval = aliased_aligned_read(haddr); + } + + template + void do_write_memory_word(host_addr haddr, uint64_t pma_index, T val) { + touch_page(m_m.get_paddr(haddr, pma_index)); + aliased_aligned_write(haddr, val); + } + + template + uint64_t do_read_tlb_vaddr_page(uint64_t slot_index) { + touch_page(shadow_tlb_get_vaddr_page_abs_addr(slot_index)); + return m_m.get_state().tlb.hot[USE][slot_index].vaddr_page; + } + + template + host_addr do_read_tlb_vp_offset(uint64_t slot_index) { + // During initialization, replay_step_state_access translates all vp_offset to corresponding vh_offset + // At deinitialization, it translates them back + // To do that, it needs the corresponding paddr_page = vaddr_page + vp_offset, and page data itself + // It will only do the translation if the slot is valid and it has access to all required fields + // Obviously, the slot we are reading will be needed during replay, so we touch all the pages involved here. + touch_page(shadow_tlb_get_vaddr_page_abs_addr(slot_index)); + touch_page(shadow_tlb_get_vp_offset_abs_addr(slot_index)); + touch_page(shadow_tlb_get_pma_index_abs_addr(slot_index)); + // writes to the TLB slot are atomic, so we know the values in a slot are ALWAYS internally consistent + const auto vaddr_page = m_m.get_state().tlb.hot[USE][slot_index].vaddr_page; + const auto vh_offset = m_m.get_state().tlb.hot[USE][slot_index].vh_offset; + if (vaddr_page != TLB_INVALID_PAGE) { + const auto pma_index = m_m.get_state().tlb.cold[USE][slot_index].pma_index; + const auto haddr_page = vaddr_page + vh_offset; + auto paddr_page = m_m.get_paddr(haddr_page, pma_index); + touch_page(paddr_page); } - *phptr = cast_addr_to_ptr(tlbhe.vh_offset + vaddr); - const tlb_cold_entry &tlbce = m_m.get_state().tlb.cold[ETYPE][eidx]; - touch_page(tlbce.paddr_page); - return true; - } - - template - bool do_read_memory_word_via_tlb(uint64_t vaddr, T *pval) { - const uint64_t eidx = tlb_get_entry_index(vaddr); - const tlb_hot_entry &tlbhe = m_m.get_state().tlb.hot[ETYPE][eidx]; - const tlb_cold_entry &tlbce = m_m.get_state().tlb.cold[ETYPE][eidx]; - touch_page(tlb_get_entry_hot_abs_addr(eidx)); - touch_page(tlb_get_entry_cold_abs_addr( - eidx)); // save cold entry to allow reconstruction of paddr during playback - if (unlikely(!tlb_is_hit(tlbhe.vaddr_page, vaddr))) { - return false; + return vh_offset; + } + + template + uint64_t do_read_tlb_pma_index(uint64_t slot_index) { + touch_page(shadow_tlb_get_pma_index_abs_addr(slot_index)); + return m_m.get_state().tlb.cold[USE][slot_index].pma_index; + } + + template + void do_write_tlb(uint64_t slot_index, uint64_t vaddr_page, host_addr vh_offset, uint64_t pma_index) { + // During initialization, replay_step_state_access translates all vp_offset to corresponding vh_offset + // At deinitialization, it translates them back + // To do that, it needs the corresponding paddr_page = vaddr_page + vp_offset, and page data itself + // It will only do the translation if the slot is valid and it has access to all required fields + // Obviously, the slot we are modifying will be needed during replay, so we touch all the pages involved here. + touch_page(shadow_tlb_get_vaddr_page_abs_addr(slot_index)); + touch_page(shadow_tlb_get_vp_offset_abs_addr(slot_index)); + touch_page(shadow_tlb_get_pma_index_abs_addr(slot_index)); + m_m.get_state().tlb.hot[USE][slot_index].vaddr_page = vaddr_page; + m_m.get_state().tlb.hot[USE][slot_index].vh_offset = vh_offset; + m_m.get_state().tlb.cold[USE][slot_index].pma_index = pma_index; + if (vaddr_page != TLB_INVALID_PAGE) { + const auto haddr_page = vaddr_page + vh_offset; + auto paddr_page = m_m.get_paddr(haddr_page, pma_index); + touch_page(paddr_page); } - const auto *h = cast_addr_to_ptr(tlbhe.vh_offset + vaddr); - *pval = cartesi::aliased_aligned_read(h); - touch_page(tlbce.paddr_page); - - return true; } - template - bool do_write_memory_word_via_tlb(uint64_t vaddr, T val) { - const uint64_t eidx = tlb_get_entry_index(vaddr); - const tlb_hot_entry &tlbhe = m_m.get_state().tlb.hot[ETYPE][eidx]; - touch_page(tlb_get_entry_hot_abs_addr(eidx)); - touch_page(tlb_get_entry_cold_abs_addr(eidx)); - if (unlikely(!tlb_is_hit(tlbhe.vaddr_page, vaddr))) { - return false; - } - touch_page(tlb_get_entry_cold_abs_addr( - eidx)); // save cold entry to allow reconstruction of paddr during playback - const tlb_cold_entry &tlbce = m_m.get_state().tlb.cold[ETYPE][eidx]; - touch_page(tlbce.paddr_page); - - auto *h = cast_addr_to_ptr(tlbhe.vh_offset + vaddr); - aliased_aligned_write(h, val); - return true; - } - - template - unsigned char *do_replace_tlb_entry(uint64_t vaddr, uint64_t paddr, pma_entry &pma) { - const uint64_t eidx = tlb_get_entry_index(vaddr); - touch_page(tlb_get_entry_hot_abs_addr(eidx)); - touch_page(tlb_get_entry_cold_abs_addr( - eidx)); // save cold entry to allow reconstruction of paddr during playback - tlb_hot_entry &tlbhe = m_m.get_state().tlb.hot[ETYPE][eidx]; - tlb_cold_entry &tlbce = m_m.get_state().tlb.cold[ETYPE][eidx]; - // Mark page that was on TLB as dirty so we know to update the Merkle tree - if constexpr (ETYPE == TLB_WRITE) { - if (tlbhe.vaddr_page != TLB_INVALID_PAGE) { - pma_entry &pma = do_read_pma_entry(tlbce.pma_index); - pma.mark_dirty_page(tlbce.paddr_page - pma.get_start()); - } - } - const uint64_t vaddr_page = vaddr & ~PAGE_OFFSET_MASK; - const uint64_t paddr_page = paddr & ~PAGE_OFFSET_MASK; - unsigned char *hpage = pma.get_memory_noexcept().get_host_memory() + (paddr_page - pma.get_start()); - tlbhe.vaddr_page = vaddr_page; - tlbhe.vh_offset = cast_ptr_to_addr(hpage) - vaddr_page; - tlbce.paddr_page = paddr_page; - tlbce.pma_index = static_cast(pma.get_index()); - touch_page(tlbce.paddr_page); - return hpage; - } - - template - void do_flush_tlb_entry(uint64_t eidx) { - touch_page(tlb_get_entry_hot_abs_addr(eidx)); - touch_page(tlb_get_entry_cold_abs_addr( - eidx)); // save cold entry to allow reconstruction of paddr during playback - tlb_hot_entry &tlbhe = m_m.get_state().tlb.hot[ETYPE][eidx]; - // Mark page that was on TLB as dirty so we know to update the Merkle tree - if constexpr (ETYPE == TLB_WRITE) { - if (tlbhe.vaddr_page != TLB_INVALID_PAGE) { - tlbhe.vaddr_page = TLB_INVALID_PAGE; - const tlb_cold_entry &tlbce = m_m.get_state().tlb.cold[ETYPE][eidx]; - pma_entry &pma = do_read_pma_entry(tlbce.pma_index); - pma.mark_dirty_page(tlbce.paddr_page - pma.get_start()); - } else { - tlbhe.vaddr_page = TLB_INVALID_PAGE; - } - } else { - tlbhe.vaddr_page = TLB_INVALID_PAGE; - } - } - - template - void do_flush_tlb_type() { - for (uint64_t i = 0; i < PMA_TLB_SIZE; ++i) { - do_flush_tlb_entry(i); - } - } - - void do_flush_tlb_vaddr(uint64_t vaddr) { - (void) vaddr; - // We can't flush just one TLB entry for that specific virtual address, - // because megapages/gigapages may be in use while this TLB implementation ignores it, - // so we have to flush all addresses. - do_flush_tlb_type(); - do_flush_tlb_type(); - do_flush_tlb_type(); + fast_addr do_get_faddr(uint64_t paddr, uint64_t pma_index) const { + // replay_step_state_access needs the corresponding page to perform a + // translation between paddr and its own haddr, so we touch the page here + touch_page(paddr); + return m_m.get_host_addr(paddr, pma_index); } - bool do_get_soft_yield() { - return m_m.get_state().soft_yield; + void do_mark_dirty_page(host_addr haddr, uint64_t pma_index) { + // this is a noop in replay_step_state_access, so we do nothing else + m_m.mark_dirty_page(haddr, pma_index); } void do_putchar(uint8_t c) { // NOLINT(readability-convert-member-functions-to-static) os_putchar(c); } - int do_getchar() { // NOLINT(readability-convert-member-functions-to-static) - return -1; - } }; } // namespace cartesi diff --git a/src/replay-send-cmio-state-access.h b/src/replay-send-cmio-state-access.h index 29242b308..155c3a909 100644 --- a/src/replay-send-cmio-state-access.h +++ b/src/replay-send-cmio-state-access.h @@ -34,21 +34,35 @@ #include "i-state-access.h" #include "machine-merkle-tree.h" #include "meta.h" -#include "pma.h" +#include "mock-pma-entry.h" #include "riscv-constants.h" #include "shadow-state.h" #include "unique-c-ptr.h" namespace cartesi { +class replay_send_cmio_state_access; + +// Type trait that should return the pma_entry type for a state access class +template <> +struct i_state_access_pma_entry { + using type = mock_pma_entry; +}; +// Type trait that should return the fast_addr type for a state access class +template <> +struct i_state_access_fast_addr { + using type = uint64_t; +}; + /// \brief Allows replaying a machine::send_cmio_response() from an access log. -class replay_send_cmio_state_access : public i_state_access { -public: +class replay_send_cmio_state_access : public i_state_access { using tree_type = machine_merkle_tree; using hash_type = tree_type::hash_type; using hasher_type = tree_type::hasher_type; using proof_type = tree_type::proof_type; +public: + struct context { /// \brief Constructor replay_send_cmio_state_access context /// \param log Access log to be replayed @@ -91,7 +105,7 @@ class replay_send_cmio_state_access : public i_state_access; + friend i_state_access; std::string access_to_report() const { auto index = m_context.next_access + 1; diff --git a/src/replay-step-state-access.h b/src/replay-step-state-access.h index d219a0c4c..0d757ac41 100644 --- a/src/replay-step-state-access.h +++ b/src/replay-step-state-access.h @@ -22,6 +22,7 @@ #include "compiler-defines.h" #include "device-state-access.h" +#include "host-addr.h" #include "i-state-access.h" #include "mock-pma-entry.h" #include "pma-constants.h" @@ -40,6 +41,19 @@ namespace cartesi { // \file this code is designed to be compiled for a free-standing environment. // Environment-specific functions have the prefix "interop_" and are declared in "replay-step-state-access-interop.h" +class replay_step_state_access; + +// Type trait that should return the pma_entry type for a state access class +template <> +struct i_state_access_pma_entry { + using type = mock_pma_entry; +}; +// Type trait that should return the fast_addr type for a state access class +template <> +struct i_state_access_fast_addr { + using type = host_addr; +}; + // \brief checks if a buffer is large enough to hold a data block of N elements of size S starting at a given offset // \param max The maximum offset allowed // \param current The current offset @@ -59,13 +73,8 @@ static inline bool validate_and_advance_offset(uint64_t max, uint64_t current, u return *next <= max; } -static_assert(sizeof(shadow_tlb_state::hot[0]) == PMA_PAGE_SIZE, "size of hot tlb cache must be PM_PAGE_SIZE bytes"); -static_assert(sizeof(shadow_tlb_state::cold[0]) == PMA_PAGE_SIZE, "size of cold tlb cache must be PM_PAGE_SIZE bytes"); -static_assert(sizeof(shadow_tlb_state::hot) + sizeof(shadow_tlb_state::cold) == sizeof(shadow_tlb_state), - "size of shadow tlb state"); - // \brief Provides machine state from a step log file -class replay_step_state_access : public i_state_access { +class replay_step_state_access : public i_state_access { public: using address_type = uint64_t; using data_type = unsigned char[PMA_PAGE_SIZE]; @@ -101,15 +110,10 @@ class replay_step_state_access : public i_state_access(log_image), log_size, &end_offset)) { - interop_throw_runtime_error("step log size overflow"); - } - // set page count if (!validate_and_advance_offset(log_size, 0, sizeof(m_context.page_count), 1, &first_page_offset)) { interop_throw_runtime_error("page count past end of step log"); @@ -129,18 +133,18 @@ class replay_step_state_access : public i_state_access(log_image + first_siblng_offset); + m_context.sibling_hashes = reinterpret_cast(log_image + first_sibling_offset); // ensure that we read exactly the expected log size if (end_offset != log_size) { @@ -165,9 +169,9 @@ class replay_step_state_access : public i_state_access(); - relocate_all_tlb_vh_offset(); - relocate_all_tlb_vh_offset(); + relocate_tlb_vp_offset_to_vh_offset(); + relocate_tlb_vp_offset_to_vh_offset(); + relocate_tlb_vp_offset_to_vh_offset(); } // \brief Finish the replay and check the final machine root hash @@ -176,9 +180,9 @@ class replay_step_state_access : public i_state_access(); - reset_all_tlb_vh_offset(); - reset_all_tlb_vh_offset(); + relocate_tlb_vh_offset_to_vp_offset(); + relocate_tlb_vh_offset_to_vp_offset(); + relocate_tlb_vh_offset_to_vp_offset(); // compute and check machine root hash after the replay auto computed_final_root_hash = compute_root_hash(); if (computed_final_root_hash != root_hash_after) { @@ -187,72 +191,42 @@ class replay_step_state_access : public i_state_access; + friend i_state_access; - // \brief Relocate all TLB virtual to host offsets - // \details Points the vh_offset relative to the logged page data - template - void relocate_all_tlb_vh_offset() { - // NOLINTBEGIN(cppcoreguidelines-pro-type-reinterpret-cast)) - for (size_t i = 0; i < PMA_TLB_SIZE; ++i) { - const auto he_address = tlb_get_entry_hot_abs_addr(i); - const auto he_page = he_address & ~(PMA_PAGE_SIZE - 1); - const auto he_offset = he_address - he_page; - auto *he_log = try_find_page(he_page); - // if the page containing the tlb hot entries is present in the log - if (he_log) { - volatile tlb_hot_entry *tlbhe = reinterpret_cast(he_log->data + he_offset); - if (tlbhe->vh_offset != 0) { - interop_throw_runtime_error("expected vh_offset to be zero"); - } - // find the logged cold entry page - const auto ce_addr = tlb_get_entry_cold_abs_addr(i); - const auto ce_page = ce_addr & ~(PMA_PAGE_SIZE - 1); - const auto ce_offset = ce_addr - ce_page; - auto *ce_log = find_page(ce_page); - volatile tlb_cold_entry *tlbce = reinterpret_cast(ce_log->data + ce_offset); - // find the logged page pointed by the cold entry - auto *log = try_find_page(tlbce->paddr_page); - if (log) { - // point vh_offset to the logged page data - tlbhe->vh_offset = cast_ptr_to_addr(log->data) - tlbhe->vaddr_page; - } + /// \brief Try to find a page in the logged data by its physical address + /// \param paddr The physical address of the page + /// \return A pointer to the page_type structure if found, nullptr otherwise + page_type *try_find_page(uint64_t paddr_page) const { + const auto page_index = paddr_page >> PMA_PAGE_SIZE_LOG2; + uint64_t min{0}; + uint64_t max{m_context.page_count}; + while (min < max) { + auto mid = (min + max) >> 1; + if (m_context.pages[mid].index == page_index) { + return &m_context.pages[mid]; } - } - // NOLINTEND(cppcoreguidelines-pro-type-reinterpret-cast)) - } - - // \brief Reset all TLB virtual to host offsets - // \details Points the vh_offset to zero, replicating the behavior of the tlb pma device - template - void reset_all_tlb_vh_offset() { - // NOLINTBEGIN(cppcoreguidelines-pro-type-reinterpret-cast)) - for (size_t i = 0; i < PMA_TLB_SIZE; ++i) { - const auto addr = tlb_get_entry_hot_abs_addr(i); - const auto page = addr & ~(PMA_PAGE_SIZE - 1); - const auto offset = addr - page; - auto *p = try_find_page(page); - if (p) { - volatile tlb_hot_entry *tlbhe = reinterpret_cast(p->data + offset); - tlbhe->vh_offset = 0; + if (m_context.pages[mid].index < page_index) { + min = mid + 1; + } else { + max = mid; } } - // NOLINTEND(cppcoreguidelines-pro-type-reinterpret-cast)) + return nullptr; } - /// \brief Try to find a page in the logged data - /// \param address The physical address of the page + /// \brief Try to find a page in the logged data by the host address of its data + /// \param haddr Host address of page data /// \return A pointer to the page_type structure if found, nullptr otherwise - page_type *try_find_page(uint64_t address) const { - const auto page_index = address >> PMA_PAGE_SIZE_LOG2; + page_type *try_find_page(host_addr haddr_page) const { uint64_t min{0}; uint64_t max{m_context.page_count}; while (min < max) { auto mid = (min + max) >> 1; - if (m_context.pages[mid].index == page_index) { + auto mid_page_data = cast_ptr_to_host_addr(m_context.pages[mid].data); + if (mid_page_data == haddr_page) { return &m_context.pages[mid]; } - if (m_context.pages[mid].index < page_index) { + if (mid_page_data < haddr_page) { min = mid + 1; } else { max = mid; @@ -261,63 +235,107 @@ class replay_step_state_access : public i_state_access + void relocate_tlb_vp_offset_to_vh_offset() { + for (uint64_t slot_index = 0; slot_index < TLB_SET_SIZE; ++slot_index) { + const auto vp_offset_field_addr = shadow_tlb_get_vp_offset_abs_addr(slot_index); + auto *vp_offset_log = try_find_page(vp_offset_field_addr & ~PAGE_OFFSET_MASK); + const auto vaddr_page_field_addr = shadow_tlb_get_vaddr_page_abs_addr(slot_index); + auto *vaddr_page_log = try_find_page(vaddr_page_field_addr & ~PAGE_OFFSET_MASK); + // If vp_offset was accessed during record, both it and vaddr_apge will appear in the log + // (record_step_state_access makes sure of it) + // Otherwise, we do not need to translate + if (vp_offset_log == nullptr || vaddr_page_log == nullptr) { + continue; + } + const auto vaddr_page_field_haddr = + cast_ptr_to_host_addr(vaddr_page_log->data) + (vaddr_page_field_addr & PAGE_OFFSET_MASK); + const auto vaddr_page = aliased_aligned_read(vaddr_page_field_haddr); + // If, moreover, the slot was valid, the corresponding page will also appear in the log + // (record_step_state_access makes sure of it) + // Otherwise, we do not need to translate + if (vaddr_page != TLB_INVALID_PAGE) { + const auto vp_offset_field_haddr = + cast_ptr_to_host_addr(vp_offset_log->data) + (vp_offset_field_addr & PAGE_OFFSET_MASK); + const auto vp_offset = aliased_aligned_read(vp_offset_field_haddr); + const auto paddr_page = vaddr_page + vp_offset; + auto *page_log = try_find_page(paddr_page); + if (page_log == nullptr) { + continue; + } + const auto haddr_page = cast_ptr_to_host_addr(page_log->data); + const auto vh_offset = haddr_page - vaddr_page; + aliased_aligned_write(vp_offset_field_haddr, vh_offset); + } } - return page; } - /// \brief Find a page in the logged data - /// \param address The physical address of the page - /// \return A pointer to the page_type structure - page_type *find_page(uint64_t address) { - // NOLINTNEXTLINE(cppcoreguidelines-pro-type-const-cast): remove const to reuse code - return static_cast(this)->find_page(address); + // \brief Reverses changes to TLB so we have vp_offset fields again instead of vh_offset + // \details This makes the translation point back to target physical addresses + template + void relocate_tlb_vh_offset_to_vp_offset() { + for (uint64_t slot_index = 0; slot_index < TLB_SET_SIZE; ++slot_index) { + const auto vp_offset_field_addr = shadow_tlb_get_vp_offset_abs_addr(slot_index); + auto *vp_offset_log = try_find_page(vp_offset_field_addr & ~PAGE_OFFSET_MASK); + const auto vaddr_page_field_addr = shadow_tlb_get_vaddr_page_abs_addr(slot_index); + auto *vaddr_page_log = try_find_page(vaddr_page_field_addr & ~PAGE_OFFSET_MASK); + // If vp_offset was accessed during record, both it and vaddr_apge will appear in the log + // (record_step_state_access makes sure of it) + // Otherwise, we do not need to translate + if (vp_offset_log == nullptr || vaddr_page_log == nullptr) { + continue; + } + const auto vaddr_page_field_haddr = + cast_ptr_to_host_addr(vaddr_page_log->data) + (vaddr_page_field_addr & PAGE_OFFSET_MASK); + const auto vaddr_page = aliased_aligned_read(vaddr_page_field_haddr); + // If, moreover, the slot was valid, the corresponding page will also appear in the log + // (record_step_state_access makes sure of it) + // Otherwise, we do not need to translate + // We also don't need to translate back if it is no longer valid + if (vaddr_page != TLB_INVALID_PAGE) { + const auto vp_offset_field_haddr = + cast_ptr_to_host_addr(vp_offset_log->data) + (vp_offset_field_addr & PAGE_OFFSET_MASK); + const auto vh_offset = aliased_aligned_read(vp_offset_field_haddr); + const auto haddr_page = vaddr_page + vh_offset; + auto *page_log = try_find_page(haddr_page); + if (page_log == nullptr) { + continue; + } + const auto paddr_page = page_log->index << PMA_PAGE_SIZE_LOG2; + const auto vp_offset = paddr_page - vaddr_page; + aliased_aligned_write(vp_offset_field_haddr, vp_offset); + } + } } - /// \brief Get the raw memory pointer for a given physical address - /// \param paddr The physical address - /// \param size The size of the memory region - /// \return A pointer to the raw memory - void *get_raw_memory_pointer(uint64_t paddr, size_t size) const { - auto page = paddr & ~(PMA_PAGE_SIZE - 1); - const auto offset = paddr - page; - auto end_page = (paddr + size - 1) & ~(PMA_PAGE_SIZE - 1); - if (end_page != page) { - interop_throw_runtime_error("get_raw_memory_pointer: paddr crosses page boundary"); + page_type *find_page(uint64_t paddr_page) const { + auto *page_log = try_find_page(paddr_page); + if (page_log == nullptr) { + interop_throw_runtime_error("find_page: page not found"); } - auto *data = find_page(page); - auto *p = data->data + offset; - return p; + return page_log; } - /// \brief Read a value from raw memory - /// \tparam T The type of the value to read - /// \param paddr The physical address - /// \return The value read - template - T raw_read_memory(uint64_t paddr) const { - auto size = sizeof(T); - // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) - volatile T *ptr = reinterpret_cast(get_raw_memory_pointer(paddr, size)); - return *ptr; + page_type *find_page(host_addr haddr_page) const { + auto *page_log = try_find_page(haddr_page); + if (page_log == nullptr) { + interop_throw_runtime_error("find_page: page not found"); + } + return page_log; } - /// \brief Write a value to raw memory - /// \tparam T The type of the value to write + /// \brief Convert physical address to host address /// \param paddr The physical address - /// \param val The value to write - template - void raw_write_memory(uint64_t paddr, T val) { - auto size = sizeof(T); - // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) - volatile T *ptr = reinterpret_cast(get_raw_memory_pointer(paddr, size)); - *ptr = val; + /// \return Host address + host_addr do_get_faddr(uint64_t paddr, uint64_t /* pma_index */ = 0) const { + // This assumes the corresponding page has been touched + // (replay_step_state_access makes sure of it for any address we try to convert) + const auto paddr_page = paddr & ~PAGE_OFFSET_MASK; + auto *page_log = find_page(paddr_page); + const auto offset = paddr & PAGE_OFFSET_MASK; + return cast_ptr_to_host_addr(page_log->data) + offset; } // \brief Compute the current machine root hash @@ -388,353 +406,417 @@ class replay_step_state_access : public i_state_access(machine_reg_address(machine_reg::x0, reg)); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::x0, reg)); + return aliased_aligned_read(haddr); } void do_write_x(int reg, uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::x0, reg), val); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::x0, reg)); + aliased_aligned_write(haddr, val); } uint64_t do_read_f(int reg) { - return raw_read_memory(machine_reg_address(machine_reg::f0, reg)); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::f0, reg)); + return aliased_aligned_read(haddr); } void do_write_f(int reg, uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::f0, reg), val); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::f0, reg)); + aliased_aligned_write(haddr, val); } uint64_t do_read_pc() { - return raw_read_memory(machine_reg_address(machine_reg::pc)); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::pc)); + return aliased_aligned_read(haddr); } void do_write_pc(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::pc), val); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::pc)); + aliased_aligned_write(haddr, val); } uint64_t do_read_fcsr() { - return raw_read_memory(machine_reg_address(machine_reg::fcsr)); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::fcsr)); + return aliased_aligned_read(haddr); } void do_write_fcsr(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::fcsr), val); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::fcsr)); + aliased_aligned_write(haddr, val); } uint64_t do_read_icycleinstret() { - return raw_read_memory(machine_reg_address(machine_reg::icycleinstret)); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::icycleinstret)); + return aliased_aligned_read(haddr); } void do_write_icycleinstret(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::icycleinstret), val); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::icycleinstret)); + aliased_aligned_write(haddr, val); } uint64_t do_read_mvendorid() { - return raw_read_memory(machine_reg_address(machine_reg::mvendorid)); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::mvendorid)); + return aliased_aligned_read(haddr); } uint64_t do_read_marchid() { - return raw_read_memory(machine_reg_address(machine_reg::marchid)); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::marchid)); + return aliased_aligned_read(haddr); } uint64_t do_read_mimpid() { - return raw_read_memory(machine_reg_address(machine_reg::mimpid)); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::mimpid)); + return aliased_aligned_read(haddr); } uint64_t do_read_mcycle() { - return raw_read_memory(machine_reg_address(machine_reg::mcycle)); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::mcycle)); + return aliased_aligned_read(haddr); } void do_write_mcycle(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::mcycle), val); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::mcycle)); + aliased_aligned_write(haddr, val); } uint64_t do_read_mstatus() { - return raw_read_memory(machine_reg_address(machine_reg::mstatus)); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::mstatus)); + return aliased_aligned_read(haddr); } void do_write_mstatus(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::mstatus), val); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::mstatus)); + aliased_aligned_write(haddr, val); } uint64_t do_read_mtvec() { - return raw_read_memory(machine_reg_address(machine_reg::mtvec)); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::mtvec)); + return aliased_aligned_read(haddr); } void do_write_mtvec(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::mtvec), val); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::mtvec)); + aliased_aligned_write(haddr, val); } uint64_t do_read_mscratch() { - return raw_read_memory(machine_reg_address(machine_reg::mscratch)); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::mscratch)); + return aliased_aligned_read(haddr); } void do_write_mscratch(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::mscratch), val); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::mscratch)); + aliased_aligned_write(haddr, val); } uint64_t do_read_mepc() { - return raw_read_memory(machine_reg_address(machine_reg::mepc)); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::mepc)); + return aliased_aligned_read(haddr); } void do_write_mepc(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::mepc), val); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::mepc)); + aliased_aligned_write(haddr, val); } uint64_t do_read_mcause() { - return raw_read_memory(machine_reg_address(machine_reg::mcause)); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::mcause)); + return aliased_aligned_read(haddr); } void do_write_mcause(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::mcause), val); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::mcause)); + aliased_aligned_write(haddr, val); } uint64_t do_read_mtval() { - return raw_read_memory(machine_reg_address(machine_reg::mtval)); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::mtval)); + return aliased_aligned_read(haddr); } void do_write_mtval(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::mtval), val); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::mtval)); + aliased_aligned_write(haddr, val); } uint64_t do_read_misa() { - return raw_read_memory(machine_reg_address(machine_reg::misa)); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::misa)); + return aliased_aligned_read(haddr); } void do_write_misa(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::misa), val); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::misa)); + aliased_aligned_write(haddr, val); } uint64_t do_read_mie() { - return raw_read_memory(machine_reg_address(machine_reg::mie)); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::mie)); + return aliased_aligned_read(haddr); } void do_write_mie(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::mie), val); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::mie)); + aliased_aligned_write(haddr, val); } uint64_t do_read_mip() { - return raw_read_memory(machine_reg_address(machine_reg::mip)); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::mip)); + return aliased_aligned_read(haddr); } void do_write_mip(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::mip), val); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::mip)); + aliased_aligned_write(haddr, val); } uint64_t do_read_medeleg() { - return raw_read_memory(machine_reg_address(machine_reg::medeleg)); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::medeleg)); + return aliased_aligned_read(haddr); } void do_write_medeleg(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::medeleg), val); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::medeleg)); + aliased_aligned_write(haddr, val); } uint64_t do_read_mideleg() { - return raw_read_memory(machine_reg_address(machine_reg::mideleg)); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::mideleg)); + return aliased_aligned_read(haddr); } void do_write_mideleg(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::mideleg), val); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::mideleg)); + aliased_aligned_write(haddr, val); } uint64_t do_read_mcounteren() { - return raw_read_memory(machine_reg_address(machine_reg::mcounteren)); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::mcounteren)); + return aliased_aligned_read(haddr); } void do_write_mcounteren(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::mcounteren), val); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::mcounteren)); + aliased_aligned_write(haddr, val); } uint64_t do_read_senvcfg() const { - return raw_read_memory(machine_reg_address(machine_reg::senvcfg)); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::senvcfg)); + return aliased_aligned_read(haddr); } void do_write_senvcfg(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::senvcfg), val); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::senvcfg)); + aliased_aligned_write(haddr, val); } uint64_t do_read_menvcfg() const { - return raw_read_memory(machine_reg_address(machine_reg::menvcfg)); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::menvcfg)); + return aliased_aligned_read(haddr); } void do_write_menvcfg(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::menvcfg), val); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::menvcfg)); + aliased_aligned_write(haddr, val); } uint64_t do_read_stvec() { - return raw_read_memory(machine_reg_address(machine_reg::stvec)); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::stvec)); + return aliased_aligned_read(haddr); } void do_write_stvec(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::stvec), val); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::stvec)); + aliased_aligned_write(haddr, val); } uint64_t do_read_sscratch() { - return raw_read_memory(machine_reg_address(machine_reg::sscratch)); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::sscratch)); + return aliased_aligned_read(haddr); } void do_write_sscratch(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::sscratch), val); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::sscratch)); + aliased_aligned_write(haddr, val); } uint64_t do_read_sepc() { - return raw_read_memory(machine_reg_address(machine_reg::sepc)); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::sepc)); + return aliased_aligned_read(haddr); } void do_write_sepc(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::sepc), val); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::sepc)); + aliased_aligned_write(haddr, val); } uint64_t do_read_scause() { - return raw_read_memory(machine_reg_address(machine_reg::scause)); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::scause)); + return aliased_aligned_read(haddr); } void do_write_scause(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::scause), val); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::scause)); + aliased_aligned_write(haddr, val); } uint64_t do_read_stval() { - return raw_read_memory(machine_reg_address(machine_reg::stval)); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::stval)); + return aliased_aligned_read(haddr); } void do_write_stval(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::stval), val); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::stval)); + aliased_aligned_write(haddr, val); } uint64_t do_read_satp() { - return raw_read_memory(machine_reg_address(machine_reg::satp)); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::satp)); + return aliased_aligned_read(haddr); } void do_write_satp(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::satp), val); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::satp)); + aliased_aligned_write(haddr, val); } uint64_t do_read_scounteren() { - return raw_read_memory(machine_reg_address(machine_reg::scounteren)); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::scounteren)); + return aliased_aligned_read(haddr); } void do_write_scounteren(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::scounteren), val); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::scounteren)); + aliased_aligned_write(haddr, val); } uint64_t do_read_ilrsc() { - return raw_read_memory(machine_reg_address(machine_reg::ilrsc)); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::ilrsc)); + return aliased_aligned_read(haddr); } void do_write_ilrsc(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::ilrsc), val); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::ilrsc)); + aliased_aligned_write(haddr, val); } uint64_t do_read_iprv() { - return raw_read_memory(machine_reg_address(machine_reg::iprv)); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::iprv)); + return aliased_aligned_read(haddr); } void do_write_iprv(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::iprv), val); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::iprv)); + aliased_aligned_write(haddr, val); } uint64_t do_read_iflags_X() { - return raw_read_memory(machine_reg_address(machine_reg::iflags_X)); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::iflags_X)); + return aliased_aligned_read(haddr); } void do_write_iflags_X(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::iflags_X), val); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::iflags_X)); + aliased_aligned_write(haddr, val); } uint64_t do_read_iflags_Y() { - return raw_read_memory(machine_reg_address(machine_reg::iflags_Y)); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::iflags_Y)); + return aliased_aligned_read(haddr); } void do_write_iflags_Y(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::iflags_Y), val); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::iflags_Y)); + aliased_aligned_write(haddr, val); } uint64_t do_read_iflags_H() { - return raw_read_memory(machine_reg_address(machine_reg::iflags_H)); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::iflags_H)); + return aliased_aligned_read(haddr); } void do_write_iflags_H(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::iflags_H), val); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::iflags_H)); + aliased_aligned_write(haddr, val); } uint64_t do_read_iunrep() { - return raw_read_memory(machine_reg_address(machine_reg::iunrep)); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::iunrep)); + return aliased_aligned_read(haddr); } void do_write_iunrep(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::iunrep), val); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::iunrep)); + aliased_aligned_write(haddr, val); } uint64_t do_read_clint_mtimecmp() { - return raw_read_memory(machine_reg_address(machine_reg::clint_mtimecmp)); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::clint_mtimecmp)); + return aliased_aligned_read(haddr); } void do_write_clint_mtimecmp(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::clint_mtimecmp), val); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::clint_mtimecmp)); + aliased_aligned_write(haddr, val); } uint64_t do_read_plic_girqpend() { - return raw_read_memory(machine_reg_address(machine_reg::plic_girqpend)); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::plic_girqpend)); + return aliased_aligned_read(haddr); } void do_write_plic_girqpend(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::plic_girqpend), val); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::plic_girqpend)); + aliased_aligned_write(haddr, val); } uint64_t do_read_plic_girqsrvd() { - return raw_read_memory(machine_reg_address(machine_reg::plic_girqsrvd)); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::plic_girqsrvd)); + return aliased_aligned_read(haddr); } void do_write_plic_girqsrvd(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::plic_girqsrvd), val); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::plic_girqsrvd)); + aliased_aligned_write(haddr, val); } uint64_t do_read_htif_fromhost() { - return raw_read_memory(machine_reg_address(machine_reg::htif_fromhost)); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::htif_fromhost)); + return aliased_aligned_read(haddr); } void do_write_htif_fromhost(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::htif_fromhost), val); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::htif_fromhost)); + aliased_aligned_write(haddr, val); } uint64_t do_read_htif_tohost() { - return raw_read_memory(machine_reg_address(machine_reg::htif_tohost)); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::htif_tohost)); + return aliased_aligned_read(haddr); } void do_write_htif_tohost(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::htif_tohost), val); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::htif_tohost)); + aliased_aligned_write(haddr, val); } uint64_t do_read_htif_ihalt() { - return raw_read_memory(machine_reg_address(machine_reg::htif_ihalt)); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::htif_ihalt)); + return aliased_aligned_read(haddr); } uint64_t do_read_htif_iconsole() { - return raw_read_memory(machine_reg_address(machine_reg::htif_iconsole)); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::htif_iconsole)); + return aliased_aligned_read(haddr); } uint64_t do_read_htif_iyield() { - return raw_read_memory(machine_reg_address(machine_reg::htif_iyield)); - } - - // NOLINTNEXTLINE(readability-convert-member-functions-to-static) - std::pair do_poll_external_interrupts(uint64_t mcycle, uint64_t mcycle_max) { - (void) mcycle_max; - return {mcycle, false}; - } - - uint64_t read_pma_istart(uint64_t i) { - return raw_read_memory(shadow_pmas_get_pma_abs_addr(i)); - } - - uint64_t read_pma_ilength(uint64_t i) { - return raw_read_memory(shadow_pmas_get_pma_abs_addr(i) + sizeof(uint64_t)); - } - - template - void do_read_memory_word(uint64_t paddr, const unsigned char *hpage, uint64_t hoffset, T *pval) { - (void) hpage; - (void) hoffset; - *pval = raw_read_memory(paddr); + const auto haddr = do_get_faddr(machine_reg_address(machine_reg::htif_iyield)); + return aliased_aligned_read(haddr); } // NOLINTNEXTLINE(readability-convert-member-functions-to-static) @@ -753,154 +835,80 @@ class replay_step_state_access : public i_state_access - void do_write_memory_word(uint64_t paddr, const unsigned char *hpage, uint64_t hoffset, T val) { - (void) hpage; - (void) hoffset; - raw_write_memory(paddr, val); + uint64_t read_pma_istart(uint64_t index) { + const auto haddr = do_get_faddr(shadow_pmas_get_pma_istart_abs_addr(index)); + return aliased_aligned_read(haddr); + } + + uint64_t read_pma_ilength(uint64_t index) { + const auto haddr = do_get_faddr(shadow_pmas_get_pma_ilength_abs_addr(index)); + return aliased_aligned_read(haddr); } mock_pma_entry &do_read_pma_entry(uint64_t index) { assert(index < PMA_MAX); + // record_step_state_access will have recorded the access to istart and + // ilength in its implementation of read_pma_entry. const uint64_t istart = read_pma_istart(index); const uint64_t ilength = read_pma_ilength(index); // NOLINTNEXTLINE(bugprone-narrowing-conversions) const int i = static_cast(index); - if (!m_pmas[i]) { - m_pmas[i] = + if (!m_context.pmas[i]) { + m_context.pmas[i] = make_mock_pma_entry(index, istart, ilength, [](const char *err) { interop_throw_runtime_error(err); }); } // NOLINTNEXTLINE(bugprone-unchecked-optional-access) return m_context.pmas[index].value(); } - unsigned char *do_get_host_memory(mock_pma_entry &pma) { // NOLINT(readability-convert-member-functions-to-static) - (void) pma; - return nullptr; + template + void do_read_memory_word(host_addr haddr, uint64_t /* pma_index */, T *pval) { + *pval = aliased_aligned_read(haddr); } - template - volatile tlb_hot_entry &do_get_tlb_hot_entry(uint64_t eidx) { - auto addr = tlb_get_entry_hot_abs_addr(eidx); - auto size = sizeof(tlb_hot_entry); - // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) - volatile tlb_hot_entry *tlbe = reinterpret_cast(get_raw_memory_pointer(addr, size)); - return *tlbe; + template + void do_write_memory_word(host_addr haddr, uint64_t /* pma_index */, T val) { + aliased_aligned_write(haddr, val); } - template - volatile tlb_cold_entry &do_get_tlb_entry_cold(uint64_t eidx) { - auto addr = tlb_get_entry_cold_abs_addr(eidx); - auto size = sizeof(tlb_cold_entry); - // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) - volatile tlb_cold_entry *tlbe = reinterpret_cast(get_raw_memory_pointer(addr, size)); - return *tlbe; + template + uint64_t do_read_tlb_vaddr_page(uint64_t slot_index) { + const auto haddr = do_get_faddr(shadow_tlb_get_vaddr_page_abs_addr(slot_index)); + return aliased_aligned_read(haddr); } - template - bool do_translate_vaddr_via_tlb(uint64_t vaddr, unsigned char **phptr) { - const uint64_t eidx = tlb_get_entry_index(vaddr); - const volatile tlb_hot_entry &tlbhe = do_get_tlb_hot_entry(eidx); - if (tlb_is_hit(tlbhe.vaddr_page, vaddr)) { - *phptr = cast_addr_to_ptr(tlbhe.vh_offset + vaddr); - return true; - } - return false; + template + host_addr do_read_tlb_vp_offset(uint64_t slot_index) { + const auto haddr = do_get_faddr(shadow_tlb_get_vp_offset_abs_addr(slot_index)); + return aliased_aligned_read(haddr); } - template - bool do_read_memory_word_via_tlb(uint64_t vaddr, T *pval) { - const uint64_t eidx = tlb_get_entry_index(vaddr); - const volatile tlb_hot_entry &tlbhe = do_get_tlb_hot_entry(eidx); - if (tlb_is_hit(tlbhe.vaddr_page, vaddr)) { - const uint64_t poffset = vaddr & PAGE_OFFSET_MASK; - const volatile tlb_cold_entry &tlbce = do_get_tlb_entry_cold(eidx); - *pval = raw_read_memory(tlbce.paddr_page + poffset); - return true; - } - return false; + template + uint64_t do_read_tlb_pma_index(uint64_t slot_index) { + const auto haddr = do_get_faddr(shadow_tlb_get_pma_index_abs_addr(slot_index)); + return aliased_aligned_read(haddr); } - template - bool do_write_memory_word_via_tlb(uint64_t vaddr, T val) { - const uint64_t eidx = tlb_get_entry_index(vaddr); - const volatile tlb_hot_entry &tlbhe = do_get_tlb_hot_entry(eidx); - if (tlb_is_hit(tlbhe.vaddr_page, vaddr)) { - const uint64_t poffset = vaddr & PAGE_OFFSET_MASK; - const volatile tlb_cold_entry &tlbce = do_get_tlb_entry_cold(eidx); - raw_write_memory(tlbce.paddr_page + poffset, val); - return true; - } - return false; + template + void do_write_tlb(uint64_t slot_index, uint64_t vaddr_page, host_addr vh_offset, uint64_t pma_index) { + const auto haddr_vaddr_page = do_get_faddr(shadow_tlb_get_vaddr_page_abs_addr(slot_index)); + aliased_aligned_write(haddr_vaddr_page, vaddr_page); + const auto haddr_vp_offset = do_get_faddr(shadow_tlb_get_vp_offset_abs_addr(slot_index)); + aliased_aligned_write(haddr_vp_offset, vh_offset); + const auto haddr_pma_index = do_get_faddr(shadow_tlb_get_pma_index_abs_addr(slot_index)); + aliased_aligned_write(haddr_pma_index, pma_index); } - template - unsigned char *do_replace_tlb_entry(uint64_t vaddr, uint64_t paddr, mock_pma_entry &pma) { - const uint64_t eidx = tlb_get_entry_index(vaddr); - volatile tlb_hot_entry &tlbhe = do_get_tlb_hot_entry(eidx); - volatile tlb_cold_entry &tlbce = do_get_tlb_entry_cold(eidx); - if constexpr (ETYPE == TLB_WRITE) { - if (tlbhe.vaddr_page != TLB_INVALID_PAGE) { - mock_pma_entry &pma = do_read_pma_entry(tlbce.pma_index); - pma.mark_dirty_page(tlbce.paddr_page - pma.get_start()); - } - } - const uint64_t vaddr_page = vaddr & ~PAGE_OFFSET_MASK; - const uint64_t paddr_page = paddr & ~PAGE_OFFSET_MASK; - - auto *page_type = find_page(paddr_page); - auto *hpage = page_type->data; - - tlbhe.vaddr_page = vaddr_page; - tlbhe.vh_offset = cast_ptr_to_addr(hpage) - vaddr_page; - tlbce.paddr_page = paddr_page; - tlbce.pma_index = pma.get_index(); - return hpage; - } - - template - void do_flush_tlb_entry(uint64_t eidx) { - volatile tlb_hot_entry &tlbhe = do_get_tlb_hot_entry(eidx); - // Mark page that was on TLB as dirty so we know to update the Merkle tree - if constexpr (ETYPE == TLB_WRITE) { - if (tlbhe.vaddr_page != TLB_INVALID_PAGE) { - tlbhe.vaddr_page = TLB_INVALID_PAGE; - const volatile tlb_cold_entry &tlbce = do_get_tlb_entry_cold(eidx); - mock_pma_entry &pma = do_read_pma_entry(tlbce.pma_index); - pma.mark_dirty_page(tlbce.paddr_page - pma.get_start()); - } else { - tlbhe.vaddr_page = TLB_INVALID_PAGE; - } - } else { - tlbhe.vaddr_page = TLB_INVALID_PAGE; - } - } - - template - void do_flush_tlb_type() { - for (uint64_t i = 0; i < PMA_TLB_SIZE; ++i) { - do_flush_tlb_entry(i); - } + void do_mark_dirty_page(host_addr /* haddr */, uint64_t /* pma_index */) { + // this is a noop since we have no host machine } - void do_flush_tlb_vaddr(uint64_t vaddr) { - (void) vaddr; - do_flush_tlb_type(); - do_flush_tlb_type(); - do_flush_tlb_type(); - } - - bool do_get_soft_yield() { // NOLINT(readability-convert-member-functions-to-static) - return false; - } - - void do_putchar(uint8_t /* c */) { // NOLINT(readability-convert-member-functions-to-static) - ; + // NOLINTNEXTLINE(readability-convert-member-functions-to-static) + std::pair do_poll_external_interrupts(uint64_t mcycle, uint64_t mcycle_max) { + (void) mcycle_max; + return {mcycle, false}; } - int do_getchar() { // NOLINT(readability-convert-member-functions-to-static) - return -1; - } }; } // namespace cartesi diff --git a/src/shadow-pmas.h b/src/shadow-pmas.h index 1aa7656da..60a2df6df 100644 --- a/src/shadow-pmas.h +++ b/src/shadow-pmas.h @@ -53,6 +53,16 @@ static inline uint64_t shadow_pmas_get_pma_abs_addr(uint64_t p) { return PMA_SHADOW_PMAS_START + shadow_pmas_get_pma_rel_addr(p); } +/// \brief Obtains the absolute address of the istart field in a PMA entry in shadow memory. +static inline uint64_t shadow_pmas_get_pma_istart_abs_addr(uint64_t p) { + return shadow_pmas_get_pma_abs_addr(p) + offsetof(shadow_pma_entry, istart); +} + +/// \brief Obtains the absolute address of the ilength field in a PMA entry in shadow memory. +static inline uint64_t shadow_pmas_get_pma_ilength_abs_addr(uint64_t p) { + return shadow_pmas_get_pma_abs_addr(p) + offsetof(shadow_pma_entry, ilength); +} + } // namespace cartesi #endif diff --git a/src/shadow-state.h b/src/shadow-state.h index fd35759f1..720112679 100644 --- a/src/shadow-state.h +++ b/src/shadow-state.h @@ -22,6 +22,7 @@ #include #include "compiler-defines.h" +#include "pma-constants.h" #include "pma-driver.h" #include "riscv-constants.h" diff --git a/src/shadow-tlb-factory.cpp b/src/shadow-tlb-factory.cpp index ba69c6746..c59cd2686 100644 --- a/src/shadow-tlb-factory.cpp +++ b/src/shadow-tlb-factory.cpp @@ -37,52 +37,42 @@ static bool shadow_tlb_peek(const pma_entry &pma, const machine &m, uint64_t pag *page_data = nullptr; return false; } - // Clear page memset(scratch, 0, PMA_PAGE_SIZE); - // Copy relevant TLB entries to the page - for (uint64_t off = 0; off < PMA_PAGE_SIZE; off += sizeof(uint64_t)) { + const auto &tlb = m.get_state().tlb; + for (uint64_t offset = 0; offset < PMA_PAGE_SIZE; offset += sizeof(uint64_t)) { uint64_t val = 0; - const uint64_t tlboff = page_offset + off; - if (tlboff < offsetof(shadow_tlb_state, cold)) { // Hot entry - const uint64_t etype = tlboff / sizeof(std::array); - const uint64_t etypeoff = tlboff % sizeof(std::array); - const uint64_t eidx = etypeoff / sizeof(tlb_hot_entry); - const uint64_t fieldoff = etypeoff % sizeof(tlb_hot_entry); - const tlb_hot_entry &tlbhe = m.get_state().tlb.hot[etype][eidx]; - switch (fieldoff) { - case offsetof(tlb_hot_entry, vaddr_page): - val = tlbhe.vaddr_page; + if (offset < sizeof(shadow_tlb_state)) { + // Figure out in which set (code/read/write) the offset falls + const uint64_t set_index = offset / sizeof(shadow_tlb_set); + const uint64_t slot_offset = offset % sizeof(shadow_tlb_set); + // Figure out in which slot index the offset falls + const uint64_t slot_index = slot_offset / sizeof(shadow_tlb_slot); + const uint64_t field_offset = slot_offset % sizeof(shadow_tlb_slot); + switch (field_offset) { + case offsetof(shadow_tlb_slot, vaddr_page): + val = tlb.hot[set_index][slot_index].vaddr_page; break; - case offsetof(tlb_hot_entry, vh_offset): - default: - // Here we skip host related fields, they are visible as 0 in the device - val = 0; + case offsetof(shadow_tlb_slot, vp_offset): { + auto vaddr_page = tlb.hot[set_index][slot_index].vaddr_page; + auto vh_offset = tlb.hot[set_index][slot_index].vh_offset; + auto haddr_page = vaddr_page + vh_offset; + auto pma_index = tlb.cold[set_index][slot_index].pma_index; + auto paddr_page = m.get_paddr(haddr_page, pma_index); + val = paddr_page - vaddr_page; break; - } - } else if (tlboff < sizeof(shadow_tlb_state)) { // Cold entry - const uint64_t coldoff = tlboff - offsetof(shadow_tlb_state, cold); - const uint64_t etype = coldoff / sizeof(std::array); - const uint64_t etypeoff = coldoff % sizeof(std::array); - const uint64_t eidx = etypeoff / sizeof(tlb_cold_entry); - const uint64_t fieldoff = etypeoff % sizeof(tlb_cold_entry); - const tlb_cold_entry &tlbce = m.get_state().tlb.cold[etype][eidx]; - switch (fieldoff) { - case offsetof(tlb_cold_entry, paddr_page): - val = tlbce.paddr_page; - break; - case offsetof(tlb_cold_entry, pma_index): - val = tlbce.pma_index; + } + case offsetof(shadow_tlb_slot, pma_index): + val = tlb.cold[set_index][slot_index].pma_index; break; default: val = 0; break; } } - aliased_aligned_write(scratch + off, val); + aliased_aligned_write(scratch + offset, val); } - *page_data = scratch; return true; } diff --git a/src/shadow-tlb.h b/src/shadow-tlb.h index 3ecff19c7..c088aff9f 100644 --- a/src/shadow-tlb.h +++ b/src/shadow-tlb.h @@ -19,123 +19,53 @@ /// \file /// \brief TLB device. -/// \details The Translation Lookaside Buffer is a small cache used to speed up translation between -/// virtual target addresses and the corresponding memory address in the host. +/// \details The Translation Lookaside Buffer is a small cache used to speed up translation between address spaces. #include #include #include -#include "pma-constants.h" +#include "compiler-defines.h" #include "pma-driver.h" -#include "riscv-constants.h" +#include "tlb.h" namespace cartesi { -extern const pma_driver shadow_tlb_driver; - -/// \brief TLB entry type. -enum TLB_entry_type : uint64_t { TLB_CODE, TLB_READ, TLB_WRITE }; - -/// \brief TLB constants. -enum TLB_constants : uint64_t { TLB_INVALID_PAGE = UINT64_C(-1), TLB_INVALID_PMA = PMA_MAX }; - -/// \brief TLB hot entry. -struct tlb_hot_entry final { - uint64_t vaddr_page; ///< Target virtual address of page start - uint64_t vh_offset; ///< Offset that maps target virtual addresses directly to host addresses. +/// \brief Shadow TLB slot +struct PACKED shadow_tlb_slot { + uint64_t vaddr_page; ///< Tag is target virtual address of start of page + uint64_t vp_offset; ///< Value is offset from target virtual address to target physical address within page + uint64_t pma_index; ///< Index of PMA where physical page falls }; -/// \brief TLB cold entry. -struct tlb_cold_entry final { - uint64_t paddr_page; ///< Target physical address of page start - uint64_t pma_index; ///< PMA entry index for corresponding range -}; +/// \brief Shadow TLB set +using shadow_tlb_set = std::array; -/// \brief TLB state. -struct shadow_tlb_state final { - // The TLB state is split in hot and cold regions. - // The hot region is accessed with very high frequency every hit check, - // while the cold region with low frequency only when replacing or flushing write TLB entries. - // - // Splitting into hold and cold regions increases host CPU cache usage when checking TLB hits, - // due to more data locality, therefore improving the TLB performance. - std::array, 3> hot; - std::array, 3> cold; -}; +/// \brief Shadow TLB memory layout +using shadow_tlb_state = std::array; // one set for code, one for read and one for write -//??E Do we even want to support 128 bit systems someday? -// Make sure uint64_t is large enough to hold host pointers, otherwise 'vh_offset' field will not work correctly. -static_assert(sizeof(uint64_t) >= sizeof(uintptr_t), "TLB expects host pointer to be at maximum 64bit"); - -// This TLB algorithm assumes the following conditions -static_assert((sizeof(tlb_hot_entry) & (sizeof(tlb_hot_entry) - 1)) == 0 && - (sizeof(tlb_cold_entry) & (sizeof(tlb_cold_entry) - 1)) == 0, - "TLB entry size must be a power of 2"); -static_assert(PMA_SHADOW_TLB_LENGTH % sizeof(std::array) == 0 && - PMA_SHADOW_TLB_LENGTH % sizeof(std::array) == 0, - "code assumes PMA TLB length is divisible by TLB entry array size"); -static_assert(PMA_SHADOW_TLB_LENGTH == sizeof(shadow_tlb_state), - "code assumes PMA TLB length is equal to TLB state size"); - -/// \brief Gets a TLB entry index. -/// \param vaddr Target virtual address. -static inline uint64_t tlb_get_entry_index(uint64_t vaddr) { - return (vaddr >> PMA_PAGE_SIZE_LOG2) & (PMA_TLB_SIZE - 1); -} +static_assert(PMA_SHADOW_TLB_LENGTH >= sizeof(shadow_tlb_state), "TLB state must fit in TLB shadow"); -/// \brief Checks for a TLB hit. -/// \tparam T Type of access needed (uint8_t, uint16_t, uint32_t, uint64_t). -/// \param vaddr_page Target virtual address of page start of a TLB entry -/// \param vaddr Target virtual address. -/// \returns True on hit, false otherwise. -template -static inline bool tlb_is_hit(uint64_t vaddr_page, uint64_t vaddr) { - // Make sure misaligned accesses are always considered a miss - // Otherwise, we could report a hit for a word that goes past the end of the PMA range. - // Aligned accesses cannot do so because the PMA ranges - // are always page-aligned. - return (vaddr_page == (vaddr & ~(PAGE_OFFSET_MASK & ~(sizeof(T) - 1)))); -} - -template -static inline uint64_t tlb_get_entry_hot_rel_addr(uint64_t eidx) { - return offsetof(shadow_tlb_state, hot) + (ETYPE * sizeof(std::array)) + - (eidx * sizeof(tlb_hot_entry)); -} - -template -static inline uint64_t tlb_get_entry_cold_rel_addr(uint64_t eidx) { - return offsetof(shadow_tlb_state, cold) + (ETYPE * sizeof(std::array)) + - (eidx * sizeof(tlb_cold_entry)); -} - -template -static inline uint64_t tlb_get_entry_hot_abs_addr(uint64_t eidx) { - return PMA_SHADOW_TLB_START + tlb_get_entry_hot_rel_addr(eidx); -} +extern const pma_driver shadow_tlb_driver; -template -static inline uint64_t tlb_get_entry_cold_abs_addr(uint64_t eidx) { - return PMA_SHADOW_TLB_START + tlb_get_entry_cold_rel_addr(eidx); +template +constexpr uint64_t shadow_tlb_get_slot_abs_addr(uint64_t slot_index) { + return PMA_SHADOW_TLB_START + USE * sizeof(shadow_tlb_set) + slot_index * sizeof(shadow_tlb_slot); } -template -static inline uint64_t tlb_get_vaddr_page_rel_addr(uint64_t eidx) { - return offsetof(shadow_tlb_state, hot) + (ETYPE * sizeof(std::array)) + - (eidx * sizeof(tlb_hot_entry)) + offsetof(tlb_hot_entry, vaddr_page); +template +constexpr uint64_t shadow_tlb_get_vaddr_page_abs_addr(uint64_t slot_index) { + return shadow_tlb_get_slot_abs_addr(slot_index) + offsetof(shadow_tlb_slot, vaddr_page); } -template -static inline uint64_t tlb_get_paddr_page_rel_addr(uint64_t eidx) { - return offsetof(shadow_tlb_state, cold) + (ETYPE * sizeof(std::array)) + - (eidx * sizeof(tlb_cold_entry)) + offsetof(tlb_cold_entry, paddr_page); +template +constexpr uint64_t shadow_tlb_get_vp_offset_abs_addr(uint64_t slot_index) { + return shadow_tlb_get_slot_abs_addr(slot_index) + offsetof(shadow_tlb_slot, vp_offset); } -template -static inline uint64_t tlb_get_pma_index_rel_addr(uint64_t eidx) { - return offsetof(shadow_tlb_state, cold) + (ETYPE * sizeof(std::array)) + - (eidx * sizeof(tlb_cold_entry)) + offsetof(tlb_cold_entry, pma_index); +template +constexpr uint64_t shadow_tlb_get_pma_index_abs_addr(uint64_t slot_index) { + return shadow_tlb_get_slot_abs_addr(slot_index) + offsetof(shadow_tlb_slot, pma_index); } } // namespace cartesi diff --git a/src/state-access.h b/src/state-access.h index 66f462efb..91fc81511 100644 --- a/src/state-access.h +++ b/src/state-access.h @@ -28,6 +28,7 @@ #include "compiler-defines.h" #include "device-state-access.h" +#include "host-addr.h" #include "i-state-access.h" #include "interpret.h" #include "machine-state.h" @@ -42,10 +43,23 @@ namespace cartesi { +class state_access; + +// Type trait that should return the pma_entry type for a state access class +template <> +struct i_state_access_pma_entry { + using type = pma_entry; +}; +// Type trait that should return the fast_addr type for a state access class +template <> +struct i_state_access_fast_addr { + using type = host_addr; +}; + /// \class state_access /// \details The state_access class implements fast, direct /// access to the machine state. No logs are kept. -class state_access : public i_state_access { +class state_access : public i_state_access { // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) machine &m_m; ///< Associated machine @@ -65,8 +79,7 @@ class state_access : public i_state_access { } private: - // Declare interface as friend to it can forward calls to the "overridden" methods. - friend i_state_access; + friend i_state_access; machine_state &do_get_naked_state() { return m_m.get_state(); @@ -409,54 +422,6 @@ class state_access : public i_state_access { return m_m.get_state().htif.iyield; } - NO_INLINE std::pair do_poll_external_interrupts(uint64_t mcycle, uint64_t mcycle_max) { - bool interrupt_raised = false; - // Only poll external interrupts if we are in unreproducible mode - if (unlikely(do_read_iunrep())) { - // Convert the relative interval of cycles we can wait to the interval of host time we can wait - uint64_t timeout_us = (mcycle_max - mcycle) / RTC_CYCLES_PER_US; - int64_t start_us = 0; - if (timeout_us > 0) { - start_us = os_now_us(); - } - device_state_access da(*this, mcycle); - // Poll virtio for events (e.g console stdin, network sockets) - // Timeout may be decremented in case a device has deadline timers (e.g network device) - if (m_m.has_virtio_devices() && m_m.has_virtio_console()) { // VirtIO + VirtIO console - interrupt_raised |= m_m.poll_virtio_devices(&timeout_us, &da); - // VirtIO console device will poll TTY - } else if (m_m.has_virtio_devices()) { // VirtIO without a console - interrupt_raised |= m_m.poll_virtio_devices(&timeout_us, &da); - if (m_m.has_htif_console()) { // VirtIO + HTIF console - // Poll tty without waiting more time, because the pool above should have waited enough time - interrupt_raised |= os_poll_tty(0); - } - } else if (m_m.has_htif_console()) { // Only HTIF console - interrupt_raised |= os_poll_tty(timeout_us); - } else if (timeout_us > 0) { // No interrupts to check, just keep the CPU idle - os_sleep_us(timeout_us); - } - // If timeout is greater than zero, we should also increment mcycle relative to the elapsed time - if (timeout_us > 0) { - const int64_t end_us = os_now_us(); - const uint64_t elapsed_us = static_cast(std::max(end_us - start_us, INT64_C(0))); - const uint64_t next_mcycle = mcycle + (elapsed_us * RTC_CYCLES_PER_US); - mcycle = std::min(std::max(next_mcycle, mcycle), mcycle_max); - } - } - return {mcycle, interrupt_raised}; - } - - template - void do_read_memory_word(uint64_t /*paddr*/, const unsigned char *hpage, uint64_t hoffset, T *pval) const { - *pval = aliased_aligned_read(hpage + hoffset); - } - - template - void do_write_memory_word(uint64_t /*paddr*/, unsigned char *hpage, uint64_t hoffset, T val) { - aliased_aligned_write(hpage + hoffset, val); - } - bool do_read_memory(uint64_t paddr, unsigned char *data, uint64_t length) const { //??(edubart): Treating exceptions here is not ideal, we should probably // move read_memory() method implementation inside state access later @@ -479,123 +444,107 @@ class state_access : public i_state_access { } } - static unsigned char *do_get_host_memory(pma_entry &pma) { - return pma.get_memory_noexcept().get_host_memory(); - } - pma_entry &do_read_pma_entry(uint64_t index) { assert(index < PMA_MAX); - return m_m.get_state().pmas[index]; + // NOLINTNEXTLINE(bugprone-narrowing-conversions) + return m_m.get_state().pmas[static_cast(index)]; } - template - bool do_translate_vaddr_via_tlb(uint64_t vaddr, unsigned char **phptr) { - const uint64_t eidx = tlb_get_entry_index(vaddr); - const tlb_hot_entry &tlbhe = m_m.get_state().tlb.hot[ETYPE][eidx]; - if (unlikely(!tlb_is_hit(tlbhe.vaddr_page, vaddr))) { - return false; + void do_write_memory_with_padding(uint64_t paddr, const unsigned char *data, uint64_t data_length, + int write_length_log2_size) { + if (data == nullptr) { + throw std::runtime_error("data is null"); + } + const uint64_t write_length = static_cast(1) << write_length_log2_size; + if (write_length < data_length) { + throw std::runtime_error("write_length is less than data_length"); + } + m_m.write_memory(paddr, data, data_length); + if (write_length > data_length) { + m_m.fill_memory(paddr + data_length, 0, write_length - data_length); } - *phptr = cast_addr_to_ptr(tlbhe.vh_offset + vaddr); - return true; } - template - bool do_read_memory_word_via_tlb(uint64_t vaddr, T *pval) { - const uint64_t eidx = tlb_get_entry_index(vaddr); - const tlb_hot_entry &tlbhe = m_m.get_state().tlb.hot[ETYPE][eidx]; - if (unlikely(!tlb_is_hit(tlbhe.vaddr_page, vaddr))) { - return false; - } - const auto *h = cast_addr_to_ptr(tlbhe.vh_offset + vaddr); - *pval = aliased_aligned_read(h); - return true; + template + void do_read_memory_word(host_addr haddr, uint64_t /* pma_index */, T *pval) { + *pval = aliased_aligned_read(haddr); } - template - bool do_write_memory_word_via_tlb(uint64_t vaddr, T val) { - const uint64_t eidx = tlb_get_entry_index(vaddr); - const tlb_hot_entry &tlbhe = m_m.get_state().tlb.hot[ETYPE][eidx]; - if (unlikely(!tlb_is_hit(tlbhe.vaddr_page, vaddr))) { - return false; - } - auto *h = cast_addr_to_ptr(tlbhe.vh_offset + vaddr); - aliased_aligned_write(h, val); - return true; - } - - template - unsigned char *do_replace_tlb_entry(uint64_t vaddr, uint64_t paddr, pma_entry &pma) { - const uint64_t eidx = tlb_get_entry_index(vaddr); - tlb_hot_entry &tlbhe = m_m.get_state().tlb.hot[ETYPE][eidx]; - tlb_cold_entry &tlbce = m_m.get_state().tlb.cold[ETYPE][eidx]; - // Mark page that was on TLB as dirty so we know to update the Merkle tree - if constexpr (ETYPE == TLB_WRITE) { - if (tlbhe.vaddr_page != TLB_INVALID_PAGE) { - pma_entry &pma = do_read_pma_entry(tlbce.pma_index); - pma.mark_dirty_page(tlbce.paddr_page - pma.get_start()); - } - } - const uint64_t vaddr_page = vaddr & ~PAGE_OFFSET_MASK; - const uint64_t paddr_page = paddr & ~PAGE_OFFSET_MASK; - unsigned char *hpage = pma.get_memory_noexcept().get_host_memory() + (paddr_page - pma.get_start()); - tlbhe.vaddr_page = vaddr_page; - tlbhe.vh_offset = cast_ptr_to_addr(hpage) - vaddr_page; - tlbce.paddr_page = paddr_page; - tlbce.pma_index = static_cast(pma.get_index()); - return hpage; - } - - template - void do_flush_tlb_entry(uint64_t eidx) { - tlb_hot_entry &tlbhe = m_m.get_state().tlb.hot[ETYPE][eidx]; - // Mark page that was on TLB as dirty so we know to update the Merkle tree - if constexpr (ETYPE == TLB_WRITE) { - if (tlbhe.vaddr_page != TLB_INVALID_PAGE) { - tlbhe.vaddr_page = TLB_INVALID_PAGE; - const tlb_cold_entry &tlbce = m_m.get_state().tlb.cold[ETYPE][eidx]; - pma_entry &pma = do_read_pma_entry(tlbce.pma_index); - pma.mark_dirty_page(tlbce.paddr_page - pma.get_start()); - } else { - tlbhe.vaddr_page = TLB_INVALID_PAGE; - } - } else { - tlbhe.vaddr_page = TLB_INVALID_PAGE; - } + template + void do_write_memory_word(host_addr haddr, uint64_t /* pma_index */, T val) { + aliased_aligned_write(haddr, val); } - template - void do_flush_tlb_type() { - for (uint64_t i = 0; i < PMA_TLB_SIZE; ++i) { - do_flush_tlb_entry(i); - } + template + uint64_t do_read_tlb_vaddr_page(uint64_t slot_index) { + return m_m.get_state().tlb.hot[USE][slot_index].vaddr_page; } - void do_flush_tlb_vaddr(uint64_t /*vaddr*/) { - // We can't flush just one TLB entry for that specific virtual address, - // because megapages/gigapages may be in use while this TLB implementation ignores it, - // so we have to flush all addresses. - do_flush_tlb_type(); - do_flush_tlb_type(); - do_flush_tlb_type(); + template + host_addr do_read_tlb_vp_offset(uint64_t slot_index) { + return m_m.get_state().tlb.hot[USE][slot_index].vh_offset; } - bool do_get_soft_yield() { - return m_m.get_state().soft_yield; + template + uint64_t do_read_tlb_pma_index(uint64_t slot_index) { + return m_m.get_state().tlb.cold[USE][slot_index].pma_index; } - void do_write_memory_with_padding(uint64_t paddr, const unsigned char *data, uint64_t data_length, - int write_length_log2_size) { - if (data == nullptr) { - throw std::runtime_error("data is null"); - } - const uint64_t write_length = static_cast(1) << write_length_log2_size; - if (write_length < data_length) { - throw std::runtime_error("write_length is less than data_length"); - } - m_m.write_memory(paddr, data, data_length); - if (write_length > data_length) { - m_m.fill_memory(paddr + data_length, 0, write_length - data_length); + template + void do_write_tlb(uint64_t slot_index, uint64_t vaddr_page, host_addr vh_offset, uint64_t pma_index) { + m_m.get_state().tlb.hot[USE][slot_index].vaddr_page = vaddr_page; + m_m.get_state().tlb.hot[USE][slot_index].vh_offset = vh_offset; + m_m.get_state().tlb.cold[USE][slot_index].pma_index = pma_index; + } + + fast_addr do_get_faddr(uint64_t paddr, uint64_t pma_index) const { + return m_m.get_host_addr(paddr, pma_index); + } + + void do_mark_dirty_page(host_addr haddr, uint64_t pma_index) { + m_m.mark_dirty_page(haddr, pma_index); + } + + NO_INLINE std::pair do_poll_external_interrupts(uint64_t mcycle, uint64_t mcycle_max) { + bool interrupt_raised = false; + // Only poll external interrupts if we are in unreproducible mode + if (unlikely(do_read_iunrep())) { + // Convert the relative interval of cycles we can wait to the interval of host time we can wait + uint64_t timeout_us = (mcycle_max - mcycle) / RTC_CYCLES_PER_US; + int64_t start_us = 0; + if (timeout_us > 0) { + start_us = os_now_us(); + } + device_state_access da(*this, mcycle); + // Poll virtio for events (e.g console stdin, network sockets) + // Timeout may be decremented in case a device has deadline timers (e.g network device) + if (m_m.has_virtio_devices() && m_m.has_virtio_console()) { // VirtIO + VirtIO console + interrupt_raised |= m_m.poll_virtio_devices(&timeout_us, &da); + // VirtIO console device will poll TTY + } else if (m_m.has_virtio_devices()) { // VirtIO without a console + interrupt_raised |= m_m.poll_virtio_devices(&timeout_us, &da); + if (m_m.has_htif_console()) { // VirtIO + HTIF console + // Poll tty without waiting more time, because the pool above should have waited enough time + interrupt_raised |= os_poll_tty(0); + } + } else if (m_m.has_htif_console()) { // Only HTIF console + interrupt_raised |= os_poll_tty(timeout_us); + } else if (timeout_us > 0) { // No interrupts to check, just keep the CPU idle + os_sleep_us(timeout_us); + } + // If timeout is greater than zero, we should also increment mcycle relative to the elapsed time + if (timeout_us > 0) { + const int64_t end_us = os_now_us(); + const uint64_t elapsed_us = static_cast(std::max(end_us - start_us, INT64_C(0))); + const uint64_t next_mcycle = mcycle + (elapsed_us * RTC_CYCLES_PER_US); + mcycle = std::min(std::max(next_mcycle, mcycle), mcycle_max); + } } + return {mcycle, interrupt_raised}; + } + + bool do_get_soft_yield() { + return m_m.get_state().soft_yield; } // NOLINTNEXTLINE(readability-convert-member-functions-to-static) diff --git a/src/strict-aliasing.h b/src/strict-aliasing.h index 5e93457c9..5313f427c 100644 --- a/src/strict-aliasing.h +++ b/src/strict-aliasing.h @@ -21,6 +21,8 @@ #include #include +#include "host-addr.h" + /// \file /// \brief Enforcement of the strict aliasing rule @@ -28,80 +30,46 @@ namespace cartesi { /// \brief Writes a value to memory. /// \tparam T Type of value. -/// \param p Where to write. Must be aligned to sizeof(T). +/// \tparam A Type to which \p haddr is aligned. +/// \param haddr Where to write. Must be aligned to sizeof(A). /// \param v Value to write. -template -static inline void aliased_aligned_write(void *p, T v) { - memcpy(__builtin_assume_aligned(p, sizeof(T)), &v, sizeof(T)); +template +static inline void aliased_aligned_write(host_addr haddr, T v) { + memcpy(__builtin_assume_aligned(cast_host_addr_to_ptr(haddr), sizeof(A)), &v, sizeof(T)); } /// \brief Reads a value from memory. /// \tparam T Type of value. -/// \param p Where to find value. Must be aligned to sizeof(T). -/// \returns Value. -template -static inline T aliased_aligned_read(const void *p) { - T v; - memcpy(&v, __builtin_assume_aligned(p, sizeof(T)), sizeof(T)); - return v; -} - -/// \brief Reads an unaligned value from memory. -/// \tparam T Type of unaligned value. -/// \tparam U Type of aligned value. -/// \tparam ALIGN Alignment of p. -/// \param p Where to find value. Must be aligned to sizeof(U). -/// \returns Value. -template -static inline T aliased_unaligned_read(const void *p) { +/// \tparam A Type to which \p haddr is aligned. +/// \param haddr Where to find value. Must be aligned to sizeof(A). +/// \returns Value read. +template +static inline T aliased_aligned_read(host_addr haddr) { T v; - memcpy(&v, __builtin_assume_aligned(p, sizeof(U)), sizeof(T)); + memcpy(&v, __builtin_assume_aligned(cast_host_addr_to_ptr(haddr), sizeof(A)), sizeof(T)); return v; } -/// \brief Casts a pointer to an unsigned integer. -/// \details The address returned by this function, -/// can later be converted to a pointer using cast_addr_to_ptr, -/// and must be read/written using aliased_aligned_read/aliased_aligned_write, -/// otherwise strict aliasing rules may be violated. -/// \tparam T Unsigned integer type to cast to. -/// \tparam PTR Pointer type to perform the cast. -/// \param ptr The pointer to retrieve its unsigned integer representation. -/// \returns An unsigned integer. -template -static inline uint64_t cast_ptr_to_addr(PTR ptr) { - // Enforcement on type arguments - static_assert(std::is_pointer_v); - static_assert(std::is_unsigned_v); - static_assert(sizeof(PTR) == sizeof(uintptr_t)); - // We can only cast to integers that large enough to contain a pointer - static_assert(sizeof(T) >= sizeof(uintptr_t), "expected sizeof(T) >= sizeof(uintptr_t)"); - // Note that bellow we cast the pointer to void* first, - // according to the C spec this is required is to ensure the same presentation, before casting to uintptr_t - // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast,bugprone-casting-through-void) - return static_cast(reinterpret_cast(static_cast(ptr))); +/// \brief Writes a value to memory. +/// \tparam T Type of value. +/// \tparam A Type to which \p haddr is aligned. +/// \param p Where to write. Must be aligned to sizeof(A). +/// \param v Value to write. +template +static inline void aliased_aligned_write(void *p, T v) { + memcpy(__builtin_assume_aligned(p, sizeof(A)), &v, sizeof(T)); } -/// \brief Casts a pointer to an unsigned integer. -/// \details The pointer returned by this function -/// must only be read/written using aliased_aligned_read/aliased_aligned_write, -/// otherwise strict aliasing rules may be violated. -/// \tparam T Unsigned integer type to cast to. -/// \tparam PTR Pointer type to perform the cast. -/// \param addr The address of the pointer represented by an unsigned integer. -/// \returns A pointer. -template -static inline PTR cast_addr_to_ptr(T addr) { - // Enforcement on type arguments - static_assert(std::is_pointer_v); - static_assert(std::is_unsigned_v); - static_assert(sizeof(PTR) == sizeof(uintptr_t)); - // We can only cast from integer that are large enough to contain a pointer - static_assert(sizeof(T) >= sizeof(uintptr_t), "expected sizeof(T) >= sizeof(uintptr_t)"); - // Note that bellow we cast the address to void* first, - // according to the C spec this is required is to ensure the same presentation, before casting to PTR - // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast,bugprone-casting-through-void,performance-no-int-to-ptr) - return static_cast(reinterpret_cast(static_cast(addr))); +/// \brief Reads a value from memory. +/// \tparam T Type of value. +/// \tparam A Type to which \p haddr is aligned. +/// \param p Where to find value. Must be aligned to sizeof(A). +/// \returns Value read. +template +static inline T aliased_aligned_read(const void *p) { + T v; + memcpy(&v, __builtin_assume_aligned(p, sizeof(A)), sizeof(T)); + return v; } } // namespace cartesi diff --git a/src/tlb.h b/src/tlb.h new file mode 100644 index 000000000..991f67d32 --- /dev/null +++ b/src/tlb.h @@ -0,0 +1,98 @@ +// Copyright Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: LGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify it under +// the terms of the GNU Lesser General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) any +// later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +// PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License along +// with this program (see COPYING). If not, see . +// + +#ifndef TLB_H +#define TLB_H + +/// \file +/// \brief TLB definitions +/// \details The Translation Lookaside Buffer is a small cache used to speed up translation between address spaces. + +#include +#include +#include + +#include "host-addr.h" +#include "pma-constants.h" +#include "pma-driver.h" +#include "riscv-constants.h" + +namespace cartesi { + +/// \brief TLB set mode. +enum TLB_set_use : uint64_t { TLB_CODE, TLB_READ, TLB_WRITE, TLB_LAST_ = TLB_WRITE }; + +/// \brief TLB constants. +enum TLB_constants : uint64_t { + TLB_SET_SIZE = 256, + TLB_INVALID_PAGE = UINT64_C(-1), + TLB_INVALID_PMA_INDEX = UINT64_C(-1) +}; + +/// \brief TLB hot slot. +struct tlb_hot_slot final { + uint64_t vaddr_page; ///< Tag is the target virtual address of page start + host_addr vh_offset; ///< Value is the offset from target virtual address in the same page to + ///< translated host address (vh_offset = haddr - vaddr) +}; + +using tlb_hot_set = std::array; + +/// \brief TLB cold slot. +struct tlb_cold_slot final { + uint64_t pma_index; ///< Index of PMA where physical address falls +}; + +using tlb_cold_set = std::array; + +/// \brief TLB state. +struct tlb_state { + std::array hot; + std::array cold; +}; + +static_assert(sizeof(uint64_t) >= sizeof(uintptr_t), "TLB expects host pointer fit in 64 bits"); +//??D why? +static_assert((sizeof(tlb_hot_slot) & (sizeof(tlb_hot_slot) - 1)) == 0, "TLB slot size must be a power of 2"); +static_assert((sizeof(tlb_cold_slot) & (sizeof(tlb_cold_slot) - 1)) == 0, "TLB slot size must be a power of 2"); + +/// \brief Gets a TLB slot index for a page. +/// \param vaddr Target virtual address. +/// \returns TLB slot index. +constexpr uint64_t tlb_slot_index(uint64_t vaddr) { + return (vaddr >> LOG2_PAGE_SIZE) & (TLB_SET_SIZE - 1); +} + +/// \brief Checks for a TLB hit. +/// \tparam T Type of access needed (uint8_t, uint16_t, uint32_t, uint64_t). +/// \param slot_vaddr_page vaddr_page in TLB slot +/// \param vaddr Target virtual address being looked up. +/// \returns True on hit, false otherwise. +template +constexpr bool tlb_is_hit(uint64_t slot_vaddr_page, uint64_t vaddr) { + // Make sure misaligned accesses are always considered a miss + // Otherwise, we could report a hit for a word that goes past the end of the page. + // Aligned accesses smaller than a page size cannot straddle two pages. + return slot_vaddr_page == (vaddr & ~(PAGE_OFFSET_MASK & ~(sizeof(T) - 1))); +} + +constexpr uint64_t tlb_addr_page(uint64_t addr) { + return addr & ~PAGE_OFFSET_MASK; +} + +} // namespace cartesi + +#endif diff --git a/src/translate-virtual-address.h b/src/translate-virtual-address.h index a5bfba31f..5a740a29a 100644 --- a/src/translate-virtual-address.h +++ b/src/translate-virtual-address.h @@ -58,18 +58,17 @@ namespace cartesi { /// \param val Value to write. /// \returns True if succeeded, false otherwise. template -static FORCE_INLINE bool write_ram_uint64(STATE_ACCESS a, uint64_t paddr, uint64_t val) { - auto &pma = find_pma_entry(a, paddr); +static inline bool write_ram_uint64(STATE_ACCESS a, uint64_t paddr, uint64_t val) { + uint64_t pma_index = 0; + auto &pma = find_pma_entry(a, paddr, pma_index); if (unlikely(!pma.get_istart_M() || !pma.get_istart_W())) { return false; } - const uint64_t paddr_page = paddr & ~PAGE_OFFSET_MASK; - unsigned char *hpage = a.get_host_memory(pma) + (paddr_page - pma.get_start()); - const uint64_t hoffset = paddr - paddr_page; + const auto faddr = a.get_faddr(paddr, pma_index); // log writes to memory - a.write_memory_word(paddr, hpage, hoffset, val); + a.write_memory_word(faddr, pma_index, val); // mark page as dirty so we know to update the Merkle tree - pma.mark_dirty_page(paddr - pma.get_start()); + a.mark_dirty_page(faddr, pma_index); return true; } @@ -80,15 +79,14 @@ static FORCE_INLINE bool write_ram_uint64(STATE_ACCESS a, uint64_t paddr, uint64 /// \param pval Pointer to word. /// \returns True if succeeded, false otherwise. template -static FORCE_INLINE bool read_ram_uint64(STATE_ACCESS a, uint64_t paddr, uint64_t *pval) { - auto &pma = find_pma_entry(a, paddr); +static inline bool read_ram_uint64(STATE_ACCESS a, uint64_t paddr, uint64_t *pval) { + uint64_t pma_index = 0; + auto &pma = find_pma_entry(a, paddr, pma_index); if (unlikely(!pma.get_istart_M() || !pma.get_istart_R())) { return false; } - const uint64_t paddr_page = paddr & ~PAGE_OFFSET_MASK; - unsigned char *hpage = a.get_host_memory(pma) + (paddr_page - pma.get_start()); - const uint64_t hoffset = paddr - paddr_page; - a.read_memory_word(paddr, hpage, hoffset, pval); + const auto faddr = a.get_faddr(paddr, pma_index); + a.read_memory_word(faddr, pma_index, pval); return true; } diff --git a/src/uarch-bridge.h b/src/uarch-bridge.h index b09ea5a39..423a2ead2 100644 --- a/src/uarch-bridge.h +++ b/src/uarch-bridge.h @@ -27,6 +27,7 @@ #include "machine-state.h" #include "pma-constants.h" #include "riscv-constants.h" +#include "shadow-pmas.h" #include "shadow-state.h" #include "shadow-tlb.h" #include "strict-aliasing.h" @@ -814,22 +815,27 @@ class uarch_bridge { return "f"; } - if (paddr >= PMA_SHADOW_PMAS_START && paddr < PMA_SHADOW_PMAS_START + (PMA_MAX * PMA_WORD_SIZE * 2)) { - auto word_index = (paddr - PMA_SHADOW_PMAS_START) >> 3; - if ((word_index & 1) == 0) { + if (paddr >= shadow_pmas_get_pma_abs_addr(0) && paddr < shadow_pmas_get_pma_abs_addr(PMA_MAX)) { + if ((paddr & 0b1111) == 0) { return "pma.istart"; } - return "pma.ilength"; + if ((paddr & 0b1111) == 0b1000) { + return "pma.ilength"; + } } - if (paddr >= PMA_SHADOW_TLB_START && paddr < PMA_SHADOW_TLB_START + PMA_SHADOW_TLB_LENGTH && - paddr % sizeof(uint64_t) == 0) { - const uint64_t tlboff = paddr - PMA_SHADOW_TLB_START; - if (tlboff < offsetof(shadow_tlb_state, cold)) { - return "cold_tlb_entry_field"; + if (paddr % sizeof(uint64_t) == 0) { + if (paddr >= shadow_tlb_get_slot_abs_addr(0) && + paddr < shadow_tlb_get_slot_abs_addr(TLB_SET_SIZE)) { + return "write tlb"; + } + if (paddr >= shadow_tlb_get_slot_abs_addr(0) && + paddr < shadow_tlb_get_slot_abs_addr(TLB_SET_SIZE)) { + return "read tlb"; } - if (tlboff < sizeof(shadow_tlb_state)) { - return "hot_tlb_entry_field"; + if (paddr >= shadow_tlb_get_slot_abs_addr(0) && + paddr < shadow_tlb_get_slot_abs_addr(TLB_SET_SIZE)) { + return "code tlb"; } } diff --git a/src/uarch-solidity-compat.h b/src/uarch-solidity-compat.h index fc31d052b..4cc48266f 100644 --- a/src/uarch-solidity-compat.h +++ b/src/uarch-solidity-compat.h @@ -100,23 +100,23 @@ static inline void resetState(UarchState &a) { a.reset_state(); } -template -static inline uint64 readIflagsY(UarchState &a) { +template +static inline uint64 readIflagsY(State &a) { return a.read_iflags_Y(); } template -static inline void writeIflagsY(UarchState &a, uint64 val) { +static inline void writeIflagsY(State &a, uint64 val) { a.write_iflags_Y(val); } -template -static inline void writeHtifFromhost(UarchState &a, uint64 val) { +template +static inline void writeHtifFromhost(State &a, uint64 val) { a.write_htif_fromhost(val); } -template -static inline void writeMemoryWithPadding(UarchState &a, uint64 paddr, bytes data, uint64_t data_length, +template +static inline void writeMemoryWithPadding(State &a, uint64 paddr, bytes data, uint64_t data_length, int32 write_length_log2_size) { a.write_memory_with_padding(paddr, data, data_length, write_length_log2_size); } @@ -131,6 +131,15 @@ static inline void putChar(UarchState & /*a*/, unsigned char c) { os_putchar(c); } +template +static inline void markDirtyPage(UarchState &a, uint64 paddr, uint64 pma_index) { + a.mark_dirty_page(paddr, pma_index); +} + +static inline void writeTlbVpOffset(UarchState &a, uint64 use, uint64 vp_offset, uint64 pma_index) { + a.write_tlb_vp_offset(use, slot_index, vp_offset, pma_index); +} + // Conversions and arithmetic functions static inline int32 uint64ToInt32(uint64 v) { diff --git a/src/uarch-state-access.h b/src/uarch-state-access.h index c0578f2e6..c840da29c 100644 --- a/src/uarch-state-access.h +++ b/src/uarch-state-access.h @@ -27,13 +27,16 @@ #include "pma.h" #include "riscv-constants.h" #include "strict-aliasing.h" +#if OKUARCH #include "uarch-bridge.h" #include "uarch-pristine.h" #include "uarch-state.h" +#endif namespace cartesi { class uarch_state_access : public i_uarch_state_access { +#ifdef OKUARCH // NOLINTBEGIN(cppcoreguidelines-avoid-const-or-ref-data-members) uarch_state &m_us; machine_state &m_s; @@ -199,6 +202,7 @@ class uarch_state_access : public i_uarch_state_access { m_us.ram.fill_memory(m_us.ram.get_start() + uarch_pristine_ram_len, 0, m_us.ram.get_length() - uarch_pristine_ram_len); } +#endif }; } // namespace cartesi diff --git a/src/uarch-step.cpp b/src/uarch-step.cpp index b1c343027..b7dbd9f4e 100644 --- a/src/uarch-step.cpp +++ b/src/uarch-step.cpp @@ -865,17 +865,35 @@ static inline void executeSD(UarchState &a, uint32 insn, uint64 pc) { template static inline void executeECALL(UarchState &a, uint64 pc) { [[maybe_unused]] auto note = a.make_scoped_note("ecall"); + // ECALL conventions + // a0--a7 are the same as x10--x17 + // syscall is passed in a7 + // arguments are passed in a0--a5 + // return value is in a0 (and maybe also in a1) uint64 fn = readX(a, 17); // a7 contains the function number if (fn == UARCH_ECALL_FN_HALT) { return setHaltFlag(a); } if (fn == UARCH_ECALL_FN_PUTCHAR) { - uint64 character = readX(a, 16); // a6 contains the character to print + uint64 character = readX(a, 10); // a0 contains the character to print putChar(a, uint8(character)); - } else { - throwRuntimeError(a, "unsupported ecall function"); - } - return advancePc(a, pc); + return advancePc(a, pc); + } + if (fn == UARCH_ECALL_FN_MARK_DIRTY_PAGE) { + uint64 paddr = readX(a, 10); // a0 contains physical address in page to be marked dirty + uint64 pma_index = readX(a, 11); // a1 contains a index of PMA where page falls + markDirtyPage(a, paddr, pma_index); + return advancePc(a, pc); + } + if (fn == UARCH_ECALL_FN_WRITE_TLB_VP_OFFSET) { + uint64 use = readX(a, 10); // a0 contains TLB set (code, read, write) + uint64 slot_index = readX(a, 11); // a1 contains slot_index to modify + uint64 vp_offset = readX(a, 12); // a2 contains vp_offset to write + uint64 pma_index = readX(a, 13); // a3 contains index of PMA where page falls + writeTlbVpOffset(a, use, slot_index, vp_offset, pma_index); + return advancePc(a, pc); + } + throwRuntimeError(a, "unsupported ecall function"); } template From a4644227615ceb0803be7bfc5bfd64a4b9e06b31 Mon Sep 17 00:00:00 2001 From: Diego Nehab <1635557+diegonehab@users.noreply.github.com> Date: Mon, 20 Jan 2025 13:21:23 +0000 Subject: [PATCH 04/82] refactor: clean up uarch state access mechanism --- Makefile | 6 +- src/Makefile | 38 +- src/access-log.h | 60 +- src/cartesi/util.lua | 3 +- src/clint-factory.cpp | 10 +- src/find-pma-entry.h | 1 + src/htif-factory.cpp | 11 +- src/htif-factory.h | 1 - src/i-state-access.h | 612 ++++++++++-- src/i-uarch-state-access.h | 125 ++- src/interpret.cpp | 69 +- src/machine-c-api.h | 26 +- src/machine-reg.h | 339 ++++--- src/machine.cpp | 641 +++++++------ src/machine.h | 111 ++- src/mock-pma-entry.h | 4 + src/plic-factory.cpp | 13 +- src/pm-type-name.h | 48 + src/pma-constants.h | 31 + src/pma.cpp | 29 +- src/pma.h | 19 +- src/record-send-cmio-state-access.h | 18 +- src/record-step-state-access.h | 371 ++++---- src/replay-send-cmio-state-access.h | 5 + src/replay-step-state-access.h | 387 ++++---- src/scoped-note.h | 52 ++ src/shadow-peek.h | 68 ++ src/shadow-pmas-factory.h | 10 +- src/shadow-pmas.h | 60 +- src/shadow-state-factory.cpp | 96 +- src/shadow-state.h | 345 +++++++ src/shadow-tlb-factory.cpp | 71 +- src/shadow-tlb.h | 75 +- src/shadow-uarch-state-factory.cpp | 52 +- src/shadow-uarch-state.h | 130 +++ src/state-access.h | 54 +- src/tlb.h | 4 +- src/uarch-bridge.h | 1038 --------------------- src/uarch-config.h | 2 +- src/uarch-constants.h | 6 +- src/uarch-defines.h | 6 +- src/uarch-interpret.h | 5 +- src/uarch-machine-bridge.cpp | 170 ++++ src/uarch-machine-bridge.h | 57 ++ src/uarch-machine.cpp | 83 -- src/uarch-machine.h | 83 -- src/uarch-record-state-access.h | 525 ++++------- src/uarch-replay-state-access.h | 443 ++++----- src/uarch-solidity-compat.h | 24 +- src/uarch-state-access.h | 189 ++-- src/uarch-state.h | 2 +- src/uarch-step.cpp | 26 +- tests/Makefile | 4 +- tests/lua/cartesi/tests/util.lua | 12 +- tests/lua/log-with-mtime-transition.lua | 4 + tests/lua/machine-bind.lua | 181 ++-- tests/misc/test-machine-c-api.cpp | 8 +- uarch/Makefile | 3 + uarch/machine-uarch-bridge-state-access.h | 507 ++++++++++ uarch/uarch-ecall.c | 73 ++ uarch/uarch-ecall.h | 36 + uarch/uarch-machine-state-access.h | 586 ------------ uarch/uarch-printf.c | 4 +- uarch/uarch-printf.h | 7 - uarch/uarch-ram-entry.S | 4 +- uarch/uarch-run.cpp | 26 +- uarch/uarch-runtime.cpp | 14 - uarch/uarch-strict-aliasing.h | 92 ++ 68 files changed, 4258 insertions(+), 3957 deletions(-) create mode 100644 src/pm-type-name.h create mode 100644 src/scoped-note.h create mode 100644 src/shadow-peek.h delete mode 100644 src/uarch-bridge.h create mode 100644 src/uarch-machine-bridge.cpp create mode 100644 src/uarch-machine-bridge.h delete mode 100644 src/uarch-machine.cpp delete mode 100644 src/uarch-machine.h create mode 100644 uarch/machine-uarch-bridge-state-access.h create mode 100644 uarch/uarch-ecall.c create mode 100644 uarch/uarch-ecall.h delete mode 100644 uarch/uarch-machine-state-access.h create mode 100644 uarch/uarch-strict-aliasing.h diff --git a/Makefile b/Makefile index 91ee8e864..98a72b129 100644 --- a/Makefile +++ b/Makefile @@ -96,6 +96,9 @@ TESTSDIR = $(abspath tests) DOWNLOADDIR = $(DEPDIR)/downloads SUBCLEAN = $(addsuffix .clean,$(SRCDIR) uarch tests) +# Pass down received UARCH_DEFS to sub-makefiles +export UARCH_DEFS + # Docker image tag TAG ?= devel DEBIAN_IMG ?= cartesi/machine-emulator:$(TAG).deb @@ -306,11 +309,12 @@ toolchain-env: check-toolchain cartesi/machine-emulator:toolchain /bin/bash toolchain-exec: check-toolchain - @docker run --hostname toolchain --rm \ + docker run --hostname toolchain --rm \ -e USER=$$(id -u -n) \ -e GROUP=$$(id -g -n) \ -e UID=$$(id -u) \ -e GID=$$(id -g) \ + -e UARCH_DEFS="$(UARCH_DEFS)" \ -v `pwd`:/opt/cartesi/machine-emulator \ -w /opt/cartesi/machine-emulator \ cartesi/machine-emulator:toolchain /bin/bash -c "$(CONTAINER_COMMAND)" diff --git a/src/Makefile b/src/Makefile index f7780c2fb..53c30cb7b 100644 --- a/src/Makefile +++ b/src/Makefile @@ -175,17 +175,22 @@ DEFS+=-D_FILE_OFFSET_BITS=64 DEFS+=-DJSON_HAS_FILESYSTEM=0 ifeq ($(dump),yes) -#DEFS+=-DDUMP_ILLEGAL_INSN_EXCEPTIONS -#DEFS+=-DDUMP_EXCEPTIONS -#DEFS+=-DDUMP_INTERRUPTS -DEFS+=-DDUMP_HIST -#DEFS+=-DDUMP_MMU_EXCEPTIONS -#DEFS+=-DDUMP_INVALID_MEM_ACCESS -#DEFS+=-DDUMP_INVALID_CSR -#DEFS+=-DDUMP_INSN -#DEFS+=-DDUMP_REGS -#DEFS+=-DDUMP_COUNTERS +#DUMP_DEFS+=-DDUMP_ILLEGAL_INSN_EXCEPTIONS +#DUMP_DEFS+=-DDUMP_EXCEPTIONS +#DUMP_DEFS+=-DDUMP_INTERRUPTS +#DUMP_DEFS+=-DDUMP_HIST +#DUMP_DEFS+=-DDUMP_MMU_EXCEPTIONS +#DUMP_DEFS+=-DDUMP_INVALID_MEM_ACCESS +#DUMP_DEFS+=-DDUMP_INVALID_CSR +#DUMP_DEFS+=-DDUMP_INSN +DUMP_DEFS+=-DDUMP_STATE_ACCESS +DUMP_DEFS+=-DDUMP_UARCH_STATE_ACCESS +#DUMP_DEFS+=-DDUMP_REGS +#DUMP_DEFS+=-DDUMP_COUNTERS endif +DEFS += $(DUMP_DEFS) +# Pass down UARCH_DEFS to sub-makefiles +export UARCH_DEFS += $(DUMP_DEFS) # By default we compile in release with debug information, # so the emulator is packaged correctly by default. @@ -282,7 +287,7 @@ CLANG_TIDY=clang-tidy CLANG_TIDY_TARGETS=$(patsubst %.cpp,%.clang-tidy,$(patsubst %.c,%.clang-tidy,$(LINTER_SOURCES))) CLANG_FORMAT=clang-format -CLANG_FORMAT_UARCH_FILES:=$(wildcard ../uarch/*.cpp, ../uarch/*.h) +CLANG_FORMAT_UARCH_FILES:=$(wildcard ../uarch/*.cpp) $(wildcard ../uarch/*.h) CLANG_FORMAT_UARCH_FILES:=$(filter-out %uarch-printf%,$(strip $(CLANG_FORMAT_UARCH_FILES))) CLANG_FORMAT_FILES:=$(wildcard *.cpp) $(wildcard *.c) $(wildcard *.h) $(wildcard *.hpp) $(CLANG_FORMAT_UARCH_FILES) CLANG_FORMAT_IGNORE_FILES:=interpret-jump-table.h @@ -381,14 +386,13 @@ LIBCARTESI_OBJS:= \ uarch-pristine-ram.o \ uarch-pristine-state-hash.o \ uarch-pristine-hash.o \ + uarch-interpret.o \ + uarch-step.o \ + uarch-machine-bridge.o \ + uarch-reset-state.o \ + send-cmio-response.o \ replay-step-state-access-interop.o - #send-cmio-response.o \ - #uarch-machine.o \ - #uarch-interpret.o \ - #uarch-step.o \ - #uarch-reset-state.o \ - CARTESI_CLUA_OBJS:= \ clua.o \ clua-i-virtual-machine.o diff --git a/src/access-log.h b/src/access-log.h index 97d439a77..62e526f6c 100644 --- a/src/access-log.h +++ b/src/access-log.h @@ -54,12 +54,12 @@ static inline void set_word_access_data(uint64_t w, access_data &ad) { ad.insert(ad.end(), p, p + sizeof(w)); } -static inline void replace_word_access_data(uint64_t w, access_data &ad, int offset = 0) { +static inline void replace_word_access_data(uint64_t w, access_data &ad, uint64_t offset = 0) { assert(ad.size() >= offset + sizeof(uint64_t)); aliased_aligned_write(ad.data() + offset, w); } -static inline uint64_t get_word_access_data(const access_data &ad, int offset = 0) { +static inline uint64_t get_word_access_data(const access_data &ad, uint64_t offset = 0) { assert(ad.size() >= offset + sizeof(uint64_t)); return aliased_aligned_read(ad.data() + offset); } @@ -258,13 +258,14 @@ class access_log { }; private: - std::vector m_accesses; ///< List of all accesses - std::vector m_brackets; ///< Begin/End annotations - std::vector m_notes; ///< Per-access annotations - type m_log_type; ///< Log type + std::vector m_accesses; ///< List of all accesses + std::vector m_brackets; ///< Begin/End annotations + std::vector m_notes; ///< Per-access annotations + type m_log_type; ///< Log type + std::vector::size_type m_outstanding_ends; ///< Number of outstanding unmatched end brackets public: - explicit access_log(type log_type) : m_log_type(log_type) { + explicit access_log(type log_type) : m_log_type(log_type), m_outstanding_ends{0} { ; } @@ -273,8 +274,16 @@ class access_log { m_accesses(std::forward(accesses)), m_brackets(std::forward(brackets)), m_notes(std::forward(notes)), - m_log_type(log_type) { - ; + m_log_type(log_type), + m_outstanding_ends(0) { + for (const auto &b : m_brackets) { + if (b.type == bracket_type::begin) { + ++m_outstanding_ends; + } + if (b.type == bracket_type::end && m_outstanding_ends > 0) { + --m_outstanding_ends; + } + }; } /// \brief Clear the log @@ -282,28 +291,43 @@ class access_log { m_accesses.clear(); m_notes.clear(); m_brackets.clear(); + m_outstanding_ends = 0; } /// \brief Adds a bracket annotation to the log (if the log type includes annotations) /// \param type Bracket type /// \param text Annotation contents - void push_bracket(bracket_type type, const char *text) { + void push_begin_bracket(const char *text) { + if (m_log_type.has_annotations()) { + // Increment number of outstanding end brackets we are expecting + ++m_outstanding_ends; + // Make sure we have room for the matching end bracket as well. + // That way, unless the user is messing with unbalanced brackets, there is no way we + // would throw an exception for lack of memory on the matching end bracket + m_brackets.push_back(bracket_note{bracket_type::begin, m_accesses.size(), text}); + m_brackets.reserve(m_brackets.size() + m_outstanding_ends); + } + } + + void push_end_bracket(const char *text) noexcept { if (m_log_type.has_annotations()) { - if (type == bracket_type::begin) { - // make sure we have room for end bracket as well. that way, - // unless the user use unbalanced brackets, there is no way we - // would throw an exception for lack of memory on and end bracket - m_brackets.reserve(m_brackets.size() + 2); + // If we failed to push, it was because the system is completely screwed anyway *and* the + // user is using unbalanced brackets. Therefore, it's OK to quietly ignore the error. + try { + m_brackets.push_back(bracket_note{bracket_type::end, m_accesses.size(), text}); + } catch (...) { // NOLINT(bugprone-empty-catch) + } + // Decrement number of outstanding end brackets we are expecting + if (m_outstanding_ends > 0) { + --m_outstanding_ends; } - m_brackets.push_back(bracket_note{.type = type, .where = m_accesses.size(), .text = text}); } } /// \brief Adds a new access to the log /// \tparam A Type of access /// \param a Access object - /// \param text Annotation contents (added if the log - /// type includes annotations, ignored otherwise) + /// \param text Annotation contents (added if the log type includes annotations, ignored otherwise) template void push_access(A &&a, const char *text) { m_accesses.push_back(std::forward(a)); diff --git a/src/cartesi/util.lua b/src/cartesi/util.lua index f050ad64d..d80d5d0d6 100644 --- a/src/cartesi/util.lua +++ b/src/cartesi/util.lua @@ -202,10 +202,11 @@ local function hexhash8(hash) return string.sub(hexhash(hash), 1, 8) end local function accessdatastring(data, data_hash, data_log2_size, address) local data_size = 1 << data_log2_size if data_log2_size == 3 then + if not data then return "???(no written data)" end if data_size < #data then -- access data is smaller than the tree leaf size -- the logged data is the entire tree leaf, but we only need the data that was accessed - local leaf_aligned_address = address & ~((1 << cartesi.TREE_LOG2_WORD_SIZE) - 1) + local leaf_aligned_address = (address >> cartesi.TREE_LOG2_WORD_SIZE) << cartesi.TREE_LOG2_WORD_SIZE local word_offset = address - leaf_aligned_address data = data:sub(word_offset + 1, word_offset + data_size) end diff --git a/src/clint-factory.cpp b/src/clint-factory.cpp index c6f34a689..24e323932 100644 --- a/src/clint-factory.cpp +++ b/src/clint-factory.cpp @@ -18,22 +18,14 @@ #include "clint-factory.h" #include "clint.h" -#include "machine.h" #include "pma-constants.h" #include "pma.h" namespace cartesi { -/// \brief CLINT device peek callback. See ::pma_peek. -static bool clint_peek(const pma_entry &pma, const machine & /*m*/, uint64_t page_offset, - const unsigned char **page_data, unsigned char * /*scratch*/) { - *page_data = nullptr; - return (page_offset % PMA_PAGE_SIZE) == 0 && page_offset < pma.get_length(); -} - pma_entry make_clint_pma_entry(uint64_t start, uint64_t length) { const pma_entry::flags f{.R = true, .W = true, .X = false, .IR = false, .IW = false, .DID = PMA_ISTART_DID::CLINT}; - return make_device_pma_entry("CLINT device", start, length, clint_peek, &clint_driver).set_flags(f); + return make_device_pma_entry("CLINT device", start, length, pma_peek_pristine, &clint_driver).set_flags(f); } } // namespace cartesi diff --git a/src/find-pma-entry.h b/src/find-pma-entry.h index 1e2b223b9..1cb857ca4 100644 --- a/src/find-pma-entry.h +++ b/src/find-pma-entry.h @@ -31,6 +31,7 @@ namespace cartesi { /// \returns PMA entry where word falls, or empty sentinel. template auto &find_pma_entry(STATE_ACCESS &a, uint64_t paddr, uint64_t &index) { + [[maybe_unused]] auto note = a.make_scoped_note("find_pma_entry"); index = 0; while (true) { auto &pma = a.read_pma_entry(index); diff --git a/src/htif-factory.cpp b/src/htif-factory.cpp index 47eff8db4..4754b8ccc 100644 --- a/src/htif-factory.cpp +++ b/src/htif-factory.cpp @@ -18,23 +18,14 @@ #include "htif-factory.h" #include "htif.h" -#include "machine-runtime-config.h" -#include "machine.h" #include "pma-constants.h" #include "pma.h" namespace cartesi { -/// \brief HTIF device peek callback. See ::pma_peek. -static bool htif_peek(const pma_entry &pma, const machine & /*m*/, uint64_t page_offset, - const unsigned char **page_data, unsigned char * /*scratch*/) { - *page_data = nullptr; - return (page_offset % PMA_PAGE_SIZE) == 0 && page_offset < pma.get_length(); -} - pma_entry make_htif_pma_entry(uint64_t start, uint64_t length) { const pma_entry::flags f{.R = true, .W = true, .X = false, .IR = false, .IW = false, .DID = PMA_ISTART_DID::HTIF}; - return make_device_pma_entry("HTIF device", start, length, htif_peek, &htif_driver).set_flags(f); + return make_device_pma_entry("HTIF device", start, length, pma_peek_pristine, &htif_driver).set_flags(f); } } // namespace cartesi diff --git a/src/htif-factory.h b/src/htif-factory.h index 9a6807682..1bba5844c 100644 --- a/src/htif-factory.h +++ b/src/htif-factory.h @@ -19,7 +19,6 @@ #include -#include "machine-runtime-config.h" #include "pma.h" namespace cartesi { diff --git a/src/i-state-access.h b/src/i-state-access.h index 2b2fad014..104e27d57 100644 --- a/src/i-state-access.h +++ b/src/i-state-access.h @@ -28,6 +28,10 @@ #include "meta.h" #include "tlb.h" +#ifdef DUMP_STATE_ACCESS +#include "pm-type-name.h" +#endif + namespace cartesi { // Forward declarations @@ -91,11 +95,22 @@ class i_state_access { // CRTP return derived().do_get_naked_state(); } - /// \brief Adds an annotation bracket to the log - /// \param type Type of bracket + /// \brief Adds a begin bracket annotation to the log + /// \param text String with the text for the annotation + void push_begin_bracket(const char *text) { +#ifdef DUMP_STATE_ACCESS + fprintf(stderr, "----> begin %s (%s)\n", text, get_name()); +#endif + return derived().do_push_begin_bracket(text); + } + + /// \brief Adds an end bracket annotation to the log /// \param text String with the text for the annotation - void push_bracket(bracket_type type, const char *text) { - return derived().do_push_bracket(type, text); + void push_end_bracket(const char *text) { +#ifdef DUMP_STATE_ACCESS + fprintf(stderr, "<---- end %s (%s)\n", text, get_name()); +#endif + return derived().do_push_end_bracket(text); } /// \brief Adds annotations to the state, bracketing a scope @@ -106,511 +121,912 @@ class i_state_access { // CRTP } /// \brief Reads from general-purpose register. - /// \param reg Register index. + /// \param i Register index. /// \returns Register value. - uint64_t read_x(int reg) { - return derived().do_read_x(reg); + uint64_t read_x(int i) { +#ifdef DUMP_STATE_ACCESS + const auto val = derived().do_read_x(i); + fprintf(stderr, "%s::read_x(%d) = %llu(0x%llx)\n", get_name(), i, val, val); + return val; +#else + return derived().do_read_x(i); +#endif } /// \brief Writes register to general-purpose register. - /// \param reg Register index. + /// \param i Register index. /// \param val New register value. /// \details Writes to register zero *break* the machine. /// There is an assertion to catch this, but NDEBUG will let the value pass through. - void write_x(int reg, uint64_t val) { - return derived().do_write_x(reg, val); + void write_x(int i, uint64_t val) { + derived().do_write_x(i, val); +#ifdef DUMP_STATE_ACCESS + fprintf(stderr, "%s::write_x(%d, %llu(0x%llx))\n", get_name(), i, val, val); +#endif } /// \brief Reads from floating-point register. - /// \param reg Register index. + /// \param i Register index. /// \returns Register value. - uint64_t read_f(int reg) { - return derived().do_read_f(reg); + uint64_t read_f(int i) { +#ifdef DUMP_STATE_ACCESS + const auto val = derived().do_read_f(i); + fprintf(stderr, "%s::read_f(%d) = %llu(0x%llx)\n", get_name(), i, val, val); + return val; +#else + return derived().do_read_f(i); +#endif } /// \brief Writes register to floating-point register. - /// \param reg Register index. + /// \param i Register index. /// \param val New register value. - void write_f(int reg, uint64_t val) { - return derived().do_write_f(reg, val); + void write_f(int i, uint64_t val) { + derived().do_write_f(i, val); +#ifdef DUMP_STATE_ACCESS + fprintf(stderr, "%s::write_f(%d, %llu(%llx))\n", get_name(), i, val, val); +#endif } /// \brief Reads the program counter. /// \returns Register value. uint64_t read_pc() { +#ifdef DUMP_STATE_ACCESS + const auto val = derived().do_read_pc(); + fprintf(stderr, "%s::read_pc() = %llu(0x%llx)\n", get_name(), val, val); + return val; +#else return derived().do_read_pc(); +#endif } /// \brief Writes the program counter. /// \param val New register value. void write_pc(uint64_t val) { - return derived().do_write_pc(val); + derived().do_write_pc(val); +#ifdef DUMP_STATE_ACCESS + fprintf(stderr, "%s::write_pc(%llu(0x%llx))\n", get_name(), val, val); +#endif } /// \brief Writes CSR fcsr. /// \param val New register value. void write_fcsr(uint64_t val) { - return derived().do_write_fcsr(val); + derived().do_write_fcsr(val); +#ifdef DUMP_STATE_ACCESS + fprintf(stderr, "%s::write_fcsr(%llu(0x%llx))\n", get_name(), val, val); +#endif } /// \brief Reads CSR fcsr. /// \returns Register value. uint64_t read_fcsr() { +#ifdef DUMP_STATE_ACCESS + const auto val = derived().do_read_fcsr(); + fprintf(stderr, "%s::read_fcsr() = %llu(0x%llx)\n", get_name(), val, val); + return val; +#else return derived().do_read_fcsr(); +#endif } /// \brief Reads CSR icycleinstret. /// \returns Register value. uint64_t read_icycleinstret() { +#ifdef DUMP_STATE_ACCESS + const auto val = derived().do_read_icycleinstret(); + fprintf(stderr, "%s::read_icycleinstret() = %llu(0x%llx)\n", get_name(), val, val); + return val; +#else return derived().do_read_icycleinstret(); +#endif } /// \brief Writes CSR icycleinstret. /// \param val New register value. void write_icycleinstret(uint64_t val) { - return derived().do_write_icycleinstret(val); + derived().do_write_icycleinstret(val); +#ifdef DUMP_STATE_ACCESS + fprintf(stderr, "%s::write_icycleinstret(%llu(0x%llx))\n", get_name(), val, val); +#endif } /// \brief Reads CSR mvendorid. /// \returns Register value. uint64_t read_mvendorid() { +#ifdef DUMP_STATE_ACCESS + const auto val = derived().do_read_mvendorid(); + fprintf(stderr, "%s::read_mvendorid() = %llu(0x%llx)\n", get_name(), val, val); + return val; +#else return derived().do_read_mvendorid(); +#endif } /// \brief Reads CSR marchid. /// \returns Register value. uint64_t read_marchid() { +#ifdef DUMP_STATE_ACCESS + const auto val = derived().do_read_marchid(); + fprintf(stderr, "%s::read_marchid() = %llu(0x%llx)\n", get_name(), val, val); + return val; +#else return derived().do_read_marchid(); +#endif } /// \brief Reads CSR mimpid. /// \returns Register value. uint64_t read_mimpid() { +#ifdef DUMP_STATE_ACCESS + const auto val = derived().do_read_mimpid(); + fprintf(stderr, "%s::read_mimpid() = %llu(0x%llx)\n", get_name(), val, val); + return val; +#else return derived().do_read_mimpid(); +#endif } /// \brief Reads CSR mcycle. /// \returns Register value. uint64_t read_mcycle() { +#ifdef DUMP_STATE_ACCESS + const auto val = derived().do_read_mcycle(); + fprintf(stderr, "%s::read_mcycle() = %llu(0x%llx)\n", get_name(), val, val); + return val; +#else return derived().do_read_mcycle(); +#endif } /// \brief Writes CSR mcycle. /// \param val New register value. void write_mcycle(uint64_t val) { - return derived().do_write_mcycle(val); + derived().do_write_mcycle(val); +#ifdef DUMP_STATE_ACCESS + fprintf(stderr, "%s::write_mcycle(%llu(0x%llx))\n", get_name(), val, val); +#endif } /// \brief Reads CSR mstatus. /// \returns Register value. uint64_t read_mstatus() { +#ifdef DUMP_STATE_ACCESS + const auto val = derived().do_read_mstatus(); + fprintf(stderr, "%s::read_mstatus() = %llu(0x%llx)\n", get_name(), val, val); + return val; +#else return derived().do_read_mstatus(); +#endif } /// \brief Writes CSR mstatus. /// \param val New register value. void write_mstatus(uint64_t val) { - return derived().do_write_mstatus(val); + derived().do_write_mstatus(val); +#ifdef DUMP_STATE_ACCESS + fprintf(stderr, "%s::write_mstatus(%llu(0x%llx))\n", get_name(), val, val); +#endif } /// \brief Reads CSR menvcfg. /// \returns Register value. uint64_t read_menvcfg() { +#ifdef DUMP_STATE_ACCESS + const auto val = derived().do_read_menvcfg(); + fprintf(stderr, "%s::read_menvcfg() = %llu(0x%llx)\n", get_name(), val, val); + return val; +#else return derived().do_read_menvcfg(); +#endif } /// \brief Writes CSR menvcfg. /// \param val New register value. void write_menvcfg(uint64_t val) { - return derived().do_write_menvcfg(val); + derived().do_write_menvcfg(val); +#ifdef DUMP_STATE_ACCESS + fprintf(stderr, "%s::write_menvcfg(%llu(0x%llx))\n", get_name(), val, val); +#endif } /// \brief Reads CSR mtvec. /// \returns Register value. uint64_t read_mtvec() { +#ifdef DUMP_STATE_ACCESS + const auto val = derived().do_read_mtvec(); + fprintf(stderr, "%s::read_mtvec() = %llu(0x%llx)\n", get_name(), val, val); + return val; +#else return derived().do_read_mtvec(); +#endif } /// \brief Writes CSR mtvec. /// \param val New register value. void write_mtvec(uint64_t val) { - return derived().do_write_mtvec(val); + derived().do_write_mtvec(val); +#ifdef DUMP_STATE_ACCESS + fprintf(stderr, "%s::write_mtvec(%llu(0x%llx))\n", get_name(), val, val); +#endif } /// \brief Reads CSR mscratch. /// \returns Register value. uint64_t read_mscratch() { +#ifdef DUMP_STATE_ACCESS + const auto val = derived().do_read_mscratch(); + fprintf(stderr, "%s::read_mscratch() = %llu(0x%llx)\n", get_name(), val, val); + return val; +#else return derived().do_read_mscratch(); +#endif } /// \brief Writes CSR mscratch. /// \param val New register value. void write_mscratch(uint64_t val) { - return derived().do_write_mscratch(val); + derived().do_write_mscratch(val); +#ifdef DUMP_STATE_ACCESS + fprintf(stderr, "%s::write_mscratch(%llu(0x%llx))\n", get_name(), val, val); +#endif } /// \brief Reads CSR mepc. /// \returns Register value. uint64_t read_mepc() { +#ifdef DUMP_STATE_ACCESS + const auto val = derived().do_read_mepc(); + fprintf(stderr, "%s::read_mepc() = %llu(0x%llx)\n", get_name(), val, val); + return val; +#else return derived().do_read_mepc(); +#endif } /// \brief Writes CSR mepc. /// \param val New register value. void write_mepc(uint64_t val) { - return derived().do_write_mepc(val); + derived().do_write_mepc(val); +#ifdef DUMP_STATE_ACCESS + fprintf(stderr, "%s::write_mepc(%llu(0x%llx))\n", get_name(), val, val); +#endif } /// \brief Reads CSR mcause. /// \returns Register value. uint64_t read_mcause() { +#ifdef DUMP_STATE_ACCESS + const auto val = derived().do_read_mcause(); + fprintf(stderr, "%s::read_mcause() = %llu(0x%llx)\n", get_name(), val, val); + return val; +#else return derived().do_read_mcause(); +#endif } /// \brief Writes CSR mcause. /// \param val New register value. void write_mcause(uint64_t val) { - return derived().do_write_mcause(val); + derived().do_write_mcause(val); +#ifdef DUMP_STATE_ACCESS + fprintf(stderr, "%s::write_mcause(%llu(0x%llx))\n", get_name(), val, val); +#endif } /// \brief Reads CSR mtval. /// \returns Register value. uint64_t read_mtval() { +#ifdef DUMP_STATE_ACCESS + const auto val = derived().do_read_mtval(); + fprintf(stderr, "%s::read_mtval() = %llu(0x%llx)\n", get_name(), val, val); + return val; +#else return derived().do_read_mtval(); +#endif } /// \brief Writes CSR mtval. /// \param val New register value. void write_mtval(uint64_t val) { - return derived().do_write_mtval(val); + derived().do_write_mtval(val); +#ifdef DUMP_STATE_ACCESS + fprintf(stderr, "%s::write_mtval(%llu(0x%llx))\n", get_name(), val, val); +#endif } /// \brief Reads CSR misa. /// \returns Register value. uint64_t read_misa() { +#ifdef DUMP_STATE_ACCESS + const auto val = derived().do_read_misa(); + fprintf(stderr, "%s::read_misa() = %llu(0x%llx)\n", get_name(), val, val); + return val; +#else return derived().do_read_misa(); +#endif } /// \brief Writes CSR misa. /// \param val New register value. void write_misa(uint64_t val) { - return derived().do_write_misa(val); + derived().do_write_misa(val); +#ifdef DUMP_STATE_ACCESS + fprintf(stderr, "%s::write_misa(%llu(0x%llx))\n", get_name(), val, val); +#endif } /// \brief Reads CSR mie. /// \returns Register value. uint64_t read_mie() { +#ifdef DUMP_STATE_ACCESS + const auto val = derived().do_read_mie(); + fprintf(stderr, "%s::read_mie() = %llu(0x%llx)\n", get_name(), val, val); + return val; +#else return derived().do_read_mie(); +#endif } /// \brief Writes CSR mie. /// \param val New register value. void write_mie(uint64_t val) { - return derived().do_write_mie(val); + derived().do_write_mie(val); +#ifdef DUMP_STATE_ACCESS + fprintf(stderr, "%s::write_mie(%llu(0x%llx))\n", get_name(), val, val); +#endif } /// \brief Reads CSR mip. /// \returns Register value. uint64_t read_mip() { +#ifdef DUMP_STATE_ACCESS + const auto val = derived().do_read_mip(); + fprintf(stderr, "%s::read_mip() = %llu(0x%llx)\n", get_name(), val, val); + return val; +#else return derived().do_read_mip(); +#endif } /// \brief Writes CSR mip. /// \param val New register value. void write_mip(uint64_t val) { - return derived().do_write_mip(val); + derived().do_write_mip(val); +#ifdef DUMP_STATE_ACCESS + fprintf(stderr, "%s::write_mip(%llu(0x%llx))\n", get_name(), val, val); +#endif } /// \brief Reads CSR medeleg. /// \returns Register value. uint64_t read_medeleg() { +#ifdef DUMP_STATE_ACCESS + const auto val = derived().do_read_medeleg(); + fprintf(stderr, "%s::read_medeleg() = %llu(0x%llx)\n", get_name(), val, val); + return val; +#else return derived().do_read_medeleg(); +#endif } /// \brief Writes CSR medeleg. /// \param val New register value. void write_medeleg(uint64_t val) { - return derived().do_write_medeleg(val); + derived().do_write_medeleg(val); +#ifdef DUMP_STATE_ACCESS + fprintf(stderr, "%s::write_medeleg(%llu(0x%llx))\n", get_name(), val, val); +#endif } /// \brief Reads CSR mideleg. /// \returns Register value. uint64_t read_mideleg() { +#ifdef DUMP_STATE_ACCESS + const auto val = derived().do_read_mideleg(); + fprintf(stderr, "%s::read_mideleg() = %llu(0x%llx)\n", get_name(), val, val); + return val; +#else return derived().do_read_mideleg(); +#endif } /// \brief Writes CSR mideleg. /// \param val New register value. void write_mideleg(uint64_t val) { - return derived().do_write_mideleg(val); + derived().do_write_mideleg(val); +#ifdef DUMP_STATE_ACCESS + fprintf(stderr, "%s::write_mideleg(%llu(0x%llx))\n", get_name(), val, val); +#endif } /// \brief Reads CSR iprv. /// \returns Register value. uint64_t read_iprv() { +#ifdef DUMP_STATE_ACCESS + const auto val = derived().do_read_iprv(); + fprintf(stderr, "%s::read_iprv() = %llu(0x%llx)\n", get_name(), val, val); + return val; +#else return derived().do_read_iprv(); +#endif } /// \brief Writes CSR iprv. /// \param val New register value. void write_iprv(uint64_t val) { - return derived().do_write_iprv(val); + derived().do_write_iprv(val); +#ifdef DUMP_STATE_ACCESS + fprintf(stderr, "%s::write_iprv(%llu(0x%llx))\n", get_name(), val, val); +#endif } /// \brief Reads CSR iflags_X. /// \returns Register value. uint64_t read_iflags_X() { +#ifdef DUMP_STATE_ACCESS + const auto val = derived().do_read_iflags_X(); + fprintf(stderr, "%s::read_iflags_X() = %llu(0x%llx)\n", get_name(), val, val); + return val; +#else return derived().do_read_iflags_X(); +#endif } /// \brief Writes CSR iflags_X. /// \param val New register value. void write_iflags_X(uint64_t val) { - return derived().do_write_iflags_X(val); + derived().do_write_iflags_X(val); +#ifdef DUMP_STATE_ACCESS + fprintf(stderr, "%s::write_iflags_X(%llu(0x%llx))\n", get_name(), val, val); +#endif } /// \brief Reads CSR iflags_Y. /// \returns Register value. uint64_t read_iflags_Y() { +#ifdef DUMP_STATE_ACCESS + const auto val = derived().do_read_iflags_Y(); + fprintf(stderr, "%s::read_iflags_Y() = %llu(0x%llx)\n", get_name(), val, val); + return val; +#else return derived().do_read_iflags_Y(); +#endif } /// \brief Writes CSR iflags_Y. /// \param val New register value. void write_iflags_Y(uint64_t val) { - return derived().do_write_iflags_Y(val); + derived().do_write_iflags_Y(val); +#ifdef DUMP_STATE_ACCESS + fprintf(stderr, "%s::write_iflags_Y(%llu(0x%llx))\n", get_name(), val, val); +#endif } /// \brief Reads CSR iflags_H. /// \returns Register value. uint64_t read_iflags_H() { +#ifdef DUMP_STATE_ACCESS + const auto val = derived().do_read_iflags_H(); + fprintf(stderr, "%s::read_iflags_H() = %llu(0x%llx)\n", get_name(), val, val); + return val; +#else return derived().do_read_iflags_H(); +#endif } /// \brief Writes CSR iflags_H. /// \param val New register value. void write_iflags_H(uint64_t val) { - return derived().do_write_iflags_H(val); + derived().do_write_iflags_H(val); +#ifdef DUMP_STATE_ACCESS + fprintf(stderr, "%s::write_iflags_H(%llu(0x%llx))\n", get_name(), val, val); +#endif } /// \brief Reads CSR mcounteren. /// \returns Register value. uint64_t read_mcounteren() { +#ifdef DUMP_STATE_ACCESS + const auto val = derived().do_read_mcounteren(); + fprintf(stderr, "%s::read_mcounteren() = %llu(0x%llx)\n", get_name(), val, val); + return val; +#else return derived().do_read_mcounteren(); +#endif } /// \brief Writes CSR mcounteren. /// \param val New register value. void write_mcounteren(uint64_t val) { - return derived().do_write_mcounteren(val); + derived().do_write_mcounteren(val); +#ifdef DUMP_STATE_ACCESS + fprintf(stderr, "%s::write_mcounteren(%llu(0x%llx))\n", get_name(), val, val); +#endif } /// \brief Reads CSR senvcfg. /// \returns Register value. uint64_t read_senvcfg() { +#ifdef DUMP_STATE_ACCESS + const auto val = derived().do_read_senvcfg(); + fprintf(stderr, "%s::read_senvcfg() = %llu(0x%llx)\n", get_name(), val, val); + return val; +#else return derived().do_read_senvcfg(); +#endif } /// \brief Writes CSR senvcfg. /// \param val New register value. void write_senvcfg(uint64_t val) { - return derived().do_write_senvcfg(val); + derived().do_write_senvcfg(val); +#ifdef DUMP_STATE_ACCESS + fprintf(stderr, "%s::write_senvcfg(%llu(0x%llx))\n", get_name(), val, val); +#endif } /// \brief Reads CSR stvec. /// \returns Register value. uint64_t read_stvec() { +#ifdef DUMP_STATE_ACCESS + const auto val = derived().do_read_stvec(); + fprintf(stderr, "%s::read_stvec() = %llu(0x%llx)\n", get_name(), val, val); + return val; +#else return derived().do_read_stvec(); +#endif } /// \brief Writes CSR stvec. /// \param val New register value. void write_stvec(uint64_t val) { - return derived().do_write_stvec(val); + derived().do_write_stvec(val); +#ifdef DUMP_STATE_ACCESS + fprintf(stderr, "%s::write_stvec(%llu(0x%llx))\n", get_name(), val, val); +#endif } /// \brief Reads CSR sscratch. /// \returns Register value. uint64_t read_sscratch() { +#ifdef DUMP_STATE_ACCESS + const auto val = derived().do_read_sscratch(); + fprintf(stderr, "%s::read_sscratch() = %llu(0x%llx)\n", get_name(), val, val); + return val; +#else return derived().do_read_sscratch(); +#endif } /// \brief Writes CSR sscratch. /// \param val New register value. void write_sscratch(uint64_t val) { - return derived().do_write_sscratch(val); + derived().do_write_sscratch(val); +#ifdef DUMP_STATE_ACCESS + fprintf(stderr, "%s::write_sscratch(%llu(0x%llx))\n", get_name(), val, val); +#endif } /// \brief Reads CSR sepc. /// \returns Register value. uint64_t read_sepc() { +#ifdef DUMP_STATE_ACCESS + const auto val = derived().do_read_sepc(); + fprintf(stderr, "%s::read_sepc() = %llu(0x%llx)\n", get_name(), val, val); + return val; +#else return derived().do_read_sepc(); +#endif } /// \brief Writes CSR sepc. /// \param val New register value. void write_sepc(uint64_t val) { - return derived().do_write_sepc(val); + derived().do_write_sepc(val); +#ifdef DUMP_STATE_ACCESS + fprintf(stderr, "%s::write_sepc(%llu(0x%llx))\n", get_name(), val, val); +#endif } /// \brief Reads CSR scause. /// \returns Register value. uint64_t read_scause() { +#ifdef DUMP_STATE_ACCESS + const auto val = derived().do_read_scause(); + fprintf(stderr, "%s::read_scause() = %llu(0x%llx)\n", get_name(), val, val); + return val; +#else return derived().do_read_scause(); +#endif } /// \brief Writes CSR scause. /// \param val New register value. void write_scause(uint64_t val) { - return derived().do_write_scause(val); + derived().do_write_scause(val); +#ifdef DUMP_STATE_ACCESS + fprintf(stderr, "%s::write_scause(%llu(0x%llx))\n", get_name(), val, val); +#endif } /// \brief Reads CSR stval. /// \returns Register value. uint64_t read_stval() { +#ifdef DUMP_STATE_ACCESS + const auto val = derived().do_read_stval(); + fprintf(stderr, "%s::read_stval() = %llu(0x%llx)\n", get_name(), val, val); + return val; +#else return derived().do_read_stval(); +#endif } /// \brief Writes CSR stval. /// \param val New register value. void write_stval(uint64_t val) { - return derived().do_write_stval(val); + derived().do_write_stval(val); +#ifdef DUMP_STATE_ACCESS + fprintf(stderr, "%s::write_stval(%llu(0x%llx))\n", get_name(), val, val); +#endif } /// \brief Reads CSR satp. /// \returns Register value. uint64_t read_satp() { +#ifdef DUMP_STATE_ACCESS + const auto val = derived().do_read_satp(); + fprintf(stderr, "%s::read_satp() = %llu(0x%llx)\n", get_name(), val, val); + return val; +#else return derived().do_read_satp(); +#endif } /// \brief Writes CSR satp. /// \param val New register value. void write_satp(uint64_t val) { - return derived().do_write_satp(val); + derived().do_write_satp(val); +#ifdef DUMP_STATE_ACCESS + fprintf(stderr, "%s::write_satp(%llu(0x%llx))\n", get_name(), val, val); +#endif } /// \brief Reads CSR scounteren. /// \returns Register value. uint64_t read_scounteren() { +#ifdef DUMP_STATE_ACCESS + const auto val = derived().do_read_scounteren(); + fprintf(stderr, "%s::read_scounteren() = %llu(0x%llx)\n", get_name(), val, val); + return val; +#else return derived().do_read_scounteren(); +#endif } /// \brief Writes CSR scounteren. /// \param val New register value. void write_scounteren(uint64_t val) { - return derived().do_write_scounteren(val); + derived().do_write_scounteren(val); +#ifdef DUMP_STATE_ACCESS + fprintf(stderr, "%s::write_scounteren(%llu(0x%llx))\n", get_name(), val, val); +#endif } /// \brief Reads CSR ilrsc. /// \returns Register value. /// \details This is Cartesi-specific. uint64_t read_ilrsc() { +#ifdef DUMP_STATE_ACCESS + const auto val = derived().do_read_ilrsc(); + fprintf(stderr, "%s::read_ilrsc() = %llu(0x%llx)\n", get_name(), val, val); + return val; +#else return derived().do_read_ilrsc(); +#endif } /// \brief Writes CSR ilrsc. /// \param val New register value. /// \details This is Cartesi-specific. void write_ilrsc(uint64_t val) { - return derived().do_write_ilrsc(val); + derived().do_write_ilrsc(val); +#ifdef DUMP_STATE_ACCESS + fprintf(stderr, "%s::write_ilrsc(%llu(0x%llx))\n", get_name(), val, val); +#endif } /// \brief Reads CSR iunrep. /// \returns Register value. /// \details This is Cartesi-specific. uint64_t read_iunrep() { +#ifdef DUMP_STATE_ACCESS + const auto val = derived().do_read_iunrep(); + fprintf(stderr, "%s::read_iunrep() = %llu(0x%llx)\n", get_name(), val, val); + return val; +#else return derived().do_read_iunrep(); +#endif } /// \brief Writes CSR iunrep. /// \param val New register value. /// \details This is Cartesi-specific. void write_iunrep(uint64_t val) { - return derived().do_write_iunrep(val); + derived().do_write_iunrep(val); +#ifdef DUMP_STATE_ACCESS + fprintf(stderr, "%s::write_iunrep(%llu(0x%llx))\n", get_name(), val, val); +#endif } /// \brief Reads CLINT's mtimecmp. /// \returns Register value. uint64_t read_clint_mtimecmp() { +#ifdef DUMP_STATE_ACCESS + const auto val = derived().do_read_clint_mtimecmp(); + fprintf(stderr, "%s::read_clint_mtimecmp() = %llu(0x%llx)\n", get_name(), val, val); + return val; +#else return derived().do_read_clint_mtimecmp(); +#endif } /// \brief Writes CLINT's mtimecmp. /// \param val New register value. void write_clint_mtimecmp(uint64_t val) { - return derived().do_write_clint_mtimecmp(val); + derived().do_write_clint_mtimecmp(val); +#ifdef DUMP_STATE_ACCESS + fprintf(stderr, "%s::write_clint_mtimecmp(%llu(0x%llx))\n", get_name(), val, val); +#endif } /// \brief Reads PLIC's girqpend. /// \returns Register value. uint64_t read_plic_girqpend() { +#ifdef DUMP_STATE_ACCESS + const auto val = derived().do_read_plic_girqpend(); + fprintf(stderr, "%s::read_plic_girqpend() = %llu(0x%llx)\n", get_name(), val, val); + return val; +#else return derived().do_read_plic_girqpend(); +#endif } /// \brief Writes PLIC's girqpend. /// \param val New register value. void write_plic_girqpend(uint64_t val) { - return derived().do_write_plic_girqpend(val); + derived().do_write_plic_girqpend(val); +#ifdef DUMP_STATE_ACCESS + fprintf(stderr, "%s::write_plic_girqpend(%llu(0x%llx))\n", get_name(), val, val); +#endif } /// \brief Reads PLIC's girqsrvd. /// \returns Register value. uint64_t read_plic_girqsrvd() { +#ifdef DUMP_STATE_ACCESS + const auto val = derived().do_read_plic_girqsrvd(); + fprintf(stderr, "%s::read_plic_girqsrvd() = %llu(0x%llx)\n", get_name(), val, val); + return val; +#else return derived().do_read_plic_girqsrvd(); +#endif } /// \brief Writes PLIC's girqsrvd. /// \param val New register value. void write_plic_girqsrvd(uint64_t val) { - return derived().do_write_plic_girqsrvd(val); + derived().do_write_plic_girqsrvd(val); +#ifdef DUMP_STATE_ACCESS + fprintf(stderr, "%s::write_plic_girqsrvd(%llu(0x%llx))\n", get_name(), val, val); +#endif } /// \brief Reads HTIF's fromhost. /// \returns Register value. uint64_t read_htif_fromhost() { +#ifdef DUMP_STATE_ACCESS + const auto val = derived().do_read_htif_fromhost(); + fprintf(stderr, "%s::read_htif_fromhost() = %llu(0x%llx)\n", get_name(), val, val); + return val; +#else return derived().do_read_htif_fromhost(); +#endif } /// \brief Writes HTIF's fromhost. /// \param val New register value. void write_htif_fromhost(uint64_t val) { - return derived().do_write_htif_fromhost(val); + derived().do_write_htif_fromhost(val); +#ifdef DUMP_STATE_ACCESS + fprintf(stderr, "%s::write_htif_fromhost(%llu(0x%llx))\n", get_name(), val, val); +#endif } /// \brief Reads HTIF's tohost. /// \returns Register value. uint64_t read_htif_tohost() { +#ifdef DUMP_STATE_ACCESS + const auto val = derived().do_read_htif_tohost(); + fprintf(stderr, "%s::read_htif_tohost() = %llu(0x%llx)\n", get_name(), val, val); + return val; +#else return derived().do_read_htif_tohost(); +#endif } /// \brief Writes HTIF's tohost. /// \param val New register value. void write_htif_tohost(uint64_t val) { - return derived().do_write_htif_tohost(val); + derived().do_write_htif_tohost(val); +#ifdef DUMP_STATE_ACCESS + fprintf(stderr, "%s::write_htif_tohost(%llu(0x%llx))\n", get_name(), val, val); +#endif } /// \brief Reads HTIF's ihalt. /// \returns Register value. uint64_t read_htif_ihalt() { +#ifdef DUMP_STATE_ACCESS + const auto val = derived().do_read_htif_ihalt(); + fprintf(stderr, "%s::read_htif_ihalt() = %llu(0x%llx)\n", get_name(), val, val); + return val; +#else return derived().do_read_htif_ihalt(); +#endif } /// \brief Reads HTIF's iconsole. /// \returns Register value. uint64_t read_htif_iconsole() { +#ifdef DUMP_STATE_ACCESS + const auto val = derived().do_read_htif_iconsole(); + fprintf(stderr, "%s::read_htif_iconsole() = %llu(0x%llx)\n", get_name(), val, val); + return val; +#else return derived().do_read_htif_iconsole(); +#endif } /// \brief Reads HTIF's iyield. /// \returns Register value. uint64_t read_htif_iyield() { +#ifdef DUMP_STATE_ACCESS + const auto val = derived().do_read_htif_iyield(); + fprintf(stderr, "%s::read_htif_iyield() = %llu(0x%llx)\n", get_name(), val, val); + return val; +#else return derived().do_read_htif_iyield(); +#endif } /// \brief Reads PMA entry at a given index. /// \param index Index of PMA pma_entry &read_pma_entry(uint64_t index) { +#ifdef DUMP_STATE_ACCESS + auto &pma = derived().do_read_pma_entry(index); + fprintf(stderr, "%s::read_pma_entry(%llu) = {%s, 0x%llx, 0x%llx}\n", get_name(), index, + pma_get_DID_name(pma.get_istart_DID()), pma.get_start(), pma.get_length()); + return pma; +#else return derived().do_read_pma_entry(index); +#endif + } + + /// \brief Converts a target physical address to the implementation-defined fast address + /// \param paddr Target physical address to convert + /// \param pma_index Index of PMA where address falls + /// \returns Corresponding implementation-defined fast address + fast_addr get_faddr(uint64_t paddr, uint64_t pma_index) const { +#ifdef DUMP_STATE_ACCESS + const auto val = derived().do_get_faddr(paddr, pma_index); + const char *fast_addr_name = std::is_same_v ? "phys_addr" : "fast_addr"; + fprintf(stderr, "%s::get_faddr(%llu(0x%llx)) = %s{%llu(0x%llx)}\n", get_name(), paddr, paddr, fast_addr_name, + val, val); + return val; +#else + return derived().do_get_faddr(paddr, pma_index); +#endif } /// \brief Reads a chunk of data from a memory PMA range. @@ -642,7 +1058,7 @@ class i_state_access { // CRTP /// \param write_length_log2_size Log2 size of the total write length. void write_memory_with_padding(uint64_t paddr, const unsigned char *data, uint64_t data_length, int write_length_log2_size) { - return derived().do_write_memory_with_padding(paddr, data, data_length, write_length_log2_size); + derived().do_write_memory_with_padding(paddr, data, data_length, write_length_log2_size); } /// \brief Reads a word from memory. @@ -655,7 +1071,15 @@ class i_state_access { // CRTP template void read_memory_word(fast_addr faddr, uint64_t pma_index, T *pval) { static_assert(std::is_integral::value && sizeof(T) <= sizeof(uint64_t), "unsupported type"); - return derived().template do_read_memory_word(faddr, pma_index, pval); +#ifdef DUMP_STATE_ACCESS + derived().template do_read_memory_word(faddr, pma_index, pval); + const char *fast_addr_name = std::is_same_v ? "phys_addr" : "fast_addr"; + fprintf(stderr, "%s::read_memory_word<%s,%s>(%s{0x%llx}, %llu) = %llu(0x%llx)\n", get_name(), pm_type_name_v, + pm_type_name_v, fast_addr_name, faddr, pma_index, static_cast(*pval), + static_cast(*pval)); +#else + derived().template do_read_memory_word(faddr, pma_index, pval); +#endif } /// \brief Writes a word to memory. @@ -670,34 +1094,60 @@ class i_state_access { // CRTP template void write_memory_word(fast_addr faddr, uint64_t pma_index, T val) { static_assert(std::is_integral::value && sizeof(T) <= sizeof(uint64_t), "unsupported type"); - return derived().template do_write_memory_word(faddr, pma_index, val); + derived().template do_write_memory_word(faddr, pma_index, val); +#ifdef DUMP_STATE_ACCESS + const char *fast_addr_name = std::is_same_v ? "phys_addr" : "fast_addr"; + fprintf(stderr, "%s::write_memory_word<%s,%s>(%s{0x%llx}, %llu, %llu(0x%llx))\n", get_name(), pm_type_name_v, + pm_type_name_v, fast_addr_name, faddr, pma_index, static_cast(val), + static_cast(val)); +#endif } /// \brief Reads TLB's vaddr_page /// \tparam USE TLB set /// \param slot_index Slot index /// \returns Value in slot. - template + template uint64_t read_tlb_vaddr_page(uint64_t slot_index) { - return derived().template do_read_tlb_vaddr_page(slot_index); +#ifdef DUMP_STATE_ACCESS + const auto val = derived().template do_read_tlb_vaddr_page(slot_index); + fprintf(stderr, "%s::read_tlb_vaddr_page<%llu>(%llu) = 0x%llx\n", get_name(), SET, slot_index, val); + return val; +#else + return derived().template do_read_tlb_vaddr_page(slot_index); +#endif } /// \brief Reads TLB's vp_offset /// \tparam USE TLB set /// \param slot_index Slot index /// \returns Value in slot. - template + template fast_addr read_tlb_vp_offset(uint64_t slot_index) { - return derived().template do_read_tlb_vp_offset(slot_index); +#ifdef DUMP_STATE_ACCESS + const char *fast_addr_name = std::is_same_v ? "phys_addr" : "fast_addr"; + const auto val = derived().template do_read_tlb_vp_offset(slot_index); + fprintf(stderr, "%s::read_tlb_vp_offset<%llu>(%llu) = %s{0x%llx}\n", get_name(), SET, slot_index, + fast_addr_name, val); + return val; +#else + return derived().template do_read_tlb_vp_offset(slot_index); +#endif } /// \brief Reads TLB's pma_index /// \tparam USE TLB set /// \param slot_index Slot index /// \returns Value in slot. - template + template uint64_t read_tlb_pma_index(uint64_t slot_index) { - return derived().template do_read_tlb_pma_index(slot_index); +#ifdef DUMP_STATE_ACCESS + const auto val = derived().template do_read_tlb_pma_index(slot_index); + fprintf(stderr, "%s::read_tlb_pma_index<%llu>(%llu) = %llu(0x%llx)\n", get_name(), SET, slot_index, val, val); + return val; +#else + return derived().template do_read_tlb_pma_index(slot_index); +#endif } /// \brief Writes to a TLB slot @@ -708,17 +1158,14 @@ class i_state_access { // CRTP /// \param pma_index Value to write /// \detail Writes to the TLB must be modify all fields atomically to prevent an inconsistent state. /// This simplifies all state access implementations. - template + template void write_tlb(uint64_t slot_index, uint64_t vaddr_page, fast_addr vp_offset, uint64_t pma_index) { - return derived().template do_write_tlb(slot_index, vaddr_page, vp_offset, pma_index); - } - - /// \brief Converts a target physical address to the implementation-defined fast address - /// \param paddr Target physical address to convert - /// \param pma_index Index of PMA where address falls - /// \returns Correspnding implementation-defined fast address - fast_addr get_faddr(uint64_t paddr, uint64_t pma_index) const { - return derived().do_get_faddr(paddr, pma_index); + derived().template do_write_tlb(slot_index, vaddr_page, vp_offset, pma_index); +#ifdef DUMP_STATE_ACCESS + const char *fast_addr_name = std::is_same_v ? "phys_addr" : "fast_addr"; + fprintf(stderr, "%s::write_tlb<%llu>(%llu, 0x%llx, %s{0x%llx}, %llu)\n", get_name(), SET, slot_index, + vaddr_page, fast_addr_name, vp_offset, pma_index); +#endif } /// \brief Marks a page as dirty @@ -730,17 +1177,17 @@ class i_state_access { // CRTP /// have been marked dirty. When a page leaves the write TLB, it is marked dirty. /// If the state belongs to a host machine, then this call MUST be forwarded to machine::mark_dirty_page(); void mark_dirty_page(fast_addr faddr, uint64_t pma_index) { - return derived().do_mark_dirty_page(faddr, pma_index); + derived().do_mark_dirty_page(faddr, pma_index); } /// \brief Writes a character to the console /// \param c Character to output void putchar(uint8_t c) { - return derived().do_putchar(c); + derived().do_putchar(c); } // --- - // These methods ONLY need to be implemented when state belongs to non-reproducible host machine + // These methods ONLY need to be implemented when state belongs to non-reproducible host machines // --- /// \brief Poll for external interrupts. @@ -773,8 +1220,11 @@ class i_state_access { // CRTP } #endif -protected: + constexpr const char *get_name() const { + return derived().do_get_name(); + } +protected: /// \brief Default implementation when state does not belong to non-reproducible host machine bool do_get_soft_yield() { // NOLINT(readability-convert-member-functions-to-static) return false; @@ -791,6 +1241,18 @@ class i_state_access { // CRTP return -1; } + /// \brief For state access classes that do not need annotations + void do_push_begin_bracket(const char * /*text*/) {} + + /// \brief For state access classes that do not need annotations + void do_push_end_bracket(const char * /*text*/) {} + +#ifndef DUMP_STATE_ACCESS + /// \brief For state access classes that do not need annotations + int do_make_scoped_note(const char * /*text*/) { + return 0; + } +#endif }; /// \brief SFINAE test implementation of the i_state_access interface diff --git a/src/i-uarch-state-access.h b/src/i-uarch-state-access.h index 0f74793d5..875fe7f35 100644 --- a/src/i-uarch-state-access.h +++ b/src/i-uarch-state-access.h @@ -20,7 +20,7 @@ #include #include "bracket-note.h" -#include "pma.h" +#include "tlb.h" namespace cartesi { @@ -39,11 +39,22 @@ class i_uarch_state_access { // CRTP } public: - /// \brief Adds an annotation bracket to the log - /// \param type Type of bracket + /// \brief Adds a begin bracket annotation to the log /// \param text String with the text for the annotation - void push_bracket(bracket_type type, const char *text) { - return derived().do_push_bracket(type, text); + void push_begin_bracket(const char *text) { +#ifdef DUMP_UARCH_STATE_ACCESS + printf("----> begin %s (%s)\n", text, get_name()); +#endif + return derived().do_push_begin_bracket(text); + } + + /// \brief Adds an end bracket annotation to the log + /// \param text String with the text for the annotation + void push_end_bracket(const char *text) { +#ifdef DUMP_UARCH_STATE_ACCESS + printf("<---- end %s (%s)\n", text, get_name()); +#endif + return derived().do_push_end_bracket(text); } /// \brief Adds annotations to the state, bracketing a scope @@ -53,54 +64,126 @@ class i_uarch_state_access { // CRTP return derived().do_make_scoped_note(text); } - auto read_x(int r) { - return derived().do_read_x(r); + auto read_x(int i) { +#ifdef DUMP_UARCH_STATE_ACCESS + const auto val = derived().do_read_x(i); + printf("%s::read_x(%d) = %llu(0x%llx)\n", get_name(), i, val, val); + return val; +#else + return derived().do_read_x(i); +#endif } - auto write_x(int r, uint64_t v) { - return derived().do_write_x(r, v); + auto write_x(int i, uint64_t val) { + derived().do_write_x(i, val); +#ifdef DUMP_UARCH_STATE_ACCESS + printf("%s::write_x(%d, %llu)\n", get_name(), i, val); +#endif } auto read_pc() { +#ifdef DUMP_UARCH_STATE_ACCESS + const auto val = derived().do_read_pc(); + printf("%s::read_pc() = %llu(0x%llx)\n", get_name(), val, val); + return val; +#else return derived().do_read_pc(); +#endif } - auto write_pc(uint64_t v) { - return derived().do_write_pc(v); + auto write_pc(uint64_t val) { + derived().do_write_pc(val); +#ifdef DUMP_UARCH_STATE_ACCESS + printf("%s::write_pc(%llu(0x%llx))\n", get_name(), val, val); +#endif } auto read_cycle() { +#ifdef DUMP_UARCH_STATE_ACCESS + const auto val = derived().do_read_cycle(); + printf("%s::read_cycle() = %llu(0x%llx)\n", get_name(), val, val); + return val; +#else return derived().do_read_cycle(); +#endif } auto read_halt_flag() { +#ifdef DUMP_UARCH_STATE_ACCESS + const auto val = derived().do_read_halt_flag(); + printf("%s::read_halt_flag() = %llu(0x%llx)\n", get_name(), val, val); + return val; +#else return derived().do_read_halt_flag(); +#endif } - auto set_halt_flag() { - return derived().do_set_halt_flag(); - } - - auto reset_halt_flag() { - return derived().do_reset_halt_flag(); + auto write_halt_flag(uint64_t val) { + derived().do_write_halt_flag(val); +#ifdef DUMP_UARCH_STATE_ACCESS + printf("%s::write_halt_flag(%llu(0x%llx))\n", get_name(), val, val); +#endif } - auto write_cycle(uint64_t v) { - return derived().do_write_cycle(v); + auto write_cycle(uint64_t val) { + derived().do_write_cycle(val); +#ifdef DUMP_UARCH_STATE_ACCESS + printf("%s::write_cycle(%llu(0x%llx))\n", get_name(), val, val); +#endif } uint64_t read_word(uint64_t paddr) { +#ifdef DUMP_UARCH_STATE_ACCESS + const auto val = derived().do_read_word(paddr); + printf("%s::read_word(phys_addr{0x%llx}) = %llu(0x%llx)\n", get_name(), paddr, val, val); + return val; +#else return derived().do_read_word(paddr); +#endif } - void write_word(uint64_t paddr, uint64_t data) { - return derived().do_write_word(paddr, data); + void write_word(uint64_t paddr, uint64_t val) { + derived().do_write_word(paddr, val); +#ifdef DUMP_UARCH_STATE_ACCESS + printf("%s::write_word(phys_addr{0x%llx}, %llu(0x%llx))\n", get_name(), paddr, val, val); +#endif } /// \brief Resets uarch to pristine state void reset_state() { return derived().do_reset_state(); } + + void putchar(uint8_t c) { + derived().do_putchar(c); + } + + void mark_dirty_page(uint64_t paddr, uint64_t pma_index) { + return derived().do_mark_dirty_page(paddr, pma_index); + } + + void write_tlb(TLB_set_index set_index, uint64_t slot_index, uint64_t vaddr_page, uint64_t vp_offset, + uint64_t pma_index) { + derived().do_write_tlb(set_index, slot_index, vaddr_page, vp_offset, pma_index); + } + + constexpr const char *get_name() const { + return derived().do_get_name(); + } + +private: + /// \brief For state access classes that do not need annotations + void do_push_begin_bracket(const char * /*text*/) {} + + /// \brief For state access classes that do not need annotations + void do_push_end_bracket(const char * /*text*/) {} + +#ifndef DUMP_UARCH_STATE_ACCESS + /// \brief For state access classes that do not need annotations + int do_make_scoped_note(const char * /*text*/) { + return 0; + } +#endif }; } // namespace cartesi diff --git a/src/interpret.cpp b/src/interpret.cpp index d986d4f7e..42a879860 100644 --- a/src/interpret.cpp +++ b/src/interpret.cpp @@ -21,10 +21,8 @@ #include #ifdef MICROARCHITECTURE -#ifdef OKUARCH -#include "uarch-machine-state-access.h" +#include "machine-uarch-bridge-state-access.h" #include "uarch-runtime.h" -#endif #else #include "record-step-state-access.h" #include "replay-step-state-access.h" @@ -360,7 +358,6 @@ static NO_INLINE uint64_t raise_exception(STATE_ACCESS a, uint64_t pc, uint64_t // this is not performed in the instruction hot loop as an optimization. tval = static_cast(tval); } - #if defined(DUMP_EXCEPTIONS) || defined(DUMP_MMU_EXCEPTIONS) || defined(DUMP_INTERRUPTS) || \ defined(DUMP_ILLEGAL_INSN_EXCEPTIONS) { @@ -831,14 +828,14 @@ static FORCE_INLINE int32_t insn_get_C_SWSP_imm(uint32_t insn) { /// \tparam STATE_ACCESS Class of machine state accessor object. /// \param a Machine state accessor object. /// \param slot_index Slot index -template +template static void flush_tlb_slot(STATE_ACCESS &a, uint64_t slot_index) { // Make sure a valid page leaving the write TLB is marked as dirty // We must do this BEFORE we modify the TLB entries themselves // (Otherwise, we could stop uarch before it marks the page dirty but after // the entry is no longer in the TLB, which would cause the Merkle tree to // miss a dirty page.) - if constexpr (USE == TLB_WRITE) { + if constexpr (SET == TLB_WRITE) { const auto old_vaddr_page = a.template read_tlb_vaddr_page(slot_index); if (old_vaddr_page != TLB_INVALID_PAGE) { auto old_pma_index = a.template read_tlb_pma_index(slot_index); @@ -851,17 +848,17 @@ static void flush_tlb_slot(STATE_ACCESS &a, uint64_t slot_index) { const auto vaddr_page = TLB_INVALID_PAGE; const auto vp_offset = i_state_access_fast_addr_t{}; const auto pma_index = TLB_INVALID_PMA_INDEX; - a.template write_tlb(slot_index, vaddr_page, vp_offset, pma_index); + a.template write_tlb(slot_index, vaddr_page, vp_offset, pma_index); } /// \brief Flushes out an entire TLB set /// \tparam USE TLB set /// \tparam STATE_ACCESS Class of machine state accessor object. /// \param a Machine state accessor object. -template +template static void flush_tlb_set(STATE_ACCESS &a) { for (uint64_t slot_index = 0; slot_index < PMA_TLB_SIZE; ++slot_index) { - flush_tlb_slot(a, slot_index); + flush_tlb_slot(a, slot_index); } } @@ -898,15 +895,16 @@ static void flush_tlb_vaddr(STATE_ACCESS &a, uint64_t /* vaddr */) { /// \param paddr Index of PMA where paddr falls /// \param vp_offset Receives the new vp_offset that will be stored in the slot /// \returns The implementation-defined fast address corresponding to paddr -template +template static i_state_access_fast_addr_t replace_tlb_entry(STATE_ACCESS &a, uint64_t vaddr, uint64_t paddr, uint64_t pma_index, i_state_access_fast_addr_t &vp_offset) { + [[maybe_unused]] auto note = a.make_scoped_note("replace_tlb_entry"); const auto slot_index = tlb_slot_index(vaddr); - flush_tlb_slot(a, slot_index); + flush_tlb_slot(a, slot_index); const auto vaddr_page = tlb_addr_page(vaddr); const auto faddr = a.get_faddr(paddr, pma_index); vp_offset = faddr - vaddr; - a.template write_tlb(slot_index, vaddr_page, vp_offset, pma_index); + a.template write_tlb(slot_index, vaddr_page, vp_offset, pma_index); return faddr; } @@ -918,10 +916,10 @@ static i_state_access_fast_addr_t replace_tlb_entry(STATE_ACCESS & /// \param paddr Corresponding physical address /// \param paddr Index of PMA where paddr falls /// \returns The implementation-defined fast address corresponding to paddr -template +template static FORCE_INLINE auto replace_tlb_entry(STATE_ACCESS &a, uint64_t vaddr, uint64_t paddr, uint64_t pma_index) { i_state_access_fast_addr_t vp_offset{0}; - return replace_tlb_entry(a, vaddr, paddr, pma_index, vp_offset); + return replace_tlb_entry(a, vaddr, paddr, pma_index, vp_offset); } /// \brief Read an aligned word from virtual memory (slow path that goes through virtual address translation). @@ -942,6 +940,7 @@ static FORCE_INLINE auto replace_tlb_entry(STATE_ACCESS &a, uint64_t vaddr, uint template static NO_INLINE std::pair read_virtual_memory_slow(STATE_ACCESS a, uint64_t pc, uint64_t mcycle, uint64_t vaddr, T *pval) { + [[maybe_unused]] auto note = a.make_scoped_note("read_virtual_memory_slow"); using U = std::make_unsigned_t; // No support for misaligned accesses: They are handled by a trap in BBL if (unlikely(vaddr & (sizeof(T) - 1))) { @@ -960,11 +959,13 @@ static NO_INLINE std::pair read_virtual_memory_slow(STATE_ACCESS const auto &pma = find_pma_entry(a, paddr, pma_index); if (likely(pma.get_istart_R())) { if (likely(pma.get_istart_M())) { + [[maybe_unused]] auto note = a.make_scoped_note("read memory"); const auto faddr = replace_tlb_entry(a, vaddr, paddr, pma_index); a.template read_memory_word(faddr, pma_index, pval); return {true, pc}; } if (likely(pma.get_istart_IO())) { + [[maybe_unused]] auto note = a.make_scoped_note("read device"); const uint64_t offset = paddr - pma.get_start(); uint64_t val{}; device_state_access da(a, mcycle); @@ -993,6 +994,7 @@ static NO_INLINE std::pair read_virtual_memory_slow(STATE_ACCESS /// \returns True if succeeded, false otherwise. template static FORCE_INLINE bool read_virtual_memory(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint64_t vaddr, T *pval) { + [[maybe_unused]] auto note = a.make_scoped_note("read_virtual_memory"); // Try hitting the TLB const auto slot_index = tlb_slot_index(vaddr); const auto slot_vaddr_page = a.template read_tlb_vaddr_page(slot_index); @@ -1031,6 +1033,7 @@ static FORCE_INLINE bool read_virtual_memory(STATE_ACCESS a, uint64_t &pc, uint6 template static NO_INLINE std::pair write_virtual_memory_slow(STATE_ACCESS a, uint64_t pc, uint64_t mcycle, uint64_t vaddr, uint64_t val64) { + [[maybe_unused]] auto note = a.make_scoped_note("write_virtual_memory_slow"); using U = std::make_unsigned_t; // No support for misaligned accesses: They are handled by a trap in BBL if (unlikely(vaddr & (sizeof(T) - 1))) { @@ -1077,6 +1080,7 @@ static NO_INLINE std::pair write_virtual_memory_slow(S template static FORCE_INLINE execute_status write_virtual_memory(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint64_t vaddr, uint64_t val64) { + [[maybe_unused]] auto note = a.make_scoped_note("write_virtual_memory"); // Try hitting the TLB const uint64_t slot_index = tlb_slot_index(vaddr); const uint64_t slot_vaddr_page = a.template read_tlb_vaddr_page(slot_index); @@ -1087,8 +1091,8 @@ static FORCE_INLINE execute_status write_virtual_memory(STATE_ACCESS a, uint64_t pc = new_pc; return status; } - const auto pma_index = a.template read_tlb_pma_index(slot_index); - const auto vp_offset = a.template read_tlb_vp_offset(slot_index); + const auto pma_index = a.template read_tlb_pma_index(slot_index); + const auto vp_offset = a.template read_tlb_vp_offset(slot_index); const auto faddr = vaddr + vp_offset; a.template write_memory_word(faddr, pma_index, static_cast(val64)); INC_COUNTER(a.get_statistics(), tlb_whit); @@ -1098,6 +1102,7 @@ static FORCE_INLINE execute_status write_virtual_memory(STATE_ACCESS a, uint64_t template static void dump_insn([[maybe_unused]] STATE_ACCESS a, [[maybe_unused]] uint64_t pc, [[maybe_unused]] uint32_t insn, [[maybe_unused]] const char *name) { + [[maybe_unused]] auto note = a.make_scoped_note("dump_insn"); #ifdef DUMP_HIST a.get_naked_state().insn_hist[name]++; #endif @@ -1105,13 +1110,8 @@ static void dump_insn([[maybe_unused]] STATE_ACCESS a, [[maybe_unused]] uint64_t dump_regs(a.get_naked_state()); #endif #ifdef DUMP_INSN - uint64_t ppc = 0; - // If we are running in the microinterpreter, we may or may not be collecting a step access log. - // To prevent additional address translation end up in the log, - // the following check will always be false when MICROARCHITECTURE is defined. - if (std::is_same_v && - !translate_virtual_address(a, &ppc, pc, PTE_XWR_X_SHIFT)) { - ppc = pc; + uint64_t ppc = pc; + if (!translate_virtual_address(a, &ppc, pc, PTE_XWR_X_SHIFT)) { fprintf(stderr, "v %08" PRIx64, ppc); } else { fprintf(stderr, "p %08" PRIx64, ppc); @@ -5389,6 +5389,7 @@ static FORCE_INLINE fetch_status fetch_translate_pc(STATE_ACCESS a, uint64_t &pc template static FORCE_INLINE fetch_status fetch_insn(STATE_ACCESS a, uint64_t &pc, uint32_t &insn, uint64_t &last_vaddr_page, i_state_access_fast_addr_t &last_vp_offset, uint64_t &last_pma_index) { + [[maybe_unused]] auto note = a.make_scoped_note("fetch_insn"); i_state_access_fast_addr_t faddr{0}; const uint64_t pc_vaddr_page = tlb_addr_page(pc); // If pc is in the same page as the last pc fetch, @@ -5397,12 +5398,16 @@ static FORCE_INLINE fetch_status fetch_insn(STATE_ACCESS a, uint64_t &pc, uint32 faddr = pc + last_vp_offset; } else { // Not in the same page as last the fetch, we need to perform address translation - if (unlikely(fetch_translate_pc(a, pc, pc, last_vp_offset, last_pma_index) == fetch_status::exception)) { + i_state_access_fast_addr_t pc_vp_offset{}; + uint64_t pc_pma_index{}; + if (unlikely(fetch_translate_pc(a, pc, pc, pc_vp_offset, pc_pma_index) == fetch_status::exception)) { return fetch_status::exception; } // Update fetch address translation cache last_vaddr_page = pc_vaddr_page; - faddr = pc + last_vp_offset; + last_vp_offset = pc_vp_offset; + last_pma_index = pc_pma_index; + faddr = pc + pc_vp_offset; } // The following code assumes pc is always 2-byte aligned, this is guaranteed by RISC-V spec. // If pc is pointing to the very last 2 bytes of a page, it's crossing a page boundary. @@ -5415,10 +5420,14 @@ static FORCE_INLINE fetch_status fetch_insn(STATE_ACCESS a, uint64_t &pc, uint32 if (unlikely(insn_is_uncompressed(insn))) { // We have to perform a new address translation to read the next 2 bytes since we changed pages. const uint64_t pc2 = pc + 2; - if (unlikely(fetch_translate_pc(a, pc, pc2, last_vp_offset, last_pma_index) == fetch_status::exception)) { + i_state_access_fast_addr_t pc2_vp_offset{}; + uint64_t pc2_pma_index{}; + if (unlikely(fetch_translate_pc(a, pc, pc2, pc2_vp_offset, pc2_pma_index) == fetch_status::exception)) { return fetch_status::exception; } last_vaddr_page = tlb_addr_page(pc2); + last_vp_offset = pc2_vp_offset; + last_pma_index = pc2_pma_index; faddr = pc2 + last_vp_offset; a.template read_memory_word(faddr, last_pma_index, &insn16); insn |= insn16 << 16; @@ -6042,16 +6051,14 @@ interpreter_break_reason interpret(STATE_ACCESS a, uint64_t mcycle_end) { } if (status == execute_status::success_and_yield) { return interpreter_break_reason::yielded_softly; - } // Reached mcycle_end + } // Reached mcycle_end assert(a.read_mcycle() == mcycle_end); // LCOV_EXCL_LINE return interpreter_break_reason::reached_target_mcycle; } #ifdef MICROARCHITECTURE -#ifdef OKUARCH -// Explicit instantiation for uarch_machine_state_access -template interpreter_break_reason interpret(uarch_machine_state_access a, uint64_t mcycle_end); -#endif +// Explicit instantiation for machine_uarch_bridge_state_access +template interpreter_break_reason interpret(machine_uarch_bridge_state_access a, uint64_t mcycle_end); #else // Explicit instantiation for state_access template interpreter_break_reason interpret(state_access a, uint64_t mcycle_end); diff --git a/src/machine-c-api.h b/src/machine-c-api.h index ae3e523ae..8ad49f0c3 100644 --- a/src/machine-c-api.h +++ b/src/machine-c-api.h @@ -522,24 +522,24 @@ CM_API cm_error cm_read_reg(const cm_machine *m, cm_reg reg, uint64_t *val); /// \returns 0 for success, non zero code for error. CM_API cm_error cm_write_reg(cm_machine *m, cm_reg reg, uint64_t val); -/// \brief Reads a chunk of data from a machine memory range, by its physical address. +/// \brief Reads a chunk of data, by its target physical address and length. /// \param m Pointer to a non-empty machine object (holds a machine instance). -/// \param address Physical address to start reading. -/// \param data Receives chunk of memory. -/// \param length Size of chunk in bytes. +/// \param paddr Target physical address to start reading from. +/// \param data Buffer that receives data to read. Must be at least \p length bytes long. +/// \param length Number of bytes to read from \p paddr to \p data. /// \returns 0 for success, non zero code for error. -/// \details The entire chunk must be inside the same memory range. -CM_API cm_error cm_read_memory(const cm_machine *m, uint64_t address, uint8_t *data, uint64_t length); +/// \details The data can be anywhere in the entire address space. +CM_API cm_error cm_read_memory(const cm_machine *m, uint64_t paddr, uint8_t *data, uint64_t length); -/// \brief Writes a chunk of data to a machine memory range, by its physical address. +/// \brief Writes a chunk of data to machine memory, by its target physical address and length. /// \param m Pointer to a non-empty machine object (holds a machine instance). -/// \param address Physical address to start writing. -/// \param data Source for chunk of data. -/// \param length Size of chunk in bytes. +/// \param paddr Target physical address to start writing to. +/// \param data Buffer that contains data to write. Must be at least \p length bytes long. +/// \param length Number of bytes to write starting from \p data to \p paddr. /// \returns 0 for success, non zero code for error. -/// \details The entire chunk must be inside the same memory range. -/// Moreover, unlike cm_read_memory(), the memory range written to must not be mapped to a device. -CM_API cm_error cm_write_memory(cm_machine *m, uint64_t address, const uint8_t *data, uint64_t length); +/// \details Unlike read_memory(), the entire chunk of data, from \p paddr to \p paddr + \p length, +/// must reside entirely in the same memory range. Moreover, it cannot be mapped to a device. +CM_API cm_error cm_write_memory(cm_machine *m, uint64_t paddr, const uint8_t *data, uint64_t length); /// \brief Reads a chunk of data from a machine memory range, by its virtual address. /// \param m Pointer to a non-empty machine object (holds a machine instance). diff --git a/src/machine-reg.h b/src/machine-reg.h index 0fc8e77bf..be84ae758 100644 --- a/src/machine-reg.h +++ b/src/machine-reg.h @@ -17,7 +17,6 @@ #ifndef MACHINE_REG_H #define MACHINE_REG_H -#include "pma-constants.h" #include "shadow-state.h" #include "shadow-uarch-state.h" @@ -29,153 +28,156 @@ namespace cartesi { /// \brief List of machine registers enum class machine_reg : uint64_t { // Processor x registers - x0 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[0]), - x1 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[1]), - x2 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[2]), - x3 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[3]), - x4 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[4]), - x5 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[5]), - x6 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[6]), - x7 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[7]), - x8 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[8]), - x9 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[9]), - x10 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[10]), - x11 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[11]), - x12 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[12]), - x13 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[13]), - x14 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[14]), - x15 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[15]), - x16 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[16]), - x17 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[17]), - x18 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[18]), - x19 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[19]), - x20 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[20]), - x21 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[21]), - x22 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[22]), - x23 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[23]), - x24 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[24]), - x25 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[25]), - x26 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[26]), - x27 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[27]), - x28 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[28]), - x29 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[29]), - x30 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[30]), - x31 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[31]), - f0 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[0]), - f1 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[1]), - f2 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[2]), - f3 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[3]), - f4 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[4]), - f5 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[5]), - f6 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[6]), - f7 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[7]), - f8 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[8]), - f9 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[9]), - f10 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[10]), - f11 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[11]), - f12 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[12]), - f13 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[13]), - f14 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[14]), - f15 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[15]), - f16 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[16]), - f17 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[17]), - f18 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[18]), - f19 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[19]), - f20 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[20]), - f21 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[21]), - f22 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[22]), - f23 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[23]), - f24 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[24]), - f25 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[25]), - f26 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[26]), - f27 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[27]), - f28 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[28]), - f29 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[29]), - f30 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[30]), - f31 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[31]), - pc = PMA_SHADOW_STATE_START + offsetof(shadow_state, pc), - fcsr = PMA_SHADOW_STATE_START + offsetof(shadow_state, fcsr), - mvendorid = PMA_SHADOW_STATE_START + offsetof(shadow_state, mvendorid), - marchid = PMA_SHADOW_STATE_START + offsetof(shadow_state, marchid), - mimpid = PMA_SHADOW_STATE_START + offsetof(shadow_state, mimpid), - mcycle = PMA_SHADOW_STATE_START + offsetof(shadow_state, mcycle), - icycleinstret = PMA_SHADOW_STATE_START + offsetof(shadow_state, icycleinstret), - mstatus = PMA_SHADOW_STATE_START + offsetof(shadow_state, mstatus), - mtvec = PMA_SHADOW_STATE_START + offsetof(shadow_state, mtvec), - mscratch = PMA_SHADOW_STATE_START + offsetof(shadow_state, mscratch), - mepc = PMA_SHADOW_STATE_START + offsetof(shadow_state, mepc), - mcause = PMA_SHADOW_STATE_START + offsetof(shadow_state, mcause), - mtval = PMA_SHADOW_STATE_START + offsetof(shadow_state, mtval), - misa = PMA_SHADOW_STATE_START + offsetof(shadow_state, misa), - mie = PMA_SHADOW_STATE_START + offsetof(shadow_state, mie), - mip = PMA_SHADOW_STATE_START + offsetof(shadow_state, mip), - medeleg = PMA_SHADOW_STATE_START + offsetof(shadow_state, medeleg), - mideleg = PMA_SHADOW_STATE_START + offsetof(shadow_state, mideleg), - mcounteren = PMA_SHADOW_STATE_START + offsetof(shadow_state, mcounteren), - menvcfg = PMA_SHADOW_STATE_START + offsetof(shadow_state, menvcfg), - stvec = PMA_SHADOW_STATE_START + offsetof(shadow_state, stvec), - sscratch = PMA_SHADOW_STATE_START + offsetof(shadow_state, sscratch), - sepc = PMA_SHADOW_STATE_START + offsetof(shadow_state, sepc), - scause = PMA_SHADOW_STATE_START + offsetof(shadow_state, scause), - stval = PMA_SHADOW_STATE_START + offsetof(shadow_state, stval), - satp = PMA_SHADOW_STATE_START + offsetof(shadow_state, satp), - scounteren = PMA_SHADOW_STATE_START + offsetof(shadow_state, scounteren), - senvcfg = PMA_SHADOW_STATE_START + offsetof(shadow_state, senvcfg), - ilrsc = PMA_SHADOW_STATE_START + offsetof(shadow_state, ilrsc), - iprv = PMA_SHADOW_STATE_START + offsetof(shadow_state, iprv), - iflags_X = PMA_SHADOW_STATE_START + offsetof(shadow_state, iflags_X), - iflags_Y = PMA_SHADOW_STATE_START + offsetof(shadow_state, iflags_Y), - iflags_H = PMA_SHADOW_STATE_START + offsetof(shadow_state, iflags_H), - iunrep = PMA_SHADOW_STATE_START + offsetof(shadow_state, iunrep), - clint_mtimecmp = PMA_SHADOW_STATE_START + offsetof(shadow_state, clint_mtimecmp), - plic_girqpend = PMA_SHADOW_STATE_START + offsetof(shadow_state, plic_girqpend), - plic_girqsrvd = PMA_SHADOW_STATE_START + offsetof(shadow_state, plic_girqsrvd), - htif_tohost = PMA_SHADOW_STATE_START + offsetof(shadow_state, htif_tohost), - htif_fromhost = PMA_SHADOW_STATE_START + offsetof(shadow_state, htif_fromhost), - htif_ihalt = PMA_SHADOW_STATE_START + offsetof(shadow_state, htif_ihalt), - htif_iconsole = PMA_SHADOW_STATE_START + offsetof(shadow_state, htif_iconsole), - htif_iyield = PMA_SHADOW_STATE_START + offsetof(shadow_state, htif_iyield), + x0 = static_cast(shadow_state_what::x0), + x1 = static_cast(shadow_state_what::x1), + x2 = static_cast(shadow_state_what::x2), + x3 = static_cast(shadow_state_what::x3), + x4 = static_cast(shadow_state_what::x4), + x5 = static_cast(shadow_state_what::x5), + x6 = static_cast(shadow_state_what::x6), + x7 = static_cast(shadow_state_what::x7), + x8 = static_cast(shadow_state_what::x8), + x9 = static_cast(shadow_state_what::x9), + x10 = static_cast(shadow_state_what::x10), + x11 = static_cast(shadow_state_what::x11), + x12 = static_cast(shadow_state_what::x12), + x13 = static_cast(shadow_state_what::x13), + x14 = static_cast(shadow_state_what::x14), + x15 = static_cast(shadow_state_what::x15), + x16 = static_cast(shadow_state_what::x16), + x17 = static_cast(shadow_state_what::x17), + x18 = static_cast(shadow_state_what::x18), + x19 = static_cast(shadow_state_what::x19), + x20 = static_cast(shadow_state_what::x20), + x21 = static_cast(shadow_state_what::x21), + x22 = static_cast(shadow_state_what::x22), + x23 = static_cast(shadow_state_what::x23), + x24 = static_cast(shadow_state_what::x24), + x25 = static_cast(shadow_state_what::x25), + x26 = static_cast(shadow_state_what::x26), + x27 = static_cast(shadow_state_what::x27), + x28 = static_cast(shadow_state_what::x28), + x29 = static_cast(shadow_state_what::x29), + x30 = static_cast(shadow_state_what::x30), + x31 = static_cast(shadow_state_what::x31), + f0 = static_cast(shadow_state_what::f0), + f1 = static_cast(shadow_state_what::f1), + f2 = static_cast(shadow_state_what::f2), + f3 = static_cast(shadow_state_what::f3), + f4 = static_cast(shadow_state_what::f4), + f5 = static_cast(shadow_state_what::f5), + f6 = static_cast(shadow_state_what::f6), + f7 = static_cast(shadow_state_what::f7), + f8 = static_cast(shadow_state_what::f8), + f9 = static_cast(shadow_state_what::f9), + f10 = static_cast(shadow_state_what::f10), + f11 = static_cast(shadow_state_what::f11), + f12 = static_cast(shadow_state_what::f12), + f13 = static_cast(shadow_state_what::f13), + f14 = static_cast(shadow_state_what::f14), + f15 = static_cast(shadow_state_what::f15), + f16 = static_cast(shadow_state_what::f16), + f17 = static_cast(shadow_state_what::f17), + f18 = static_cast(shadow_state_what::f18), + f19 = static_cast(shadow_state_what::f19), + f20 = static_cast(shadow_state_what::f20), + f21 = static_cast(shadow_state_what::f21), + f22 = static_cast(shadow_state_what::f22), + f23 = static_cast(shadow_state_what::f23), + f24 = static_cast(shadow_state_what::f24), + f25 = static_cast(shadow_state_what::f25), + f26 = static_cast(shadow_state_what::f26), + f27 = static_cast(shadow_state_what::f27), + f28 = static_cast(shadow_state_what::f28), + f29 = static_cast(shadow_state_what::f29), + f30 = static_cast(shadow_state_what::f30), + f31 = static_cast(shadow_state_what::f31), + pc = static_cast(shadow_state_what::pc), + fcsr = static_cast(shadow_state_what::fcsr), + mvendorid = static_cast(shadow_state_what::mvendorid), + marchid = static_cast(shadow_state_what::marchid), + mimpid = static_cast(shadow_state_what::mimpid), + mcycle = static_cast(shadow_state_what::mcycle), + icycleinstret = static_cast(shadow_state_what::icycleinstret), + mstatus = static_cast(shadow_state_what::mstatus), + mtvec = static_cast(shadow_state_what::mtvec), + mscratch = static_cast(shadow_state_what::mscratch), + mepc = static_cast(shadow_state_what::mepc), + mcause = static_cast(shadow_state_what::mcause), + mtval = static_cast(shadow_state_what::mtval), + misa = static_cast(shadow_state_what::misa), + mie = static_cast(shadow_state_what::mie), + mip = static_cast(shadow_state_what::mip), + medeleg = static_cast(shadow_state_what::medeleg), + mideleg = static_cast(shadow_state_what::mideleg), + mcounteren = static_cast(shadow_state_what::mcounteren), + menvcfg = static_cast(shadow_state_what::menvcfg), + stvec = static_cast(shadow_state_what::stvec), + sscratch = static_cast(shadow_state_what::sscratch), + sepc = static_cast(shadow_state_what::sepc), + scause = static_cast(shadow_state_what::scause), + stval = static_cast(shadow_state_what::stval), + satp = static_cast(shadow_state_what::satp), + scounteren = static_cast(shadow_state_what::scounteren), + senvcfg = static_cast(shadow_state_what::senvcfg), + ilrsc = static_cast(shadow_state_what::ilrsc), + iprv = static_cast(shadow_state_what::iprv), + iflags_X = static_cast(shadow_state_what::iflags_X), + iflags_Y = static_cast(shadow_state_what::iflags_Y), + iflags_H = static_cast(shadow_state_what::iflags_H), + iunrep = static_cast(shadow_state_what::iunrep), + clint_mtimecmp = static_cast(shadow_state_what::clint_mtimecmp), + plic_girqpend = static_cast(shadow_state_what::plic_girqpend), + plic_girqsrvd = static_cast(shadow_state_what::plic_girqsrvd), + htif_tohost = static_cast(shadow_state_what::htif_tohost), + htif_fromhost = static_cast(shadow_state_what::htif_fromhost), + htif_ihalt = static_cast(shadow_state_what::htif_ihalt), + htif_iconsole = static_cast(shadow_state_what::htif_iconsole), + htif_iyield = static_cast(shadow_state_what::htif_iyield), first_ = x0, last_ = htif_iyield, - uarch_halt_flag = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, halt_flag), - uarch_cycle = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, cycle), - uarch_pc = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, pc), - uarch_x0 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[0]), - uarch_x1 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[1]), - uarch_x2 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[2]), - uarch_x3 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[3]), - uarch_x4 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[4]), - uarch_x5 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[5]), - uarch_x6 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[6]), - uarch_x7 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[7]), - uarch_x8 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[8]), - uarch_x9 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[9]), - uarch_x10 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[10]), - uarch_x11 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[11]), - uarch_x12 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[12]), - uarch_x13 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[13]), - uarch_x14 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[14]), - uarch_x15 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[15]), - uarch_x16 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[16]), - uarch_x17 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[17]), - uarch_x18 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[18]), - uarch_x19 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[19]), - uarch_x20 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[20]), - uarch_x21 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[21]), - uarch_x22 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[22]), - uarch_x23 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[23]), - uarch_x24 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[24]), - uarch_x25 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[25]), - uarch_x26 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[26]), - uarch_x27 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[27]), - uarch_x28 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[28]), - uarch_x29 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[29]), - uarch_x30 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[30]), - uarch_x31 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[31]), + uarch_halt_flag = static_cast(shadow_uarch_state_what::uarch_halt_flag), + uarch_cycle = static_cast(shadow_uarch_state_what::uarch_cycle), + uarch_pc = static_cast(shadow_uarch_state_what::uarch_pc), + uarch_x0 = static_cast(shadow_uarch_state_what::uarch_x0), + uarch_x1 = static_cast(shadow_uarch_state_what::uarch_x1), + uarch_x2 = static_cast(shadow_uarch_state_what::uarch_x2), + uarch_x3 = static_cast(shadow_uarch_state_what::uarch_x3), + uarch_x4 = static_cast(shadow_uarch_state_what::uarch_x4), + uarch_x5 = static_cast(shadow_uarch_state_what::uarch_x5), + uarch_x6 = static_cast(shadow_uarch_state_what::uarch_x6), + uarch_x7 = static_cast(shadow_uarch_state_what::uarch_x7), + uarch_x8 = static_cast(shadow_uarch_state_what::uarch_x8), + uarch_x9 = static_cast(shadow_uarch_state_what::uarch_x9), + uarch_x10 = static_cast(shadow_uarch_state_what::uarch_x10), + uarch_x11 = static_cast(shadow_uarch_state_what::uarch_x11), + uarch_x12 = static_cast(shadow_uarch_state_what::uarch_x12), + uarch_x13 = static_cast(shadow_uarch_state_what::uarch_x13), + uarch_x14 = static_cast(shadow_uarch_state_what::uarch_x14), + uarch_x15 = static_cast(shadow_uarch_state_what::uarch_x15), + uarch_x16 = static_cast(shadow_uarch_state_what::uarch_x16), + uarch_x17 = static_cast(shadow_uarch_state_what::uarch_x17), + uarch_x18 = static_cast(shadow_uarch_state_what::uarch_x18), + uarch_x19 = static_cast(shadow_uarch_state_what::uarch_x19), + uarch_x20 = static_cast(shadow_uarch_state_what::uarch_x20), + uarch_x21 = static_cast(shadow_uarch_state_what::uarch_x21), + uarch_x22 = static_cast(shadow_uarch_state_what::uarch_x22), + uarch_x23 = static_cast(shadow_uarch_state_what::uarch_x23), + uarch_x24 = static_cast(shadow_uarch_state_what::uarch_x24), + uarch_x25 = static_cast(shadow_uarch_state_what::uarch_x25), + uarch_x26 = static_cast(shadow_uarch_state_what::uarch_x26), + uarch_x27 = static_cast(shadow_uarch_state_what::uarch_x27), + uarch_x28 = static_cast(shadow_uarch_state_what::uarch_x28), + uarch_x29 = static_cast(shadow_uarch_state_what::uarch_x29), + uarch_x30 = static_cast(shadow_uarch_state_what::uarch_x30), + uarch_x31 = static_cast(shadow_uarch_state_what::uarch_x31), uarch_first_ = uarch_halt_flag, uarch_last_ = uarch_x31, + // Something unkonwn + unknown_ = UINT64_C(1) << 63, // Outside of RISC-V address space + // Views of registers htif_tohost_dev, htif_tohost_cmd, @@ -185,15 +187,58 @@ enum class machine_reg : uint64_t { htif_fromhost_cmd, htif_fromhost_reason, htif_fromhost_data, - unknown_, + view_first_ = htif_tohost_dev, + view_last_ = htif_fromhost_data, + }; -constexpr uint64_t machine_reg_address(machine_reg reg, int i = 0) { - return static_cast(reg) + (i * sizeof(uint64_t)); +static constexpr uint64_t machine_reg_address(machine_reg reg, int i = 0) { + return static_cast(reg) + i * sizeof(uint64_t); +} + +static constexpr machine_reg machine_reg_enum(machine_reg reg, int i) { + return static_cast(static_cast(reg) + i * sizeof(uint64_t)); +} + +static constexpr machine_reg machine_reg_enum(shadow_state_what reg) { + return static_cast(reg); +} + +static constexpr machine_reg machine_reg_enum(shadow_uarch_state_what reg) { + return static_cast(reg); } -constexpr machine_reg machine_reg_enum(machine_reg reg, int i) { - return static_cast(static_cast(reg) + (i * sizeof(uint64_t))); +static constexpr const char *machine_reg_get_name(machine_reg reg) { + const auto ureg = static_cast(reg); + if (ureg >= static_cast(machine_reg::first_) && ureg <= static_cast(machine_reg::last_)) { + return shadow_state_get_what_name(static_cast(reg)); + } + if (ureg >= static_cast(machine_reg::uarch_first_) && + ureg <= static_cast(machine_reg::uarch_last_)) { + return shadow_uarch_state_get_what_name(static_cast(reg)); + } + switch (reg) { + case machine_reg::htif_tohost_dev: + return "htif.tohost_dev"; + case machine_reg::htif_tohost_cmd: + return "htif.tohost_cmd"; + case machine_reg::htif_tohost_reason: + return "htif.tohost_reason"; + case machine_reg::htif_tohost_data: + return "htif.tohost_data"; + case machine_reg::htif_fromhost_dev: + return "htif.fromhost_dev"; + case machine_reg::htif_fromhost_cmd: + return "htif.fromhost_cmd"; + case machine_reg::htif_fromhost_reason: + return "htif.fromhost_reason"; + case machine_reg::htif_fromhost_data: + return "htif.fromhost_data"; + case machine_reg::unknown_: + [[fallthrough]]; + default: + return "unknown"; + } } static_assert(machine_reg_address(machine_reg::uarch_first_) > machine_reg_address(machine_reg::last_)); diff --git a/src/machine.cpp b/src/machine.cpp index f18acd299..b3f101340 100644 --- a/src/machine.cpp +++ b/src/machine.cpp @@ -62,20 +62,22 @@ #include "shadow-state.h" #include "shadow-tlb-factory.h" #include "shadow-tlb.h" +#include "shadow-uarch-state-factory.h" #include "shadow-uarch-state.h" #include "state-access.h" #include "strict-aliasing.h" #include "tlb.h" #include "translate-virtual-address.h" -#include "uarch-pristine-state-hash.h" -#ifdef OKUARCH +#include "uarch-config.h" +#include "uarch-constants.h" #include "uarch-interpret.h" +#include "uarch-pristine-state-hash.h" +#include "uarch-pristine.h" #include "uarch-record-state-access.h" #include "uarch-replay-state-access.h" #include "uarch-reset-state.h" #include "uarch-state-access.h" #include "uarch-step.h" -#endif #include "unique-c-ptr.h" #include "virtio-console.h" #include "virtio-device.h" @@ -202,6 +204,7 @@ pma_entry &machine::register_pma_entry(pma_entry &&pma) { static bool DID_is_protected(PMA_ISTART_DID DID) { switch (DID) { + case PMA_ISTART_DID::memory: case PMA_ISTART_DID::flash_drive: case PMA_ISTART_DID::cmio_rx_buffer: case PMA_ISTART_DID::cmio_tx_buffer: @@ -215,7 +218,7 @@ void machine::replace_memory_range(const memory_range_config &range) { for (auto &pma : m_s.pmas) { if (pma.get_start() == range.start && pma.get_length() == range.length) { const auto curr = pma.get_istart_DID(); - if (DID_is_protected(curr)) { + if (pma.get_length() == 0 || DID_is_protected(curr)) { throw std::invalid_argument{"attempt to replace a protected range "s + pma.get_description()}; } // replace range preserving original flags @@ -227,78 +230,65 @@ void machine::replace_memory_range(const memory_range_config &range) { } void machine::init_tlb() { - for (auto use : {TLB_CODE, TLB_READ, TLB_WRITE}) { - auto &hot_set = m_s.tlb.hot[use]; - auto &cold_set = m_s.tlb.cold[use]; + for (auto set_index : {TLB_CODE, TLB_READ, TLB_WRITE}) { for (uint64_t slot_index = 0; slot_index < TLB_SET_SIZE; ++slot_index) { - auto &hot_slot = hot_set[slot_index]; - hot_slot.vaddr_page = TLB_INVALID_PAGE; - hot_slot.vh_offset = host_addr{}; - auto &cold_slot = cold_set[slot_index]; - cold_slot.pma_index = TLB_INVALID_PMA_INDEX; + write_tlb(set_index, slot_index, TLB_INVALID_PAGE, host_addr{}, TLB_INVALID_PMA_INDEX); } } } void machine::init_tlb(const shadow_tlb_state &shadow_tlb) { - const char *err_prefix = "stored TLB is corrupted: "; - for (auto use : {TLB_CODE, TLB_READ, TLB_WRITE}) { - auto &hot_set = m_s.tlb.hot[use]; - auto &cold_set = m_s.tlb.cold[use]; - auto &shadow_set = shadow_tlb[use]; + for (auto set_index : {TLB_CODE, TLB_READ, TLB_WRITE}) { for (uint64_t slot_index = 0; slot_index < TLB_SET_SIZE; ++slot_index) { - // copy from shadow to live TLB - auto &shadow_slot = shadow_set[slot_index]; - auto &cold_slot = cold_set[slot_index]; - const auto pma_index = cold_slot.pma_index = shadow_slot.pma_index; - auto &hot_slot = hot_set[slot_index]; - const auto vaddr_page = hot_slot.vaddr_page = shadow_slot.vaddr_page; - // check consistency of slot, if valid, and compute live host address offset - if (shadow_slot.vaddr_page != TLB_INVALID_PAGE) { - // virtual page address must be page-aligned - if (shadow_slot.vaddr_page & PAGE_OFFSET_MASK) { - throw std::invalid_argument{err_prefix + "invalid vaddr_page in TLB slot"s}; - } - // PMA index cannot be outside pmas array - if (pma_index >= m_s.pmas.size()) { - throw std::invalid_argument{err_prefix + "invalid pma_index in active TLB slot"s}; - } - // PMA at index must be a non-empty memory PMA - const auto &pma = m_s.pmas[pma_index]; - if (pma.get_length() == 0 || !pma.get_istart_M()) { - throw std::invalid_argument{err_prefix + "invalid pma_index in active TLB slot"s}; - } - auto paddr_page = shadow_slot.vaddr_page + shadow_slot.vp_offset; - // translated physical page address must be page-aligned - if (paddr_page & PAGE_OFFSET_MASK) { - throw std::invalid_argument{err_prefix + "invalid vp_offset in active TLB slot"s}; - } - // translated physical page address must fall entirely into designated PMA - if (paddr_page < pma.get_start() || paddr_page - pma.get_start() > pma.get_length() - PMA_PAGE_SIZE) { - throw std::invalid_argument{err_prefix + "invalid vp_offset in active TLB slot"s}; - } - hot_slot.vh_offset = get_host_addr(paddr_page, pma_index) - vaddr_page; - } else { - if (shadow_slot.vp_offset != 0) { - throw std::invalid_argument{err_prefix + "invalid vp_offset in empty TLB slot"s}; - } - if (pma_index != TLB_INVALID_PMA_INDEX) { - throw std::invalid_argument{err_prefix + "invalid pma_index in empty TLB slot"s}; - } - // We never leave garbage behind, even in invalid slots - hot_slot.vh_offset = host_addr{}; - } + const auto vaddr_page = shadow_tlb[set_index][slot_index].vaddr_page; + const auto vp_offset = shadow_tlb[set_index][slot_index].vp_offset; + const auto pma_index = shadow_tlb[set_index][slot_index].pma_index; + check_shadow_tlb(set_index, slot_index, vaddr_page, vp_offset, pma_index, "stored TLB is corrupt: "s); + write_shadow_tlb(set_index, slot_index, vaddr_page, vp_offset, pma_index); } } } -machine::machine(const machine_config &c, const machine_runtime_config &r) : m_c{c}, m_uarch{c.uarch}, m_r{r} { +void machine::init_uarch(const uarch_config &config) { + using reg = machine_reg; + write_reg(reg::uarch_pc, config.processor.pc); + write_reg(reg::uarch_cycle, config.processor.cycle); + write_reg(reg::uarch_halt_flag, config.processor.halt_flag); + // General purpose registers + for (int i = 1; i < UARCH_X_REG_COUNT; i++) { + write_reg(machine_reg_enum(reg::uarch_x0, i), config.processor.x[i]); + } + // Register shadow state + m_us.shadow_state = make_shadow_uarch_state_pma_entry(PMA_SHADOW_UARCH_STATE_START, PMA_SHADOW_UARCH_STATE_LENGTH); + // Register RAM + constexpr auto ram_description = "uarch RAM"; + if (!config.ram.image_filename.empty()) { + // Load RAM image from file + m_us.ram = make_callocd_memory_pma_entry(ram_description, PMA_UARCH_RAM_START, UARCH_RAM_LENGTH, + config.ram.image_filename) + .set_flags(m_ram_flags); + } else { + // Load embedded pristine RAM image + m_us.ram = make_callocd_memory_pma_entry(ram_description, PMA_UARCH_RAM_START, PMA_UARCH_RAM_LENGTH) + .set_flags(m_ram_flags); + if (uarch_pristine_ram_len > m_us.ram.get_length()) { + throw std::runtime_error("embedded uarch RAM image does not fit in uarch ram PMA"); + } + memcpy(m_us.ram.get_memory().get_host_memory(), uarch_pristine_ram, uarch_pristine_ram_len); + } +} + +// ??D It is best to leave the std::move() on r because it may one day be necessary! +// NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) +machine::machine(machine_config c, machine_runtime_config r) : m_c{std::move(c)}, m_r{std::move(r)} { + + init_uarch(m_c.uarch); if (m_c.processor.marchid == UINT64_C(-1)) { m_c.processor.marchid = MARCHID_INIT; } - if (m_c.processor.marchid != MARCHID_INIT && !r.skip_version_check) { + if (m_c.processor.marchid != MARCHID_INIT && !m_r.skip_version_check) { throw std::invalid_argument{"marchid mismatch, emulator version is incompatible"}; } @@ -306,7 +296,7 @@ machine::machine(const machine_config &c, const machine_runtime_config &r) : m_c m_c.processor.mvendorid = MVENDORID_INIT; } - if (m_c.processor.mvendorid != MVENDORID_INIT && !r.skip_version_check) { + if (m_c.processor.mvendorid != MVENDORID_INIT && !m_r.skip_version_check) { throw std::invalid_argument{"mvendorid mismatch, emulator version is incompatible"}; } @@ -314,11 +304,11 @@ machine::machine(const machine_config &c, const machine_runtime_config &r) : m_c m_c.processor.mimpid = MIMPID_INIT; } - if (m_c.processor.mimpid != MIMPID_INIT && !r.skip_version_check) { + if (m_c.processor.mimpid != MIMPID_INIT && !m_r.skip_version_check) { throw std::invalid_argument{"mimpid mismatch, emulator version is incompatible"}; } - m_s.soft_yield = r.soft_yield; + m_s.soft_yield = m_r.soft_yield; // General purpose registers for (int i = 1; i < X_REG_COUNT; i++) { @@ -511,14 +501,8 @@ machine::machine(const machine_config &c, const machine_runtime_config &r) : m_c dtb_init(m_c, dtb.get_memory().get_host_memory(), PMA_DTB_LENGTH); } - // Include machine PMAs in set considered by the Merkle tree. - for (auto &pma : m_s.pmas) { - m_merkle_pmas.push_back(&pma); - } - // Last, add empty sentinels until we reach capacity (need at least one sentinel) register_pma_entry(make_empty_pma_entry("sentinel"s, 0, 0)); - // NOLINTNEXTLINE(readability-static-accessed-through-instance) if (m_s.pmas.capacity() != PMA_MAX) { throw std::logic_error{"PMAs array must be able to hold PMA_MAX entries"}; @@ -530,12 +514,6 @@ machine::machine(const machine_config &c, const machine_runtime_config &r) : m_c // Populate shadow PMAs populate_shadow_pmas_state(m_s.pmas, shadow_pmas); - // Include uarch PMAs in set considered by Merkle tree - m_merkle_pmas.push_back(&m_uarch.get_state().shadow_state); - m_merkle_pmas.push_back(&m_uarch.get_state().ram); - // Last, add sentinel PMA - m_merkle_pmas.push_back(&m_s.empty_pma); - // Initialize TLB device. // This must be done after all PMA entries are already registered, so we can lookup page addresses if (!m_c.tlb.image_filename.empty()) { @@ -557,6 +535,22 @@ machine::machine(const machine_config &c, const machine_runtime_config &r) : m_c } os_silence_putchar(m_r.htif.no_console_putchar); + // Disable SIGPIPE handler, because this signal can be raised and terminate the emulator process + // when calling write() on closed file descriptors. + // This can happen with the stdout console file descriptors or network file descriptors. + os_disable_sigpipe(); + + // Include machine PMAs in set considered by the Merkle tree. + for (auto &pma : m_s.pmas) { + if (!pma.get_istart_E()) { + m_merkle_pmas.push_back(&pma); + } + } + m_merkle_pmas.push_back(&m_us.shadow_state); + m_merkle_pmas.push_back(&m_us.ram); + // Last, add sentinel PMA + m_merkle_pmas.push_back(&m_s.empty_pma); + // Initialize memory range descriptions returned by get_memory_ranges method for (const auto *pma : m_merkle_pmas) { if (pma->get_length() != 0) { @@ -568,11 +562,6 @@ machine::machine(const machine_config &c, const machine_runtime_config &r) : m_c // Sort it by increasing start address std::sort(m_mrds.begin(), m_mrds.end(), [](const machine_memory_range_descr &a, const machine_memory_range_descr &b) { return a.start < b.start; }); - - // Disable SIGPIPE handler, because this signal can be raised and terminate the emulator process - // when calling write() on closed file descriptors. - // This can happen with the stdout console file descriptors or network file descriptors. - os_disable_sigpipe(); } static void load_hash(const std::string &dir, machine::hash_type &h) { @@ -583,8 +572,10 @@ static void load_hash(const std::string &dir, machine::hash_type &h) { } } -machine::machine(const std::string &dir, const machine_runtime_config &r) : machine{machine_config::load(dir), r} { - if (r.skip_root_hash_check) { +// ??D It is best to leave the std::move() on r because it may one day be necessary! +// NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) +machine::machine(const std::string &dir, machine_runtime_config r) : machine{machine_config::load(dir), std::move(r)} { + if (m_r.skip_root_hash_check) { return; } hash_type hstored; @@ -647,10 +638,10 @@ const machine_runtime_config &machine::get_runtime_config() const { } /// \brief Changes the machine runtime config. -void machine::set_runtime_config(const machine_runtime_config &r) { - m_r = r; +void machine::set_runtime_config(machine_runtime_config r) { + m_r = std::move(r); // NOLINT(hicpp-move-const-arg,performance-move-const-arg) m_s.soft_yield = m_r.soft_yield; - os_silence_putchar(r.htif.no_console_putchar); + os_silence_putchar(m_r.htif.no_console_putchar); } machine_config machine::get_serialization_config() const { @@ -727,7 +718,7 @@ machine_config machine::get_serialization_config() const { c.cmio.rx_buffer.image_filename.clear(); c.cmio.tx_buffer.image_filename.clear(); c.uarch.processor.cycle = read_reg(reg::uarch_cycle); - c.uarch.processor.halt_flag = (read_reg(reg::uarch_halt_flag) != 0); + c.uarch.processor.halt_flag = read_reg(reg::uarch_halt_flag); c.uarch.processor.pc = read_reg(reg::uarch_pc); for (int i = 1; i < UARCH_X_REG_COUNT; i++) { c.uarch.processor.x[i] = read_reg(machine_reg_enum(reg::uarch_x0, i)); @@ -746,7 +737,7 @@ static void store_device_pma(const machine &m, const pma_entry &pma, const std:: page_start_in_range += PMA_PAGE_SIZE) { const unsigned char *page_data = nullptr; auto peek = pma.get_peek(); - if (!peek(pma, m, page_start_in_range, &page_data, scratch.get())) { + if (!peek(pma, m, page_start_in_range, PMA_PAGE_SIZE, &page_data, scratch.get())) { throw std::runtime_error{"peek failed"}; } if (page_data == nullptr) { @@ -794,19 +785,96 @@ host_addr machine::get_host_addr(uint64_t paddr, uint64_t pma_index) const { return host_addr{paddr} - get_hp_offset(pma_index); } -void machine::mark_dirty_page(host_addr haddr, uint64_t pma_index) { - auto paddr = get_paddr(haddr, pma_index); +void machine::mark_dirty_page(uint64_t paddr, uint64_t pma_index) { auto &pma = m_s.pmas[static_cast(pma_index)]; pma.mark_dirty_page(paddr - pma.get_start()); } +void machine::mark_dirty_page(host_addr haddr, uint64_t pma_index) { + auto paddr = get_paddr(haddr, pma_index); + mark_dirty_page(paddr, pma_index); +} + +uint64_t machine::read_shadow_tlb(TLB_set_index set_index, uint64_t slot_index, shadow_tlb_what reg) const { + switch (reg) { + case shadow_tlb_what::vaddr_page: + return m_s.tlb.hot[set_index][slot_index].vaddr_page; + case shadow_tlb_what::vp_offset: { + const auto vaddr_page = m_s.tlb.hot[set_index][slot_index].vaddr_page; + if (vaddr_page != TLB_INVALID_PAGE) { + const auto vh_offset = m_s.tlb.hot[set_index][slot_index].vh_offset; + const auto haddr_page = vaddr_page + vh_offset; + const auto pma_index = m_s.tlb.cold[set_index][slot_index].pma_index; + return get_paddr(haddr_page, pma_index) - vaddr_page; + } + return 0; + } + case shadow_tlb_what::pma_index: + return m_s.tlb.cold[set_index][slot_index].pma_index; + case shadow_tlb_what::zero_padding_: + return 0; + default: + throw std::domain_error{"unknown shadow TLB register"}; + } +} + +void machine::check_shadow_tlb(TLB_set_index set_index, uint64_t slot_index, uint64_t vaddr_page, uint64_t vp_offset, + uint64_t pma_index, const std::string &prefix) const { + if (set_index > TLB_LAST_) { + throw std::domain_error{prefix + "TLB set index is out of range"s}; + } + if (slot_index >= TLB_SET_SIZE) { + throw std::domain_error{prefix + "TLB slot index is out of range"s}; + } + if (vaddr_page != TLB_INVALID_PAGE) { + if (pma_index >= m_s.pmas.size()) { + throw std::domain_error{prefix + "pma_index is out of range"s}; + } + const auto &pma = m_s.pmas[pma_index]; + if (pma.get_length() == 0 || !pma.get_istart_M()) { + throw std::invalid_argument{prefix + "pma_index does not point to memory range"s}; + } + if ((vaddr_page & PAGE_OFFSET_MASK) != 0) { + throw std::invalid_argument{prefix + "vaddr_page is not aligned"s}; + } + const auto paddr_page = vaddr_page + vp_offset; + if ((paddr_page & PAGE_OFFSET_MASK) != 0) { + throw std::invalid_argument{prefix + "vp_offset is not aligned"s}; + } + const auto pma_end = pma.get_start() + (pma.get_length() - PMA_PAGE_SIZE); + if (paddr_page < pma.get_start() || paddr_page > pma_end) { + throw std::invalid_argument{prefix + "vp_offset is inconsistent with pma_index"s}; + } + } else if (pma_index != TLB_INVALID_PMA_INDEX || vp_offset != 0) { + throw std::domain_error{prefix + "inconsistent empty TLB slot"}; + } +} + +void machine::write_tlb(TLB_set_index set_index, uint64_t slot_index, uint64_t vaddr_page, host_addr vh_offset, + uint64_t pma_index) { + m_s.tlb.hot[set_index][slot_index].vaddr_page = vaddr_page; + m_s.tlb.hot[set_index][slot_index].vh_offset = vh_offset; + m_s.tlb.cold[set_index][slot_index].pma_index = pma_index; +} + +void machine::write_shadow_tlb(TLB_set_index set_index, uint64_t slot_index, uint64_t vaddr_page, uint64_t vp_offset, + uint64_t pma_index) { + if (vaddr_page != TLB_INVALID_PAGE) { + auto paddr_page = vaddr_page + vp_offset; + const auto vh_offset = get_host_addr(paddr_page, pma_index) - vaddr_page; + write_tlb(set_index, slot_index, vaddr_page, vh_offset, pma_index); + } else { + write_tlb(set_index, slot_index, TLB_INVALID_PAGE, host_addr{0}, TLB_INVALID_PMA_INDEX); + } +} + host_addr machine::get_hp_offset(uint64_t pma_index) const { if (pma_index >= m_s.pmas.size()) { - throw std::domain_error{"PMA is out of range"}; + throw std::domain_error{"PMA index is out of range (" + std::to_string(pma_index) + ")"}; } const auto &pma = m_s.pmas[static_cast(pma_index)]; if (!pma.get_istart_M()) { - throw std::domain_error{"PMA is not memory"}; + throw std::domain_error{"PMA is not memory (" + pma.get_description() + ")"}; } auto haddr = cast_ptr_to_host_addr(pma.get_memory().get_host_memory()); auto paddr = pma.get_start(); @@ -856,8 +924,8 @@ void machine::store_pmas(const machine_config &c, const std::string &dir) const } store_memory_pma(find_pma_entry(PMA_CMIO_RX_BUFFER_START), dir); store_memory_pma(find_pma_entry(PMA_CMIO_TX_BUFFER_START), dir); - if (!m_uarch.get_state().ram.get_istart_E()) { - store_memory_pma(m_uarch.get_state().ram, dir); + if (!m_us.ram.get_istart_E()) { + store_memory_pma(m_us.ram, dir); } } @@ -940,6 +1008,7 @@ machine::~machine() { } uint64_t machine::read_reg(reg r) const { + using reg = machine_reg; switch (r) { case reg::x0: return m_s.x[0]; @@ -1154,75 +1223,75 @@ uint64_t machine::read_reg(reg r) const { case reg::htif_iyield: return m_s.htif.iyield; case reg::uarch_x0: - return m_uarch.get_state().x[0]; + return m_us.x[0]; case reg::uarch_x1: - return m_uarch.get_state().x[1]; + return m_us.x[1]; case reg::uarch_x2: - return m_uarch.get_state().x[2]; + return m_us.x[2]; case reg::uarch_x3: - return m_uarch.get_state().x[3]; + return m_us.x[3]; case reg::uarch_x4: - return m_uarch.get_state().x[4]; + return m_us.x[4]; case reg::uarch_x5: - return m_uarch.get_state().x[5]; + return m_us.x[5]; case reg::uarch_x6: - return m_uarch.get_state().x[6]; + return m_us.x[6]; case reg::uarch_x7: - return m_uarch.get_state().x[7]; + return m_us.x[7]; case reg::uarch_x8: - return m_uarch.get_state().x[8]; + return m_us.x[8]; case reg::uarch_x9: - return m_uarch.get_state().x[9]; + return m_us.x[9]; case reg::uarch_x10: - return m_uarch.get_state().x[10]; + return m_us.x[10]; case reg::uarch_x11: - return m_uarch.get_state().x[11]; + return m_us.x[11]; case reg::uarch_x12: - return m_uarch.get_state().x[12]; + return m_us.x[12]; case reg::uarch_x13: - return m_uarch.get_state().x[13]; + return m_us.x[13]; case reg::uarch_x14: - return m_uarch.get_state().x[14]; + return m_us.x[14]; case reg::uarch_x15: - return m_uarch.get_state().x[15]; + return m_us.x[15]; case reg::uarch_x16: - return m_uarch.get_state().x[16]; + return m_us.x[16]; case reg::uarch_x17: - return m_uarch.get_state().x[17]; + return m_us.x[17]; case reg::uarch_x18: - return m_uarch.get_state().x[18]; + return m_us.x[18]; case reg::uarch_x19: - return m_uarch.get_state().x[19]; + return m_us.x[19]; case reg::uarch_x20: - return m_uarch.get_state().x[20]; + return m_us.x[20]; case reg::uarch_x21: - return m_uarch.get_state().x[21]; + return m_us.x[21]; case reg::uarch_x22: - return m_uarch.get_state().x[22]; + return m_us.x[22]; case reg::uarch_x23: - return m_uarch.get_state().x[23]; + return m_us.x[23]; case reg::uarch_x24: - return m_uarch.get_state().x[24]; + return m_us.x[24]; case reg::uarch_x25: - return m_uarch.get_state().x[25]; + return m_us.x[25]; case reg::uarch_x26: - return m_uarch.get_state().x[26]; + return m_us.x[26]; case reg::uarch_x27: - return m_uarch.get_state().x[27]; + return m_us.x[27]; case reg::uarch_x28: - return m_uarch.get_state().x[28]; + return m_us.x[28]; case reg::uarch_x29: - return m_uarch.get_state().x[29]; + return m_us.x[29]; case reg::uarch_x30: - return m_uarch.get_state().x[30]; + return m_us.x[30]; case reg::uarch_x31: - return m_uarch.get_state().x[31]; + return m_us.x[31]; case reg::uarch_pc: - return m_uarch.get_state().pc; + return m_us.pc; case reg::uarch_cycle: - return m_uarch.get_state().cycle; + return m_us.cycle; case reg::uarch_halt_flag: - return static_cast(m_uarch.get_state().halt_flag); + return m_us.halt_flag; case reg::htif_tohost_dev: return HTIF_DEV_FIELD(m_s.htif.tohost); case reg::htif_tohost_cmd: @@ -1564,106 +1633,106 @@ void machine::write_reg(reg w, uint64_t value) { case reg::uarch_x0: throw std::invalid_argument{"register is read-only"}; case reg::uarch_x1: - m_uarch.get_state().x[1] = value; + m_us.x[1] = value; break; case reg::uarch_x2: - m_uarch.get_state().x[2] = value; + m_us.x[2] = value; break; case reg::uarch_x3: - m_uarch.get_state().x[3] = value; + m_us.x[3] = value; break; case reg::uarch_x4: - m_uarch.get_state().x[4] = value; + m_us.x[4] = value; break; case reg::uarch_x5: - m_uarch.get_state().x[5] = value; + m_us.x[5] = value; break; case reg::uarch_x6: - m_uarch.get_state().x[6] = value; + m_us.x[6] = value; break; case reg::uarch_x7: - m_uarch.get_state().x[7] = value; + m_us.x[7] = value; break; case reg::uarch_x8: - m_uarch.get_state().x[8] = value; + m_us.x[8] = value; break; case reg::uarch_x9: - m_uarch.get_state().x[9] = value; + m_us.x[9] = value; break; case reg::uarch_x10: - m_uarch.get_state().x[10] = value; + m_us.x[10] = value; break; case reg::uarch_x11: - m_uarch.get_state().x[11] = value; + m_us.x[11] = value; break; case reg::uarch_x12: - m_uarch.get_state().x[12] = value; + m_us.x[12] = value; break; case reg::uarch_x13: - m_uarch.get_state().x[13] = value; + m_us.x[13] = value; break; case reg::uarch_x14: - m_uarch.get_state().x[14] = value; + m_us.x[14] = value; break; case reg::uarch_x15: - m_uarch.get_state().x[15] = value; + m_us.x[15] = value; break; case reg::uarch_x16: - m_uarch.get_state().x[16] = value; + m_us.x[16] = value; break; case reg::uarch_x17: - m_uarch.get_state().x[17] = value; + m_us.x[17] = value; break; case reg::uarch_x18: - m_uarch.get_state().x[18] = value; + m_us.x[18] = value; break; case reg::uarch_x19: - m_uarch.get_state().x[19] = value; + m_us.x[19] = value; break; case reg::uarch_x20: - m_uarch.get_state().x[20] = value; + m_us.x[20] = value; break; case reg::uarch_x21: - m_uarch.get_state().x[21] = value; + m_us.x[21] = value; break; case reg::uarch_x22: - m_uarch.get_state().x[22] = value; + m_us.x[22] = value; break; case reg::uarch_x23: - m_uarch.get_state().x[23] = value; + m_us.x[23] = value; break; case reg::uarch_x24: - m_uarch.get_state().x[24] = value; + m_us.x[24] = value; break; case reg::uarch_x25: - m_uarch.get_state().x[25] = value; + m_us.x[25] = value; break; case reg::uarch_x26: - m_uarch.get_state().x[26] = value; + m_us.x[26] = value; break; case reg::uarch_x27: - m_uarch.get_state().x[27] = value; + m_us.x[27] = value; break; case reg::uarch_x28: - m_uarch.get_state().x[28] = value; + m_us.x[28] = value; break; case reg::uarch_x29: - m_uarch.get_state().x[29] = value; + m_us.x[29] = value; break; case reg::uarch_x30: - m_uarch.get_state().x[30] = value; + m_us.x[30] = value; break; case reg::uarch_x31: - m_uarch.get_state().x[31] = value; + m_us.x[31] = value; break; case reg::uarch_pc: - m_uarch.get_state().pc = value; + m_us.pc = value; break; case reg::uarch_cycle: - m_uarch.get_state().cycle = value; + m_us.cycle = value; break; case reg::uarch_halt_flag: - m_uarch.get_state().halt_flag = static_cast(value); + m_us.halt_flag = value; break; case reg::htif_tohost_dev: m_s.htif.tohost = HTIF_REPLACE_DEV(m_s.htif.tohost, value); @@ -1746,7 +1815,7 @@ bool machine::verify_dirty_page_maps() const { const uint64_t page_address = pma.get_start() + page_start_in_range; if (pma.get_istart_M()) { const unsigned char *page_data = nullptr; - peek(pma, *this, page_start_in_range, &page_data, scratch.get()); + peek(pma, *this, page_start_in_range, PMA_PAGE_SIZE, &page_data, scratch.get()); hash_type stored; hash_type real; m_t.get_page_node_hash(page_address, stored); @@ -1812,7 +1881,7 @@ bool machine::update_merkle_tree() const { } // If the peek failed, or if it returned a page for update but // we failed updating it, the entire process failed - if (!peek(*pma, *this, page_start_in_range, &page_data, scratch.get())) { + if (!peek(*pma, *this, page_start_in_range, PMA_PAGE_SIZE, &page_data, scratch.get())) { return false; } if (page_data != nullptr) { @@ -1867,7 +1936,7 @@ bool machine::update_merkle_tree_page(uint64_t address) { m_t.begin_update(); const unsigned char *page_data = nullptr; auto peek = pma.get_peek(); - if (!peek(pma, *this, page_start_in_range, &page_data, scratch.get())) { + if (!peek(pma, *this, page_start_in_range, PMA_PAGE_SIZE, &page_data, scratch.get())) { m_t.end_update(h); return false; } @@ -1900,7 +1969,22 @@ void machine::get_root_hash(hash_type &hash) const { machine::hash_type machine::get_merkle_tree_node_hash(uint64_t address, int log2_size, skip_merkle_tree_update_t /* unused */) const { - return m_t.get_node_hash(address, log2_size); + if (log2_size < 0 || log2_size >= machine_merkle_tree::get_log2_root_size()) { + throw std::domain_error{"log2_size is out of bounds"}; + } + if (log2_size >= machine_merkle_tree::get_log2_page_size()) { + return m_t.get_node_hash(address, log2_size); + } + if ((address >> log2_size) << log2_size != address) { + throw std::domain_error{"address not aligned to log2_size"}; + } + const auto size = UINT64_C(1) << log2_size; + auto scratch = unique_calloc(size); + read_memory(address, scratch.get(), size); + machine_merkle_tree::hasher_type h; + machine_merkle_tree::hash_type hash; + get_merkle_tree_hash(h, scratch.get(), size, machine_merkle_tree::get_word_size(), hash); + return hash; } machine::hash_type machine::get_merkle_tree_node_hash(uint64_t address, int log2_size) const { @@ -1925,7 +2009,7 @@ machine_merkle_tree::proof_type machine::get_proof(uint64_t address, int log2_si } // Check target address alignment if ((address & ((~UINT64_C(0)) >> (64 - log2_size))) != 0) { - throw std::invalid_argument{"address not aligned to log2_size"}; + throw std::domain_error{"address not aligned to log2_size"}; } // If proof concerns range smaller than a page, we may need to rebuild part // of the proof from the contents of a page inside some PMA range. @@ -1947,7 +2031,7 @@ machine_merkle_tree::proof_type machine::get_proof(uint64_t address, int log2_si if (!pma.get_istart_E()) { const uint64_t page_start_in_range = (address - pma.get_start()) & (~(PMA_PAGE_SIZE - 1)); auto peek = pma.get_peek(); - if (!peek(pma, *this, page_start_in_range, &page_data, scratch.get())) { + if (!peek(pma, *this, page_start_in_range, PMA_PAGE_SIZE, &page_data, scratch.get())) { throw std::runtime_error{"PMA peek failed"}; } } @@ -1965,70 +2049,91 @@ machine_merkle_tree::proof_type machine::get_proof(uint64_t address, int log2_si return get_proof(address, log2_size, skip_merkle_tree_update); } -void machine::read_memory(uint64_t address, unsigned char *data, uint64_t length) const { +void machine::read_memory(uint64_t paddr, unsigned char *data, uint64_t length) const { if (length == 0) { return; } if (data == nullptr) { throw std::invalid_argument{"invalid data buffer"}; } - const pma_entry &pma = find_pma_entry(m_merkle_pmas, address, length); - if (pma.get_istart_M()) { - memcpy(data, pma.get_memory().get_host_memory() + (address - pma.get_start()), length); - return; - } - auto scratch = unique_calloc(PMA_PAGE_SIZE); - // relative request address inside pma - uint64_t shift = address - pma.get_start(); - // relative page address inside pma - constexpr const auto log2_page_size = PMA_constants::PMA_PAGE_SIZE_LOG2; - uint64_t page_address = (shift >> log2_page_size) << log2_page_size; - // relative request address inside page - shift -= page_address; - - const unsigned char *page_data = nullptr; - auto peek = pma.get_peek(); - - while (length != 0) { - const uint64_t bytes_to_write = std::min(length, PMA_PAGE_SIZE - shift); - // avoid copying to the intermediate buffer when getting the whole page - if (bytes_to_write == PMA_PAGE_SIZE) { - if (!peek(pma, *this, page_address, &page_data, data)) { + // Compute the distance between the initial paddr and the first page boundary + const uint64_t align_paddr = (paddr & PAGE_OFFSET_MASK) != 0 ? (paddr | PAGE_OFFSET_MASK) + 1 : paddr; + uint64_t align_length = align_paddr - paddr; + const uint64_t page_size = PMA_PAGE_SIZE; + align_length = (align_length == 0) ? page_size : align_length; + // First peek goes at most to the next page boundary, or up to length + uint64_t peek_length = std::min(align_length, length); + // The outer loop finds the PMA for all peeks performed by the inner loop + // The inner loop peeks at most min(page_size, length) from the PMA per iteration + // All peeks but the absolute first peek start at a page boundary. + // That first peek reads at most up to the next page boundary. + // So the inner loop iterations never cross page boundaries. + for (;;) { + const pma_entry &pma = find_pma_entry(m_merkle_pmas, paddr, peek_length); + const auto peek = pma.get_peek(); + const auto pma_start = pma.get_start(); + const auto pma_empty = pma.get_istart_E(); + const auto pma_length = pma.get_length(); + // If the PMA is empty, the inner loop will break after a single iteration. + // But it is safe to return pristine data for that one iteration, without even peeking. + // This is because the inner iteration never reads past a page boundary, and the next + // non-empty PMA starts at the earliest on the next page boundary after paddr. + for (;;) { + const unsigned char *peek_data = nullptr; + // If non-empty PMA, peek, otherwise leave peek_data as nullptr (i.e. pristine) + if (!pma_empty && !peek(pma, *this, paddr - pma_start, peek_length, &peek_data, data)) { throw std::runtime_error{"peek failed"}; } - if (page_data == nullptr) { - memset(data, 0, bytes_to_write); + // If the chunk is pristine, copy zero data to buffer + if (peek_data == nullptr) { + memset(data, 0, peek_length); + // If peek returned pointer to internal buffer, copy to data buffer + } else if (peek_data != data) { + memcpy(data, peek_data, peek_length); } - } else { - if (!peek(pma, *this, page_address, &page_data, scratch.get())) { - throw std::runtime_error{"peek failed"}; + // Otherwise, peek copied data straight into the data buffer + // If we read everything we wanted to read, we are done + length -= peek_length; + if (length == 0) { + return; + } + paddr += peek_length; + data += peek_length; + peek_length = std::min(page_size, length); + // If the PMA was empty, break to check if next read is in another PMA + if (pma_empty) { + break; } - if (page_data == nullptr) { - memset(data, 0, bytes_to_write); - } else { - memcpy(data, page_data + shift, bytes_to_write); + // If the next read does not fit in current PMA, break to get the next one + // There can be no overflow in the condition. + // Since the PMA is non-empty, (paddr-pma_start) >= 0. + // Moreover, pma_length >= page_size. + // Since, peek_length <= page_size, we get (pma_length-peek_length) >= 0. + if (paddr - pma_start >= pma_length - peek_length) { + break; } } - - page_address += PMA_PAGE_SIZE; - length -= bytes_to_write; - data += bytes_to_write; - shift = 0; } } -void machine::write_memory(uint64_t address, const unsigned char *data, uint64_t length) { +void machine::write_memory(uint64_t paddr, const unsigned char *data, uint64_t length) { if (length == 0) { return; } if (data == nullptr) { throw std::invalid_argument{"invalid data buffer"}; } - pma_entry &pma = find_pma_entry(m_merkle_pmas, address, length); + pma_entry &pma = find_pma_entry(m_merkle_pmas, paddr, length); + if (pma.get_istart_IO()) { + throw std::invalid_argument{"attempted write to device memory range"}; + } if (!pma.get_istart_M() || pma.get_istart_E()) { - throw std::invalid_argument{"address range not entirely in memory PMA"}; + throw std::invalid_argument{"address range not entirely in single memory range"}; + } + if (DID_is_protected(pma.get_istart_DID())) { + throw std::invalid_argument{"attempt to write to protected memory range"}; } - pma.write_memory(address, data, length); + pma.write_memory(paddr, data, length); } void machine::fill_memory(uint64_t address, uint8_t data, uint64_t length) { @@ -2116,29 +2221,15 @@ uint64_t machine::translate_virtual_address(uint64_t vaddr) { return paddr; } -uint64_t machine::read_word(uint64_t word_address) const { +uint64_t machine::read_word(uint64_t paddr) const { // Make sure address is aligned - if ((word_address & (PMA_WORD_SIZE - 1)) != 0) { - throw std::invalid_argument{"address not aligned"}; - } - const pma_entry &pma = find_pma_entry(word_address); - // ??D We should split peek into peek_word and peek_page - // for performance. On the other hand, this function - // will almost never be used, so one wonders if it is worth it... - auto scratch = unique_calloc(PMA_PAGE_SIZE); - const unsigned char *page_data = nullptr; - const uint64_t page_start_in_range = (word_address - pma.get_start()) & (~(PMA_PAGE_SIZE - 1)); - auto peek = pma.get_peek(); - if (!peek(pma, *this, page_start_in_range, &page_data, scratch.get())) { - throw std::invalid_argument{"peek failed"}; - } - // If peek returns a page, read from it - if (page_data != nullptr) { - const uint64_t word_start_in_range = (word_address - pma.get_start()) & (PMA_PAGE_SIZE - 1); - return aliased_aligned_read(page_data + word_start_in_range); - // Otherwise, page is always pristine + if ((paddr & (sizeof(uint64_t) - 1)) != 0) { + throw std::domain_error{"address not aligned"}; } - return 0; + // Use read_memory + alignas(sizeof(uint64_t)) std::array scratch{}; + read_memory(paddr, scratch.data(), scratch.size()); + return aliased_aligned_read(scratch.data()); } void machine::send_cmio_response(uint16_t reason, const unsigned char *data, uint64_t length) { @@ -2153,9 +2244,10 @@ access_log machine::log_send_cmio_response(uint16_t reason, const unsigned char access_log log(log_type); // Call send_cmio_response with the recording state accessor record_send_cmio_state_access a(*this, log); - a.push_bracket(bracket_type::begin, "send cmio response"); - cartesi::send_cmio_response(a, reason, data, length); - a.push_bracket(bracket_type::end, "send cmio response"); + { + [[maybe_unused]] auto note = a.make_scoped_note("send_cmio_response"); + cartesi::send_cmio_response(a, reason, data, length); + } // Verify access log before returning hash_type root_hash_after; update_merkle_tree(); @@ -2166,16 +2258,11 @@ access_log machine::log_send_cmio_response(uint16_t reason, const unsigned char void machine::verify_send_cmio_response(uint16_t reason, const unsigned char *data, uint64_t length, const hash_type &root_hash_before, const access_log &log, const hash_type &root_hash_after) { - // There must be at least one access in log - if (log.get_accesses().empty()) { - throw std::invalid_argument{"too few accesses in log"}; - } replay_send_cmio_state_access::context context(log, root_hash_before); // Verify all intermediate state transitions replay_send_cmio_state_access a(context); cartesi::send_cmio_response(a, reason, data, length); a.finish(); - // Make sure the access log ends at the same root hash as the state hash_type obtained_root_hash; a.get_root_hash(obtained_root_hash); @@ -2185,38 +2272,42 @@ void machine::verify_send_cmio_response(uint16_t reason, const unsigned char *da } void machine::reset_uarch() { -#if OKUARCH - uarch_state_access a(m_uarch.get_state(), get_state()); - uarch_reset_state(a); -#endif + using reg = machine_reg; + write_reg(reg::uarch_halt_flag, UARCH_HALT_FLAG_INIT); + write_reg(reg::uarch_pc, UARCH_PC_INIT); + write_reg(reg::uarch_cycle, UARCH_CYCLE_INIT); + // General purpose registers + for (int i = 1; i < UARCH_X_REG_COUNT; i++) { + write_reg(machine_reg_enum(reg::uarch_x0, i), UARCH_X_INIT); + } + // Load embedded pristine RAM image + if (uarch_pristine_ram_len > m_us.ram.get_length()) { + throw std::runtime_error("embedded uarch ram image does not fit in uarch ram pma"); + } + // Reset RAM to initial state + m_us.ram.fill_memory(m_us.ram.get_start(), 0, m_us.ram.get_length()); + m_us.ram.write_memory(m_us.ram.get_start(), uarch_pristine_ram, uarch_pristine_ram_len); } access_log machine::log_reset_uarch(const access_log::type &log_type) { -#if OKUARCH hash_type root_hash_before; get_root_hash(root_hash_before); // Call uarch_reset_state with a uarch_record_state_access object - uarch_record_state_access a(m_uarch.get_state(), *this, log_type); - a.push_bracket(bracket_type::begin, "reset uarch state"); - uarch_reset_state(a); - a.push_bracket(bracket_type::end, "reset uarch state"); + uarch_record_state_access a(m_us, *this, log_type); + { + [[maybe_unused]] auto note = a.make_scoped_note("reset_uarch_state"); + uarch_reset_state(a); + } // Verify access log before returning hash_type root_hash_after; update_merkle_tree(); get_root_hash(root_hash_after); verify_reset_uarch(root_hash_before, *a.get_log(), root_hash_after); return std::move(*a.get_log()); -#endif - return access_log{log_type}; } void machine::verify_reset_uarch(const hash_type &root_hash_before, const access_log &log, const hash_type &root_hash_after) { -#if OKUARCH - // There must be at least one access in log - if (log.get_accesses().empty()) { - throw std::invalid_argument{"too few accesses in log"}; - } // Verify all intermediate state transitions uarch_replay_state_access a(log, root_hash_before); uarch_reset_state(a); @@ -2227,50 +2318,35 @@ void machine::verify_reset_uarch(const hash_type &root_hash_before, const access if (obtained_root_hash != root_hash_after) { throw std::invalid_argument{"mismatch in root hash after replay"}; } -#endif - (void) root_hash_before; - (void) log; - (void) root_hash_after; } // Declaration of explicit instantiation in module uarch-step.cpp -#if OKUARCH extern template UArchStepStatus uarch_step(uarch_record_state_access &a); -#endif access_log machine::log_step_uarch(const access_log::type &log_type) { -#if OKUARCH - if (m_uarch.get_state().ram.get_istart_E()) { + if (m_us.ram.get_istart_E()) { throw std::runtime_error("microarchitecture RAM is not present"); } hash_type root_hash_before; get_root_hash(root_hash_before); // Call interpret with a logged state access object - uarch_record_state_access a(m_uarch.get_state(), *this, log_type); - a.push_bracket(bracket_type::begin, "step"); - uarch_step(a); - a.push_bracket(bracket_type::end, "step"); + uarch_record_state_access a(m_us, *this, log_type); + { + [[maybe_unused]] auto note = a.make_scoped_note("step"); + uarch_step(a); + } // Verify access log before returning hash_type root_hash_after; get_root_hash(root_hash_after); verify_step_uarch(root_hash_before, *a.get_log(), root_hash_after); return std::move(*a.get_log()); -#endif - return access_log{log_type}; } // Declaration of explicit instantiation in module uarch-step.cpp -#if OKUARCH extern template UArchStepStatus uarch_step(uarch_replay_state_access &a); -#endif void machine::verify_step_uarch(const hash_type &root_hash_before, const access_log &log, const hash_type &root_hash_after) { -#if OKUARCH - // There must be at least one access in log - if (log.get_accesses().empty()) { - throw std::invalid_argument{"too few accesses in log"}; - } // Verify all intermediate state transitions uarch_replay_state_access a(log, root_hash_before); uarch_step(a); @@ -2281,10 +2357,6 @@ void machine::verify_step_uarch(const hash_type &root_hash_before, const access_ if (obtained_root_hash != root_hash_after) { throw std::invalid_argument{"mismatch in root hash after replay"}; } -#endif - (void) root_hash_before; - (void) log; - (void) root_hash_after; } machine_config machine::get_default_config() { @@ -2293,18 +2365,14 @@ machine_config machine::get_default_config() { // NOLINTNEXTLINE(readability-convert-member-functions-to-static) uarch_interpreter_break_reason machine::run_uarch(uint64_t uarch_cycle_end) { -#if OKUARCH if (read_reg(reg::iunrep) != 0) { throw std::runtime_error("microarchitecture cannot be used with unreproducible machines"); } - if (m_uarch.get_state().ram.get_istart_E()) { + if (m_us.ram.get_istart_E()) { throw std::runtime_error("microarchitecture RAM is not present"); } - uarch_state_access a(m_uarch.get_state(), get_state()); + uarch_state_access a(*this); return uarch_interpret(a, uarch_cycle_end); -#endif - (void) uarch_cycle_end; - return uarch_interpreter_break_reason::uarch_halted; } interpreter_break_reason machine::log_step(uint64_t mcycle_count, const std::string &filename) { @@ -2350,7 +2418,8 @@ interpreter_break_reason machine::verify_step(const hash_type &root_hash_before, } interpreter_break_reason machine::run(uint64_t mcycle_end) { - if (mcycle_end < read_reg(reg::mcycle)) { + const auto mcycle = read_reg(reg::mcycle); + if (mcycle_end < mcycle) { throw std::invalid_argument{"mcycle is past"}; } const state_access a(*this); diff --git a/src/machine.h b/src/machine.h index 07ea6e104..31b7a7063 100644 --- a/src/machine.h +++ b/src/machine.h @@ -40,8 +40,9 @@ #include "os.h" #include "pma-constants.h" #include "pma.h" +#include "shadow-tlb.h" #include "uarch-interpret.h" -#include "uarch-machine.h" +#include "uarch-state.h" #include "virtio-device.h" namespace cartesi { @@ -58,18 +59,11 @@ constexpr skip_merkle_tree_update_t skip_merkle_tree_update; /// \brief Cartesi Machine implementation class machine final { private: - //??D Ideally, we would hold a unique_ptr to the state. This - // would allow us to remove the machine-state.h include and - // therefore hide its contents from anyone who includes only - // machine.h. Maybe the compiler can do a good job we we are - // not constantly going through the extra indirection. We - // should test this. - - mutable machine_state m_s; ///< Opaque machine state + mutable machine_state m_s; ///< Big machine state + mutable uarch_state m_us; ///< Microarchitecture state mutable machine_merkle_tree m_t; ///< Merkle tree of state std::vector m_merkle_pmas; ///< PMAs considered by the Merkle tree: from big machine and uarch machine_config m_c; ///< Copy of initialization config - uarch_machine m_uarch; ///< Microarchitecture machine machine_runtime_config m_r; ///< Copy of initialization runtime config machine_memory_range_descrs m_mrds; ///< List of memory ranges returned by get_memory_ranges(). @@ -137,6 +131,10 @@ class machine final { /// \param shadow_tlb Shadow TLB loaded from disk void init_tlb(const shadow_tlb_state &shadow_tlb); + /// \brief Initializes microarchitecture + /// \param config Microarchitecture configuration + void init_uarch(const uarch_config &config); + public: /// \brief Type of hash using hash_type = machine_merkle_tree::hash_type; @@ -146,12 +144,12 @@ class machine final { /// \brief Constructor from machine configuration /// \param config Machine config to use instantiating machine /// \param runtime Runtime config to use with machine - explicit machine(const machine_config &config, const machine_runtime_config &runtime = {}); + explicit machine(machine_config config, machine_runtime_config runtime = {}); /// \brief Constructor from previously serialized directory /// \param directory Directory to load stored machine from /// \param runtime Runtime config to use with machine - explicit machine(const std::string &directory, const machine_runtime_config &runtime = {}); + explicit machine(const std::string &directory, machine_runtime_config runtime = {}); /// \brief Serialize entire state to directory /// \param directory Directory to store machine into @@ -235,6 +233,16 @@ class machine final { return m_s; } + /// \brief Returns uarch state for direct access. + uarch_state &get_uarch_state() { + return m_us; + } + + /// \brief Returns uarch state for direct read-only access. + const uarch_state &get_uarch_state() const { + return m_us; + } + /// \brief Returns a list of descriptions for all PMA entries registered in the machine, sorted by start machine_memory_range_descrs get_memory_ranges() const { return m_mrds; @@ -337,27 +345,25 @@ class machine final { static uint64_t get_reg_address(reg r); /// \brief Read the value of a word in the machine state. - /// \param address Word address (aligned to 64-bit boundary). + /// \param paddr Word address (aligned to 64-bit boundary). /// \returns The value of word at address. - /// \warning The current implementation of this function is very slow! - uint64_t read_word(uint64_t address) const; - - /// \brief Reads a chunk of data from the machine memory. - /// \param address Physical address to start reading. - /// \param data Receives chunk of memory. - /// \param length Size of chunk. - /// \details The entire chunk, from \p address to \p address + \p length must - /// be inside the same PMA region. - void read_memory(uint64_t address, unsigned char *data, uint64_t length) const; - - /// \brief Writes a chunk of data to the machine memory. - /// \param address Physical address to start writing. - /// \param data Source for chunk of data. - /// \param length Size of chunk. - /// \details The entire chunk, from \p address to \p address + \p length must - /// be inside the same PMA region. Moreover, this PMA must be a memory PMA, - /// and not a device PMA. - void write_memory(uint64_t address, const unsigned char *data, uint64_t length); + /// \details The word can be anywhere in the entire address space. + uint64_t read_word(uint64_t paddr) const; + + /// \brief Reads a chunk of data, by its target physical address and length. + /// \param paddr Target physical address to start reading from. + /// \param data Buffer that receives data to read. Must be at least \p length bytes long. + /// \param length Number of bytes to read from \p paddr to \p data. + /// \details The data can be anywhere in the entire address space. + void read_memory(uint64_t paddr, unsigned char *data, uint64_t length) const; + + /// \brief Writes a chunk of data to machine memory, by its target physical address and length. + /// \param paddr Target physical address to start writing to. + /// \param data Buffer that contains data to write. Must be at least \p length bytes long. + /// \param length Number of bytes to write starting from \p data to \p paddr. + /// \details Unlike read_memory(), the entire chunk of data, from \p paddr to \p paddr + \p length, + /// must reside entirely in the same memory range. Moreover, it cannot be mapped to a device. + void write_memory(uint64_t paddr, const unsigned char *data, uint64_t length); /// \brief Fills a memory range with a single byte. /// \param address Physical address to start filling. @@ -428,7 +434,7 @@ class machine final { /// \brief Changes the machine runtime config. /// \param range Configuration of the new memory range. /// \details Some runtime options cannot be changed. - void set_runtime_config(const machine_runtime_config &r); + void set_runtime_config(machine_runtime_config r); /// \brief Replaces a memory range. /// \param range Configuration of the new memory range. @@ -461,6 +467,45 @@ class machine final { /// \param pma_index Index of PMA where address falls void mark_dirty_page(host_addr haddr, uint64_t pma_index); + /// \brief Marks a page as dirty + /// \param paddr Target phyislcal address within page + /// \param pma_index Index of PMA where address falls + void mark_dirty_page(uint64_t paddr, uint64_t pma_index); + + /// \brief Updates a TLB slot + /// \param set_index TLB_CODE, TLB_READ, or TLB_WRITE + /// \param slot_index Index of slot to update + /// \param vaddr_page Virtual address of page to map + /// \param vh_offset Offset from target virtual addresses to host addresses within page + /// \param pma_index Index of PMA where address falls + void write_tlb(TLB_set_index set_index, uint64_t slot_index, uint64_t vaddr_page, host_addr vh_offset, + uint64_t pma_index); + + /// \brief Updates a TLB slot + /// \param set_index TLB_CODE, TLB_READ, or TLB_WRITE + /// \param slot_index Index of slot to update + /// \param vaddr_page Virtual address of page to map + /// \param vp_offset Offset from target virtual addresses to target physical addresses within page + /// \param pma_index Index of PMA where address falls + void write_shadow_tlb(TLB_set_index set_index, uint64_t slot_index, uint64_t vaddr_page, uint64_t vp_offset, + uint64_t pma_index); + + /// \brief Check consistency of TLB slot + /// \param set_index TLB_CODE, TLB_READ, or TLB_WRITE + /// \param slot_index Index of slot to update + /// \param vaddr_page Virtual address of page to map + /// \param vp_offset Offset from target virtual addresses to target physical addresses within page + /// \param pma_index Index of PMA where address falls + void check_shadow_tlb(TLB_set_index set_index, uint64_t slot_index, uint64_t vaddr_page, uint64_t vp_offset, + uint64_t pma_index, const std::string &prefix = "") const; + + /// \brief Reads a TLB register + /// \param set_index TLB_CODE, TLB_READ, or TLB_WRITE + /// \param slot_index Index of slot to read + /// \param reg Register to read from slot + /// \returns Value of register + uint64_t read_shadow_tlb(TLB_set_index set_index, uint64_t slot_index, shadow_tlb_what reg) const; + /// \brief Sends cmio response and returns an access log /// \param reason Reason for sending response. /// \param data Response data. diff --git a/src/mock-pma-entry.h b/src/mock-pma-entry.h index 883bd6725..75cda69cb 100644 --- a/src/mock-pma-entry.h +++ b/src/mock-pma-entry.h @@ -136,6 +136,10 @@ class mock_pma_entry { return m_flags.IR; } + PMA_ISTART_DID get_istart_DID() const { + return m_flags.DID; + } + const auto *get_driver() const { return m_driver; } diff --git a/src/plic-factory.cpp b/src/plic-factory.cpp index d0f139e3e..c68f7b6b1 100644 --- a/src/plic-factory.cpp +++ b/src/plic-factory.cpp @@ -14,27 +14,18 @@ // with this program (see COPYING). If not, see . // -#include "plic-factory.h" - #include +#include "plic-factory.h" #include "plic.h" #include "pma-constants.h" #include "pma.h" namespace cartesi { -/// \brief PLIC device peek callback. See ::pma_peek. -static bool plic_peek(const pma_entry &pma, const machine & /*m*/, uint64_t page_offset, - const unsigned char **page_data, unsigned char * /*context*/) { - // PLIC range can be represented as pristine because its state is already represented in shadow CSRs - *page_data = nullptr; - return (page_offset % PMA_PAGE_SIZE) == 0 && page_offset < pma.get_length(); -} - pma_entry make_plic_pma_entry(uint64_t start, uint64_t length) { const pma_entry::flags f{.R = true, .W = true, .X = false, .IR = false, .IW = false, .DID = PMA_ISTART_DID::PLIC}; - return make_device_pma_entry("PLIC device", start, length, plic_peek, &plic_driver).set_flags(f); + return make_device_pma_entry("PLIC device", start, length, pma_peek_pristine, &plic_driver).set_flags(f); } } // namespace cartesi diff --git a/src/pm-type-name.h b/src/pm-type-name.h new file mode 100644 index 000000000..6f3d61e92 --- /dev/null +++ b/src/pm-type-name.h @@ -0,0 +1,48 @@ +// Copyright Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: LGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify it under +// the terms of the GNU Lesser General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) any +// later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +// PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License along +// with this program (see COPYING). If not, see . +// + +#ifndef PM_TYPE_NAME_H +#define PM_TYPE_NAME_H + +namespace cartesi { + +//?DD Poor man's rtti that works in microarchitecture +template +struct pm_type_name { + static constexpr const char *value = "unknown type"; +}; + +template +constexpr const char *pm_type_name_v = pm_type_name::value; + +#define PM_TYPE_NAME(type) \ + template <> \ + struct pm_type_name { \ + static constexpr const char *value = #type; \ + } + +PM_TYPE_NAME(uint8_t); +PM_TYPE_NAME(int8_t); +PM_TYPE_NAME(uint16_t); +PM_TYPE_NAME(int16_t); +PM_TYPE_NAME(uint32_t); +PM_TYPE_NAME(int32_t); +PM_TYPE_NAME(uint64_t); +PM_TYPE_NAME(int64_t); + +} // namespace cartesi + +#endif diff --git a/src/pma-constants.h b/src/pma-constants.h index be97580eb..8be8edd4a 100644 --- a/src/pma-constants.h +++ b/src/pma-constants.h @@ -132,6 +132,37 @@ enum class PMA_ISTART_DID { shadow_uarch = PMA_SHADOW_UARCH_STATE_DID_DEF, ///< DID for shadow uarch state device }; +static constexpr const char *pma_get_DID_name(PMA_ISTART_DID did) { + switch (did) { + case PMA_ISTART_DID::memory: + return "DID.memory"; + case PMA_ISTART_DID::shadow_state: + return "DID.shadow_state"; + case PMA_ISTART_DID::shadow_pmas: + return "DID.shadow_pmas"; + case PMA_ISTART_DID::shadow_TLB: + return "DID.shadow_TLB"; + case PMA_ISTART_DID::flash_drive: + return "DID.flash_drive"; + case PMA_ISTART_DID::CLINT: + return "DID.CLINT"; + case PMA_ISTART_DID::PLIC: + return "DID.PLIC"; + case PMA_ISTART_DID::HTIF: + return "DID.HTIF"; + case PMA_ISTART_DID::VIRTIO: + return "DID.VIRTIO"; + case PMA_ISTART_DID::cmio_rx_buffer: + return "DID.cmio_rx_buffer"; + case PMA_ISTART_DID::cmio_tx_buffer: + return "DID.cmio_tx_buffer"; + case PMA_ISTART_DID::shadow_uarch: + return "DID.shadow_uarch"; + default: + return "DID.unkown"; + } +} + static_assert(PMA_CMIO_RX_BUFFER_START_DEF == CM_PMA_CMIO_RX_BUFFER_START); static_assert(PMA_CMIO_RX_BUFFER_LOG2_SIZE_DEF == CM_PMA_CMIO_RX_BUFFER_LOG2_SIZE); static_assert(PMA_CMIO_TX_BUFFER_START_DEF == CM_PMA_CMIO_TX_BUFFER_START); diff --git a/src/pma.cpp b/src/pma.cpp index 9a112b4dd..5edb43f46 100644 --- a/src/pma.cpp +++ b/src/pma.cpp @@ -204,28 +204,27 @@ void pma_entry::fill_memory(uint64_t paddr, unsigned char value, uint64_t size) } } -bool pma_peek_error(const pma_entry & /*pma*/, const machine & /*m*/, uint64_t /*page_address*/, +bool pma_peek_error(const pma_entry & /*pma*/, const machine & /*m*/, uint64_t /*offset*/, uint64_t /*length*/, const unsigned char ** /*page_data*/, unsigned char * /*scratch*/) { return false; } +bool pma_peek_pristine(const pma_entry &pma, const machine & /*m*/, uint64_t offset, uint64_t length, + const unsigned char **data, unsigned char * /*scratch*/) { + *data = nullptr; + return length <= pma.get_length() && offset <= pma.get_length() - length; +} + /// \brief Memory range peek callback. See pma_peek. -static bool memory_peek(const pma_entry &pma, const machine & /*m*/, uint64_t page_address, - const unsigned char **page_data, unsigned char *scratch) { - // If page_address is not aligned, or if it is out of range, return error - if ((page_address & (PMA_PAGE_SIZE - 1)) != 0 || page_address > pma.get_length()) { - *page_data = nullptr; +static bool memory_peek(const pma_entry &pma, const machine & /*m*/, uint64_t offset, uint64_t length, + const unsigned char **data, unsigned char * /*scratch*/) { + // If desired data does not fit in range, return error + if (length > pma.get_length() || offset > pma.get_length() - length) { + *data = nullptr; return false; } - // If page is only partially inside range, copy to scratch - if (page_address + PMA_PAGE_SIZE > pma.get_length()) { - memset(scratch, 0, PMA_PAGE_SIZE); - memcpy(scratch, pma.get_memory().get_host_memory() + page_address, pma.get_length() - page_address); - *page_data = scratch; - return true; - // Otherwise, return pointer directly into host memory - } - *page_data = pma.get_memory().get_host_memory() + page_address; + // Otherwise, return pointer directly into host memory + *data = pma.get_memory().get_host_memory() + offset; return true; } diff --git a/src/pma.h b/src/pma.h index 69d53d035..74f96fa05 100644 --- a/src/pma.h +++ b/src/pma.h @@ -41,16 +41,21 @@ class machine; /// \brief Prototype for callback invoked when machine wants to peek into a range with no side-effects. /// \param pma Reference to corresponding PMA entry. /// \param m Reference to associated machine. -/// \param page_offset Offset of page start within range. Must be aligned to PMA_PAGE_SIZE. -/// \param page_data Receives pointer to start of page data, or nullptr if page is constant *and* pristine. -/// \param scratch Pointer to memory buffer that must be able to hold PMA_PAGE_SIZE bytes. +/// \param offset Offset within range to start reading. +/// \param length Number of bytes to read. +/// \param data Receives pointer to start of data, or nullptr if data is constant *and* pristine. +/// \param scratch Pointer to memory buffer that must be able to hold \p length bytes. /// \returns True if operation succeeded, false otherwise. -using pma_peek = bool (*)(const pma_entry &pma, const machine &m, uint64_t page_offset, const unsigned char **page_data, - unsigned char *scratch); +using pma_peek = bool (*)(const pma_entry &pma, const machine &m, uint64_t offset, uint64_t length, + const unsigned char **data, unsigned char *scratch); /// \brief Default peek callback issues error on peeks. -bool pma_peek_error(const pma_entry & /*pma*/, const machine & /*m*/, uint64_t /*page_offset*/, - const unsigned char ** /*page_data*/, unsigned char * /*scratch*/); +bool pma_peek_error(const pma_entry & /*pma*/, const machine & /*m*/, uint64_t /*offset*/, uint64_t /*length*/, + const unsigned char ** /*data*/, unsigned char * /*scratch*/); + +/// \brief Default peek callback for pristine ranges. +bool pma_peek_pristine(const pma_entry & /*pma*/, const machine & /*m*/, uint64_t offset, uint64_t length, + const unsigned char **data, unsigned char * /*scratch*/); /// \brief Data for IO ranges. class pma_device final { diff --git a/src/record-send-cmio-state-access.h b/src/record-send-cmio-state-access.h index 3c595e53a..f9964a5f5 100644 --- a/src/record-send-cmio-state-access.h +++ b/src/record-send-cmio-state-access.h @@ -35,6 +35,7 @@ #include "machine.h" #include "meta.h" #include "pma.h" +#include "scoped-note.h" #include "shadow-state.h" namespace cartesi { @@ -184,8 +185,16 @@ class record_send_cmio_state_access : public i_state_access{*this, text}; } void do_write_iflags_Y(uint64_t val) { @@ -262,6 +271,11 @@ class record_send_cmio_state_access : public i_state_access +#include +#include +#include "compiler-defines.h" #include "device-state-access.h" #include "i-state-access.h" +#include "machine.h" #include "shadow-pmas.h" #include "shadow-tlb.h" #include "unique-c-ptr.h" -#include -#include -#include + +#if DUMP_STATE_ACCESS +#include "scoped-note.h" +#endif namespace cartesi { @@ -184,427 +188,361 @@ class record_step_state_access : public i_state_access } } - // NOLINTNEXTLINE(readability-convert-member-functions-to-static) - void do_push_bracket(bracket_type type, const char *text) { - (void) type; - (void) text; + uint64_t log_read_reg(machine_reg reg) const { + touch_page(machine_reg_address(reg)); + return m_m.read_reg(reg); } - int do_make_scoped_note(const char *text) { // NOLINT(readability-convert-member-functions-to-static) - (void) text; - return 0; + void log_write_reg(machine_reg reg, uint64_t val) { + touch_page(machine_reg_address(reg)); + m_m.write_reg(reg, val); } - uint64_t do_read_x(int reg) const { - touch_page(machine_reg_address(machine_reg::x0, reg)); - return m_m.get_state().x[reg]; + uint64_t log_read_tlb(TLB_set_index set_index, uint64_t slot_index, shadow_tlb_what what) { + touch_page(shadow_tlb_get_abs_addr(set_index, slot_index, what)); + return m_m.read_shadow_tlb(set_index, slot_index, what); } - void do_write_x(int reg, uint64_t val) { - assert(reg != 0); - touch_page(machine_reg_address(machine_reg::x0, reg)); - m_m.get_state().x[reg] = val; + // NOLINTNEXTLINE(readability-convert-member-functions-to-static) + void do_push_begin_bracket(const char * /*text*/) {} + + // NOLINTNEXTLINE(readability-convert-member-functions-to-static) + void do_push_end_bracket(const char * /*text*/) {} + +#ifdef DUMP_STATE_ACCESS + // NOLINTNEXTLINE(readability-convert-member-functions-to-static) + auto do_make_scoped_note([[maybe_unused]] const char *text) { + return scoped_note{*this, text}; + } +#endif + + uint64_t do_read_x(int i) const { + return log_read_reg(machine_reg_enum(machine_reg::x0, i)); + } + + void do_write_x(int i, uint64_t val) { + assert(i != 0); + log_write_reg(machine_reg_enum(machine_reg::x0, i), val); } - uint64_t do_read_f(int reg) const { - touch_page(machine_reg_address(machine_reg::f0, reg)); - return m_m.get_state().f[reg]; + uint64_t do_read_f(int i) const { + return log_read_reg(machine_reg_enum(machine_reg::f0, i)); } - void do_write_f(int reg, uint64_t val) { - touch_page(machine_reg_address(machine_reg::f0, reg)); - m_m.get_state().f[reg] = val; + void do_write_f(int i, uint64_t val) { + log_write_reg(machine_reg_enum(machine_reg::f0, i), val); } uint64_t do_read_pc() const { - // get phys address of pc in dhadow - touch_page(machine_reg_address(machine_reg::pc)); - return m_m.get_state().pc; + return log_read_reg(machine_reg::pc); } void do_write_pc(uint64_t val) { - touch_page(machine_reg_address(machine_reg::pc)); - m_m.get_state().pc = val; + log_write_reg(machine_reg::pc, val); } uint64_t do_read_fcsr() const { - touch_page(machine_reg_address(machine_reg::fcsr)); - return m_m.get_state().fcsr; + return log_read_reg(machine_reg::fcsr); } void do_write_fcsr(uint64_t val) { - touch_page(machine_reg_address(machine_reg::fcsr)); - m_m.get_state().fcsr = val; + log_write_reg(machine_reg::fcsr, val); } uint64_t do_read_icycleinstret() const { - touch_page(machine_reg_address(machine_reg::icycleinstret)); - return m_m.get_state().icycleinstret; + return log_read_reg(machine_reg::icycleinstret); } void do_write_icycleinstret(uint64_t val) { - touch_page(machine_reg_address(machine_reg::icycleinstret)); - m_m.get_state().icycleinstret = val; + log_write_reg(machine_reg::icycleinstret, val); } - uint64_t do_read_mvendorid() const { // NOLINT(readability-convert-member-functions-to-static) - touch_page(machine_reg_address(machine_reg::mvendorid)); - return MVENDORID_INIT; + uint64_t do_read_mvendorid() const { + return log_read_reg(machine_reg::mvendorid); } - uint64_t do_read_marchid() const { // NOLINT(readability-convert-member-functions-to-static) - touch_page(machine_reg_address(machine_reg::marchid)); - return MARCHID_INIT; + uint64_t do_read_marchid() const { + return log_read_reg(machine_reg::marchid); } - uint64_t do_read_mimpid() const { // NOLINT(readability-convert-member-functions-to-static) - touch_page(machine_reg_address(machine_reg::mimpid)); - return MIMPID_INIT; + uint64_t do_read_mimpid() const { + return log_read_reg(machine_reg::mimpid); } uint64_t do_read_mcycle() const { - touch_page(machine_reg_address(machine_reg::mcycle)); - return m_m.get_state().mcycle; + return log_read_reg(machine_reg::mcycle); } void do_write_mcycle(uint64_t val) { - touch_page(machine_reg_address(machine_reg::mcycle)); - m_m.get_state().mcycle = val; + log_write_reg(machine_reg::mcycle, val); } uint64_t do_read_mstatus() const { - touch_page(machine_reg_address(machine_reg::mstatus)); - return m_m.get_state().mstatus; + return log_read_reg(machine_reg::mstatus); } void do_write_mstatus(uint64_t val) { - touch_page(machine_reg_address(machine_reg::mstatus)); - m_m.get_state().mstatus = val; + log_write_reg(machine_reg::mstatus, val); } uint64_t do_read_menvcfg() const { - touch_page(machine_reg_address(machine_reg::menvcfg)); - return m_m.get_state().menvcfg; + return log_read_reg(machine_reg::menvcfg); } void do_write_menvcfg(uint64_t val) { - touch_page(machine_reg_address(machine_reg::menvcfg)); - m_m.get_state().menvcfg = val; + log_write_reg(machine_reg::menvcfg, val); } uint64_t do_read_mtvec() const { - touch_page(machine_reg_address(machine_reg::mtvec)); - return m_m.get_state().mtvec; + return log_read_reg(machine_reg::mtvec); } void do_write_mtvec(uint64_t val) { - touch_page(machine_reg_address(machine_reg::mtvec)); - m_m.get_state().mtvec = val; + log_write_reg(machine_reg::mtvec, val); } uint64_t do_read_mscratch() const { - touch_page(machine_reg_address(machine_reg::mscratch)); - return m_m.get_state().mscratch; + return log_read_reg(machine_reg::mscratch); } void do_write_mscratch(uint64_t val) { - touch_page(machine_reg_address(machine_reg::mscratch)); - m_m.get_state().mscratch = val; + log_write_reg(machine_reg::mscratch, val); } uint64_t do_read_mepc() const { - touch_page(machine_reg_address(machine_reg::mepc)); - return m_m.get_state().mepc; + return log_read_reg(machine_reg::mepc); } void do_write_mepc(uint64_t val) { - touch_page(machine_reg_address(machine_reg::mepc)); - m_m.get_state().mepc = val; + log_write_reg(machine_reg::mepc, val); } uint64_t do_read_mcause() const { - touch_page(machine_reg_address(machine_reg::mcause)); - return m_m.get_state().mcause; + return log_read_reg(machine_reg::mcause); } void do_write_mcause(uint64_t val) { - touch_page(machine_reg_address(machine_reg::mcause)); - m_m.get_state().mcause = val; + log_write_reg(machine_reg::mcause, val); } uint64_t do_read_mtval() const { - touch_page(machine_reg_address(machine_reg::mtval)); - return m_m.get_state().mtval; + return log_read_reg(machine_reg::mtval); } void do_write_mtval(uint64_t val) { - touch_page(machine_reg_address(machine_reg::mtval)); - m_m.get_state().mtval = val; + log_write_reg(machine_reg::mtval, val); } uint64_t do_read_misa() const { - touch_page(machine_reg_address(machine_reg::misa)); - return m_m.get_state().misa; + return log_read_reg(machine_reg::misa); } void do_write_misa(uint64_t val) { - touch_page(machine_reg_address(machine_reg::misa)); - m_m.get_state().misa = val; + log_write_reg(machine_reg::misa, val); } uint64_t do_read_mie() const { - touch_page(machine_reg_address(machine_reg::mie)); - return m_m.get_state().mie; + return log_read_reg(machine_reg::mie); } void do_write_mie(uint64_t val) { - touch_page(machine_reg_address(machine_reg::mie)); - m_m.get_state().mie = val; + log_write_reg(machine_reg::mie, val); } uint64_t do_read_mip() const { - touch_page(machine_reg_address(machine_reg::mip)); - return m_m.get_state().mip; + return log_read_reg(machine_reg::mip); } void do_write_mip(uint64_t val) { - touch_page(machine_reg_address(machine_reg::mip)); - m_m.get_state().mip = val; + log_write_reg(machine_reg::mip, val); } uint64_t do_read_medeleg() const { - touch_page(machine_reg_address(machine_reg::medeleg)); - return m_m.get_state().medeleg; + return log_read_reg(machine_reg::medeleg); } void do_write_medeleg(uint64_t val) { - touch_page(machine_reg_address(machine_reg::medeleg)); - m_m.get_state().medeleg = val; + log_write_reg(machine_reg::medeleg, val); } uint64_t do_read_mideleg() const { - touch_page(machine_reg_address(machine_reg::mideleg)); - return m_m.get_state().mideleg; + return log_read_reg(machine_reg::mideleg); } void do_write_mideleg(uint64_t val) { - touch_page(machine_reg_address(machine_reg::mideleg)); - m_m.get_state().mideleg = val; + log_write_reg(machine_reg::mideleg, val); } uint64_t do_read_mcounteren() const { - touch_page(machine_reg_address(machine_reg::mcounteren)); - return m_m.get_state().mcounteren; + return log_read_reg(machine_reg::mcounteren); } void do_write_mcounteren(uint64_t val) { - touch_page(machine_reg_address(machine_reg::mcounteren)); - m_m.get_state().mcounteren = val; + log_write_reg(machine_reg::mcounteren, val); } uint64_t do_read_senvcfg() const { - touch_page(machine_reg_address(machine_reg::senvcfg)); - return m_m.get_state().senvcfg; + return log_read_reg(machine_reg::senvcfg); } void do_write_senvcfg(uint64_t val) { - touch_page(machine_reg_address(machine_reg::senvcfg)); - m_m.get_state().senvcfg = val; + log_write_reg(machine_reg::senvcfg, val); } uint64_t do_read_stvec() const { - touch_page(machine_reg_address(machine_reg::stvec)); - return m_m.get_state().stvec; + return log_read_reg(machine_reg::stvec); } void do_write_stvec(uint64_t val) { - touch_page(machine_reg_address(machine_reg::stvec)); - m_m.get_state().stvec = val; + log_write_reg(machine_reg::stvec, val); } uint64_t do_read_sscratch() const { - touch_page(machine_reg_address(machine_reg::sscratch)); - return m_m.get_state().sscratch; + return log_read_reg(machine_reg::sscratch); } void do_write_sscratch(uint64_t val) { - touch_page(machine_reg_address(machine_reg::sscratch)); - m_m.get_state().sscratch = val; + log_write_reg(machine_reg::sscratch, val); } uint64_t do_read_sepc() const { - touch_page(machine_reg_address(machine_reg::sepc)); - return m_m.get_state().sepc; + return log_read_reg(machine_reg::sepc); } void do_write_sepc(uint64_t val) { - touch_page(machine_reg_address(machine_reg::sepc)); - m_m.get_state().sepc = val; + log_write_reg(machine_reg::sepc, val); } uint64_t do_read_scause() const { - touch_page(machine_reg_address(machine_reg::scause)); - return m_m.get_state().scause; + return log_read_reg(machine_reg::scause); } void do_write_scause(uint64_t val) { - touch_page(machine_reg_address(machine_reg::scause)); - m_m.get_state().scause = val; + log_write_reg(machine_reg::scause, val); } uint64_t do_read_stval() const { - touch_page(machine_reg_address(machine_reg::stval)); - return m_m.get_state().stval; + return log_read_reg(machine_reg::stval); } void do_write_stval(uint64_t val) { - touch_page(machine_reg_address(machine_reg::stval)); - m_m.get_state().stval = val; + log_write_reg(machine_reg::stval, val); } uint64_t do_read_satp() const { - touch_page(machine_reg_address(machine_reg::satp)); - return m_m.get_state().satp; + return log_read_reg(machine_reg::satp); } void do_write_satp(uint64_t val) { - touch_page(machine_reg_address(machine_reg::satp)); - m_m.get_state().satp = val; + log_write_reg(machine_reg::satp, val); } uint64_t do_read_scounteren() const { - touch_page(machine_reg_address(machine_reg::scounteren)); - return m_m.get_state().scounteren; + return log_read_reg(machine_reg::scounteren); } void do_write_scounteren(uint64_t val) { - touch_page(machine_reg_address(machine_reg::scounteren)); - m_m.get_state().scounteren = val; + log_write_reg(machine_reg::scounteren, val); } uint64_t do_read_ilrsc() const { - touch_page(machine_reg_address(machine_reg::ilrsc)); - return m_m.get_state().ilrsc; + return log_read_reg(machine_reg::ilrsc); } void do_write_ilrsc(uint64_t val) { - touch_page(machine_reg_address(machine_reg::ilrsc)); - m_m.get_state().ilrsc = val; + log_write_reg(machine_reg::ilrsc, val); } uint64_t do_read_iprv() const { - touch_page(machine_reg_address(machine_reg::iprv)); - return m_m.get_state().iprv; + return log_read_reg(machine_reg::iprv); } void do_write_iprv(uint64_t val) { - touch_page(machine_reg_address(machine_reg::iprv)); - m_m.get_state().iprv = val; + log_write_reg(machine_reg::iprv, val); } void do_write_iflags_X(uint64_t val) { - touch_page(machine_reg_address(machine_reg::iflags_X)); - m_m.get_state().iflags.X = val; + log_write_reg(machine_reg::iflags_X, val); } uint64_t do_read_iflags_X() const { - touch_page(machine_reg_address(machine_reg::iflags_X)); - return m_m.get_state().iflags.X; + return log_read_reg(machine_reg::iflags_X); } void do_write_iflags_Y(uint64_t val) { - touch_page(machine_reg_address(machine_reg::iflags_Y)); - m_m.get_state().iflags.Y = val; + log_write_reg(machine_reg::iflags_Y, val); } uint64_t do_read_iflags_Y() const { - touch_page(machine_reg_address(machine_reg::iflags_Y)); - return m_m.get_state().iflags.Y; + return log_read_reg(machine_reg::iflags_Y); } void do_write_iflags_H(uint64_t val) { - touch_page(machine_reg_address(machine_reg::iflags_H)); - m_m.get_state().iflags.H = val; + log_write_reg(machine_reg::iflags_H, val); } uint64_t do_read_iflags_H() const { - touch_page(machine_reg_address(machine_reg::iflags_H)); - return m_m.get_state().iflags.H; + return log_read_reg(machine_reg::iflags_H); } uint64_t do_read_iunrep() const { - touch_page(machine_reg_address(machine_reg::iunrep)); - return m_m.get_state().iunrep; + return log_read_reg(machine_reg::iunrep); } void do_write_iunrep(uint64_t val) { - touch_page(machine_reg_address(machine_reg::iunrep)); - m_m.get_state().iunrep = val; + log_write_reg(machine_reg::iunrep, val); } uint64_t do_read_clint_mtimecmp() const { - touch_page(machine_reg_address(machine_reg::clint_mtimecmp)); - return m_m.get_state().clint.mtimecmp; + return log_read_reg(machine_reg::clint_mtimecmp); } void do_write_clint_mtimecmp(uint64_t val) { - touch_page(machine_reg_address(machine_reg::clint_mtimecmp)); - m_m.get_state().clint.mtimecmp = val; + log_write_reg(machine_reg::clint_mtimecmp, val); } uint64_t do_read_plic_girqpend() const { - touch_page(machine_reg_address(machine_reg::plic_girqpend)); - return m_m.get_state().plic.girqpend; + return log_read_reg(machine_reg::plic_girqpend); } void do_write_plic_girqpend(uint64_t val) { - touch_page(machine_reg_address(machine_reg::plic_girqpend)); - m_m.get_state().plic.girqpend = val; + log_write_reg(machine_reg::plic_girqpend, val); } uint64_t do_read_plic_girqsrvd() const { - touch_page(machine_reg_address(machine_reg::plic_girqsrvd)); - return m_m.get_state().plic.girqsrvd; + return log_read_reg(machine_reg::plic_girqsrvd); } void do_write_plic_girqsrvd(uint64_t val) { - touch_page(machine_reg_address(machine_reg::plic_girqsrvd)); - m_m.get_state().plic.girqsrvd = val; + log_write_reg(machine_reg::plic_girqsrvd, val); } uint64_t do_read_htif_fromhost() const { - touch_page(machine_reg_address(machine_reg::htif_fromhost)); - return m_m.get_state().htif.fromhost; + return log_read_reg(machine_reg::htif_fromhost); } void do_write_htif_fromhost(uint64_t val) { - touch_page(machine_reg_address(machine_reg::htif_fromhost)); - m_m.get_state().htif.fromhost = val; + log_write_reg(machine_reg::htif_fromhost, val); } uint64_t do_read_htif_tohost() const { - touch_page(machine_reg_address(machine_reg::htif_tohost)); - return m_m.get_state().htif.tohost; + return log_read_reg(machine_reg::htif_tohost); } void do_write_htif_tohost(uint64_t val) { - touch_page(machine_reg_address(machine_reg::htif_tohost)); - m_m.get_state().htif.tohost = val; + log_write_reg(machine_reg::htif_tohost, val); } uint64_t do_read_htif_ihalt() const { - touch_page(machine_reg_address(machine_reg::htif_ihalt)); - return m_m.get_state().htif.ihalt; + return log_read_reg(machine_reg::htif_ihalt); } uint64_t do_read_htif_iconsole() const { - touch_page(machine_reg_address(machine_reg::htif_iconsole)); - return m_m.get_state().htif.iconsole; + return log_read_reg(machine_reg::htif_iconsole); } uint64_t do_read_htif_iyield() const { - touch_page(machine_reg_address(machine_reg::htif_iyield)); - return m_m.get_state().htif.iyield; + return log_read_reg(machine_reg::htif_iyield); } // NOLINTNEXTLINE(readability-convert-member-functions-to-static) @@ -628,8 +566,8 @@ class record_step_state_access : public i_state_access // replay_step_state_access reconstructs a mock_pma_entry from the // corresponding istart and ilength fields in the shadow pmas // so we mark the page where they live here - touch_page(shadow_pmas_get_pma_istart_abs_addr(index)); - touch_page(shadow_pmas_get_pma_ilength_abs_addr(index)); + touch_page(shadow_pmas_get_pma_abs_addr(index, shadow_pmas_what::istart)); + touch_page(shadow_pmas_get_pma_abs_addr(index, shadow_pmas_what::ilength)); // NOLINTNEXTLINE(bugprone-narrowing-conversions) return m_m.get_state().pmas[static_cast(index)]; } @@ -646,27 +584,32 @@ class record_step_state_access : public i_state_access aliased_aligned_write(haddr, val); } - template + template uint64_t do_read_tlb_vaddr_page(uint64_t slot_index) { - touch_page(shadow_tlb_get_vaddr_page_abs_addr(slot_index)); - return m_m.get_state().tlb.hot[USE][slot_index].vaddr_page; + return log_read_tlb(SET, slot_index, shadow_tlb_what::vaddr_page); } - template + template + uint64_t do_read_tlb_pma_index(uint64_t slot_index) { + return log_read_tlb(SET, slot_index, shadow_tlb_what::pma_index); + } + + template host_addr do_read_tlb_vp_offset(uint64_t slot_index) { // During initialization, replay_step_state_access translates all vp_offset to corresponding vh_offset // At deinitialization, it translates them back // To do that, it needs the corresponding paddr_page = vaddr_page + vp_offset, and page data itself - // It will only do the translation if the slot is valid and it has access to all required fields - // Obviously, the slot we are reading will be needed during replay, so we touch all the pages involved here. - touch_page(shadow_tlb_get_vaddr_page_abs_addr(slot_index)); - touch_page(shadow_tlb_get_vp_offset_abs_addr(slot_index)); - touch_page(shadow_tlb_get_pma_index_abs_addr(slot_index)); - // writes to the TLB slot are atomic, so we know the values in a slot are ALWAYS internally consistent - const auto vaddr_page = m_m.get_state().tlb.hot[USE][slot_index].vaddr_page; - const auto vh_offset = m_m.get_state().tlb.hot[USE][slot_index].vh_offset; + // It will only do the translation if the slot is valid and the log has all required fields + // Obviously, the slot we are reading will be needed during replay + // vaddr_page, vp_offset, and pma_index are on the same page, so we only need touch one of them. + touch_page(shadow_tlb_get_abs_addr(SET, slot_index, shadow_tlb_what::vaddr_page)); + // We still need to touch the page data + // Writes to the TLB slot are atomic, so we know the values in a slot are ALWAYS internally consistent. + // This means we can safely use all other fields to find paddr_page. + const auto vaddr_page = m_m.get_state().tlb.hot[SET][slot_index].vaddr_page; + const auto vh_offset = m_m.get_state().tlb.hot[SET][slot_index].vh_offset; if (vaddr_page != TLB_INVALID_PAGE) { - const auto pma_index = m_m.get_state().tlb.cold[USE][slot_index].pma_index; + const auto pma_index = m_m.get_state().tlb.cold[SET][slot_index].pma_index; const auto haddr_page = vaddr_page + vh_offset; auto paddr_page = m_m.get_paddr(haddr_page, pma_index); touch_page(paddr_page); @@ -674,30 +617,24 @@ class record_step_state_access : public i_state_access return vh_offset; } - template - uint64_t do_read_tlb_pma_index(uint64_t slot_index) { - touch_page(shadow_tlb_get_pma_index_abs_addr(slot_index)); - return m_m.get_state().tlb.cold[USE][slot_index].pma_index; - } - - template + template void do_write_tlb(uint64_t slot_index, uint64_t vaddr_page, host_addr vh_offset, uint64_t pma_index) { // During initialization, replay_step_state_access translates all vp_offset to corresponding vh_offset // At deinitialization, it translates them back // To do that, it needs the corresponding paddr_page = vaddr_page + vp_offset, and page data itself - // It will only do the translation if the slot is valid and it has access to all required fields - // Obviously, the slot we are modifying will be needed during replay, so we touch all the pages involved here. - touch_page(shadow_tlb_get_vaddr_page_abs_addr(slot_index)); - touch_page(shadow_tlb_get_vp_offset_abs_addr(slot_index)); - touch_page(shadow_tlb_get_pma_index_abs_addr(slot_index)); - m_m.get_state().tlb.hot[USE][slot_index].vaddr_page = vaddr_page; - m_m.get_state().tlb.hot[USE][slot_index].vh_offset = vh_offset; - m_m.get_state().tlb.cold[USE][slot_index].pma_index = pma_index; + // It will only do the translation if the slot is valid and the log has all required fields + // Obviously, the slot we are writing will be needed during replay + // vaddr_page, vp_offset, and pma_index are on the same page, so we only need touch one of them. + touch_page(shadow_tlb_get_abs_addr(SET, slot_index, shadow_tlb_what::vaddr_page)); + // We still need to touch the page data if (vaddr_page != TLB_INVALID_PAGE) { const auto haddr_page = vaddr_page + vh_offset; auto paddr_page = m_m.get_paddr(haddr_page, pma_index); touch_page(paddr_page); } + m_m.get_state().tlb.hot[SET][slot_index].vaddr_page = vaddr_page; + m_m.get_state().tlb.hot[SET][slot_index].vh_offset = vh_offset; + m_m.get_state().tlb.cold[SET][slot_index].pma_index = pma_index; } fast_addr do_get_faddr(uint64_t paddr, uint64_t pma_index) const { @@ -716,6 +653,10 @@ class record_step_state_access : public i_state_access os_putchar(c); } + // NOLINTNEXTLINE(readability-convert-member-functions-to-static) + constexpr const char *do_get_name() const { // NOLINT(readability-convert-member-functions-to-static) + return "record_step_state_access"; + } }; } // namespace cartesi diff --git a/src/replay-send-cmio-state-access.h b/src/replay-send-cmio-state-access.h index 155c3a909..418f5ef9e 100644 --- a/src/replay-send-cmio-state-access.h +++ b/src/replay-send-cmio-state-access.h @@ -362,6 +362,11 @@ class replay_send_cmio_state_access : public i_state_access if (i > 0 && m_context.pages[i - 1].index >= m_context.pages[i].index) { interop_throw_runtime_error("invalid log format: page index is not in increasing order"); } + // In the current implementaiton, this check is unecessary + // But we may in the future change the data field to point to independently allocated pages + // This would break the code that uses binary search to find the page based on the address of its data + if (i > 0 && +m_context.pages[i - 1].data >= +m_context.pages[i].data) { + interop_throw_runtime_error("invalid log format: page data is not in increasing order"); + } if (m_context.pages[i].hash != all_zeros) { interop_throw_runtime_error("invalid log format: page scratch hash area is not zero"); } } - // compute and check the machine root hash before the replay auto computed_root_hash_before = compute_root_hash(); if (computed_root_hash_before != root_hash_before) { @@ -193,6 +202,10 @@ class replay_step_state_access : public i_state_access private: friend i_state_access; + void do_putchar(uint8_t /*c*/) { // NOLINT(readability-convert-member-functions-to-static) + ; // do nothing + } + /// \brief Try to find a page in the logged data by its physical address /// \param paddr The physical address of the page /// \return A pointer to the page_type structure if found, nullptr otherwise @@ -237,12 +250,12 @@ class replay_step_state_access : public i_state_access // \brief Relocate all TLB vp_offset fields to vh_offset // \details This makes the translation point directly to logged page data - template + template void relocate_tlb_vp_offset_to_vh_offset() { for (uint64_t slot_index = 0; slot_index < TLB_SET_SIZE; ++slot_index) { - const auto vp_offset_field_addr = shadow_tlb_get_vp_offset_abs_addr(slot_index); + const auto vp_offset_field_addr = shadow_tlb_get_abs_addr(SET, slot_index, shadow_tlb_what::vp_offset); auto *vp_offset_log = try_find_page(vp_offset_field_addr & ~PAGE_OFFSET_MASK); - const auto vaddr_page_field_addr = shadow_tlb_get_vaddr_page_abs_addr(slot_index); + const auto vaddr_page_field_addr = shadow_tlb_get_abs_addr(SET, slot_index, shadow_tlb_what::vaddr_page); auto *vaddr_page_log = try_find_page(vaddr_page_field_addr & ~PAGE_OFFSET_MASK); // If vp_offset was accessed during record, both it and vaddr_apge will appear in the log // (record_step_state_access makes sure of it) @@ -274,12 +287,12 @@ class replay_step_state_access : public i_state_access // \brief Reverses changes to TLB so we have vp_offset fields again instead of vh_offset // \details This makes the translation point back to target physical addresses - template + template void relocate_tlb_vh_offset_to_vp_offset() { for (uint64_t slot_index = 0; slot_index < TLB_SET_SIZE; ++slot_index) { - const auto vp_offset_field_addr = shadow_tlb_get_vp_offset_abs_addr(slot_index); + const auto vp_offset_field_addr = shadow_tlb_get_abs_addr(SET, slot_index, shadow_tlb_what::vp_offset); auto *vp_offset_log = try_find_page(vp_offset_field_addr & ~PAGE_OFFSET_MASK); - const auto vaddr_page_field_addr = shadow_tlb_get_vaddr_page_abs_addr(slot_index); + const auto vaddr_page_field_addr = shadow_tlb_get_abs_addr(SET, slot_index, shadow_tlb_what::vaddr_page); auto *vaddr_page_log = try_find_page(vaddr_page_field_addr & ~PAGE_OFFSET_MASK); // If vp_offset was accessed during record, both it and vaddr_apge will appear in the log // (record_step_state_access makes sure of it) @@ -313,7 +326,7 @@ class replay_step_state_access : public i_state_access page_type *find_page(uint64_t paddr_page) const { auto *page_log = try_find_page(paddr_page); if (page_log == nullptr) { - interop_throw_runtime_error("find_page: page not found"); + interop_throw_runtime_error("required page not found"); } return page_log; } @@ -321,7 +334,7 @@ class replay_step_state_access : public i_state_access page_type *find_page(host_addr haddr_page) const { auto *page_log = try_find_page(haddr_page); if (page_log == nullptr) { - interop_throw_runtime_error("find_page: page not found"); + interop_throw_runtime_error("required page not found"); } return page_log; } @@ -340,46 +353,48 @@ class replay_step_state_access : public i_state_access // \brief Compute the current machine root hash hash_type compute_root_hash() { + //??D Here we should only do this for dirty pages, right? + //??D Initially, all pages are dirty, because we don't know their hashes + //??D But in the end, we should only update those pages that we touched + //??D May improve performance when we are running this on ZK for (uint64_t i = 0; i < m_context.page_count; i++) { interop_merkle_tree_hash(m_context.pages[i].data, PMA_PAGE_SIZE, // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) reinterpret_cast(&m_context.pages[i].hash)); } - size_t next_page = 0; size_t next_sibling = 0; auto root_hash = compute_root_hash_impl(0, interop_log2_root_size - PMA_PAGE_SIZE_LOG2, next_page, next_sibling); if (next_page != m_context.page_count) { - interop_throw_runtime_error("compute_root_hash: next_page != m_context.page_count"); + interop_throw_runtime_error("too many pages in log"); } if (next_sibling != m_context.sibling_count) { - interop_throw_runtime_error("compute_root_hash: sibling hashes not totally consumed"); + interop_throw_runtime_error("too many sibling hashes in log"); } return root_hash; } // \brief Compute the root hash of a memory range recursively // \param page_index Index of the first page in the range - // \param page_count_log2_size Log2 of the size of number of pages in the range + // \param log2_page_count Log2 of the size of number of pages in the range // \param next_page Index of the next page to be visited // \param next_sibling Index of the next sibling hash to be visited // \return Resulting root hash of the range - hash_type compute_root_hash_impl(address_type page_index, int page_count_log2_size, size_t &next_page, + hash_type compute_root_hash_impl(address_type page_index, int log2_page_count, size_t &next_page, size_t &next_sibling) { // NOLINTBEGIN(cppcoreguidelines-pro-type-reinterpret-cast)) - auto page_count = UINT64_C(1) << page_count_log2_size; + auto page_count = UINT64_C(1) << log2_page_count; if (next_page >= m_context.page_count || page_index + page_count <= m_context.pages[next_page].index) { if (next_sibling >= m_context.sibling_count) { - interop_throw_runtime_error( - "compute_root_hash_impl: trying to access beyond sibling count while skipping range"); + interop_throw_runtime_error("too few sibling hashes in log"); } return m_context.sibling_hashes[next_sibling++]; } - if (page_count_log2_size > 0) { - auto left = compute_root_hash_impl(page_index, page_count_log2_size - 1, next_page, next_sibling); - auto right = compute_root_hash_impl(page_index + (UINT64_C(1) << (page_count_log2_size - 1)), - page_count_log2_size - 1, next_page, next_sibling); + if (log2_page_count > 0) { + auto left = compute_root_hash_impl(page_index, log2_page_count - 1, next_page, next_sibling); + const auto halfway_page_index = page_index + (page_count >> 1); + auto right = compute_root_hash_impl(halfway_page_index, log2_page_count - 1, next_page, next_sibling); hash_type hash{}; interop_concat_hash(reinterpret_cast(&left), reinterpret_cast(&right), reinterpret_cast(&hash)); @@ -389,434 +404,362 @@ class replay_step_state_access : public i_state_access return m_context.pages[next_page++].hash; } if (next_sibling >= m_context.sibling_count) { - interop_throw_runtime_error("compute_root_hash_impl: trying to access beyond sibling count"); + interop_throw_runtime_error("too few sibling hashes in log"); } return m_context.sibling_hashes[next_sibling++]; // NOLINTEND(cppcoreguidelines-pro-type-reinterpret-cast)) } +#ifdef DUMP_STATE_ACCESS // NOLINTNEXTLINE(readability-convert-member-functions-to-static) - void do_push_bracket(bracket_type type, const char *text) { - (void) type; - (void) text; - } - - int do_make_scoped_note(const char *text) { // NOLINT(readability-convert-member-functions-to-static) - (void) text; - return 0; + auto do_make_scoped_note([[maybe_unused]] const char *text) { + return scoped_note{*this, text}; } +#endif //??D we should probably optimize access to the shadow so it doesn't perform a translation every time // We can do this by caching the vh_offset trasnslation of the processor shadow page. This is easy if // static_assert(sizeof(shadow_state) <= PMA_PAGE_SIZE, "shadow state must fit in single page"); - uint64_t do_read_x(int reg) { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::x0, reg)); + uint64_t check_read_reg(machine_reg reg) const { + const auto haddr = do_get_faddr(machine_reg_address(reg)); return aliased_aligned_read(haddr); } - void do_write_x(int reg, uint64_t val) { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::x0, reg)); + //??D we should probably optimize access to the shadow so it doesn't perform a translation every time + // We can do this by caching the vh_offset trasnslation of the processor shadow page. This is easy if + // static_assert(sizeof(shadow_state) <= PMA_PAGE_SIZE, "shadow state must fit in single page"); + void check_write_reg(machine_reg reg, uint64_t val) { + const auto haddr = do_get_faddr(machine_reg_address(reg)); aliased_aligned_write(haddr, val); } - uint64_t do_read_f(int reg) { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::f0, reg)); - return aliased_aligned_read(haddr); + uint64_t do_read_x(int i) { + return check_read_reg(machine_reg_enum(machine_reg::x0, i)); } - void do_write_f(int reg, uint64_t val) { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::f0, reg)); - aliased_aligned_write(haddr, val); + void do_write_x(int i, uint64_t val) { + assert(i != 0); + check_write_reg(machine_reg_enum(machine_reg::x0, i), val); + } + + uint64_t do_read_f(int i) { + return check_read_reg(machine_reg_enum(machine_reg::f0, i)); + } + + void do_write_f(int i, uint64_t val) { + check_write_reg(machine_reg_enum(machine_reg::f0, i), val); } uint64_t do_read_pc() { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::pc)); - return aliased_aligned_read(haddr); + return check_read_reg(machine_reg::pc); } void do_write_pc(uint64_t val) { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::pc)); - aliased_aligned_write(haddr, val); + check_write_reg(machine_reg::pc, val); } uint64_t do_read_fcsr() { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::fcsr)); - return aliased_aligned_read(haddr); + return check_read_reg(machine_reg::fcsr); } void do_write_fcsr(uint64_t val) { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::fcsr)); - aliased_aligned_write(haddr, val); + check_write_reg(machine_reg::fcsr, val); } uint64_t do_read_icycleinstret() { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::icycleinstret)); - return aliased_aligned_read(haddr); + return check_read_reg(machine_reg::icycleinstret); } void do_write_icycleinstret(uint64_t val) { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::icycleinstret)); - aliased_aligned_write(haddr, val); + check_write_reg(machine_reg::icycleinstret, val); } uint64_t do_read_mvendorid() { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::mvendorid)); - return aliased_aligned_read(haddr); + return check_read_reg(machine_reg::mvendorid); } uint64_t do_read_marchid() { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::marchid)); - return aliased_aligned_read(haddr); + return check_read_reg(machine_reg::marchid); } uint64_t do_read_mimpid() { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::mimpid)); - return aliased_aligned_read(haddr); + return check_read_reg(machine_reg::mimpid); } uint64_t do_read_mcycle() { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::mcycle)); - return aliased_aligned_read(haddr); + return check_read_reg(machine_reg::mcycle); } void do_write_mcycle(uint64_t val) { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::mcycle)); - aliased_aligned_write(haddr, val); + check_write_reg(machine_reg::mcycle, val); } uint64_t do_read_mstatus() { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::mstatus)); - return aliased_aligned_read(haddr); + return check_read_reg(machine_reg::mstatus); } void do_write_mstatus(uint64_t val) { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::mstatus)); - aliased_aligned_write(haddr, val); + check_write_reg(machine_reg::mstatus, val); } uint64_t do_read_mtvec() { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::mtvec)); - return aliased_aligned_read(haddr); + return check_read_reg(machine_reg::mtvec); } void do_write_mtvec(uint64_t val) { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::mtvec)); - aliased_aligned_write(haddr, val); + check_write_reg(machine_reg::mtvec, val); } uint64_t do_read_mscratch() { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::mscratch)); - return aliased_aligned_read(haddr); + return check_read_reg(machine_reg::mscratch); } void do_write_mscratch(uint64_t val) { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::mscratch)); - aliased_aligned_write(haddr, val); + check_write_reg(machine_reg::mscratch, val); } uint64_t do_read_mepc() { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::mepc)); - return aliased_aligned_read(haddr); + return check_read_reg(machine_reg::mepc); } void do_write_mepc(uint64_t val) { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::mepc)); - aliased_aligned_write(haddr, val); + check_write_reg(machine_reg::mepc, val); } uint64_t do_read_mcause() { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::mcause)); - return aliased_aligned_read(haddr); + return check_read_reg(machine_reg::mcause); } void do_write_mcause(uint64_t val) { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::mcause)); - aliased_aligned_write(haddr, val); + check_write_reg(machine_reg::mcause, val); } uint64_t do_read_mtval() { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::mtval)); - return aliased_aligned_read(haddr); + return check_read_reg(machine_reg::mtval); } void do_write_mtval(uint64_t val) { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::mtval)); - aliased_aligned_write(haddr, val); + check_write_reg(machine_reg::mtval, val); } uint64_t do_read_misa() { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::misa)); - return aliased_aligned_read(haddr); + return check_read_reg(machine_reg::misa); } void do_write_misa(uint64_t val) { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::misa)); - aliased_aligned_write(haddr, val); + check_write_reg(machine_reg::misa, val); } uint64_t do_read_mie() { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::mie)); - return aliased_aligned_read(haddr); + return check_read_reg(machine_reg::mie); } void do_write_mie(uint64_t val) { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::mie)); - aliased_aligned_write(haddr, val); + check_write_reg(machine_reg::mie, val); } uint64_t do_read_mip() { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::mip)); - return aliased_aligned_read(haddr); + return check_read_reg(machine_reg::mip); } void do_write_mip(uint64_t val) { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::mip)); - aliased_aligned_write(haddr, val); + check_write_reg(machine_reg::mip, val); } uint64_t do_read_medeleg() { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::medeleg)); - return aliased_aligned_read(haddr); + return check_read_reg(machine_reg::medeleg); } void do_write_medeleg(uint64_t val) { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::medeleg)); - aliased_aligned_write(haddr, val); + check_write_reg(machine_reg::medeleg, val); } uint64_t do_read_mideleg() { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::mideleg)); - return aliased_aligned_read(haddr); + return check_read_reg(machine_reg::mideleg); } void do_write_mideleg(uint64_t val) { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::mideleg)); - aliased_aligned_write(haddr, val); + check_write_reg(machine_reg::mideleg, val); } uint64_t do_read_mcounteren() { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::mcounteren)); - return aliased_aligned_read(haddr); + return check_read_reg(machine_reg::mcounteren); } void do_write_mcounteren(uint64_t val) { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::mcounteren)); - aliased_aligned_write(haddr, val); + check_write_reg(machine_reg::mcounteren, val); } - uint64_t do_read_senvcfg() const { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::senvcfg)); - return aliased_aligned_read(haddr); + uint64_t do_read_senvcfg() { + return check_read_reg(machine_reg::senvcfg); } void do_write_senvcfg(uint64_t val) { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::senvcfg)); - aliased_aligned_write(haddr, val); + check_write_reg(machine_reg::senvcfg, val); } - uint64_t do_read_menvcfg() const { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::menvcfg)); - return aliased_aligned_read(haddr); + uint64_t do_read_menvcfg() { + return check_read_reg(machine_reg::menvcfg); } void do_write_menvcfg(uint64_t val) { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::menvcfg)); - aliased_aligned_write(haddr, val); + check_write_reg(machine_reg::menvcfg, val); } uint64_t do_read_stvec() { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::stvec)); - return aliased_aligned_read(haddr); + return check_read_reg(machine_reg::stvec); } void do_write_stvec(uint64_t val) { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::stvec)); - aliased_aligned_write(haddr, val); + check_write_reg(machine_reg::stvec, val); } uint64_t do_read_sscratch() { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::sscratch)); - return aliased_aligned_read(haddr); + return check_read_reg(machine_reg::sscratch); } void do_write_sscratch(uint64_t val) { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::sscratch)); - aliased_aligned_write(haddr, val); + check_write_reg(machine_reg::sscratch, val); } uint64_t do_read_sepc() { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::sepc)); - return aliased_aligned_read(haddr); + return check_read_reg(machine_reg::sepc); } void do_write_sepc(uint64_t val) { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::sepc)); - aliased_aligned_write(haddr, val); + check_write_reg(machine_reg::sepc, val); } uint64_t do_read_scause() { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::scause)); - return aliased_aligned_read(haddr); + return check_read_reg(machine_reg::scause); } void do_write_scause(uint64_t val) { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::scause)); - aliased_aligned_write(haddr, val); + check_write_reg(machine_reg::scause, val); } uint64_t do_read_stval() { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::stval)); - return aliased_aligned_read(haddr); + return check_read_reg(machine_reg::stval); } void do_write_stval(uint64_t val) { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::stval)); - aliased_aligned_write(haddr, val); + check_write_reg(machine_reg::stval, val); } uint64_t do_read_satp() { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::satp)); - return aliased_aligned_read(haddr); + return check_read_reg(machine_reg::satp); } void do_write_satp(uint64_t val) { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::satp)); - aliased_aligned_write(haddr, val); + check_write_reg(machine_reg::satp, val); } uint64_t do_read_scounteren() { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::scounteren)); - return aliased_aligned_read(haddr); + return check_read_reg(machine_reg::scounteren); } void do_write_scounteren(uint64_t val) { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::scounteren)); - aliased_aligned_write(haddr, val); + check_write_reg(machine_reg::scounteren, val); } uint64_t do_read_ilrsc() { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::ilrsc)); - return aliased_aligned_read(haddr); + return check_read_reg(machine_reg::ilrsc); } void do_write_ilrsc(uint64_t val) { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::ilrsc)); - aliased_aligned_write(haddr, val); + check_write_reg(machine_reg::ilrsc, val); } uint64_t do_read_iprv() { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::iprv)); - return aliased_aligned_read(haddr); + return check_read_reg(machine_reg::iprv); } void do_write_iprv(uint64_t val) { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::iprv)); - aliased_aligned_write(haddr, val); + check_write_reg(machine_reg::iprv, val); } uint64_t do_read_iflags_X() { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::iflags_X)); - return aliased_aligned_read(haddr); + return check_read_reg(machine_reg::iflags_X); } void do_write_iflags_X(uint64_t val) { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::iflags_X)); - aliased_aligned_write(haddr, val); + check_write_reg(machine_reg::iflags_X, val); } uint64_t do_read_iflags_Y() { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::iflags_Y)); - return aliased_aligned_read(haddr); + return check_read_reg(machine_reg::iflags_Y); } void do_write_iflags_Y(uint64_t val) { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::iflags_Y)); - aliased_aligned_write(haddr, val); + check_write_reg(machine_reg::iflags_Y, val); } uint64_t do_read_iflags_H() { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::iflags_H)); - return aliased_aligned_read(haddr); + return check_read_reg(machine_reg::iflags_H); } void do_write_iflags_H(uint64_t val) { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::iflags_H)); - aliased_aligned_write(haddr, val); + check_write_reg(machine_reg::iflags_H, val); } uint64_t do_read_iunrep() { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::iunrep)); - return aliased_aligned_read(haddr); + return check_read_reg(machine_reg::iunrep); } void do_write_iunrep(uint64_t val) { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::iunrep)); - aliased_aligned_write(haddr, val); + check_write_reg(machine_reg::iunrep, val); } uint64_t do_read_clint_mtimecmp() { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::clint_mtimecmp)); - return aliased_aligned_read(haddr); + return check_read_reg(machine_reg::clint_mtimecmp); } void do_write_clint_mtimecmp(uint64_t val) { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::clint_mtimecmp)); - aliased_aligned_write(haddr, val); + check_write_reg(machine_reg::clint_mtimecmp, val); } uint64_t do_read_plic_girqpend() { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::plic_girqpend)); - return aliased_aligned_read(haddr); + return check_read_reg(machine_reg::plic_girqpend); } void do_write_plic_girqpend(uint64_t val) { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::plic_girqpend)); - aliased_aligned_write(haddr, val); + check_write_reg(machine_reg::plic_girqpend, val); } uint64_t do_read_plic_girqsrvd() { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::plic_girqsrvd)); - return aliased_aligned_read(haddr); + return check_read_reg(machine_reg::plic_girqsrvd); } void do_write_plic_girqsrvd(uint64_t val) { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::plic_girqsrvd)); - aliased_aligned_write(haddr, val); + check_write_reg(machine_reg::plic_girqsrvd, val); } uint64_t do_read_htif_fromhost() { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::htif_fromhost)); - return aliased_aligned_read(haddr); + return check_read_reg(machine_reg::htif_fromhost); } void do_write_htif_fromhost(uint64_t val) { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::htif_fromhost)); - aliased_aligned_write(haddr, val); + check_write_reg(machine_reg::htif_fromhost, val); } uint64_t do_read_htif_tohost() { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::htif_tohost)); - return aliased_aligned_read(haddr); + return check_read_reg(machine_reg::htif_tohost); } void do_write_htif_tohost(uint64_t val) { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::htif_tohost)); - aliased_aligned_write(haddr, val); + check_write_reg(machine_reg::htif_tohost, val); } uint64_t do_read_htif_ihalt() { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::htif_ihalt)); - return aliased_aligned_read(haddr); + return check_read_reg(machine_reg::htif_ihalt); } uint64_t do_read_htif_iconsole() { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::htif_iconsole)); - return aliased_aligned_read(haddr); + return check_read_reg(machine_reg::htif_iconsole); } uint64_t do_read_htif_iyield() { - const auto haddr = do_get_faddr(machine_reg_address(machine_reg::htif_iyield)); - return aliased_aligned_read(haddr); + return check_read_reg(machine_reg::htif_iyield); } // NOLINTNEXTLINE(readability-convert-member-functions-to-static) @@ -836,12 +779,12 @@ class replay_step_state_access : public i_state_access } uint64_t read_pma_istart(uint64_t index) { - const auto haddr = do_get_faddr(shadow_pmas_get_pma_istart_abs_addr(index)); + const auto haddr = do_get_faddr(shadow_pmas_get_pma_abs_addr(index, shadow_pmas_what::istart)); return aliased_aligned_read(haddr); } uint64_t read_pma_ilength(uint64_t index) { - const auto haddr = do_get_faddr(shadow_pmas_get_pma_ilength_abs_addr(index)); + const auto haddr = do_get_faddr(shadow_pmas_get_pma_abs_addr(index, shadow_pmas_what::ilength)); return aliased_aligned_read(haddr); } @@ -871,32 +814,38 @@ class replay_step_state_access : public i_state_access aliased_aligned_write(haddr, val); } - template + template + auto check_read_tlb(TLB_set_index set_index, uint64_t slot_index, shadow_tlb_what what) { + const auto haddr = do_get_faddr(shadow_tlb_get_abs_addr(set_index, slot_index, what)); + return aliased_aligned_read(haddr); + } + + template uint64_t do_read_tlb_vaddr_page(uint64_t slot_index) { - const auto haddr = do_get_faddr(shadow_tlb_get_vaddr_page_abs_addr(slot_index)); - return aliased_aligned_read(haddr); + return check_read_tlb(SET, slot_index, shadow_tlb_what::vaddr_page); } - template + template host_addr do_read_tlb_vp_offset(uint64_t slot_index) { - const auto haddr = do_get_faddr(shadow_tlb_get_vp_offset_abs_addr(slot_index)); - return aliased_aligned_read(haddr); + return check_read_tlb(SET, slot_index, shadow_tlb_what::vp_offset); } - template + template uint64_t do_read_tlb_pma_index(uint64_t slot_index) { - const auto haddr = do_get_faddr(shadow_tlb_get_pma_index_abs_addr(slot_index)); - return aliased_aligned_read(haddr); + return check_read_tlb(SET, slot_index, shadow_tlb_what::pma_index); } - template + template + auto check_write_tlb(TLB_set_index set_index, uint64_t slot_index, shadow_tlb_what what, TYPE val) { + const auto haddr = do_get_faddr(shadow_tlb_get_abs_addr(set_index, slot_index, what)); + aliased_aligned_write(haddr, val); + } + + template void do_write_tlb(uint64_t slot_index, uint64_t vaddr_page, host_addr vh_offset, uint64_t pma_index) { - const auto haddr_vaddr_page = do_get_faddr(shadow_tlb_get_vaddr_page_abs_addr(slot_index)); - aliased_aligned_write(haddr_vaddr_page, vaddr_page); - const auto haddr_vp_offset = do_get_faddr(shadow_tlb_get_vp_offset_abs_addr(slot_index)); - aliased_aligned_write(haddr_vp_offset, vh_offset); - const auto haddr_pma_index = do_get_faddr(shadow_tlb_get_pma_index_abs_addr(slot_index)); - aliased_aligned_write(haddr_pma_index, pma_index); + check_write_tlb(SET, slot_index, shadow_tlb_what::vaddr_page, vaddr_page); + check_write_tlb(SET, slot_index, shadow_tlb_what::vp_offset, vh_offset); + check_write_tlb(SET, slot_index, shadow_tlb_what::pma_index, pma_index); } void do_mark_dirty_page(host_addr /* haddr */, uint64_t /* pma_index */) { @@ -904,11 +853,9 @@ class replay_step_state_access : public i_state_access } // NOLINTNEXTLINE(readability-convert-member-functions-to-static) - std::pair do_poll_external_interrupts(uint64_t mcycle, uint64_t mcycle_max) { - (void) mcycle_max; - return {mcycle, false}; + constexpr const char *do_get_name() const { // NOLINT(readability-convert-member-functions-to-static) + return "replay_step_state_access"; } - }; } // namespace cartesi diff --git a/src/scoped-note.h b/src/scoped-note.h new file mode 100644 index 000000000..b5286b092 --- /dev/null +++ b/src/scoped-note.h @@ -0,0 +1,52 @@ +// Copyright Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: LGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify it under +// the terms of the GNU Lesser General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) any +// later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +// PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License along +// with this program (see COPYING). If not, see . +// + +#ifndef SCOPED_NOTE_H +#define SCOPED_NOTE_H + +namespace cartesi { + +/// \brief Adds annotations to the state, bracketing a scope +template +class scoped_note { + + STATE_ACCESS &m_a; + const char *const m_text; ///< String with the text for the annotation + +public: + /// \brief Constructor adds the "begin" bracketing note + /// \param a State access receiving annotations + /// \param text Pointer to annotation text (must be valid until destruction) + /// \details A note is added at the moment of construction and destruction + scoped_note(STATE_ACCESS &a, const char *text) : m_a{a}, m_text(text) { + m_a.push_begin_bracket(m_text); + } + + /// \brief No assignments/copies + scoped_note(const scoped_note &) = delete; + scoped_note &operator=(const scoped_note &) = delete; + scoped_note(scoped_note &&) = delete; + scoped_note &operator=(scoped_note &&) = delete; + + /// \brief Destructor adds the "end" bracketing note + ~scoped_note() { + m_a.push_end_bracket(m_text); + } +}; + +} // namespace cartesi + +#endif // SCOPED_NOTE_H diff --git a/src/shadow-peek.h b/src/shadow-peek.h new file mode 100644 index 000000000..9275edfda --- /dev/null +++ b/src/shadow-peek.h @@ -0,0 +1,68 @@ +// Copyright Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: LGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify it under +// the terms of the GNU Lesser General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) any +// later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +// PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License along +// with this program (see COPYING). If not, see . +// + +#ifndef SHADOW_PEEK_H +#define SHADOW_PEEK_H + +#include +#include + +#include "machine-reg.h" +#include "machine.h" +#include "pma.h" + +namespace cartesi { + +/// \file +/// \brief Peeks part of a shadow +template +static bool shadow_peek(READ_REG_F read_reg, const pma_entry &pma, const machine &m, uint64_t offset, uint64_t length, + const unsigned char **data, unsigned char *scratch) { + // Must be inside range + if (!pma.contains(pma.get_start() + offset, length)) { + *data = nullptr; + return false; + } + // Initial, potentially partial register read + const uint64_t offset_aligned = offset & ~(sizeof(uint64_t) - 1); + if (offset > offset_aligned) { + const auto val = read_reg(m, pma.get_start() + offset_aligned); + const auto partial = offset - offset_aligned; + memcpy(scratch, &val, partial); + length -= partial; + offset += partial; + } + // Now we are aligned, do all complete registers + const uint64_t paddr_start = pma.get_start() + offset; + const uint64_t length_aligned = length & ~(sizeof(uint64_t) - 1); + const uint64_t paddr_end = paddr_start + length_aligned; + for (uint64_t paddr = paddr_start; paddr < paddr_end; paddr += sizeof(uint64_t)) { + const auto val = read_reg(m, paddr); + aliased_aligned_write(scratch + (paddr - paddr_start), val); + } + // Final, potentially partial register read + if (length > length_aligned) { + const auto partial = length - length_aligned; + const auto val = read_reg(m, paddr_end); + memcpy(scratch + length_aligned, &val, partial); + } + *data = scratch; + return true; +} + +} // namespace cartesi + +#endif diff --git a/src/shadow-pmas-factory.h b/src/shadow-pmas-factory.h index 35514872f..008f37bed 100644 --- a/src/shadow-pmas-factory.h +++ b/src/shadow-pmas-factory.h @@ -31,11 +31,15 @@ pma_entry make_shadow_pmas_pma_entry(uint64_t start, uint64_t length); template void populate_shadow_pmas_state(const PMAS &pmas, shadow_pmas_state *shadow) { - static_assert(PMA_SHADOW_PMAS_LENGTH >= sizeof(shadow_pmas_state), "shadow PMAs length is too small"); + static_assert(sizeof(shadow_pmas_state) == PMA_MAX * 2 * sizeof(uint64_t), "inconsistent shadow PMAs length"); + static_assert(PMA_SHADOW_PMAS_LENGTH >= sizeof(shadow_pmas_state), "shadow PMAs not long enough"); + if (pmas.size() > PMA_MAX) { + throw std::invalid_argument{"more PMAs than shadow PMAs can hold"}; + } unsigned index = 0; for (const auto &pma : pmas) { - shadow->pmas[index].istart = pma.get_istart(); - shadow->pmas[index].ilength = pma.get_ilength(); + (*shadow)[index].istart = pma.get_istart(); + (*shadow)[index].ilength = pma.get_ilength(); ++index; } } diff --git a/src/shadow-pmas.h b/src/shadow-pmas.h index 60a2df6df..662d0fcd5 100644 --- a/src/shadow-pmas.h +++ b/src/shadow-pmas.h @@ -17,6 +17,7 @@ #ifndef SHADOW_PMAS_H #define SHADOW_PMAS_H +#include #include #include @@ -36,31 +37,60 @@ struct PACKED shadow_pma_entry { uint64_t ilength; }; -struct PACKED shadow_pmas_state { - shadow_pma_entry pmas[PMA_MAX]; +using shadow_pmas_state = std::array; + +/// \brief List of field types +enum class shadow_pmas_what : uint64_t { + istart = offsetof(shadow_pma_entry, istart), + ilength = offsetof(shadow_pma_entry, ilength), + unknown_ = UINT64_C(1) << 63, // Outside of RISC-V address space }; -/// \brief Obtains the relative address of a PMA entry in shadow memory. -/// \param p Index of desired shadow PMA entry, in 0..31. +/// \brief Obtains the absolute address of a PMA entry in shadow memory. +/// \param p Index of desired shadow PMA entry /// \returns The address. -static inline uint64_t shadow_pmas_get_pma_rel_addr(uint64_t p) { - assert(p < (int) PMA_MAX); - return p * sizeof(shadow_pma_entry); +static constexpr uint64_t shadow_pmas_get_pma_abs_addr(uint64_t p) { + return PMA_SHADOW_PMAS_START + p * sizeof(shadow_pma_entry); } /// \brief Obtains the absolute address of a PMA entry in shadow memory. -static inline uint64_t shadow_pmas_get_pma_abs_addr(uint64_t p) { - return PMA_SHADOW_PMAS_START + shadow_pmas_get_pma_rel_addr(p); +/// \param p Index of desired shadow PMA entry +/// \param what Desired field +/// \returns The address. +static constexpr uint64_t shadow_pmas_get_pma_abs_addr(uint64_t p, shadow_pmas_what what) { + return shadow_pmas_get_pma_abs_addr(p) + static_cast(what); } -/// \brief Obtains the absolute address of the istart field in a PMA entry in shadow memory. -static inline uint64_t shadow_pmas_get_pma_istart_abs_addr(uint64_t p) { - return shadow_pmas_get_pma_abs_addr(p) + offsetof(shadow_pma_entry, istart); +static constexpr shadow_pmas_what shadow_pmas_get_what(uint64_t paddr) { + if (paddr < PMA_SHADOW_PMAS_START || paddr - PMA_SHADOW_PMAS_START >= sizeof(shadow_pmas_state) || + (paddr & (sizeof(uint64_t) - 1)) != 0) { + return shadow_pmas_what::unknown_; + } + //??D First condition ensures offset = (paddr-PMA_SHADOW_PMAS_START) >= 0 + //??D Second ensures offset < sizeof(shadow_pmas_state) + //??D Third ensures offset is aligned to sizeof(uint64_t) + //??D shadow_pma_entry only contains uint64_t fields + //??D shadow_pmas_state_what contains one entry with the offset of each field in shadow_pma_entry + //??D I don't see how the cast can produce something outside the enum... + // NOLINTNEXTLINE(clang-analyzer-optin.core.EnumCastOutOfRange) + return shadow_pmas_what{(paddr - PMA_SHADOW_PMAS_START) % sizeof(shadow_pma_entry)}; } -/// \brief Obtains the absolute address of the ilength field in a PMA entry in shadow memory. -static inline uint64_t shadow_pmas_get_pma_ilength_abs_addr(uint64_t p) { - return shadow_pmas_get_pma_abs_addr(p) + offsetof(shadow_pma_entry, ilength); +static constexpr const char *shadow_pmas_get_what_name(shadow_pmas_what what) { + const auto paddr = static_cast(what); + if (paddr < PMA_SHADOW_PMAS_START || paddr - PMA_SHADOW_PMAS_START >= sizeof(shadow_pmas_state) || + (paddr & (sizeof(uint64_t) - 1)) != 0) { + return "pma.unknown"; + } + using reg = shadow_pmas_what; + switch (what) { + case reg::istart: + return "pma.istart"; + case reg::ilength: + return "pma.ilength"; + case reg::unknown_: + return "pma.unknown"; + } } } // namespace cartesi diff --git a/src/shadow-state-factory.cpp b/src/shadow-state-factory.cpp index d055a63da..e87c975f1 100644 --- a/src/shadow-state-factory.cpp +++ b/src/shadow-state-factory.cpp @@ -14,91 +14,36 @@ // with this program (see COPYING). If not, see . // -#include "shadow-state-factory.h" - #include -#include #include "machine.h" #include "pma-constants.h" #include "pma.h" -#include "riscv-constants.h" +#include "shadow-peek.h" +#include "shadow-state-factory.h" #include "shadow-state.h" namespace cartesi { /// \brief Shadow device peek callback. See ::pma_peek. -static bool shadow_state_peek(const pma_entry & /*pma*/, const machine &m, uint64_t page_offset, - const unsigned char **page_data, unsigned char *scratch) { - static_assert(sizeof(shadow_state) <= PMA_PAGE_SIZE); - - // There is only one page: 0 - if (page_offset != 0) { - *page_data = nullptr; - return false; - } - // Clear page - memset(scratch, 0, PMA_PAGE_SIZE); - - // The page will reflect the shadow structure - // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) - auto *s = reinterpret_cast(scratch); - - // Copy general-purpose registers - for (int i = 0; i < X_REG_COUNT; ++i) { - s->x[i] = m.read_reg(machine_reg_enum(machine_reg::x0, i)); +static bool shadow_state_peek(const pma_entry &pma, const machine &m, uint64_t offset, uint64_t length, + const unsigned char **data, unsigned char *scratch) { + // If past useful range + if (offset >= sizeof(shadow_state)) { + *data = nullptr; + return length <= pma.get_length() && offset <= pma.get_length() - length; } - - // Copy floating-point registers - for (int i = 0; i < F_REG_COUNT; ++i) { - s->f[i] = m.read_reg(machine_reg_enum(machine_reg::f0, i)); - } - - // Copy named registers - s->pc = m.read_reg(machine_reg::pc); - s->fcsr = m.read_reg(machine_reg::fcsr); - s->mvendorid = m.read_reg(machine_reg::mvendorid); - s->marchid = m.read_reg(machine_reg::marchid); - s->mimpid = m.read_reg(machine_reg::mimpid); - s->mcycle = m.read_reg(machine_reg::mcycle); - s->icycleinstret = m.read_reg(machine_reg::icycleinstret); - s->mstatus = m.read_reg(machine_reg::mstatus); - s->mtvec = m.read_reg(machine_reg::mtvec); - s->mscratch = m.read_reg(machine_reg::mscratch); - s->mepc = m.read_reg(machine_reg::mepc); - s->mcause = m.read_reg(machine_reg::mcause); - s->mtval = m.read_reg(machine_reg::mtval); - s->misa = m.read_reg(machine_reg::misa); - s->mie = m.read_reg(machine_reg::mie); - s->mip = m.read_reg(machine_reg::mip); - s->medeleg = m.read_reg(machine_reg::medeleg); - s->mideleg = m.read_reg(machine_reg::mideleg); - s->mcounteren = m.read_reg(machine_reg::mcounteren); - s->menvcfg = m.read_reg(machine_reg::menvcfg); - s->stvec = m.read_reg(machine_reg::stvec); - s->sscratch = m.read_reg(machine_reg::sscratch); - s->sepc = m.read_reg(machine_reg::sepc); - s->scause = m.read_reg(machine_reg::scause); - s->stval = m.read_reg(machine_reg::stval); - s->satp = m.read_reg(machine_reg::satp); - s->scounteren = m.read_reg(machine_reg::scounteren); - s->senvcfg = m.read_reg(machine_reg::senvcfg); - s->ilrsc = m.read_reg(machine_reg::ilrsc); - s->iprv = m.read_reg(machine_reg::iprv); - s->iflags_X = m.read_reg(machine_reg::iflags_X); - s->iflags_Y = m.read_reg(machine_reg::iflags_Y); - s->iflags_H = m.read_reg(machine_reg::iflags_H); - s->iunrep = m.read_reg(machine_reg::iunrep); - s->clint_mtimecmp = m.read_reg(machine_reg::clint_mtimecmp); - s->plic_girqpend = m.read_reg(machine_reg::plic_girqpend); - s->plic_girqsrvd = m.read_reg(machine_reg::plic_girqsrvd); - s->htif_tohost = m.read_reg(machine_reg::htif_tohost); - s->htif_fromhost = m.read_reg(machine_reg::htif_fromhost); - s->htif_ihalt = m.read_reg(machine_reg::htif_ihalt); - s->htif_iconsole = m.read_reg(machine_reg::htif_iconsole); - s->htif_iyield = m.read_reg(machine_reg::htif_iyield); - *page_data = scratch; - return true; + // Otherwise, copy and return register data + return shadow_peek( + [](const machine &m, uint64_t paddr) { + auto reg = shadow_state_get_what(paddr); + uint64_t val = 0; + if (reg != shadow_state_what::unknown_) { + val = m.read_reg(machine_reg_enum(reg)); + } + return val; + }, + pma, m, offset, length, data, scratch); } pma_entry make_shadow_state_pma_entry(uint64_t start, uint64_t length) { @@ -108,8 +53,7 @@ pma_entry make_shadow_state_pma_entry(uint64_t start, uint64_t length) { .IR = false, .IW = false, .DID = PMA_ISTART_DID::shadow_state}; - return make_device_pma_entry("shadow state device", start, length, shadow_state_peek, &shadow_state_driver) - .set_flags(f); + return make_device_pma_entry("shadow state", start, length, shadow_state_peek, &shadow_state_driver).set_flags(f); } } // namespace cartesi diff --git a/src/shadow-state.h b/src/shadow-state.h index 720112679..0d22ea2c6 100644 --- a/src/shadow-state.h +++ b/src/shadow-state.h @@ -79,6 +79,351 @@ struct PACKED shadow_state { uint64_t htif_iyield; }; +enum class shadow_state_what : uint64_t { + x0 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[0]), + x1 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[1]), + x2 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[2]), + x3 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[3]), + x4 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[4]), + x5 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[5]), + x6 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[6]), + x7 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[7]), + x8 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[8]), + x9 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[9]), + x10 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[10]), + x11 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[11]), + x12 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[12]), + x13 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[13]), + x14 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[14]), + x15 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[15]), + x16 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[16]), + x17 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[17]), + x18 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[18]), + x19 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[19]), + x20 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[20]), + x21 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[21]), + x22 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[22]), + x23 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[23]), + x24 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[24]), + x25 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[25]), + x26 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[26]), + x27 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[27]), + x28 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[28]), + x29 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[29]), + x30 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[30]), + x31 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[31]), + f0 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[0]), + f1 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[1]), + f2 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[2]), + f3 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[3]), + f4 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[4]), + f5 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[5]), + f6 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[6]), + f7 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[7]), + f8 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[8]), + f9 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[9]), + f10 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[10]), + f11 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[11]), + f12 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[12]), + f13 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[13]), + f14 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[14]), + f15 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[15]), + f16 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[16]), + f17 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[17]), + f18 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[18]), + f19 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[19]), + f20 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[20]), + f21 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[21]), + f22 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[22]), + f23 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[23]), + f24 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[24]), + f25 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[25]), + f26 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[26]), + f27 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[27]), + f28 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[28]), + f29 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[29]), + f30 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[30]), + f31 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[31]), + pc = PMA_SHADOW_STATE_START + offsetof(shadow_state, pc), + fcsr = PMA_SHADOW_STATE_START + offsetof(shadow_state, fcsr), + mvendorid = PMA_SHADOW_STATE_START + offsetof(shadow_state, mvendorid), + marchid = PMA_SHADOW_STATE_START + offsetof(shadow_state, marchid), + mimpid = PMA_SHADOW_STATE_START + offsetof(shadow_state, mimpid), + mcycle = PMA_SHADOW_STATE_START + offsetof(shadow_state, mcycle), + icycleinstret = PMA_SHADOW_STATE_START + offsetof(shadow_state, icycleinstret), + mstatus = PMA_SHADOW_STATE_START + offsetof(shadow_state, mstatus), + mtvec = PMA_SHADOW_STATE_START + offsetof(shadow_state, mtvec), + mscratch = PMA_SHADOW_STATE_START + offsetof(shadow_state, mscratch), + mepc = PMA_SHADOW_STATE_START + offsetof(shadow_state, mepc), + mcause = PMA_SHADOW_STATE_START + offsetof(shadow_state, mcause), + mtval = PMA_SHADOW_STATE_START + offsetof(shadow_state, mtval), + misa = PMA_SHADOW_STATE_START + offsetof(shadow_state, misa), + mie = PMA_SHADOW_STATE_START + offsetof(shadow_state, mie), + mip = PMA_SHADOW_STATE_START + offsetof(shadow_state, mip), + medeleg = PMA_SHADOW_STATE_START + offsetof(shadow_state, medeleg), + mideleg = PMA_SHADOW_STATE_START + offsetof(shadow_state, mideleg), + mcounteren = PMA_SHADOW_STATE_START + offsetof(shadow_state, mcounteren), + menvcfg = PMA_SHADOW_STATE_START + offsetof(shadow_state, menvcfg), + stvec = PMA_SHADOW_STATE_START + offsetof(shadow_state, stvec), + sscratch = PMA_SHADOW_STATE_START + offsetof(shadow_state, sscratch), + sepc = PMA_SHADOW_STATE_START + offsetof(shadow_state, sepc), + scause = PMA_SHADOW_STATE_START + offsetof(shadow_state, scause), + stval = PMA_SHADOW_STATE_START + offsetof(shadow_state, stval), + satp = PMA_SHADOW_STATE_START + offsetof(shadow_state, satp), + scounteren = PMA_SHADOW_STATE_START + offsetof(shadow_state, scounteren), + senvcfg = PMA_SHADOW_STATE_START + offsetof(shadow_state, senvcfg), + ilrsc = PMA_SHADOW_STATE_START + offsetof(shadow_state, ilrsc), + iprv = PMA_SHADOW_STATE_START + offsetof(shadow_state, iprv), + iflags_X = PMA_SHADOW_STATE_START + offsetof(shadow_state, iflags_X), + iflags_Y = PMA_SHADOW_STATE_START + offsetof(shadow_state, iflags_Y), + iflags_H = PMA_SHADOW_STATE_START + offsetof(shadow_state, iflags_H), + iunrep = PMA_SHADOW_STATE_START + offsetof(shadow_state, iunrep), + clint_mtimecmp = PMA_SHADOW_STATE_START + offsetof(shadow_state, clint_mtimecmp), + plic_girqpend = PMA_SHADOW_STATE_START + offsetof(shadow_state, plic_girqpend), + plic_girqsrvd = PMA_SHADOW_STATE_START + offsetof(shadow_state, plic_girqsrvd), + htif_tohost = PMA_SHADOW_STATE_START + offsetof(shadow_state, htif_tohost), + htif_fromhost = PMA_SHADOW_STATE_START + offsetof(shadow_state, htif_fromhost), + htif_ihalt = PMA_SHADOW_STATE_START + offsetof(shadow_state, htif_ihalt), + htif_iconsole = PMA_SHADOW_STATE_START + offsetof(shadow_state, htif_iconsole), + htif_iyield = PMA_SHADOW_STATE_START + offsetof(shadow_state, htif_iyield), + unknown_ = UINT64_C(1) << 63, // Outside of RISC-V address space +}; + +static constexpr shadow_state_what shadow_state_get_what(uint64_t paddr) { + if (paddr < PMA_SHADOW_STATE_START || paddr - PMA_SHADOW_STATE_START >= sizeof(shadow_state) || + (paddr & (sizeof(uint64_t) - 1)) != 0) { + return shadow_state_what::unknown_; + } + return static_cast(paddr); +} + +static constexpr const char *shadow_state_get_what_name(shadow_state_what what) { + const auto paddr = static_cast(what); + if (paddr < PMA_SHADOW_STATE_START || paddr - PMA_SHADOW_STATE_START >= sizeof(shadow_state) || + (paddr & (sizeof(uint64_t) - 1)) != 0) { + return "state.unknown"; + } + using reg = shadow_state_what; + switch (what) { + case reg::x0: + return "x0"; + case reg::x1: + return "x1"; + case reg::x2: + return "x2"; + case reg::x3: + return "x3"; + case reg::x4: + return "x4"; + case reg::x5: + return "x5"; + case reg::x6: + return "x6"; + case reg::x7: + return "x7"; + case reg::x8: + return "x8"; + case reg::x9: + return "x9"; + case reg::x10: + return "x10"; + case reg::x11: + return "x11"; + case reg::x12: + return "x12"; + case reg::x13: + return "x13"; + case reg::x14: + return "x14"; + case reg::x15: + return "x15"; + case reg::x16: + return "x16"; + case reg::x17: + return "x17"; + case reg::x18: + return "x18"; + case reg::x19: + return "x19"; + case reg::x20: + return "x20"; + case reg::x21: + return "x21"; + case reg::x22: + return "x22"; + case reg::x23: + return "x23"; + case reg::x24: + return "x24"; + case reg::x25: + return "x25"; + case reg::x26: + return "x26"; + case reg::x27: + return "x27"; + case reg::x28: + return "x28"; + case reg::x29: + return "x29"; + case reg::x30: + return "x30"; + case reg::x31: + return "x31"; + case reg::f0: + return "f0"; + case reg::f1: + return "f1"; + case reg::f2: + return "f2"; + case reg::f3: + return "f3"; + case reg::f4: + return "f4"; + case reg::f5: + return "f5"; + case reg::f6: + return "f6"; + case reg::f7: + return "f7"; + case reg::f8: + return "f8"; + case reg::f9: + return "f9"; + case reg::f10: + return "f10"; + case reg::f11: + return "f11"; + case reg::f12: + return "f12"; + case reg::f13: + return "f13"; + case reg::f14: + return "f14"; + case reg::f15: + return "f15"; + case reg::f16: + return "f16"; + case reg::f17: + return "f17"; + case reg::f18: + return "f18"; + case reg::f19: + return "f19"; + case reg::f20: + return "f20"; + case reg::f21: + return "f21"; + case reg::f22: + return "f22"; + case reg::f23: + return "f23"; + case reg::f24: + return "f24"; + case reg::f25: + return "f25"; + case reg::f26: + return "f26"; + case reg::f27: + return "f27"; + case reg::f28: + return "f28"; + case reg::f29: + return "f29"; + case reg::f30: + return "f30"; + case reg::f31: + return "f31"; + case reg::pc: + return "pc"; + case reg::fcsr: + return "fcsr"; + case reg::mvendorid: + return "mvendorid"; + case reg::marchid: + return "marchid"; + case reg::mimpid: + return "mimpid"; + case reg::mcycle: + return "mcycle"; + case reg::icycleinstret: + return "icycleinstret"; + case reg::mstatus: + return "mstatus"; + case reg::mtvec: + return "mtvec"; + case reg::mscratch: + return "mscratch"; + case reg::mepc: + return "mepc"; + case reg::mcause: + return "mcause"; + case reg::mtval: + return "mtval"; + case reg::misa: + return "misa"; + case reg::mie: + return "mie"; + case reg::mip: + return "mip"; + case reg::medeleg: + return "medeleg"; + case reg::mideleg: + return "mideleg"; + case reg::mcounteren: + return "mcounteren"; + case reg::menvcfg: + return "menvcfg"; + case reg::stvec: + return "stvec"; + case reg::sscratch: + return "sscratch"; + case reg::sepc: + return "sepc"; + case reg::scause: + return "scause"; + case reg::stval: + return "stval"; + case reg::satp: + return "satp"; + case reg::scounteren: + return "scounteren"; + case reg::senvcfg: + return "senvcfg"; + case reg::ilrsc: + return "ilrsc"; + case reg::iprv: + return "iprv"; + case reg::iflags_X: + return "iflags.X"; + case reg::iflags_Y: + return "iflags.Y"; + case reg::iflags_H: + return "iflags.H"; + case reg::iunrep: + return "iunrep"; + case reg::clint_mtimecmp: + return "clint.mtimecmp"; + case reg::plic_girqpend: + return "plic.girqpend"; + case reg::plic_girqsrvd: + return "plic.girqsrvd"; + case reg::htif_tohost: + return "htif.tohost"; + case reg::htif_fromhost: + return "htif.fromhost"; + case reg::htif_ihalt: + return "htif.ihalt"; + case reg::htif_iconsole: + return "htif.iconsole"; + case reg::htif_iyield: + return "htif.iyield"; + case reg::unknown_: + [[fallthrough]]; + default: + return "state.unknown"; + } +} + /// \brief Global instance of the processor shadow device driver. extern const pma_driver shadow_state_driver; diff --git a/src/shadow-tlb-factory.cpp b/src/shadow-tlb-factory.cpp index c59cd2686..915cfb99d 100644 --- a/src/shadow-tlb-factory.cpp +++ b/src/shadow-tlb-factory.cpp @@ -14,67 +14,38 @@ // with this program (see COPYING). If not, see . // -#include "shadow-tlb-factory.h" - -#include -#include #include -#include #include "machine.h" #include "pma-constants.h" #include "pma.h" +#include "shadow-peek.h" +#include "shadow-tlb-factory.h" #include "shadow-tlb.h" -#include "strict-aliasing.h" namespace cartesi { /// \brief TLB device peek callback. See ::pma_peek. -static bool shadow_tlb_peek(const pma_entry &pma, const machine &m, uint64_t page_offset, - const unsigned char **page_data, unsigned char *scratch) { - // Check for alignment and range - if (page_offset % PMA_PAGE_SIZE != 0 || page_offset >= pma.get_length()) { - *page_data = nullptr; - return false; +static bool shadow_tlb_peek(const pma_entry &pma, const machine &m, uint64_t offset, uint64_t length, + const unsigned char **data, unsigned char *scratch) { + // If past useful range + if (offset >= sizeof(shadow_tlb_state)) { + *data = nullptr; + return length <= pma.get_length() && offset <= pma.get_length() - length; } - // Clear page - memset(scratch, 0, PMA_PAGE_SIZE); - // Copy relevant TLB entries to the page - const auto &tlb = m.get_state().tlb; - for (uint64_t offset = 0; offset < PMA_PAGE_SIZE; offset += sizeof(uint64_t)) { - uint64_t val = 0; - if (offset < sizeof(shadow_tlb_state)) { - // Figure out in which set (code/read/write) the offset falls - const uint64_t set_index = offset / sizeof(shadow_tlb_set); - const uint64_t slot_offset = offset % sizeof(shadow_tlb_set); - // Figure out in which slot index the offset falls - const uint64_t slot_index = slot_offset / sizeof(shadow_tlb_slot); - const uint64_t field_offset = slot_offset % sizeof(shadow_tlb_slot); - switch (field_offset) { - case offsetof(shadow_tlb_slot, vaddr_page): - val = tlb.hot[set_index][slot_index].vaddr_page; - break; - case offsetof(shadow_tlb_slot, vp_offset): { - auto vaddr_page = tlb.hot[set_index][slot_index].vaddr_page; - auto vh_offset = tlb.hot[set_index][slot_index].vh_offset; - auto haddr_page = vaddr_page + vh_offset; - auto pma_index = tlb.cold[set_index][slot_index].pma_index; - auto paddr_page = m.get_paddr(haddr_page, pma_index); - val = paddr_page - vaddr_page; - break; - } - case offsetof(shadow_tlb_slot, pma_index): - val = tlb.cold[set_index][slot_index].pma_index; - break; - default: - val = 0; - break; + // Otherwise, copy and return register data + return shadow_peek( + [](const machine &m, uint64_t paddr) { + TLB_set_index set_index{}; + uint64_t slot_index{}; + auto reg = shadow_tlb_get_what(paddr, set_index, slot_index); + uint64_t val = 0; + if (reg != shadow_tlb_what::unknown_) { + val = m.read_shadow_tlb(set_index, slot_index, reg); } - } - aliased_aligned_write(scratch + offset, val); - } - *page_data = scratch; - return true; + return val; + }, + pma, m, offset, length, data, scratch); } pma_entry make_shadow_tlb_pma_entry(uint64_t start, uint64_t length) { @@ -84,7 +55,7 @@ pma_entry make_shadow_tlb_pma_entry(uint64_t start, uint64_t length) { .IR = false, .IW = false, .DID = PMA_ISTART_DID::shadow_TLB}; - return make_device_pma_entry("shadow TLB device", start, length, shadow_tlb_peek, &shadow_tlb_driver).set_flags(f); + return make_device_pma_entry("shadow TLB", start, length, shadow_tlb_peek, &shadow_tlb_driver).set_flags(f); } } // namespace cartesi diff --git a/src/shadow-tlb.h b/src/shadow-tlb.h index c088aff9f..36d8f82d0 100644 --- a/src/shadow-tlb.h +++ b/src/shadow-tlb.h @@ -32,12 +32,21 @@ namespace cartesi { /// \brief Shadow TLB slot +// Writes to TLB slots have to be atomic. +// We can only do /aligned/ atomic writes. +// Therefore, TLB slot cannot be misaligned. struct PACKED shadow_tlb_slot { - uint64_t vaddr_page; ///< Tag is target virtual address of start of page - uint64_t vp_offset; ///< Value is offset from target virtual address to target physical address within page - uint64_t pma_index; ///< Index of PMA where physical page falls + uint64_t vaddr_page; ///< Tag is target virtual address of start of page + uint64_t vp_offset; ///< Value is offset from target virtual address to target physical address within page + uint64_t pma_index; ///< Index of PMA where physical page falls + uint64_t zero_padding_; ///< Padding to make sure the sizeof(shadow_tlb_slot) is a power of 2 }; +constexpr uint64_t SHADOW_TLB_SLOT_SIZE = sizeof(shadow_tlb_slot); +static_assert((SHADOW_TLB_SLOT_SIZE & (SHADOW_TLB_SLOT_SIZE - 1)) == 0, "shadow TLB slot size must be power of two"); +constexpr uint64_t SHADOW_TLB_SLOT_LOG2_SIZE = 5; +static_assert((UINT64_C(1) << SHADOW_TLB_SLOT_LOG2_SIZE) == SHADOW_TLB_SLOT_SIZE, "shadow TLB slot log2 size is wrong"); + /// \brief Shadow TLB set using shadow_tlb_set = std::array; @@ -48,24 +57,60 @@ static_assert(PMA_SHADOW_TLB_LENGTH >= sizeof(shadow_tlb_state), "TLB state must extern const pma_driver shadow_tlb_driver; -template -constexpr uint64_t shadow_tlb_get_slot_abs_addr(uint64_t slot_index) { - return PMA_SHADOW_TLB_START + USE * sizeof(shadow_tlb_set) + slot_index * sizeof(shadow_tlb_slot); +/// \brief List of field types +enum class shadow_tlb_what : uint64_t { + vaddr_page = offsetof(shadow_tlb_slot, vaddr_page), + vp_offset = offsetof(shadow_tlb_slot, vp_offset), + pma_index = offsetof(shadow_tlb_slot, pma_index), + zero_padding_ = offsetof(shadow_tlb_slot, zero_padding_), + unknown_ = UINT64_C(1) << 63, // Outside of RISC-V address space +}; + +static constexpr uint64_t shadow_tlb_get_abs_addr(TLB_set_index set_index, uint64_t slot_index) { + return PMA_SHADOW_TLB_START + set_index * sizeof(shadow_tlb_set) + slot_index * sizeof(shadow_tlb_slot); +} + +static constexpr uint64_t shadow_tlb_get_abs_addr(TLB_set_index set_index, uint64_t slot_index, shadow_tlb_what what) { + return shadow_tlb_get_abs_addr(set_index, slot_index) + static_cast(what); } -template -constexpr uint64_t shadow_tlb_get_vaddr_page_abs_addr(uint64_t slot_index) { - return shadow_tlb_get_slot_abs_addr(slot_index) + offsetof(shadow_tlb_slot, vaddr_page); +static constexpr shadow_tlb_what shadow_tlb_get_what(uint64_t paddr, TLB_set_index &set_index, uint64_t &slot_index) { + if (paddr < PMA_SHADOW_TLB_START || paddr - PMA_SHADOW_TLB_START >= sizeof(shadow_tlb_state) || + (paddr & (sizeof(uint64_t) - 1)) != 0) { + return shadow_tlb_what::unknown_; + } + paddr -= PMA_SHADOW_TLB_START; + set_index = TLB_set_index{paddr / sizeof(shadow_tlb_set)}; + slot_index = (paddr % sizeof(shadow_tlb_set)) / sizeof(shadow_tlb_slot); + return shadow_tlb_what{paddr % sizeof(shadow_tlb_slot)}; } -template -constexpr uint64_t shadow_tlb_get_vp_offset_abs_addr(uint64_t slot_index) { - return shadow_tlb_get_slot_abs_addr(slot_index) + offsetof(shadow_tlb_slot, vp_offset); +static constexpr const char *shadow_tlb_get_what_name(shadow_tlb_what what) { + const auto offset = static_cast(what); + using reg = shadow_tlb_what; + if (offset > static_cast(reg::unknown_) || (offset & (sizeof(uint64_t) - 1)) != 0) { + return "tlb.unknown"; + } + switch (what) { + case reg::vaddr_page: + return "tlb.slot.vaddr_page"; + case reg::vp_offset: + return "tlb.slot.vp_offset"; + case reg::pma_index: + return "tlb.slot.pma_index"; + case reg::zero_padding_: + return "tlb.slot.zero_padding_"; + case reg::unknown_: + return "tlb.unknown"; + } } -template -constexpr uint64_t shadow_tlb_get_pma_index_abs_addr(uint64_t slot_index) { - return shadow_tlb_get_slot_abs_addr(slot_index) + offsetof(shadow_tlb_slot, pma_index); +[[maybe_unused]] static void shadow_tlb_fill_slot(uint64_t vaddr_page, uint64_t vp_offset, uint64_t pma_index, + shadow_tlb_slot &slot) { + slot.vaddr_page = vaddr_page; + slot.vp_offset = vp_offset; + slot.pma_index = pma_index; + slot.zero_padding_ = 0; } } // namespace cartesi diff --git a/src/shadow-uarch-state-factory.cpp b/src/shadow-uarch-state-factory.cpp index 376ad69ee..de98f4a6f 100644 --- a/src/shadow-uarch-state-factory.cpp +++ b/src/shadow-uarch-state-factory.cpp @@ -14,44 +14,36 @@ // with this program (see COPYING). If not, see . // -#include "shadow-uarch-state-factory.h" +#include +#include "machine.h" #include "pma-constants.h" #include "pma.h" -#include "riscv-constants.h" +#include "shadow-peek.h" +#include "shadow-uarch-state-factory.h" #include "shadow-uarch-state.h" -#include -#include - -#include "machine.h" namespace cartesi { /// \brief Shadow uarch state device peek callback. See ::pma_peek. -static bool shadow_uarch_state_peek(const pma_entry & /*pma*/, const machine &m, uint64_t page_offset, - const unsigned char **page_data, unsigned char *scratch) { - static_assert(sizeof(shadow_uarch_state) <= PMA_PAGE_SIZE); - - // There is only one page: 0 - if (page_offset != 0) { - *page_data = nullptr; - return false; - } - // Clear page - memset(scratch, 0, PMA_PAGE_SIZE); - - // The page will reflect the shadow uarch state structure - // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) - auto *s = reinterpret_cast(scratch); - - s->halt_flag = m.read_reg(machine_reg::uarch_halt_flag); - s->cycle = m.read_reg(machine_reg::uarch_cycle); - s->pc = m.read_reg(machine_reg::uarch_pc); - for (int i = 0; i < UARCH_X_REG_COUNT; ++i) { - s->x[i] = m.read_reg(machine_reg_enum(machine_reg::uarch_x0, i)); +static bool shadow_uarch_state_peek(const pma_entry &pma, const machine &m, uint64_t offset, uint64_t length, + const unsigned char **data, unsigned char *scratch) { + // If past useful range + if (offset >= sizeof(shadow_uarch_state)) { + *data = nullptr; + return length <= pma.get_length() && offset <= pma.get_length() - length; } - *page_data = scratch; - return true; + // Otherwise, copy and return register data + return shadow_peek( + [](const machine &m, uint64_t paddr) { + const auto reg = shadow_uarch_state_get_what(paddr); + uint64_t val = 0; + if (reg != shadow_uarch_state_what::unknown_) { + val = m.read_reg(machine_reg_enum(reg)); + } + return val; + }, + pma, m, offset, length, data, scratch); } pma_entry make_shadow_uarch_state_pma_entry(uint64_t start, uint64_t length) { @@ -61,7 +53,7 @@ pma_entry make_shadow_uarch_state_pma_entry(uint64_t start, uint64_t length) { .IR = false, .IW = false, .DID = PMA_ISTART_DID::shadow_uarch}; - return make_device_pma_entry("shadow uarch state device", start, length, shadow_uarch_state_peek, + return make_device_pma_entry("shadow uarch state", start, length, shadow_uarch_state_peek, &shadow_uarch_state_driver) .set_flags(f); } diff --git a/src/shadow-uarch-state.h b/src/shadow-uarch-state.h index 36c7ea34f..efcc72c73 100644 --- a/src/shadow-uarch-state.h +++ b/src/shadow-uarch-state.h @@ -38,6 +38,136 @@ struct PACKED shadow_uarch_state { uint64_t x[UARCH_X_REG_COUNT]; }; +enum class shadow_uarch_state_what : uint64_t { + uarch_halt_flag = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, halt_flag), + uarch_cycle = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, cycle), + uarch_pc = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, pc), + uarch_x0 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[0]), + uarch_x1 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[1]), + uarch_x2 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[2]), + uarch_x3 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[3]), + uarch_x4 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[4]), + uarch_x5 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[5]), + uarch_x6 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[6]), + uarch_x7 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[7]), + uarch_x8 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[8]), + uarch_x9 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[9]), + uarch_x10 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[10]), + uarch_x11 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[11]), + uarch_x12 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[12]), + uarch_x13 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[13]), + uarch_x14 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[14]), + uarch_x15 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[15]), + uarch_x16 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[16]), + uarch_x17 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[17]), + uarch_x18 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[18]), + uarch_x19 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[19]), + uarch_x20 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[20]), + uarch_x21 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[21]), + uarch_x22 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[22]), + uarch_x23 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[23]), + uarch_x24 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[24]), + uarch_x25 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[25]), + uarch_x26 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[26]), + uarch_x27 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[27]), + uarch_x28 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[28]), + uarch_x29 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[29]), + uarch_x30 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[30]), + uarch_x31 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[31]), + unknown_ = UINT64_C(1) << 63, // Outside of RISC-V address space +}; + +static constexpr shadow_uarch_state_what shadow_uarch_state_get_what(uint64_t paddr) { + if (paddr < PMA_SHADOW_UARCH_STATE_START || paddr - PMA_SHADOW_UARCH_STATE_START >= sizeof(shadow_uarch_state) || + (paddr & (sizeof(uint64_t) - 1)) != 0) { + return shadow_uarch_state_what::unknown_; + } + return static_cast(paddr); +} + +static constexpr const char *shadow_uarch_state_get_what_name(shadow_uarch_state_what what) { + const auto paddr = static_cast(what); + if (paddr < PMA_SHADOW_UARCH_STATE_START || paddr - PMA_SHADOW_UARCH_STATE_START >= sizeof(shadow_uarch_state) || + (paddr & (sizeof(uint64_t) - 1)) != 0) { + return "uarch.unknown"; + } + using reg = shadow_uarch_state_what; + switch (what) { + case reg::uarch_halt_flag: + return "uarch.halt_flag"; + case reg::uarch_cycle: + return "uarch.cycle"; + case reg::uarch_pc: + return "uarch.pc"; + case reg::uarch_x0: + return "uarch.x0"; + case reg::uarch_x1: + return "uarch.x1"; + case reg::uarch_x2: + return "uarch.x2"; + case reg::uarch_x3: + return "uarch.x3"; + case reg::uarch_x4: + return "uarch.x4"; + case reg::uarch_x5: + return "uarch.x5"; + case reg::uarch_x6: + return "uarch.x6"; + case reg::uarch_x7: + return "uarch.x7"; + case reg::uarch_x8: + return "uarch.x8"; + case reg::uarch_x9: + return "uarch.x9"; + case reg::uarch_x10: + return "uarch.x10"; + case reg::uarch_x11: + return "uarch.x11"; + case reg::uarch_x12: + return "uarch.x12"; + case reg::uarch_x13: + return "uarch.x13"; + case reg::uarch_x14: + return "uarch.x14"; + case reg::uarch_x15: + return "uarch.x15"; + case reg::uarch_x16: + return "uarch.x16"; + case reg::uarch_x17: + return "uarch.x17"; + case reg::uarch_x18: + return "uarch.x18"; + case reg::uarch_x19: + return "uarch.x19"; + case reg::uarch_x20: + return "uarch.x20"; + case reg::uarch_x21: + return "uarch.x21"; + case reg::uarch_x22: + return "uarch.x22"; + case reg::uarch_x23: + return "uarch.x23"; + case reg::uarch_x24: + return "uarch.x24"; + case reg::uarch_x25: + return "uarch.x25"; + case reg::uarch_x26: + return "uarch.x26"; + case reg::uarch_x27: + return "uarch.x27"; + case reg::uarch_x28: + return "uarch.x28"; + case reg::uarch_x29: + return "uarch.x29"; + case reg::uarch_x30: + return "uarch.x30"; + case reg::uarch_x31: + return "uarch.x31"; + case reg::unknown_: + return "uarch.unknown_"; + } +} + /// \brief Global instance of theprocessor shadow uarch state device driver. extern const pma_driver shadow_uarch_state_driver; diff --git a/src/state-access.h b/src/state-access.h index 91fc81511..5a8540bcb 100644 --- a/src/state-access.h +++ b/src/state-access.h @@ -41,6 +41,10 @@ #include "shadow-tlb.h" #include "strict-aliasing.h" +#if DUMP_STATE_ACCESS +#include "scoped-note.h" +#endif + namespace cartesi { class state_access; @@ -85,29 +89,28 @@ class state_access : public i_state_access { return m_m.get_state(); } +#ifdef DUMP_STATE_ACCESS // NOLINTNEXTLINE(readability-convert-member-functions-to-static) - void do_push_bracket(bracket_type /*type*/, const char * /*text*/) {} - - // NOLINTNEXTLINE(readability-convert-member-functions-to-static) - int do_make_scoped_note(const char * /*text*/) { - return 0; + auto do_make_scoped_note([[maybe_unused]] const char *text) { + return scoped_note{*this, text}; } +#endif - uint64_t do_read_x(int reg) const { - return m_m.get_state().x[reg]; + uint64_t do_read_x(int i) const { + return m_m.get_state().x[i]; } - void do_write_x(int reg, uint64_t val) { - assert(reg != 0); - m_m.get_state().x[reg] = val; + void do_write_x(int i, uint64_t val) { + assert(i != 0); + m_m.get_state().x[i] = val; } - uint64_t do_read_f(int reg) const { - return m_m.get_state().f[reg]; + uint64_t do_read_f(int i) const { + return m_m.get_state().f[i]; } - void do_write_f(int reg, uint64_t val) { - m_m.get_state().f[reg] = val; + void do_write_f(int i, uint64_t val) { + m_m.get_state().f[i] = val; } uint64_t do_read_pc() const { @@ -475,26 +478,24 @@ class state_access : public i_state_access { aliased_aligned_write(haddr, val); } - template + template uint64_t do_read_tlb_vaddr_page(uint64_t slot_index) { - return m_m.get_state().tlb.hot[USE][slot_index].vaddr_page; + return m_m.get_state().tlb.hot[SET][slot_index].vaddr_page; } - template + template host_addr do_read_tlb_vp_offset(uint64_t slot_index) { - return m_m.get_state().tlb.hot[USE][slot_index].vh_offset; + return m_m.get_state().tlb.hot[SET][slot_index].vh_offset; } - template + template uint64_t do_read_tlb_pma_index(uint64_t slot_index) { - return m_m.get_state().tlb.cold[USE][slot_index].pma_index; + return m_m.get_state().tlb.cold[SET][slot_index].pma_index; } - template + template void do_write_tlb(uint64_t slot_index, uint64_t vaddr_page, host_addr vh_offset, uint64_t pma_index) { - m_m.get_state().tlb.hot[USE][slot_index].vaddr_page = vaddr_page; - m_m.get_state().tlb.hot[USE][slot_index].vh_offset = vh_offset; - m_m.get_state().tlb.cold[USE][slot_index].pma_index = pma_index; + m_m.write_tlb(SET, slot_index, vaddr_page, vh_offset, pma_index); } fast_addr do_get_faddr(uint64_t paddr, uint64_t pma_index) const { @@ -558,6 +559,11 @@ class state_access : public i_state_access { return os_getchar(); } + // NOLINTNEXTLINE(readability-convert-member-functions-to-static) + constexpr const char *do_get_name() const { + return "state_access"; + } + #ifdef DUMP_COUNTERS machine_statistics &do_get_statistics() { return m_m.get_state().stats; diff --git a/src/tlb.h b/src/tlb.h index 991f67d32..ab7336bcf 100644 --- a/src/tlb.h +++ b/src/tlb.h @@ -32,8 +32,8 @@ namespace cartesi { -/// \brief TLB set mode. -enum TLB_set_use : uint64_t { TLB_CODE, TLB_READ, TLB_WRITE, TLB_LAST_ = TLB_WRITE }; +/// \brief Index of TLB set +enum TLB_set_index : uint64_t { TLB_CODE, TLB_READ, TLB_WRITE, TLB_LAST_ = TLB_WRITE }; /// \brief TLB constants. enum TLB_constants : uint64_t { diff --git a/src/uarch-bridge.h b/src/uarch-bridge.h deleted file mode 100644 index 423a2ead2..000000000 --- a/src/uarch-bridge.h +++ /dev/null @@ -1,1038 +0,0 @@ -// Copyright Cartesi and individual authors (see AUTHORS) -// SPDX-License-Identifier: LGPL-3.0-or-later -// -// This program is free software: you can redistribute it and/or modify it under -// the terms of the GNU Lesser General Public License as published by the Free -// Software Foundation, either version 3 of the License, or (at your option) any -// later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT ANY -// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A -// PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License along -// with this program (see COPYING). If not, see . -// - -#ifndef UARCH_BRIDGE_H -#define UARCH_BRIDGE_H - -#include -#include -#include -#include -#include - -#include "machine-reg.h" -#include "machine-state.h" -#include "pma-constants.h" -#include "riscv-constants.h" -#include "shadow-pmas.h" -#include "shadow-state.h" -#include "shadow-tlb.h" -#include "strict-aliasing.h" - -namespace cartesi { - -/// \brief Allows microarchitecture code to access the machine state -class uarch_bridge { -public: - /// \brief Updates the value of a machine state register. - /// \param s Machine state. - /// \param paddr Address that identifies the machine register to be written. - /// \param data Data to write. - /// \details \{ - /// An exception is thrown if paddr can't me mapped to a valid state register. - //// \} - static void write_register(uint64_t paddr, machine_state &s, uint64_t data) { - using reg = machine_reg; - switch (static_cast(paddr)) { - case reg::x0: - s.x[0] = data; - return; - case reg::x1: - s.x[1] = data; - return; - case reg::x2: - s.x[2] = data; - return; - case reg::x3: - s.x[3] = data; - return; - case reg::x4: - s.x[4] = data; - return; - case reg::x5: - s.x[5] = data; - return; - case reg::x6: - s.x[6] = data; - return; - case reg::x7: - s.x[7] = data; - return; - case reg::x8: - s.x[8] = data; - return; - case reg::x9: - s.x[9] = data; - return; - case reg::x10: - s.x[10] = data; - return; - case reg::x11: - s.x[11] = data; - return; - case reg::x12: - s.x[12] = data; - return; - case reg::x13: - s.x[13] = data; - return; - case reg::x14: - s.x[14] = data; - return; - case reg::x15: - s.x[15] = data; - return; - case reg::x16: - s.x[16] = data; - return; - case reg::x17: - s.x[17] = data; - return; - case reg::x18: - s.x[18] = data; - return; - case reg::x19: - s.x[19] = data; - return; - case reg::x20: - s.x[20] = data; - return; - case reg::x21: - s.x[21] = data; - return; - case reg::x22: - s.x[22] = data; - return; - case reg::x23: - s.x[23] = data; - return; - case reg::x24: - s.x[24] = data; - return; - case reg::x25: - s.x[25] = data; - return; - case reg::x26: - s.x[26] = data; - return; - case reg::x27: - s.x[27] = data; - return; - case reg::x28: - s.x[28] = data; - return; - case reg::x29: - s.x[29] = data; - return; - case reg::x30: - s.x[30] = data; - return; - case reg::x31: - s.x[31] = data; - return; - case reg::f0: - s.f[0] = data; - return; - case reg::f1: - s.f[1] = data; - return; - case reg::f2: - s.f[2] = data; - return; - case reg::f3: - s.f[3] = data; - return; - case reg::f4: - s.f[4] = data; - return; - case reg::f5: - s.f[5] = data; - return; - case reg::f6: - s.f[6] = data; - return; - case reg::f7: - s.f[7] = data; - return; - case reg::f8: - s.f[8] = data; - return; - case reg::f9: - s.f[9] = data; - return; - case reg::f10: - s.f[10] = data; - return; - case reg::f11: - s.f[11] = data; - return; - case reg::f12: - s.f[12] = data; - return; - case reg::f13: - s.f[13] = data; - return; - case reg::f14: - s.f[14] = data; - return; - case reg::f15: - s.f[15] = data; - return; - case reg::f16: - s.f[16] = data; - return; - case reg::f17: - s.f[17] = data; - return; - case reg::f18: - s.f[18] = data; - return; - case reg::f19: - s.f[19] = data; - return; - case reg::f20: - s.f[20] = data; - return; - case reg::f21: - s.f[21] = data; - return; - case reg::f22: - s.f[22] = data; - return; - case reg::f23: - s.f[23] = data; - return; - case reg::f24: - s.f[24] = data; - return; - case reg::f25: - s.f[25] = data; - return; - case reg::f26: - s.f[26] = data; - return; - case reg::f27: - s.f[27] = data; - return; - case reg::f28: - s.f[28] = data; - return; - case reg::f29: - s.f[29] = data; - return; - case reg::f30: - s.f[30] = data; - return; - case reg::f31: - s.f[31] = data; - return; - case reg::pc: - s.pc = data; - return; - case reg::fcsr: - s.fcsr = data; - return; - case reg::mcycle: - s.mcycle = data; - return; - case reg::icycleinstret: - s.icycleinstret = data; - return; - case reg::mstatus: - s.mstatus = data; - return; - case reg::mtvec: - s.mtvec = data; - return; - case reg::mscratch: - s.mscratch = data; - return; - case reg::mepc: - s.mepc = data; - return; - case reg::mcause: - s.mcause = data; - return; - case reg::mtval: - s.mtval = data; - return; - case reg::misa: - s.misa = data; - return; - case reg::mie: - s.mie = data; - return; - case reg::mip: - s.mip = data; - return; - case reg::medeleg: - s.medeleg = data; - return; - case reg::mideleg: - s.mideleg = data; - return; - case reg::mcounteren: - s.mcounteren = data; - return; - case reg::menvcfg: - s.menvcfg = data; - return; - case reg::stvec: - s.stvec = data; - return; - case reg::sscratch: - s.sscratch = data; - return; - case reg::sepc: - s.sepc = data; - return; - case reg::scause: - s.scause = data; - return; - case reg::stval: - s.stval = data; - return; - case reg::satp: - s.satp = data; - return; - case reg::scounteren: - s.scounteren = data; - return; - case reg::senvcfg: - s.senvcfg = data; - return; - case reg::ilrsc: - s.ilrsc = data; - return; - case reg::iprv: - s.iprv = data; - return; - case reg::iflags_X: - s.iflags.X = data; - return; - case reg::iflags_Y: - s.iflags.Y = data; - return; - case reg::iflags_H: - s.iflags.H = data; - return; - case reg::clint_mtimecmp: - s.clint.mtimecmp = data; - return; - case reg::plic_girqpend: - s.plic.girqpend = data; - return; - case reg::plic_girqsrvd: - s.plic.girqsrvd = data; - return; - case reg::htif_tohost: - s.htif.tohost = data; - return; - case reg::htif_fromhost: - s.htif.fromhost = data; - return; - default: - break; - } - if (try_write_tlb(s, paddr, data)) { - return; - } - throw std::runtime_error("invalid write memory access from microarchitecture"); - } - - /// \brief Reads a machine state register. - /// \param s Machine state. - /// \param paddr Address that identifies the machine register to be read. - /// \param data Receives the state register value. - /// \details \{ - /// An exception is thrown if paddr can't me mapped to a valid state register. - //// \} - static uint64_t read_register(uint64_t paddr, machine_state &s) { - using reg = machine_reg; - switch (static_cast(paddr)) { - case reg::x0: - return s.x[0]; - case reg::x1: - return s.x[1]; - case reg::x2: - return s.x[2]; - case reg::x3: - return s.x[3]; - case reg::x4: - return s.x[4]; - case reg::x5: - return s.x[5]; - case reg::x6: - return s.x[6]; - case reg::x7: - return s.x[7]; - case reg::x8: - return s.x[8]; - case reg::x9: - return s.x[9]; - case reg::x10: - return s.x[10]; - case reg::x11: - return s.x[11]; - case reg::x12: - return s.x[12]; - case reg::x13: - return s.x[13]; - case reg::x14: - return s.x[14]; - case reg::x15: - return s.x[15]; - case reg::x16: - return s.x[16]; - case reg::x17: - return s.x[17]; - case reg::x18: - return s.x[18]; - case reg::x19: - return s.x[19]; - case reg::x20: - return s.x[20]; - case reg::x21: - return s.x[21]; - case reg::x22: - return s.x[22]; - case reg::x23: - return s.x[23]; - case reg::x24: - return s.x[24]; - case reg::x25: - return s.x[25]; - case reg::x26: - return s.x[26]; - case reg::x27: - return s.x[27]; - case reg::x28: - return s.x[28]; - case reg::x29: - return s.x[29]; - case reg::x30: - return s.x[30]; - case reg::x31: - return s.x[31]; - case reg::f0: - return s.f[0]; - case reg::f1: - return s.f[1]; - case reg::f2: - return s.f[2]; - case reg::f3: - return s.f[3]; - case reg::f4: - return s.f[4]; - case reg::f5: - return s.f[5]; - case reg::f6: - return s.f[6]; - case reg::f7: - return s.f[7]; - case reg::f8: - return s.f[8]; - case reg::f9: - return s.f[9]; - case reg::f10: - return s.f[10]; - case reg::f11: - return s.f[11]; - case reg::f12: - return s.f[12]; - case reg::f13: - return s.f[13]; - case reg::f14: - return s.f[14]; - case reg::f15: - return s.f[15]; - case reg::f16: - return s.f[16]; - case reg::f17: - return s.f[17]; - case reg::f18: - return s.f[18]; - case reg::f19: - return s.f[19]; - case reg::f20: - return s.f[20]; - case reg::f21: - return s.f[21]; - case reg::f22: - return s.f[22]; - case reg::f23: - return s.f[23]; - case reg::f24: - return s.f[24]; - case reg::f25: - return s.f[25]; - case reg::f26: - return s.f[26]; - case reg::f27: - return s.f[27]; - case reg::f28: - return s.f[28]; - case reg::f29: - return s.f[29]; - case reg::f30: - return s.f[30]; - case reg::f31: - return s.f[31]; - case reg::pc: - return s.pc; - case reg::fcsr: - return s.fcsr; - case reg::mvendorid: - return MVENDORID_INIT; - case reg::marchid: - return MARCHID_INIT; - case reg::mimpid: - return MIMPID_INIT; - case reg::mcycle: - return s.mcycle; - case reg::icycleinstret: - return s.icycleinstret; - case reg::mstatus: - return s.mstatus; - case reg::mtvec: - return s.mtvec; - case reg::mscratch: - return s.mscratch; - case reg::mepc: - return s.mepc; - case reg::mcause: - return s.mcause; - case reg::mtval: - return s.mtval; - case reg::misa: - return s.misa; - case reg::mie: - return s.mie; - case reg::mip: - return s.mip; - case reg::medeleg: - return s.medeleg; - case reg::mideleg: - return s.mideleg; - case reg::mcounteren: - return s.mcounteren; - case reg::menvcfg: - return s.menvcfg; - case reg::stvec: - return s.stvec; - case reg::sscratch: - return s.sscratch; - case reg::sepc: - return s.sepc; - case reg::scause: - return s.scause; - case reg::stval: - return s.stval; - case reg::satp: - return s.satp; - case reg::scounteren: - return s.scounteren; - case reg::senvcfg: - return s.senvcfg; - case reg::ilrsc: - return s.ilrsc; - case reg::iprv: - return s.iprv; - case reg::iflags_X: - return s.iflags.X; - case reg::iflags_Y: - return s.iflags.Y; - case reg::iflags_H: - return s.iflags.H; - case reg::clint_mtimecmp: - return s.clint.mtimecmp; - case reg::plic_girqpend: - return s.plic.girqpend; - case reg::plic_girqsrvd: - return s.plic.girqsrvd; - case reg::htif_tohost: - return s.htif.tohost; - case reg::htif_fromhost: - return s.htif.fromhost; - case reg::htif_ihalt: - return s.htif.ihalt; - case reg::htif_iconsole: - return s.htif.iconsole; - case reg::htif_iyield: - return s.htif.iyield; - default: - break; - } - uint64_t data = 0; - if (try_read_tlb(s, paddr, &data)) { - return data; - } - if (try_read_pma(s, paddr, &data)) { - return data; - } - throw std::runtime_error("invalid read memory access from microarchitecture"); - } - - /// \brief Reads the name of a machine state register. - /// \param paddr Address of the state register. - /// \returns The register name, if paddr maps to a register, or nullptr otherwise. - static const char *get_register_name(uint64_t paddr) { - using reg = machine_reg; - switch (static_cast(paddr)) { - case reg::x0: - return "x0"; - case reg::x1: - return "x1"; - case reg::x2: - return "x2"; - case reg::x3: - return "x3"; - case reg::x4: - return "x4"; - case reg::x5: - return "x5"; - case reg::x6: - return "x6"; - case reg::x7: - return "x7"; - case reg::x8: - return "x8"; - case reg::x9: - return "x9"; - case reg::x10: - return "x10"; - case reg::x11: - return "x11"; - case reg::x12: - return "x12"; - case reg::x13: - return "x13"; - case reg::x14: - return "x14"; - case reg::x15: - return "x15"; - case reg::x16: - return "x16"; - case reg::x17: - return "x17"; - case reg::x18: - return "x18"; - case reg::x19: - return "x19"; - case reg::x20: - return "x20"; - case reg::x21: - return "x21"; - case reg::x22: - return "x22"; - case reg::x23: - return "x23"; - case reg::x24: - return "x24"; - case reg::x25: - return "x25"; - case reg::x26: - return "x26"; - case reg::x27: - return "x27"; - case reg::x28: - return "x28"; - case reg::x29: - return "x29"; - case reg::x30: - return "x30"; - case reg::x31: - return "x31"; - case reg::f0: - return "f0"; - case reg::f1: - return "f1"; - case reg::f2: - return "f2"; - case reg::f3: - return "f3"; - case reg::f4: - return "f4"; - case reg::f5: - return "f5"; - case reg::f6: - return "f6"; - case reg::f7: - return "f7"; - case reg::f8: - return "f8"; - case reg::f9: - return "f9"; - case reg::f10: - return "f10"; - case reg::f11: - return "f11"; - case reg::f12: - return "f12"; - case reg::f13: - return "f13"; - case reg::f14: - return "f14"; - case reg::f15: - return "f15"; - case reg::f16: - return "f16"; - case reg::f17: - return "f17"; - case reg::f18: - return "f18"; - case reg::f19: - return "f19"; - case reg::f20: - return "f20"; - case reg::f21: - return "f21"; - case reg::f22: - return "f22"; - case reg::f23: - return "f23"; - case reg::f24: - return "f24"; - case reg::f25: - return "f25"; - case reg::f26: - return "f26"; - case reg::f27: - return "f27"; - case reg::f28: - return "f28"; - case reg::f29: - return "f29"; - case reg::f30: - return "f30"; - case reg::f31: - return "f31"; - case reg::pc: - return "pc"; - case reg::fcsr: - return "fcsr"; - case reg::mvendorid: - return "mvendorid"; - case reg::marchid: - return "marchid"; - case reg::mimpid: - return "mimpid"; - case reg::mcycle: - return "mcycle"; - case reg::icycleinstret: - return "icycleinstret"; - case reg::mstatus: - return "mstatus"; - case reg::mtvec: - return "mtvec"; - case reg::mscratch: - return "mscratch"; - case reg::mepc: - return "mepc"; - case reg::mcause: - return "mcause"; - case reg::mtval: - return "mtval"; - case reg::misa: - return "misa"; - case reg::mie: - return "mie"; - case reg::mip: - return "mip"; - case reg::medeleg: - return "medeleg"; - case reg::mideleg: - return "mideleg"; - case reg::mcounteren: - return "mcounteren"; - case reg::menvcfg: - return "menvcfg"; - case reg::stvec: - return "stvec"; - case reg::sscratch: - return "sscratch"; - case reg::sepc: - return "sepc"; - case reg::scause: - return "scause"; - case reg::stval: - return "stval"; - case reg::satp: - return "satp"; - case reg::scounteren: - return "scounteren"; - case reg::senvcfg: - return "senvcfg"; - case reg::ilrsc: - return "ilrsc"; - case reg::iprv: - return "iprv"; - case reg::iflags_X: - return "iflags.X"; - case reg::iflags_Y: - return "iflags.Y"; - case reg::iflags_H: - return "iflags.H"; - case reg::clint_mtimecmp: - return "clint.mtimecmp"; - case reg::plic_girqpend: - return "plic.girqpend"; - case reg::plic_girqsrvd: - return "plic.girqsrvd"; - case reg::htif_tohost: - return "htif.tohost"; - case reg::htif_fromhost: - return "htif.fromhost"; - case reg::htif_ihalt: - return "htif.ihalt"; - case reg::htif_iconsole: - return "htif.iconsole"; - case reg::htif_iyield: - return "htif.iyield"; - default: - break; - } - if (paddr >= machine_reg_address(machine_reg::x0) && paddr <= machine_reg_address(machine_reg::x31) && - (paddr & 0b111) == 0) { - return "x"; - } - - if (paddr >= machine_reg_address(machine_reg::f0) && paddr <= machine_reg_address(machine_reg::f31) && - (paddr & 0b111) == 0) { - return "f"; - } - - if (paddr >= shadow_pmas_get_pma_abs_addr(0) && paddr < shadow_pmas_get_pma_abs_addr(PMA_MAX)) { - if ((paddr & 0b1111) == 0) { - return "pma.istart"; - } - if ((paddr & 0b1111) == 0b1000) { - return "pma.ilength"; - } - } - - if (paddr % sizeof(uint64_t) == 0) { - if (paddr >= shadow_tlb_get_slot_abs_addr(0) && - paddr < shadow_tlb_get_slot_abs_addr(TLB_SET_SIZE)) { - return "write tlb"; - } - if (paddr >= shadow_tlb_get_slot_abs_addr(0) && - paddr < shadow_tlb_get_slot_abs_addr(TLB_SET_SIZE)) { - return "read tlb"; - } - if (paddr >= shadow_tlb_get_slot_abs_addr(0) && - paddr < shadow_tlb_get_slot_abs_addr(TLB_SET_SIZE)) { - return "code tlb"; - } - } - - return nullptr; - } - -private: - /// \brief Tries to read a PMA entry field. - /// \param s Machine state. - /// \param paddr Absolute address of the PMA entry field within shadow PMAs range - /// \param data Pointer to word receiving value. - /// \return true if the register was successfully read - static bool try_read_pma(machine_state &s, uint64_t paddr, uint64_t *data) { - if (paddr < PMA_SHADOW_PMAS_START || paddr >= PMA_SHADOW_PMAS_START + (PMA_MAX * PMA_WORD_SIZE * 2)) { - return false; - } - auto word_index = (paddr - PMA_SHADOW_PMAS_START) >> 3; - auto pma_index = word_index >> 1; - if (pma_index >= s.pmas.size()) { - *data = 0; - return true; - } - auto &pma = s.pmas[pma_index]; - if ((word_index & 1) == 0) { - *data = pma.get_istart(); - } else { - *data = pma.get_ilength(); - } - return true; - } - - /// \brief Tries to read a TLB entry field. - /// \param s Machine state. - /// \param paddr Absolute address of the TLB entry fieldwithin shadow TLB range - /// \param data Pointer to word receiving value. - /// \return true if the register was successfully read - static bool try_read_tlb(machine_state &s, uint64_t paddr, uint64_t *data) { - if (paddr < PMA_SHADOW_TLB_START || - paddr >= PMA_SHADOW_TLB_START + PMA_SHADOW_TLB_LENGTH) { // In PMA TLB range? - return false; - } - if (paddr % sizeof(uint64_t) != 0) { // Misaligned field? - return false; - } - const uint64_t tlboff = paddr - PMA_SHADOW_TLB_START; - if (tlboff < offsetof(shadow_tlb_state, cold)) { // Hot entry - const uint64_t etype = tlboff / sizeof(std::array); - const uint64_t etypeoff = tlboff % sizeof(std::array); - const uint64_t eidx = etypeoff / sizeof(tlb_hot_entry); - const uint64_t fieldoff = etypeoff % sizeof(tlb_hot_entry); - return read_tlb_entry_field(s, true, etype, eidx, fieldoff, data); - } - if (tlboff < sizeof(shadow_tlb_state)) { // Cold entry - const uint64_t coldoff = tlboff - offsetof(shadow_tlb_state, cold); - const uint64_t etype = coldoff / sizeof(std::array); - const uint64_t etypeoff = coldoff % sizeof(std::array); - const uint64_t eidx = etypeoff / sizeof(tlb_cold_entry); - const uint64_t fieldoff = etypeoff % sizeof(tlb_cold_entry); - return read_tlb_entry_field(s, false, etype, eidx, fieldoff, data); - } - return false; - } - - /// \brief Tries to update a PMA entry field. - /// \param s Machine state. - /// \param paddr Absolute address of the PMA entry property within shadow PMAs range - /// \param data Data to write - /// \return true if the register was successfully written - static bool try_write_tlb(machine_state &s, uint64_t paddr, uint64_t data) { - if (paddr < PMA_SHADOW_TLB_START || - paddr >= PMA_SHADOW_TLB_START + PMA_SHADOW_TLB_LENGTH) { // In PMA TLB range? - return false; - } - if (paddr % sizeof(uint64_t) != 0) { // Misaligned field? - return false; - } - const uint64_t tlboff = paddr - PMA_SHADOW_TLB_START; - if (tlboff < offsetof(shadow_tlb_state, cold)) { // Hot entry - const uint64_t etype = tlboff / sizeof(std::array); - const uint64_t etypeoff = tlboff % sizeof(std::array); - const uint64_t eidx = etypeoff / sizeof(tlb_hot_entry); - const uint64_t fieldoff = etypeoff % sizeof(tlb_hot_entry); - return write_tlb_entry_field(s, true, etype, eidx, fieldoff, data); - } - if (tlboff < sizeof(shadow_tlb_state)) { // Cold entry - const uint64_t coldoff = tlboff - offsetof(shadow_tlb_state, cold); - const uint64_t etype = coldoff / sizeof(std::array); - const uint64_t etypeoff = coldoff % sizeof(std::array); - const uint64_t eidx = etypeoff / sizeof(tlb_cold_entry); - const uint64_t fieldoff = etypeoff % sizeof(tlb_cold_entry); - return write_tlb_entry_field(s, false, etype, eidx, fieldoff, data); - } - return false; - } - - /// \brief Reads a field of a TLB entry. - /// \param s Machine state. - /// \param hot If true read from hot TLB entries, otherwise from cold TLB entries. - /// \param etype TLB entry type. - /// \param eidx TLB entry index. - /// \param fieldoff TLB entry field offset. - /// \param pval Pointer to word receiving value. - /// \returns True if the field was read, false otherwise. - static bool read_tlb_entry_field(machine_state &s, bool hot, uint64_t etype, uint64_t eidx, uint64_t fieldoff, - uint64_t *pval) { - if (etype > TLB_WRITE || eidx >= PMA_TLB_SIZE) { - return false; - } - const tlb_hot_entry &tlbhe = s.tlb.hot[etype][eidx]; - const tlb_cold_entry &tlbce = s.tlb.cold[etype][eidx]; - if (hot) { - if (fieldoff == offsetof(tlb_hot_entry, vaddr_page)) { - *pval = tlbhe.vaddr_page; - return true; - } - // Other fields like vh_offset contains host data, and cannot be read - return false; - } - // Cold - switch (fieldoff) { - case offsetof(tlb_cold_entry, paddr_page): - *pval = tlbce.paddr_page; - return true; - case offsetof(tlb_cold_entry, pma_index): - *pval = tlbce.pma_index; - return true; - default: - return false; - } - } - - /// \brief Writes a field of a TLB entry. - /// \param s Machine state. - /// \param hot If true write to hot TLB entries, otherwise to cold TLB entries. - /// \param etype TLB entry type. - /// \param eidx TLB entry index. - /// \param fieldoff TLB entry field offset. - /// \param val Value to be written. - /// \returns True if the field was written, false otherwise. - static bool write_tlb_entry_field(machine_state &s, bool hot, uint64_t etype, uint64_t eidx, uint64_t fieldoff, - uint64_t val) { - if (etype > TLB_WRITE || eidx >= PMA_TLB_SIZE) { - return false; - } - tlb_hot_entry &tlbhe = s.tlb.hot[etype][eidx]; - tlb_cold_entry &tlbce = s.tlb.cold[etype][eidx]; - if (hot) { - if (fieldoff == offsetof(tlb_hot_entry, vaddr_page)) { - tlbhe.vaddr_page = val; - // Update vh_offset - if (val != TLB_INVALID_PAGE) { - const pma_entry &pma = find_pma_entry(s, tlbce.paddr_page); - assert(pma.get_istart_M()); // TLB only works for memory mapped PMAs - const unsigned char *hpage = - pma.get_memory().get_host_memory() + (tlbce.paddr_page - pma.get_start()); - tlbhe.vh_offset = cast_ptr_to_addr(hpage) - tlbhe.vaddr_page; - } - return true; - } - // Other fields like vh_offset contains host data, and cannot be written - return false; - } - // Cold - switch (fieldoff) { - case offsetof(tlb_cold_entry, paddr_page): - tlbce.paddr_page = val; - return true; - case offsetof(tlb_cold_entry, pma_index): - tlbce.pma_index = val; - return true; - default: - return false; - } - } - - /// \brief Obtain PMA entry that covers a given physical memory region - /// \tparam T Type of word. - /// \param s Mmachine state. - /// \param paddr Start of physical memory region. - /// \returns Corresponding entry if found, or a sentinel entry - /// for an empty range. - template - static const pma_entry &find_pma_entry(machine_state &s, uint64_t paddr) { - for (const auto &pma : s.pmas) { - // Stop at first empty PMA - if (pma.get_length() == 0) { - return pma; - } - if (pma.contains(paddr, sizeof(T))) { - return pma; - } - } - // Last PMA is always the empty range - return s.pmas.back(); - } -}; - -} // namespace cartesi - -#endif diff --git a/src/uarch-config.h b/src/uarch-config.h index 42333e059..6e1e29871 100644 --- a/src/uarch-config.h +++ b/src/uarch-config.h @@ -35,7 +35,7 @@ struct uarch_processor_config final { std::array x{}; ///< Value of general-purpose registers uint64_t pc{UARCH_PC_INIT}; ///< Value of pc uint64_t cycle{UARCH_CYCLE_INIT}; ///< Value of ucycle counter - bool halt_flag{}; + uint64_t halt_flag{}; }; /// \brief Microarchitecture configuration diff --git a/src/uarch-constants.h b/src/uarch-constants.h index 34a597cab..da52db2e2 100644 --- a/src/uarch-constants.h +++ b/src/uarch-constants.h @@ -64,8 +64,10 @@ static_assert((UARCH_RAM_LENGTH & (PMA_PAGE_SIZE - 1)) == 0, "UARCH_RAM_LENGTH m /// \brief ecall function codes enum uarch_ecall_functions : uint64_t { - UARCH_ECALL_FN_HALT = EXPAND_UINT64_C(UARCH_ECALL_FN_HALT_DEF), ///< halt uarch execution - UARCH_ECALL_FN_PUTCHAR = EXPAND_UINT64_C(UARCH_ECALL_FN_PUTCHAR_DEF), //< putchar + UARCH_ECALL_FN_HALT = EXPAND_UINT64_C(UARCH_ECALL_FN_HALT_DEF), ///< halt uarch execution + UARCH_ECALL_FN_PUTCHAR = EXPAND_UINT64_C(UARCH_ECALL_FN_PUTCHAR_DEF), ///< putchar + UARCH_ECALL_FN_MARK_DIRTY_PAGE = EXPAND_UINT64_C(UARCH_ECALL_FN_MARK_DIRTY_PAGE_DEF), ///< mark_dirty_page + UARCH_ECALL_FN_WRITE_TLB = EXPAND_UINT64_C(UARCH_ECALL_FN_WRITE_TLB_DEF), ///< write_tlb }; } // namespace cartesi diff --git a/src/uarch-defines.h b/src/uarch-defines.h index 855bfbbba..c5d3abb8b 100644 --- a/src/uarch-defines.h +++ b/src/uarch-defines.h @@ -26,8 +26,10 @@ #define UARCH_STATE_LOG2_SIZE_DEF 22 // microarchitecture ecall function codes -#define UARCH_ECALL_FN_HALT_DEF 1 // halt uarch -#define UARCH_ECALL_FN_PUTCHAR_DEF 2 // putchar +#define UARCH_ECALL_FN_HALT_DEF 1 // halt uarch +#define UARCH_ECALL_FN_PUTCHAR_DEF 2 // putchar +#define UARCH_ECALL_FN_MARK_DIRTY_PAGE_DEF 3 // mark_dirty_page +#define UARCH_ECALL_FN_WRITE_TLB_DEF 4 // write_tlb // NOLINTEND(cppcoreguidelines-macro-usage,cppcoreguidelines-macro-to-enum,modernize-macro-to-enum) #endif /* end of include guard: UARCH_DEFINES_H */ diff --git a/src/uarch-interpret.h b/src/uarch-interpret.h index 00c676529..295822394 100644 --- a/src/uarch-interpret.h +++ b/src/uarch-interpret.h @@ -19,10 +19,11 @@ #include -#include "uarch-state-access.h" - namespace cartesi { +// Forward declaration +class uarch_state_access; + enum class uarch_interpreter_break_reason : int { reached_target_cycle, uarch_halted }; // Run the microarchitecture interpreter until cycle hits a target or a fixed point is reached diff --git a/src/uarch-machine-bridge.cpp b/src/uarch-machine-bridge.cpp new file mode 100644 index 000000000..4583230b0 --- /dev/null +++ b/src/uarch-machine-bridge.cpp @@ -0,0 +1,170 @@ +// Copyright Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: LGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify it under +// the terms of the GNU Lesser General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) any +// later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +// PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License along +// with this program (see COPYING). If not, see . +// + +#include +#include + +#include "machine.h" +#include "riscv-constants.h" +#include "shadow-pmas.h" +#include "shadow-state.h" +#include "shadow-tlb.h" +#include "strict-aliasing.h" +#include "uarch-machine-bridge.h" + +namespace cartesi { + +const char *uarch_machine_bridge::get_what_name(uint64_t paddr) { + if (paddr >= PMA_UARCH_RAM_START && paddr - PMA_UARCH_RAM_START < PMA_UARCH_RAM_LENGTH) { + return "uarch.ram"; + } + // If in shadow, return refined name + if (paddr >= PMA_SHADOW_TLB_START && paddr - PMA_SHADOW_TLB_START < PMA_SHADOW_TLB_LENGTH) { + [[maybe_unused]] TLB_set_index set_index{}; + [[maybe_unused]] uint64_t slot_index{}; + return shadow_tlb_get_what_name(shadow_tlb_get_what(paddr, set_index, slot_index)); + } + if (paddr >= PMA_SHADOW_STATE_START && paddr - PMA_SHADOW_STATE_START < PMA_SHADOW_STATE_LENGTH) { + return shadow_state_get_what_name(shadow_state_get_what(paddr)); + } + if (paddr >= PMA_SHADOW_PMAS_START && paddr - PMA_SHADOW_PMAS_START < PMA_SHADOW_PMAS_LENGTH) { + return shadow_pmas_get_what_name(shadow_pmas_get_what(paddr)); + } + if (paddr >= PMA_SHADOW_UARCH_STATE_START && paddr - PMA_SHADOW_UARCH_STATE_START < PMA_SHADOW_UARCH_STATE_LENGTH) { + return shadow_uarch_state_get_what_name(shadow_uarch_state_get_what(paddr)); + } + return "memory"; +} + +void uarch_machine_bridge::write_word(uint64_t paddr, uint64_t val) { + if ((paddr & (sizeof(uint64_t) - 1)) != 0) { + throw std::runtime_error("misaligned write via uarch-machine bridge"); + } + if (paddr >= PMA_UARCH_RAM_START && paddr - PMA_UARCH_RAM_START < PMA_UARCH_RAM_LENGTH) { + write_uarch_memory_word(paddr, val); + return; + } + if (paddr >= PMA_SHADOW_STATE_START && paddr - PMA_SHADOW_STATE_START < PMA_SHADOW_STATE_LENGTH) { + write_shadow_state(paddr, val); + return; + } + write_memory_word(paddr, val); +} + +uint64_t uarch_machine_bridge::read_word(uint64_t paddr) const { + if ((paddr & (sizeof(uint64_t) - 1)) != 0) { + throw std::runtime_error("misaligned read via uarch-machine bridge"); + } + if (paddr >= PMA_UARCH_RAM_START && paddr - PMA_UARCH_RAM_START < PMA_UARCH_RAM_LENGTH) { + return read_uarch_memory_word(paddr); + } + if (paddr >= PMA_SHADOW_STATE_START && paddr - PMA_SHADOW_STATE_START < PMA_SHADOW_STATE_LENGTH) { + return read_shadow_state(paddr); + } + if (paddr >= PMA_SHADOW_TLB_START && paddr - PMA_SHADOW_TLB_START < PMA_SHADOW_TLB_LENGTH) { + return read_shadow_tlb(paddr); + } + return read_memory_word(paddr); +} + +uint64_t uarch_machine_bridge::read_memory_word(uint64_t paddr) const { + const auto &pma = m_m.find_pma_entry(paddr, sizeof(uint64_t)); + if (pma.get_istart_E() || !pma.get_istart_M()) { + std::ostringstream err; + err << "unhandled memory read from " << pma.get_description() << " at address 0x" << std::hex << paddr << "(" + << std::dec << paddr << ")"; + } + if (!pma.get_istart_R()) { + std::ostringstream err; + err << "attempted memory read from (non-readable) " << pma.get_description() << " at address 0x" << std::hex + << paddr << "(" << std::dec << paddr << ")"; + throw std::runtime_error{err.str()}; + } + const auto offset = paddr - pma.get_start(); + return aliased_aligned_read(pma.get_memory().get_host_memory() + offset); +} + +uint64_t uarch_machine_bridge::read_uarch_memory_word(uint64_t paddr) const { + const auto &pma = m_m.get_uarch_state().ram; + const auto offset = paddr - pma.get_start(); + return aliased_aligned_read(pma.get_memory().get_host_memory() + offset); +} + +void uarch_machine_bridge::write_uarch_memory_word(uint64_t paddr, uint64_t val) { + auto &pma = m_m.get_uarch_state().ram; + const auto offset = paddr - pma.get_start(); + aliased_aligned_write(pma.get_memory().get_host_memory() + offset, val); +} + +uint64_t uarch_machine_bridge::read_shadow_state(uint64_t paddr) const { + auto reg = shadow_state_get_what(paddr); + if (reg == shadow_state_what::unknown_) { + throw std::runtime_error("unhandled shadow state read via uarch-machine bridge"); + } + return m_m.read_reg(machine_reg_enum(reg)); +} + +uint64_t uarch_machine_bridge::read_shadow_tlb(uint64_t paddr) const { + TLB_set_index set_index{}; + uint64_t slot_index{}; + auto reg = shadow_tlb_get_what(paddr, set_index, slot_index); + if (reg == shadow_tlb_what::unknown_) { + throw std::runtime_error("unhandled shadow TLB read via uarch-machine bridge"); + } + return m_m.read_shadow_tlb(set_index, slot_index, reg); +} + +void uarch_machine_bridge::write_shadow_state(uint64_t paddr, uint64_t val) { + auto reg = shadow_state_get_what(paddr); + if (reg == shadow_state_what::unknown_) { + throw std::runtime_error("unhandled shadow state write via uarch-machine bridge"); + } + if (reg == shadow_state_what::x0) { + throw std::runtime_error("invalid write to shadow state x0 via uarch-machine bridge"); + } + m_m.write_reg(machine_reg_enum(reg), val); +} + +void uarch_machine_bridge::mark_dirty_page(uint64_t paddr, uint64_t pma_index) { + m_m.mark_dirty_page(paddr, pma_index); +} + +void uarch_machine_bridge::write_shadow_tlb(TLB_set_index set_index, uint64_t slot_index, uint64_t vaddr_page, + uint64_t vp_offset, uint64_t pma_index) { + m_m.check_shadow_tlb(set_index, slot_index, vaddr_page, vp_offset, pma_index, + "invalid write to shadow TLB via uarch-machine bridge: "); + m_m.write_shadow_tlb(set_index, slot_index, vaddr_page, vp_offset, pma_index); +} + +void uarch_machine_bridge::write_memory_word(uint64_t paddr, uint64_t val) { + auto &pma = m_m.find_pma_entry(paddr, sizeof(uint64_t)); + if (pma.get_istart_E() || !pma.get_istart_M()) { + std::ostringstream err; + err << "unhandled memory write to " << pma.get_description() << " at address 0x" << std::hex << paddr << "(" + << std::dec << paddr << ")"; + throw std::runtime_error{err.str()}; + } + if (!pma.get_istart_W()) { + std::ostringstream err; + err << "attempted memory write to (non-writeable) " << pma.get_description() << " at address 0x" << std::hex + << paddr << "(" << std::dec << paddr << ")"; + throw std::runtime_error{err.str()}; + } + const auto offset = paddr - pma.get_start(); + aliased_aligned_write(pma.get_memory().get_host_memory() + offset, val); +} + +} // namespace cartesi diff --git a/src/uarch-machine-bridge.h b/src/uarch-machine-bridge.h new file mode 100644 index 000000000..0525f0d45 --- /dev/null +++ b/src/uarch-machine-bridge.h @@ -0,0 +1,57 @@ +// Copyright Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: LGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify it under +// the terms of the GNU Lesser General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) any +// later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +// PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License along +// with this program (see COPYING). If not, see . +// + +#ifndef UARCH_MACHINE_BRIDGE_H +#define UARCH_MACHINE_BRIDGE_H + +#include +#include + +#include "tlb.h" + +namespace cartesi { + +class machine; + +/// \brief Allows microarchitecture code to access the machine state +class uarch_machine_bridge { + + machine &m_m; // NOLINT(cppcoreguidelines-avoid-const-or-ref-data-members) + +public: + explicit uarch_machine_bridge(machine &m) : m_m(m) { + ; + } + uint64_t read_word(uint64_t paddr) const; + void write_word(uint64_t paddr, uint64_t val); + void mark_dirty_page(uint64_t paddr, uint64_t pma_index); + void write_shadow_tlb(TLB_set_index set_index, uint64_t slot_index, uint64_t vaddr_page, uint64_t vp_offset, + uint64_t pma_index); + static const char *get_what_name(uint64_t paddr); + +private: + void write_shadow_state(uint64_t paddr, uint64_t val); + void write_memory_word(uint64_t paddr, uint64_t val); + void write_uarch_memory_word(uint64_t paddr, uint64_t val); + uint64_t read_shadow_state(uint64_t paddr) const; + uint64_t read_memory_word(uint64_t paddr) const; + uint64_t read_uarch_memory_word(uint64_t paddr) const; + uint64_t read_shadow_tlb(uint64_t paddr) const; +}; + +} // namespace cartesi + +#endif diff --git a/src/uarch-machine.cpp b/src/uarch-machine.cpp deleted file mode 100644 index dbff4c580..000000000 --- a/src/uarch-machine.cpp +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright Cartesi and individual authors (see AUTHORS) -// SPDX-License-Identifier: LGPL-3.0-or-later -// -// This program is free software: you can redistribute it and/or modify it under -// the terms of the GNU Lesser General Public License as published by the Free -// Software Foundation, either version 3 of the License, or (at your option) any -// later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT ANY -// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A -// PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License along -// with this program (see COPYING). If not, see . -// - -#include "uarch-machine.h" - -#include -#include -#include -#include - -#include "pma-constants.h" -#include "pma.h" -#include "riscv-constants.h" -#include "shadow-uarch-state-factory.h" -#include "uarch-config.h" -#include "uarch-constants.h" -#include "uarch-pristine.h" - -namespace cartesi { - -using namespace std::string_literals; - -const pma_entry::flags ram_flags{.R = true, - .W = true, - .X = true, - .IR = true, - .IW = true, - .DID = PMA_ISTART_DID::memory}; - -uarch_machine::uarch_machine(uarch_config c) : m_c{c} { - m_s.pc = c.processor.pc; - m_s.cycle = c.processor.cycle; - m_s.halt_flag = c.processor.halt_flag; - // General purpose registers - for (int i = 1; i < UARCH_X_REG_COUNT; i++) { - m_s.x[i] = c.processor.x[i]; - } - // Register shadow state - m_s.shadow_state = make_shadow_uarch_state_pma_entry(PMA_SHADOW_UARCH_STATE_START, PMA_SHADOW_UARCH_STATE_LENGTH); - // Register RAM - constexpr auto ram_description = "uarch RAM"; - if (!c.ram.image_filename.empty()) { - // Load RAM image from file - m_s.ram = - make_callocd_memory_pma_entry(ram_description, PMA_UARCH_RAM_START, UARCH_RAM_LENGTH, c.ram.image_filename) - .set_flags(ram_flags); - } else { - // Load embedded pristine RAM image - m_s.ram = make_callocd_memory_pma_entry(ram_description, PMA_UARCH_RAM_START, PMA_UARCH_RAM_LENGTH) - .set_flags(ram_flags); - if (uarch_pristine_ram_len > m_s.ram.get_length()) { - throw std::runtime_error("embedded uarch ram image does not fit in uarch ram pma"); - } - memcpy(m_s.ram.get_memory().get_host_memory(), uarch_pristine_ram, uarch_pristine_ram_len); - } -} - -pma_entry &uarch_machine::find_pma_entry(uint64_t paddr, uint64_t length) { - // NOLINTNEXTLINE(cppcoreguidelines-pro-type-const-cast): remove const to reuse code - return const_cast(std::as_const(*this).find_pma_entry(paddr, length)); -} - -const pma_entry &uarch_machine::find_pma_entry(uint64_t paddr, uint64_t length) const { - if (m_s.ram.contains(paddr, length)) { - return m_s.ram; - } - return m_s.empty_pma; -} - -} // namespace cartesi diff --git a/src/uarch-machine.h b/src/uarch-machine.h deleted file mode 100644 index 33e2a7f34..000000000 --- a/src/uarch-machine.h +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright Cartesi and individual authors (see AUTHORS) -// SPDX-License-Identifier: LGPL-3.0-or-later -// -// This program is free software: you can redistribute it and/or modify it under -// the terms of the GNU Lesser General Public License as published by the Free -// Software Foundation, either version 3 of the License, or (at your option) any -// later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT ANY -// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A -// PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License along -// with this program (see COPYING). If not, see . -// - -#ifndef UARCH_MACHINE_H -#define UARCH_MACHINE_H - -/// \file -/// \brief Cartesi microarchitecture machine -#include - -#include "pma.h" -#include "uarch-config.h" -#include "uarch-state.h" - -namespace cartesi { - -/// \class uarch_machine -/// \brief Cartesi Machine Microarchitecture implementation -class uarch_machine final { - uarch_state m_s; ///< Opaque microarchitecture machine state - uarch_config m_c; ///< Copy of initialization config - -public: - /// \brief Constructor from machine configuration - // I will deal with clang-tidy later. - explicit uarch_machine(uarch_config c); - /// \brief Destructor. - ~uarch_machine() = default; - - /// \brief No default constructor - uarch_machine() = delete; - /// \brief No copy constructor - uarch_machine(const uarch_machine &other) = delete; - /// \brief No move constructor - uarch_machine(uarch_machine &&other) = delete; - /// \brief No copy assignment - uarch_machine &operator=(const uarch_machine &other) = delete; - /// \brief No move assignment - uarch_machine &operator=(uarch_machine &&other) = delete; - - /// \brief Returns machine state for direct access. - uarch_state &get_state() { - return m_s; - } - - /// \brief Returns machine state for direct read-only access. - const uarch_state &get_state() const { - return m_s; - } - - /// \brief Obtain PMA entry that covers a given physical memory region - /// \param s Pointer to machine state. - /// \param paddr Start of physical memory region. - /// \param length Length of physical memory region. - /// \returns Corresponding entry if found, or a sentinel entry - /// for an empty range. - pma_entry &find_pma_entry(uint64_t paddr, uint64_t length); - - /// \brief Obtain PMA entry that covers a given physical memory region - const pma_entry &find_pma_entry(uint64_t paddr, uint64_t length) const; - - /// \brief Returns copy of initialization config. - const uarch_config &get_initial_config() const { - return m_c; - } -}; - -} // namespace cartesi - -#endif diff --git a/src/uarch-record-state-access.h b/src/uarch-record-state-access.h index f73c313f2..7a8176650 100644 --- a/src/uarch-record-state-access.h +++ b/src/uarch-record-state-access.h @@ -28,66 +28,46 @@ #include #include "access-log.h" +#include "host-addr.h" #include "i-hasher.h" #include "i-state-access.h" #include "i-uarch-state-access.h" #include "machine-merkle-tree.h" -#include "machine-state.h" #include "machine.h" #include "meta.h" #include "pma.h" #include "riscv-constants.h" +#include "scoped-note.h" +#include "shadow-tlb.h" #include "shadow-uarch-state.h" #include "strict-aliasing.h" -#include "uarch-bridge.h" #include "uarch-constants.h" +#include "uarch-machine-bridge.h" #include "uarch-pristine-state-hash.h" #include "uarch-pristine.h" #include "uarch-state.h" namespace cartesi { +using namespace std::string_literals; + /// \details The uarch_record_state_access logs all access to the machine state. class uarch_record_state_access : public i_uarch_state_access { + using hasher_type = machine_merkle_tree::hasher_type; using hash_type = machine_merkle_tree::hash_type; // NOLINTBEGIN(cppcoreguidelines-avoid-const-or-ref-data-members) uarch_state &m_us; machine &m_m; ///< Macro machine - machine_state &m_s; + uarch_machine_bridge m_b; // NOLINTEND(cppcoreguidelines-avoid-const-or-ref-data-members) std::shared_ptr m_log; ///< Pointer to access log - /// \brief Obtain Memory PMA entry that covers a given physical memory region - /// \param paddr Start of physical memory region. - /// \param length Length of physical memory region. - /// \returns Corresponding entry if found, or a sentinel entry - /// for an empty range. - pma_entry &find_memory_pma_entry(uint64_t paddr, uint64_t length) { - // First, search microarchitecture's private PMA entries - if (m_us.ram.contains(paddr, length)) { - return m_us.ram; - } - int i = 0; - // Search machine memory PMA entries (not devices or anything else) - while (true) { - auto &pma = m_s.pmas[i]; - // The pmas array always contain a sentinel. It is an entry with - // zero length. If we hit it, return it - if (pma.get_length() == 0) { - return pma; - } - if (pma.get_istart_M() && pma.contains(paddr, length)) { - return pma; - } - i++; - } - } - - static void get_hash(const access_data &data, hash_type &hash) { - hasher_type hasher; + static auto get_hash(hasher_type &hasher, const access_data &data) { + hash_type hash{}; get_merkle_tree_hash(hasher, data.data(), data.size(), machine_merkle_tree::get_word_size(), hash); + return hash; } public: @@ -97,7 +77,7 @@ class uarch_record_state_access : public i_uarch_state_access(log_type)) { ; } @@ -123,356 +103,253 @@ class uarch_record_state_access : public i_uarch_state_access m_log; ///< Pointer to log receiving annotations - std::string m_text; ///< String with the text for the annotation - - public: - /// \brief Constructor adds the "begin" bracketing note - /// \param log Pointer to access log receiving annotations - /// \param text Pointer to annotation text - /// \details A note is added at the moment of construction - scoped_note(std::shared_ptr log, const char *text) : m_log(std::move(log)), m_text(text) { - if (m_log) { - m_log->push_bracket(bracket_type::begin, text); - } - } +private: + void do_push_begin_bracket(const char *text) { + m_log->push_begin_bracket(text); + } - /// \brief No copy constructors - scoped_note(const scoped_note &) = delete; - - /// \brief No copy assignment - scoped_note &operator=(const scoped_note &) = delete; - - /// \brief Default move constructor - /// \details This is OK because the shared_ptr to log will be - /// empty afterwards and we explicitly test for this - /// condition before writing the "end" bracketing note - scoped_note(scoped_note &&) = default; - - /// \brief Default move assignment - /// \details This is OK because the shared_ptr to log will be - /// empty afterwards and we explicitly test for this - /// condition before writing the "end" bracketing note - scoped_note &operator=(scoped_note &&) = default; - - /// \brief Destructor adds the "end" bracketing note - /// if the log shared_ptr is not empty - ~scoped_note() { - if (m_log) { - try { - m_log->push_bracket(bracket_type::end, m_text.c_str()); - } catch (...) { // NOLINT(bugprone-empty-catch) - // push_bracket with type begin always reserves space for the end bracket - // so either the user using unbalanced with begin/end, or there - // is no way we ran out of memory. therefore, if we did run out of - // memory, it was because the system is completely screwed anyway *and* the - // user is using unbalanced brackets. it's ok to quietly ignore, as the user's - // brackets were already unbalanced anyway... - } - } - } - }; + void do_push_end_bracket(const char *text) { + m_log->push_end_bracket(text); + } -private: - /// \brief Logs a read access of a uint64_t word from the machine state. - /// \param paligned Physical address in the machine state, aligned to a 64-bit word. - /// \param val Value read. - /// \param text Textual description of the access. - uint64_t log_read(uint64_t paligned, uint64_t val, const char *text) const { - static_assert(machine_merkle_tree::get_log2_word_size() >= log2_size::value, - "Merkle tree word size must be at least as large as a machine word"); - if ((paligned & (sizeof(uint64_t) - 1)) != 0) { - throw std::invalid_argument{"paligned is not aligned to word size"}; + auto do_make_scoped_note(const char *text) { + return scoped_note{*this, text}; + } + + static std::pair adjust_access(uint64_t paddr, int log2_size) { + static_assert(cartesi::log2_size::value <= machine_merkle_tree::get_log2_word_size(), + "Merkle tree word size must not be smaller than machine word size"); + if (((paddr >> log2_size) << log2_size) != paddr) { + throw std::invalid_argument{"misaligned access"}; } - const uint64_t pleaf_aligned = paligned & ~(machine_merkle_tree::get_word_size() - 1); - access a; + const auto log2_word_size = machine_merkle_tree::get_log2_word_size(); + const auto log2_access_size = std::max(log2_size, log2_word_size); + const auto access_paddr = (paddr >> log2_access_size) << log2_access_size; + return {access_paddr, log2_access_size}; + } + + void log_access(access &&a, const char *text) const { + m_log->push_access(std::move(a), text); + } - // We can skip updating the merkle tree while getting the proof because we assume that: - // 1) A full merkle tree update was called at the beginning of machine::log_step_uarch() - // 2) We called update_merkle_tree_page on all write accesses - const auto proof = - m_m.get_proof(pleaf_aligned, machine_merkle_tree::get_log2_word_size(), skip_merkle_tree_update); - // We just store the sibling hashes in the access because this is the only missing piece of data needed to - // reconstruct the proof + static void log_access_type(access &a, access_type type) { + a.set_type(type); + } + + static void log_access_range(access &a, uint64_t paddr, int log2_size) { + a.set_address(paddr); + a.set_log2_size(log2_size); + } + + void log_access_siblings_and_read_hash(access &a, uint64_t paddr, int log2_size) const { + // Since the tree was updated before we started collecting the log, we only update after writes + const auto proof = m_m.get_proof(paddr, log2_size, skip_merkle_tree_update); + // The only pieces of data we use from the proof are the target hash and the siblings a.set_sibling_hashes(proof.get_sibling_hashes()); + a.set_read_hash(proof.get_target_hash()); + } + + static void log_written_hash(access &a, const hash_type &written_hash) { + a.get_written_hash().emplace(written_hash); + } - a.set_type(access_type::read); - a.set_address(paligned); - a.set_log2_size(log2_size::value); + const auto &log_read_data(access &a, uint64_t paddr, int log2_size) const { // NOLINTBEGIN(bugprone-unchecked-optional-access) - // we log the leaf data at pleaf_aligned that contains the word at paligned + const auto size = UINT64_C(1) << log2_size; a.get_read().emplace(); - a.get_read().value().resize(machine_merkle_tree::get_word_size()); - // read the entire leaf where the word is located - m_m.read_memory(pleaf_aligned, a.get_read().value().data(), machine_merkle_tree::get_word_size()); - get_hash(a.get_read().value(), a.get_read_hash()); - // ensure that the read data is the same as the value read - const int word_offset = static_cast(paligned - pleaf_aligned); // offset of word in leaf - const uint64_t logged_val = get_word_access_data(a.get_read().value(), word_offset); - if (val != logged_val) { - throw std::runtime_error("read value does not match logged value"); - } + a.get_read().value().resize(size); + m_m.read_memory(paddr, a.get_read().value().data(), size); + return a.get_read().value(); // NOLINTEND(bugprone-unchecked-optional-access) - m_log->push_access(std::move(a), text); - return val; } - /// \brief Logs a write access to a uint64_t word before it happens. - /// \param paligned Physical address of the word in the machine state (Must be aligned to a 64-bit word). - /// \param val Value to write. - /// \param text Textual description of the access. - void log_before_write(uint64_t paligned, uint64_t word, const char *text) { - static_assert(machine_merkle_tree::get_log2_word_size() >= log2_size::value, - "Merkle tree word size must be at least as large as a machine word"); - if ((paligned & (sizeof(uint64_t) - 1)) != 0) { - throw std::invalid_argument{"paligned is not aligned to word size"}; + void log_read_data_if_requested(access &a, uint64_t paddr, int log2_size) const { + if (m_log->get_log_type().has_large_data()) { + std::ignore = log_read_data(a, paddr, log2_size); } - const uint64_t pleaf_aligned = paligned & ~(machine_merkle_tree::get_word_size() - 1); - access a; - - // We can skip updating the merkle tree while getting the proof because we assume that: - // 1) A full merkle tree update was called at the beginning of machine::log_step_uarch() - // 2) We called update_merkle_tree_page on all write accesses - const auto proof = - m_m.get_proof(pleaf_aligned, machine_merkle_tree::get_log2_word_size(), skip_merkle_tree_update); - // We just store the sibling hashes in the access because this is the only missing piece of data needed to - // reconstruct the proof - a.set_sibling_hashes(proof.get_sibling_hashes()); + } - a.set_type(access_type::write); - a.set_address(paligned); - a.set_log2_size(log2_size::value); + void log_written_data(access &a, uint64_t paddr, int log2_size) const { // NOLINTBEGIN(bugprone-unchecked-optional-access) - // we log the leaf data at pleaf_aligned that contains the word at paligned - a.get_read().emplace(); - a.get_read().value().resize(machine_merkle_tree::get_word_size()); - m_m.read_memory(pleaf_aligned, a.get_read().value().data(), machine_merkle_tree::get_word_size()); - get_hash(a.get_read().value(), a.get_read_hash()); - // the logged written data is the same as the read data, but with the word at paligned replaced by word - a.set_written(access_data(a.get_read().value())); // copy the read data - const int word_offset = static_cast(paligned - pleaf_aligned); // offset of word in leaf - replace_word_access_data(word, a.get_written().value(), word_offset); // replace the word - // compute the hash of the written data - a.get_written_hash().emplace(); - get_hash(a.get_written().value(), a.get_written_hash().value()); + const auto size = UINT64_C(1) << log2_size; + a.get_written().emplace(); + a.get_written().value().resize(size); + m_m.read_memory(paddr, a.get_written().value().data(), size); // NOLINTEND(bugprone-unchecked-optional-access) - m_log->push_access(std::move(a), text); } - /// \brief Updates the Merkle tree after the modification of a word in the machine state. - /// \param paligned Physical address in the machine state, aligned to a 64-bit word. - void update_after_write(uint64_t paligned) { - assert((paligned & (sizeof(uint64_t) - 1)) == 0); - [[maybe_unused]] const bool updated = m_m.update_merkle_tree_page(paligned); - assert(updated); + void log_written_data_if_requested(access &a, uint64_t paddr, int log2_size) const { + if (m_log->get_log_type().has_large_data()) { + log_written_data(a, paddr, log2_size); + } } - /// \brief Logs a write access before it happens, writes, and then update the Merkle tree. - /// \param paligned Physical address of the word in the machine state (Must be aligned to a 64-bit word). - /// \param dest Reference to value before writing. - /// \param val Value to write to \p dest. - /// \param text Textual description of the access. - void log_before_write_write_and_update(uint64_t paligned, uint64_t &dest, uint64_t word, const char *text) { - assert((paligned & (sizeof(uint64_t) - 1)) == 0); - log_before_write(paligned, word, text); - dest = word; - update_after_write(paligned); + uint64_t log_read_word_access(uint64_t paddr, const char *text) const { + const auto log2_size = cartesi::log2_size::value; + access a; + log_access_type(a, access_type::read); + log_access_range(a, paddr, log2_size); + const auto [access_paddr, access_log2_size] = adjust_access(paddr, log2_size); + log_access_siblings_and_read_hash(a, access_paddr, access_log2_size); + const auto &read_data = log_read_data(a, access_paddr, access_log2_size); + const auto val_offset = paddr - access_paddr; + const auto val = get_word_access_data(read_data, val_offset); + log_access(std::move(a), text); + return val; } - void log_before_write_write_and_update(uint64_t paligned, bool &dest, bool val, const char *text) { - auto dest64 = static_cast(dest); - log_before_write_write_and_update(paligned, dest64, static_cast(val), text); - dest = (dest64 != 0); - update_after_write(paligned); + uint64_t log_read_reg_access(machine_reg reg) const { + return log_read_word_access(machine_reg_address(reg), machine_reg_get_name(reg)); } - // Declare interface as friend to it can forward calls to the "overridden" methods. - friend i_uarch_state_access; - - void do_push_bracket(bracket_type &type, const char *text) { - m_log->push_bracket(type, text); + template + void log_write_access(uint64_t paddr, int log2_size, WRITE_UPDATE_F write_and_update, const char *text) { + access a; + log_access_type(a, access_type::write); + log_access_range(a, paddr, log2_size); + const auto [access_paddr, access_log2_size] = adjust_access(paddr, log2_size); + log_access_siblings_and_read_hash(a, access_paddr, access_log2_size); + // We *need* the read data for small writes, because we splice the written into it + if (log2_size < machine_merkle_tree::get_log2_word_size()) { + std::ignore = log_read_data(a, access_paddr, access_log2_size); + } else { + log_read_data_if_requested(a, access_paddr, access_log2_size); + } + // Call functor to perform the write and update the tree + write_and_update(); + // The functor updated the tree, so we don't do it again + log_written_hash(a, m_m.get_merkle_tree_node_hash(access_paddr, access_log2_size, skip_merkle_tree_update)); + // We don't *need* the written for small writes, but it is convenient to always have it (for debugging purposes) + if (log2_size < machine_merkle_tree::get_log2_word_size()) { + log_written_data(a, access_paddr, access_log2_size); + } else { + log_written_data_if_requested(a, access_paddr, access_log2_size); + } + log_access(std::move(a), text); } - scoped_note do_make_scoped_note(const char *text) { - return scoped_note{m_log, text}; + void log_write_reg_access(machine_reg reg, uint64_t val) { + log_write_access( + machine_reg_address(reg), cartesi::log2_size::value, + [this, reg, val]() { + m_m.write_reg(reg, val); + if (!m_m.update_merkle_tree_page(machine_reg_address(reg))) { + throw std::invalid_argument{"error updating Merkle tree"}; + }; + }, + machine_reg_get_name(reg)); } - uint64_t do_read_x(int reg) const { - return log_read(machine_reg_address(machine_reg::uarch_x0, reg), m_us.x[reg], "uarch.x"); + // Declare interface as friend to it can forward calls to the "overridden" methods. + friend i_uarch_state_access; + + uint64_t do_read_x(int i) const { + return log_read_reg_access(machine_reg_enum(machine_reg::uarch_x0, i)); } - void do_write_x(int reg, uint64_t val) { - assert(reg != 0); - log_before_write_write_and_update(machine_reg_address(machine_reg::uarch_x0, reg), m_us.x[reg], val, "uarch.x"); + void do_write_x(int i, uint64_t val) { + assert(i != 0); + log_write_reg_access(machine_reg_enum(machine_reg::uarch_x0, i), val); } uint64_t do_read_pc() const { - return log_read(machine_reg_address(machine_reg::uarch_pc), m_us.pc, "uarch.pc"); + return log_read_reg_access(machine_reg::uarch_pc); } void do_write_pc(uint64_t val) { - log_before_write_write_and_update(machine_reg_address(machine_reg::uarch_pc), m_us.pc, val, "uarch.pc"); + log_write_reg_access(machine_reg::uarch_pc, val); } uint64_t do_read_cycle() const { - return log_read(machine_reg_address(machine_reg::uarch_cycle), m_us.cycle, "uarch.cycle"); + return log_read_reg_access(machine_reg::uarch_cycle); } void do_write_cycle(uint64_t val) { - log_before_write_write_and_update(machine_reg_address(machine_reg::uarch_cycle), m_us.cycle, val, - "uarch.cycle"); - } - - bool do_read_halt_flag() const { - return log_read(machine_reg_address(machine_reg::uarch_halt_flag), static_cast(m_us.halt_flag), - "uarch.halt_flag") != 0; + log_write_reg_access(machine_reg::uarch_cycle, val); } - void do_set_halt_flag() { - log_before_write_write_and_update(machine_reg_address(machine_reg::uarch_halt_flag), m_us.halt_flag, true, - "uarch.halt_flag"); + uint64_t do_read_halt_flag() const { + return log_read_reg_access(machine_reg::uarch_halt_flag); } - void do_reset_halt_flag() { - log_before_write_write_and_update(machine_reg_address(machine_reg::uarch_halt_flag), m_us.halt_flag, false, - "uarch.halt_flag"); + void do_write_halt_flag(uint64_t val) { + log_write_reg_access(machine_reg::uarch_halt_flag, val); } - uint64_t do_read_word(uint64_t paddr) { - assert((paddr & (sizeof(uint64_t) - 1)) == 0); - // Find a memory range that contains the specified address - auto &pma = find_memory_pma_entry(paddr, sizeof(uint64_t)); - if (pma.get_istart_E()) { - // Memory not found. Try reading a machine state register - return read_register(paddr); - } - if (!pma.get_istart_R()) { - throw std::runtime_error("pma is not readable"); + uint64_t do_read_word(uint64_t paddr) const { + if ((paddr & (sizeof(uint64_t) - 1)) != 0) { + throw std::runtime_error("misaligned read from uarch"); } - // Found a readable memory range. Access host memory accordingly. - const uint64_t hoffset = paddr - pma.get_start(); - auto *hmem = pma.get_memory().get_host_memory(); - auto data = aliased_aligned_read(hmem + hoffset); - log_read(paddr, data, "memory"); - return data; - } - - /// \brief Reads a uint64 machine state register mapped to a memory address - /// \param paddr Address of the state register - /// \param data Pointer receiving register value - uint64_t read_register(uint64_t paddr) { - auto data = uarch_bridge::read_register(paddr, m_s); - const auto *name = uarch_bridge::get_register_name(paddr); - log_read(paddr, data, name); - return data; - } - - void do_write_word(uint64_t paddr, uint64_t data) { - assert((paddr & (sizeof(uint64_t) - 1)) == 0); - // Find a memory range that contains the specified address - auto &pma = find_memory_pma_entry(paddr, sizeof(uint64_t)); - if (pma.get_istart_E()) { - // Memory not found. Try to write a machine state register - write_register(paddr, data); - return; - } - if (!pma.get_istart_W()) { - throw std::runtime_error("pma is not writable"); - } - // Found a writable memory range. Access host memory accordingly. - - // The proof in the log uses the Merkle tree before the state is modified. - // But log needs the word value before and after the change. - // So we first get value before the write - const uint64_t hoffset = paddr - pma.get_start(); - unsigned char *hmem = pma.get_memory().get_host_memory(); - void *hdata = hmem + hoffset; - // Log the write access - log_before_write(paddr, data, "memory"); - // Actually modify the state - aliased_aligned_write(hdata, data); - - // When proofs are requested, we always want to update the Merkle tree - update_after_write(paddr); + return log_read_word_access(paddr, uarch_machine_bridge::get_what_name(paddr)); } - /// \brief Writes a uint64 machine state register mapped to a memory address - /// \param paddr Address of the state register - /// \param data New register value - void write_register(uint64_t paddr, uint64_t data) { - const auto *name = uarch_bridge::get_register_name(paddr); - log_before_write(paddr, data, name); - uarch_bridge::write_register(paddr, m_s, data); - update_after_write(paddr); + void do_write_word(uint64_t paddr, uint64_t val) { + if ((paddr & (sizeof(uint64_t) - 1)) != 0) { + throw std::runtime_error("misaligned write to uarch"); + } + log_write_access( + paddr, cartesi::log2_size::value, + [this, paddr, val]() { + m_b.write_word(paddr, val); + if (!m_m.update_merkle_tree_page(paddr)) { + throw std::invalid_argument{"error updating Merkle tree"}; + }; + }, + uarch_machine_bridge::get_what_name(paddr)); } - /// \brief Fallback to error on all other word sizes - template - void write_register(uint64_t /*paddr*/, T /*data*/) { - throw std::runtime_error("invalid memory write access from microarchitecture"); + void do_write_tlb(TLB_set_index set_index, uint64_t slot_index, uint64_t vaddr_page, uint64_t vp_offset, + uint64_t pma_index) { + const auto slot_paddr = shadow_tlb_get_abs_addr(set_index, slot_index); + log_write_access( + slot_paddr, SHADOW_TLB_SLOT_LOG2_SIZE, + [this, set_index, slot_index, vaddr_page, vp_offset, pma_index]() { + m_b.write_shadow_tlb(set_index, slot_index, vaddr_page, vp_offset, pma_index); + // Entire slot is in a single page + if (!m_m.update_merkle_tree_page(shadow_tlb_get_abs_addr(set_index, slot_index))) { + throw std::invalid_argument{"error updating Merkle tree"}; + }; + }, + "tlb.slot"); + // Writes to TLB slots have to be atomic. + // We can only do atomic writes of entire Merkle tree nodes. + // Therefore, TLB slot must have a power-of-two size, or at least be aligned to it. + static_assert(SHADOW_TLB_SLOT_SIZE == sizeof(shadow_tlb_slot), "shadow TLB slot size is wrong"); + static_assert((UINT64_C(1) << SHADOW_TLB_SLOT_LOG2_SIZE) == SHADOW_TLB_SLOT_SIZE, + "shadow TLB slot log2 size is wrong"); + static_assert(SHADOW_TLB_SLOT_LOG2_SIZE >= machine_merkle_tree::get_log2_word_size(), + "shadow TLB slot must fill at least an entire Merkle tree word"); } void do_reset_state() { - // The pristine uarch state decided at compile time and never changes. - // We set all uarch registers and RAM to their initial values and - // log a single write access to the entire uarch memory range. - // This write access does not contain any data, just hashes, unless log_type.large_data is enabled. - access a; - a.set_type(access_type::write); - a.set_address(UARCH_STATE_START_ADDRESS); - a.set_log2_size(UARCH_STATE_LOG2_SIZE); - - // Always compute the proof, even if we are not logging it, because: - // (1) we always need to compute read_hash. - // (2) Depending on the log type, we may also need to compute the proof. - // (3) proof.target_hash is the value that we need for a.read_hash in (1). - auto proof = m_m.get_proof(UARCH_STATE_START_ADDRESS, UARCH_STATE_LOG2_SIZE); - a.set_read_hash(proof.get_target_hash()); - if (m_log->get_log_type().has_large_data()) { - // log read data, if debug info is enabled - a.get_read().emplace(get_uarch_state_image()); - } + //??D I'd like to add an static_assert or some other guard mechanism to + // guarantee that uarch.ram and uarch.shadow are alone in the entire + // span of their common Merkle tree parent node + log_write_access( + UARCH_STATE_START_ADDRESS, UARCH_STATE_LOG2_SIZE, + [this]() { + m_m.reset_uarch(); + // reset_uarch() marks all modified pages as dirty + if (!m_m.update_merkle_tree()) { + throw std::invalid_argument{"error updating Merkle tree"}; + } + }, + "uarch.state"); + } - // We just store the sibling hashes in the access because this is the only missing piece of data needed to - // reconstruct the proof - a.set_sibling_hashes(proof.get_sibling_hashes()); + // NOLINTNEXTLINE(readability-convert-member-functions-to-static) + void do_putchar(uint8_t c) { + os_putchar(c); + } - a.set_written_hash(get_uarch_pristine_state_hash()); + void do_mark_dirty_page(uint64_t paddr, uint64_t pma_index) { + // forward to bridge, and no need to log + m_b.mark_dirty_page(paddr, pma_index); + } - // Restore uarch to pristine state - m_us.halt_flag = false; - m_us.pc = UARCH_PC_INIT; - m_us.cycle = UARCH_CYCLE_INIT; - for (int i = 1; i < UARCH_X_REG_COUNT; i++) { - m_us.x[i] = UARCH_X_INIT; - } - m_us.ram.write_memory(m_us.ram.get_start(), uarch_pristine_ram, uarch_pristine_ram_len); - m_us.ram.fill_memory(m_us.ram.get_start() + uarch_pristine_ram_len, 0, - m_us.ram.get_length() - uarch_pristine_ram_len); - if (m_log->get_log_type().has_large_data()) { - // log written data, if debug info is enabled - a.get_written().emplace(get_uarch_state_image()); - } - m_log->push_access(a, "uarch_state"); - } - - /// \brief Returns the image of the entire uarch state - /// \return access_data containing the image of the current uarch state - access_data get_uarch_state_image() { - constexpr int uarch_data_len = uint64_t{1} << UARCH_STATE_LOG2_SIZE; - access_data data(uarch_data_len, 0); - constexpr auto ram_offset = UARCH_RAM_START_ADDRESS - UARCH_STATE_START_ADDRESS; - // copy shadow state data - const unsigned char *page_data = nullptr; - auto peek = m_us.shadow_state.get_peek(); - if (!peek(m_us.shadow_state, m_m, 0, &page_data, data.data())) { - throw std::runtime_error{"peek failed"}; - } - // copy ram data - memcpy(data.data() + ram_offset, m_us.ram.get_memory().get_host_memory(), m_us.ram.get_length()); - return data; + // NOLINTNEXTLINE(readability-convert-member-functions-to-static) + constexpr const char *do_get_name() const { + return "uarch_record_state_access"; } }; diff --git a/src/uarch-replay-state-access.h b/src/uarch-replay-state-access.h index 437ff40c1..3f2d3e35c 100644 --- a/src/uarch-replay-state-access.h +++ b/src/uarch-replay-state-access.h @@ -34,8 +34,8 @@ #include "i-uarch-state-access.h" #include "machine-merkle-tree.h" #include "meta.h" +#include "shadow-tlb.h" #include "shadow-uarch-state.h" -#include "uarch-bridge.h" #include "uarch-constants.h" #include "uarch-pristine-state-hash.h" @@ -64,9 +64,7 @@ class uarch_replay_state_access : public i_uarch_state_access= log2_size::value, - "Merkle tree word size must be at least as large as a machine word"); - if ((paligned & (sizeof(uint64_t) - 1)) != 0) { - throw std::invalid_argument{"address not aligned to word size"}; - } + static std::pair adjust_access(uint64_t paddr, int log2_size) { + static_assert(cartesi::log2_size::value <= machine_merkle_tree::get_log2_word_size(), + "Merkle tree word size must not be smaller than machine word size"); + const auto log2_word_size = machine_merkle_tree::get_log2_word_size(); + const auto log2_access_size = std::max(log2_size, log2_word_size); + const auto access_paddr = (paddr >> log2_access_size) << log2_access_size; + return {access_paddr, log2_access_size}; + } + + const access &check_access(const char *text) { if (m_next_access >= m_accesses.size()) { - throw std::invalid_argument{"too few accesses in log"}; + throw std::invalid_argument{"log is missing access " + access_to_report() + " to " + text}; + } + return m_accesses[m_next_access]; + } + + void check_access_type(const access &a, access_type type, const char *text) { + if (a.get_type() != type) { + throw std::invalid_argument{ + "expected " + access_to_report() + " to " + access_type_name(type) + " " + text}; } - const auto &access = m_accesses[m_next_access]; - if (access.get_type() != access_type::read) { - throw std::invalid_argument{"expected " + access_to_report() + " to read " + text}; + if (type == access_type::read) { + if (a.get_written().has_value()) { + throw std::invalid_argument{ + "unexpected written data in " + access_to_report() + " read access to " + text}; + } + if (a.get_written_hash().has_value()) { + throw std::invalid_argument{ + "unexpected written hash in " + access_to_report() + " read access to " + text}; + } } - if (access.get_address() != paligned) { + } + + void check_access_range(const access &a, access_type type, uint64_t paddr, uint64_t log2_size, const char *text) { + if (a.get_address() != paddr) { std::ostringstream err; - err << "expected " << access_to_report() << " to read " << text << " address 0x" << std::hex << paligned - << "(" << std::dec << paligned << ")"; + err << "expected " << access_to_report() << " to " << access_type_name(type) << " " << text + << " at address 0x" << std::hex << paddr << "(" << std::dec << paddr << ")"; throw std::invalid_argument{err.str()}; } - if (access.get_log2_size() != log2_size::value) { - throw std::invalid_argument{"expected " + access_to_report() + " to read 2^" + - std::to_string(machine_merkle_tree::get_log2_word_size()) + " bytes from " + text}; - } - if (!access.get_read().has_value()) { - throw std::invalid_argument{"missing read " + std::string(text) + " data at " + access_to_report()}; + if (a.get_log2_size() != static_cast(log2_size)) { + throw std::invalid_argument{"expected " + access_to_report() + " to " + text + " to " + + access_type_name(type) + " 2^" + std::to_string(log2_size) + " bytes"}; } - // NOLINTBEGIN(bugprone-unchecked-optional-access) - const auto &read_data = access.get_read().value(); - if (read_data.size() != machine_merkle_tree::get_word_size()) { - throw std::invalid_argument{"expected read " + std::string(text) + " data to contain 2^" + - std::to_string(machine_merkle_tree::get_log2_word_size()) + " bytes at " + access_to_report()}; - } - // check if logged read data hashes to the logged read hash - hash_type computed_read_hash{}; - get_hash(m_hasher, read_data, computed_read_hash); - if (access.get_read_hash() != computed_read_hash) { - throw std::invalid_argument{"logged read data of " + std::string(text) + - " data does not hash to the logged read hash at " + access_to_report()}; - } - auto proof = access.make_proof(m_root_hash); + } + + auto check_access_siblings_and_read_hash(const access &a, const char *text) { + const auto proof = a.make_proof(m_root_hash); if (!proof.verify(m_hasher)) { - throw std::invalid_argument{"Mismatch in root hash of " + access_to_report()}; + throw std::invalid_argument{ + "siblings and read hash do not match root hash before " + access_to_report() + " to " + text}; } - m_next_access++; - const uint64_t pleaf_aligned = paligned & ~(machine_merkle_tree::get_word_size() - 1); - const int word_offset = static_cast(paligned - pleaf_aligned); - return get_word_access_data(read_data, word_offset); - // NOLINTEND(bugprone-unchecked-optional-access) + return proof; } - /// \brief Checks a logged word write and advances log. - /// \param paligned Physical address in the machine state, - /// aligned to a 64-bit word. - /// \param word Word value to write. - /// \param text Textual description of the access. - void check_write(uint64_t paligned, uint64_t word, const char *text) { - static_assert(machine_merkle_tree::get_log2_word_size() >= log2_size::value, - "Merkle tree word size must be at least as large as a machine word"); - if ((paligned & (sizeof(uint64_t) - 1)) != 0) { - throw std::invalid_argument{"paligned not aligned to word size"}; - } - if (m_next_access >= m_accesses.size()) { - throw std::invalid_argument{"too few accesses in log"}; - } - const auto &access = m_accesses[m_next_access]; - if (access.get_type() != access_type::write) { - throw std::invalid_argument{"expected " + access_to_report() + " to write " + text}; - } - if (access.get_address() != paligned) { - std::ostringstream err; - err << "expected " << access_to_report() << " to write " << text << " to address 0x" << std::hex << paligned - << "(" << std::dec << paligned << ")"; - throw std::invalid_argument{err.str()}; - } - if (access.get_log2_size() != log2_size::value) { - throw std::invalid_argument{"expected " + access_to_report() + " to write 2^" + - std::to_string(machine_merkle_tree::get_log2_word_size()) + " bytes to " + text}; + const auto &check_written_hash(const access &a, const hash_type &expected_hash, const char *text) { + if (!a.get_written_hash().has_value()) { + throw std::invalid_argument{"missing written hash of " + std::string(text) + " in " + access_to_report()}; } // NOLINTBEGIN(bugprone-unchecked-optional-access) - // check read - if (!access.get_read().has_value()) { - throw std::invalid_argument{"missing read " + std::string(text) + " data at " + access_to_report()}; - } - const auto &read_data = access.get_read().value(); - if (read_data.size() != machine_merkle_tree::get_word_size()) { - throw std::invalid_argument{"expected overwritten data from " + std::string(text) + " to contain 2^" + - std::to_string(access.get_log2_size()) + " bytes at " + access_to_report()}; - } - // check if read data hashes to the logged read hash - hash_type computed_read_hash{}; - get_hash(m_hasher, read_data, computed_read_hash); - if (access.get_read_hash() != computed_read_hash) { - throw std::invalid_argument{"logged read data of " + std::string(text) + - " does not hash to the logged read hash at " + access_to_report()}; + if (a.get_written_hash().value() != expected_hash) { + throw std::invalid_argument{ + "written hash for " + std::string(text) + " does not match expected hash in " + access_to_report()}; } - // check write - if (!access.get_written_hash().has_value()) { - throw std::invalid_argument{"missing written " + std::string(text) + " hash at " + access_to_report()}; + return a.get_written_hash().value(); + // NOLINTEND(bugprone-unchecked-optional-access) + } + + const auto &check_read_data(const access &a, const char *text) { + if (!a.get_read().has_value()) { + throw std::invalid_argument{"missing read data for " + std::string(text) + " in " + access_to_report()}; } - const auto &written_hash = access.get_written_hash().value(); - if (!access.get_written().has_value()) { - throw std::invalid_argument{"missing written " + std::string(text) + " data at " + access_to_report()}; + // check if logged read data hashes to the logged read hash + // NOLINTBEGIN(bugprone-unchecked-optional-access) + const auto computed_read_hash = get_hash(m_hasher, a.get_read().value()); + if (a.get_read_hash() != computed_read_hash) { + throw std::invalid_argument{ + "read data for " + std::string(text) + " does not match read hash in " + access_to_report()}; } - const auto &written_data = access.get_written().value(); - if (written_data.size() != read_data.size()) { - throw std::invalid_argument{"expected written " + std::string(text) + " data to contain 2^" + - std::to_string(access.get_log2_size()) + " bytes at " + access_to_report()}; + return a.get_read().value(); + // NOLINTEND(bugprone-unchecked-optional-access) + } + + void check_written_data_if_there(const access &a, const hash_type &written_hash, const char *text) { + if (!a.get_written().has_value()) { + return; } - // check if written data hashes to the logged written hash - hash_type computed_written_hash{}; - get_hash(m_hasher, written_data, computed_written_hash); - if (written_hash != computed_written_hash) { - throw std::invalid_argument{"logged written data of " + std::string(text) + - " does not hash to the logged written hash at " + access_to_report()}; + // NOLINTBEGIN(bugprone-unchecked-optional-access) + if (written_hash != get_hash(m_hasher, a.get_written().value())) { + throw std::invalid_argument{ + "written data for " + std::string(text) + " does not match written hash in " + access_to_report()}; } - // check if word being written matches the logged data - const uint64_t pleaf_aligned = paligned & ~(machine_merkle_tree::get_word_size() - 1); - const int word_offset = static_cast(paligned - pleaf_aligned); - const uint64_t logged_word = get_word_access_data(written_data, word_offset); - if (word != logged_word) { - throw std::invalid_argument{"value being written to " + std::string(text) + - " does not match the logged written value at " + access_to_report()}; + // NOLINTEND(bugprone-unchecked-optional-access) + } + + void check_read_data_if_there(const access &a, const char *text) { + if (!a.get_read().has_value()) { + return; } - // check if logged written data differs from the logged read data only by the written word - access_data expected_written_data(read_data); // make a copy of read data - replace_word_access_data(word, expected_written_data, word_offset); // patch with written word - if (written_data != expected_written_data) { - throw std::invalid_argument{"logged written data of " + std::string(text) + - " doesn't differ from the logged read data only by the written word at " + access_to_report()}; + // NOLINTBEGIN(bugprone-unchecked-optional-access) + if (a.get_read_hash() != get_hash(m_hasher, a.get_read().value())) { + throw std::invalid_argument{ + "read data for " + std::string(text) + " does not match read hash in " + access_to_report()}; } // NOLINTEND(bugprone-unchecked-optional-access) - // check proof - auto proof = access.make_proof(m_root_hash); - if (!proof.verify(m_hasher)) { - throw std::invalid_argument{"Mismatch in root hash of " + access_to_report()}; - } - // Update root hash to reflect the data written by this access + } + + void update_root_hash(const proof_type &proof, const hash_type &written_hash) { m_root_hash = proof.bubble_up(m_hasher, written_hash); + } + + void check_write_access(uint64_t paddr, uint64_t log2_size, const hash_type &expected_hash, const char *text) { + const auto &a = check_access(text); + check_access_type(a, access_type::write, text); + check_access_range(a, access_type::write, paddr, log2_size, text); + const auto proof = check_access_siblings_and_read_hash(a, text); + const auto &written_hash = check_written_hash(a, expected_hash, text); + check_read_data_if_there(a, text); + check_written_data_if_there(a, written_hash, text); + update_root_hash(proof, written_hash); + m_next_access++; + } + + void check_write_word_access(uint64_t paddr, uint64_t val, const char *text) { + const auto log2_size = cartesi::log2_size::value; + const auto &a = check_access(text); + check_access_type(a, access_type::write, text); + check_access_range(a, access_type::write, paddr, log2_size, text); + const auto proof = check_access_siblings_and_read_hash(a, text); + auto written_data = check_read_data(a, text); + [[maybe_unused]] const auto [access_paddr, access_log2_size] = adjust_access(paddr, log2_size); + const auto val_offset = paddr - access_paddr; + replace_word_access_data(val, written_data, val_offset); + const auto &written_hash = check_written_hash(a, get_hash(m_hasher, written_data), text); + check_written_data_if_there(a, written_hash, text); + update_root_hash(proof, written_hash); + m_next_access++; + } + + uint64_t check_read_word_access(uint64_t paddr, const char *text) { + const auto log2_size = cartesi::log2_size::value; + const auto &a = check_access(text); + check_access_type(a, access_type::read, text); + check_access_range(a, access_type::read, paddr, log2_size, text); + std::ignore = check_access_siblings_and_read_hash(a, text); + const auto &read_data = check_read_data(a, text); + [[maybe_unused]] const auto [access_paddr, access_log2_size] = adjust_access(paddr, log2_size); + const auto val_offset = paddr - access_paddr; + const auto val = get_word_access_data(read_data, val_offset); m_next_access++; + return val; } friend i_uarch_state_access; @@ -260,123 +283,105 @@ class uarch_replay_state_access : public i_uarch_state_access{*this, text}; } +#endif - uint64_t do_read_x(int reg) { - return check_read(machine_reg_address(machine_reg::uarch_x0, reg), "uarch.x"); + uint64_t do_read_x(int i) { + const auto reg = machine_reg_enum(machine_reg::uarch_x0, i); + return check_read_word_access(machine_reg_address(reg), machine_reg_get_name(reg)); } - void do_write_x(int reg, uint64_t val) { - assert(reg != 0); - check_write(machine_reg_address(machine_reg::uarch_x0, reg), val, "uarch.x"); + void do_write_x(int i, uint64_t val) { + assert(i != 0); + const auto reg = machine_reg_enum(machine_reg::uarch_x0, i); + check_write_word_access(machine_reg_address(reg), val, machine_reg_get_name(reg)); } uint64_t do_read_pc() { - return check_read(machine_reg_address(machine_reg::uarch_pc), "uarch.pc"); + const auto reg = machine_reg::uarch_pc; + return check_read_word_access(machine_reg_address(reg), machine_reg_get_name(reg)); } void do_write_pc(uint64_t val) { - check_write(machine_reg_address(machine_reg::uarch_pc), val, "uarch.pc"); + const auto reg = machine_reg::uarch_pc; + check_write_word_access(machine_reg_address(reg), val, machine_reg_get_name(reg)); } uint64_t do_read_cycle() { - return check_read(machine_reg_address(machine_reg::uarch_cycle), "uarch.uarch_cycle"); + const auto reg = machine_reg::uarch_cycle; + return check_read_word_access(machine_reg_address(reg), machine_reg_get_name(reg)); } void do_write_cycle(uint64_t val) { - check_write(machine_reg_address(machine_reg::uarch_cycle), val, "uarch.cycle"); + const auto reg = machine_reg::uarch_cycle; + check_write_word_access(machine_reg_address(reg), val, machine_reg_get_name(reg)); } - bool do_read_halt_flag() { - return check_read(machine_reg_address(machine_reg::uarch_halt_flag), "uarch.halt_flag") != 0; + uint64_t do_read_halt_flag() { + const auto reg = machine_reg::uarch_halt_flag; + return check_read_word_access(machine_reg_address(reg), machine_reg_get_name(reg)); } - void do_set_halt_flag() { - check_write(machine_reg_address(machine_reg::uarch_halt_flag), static_cast(true), "uarch.halt_flag"); + void do_write_halt_flag(uint64_t val) { + const auto reg = machine_reg::uarch_halt_flag; + check_write_word_access(machine_reg_address(reg), val, machine_reg_get_name(reg)); } - void do_reset_halt_flag() { - check_write(machine_reg_address(machine_reg::uarch_halt_flag), static_cast(false), "uarch.halt_flag"); + uint64_t do_read_word(uint64_t paddr) { + return check_read_word_access(paddr, uarch_machine_bridge::get_what_name(paddr)); } - uint64_t do_read_word(uint64_t paddr) { - assert((paddr & (sizeof(uint64_t) - 1)) == 0); - // Get the name of the state register identified by this address - const auto *name = uarch_bridge::get_register_name(paddr); - if (name == nullptr) { - // this is a regular memory access - name = "memory"; - } - return check_read(paddr, name); + void do_write_word(uint64_t paddr, uint64_t val) { + check_write_word_access(paddr, val, uarch_machine_bridge::get_what_name(paddr)); } - void do_write_word(uint64_t paddr, uint64_t data) { - assert((paddr & (sizeof(uint64_t) - 1)) == 0); - // Get the name of the state register identified by this address - const auto *name = uarch_bridge::get_register_name(paddr); - if (name == nullptr) { - // this is a regular memory access - name = "memory"; - } - check_write(paddr, data, name); + // NOLINTNEXTLINE(readability-convert-member-functions-to-static) + void do_putchar(uint8_t /*c*/) { + ; // do nothing + } + + // NOLINTNEXTLINE(readability-convert-member-functions-to-static) + void do_mark_dirty_page(uint64_t /*paddr*/, uint64_t /*pma_index*/) { + ; // do nothing } void do_reset_state() { - hasher_type hasher; - auto text = std::string("uarchState"); - if (m_next_access >= m_accesses.size()) { - throw std::invalid_argument{"too few accesses in log"}; - } - const auto &access = m_accesses[m_next_access]; - if (access.get_address() != UARCH_STATE_START_ADDRESS) { - throw std::invalid_argument{ - "expected address of " + access_to_report() + " to be the start address of the uarch state"}; - } - if (access.get_log2_size() != UARCH_STATE_LOG2_SIZE) { - throw std::invalid_argument{"expected " + access_to_report() + " to write 2^" + - std::to_string(UARCH_STATE_LOG2_SIZE) + " bytes to " + text}; - } - if (access.get_type() != access_type::write) { - throw std::invalid_argument{"expected " + access_to_report() + " to write " + text}; - } - // NOLINTBEGIN(bugprone-unchecked-optional-access) - if (access.get_read().has_value()) { - // if read data is available then its hash and the logged read hash must match - hash_type computed_hash; - get_hash(hasher, access.get_read().value(), computed_hash); - if (computed_hash != access.get_read_hash()) { - throw std::invalid_argument{ - "hash of read data and read hash at " + access_to_report() + " does not match read hash"}; - } - } - if (!access.get_written_hash().has_value()) { - throw std::invalid_argument{"write " + access_to_report() + " has no written hash"}; - } - const auto &written_hash = access.get_written_hash().value(); - if (written_hash != get_uarch_pristine_state_hash()) { - throw std::invalid_argument{ - "expected written hash of " + access_to_report() + " to be the start hash of the pristine uarch state"}; - } - if (access.get_written().has_value()) { - // if written data is available then its hash and the logged written hash must match - hash_type computed_hash; - get_hash(hasher, access.get_written().value(), computed_hash); - if (computed_hash != written_hash) { - throw std::invalid_argument{"written hash and written data mismatch at " + access_to_report()}; - } - } - // NOLINTEND(bugprone-unchecked-optional-access) - // check proof - auto proof = access.make_proof(m_root_hash); - if (!proof.verify(m_hasher)) { - throw std::invalid_argument{"Mismatch in root hash of " + access_to_report()}; - } - // Update root hash to reflect the data written by this access - m_root_hash = proof.bubble_up(m_hasher, written_hash); - m_next_access++; + check_write_access(UARCH_STATE_START_ADDRESS, UARCH_STATE_LOG2_SIZE, uarch_pristine_state_hash, "uarch.state"); + } + + auto get_write_tlb_slot_hash(uint64_t vaddr_page, uint64_t vp_offset, uint64_t pma_index) { + // Writes to TLB slots have to be atomic. + // We can only do atomic writes of entire Merkle tree nodes. + // Therefore, TLB slot must have a power-of-two size, or at least be aligned to it. + static_assert(SHADOW_TLB_SLOT_SIZE == sizeof(shadow_tlb_slot), "shadow TLB slot size is wrong"); + static_assert((UINT64_C(1) << SHADOW_TLB_SLOT_LOG2_SIZE) == SHADOW_TLB_SLOT_SIZE, + "shadow TLB slot log2 size is wrong"); + static_assert(SHADOW_TLB_SLOT_LOG2_SIZE >= machine_merkle_tree::get_log2_word_size(), + "shadow TLB slot must fill at least an entire Merkle tree word"); + shadow_tlb_slot slot_data{}; + shadow_tlb_fill_slot(vaddr_page, vp_offset, pma_index, slot_data); + hash_type slot_hash{}; + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + get_merkle_tree_hash(m_hasher, reinterpret_cast(&slot_data), sizeof(slot_data), + machine_merkle_tree::get_word_size(), slot_hash); + return slot_hash; + } + + void do_write_tlb(TLB_set_index set_index, uint64_t slot_index, uint64_t vaddr_page, uint64_t vp_offset, + uint64_t pma_index) { + const auto slot_paddr = shadow_tlb_get_abs_addr(set_index, slot_index); + const auto slot_hash = get_write_tlb_slot_hash(vaddr_page, vp_offset, pma_index); + check_write_access(slot_paddr, SHADOW_TLB_SLOT_LOG2_SIZE, slot_hash, "tlb.slot"); + } + + // NOLINTNEXTLINE(readability-convert-member-functions-to-static) + constexpr const char *do_get_name() const { + return "uarch_replay_state_access"; +>>>>>>> 1e5dc22 (refactor: clean up uarch state access mechanism) } }; diff --git a/src/uarch-solidity-compat.h b/src/uarch-solidity-compat.h index 4cc48266f..1be7df99c 100644 --- a/src/uarch-solidity-compat.h +++ b/src/uarch-solidity-compat.h @@ -17,11 +17,13 @@ #ifndef UARCH_INTERPRET_SOLIDITY_COMPAT_H #define UARCH_INTERPRET_SOLIDITY_COMPAT_H -#include "os.h" #include #include #include #include +#ifdef DUMP_INSN +#include +#endif /// \file /// \brief Solidity Compatibility Layer @@ -66,13 +68,13 @@ static inline void writeCycle(UarchState &a, uint64 val) { } template -static inline bool readHaltFlag(UarchState &a) { +static inline uint64 readHaltFlag(UarchState &a) { return a.read_halt_flag(); } template -static inline void setHaltFlag(UarchState &a) { - return a.set_halt_flag(); +static inline void writeHaltFlag(UarchState &a, uint64 val) { + a.write_halt_flag(val); } template @@ -105,7 +107,7 @@ static inline uint64 readIflagsY(State &a) { return a.read_iflags_Y(); } -template +template static inline void writeIflagsY(State &a, uint64 val) { a.write_iflags_Y(val); } @@ -127,17 +129,19 @@ static inline void throwRuntimeError(UarchState & /*a*/, const char *message) { } template -static inline void putChar(UarchState & /*a*/, unsigned char c) { - os_putchar(c); +static inline void putCharECALL(UarchState &a, uint8 c) { + a.putchar(c); } template -static inline void markDirtyPage(UarchState &a, uint64 paddr, uint64 pma_index) { +static inline void markDirtyPageECALL(UarchState &a, uint64 paddr, uint64 pma_index) { a.mark_dirty_page(paddr, pma_index); } -static inline void writeTlbVpOffset(UarchState &a, uint64 use, uint64 vp_offset, uint64 pma_index) { - a.write_tlb_vp_offset(use, slot_index, vp_offset, pma_index); +template +static inline void writeTlbECALL(UarchState &a, uint64 set_index, uint64 slot_index, uint64 vaddr_page, + uint64 vp_offset, uint64 pma_index) { + a.write_tlb(static_cast(set_index), slot_index, vaddr_page, vp_offset, pma_index); } // Conversions and arithmetic functions diff --git a/src/uarch-state-access.h b/src/uarch-state-access.h index c840da29c..94d66784b 100644 --- a/src/uarch-state-access.h +++ b/src/uarch-state-access.h @@ -22,58 +22,38 @@ #include #include "bracket-note.h" +#include "host-addr.h" #include "i-uarch-state-access.h" -#include "machine-state.h" -#include "pma.h" +#include "machine.h" +#include "os.h" #include "riscv-constants.h" #include "strict-aliasing.h" -#if OKUARCH -#include "uarch-bridge.h" +#include "uarch-machine-bridge.h" #include "uarch-pristine.h" -#include "uarch-state.h" + +#if DUMP_UARCH_STATE_ACCESS +#include "scoped-note.h" #endif namespace cartesi { class uarch_state_access : public i_uarch_state_access { -#ifdef OKUARCH // NOLINTBEGIN(cppcoreguidelines-avoid-const-or-ref-data-members) - uarch_state &m_us; - machine_state &m_s; + machine &m_m; + uarch_machine_bridge m_b; + host_addr m_uram_ph_offset; // NOLINTEND(cppcoreguidelines-avoid-const-or-ref-data-members)1 - /// \brief Obtain Memory PMA entry that covers a given physical memory region - /// \param paddr Start of physical memory region. - /// \param length Length of physical memory region. - /// \returns Corresponding entry if found, or a sentinel entry - /// for an empty range. - pma_entry &find_memory_pma_entry(uint64_t paddr, uint64_t length) { - // First, search microarchitecture private PMA entries - if (m_us.ram.contains(paddr, length)) { - return m_us.ram; - } - int i = 0; - // Search machine memory PMA entries (not devices or anything else) - while (true) { - auto &pma = m_s.pmas[i]; - // The pmas array always contain a sentinel. It is an entry with - // zero length. If we hit it, return it - if (pma.get_length() == 0) { - return pma; - } - if (pma.get_istart_M() && pma.contains(paddr, length)) { - return pma; - } - i++; - } - } - public: /// \brief Constructor from machine and uarch states. /// \param um Reference to uarch state. /// \param m Reference to machine state. - explicit uarch_state_access(uarch_state &us, machine_state &s) : m_us(us), m_s(s) { - ; + explicit uarch_state_access(machine &m) : m_m(m), m_b(m) { + const auto &uram = m_m.get_uarch_state().ram; + const auto haddr = cast_ptr_to_host_addr(uram.get_memory_noexcept().get_host_memory()); + const auto paddr = uram.get_start(); + // initialize translation cache from paddr in uarch RAM to host address + m_uram_ph_offset = haddr - paddr; } /// \brief No copy constructor @@ -90,119 +70,98 @@ class uarch_state_access : public i_uarch_state_access { private: friend i_uarch_state_access; +#ifdef DUMP_UARCH_STATE_ACCESS // NOLINTNEXTLINE(readability-convert-member-functions-to-static) - void do_push_bracket(bracket_type /*type*/, const char * /*text*/) {} - - // NOLINTNEXTLINE(readability-convert-member-functions-to-static) - int do_make_scoped_note(const char * /*text*/) { - return 0; + auto do_make_scoped_note([[maybe_unused]] const char *text) { + return scoped_note{*this, text}; } +#endif - uint64_t do_read_x(int reg) const { - return m_us.x[reg]; + uint64_t do_read_x(int i) const { + return m_m.get_uarch_state().x[i]; } - void do_write_x(int reg, uint64_t val) { - assert(reg != 0); - m_us.x[reg] = val; + void do_write_x(int i, uint64_t val) { + assert(i != 0); + m_m.get_uarch_state().x[i] = val; } uint64_t do_read_pc() const { - return m_us.pc; + return m_m.get_uarch_state().pc; } void do_write_pc(uint64_t val) { - m_us.pc = val; + m_m.get_uarch_state().pc = val; } uint64_t do_read_cycle() const { - return m_us.cycle; + return m_m.get_uarch_state().cycle; } void do_write_cycle(uint64_t val) { - m_us.cycle = val; - } - - bool do_read_halt_flag() const { - return m_us.halt_flag; + m_m.get_uarch_state().cycle = val; } - void do_set_halt_flag() { - m_us.halt_flag = true; + uint64_t do_read_halt_flag() const { + return m_m.get_uarch_state().halt_flag; } - void do_reset_halt_flag() { - m_us.halt_flag = false; + void do_write_halt_flag(uint64_t v) { + m_m.get_uarch_state().halt_flag = v; } uint64_t do_read_word(uint64_t paddr) { - // Find a memory range that contains the specified address - auto &pma = find_memory_pma_entry(paddr, sizeof(uint64_t)); - if (pma.get_istart_E()) { - // This word doesn't fall within any memory PMA range. - // Check if uarch is trying to access a machine state register - return read_register(paddr); + if ((paddr & (sizeof(uint64_t) - 1)) != 0) { + throw std::runtime_error("misaligned read from uarch"); } - if (!pma.get_istart_R()) { - throw std::runtime_error("pma is not readable"); + // If the word is in UARCH_RAM, read it + if (m_m.get_uarch_state().ram.contains(paddr, sizeof(uint64_t))) { + auto haddr = m_uram_ph_offset + paddr; + return aliased_aligned_read(haddr); } - // Found a writable memory range. Access host memory accordingly. - const uint64_t hoffset = paddr - pma.get_start(); - unsigned char *hmem = pma.get_memory().get_host_memory() + hoffset; - return aliased_aligned_read(hmem); - } - - /// \brief Reads a uint64 machine state register mapped to a memory address - /// \param paddr Address of the state register - /// \param data Pointer receiving register value - uint64_t read_register(uint64_t paddr) { - return uarch_bridge::read_register(paddr, m_s); - } - - /// \brief Fallback to error on all other word sizes - void do_write_word(uint64_t paddr, uint64_t data) { - // Find a memory range that contains the specified address - auto &pma = find_memory_pma_entry(paddr, sizeof(uint64_t)); - if (pma.get_istart_E()) { - // This word doesn't fall within any memory PMA range. - // Check if uarch is trying to access a machine state register - write_register(paddr, data); - return; + // Otherwise, forward to bridge + return m_b.read_word(paddr); + } + + void do_write_word(uint64_t paddr, uint64_t val) { + if ((paddr & (sizeof(uint64_t) - 1)) != 0) { + throw std::runtime_error("misaligned write to uarch"); } - if (!pma.get_istart_W()) { - throw std::runtime_error("pma is not writable"); + // If the word is in UARCH_RAM, write it + if (m_m.get_uarch_state().ram.contains(paddr, sizeof(uint64_t))) { + auto haddr = m_uram_ph_offset + paddr; + aliased_aligned_write(haddr, val); + return; } - // Found a writable memory range. Access host memory accordingly. - const uint64_t hoffset = paddr - pma.get_start(); - unsigned char *hmem = pma.get_memory().get_host_memory() + hoffset; - aliased_aligned_write(hmem, data); - const uint64_t paddr_page = paddr & ~PAGE_OFFSET_MASK; - pma.mark_dirty_page(paddr_page - pma.get_start()); + // Otherwise, forward to bridge + m_b.write_word(paddr, val); + } + + // NOLINTNEXTLINE(readability-convert-member-functions-to-static) + void do_putchar(uint8_t c) { + os_putchar(c); + } + + // NOLINTNEXTLINE(readability-convert-member-functions-to-static) + void do_mark_dirty_page(uint64_t paddr, uint64_t pma_index) { + // forward to bridge + m_b.mark_dirty_page(paddr, pma_index); } - /// \brief Writes a uint64 machine state register mapped to a memory address - /// \param paddr Address of the state register - /// \param data New register value - void write_register(uint64_t paddr, uint64_t data) { - uarch_bridge::write_register(paddr, m_s, data); + void do_write_tlb(TLB_set_index set_index, uint64_t slot_index, uint64_t vaddr_page, uint64_t vp_offset, + uint64_t pma_index) { + // forward to bridge + m_b.write_shadow_tlb(set_index, slot_index, vaddr_page, vp_offset, pma_index); } void do_reset_state() { - m_us.halt_flag = false; - m_us.pc = UARCH_PC_INIT; - m_us.cycle = UARCH_CYCLE_INIT; - for (int i = 1; i < UARCH_X_REG_COUNT; i++) { - m_us.x[i] = UARCH_X_INIT; - } - // Load embedded pristine RAM image - if (uarch_pristine_ram_len > m_us.ram.get_length()) { - throw std::runtime_error("embedded uarch ram image does not fit in uarch ram pma"); - } - m_us.ram.write_memory(m_us.ram.get_start(), uarch_pristine_ram, uarch_pristine_ram_len); - m_us.ram.fill_memory(m_us.ram.get_start() + uarch_pristine_ram_len, 0, - m_us.ram.get_length() - uarch_pristine_ram_len); + m_m.reset_uarch(); + } + + // NOLINTNEXTLINE(readability-convert-member-functions-to-static) + constexpr const char *do_get_name() const { + return "uarch_state_access"; } -#endif }; } // namespace cartesi diff --git a/src/uarch-state.h b/src/uarch-state.h index 3c4e0f18a..f028c5027 100644 --- a/src/uarch-state.h +++ b/src/uarch-state.h @@ -41,7 +41,7 @@ struct uarch_state { uint64_t pc{}; ///< Program counter. std::array x{}; ///< Register file. uint64_t cycle{}; ///< Cycles counter - bool halt_flag{}; + uint64_t halt_flag{}; pma_entry shadow_state; ///< Shadow uarch state pma_entry ram; ///< Memory range for micro RAM pma_entry empty_pma; ///< Empty range fallback diff --git a/src/uarch-step.cpp b/src/uarch-step.cpp index b7dbd9f4e..6e2466aff 100644 --- a/src/uarch-step.cpp +++ b/src/uarch-step.cpp @@ -872,25 +872,27 @@ static inline void executeECALL(UarchState &a, uint64 pc) { // return value is in a0 (and maybe also in a1) uint64 fn = readX(a, 17); // a7 contains the function number if (fn == UARCH_ECALL_FN_HALT) { - return setHaltFlag(a); + return writeHaltFlag(a, 1); } if (fn == UARCH_ECALL_FN_PUTCHAR) { - uint64 character = readX(a, 10); // a0 contains the character to print - putChar(a, uint8(character)); + uint64 c = readX(a, 10); // a0 contains the character to print + putCharECALL(a, uint8(c)); // Can be a NOOP in Solidity return advancePc(a, pc); } if (fn == UARCH_ECALL_FN_MARK_DIRTY_PAGE) { - uint64 paddr = readX(a, 10); // a0 contains physical address in page to be marked dirty - uint64 pma_index = readX(a, 11); // a1 contains a index of PMA where page falls - markDirtyPage(a, paddr, pma_index); + uint64 paddr = readX(a, 10); // a0 contains physical address in page to be marked dirty + uint64 pma_index = readX(a, 11); // a1 contains a index of PMA where page falls + markDirtyPageECALL(a, paddr, pma_index); // This MUST be be a NOOP in Solidity return advancePc(a, pc); } - if (fn == UARCH_ECALL_FN_WRITE_TLB_VP_OFFSET) { - uint64 use = readX(a, 10); // a0 contains TLB set (code, read, write) + if (fn == UARCH_ECALL_FN_WRITE_TLB) { + uint64 set_index = readX(a, 10); // a0 contains TLB set (code, read, write) uint64 slot_index = readX(a, 11); // a1 contains slot_index to modify - uint64 vp_offset = readX(a, 12); // a2 contains vp_offset to write - uint64 pma_index = readX(a, 13); // a3 contains index of PMA where page falls - writeTlbVpOffset(a, use, slot_index, vp_offset, pma_index); + uint64 vaddr_page = readX(a, 12); // a2 contains vaddr_page to write + uint64 vp_offset = readX(a, 13); // a3 contains vp_offset to write + uint64 pma_index = readX(a, 14); // a4 contains index of PMA where page falls + writeTlbECALL(a, set_index, slot_index, vaddr_page, vp_offset, + pma_index); // WARNING: This CANNOT be a NOOP in Solidity return advancePc(a, pc); } throwRuntimeError(a, "unsupported ecall function"); @@ -1097,7 +1099,7 @@ UArchStepStatus uarch_step(UarchState &a) { return UArchStepStatus::CycleOverflow; } // do not advance if machine is halted - if (readHaltFlag(a)) { + if (readHaltFlag(a) != 0) { return UArchStepStatus::UArchHalted; } // execute next instruction diff --git a/tests/Makefile b/tests/Makefile index e54068cdd..fadc12612 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -44,12 +44,12 @@ ifeq ($(TARGET_OS),Darwin) LIBCARTESI_SO=libcartesi-$(MACHINE_EMULATOR_SO_VERSION).dylib LIBCARTESI_SO_JSONRPC=libcartesi_jsonrpc-$(MACHINE_EMULATOR_SO_VERSION).dylib -NUM_JOBS := $(shell sysctl -n hw.ncpu) +NUM_JOBS ?= $(shell sysctl -n hw.ncpu) else LIBCARTESI_SO=libcartesi-$(MACHINE_EMULATOR_SO_VERSION).so LIBCARTESI_SO_JSONRPC=libcartesi_jsonrpc-$(MACHINE_EMULATOR_SO_VERSION).so -NUM_JOBS := $(shell nproc) +NUM_JOBS ?= $(shell nproc) endif ifeq ($(coverage),yes) diff --git a/tests/lua/cartesi/tests/util.lua b/tests/lua/cartesi/tests/util.lua index c98e13138..b63eac9b6 100644 --- a/tests/lua/cartesi/tests/util.lua +++ b/tests/lua/cartesi/tests/util.lua @@ -26,10 +26,10 @@ local function adjust_path(path) end local test_util = { - images_path = adjust_path(assert(os.getenv("CARTESI_IMAGES_PATH"))), - tests_path = adjust_path(assert(os.getenv("CARTESI_TESTS_PATH"))), - cmio_path = adjust_path(assert(os.getenv("CARTESI_CMIO_PATH"))), - tests_uarch_path = adjust_path(assert(os.getenv("CARTESI_TESTS_UARCH_PATH"))), + images_path = adjust_path(assert(os.getenv("CARTESI_IMAGES_PATH"), "must set CARTESI_IMAGES_PATH")), + tests_path = adjust_path(assert(os.getenv("CARTESI_TESTS_PATH"), "must set CARTESI_TESTS_PATH")), + cmio_path = adjust_path(assert(os.getenv("CARTESI_CMIO_PATH"), "must set CARTESI_CMIO_PATH")), + tests_uarch_path = adjust_path(assert(os.getenv("CARTESI_TESTS_UARCH_PATH"), "must set CARTESI_TESTS_UARCH_PATH")), } local zero_keccak_hash_table = { @@ -60,9 +60,7 @@ test_util.uarch_programs.default = { } function test_util.create_test_uarch_program(instructions) - if not instructions then - instructions = test_util.uarch_programs.default - end + assert(instructions, "missing instructions") local file_path = os.tmpname() local f = assert(io.open(file_path, "wb")) for _, insn in pairs(instructions) do diff --git a/tests/lua/log-with-mtime-transition.lua b/tests/lua/log-with-mtime-transition.lua index ba402519d..4d3298952 100755 --- a/tests/lua/log-with-mtime-transition.lua +++ b/tests/lua/log-with-mtime-transition.lua @@ -12,8 +12,12 @@ local config = { } local machine = cartesi.machine(config) +io.stderr:write("getting root hash\n") local old_hash = machine:get_root_hash() +io.stderr:write("getting uarch step log\n") local access_log = machine:log_step_uarch() +io.stderr:write("getting new root hash\n") local new_hash = machine:get_root_hash() +io.stderr:write("verifying step log\n") cartesi.machine:verify_step_uarch(old_hash, access_log, new_hash, {}) print("ok") diff --git a/tests/lua/machine-bind.lua b/tests/lua/machine-bind.lua index c4259c1f2..8e050748b 100755 --- a/tests/lua/machine-bind.lua +++ b/tests/lua/machine-bind.lua @@ -242,6 +242,15 @@ local function get_cpu_reg_test_values() return reg_values end +local function check_error_find(err, expected_err) + if not err then + error("expected error with '" .. expected_err .. "' (got no error)") + end + if not err:find(expected_err, 1, true) then + error("expected error with '" .. expected_err .. "' (got '" .. tostring(err) .. "')") + end +end + local machine_type = assert(arguments[1], "missing machine type") assert(machine_type == "local" or machine_type == "jsonrpc", "unknown machine type, should be 'local' or 'jsonrpc'") local to_shutdown -- luacheck: no unused @@ -268,7 +277,7 @@ local function build_machine_config(config_options) processor = get_uarch_cpu_reg_test_values(), ram = { length = 0x1000, - image_filename = test_util.create_test_uarch_program(), + image_filename = test_util.create_test_uarch_program(test_util.uarch_programs.default), }, }, } @@ -551,7 +560,7 @@ do_test("should error if target mcycle is smaller than current mcycle", function machine:run(MAX_MCYCLE - 1) end) assert(success == false) - assert(err and err:match("mcycle is past")) + check_error_find(err, "mcycle is past") assert(machine:read_reg("mcycle") == MAX_MCYCLE) end) @@ -562,7 +571,7 @@ do_test("should error if target uarch_cycle is smaller than current uarch_cycle" machine:run_uarch(MAX_UARCH_CYCLE - 1) end) assert(success == false) - assert(err and err:match("uarch_cycle is past")) + check_error_find(err, "uarch_cycle is past") assert(machine:read_reg("uarch_cycle") == MAX_UARCH_CYCLE) end) @@ -638,9 +647,9 @@ do_test("written and read values should match", function(machine) assert(memory_read == "mydataol12345678") end) -print("\n\n dump step log to console") +print("\n\n dump step log to console") do_test("dumped step log content should match", function(machine) - local log = machine:log_step_uarch(cartesi.ACCESS_LOG_TYPE_ANNOTATIONS) + local log = machine:log_step_uarch(cartesi.ACCESS_LOG_TYPE_ANNOTATIONS | cartesi.ACCESS_LOG_TYPE_LARGE_DATA) local temp_file = test_util.new_temp_file() util.dump_log(log, temp_file) local log_output = temp_file:read_all() @@ -649,10 +658,10 @@ do_test("dumped step log content should match", function(machine) .. " 1: read uarch.cycle@0x400008(4194312): 0x0(0)\n" .. " 2: read uarch.halt_flag@0x400000(4194304): 0x0(0)\n" .. " 3: read uarch.pc@0x400010(4194320): 0x600000(6291456)\n" - .. " 4: read memory@0x600000(6291456): 0x10089307b00513(4513027209561363)\n" + .. " 4: read uarch.ram@0x600000(6291456): 0x10089307b00513(4513027209561363)\n" .. " begin addi\n" - .. " 5: read uarch.x@0x400018(4194328): 0x0(0)\n" - .. " 6: write uarch.x@0x400068(4194408): 0x10050(65616) -> 0x7b(123)\n" + .. " 5: read uarch.x0@0x400018(4194328): 0x0(0)\n" + .. " 6: write uarch.x10@0x400068(4194408): 0x10050(65616) -> 0x7b(123)\n" .. " 7: write uarch.pc@0x400010(4194320): 0x600000(6291456) -> 0x600004(6291460)\n" .. " end addi\n" .. " 8: write uarch.cycle@0x400008(4194312): 0x0(0) -> 0x1(1)\n" @@ -686,7 +695,7 @@ do_test("Step log must contain conssitent data hashes", function(machine) -- ensure that verification fails with wrong read hash read_access.read_hash = wrong_hash local _, err = pcall(machine.verify_step_uarch, machine, initial_hash, log, final_hash) - assert(err:match("logged read data of uarch.uarch_cycle data does not hash to the logged read hash at 1st access")) + check_error_find(err, "siblings and read hash do not match root hash before 1st access to uarch.cycle") read_access.read_hash = read_hash -- restore correct value -- ensure that verification fails with wrong read hash @@ -695,13 +704,13 @@ do_test("Step log must contain conssitent data hashes", function(machine) read_hash = write_access.read_hash write_access.read_hash = wrong_hash _, err = pcall(machine.verify_step_uarch, machine, initial_hash, log, final_hash) - assert(err:match("logged read data of uarch.cycle does not hash to the logged read hash at 8th access")) + check_error_find(err, "siblings and read hash do not match root hash before 8th access to uarch.cycle") write_access.read_hash = read_hash -- restore correct value -- ensure that verification fails with wrong written hash write_access.written_hash = wrong_hash _, err = pcall(machine.verify_step_uarch, machine, initial_hash, log, final_hash) - assert(err:match("logged written data of uarch.cycle does not hash to the logged written hash at 8th access")) + check_error_find(err, "written hash for uarch.cycle does not match expected hash in 8th access") end) do_test("step when uarch cycle is max", function(machine) @@ -819,7 +828,7 @@ test_util.make_do_test(build_machine, machine_type, { uarch = {} })( local test_reset_uarch_config = { processor = { - halt_flag = true, + halt_flag = 1, cycle = 1, pc = 0, }, @@ -901,10 +910,10 @@ test_util.make_do_test(build_machine, machine_type, { uarch = test_reset_uarch_c -- verifying incorrect initial hash local wrong_hash = string.rep("0", cartesi.HASH_SIZE) local _, err = pcall(machine.verify_reset_uarch, machine, wrong_hash, log, final_hash) - assert(err:match("Mismatch in root hash of 1st access")) + check_error_find(err, "siblings and read hash do not match root hash before 1st access to uarch.state") -- verifying incorrect final hash _, err = pcall(machine.verify_reset_uarch, machine, initial_hash, log, wrong_hash) - assert(err:match("mismatch in root hash after replay")) + check_error_find(err, "mismatch in root hash after replay") end ) @@ -922,10 +931,10 @@ test_util.make_do_test(build_machine, machine_type, { uarch = test_reset_uarch_c "Dump of log produced by log_reset_uarch should match", function(machine) local log = machine:log_reset_uarch(cartesi.ACCESS_LOG_TYPE_ANNOTATIONS) - local expected_dump_pattern = "begin reset uarch state\n" - .. " 1: write uarch_state@0x400000%(4194304%): " + local expected_dump_pattern = "begin reset_uarch_state\n" + .. " 1: write uarch.state@0x400000%(4194304%): " .. 'hash:"[0-9a-f]+"%(2%^22 bytes%) %-> hash:"[0-9a-fA-F]+"%(2%^22 bytes%)\n' - .. "end reset uarch state\n" + .. "end reset_uarch_state\n" local tmpname = os.tmpname() local deleter = {} @@ -959,7 +968,7 @@ test_util.make_do_test(build_machine, machine_type, { uarch = test_reset_uarch_c local final_hash = machine:get_root_hash() assert(#log.accesses == 1, "log should have 1 access") local access = log.accesses[1] - -- in debug mode, the log must include read and written data + -- when large data is requested, the log must include read and written data assert(access.read ~= nil, "read data should not be nil") assert(access.written ~= nil, "written data should not be nil") -- verify returned log @@ -969,13 +978,13 @@ test_util.make_do_test(build_machine, machine_type, { uarch = test_reset_uarch_c -- tamper with read data to produce a hash mismatch access.read = "X" .. access.read:sub(2) local _, err = pcall(machine.verify_reset_uarch, machine, initial_hash, log, final_hash) - assert(err:match("hash of read data and read hash at 1st access does not match read hash")) + check_error_find(err, "read data for uarch.state does not match read hash in 1st access") -- restore correct read access.read = original_read -- change written data to produce a hash mismatch access.written = "X" .. access.written:sub(2) _, err = pcall(machine.verify_reset_uarch, machine, initial_hash, log, final_hash) - assert(err:match("written hash and written data mismatch at 1st access")) + check_error_find(err, "written data for uarch.state does not match written hash in 1st access") end ) @@ -988,41 +997,38 @@ do_test("Test unhappy paths of verify_reset_uarch", function(machine) local final_hash = machine:get_root_hash() callback(log) local _, err = pcall(machine.verify_reset_uarch, machine, initial_hash, log, final_hash) - assert( - err:match(expected_error), - 'Error text "' .. err .. '" does not match expected "' .. expected_error .. '"' - ) + check_error_find(err, expected_error) end - assert_error("too few accesses in log", function(log) + assert_error("log is missing access 1st access to uarch.state", function(log) log.accesses = {} end) - assert_error("expected address of 1st access to be the start address of the uarch state", function(log) + assert_error("expected 1st access to write uarch.state at address 0x400000(4194304)", function(log) log.accesses[1].address = 0 end) - assert_error("is out of bounds", function(log) + assert_error('field "value/accesses/0/log2_size" is out of bounds', function(log) log.accesses[1].log2_size = 64 end) - assert_error("missing field", function(log) + assert_error('missing field "value/accesses/0/read_hash"', function(log) log.accesses[#log.accesses].read_hash = nil end) - assert_error("Mismatch in root hash of 1st access", function(log) + assert_error("siblings and read hash do not match root hash before 1st access to uarch.state", function(log) log.accesses[1].read_hash = bad_hash end) assert_error("access log was not fully consumed", function(log) log.accesses[#log.accesses + 1] = log.accesses[1] end) - assert_error("write 1st access has no written hash", function(log) + assert_error("missing written hash of uarch.state in 1st access", function(log) log.accesses[#log.accesses].written_hash = nil end) - assert_error("has wrong length", function(log) + assert_error('field "value/accesses/0/written" has wrong length', function(log) log.accesses[#log.accesses].written = "\0" end) - assert_error("written hash and written data mismatch at 1st access", function(log) + assert_error("written data for uarch.state does not match written hash in 1st access", function(log) log.accesses[#log.accesses].written = string.rep("\0", 2 ^ 22) end) - assert_error("Mismatch in root hash of 1st access", function(log) + assert_error("siblings and read hash do not match root hash before 1st access to uarch.state", function(log) log.accesses[1].sibling_hashes[1] = bad_hash end) end) @@ -1036,54 +1042,45 @@ do_test("Test unhappy paths of verify_step_uarch", function(machine) local final_hash = machine:get_root_hash() callback(log) local _, err = pcall(machine.verify_step_uarch, machine, initial_hash, log, final_hash) - assert( - err:match(expected_error), - 'Error text "' .. err .. '" does not match expected "' .. expected_error .. '"' - ) + check_error_find(err, expected_error) end - assert_error("too few accesses in log", function(log) + assert_error("log is missing access 1st access to uarch.cycle", function(log) log.accesses = {} end) - assert_error("expected 1st access to read uarch.uarch_cycle", function(log) + assert_error("expected 1st access to read uarch.cycle", function(log) log.accesses[1].address = 0 end) - assert_error("expected 1st access to read 2%^5 bytes from uarch.uarch_cycle", function(log) + assert_error("expected 1st access to uarch.cycle to read 2^3 bytes", function(log) log.accesses[1].log2_size = 2 end) assert_error("is out of bounds", function(log) log.accesses[1].log2_size = 65 end) - assert_error("missing read uarch.uarch_cycle data at 1st access", function(log) + assert_error("missing read data for uarch.cycle in 1st access", function(log) log.accesses[1].read = nil end) assert_error("has wrong length", function(log) log.accesses[1].read = "\0" end) - assert_error( - "logged read data of uarch.uarch_cycle data does not hash to the logged read hash at 1st access", - function(log) - log.accesses[1].read_hash = bad_hash - end - ) + assert_error("siblings and read hash do not match root hash before 1st access to uarch.cycle", function(log) + log.accesses[1].read_hash = bad_hash + end) assert_error("missing field", function(log) log.accesses[#log.accesses].read_hash = nil end) assert_error("access log was not fully consumed", function(log) log.accesses[#log.accesses + 1] = log.accesses[1] end) - assert_error("missing written uarch.cycle hash at 7th access", function(log) + assert_error("missing written hash of uarch.cycle in 7th access", function(log) log.accesses[#log.accesses].written_hash = nil end) assert_error("has wrong length", function(log) log.accesses[#log.accesses].written = "\0" end) - assert_error( - "logged written data of uarch.cycle does not hash to the logged written hash at 7th access", - function(log) - log.accesses[#log.accesses].written = string.rep("\0", cartesi.HASH_SIZE) - end - ) - assert_error("Mismatch in root hash of 1st access", function(log) + assert_error("written data for uarch.cycle does not match written hash in 7th access", function(log) + log.accesses[#log.accesses].written = string.rep("\0", cartesi.HASH_SIZE) + end) + assert_error("siblings and read hash do not match root hash before 1st access to uarch.cycle", function(log) log.accesses[1].sibling_hashes[1] = bad_hash end) end) @@ -1101,7 +1098,7 @@ test_util.make_do_test(build_machine, machine_type, { })("Detect illegal instruction", function(machine) local success, err = pcall(machine.run_uarch, machine) assert(success == false) - assert(err:match("illegal instruction")) + check_error_find(err, "illegal instruction") end) --[==[ @@ -1261,12 +1258,12 @@ do_test("Dump of log produced by send_cmio_response should match", function(mach local reason = 7 local log = machine:log_send_cmio_response(reason, data, cartesi.ACCESS_LOG_TYPE_ANNOTATIONS) -- luacheck: push no max line length - local expected_dump = "begin send cmio response\n" + local expected_dump = "begin send_cmio_response\n" .. " 1: read iflags.Y@0x2f8(760): 0x1(1)\n" .. ' 2: write cmio rx buffer@0x60000000(1610612736): hash:"290decd9"(2^5 bytes) -> hash:"555b1f6d"(2^5 bytes)\n' .. " 3: write htif.fromhost@0x330(816): 0x0(0) -> 0x70000000a(30064771082)\n" .. " 4: write iflags.Y@0x2f8(760): 0x1(1) -> 0x0(0)\n" - .. "end send cmio response\n" + .. "end send_cmio_response\n" -- luacheck: pop local temp_file = test_util.new_temp_file() util.dump_log(log, temp_file) @@ -1459,7 +1456,14 @@ test_util.make_do_test(build_machine, machine_type, { machine:write_reg("uarch_x" .. t1, leaf_address) machine:write_reg("uarch_x" .. t0, 0xaaaaaaaaaaaaaaaa) local log, dump = log_step() - assert(dump:match("7: write memory@0x%x+%(%d+%): 0x1111111111111111%(%d+%) %-> 0xaaaaaaaaaaaaaaaa%(%d+%)")) + assert( + dump:find( + "write uarch.ram@0x600020(6291488): 0x1111111111111111(1229782938247303441)" + .. " -> 0xaaaaaaaaaaaaaaaa(12297829382473034410)", + 1, + true + ) + ) assert(log.accesses[7].read == leaf_data) leaf_data = machine:read_memory(leaf_address, leaf_size) -- read and check written data assert(leaf_data == make_leaf("\xaa", "\x22", "\x33", "\x44")) @@ -1470,7 +1474,14 @@ test_util.make_do_test(build_machine, machine_type, { machine:write_reg("uarch_x" .. t1, machine:read_reg("uarch_x" .. t1) + word_size) machine:write_reg("uarch_x" .. t0, 0xbbbbbbbbbbbbbbbb) log, dump = log_step() - assert(dump:match("7: write memory@0x%x+%(%d+%): 0x2222222222222222%(%d+%) %-> 0xbbbbbbbbbbbbbbbb%(%d+%)")) + assert( + dump:find( + "write uarch.ram@0x600028(6291496): 0x2222222222222222(2459565876494606882)" + .. " -> 0xbbbbbbbbbbbbbbbb(13527612320720337851)", + 1, + true + ) + ) assert(log.accesses[7].read == leaf_data) leaf_data = machine:read_memory(leaf_address, leaf_size) assert(leaf_data == make_leaf("\xaa", "\xbb", "\x33", "\x44")) @@ -1481,7 +1492,14 @@ test_util.make_do_test(build_machine, machine_type, { machine:write_reg("uarch_x" .. t1, machine:read_reg("uarch_x" .. t1) + word_size) machine:write_reg("uarch_x" .. t0, 0xcccccccccccccccc) log, dump = log_step() - assert(dump:match("7: write memory@0x%x+%(%d+%): 0x3333333333333333%(%d+%) %-> 0xcccccccccccccccc%(%d+%)")) + assert( + dump:find( + "7: write uarch.ram@0x600030(6291504): 0x3333333333333333(3689348814741910323)" + .. " -> 0xcccccccccccccccc(14757395258967641292)", + 1, + true + ) + ) assert(log.accesses[7].read == leaf_data) leaf_data = machine:read_memory(leaf_address, leaf_size) assert(leaf_data == make_leaf("\xaa", "\xbb", "\xcc", "\x44")) @@ -1492,7 +1510,14 @@ test_util.make_do_test(build_machine, machine_type, { machine:write_reg("uarch_x" .. t1, machine:read_reg("uarch_x" .. t1) + word_size) machine:write_reg("uarch_x" .. t0, 0xdddddddddddddddd) log, dump = log_step() - assert(dump:match("7: write memory@0x%x+%(%d+%): 0x4444444444444444%(%d+%) %-> 0xdddddddddddddddd%(%d+%)")) + assert( + dump:find( + "7: write uarch.ram@0x600038(6291512): 0x4444444444444444(4919131752989213764)" + .. " -> 0xdddddddddddddddd(15987178197214944733)", + 1, + true + ) + ) assert(log.accesses[7].read == leaf_data) leaf_data = machine:read_memory(leaf_address, leaf_size) assert(leaf_data == make_leaf("\xaa", "\xbb", "\xcc", "\xdd")) @@ -1568,7 +1593,7 @@ test_util.make_do_test(build_machine, machine_type, { uarch = {} })("log_step sa machine:log_step(1, filename1) end) assert(not success) - assert(err:match("file already exists")) + check_error_find(err, "file already exists") -- delete file and confirm that machine is on same mcycle os.remove(filename1) assert(machine:read_reg("mcycle") == 0) @@ -1588,11 +1613,11 @@ test_util.make_do_test(build_machine, machine_type, { uarch = {} })("log_step sa _, err = pcall(function() machine:verify_step(bad_hash, filename1, mcycle_count, root_hash_after) end) - assert(err:match("initial root hash mismatch")) + check_error_find(err, "initial root hash mismatch") _, err = pcall(function() machine:verify_step(root_hash_before, filename1, mcycle_count, bad_hash) end) - assert(err:match("final root hash mismatch")) + check_error_find(err, "final root hash mismatch") -- ensure that copy_step_log() works copy_step_log(filename1, filename2, function() -- copy original file without modifications @@ -1605,7 +1630,7 @@ test_util.make_do_test(build_machine, machine_type, { uarch = {} })("log_step sa _, err = pcall(function() machine:verify_step(root_hash_before, filename2, mcycle_count, root_hash_after) end) - assert(err:match("initial root hash mismatch")) + check_error_find(err, "initial root hash mismatch") -- page indices not in ascending order should fail copy_step_log(filename1, filename2, function(log_data) log_data.pages[2].index = log_data.pages[1].index @@ -1613,7 +1638,7 @@ test_util.make_do_test(build_machine, machine_type, { uarch = {} })("log_step sa _, err = pcall(function() machine:verify_step(root_hash_before, filename2, mcycle_count, bad_hash) end) - assert(err:match("invalid log format: page index is not in increasing order")) + check_error_find(err, "invalid log format: page index is not in increasing order") -- page scratch hash area not zeroed copy_step_log(filename1, filename2, function(log_data) log_data.pages[1].hash = string.rep("\1", 32) @@ -1621,7 +1646,7 @@ test_util.make_do_test(build_machine, machine_type, { uarch = {} })("log_step sa _, err = pcall(function() machine:verify_step(root_hash_before, filename2, mcycle_count, bad_hash) end) - assert(err:match("invalid log format: page scratch hash area is not zero")) + check_error_find(err, "invalid log format: page scratch hash area is not zero") -- add one extra page copy_step_log(filename1, filename2, function(log_data) table.insert(log_data.pages, { @@ -1633,7 +1658,7 @@ test_util.make_do_test(build_machine, machine_type, { uarch = {} })("log_step sa _, err = pcall(function() machine:verify_step(root_hash_before, filename2, mcycle_count, bad_hash) end) - assert(err:match("trying to access beyond sibling count while skipping range")) + check_error_find(err, "too many sibling hashes in log") -- remove one page copy_step_log(filename1, filename2, function(log_data) table.remove(log_data.pages) @@ -1641,7 +1666,7 @@ test_util.make_do_test(build_machine, machine_type, { uarch = {} })("log_step sa _, err = pcall(function() machine:verify_step(root_hash_before, filename2, mcycle_count, bad_hash) end) - assert(err:match("initial root hash mismatch")) + check_error_find(err, "too many sibling hashes in log") -- override page count to zero copy_step_log(filename1, filename2, function(log_data) log_data.override_page_count = 0 @@ -1649,7 +1674,7 @@ test_util.make_do_test(build_machine, machine_type, { uarch = {} })("log_step sa _, err = pcall(function() machine:verify_step(root_hash_before, filename2, mcycle_count, bad_hash) end) - assert(err:match("page count is zero")) + check_error_find(err, "page count is zero") -- override page count to overflow copy_step_log(filename1, filename2, function(log_data) -- There is no UINT64_MAX in Lua, so we have to use the signed representation @@ -1659,7 +1684,7 @@ test_util.make_do_test(build_machine, machine_type, { uarch = {} })("log_step sa _, err = pcall(function() machine:verify_step(root_hash_before, filename2, mcycle_count, bad_hash) end) - assert(err:match("page data past end of step log")) + check_error_find(err, "page data past end of step log") -- remove one sibling copy_step_log(filename1, filename2, function(log_data) table.remove(log_data.siblings) @@ -1667,7 +1692,7 @@ test_util.make_do_test(build_machine, machine_type, { uarch = {} })("log_step sa _, err = pcall(function() machine:verify_step(root_hash_before, filename2, mcycle_count, bad_hash) end) - assert(err:match("trying to access beyond sibling count while skipping range")) + check_error_find(err, "too few sibling hashes in log") -- add an extra sibling copy_step_log(filename1, filename2, function(log_data) table.insert(log_data.siblings, bad_hash) @@ -1675,7 +1700,7 @@ test_util.make_do_test(build_machine, machine_type, { uarch = {} })("log_step sa _, err = pcall(function() machine:verify_step(root_hash_before, filename2, mcycle_count, bad_hash) end) - assert(err:match("sibling hashes not totally consumed")) + check_error_find(err, "too many sibling hashes in log") -- modify one sibling hash copy_step_log(filename1, filename2, function(log_data) log_data.siblings[1] = bad_hash @@ -1683,7 +1708,7 @@ test_util.make_do_test(build_machine, machine_type, { uarch = {} })("log_step sa _, err = pcall(function() machine:verify_step(root_hash_before, filename2, mcycle_count, bad_hash) end) - assert(err:match("initial root hash mismatch")) + check_error_find(err, "initial root hash mismatch") -- empty siblings copy_step_log(filename1, filename2, function(log_data) log_data.siblings = {} @@ -1691,7 +1716,7 @@ test_util.make_do_test(build_machine, machine_type, { uarch = {} })("log_step sa _, err = pcall(function() machine:verify_step(root_hash_before, filename2, mcycle_count, bad_hash) end) - assert(err:match("compute_root_hash_impl: trying to access beyond sibling count while skipping range")) + check_error_find(err, "too few sibling hashes in log") -- override sibling count to overflow copy_step_log(filename1, filename2, function(log_data) log_data.override_sibling_count = 0xffffffff @@ -1699,14 +1724,14 @@ test_util.make_do_test(build_machine, machine_type, { uarch = {} })("log_step sa _, err = pcall(function() machine:verify_step(root_hash_before, filename2, mcycle_count, bad_hash) end) - assert(err:match("sibling hashes past end of step log")) + check_error_find(err, "sibling hashes past end of step log") -- log_step should fail if uarch is not reset machine:run_uarch(1) -- advance 1 micro step 0< uarch is not reset os.remove(filename1) _, err = pcall(function() machine:log_step(1, filename1) end) - assert(err:match("microarchitecture is not reset")) + check_error_find(err, "microarchitecture is not reset") -- after uarch is reset, log_step should work machine:reset_uarch() status = machine:log_step(1, filename1) diff --git a/tests/misc/test-machine-c-api.cpp b/tests/misc/test-machine-c-api.cpp index 313eb0e3d..980d4d595 100644 --- a/tests/misc/test-machine-c-api.cpp +++ b/tests/misc/test-machine-c-api.cpp @@ -390,7 +390,7 @@ BOOST_AUTO_TEST_CASE_NOLINT(get_proof_null_machine_test) { BOOST_FIXTURE_TEST_CASE_NOLINT(get_proof_invalid_address_test, ordinary_machine_fixture) { const char *proof{}; cm_error error_code = cm_get_proof(_machine, 1, 12, &proof); - BOOST_CHECK_EQUAL(error_code, CM_ERROR_INVALID_ARGUMENT); + BOOST_CHECK_EQUAL(error_code, CM_ERROR_DOMAIN_ERROR); std::string result = cm_get_last_error_message(); std::string origin("address not aligned to log2_size"); @@ -460,7 +460,7 @@ BOOST_AUTO_TEST_CASE_NOLINT(read_word_null_machine_test) { BOOST_FIXTURE_TEST_CASE_NOLINT(read_word_invalid_address_test, ordinary_machine_fixture) { uint64_t word_value = 0; cm_error error_code = cm_read_word(_machine, 0xffffffff, &word_value); - BOOST_CHECK_EQUAL(error_code, CM_ERROR_INVALID_ARGUMENT); + BOOST_CHECK_EQUAL(error_code, CM_ERROR_DOMAIN_ERROR); std::string result = cm_get_last_error_message(); std::string origin("address not aligned"); @@ -538,7 +538,7 @@ BOOST_FIXTURE_TEST_CASE_NOLINT(write_memory_invalid_address_range_test, ordinary cm_error error_code = cm_write_memory(_machine, address, write_data.data(), write_data.size()); BOOST_CHECK_EQUAL(error_code, CM_ERROR_INVALID_ARGUMENT); std::string result = cm_get_last_error_message(); - std::string origin("address range not entirely in memory PMA"); + std::string origin("attempted write to device memory range"); BOOST_CHECK_EQUAL(origin, result); } @@ -631,7 +631,7 @@ BOOST_FIXTURE_TEST_CASE_NOLINT(write_virtual_memory_invalid_address_range_test, cm_error error_code = cm_write_virtual_memory(_machine, address, write_data.data(), write_data.size()); BOOST_CHECK_EQUAL(error_code, CM_ERROR_INVALID_ARGUMENT); std::string result = cm_get_last_error_message(); - std::string origin("address range not entirely in memory PMA"); + std::string origin("attempted write to device memory range"); BOOST_CHECK_EQUAL(origin, result); } diff --git a/uarch/Makefile b/uarch/Makefile index 392642d7c..273811c30 100644 --- a/uarch/Makefile +++ b/uarch/Makefile @@ -53,6 +53,7 @@ CFLAGS := -march=rv64i -mabi=lp64 -Wl,--gc-sections $(OPTFLAGS) $(UBFLAGS) $(WAR -I. \ -I$(THIRD_PARTY_DIR)/llvm-flang-uint128 \ -I$(EMULATOR_SRC_DIR) \ + $(UARCH_DEFS) \ $(TOOLCHAIN_INCS) CXXFLAGS := -std=c++20 -fno-rtti @@ -60,6 +61,7 @@ CXXFLAGS := -std=c++20 -fno-rtti UARCH_SOURCES=\ uarch-printf.c \ uarch-run.cpp \ + uarch-ecall.c \ uarch-runtime.cpp EMULATOR_SOURCES=\ @@ -79,6 +81,7 @@ COMPUTE_UARCH_CPP_SOURCES=\ $(EMULATOR_SRC_DIR)/pristine-merkle-tree.cpp \ $(EMULATOR_SRC_DIR)/complete-merkle-tree.cpp \ $(EMULATOR_SRC_DIR)/full-merkle-tree.cpp + COMPUTE_UARCH_C_SOURCES=\ $(THIRD_PARTY_DIR)/tiny_sha3/sha3.c \ uarch-pristine-ram.c diff --git a/uarch/machine-uarch-bridge-state-access.h b/uarch/machine-uarch-bridge-state-access.h new file mode 100644 index 000000000..bf08fb53a --- /dev/null +++ b/uarch/machine-uarch-bridge-state-access.h @@ -0,0 +1,507 @@ +// Copyright Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: LGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify it under +// the terms of the GNU Lesser General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) any +// later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +// PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License along +// with this program (see COPYING). If not, see . +// + +#ifndef MACHINE_UARCH_BRIDGE_STATE_ACCESS_H +#define MACHINE_UARCH_BRIDGE_STATE_ACCESS_H + +#include "uarch-runtime.h" // must be included first, because of assert + +#include + +#include "compiler-defines.h" +#include "device-state-access.h" +#include "i-state-access.h" +#include "machine-reg.h" +#include "mock-pma-entry.h" +#include "pma-constants.h" +#include "riscv-constants.h" +#include "shadow-pmas.h" +#include "uarch-constants.h" +#include "uarch-defines.h" +#include "uarch-ecall.h" +#include "uarch-strict-aliasing.h" + +#if DUMP_UARCH_STATE_ACCESS +#include "scoped-note.h" +#endif + +namespace cartesi { + +class machine_uarch_bridge_state_access; + +// Type trait that should return the pma_entry type for a state access class +template <> +struct i_state_access_pma_entry { + using type = mock_pma_entry; +}; +// Type trait that should return the fast_addr type for a state access class +template <> +struct i_state_access_fast_addr { + using type = uint64_t; +}; + +// Provides access to the state of the big emulator from microcode +class machine_uarch_bridge_state_access : public i_state_access { + + //NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) + std::array, PMA_MAX> &m_pmas; + +public: + machine_uarch_bridge_state_access(std::array, PMA_MAX> &pmas): m_pmas(pmas) {} + machine_uarch_bridge_state_access(const machine_uarch_bridge_state_access &other) = default; + machine_uarch_bridge_state_access(machine_uarch_bridge_state_access &&other) = default; + machine_uarch_bridge_state_access &operator=(const machine_uarch_bridge_state_access &other) = delete; + machine_uarch_bridge_state_access &operator=(machine_uarch_bridge_state_access &&other) = delete; + ~machine_uarch_bridge_state_access() = default; + +private: + friend i_state_access; + +#ifdef DUMP_UARCH_STATE_ACCESS + // NOLINTNEXTLINE(readability-convert-member-functions-to-static) + auto do_make_scoped_note([[maybe_unused]] const char *text) { + return scoped_note{*this, text}; + } +#endif + + uint64_t bridge_read_reg(machine_reg reg) { + return ua_aliased_aligned_read(machine_reg_address(reg)); + } + + void bridge_write_reg(machine_reg reg, uint64_t val) { + ua_aliased_aligned_write(machine_reg_address(reg), val); + } + + uint64_t do_read_x(int i) { + return bridge_read_reg(machine_reg_enum(machine_reg::x0, i)); + } + + void do_write_x(int i, uint64_t val) { + bridge_write_reg(machine_reg_enum(machine_reg::x0, i), val); + } + + uint64_t do_read_f(int i) { + return bridge_read_reg(machine_reg_enum(machine_reg::f0, i)); + } + + void do_write_f(int i, uint64_t val) { + bridge_write_reg(machine_reg_enum(machine_reg::f0, i), val); + } + + uint64_t do_read_pc() { + return bridge_read_reg(machine_reg::pc); + } + + void do_write_pc(uint64_t val) { + bridge_write_reg(machine_reg::pc, val); + } + + uint64_t do_read_fcsr() { + return bridge_read_reg(machine_reg::fcsr); + } + + void do_write_fcsr(uint64_t val) { + bridge_write_reg(machine_reg::fcsr, val); + } + + uint64_t do_read_icycleinstret() { + return bridge_read_reg(machine_reg::icycleinstret); + } + + void do_write_icycleinstret(uint64_t val) { + bridge_write_reg(machine_reg::icycleinstret, val); + } + + uint64_t do_read_mvendorid() { + return bridge_read_reg(machine_reg::mvendorid); + } + + uint64_t do_read_marchid() { + return bridge_read_reg(machine_reg::marchid); + } + + uint64_t do_read_mimpid() { + return bridge_read_reg(machine_reg::mimpid); + } + + uint64_t do_read_mcycle() { + return bridge_read_reg(machine_reg::mcycle); + } + + void do_write_mcycle(uint64_t val) { + bridge_write_reg(machine_reg::mcycle, val); + } + + uint64_t do_read_mstatus() { + return bridge_read_reg(machine_reg::mstatus); + } + + void do_write_mstatus(uint64_t val) { + bridge_write_reg(machine_reg::mstatus, val); + } + + uint64_t do_read_mtvec() { + return bridge_read_reg(machine_reg::mtvec); + } + + void do_write_mtvec(uint64_t val) { + bridge_write_reg(machine_reg::mtvec, val); + } + + uint64_t do_read_mscratch() { + return bridge_read_reg(machine_reg::mscratch); + } + + void do_write_mscratch(uint64_t val) { + bridge_write_reg(machine_reg::mscratch, val); + } + + uint64_t do_read_mepc() { + return bridge_read_reg(machine_reg::mepc); + } + + void do_write_mepc(uint64_t val) { + bridge_write_reg(machine_reg::mepc, val); + } + + uint64_t do_read_mcause() { + return bridge_read_reg(machine_reg::mcause); + } + + void do_write_mcause(uint64_t val) { + bridge_write_reg(machine_reg::mcause, val); + } + + uint64_t do_read_mtval() { + return bridge_read_reg(machine_reg::mtval); + } + + void do_write_mtval(uint64_t val) { + bridge_write_reg(machine_reg::mtval, val); + } + + uint64_t do_read_misa() { + return bridge_read_reg(machine_reg::misa); + } + + void do_write_misa(uint64_t val) { + bridge_write_reg(machine_reg::misa, val); + } + + uint64_t do_read_mie() { + return bridge_read_reg(machine_reg::mie); + } + + void do_write_mie(uint64_t val) { + bridge_write_reg(machine_reg::mie, val); + } + + uint64_t do_read_mip() { + return bridge_read_reg(machine_reg::mip); + } + + void do_write_mip(uint64_t val) { + bridge_write_reg(machine_reg::mip, val); + } + + uint64_t do_read_medeleg() { + return bridge_read_reg(machine_reg::medeleg); + } + + void do_write_medeleg(uint64_t val) { + bridge_write_reg(machine_reg::medeleg, val); + } + + uint64_t do_read_mideleg() { + return bridge_read_reg(machine_reg::mideleg); + } + + void do_write_mideleg(uint64_t val) { + bridge_write_reg(machine_reg::mideleg, val); + } + + uint64_t do_read_mcounteren() { + return bridge_read_reg(machine_reg::mcounteren); + } + + void do_write_mcounteren(uint64_t val) { + bridge_write_reg(machine_reg::mcounteren, val); + } + + uint64_t do_read_senvcfg() { + return bridge_read_reg(machine_reg::senvcfg); + } + + void do_write_senvcfg(uint64_t val) { + bridge_write_reg(machine_reg::senvcfg, val); + } + + uint64_t do_read_menvcfg() { + return bridge_read_reg(machine_reg::menvcfg); + } + + void do_write_menvcfg(uint64_t val) { + bridge_write_reg(machine_reg::menvcfg, val); + } + + uint64_t do_read_stvec() { + return bridge_read_reg(machine_reg::stvec); + } + + void do_write_stvec(uint64_t val) { + bridge_write_reg(machine_reg::stvec, val); + } + + uint64_t do_read_sscratch() { + return bridge_read_reg(machine_reg::sscratch); + } + + void do_write_sscratch(uint64_t val) { + bridge_write_reg(machine_reg::sscratch, val); + } + + uint64_t do_read_sepc() { + return bridge_read_reg(machine_reg::sepc); + } + + void do_write_sepc(uint64_t val) { + bridge_write_reg(machine_reg::sepc, val); + } + + uint64_t do_read_scause() { + return bridge_read_reg(machine_reg::scause); + } + + void do_write_scause(uint64_t val) { + bridge_write_reg(machine_reg::scause, val); + } + + uint64_t do_read_stval() { + return bridge_read_reg(machine_reg::stval); + } + + void do_write_stval(uint64_t val) { + bridge_write_reg(machine_reg::stval, val); + } + + uint64_t do_read_satp() { + return bridge_read_reg(machine_reg::satp); + } + + void do_write_satp(uint64_t val) { + bridge_write_reg(machine_reg::satp, val); + } + + uint64_t do_read_scounteren() { + return bridge_read_reg(machine_reg::scounteren); + } + + void do_write_scounteren(uint64_t val) { + bridge_write_reg(machine_reg::scounteren, val); + } + + uint64_t do_read_ilrsc() { + return bridge_read_reg(machine_reg::ilrsc); + } + + void do_write_ilrsc(uint64_t val) { + bridge_write_reg(machine_reg::ilrsc, val); + } + + uint64_t do_read_iprv() { + return bridge_read_reg(machine_reg::iprv); + } + + void do_write_iprv(uint64_t val) { + bridge_write_reg(machine_reg::iprv, val); + } + + uint64_t do_read_iflags_X() { + return bridge_read_reg(machine_reg::iflags_X); + } + + void do_write_iflags_X(uint64_t val) { + bridge_write_reg(machine_reg::iflags_X, val); + } + + uint64_t do_read_iflags_Y() { + return bridge_read_reg(machine_reg::iflags_Y); + } + + void do_write_iflags_Y(uint64_t val) { + bridge_write_reg(machine_reg::iflags_Y, val); + } + + uint64_t do_read_iflags_H() { + return bridge_read_reg(machine_reg::iflags_H); + } + + void do_write_iflags_H(uint64_t val) { + bridge_write_reg(machine_reg::iflags_H, val); + } + + uint64_t do_read_iunrep() { + return bridge_read_reg(machine_reg::iunrep); + } + + void do_write_iunrep(uint64_t val) { + bridge_write_reg(machine_reg::iunrep, val); + } + + uint64_t do_read_clint_mtimecmp() { + return bridge_read_reg(machine_reg::clint_mtimecmp); + } + + void do_write_clint_mtimecmp(uint64_t val) { + bridge_write_reg(machine_reg::clint_mtimecmp, val); + } + + uint64_t do_read_plic_girqpend() { + return bridge_read_reg(machine_reg::plic_girqpend); + } + + void do_write_plic_girqpend(uint64_t val) { + bridge_write_reg(machine_reg::plic_girqpend, val); + } + + uint64_t do_read_plic_girqsrvd() { + return bridge_read_reg(machine_reg::plic_girqsrvd); + } + + void do_write_plic_girqsrvd(uint64_t val) { + bridge_write_reg(machine_reg::plic_girqsrvd, val); + } + + uint64_t do_read_htif_fromhost() { + return bridge_read_reg(machine_reg::htif_fromhost); + } + + void do_write_htif_fromhost(uint64_t val) { + bridge_write_reg(machine_reg::htif_fromhost, val); + } + + uint64_t do_read_htif_tohost() { + return bridge_read_reg(machine_reg::htif_tohost); + } + + void do_write_htif_tohost(uint64_t val) { + bridge_write_reg(machine_reg::htif_tohost, val); + } + + uint64_t do_read_htif_ihalt() { + return bridge_read_reg(machine_reg::htif_ihalt); + } + + uint64_t do_read_htif_iconsole() { + return bridge_read_reg(machine_reg::htif_iconsole); + } + + uint64_t do_read_htif_iyield() { + return bridge_read_reg(machine_reg::htif_iyield); + } + + std::pair do_poll_external_interrupts(uint64_t mcycle, uint64_t /*mcycle_max*/) { + return {mcycle, false}; + } + + template + void do_read_memory_word(uint64_t paddr, uint64_t /* pma_index */, T *pval) { + *pval = ua_aliased_aligned_read(paddr); + } + + template + void do_write_memory_word(uint64_t paddr, uint64_t /* pma_index */, T val) { + ua_aliased_aligned_write(paddr, val); + } + + bool do_read_memory(uint64_t /*paddr*/, unsigned char * /*data*/, uint64_t /*length*/) { + // This is not implemented yet because it's not being used + abort(); + return false; + } + + bool do_write_memory(uint64_t /*paddr*/, const unsigned char * /*data*/, uint64_t /*length*/) { + // This is not implemented yet because it's not being used + abort(); + return false; + } + + uint64_t read_pma_istart(int i) { + return ua_aliased_aligned_read(shadow_pmas_get_pma_abs_addr(i, shadow_pmas_what::istart)); + } + + uint64_t read_pma_ilength(int i) { + return ua_aliased_aligned_read(shadow_pmas_get_pma_abs_addr(i, shadow_pmas_what::ilength)); + } + + mock_pma_entry &do_read_pma_entry(uint64_t index) { + const uint64_t istart = read_pma_istart(index); + const uint64_t ilength = read_pma_ilength(index); + // NOLINTNEXTLINE(bugprone-narrowing-conversions) + int i = static_cast(index); + if (!m_pmas[i]) { + m_pmas[i] = make_mock_pma_entry(index, istart, ilength, [](const char * /*err*/) { abort(); }); + } + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + return m_pmas[i].value(); + } + + uint64_t do_get_faddr(uint64_t paddr, uint64_t /* pma_index */) const { + return paddr; + } + + uint64_t bridge_read_shadow_tlb(TLB_set_index set_index, uint64_t slot_index, shadow_tlb_what what) { + return ua_aliased_aligned_read(shadow_tlb_get_abs_addr(set_index, slot_index, what)); + } + + template + uint64_t do_read_tlb_vaddr_page(uint64_t slot_index) { + return bridge_read_shadow_tlb(SET, slot_index, shadow_tlb_what::vaddr_page); + } + + template + uint64_t do_read_tlb_vp_offset(uint64_t slot_index) { + return bridge_read_shadow_tlb(SET, slot_index, shadow_tlb_what::vp_offset); + } + + template + uint64_t do_read_tlb_pma_index(uint64_t slot_index) { + return bridge_read_shadow_tlb(SET, slot_index, shadow_tlb_what::pma_index); + } + + template + void do_write_tlb(uint64_t slot_index, uint64_t vaddr_page, uint64_t vp_offset, uint64_t pma_index) { + ua_write_tlb_ECALL(SET, slot_index, vaddr_page, vp_offset, pma_index); + } + + void do_putchar(uint8_t c) { + ua_putchar_ECALL(c); + } + + void do_mark_dirty_page(uint64_t paddr, uint64_t pma_index) { + ua_mark_dirty_page_ECALL(paddr, pma_index); + } + + constexpr const char *do_get_name() const { // NOLINT(readability-convert-member-functions-to-static) + return "machine_uarch_bridge_state_access"; + } + + // NOLINTEND(readability-convert-member-functions-to-static) +}; + +} // namespace cartesi + +#endif diff --git a/uarch/uarch-ecall.c b/uarch/uarch-ecall.c new file mode 100644 index 000000000..33dc8790c --- /dev/null +++ b/uarch/uarch-ecall.c @@ -0,0 +1,73 @@ +// Copyright Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: LGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify it under +// the terms of the GNU Lesser General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) any +// later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +// PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License along +// with this program (see COPYING). If not, see . +// + +#include "uarch-ecall.h" +#include "compiler-defines.h" +#include "uarch-defines.h" + +#include +#include + +void ua_halt_ECALL() { + // NOLINTNEXTLINE(hicpp-no-assembler) + asm volatile("mv a7, %0\n" + "ecall\n" + : // no output + : "r"(UARCH_ECALL_FN_HALT_DEF) + : "a7" // modified registers + ); +} + +void ua_putchar_ECALL(uint8_t c) { + // NOLINTNEXTLINE(hicpp-no-assembler) + asm volatile("mv a7, %0\n" + "mv a0, %1\n" + "ecall\n" + : // no output + : "r"(UARCH_ECALL_FN_PUTCHAR_DEF), + "r"(c) // character to print + : "a7", "a0" // clobbered registers + ); +} + +void ua_mark_dirty_page_ECALL(uint64_t paddr, uint64_t pma_index) { + // NOLINTNEXTLINE(hicpp-no-assembler) + asm volatile("mv a7, %0\n" + "mv a0, %1\n" + "mv a1, %2\n" + "ecall\n" + : // no output + : "r"(UARCH_ECALL_FN_MARK_DIRTY_PAGE_DEF), "r"(paddr), "r"(pma_index) + : "a7", "a0", "a1" // clobbered registers + ); +} + +void ua_write_tlb_ECALL(uint64_t use, uint64_t slot_index, uint64_t vaddr_page, uint64_t vp_offset, + uint64_t pma_index) { + // NOLINTNEXTLINE(hicpp-no-assembler) + asm volatile("mv a7, %0\n" + "mv a0, %1\n" + "mv a1, %2\n" + "mv a2, %3\n" + "mv a3, %4\n" + "mv a4, %5\n" + "ecall\n" + : // no output + : "r"(UARCH_ECALL_FN_WRITE_TLB_DEF), "r"(use), "r"(slot_index), + "r"(vaddr_page), "r"(vp_offset), "r"(pma_index) + : "a7", "a0", "a1", "a2", "a3", "a4" // clobbered registers + ); +} diff --git a/uarch/uarch-ecall.h b/uarch/uarch-ecall.h new file mode 100644 index 000000000..08bdc27af --- /dev/null +++ b/uarch/uarch-ecall.h @@ -0,0 +1,36 @@ +// Copyright Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: LGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify it under +// the terms of the GNU Lesser General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) any +// later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +// PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License along +// with this program (see COPYING). If not, see . +// + +#ifndef UARCH_ECALL_H +#define UARCH_ECALL_H + +#ifdef __cplusplus +#include +extern "C" { +#else +#include +#endif + +void ua_halt_ECALL(); +void ua_putchar_ECALL(uint8_t c); +void ua_mark_dirty_page_ECALL(uint64_t paddr, uint64_t pma_index); +void ua_write_tlb_ECALL(uint64_t use, uint64_t slot_index, uint64_t vaddr_page, uint64_t vp_offset, uint64_t pma_index); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/uarch/uarch-machine-state-access.h b/uarch/uarch-machine-state-access.h deleted file mode 100644 index 6d21dae88..000000000 --- a/uarch/uarch-machine-state-access.h +++ /dev/null @@ -1,586 +0,0 @@ -// Copyright Cartesi and individual authors (see AUTHORS) -// SPDX-License-Identifier: LGPL-3.0-or-later -// -// This program is free software: you can redistribute it and/or modify it under -// the terms of the GNU Lesser General Public License as published by the Free -// Software Foundation, either version 3 of the License, or (at your option) any -// later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT ANY -// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A -// PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License along -// with this program (see COPYING). If not, see . -// - -#ifndef UARCH_MACHINE_STATE_ACCESS_H -#define UARCH_MACHINE_STATE_ACCESS_H - -#include "uarch-runtime.h" // must be included first, because of assert - -#define MOCK_THROW_RUNTIME_ERROR(err) abort() -#include "compiler-defines.h" -#include "device-state-access.h" -#include "i-state-access.h" -#include "machine-reg.h" -#include "mock-pma-entry.h" -#include "pma-constants.h" -#include "riscv-constants.h" -#include "shadow-pmas.h" -#include "strict-aliasing.h" -#include "uarch-constants.h" -#include "uarch-defines.h" -#include - -namespace cartesi { - -template -static T raw_read_memory(uint64_t paddr) { - // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast,performance-no-int-to-ptr) - volatile T *p = reinterpret_cast(paddr); - return *p; -} - -template -static void raw_write_memory(uint64_t paddr, T val) { - // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast,performance-no-int-to-ptr) - volatile T *p = reinterpret_cast(paddr); - *p = val; -} - -// Provides access to the state of the big emulator from microcode -class uarch_machine_state_access : public i_state_access { - std::array, PMA_MAX> m_pmas; - -public: - explicit uarch_machine_state_access(std::array, PMA_MAX>& pmas) : m_pmas(pmas) {} - uarch_machine_state_access(const uarch_machine_state_access &other) = default; - uarch_machine_state_access(uarch_machine_state_access &&other) = default; - uarch_machine_state_access &operator=(const uarch_machine_state_access &other) = delete; - uarch_machine_state_access &operator=(uarch_machine_state_access &&other) = delete; - ~uarch_machine_state_access() = default; - -private: - friend i_state_access; - - // NOLINTBEGIN(readability-convert-member-functions-to-static) - - void do_push_bracket(bracket_type /*type*/, const char * /*text*/) {} - - int do_make_scoped_note(const char * /*text*/) { - return 0; - } - - uint64_t do_read_x(int reg) { - return raw_read_memory(machine_reg_address(machine_reg::x0, reg)); - } - - void do_write_x(int reg, uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::x0, reg), val); - } - - uint64_t do_read_f(int reg) { - return raw_read_memory(machine_reg_address(machine_reg::f0, reg)); - } - - void do_write_f(int reg, uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::f0, reg), val); - } - - uint64_t do_read_pc() { - return raw_read_memory(machine_reg_address(machine_reg::pc)); - } - - void do_write_pc(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::pc), val); - } - - uint64_t do_read_fcsr() { - return raw_read_memory(machine_reg_address(machine_reg::fcsr)); - } - - void do_write_fcsr(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::fcsr), val); - } - - uint64_t do_read_icycleinstret() { - return raw_read_memory(machine_reg_address(machine_reg::icycleinstret)); - } - - void do_write_icycleinstret(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::icycleinstret), val); - } - - uint64_t do_read_mvendorid() { - return raw_read_memory(machine_reg_address(machine_reg::mvendorid)); - } - - uint64_t do_read_marchid() { - return raw_read_memory(machine_reg_address(machine_reg::marchid)); - } - - uint64_t do_read_mimpid() { - return raw_read_memory(machine_reg_address(machine_reg::mimpid)); - } - - uint64_t do_read_mcycle() { - return raw_read_memory(machine_reg_address(machine_reg::mcycle)); - } - - void do_write_mcycle(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::mcycle), val); - } - - uint64_t do_read_mstatus() { - return raw_read_memory(machine_reg_address(machine_reg::mstatus)); - } - - void do_write_mstatus(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::mstatus), val); - } - - uint64_t do_read_mtvec() { - return raw_read_memory(machine_reg_address(machine_reg::mtvec)); - } - - void do_write_mtvec(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::mtvec), val); - } - - uint64_t do_read_mscratch() { - return raw_read_memory(machine_reg_address(machine_reg::mscratch)); - } - - void do_write_mscratch(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::mscratch), val); - } - - uint64_t do_read_mepc() { - return raw_read_memory(machine_reg_address(machine_reg::mepc)); - } - - void do_write_mepc(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::mepc), val); - } - - uint64_t do_read_mcause() { - return raw_read_memory(machine_reg_address(machine_reg::mcause)); - } - - void do_write_mcause(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::mcause), val); - } - - uint64_t do_read_mtval() { - return raw_read_memory(machine_reg_address(machine_reg::mtval)); - } - - void do_write_mtval(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::mtval), val); - } - - uint64_t do_read_misa() { - return raw_read_memory(machine_reg_address(machine_reg::misa)); - } - - void do_write_misa(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::misa), val); - } - - uint64_t do_read_mie() { - return raw_read_memory(machine_reg_address(machine_reg::mie)); - } - - void do_write_mie(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::mie), val); - } - - uint64_t do_read_mip() { - return raw_read_memory(machine_reg_address(machine_reg::mip)); - } - - void do_write_mip(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::mip), val); - } - - uint64_t do_read_medeleg() { - return raw_read_memory(machine_reg_address(machine_reg::medeleg)); - } - - void do_write_medeleg(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::medeleg), val); - } - - uint64_t do_read_mideleg() { - return raw_read_memory(machine_reg_address(machine_reg::mideleg)); - } - - void do_write_mideleg(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::mideleg), val); - } - - uint64_t do_read_mcounteren() { - return raw_read_memory(machine_reg_address(machine_reg::mcounteren)); - } - - void do_write_mcounteren(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::mcounteren), val); - } - - uint64_t do_read_senvcfg() const { - return raw_read_memory(machine_reg_address(machine_reg::senvcfg)); - } - - void do_write_senvcfg(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::senvcfg), val); - } - - uint64_t do_read_menvcfg() const { - return raw_read_memory(machine_reg_address(machine_reg::menvcfg)); - } - - void do_write_menvcfg(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::menvcfg), val); - } - - uint64_t do_read_stvec() { - return raw_read_memory(machine_reg_address(machine_reg::stvec)); - } - - void do_write_stvec(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::stvec), val); - } - - uint64_t do_read_sscratch() { - return raw_read_memory(machine_reg_address(machine_reg::sscratch)); - } - - void do_write_sscratch(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::sscratch), val); - } - - uint64_t do_read_sepc() { - return raw_read_memory(machine_reg_address(machine_reg::sepc)); - } - - void do_write_sepc(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::sepc), val); - } - - uint64_t do_read_scause() { - return raw_read_memory(machine_reg_address(machine_reg::scause)); - } - - void do_write_scause(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::scause), val); - } - - uint64_t do_read_stval() { - return raw_read_memory(machine_reg_address(machine_reg::stval)); - } - - void do_write_stval(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::stval), val); - } - - uint64_t do_read_satp() { - return raw_read_memory(machine_reg_address(machine_reg::satp)); - } - - void do_write_satp(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::satp), val); - } - - uint64_t do_read_scounteren() { - return raw_read_memory(machine_reg_address(machine_reg::scounteren)); - } - - void do_write_scounteren(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::scounteren), val); - } - - uint64_t do_read_ilrsc() { - return raw_read_memory(machine_reg_address(machine_reg::ilrsc)); - } - - void do_write_ilrsc(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::ilrsc), val); - } - - uint64_t do_read_iprv() { - return raw_read_memory(machine_reg_address(machine_reg::iprv)); - } - - void do_write_iprv(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::iprv), val); - } - - uint64_t do_read_iflags_X() { - return raw_read_memory(machine_reg_address(machine_reg::iflags_X)); - } - - void do_write_iflags_X(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::iflags_X), val); - } - - uint64_t do_read_iflags_Y() { - return raw_read_memory(machine_reg_address(machine_reg::iflags_Y)); - } - - void do_write_iflags_Y(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::iflags_Y), val); - } - - uint64_t do_read_iflags_H() { - return raw_read_memory(machine_reg_address(machine_reg::iflags_H)); - } - - void do_write_iflags_H(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::iflags_H), val); - } - - uint64_t do_read_iunrep() { - return raw_read_memory(machine_reg_address(machine_reg::iunrep)); - } - - void do_write_iunrep(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::iunrep), val); - } - - uint64_t do_read_clint_mtimecmp() { - return raw_read_memory(machine_reg_address(machine_reg::clint_mtimecmp)); - } - - void do_write_clint_mtimecmp(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::clint_mtimecmp), val); - } - - uint64_t do_read_plic_girqpend() { - return raw_read_memory(machine_reg_address(machine_reg::plic_girqpend)); - } - - void do_write_plic_girqpend(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::plic_girqpend), val); - } - - uint64_t do_read_plic_girqsrvd() { - return raw_read_memory(machine_reg_address(machine_reg::plic_girqsrvd)); - } - - void do_write_plic_girqsrvd(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::plic_girqsrvd), val); - } - - uint64_t do_read_htif_fromhost() { - return raw_read_memory(machine_reg_address(machine_reg::htif_fromhost)); - } - - void do_write_htif_fromhost(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::htif_fromhost), val); - } - - uint64_t do_read_htif_tohost() { - return raw_read_memory(machine_reg_address(machine_reg::htif_tohost)); - } - - void do_write_htif_tohost(uint64_t val) { - raw_write_memory(machine_reg_address(machine_reg::htif_tohost), val); - } - - uint64_t do_read_htif_ihalt() { - return raw_read_memory(machine_reg_address(machine_reg::htif_ihalt)); - } - - uint64_t do_read_htif_iconsole() { - return raw_read_memory(machine_reg_address(machine_reg::htif_iconsole)); - } - - uint64_t do_read_htif_iyield() { - return raw_read_memory(machine_reg_address(machine_reg::htif_iyield)); - } - - std::pair do_poll_external_interrupts(uint64_t mcycle, uint64_t /*mcycle_max*/) { - return {mcycle, false}; - } - - uint64_t read_pma_istart(uint64_t i) { - return raw_read_memory(shadow_pmas_get_pma_abs_addr(i)); - } - - uint64_t read_pma_ilength(uint64_t i) { - return raw_read_memory(shadow_pmas_get_pma_abs_addr(i) + sizeof(uint64_t)); - } - - template - void do_read_memory_word(uint64_t paddr, const unsigned char * /*hpage*/, uint64_t /*hoffset*/, T *pval) { - *pval = raw_read_memory(paddr); - } - - bool do_read_memory(uint64_t /*paddr*/, unsigned char * /*data*/, uint64_t /*length*/) { - // This is not implemented yet because it's not being used - abort(); - return false; - } - - bool do_write_memory(uint64_t /*paddr*/, const unsigned char * /*data*/, uint64_t /*length*/) { - // This is not implemented yet because it's not being used - abort(); - return false; - } - - template - void do_write_memory_word(uint64_t paddr, const unsigned char * /*hpage*/, uint64_t /*hoffset*/, T val) { - raw_write_memory(paddr, val); - } - - mock_pma_entry &do_read_pma_entry(uint64_t index) { - const uint64_t istart = read_pma_istart(index); - const uint64_t ilength = read_pma_ilength(index); - // NOLINTNEXTLINE(bugprone-narrowing-conversions) - int i = static_cast(index); - if (!m_pmas[i]) { - m_pmas[i] = make_mock_pma_entry(index, istart, ilength, [](const char * /*err*/) { abort(); }); - } - // NOLINTNEXTLINE(bugprone-unchecked-optional-access) - return m_pmas[index].value(); - } - - unsigned char *do_get_host_memory(mock_pma_entry & /*pma*/) { - return nullptr; - } - - template - volatile tlb_hot_entry &do_get_tlb_hot_entry(uint64_t eidx) { - // Volatile is used, so the compiler does not optimize out, or do of order writes. - // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast,performance-no-int-to-ptr) - volatile tlb_hot_entry *tlbe = reinterpret_cast(tlb_get_entry_hot_abs_addr(eidx)); - return *tlbe; - } - - template - volatile tlb_cold_entry &do_get_tlb_entry_cold(uint64_t eidx) { - // Volatile is used, so the compiler does not optimize out, or do of order writes. - // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast,performance-no-int-to-ptr) - volatile tlb_cold_entry *tlbe = reinterpret_cast(tlb_get_entry_cold_abs_addr(eidx)); - return *tlbe; - } - - template - bool do_translate_vaddr_via_tlb(uint64_t vaddr, unsigned char **phptr) { - uint64_t eidx = tlb_get_entry_index(vaddr); - const volatile tlb_hot_entry &tlbhe = do_get_tlb_hot_entry(eidx); - if (tlb_is_hit(tlbhe.vaddr_page, vaddr)) { - const uint64_t poffset = vaddr & PAGE_OFFSET_MASK; - const volatile tlb_cold_entry &tlbce = do_get_tlb_entry_cold(eidx); - *phptr = cast_addr_to_ptr(tlbce.paddr_page + poffset); - return true; - } - return false; - } - - template - bool do_read_memory_word_via_tlb(uint64_t vaddr, T *pval) { - uint64_t eidx = tlb_get_entry_index(vaddr); - const volatile tlb_hot_entry &tlbhe = do_get_tlb_hot_entry(eidx); - if (tlb_is_hit(tlbhe.vaddr_page, vaddr)) { - const uint64_t poffset = vaddr & PAGE_OFFSET_MASK; - const volatile tlb_cold_entry &tlbce = do_get_tlb_entry_cold(eidx); - *pval = raw_read_memory(tlbce.paddr_page + poffset); - return true; - } - return false; - } - - template - bool do_write_memory_word_via_tlb(uint64_t vaddr, T val) { - uint64_t eidx = tlb_get_entry_index(vaddr); - volatile tlb_hot_entry &tlbhe = do_get_tlb_hot_entry(eidx); - if (tlb_is_hit(tlbhe.vaddr_page, vaddr)) { - const uint64_t poffset = vaddr & PAGE_OFFSET_MASK; - const volatile tlb_cold_entry &tlbce = do_get_tlb_entry_cold(eidx); - raw_write_memory(tlbce.paddr_page + poffset, val); - return true; - } - return false; - } - - template - unsigned char *do_replace_tlb_entry(uint64_t vaddr, uint64_t paddr, mock_pma_entry &pma) { - uint64_t eidx = tlb_get_entry_index(vaddr); - volatile tlb_cold_entry &tlbce = do_get_tlb_entry_cold(eidx); - volatile tlb_hot_entry &tlbhe = do_get_tlb_hot_entry(eidx); - // Mark page that was on TLB as dirty so we know to update the Merkle tree - if constexpr (ETYPE == TLB_WRITE) { - if (tlbhe.vaddr_page != TLB_INVALID_PAGE) { - mock_pma_entry &pma = do_read_pma_entry(tlbce.pma_index); - pma.mark_dirty_page(tlbce.paddr_page - pma.get_start()); - } - } - const uint64_t vaddr_page = vaddr & ~PAGE_OFFSET_MASK; - const uint64_t paddr_page = paddr & ~PAGE_OFFSET_MASK; - // Both pma_index and paddr_page MUST BE written while its state is invalidated, - // otherwise TLB entry may be read in an incomplete state when computing root hash - // while stepping over this function. - // To do this we first invalidate TLB state before these fields are written to "lock", - // and "unlock" by writing a valid vaddr_page. - tlbhe.vaddr_page = TLB_INVALID_PAGE; // "lock", DO NOT OPTIMIZE OUT THIS LINE - tlbce.pma_index = pma.get_index(); - tlbce.paddr_page = paddr_page; - // The write to vaddr_page MUST BE the last TLB entry write. - tlbhe.vaddr_page = vaddr_page; // "unlock" - // Note that we can't write here the correct vh_offset value, because it depends in a host pointer, - // however the uarch memory bridge will take care of updating it. - return cast_addr_to_ptr(paddr_page); - } - - template - void do_flush_tlb_entry(uint64_t eidx) { - volatile tlb_hot_entry &tlbhe = do_get_tlb_hot_entry(eidx); - // Mark page that was on TLB as dirty so we know to update the Merkle tree - if constexpr (ETYPE == TLB_WRITE) { - if (tlbhe.vaddr_page != TLB_INVALID_PAGE) { - tlbhe.vaddr_page = TLB_INVALID_PAGE; - const volatile tlb_cold_entry &tlbce = do_get_tlb_entry_cold(eidx); - mock_pma_entry &pma = do_read_pma_entry(tlbce.pma_index); - pma.mark_dirty_page(tlbce.paddr_page - pma.get_start()); - } else { - tlbhe.vaddr_page = TLB_INVALID_PAGE; - } - } else { - tlbhe.vaddr_page = TLB_INVALID_PAGE; - } - } - - template - void do_flush_tlb_type() { - for (uint64_t i = 0; i < PMA_TLB_SIZE; ++i) { - do_flush_tlb_entry(i); - } - } - - void do_flush_tlb_vaddr(uint64_t /*vaddr*/) { - do_flush_tlb_type(); - do_flush_tlb_type(); - do_flush_tlb_type(); - } - - bool do_get_soft_yield() { - // Soft yield is meaningless in microarchitecture - return false; - } - - void do_putchar(uint8_t c) { - _putchar(c); - } - - int do_getchar() { - return -1; - } - - // NOLINTEND(readability-convert-member-functions-to-static) -}; - -} // namespace cartesi - -#endif diff --git a/uarch/uarch-printf.c b/uarch/uarch-printf.c index d328b95c0..70254e586 100644 --- a/uarch/uarch-printf.c +++ b/uarch/uarch-printf.c @@ -37,7 +37,7 @@ #include #include "uarch-printf.h" - +#include "uarch-ecall.h" // define this globally (e.g. gcc -DPRINTF_INCLUDE_CONFIG_H ...) to include the // printf_config.h header file @@ -153,7 +153,7 @@ static inline void _out_char(char character, void* buffer, size_t idx, size_t ma { (void)buffer; (void)idx; (void)maxlen; if (character) { - _putchar(character); + ua_putchar_ECALL(character); } } diff --git a/uarch/uarch-printf.h b/uarch/uarch-printf.h index 43341e9ef..a68a6141b 100644 --- a/uarch/uarch-printf.h +++ b/uarch/uarch-printf.h @@ -39,13 +39,6 @@ extern "C" { #endif -/** - * Output a character to a custom device like UART, used by the printf() function - * This function is declared here only. You have to write your custom implementation somewhere - * \param character Character to output - */ -void _putchar(char character); - /** * Tiny printf implementation * You have to implement _putchar if you use printf() diff --git a/uarch/uarch-ram-entry.S b/uarch/uarch-ram-entry.S index 81fc85960..f8fca7184 100644 --- a/uarch/uarch-ram-entry.S +++ b/uarch/uarch-ram-entry.S @@ -20,9 +20,9 @@ .section .text.init; .align 3; .global _start; -_start: +_start: // Initialize stack - li sp, PMA_UARCH_RAM_START_DEF + li sp, PMA_UARCH_RAM_START_DEF li t0, PMA_UARCH_RAM_LENGTH_DEF add sp, sp, t0 // stack pointer at the end of RAM call interpret_next_mcycle_with_uarch diff --git a/uarch/uarch-run.cpp b/uarch/uarch-run.cpp index e3871095a..1248daa0f 100644 --- a/uarch/uarch-run.cpp +++ b/uarch/uarch-run.cpp @@ -18,40 +18,36 @@ #include "compiler-defines.h" #include "interpret.h" +#include "mock-pma-entry.h" +#include "machine-uarch-bridge-state-access.h" #include "uarch-constants.h" -#include "uarch-machine-state-access.h" +#include "uarch-ecall.h" +#include +#include #include using namespace cartesi; -static void set_uarch_halt_flag() { - // NOLINTNEXTLINE(hicpp-no-assembler) - asm volatile("mv a7, %0\n" - "ecall\n" - : // no output - : "r"(cartesi::uarch_ecall_functions::UARCH_ECALL_FN_HALT) - : "a7" // modified registers - ); -} +// Let the state accessor be on static memory storage to speed up uarch initialization +// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) +std::array, PMA_MAX> pmas; namespace cartesi { // Declaration of explicit instantiation in module interpret.cpp when compiled with microarchitecture -extern template interpreter_break_reason interpret(uarch_machine_state_access a, uint64_t mcycle_end); +extern template interpreter_break_reason interpret(machine_uarch_bridge_state_access a, uint64_t mcycle_end); } // namespace cartesi /// \brief Advances one mcycle by executing the "big machine interpreter" compiled to the microarchitecture /// \return This function never returns extern "C" NO_RETURN void interpret_next_mcycle_with_uarch() { - // Let the state accessor be on static memory storage to speed up uarch initialization - static std::array, PMA_MAX> pmas; - uarch_machine_state_access a(pmas); + machine_uarch_bridge_state_access a(pmas); const uint64_t mcycle_end = a.read_mcycle() + 1; interpret(a, mcycle_end); // Finished executing a whole mcycle: halt the microarchitecture - set_uarch_halt_flag(); + ua_halt_ECALL(); // The micro interpreter will never execute this line because the micro machine is halted __builtin_trap(); } diff --git a/uarch/uarch-runtime.cpp b/uarch/uarch-runtime.cpp index bb8758875..932544cef 100644 --- a/uarch/uarch-runtime.cpp +++ b/uarch/uarch-runtime.cpp @@ -21,8 +21,6 @@ #include #include -using namespace cartesi; - // NOLINTNEXTLINE(bugprone-reserved-identifier,cert-dcl37-c,cert-dcl51-cpp) extern "C" void __cxa_pure_virtual() { abort(); @@ -81,18 +79,6 @@ extern "C" void *memset(void *ptr, int value, size_t num) { return ptr; } -extern "C" void _putchar(char c) { - // NOLINTNEXTLINE(hicpp-no-assembler) - asm volatile("mv a7, %0\n" - "mv a6, %1\n" - "ecall\n" - : // no output - : "r"(cartesi::uarch_ecall_functions::UARCH_ECALL_FN_PUTCHAR), - "r"(c) // character to print - : "a7", "a6" // modified registers - ); -} - extern "C" NO_RETURN void abort() { // NOLINTNEXTLINE(hicpp-no-assembler) asm volatile("ebreak" diff --git a/uarch/uarch-strict-aliasing.h b/uarch/uarch-strict-aliasing.h new file mode 100644 index 000000000..7b166eb90 --- /dev/null +++ b/uarch/uarch-strict-aliasing.h @@ -0,0 +1,92 @@ +// Copyright Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: LGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify it under +// the terms of the GNU Lesser General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) any +// later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +// PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License along +// with this program (see COPYING). If not, see . +// + +#ifndef UARCH_STRICT_ALIASING_H +#define UARCH_STRICT_ALIASING_H + +#include "compiler-defines.h" +#include + +template +static inline void ua_aliased_aligned_write(uint64_t paddr, T val); + +template +static inline T ua_aliased_aligned_read(uint64_t paddr); + +#define UA_ALIASED_ALIGNED_WRITE(TYPE, INSN) \ + template <> \ + [[maybe_unused]] void ua_aliased_aligned_write(uint64_t paddr, TYPE val) { \ + /* NOLINTNEXTLINE(hicpp-no-assembler) */ \ + asm volatile("mv a0, %0\n" \ + "mv a1, %1\n" INSN " a1, (a0)\n" \ + : /* no output */ \ + : "r"(paddr), "r"(val) \ + : "a0", "a1" /* clobbered registers */ \ + ); \ + } + +UA_ALIASED_ALIGNED_WRITE(uint64_t, "sd") +UA_ALIASED_ALIGNED_WRITE(int64_t, "sd") +UA_ALIASED_ALIGNED_WRITE(uint32_t, "sw") +UA_ALIASED_ALIGNED_WRITE(int32_t, "sw") +UA_ALIASED_ALIGNED_WRITE(uint16_t, "sh") +UA_ALIASED_ALIGNED_WRITE(int16_t, "sh") +UA_ALIASED_ALIGNED_WRITE(uint8_t, "sb") +UA_ALIASED_ALIGNED_WRITE(int8_t, "sb") + +//??D see if this is the best we can do +#define UA_ALIASED_ALIGNED_READ(TYPE, INSN) \ + template <> \ + [[maybe_unused]] TYPE ua_aliased_aligned_read(uint64_t paddr) { \ + /* NOLINTNEXTLINE(hicpp-no-assembler) */ \ + TYPE ret = 0; \ + asm volatile("mv a0, %1\n" INSN " a1, (a0)\n" \ + "mv %0, a1\n" \ + : "=r"(ret) \ + : "r"(paddr), "r"(ret) \ + : "a0", "a1" /* clobbered registers */ \ + ); \ + return ret; \ + } + +UA_ALIASED_ALIGNED_READ(uint64_t, "ld") +UA_ALIASED_ALIGNED_READ(int64_t, "ld") +UA_ALIASED_ALIGNED_READ(uint32_t, "lwu") +UA_ALIASED_ALIGNED_READ(int32_t, "lw") +UA_ALIASED_ALIGNED_READ(uint16_t, "lhu") +UA_ALIASED_ALIGNED_READ(int16_t, "lh") +UA_ALIASED_ALIGNED_READ(uint8_t, "lbu") +UA_ALIASED_ALIGNED_READ(int8_t, "lb") + +//??D see if this is the best we can do +template <> +[[maybe_unused]] uint32_t ua_aliased_aligned_read(uint64_t paddr) { + // NOLINTNEXTLINE(hicpp-no-assembler) + uint32_t ret = 0; + asm volatile("mv a0, %1\n" + "lhu a1, (a0)\n" + "lhu a2, 2(a0)\n" + "slli a2, a2, 16\n" + "or a1, a2, a1\n" + "mv %0, a1\n" + : "=r"(ret) + : "r"(paddr), "r"(ret) + : "a0", "a1", "a2" // clobbered registers + ); + return ret; +} + +#endif From b3190aa0d94e6a5b5551b8ac7ddb83668c32a209 Mon Sep 17 00:00:00 2001 From: Diego Nehab <1635557+diegonehab@users.noreply.github.com> Date: Sat, 1 Feb 2025 22:46:16 +0000 Subject: [PATCH 05/82] refactor: remove some inline asm from uarch Using __builtin_memcpy instead. --- src/host-addr.h | 22 +++++++ src/strict-aliasing.h | 28 +-------- uarch/uarch-strict-aliasing.h | 111 ++++++++++++++-------------------- 3 files changed, 70 insertions(+), 91 deletions(-) diff --git a/src/host-addr.h b/src/host-addr.h index b6a33f6d1..af5f70dbd 100644 --- a/src/host-addr.h +++ b/src/host-addr.h @@ -17,6 +17,8 @@ #ifndef HOST_ADDR_H #define HOST_ADDR_H +#include "strict-aliasing.h" + namespace cartesi { // This is simply an uint64_t as a separate type, not automatically convertible to uint64_t @@ -93,6 +95,26 @@ static inline void *cast_host_addr_to_ptr(host_addr host_addr) { return reinterpret_cast(static_cast(host_addr)); } +/// \brief Writes a value to memory. +/// \tparam T Type of value. +/// \tparam A Type to which \p haddr is aligned. +/// \param haddr Where to write. Must be aligned to sizeof(A). +/// \param v Value to write. +template +static inline void aliased_aligned_write(host_addr haddr, T v) { + aliased_aligned_write(cast_host_addr_to_ptr(haddr), v); +} + +/// \brief Reads a value from memory. +/// \tparam T Type of value. +/// \tparam A Type to which \p haddr is aligned. +/// \param haddr Where to find value. Must be aligned to sizeof(A). +/// \returns Value read. +template +static inline T aliased_aligned_read(host_addr haddr) { + return aliased_aligned_read(cast_host_addr_to_ptr(haddr)); +} + } // namespace cartesi #endif diff --git a/src/strict-aliasing.h b/src/strict-aliasing.h index 5313f427c..a35becf52 100644 --- a/src/strict-aliasing.h +++ b/src/strict-aliasing.h @@ -21,35 +21,11 @@ #include #include -#include "host-addr.h" - /// \file /// \brief Enforcement of the strict aliasing rule namespace cartesi { -/// \brief Writes a value to memory. -/// \tparam T Type of value. -/// \tparam A Type to which \p haddr is aligned. -/// \param haddr Where to write. Must be aligned to sizeof(A). -/// \param v Value to write. -template -static inline void aliased_aligned_write(host_addr haddr, T v) { - memcpy(__builtin_assume_aligned(cast_host_addr_to_ptr(haddr), sizeof(A)), &v, sizeof(T)); -} - -/// \brief Reads a value from memory. -/// \tparam T Type of value. -/// \tparam A Type to which \p haddr is aligned. -/// \param haddr Where to find value. Must be aligned to sizeof(A). -/// \returns Value read. -template -static inline T aliased_aligned_read(host_addr haddr) { - T v; - memcpy(&v, __builtin_assume_aligned(cast_host_addr_to_ptr(haddr), sizeof(A)), sizeof(T)); - return v; -} - /// \brief Writes a value to memory. /// \tparam T Type of value. /// \tparam A Type to which \p haddr is aligned. @@ -57,7 +33,7 @@ static inline T aliased_aligned_read(host_addr haddr) { /// \param v Value to write. template static inline void aliased_aligned_write(void *p, T v) { - memcpy(__builtin_assume_aligned(p, sizeof(A)), &v, sizeof(T)); + __builtin_memcpy(__builtin_assume_aligned(p, sizeof(A)), &v, sizeof(T)); } /// \brief Reads a value from memory. @@ -68,7 +44,7 @@ static inline void aliased_aligned_write(void *p, T v) { template static inline T aliased_aligned_read(const void *p) { T v; - memcpy(&v, __builtin_assume_aligned(p, sizeof(A)), sizeof(T)); + __builtin_memcpy(&v, __builtin_assume_aligned(p, sizeof(A)), sizeof(T)); return v; } diff --git a/uarch/uarch-strict-aliasing.h b/uarch/uarch-strict-aliasing.h index 7b166eb90..c3bfb2ffc 100644 --- a/uarch/uarch-strict-aliasing.h +++ b/uarch/uarch-strict-aliasing.h @@ -17,76 +17,57 @@ #ifndef UARCH_STRICT_ALIASING_H #define UARCH_STRICT_ALIASING_H -#include "compiler-defines.h" -#include - -template -static inline void ua_aliased_aligned_write(uint64_t paddr, T val); +/// \file +/// \brief Enforcement of the strict aliasing rule -template -static inline T ua_aliased_aligned_read(uint64_t paddr); - -#define UA_ALIASED_ALIGNED_WRITE(TYPE, INSN) \ - template <> \ - [[maybe_unused]] void ua_aliased_aligned_write(uint64_t paddr, TYPE val) { \ - /* NOLINTNEXTLINE(hicpp-no-assembler) */ \ - asm volatile("mv a0, %0\n" \ - "mv a1, %1\n" INSN " a1, (a0)\n" \ - : /* no output */ \ - : "r"(paddr), "r"(val) \ - : "a0", "a1" /* clobbered registers */ \ - ); \ - } +#include "compiler-defines.h" +#include "strict-aliasing.h" -UA_ALIASED_ALIGNED_WRITE(uint64_t, "sd") -UA_ALIASED_ALIGNED_WRITE(int64_t, "sd") -UA_ALIASED_ALIGNED_WRITE(uint32_t, "sw") -UA_ALIASED_ALIGNED_WRITE(int32_t, "sw") -UA_ALIASED_ALIGNED_WRITE(uint16_t, "sh") -UA_ALIASED_ALIGNED_WRITE(int16_t, "sh") -UA_ALIASED_ALIGNED_WRITE(uint8_t, "sb") -UA_ALIASED_ALIGNED_WRITE(int8_t, "sb") +namespace cartesi { -//??D see if this is the best we can do -#define UA_ALIASED_ALIGNED_READ(TYPE, INSN) \ - template <> \ - [[maybe_unused]] TYPE ua_aliased_aligned_read(uint64_t paddr) { \ - /* NOLINTNEXTLINE(hicpp-no-assembler) */ \ - TYPE ret = 0; \ - asm volatile("mv a0, %1\n" INSN " a1, (a0)\n" \ - "mv %0, a1\n" \ - : "=r"(ret) \ - : "r"(paddr), "r"(ret) \ - : "a0", "a1" /* clobbered registers */ \ - ); \ - return ret; \ - } +/// \brief Casts a pointer to an unsigned integer. +/// \details The pointer returned by this function +/// must only be read/written using aliased_aligned_read/aliased_aligned_write, +/// otherwise strict aliasing rules may be violated. +/// \tparam T Unsigned integer type to cast to. +/// \tparam PTR Pointer type to perform the cast. +/// \param addr The address of the pointer represented by an unsigned integer. +/// \returns A pointer. +static inline void *cast_phys_addr_to_ptr(uint64_t paddr) { + // Enforcement on type arguments + static_assert(sizeof(void *) == sizeof(uintptr_t)); + static_assert(sizeof(paddr) >= sizeof(uintptr_t)); + // Note that bellow we cast the address to void* first, + // according to the C spec this is required is to ensure the same presentation, before casting to PTR + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast,bugprone-casting-through-void,performance-no-int-to-ptr) + return reinterpret_cast(static_cast(paddr)); +} -UA_ALIASED_ALIGNED_READ(uint64_t, "ld") -UA_ALIASED_ALIGNED_READ(int64_t, "ld") -UA_ALIASED_ALIGNED_READ(uint32_t, "lwu") -UA_ALIASED_ALIGNED_READ(int32_t, "lw") -UA_ALIASED_ALIGNED_READ(uint16_t, "lhu") -UA_ALIASED_ALIGNED_READ(int16_t, "lh") -UA_ALIASED_ALIGNED_READ(uint8_t, "lbu") -UA_ALIASED_ALIGNED_READ(int8_t, "lb") +//??D I don't know why GCC warns about this overflow when there is none. +//??D The code generated seems to be pretty good as well. +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wstringop-overflow" +/// \brief Writes a value to memory. +/// \tparam T Type of value. +/// \tparam A Type to which \p paddr is aligned. +/// \param paddr Where to write. Must be aligned to sizeof(A). +/// \param v Value to write. +template +static inline void ua_aliased_aligned_write(uint64_t paddr, T v) { + aliased_aligned_write(cast_phys_addr_to_ptr(paddr), v); +} -//??D see if this is the best we can do -template <> -[[maybe_unused]] uint32_t ua_aliased_aligned_read(uint64_t paddr) { - // NOLINTNEXTLINE(hicpp-no-assembler) - uint32_t ret = 0; - asm volatile("mv a0, %1\n" - "lhu a1, (a0)\n" - "lhu a2, 2(a0)\n" - "slli a2, a2, 16\n" - "or a1, a2, a1\n" - "mv %0, a1\n" - : "=r"(ret) - : "r"(paddr), "r"(ret) - : "a0", "a1", "a2" // clobbered registers - ); - return ret; +/// \brief Reads a value from memory. +/// \tparam T Type of value. +/// \tparam A Type to which \p paddr is aligned. +/// \param paddr Where to find value. Must be aligned to sizeof(A). +/// \returns Value read. +template +static inline T ua_aliased_aligned_read(uint64_t paddr) { + return aliased_aligned_read(cast_phys_addr_to_ptr(paddr)); } +#pragma GCC diagnostic pop + +} // namespace cartesi #endif From 9dd83e0e74977ab1387356aaf42fb3230eb99c3a Mon Sep 17 00:00:00 2001 From: Diego Nehab <1635557+diegonehab@users.noreply.github.com> Date: Mon, 3 Feb 2025 09:04:04 +0000 Subject: [PATCH 06/82] refactor: remove machine bridge The machine class took over the role. --- src/Makefile | 1 - src/machine.cpp | 75 +++++++++- src/machine.h | 14 +- src/uarch-machine-bridge.cpp | 170 ---------------------- src/uarch-machine-bridge.h | 57 -------- src/uarch-record-state-access.h | 19 +-- src/uarch-replay-state-access.h | 4 +- src/uarch-state-access.h | 31 ++-- tests/misc/test-machine-c-api.cpp | 2 +- uarch/machine-uarch-bridge-state-access.h | 4 - 10 files changed, 111 insertions(+), 266 deletions(-) delete mode 100644 src/uarch-machine-bridge.cpp delete mode 100644 src/uarch-machine-bridge.h diff --git a/src/Makefile b/src/Makefile index 53c30cb7b..08b00ae82 100644 --- a/src/Makefile +++ b/src/Makefile @@ -388,7 +388,6 @@ LIBCARTESI_OBJS:= \ uarch-pristine-hash.o \ uarch-interpret.o \ uarch-step.o \ - uarch-machine-bridge.o \ uarch-reset-state.o \ send-cmio-response.o \ replay-step-state-access-interop.o diff --git a/src/machine.cpp b/src/machine.cpp index b3f101340..68447e514 100644 --- a/src/machine.cpp +++ b/src/machine.cpp @@ -1987,6 +1987,28 @@ machine::hash_type machine::get_merkle_tree_node_hash(uint64_t address, int log2 return hash; } +const char *machine::get_what_name(uint64_t paddr) { + if (paddr >= PMA_UARCH_RAM_START && paddr - PMA_UARCH_RAM_START < PMA_UARCH_RAM_LENGTH) { + return "uarch.ram"; + } + // If in shadow, return refined name + if (paddr >= PMA_SHADOW_TLB_START && paddr - PMA_SHADOW_TLB_START < PMA_SHADOW_TLB_LENGTH) { + [[maybe_unused]] TLB_set_index set_index{}; + [[maybe_unused]] uint64_t slot_index{}; + return shadow_tlb_get_what_name(shadow_tlb_get_what(paddr, set_index, slot_index)); + } + if (paddr >= PMA_SHADOW_STATE_START && paddr - PMA_SHADOW_STATE_START < PMA_SHADOW_STATE_LENGTH) { + return shadow_state_get_what_name(shadow_state_get_what(paddr)); + } + if (paddr >= PMA_SHADOW_PMAS_START && paddr - PMA_SHADOW_PMAS_START < PMA_SHADOW_PMAS_LENGTH) { + return shadow_pmas_get_what_name(shadow_pmas_get_what(paddr)); + } + if (paddr >= PMA_SHADOW_UARCH_STATE_START && paddr - PMA_SHADOW_UARCH_STATE_START < PMA_SHADOW_UARCH_STATE_LENGTH) { + return shadow_uarch_state_get_what_name(shadow_uarch_state_get_what(paddr)); + } + return "memory"; +} + machine::hash_type machine::get_merkle_tree_node_hash(uint64_t address, int log2_size) const { if (!update_merkle_tree()) { throw std::runtime_error{"error updating Merkle tree"}; @@ -2224,7 +2246,7 @@ uint64_t machine::translate_virtual_address(uint64_t vaddr) { uint64_t machine::read_word(uint64_t paddr) const { // Make sure address is aligned if ((paddr & (sizeof(uint64_t) - 1)) != 0) { - throw std::domain_error{"address not aligned"}; + throw std::domain_error{"attempted misaligned read from word"}; } // Use read_memory alignas(sizeof(uint64_t)) std::array scratch{}; @@ -2232,6 +2254,53 @@ uint64_t machine::read_word(uint64_t paddr) const { return aliased_aligned_read(scratch.data()); } +void machine::write_word(uint64_t paddr, uint64_t val) { + // Make sure address is aligned + if ((paddr & (sizeof(uint64_t) - 1)) != 0) { + throw std::domain_error{"attempted misaligned write to word"}; + } + // If in shadow, forward to write_reg + if (paddr >= PMA_SHADOW_STATE_START && paddr - PMA_SHADOW_STATE_START < PMA_SHADOW_STATE_LENGTH) { + auto reg = shadow_state_get_what(paddr); + if (reg == shadow_state_what::unknown_) { + throw std::runtime_error("unhandled write to shadow state"); + } + if (reg == shadow_state_what::x0) { + throw std::runtime_error("invalid shadow state write to x0"); + } + write_reg(machine_reg_enum(reg), val); + return; + } + // If in uarch shadow, forward to write_reg + if (paddr >= PMA_SHADOW_UARCH_STATE_START && paddr - PMA_SHADOW_UARCH_STATE_START < PMA_SHADOW_UARCH_STATE_LENGTH) { + auto reg = shadow_uarch_state_get_what(paddr); + if (reg == shadow_uarch_state_what::unknown_) { + throw std::runtime_error("unhandled write to shadow uarch state"); + } + if (reg == shadow_uarch_state_what::uarch_x0) { + throw std::runtime_error("invalid shadow state write to uarch.x0"); + } + write_reg(machine_reg_enum(reg), val); + return; + } + // Otherwise, try the slow path + auto &pma = find_pma_entry(m_merkle_pmas, paddr, sizeof(uint64_t)); + if (pma.get_istart_E() || !pma.get_istart_M()) { + std::ostringstream err; + err << "attempted memory write to " << pma.get_description() << " at address 0x" << std::hex << paddr << "(" + << std::dec << paddr << ")"; + throw std::runtime_error{err.str()}; + } + if (!pma.get_istart_W()) { + std::ostringstream err; + err << "attempted memory write to (non-writeable) " << pma.get_description() << " at address 0x" << std::hex + << paddr << "(" << std::dec << paddr << ")"; + throw std::runtime_error{err.str()}; + } + const auto offset = paddr - pma.get_start(); + aliased_aligned_write(pma.get_memory().get_host_memory() + offset, val); +} + void machine::send_cmio_response(uint16_t reason, const unsigned char *data, uint64_t length) { const state_access a(*this); cartesi::send_cmio_response(a, reason, data, length); @@ -2293,7 +2362,7 @@ access_log machine::log_reset_uarch(const access_log::type &log_type) { hash_type root_hash_before; get_root_hash(root_hash_before); // Call uarch_reset_state with a uarch_record_state_access object - uarch_record_state_access a(m_us, *this, log_type); + uarch_record_state_access a(*this, log_type); { [[maybe_unused]] auto note = a.make_scoped_note("reset_uarch_state"); uarch_reset_state(a); @@ -2330,7 +2399,7 @@ access_log machine::log_step_uarch(const access_log::type &log_type) { hash_type root_hash_before; get_root_hash(root_hash_before); // Call interpret with a logged state access object - uarch_record_state_access a(m_us, *this, log_type); + uarch_record_state_access a(*this, log_type); { [[maybe_unused]] auto note = a.make_scoped_note("step"); uarch_step(a); diff --git a/src/machine.h b/src/machine.h index 31b7a7063..5ea27047f 100644 --- a/src/machine.h +++ b/src/machine.h @@ -344,12 +344,19 @@ class machine final { /// \returns The address of the register static uint64_t get_reg_address(reg r); - /// \brief Read the value of a word in the machine state. + /// \brief Read the value of a word from the machine state. /// \param paddr Word address (aligned to 64-bit boundary). /// \returns The value of word at address. /// \details The word can be anywhere in the entire address space. uint64_t read_word(uint64_t paddr) const; + /// \brief Writes the value of a word to the machine state. + /// \param paddr Word address (aligned to 64-bit boundary). + /// \details The word can be in a writeable area of the address space. + /// This includes the shadow state and the shadow uarch state. + /// (But does NOT include memory-mapped devices, the shadow tlb, shadow PMAs, or unnocupied memory regions.) + void write_word(uint64_t paddr, uint64_t val); + /// \brief Reads a chunk of data, by its target physical address and length. /// \param paddr Target physical address to start reading from. /// \param data Buffer that receives data to read. Must be at least \p length bytes long. @@ -524,6 +531,11 @@ class machine final { /// \param root_hash_after State hash after response was sent. static void verify_send_cmio_response(uint16_t reason, const unsigned char *data, uint64_t length, const hash_type &root_hash_before, const access_log &log, const hash_type &root_hash_after); + + /// \brief Returns a description of what is at a given target physical address + /// \param paddr Target physical address of interest + /// \returns Description of what is at that address + static const char *get_what_name(uint64_t paddr); }; } // namespace cartesi diff --git a/src/uarch-machine-bridge.cpp b/src/uarch-machine-bridge.cpp deleted file mode 100644 index 4583230b0..000000000 --- a/src/uarch-machine-bridge.cpp +++ /dev/null @@ -1,170 +0,0 @@ -// Copyright Cartesi and individual authors (see AUTHORS) -// SPDX-License-Identifier: LGPL-3.0-or-later -// -// This program is free software: you can redistribute it and/or modify it under -// the terms of the GNU Lesser General Public License as published by the Free -// Software Foundation, either version 3 of the License, or (at your option) any -// later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT ANY -// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A -// PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License along -// with this program (see COPYING). If not, see . -// - -#include -#include - -#include "machine.h" -#include "riscv-constants.h" -#include "shadow-pmas.h" -#include "shadow-state.h" -#include "shadow-tlb.h" -#include "strict-aliasing.h" -#include "uarch-machine-bridge.h" - -namespace cartesi { - -const char *uarch_machine_bridge::get_what_name(uint64_t paddr) { - if (paddr >= PMA_UARCH_RAM_START && paddr - PMA_UARCH_RAM_START < PMA_UARCH_RAM_LENGTH) { - return "uarch.ram"; - } - // If in shadow, return refined name - if (paddr >= PMA_SHADOW_TLB_START && paddr - PMA_SHADOW_TLB_START < PMA_SHADOW_TLB_LENGTH) { - [[maybe_unused]] TLB_set_index set_index{}; - [[maybe_unused]] uint64_t slot_index{}; - return shadow_tlb_get_what_name(shadow_tlb_get_what(paddr, set_index, slot_index)); - } - if (paddr >= PMA_SHADOW_STATE_START && paddr - PMA_SHADOW_STATE_START < PMA_SHADOW_STATE_LENGTH) { - return shadow_state_get_what_name(shadow_state_get_what(paddr)); - } - if (paddr >= PMA_SHADOW_PMAS_START && paddr - PMA_SHADOW_PMAS_START < PMA_SHADOW_PMAS_LENGTH) { - return shadow_pmas_get_what_name(shadow_pmas_get_what(paddr)); - } - if (paddr >= PMA_SHADOW_UARCH_STATE_START && paddr - PMA_SHADOW_UARCH_STATE_START < PMA_SHADOW_UARCH_STATE_LENGTH) { - return shadow_uarch_state_get_what_name(shadow_uarch_state_get_what(paddr)); - } - return "memory"; -} - -void uarch_machine_bridge::write_word(uint64_t paddr, uint64_t val) { - if ((paddr & (sizeof(uint64_t) - 1)) != 0) { - throw std::runtime_error("misaligned write via uarch-machine bridge"); - } - if (paddr >= PMA_UARCH_RAM_START && paddr - PMA_UARCH_RAM_START < PMA_UARCH_RAM_LENGTH) { - write_uarch_memory_word(paddr, val); - return; - } - if (paddr >= PMA_SHADOW_STATE_START && paddr - PMA_SHADOW_STATE_START < PMA_SHADOW_STATE_LENGTH) { - write_shadow_state(paddr, val); - return; - } - write_memory_word(paddr, val); -} - -uint64_t uarch_machine_bridge::read_word(uint64_t paddr) const { - if ((paddr & (sizeof(uint64_t) - 1)) != 0) { - throw std::runtime_error("misaligned read via uarch-machine bridge"); - } - if (paddr >= PMA_UARCH_RAM_START && paddr - PMA_UARCH_RAM_START < PMA_UARCH_RAM_LENGTH) { - return read_uarch_memory_word(paddr); - } - if (paddr >= PMA_SHADOW_STATE_START && paddr - PMA_SHADOW_STATE_START < PMA_SHADOW_STATE_LENGTH) { - return read_shadow_state(paddr); - } - if (paddr >= PMA_SHADOW_TLB_START && paddr - PMA_SHADOW_TLB_START < PMA_SHADOW_TLB_LENGTH) { - return read_shadow_tlb(paddr); - } - return read_memory_word(paddr); -} - -uint64_t uarch_machine_bridge::read_memory_word(uint64_t paddr) const { - const auto &pma = m_m.find_pma_entry(paddr, sizeof(uint64_t)); - if (pma.get_istart_E() || !pma.get_istart_M()) { - std::ostringstream err; - err << "unhandled memory read from " << pma.get_description() << " at address 0x" << std::hex << paddr << "(" - << std::dec << paddr << ")"; - } - if (!pma.get_istart_R()) { - std::ostringstream err; - err << "attempted memory read from (non-readable) " << pma.get_description() << " at address 0x" << std::hex - << paddr << "(" << std::dec << paddr << ")"; - throw std::runtime_error{err.str()}; - } - const auto offset = paddr - pma.get_start(); - return aliased_aligned_read(pma.get_memory().get_host_memory() + offset); -} - -uint64_t uarch_machine_bridge::read_uarch_memory_word(uint64_t paddr) const { - const auto &pma = m_m.get_uarch_state().ram; - const auto offset = paddr - pma.get_start(); - return aliased_aligned_read(pma.get_memory().get_host_memory() + offset); -} - -void uarch_machine_bridge::write_uarch_memory_word(uint64_t paddr, uint64_t val) { - auto &pma = m_m.get_uarch_state().ram; - const auto offset = paddr - pma.get_start(); - aliased_aligned_write(pma.get_memory().get_host_memory() + offset, val); -} - -uint64_t uarch_machine_bridge::read_shadow_state(uint64_t paddr) const { - auto reg = shadow_state_get_what(paddr); - if (reg == shadow_state_what::unknown_) { - throw std::runtime_error("unhandled shadow state read via uarch-machine bridge"); - } - return m_m.read_reg(machine_reg_enum(reg)); -} - -uint64_t uarch_machine_bridge::read_shadow_tlb(uint64_t paddr) const { - TLB_set_index set_index{}; - uint64_t slot_index{}; - auto reg = shadow_tlb_get_what(paddr, set_index, slot_index); - if (reg == shadow_tlb_what::unknown_) { - throw std::runtime_error("unhandled shadow TLB read via uarch-machine bridge"); - } - return m_m.read_shadow_tlb(set_index, slot_index, reg); -} - -void uarch_machine_bridge::write_shadow_state(uint64_t paddr, uint64_t val) { - auto reg = shadow_state_get_what(paddr); - if (reg == shadow_state_what::unknown_) { - throw std::runtime_error("unhandled shadow state write via uarch-machine bridge"); - } - if (reg == shadow_state_what::x0) { - throw std::runtime_error("invalid write to shadow state x0 via uarch-machine bridge"); - } - m_m.write_reg(machine_reg_enum(reg), val); -} - -void uarch_machine_bridge::mark_dirty_page(uint64_t paddr, uint64_t pma_index) { - m_m.mark_dirty_page(paddr, pma_index); -} - -void uarch_machine_bridge::write_shadow_tlb(TLB_set_index set_index, uint64_t slot_index, uint64_t vaddr_page, - uint64_t vp_offset, uint64_t pma_index) { - m_m.check_shadow_tlb(set_index, slot_index, vaddr_page, vp_offset, pma_index, - "invalid write to shadow TLB via uarch-machine bridge: "); - m_m.write_shadow_tlb(set_index, slot_index, vaddr_page, vp_offset, pma_index); -} - -void uarch_machine_bridge::write_memory_word(uint64_t paddr, uint64_t val) { - auto &pma = m_m.find_pma_entry(paddr, sizeof(uint64_t)); - if (pma.get_istart_E() || !pma.get_istart_M()) { - std::ostringstream err; - err << "unhandled memory write to " << pma.get_description() << " at address 0x" << std::hex << paddr << "(" - << std::dec << paddr << ")"; - throw std::runtime_error{err.str()}; - } - if (!pma.get_istart_W()) { - std::ostringstream err; - err << "attempted memory write to (non-writeable) " << pma.get_description() << " at address 0x" << std::hex - << paddr << "(" << std::dec << paddr << ")"; - throw std::runtime_error{err.str()}; - } - const auto offset = paddr - pma.get_start(); - aliased_aligned_write(pma.get_memory().get_host_memory() + offset, val); -} - -} // namespace cartesi diff --git a/src/uarch-machine-bridge.h b/src/uarch-machine-bridge.h deleted file mode 100644 index 0525f0d45..000000000 --- a/src/uarch-machine-bridge.h +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright Cartesi and individual authors (see AUTHORS) -// SPDX-License-Identifier: LGPL-3.0-or-later -// -// This program is free software: you can redistribute it and/or modify it under -// the terms of the GNU Lesser General Public License as published by the Free -// Software Foundation, either version 3 of the License, or (at your option) any -// later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT ANY -// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A -// PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License along -// with this program (see COPYING). If not, see . -// - -#ifndef UARCH_MACHINE_BRIDGE_H -#define UARCH_MACHINE_BRIDGE_H - -#include -#include - -#include "tlb.h" - -namespace cartesi { - -class machine; - -/// \brief Allows microarchitecture code to access the machine state -class uarch_machine_bridge { - - machine &m_m; // NOLINT(cppcoreguidelines-avoid-const-or-ref-data-members) - -public: - explicit uarch_machine_bridge(machine &m) : m_m(m) { - ; - } - uint64_t read_word(uint64_t paddr) const; - void write_word(uint64_t paddr, uint64_t val); - void mark_dirty_page(uint64_t paddr, uint64_t pma_index); - void write_shadow_tlb(TLB_set_index set_index, uint64_t slot_index, uint64_t vaddr_page, uint64_t vp_offset, - uint64_t pma_index); - static const char *get_what_name(uint64_t paddr); - -private: - void write_shadow_state(uint64_t paddr, uint64_t val); - void write_memory_word(uint64_t paddr, uint64_t val); - void write_uarch_memory_word(uint64_t paddr, uint64_t val); - uint64_t read_shadow_state(uint64_t paddr) const; - uint64_t read_memory_word(uint64_t paddr) const; - uint64_t read_uarch_memory_word(uint64_t paddr) const; - uint64_t read_shadow_tlb(uint64_t paddr) const; -}; - -} // namespace cartesi - -#endif diff --git a/src/uarch-record-state-access.h b/src/uarch-record-state-access.h index 7a8176650..c6ad12cef 100644 --- a/src/uarch-record-state-access.h +++ b/src/uarch-record-state-access.h @@ -42,7 +42,6 @@ #include "shadow-uarch-state.h" #include "strict-aliasing.h" #include "uarch-constants.h" -#include "uarch-machine-bridge.h" #include "uarch-pristine-state-hash.h" #include "uarch-pristine.h" #include "uarch-state.h" @@ -58,9 +57,7 @@ class uarch_record_state_access : public i_uarch_state_access m_log; ///< Pointer to access log @@ -74,10 +71,8 @@ class uarch_record_state_access : public i_uarch_state_access(log_type)) { ; } @@ -280,7 +275,7 @@ class uarch_record_state_access : public i_uarch_state_access::value, [this, paddr, val]() { - m_b.write_word(paddr, val); + m_m.write_word(paddr, val); if (!m_m.update_merkle_tree_page(paddr)) { throw std::invalid_argument{"error updating Merkle tree"}; }; }, - uarch_machine_bridge::get_what_name(paddr)); + machine::get_what_name(paddr)); } void do_write_tlb(TLB_set_index set_index, uint64_t slot_index, uint64_t vaddr_page, uint64_t vp_offset, @@ -304,7 +299,7 @@ class uarch_record_state_access : public i_uarch_state_access { // NOLINTBEGIN(cppcoreguidelines-avoid-const-or-ref-data-members) machine &m_m; - uarch_machine_bridge m_b; + // NOLINTEND(cppcoreguidelines-avoid-const-or-ref-data-members) host_addr m_uram_ph_offset; - // NOLINTEND(cppcoreguidelines-avoid-const-or-ref-data-members)1 public: /// \brief Constructor from machine and uarch states. /// \param um Reference to uarch state. /// \param m Reference to machine state. - explicit uarch_state_access(machine &m) : m_m(m), m_b(m) { + explicit uarch_state_access(machine &m) : m_m(m) { const auto &uram = m_m.get_uarch_state().ram; const auto haddr = cast_ptr_to_host_addr(uram.get_memory_noexcept().get_host_memory()); const auto paddr = uram.get_start(); @@ -114,27 +112,29 @@ class uarch_state_access : public i_uarch_state_access { if ((paddr & (sizeof(uint64_t) - 1)) != 0) { throw std::runtime_error("misaligned read from uarch"); } - // If the word is in UARCH_RAM, read it + // If the word is in UARCH_RAM, read it directly + // ??D This is an optimization that makes 15% difference in run_uarch tests if (m_m.get_uarch_state().ram.contains(paddr, sizeof(uint64_t))) { auto haddr = m_uram_ph_offset + paddr; return aliased_aligned_read(haddr); } - // Otherwise, forward to bridge - return m_b.read_word(paddr); + // Forward to machine + return m_m.read_word(paddr); } void do_write_word(uint64_t paddr, uint64_t val) { if ((paddr & (sizeof(uint64_t) - 1)) != 0) { - throw std::runtime_error("misaligned write to uarch"); + throw std::runtime_error("misaligned read from uarch"); } - // If the word is in UARCH_RAM, write it + // If the word is in UARCH_RAM, write it directly + // ??D This is an optimization that makes 15% difference in run_uarch tests if (m_m.get_uarch_state().ram.contains(paddr, sizeof(uint64_t))) { auto haddr = m_uram_ph_offset + paddr; aliased_aligned_write(haddr, val); return; } - // Otherwise, forward to bridge - m_b.write_word(paddr, val); + // Forward to machine + m_m.write_word(paddr, val); } // NOLINTNEXTLINE(readability-convert-member-functions-to-static) @@ -144,17 +144,18 @@ class uarch_state_access : public i_uarch_state_access { // NOLINTNEXTLINE(readability-convert-member-functions-to-static) void do_mark_dirty_page(uint64_t paddr, uint64_t pma_index) { - // forward to bridge - m_b.mark_dirty_page(paddr, pma_index); + // Forward to machine + m_m.mark_dirty_page(paddr, pma_index); } void do_write_tlb(TLB_set_index set_index, uint64_t slot_index, uint64_t vaddr_page, uint64_t vp_offset, uint64_t pma_index) { - // forward to bridge - m_b.write_shadow_tlb(set_index, slot_index, vaddr_page, vp_offset, pma_index); + // Forward to machine + m_m.write_shadow_tlb(set_index, slot_index, vaddr_page, vp_offset, pma_index); } void do_reset_state() { + // Forward to machine m_m.reset_uarch(); } diff --git a/tests/misc/test-machine-c-api.cpp b/tests/misc/test-machine-c-api.cpp index 980d4d595..2be54e96c 100644 --- a/tests/misc/test-machine-c-api.cpp +++ b/tests/misc/test-machine-c-api.cpp @@ -463,7 +463,7 @@ BOOST_FIXTURE_TEST_CASE_NOLINT(read_word_invalid_address_test, ordinary_machine_ BOOST_CHECK_EQUAL(error_code, CM_ERROR_DOMAIN_ERROR); std::string result = cm_get_last_error_message(); - std::string origin("address not aligned"); + std::string origin("attempted misaligned read from word"); BOOST_CHECK_EQUAL(origin, result); } diff --git a/uarch/machine-uarch-bridge-state-access.h b/uarch/machine-uarch-bridge-state-access.h index bf08fb53a..e0fcac1fe 100644 --- a/uarch/machine-uarch-bridge-state-access.h +++ b/uarch/machine-uarch-bridge-state-access.h @@ -413,10 +413,6 @@ class machine_uarch_bridge_state_access : public i_state_access do_poll_external_interrupts(uint64_t mcycle, uint64_t /*mcycle_max*/) { - return {mcycle, false}; - } - template void do_read_memory_word(uint64_t paddr, uint64_t /* pma_index */, T *pval) { *pval = ua_aliased_aligned_read(paddr); From e83beaa64ba77788ae8d87317bc0e2c7c4638e99 Mon Sep 17 00:00:00 2001 From: Diego Nehab <1635557+diegonehab@users.noreply.github.com> Date: Mon, 3 Feb 2025 11:54:19 +0000 Subject: [PATCH 07/82] refactor: split out i-interactive-state-access.h --- src/complete-merkle-tree.h | 2 +- src/device-state-access.h | 17 +++--- src/i-hasher.h | 10 ++-- src/i-interactive-state-access.h | 80 +++++++++++++++++++++++++++++ src/i-state-access.h | 52 +++---------------- src/interpret.cpp | 38 ++++++++------ src/machine.cpp | 43 +++++++++++++++- src/machine.h | 10 ++++ src/merkle-tree-proof.h | 8 +-- src/meta.h | 14 +++-- src/record-send-cmio-state-access.h | 8 +-- src/record-step-state-access.h | 1 - src/replay-send-cmio-state-access.h | 8 +-- src/replay-step-state-access.h | 1 - src/state-access.h | 72 ++++++++------------------ src/uarch-record-state-access.h | 8 +-- src/uarch-replay-state-access.h | 6 +-- uarch/Makefile | 53 ++++++++++--------- 18 files changed, 251 insertions(+), 180 deletions(-) create mode 100644 src/i-interactive-state-access.h diff --git a/src/complete-merkle-tree.h b/src/complete-merkle-tree.h index f56783bc3..72d3e8fea 100644 --- a/src/complete-merkle-tree.h +++ b/src/complete-merkle-tree.h @@ -66,7 +66,7 @@ class complete_merkle_tree { template complete_merkle_tree(int log2_root_size, int log2_leaf_size, int log2_word_size, L &&leaves) : complete_merkle_tree{log2_root_size, log2_leaf_size, log2_word_size} { - static_assert(std::is_same_v::type>, "not a leaves vector"); + static_assert(std::is_same_v>, "not a leaves vector"); get_level(get_log2_leaf_size()) = std::forward(leaves); bubble_up(); } diff --git a/src/device-state-access.h b/src/device-state-access.h index 812d2c1cb..b424470c4 100644 --- a/src/device-state-access.h +++ b/src/device-state-access.h @@ -23,21 +23,19 @@ #include #include "i-device-state-access.h" +#include "i-interactive-state-access.h" #include "i-state-access.h" namespace cartesi { -/// \details The device_state_access class implements a -/// virtual interface to the state on top of the static -/// interface provided by any class implementing the -/// i_state_access interface. -/// \tparam STATE_ACCESS Class implementing the -/// i_state_access interface. +/// \details The device_state_access class implements a virtual interface to the state on top of the static +/// interface provided by any class implementing the i_state_access interface. +/// \tparam STATE_ACCESS Class implementing the i_state_access interface. template class device_state_access : public i_device_state_access { public: explicit device_state_access(STATE_ACCESS a, uint64_t mcycle) : m_a(a), m_mcycle(mcycle) { - static_assert(is_an_i_state_access::value, "not an i_state_access"); + static_assert(is_an_i_state_access_v, "not an i_state_access"); } /// \brief No copy constructor @@ -152,7 +150,10 @@ class device_state_access : public i_device_state_access { } int do_getchar() override { - return m_a.getchar(); + if constexpr (is_an_i_interactive_state_access_v) { + return m_a.getchar(); + } + return -1; } }; diff --git a/src/i-hasher.h b/src/i-hasher.h index 70fc17390..790d98b2f 100644 --- a/src/i-hasher.h +++ b/src/i-hasher.h @@ -67,8 +67,10 @@ class i_hasher { // CRTP }; template -using is_an_i_hasher = - std::integral_constant::type>::value>; +using is_an_i_hasher = std::integral_constant>>; + +template +constexpr bool is_an_i_hasher_v = is_an_i_hasher::value; /// \brief Computes the hash of concatenated hashes /// \tparam H Hasher class @@ -79,7 +81,7 @@ using is_an_i_hasher = template inline static void get_concat_hash(H &h, const typename H::hash_type &left, const typename H::hash_type &right, typename H::hash_type &result) { - static_assert(is_an_i_hasher::value, "not an i_hasher"); + static_assert(is_an_i_hasher_v, "not an i_hasher"); h.begin(); h.add_data(left.data(), static_cast(left.size())); h.add_data(right.data(), static_cast(right.size())); @@ -95,7 +97,7 @@ inline static void get_concat_hash(H &h, const typename H::hash_type &left, cons template inline static typename H::hash_type get_concat_hash(H &h, const typename H::hash_type &left, const typename H::hash_type &right) { - static_assert(is_an_i_hasher::value, "not an i_hasher"); + static_assert(is_an_i_hasher_v, "not an i_hasher"); h.begin(); h.add_data(left.data(), left.size()); h.add_data(right.data(), right.size()); diff --git a/src/i-interactive-state-access.h b/src/i-interactive-state-access.h new file mode 100644 index 000000000..f08450857 --- /dev/null +++ b/src/i-interactive-state-access.h @@ -0,0 +1,80 @@ +// Copyright Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: LGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify it under +// the terms of the GNU Lesser General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) any +// later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +// PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License along +// with this program (see COPYING). If not, see . +// + +#ifndef I_INTERACTIVE_STATE_ACCESS_H +#define I_INTERACTIVE_STATE_ACCESS_H + +/// \file +/// \brief State access interface + +#include +#include + +#include "meta.h" + +namespace cartesi { + +/// \class i_interactive_state_access +/// \brief Interface for interactive machine state access. +/// \tparam DERIVED Derived class implementing the interface. (An example of CRTP.) +template +class i_interactive_state_access { // CRTP + + /// \brief Returns object cast as the derived class + DERIVED &derived() { + return *static_cast(this); + } + + /// \brief Returns object cast as the derived class + const DERIVED &derived() const { + return *static_cast(this); + } + +public: + /// \brief Wait for external interrupts requests. + /// \param mcycle Current value of mcycle. + /// \param mcycle_max Maximum mcycle after wait. + auto poll_external_interrupts(uint64_t mcycle, uint64_t mcycle_max) { + return derived().do_poll_external_interrupts(mcycle, mcycle_max); + } + + /// \brief Returns true if soft yield HINT instruction is enabled at runtime + bool get_soft_yield() { + return derived().do_get_soft_yield(); + } + + /// \brief Reads a character from the console + /// \returns Character read if any, -1 otherwise + int getchar() { + return derived().do_getchar(); + } + + constexpr const char *get_name() const { + return derived().do_get_name(); + } +}; + +/// \brief SFINAE test implementation of the i_state_access interface +template +using is_an_i_interactive_state_access = + std::integral_constant>>; + +template +constexpr bool is_an_i_interactive_state_access_v = is_an_i_interactive_state_access::value; + +} // namespace cartesi + +#endif diff --git a/src/i-state-access.h b/src/i-state-access.h index 104e27d57..053330297 100644 --- a/src/i-state-access.h +++ b/src/i-state-access.h @@ -1070,7 +1070,7 @@ class i_state_access { // CRTP /// \warning T may or may not cross a Merkle tree word boundary starting from \p faddr! template void read_memory_word(fast_addr faddr, uint64_t pma_index, T *pval) { - static_assert(std::is_integral::value && sizeof(T) <= sizeof(uint64_t), "unsupported type"); + static_assert(std::is_integral_v && sizeof(T) <= sizeof(uint64_t), "unsupported type"); #ifdef DUMP_STATE_ACCESS derived().template do_read_memory_word(faddr, pma_index, pval); const char *fast_addr_name = std::is_same_v ? "phys_addr" : "fast_addr"; @@ -1093,7 +1093,7 @@ class i_state_access { // CRTP /// \warning T may or may not cross a Merkle tree word boundary starting from \p faddr! template void write_memory_word(fast_addr faddr, uint64_t pma_index, T val) { - static_assert(std::is_integral::value && sizeof(T) <= sizeof(uint64_t), "unsupported type"); + static_assert(std::is_integral_v && sizeof(T) <= sizeof(uint64_t), "unsupported type"); derived().template do_write_memory_word(faddr, pma_index, val); #ifdef DUMP_STATE_ACCESS const char *fast_addr_name = std::is_same_v ? "phys_addr" : "fast_addr"; @@ -1186,33 +1186,6 @@ class i_state_access { // CRTP derived().do_putchar(c); } - // --- - // These methods ONLY need to be implemented when state belongs to non-reproducible host machines - // --- - - /// \brief Poll for external interrupts. - /// \param mcycle Current machine mcycle. - /// \param mcycle_max Maximum mcycle to wait for interrupts. - /// \returns A pair, the first value is the new machine mcycle advanced by the relative elapsed time while - /// polling, the second value is a boolean that is true when the poll is stopped due do an external interrupt - /// request. - /// \details When mcycle_max is greater than mcycle, this function will sleep until an external interrupt - /// is triggered or mcycle_max relative elapsed time is reached. - std::pair poll_external_interrupts(uint64_t mcycle, uint64_t mcycle_max) { - return derived().do_poll_external_interrupts(mcycle, mcycle_max); - } - - /// \brief Returns true if soft yield HINT instruction is enabled at runtime - bool get_soft_yield() { - return derived().do_get_soft_yield(); - } - - /// \brief Reads a character from the console - /// \returns Character read if any, -1 otherwise - int getchar() { - return derived().do_getchar(); - } - #ifdef DUMP_COUNTERS //??D we should probably remove this from the interface auto &get_statistics() { @@ -1225,22 +1198,6 @@ class i_state_access { // CRTP } protected: - /// \brief Default implementation when state does not belong to non-reproducible host machine - bool do_get_soft_yield() { // NOLINT(readability-convert-member-functions-to-static) - return false; - } - - /// \brief Default implementation when state does not belong to non-reproducible host machine - std::pair do_poll_external_interrupts(uint64_t mcycle, - uint64_t /* mcycle_max */) { // NOLINT(readability-convert-member-functions-to-static) - return {mcycle, false}; - } - - /// \brief Default implementation when state does not belong to non-reproducible host machine - int do_getchar() { // NOLINT(readability-convert-member-functions-to-static) - return -1; - } - /// \brief For state access classes that do not need annotations void do_push_begin_bracket(const char * /*text*/) {} @@ -1258,7 +1215,10 @@ class i_state_access { // CRTP /// \brief SFINAE test implementation of the i_state_access interface template using is_an_i_state_access = - std::integral_constant::type>::value>; + std::integral_constant>>; + +template +constexpr bool is_an_i_state_access_v = is_an_i_state_access::value; } // namespace cartesi diff --git a/src/interpret.cpp b/src/interpret.cpp index 42a879860..442926ada 100644 --- a/src/interpret.cpp +++ b/src/interpret.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -32,6 +33,7 @@ #include "compiler-defines.h" #include "device-state-access.h" +#include "i-interactive-state-access.h" #include "i-state-access.h" #include "machine-statistics.h" #include "tlb.h" @@ -971,7 +973,7 @@ static NO_INLINE std::pair read_virtual_memory_slow(STATE_ACCESS device_state_access da(a, mcycle); // If we do not know how to read, we treat this as a PMA violation const bool status = pma.get_device_noexcept().get_driver()->read(pma.get_device_noexcept().get_context(), - &da, offset, &val, log2_size::value); + &da, offset, &val, log2_size_v); if (likely(status)) { *pval = static_cast(val); // device logs its own state accesses @@ -1058,7 +1060,7 @@ static NO_INLINE std::pair write_virtual_memory_slow(S const uint64_t offset = paddr - pma.get_start(); device_state_access da(a, mcycle); auto status = pma.get_device_noexcept().get_driver()->write(pma.get_device_noexcept().get_context(), &da, - offset, static_cast(static_cast(val64)), log2_size::value); + offset, static_cast(static_cast(val64)), log2_size_v); // If we do not know how to write, we treat this as a PMA violation if (likely(status != execute_status::failure)) { return {status, pc}; @@ -2757,6 +2759,7 @@ static FORCE_INLINE execute_status execute_MRET(STATE_ACCESS a, uint64_t &pc, ui template static FORCE_INLINE execute_status execute_WFI(STATE_ACCESS a, uint64_t &pc, uint64_t &mcycle, uint32_t insn) { dump_insn(a, pc, insn, "wfi"); + auto status = execute_status::success; // Check privileges and do nothing else auto prv = a.read_iprv(); const uint64_t mstatus = a.read_mstatus(); @@ -2766,14 +2769,11 @@ static FORCE_INLINE execute_status execute_WFI(STATE_ACCESS a, uint64_t &pc, uin } // We wait for interrupts until the next timer interrupt. const uint64_t mcycle_max = rtc_time_to_cycle(a.read_clint_mtimecmp()); - execute_status status = execute_status::success; - if (mcycle_max > mcycle) { - // Poll for external interrupts (e.g console or network), - // this may advance mcycle only when interactive mode is enabled - const auto [next_mcycle, interrupted] = a.poll_external_interrupts(mcycle, mcycle_max); - mcycle = next_mcycle; - if (interrupted) { - status = execute_status::success_and_serve_interrupts; + if constexpr (is_an_i_interactive_state_access_v) { + if (mcycle_max > mcycle) { + // Poll for external interrupts (e.g console or network), + // this may advance mcycle only when interactive mode is enabled + std::tie(mcycle, status) = a.poll_external_interrupts(mcycle, mcycle_max); } } return advance_to_next_insn(a, pc, status); @@ -3215,11 +3215,13 @@ static FORCE_INLINE execute_status execute_SRLIW(STATE_ACCESS a, uint64_t &pc, u template static FORCE_INLINE execute_status execute_SRAIW(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { dump_insn(a, pc, insn, "sraiw"); + // When rd=0 the instruction is a HINT, and we consider it as a soft yield when rs1 == 31 if constexpr (rd_kind == rd_kind::x0) { - // When rd=0 the instruction is a HINT, and we consider it as a soft yield when rs1 == 31 - if (unlikely(insn_get_rs1(insn) == 31 && a.get_soft_yield())) { - // Force the main interpreter loop to break - return advance_to_next_insn(a, pc, execute_status::success_and_yield); + if constexpr (is_an_i_interactive_state_access_v) { + if (unlikely(insn_get_rd(insn) == 0 && insn_get_rs1(insn) == 31 && a.get_soft_yield())) { + // Force the main interpreter loop to break + return advance_to_next_insn(a, pc, execute_status::success_and_yield); + } } return advance_to_next_insn(a, pc); } @@ -5485,8 +5487,10 @@ static NO_INLINE execute_status interpret_loop(STATE_ACCESS a, uint64_t mcycle_e // Polling external interrupts only in WFI instructions is not enough // because Linux won't execute WFI instructions while under heavy load, // yet external interrupts still need to be triggered. - // Therefore we poll for external interrupt once a while in the interpreter loop. - a.poll_external_interrupts(mcycle, mcycle); + // Therefore we poll for external interrupt once in a while in the interpreter loop. + if constexpr (is_an_i_interactive_state_access_v) { + a.poll_external_interrupts(mcycle, mcycle); + } } // Raise the highest priority pending interrupt, if any @@ -6013,7 +6017,7 @@ static NO_INLINE execute_status interpret_loop(STATE_ACCESS a, uint64_t mcycle_e template interpreter_break_reason interpret(STATE_ACCESS a, uint64_t mcycle_end) { static_assert(__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__, "code assumes little-endian byte ordering"); - static_assert(is_an_i_state_access::value, "not an i_state_access"); + static_assert(is_an_i_state_access_v, "not an i_state_access"); const uint64_t mcycle = a.read_mcycle(); diff --git a/src/machine.cpp b/src/machine.cpp index 68447e514..26052f4cd 100644 --- a/src/machine.cpp +++ b/src/machine.cpp @@ -37,10 +37,10 @@ #include "access-log.h" #include "bracket-note.h" #include "clint-factory.h" +#include "device-state-access.h" #include "dtb.h" #include "htif-factory.h" #include "htif.h" -#include "i-device-state-access.h" #include "interpret.h" #include "is-pristine.h" #include "machine-config.h" @@ -56,6 +56,7 @@ #include "replay-send-cmio-state-access.h" #include "replay-step-state-access.h" #include "riscv-constants.h" +#include "rtc.h" #include "send-cmio-response.h" #include "shadow-pmas-factory.h" #include "shadow-state-factory.h" @@ -2495,4 +2496,44 @@ interpreter_break_reason machine::run(uint64_t mcycle_end) { return interpret(a, mcycle_end); } +//??D How come this function seems to never signal we have an inteerrupt??? +std::pair machine::poll_external_interrupts(uint64_t mcycle, uint64_t mcycle_max) { + const auto status = execute_status::success; + // Only poll external interrupts if we are in unreproducible mode + if (unlikely(m_s.iunrep)) { + // Convert the relative interval of cycles we can wait to the interval of host time we can wait + uint64_t timeout_us = (mcycle_max - mcycle) / RTC_CYCLES_PER_US; + int64_t start_us = 0; + if (timeout_us > 0) { + start_us = os_now_us(); + } + state_access a(*this); + device_state_access da(a, mcycle); + // Poll virtio for events (e.g console stdin, network sockets) + // Timeout may be decremented in case a device has deadline timers (e.g network device) + if (has_virtio_devices() && has_virtio_console()) { // VirtIO + VirtIO console + poll_virtio_devices(&timeout_us, &da); + // VirtIO console device will poll TTY + } else if (has_virtio_devices()) { // VirtIO without a console + poll_virtio_devices(&timeout_us, &da); + if (has_htif_console()) { // VirtIO + HTIF console + // Poll tty without waiting more time, because the pool above should have waited enough time + os_poll_tty(0); + } + } else if (has_htif_console()) { // Only HTIF console + os_poll_tty(timeout_us); + } else if (timeout_us > 0) { // No interrupts to check, just keep the CPU idle + os_sleep_us(timeout_us); + } + // If timeout is greater than zero, we should also increment mcycle relative to the elapsed time + if (timeout_us > 0) { + const int64_t end_us = os_now_us(); + const uint64_t elapsed_us = static_cast(std::max(end_us - start_us, INT64_C(0))); + const uint64_t next_mcycle = mcycle + (elapsed_us * RTC_CYCLES_PER_US); + mcycle = std::min(std::max(next_mcycle, mcycle), mcycle_max); + } + } + return {mcycle, status}; +} + } // namespace cartesi diff --git a/src/machine.h b/src/machine.h index 5ea27047f..26eca98aa 100644 --- a/src/machine.h +++ b/src/machine.h @@ -248,6 +248,16 @@ class machine final { return m_mrds; } + /// \brief Wait for external interrupts requests. + /// \param mcycle Current value of mcycle. + /// \param mcycle_max Maximum mcycle after wait. + /// \returns A pair {new_mcycle, status}, where new_mcycle gives new value for mcycle after wait, + /// and status will be execute_status::success_and_serve_interrupts if wait was stopped by an + /// external interrupt resquest. + /// \details When mcycle_max is greater than mcycle, this function will sleep until an external interrupt + /// is triggered or until the amount of time estimated for mcycle to reach mcycle_max has elapsed. + std::pair poll_external_interrupts(uint64_t mcycle, uint64_t mcycle_max); + /// \brief Destructor. ~machine(); diff --git a/src/merkle-tree-proof.h b/src/merkle-tree-proof.h index 14de54041..29d7597e5 100644 --- a/src/merkle-tree-proof.h +++ b/src/merkle-tree-proof.h @@ -191,8 +191,8 @@ class merkle_tree_proof final { ///< \return New root hash template hash_type bubble_up(HASHER_TYPE &h, const hash_type &new_target_hash) const { - static_assert(is_an_i_hasher::value, "not an i_hasher"); - static_assert(std::is_same_v::type::hash_type, hash_type>, + static_assert(is_an_i_hasher_v, "not an i_hasher"); + static_assert(std::is_same_v::hash_type, hash_type>, "incompatible hash types"); hash_type hash = new_target_hash; for (int log2_size = get_log2_target_size(); log2_size < get_log2_root_size(); ++log2_size) { @@ -209,8 +209,8 @@ class merkle_tree_proof final { template merkle_tree_proof slice(HASHER_TYPE &h, int new_log2_root_size, int new_log2_target_size) const { - static_assert(is_an_i_hasher::value, "not an i_hasher"); - static_assert(std::is_same_v::type::hash_type, hash_type>, + static_assert(is_an_i_hasher_v, "not an i_hasher"); + static_assert(std::is_same_v::hash_type, hash_type>, "incompatible hash types"); if (new_log2_root_size <= 0) { throw std::out_of_range{"log2_root_size is not positive"}; diff --git a/src/meta.h b/src/meta.h index c0d5023d7..1a1f28a10 100644 --- a/src/meta.h +++ b/src/meta.h @@ -42,14 +42,6 @@ struct is_template_base_of_helper { }; } // namespace detail -/// \class remove_cvref -/// \brief Provides a member typedef type with reference and topmost cv-qualifiers removed. -/// \note (This is directly available in C++20.) -template -struct remove_cvref { - using type = std::remove_reference_t>; -}; - /// \class is_template_base_of /// \brief SFINAE test if class is derived from from a base template class. /// \tparam BASE Base template. @@ -59,12 +51,18 @@ using is_template_base_of = std::integral_constant, const DERIVED &>, typename detail::is_template_base_of_helper::yes>>; +template