diff --git a/.clang-format b/.clang-format index 0255bce9c..555606d8e 100644 --- a/.clang-format +++ b/.clang-format @@ -16,4 +16,4 @@ ConstructorInitializerAllOnOneLineOrOnePerLine: true IndentCaseLabels: true IndentWidth: 4 SpaceAfterCStyleCast: true -Standard: c++20 +Standard: Latest diff --git a/.clang-tidy b/.clang-tidy index bdd70a8f7..9b98f5647 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -1,12 +1,10 @@ Checks: >- boost*, - -boost-use-ranges, bugprone*, -bugprone-easily-swappable-parameters, cert*, -cert-int09-c, clang-analyzer*, - -clang-analyzer-optin.cplusplus.VirtualCall, cppcoreguidelines*, -cppcoreguidelines-avoid-magic-numbers, -cppcoreguidelines-owning-memory, @@ -32,7 +30,6 @@ Checks: >- -modernize-use-nodiscard, -modernize-use-trailing-return-type, -modernize-avoid-c-arrays, - -modernize-use-ranges, performance*, -performance-enum-size, portability*, @@ -41,16 +38,6 @@ Checks: >- -readability-identifier-length, -readability-magic-numbers, -readability-enum-initial-value, + -readability-use-concise-preprocessor-directives, WarningsAsErrors: >- - boost*, - bugprone*, - cert*, - clang-analyzer*, - cppcoreguidelines*, - google*, - hicpp*, - misc*, - modernize*, - performance*, - portability*, - readability*, + * diff --git a/.dockerignore b/.dockerignore index 99ea8dfa7..d39f81a13 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,8 +2,6 @@ **/*.a **/*.swp **/*.d -**/*.pb.cc -**/*.pb.h **/*.gcno **/*.so **/*.dtb @@ -15,7 +13,7 @@ build third-party/downloads src/cartesi-jsonrpc-machine -src/cartesi-merkle-tree-hash +src/cartesi-hash-tree-hash doc/html doc/api.md diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 62fe73d45..95abae7ed 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -209,10 +209,6 @@ jobs: run: | docker run --rm -t -v cmio-templates:/tmp/cartesi-machine/tests/data cartesi/machine-emulator:tests /usr/share/cartesi-machine/tests/scripts/test-cmio.sh cartesi-jsonrpc-machine cartesi-machine lua - - name: Run Merkle tree tests - run: | - docker run --rm -t ${{ github.repository_owner }}/machine-emulator:tests test-merkle-tree-hash --log2-root-size=30 --log2-leaf-size=12 --input=/usr/bin/test-merkle-tree-hash - - name: Run C API tests run: | docker run --rm -t ${{ github.repository_owner }}/machine-emulator:tests test-machine-c-api @@ -227,7 +223,7 @@ jobs: - name: Run test suite with microarchitecture and host based interpreters comparing machine hashes at every step run: | - docker run --rm -t ${{ github.repository_owner }}/machine-emulator:tests cartesi-machine-tests --concurrency=update_merkle_tree:1 --test="^rv64ui.*$" --jobs=$(nproc) run_host_and_uarch + docker run --rm -t ${{ github.repository_owner }}/machine-emulator:tests cartesi-machine-tests --concurrency=update_hash_tree:1 --test="^rv64ui.*$" --jobs=$(nproc) run_host_and_uarch - name: Create uarch json logs to be used to test the Solidity based microarchitecture interpreter run: | @@ -377,10 +373,6 @@ jobs: run: | docker run --platform linux/arm64 --rm -t -v cmio-templates:/tmp/cartesi-machine/tests/data cartesi/machine-emulator:tests /usr/share/cartesi-machine/tests/scripts/test-cmio.sh cartesi-jsonrpc-machine cartesi-machine lua - - name: Run Merkle tree tests - run: | - docker run --platform linux/arm64 --rm -t ${{ github.repository_owner }}/machine-emulator:tests test-merkle-tree-hash --log2-root-size=30 --log2-leaf-size=12 --input=/usr/bin/test-merkle-tree-hash - - name: Run C API tests run: | docker run --platform linux/arm64 --rm -t ${{ github.repository_owner }}/machine-emulator:tests test-machine-c-api @@ -395,7 +387,7 @@ jobs: - name: Run test suite with microarchitecture and host based interpreters comparing machine hashes at every step run: | - docker run --platform linux/arm64 --rm -t ${{ github.repository_owner }}/machine-emulator:tests cartesi-machine-tests --test="^rv64ui%-v%-add.bin$" --concurrency=update_merkle_tree:1 --jobs=$(nproc) run_host_and_uarch + docker run --platform linux/arm64 --rm -t ${{ github.repository_owner }}/machine-emulator:tests cartesi-machine-tests --test="^rv64ui%-v%-add.bin$" --concurrency=update_hash_tree:1 --jobs=$(nproc) run_host_and_uarch - name: Build machine-emulator "tests" docker image uses: docker/build-push-action@v5 @@ -529,6 +521,7 @@ jobs: GIT_COMMIT=${GITHUB_SHA} DEBUG=yes COVERAGE=yes + THREADS=no MACHINE_EMULATOR_VERSION=${{ env.MACHINE_EMULATOR_VERSION }} project: ${{ vars.DEPOT_PROJECT }} token: ${{ secrets.DEPOT_TOKEN }} @@ -546,11 +539,12 @@ jobs: build-args: | DEBUG=yes COVERAGE=yes + THREADS=no MACHINE_EMULATOR_VERSION=${{ env.MACHINE_EMULATOR_VERSION }} - name: Run coverage run: | - docker run --name coverage-report -t ${{ github.repository_owner }}/machine-emulator:coverage make -j1 test-save-and-load test-machine test-hash test-lua test-jsonrpc test-c-api coverage-machine test-uarch-rv64ui test-uarch-interpreter coverage-uarch coverage-report coverage=yes + docker run --name coverage-report -t ${{ github.repository_owner }}/machine-emulator:coverage make -j1 test-save-and-load test-machine test-lua test-jsonrpc test-c-api coverage-machine test-uarch-rv64ui test-uarch-interpreter coverage-uarch coverage-report coverage=yes docker cp coverage-report:/usr/src/emulator/tests/build/coverage . docker rm coverage-report @@ -569,7 +563,7 @@ jobs: sanitize: name: Sanitize - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: @@ -628,17 +622,17 @@ jobs: SANITIZE=yes MACHINE_EMULATOR_VERSION=${{ env.MACHINE_EMULATOR_VERSION }} - - name: MMAP rnd_bits workaround for the new GitHub ubuntu-22 runner + - name: MMAP rnd_bits workaround for the new GitHub Ubuntu runner run: sudo sysctl vm.mmap_rnd_bits=28 - name: Run tests with sanitizer run: | - docker run --rm -t ${{ github.repository_owner }}/machine-emulator:sanitizer make sanitize=yes test-save-and-load test-machine test-hash test-lua test-jsonrpc test-c-api coverage-machine test-uarch-rv64ui test-uarch-interpreter coverage-uarch + docker run --rm -t ${{ github.repository_owner }}/machine-emulator:sanitizer make sanitize=yes test-save-and-load test-machine test-lua test-jsonrpc test-c-api coverage-machine test-uarch-rv64ui test-uarch-interpreter coverage-uarch publish_artifacts: name: Publish artifacts needs: [build, static-analysis, coverage, sanitize, test_amd64, test_arm64] - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout emulator source code uses: actions/checkout@v4 diff --git a/.gitignore b/.gitignore index 0772c58de..6d5078aa6 100644 --- a/.gitignore +++ b/.gitignore @@ -6,14 +6,15 @@ *.a *.lib *.wasm +*.TODO +*.tmp build pkg third-party/downloads src/cartesi-jsonrpc-machine -src/cartesi-merkle-tree-hash +src/cartesi-hash-tree-hash src/tests/test-machine-c-api -src/tests/test-merkle-tree-hash doc/html doc/api.md diff --git a/.typos.toml b/.typos.toml index ccb29e1e3..4f2054dcd 100644 --- a/.typos.toml +++ b/.typos.toml @@ -9,3 +9,5 @@ stap = "stap" wronly = "wronly" optin = "optin" sxl = "sxl" +nd = "nd" +mke = "mke" diff --git a/Dockerfile b/Dockerfile index 90b21a40a..f12877818 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,35 +1,17 @@ -FROM debian:bookworm-20250407 AS toolchain +FROM debian:trixie-20250811 AS toolchain RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ - build-essential vim wget git lcov \ - libboost1.81-dev libssl-dev libslirp-dev \ + build-essential vim wget git gcovr \ + libomp-19-dev libboost1.83-dev libssl-dev libslirp-dev \ ca-certificates pkg-config lua5.4 liblua5.4-dev \ - luarocks xxd procps \ - g++-12-riscv64-linux-gnu=12.2.0-13cross1 \ - gcc-riscv64-unknown-elf=12.2.0-14+11+b1 && \ + lua-check lua-socket lua-posix lua-lpeg \ + xxd procps unzip gosu \ + clang-tidy clang-format \ + g++-14-riscv64-linux-gnu=14.2.0-19cross1 \ + gcc-riscv64-unknown-elf=14.2.0+19 && \ rm -rf /var/lib/apt/lists/* -# Install clang 19 -RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ - wget software-properties-common gnupg && \ - wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc && \ - add-apt-repository -y 'deb http://apt.llvm.org/bookworm/ llvm-toolchain-bookworm-19 main' && \ - add-apt-repository -y 'deb http://apt.llvm.org/bookworm/ llvm-toolchain-bookworm-19 main' && \ - apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ - clang-tidy-19 clang-format-19 && \ - update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-19 120 && \ - update-alternatives --install /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-19 120 && \ - rm -rf /var/lib/apt/lists/* - -# Install lua packages -RUN luarocks install --lua-version=5.4 luasocket && \ - luarocks install --lua-version=5.4 luasec && \ - luarocks install --lua-version=5.4 luaposix && \ - luarocks install --lua-version=5.4 luacheck - # Install stylua RUN cd /tmp && \ wget https://github.com/JohnnyMorganz/StyLua/releases/download/v0.20.0/stylua-linux-`uname -m`.zip && \ @@ -44,15 +26,6 @@ RUN cd /tmp && \ # Environment has the riscv64 toolchains ENV DEV_ENV_HAS_TOOLCHAIN=yes -# Install su-exec -RUN cd /tmp && \ - git clone --branch v0.2 --depth 1 https://github.com/ncopa/su-exec.git && \ - cd su-exec && \ - if [ `git rev-parse --verify HEAD` != 'f85e5bde1afef399021fbc2a99c837cf851ceafa' ]; then exit 1; fi && \ - make && \ - cp su-exec /usr/local/bin/ && \ - rm -rf /tmp/su-exec - # Install workaround to run as current user COPY tools/docker-entrypoint.sh /usr/local/bin/entrypoint.sh RUN chmod +x /usr/local/bin/entrypoint.sh @@ -69,10 +42,11 @@ FROM toolchain AS builder ARG GIT_COMMIT="" ARG DEBUG=no ARG COVERAGE=no +ARG THREADS=yes ARG SANITIZE=no COPY . . -RUN make -j$(nproc) git_commit=$GIT_COMMIT debug=$DEBUG coverage=$COVERAGE sanitize=$SANITIZE +RUN make -j$(nproc) git_commit=$GIT_COMMIT debug=$DEBUG coverage=$COVERAGE threads=$THREADS sanitize=$SANITIZE #################################################################################################### FROM builder AS debian-packager @@ -80,21 +54,17 @@ FROM builder AS debian-packager RUN make install-uarch debian-package DESTDIR=$PWD/_install #################################################################################################### -FROM debian:bookworm-20250407-slim +FROM debian:trixie-20250811-slim ARG TARGETARCH -COPY --from=debian-packager \ - /usr/src/emulator/machine-emulator_${TARGETARCH}.deb \ - machine-emulator.deb -COPY --from=debian-packager /usr/local/lib/lua /usr/local/lib/lua -COPY --from=debian-packager /usr/local/share/lua /usr/local/share/lua +COPY --from=debian-packager /usr/src/emulator/machine-emulator_${TARGETARCH}.deb machine-emulator.deb RUN apt-get update && \ apt-get install -y ./machine-emulator.deb && \ rm -rf /var/lib/apt/lists/* /var/cache/apt/* machine-emulator.deb -RUN addgroup --system --gid 102 cartesi && \ - adduser --system --uid 102 --ingroup cartesi --disabled-login --no-create-home --home /nonexistent --gecos "cartesi user" --shell /bin/false cartesi +RUN groupadd --system --gid 102 cartesi && \ + useradd --system --uid 102 --gid 102 --no-create-home --home /nonexistent --comment "cartesi user" --shell /bin/false cartesi WORKDIR /opt/cartesi diff --git a/LICENSES.md b/LICENSES.md index 0c12e43b3..0569ab0ba 100644 --- a/LICENSES.md +++ b/LICENSES.md @@ -8,11 +8,11 @@ This project includes several submodules and dependencies, each with its own lic - `tests/machine`: Licensed under the Apache License 2.0. See the license terms in [tests/machine/LICENSE](tests/machine/LICENSE). - `tests/uarch`: Licensed under the Apache License 2.0. Licensing details are available in [tests/uarch/LICENSE](tests/uarch/LICENSE). +- `third-party/ankerl`: Licensed under the MIT License. The license can be found at [third-party/ankerl/LICENSE](third-party/ankerl/LICENSE). - `third-party/llvm-flang-uint128`: Licensed under the Apache License 2.0 with LLVM exceptions. The license can be found at [third-party/llvm-flang-uint128/LICENSE](third-party/llvm-flang-uint128/LICENSE). - `third-party/riscv-arch-test`: Source code licensed under the Apache 2.0 and BSD 3-Clause licenses. Documentation under `CC-BY-4.0`. License information is provided in README.md and other COPYING.* files like [third-party/riscv-arch-test/COPYING.APACHE](third-party/riscv-arch-test/COPYING.APACHE). - `third-party/riscv-tests`: Licensed under the BSD 3-Clause "New" or "Revised" License. See [third-party/riscv-tests/LICENSE](third-party/riscv-tests/LICENSE) for license details. - `third-party/riscv-tests/env`: Licensed under the BSD 3-Clause "New" or "Revised" License. License details are in [third-party/riscv-tests/env/LICENSE](third-party/riscv-tests/env/LICENSE). -- `third-party/tiny_sha3`: Licensed under the MIT License. The license can be found at [third-party/tiny_sha3/LICENSE](third-party/tiny_sha3/LICENSE). - `third-party/nlohmann-json`: Licensed under the MIT License. The license can be found at [third-party/nlohmann-json/LICENSE.MIT](third-party/nlohmann-json/LICENSE.MIT). ## Debian Packages diff --git a/Makefile b/Makefile index 91ee8e864..3ec08cf1d 100644 --- a/Makefile +++ b/Makefile @@ -73,17 +73,17 @@ INSTALL_DIR= cp -RP SYMLINK= ln -sf CHMOD_EXEC= chmod 0755 -EMU_TO_BIN= src/cartesi-jsonrpc-machine src/cartesi-merkle-tree-hash +EMU_TO_BIN= src/cartesi-jsonrpc-machine src/cartesi-hash-tree-hash EMU_TO_LIB= src/$(LIBCARTESI_SO) src/$(LIBCARTESI_SO_JSONRPC) EMU_TO_LIB_A= src/libcartesi.a src/libcartesi_jsonrpc.a src/libluacartesi.a src/libluacartesi_jsonrpc.a EMU_LUA_TO_BIN= src/cartesi-machine.lua src/cartesi-machine-stored-hash.lua -EMU_TO_LUA_PATH= src/cartesi/util.lua src/cartesi/proof.lua src/cartesi/gdbstub.lua +EMU_TO_LUA_PATH= src/cartesi/util.lua src/cartesi/gdbstub.lua EMU_TO_LUA_CPATH= src/cartesi.so EMU_TO_LUA_CARTESI_CPATH= src/cartesi/jsonrpc.so EMU_TO_INC= $(addprefix src/,jsonrpc-machine-c-api.h machine-c-api.h machine-c-version.h) UARCH_TO_SHARE= uarch-ram.bin -TESTS_TO_BIN= tests/build/misc/test-merkle-tree-hash tests/build/misc/test-machine-c-api +TESTS_TO_BIN= tests/build/misc/test-machine-c-api TESTS_LUA_TO_LUA_PATH=tests/lua/cartesi TESTS_LUA_TO_TEST_LUA_PATH=$(wildcard tests/lua/*.lua) TESTS_SCRIPTS_TO_TEST_SCRIPTS_PATH=$(wildcard tests/scripts/*.sh) @@ -96,6 +96,9 @@ TESTSDIR = $(abspath tests) DOWNLOADDIR = $(DEPDIR)/downloads SUBCLEAN = $(addsuffix .clean,$(SRCDIR) uarch tests) +# Pass down received UARCH_DEFS to sub-makefiles +export UARCH_DEFS + # Docker image tag TAG ?= devel DEBIAN_IMG ?= cartesi/machine-emulator:$(TAG).deb @@ -206,10 +209,10 @@ doc: bundle-boost: $(DEPDIR)/downloads/boost $(DEPDIR)/downloads/boost: mkdir -p $(DOWNLOADDIR) - wget -O $(DEPDIR)/downloads/boost_1_81_0.tar.gz https://archives.boost.io/release/1.81.0/source/boost_1_81_0.tar.gz - tar -C $(DEPDIR)/downloads -xzf $(DEPDIR)/downloads/boost_1_81_0.tar.gz boost_1_81_0/boost - mv $(DEPDIR)/downloads/boost_1_81_0/boost $(DEPDIR)/downloads/boost - rm -rf $(DEPDIR)/downloads/boost_1_81_0.tar.gz $(DEPDIR)/downloads/boost_1_81_0 + wget -O $(DEPDIR)/downloads/boost_1_83_0.tar.gz https://archives.boost.io/release/1.83.0/source/boost_1_83_0.tar.gz + tar -C $(DEPDIR)/downloads -xzf $(DEPDIR)/downloads/boost_1_83_0.tar.gz boost_1_83_0/boost + mv $(DEPDIR)/downloads/boost_1_83_0/boost $(DEPDIR)/downloads/boost + rm -rf $(DEPDIR)/downloads/boost_1_83_0.tar.gz $(DEPDIR)/downloads/boost_1_83_0 submodules: git submodule update --init --recursive @@ -250,22 +253,22 @@ $(SRCDIR)/interpret-jump-table.h: @eval $$($(MAKE) -s --no-print-directory env); $(MAKE) -C $(SRCDIR) interpret-jump-table.h build-emulator-builder-image: - docker build $(DOCKER_PLATFORM) --build-arg DEBUG=$(debug) --build-arg COVERAGE=$(coverage) --build-arg SANITIZE=$(sanitize) --target builder -t cartesi/machine-emulator:builder -f Dockerfile . + docker build $(DOCKER_PLATFORM) --build-arg DEBUG=$(debug) --build-arg COVERAGE=$(coverage) --build-arg THREADS=$(threads) --build-arg SANITIZE=$(sanitize) --target builder -t cartesi/machine-emulator:builder -f Dockerfile . build-emulator-toolchain-image build-toolchain: docker build $(DOCKER_PLATFORM) --target toolchain -t cartesi/machine-emulator:toolchain -f Dockerfile . build-emulator-image: - docker build $(DOCKER_PLATFORM) --build-arg DEBUG=$(debug) --build-arg COVERAGE=$(coverage) --build-arg SANITIZE=$(sanitize) -t cartesi/machine-emulator:$(TAG) -f Dockerfile . + docker build $(DOCKER_PLATFORM) --build-arg DEBUG=$(debug) --build-arg COVERAGE=$(coverage) --build-arg THREADS=$(threads) --build-arg SANITIZE=$(sanitize) -t cartesi/machine-emulator:$(TAG) -f Dockerfile . build-emulator-tests-image: build-emulator-builder-image build-emulator-image - docker build $(DOCKER_PLATFORM) --build-arg DEBUG=$(debug) --build-arg COVERAGE=$(coverage) --build-arg SANITIZE=$(sanitize) --build-arg TAG=$(TAG) -t cartesi/machine-emulator:tests -f tests/Dockerfile . + docker build $(DOCKER_PLATFORM) --build-arg DEBUG=$(debug) --build-arg COVERAGE=$(coverage) --build-arg THREADS=$(threads) --build-arg SANITIZE=$(sanitize) --build-arg TAG=$(TAG) -t cartesi/machine-emulator:tests -f tests/Dockerfile . build-emulator-tests-builder-image: build-emulator-builder-image - docker build $(DOCKER_PLATFORM) --target tests-builder --build-arg DEBUG=$(debug) --build-arg COVERAGE=$(coverage) --build-arg SANITIZE=$(sanitize) --build-arg TAG=$(TAG) -t cartesi/machine-emulator:tests-builder -f tests/Dockerfile . + docker build $(DOCKER_PLATFORM) --target tests-builder --build-arg DEBUG=$(debug) --build-arg COVERAGE=$(coverage) --build-arg THREADS=$(threads) --build-arg SANITIZE=$(sanitize) --build-arg TAG=$(TAG) -t cartesi/machine-emulator:tests-builder -f tests/Dockerfile . build-debian-package: - docker build $(DOCKER_PLATFORM) --target debian-packager --build-arg DEBUG=$(debug) --build-arg COVERAGE=$(coverage) --build-arg SANITIZE=$(sanitize) -t $(DEBIAN_IMG) -f Dockerfile . + docker build $(DOCKER_PLATFORM) --target debian-packager --build-arg DEBUG=$(debug) --build-arg COVERAGE=$(coverage) --build-arg THREADS=$(threads) --build-arg SANITIZE=$(sanitize) -t $(DEBIAN_IMG) -f Dockerfile . build-tests-debian-packages: build-emulator-builder-image docker build $(DOCKER_PLATFORM) --target tests-debian-packager --build-arg TAG=$(TAG) -t cartesi/machine-emulator:tests-debian-packager -f tests/Dockerfile . @@ -306,11 +309,12 @@ toolchain-env: check-toolchain cartesi/machine-emulator:toolchain /bin/bash toolchain-exec: check-toolchain - @docker run --hostname toolchain --rm \ + docker run --hostname toolchain --rm \ -e USER=$$(id -u -n) \ -e GROUP=$$(id -g -n) \ -e UID=$$(id -u) \ -e GID=$$(id -g) \ + -e UARCH_DEFS="$(UARCH_DEFS)" \ -v `pwd`:/opt/cartesi/machine-emulator \ -w /opt/cartesi/machine-emulator \ cartesi/machine-emulator:toolchain /bin/bash -c "$(CONTAINER_COMMAND)" diff --git a/README.md b/README.md index b6bc506cf..c2cf67d7b 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,7 @@ sudo apt-get update sudo apt-get install cartesi-machine ``` -The packages provided in this APT repository are known to work with **Debian 12** (Bookworm) and **Ubuntu 24.04** (Noble). +The packages provided in this APT repository are known to work with **Debian 13** (Trixie) and **Ubuntu 24.04** (Noble). #### Alpine Linux @@ -116,28 +116,28 @@ brew install cartesi-machine ##### System Requirements -- C++ Compiler with support for C++20 (tested with GCC >= 11.x and Clang >= 14.x). +- C++ Compiler with support for C++23 (tested with GCC >= 14.x and Clang >= 19.x). - GNU Make >= 3.81 -- Boost >= 1.81 -- Lua >= 5.4.4 (optional, required for scripting support and interactive terminal) +- Boost >= 1.83 +- Lua >= 5.4.6 (optional, required for scripting support and interactive terminal) - Libslirp >= 4.6.0 (optional, required for networking support) ###### Debian Requirements ```sh -sudo apt-get install build-essential git wget libboost1.81-dev liblua5.4-dev libslirp-dev lua5.4 +sudo apt-get install build-essential git wget libgomp-dev libboost1.83-dev liblua5.4-dev libslirp-dev lua5.4 ``` ###### MacPorts Requirements ```sh -sudo port install clang boost181 wget pkgconfig lua54 libslirp +sudo port install clang libomp boost181 wget pkgconfig lua54 libslirp ``` ###### Homebrew Requirements ```sh -brew install llvm boost wget pkg-config lua libslirp +brew install llvm libomp boost wget pkg-config lua libslirp ``` #### Build diff --git a/doc/Doxyfile b/doc/Doxyfile index c7d38e32c..154b51eae 100644 --- a/doc/Doxyfile +++ b/doc/Doxyfile @@ -924,7 +924,7 @@ EXCLUDE_SYMLINKS = NO # Note that the wildcards are matched against the file with absolute path, so to # exclude all test directories for example use the pattern */test/* -EXCLUDE_PATTERNS = *.pb.* +EXCLUDE_PATTERNS = # The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names # (namespaces, classes, functions, etc.) that should be excluded from the diff --git a/src/.styluaignore b/src/.styluaignore new file mode 100644 index 000000000..67d08e3a1 --- /dev/null +++ b/src/.styluaignore @@ -0,0 +1 @@ +cartesi/third-party/* diff --git a/src/Makefile b/src/Makefile index eff7a414c..595bca814 100644 --- a/src/Makefile +++ b/src/Makefile @@ -14,11 +14,11 @@ # with this program (see COPYING). If not, see . # -EMULATOR_MARCHID=19 +EMULATOR_MARCHID=20 # Every new emulator release should bump these constants EMULATOR_VERSION_MAJOR=0 -EMULATOR_VERSION_MINOR=19 +EMULATOR_VERSION_MINOR=20 EMULATOR_VERSION_PATCH=0 EMULATOR_VERSION_LABEL= @@ -61,8 +61,8 @@ PICCFLAGS=-fPIC SOLDFLAGS=-dynamiclib -undefined dynamic_lookup LIBLDFLAGS=-dynamiclib EXELDFLAGS= -PTHREAD_CFLAGS= -PTHREAD_LDFLAGS=-lpthread +PTHREAD_CFLAGS=-I/opt/local/include/libomp -fopenmp +PTHREAD_LDFLAGS=-L/opt/local/lib/libomp -lgomp CC=clang CXX=clang++ AR=libtool -static -o @@ -86,8 +86,8 @@ SLIRP_INC=-I$(BREW_PREFIX)/libslirp/include # Macports installation else ifneq (,$(PORT_PREFIX)) INSTALL_PREFIX=/opt/local -BOOST_LIB_DIR=-L$(INSTALL_PREFIX)/libexec/boost/1.81/lib -BOOST_INC=-I$(INSTALL_PREFIX)/libexec/boost/1.81/include +BOOST_LIB_DIR=-L$(INSTALL_PREFIX)/libexec/boost/1.87/lib +BOOST_INC=-I$(INSTALL_PREFIX)/libexec/boost/1.87/include SLIRP_LIB=-L$(INSTALL_PREFIX)/lib -lslirp SLIRP_INC=-I$(INSTALL_PREFIX)/include @@ -97,7 +97,7 @@ endif SO_EXT=dylib LIBCARTESI_LDFLAGS=-install_name '@rpath/$(LIBCARTESI)' -LIBCARTESI_MERKLE_TREE_LDFLAGS=-install_name '@rpath/$(LIBCARTESI_MERKLE_TREE)' +LIBCARTESI_HASH_TREE_LDFLAGS=-install_name '@rpath/$(LIBCARTESI_HASH_TREE)' LIBCARTESI_JSONRPC_LDFLAGS=-install_name '@rpath/$(LIBCARTESI_JSONRPC)' -Wl,-rpath,@loader_path LUACARTESI_LDFLAGS=-install_name '@rpath/cartesi.so' LUACARTESI_JSONRPC_LDFLAGS=-install_name '@rpath/cartesi/jsonrpc.so' -Wl,-rpath,@loader_path/.. @@ -111,8 +111,8 @@ PICCFLAGS=-fPIC SOLDFLAGS=-shared $(PICCFLAGS) $(GCLDFLAGS) LIBLDFLAGS=$(SOLDFLAGS) -Wl,--no-undefined EXELDFLAGS=$(GCLDFLAGS) -Wl,--no-undefined -PTHREAD_CFLAGS=-pthread -PTHREAD_LDFLAGS=-pthread -lpthread +PTHREAD_CFLAGS=-fopenmp +PTHREAD_LDFLAGS=-fopenmp CC=gcc CXX=g++ AR=ar rcs @@ -123,7 +123,7 @@ SLIRP_INC= SLIRP_LIB=-lslirp SO_EXT=so LIBCARTESI_LDFLAGS= -LIBCARTESI_MERKLE_TREE_LDFLAGS= +LIBCARTESI_HASH_TREE_LDFLAGS= LIBCARTESI_JSONRPC_LDFLAGS=-Wl,-rpath,'$$ORIGIN' LUACARTESI_LDFLAGS= LUACARTESI_JSONRPC_LDFLAGS=-Wl,-rpath,'$$ORIGIN/..' @@ -132,15 +132,15 @@ PROFILE_DATA= endif LIBCARTESI=libcartesi-$(EMULATOR_VERSION_MAJOR).$(EMULATOR_VERSION_MINOR).$(SO_EXT) -LIBCARTESI_MERKLE_TREE=libcartesi_merkle_tree-$(EMULATOR_VERSION_MAJOR).$(EMULATOR_VERSION_MINOR).$(SO_EXT) +LIBCARTESI_HASH_TREE=libcartesi_hash_tree-$(EMULATOR_VERSION_MAJOR).$(EMULATOR_VERSION_MINOR).$(SO_EXT) LIBCARTESI_JSONRPC=libcartesi_jsonrpc-$(EMULATOR_VERSION_MAJOR).$(EMULATOR_VERSION_MINOR).$(SO_EXT) ifeq ($(slirp),yes) # Workaround for building with macports lua-luarocks installation -machine.o: INCS+=$(SLIRP_INC) -machine.clang-tidy: INCS+=$(SLIRP_INC) -virtio-net-carrier-slirp.o: INCS+=$(SLIRP_INC) -virtio-net-carrier-slirp.clang-tidy: INCS+=$(SLIRP_INC) +machine-address-ranges.o: INCS+=$(SLIRP_INC) +machine-address-ranges.clang-tidy: INCS+=$(SLIRP_INC) +virtio-net-user-address-range.o: INCS+=$(SLIRP_INC) +virtio-net-user-address-range.clang-tidy: INCS+=$(SLIRP_INC) #INCS+=$(SLIRP_INC) LIBCARTESI_COMMON_LIBS+=$(SLIRP_LIB) else @@ -148,22 +148,23 @@ DEFS+=-DNO_SLIRP endif LIBCARTESI_LIBS=$(LIBCARTESI_COMMON_LIBS) -LIBCARTESI_MERKLE_TREE_LIBS= +LIBCARTESI_HASH_TREE_LIBS= LIBCARTESI_JSONRPC_LIBS= LUACARTESI_LIBS=$(LIBCARTESI_COMMON_LIBS) LUACARTESI_JSONRPC_LIBS= CARTESI_JSONRPC_MACHINE_LIBS=$(LIBCARTESI_COMMON_LIBS) -CARTESI_MERKLE_TREE_HASH_LIBS= +CARTESI_HASH_TREE_HASH_LIBS= -#DEFS+= -DMT_ALL_DIRTY +C_WARNS=-Wall -Wextra -Wpedantic +# C_WARNS+=-Wshadow -Wconversion -Wsign-conversion -Wsign-promo -Wuseless-cast -Wformat=2 +CXX_WARNS=$(C_WARNS) -WARNS=-Wall -Wextra -Wpedantic -CLANG_TIDY_WARNS=-Wthread-safety -Wglobal-constructors +CLANG_TIDY_WARNS=-Wthread-safety -Wglobal-constructors -Wundef -Wredundant-decls -Wextra-semi # Place our include directories before the system's INCS+= \ + -I../third-party/ankerl \ -I../third-party/llvm-flang-uint128 \ - -I../third-party/tiny_sha3 \ -I../third-party/nlohmann-json \ -I../third-party/downloads \ $(BOOST_INC) @@ -175,25 +176,35 @@ DEFS+=-D_FILE_OFFSET_BITS=64 DEFS+=-DJSON_HAS_FILESYSTEM=0 ifeq ($(dump),yes) -#DEFS+=-DDUMP_ILLEGAL_INSN_EXCEPTIONS -#DEFS+=-DDUMP_EXCEPTIONS -#DEFS+=-DDUMP_INTERRUPTS -DEFS+=-DDUMP_HIST -#DEFS+=-DDUMP_MMU_EXCEPTIONS -#DEFS+=-DDUMP_INVALID_MEM_ACCESS -#DEFS+=-DDUMP_INVALID_CSR -#DEFS+=-DDUMP_INSN -#DEFS+=-DDUMP_REGS -#DEFS+=-DDUMP_COUNTERS +DUMP_DEFS+=-DDUMP_HASH_TREE_STATS +DUMP_DEFS+=-DDUMP_ILLEGAL_INSN_EXCEPTIONS +DUMP_DEFS+=-DDUMP_EXCEPTIONS +DUMP_DEFS+=-DDUMP_INTERRUPTS +DUMP_DEFS+=-DDUMP_MMU_EXCEPTIONS +DUMP_DEFS+=-DDUMP_INVALID_CSR +DUMP_DEFS+=-DDUMP_REGS +DUMP_DEFS+=-DDUMP_INSN_HIST +DUMP_DEFS+=-DDUMP_STATS +DUMP_DEFS+=-DDUMP_INSN +DUMP_DEFS+=-DDUMP_UARCH_INSN +DUMP_DEFS+=-DDUMP_SCOPED_NOTE +DUMP_DEFS+=-DDUMP_STATE_ACCESS +DUMP_DEFS+=-DDUMP_UARCH_STATE_ACCESS endif +DEFS += $(DUMP_DEFS) +# Pass down UARCH_DEFS to sub-makefiles +export UARCH_DEFS += $(DUMP_DEFS) # By default we compile in release with debug information, # so the emulator is packaged correctly by default. -ifeq (,$(filter yes,$(relwithdebinfo) $(release) $(debug) $(sanitize))) +ifeq (,$(filter yes,$(relwithdebinfo) $(release) $(debug) $(coverage) $(sanitize))) relwithdebinfo=yes endif -ifeq ($(relwithdebinfo),yes) +ifeq ($(coverage),yes) +OPTFLAGS+=-Og -g -fno-omit-frame-pointer -fno-dce -fno-inline +DEFS+=-DCODE_COVERAGE +else ifeq ($(relwithdebinfo),yes) OPTFLAGS+=-O2 -g INTERPRET_CXXFLAGS+=-DNDEBUG # disable asserts only for interpret.cpp else ifeq ($(release),yes) @@ -213,8 +224,8 @@ ifneq ($(git_commit),) DEFS+=-DGIT_COMMIT='"$(git_commit)"' endif -# The SHA3 is third party library we always want to compile with O3 -SHA3_CFLAGS=-O3 +# Hashing libraries have special optimizations flags +HASH_CFLAGS=-O3 -DNDEBUG -funroll-loops -fno-stack-protector # Optimization flags for the interpreter ifneq (,$(filter yes,$(relwithdebinfo) $(release))) @@ -274,15 +285,13 @@ PGO_WORKLOAD=\ whetstone 25000 LINTER_IGNORE_SOURCES= -LINTER_IGNORE_HEADERS=interpret-jump-table.h LINTER_SOURCES=$(filter-out $(LINTER_IGNORE_SOURCES),$(strip $(wildcard *.cpp) $(wildcard *.c))) -LINTER_HEADERS=$(filter-out $(LINTER_IGNORE_HEADERS),$(strip $(wildcard *.hpp) $(wildcard *.h))) CLANG_TIDY=clang-tidy CLANG_TIDY_TARGETS=$(patsubst %.cpp,%.clang-tidy,$(patsubst %.c,%.clang-tidy,$(LINTER_SOURCES))) CLANG_FORMAT=clang-format -CLANG_FORMAT_UARCH_FILES:=$(wildcard ../uarch/*.cpp) +CLANG_FORMAT_UARCH_FILES:=$(wildcard ../uarch/*.cpp) $(wildcard ../uarch/*.h) CLANG_FORMAT_UARCH_FILES:=$(filter-out %uarch-printf%,$(strip $(CLANG_FORMAT_UARCH_FILES))) CLANG_FORMAT_FILES:=$(wildcard *.cpp) $(wildcard *.c) $(wildcard *.h) $(wildcard *.hpp) $(CLANG_FORMAT_UARCH_FILES) CLANG_FORMAT_IGNORE_FILES:=interpret-jump-table.h @@ -290,11 +299,10 @@ CLANG_FORMAT_FILES:=$(strip $(CLANG_FORMAT_FILES)) CLANG_FORMAT_FILES:=$(filter-out $(CLANG_FORMAT_IGNORE_FILES),$(strip $(CLANG_FORMAT_FILES))) STYLUA=stylua -STYLUA_FLAGS=--indent-type Spaces --collapse-simple-statement Always +STYLUA_FLAGS=--indent-type Spaces --collapse-simple-statement Always --respect-ignores EMPTY:= SPACE:=$(EMPTY) $(EMPTY) -CLANG_TIDY_HEADER_FILTER=$(CURDIR)/($(subst $(SPACE),|,$(LINTER_HEADERS))) ifeq ($(threads),yes) CFLAGS+=$(PTHREAD_CFLAGS) @@ -302,22 +310,24 @@ CXXFLAGS+=$(PTHREAD_CFLAGS) LDFLAGS+=$(PTHREAD_LDFLAGS) else DEFS+=-DNO_THREADS +C_WARNS+=-Wno-unknown-pragmas +CXX_WARNS+=-Wno-unknown-pragmas endif -CXXFLAGS+=$(OPTFLAGS) -std=gnu++20 -fvisibility=hidden -MMD $(PICCFLAGS) $(CC_MARCH) $(INCS) $(GCFLAGS) $(UBCFLAGS) $(DEFS) $(WARNS) -CFLAGS+=$(OPTFLAGS) -std=gnu99 -fvisibility=hidden -MMD $(PICCFLAGS) $(CC_MARCH) $(INCS) $(GCFLAGS) $(UBCFLAGS) $(DEFS) $(WARNS) +CXXFLAGS+=$(OPTFLAGS) -std=gnu++23 -fvisibility=hidden -MMD $(PICCFLAGS) $(CC_MARCH) $(INCS) $(GCFLAGS) $(UBCFLAGS) $(DEFS) $(CXX_WARNS) +CFLAGS+=$(OPTFLAGS) -std=gnu99 -fvisibility=hidden -MMD $(PICCFLAGS) $(CC_MARCH) $(INCS) $(GCFLAGS) $(UBCFLAGS) $(DEFS) $(C_WARNS) LDFLAGS+=$(UBLDFLAGS) ifeq ($(coverage),yes) ifeq ($(COVERAGE_TOOLCHAIN),gcc) CC=gcc CXX=g++ -CXXFLAGS+=-g -Og -fno-dce -fno-inline -DCODE_COVERAGE --coverage +CXXFLAGS+=--coverage LDFLAGS+=--coverage else ifeq ($(COVERAGE_TOOLCHAIN),clang) CC=clang CXX=clang++ -CXXFLAGS+=-g -O0 -DCODE_COVERAGE -fprofile-instr-generate -fcoverage-mapping +CXXFLAGS+=-fprofile-instr-generate -fcoverage-mapping LDFLAGS+=-fprofile-instr-generate -fcoverage-mapping else ifneq ($(COVERAGE_TOOLCHAIN),) $(error invalid value for COVERAGE_TOOLCHAIN: $(COVERAGE_TOOLCHAIN)) @@ -331,84 +341,77 @@ SOLDFLAGS+=$(MYSOLDFLAGS) LIBLDFLAGS+=$(MYLIBLDFLAGS) EXELDFLAGS+=$(MYEXELDFLAGS) -all: libcartesi libcartesi_merkle_tree libcartesi_jsonrpc c-api luacartesi cartesi-jsonrpc-machine hash +all: libcartesi libcartesi_hash_tree libcartesi_jsonrpc c-api luacartesi cartesi-jsonrpc-machine hash luacartesi: libluacartesi.a cartesi.so libluacartesi_jsonrpc.a cartesi/jsonrpc.so jsonrpc: cartesi/jsonrpc.so cartesi-jsonrpc-machine -hash: cartesi-merkle-tree-hash +hash: cartesi-hash-tree-hash -c-api: $(LIBCARTESI) $(LIBCARTESI_MERKLE_TREE) $(LIBCARTESI_JSONRPC) +c-api: $(LIBCARTESI) $(LIBCARTESI_HASH_TREE) $(LIBCARTESI_JSONRPC) .PHONY: all generate use clean lint format format-lua check-format check-format-lua luacartesi hash c-api compile_flags.txt LIBCARTESI_OBJS:= \ - pma-driver.o \ - clint.o \ - clint-factory.o \ - plic.o \ - plic-factory.o \ - virtio-factory.o \ - virtio-device.o \ - virtio-console.o \ - virtio-p9fs.o \ - virtio-net.o \ - virtio-net-carrier-tuntap.o \ - virtio-net-carrier-slirp.o \ - dtb.o \ - os.o \ - htif.o \ - htif-factory.o \ - shadow-state.o \ - shadow-state-factory.o \ - shadow-pmas-factory.o \ - shadow-tlb.o \ - shadow-tlb-factory.o \ - shadow-uarch-state.o \ - shadow-uarch-state-factory.o \ - pma.o \ - machine.o \ - machine-config.o \ - json-util.o \ base64.o \ + clint-address-range.o \ + dtb.o \ + hash-tree.o \ + htif-address-range.o \ interpret.o \ - virtual-machine.o \ - uarch-machine.o \ - uarch-step.o \ - uarch-reset-state.o \ - sha3.o \ - machine-merkle-tree.o \ - pristine-merkle-tree.o \ - uarch-interpret.o \ + json-util.o \ machine-c-api.o \ + machine-config.o \ + machine.o \ + machine-address-ranges.o \ + memory-address-range.o \ + os.o \ + os-mapped-memory.o \ + os-filesystem.o \ + plic-address-range.o \ + back-merkle-tree.o \ + replay-step-state-access-interop.o \ + send-cmio-response.o \ + keccak-256-hasher.o \ + sha-256-hasher.o \ + is-pristine.o \ + uarch-pristine-hash.o \ uarch-pristine-ram.o \ uarch-pristine-state-hash.o \ - uarch-pristine-hash.o \ - send-cmio-response.o \ - replay-step-state-access-interop.o + uarch-reset-state.o \ + uarch-step.o \ + local-machine.o \ + uarch-interpret.o \ + virtio-address-range.o \ + virtio-console-address-range.o \ + virtio-p9fs-address-range.o \ + virtio-net-address-range.o \ + virtio-net-tuntap-address-range.o \ + virtio-net-user-address-range.o CARTESI_CLUA_OBJS:= \ clua.o \ - clua-i-virtual-machine.o + clua-i-machine.o \ + uarch-pristine-ram.o \ + uarch-pristine-state-hash.o \ + uarch-pristine-hash.o LUACARTESI_OBJS:= \ clua-cartesi.o \ $(CARTESI_CLUA_OBJS) -LIBCARTESI_MERKLE_TREE_OBJS:= \ - sha3.o \ - machine-merkle-tree.o \ +LIBCARTESI_HASH_TREE_OBJS:= \ + keccak-256-hasher.o \ + sha-256-hasher.o \ + is-pristine.o \ back-merkle-tree.o \ - pristine-merkle-tree.o \ - complete-merkle-tree.o \ - full-merkle-tree.o -CARTESI_MERKLE_TREE_HASH_OBJS:= \ - merkle-tree-hash.o +CARTESI_HASH_TREE_HASH_OBJS:= \ + hash-tree-hash.o LIBCARTESI_JSONRPC_OBJS:= \ - jsonrpc-virtual-machine.o \ + jsonrpc-machine.o \ os.o \ jsonrpc-machine-c-api.o \ base64.o \ @@ -426,7 +429,7 @@ CARTESI_JSONRPC_MACHINE_OBJS:= \ ifeq ($(gperf),yes) DEFS+=-DGPERF LIBCARTESI_LIBS+=-lprofiler -LIBCARTESI_MERKLE_TREE_LIBS+=-lprofiler +LIBCARTESI_HASH_TREE_LIBS+=-lprofiler LIBCARTESI_JSONRPC_LIBS+=-lprofiler LUACARTESI_LIBS+=-lprofiler LUACARTESI_JSONRPC_LIBS+=-lprofiler @@ -439,10 +442,10 @@ so-version: @echo $(EMULATOR_VERSION_MAJOR).$(EMULATOR_VERSION_MINOR) libcartesi: libcartesi.a libcartesi.$(SO_EXT) -libcartesi.$(SO_EXT): $(LIBCARTESI) $(LIBCARTESI_MERKLE_TREE) +libcartesi.$(SO_EXT): $(LIBCARTESI) $(LIBCARTESI_HASH_TREE) ln -sf $< $@ -libcartesi_merkle_tree: libcartesi_merkle_tree.a libcartesi_merkle_tree.$(SO_EXT) -libcartesi_merkle_tree.$(SO_EXT): $(LIBCARTESI_MERKLE_TREE) +libcartesi_hash_tree: libcartesi_hash_tree.a libcartesi_hash_tree.$(SO_EXT) +libcartesi_hash_tree.$(SO_EXT): $(LIBCARTESI_HASH_TREE) ln -sf $< $@ libcartesi_jsonrpc: libcartesi_jsonrpc.a libcartesi_jsonrpc.$(SO_EXT) libcartesi_jsonrpc.$(SO_EXT): $(LIBCARTESI_JSONRPC) @@ -451,7 +454,7 @@ libcartesi_jsonrpc.$(SO_EXT): $(LIBCARTESI_JSONRPC) libcartesi.a: $(LIBCARTESI_OBJS) $(AR) $@ $^ -libcartesi_merkle_tree.a: $(LIBCARTESI_MERKLE_TREE_OBJS) +libcartesi_hash_tree.a: $(LIBCARTESI_HASH_TREE_OBJS) $(AR) $@ $^ libcartesi_jsonrpc.a: $(LIBCARTESI_JSONRPC_OBJS) @@ -463,8 +466,8 @@ libluacartesi.a: $(LUACARTESI_OBJS) libluacartesi_jsonrpc.a: $(LUACARTESI_JSONRPC_OBJS) $(AR) $@ $^ -$(LIBCARTESI_MERKLE_TREE): $(LIBCARTESI_MERKLE_TREE_OBJS) - $(CXX) -o $@ $^ $(LIBCARTESI_MERKLE_TREE_LIBS) $(LDFLAGS) $(LIBCARTESI_MERKLE_TREE_LDFLAGS) $(LIBLDFLAGS) +$(LIBCARTESI_HASH_TREE): $(LIBCARTESI_HASH_TREE_OBJS) + $(CXX) -o $@ $^ $(LIBCARTESI_HASH_TREE_LIBS) $(LDFLAGS) $(LIBCARTESI_HASH_TREE_LDFLAGS) $(LIBLDFLAGS) $(LIBCARTESI): $(LIBCARTESI_OBJS) $(CXX) -o $@ $^ $(LIBCARTESI_LIBS) $(LDFLAGS) $(LIBCARTESI_LDFLAGS) $(LIBLDFLAGS) @@ -527,8 +530,8 @@ $(PROFILE_DATA): llvm-profdata merge -output=default.profdata default*.profraw endif -cartesi-merkle-tree-hash: $(CARTESI_MERKLE_TREE_HASH_OBJS) libcartesi_merkle_tree.a - $(CXX) -o $@ $^ $(CARTESI_MERKLE_TREE_HASH_LIBS) $(LDFLAGS) $(EXELDFLAGS) +cartesi-hash-tree-hash: $(CARTESI_HASH_TREE_HASH_OBJS) libcartesi_hash_tree.a + $(CXX) -o $@ $^ $(CARTESI_HASH_TREE_HASH_LIBS) $(LDFLAGS) $(EXELDFLAGS) cartesi-jsonrpc-machine: $(CARTESI_JSONRPC_MACHINE_OBJS) libcartesi_jsonrpc.a libcartesi.a $(CXX) -o $@ $^ $(CARTESI_JSONRPC_MACHINE_LIBS) $(LDFLAGS) $(EXELDFLAGS) @@ -547,17 +550,15 @@ jsonrpc-discover.cpp: jsonrpc-discover.json echo '} // namespace cartesi' >> jsonrpc-discover.cpp %.clang-tidy: %.cpp machine-c-version.h interpret-jump-table.h - @$(CLANG_TIDY) --header-filter='$(CLANG_TIDY_HEADER_FILTER)' $(CLANG_TIDY_FLAGS) $< -- $(CXXFLAGS) $(CLANG_TIDY_WARNS) $(LUA_INC) -DCLANG_TIDY_LINT 2>/dev/null + @$(CLANG_TIDY) $(CLANG_TIDY_FLAGS) $< -- $(CXXFLAGS) $(CLANG_TIDY_WARNS) $(LUA_INC) -DCLANG_TIDY_LINT 2>/dev/null @$(CXX) $(CXXFLAGS) $(LUA_INC) $< -MM -MT $@ -MF $@.d > /dev/null 2>&1 @touch $@ %.clang-tidy: %.c - @$(CLANG_TIDY) --header-filter='$(CLANG_TIDY_HEADER_FILTER)' $(CLANG_TIDY_FLAGS) $< -- $(CFLAGS) $(CLANG_TIDY_WARNS) -DCLANG_TIDY_LINT 2>/dev/null + @$(CLANG_TIDY) $(CLANG_TIDY_FLAGS) $< -- $(CFLAGS) $(CLANG_TIDY_WARNS) -DCLANG_TIDY_LINT 2>/dev/null @$(CC) $(CFLAGS) $< -MM -MT $@ -MF $@.d > /dev/null 2>&1 @touch $@ -sha3.o: ../third-party/tiny_sha3/sha3.c - $(CC) $(CFLAGS) $(SHA3_CFLAGS) -c -o $@ $< uarch-pristine-ram.o: $(UARCH_PRISTINE_RAM_C) $(CC) $(CFLAGS) -c -o $@ $< @@ -571,6 +572,15 @@ interpret-jump-table.h: ../tools/gen-interpret-jump-table.lua interpret.o: interpret.cpp machine-c-version.h interpret-jump-table.h $(CXX) $(CXXFLAGS) $(INTERPRET_CXXFLAGS) -c -o $@ $< +keccak-256-hasher.o: keccak-256-hasher.cpp + $(CXX) $(CXXFLAGS) $(HASH_CFLAGS) -c -o $@ $< + +sha-256-hasher.o: sha-256-hasher.cpp + $(CXX) $(CXXFLAGS) $(HASH_CFLAGS) -c -o $@ $< + +is-pristine.o: is-pristine.cpp + $(CXX) $(CXXFLAGS) $(HASH_CFLAGS) -c -o $@ $< + %.o: %.cpp machine-c-version.h $(CXX) $(CXXFLAGS) -c -o $@ $< @@ -603,7 +613,7 @@ clean-libcartesi: clean-objs @rm -f *.so *.a cartesi/*.so *.dylib clean-executables: - @rm -f cartesi-jsonrpc-machine cartesi-merkle-tree-hash compute-uarch-pristine-hash + @rm -f cartesi-jsonrpc-machine cartesi-hash-tree-hash compute-uarch-pristine-hash clean-coverage: @rm -f *.profdata *.profraw *.gcda *.gcov coverage.info coverage.txt diff --git a/src/access-log.h b/src/access-log.h index 97d439a77..0c1b126b8 100644 --- a/src/access-log.h +++ b/src/access-log.h @@ -21,9 +21,7 @@ /// \brief State access log implementation #include -#include #include -#include #include #include #include @@ -32,9 +30,12 @@ #include +#include "assert-printf.h" #include "bracket-note.h" +#include "hash-tree-constants.h" +#include "hash-tree.h" #include "machine-c-api.h" -#include "machine-merkle-tree.h" +#include "machine-hash.h" #include "strict-aliasing.h" namespace cartesi { @@ -54,12 +55,12 @@ static inline void set_word_access_data(uint64_t w, access_data &ad) { ad.insert(ad.end(), p, p + sizeof(w)); } -static inline void replace_word_access_data(uint64_t w, access_data &ad, int offset = 0) { +static inline void replace_word_access_data(uint64_t w, access_data &ad, uint64_t offset = 0) { assert(ad.size() >= offset + sizeof(uint64_t)); aliased_aligned_write(ad.data() + offset, w); } -static inline uint64_t get_word_access_data(const access_data &ad, int offset = 0) { +static inline uint64_t get_word_access_data(const access_data &ad, uint64_t offset = 0) { assert(ad.size() >= offset + sizeof(uint64_t)); return aliased_aligned_read(ad.data() + offset); } @@ -67,12 +68,9 @@ static inline uint64_t get_word_access_data(const access_data &ad, int offset = /// \brief Records an access to the machine state class access { - using hasher_type = machine_merkle_tree::hasher_type; - public: - using hash_type = machine_merkle_tree::hash_type; - using sibling_hashes_type = std::vector; - using proof_type = machine_merkle_tree::proof_type; + using proof_type = hash_tree::proof_type; + using sibling_hashes_type = hash_tree::sibling_hashes_type; void set_type(access_type type) { m_type = type; @@ -143,43 +141,43 @@ class access { /// \brief Sets hash of data that was written at address after access. /// \param hash Hash of new data at address. - void set_written_hash(const hash_type &hash) { + void set_written_hash(const machine_hash &hash) { m_written_hash = hash; } /// \brief Gets hash of data that was written at address after access. /// \returns Hash of written data at address. - const std::optional &get_written_hash() const { + const std::optional &get_written_hash() const { return m_written_hash; } - std::optional &get_written_hash() { + std::optional &get_written_hash() { return m_written_hash; } /// \brief Sets hash of data that can be read at address before access. /// \param hash Hash of data at address. - void set_read_hash(const hash_type &hash) { + void set_read_hash(const machine_hash &hash) { m_read_hash = hash; } /// \brief Gets hash of data that can be read at address before access. /// \returns Hash of data at address. - const hash_type &get_read_hash() const { + const machine_hash &get_read_hash() const { return m_read_hash; } - hash_type &get_read_hash() { + machine_hash &get_read_hash() { return m_read_hash; } /// \brief Constructs a proof using this access' data and a given root hash. /// \param root_hash Hash to be used as the root of the proof. /// \return The corresponding proof - proof_type make_proof(const hash_type root_hash) const { - // the access can be of data smaller than the merkle tree word size - // however, the proof must be at least as big as the merkle tree word size - const int proof_log2_size = std::max(m_log2_size, machine_merkle_tree::get_log2_word_size()); - // the proof address is the access address aligned to the merkle tree word size - const uint64_t proof_address = m_address & ~(machine_merkle_tree::get_word_size() - 1); + proof_type make_proof(const machine_hash root_hash) const { + // the access can be of data smaller than the hash tree word size + // however, the proof must be at least as big as the hash tree word size + const int proof_log2_size = std::max(m_log2_size, HASH_TREE_LOG2_WORD_SIZE); + // the proof address is the access address aligned to the hash tree word size + const uint64_t proof_address = m_address & ~(HASH_TREE_WORD_SIZE - 1); if (!m_sibling_hashes.has_value()) { throw std::runtime_error("can't make proof if access doesn't have sibling hashes"); } @@ -218,9 +216,9 @@ class access { uint64_t m_address{0}; ///< Address of access int m_log2_size{0}; ///< Log2 of size of access std::optional m_read; ///< Data before access - hash_type m_read_hash{}; ///< Hash of data before access + machine_hash m_read_hash{}; ///< Hash of data before access std::optional m_written; ///< Written data - std::optional m_written_hash; ///< Hash of written data + std::optional m_written_hash; ///< Hash of written data std::optional m_sibling_hashes; ///< Hashes of siblings in path from address to root }; @@ -258,13 +256,14 @@ class access_log { }; private: - std::vector m_accesses; ///< List of all accesses - std::vector m_brackets; ///< Begin/End annotations - std::vector m_notes; ///< Per-access annotations - type m_log_type; ///< Log type + std::vector m_accesses; ///< List of all accesses + std::vector m_brackets; ///< Begin/End annotations + std::vector m_notes; ///< Per-access annotations + type m_log_type; ///< Log type + std::vector::size_type m_outstanding_ends; ///< Number of outstanding unmatched end brackets public: - explicit access_log(type log_type) : m_log_type(log_type) { + explicit access_log(type log_type) : m_log_type(log_type), m_outstanding_ends{0} { ; } @@ -273,8 +272,16 @@ class access_log { m_accesses(std::forward(accesses)), m_brackets(std::forward(brackets)), m_notes(std::forward(notes)), - m_log_type(log_type) { - ; + m_log_type(log_type), + m_outstanding_ends(0) { + for (const auto &b : m_brackets) { + if (b.type == bracket_type::begin) { + ++m_outstanding_ends; + } + if (b.type == bracket_type::end && m_outstanding_ends > 0) { + --m_outstanding_ends; + } + }; } /// \brief Clear the log @@ -282,28 +289,43 @@ class access_log { m_accesses.clear(); m_notes.clear(); m_brackets.clear(); + m_outstanding_ends = 0; } /// \brief Adds a bracket annotation to the log (if the log type includes annotations) /// \param type Bracket type /// \param text Annotation contents - void push_bracket(bracket_type type, const char *text) { + void push_begin_bracket(const char *text) { if (m_log_type.has_annotations()) { - if (type == bracket_type::begin) { - // make sure we have room for end bracket as well. that way, - // unless the user use unbalanced brackets, there is no way we - // would throw an exception for lack of memory on and end bracket - m_brackets.reserve(m_brackets.size() + 2); + // Increment number of outstanding end brackets we are expecting + ++m_outstanding_ends; + // Make sure we have room for the matching end bracket as well. + // That way, unless the user is messing with unbalanced brackets, there is no way we + // would throw an exception for lack of memory on the matching end bracket + m_brackets.push_back(bracket_note{.type = bracket_type::begin, .where = m_accesses.size(), .text = text}); + m_brackets.reserve(m_brackets.size() + m_outstanding_ends); + } + } + + void push_end_bracket(const char *text) noexcept { + if (m_log_type.has_annotations()) { + // If we failed to push, it was because the system is completely screwed anyway *and* the + // user is using unbalanced brackets. Therefore, it's OK to quietly ignore the error. + try { + m_brackets.push_back(bracket_note{.type = bracket_type::end, .where = m_accesses.size(), .text = text}); + } catch (...) { // NOLINT(bugprone-empty-catch) + } + // Decrement number of outstanding end brackets we are expecting + if (m_outstanding_ends > 0) { + --m_outstanding_ends; } - m_brackets.push_back(bracket_note{.type = type, .where = m_accesses.size(), .text = text}); } } /// \brief Adds a new access to the log /// \tparam A Type of access /// \param a Access object - /// \param text Annotation contents (added if the log - /// type includes annotations, ignored otherwise) + /// \param text Annotation contents (added if the log type includes annotations, ignored otherwise) template void push_access(A &&a, const char *text) { m_accesses.push_back(std::forward(a)); diff --git a/src/address-range-constants.h b/src/address-range-constants.h new file mode 100644 index 000000000..f09a7d47c --- /dev/null +++ b/src/address-range-constants.h @@ -0,0 +1,85 @@ +// Copyright Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: LGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify it under +// the terms of the GNU Lesser General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) any +// later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +// PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License along +// with this program (see COPYING). If not, see . +// + +#ifndef ADDRESS_RANGE_CONSTANTS_H +#define ADDRESS_RANGE_CONSTANTS_H + +#include + +#include "address-range-defines.h" + +namespace cartesi { + +/// \brief Fixed address ranges. +enum AR_ranges : uint64_t { + AR_SHADOW_STATE_START = EXPAND_UINT64_C(AR_SHADOW_STATE_START_DEF), ///< Start of shadow state range + AR_SHADOW_STATE_LENGTH = EXPAND_UINT64_C(AR_SHADOW_STATE_LENGTH_DEF), ///< Length of shadow state range + AR_SHADOW_REGISTERS_START = EXPAND_UINT64_C(AR_SHADOW_REGISTERS_START_DEF), ///< Start of shadow registers range + AR_SHADOW_REGISTERS_LENGTH = EXPAND_UINT64_C(AR_SHADOW_REGISTERS_LENGTH_DEF), ///< Length of shadow registers range + AR_SHADOW_REVERT_ROOT_HASH_START = + EXPAND_UINT64_C(AR_SHADOW_REVERT_ROOT_HASH_START_DEF), ///< Start of revert root hash range + AR_SHADOW_TLB_START = EXPAND_UINT64_C(AR_SHADOW_TLB_START_DEF), ///< Start of shadow TLB range + AR_SHADOW_TLB_LENGTH = EXPAND_UINT64_C(AR_SHADOW_TLB_LENGTH_DEF), ///< Length of shadow TLB range + AR_PMAS_START = EXPAND_UINT64_C(AR_PMAS_START_DEF), ///< Start of PMAS list range + AR_PMAS_LENGTH = EXPAND_UINT64_C(AR_PMAS_LENGTH_DEF), ///< Length of PMAS list range + AR_DTB_START = EXPAND_UINT64_C(AR_DTB_START_DEF), ///< Start of DTB range + AR_DTB_LENGTH = EXPAND_UINT64_C(AR_DTB_LENGTH_DEF), ///< Length of DTB range + AR_SHADOW_UARCH_STATE_START = + EXPAND_UINT64_C(AR_SHADOW_UARCH_STATE_START_DEF), ///< Start of uarch shadow state range + AR_SHADOW_UARCH_STATE_LENGTH = + EXPAND_UINT64_C(AR_SHADOW_UARCH_STATE_LENGTH_DEF), ///< Length of uarch shadow state range + AR_CLINT_START = EXPAND_UINT64_C(AR_CLINT_START_DEF), ///< Start of CLINT range + AR_CLINT_LENGTH = EXPAND_UINT64_C(AR_CLINT_LENGTH_DEF), ///< Length of CLINT range + AR_PLIC_START = EXPAND_UINT64_C(AR_PLIC_START_DEF), ///< Start of PLIC range + AR_PLIC_LENGTH = EXPAND_UINT64_C(AR_PLIC_LENGTH_DEF), ///< Length of PLIC range + AR_HTIF_START = EXPAND_UINT64_C(AR_HTIF_START_DEF), ///< Start of HTIF range + AR_HTIF_LENGTH = EXPAND_UINT64_C(AR_HTIF_LENGTH_DEF), ///< Length of HTIF range + AR_UARCH_RAM_START = EXPAND_UINT64_C(AR_UARCH_RAM_START_DEF), ///< Start of uarch RAM range + AR_UARCH_RAM_LENGTH = EXPAND_UINT64_C(AR_UARCH_RAM_LENGTH_DEF), ///< Length of uarch RAM range + AR_CMIO_RX_BUFFER_START = EXPAND_UINT64_C(AR_CMIO_RX_BUFFER_START_DEF), ///< Start of CMIO RX buffer range + AR_CMIO_RX_BUFFER_LOG2_SIZE = EXPAND_UINT64_C(AR_CMIO_RX_BUFFER_LOG2_SIZE_DEF), ///< Log2 of CMIO RX buffer range + AR_CMIO_RX_BUFFER_LENGTH = (UINT64_C(1) << AR_CMIO_RX_BUFFER_LOG2_SIZE_DEF), ///< Length of CMIO RX buffer range + AR_CMIO_TX_BUFFER_START = EXPAND_UINT64_C(AR_CMIO_TX_BUFFER_START_DEF), ///< Start of CMIO TX buffer range + AR_CMIO_TX_BUFFER_LOG2_SIZE = EXPAND_UINT64_C(AR_CMIO_TX_BUFFER_LOG2_SIZE_DEF), ///< Log2 of CMIO TX buffer range + AR_CMIO_TX_BUFFER_LENGTH = (UINT64_C(1) << AR_CMIO_TX_BUFFER_LOG2_SIZE_DEF), ///< Length of CMIO TX buffer range + AR_DRIVE_START = EXPAND_UINT64_C(AR_DRIVE_START_DEF), ///< Start address for flash drive ranges + AR_DRIVE_OFFSET = EXPAND_UINT64_C(AR_DRIVE_OFFSET_DEF), ///< Offset for extra flash drive ranges + + AR_FIRST_VIRTIO_START = EXPAND_UINT64_C(AR_FIRST_VIRTIO_START_DEF), ///< Start of first VIRTIO range + AR_VIRTIO_LENGTH = EXPAND_UINT64_C(AR_VIRTIO_LENGTH_DEF), ///< Length of each VIRTIO range + AR_LAST_VIRTIO_END = EXPAND_UINT64_C(AR_LAST_VIRTIO_END_DEF), ///< End of last VIRTIO range + + AR_RAM_START = EXPAND_UINT64_C(AR_RAM_START_DEF), ///< Start of RAM range +}; + +static_assert(AR_SHADOW_STATE_LENGTH >= AR_SHADOW_REGISTERS_LENGTH + AR_SHADOW_TLB_LENGTH); +static_assert(AR_SHADOW_TLB_START == AR_SHADOW_REGISTERS_START + AR_SHADOW_REGISTERS_LENGTH); +static_assert(AR_SHADOW_STATE_START == AR_SHADOW_REGISTERS_START); + +/// \brief PMA constants. +enum AR_constants : uint64_t { + AR_LOG2_PAGE_SIZE = EXPAND_UINT64_C(AR_LOG2_PAGE_SIZE_DEF), ///< Log2 of physical memory page size. + AR_PAGE_SIZE = (UINT64_C(1) << AR_LOG2_PAGE_SIZE_DEF), ///< Physical memory page size. +}; + +/// \brief PMA masks. +enum AR_masks : uint64_t { + AR_ADDRESSABLE_MASK = ((UINT64_C(1) << 56) - 1) ///< Mask for addressable ranges. +}; + +} // namespace cartesi + +#endif diff --git a/src/address-range-defines.h b/src/address-range-defines.h new file mode 100644 index 000000000..c4366c278 --- /dev/null +++ b/src/address-range-defines.h @@ -0,0 +1,62 @@ +// Copyright Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: LGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify it under +// the terms of the GNU Lesser General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) any +// later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +// PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License along +// with this program (see COPYING). If not, see . +// + +#ifndef AR_DEFINES_H +#define AR_DEFINES_H + +// NOLINTBEGIN(cppcoreguidelines-macro-usage,cppcoreguidelines-macro-to-enum,modernize-macro-to-enum) +#define AR_SHADOW_STATE_START_DEF 0x0 ///< Shadow start address +#define AR_SHADOW_STATE_LENGTH_DEF 0x8000 ///< Shadow length in bytes +#define AR_SHADOW_REGISTERS_START_DEF 0x0 ///< Shadow registers start address +#define AR_SHADOW_REGISTERS_LENGTH_DEF 0x1000 ///< Shadow registers length in bytes +#define AR_SHADOW_REVERT_ROOT_HASH_START_DEF 0xfe0 ///< Shadow revert root hash start address +#define AR_SHADOW_TLB_START_DEF 0x1000 ///< Shadow TLB start address +#define AR_SHADOW_TLB_LENGTH_DEF 0x6000 ///< Shadow TLB length in bytes +#define AR_PMAS_START_DEF 0x10000 ///< PMA Array start address +#define AR_PMAS_LENGTH_DEF 0x1000 ///< PMA Array length in bytes +#define AR_SHADOW_UARCH_STATE_START_DEF 0x400000 ///< microarchitecture shadow state start address +#define AR_SHADOW_UARCH_STATE_LENGTH_DEF 0x1000 ///< microarchitecture shadow state length +#define AR_UARCH_RAM_START_DEF 0x600000 ///< microarchitecture RAM start address +#define AR_UARCH_RAM_LENGTH_DEF 0x200000 ///< microarchitecture RAM length +#define AR_CLINT_START_DEF 0x2000000 ///< CLINT start address +#define AR_CLINT_LENGTH_DEF 0xC0000 ///< CLINT length in bytes +#define AR_PLIC_START_DEF 0x40100000 ///< Start of PLIC range +#define AR_PLIC_LENGTH_DEF 0x00400000 ///< Length of PLIC range +#define AR_HTIF_START_DEF 0x40008000 ///< HTIF base address (to_host) +#define AR_HTIF_LENGTH_DEF 0x1000 ///< HTIF length in bytes +#define AR_FIRST_VIRTIO_START_DEF 0x40010000 ///< Start of first VIRTIO range +#define AR_VIRTIO_LENGTH_DEF 0x1000 ///< Length of each VIRTIO range +#define AR_LAST_VIRTIO_END_DEF 0x40020000 ///< End of last VIRTIO range +#define AR_DTB_START_DEF 0x7ff00000 ///< DTB start address +#define AR_DTB_LENGTH_DEF 0x100000 ///< DTB length in bytes +#define AR_CMIO_RX_BUFFER_START_DEF 0x60000000 ///< CMIO RX buffer start address +#define AR_CMIO_RX_BUFFER_LOG2_SIZE_DEF 21 ///< log2 of CMIO RX buffer length in bytes +#define AR_CMIO_TX_BUFFER_START_DEF 0x60800000 ///< CMIO TX buffer start address +#define AR_CMIO_TX_BUFFER_LOG2_SIZE_DEF 21 ///< log2 of CMIO TX buffer length in bytes +#define AR_DRIVE_START_DEF 0x80000000000000 ///< Start PMA address for flash drives +#define AR_DRIVE_OFFSET_DEF 0x10000000000000 ///< PMA offset for extra flash drives + +#define AR_RAM_START_DEF 0x80000000 ///< RAM start address + +#define AR_LOG2_PAGE_SIZE_DEF 12 ///< log2 of physical memory page size. + +// helper for using UINT64_C with defines +#ifndef EXPAND_UINT64_C +#define EXPAND_UINT64_C(a) UINT64_C(a) +#endif + +// NOLINTEND(cppcoreguidelines-macro-usage,cppcoreguidelines-macro-to-enum,modernize-macro-to-enum) +#endif /* end of include guard: AR_DEFINES_H */ diff --git a/src/machine-memory-range-descr.h b/src/address-range-description.h similarity index 74% rename from src/machine-memory-range-descr.h rename to src/address-range-description.h index 787275b9d..325425b98 100644 --- a/src/machine-memory-range-descr.h +++ b/src/address-range-description.h @@ -14,8 +14,8 @@ // with this program (see COPYING). If not, see . // -#ifndef MACHINE_MEMORY_RANGE_DESCR_H -#define MACHINE_MEMORY_RANGE_DESCR_H +#ifndef ADDRESS_RANGE_DESCRIPTION_H +#define ADDRESS_RANGE_DESCRIPTION_H #include #include @@ -23,15 +23,15 @@ namespace cartesi { -/// \brief Description of memory range used for introspection (i.e., get_memory_ranges()) -struct machine_memory_range_descr { +/// \brief Description of an address range used for introspection (i.e., get_address_ranges()) +struct address_range_description { uint64_t start = 0; ///< Start of memory range uint64_t length = 0; ///< Length of memory range std::string description; ///< User-friendly description for memory range }; -/// \brief List of memory range descriptions used for introspection (i.e., get_memory_ranges()) -using machine_memory_range_descrs = std::vector; +/// \brief List of address range descriptions used for introspection (i.e., get_address_ranges()) +using address_range_descriptions = std::vector; } // namespace cartesi diff --git a/src/address-range.h b/src/address-range.h new file mode 100644 index 000000000..b6c82cec7 --- /dev/null +++ b/src/address-range.h @@ -0,0 +1,394 @@ +// Copyright Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: LGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify it under +// the terms of the GNU Lesser General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) any +// later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +// PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License along +// with this program (see COPYING). If not, see . +// + +#ifndef ADDRESS_RANGE_H +#define ADDRESS_RANGE_H + +#include +#include +#include +#include +#include +#include +#include + +#include "address-range-constants.h" +#include "assert-printf.h" +#include "i-device-state-access.h" +#include "interpret.h" +#include "pmas-constants.h" +#include "pmas.h" + +#ifndef MICROARCHITECTURE +#include "i-dense-hash-tree.h" +#include "i-dirty-page-tree.h" +#endif + +namespace cartesi { + +/// \file +/// \brief Physical address range + +/// \brief Physical Address Range. +/// \details The target's physical address layout is described by an array of specializations of such ranges. +class address_range { + + std::array m_description; ///< Description of address range for use in error messages. + uint64_t m_start; ///< Target physical address where range starts. + uint64_t m_end; ///< Target physical address where range ends. + pmas_flags m_flags; ///< Physical memory attribute flags for range. + +public: + /// \brief Noexcept constexpr constructor for empty ranges with description + /// \detail Can be used to initialize a constexpr empty range + template + explicit constexpr address_range(const char (&description)[N]) noexcept : + m_description{}, + m_start{0}, + m_end{0}, + m_flags{} { + for (unsigned i = 0; i < std::min(N, m_description.size() - 1); ++i) { + m_description[i] = description[i]; + } + } + + // NOLINTNEXTLINE(hicpp-use-equals-default,modernize-use-equals-default) + constexpr virtual ~address_range() {} // = default; // doesn't work due to bug in gcc + + template + [[noreturn]] + static void ABRTF(ABRT abrt, const char (&fmt)[N], ARGS... args) { + char buf[256]{}; + std::ignore = snprintf(buf, std::size(buf), fmt, args...); + abrt(buf); + __builtin_trap(); + } + + /// \brief Constructor + /// \tparam ABRT type of function used to abort and report errors + /// \param description Description of address range for use in error messages (will be copied) + /// \param start Target physical address where range starts + /// \param length Length of range, in bytes + /// \param f Physical memory attribute flags for range + template + constexpr address_range(const char *description, uint64_t start, uint64_t length, const pmas_flags &flags, + ABRT abrt) : + m_description{}, + m_start{start}, + m_end{start + length}, + m_flags{flags} { + // Non-empty description is mandatory + if (description == nullptr || *description == '\0') { + ABRTF(abrt, "address range 0x%" PRIx64 ":0x%" PRIx64 " has empty description", start, length); + } + for (unsigned i = 0; i < m_description.size() - 1 && description[i] != '\0'; ++i) { + m_description[i] = description[i]; + } + // End = start + length cannot overflow + if (start >= UINT64_MAX - length) { + ABRTF(abrt, "0x%" PRIx64 ":0x%" PRIx64 " is out of bounds when initializing %s", start, length, + description); + } + // All address ranges must be page-aligned + if ((m_start & ~PMA_ISTART_START_MASK) != 0) { + ABRTF(abrt, "start of %s (0x%" PRIx64 ") must be aligned to page boundary (every %" PRId64 " bytes)", + description, start, AR_PAGE_SIZE); + } + if ((m_end & ~PMA_ISTART_START_MASK) != 0) { + ABRTF(abrt, "length of %s (0x% " PRIx64 ") must be multiple of page length (%" PRId64 " bytes)", + description, length, AR_PAGE_SIZE); + } + // Empty range must really be empty + if (length == 0) { + if (start != 0) { + ABRTF(abrt, "empty range with length 0 must start at 0 when initializing %s", description); + } + if (get_istart() != 0) { + ABRTF(abrt, "empty range must have clear flags when initializing %s", description); + } + } + } + + address_range(const address_range &other) = default; + address_range &operator=(const address_range &other) = default; + address_range(address_range &&other) = default; + address_range &operator=(address_range &&other) = default; + + /// \brief Checks if a range of addresses is entirely contained within this range + /// \param offset Start of range of interest, relative to start of this range + /// \param length Length of range of interest, in bytes + /// \returns True if and only if range of interest is entirely contained within this range + constexpr bool contains_relative(uint64_t offset, uint64_t length) const noexcept { + return get_length() >= length && offset <= get_length() - length; + } + + /// \brief Checks if a range of addresses is entirely contained within this range + /// \param start Target phyisical address of start of range of interest + /// \param length Length of range of interest, in bytes + /// \returns True if and only if range of interest is entirely contained within this range + constexpr bool contains_absolute(uint64_t start, uint64_t length) const noexcept { + return start >= get_start() && contains_relative(start - get_start(), length); + } + + /// \brief Returns PMA flags used during construction + /// \returns Flags + constexpr const pmas_flags &get_flags() const noexcept { + return m_flags; + } + + /// \brief Returns description of address range for use in error messages. + /// \returns Description + constexpr const char *get_description() const noexcept { + return m_description.data(); + } + + /// \brief Returns target physical address where range starts. + /// \returns Start of range + constexpr uint64_t get_start() const noexcept { + return m_start; + } + + /// \brief Returns target physical address right past end of range. + /// \returns End of range + constexpr uint64_t get_end() const noexcept { + return m_end; + } + + /// \brief Returns length of range, in bytes. + /// \returns Length of range + constexpr uint64_t get_length() const noexcept { + return m_end - m_start; + } + + /// \brief Test if address range is occupied by memory + /// \returns True if and only if range is occupied by memory + /// \details In this case, get_host_memory() is guaranteed not to return nullptr. + constexpr bool is_memory() const noexcept { + return m_flags.M; + } + + /// \brief Test if address range is occupied by a device + /// \returns True if and only if range is occupied by a device + /// \details In this case, read_device() and write_device() are operational. + constexpr bool is_device() const noexcept { + return m_flags.IO; + } + + /// \brief Test if address range is empty + /// \returns True if and only if range is empty + /// \details Empty ranges should be used only for sentinels. + constexpr bool is_empty() const noexcept { + return m_end == 0; + } + + /// \brief Tests if range is readable + /// \returns True if and only if range is readable from within the machine. + constexpr bool is_readable() const noexcept { + return m_flags.R; + } + + /// \brief Tests if range is writeable + /// \returns True if and only if range is writeable from within the machine. + constexpr bool is_writeable() const noexcept { + return m_flags.W; + } + + /// \brief Tests if range is executable + /// \returns True if and only if range is executable from within the machine. + constexpr bool is_executable() const noexcept { + return m_flags.X; + } + + /// \brief Tests if range is read-idempotent + /// \returns True if and only if what is read from range remains there until written to + constexpr bool is_read_idempotent() const noexcept { + return m_flags.IR; + } + + /// \brief Tests if range is write-idempotent + /// \returns True if and only if what is written to range remains there and can be read until written to again + constexpr bool is_write_idempotent() const noexcept { + return m_flags.IW; + } + + /// \brief Returns driver ID associated to range + /// \returns The driver ID + constexpr PMA_ISTART_DID get_driver_id() const noexcept { + return m_flags.DID; + } + + /// \brief Returns packed address range istart field as per whitepaper + /// \returns Packed address range istart + uint64_t get_istart() const noexcept { + return pmas_pack_istart(m_flags, m_start); + } + + /// \brief Returns encoded address range ilength field as per whitepaper + /// \returns Packed address range ilength + /// \details This currently contains only the length itself + uint64_t get_ilength() const noexcept { + return get_length(); + } + + /// \brief Returns number of levels in a tree where each leaf is a page + int get_level_count() const noexcept { + return get_level_count(get_length()); + } + +#ifndef MICROARCHITECTURE + /// \brief Returns reference to dirty page tree. + i_dirty_page_tree &get_dirty_page_tree() noexcept { + return do_get_dirty_page_tree(); + } + + /// \brief Returns const reference to dirty page tree. + const i_dirty_page_tree &get_dirty_page_tree() const noexcept { + return do_get_dirty_page_tree(); + } + + /// \brief Returns reference to dense hash tree. + i_dense_hash_tree &get_dense_hash_tree() noexcept { + return do_get_dense_hash_tree(); + } + + /// \brief Returns const reference to dense hash tree tree. + const i_dense_hash_tree &get_dense_hash_tree() const noexcept { + return do_get_dense_hash_tree(); + } +#endif + + // ----- + // These are only for device ranges + // ----- + + /// \brief Reads a word from a device + /// \param da State access object through which the machine state can be accessed. + /// \param offset Where to start reading, relative to start of this range. + /// \param log2_size Log2 of size of value to read (0=uint8_t, 1=uint16_t, 2=uint32_t, 3=uint64_t). + /// \param pval Pointer to word where value will be stored. + /// \returns True if operation succeeded, false otherwise. + bool read_device(i_device_state_access *da, uint64_t offset, int log2_size, uint64_t *pval) const noexcept { + return do_read_device(da, offset, log2_size, pval); + } + + /// \brief Writes a word to a device + /// \param da State access object through which the machine state can be accessed. + /// \param offset Where to start reading, relative to start of this range. + /// \param log2_size Log2 of size of value to write (0=uint8_t, 1=uint16_t, 2=uint32_t, 3=uint64_t). + /// \param val Value to write. + /// \returns execute::failure if operation failed, otherwise a success code if operation succeeded. + execute_status write_device(i_device_state_access *da, uint64_t offset, int log2_size, uint64_t val) noexcept { + return do_write_device(da, offset, log2_size, val); + } + + // ----- + // These are only for memory ranges + // ----- + + /// \brief Returns start of associated memory region in host + /// \returns Pointer to memory + const unsigned char *get_host_memory() const noexcept { + return do_get_host_memory(); + } + + /// \brief Returns start of associated memory region in host + /// \returns Pointer to memory + unsigned char *get_host_memory() noexcept { + return do_get_host_memory(); + } + + /// \brief Returns true if the mapped memory is read-only on the host + /// \returns True if the memory is read-only in the host + bool is_host_read_only() const noexcept { + return do_is_host_read_only(); + } + + /// \brief Returns true if the mapped memory is shared with a backing store in the host + /// \returns True if the memory is shared in the host + bool is_backing_store_shared() const noexcept { + return do_is_backing_store_shared(); + } + +protected: + /// \brief Returns number of levels in a tree where each leaf is a page + /// \param length Length of range, in bytes + static constexpr int get_level_count(uint64_t length) noexcept { + auto page_count = length >> AR_LOG2_PAGE_SIZE; + if (page_count == 0) { + return 0; + } + // NOLINTNEXTLINE(bugprone-narrowing-conversions,cppcoreguidelines-narrowing-conversions) + return std::bit_width(std::bit_ceil(page_count)); + } + +private: + // Default implementation of read_device() for non-device ranges always fails + virtual bool do_read_device(i_device_state_access * /*a*/, uint64_t /*offset*/, int /*log2_size*/, + uint64_t * /*val*/) const noexcept { + return false; + } + + // Default implementation of write_device() for non-device ranges always fails + virtual execute_status do_write_device(i_device_state_access * /*a*/, uint64_t /*offset*/, int /* log2_size */, + uint64_t /*val*/) noexcept { + return execute_status::failure; + } + + // Default implementation of get_host_memory() for non-memory ranges returns nullptr + virtual const unsigned char *do_get_host_memory() const noexcept { + return nullptr; + } + + virtual unsigned char *do_get_host_memory() noexcept { + return nullptr; + } + + virtual bool do_is_host_read_only() const noexcept { + return false; + } + + virtual bool do_is_backing_store_shared() const noexcept { + return false; + } + +#ifndef MICROARCHITECTURE + // Default implemenationt returns always dirty tree + virtual const i_dirty_page_tree &do_get_dirty_page_tree() const noexcept { + const static empty_dirty_page_tree no_dirty{}; + return no_dirty; + } + + virtual i_dirty_page_tree &do_get_dirty_page_tree() noexcept { + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-const-cast) + return const_cast(std::as_const(*this).do_get_dirty_page_tree()); + } + + // Default implemenationt returns no hashes + virtual const i_dense_hash_tree &do_get_dense_hash_tree() const noexcept { + const static empty_dense_hash_tree no_hashes{}; + return no_hashes; + } + + virtual i_dense_hash_tree &do_get_dense_hash_tree() noexcept { + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-const-cast) + return const_cast(std::as_const(*this).do_get_dense_hash_tree()); + } +#endif +}; + +} // namespace cartesi + +#endif // ADDRESS_RANGE_H diff --git a/src/algorithm.h b/src/algorithm.h new file mode 100644 index 000000000..60b7ccd0f --- /dev/null +++ b/src/algorithm.h @@ -0,0 +1,58 @@ +// Copyright Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: LGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify it under +// the terms of the GNU Lesser General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) any +// later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +// PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License along +// with this program (see COPYING). If not, see . +// + +#ifndef ALGORITHM_H +#define ALGORITHM_H + +#include +#include +#include + +#include "concepts.h" + +namespace cartesi { + +/// \brief Adds new entry to back of container, if not already there +/// \tparam Container Container type +/// \tparam T Value type +/// \param container Container to push back into +/// \param value Value to push back +template + requires BackInsertableWith +constexpr void try_push_back(Container &container, T &&value) { + if (container.empty() || container.back() != value) { + container.push_back(std::forward(value)); + } +} + +/// \brief Performs saturating addition of two unsigned integers +/// \tparam T Unsigned integer type +/// \param a First addend +/// \param b Second addend +/// \param max Maximum value of T (default: std::numeric_limits::max()) +/// \returns The sum of a and b, or the maximum value of T if overflow occurs +template + requires std::is_unsigned_v +static constexpr T saturating_add(T a, T b, T max = std::numeric_limits::max()) noexcept { + if (b > max || a > max - b) [[unlikely]] { + return max; + } + return a + b; +} + +} // namespace cartesi + +#endif diff --git a/src/htif-factory.h b/src/array2d.h similarity index 74% rename from src/htif-factory.h rename to src/array2d.h index 9a6807682..91e67aad2 100644 --- a/src/htif-factory.h +++ b/src/array2d.h @@ -14,19 +14,18 @@ // with this program (see COPYING). If not, see . // -#ifndef HTIF_FACTORY_H -#define HTIF_FACTORY_H +#ifndef ARRAY2D_H +#define ARRAY2D_H -#include - -#include "machine-runtime-config.h" -#include "pma.h" +#include +#include namespace cartesi { -/// \brief Creates a PMA entry for the HTIF device -pma_entry make_htif_pma_entry(uint64_t start, uint64_t length); +//??(edubart): In future C++ standards we should switch to `std::mdarray` or `std::mdspan` +template +using array2d = std::array, M>; } // namespace cartesi -#endif +#endif // ARRAY2D_H diff --git a/src/shadow-pmas-factory.h b/src/assert-printf.h similarity index 51% rename from src/shadow-pmas-factory.h rename to src/assert-printf.h index 35514872f..1bf7bb85b 100644 --- a/src/shadow-pmas-factory.h +++ b/src/assert-printf.h @@ -14,32 +14,34 @@ // with this program (see COPYING). If not, see . // -#ifndef SHADOW_PMAS_FACTORY_H -#define SHADOW_PMAS_FACTORY_H - -#include +#ifndef ASSERT_PRINTF_H +#define ASSERT_PRINTF_H /// \file -/// \brief Shadow device. - -#include "pma.h" -#include "shadow-pmas.h" +/// \brief Microarchitecture-dependent includes for printf and assert -namespace cartesi { +#ifdef MICROARCHITECTURE +#include "../uarch/uarch-runtime.h" // IWYU pragma: export +#else +#include // IWYU pragma: export +#include // IWYU pragma: export +#endif -pma_entry make_shadow_pmas_pma_entry(uint64_t start, uint64_t length); +#include // IWYU pragma: export +#include +#include -template -void populate_shadow_pmas_state(const PMAS &pmas, shadow_pmas_state *shadow) { - static_assert(PMA_SHADOW_PMAS_LENGTH >= sizeof(shadow_pmas_state), "shadow PMAs length is too small"); - unsigned index = 0; - for (const auto &pma : pmas) { - shadow->pmas[index].istart = pma.get_istart(); - shadow->pmas[index].ilength = pma.get_ilength(); - ++index; - } +static inline void d_vprintf(const char *fmt, va_list ap) { + std::ignore = vfprintf(stderr, fmt, ap); } -} // namespace cartesi +// Better to use C-style variadic function that checks for format! +// NOLINTNEXTLINE(cert-dcl50-cpp) +__attribute__((__format__(__printf__, 1, 2))) static inline void d_printf(const char *fmt, ...) { + va_list ap{}; + va_start(ap, fmt); + d_vprintf(fmt, ap); + va_end(ap); +} #endif diff --git a/src/back-merkle-tree.cpp b/src/back-merkle-tree.cpp index 027edccb7..1c3c36e6d 100644 --- a/src/back-merkle-tree.cpp +++ b/src/back-merkle-tree.cpp @@ -15,25 +15,23 @@ // /// \file -/// \brief Back Merkle tree implementation. +/// \brief Back merkle tree implementation. + +#include "back-merkle-tree.h" -#include #include +#include #include #include #include +#include -#include "back-merkle-tree.h" -#include "i-hasher.h" +#include "machine-hash.h" +#include "variant-hasher.h" namespace cartesi { -back_merkle_tree::back_merkle_tree(int log2_root_size, int log2_leaf_size, int log2_word_size) : - m_log2_root_size{log2_root_size}, - m_log2_leaf_size{log2_leaf_size}, - m_max_leaves{address_type{1} << (log2_root_size - log2_leaf_size)}, - m_context(std::max(1, log2_root_size - log2_leaf_size + 1)), - m_pristine_hashes{log2_root_size, log2_word_size} { +int back_merkle_tree::validate_log2_max_leaves_size(int log2_root_size, int log2_leaf_size, int log2_word_size) { if (log2_root_size < 0) { throw std::out_of_range{"log2_root_size is negative"}; } @@ -47,24 +45,26 @@ back_merkle_tree::back_merkle_tree(int log2_root_size, int log2_leaf_size, int l throw std::out_of_range{"log2_leaf_size is greater than log2_root_size"}; } if (log2_word_size > log2_leaf_size) { - throw std::out_of_range{"log2_word_size is greater than log2_word_size"}; + throw std::out_of_range{"log2_word_size is greater than log2_leaf_size"}; } - if (log2_root_size - m_log2_leaf_size >= std::numeric_limits::digits) { - throw std::out_of_range{"tree is too large for address type"}; + if (log2_root_size - log2_leaf_size >= std::numeric_limits::digits) { + throw std::out_of_range{"log2_root_size is too large"}; } + return log2_root_size - log2_leaf_size; } -void back_merkle_tree::push_back(const hash_type &new_leaf_hash) { - hasher_type h; - hash_type right = new_leaf_hash; - if (m_leaf_count >= m_max_leaves) { +void back_merkle_tree::push_back(const machine_hash &new_leaf_hash) { + variant_hasher h{m_hash_function}; + machine_hash right = new_leaf_hash; + if (m_leaf_count >= get_max_leaves()) { throw std::out_of_range{"too many leaves"}; } - const int depth = m_log2_root_size - m_log2_leaf_size; - for (int i = 0; i <= depth; ++i) { - if ((m_leaf_count & (address_type{1} << i)) != 0) { + const size_t log2_max_leaves = get_log2_max_leaves(); + for (size_t i = 0; i <= log2_max_leaves; ++i) { + const auto i_span = UINT64_C(1) << i; + if ((m_leaf_count & i_span) != 0) { const auto &left = m_context[i]; - get_concat_hash(h, left, right, right); + h.concat_hash(left, right, right); } else { m_context[i] = right; break; @@ -73,99 +73,101 @@ void back_merkle_tree::push_back(const hash_type &new_leaf_hash) { ++m_leaf_count; } -void back_merkle_tree::pad_back(uint64_t new_leaf_count) { - hasher_type h; - if (new_leaf_count > m_max_leaves || m_leaf_count + new_leaf_count > m_max_leaves) { - throw std::invalid_argument("too many leaves"); +void back_merkle_tree::pad_back(uint64_t new_leaf_count, const machine_hashes &pad_hashes) { + if (new_leaf_count == 0) { + return; + } + const size_t max_leaves = get_max_leaves(); + const size_t log2_max_leaves = get_log2_max_leaves(); + + // Validate inputs + if (new_leaf_count > max_leaves || m_leaf_count + new_leaf_count > max_leaves) { + throw std::invalid_argument{"too many leaves"}; + } + if (pad_hashes.size() != log2_max_leaves + 1) { + throw std::invalid_argument{"pad hashes does not have expected size"}; } - const int depth = m_log2_root_size - m_log2_leaf_size; - int j = 0; - while (j <= depth) { - const uint64_t j_span = address_type{1} << j; + + variant_hasher h{m_hash_function}; + + // Process each bit position from LSB to MSB + for (size_t j = 0; j <= log2_max_leaves;) { + const uint64_t j_span = UINT64_C(1) << j; + + // Skip if this bit isn't set in new_leaf_count if (j_span > new_leaf_count) { break; } - // is our smallest tree at depth j? - if ((m_leaf_count & j_span) != 0) { - // if so, we can add 2^j pristine leaves directly - auto right = m_pristine_hashes.get_hash(m_log2_leaf_size + j); - for (int i = j; i <= depth; ++i) { - const uint64_t i_span = address_type{1} << i; + + // Check if we have an existing subtree at this position + if ((m_leaf_count & j_span) != 0) { // Is our smallest tree at depth j? + // Combine existing subtree with padding and propagate upward + auto right = pad_hashes[j]; + for (size_t i = j; i <= log2_max_leaves; ++i) { + const uint64_t i_span = UINT64_C(1) << i; if ((m_leaf_count & i_span) != 0) { const auto &left = m_context[i]; - get_concat_hash(h, left, right, right); + h.concat_hash(left, right, right); } else { m_context[i] = right; - // outer loop continues where we left off + // Outer loop continues where we left off j = i; break; } } - new_leaf_count = new_leaf_count - j_span; - m_leaf_count = m_leaf_count + j_span; + m_leaf_count += j_span; + new_leaf_count -= j_span; } else { ++j; } } - // now add the rest of the padding directly to the context - for (int i = 0; i <= depth; ++i) { - const uint64_t i_span = address_type{1} << i; + + // Add the rest of the padding directly to the context + for (size_t i = 0; i <= log2_max_leaves && new_leaf_count > 0; ++i) { + const uint64_t i_span = UINT64_C(1) << i; + // Check if we have to set the subtree at this position if ((new_leaf_count & i_span) != 0) { - m_context[i] = m_pristine_hashes.get_hash(m_log2_leaf_size + i); - new_leaf_count = new_leaf_count - i_span; - m_leaf_count = m_leaf_count + i_span; + m_context[i] = pad_hashes[i]; + m_leaf_count += i_span; + new_leaf_count -= i_span; } } + + assert(new_leaf_count == 0); + assert(m_leaf_count <= get_max_leaves()); } -back_merkle_tree::hash_type back_merkle_tree::get_root_hash() const { - hasher_type h; - assert(m_leaf_count <= m_max_leaves); - const int depth = m_log2_root_size - m_log2_leaf_size; - if (m_leaf_count < m_max_leaves) { - auto root = m_pristine_hashes.get_hash(m_log2_leaf_size); - for (int i = 0; i < depth; ++i) { - if ((m_leaf_count & (address_type{1} << i)) != 0) { - const auto &left = m_context[i]; - get_concat_hash(h, left, root, root); - } else { - const auto &right = m_pristine_hashes.get_hash(m_log2_leaf_size + i); - get_concat_hash(h, root, right, root); - } - } - return root; +machine_hashes back_merkle_tree::make_pad_hashes(const machine_hash &leaf_hash, int log2_max_leaves, + hash_function_type hash_function) { + assert(log2_max_leaves >= 0); + machine_hashes hashes; + hashes.resize(log2_max_leaves + 1); + hashes[0] = leaf_hash; + variant_hasher h{hash_function}; + for (size_t i = 1; i < hashes.size(); ++i) { + h.concat_hash(hashes[i - 1], hashes[i - 1], hashes[i]); } - return m_context[depth]; + return hashes; } -back_merkle_tree::proof_type back_merkle_tree::get_next_leaf_proof() const { - const int depth = m_log2_root_size - m_log2_leaf_size; - if (m_leaf_count >= m_max_leaves) { - throw std::out_of_range{"tree is full"}; - } - hasher_type h; - proof_type proof{m_log2_root_size, m_log2_leaf_size}; - proof.set_target_address(m_leaf_count << m_log2_leaf_size); - proof.set_target_hash(m_pristine_hashes.get_hash(m_log2_leaf_size)); - hash_type hash = m_pristine_hashes.get_hash(m_log2_leaf_size); - for (int i = 0; i < depth; ++i) { - if ((m_leaf_count & (address_type{1} << i)) != 0) { - const auto &left = m_context[i]; - proof.set_sibling_hash(left, m_log2_leaf_size + i); - get_concat_hash(h, left, hash, hash); - } else { - const auto &right = m_pristine_hashes.get_hash(m_log2_leaf_size + i); - proof.set_sibling_hash(right, m_log2_leaf_size + i); - get_concat_hash(h, hash, right, hash); - } - } - proof.set_root_hash(hash); -#ifndef NDEBUG - if (!proof.verify(h)) { - throw std::runtime_error{"produced invalid proof"}; +static machine_hash get_pristine_word_hash(int log2_word_size, hash_function_type hash_function) { + std::vector word(UINT64_C(1) << log2_word_size, 0); + machine_hash hash{}; + variant_hasher h{hash_function}; + h.hash(word, hash); + return hash; +} + +machine_hashes back_merkle_tree::make_pristine_pad_hashes(int log2_root_size, int log2_leaf_size, int log2_word_size, + hash_function_type hash_function) { + validate_log2_max_leaves_size(log2_root_size, log2_leaf_size, log2_word_size); + auto pristine_pad_hashes = make_pad_hashes(get_pristine_word_hash(log2_word_size, hash_function), + log2_root_size - log2_word_size, hash_function); + if (log2_leaf_size > log2_word_size) { + pristine_pad_hashes.erase(pristine_pad_hashes.begin(), + pristine_pad_hashes.begin() + (log2_leaf_size - log2_word_size)); } -#endif - return proof; + return pristine_pad_hashes; } } // namespace cartesi diff --git a/src/back-merkle-tree.h b/src/back-merkle-tree.h index fd1902b80..28b0c1763 100644 --- a/src/back-merkle-tree.h +++ b/src/back-merkle-tree.h @@ -17,135 +17,191 @@ #ifndef BACK_MERKLE_TREE_H #define BACK_MERKLE_TREE_H +#include #include -#include +#include +#include +#include -#include "keccak-256-hasher.h" -#include "merkle-tree-proof.h" -#include "pristine-merkle-tree.h" +#include "machine-hash.h" +#include "variant-hasher.h" /// \file -/// \brief Back Merkle tree interface. +/// \brief Back back_hash_tree tree interface. namespace cartesi { -/// \brief Incremental way of maintaining a Merkle tree for a stream of -/// leaf hashes -/// \details This is surprisingly efficient in both time and space. -/// Adding the next leaf takes O(log(n)) in the worst case, but is -/// this is amortized to O(1) time when adding n leaves. -/// Obtaining the proof for the current leaf takes theta(log(n)) time. -/// Computing the tree root hash also takes theta(log(n)) time. -/// The class only ever stores log(n) hashes (1 for each tree level). +/// \brief Incremental hash tree that efficiently maintains hashes for a stream of leaves +/// \details Space-efficient design stores only O(log n) hashes (one per tree level). +/// Leaf insertion is O(log n) worst-case but amortizes to O(1) over n operations. class back_merkle_tree { public: - /// \brief Hasher class. - using hasher_type = keccak_256_hasher; - - /// \brief Storage for a hash. - using hash_type = hasher_type::hash_type; - - /// \brief Storage for a hash. - using address_type = uint64_t; - - /// \brief Storage for the proof of a word value. - using proof_type = merkle_tree_proof; - - /// \brief Constructor - /// \param log2_root_size Log2 of root node - /// \param log2_leaf_size Log2 of leaf node - /// \param log2_word_size Log2 of word node - back_merkle_tree(int log2_root_size, int log2_leaf_size, int log2_word_size); + /// \brief Constructor from an existing leaves context + /// \param log2_max_leaves Log base 2 of maximum amount of leaves + /// \param hash_function Hash function to use + /// \param leaf_count Amount of leaves already added to the tree + /// \param context Context representing the leaves hashes + back_merkle_tree(int log2_max_leaves, hash_function_type hash_function, uint64_t leaf_count = 0, + machine_hashes context = {}) : + m_hash_function(hash_function), + m_leaf_count{leaf_count}, + m_context(std::max(1, log2_max_leaves + 1)) { + if (log2_max_leaves < 0) { + throw std::out_of_range{"log2_max_leaves is negative"}; + } + if (log2_max_leaves >= std::numeric_limits::digits) { + throw std::out_of_range{"log2_max_leaves is too large"}; + } + if (leaf_count >= get_max_leaves()) { + throw std::out_of_range{"leaf count is greater than or equal to max leaves"}; + } + // Unpack context + size_t j = 0; + if (leaf_count > 0) { + for (int i = 0; i <= log2_max_leaves; ++i) { + const auto i_span = UINT64_C(1) << i; + if ((leaf_count & i_span) != 0) { + if (j >= context.size()) { + throw std::out_of_range{"leaves context is incompatible"}; + } + m_context[i] = context[j++]; + } + } + } + if (j != context.size()) { + throw std::out_of_range{"leaves context is incompatible"}; + } + } + /// \brief Constructor from known root, leaf and word sizes + /// \param log2_root_size Log base 2 of root node + /// \param log2_leaf_size Log base 2 of leaf node + /// \param log2_word_size Log base 2 of word node + /// \param hash_function Hash function to use + back_merkle_tree(int log2_root_size, int log2_leaf_size, int log2_word_size, hash_function_type hash_function) : + back_merkle_tree(validate_log2_max_leaves_size(log2_root_size, log2_leaf_size, log2_word_size), hash_function) { + } /// \brief Appends a new hash to the tree /// \param new_leaf_hash Hash of new leaf data /// \details - /// Consider the tree down to the leaf level. - /// The tree is only complete after 2^(log2_root_size-log2_leaf_size) - /// leaves have been added. - /// Before that, when leaf_count leaves have been added, we assume the rest - /// of the leaves are filled with zeros (i.e., they are pristine). - /// The trick is that we do not need to store the hashes of all leaf_count - /// leaves already added to the stream. - /// This is because, whenever a subtree is complete, all we need is its - /// root hash. - /// The complete subtrees are disjoint, abutting, and appear in decreasing - /// size. - /// In fact, there is exactly one complete subtree for each bit set in - /// leaf_count. - /// We only need log2_root_size-log2_leaf_size+1 bits to represent - /// leaf_count. - /// So our context is a vector with log2_root_size-log2_leaf_size+1 entries, - /// where entry i contains the hash for a complete subtree of - /// size 2^i leaves. - /// We will only use the entries i if the corresponding bit is set - /// in leaf_count. - /// Adding a new leaf hash exactly like adding 1 to leaf_count. - /// We scan from least to most significant bit in leaf_count. - /// We start with the right = leaf_hash and i = 0. - /// If the bit i is set in leaf_count, we replace - /// context[i] = hash(context[i], right) and move up a bit. - /// If the bit is not set, we simply store context[i] = right and break - /// In other words, we can update the context in - /// log time (log2_root_size-log2_leaf_size) - void push_back(const hash_type &new_leaf_hash); + /// The algorithm efficiently maintains only the root hashes of complete subtrees. + /// Each bit set in leaf_count corresponds to a complete subtree of size 2^i, + /// with its hash stored in context[i]. + /// + /// Adding a leaf is equivalent to binary addition: scan bits from LSB to MSB. + /// For each set bit i in leaf_count, combine context[i] with the new hash + /// and propagate upward. Store the result at the first unset bit position. + /// This achieves O(log n) worst-case, O(1) amortized time complexity. + void push_back(const machine_hash &new_leaf_hash); /// \brief Appends a number of padding hashes to the tree - /// \param leaf_count Number of padding hashes to append + /// \param new_leaf_count Number of padding hashes to append + /// \param pad_hashes Array containing the padding hashes /// \details - /// Recall that a bit i set in leaf_count represents a complete subtree - /// of size 2^i for which we have a hash in context[i]. - /// The remaining entries in the context are unused. - /// The base case is when the least significant bit set in leaf_count is - /// bigger than new_leaf_count. - /// We can simply add to context[j] a pristine subtree of size 2^j - /// for each bit j set in new_leaf_count. - /// No used used entry in the context will be overwritten. - /// We can then simply add new_leaf_count to leaf_count and we are done. - /// In the general case, the least significant bit set i in leaf_count is - /// less than or equal to new_leaf_count. - /// Here, we add a pristine subtree of size 2^i to the context and - /// bubble up. - /// We add 2^i to leaf_count and subtract 2^i from new_leaf_count. - /// Then we repeat this process until we reach the base case. - void pad_back(uint64_t new_leaf_count); + /// Uses binary representation of leaf counts to efficiently add padding. + /// Each set bit i in leaf_count represents a complete subtree of size 2^i. + /// + /// Base case: When the least significant set bit in leaf_count exceeds new_leaf_count, + /// directly place pad subtrees at positions corresponding to bits set in new_leaf_count. + /// + /// General case: When overlap exists, combine the smallest existing subtree with + /// a matching pad subtree, bubble up the result, and repeat until base case is reached. + void pad_back(uint64_t new_leaf_count, const machine_hashes &pad_hashes); /// \brief Returns the root tree hash /// \returns Root tree hash - /// \details - /// We can produce the tree root hash from the context at any time, also - /// in log time - /// Ostensibly, we add pristine leaves until the leaf_count - /// hits 2^(log2_root_size-log2_leaf_size) - /// To do this in log time, we start by precomputing the hashes for all - /// completely pristine subtree sizes - /// If leaf_count is already 2^(log2_root_size-log2_leaf_size), we - /// return context[i] - /// Otherwise, we start with i = 0 and root = pristine[i+log2_leaf_size] - /// (i.e., the invariant is that root contains the hash of the rightmost - /// subtree whose log size is i + log2_leaf_size) - /// If bit i is set, we set root = hash(context[i], root) and move up a bit - /// (i.e., the subtree we are growing is to the right of what is - /// in the context) - /// If bit i is not set, we set - /// root = hash(root, pristine[i+log2_leaf_size]) and move up a bit - /// (i.e., to grow our subtree, we need to pad it on the right with - /// a pristine subtree of the same size) - hash_type get_root_hash() const; - - /// \brief Returns proof for the next pristine leaf - /// \returns Proof for leaf at given index, or throws exception - /// \details This is basically the same algorithm as - /// back_merkle_tree::get_root_hash. - proof_type get_next_leaf_proof() const; + /// \details The tree must be complete, otherwise an exception is thrown + machine_hash get_root_hash() const { + if (!full()) { + throw std::runtime_error{"attempt to get root hash of an incomplete back tree"}; + } + return m_context.back(); + } + + /// \brief Clears the tree, making it empty (as if no leaves were added) + void clear() noexcept { + m_leaf_count = 0; + } + + /// \brief Returns true if the tree is complete (reached maximum amount of leaves) + bool full() const noexcept { + return m_leaf_count >= get_max_leaves(); + } + + /// \brief Returns true if the tree is empty (no leaves were added) + bool empty() const noexcept { + return m_leaf_count == 0; + } + + /// \brief Returns log base 2 of maximum amount of leaves that can be held by the tree + int get_log2_max_leaves() const noexcept { + return static_cast(m_context.size()) - 1; + } + + /// \brief Returns maximum amount of leaves that can be held by the tree + uint64_t get_max_leaves() const noexcept { + return static_cast(1) << get_log2_max_leaves(); + } + + /// \brief Returns the hash function used by the tree + hash_function_type get_hash_function() const noexcept { + return m_hash_function; + } + + /// \brief Returns amount of leaves already added to the tree + uint64_t get_leaf_count() const noexcept { + return m_leaf_count; + } + + /// \brief Returns amount of leaves that can yet be added to the tree + uint64_t get_remaining_leaf_count() const noexcept { + return get_max_leaves() - m_leaf_count; + } + + /// \brief Returns the leaves context + machine_hashes get_context() const { + // Pack context + machine_hashes context; + if (m_leaf_count > 0) { + const int log2_max_leaves = get_log2_max_leaves(); + for (int i = 0; i <= log2_max_leaves; ++i) { + const auto i_span = UINT64_C(1) << i; + if ((m_leaf_count & i_span) != 0) { + context.push_back(m_context[i]); + } + } + } + return context; + } + + /// \brief Creates an array of pad hashes to be used with pad_back() + /// \param leaf_hash Hash of the leaf node + /// \param log2_max_leaves Log base 2 of maximum amount of leaves + /// \param hash_function Hash function to use + /// \returns Array of pad hashes + static machine_hashes make_pad_hashes(const machine_hash &leaf_hash, int log2_max_leaves, + hash_function_type hash_function); + + /// \brief Creates an array of pristine pad hashes to be used with pad_back() + /// \param log2_root_size Log base 2 of root node + /// \param log2_leaf_size Log base 2 of leaf node + /// \param log2_word_size Log base 2 of word node + /// \param hash_function Hash function to use + /// \returns Array of pad hashes + static machine_hashes make_pristine_pad_hashes(int log2_root_size, int log2_leaf_size, int log2_word_size, + hash_function_type hash_function); + + /// \brief Validates and computes log2_max_leaves from root, leaf and word sizes + /// \param log2_root_size Log base 2 of root node + /// \param log2_leaf_size Log base 2 of leaf node + /// \param log2_word_size Log base 2 of word node + /// \returns Log base 2 of maximum amount of leaves + static int validate_log2_max_leaves_size(int log2_root_size, int log2_leaf_size, int log2_word_size); private: - int m_log2_root_size; ///< Log2 of tree size - int m_log2_leaf_size; ///< Log2 of leaf size - address_type m_leaf_count{0}; ///< Number of leaves already added - address_type m_max_leaves; ///< Maximum number of leaves - std::vector m_context; ///< Hashes of bits set in leaf_count - pristine_merkle_tree m_pristine_hashes; ///< Hash of pristine subtrees of all sizes + hash_function_type m_hash_function; ///< Hash function + uint64_t m_leaf_count; ///< Number of leaves already added + machine_hashes m_context; ///< Hashes of bits set in leaf_count }; } // namespace cartesi diff --git a/src/base64.cpp b/src/base64.cpp index 1fb10354d..0fcb4aec2 100644 --- a/src/base64.cpp +++ b/src/base64.cpp @@ -14,16 +14,15 @@ // with this program (see COPYING). If not, see . // +#include "base64.h" + #include #include #include #include #include -#include -#include "base64.h" - -namespace cartesi { +namespace cartesi::detail { // Base64 globals static constexpr uint8_t b64base[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; @@ -43,7 +42,7 @@ static constexpr uint8_t b64unbase[] = {255, 255, 255, 255, 255, 255, 255, 255, // Accumulates bytes in input buffer until 3 bytes are available. // Translate the 3 bytes into Base64 form and append to buffer. // Returns new number of bytes in buffer. -static size_t b64encode(uint8_t c, uint8_t *input, size_t size, std::ostringstream &sout) { +size_t b64encode(uint8_t c, uint8_t *input, size_t size, std::ostringstream &sout) { input[size++] = c; if (size == 3) { uint8_t code[4]; @@ -70,7 +69,7 @@ static size_t b64encode(uint8_t c, uint8_t *input, size_t size, std::ostringstre // Encodes the Base64 last 1 or 2 bytes and adds padding '=' // Result, if any, is appended to buffer. // Returns 0. -static size_t b64pad(const uint8_t *input, size_t size, std::ostringstream &sout) { +size_t b64pad(const uint8_t *input, size_t size, std::ostringstream &sout) { uint64_t value = 0; uint8_t code[4] = {'=', '=', '=', '='}; switch (size) { @@ -104,7 +103,7 @@ static size_t b64pad(const uint8_t *input, size_t size, std::ostringstream &sout // Accumulates bytes in input buffer until 4 bytes are available. // Translate the 4 bytes from Base64 form and append to buffer. // Returns new number of bytes in buffer. -static size_t b64decode(uint8_t c, uint8_t *input, size_t size, std::ostringstream &sout) { +size_t b64decode(uint8_t c, uint8_t *input, size_t size, std::ostringstream &sout) { if (b64unbase[c] > 64) { if (c == ' ' || c == '\t' || c == '\n' || c == '\v' || c == '\f' || c == '\r') { // ignore whitespace characters return size; @@ -145,25 +144,4 @@ static size_t b64decode(uint8_t c, uint8_t *input, size_t size, std::ostringstre return size; } -std::string encode_base64(const std::string_view &input) { - std::ostringstream sout; - uint8_t ctx[4]{}; - size_t ctxlen = 0; - for (const char b : input) { - ctxlen = b64encode(static_cast(b), ctx, ctxlen, sout); - } - b64pad(ctx, ctxlen, sout); - return sout.str(); -} - -std::string decode_base64(const std::string_view &input) { - std::ostringstream sout; - uint8_t ctx[4]{}; - size_t ctxlen = 0; - for (const char b : input) { - ctxlen = b64decode(static_cast(b), ctx, ctxlen, sout); - } - return sout.str(); -} - -} // namespace cartesi +} // namespace cartesi::detail diff --git a/src/base64.h b/src/base64.h index b2ebe8d20..6b7cf7e9b 100644 --- a/src/base64.h +++ b/src/base64.h @@ -17,14 +17,52 @@ #ifndef BASE64_H #define BASE64_H +#include +#include +#include #include -#include + +#include "concepts.h" namespace cartesi { -std::string encode_base64(const std::string_view &input); +namespace detail { + +size_t b64encode(uint8_t c, uint8_t *input, size_t size, std::ostringstream &sout); +size_t b64pad(const uint8_t *input, size_t size, std::ostringstream &sout); +size_t b64decode(uint8_t c, uint8_t *input, size_t size, std::ostringstream &sout); + +} // namespace detail + +/// \brief Encodes binary data into base64 +/// \param data Input data range +/// \returns String with encoded data +template +std::string encode_base64(R &&data) { // NOLINT(cppcoreguidelines-missing-std-forward) + //??D we could make this faster by avoiding ostringstream altogether... + std::ostringstream sout; + uint8_t ctx[4]{}; + size_t ctxlen = 0; + for (auto b : data) { + ctxlen = detail::b64encode(static_cast(b), ctx, ctxlen, sout); + } + detail::b64pad(ctx, ctxlen, sout); + return sout.str(); +} -std::string decode_base64(const std::string_view &input); +/// \brief Decodes binary data from base64 +/// \param data Input data range +/// \returns String with decoded data +template +std::string decode_base64(R &&data) { // NOLINT(cppcoreguidelines-missing-std-forward) + std::ostringstream sout; + uint8_t ctx[4]{}; + size_t ctxlen = 0; + for (auto b : data) { + ctxlen = detail::b64decode(static_cast(b), ctx, ctxlen, sout); + } + return sout.str(); +} } // namespace cartesi diff --git a/src/cartesi-machine-stored-hash.lua b/src/cartesi-machine-stored-hash.lua index 25beca28f..04f9a672b 100755 --- a/src/cartesi-machine-stored-hash.lua +++ b/src/cartesi-machine-stored-hash.lua @@ -16,12 +16,11 @@ -- with this program (see COPYING). If not, see . -- +local cartesi = require("cartesi") local util = require("cartesi.util") -local f = assert( - io.open(assert(arg[1], "missing machine name") .. "/hash", "rb"), - string.format("unable to open machine '%s'", tostring(arg[1])) -) -local h = assert(f:read("a"), "unable to read hash") -f:close() -print(util.hexhash(h)) +local dir = assert(arg[1], "missing machine directory") +local machine = cartesi.machine(dir) +local root_hash = machine:get_root_hash() +local hex_root_hash = util.hexhash(root_hash) +print(hex_root_hash) diff --git a/src/cartesi-machine.lua b/src/cartesi-machine.lua index ff5cfbc66..6b598c742 100755 --- a/src/cartesi-machine.lua +++ b/src/cartesi-machine.lua @@ -87,10 +87,6 @@ where options are: --ram-length= set RAM length. - --dtb-image= - name of file containing DTB image - (default: auto generated flattened device tree). - --no-bootargs clear default bootargs. @@ -106,19 +102,22 @@ where options are: : is one of label:, fast_addr_name, static_cast(faddr), + pma_index, static_cast(*pval), static_cast(*pval)); } - /// \brief Try to write a word to memory through the TLB. - /// \tparam ETYPE TLB entry type. + /// \brief Writes a word to memory. /// \tparam T Type of word to write. - /// \param vaddr Target virtual address. + /// \tparam A Type to which \p paddr and \p faddr are known to be aligned. + /// \param faddr Implementation-defined fast address. /// \param val Value to be written. - /// \returns True if successful (TLB hit), false otherwise. - template - bool write_memory_word_via_tlb(uint64_t vaddr, T val) { + /// \warning T must not cross page boundary starting from \p faddr. + /// \warning T (when A is smaller) may or may not cross a hash tree word boundary starting from \p faddr! + template + void write_memory_word(fast_addr faddr, uint64_t pma_index, T val) const { static_assert(std::is_integral_v && sizeof(T) <= sizeof(uint64_t), "unsupported type"); - return derived().template do_write_memory_word_via_tlb(vaddr, val); + derived().template do_write_memory_word(faddr, pma_index, val); + [[maybe_unused]] const auto fast_addr_name = std::is_same_v ? "phys_addr" : "fast_addr"; + dsa_printf("%s::write_memory_word<%s,%s>(%s{0x%" PRIx64 "}, %" PRIu64 ", %" PRIu64 "(0x%" PRIx64 "))\n", + get_name(), poor_type_name_v, poor_type_name_v, fast_addr_name, static_cast(faddr), + pma_index, static_cast(val), static_cast(val)); + } + + /// \brief Reads TLB's vaddr_page + /// \tparam USE TLB set + /// \param slot_index Slot index + /// \returns Value in slot. + template + uint64_t read_tlb_vaddr_page(uint64_t slot_index) const { + const auto val = derived().template do_read_tlb_vaddr_page(slot_index); + dsa_printf("%s::read_tlb_vaddr_page<%" PRIu64 ">(%" PRIu64 ") = 0x%" PRIx64 "\n", get_name(), SET, slot_index, + val); + return val; + } + + /// \brief Reads TLB's vf_offset + /// \tparam USE TLB set + /// \param slot_index Slot index + /// \returns Value in slot. + template + fast_addr read_tlb_vf_offset(uint64_t slot_index) const { + [[maybe_unused]] const auto fast_addr_name = std::is_same_v ? "phys_addr" : "fast_addr"; + const auto val = derived().template do_read_tlb_vf_offset(slot_index); + dsa_printf("%s::read_tlb_vf_offset<%" PRIu64 ">(%" PRIu64 ") = %s{0x%" PRIx64 "}\n", get_name(), SET, + slot_index, fast_addr_name, static_cast(val)); + return val; + } + + /// \brief Reads TLB's pma_index + /// \tparam USE TLB set + /// \param slot_index Slot index + /// \returns Value in slot. + template + uint64_t read_tlb_pma_index(uint64_t slot_index) const { + const auto val = derived().template do_read_tlb_pma_index(slot_index); + dsa_printf("%s::read_tlb_pma_index<%" PRIu64 ">(%" PRIu64 ") = %" PRIu64 "(0x%" PRIx64 ")\n", get_name(), SET, + slot_index, val, val); + return val; + } + + /// \brief Writes to a TLB slot + /// \tparam USE TLB set + /// \param slot_index Slot index + /// \param vaddr_page Value to write + /// \param vf_offset Value to write + /// \param pma_index Value to write + /// \detail Writes to the TLB must be modify all fields atomically to prevent an inconsistent state. + /// This simplifies all state access implementations. + template + void write_tlb(uint64_t slot_index, uint64_t vaddr_page, fast_addr vf_offset, uint64_t pma_index) const { + derived().template do_write_tlb(slot_index, vaddr_page, vf_offset, pma_index); + [[maybe_unused]] const auto fast_addr_name = std::is_same_v ? "phys_addr" : "fast_addr"; + dsa_printf("%s::write_tlb<%" PRIu64 ">(%" PRIu64 ", 0x%" PRIx64 ", %s{0x%" PRIx64 "}, %" PRIu64 ")\n", + get_name(), SET, slot_index, vaddr_page, fast_addr_name, static_cast(vf_offset), pma_index); + } + + /// \brief Marks a page as dirty + /// \param faddr Implementation-defined fast address. + /// \param pma_index Index of PMA where page falls + /// \details When there is a host machine, the hash tree only updates the hashes for pages that + /// have been modified. Pages can only be written to if they appear in the write TLB. Therefore, + /// the hash tree only considers the pages that are currently in the write TLB and those that + /// have been marked dirty. When a page leaves the write TLB, it is marked dirty. + /// If the state belongs to a host machine, then this call MUST be forwarded to machine::mark_dirty_page(); + void mark_dirty_page(fast_addr faddr, uint64_t pma_index) const { + derived().do_mark_dirty_page(faddr, pma_index); + } + + /// \brief Writes a character to the console + /// \param c Character to output + void putchar(uint8_t c) const { + derived().do_putchar(c); + } + + constexpr const char *get_name() const { + return derived().do_get_name(); } - - /// \brief Replaces an entry in the TLB. - /// \tparam ETYPE TLB entry type to replace. - /// \param vaddr Target virtual address. - /// \param paddr Target physical address. - /// \param pma PMA entry for the physical address. - /// \returns Pointer to page start in host memory. - template - unsigned char *replace_tlb_entry(uint64_t vaddr, uint64_t paddr, PMA_ENTRY_TYPE &pma) { - return derived().template do_replace_tlb_entry(vaddr, paddr, pma); - } - - /// \brief Invalidates all TLB entries of a type. - /// \tparam ETYPE TLB entry type to flush. - template - void flush_tlb_type() { - return derived().template do_flush_tlb_type(); - } - - /// \brief Invalidates all TLB entries of all types. - NO_INLINE void flush_all_tlb() { - derived().template flush_tlb_type(); - derived().template flush_tlb_type(); - derived().template flush_tlb_type(); - } - - /// \brief Invalidates TLB entries for a specific virtual address. - /// \param vaddr Target virtual address. - NO_INLINE void flush_tlb_vaddr(uint64_t vaddr) { - return derived().do_flush_tlb_vaddr(vaddr); - } - - /// \brief Returns true if soft yield HINT instruction is enabled at runtime - bool get_soft_yield() { - return derived().do_get_soft_yield(); - } - - /// \brief Write a data buffer to memory padded with 0 - /// \param paddr Destination physical address. - /// \param data Pointer to source data buffer. - /// \param data_length Length of data buffer. - /// \param write_length_log2_size Log2 size of the total write length. - void write_memory_with_padding(uint64_t paddr, const unsigned char *data, uint64_t data_length, - int write_length_log2_size) { - return derived().do_write_memory_with_padding(paddr, data, data_length, write_length_log2_size); - } - -#ifdef DUMP_COUNTERS - auto &get_statistics() { - return derived().do_get_statistics(); - } -#endif }; /// \brief SFINAE test implementation of the i_state_access interface template using is_an_i_state_access = - std::integral_constant::type>::value>; + std::integral_constant>>; + +template +constexpr bool is_an_i_state_access_v = is_an_i_state_access::value; } // namespace cartesi diff --git a/src/i-uarch-state-access.h b/src/i-uarch-state-access.h index 0f74793d5..a80adb2ef 100644 --- a/src/i-uarch-state-access.h +++ b/src/i-uarch-state-access.h @@ -17,10 +17,39 @@ #ifndef I_UARCH_STATE_ACCESS_H #define I_UARCH_STATE_ACCESS_H +#include +#include #include - -#include "bracket-note.h" -#include "pma.h" +#include + +#include "assert-printf.h" // IWYU pragma: keep +#include "i-prefer-shadow-uarch-state.h" +#include "meta.h" +#include "shadow-tlb.h" +#include "shadow-uarch-state.h" + +// NOLINTBEGIN(cppcoreguidelines-macro-usage) +#define DEFINE_USA_READ(REG) \ + uint64_t read_##REG() const { \ + if constexpr (!is_an_i_prefer_shadow_uarch_state_v) { \ + const auto val = derived().do_read_##REG(); \ + dusa_printf("%s::read_" #REG "() = %" PRIu64 "(0x%" PRIx64 ")\n", get_name(), val, val); \ + return val; \ + } else { \ + return prefer_read_shadow_uarch_state(shadow_uarch_state_what::REG); \ + } \ + } + +#define DEFINE_USA_WRITE(REG) \ + void write_##REG(uint64_t val) const { \ + if constexpr (!is_an_i_prefer_shadow_uarch_state_v) { \ + derived().do_write_##REG(val); \ + dusa_printf("%s::write_" #REG "(%" PRIu64 "(0x%" PRIx64 "))\n", get_name(), val, val); \ + } else { \ + prefer_write_shadow_uarch_state(shadow_uarch_state_what::REG, val); \ + } \ + } +// NOLINTEND(cppcoreguidelines-macro-usage) namespace cartesi { @@ -38,71 +67,112 @@ class i_uarch_state_access { // CRTP return *static_cast(this); } -public: - /// \brief Adds an annotation bracket to the log - /// \param type Type of bracket - /// \param text String with the text for the annotation - void push_bracket(bracket_type type, const char *text) { - return derived().do_push_bracket(type, text); + uint64_t prefer_read_shadow_uarch_state(shadow_uarch_state_what what) const { + const auto val = derived().read_shadow_uarch_state(what); + [[maybe_unused]] const auto *const what_name = shadow_uarch_state_get_what_name(what); + dusa_printf("%s::read_shadow_uarch_state(%s) = %" PRIu64 "(0x%" PRIx64 ")\n", get_name(), what_name, val, val); + return val; } - /// \brief Adds annotations to the state, bracketing a scope - /// \param text String with the text for the annotation - /// \returns An object that, when constructed and destroyed issues an annonation. - auto make_scoped_note(const char *text) { - return derived().do_make_scoped_note(text); + void prefer_write_shadow_uarch_state(shadow_uarch_state_what what, uint64_t val) const { + derived().write_shadow_uarch_state(what, val); + [[maybe_unused]] const auto *const what_name = shadow_uarch_state_get_what_name(what); + dusa_printf("%s::write_shadow_uarch_state(%s, %" PRIu64 "(0x%" PRIx64 "))\n", get_name(), what_name, val, val); } - auto read_x(int r) { - return derived().do_read_x(r); +public: + /// \brief Works as vprintf if we are dumping uarch state accesses, otherwise does nothing + static void dusa_vprintf([[maybe_unused]] const char *fmt, [[maybe_unused]] va_list ap) { +#ifdef DUMP_UARCH_STATE_ACCESS + d_vprintf(fmt, ap); +#endif } - auto write_x(int r, uint64_t v) { - return derived().do_write_x(r, v); + /// \brief Works as printf if we are dumping uarch state accesses, otherwise does nothing + // Better to use C-style variadic function that checks for format! + // NOLINTNEXTLINE(cert-dcl50-cpp) + __attribute__((__format__(__printf__, 1, 2))) static void dusa_printf([[maybe_unused]] const char *fmt, ...) { +#ifdef DUMP_UARCH_STATE_ACCESS + va_list ap; + va_start(ap, fmt); + dusa_vprintf(fmt, ap); + va_end(ap); +#endif } - auto read_pc() { - return derived().do_read_pc(); + uint64_t read_uarch_x(int i) const { + if constexpr (!is_an_i_prefer_shadow_uarch_state_v) { + const auto val = derived().do_read_uarch_x(i); + dusa_printf("%s::read_uarch_x(%d) = %" PRIu64 "(0x%" PRIx64 ")\n", get_name(), i, val, val); + return val; + } else { + return prefer_read_shadow_uarch_state(shadow_uarch_state_get_what(shadow_uarch_state_what::uarch_x0, i)); + } } - auto write_pc(uint64_t v) { - return derived().do_write_pc(v); + void write_uarch_x(int i, uint64_t val) const { + if constexpr (!is_an_i_prefer_shadow_uarch_state_v) { + derived().do_write_uarch_x(i, val); + dusa_printf("%s::write_uarch_x(%d, %" PRIu64 ")\n", get_name(), i, val); + } else { + prefer_write_shadow_uarch_state(shadow_uarch_state_get_what(shadow_uarch_state_what::uarch_x0, i), val); + } } - auto read_cycle() { - return derived().do_read_cycle(); - } + // Define read and write methods for each register in the shadow uarch state + // NOLINTBEGIN(cppcoreguidelines-macro-usage) + DEFINE_USA_READ(uarch_halt_flag) + DEFINE_USA_WRITE(uarch_halt_flag) + DEFINE_USA_READ(uarch_cycle) + DEFINE_USA_WRITE(uarch_cycle) + DEFINE_USA_READ(uarch_pc) + DEFINE_USA_WRITE(uarch_pc) + // NOLINTEND(cppcoreguidelines-macro-usage) - auto read_halt_flag() { - return derived().do_read_halt_flag(); + uint64_t read_word(uint64_t paddr) const { + const auto val = derived().do_read_word(paddr); + dusa_printf("%s::read_word(phys_addr{0x%" PRIx64 "}) = %" PRIu64 "(0x%" PRIx64 ")\n", get_name(), paddr, val, + val); + return val; } - auto set_halt_flag() { - return derived().do_set_halt_flag(); + void write_word(uint64_t paddr, uint64_t val) const { + derived().do_write_word(paddr, val); + dusa_printf("%s::write_word(phys_addr{0x%" PRIx64 "}, %" PRIu64 "(0x%" PRIx64 "))\n", get_name(), paddr, val, + val); } - auto reset_halt_flag() { - return derived().do_reset_halt_flag(); + /// \brief Resets uarch to pristine state + void reset_uarch() const { + return derived().do_reset_uarch(); } - auto write_cycle(uint64_t v) { - return derived().do_write_cycle(v); + void putchar(uint8_t c) const { + derived().do_putchar(c); } - uint64_t read_word(uint64_t paddr) { - return derived().do_read_word(paddr); + void mark_dirty_page(uint64_t paddr, uint64_t pma_index) const { + return derived().do_mark_dirty_page(paddr, pma_index); } - void write_word(uint64_t paddr, uint64_t data) { - return derived().do_write_word(paddr, data); + void write_tlb(TLB_set_index set_index, uint64_t slot_index, uint64_t vaddr_page, uint64_t vp_offset, + uint64_t pma_index) const { + derived().do_write_tlb(set_index, slot_index, vaddr_page, vp_offset, pma_index); } - /// \brief Resets uarch to pristine state - void reset_state() { - return derived().do_reset_state(); + constexpr const char *get_name() const { + return derived().do_get_name(); } }; +/// \brief SFINAE test implementation of the i_uarch_state_access interface +template +using is_an_i_uarch_state_access = + std::integral_constant>>; + +template +constexpr bool is_an_i_uarch_state_access_v = is_an_i_uarch_state_access::value; + } // namespace cartesi #endif diff --git a/src/interpret.cpp b/src/interpret.cpp index 231f8bb3b..52e0711ac 100644 --- a/src/interpret.cpp +++ b/src/interpret.cpp @@ -14,27 +14,6 @@ // with this program (see COPYING). If not, see . // -#include -#include -#include -#include -#include - -#ifdef MICROARCHITECTURE -#include "uarch-machine-state-access.h" -#include "uarch-runtime.h" -#else -#include "record-step-state-access.h" -#include "replay-step-state-access.h" -#include "state-access.h" -#include -#endif // MICROARCHITECTURE - -#include "compiler-defines.h" -#include "i-state-access.h" -#include "machine-statistics.h" -#include "shadow-tlb.h" - /// \file /// \brief Interpreter implementation. /// \details \{ @@ -102,13 +81,38 @@ /// https://gcc.gnu.org/onlinedocs/gcc-7.3.0/gcc/Arrays-and-pointers-implementation.html#Arrays-and-pointers-implementation /// \} -#include "find-pma-entry.h" #include "interpret.h" + +#include +#include +#include +#include +#include +#include + +#ifdef MICROARCHITECTURE +#include "../uarch/uarch-runtime.h" +#include "machine-uarch-bridge-state-access.h" +#else +#include "collect-mcycle-hashes-state-access.h" // IWYU pragma: keep +#include "record-step-state-access.h" // IWYU pragma: keep +#include "replay-step-state-access.h" // IWYU pragma: keep +#include "state-access.h" // IWYU pragma: keep + +#endif // MICROARCHITECTURE + +#include "assert-printf.h" +#include "compiler-defines.h" +#include "device-state-access.h" +#include "find-pma.h" +#include "i-accept-counters.h" // IWYU pragma: keep +#include "i-interactive-state-access.h" +#include "i-state-access.h" #include "meta.h" #include "riscv-constants.h" #include "rtc.h" +#include "shadow-tlb.h" #include "soft-float.h" -#include "strict-aliasing.h" #include "translate-virtual-address.h" #include "uint128.h" @@ -119,6 +123,24 @@ enum class rd_kind { xN, // rd is a positive natural number (1, 2, 3 ... 31) }; +template +static inline void DUMP_STATS_INCR([[maybe_unused]] const STATE_ACCESS a, [[maybe_unused]] const char *name) { +#ifdef DUMP_STATS + if constexpr (is_an_i_accept_counters_v) { + a.increment_counter(name, "stats."); + } +#endif +} + +template +static inline void DUMP_INSN_HIST_INCR([[maybe_unused]] const STATE_ACCESS a, [[maybe_unused]] const char *name) { +#ifdef DUMP_INSN_HIST + if constexpr (is_an_i_accept_counters_v) { + a.increment_counter(name, "insn."); + } +#endif +} + #ifdef DUMP_REGS static const std::array reg_name{"zero", "ra", "sp", "gp", "tp", "t0", "t1", "t2", "s0", "s1", "a0", "a1", "a2", "a3", "a4", "a5", "a6", "a7", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10", "s11", @@ -128,9 +150,6 @@ static const std::array f_reg_name{"ft0", "ft1", "ft2 "fs0", "fs1", "fa0", "fa1", "fa2", "fa3", "fa4", "fa5", "fa6", "fa7", "fs2", "fs3", "fs4", "fs5", "fs6", "fs7", "fs8", "fs9", "fs10", "fs11", "ft8", "ft9", "ft10", "ft11"}; -static void print_uint64_t(uint64_t a) { - std::ignore = fprintf(stderr, "%016" PRIx64, a); -} #endif #if defined(DUMP_EXCEPTIONS) || defined(DUMP_MMU_EXCEPTIONS) || defined(DUMP_INTERRUPTS) || \ @@ -160,145 +179,165 @@ static const char *sbi_ecall_name(uint64_t a7) { } } -template -static void dump_exception_or_interrupt(uint64_t cause, STATE &s) { - uint64_t a7 = s.x[17]; +static void dump_exception_or_interrupt(uint64_t cause, uint64_t a7) { if ((cause & MCAUSE_INTERRUPT_FLAG) != 0) { switch (cause & ~MCAUSE_INTERRUPT_FLAG) { case 0: - std::ignore = fprintf(stderr, "reserved software interrupt"); + d_printf("reserved software interrupt"); break; case 1: - std::ignore = fprintf(stderr, "supervisor software interrupt"); + d_printf("supervisor software interrupt"); break; case 2: - std::ignore = fprintf(stderr, "reserved software interrupt"); + d_printf("reserved software interrupt"); break; case 3: - std::ignore = fprintf(stderr, "machine software interrupt"); + d_printf("machine software interrupt"); break; case 4: - std::ignore = fprintf(stderr, "reserved timer interrupt"); + d_printf("reserved timer interrupt"); break; case 5: - std::ignore = fprintf(stderr, "supervisor timer interrupt"); + d_printf("supervisor timer interrupt"); break; case 6: - std::ignore = fprintf(stderr, "reserved timer interrupt"); + d_printf("reserved timer interrupt"); break; case 7: - std::ignore = fprintf(stderr, "machine timer interrupt"); + d_printf("machine timer interrupt"); break; case 8: - std::ignore = fprintf(stderr, "reserved external interrupt"); + d_printf("reserved external interrupt"); break; case 9: - std::ignore = fprintf(stderr, "supervisor external interrupt"); + d_printf("supervisor external interrupt"); break; case 10: - std::ignore = fprintf(stderr, "reserved external interrupt"); + d_printf("reserved external interrupt"); break; case 11: - std::ignore = fprintf(stderr, "machine external interrupt"); + d_printf("machine external interrupt"); break; default: - std::ignore = fprintf(stderr, "unknown interrupt"); + d_printf("unknown interrupt"); break; } } else { switch (cause) { case 0: - std::ignore = fprintf(stderr, "instruction address misaligned"); + d_printf("instruction address misaligned"); break; case 1: - std::ignore = fprintf(stderr, "instruction access fault"); + d_printf("instruction access fault"); break; case 2: - std::ignore = fprintf(stderr, "illegal instruction"); + d_printf("illegal instruction"); break; case 3: - std::ignore = fprintf(stderr, "breakpoint"); + d_printf("breakpoint"); break; case 4: - std::ignore = fprintf(stderr, "load address misaligned"); + d_printf("load address misaligned"); break; case 5: - std::ignore = fprintf(stderr, "load access fault"); + d_printf("load access fault"); break; case 6: - std::ignore = fprintf(stderr, "store/amo address misaligned"); + d_printf("store/amo address misaligned"); break; case 7: - std::ignore = fprintf(stderr, "store/amo access fault"); + d_printf("store/amo access fault"); break; case 8: - std::ignore = fprintf(stderr, "ecall %d from u-mode", static_cast(a7)); + d_printf("ecall %d from u-mode", static_cast(a7)); break; case 9: - std::ignore = fprintf(stderr, "ecall %s(%d) from s-mode", sbi_ecall_name(a7), static_cast(a7)); + d_printf("ecall %s(%d) from s-mode", sbi_ecall_name(a7), static_cast(a7)); break; case 10: - std::ignore = fprintf(stderr, "ecall %d reserved", static_cast(a7)); + d_printf("ecall %d reserved", static_cast(a7)); break; case 11: - std::ignore = fprintf(stderr, "ecall %s(%d) from m-mode", sbi_ecall_name(a7), static_cast(a7)); + d_printf("ecall %s(%d) from m-mode", sbi_ecall_name(a7), static_cast(a7)); break; case 12: - std::ignore = fprintf(stderr, "instruction page fault"); + d_printf("instruction page fault"); break; case 13: - std::ignore = fprintf(stderr, "load page fault"); + d_printf("load page fault"); break; case 15: - std::ignore = fprintf(stderr, "store/amo page fault"); + d_printf("store/amo page fault"); break; default: - std::ignore = fprintf(stderr, "reserved"); + d_printf("reserved"); break; } } } #endif +/// \brief Returns the name for a given privilege level +/// \param prv Privilege level +static constexpr const char *prv_get_name(uint64_t prv) { + switch (prv) { + case PRV_U: + return "prv.U"; + case PRV_S: + return "prv.S"; + case PRV_HS: + return "prv.HS"; + case PRV_M: + return "prv.M"; + default: + return "prv.unknown"; + } +} + #ifdef DUMP_REGS -template -static void dump_regs(const STATE &s) { - const std::array prv_str{"USHM"}; - int cols = 256 / XLEN; - std::ignore = fprintf(stderr, "pc = "); - print_uint64_t(s.pc); - std::ignore = fprintf(stderr, " "); +template +static void dump_regs(STATE_ACCESS &a) { + [[maybe_unused]] auto note = a.make_scoped_note("dump_regs"); +#define PRIxREG "0x%016" PRIx64 +#define PRIuREG "%" PRIu64 + const int cols = 256 / XLEN; + // First, read all needed registers from state access + // otherwise, DUMP_STATE_ACCESS output would get mixed in with DUMP_REGS + const auto pc = a.read_pc(); + const auto iprv = a.read_iprv(); + const auto mstatus = a.read_mstatus(); + const auto mcycle = a.read_mcycle(); + const auto icycleinstret = a.read_icycleinstret(); + const auto mideleg = a.read_mideleg(); + const auto mie = a.read_mie(); + const auto mip = a.read_mip(); + std::array x{}; for (int i = 1; i < X_REG_COUNT; i++) { - std::ignore = fprintf(stderr, "%-3s= ", reg_name[i]); - print_uint64_t(s.x[i]); - if ((i & (cols - 1)) == (cols - 1)) { - std::ignore = fprintf(stderr, "\n"); - } else { - std::ignore = fprintf(stderr, " "); - } + x[i] = a.read_x(i); } + std::array f{}; for (int i = 0; i < F_REG_COUNT; i++) { - std::ignore = fprintf(stderr, "%-3s= ", f_reg_name[i]); - print_uint64_t(s.f[i]); - if ((i & (cols - 1)) == (cols - 1)) { - std::ignore = fprintf(stderr, "\n"); - } else { - std::ignore = fprintf(stderr, " "); - } + f[i] = a.read_f(i); } - std::ignore = fprintf(stderr, "prv=%c", prv_str[s.iprv]); - std::ignore = fprintf(stderr, " mstatus="); - print_uint64_t(s.mstatus); - std::ignore = fprintf(stderr, " cycles=%" PRId64, s.mcycle); - std::ignore = fprintf(stderr, " insns=%" PRId64, s.mcycle - s.icycleinstret); - std::ignore = fprintf(stderr, "\n"); - std::ignore = fprintf(stderr, "mideleg="); - print_uint64_t(s.mideleg); - std::ignore = fprintf(stderr, " mie="); - print_uint64_t(s.mie); - std::ignore = fprintf(stderr, " mip="); - print_uint64_t(s.mip); - std::ignore = fprintf(stderr, "\n"); + // Now print them + d_printf("pc = " PRIxREG " ", pc); + for (int i = 1; i < X_REG_COUNT; i++) { + const char sep = ((i & (cols - 1)) == (cols - 1)) ? '\n' : ' '; + d_printf("%-4s= " PRIxREG "%c", reg_name[i], x[i], sep); + } + for (int i = 0; i < F_REG_COUNT; i++) { + const char sep = ((i & (cols - 1)) == (cols - 1)) ? '\n' : ' '; + d_printf("%-4s= " PRIxREG "%c", f_reg_name[i], f[i], sep); + } + d_printf("prv=%s", prv_get_name(iprv)); + d_printf(" mstatus=" PRIxREG "\n", mstatus); + d_printf(" cycles=" PRIuREG, mcycle); + d_printf(" insns=" PRIuREG "\n", mcycle - icycleinstret); + d_printf("mideleg=" PRIxREG, mideleg); + d_printf(" mie=" PRIxREG, mie); + d_printf(" mip=" PRIxREG "\n", mip); +#undef PRIxREG +#undef PRIuREG } #endif @@ -330,13 +369,13 @@ static inline uint32_t csr_prv(CSR_address csr) { /// \param new_prv New privilege level. /// \details This function is outlined to minimize host CPU code cache pressure. template -static FORCE_INLINE void set_prv(STATE_ACCESS a, int new_prv) { - INC_COUNTER(a.get_statistics(), priv_level[new_prv]); +static FORCE_INLINE void set_prv(STATE_ACCESS &a, uint64_t new_prv) { + DUMP_STATS_INCR(a, prv_get_name(new_prv)); a.write_iprv(new_prv); // Invalidate all TLB entries - a.flush_all_tlb(); - INC_COUNTER(a.get_statistics(), tlb_flush_all); - INC_COUNTER(a.get_statistics(), tlb_flush_set_prv); + flush_all_tlb(a); + DUMP_STATS_INCR(a, "tlb.flush_all"); + DUMP_STATS_INCR(a, "tlb.flush_set_prv"); //??D new privileged 1.11 draft says invalidation should // happen within a trap handler, although it could // also happen in xRET insn. @@ -351,23 +390,22 @@ static FORCE_INLINE void set_prv(STATE_ACCESS a, int new_prv) { /// \returns The new program counter, pointing to the raised exception trap handler. /// \details This function is outlined to minimize host CPU code cache pressure. template -static NO_INLINE uint64_t raise_exception(STATE_ACCESS a, uint64_t pc, uint64_t cause, uint64_t tval) { +static NO_INLINE uint64_t raise_exception(const STATE_ACCESS a, uint64_t pc, uint64_t cause, uint64_t tval) { if (cause == MCAUSE_ILLEGAL_INSN && !insn_is_uncompressed(static_cast(tval))) { // Discard high bits of compressed instructions, // this is not performed in the instruction hot loop as an optimization. tval = static_cast(tval); } - #if defined(DUMP_EXCEPTIONS) || defined(DUMP_MMU_EXCEPTIONS) || defined(DUMP_INTERRUPTS) || \ defined(DUMP_ILLEGAL_INSN_EXCEPTIONS) { - int flag; - flag = 0; + int flag = 0; #ifdef DUMP_MMU_EXCEPTIONS if (cause == MCAUSE_INSN_ACCESS_FAULT || cause == MCAUSE_LOAD_ACCESS_FAULT || cause == MCAUSE_STORE_AMO_ACCESS_FAULT || cause == MCAUSE_FETCH_PAGE_FAULT || - cause == MCAUSE_LOAD_PAGE_FAULT || cause == MCAUSE_STORE_AMO_PAGE_FAULT) + cause == MCAUSE_LOAD_PAGE_FAULT || cause == MCAUSE_STORE_AMO_PAGE_FAULT) { flag = 1; + } #endif #ifdef DUMP_INTERRUPTS flag |= (cause & MCAUSE_INTERRUPT_FLAG) != 0; @@ -376,19 +414,19 @@ static NO_INLINE uint64_t raise_exception(STATE_ACCESS a, uint64_t pc, uint64_t flag |= (cause & MCAUSE_INTERRUPT_FLAG) == 0; #endif #ifdef DUMP_ILLEGAL_INSN_EXCEPTIONS - if (cause == MCAUSE_ILLEGAL_INSN) + if (cause == MCAUSE_ILLEGAL_INSN) { flag = 1; + } #endif if (flag) { - std::ignore = fprintf(stderr, "raise_exception: cause=0x"); - print_uint64_t(cause); - std::ignore = fprintf(stderr, " tval=0x"); - print_uint64_t(tval); - std::ignore = fprintf(stderr, " ("); - dump_exception_or_interrupt(cause, a.get_naked_state()); - std::ignore = fprintf(stderr, ")\n"); + [[maybe_unused]] auto dnote = a.make_scoped_note("dump_exception"); + const auto a7 = a.read_x(17); + d_printf("raise_exception: cause=0x%016" PRIx64, cause); + d_printf(" tval=0x%016" PRIx64 " (", tval); + dump_exception_or_interrupt(cause, a7); + d_printf(")\n"); #ifdef DUMP_REGS - dump_regs(a.get_naked_state()); + dump_regs(a); #endif } } @@ -425,11 +463,12 @@ static NO_INLINE uint64_t raise_exception(STATE_ACCESS a, uint64_t pc, uint64_t set_prv(a, PRV_S); } new_pc = a.read_stvec(); -#ifdef DUMP_COUNTERS +#ifdef DUMP_STATS if (cause & MCAUSE_INTERRUPT_FLAG) { - INC_COUNTER(a.get_statistics(), sv_int); - } else if (cause >= MCAUSE_ECALL_BASE && cause <= MCAUSE_ECALL_BASE + PRV_M) { // Do not count environment calls - INC_COUNTER(a.get_statistics(), sv_ex); + DUMP_STATS_INCR(a, "sv_int"); + } else if (cause >= MCAUSE_ECALL_BASE && + cause <= +MCAUSE_ECALL_BASE + +PRV_M) { // Do not count environment calls + DUMP_STATS_INCR(a, "sv_ex"); } #endif } else { @@ -445,11 +484,12 @@ static NO_INLINE uint64_t raise_exception(STATE_ACCESS a, uint64_t pc, uint64_t set_prv(a, PRV_M); } new_pc = a.read_mtvec(); -#ifdef DUMP_COUNTERS +#ifdef DUMP_STATS if (cause & MCAUSE_INTERRUPT_FLAG) { - INC_COUNTER(a.get_statistics(), m_int); - } else if (cause >= MCAUSE_ECALL_BASE && cause <= MCAUSE_ECALL_BASE + PRV_M) { // Do not count environment calls - INC_COUNTER(a.get_statistics(), m_ex); + DUMP_STATS_INCR(a, "m_int"); + } else if (cause >= MCAUSE_ECALL_BASE && + cause <= +MCAUSE_ECALL_BASE + +PRV_M) { // Do not count environment calls + DUMP_STATS_INCR(a, "m_ex"); } #endif } @@ -460,7 +500,7 @@ static NO_INLINE uint64_t raise_exception(STATE_ACCESS a, uint64_t pc, uint64_t /// \param a Machine state accessor object. /// \returns The mask. template -static inline uint32_t get_pending_irq_mask(STATE_ACCESS a) { +static inline uint32_t get_pending_irq_mask(const STATE_ACCESS a) { const uint64_t mip = a.read_mip(); const uint64_t mie = a.read_mie(); @@ -540,7 +580,7 @@ static inline uint32_t get_highest_priority_irq_num(uint32_t v) { /// \param a Machine state accessor object. /// \param pc Machine current program counter. template -static inline uint64_t raise_interrupt_if_any(STATE_ACCESS a, uint64_t pc) { +static inline uint64_t raise_interrupt_if_any(const STATE_ACCESS a, uint64_t pc) { const uint32_t mask = get_pending_irq_mask(a); if (unlikely(mask != 0)) { const uint64_t irq_num = get_highest_priority_irq_num(mask); @@ -553,7 +593,7 @@ static inline uint64_t raise_interrupt_if_any(STATE_ACCESS a, uint64_t pc) { /// \param a Machine state accessor object. /// \param mcycle Machine current cycle. template -static inline void set_rtc_interrupt(STATE_ACCESS a, uint64_t mcycle) { +static inline void set_rtc_interrupt(const STATE_ACCESS a, uint64_t mcycle) { const uint64_t timecmp_cycle = rtc_time_to_cycle(a.read_clint_mtimecmp()); if (timecmp_cycle <= mcycle && timecmp_cycle != 0) { const uint64_t mip = a.read_mip(); @@ -823,6 +863,105 @@ static FORCE_INLINE int32_t insn_get_C_SWSP_imm(uint32_t insn) { return static_cast(((insn >> (9 - 2)) & 0x3c) | ((insn >> (7 - 6)) & 0xc0)); } +/// \brief Flushes out a TLB slot +/// \tparam USE TLB set +/// \tparam STATE_ACCESS Class of machine state accessor object. +/// \param a Machine state accessor object. +/// \param slot_index Slot index +template +static void flush_tlb_slot(const STATE_ACCESS a, uint64_t slot_index) { + // Make sure a valid page leaving the write TLB is marked as dirty + // We must do this BEFORE we modify the TLB entries themselves + // (Otherwise, we could stop uarch before it marks the page dirty but after + // the entry is no longer in the TLB, which would cause the hash tree to + // miss a dirty page.) + if constexpr (SET == TLB_WRITE) { + const auto old_vaddr_page = a.template read_tlb_vaddr_page(slot_index); + if (old_vaddr_page != TLB_INVALID_PAGE) { + auto old_pma_index = a.template read_tlb_pma_index(slot_index); + const auto old_faddr_page = old_vaddr_page + a.template read_tlb_vf_offset(slot_index); + a.mark_dirty_page(old_faddr_page, old_pma_index); + } + } + // We do not leave garbage behind in empty slots + // (It would make state access classes trickier to implement) + const auto vaddr_page = TLB_INVALID_PAGE; + const auto vf_offset = i_state_access_fast_addr_t{}; + const auto pma_index = TLB_INVALID_PMA_INDEX; + a.template write_tlb(slot_index, vaddr_page, vf_offset, pma_index); +} + +/// \brief Flushes out an entire TLB set +/// \tparam USE TLB set +/// \tparam STATE_ACCESS Class of machine state accessor object. +/// \param a Machine state accessor object. +template +static void flush_tlb_set(const STATE_ACCESS a) { + for (uint64_t slot_index = 0; slot_index < TLB_SET_SIZE; ++slot_index) { + flush_tlb_slot(a, slot_index); + } +} + +/// \brief Flushes out the entire TLB +/// \tparam STATE_ACCESS Class of machine state accessor object. +/// \param a Machine state accessor object. +template +static void flush_all_tlb(const STATE_ACCESS a) { + flush_tlb_set(a); + flush_tlb_set(a); + flush_tlb_set(a); +} + +/// \brief Flushes out a single virtual mapping in the TLB +/// \tparam STATE_ACCESS Class of machine state accessor object. +/// \param a Machine state accessor object. +/// \param vaddr Virtual address to remove form mapping +template +static void flush_tlb_vaddr(const STATE_ACCESS a, uint64_t /* vaddr */) { + // We can't flush just one TLB entry for that specific virtual address, + // because megapages/gigapages may be in use while this TLB implementation ignores it, + // so we have to flush all addresses. + flush_tlb_set(a); + flush_tlb_set(a); + flush_tlb_set(a); +} + +/// \brief Replaces a virtual mapping in a TLB set +/// \tparam USE TLB set +/// \tparam STATE_ACCESS Class of machine state accessor object. +/// \param a Machine state accessor object. +/// \param vaddr Virtual address of new mapping +/// \param paddr Corresponding physical address +/// \param paddr Index of PMA where paddr falls +/// \param vf_offset Receives the new vf_offset that will be stored in the slot +/// \returns The implementation-defined fast address corresponding to paddr +template +static i_state_access_fast_addr_t replace_tlb_entry(const STATE_ACCESS a, uint64_t vaddr, uint64_t paddr, + uint64_t pma_index, i_state_access_fast_addr_t &vf_offset) { + [[maybe_unused]] auto note = a.make_scoped_note("replace_tlb_entry"); + const auto slot_index = tlb_slot_index(vaddr); + flush_tlb_slot(a, slot_index); + const auto vaddr_page = tlb_addr_page(vaddr); + const auto faddr = a.get_faddr(paddr, pma_index); + vf_offset = faddr - vaddr; + a.template write_tlb(slot_index, vaddr_page, vf_offset, pma_index); + return faddr; +} + +/// \brief Replaces a virtual mapping in a TLB set +/// \tparam USE TLB set +/// \tparam STATE_ACCESS Class of machine state accessor object. +/// \param a Machine state accessor object. +/// \param vaddr Virtual address of new mapping +/// \param paddr Corresponding physical address +/// \param paddr Index of PMA where paddr falls +/// \returns The implementation-defined fast address corresponding to paddr +template +static FORCE_INLINE auto replace_tlb_entry(const STATE_ACCESS a, uint64_t vaddr, uint64_t paddr, uint64_t pma_index) { + i_state_access_fast_addr_t vf_offset{0}; + return replace_tlb_entry(a, vaddr, paddr, pma_index, vf_offset); +} + /// \brief Read an aligned word from virtual memory (slow path that goes through virtual address translation). /// \tparam T uint8_t, uint16_t, uint32_t, or uint64_t. /// \tparam STATE_ACCESS Class of machine state accessor object. @@ -839,8 +978,9 @@ static FORCE_INLINE int32_t insn_get_C_SWSP_imm(uint32_t insn) { /// is outlined, and taking PC by reference would cause the compiler to store it in a stack variable /// instead of always storing it in register (this is an optimization). template -static NO_INLINE std::pair read_virtual_memory_slow(STATE_ACCESS a, uint64_t pc, uint64_t mcycle, +static NO_INLINE std::pair read_virtual_memory_slow(const STATE_ACCESS a, uint64_t pc, uint64_t mcycle, uint64_t vaddr, T *pval) { + [[maybe_unused]] auto note = a.make_scoped_note("read_virtual_memory_slow"); using U = std::make_unsigned_t; // No support for misaligned accesses: They are handled by a trap in BBL if (unlikely(vaddr & (sizeof(T) - 1))) { @@ -855,21 +995,22 @@ static NO_INLINE std::pair read_virtual_memory_slow(STATE_ACCESS vaddr); return {false, pc}; } - auto &pma = find_pma_entry(a, paddr); - if (likely(pma.get_istart_R())) { - if (likely(pma.get_istart_M())) { - unsigned char *hpage = a.template replace_tlb_entry(vaddr, paddr, pma); - const uint64_t hoffset = vaddr & PAGE_OFFSET_MASK; - a.read_memory_word(paddr, hpage, hoffset, pval); + uint64_t pma_index = 0; + const auto &ar = find_pma(a, paddr, pma_index); + if (likely(ar.is_readable())) { + if (likely(ar.is_memory())) { + [[maybe_unused]] auto note = a.make_scoped_note("read memory"); + const auto faddr = replace_tlb_entry(a, vaddr, paddr, pma_index); + a.template read_memory_word(faddr, pma_index, pval); return {true, pc}; } - if (likely(pma.get_istart_IO())) { - const uint64_t offset = paddr - pma.get_start(); + if (likely(ar.is_device())) { + [[maybe_unused]] auto note = a.make_scoped_note("read device"); + const uint64_t offset = paddr - ar.get_start(); uint64_t val{}; device_state_access da(a, mcycle); // If we do not know how to read, we treat this as a PMA violation - const bool status = pma.get_device_noexcept().get_driver()->read(pma.get_device_noexcept().get_context(), - &da, offset, &val, log2_size::value); + const bool status = ar.read_device(&da, offset, log2_size_v, &val); if (likely(status)) { *pval = static_cast(val); // device logs its own state accesses @@ -891,19 +1032,27 @@ static NO_INLINE std::pair read_virtual_memory_slow(STATE_ACCESS /// \param pval Pointer to word receiving value. /// \returns True if succeeded, false otherwise. template -static FORCE_INLINE bool read_virtual_memory(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint64_t vaddr, T *pval) { +static FORCE_INLINE bool read_virtual_memory(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint64_t vaddr, + T *pval) { + [[maybe_unused]] auto note = a.make_scoped_note("read_virtual_memory"); // Try hitting the TLB - if (unlikely(!(a.template read_memory_word_via_tlb(vaddr, pval)))) { + const auto slot_index = tlb_slot_index(vaddr); + const auto slot_vaddr_page = a.template read_tlb_vaddr_page(slot_index); + if (unlikely(!tlb_is_hit(slot_vaddr_page, vaddr))) { // Outline the slow path into a function call to minimize host CPU code cache pressure - INC_COUNTER(a.get_statistics(), tlb_rmiss); T val = 0; // Don't pass pval reference directly so the compiler can store it in a register + DUMP_STATS_INCR(a, "tlb.rmiss"); auto [status, new_pc] = read_virtual_memory_slow(a, pc, mcycle, vaddr, &val); *pval = val; pc = new_pc; return status; } - INC_COUNTER(a.get_statistics(), tlb_rhit); + const auto pma_index = a.template read_tlb_pma_index(slot_index); + const auto vf_offset = a.template read_tlb_vf_offset(slot_index); + const auto faddr = vaddr + vf_offset; + a.template read_memory_word(faddr, pma_index, pval); + DUMP_STATS_INCR(a, "tlb.rhit"); return true; } @@ -922,8 +1071,9 @@ static FORCE_INLINE bool read_virtual_memory(STATE_ACCESS a, uint64_t &pc, uint6 /// is outlined, and taking PC by reference would cause the compiler to store it in a stack variable /// instead of always storing it in register (this is an optimization). template -static NO_INLINE std::pair write_virtual_memory_slow(STATE_ACCESS a, uint64_t pc, +static NO_INLINE std::pair write_virtual_memory_slow(const STATE_ACCESS a, uint64_t pc, uint64_t mcycle, uint64_t vaddr, uint64_t val64) { + [[maybe_unused]] auto note = a.make_scoped_note("write_virtual_memory_slow"); using U = std::make_unsigned_t; // No support for misaligned accesses: They are handled by a trap in BBL if (unlikely(vaddr & (sizeof(T) - 1))) { @@ -936,19 +1086,18 @@ static NO_INLINE std::pair write_virtual_memory_slow(S pc = raise_exception(a, pc, MCAUSE_STORE_AMO_PAGE_FAULT, vaddr); return {execute_status::failure, pc}; } - auto &pma = find_pma_entry(a, paddr); - if (likely(pma.get_istart_W())) { - if (likely(pma.get_istart_M())) { - unsigned char *hpage = a.template replace_tlb_entry(vaddr, paddr, pma); - const uint64_t hoffset = vaddr & PAGE_OFFSET_MASK; - a.write_memory_word(paddr, hpage, hoffset, static_cast(val64)); + uint64_t pma_index = 0; + auto &ar = find_pma(a, paddr, pma_index); + if (likely(ar.is_writeable())) { + if (likely(ar.is_memory())) { + const auto faddr = replace_tlb_entry(a, vaddr, paddr, pma_index); + a.write_memory_word(faddr, pma_index, static_cast(val64)); return {execute_status::success, pc}; } - if (likely(pma.get_istart_IO())) { - const uint64_t offset = paddr - pma.get_start(); + if (likely(ar.is_device())) { + const uint64_t offset = paddr - ar.get_start(); device_state_access da(a, mcycle); - auto status = pma.get_device_noexcept().get_driver()->write(pma.get_device_noexcept().get_context(), &da, - offset, static_cast(static_cast(val64)), log2_size::value); + auto status = ar.write_device(&da, offset, log2_size_v, static_cast(static_cast(val64))); // If we do not know how to write, we treat this as a PMA violation if (likely(status != execute_status::failure)) { return {status, pc}; @@ -968,44 +1117,45 @@ static NO_INLINE std::pair write_virtual_memory_slow(S /// \param val64 Value to write. /// \returns True if succeeded, false if exception raised. template -static FORCE_INLINE execute_status write_virtual_memory(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint64_t vaddr, - uint64_t val64) { +static FORCE_INLINE execute_status write_virtual_memory(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, + uint64_t vaddr, uint64_t val64) { + [[maybe_unused]] auto note = a.make_scoped_note("write_virtual_memory"); // Try hitting the TLB - if (unlikely((!a.template write_memory_word_via_tlb(vaddr, static_cast(val64))))) { - INC_COUNTER(a.get_statistics(), tlb_wmiss); + const uint64_t slot_index = tlb_slot_index(vaddr); + const uint64_t slot_vaddr_page = a.template read_tlb_vaddr_page(slot_index); + if (unlikely(!tlb_is_hit(slot_vaddr_page, vaddr))) { // Outline the slow path into a function call to minimize host CPU code cache pressure + DUMP_STATS_INCR(a, "tlb.wmiss"); auto [status, new_pc] = write_virtual_memory_slow(a, pc, mcycle, vaddr, val64); pc = new_pc; return status; } - INC_COUNTER(a.get_statistics(), tlb_whit); + const auto pma_index = a.template read_tlb_pma_index(slot_index); + const auto vf_offset = a.template read_tlb_vf_offset(slot_index); + const auto faddr = vaddr + vf_offset; + a.template write_memory_word(faddr, pma_index, static_cast(val64)); + DUMP_STATS_INCR(a, "tlb.whit"); return execute_status::success; } template -static void dump_insn([[maybe_unused]] STATE_ACCESS a, [[maybe_unused]] uint64_t pc, [[maybe_unused]] uint32_t insn, - [[maybe_unused]] const char *name) { -#ifdef DUMP_HIST - a.get_naked_state().insn_hist[name]++; -#endif +static auto dump_insn([[maybe_unused]] const STATE_ACCESS a, [[maybe_unused]] uint64_t pc, + [[maybe_unused]] uint32_t insn, [[maybe_unused]] const char *name) { + DUMP_INSN_HIST_INCR(a, name); #ifdef DUMP_REGS - dump_regs(a.get_naked_state()); + dump_regs(a); #endif #ifdef DUMP_INSN - uint64_t ppc = 0; - // If we are running in the microinterpreter, we may or may not be collecting a step access log. - // To prevent additional address translation end up in the log, - // the following check will always be false when MICROARCHITECTURE is defined. - if (std::is_same_v && - !translate_virtual_address(a, &ppc, pc, PTE_XWR_X_SHIFT)) { - ppc = pc; - fprintf(stderr, "v %08" PRIx64, ppc); + [[maybe_unused]] auto note = a.make_scoped_note("dump_insn"); + uint64_t ppc = pc; + if (!translate_virtual_address(a, &ppc, pc, PTE_XWR_X_SHIFT)) { + d_printf("v %08" PRIx64, ppc); } else { - fprintf(stderr, "p %08" PRIx64, ppc); + d_printf("p %08" PRIx64, ppc); } - fprintf(stderr, ": %08" PRIx32 " ", insn); - fprintf(stderr, "%s\n", name); + d_printf(": %08" PRIx32 " %s\n", insn, name); #endif + return a.make_scoped_note(name); } /// \brief Raises an illegal instruction exception, updating the pc. @@ -1017,7 +1167,7 @@ static void dump_insn([[maybe_unused]] STATE_ACCESS a, [[maybe_unused]] uint64_t /// \details This function is tail-called whenever the caller decoded enough of the instruction to identify it as /// illegal. template -static FORCE_INLINE execute_status raise_illegal_insn_exception(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { +static FORCE_INLINE execute_status raise_illegal_insn_exception(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { pc = raise_exception(a, pc, MCAUSE_ILLEGAL_INSN, insn); return execute_status::failure; } @@ -1029,7 +1179,8 @@ static FORCE_INLINE execute_status raise_illegal_insn_exception(STATE_ACCESS a, /// \return execute_status::failure /// \details This function is tail-called whenever the caller identified that the next value of pc is misaligned. template -static FORCE_INLINE execute_status raise_misaligned_fetch_exception(STATE_ACCESS a, uint64_t &pc, uint64_t new_pc) { +static FORCE_INLINE execute_status raise_misaligned_fetch_exception(const STATE_ACCESS a, uint64_t &pc, + uint64_t new_pc) { pc = raise_exception(a, pc, MCAUSE_INSN_ADDRESS_MISALIGNED, new_pc); return execute_status::failure; } @@ -1078,7 +1229,7 @@ static FORCE_INLINE execute_status execute_jump(STATE_ACCESS /*a*/, uint64_t &pc /// \param pc Interpreter loop program counter (will be overwritten). /// \param insn Instruction. template -static FORCE_INLINE execute_status execute_LR(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { +static FORCE_INLINE execute_status execute_LR(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { const uint64_t vaddr = a.read_x(insn_get_rs1(insn)); T val = 0; if (unlikely(!read_virtual_memory(a, pc, mcycle, vaddr, &val))) { @@ -1099,7 +1250,7 @@ static FORCE_INLINE execute_status execute_LR(STATE_ACCESS a, uint64_t &pc, uint /// \param pc Interpreter loop program counter (will be overwritten). /// \param insn Instruction. template -static FORCE_INLINE execute_status execute_SC(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { +static FORCE_INLINE execute_status execute_SC(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { uint64_t val = 0; const uint64_t vaddr = a.read_x(insn_get_rs1(insn)); execute_status status = execute_status::success; @@ -1122,23 +1273,23 @@ static FORCE_INLINE execute_status execute_SC(STATE_ACCESS a, uint64_t &pc, uint /// \brief Implementation of the LR.W instruction. template -static FORCE_INLINE execute_status execute_LR_W(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { +static FORCE_INLINE execute_status execute_LR_W(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { if (unlikely((insn & 0b00000001111100000000000000000000) != 0)) { return raise_illegal_insn_exception(a, pc, insn); } - dump_insn(a, pc, insn, "lr.w"); + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "lr.w"); return execute_LR(a, pc, mcycle, insn); } /// \brief Implementation of the SC.W instruction. template -static FORCE_INLINE execute_status execute_SC_W(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { - dump_insn(a, pc, insn, "sc.w"); +static FORCE_INLINE execute_status execute_SC_W(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "sc.w"); return execute_SC(a, pc, mcycle, insn); } template -static FORCE_INLINE execute_status execute_AMO(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn, +static FORCE_INLINE execute_status execute_AMO(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn, const F &f) { const uint64_t vaddr = a.read_x(insn_get_rs1(insn)); T valm = 0; @@ -1163,15 +1314,17 @@ static FORCE_INLINE execute_status execute_AMO(STATE_ACCESS a, uint64_t &pc, uin /// \brief Implementation of the AMOSWAP.W instruction. template -static FORCE_INLINE execute_status execute_AMOSWAP_W(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { - dump_insn(a, pc, insn, "amoswap.w"); +static FORCE_INLINE execute_status execute_AMOSWAP_W(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, + uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "amoswap.w"); return execute_AMO(a, pc, mcycle, insn, [](int32_t /*valm*/, int32_t valr) -> int32_t { return valr; }); } /// \brief Implementation of the AMOADD.W instruction. template -static FORCE_INLINE execute_status execute_AMOADD_W(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { - dump_insn(a, pc, insn, "amoadd.w"); +static FORCE_INLINE execute_status execute_AMOADD_W(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, + uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "amoadd.w"); return execute_AMO(a, pc, mcycle, insn, [](int32_t valm, int32_t valr) -> int32_t { int32_t val = 0; __builtin_add_overflow(valm, valr, &val); @@ -1180,29 +1333,32 @@ static FORCE_INLINE execute_status execute_AMOADD_W(STATE_ACCESS a, uint64_t &pc } template -static FORCE_INLINE execute_status execute_AMOXOR_W(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { - dump_insn(a, pc, insn, "amoxor.w"); +static FORCE_INLINE execute_status execute_AMOXOR_W(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, + uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "amoxor.w"); return execute_AMO(a, pc, mcycle, insn, [](int32_t valm, int32_t valr) -> int32_t { return valm ^ valr; }); } /// \brief Implementation of the AMOAND.W instruction. template -static FORCE_INLINE execute_status execute_AMOAND_W(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { - dump_insn(a, pc, insn, "amoand.w"); +static FORCE_INLINE execute_status execute_AMOAND_W(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, + uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "amoand.w"); return execute_AMO(a, pc, mcycle, insn, [](int32_t valm, int32_t valr) -> int32_t { return valm & valr; }); } /// \brief Implementation of the AMOOR.W instruction. template -static FORCE_INLINE execute_status execute_AMOOR_W(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { - dump_insn(a, pc, insn, "amoor.w"); +static FORCE_INLINE execute_status execute_AMOOR_W(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "amoor.w"); return execute_AMO(a, pc, mcycle, insn, [](int32_t valm, int32_t valr) -> int32_t { return valm | valr; }); } /// \brief Implementation of the AMOMIN.W instruction. template -static FORCE_INLINE execute_status execute_AMOMIN_W(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { - dump_insn(a, pc, insn, "amomin.w"); +static FORCE_INLINE execute_status execute_AMOMIN_W(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, + uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "amomin.w"); return execute_AMO(a, pc, mcycle, insn, [](int32_t valm, int32_t valr) -> int32_t { if (valm < valr) { return valm; @@ -1213,8 +1369,9 @@ static FORCE_INLINE execute_status execute_AMOMIN_W(STATE_ACCESS a, uint64_t &pc /// \brief Implementation of the AMOMAX.W instruction. template -static FORCE_INLINE execute_status execute_AMOMAX_W(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { - dump_insn(a, pc, insn, "amomax.w"); +static FORCE_INLINE execute_status execute_AMOMAX_W(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, + uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "amomax.w"); return execute_AMO(a, pc, mcycle, insn, [](int32_t valm, int32_t valr) -> int32_t { if (valm > valr) { return valm; @@ -1225,8 +1382,9 @@ static FORCE_INLINE execute_status execute_AMOMAX_W(STATE_ACCESS a, uint64_t &pc /// \brief Implementation of the AMOMINU.W instruction. template -static FORCE_INLINE execute_status execute_AMOMINU_W(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { - dump_insn(a, pc, insn, "amominu.w"); +static FORCE_INLINE execute_status execute_AMOMINU_W(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, + uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "amominu.w"); return execute_AMO(a, pc, mcycle, insn, [](int32_t valm, int32_t valr) -> int32_t { if (static_cast(valm) < static_cast(valr)) { return valm; @@ -1237,8 +1395,9 @@ static FORCE_INLINE execute_status execute_AMOMINU_W(STATE_ACCESS a, uint64_t &p /// \brief Implementation of the AMOMAXU.W instruction. template -static FORCE_INLINE execute_status execute_AMOMAXU_W(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { - dump_insn(a, pc, insn, "amomaxu.w"); +static FORCE_INLINE execute_status execute_AMOMAXU_W(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, + uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "amomaxu.w"); return execute_AMO(a, pc, mcycle, insn, [](int32_t valm, int32_t valr) -> int32_t { if (static_cast(valm) > static_cast(valr)) { return valm; @@ -1249,32 +1408,34 @@ static FORCE_INLINE execute_status execute_AMOMAXU_W(STATE_ACCESS a, uint64_t &p /// \brief Implementation of the LR.D instruction. template -static FORCE_INLINE execute_status execute_LR_D(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { +static FORCE_INLINE execute_status execute_LR_D(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { if (unlikely((insn & 0b00000001111100000000000000000000) != 0)) { return raise_illegal_insn_exception(a, pc, insn); } - dump_insn(a, pc, insn, "lr.d"); + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "lr.d"); return execute_LR(a, pc, mcycle, insn); } /// \brief Implementation of the SC.D instruction. template -static FORCE_INLINE execute_status execute_SC_D(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { - dump_insn(a, pc, insn, "sc.d"); +static FORCE_INLINE execute_status execute_SC_D(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "sc.d"); return execute_SC(a, pc, mcycle, insn); } /// \brief Implementation of the AMOSWAP.D instruction. template -static FORCE_INLINE execute_status execute_AMOSWAP_D(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { - dump_insn(a, pc, insn, "amoswap.d"); +static FORCE_INLINE execute_status execute_AMOSWAP_D(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, + uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "amoswap.d"); return execute_AMO(a, pc, mcycle, insn, [](int64_t /*valm*/, int64_t valr) -> int64_t { return valr; }); } /// \brief Implementation of the AMOADD.D instruction. template -static FORCE_INLINE execute_status execute_AMOADD_D(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { - dump_insn(a, pc, insn, "amoadd.d"); +static FORCE_INLINE execute_status execute_AMOADD_D(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, + uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "amoadd.d"); return execute_AMO(a, pc, mcycle, insn, [](int64_t valm, int64_t valr) -> int64_t { int64_t val = 0; __builtin_add_overflow(valm, valr, &val); @@ -1283,29 +1444,32 @@ static FORCE_INLINE execute_status execute_AMOADD_D(STATE_ACCESS a, uint64_t &pc } template -static FORCE_INLINE execute_status execute_AMOXOR_D(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { - dump_insn(a, pc, insn, "amoxor.d"); +static FORCE_INLINE execute_status execute_AMOXOR_D(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, + uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "amoxor.d"); return execute_AMO(a, pc, mcycle, insn, [](int64_t valm, int64_t valr) -> int64_t { return valm ^ valr; }); } /// \brief Implementation of the AMOAND.D instruction. template -static FORCE_INLINE execute_status execute_AMOAND_D(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { - dump_insn(a, pc, insn, "amoand.d"); +static FORCE_INLINE execute_status execute_AMOAND_D(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, + uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "amoand.d"); return execute_AMO(a, pc, mcycle, insn, [](int64_t valm, int64_t valr) -> int64_t { return valm & valr; }); } /// \brief Implementation of the AMOOR.D instruction. template -static FORCE_INLINE execute_status execute_AMOOR_D(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { - dump_insn(a, pc, insn, "amoor.d"); +static FORCE_INLINE execute_status execute_AMOOR_D(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "amoor.d"); return execute_AMO(a, pc, mcycle, insn, [](int64_t valm, int64_t valr) -> int64_t { return valm | valr; }); } /// \brief Implementation of the AMOMIN.D instruction. template -static FORCE_INLINE execute_status execute_AMOMIN_D(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { - dump_insn(a, pc, insn, "amomin.d"); +static FORCE_INLINE execute_status execute_AMOMIN_D(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, + uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "amomin.d"); return execute_AMO(a, pc, mcycle, insn, [](int64_t valm, int64_t valr) -> int64_t { if (valm < valr) { return valm; @@ -1316,8 +1480,9 @@ static FORCE_INLINE execute_status execute_AMOMIN_D(STATE_ACCESS a, uint64_t &pc /// \brief Implementation of the AMOMAX.D instruction. template -static FORCE_INLINE execute_status execute_AMOMAX_D(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { - dump_insn(a, pc, insn, "amomax.d"); +static FORCE_INLINE execute_status execute_AMOMAX_D(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, + uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "amomax.d"); return execute_AMO(a, pc, mcycle, insn, [](int64_t valm, int64_t valr) -> int64_t { if (valm > valr) { return valm; @@ -1328,8 +1493,9 @@ static FORCE_INLINE execute_status execute_AMOMAX_D(STATE_ACCESS a, uint64_t &pc /// \brief Implementation of the AMOMINU.D instruction. template -static FORCE_INLINE execute_status execute_AMOMINU_D(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { - dump_insn(a, pc, insn, "amominu.d"); +static FORCE_INLINE execute_status execute_AMOMINU_D(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, + uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "amominu.d"); return execute_AMO(a, pc, mcycle, insn, [](uint64_t valm, uint64_t valr) -> uint64_t { if (valm < valr) { return valm; @@ -1340,8 +1506,9 @@ static FORCE_INLINE execute_status execute_AMOMINU_D(STATE_ACCESS a, uint64_t &p /// \brief Implementation of the AMOMAXU.D instruction. template -static FORCE_INLINE execute_status execute_AMOMAXU_D(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { - dump_insn(a, pc, insn, "amomaxu.d"); +static FORCE_INLINE execute_status execute_AMOMAXU_D(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, + uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "amomaxu.d"); return execute_AMO(a, pc, mcycle, insn, [](uint64_t valm, uint64_t valr) -> uint64_t { if (valm > valr) { return valm; @@ -1352,8 +1519,8 @@ static FORCE_INLINE execute_status execute_AMOMAXU_D(STATE_ACCESS a, uint64_t &p /// \brief Implementation of the ADDW instruction. template -static FORCE_INLINE execute_status execute_ADDW(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "addw"); +static FORCE_INLINE execute_status execute_ADDW(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "addw"); if constexpr (rd_kind == rd_kind::x0) { return advance_to_next_insn(a, pc); } @@ -1369,8 +1536,8 @@ static FORCE_INLINE execute_status execute_ADDW(STATE_ACCESS a, uint64_t &pc, ui /// \brief Implementation of the SUBW instruction. template -static FORCE_INLINE execute_status execute_SUBW(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "subw"); +static FORCE_INLINE execute_status execute_SUBW(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "subw"); if constexpr (rd_kind == rd_kind::x0) { return advance_to_next_insn(a, pc); } @@ -1386,11 +1553,11 @@ static FORCE_INLINE execute_status execute_SUBW(STATE_ACCESS a, uint64_t &pc, ui /// \brief Implementation of the SLLW instruction. template -static FORCE_INLINE execute_status execute_SLLW(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { +static FORCE_INLINE execute_status execute_SLLW(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { if (unlikely((insn & 0b11111110000000000111000001111111) != 0b00000000000000000001000000111011)) { return raise_illegal_insn_exception(a, pc, insn); } - dump_insn(a, pc, insn, "sllw"); + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "sllw"); if constexpr (rd_kind == rd_kind::x0) { return advance_to_next_insn(a, pc); } @@ -1402,8 +1569,8 @@ static FORCE_INLINE execute_status execute_SLLW(STATE_ACCESS a, uint64_t &pc, ui /// \brief Implementation of the SRLW instruction. template -static FORCE_INLINE execute_status execute_SRLW(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "srlw"); +static FORCE_INLINE execute_status execute_SRLW(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "srlw"); if constexpr (rd_kind == rd_kind::x0) { return advance_to_next_insn(a, pc); } @@ -1415,8 +1582,8 @@ static FORCE_INLINE execute_status execute_SRLW(STATE_ACCESS a, uint64_t &pc, ui /// \brief Implementation of the SRAW instruction. template -static FORCE_INLINE execute_status execute_SRAW(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "sraw"); +static FORCE_INLINE execute_status execute_SRAW(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "sraw"); if constexpr (rd_kind == rd_kind::x0) { return advance_to_next_insn(a, pc); } @@ -1428,8 +1595,8 @@ static FORCE_INLINE execute_status execute_SRAW(STATE_ACCESS a, uint64_t &pc, ui /// \brief Implementation of the MULW instruction. template -static FORCE_INLINE execute_status execute_MULW(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "mulw"); +static FORCE_INLINE execute_status execute_MULW(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "mulw"); if constexpr (rd_kind == rd_kind::x0) { return advance_to_next_insn(a, pc); } @@ -1444,11 +1611,11 @@ static FORCE_INLINE execute_status execute_MULW(STATE_ACCESS a, uint64_t &pc, ui /// \brief Implementation of the DIVW instruction. template -static FORCE_INLINE execute_status execute_DIVW(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { +static FORCE_INLINE execute_status execute_DIVW(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { if (unlikely((insn & 0b11111110000000000111000001111111) != 0b00000010000000000100000000111011)) { return raise_illegal_insn_exception(a, pc, insn); } - dump_insn(a, pc, insn, "divw"); + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "divw"); if constexpr (rd_kind == rd_kind::x0) { return advance_to_next_insn(a, pc); } @@ -1467,8 +1634,8 @@ static FORCE_INLINE execute_status execute_DIVW(STATE_ACCESS a, uint64_t &pc, ui /// \brief Implementation of the DIVUW instruction. template -static FORCE_INLINE execute_status execute_DIVUW(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "divuw"); +static FORCE_INLINE execute_status execute_DIVUW(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "divuw"); if constexpr (rd_kind == rd_kind::x0) { return advance_to_next_insn(a, pc); } @@ -1484,11 +1651,11 @@ static FORCE_INLINE execute_status execute_DIVUW(STATE_ACCESS a, uint64_t &pc, u /// \brief Implementation of the REMW instruction. template -static FORCE_INLINE execute_status execute_REMW(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { +static FORCE_INLINE execute_status execute_REMW(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { if (unlikely((insn & 0b11111110000000000111000001111111) != 0b00000010000000000110000000111011)) { return raise_illegal_insn_exception(a, pc, insn); } - dump_insn(a, pc, insn, "remw"); + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "remw"); if constexpr (rd_kind == rd_kind::x0) { return advance_to_next_insn(a, pc); } @@ -1507,11 +1674,11 @@ static FORCE_INLINE execute_status execute_REMW(STATE_ACCESS a, uint64_t &pc, ui /// \brief Implementation of the REMUW instruction. template -static FORCE_INLINE execute_status execute_REMUW(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { +static FORCE_INLINE execute_status execute_REMUW(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { if (unlikely((insn & 0b11111110000000000111000001111111) != 0b00000010000000000111000000111011)) { return raise_illegal_insn_exception(a, pc, insn); } - dump_insn(a, pc, insn, "remuw"); + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "remuw"); if constexpr (rd_kind == rd_kind::x0) { return advance_to_next_insn(a, pc); } @@ -1536,7 +1703,7 @@ static inline uint64_t read_csr_success(uint64_t val, bool *status) { } template -static inline bool rdcounteren(STATE_ACCESS a, uint64_t mask) { +static inline bool rdcounteren(const STATE_ACCESS a, uint64_t mask) { uint64_t counteren = MCOUNTEREN_R_MASK; auto prv = a.read_iprv(); if (prv <= PRV_S) { @@ -1549,7 +1716,7 @@ static inline bool rdcounteren(STATE_ACCESS a, uint64_t mask) { } template -static inline uint64_t read_csr_cycle(STATE_ACCESS a, uint64_t mcycle, bool *status) { +static inline uint64_t read_csr_cycle(const STATE_ACCESS a, uint64_t mcycle, bool *status) { if (rdcounteren(a, MCOUNTEREN_CY_MASK)) { return read_csr_success(mcycle, status); } @@ -1557,7 +1724,7 @@ static inline uint64_t read_csr_cycle(STATE_ACCESS a, uint64_t mcycle, bool *sta } template -static inline uint64_t read_csr_instret(STATE_ACCESS a, uint64_t mcycle, bool *status) { +static inline uint64_t read_csr_instret(const STATE_ACCESS a, uint64_t mcycle, bool *status) { if (unlikely(!rdcounteren(a, MCOUNTEREN_IR_MASK))) { return read_csr_fail(status); } @@ -1567,7 +1734,7 @@ static inline uint64_t read_csr_instret(STATE_ACCESS a, uint64_t mcycle, bool *s } template -static inline uint64_t read_csr_time(STATE_ACCESS a, uint64_t mcycle, bool *status) { +static inline uint64_t read_csr_time(const STATE_ACCESS a, uint64_t mcycle, bool *status) { if (unlikely(!rdcounteren(a, MCOUNTEREN_TM_MASK))) { return read_csr_fail(status); } @@ -1576,54 +1743,54 @@ static inline uint64_t read_csr_time(STATE_ACCESS a, uint64_t mcycle, bool *stat } template -static inline uint64_t read_csr_sstatus(STATE_ACCESS a, bool *status) { +static inline uint64_t read_csr_sstatus(const STATE_ACCESS a, bool *status) { return read_csr_success(a.read_mstatus() & SSTATUS_R_MASK, status); } template -static inline uint64_t read_csr_senvcfg(STATE_ACCESS a, bool *status) { +static inline uint64_t read_csr_senvcfg(const STATE_ACCESS a, bool *status) { return read_csr_success(a.read_senvcfg() & SENVCFG_R_MASK, status); } template -static inline uint64_t read_csr_sie(STATE_ACCESS a, bool *status) { +static inline uint64_t read_csr_sie(const STATE_ACCESS a, bool *status) { const uint64_t mie = a.read_mie(); const uint64_t mideleg = a.read_mideleg(); return read_csr_success(mie & mideleg, status); } template -static inline uint64_t read_csr_stvec(STATE_ACCESS a, bool *status) { +static inline uint64_t read_csr_stvec(const STATE_ACCESS a, bool *status) { return read_csr_success(a.read_stvec(), status); } template -static inline uint64_t read_csr_scounteren(STATE_ACCESS a, bool *status) { +static inline uint64_t read_csr_scounteren(const STATE_ACCESS a, bool *status) { return read_csr_success(a.read_scounteren(), status); } template -static inline uint64_t read_csr_sscratch(STATE_ACCESS a, bool *status) { +static inline uint64_t read_csr_sscratch(const STATE_ACCESS a, bool *status) { return read_csr_success(a.read_sscratch(), status); } template -static inline uint64_t read_csr_sepc(STATE_ACCESS a, bool *status) { +static inline uint64_t read_csr_sepc(const STATE_ACCESS a, bool *status) { return read_csr_success(a.read_sepc(), status); } template -static inline uint64_t read_csr_scause(STATE_ACCESS a, bool *status) { +static inline uint64_t read_csr_scause(const STATE_ACCESS a, bool *status) { return read_csr_success(a.read_scause(), status); } template -static inline uint64_t read_csr_stval(STATE_ACCESS a, bool *status) { +static inline uint64_t read_csr_stval(const STATE_ACCESS a, bool *status) { return read_csr_success(a.read_stval(), status); } template -static inline uint64_t read_csr_sip(STATE_ACCESS a, bool *status) { +static inline uint64_t read_csr_sip(const STATE_ACCESS a, bool *status) { // Ensure values are are loaded in order: do not nest with operator const uint64_t mip = a.read_mip(); const uint64_t mideleg = a.read_mideleg(); @@ -1631,7 +1798,7 @@ static inline uint64_t read_csr_sip(STATE_ACCESS a, bool *status) { } template -static inline uint64_t read_csr_satp(STATE_ACCESS a, bool *status) { +static inline uint64_t read_csr_satp(const STATE_ACCESS a, bool *status) { const uint64_t mstatus = a.read_mstatus(); auto prv = a.read_iprv(); // When TVM=1, attempts to read or write the satp CSR @@ -1643,67 +1810,67 @@ static inline uint64_t read_csr_satp(STATE_ACCESS a, bool *status) { } template -static inline uint64_t read_csr_mstatus(STATE_ACCESS a, bool *status) { +static inline uint64_t read_csr_mstatus(const STATE_ACCESS a, bool *status) { return read_csr_success(a.read_mstatus() & MSTATUS_R_MASK, status); } template -static inline uint64_t read_csr_menvcfg(STATE_ACCESS a, bool *status) { +static inline uint64_t read_csr_menvcfg(const STATE_ACCESS a, bool *status) { return read_csr_success(a.read_menvcfg() & MENVCFG_R_MASK, status); } template -static inline uint64_t read_csr_misa(STATE_ACCESS a, bool *status) { +static inline uint64_t read_csr_misa(const STATE_ACCESS a, bool *status) { return read_csr_success(a.read_misa(), status); } template -static inline uint64_t read_csr_medeleg(STATE_ACCESS a, bool *status) { +static inline uint64_t read_csr_medeleg(const STATE_ACCESS a, bool *status) { return read_csr_success(a.read_medeleg(), status); } template -static inline uint64_t read_csr_mideleg(STATE_ACCESS a, bool *status) { +static inline uint64_t read_csr_mideleg(const STATE_ACCESS a, bool *status) { return read_csr_success(a.read_mideleg(), status); } template -static inline uint64_t read_csr_mie(STATE_ACCESS a, bool *status) { +static inline uint64_t read_csr_mie(const STATE_ACCESS a, bool *status) { return read_csr_success(a.read_mie(), status); } template -static inline uint64_t read_csr_mtvec(STATE_ACCESS a, bool *status) { +static inline uint64_t read_csr_mtvec(const STATE_ACCESS a, bool *status) { return read_csr_success(a.read_mtvec(), status); } template -static inline uint64_t read_csr_mcounteren(STATE_ACCESS a, bool *status) { +static inline uint64_t read_csr_mcounteren(const STATE_ACCESS a, bool *status) { return read_csr_success(a.read_mcounteren(), status); } template -static inline uint64_t read_csr_mscratch(STATE_ACCESS a, bool *status) { +static inline uint64_t read_csr_mscratch(const STATE_ACCESS a, bool *status) { return read_csr_success(a.read_mscratch(), status); } template -static inline uint64_t read_csr_mepc(STATE_ACCESS a, bool *status) { +static inline uint64_t read_csr_mepc(const STATE_ACCESS a, bool *status) { return read_csr_success(a.read_mepc(), status); } template -static inline uint64_t read_csr_mcause(STATE_ACCESS a, bool *status) { +static inline uint64_t read_csr_mcause(const STATE_ACCESS a, bool *status) { return read_csr_success(a.read_mcause(), status); } template -static inline uint64_t read_csr_mtval(STATE_ACCESS a, bool *status) { +static inline uint64_t read_csr_mtval(const STATE_ACCESS a, bool *status) { return read_csr_success(a.read_mtval(), status); } template -static inline uint64_t read_csr_mip(STATE_ACCESS a, bool *status) { +static inline uint64_t read_csr_mip(const STATE_ACCESS a, bool *status) { return read_csr_success(a.read_mip(), status); } @@ -1712,29 +1879,29 @@ static inline uint64_t read_csr_mcycle(uint64_t mcycle, bool *status) { } template -static inline uint64_t read_csr_minstret(STATE_ACCESS a, uint64_t mcycle, bool *status) { +static inline uint64_t read_csr_minstret(const STATE_ACCESS a, uint64_t mcycle, bool *status) { const uint64_t icycleinstret = a.read_icycleinstret(); const uint64_t minstret = mcycle - icycleinstret; return read_csr_success(minstret, status); } template -static inline uint64_t read_csr_mvendorid(STATE_ACCESS a, bool *status) { +static inline uint64_t read_csr_mvendorid(const STATE_ACCESS a, bool *status) { return read_csr_success(a.read_mvendorid(), status); } template -static inline uint64_t read_csr_marchid(STATE_ACCESS a, bool *status) { +static inline uint64_t read_csr_marchid(const STATE_ACCESS a, bool *status) { return read_csr_success(a.read_marchid(), status); } template -static inline uint64_t read_csr_mimpid(STATE_ACCESS a, bool *status) { +static inline uint64_t read_csr_mimpid(const STATE_ACCESS a, bool *status) { return read_csr_success(a.read_mimpid(), status); } template -static inline uint64_t read_csr_fflags(STATE_ACCESS a, bool *status) { +static inline uint64_t read_csr_fflags(const STATE_ACCESS a, bool *status) { // If FS is OFF, attempts to read or write the float state will cause an illegal instruction exception. if (unlikely((a.read_mstatus() & MSTATUS_FS_MASK) == MSTATUS_FS_OFF)) { return read_csr_fail(status); @@ -1744,7 +1911,7 @@ static inline uint64_t read_csr_fflags(STATE_ACCESS a, bool *status) { } template -static inline uint64_t read_csr_frm(STATE_ACCESS a, bool *status) { +static inline uint64_t read_csr_frm(const STATE_ACCESS a, bool *status) { // If FS is OFF, attempts to read or write the float state will cause an illegal instruction exception. if (unlikely((a.read_mstatus() & MSTATUS_FS_MASK) == MSTATUS_FS_OFF)) { return read_csr_fail(status); @@ -1754,7 +1921,7 @@ static inline uint64_t read_csr_frm(STATE_ACCESS a, bool *status) { } template -static inline uint64_t read_csr_fcsr(STATE_ACCESS a, bool *status) { +static inline uint64_t read_csr_fcsr(const STATE_ACCESS a, bool *status) { // If FS is OFF, attempts to read or write the float state will cause an illegal instruction exception. if (unlikely((a.read_mstatus() & MSTATUS_FS_MASK) == MSTATUS_FS_OFF)) { return read_csr_fail(status); @@ -1769,7 +1936,7 @@ static inline uint64_t read_csr_fcsr(STATE_ACCESS a, bool *status) { /// \returns Register value. /// \details This function is outlined to minimize host CPU code cache pressure. template -static NO_INLINE uint64_t read_csr(STATE_ACCESS a, uint64_t mcycle, CSR_address csraddr, bool *status) { +static NO_INLINE uint64_t read_csr(const STATE_ACCESS a, uint64_t mcycle, CSR_address csraddr, bool *status) { if (unlikely(csr_prv(csraddr) > a.read_iprv())) { return read_csr_fail(status); } @@ -1923,27 +2090,27 @@ static NO_INLINE uint64_t read_csr(STATE_ACCESS a, uint64_t mcycle, CSR_address default: // Invalid CSRs #ifdef DUMP_INVALID_CSR - fprintf(stderr, "csr_read: invalid CSR=0x%x\n", static_cast(csraddr)); + d_printf("csr_read: invalid CSR=0x%x\n", static_cast(csraddr)); #endif return read_csr_fail(status); } } template -static execute_status write_csr_sstatus(STATE_ACCESS a, uint64_t val) { +static execute_status write_csr_sstatus(const STATE_ACCESS a, uint64_t val) { const uint64_t mstatus = a.read_mstatus(); return write_csr_mstatus(a, (mstatus & ~SSTATUS_W_MASK) | (val & SSTATUS_W_MASK)); } template -static execute_status write_csr_senvcfg(STATE_ACCESS a, uint64_t val) { +static execute_status write_csr_senvcfg(const STATE_ACCESS a, uint64_t val) { const uint64_t senvcfg = a.read_senvcfg(); a.write_senvcfg((senvcfg & ~SENVCFG_W_MASK) | (val & SENVCFG_W_MASK)); return execute_status::success; } template -static execute_status write_csr_sie(STATE_ACCESS a, uint64_t val) { +static execute_status write_csr_sie(const STATE_ACCESS a, uint64_t val) { uint64_t mie = a.read_mie(); const uint64_t mask = a.read_mideleg(); mie = (mie & ~mask) | (val & mask); @@ -1952,43 +2119,43 @@ static execute_status write_csr_sie(STATE_ACCESS a, uint64_t val) { } template -static execute_status write_csr_stvec(STATE_ACCESS a, uint64_t val) { +static execute_status write_csr_stvec(const STATE_ACCESS a, uint64_t val) { a.write_stvec(val & ~1); return execute_status::success; } template -static execute_status write_csr_scounteren(STATE_ACCESS a, uint64_t val) { +static execute_status write_csr_scounteren(const STATE_ACCESS a, uint64_t val) { a.write_scounteren(val & SCOUNTEREN_RW_MASK); return execute_status::success; } template -static execute_status write_csr_sscratch(STATE_ACCESS a, uint64_t val) { +static execute_status write_csr_sscratch(const STATE_ACCESS a, uint64_t val) { a.write_sscratch(val); return execute_status::success; } template -static execute_status write_csr_sepc(STATE_ACCESS a, uint64_t val) { +static execute_status write_csr_sepc(const STATE_ACCESS a, uint64_t val) { a.write_sepc(val & ~1); return execute_status::success; } template -static execute_status write_csr_scause(STATE_ACCESS a, uint64_t val) { +static execute_status write_csr_scause(const STATE_ACCESS a, uint64_t val) { a.write_scause(val); return execute_status::success; } template -static execute_status write_csr_stval(STATE_ACCESS a, uint64_t val) { +static execute_status write_csr_stval(const STATE_ACCESS a, uint64_t val) { a.write_stval(val); return execute_status::success; } template -static execute_status write_csr_sip(STATE_ACCESS a, uint64_t val) { +static execute_status write_csr_sip(const STATE_ACCESS a, uint64_t val) { const uint64_t mask = a.read_mideleg(); uint64_t mip = a.read_mip(); mip = (mip & ~mask) | (val & mask); @@ -1997,7 +2164,7 @@ static execute_status write_csr_sip(STATE_ACCESS a, uint64_t val) { } template -static NO_INLINE execute_status write_csr_satp(STATE_ACCESS a, uint64_t val) { +static NO_INLINE execute_status write_csr_satp(const STATE_ACCESS a, uint64_t val) { const uint64_t mstatus = a.read_mstatus(); auto prv = a.read_iprv(); @@ -2029,10 +2196,12 @@ static NO_INLINE execute_status write_csr_satp(STATE_ACCESS a, uint64_t val) { } a.write_satp(stap); -#ifdef DUMP_COUNTERS - uint64_t asid = (stap & SATP_ASID_MASK) >> SATP_ASID_SHIFT; - if (asid != ASID_MAX_MASK) { // Software is not testing ASID bits - a.get_statistics().max_asid = std::max(a.get_statistics().max_asid, asid); +#ifdef DUMP_STATS + if constexpr (is_an_i_accept_counters_v) { + const uint64_t asid = (stap & SATP_ASID_MASK) >> SATP_ASID_SHIFT; + if (asid != ASID_MAX_MASK) { // Software is not testing ASID bits + a.write_counter(std::max(a.read_counter("stats.max_asid"), asid), "stats.max_asid"); + } } #endif @@ -2041,16 +2210,16 @@ static NO_INLINE execute_status write_csr_satp(STATE_ACCESS a, uint64_t val) { // because software is required to execute SFENCE.VMA when recycling an ASID. const uint64_t mod = old_satp ^ stap; if (mod & (SATP_ASID_MASK | SATP_MODE_MASK)) { - a.flush_all_tlb(); - INC_COUNTER(a.get_statistics(), tlb_flush_all); - INC_COUNTER(a.get_statistics(), tlb_flush_satp); + flush_all_tlb(a); + DUMP_STATS_INCR(a, "tlb.flush_all"); + DUMP_STATS_INCR(a, "tlb.flush_satp"); return execute_status::success_and_flush_fetch; } return execute_status::success; } template -static NO_INLINE execute_status write_csr_mstatus(STATE_ACCESS a, uint64_t val) { +static NO_INLINE execute_status write_csr_mstatus(const STATE_ACCESS a, uint64_t val) { const uint64_t old_mstatus = a.read_mstatus() & MSTATUS_R_MASK; // M-mode software can determine whether a privilege mode is implemented @@ -2103,15 +2272,15 @@ static NO_INLINE execute_status write_csr_mstatus(STATE_ACCESS a, uint64_t val) // Flush TLBs when needed if (flush_tlb_read) { - a.template flush_tlb_type(); - INC_COUNTER(a.get_statistics(), tlb_flush_read); + flush_tlb_set(a); + DUMP_STATS_INCR(a, "tlb.flush_read"); } if (flush_tlb_write) { - a.template flush_tlb_type(); - INC_COUNTER(a.get_statistics(), tlb_flush_write); + flush_tlb_set(a); + DUMP_STATS_INCR(a, "tlb.flush_write"); } if (flush_tlb_read || flush_tlb_write) { - INC_COUNTER(a.get_statistics(), tlb_flush_mstatus); + DUMP_STATS_INCR(a, "tlb.flush_mstatus"); } // When changing an interrupt enabled bit, we may have to service any pending interrupt @@ -2123,7 +2292,7 @@ static NO_INLINE execute_status write_csr_mstatus(STATE_ACCESS a, uint64_t val) } template -static execute_status write_csr_menvcfg(STATE_ACCESS a, uint64_t val) { +static execute_status write_csr_menvcfg(const STATE_ACCESS a, uint64_t val) { uint64_t menvcfg = a.read_menvcfg() & MENVCFG_R_MASK; // Modify only bits that can be written to @@ -2134,7 +2303,7 @@ static execute_status write_csr_menvcfg(STATE_ACCESS a, uint64_t val) { } template -static execute_status write_csr_medeleg(STATE_ACCESS a, uint64_t val) { +static execute_status write_csr_medeleg(const STATE_ACCESS a, uint64_t val) { // For exceptions that cannot occur in less privileged modes, // the corresponding medeleg bits should be read-only zero a.write_medeleg((a.read_medeleg() & ~MEDELEG_W_MASK) | (val & MEDELEG_W_MASK)); @@ -2142,7 +2311,7 @@ static execute_status write_csr_medeleg(STATE_ACCESS a, uint64_t val) { } template -static execute_status write_csr_mideleg(STATE_ACCESS a, uint64_t val) { +static execute_status write_csr_mideleg(const STATE_ACCESS a, uint64_t val) { const uint64_t mask = MIP_SSIP_MASK | MIP_STIP_MASK | MIP_SEIP_MASK; uint64_t mideleg = a.read_mideleg(); mideleg = (mideleg & ~mask) | (val & mask); @@ -2151,7 +2320,7 @@ static execute_status write_csr_mideleg(STATE_ACCESS a, uint64_t val) { } template -static execute_status write_csr_mie(STATE_ACCESS a, uint64_t val) { +static execute_status write_csr_mie(const STATE_ACCESS a, uint64_t val) { const uint64_t mask = MIP_MSIP_MASK | MIP_MTIP_MASK | MIP_MEIP_MASK | MIP_SSIP_MASK | MIP_STIP_MASK | MIP_SEIP_MASK; uint64_t mie = a.read_mie(); mie = (mie & ~mask) | (val & mask); @@ -2160,19 +2329,19 @@ static execute_status write_csr_mie(STATE_ACCESS a, uint64_t val) { } template -static execute_status write_csr_mtvec(STATE_ACCESS a, uint64_t val) { +static execute_status write_csr_mtvec(const STATE_ACCESS a, uint64_t val) { a.write_mtvec(val & ~1); return execute_status::success; } template -static execute_status write_csr_mcounteren(STATE_ACCESS a, uint64_t val) { +static execute_status write_csr_mcounteren(const STATE_ACCESS a, uint64_t val) { a.write_mcounteren(val & MCOUNTEREN_RW_MASK); return execute_status::success; } template -static execute_status write_csr_minstret(STATE_ACCESS a, uint64_t mcycle, uint64_t val) { +static execute_status write_csr_minstret(const STATE_ACCESS a, uint64_t mcycle, uint64_t val) { // Note that mcycle will only be incremented after the instruction is executed, // but we have to compute this in advance const uint64_t icycleinstret = (mcycle + 1) - val; @@ -2192,31 +2361,31 @@ static execute_status write_csr_mcycle(STATE_ACCESS /*a*/, uint64_t /*val*/) { } template -static execute_status write_csr_mscratch(STATE_ACCESS a, uint64_t val) { +static execute_status write_csr_mscratch(const STATE_ACCESS a, uint64_t val) { a.write_mscratch(val); return execute_status::success; } template -static execute_status write_csr_mepc(STATE_ACCESS a, uint64_t val) { +static execute_status write_csr_mepc(const STATE_ACCESS a, uint64_t val) { a.write_mepc(val & ~1); return execute_status::success; } template -static execute_status write_csr_mcause(STATE_ACCESS a, uint64_t val) { +static execute_status write_csr_mcause(const STATE_ACCESS a, uint64_t val) { a.write_mcause(val); return execute_status::success; } template -static execute_status write_csr_mtval(STATE_ACCESS a, uint64_t val) { +static execute_status write_csr_mtval(const STATE_ACCESS a, uint64_t val) { a.write_mtval(val); return execute_status::success; } template -static execute_status write_csr_mip(STATE_ACCESS a, uint64_t val) { +static execute_status write_csr_mip(const STATE_ACCESS a, uint64_t val) { const uint64_t mask = MIP_SSIP_MASK | MIP_STIP_MASK | MIP_SEIP_MASK; auto mip = a.read_mip(); mip = (mip & ~mask) | (val & mask); @@ -2225,7 +2394,7 @@ static execute_status write_csr_mip(STATE_ACCESS a, uint64_t val) { } template -static inline execute_status write_csr_fflags(STATE_ACCESS a, uint64_t val) { +static inline execute_status write_csr_fflags(const STATE_ACCESS a, uint64_t val) { const uint64_t mstatus = a.read_mstatus(); // If FS is OFF, attempts to read or write the float state will cause an illegal instruction exception. if (unlikely((mstatus & MSTATUS_FS_MASK) == MSTATUS_FS_OFF)) { @@ -2237,7 +2406,7 @@ static inline execute_status write_csr_fflags(STATE_ACCESS a, uint64_t val) { } template -static inline execute_status write_csr_frm(STATE_ACCESS a, uint64_t val) { +static inline execute_status write_csr_frm(const STATE_ACCESS a, uint64_t val) { const uint64_t mstatus = a.read_mstatus(); // If FS is OFF, attempts to read or write the float state will cause an illegal instruction exception. if (unlikely((mstatus & MSTATUS_FS_MASK) == MSTATUS_FS_OFF)) { @@ -2249,7 +2418,7 @@ static inline execute_status write_csr_frm(STATE_ACCESS a, uint64_t val) { } template -static inline execute_status write_csr_fcsr(STATE_ACCESS a, uint64_t val) { +static inline execute_status write_csr_fcsr(const STATE_ACCESS a, uint64_t val) { const uint64_t mstatus = a.read_mstatus(); // If FS is OFF, attempts to read or write the float state will cause an illegal instruction exception. if (unlikely((mstatus & MSTATUS_FS_MASK) == MSTATUS_FS_OFF)) { @@ -2267,11 +2436,11 @@ static inline execute_status write_csr_fcsr(STATE_ACCESS a, uint64_t val) { /// \returns The status of the operation (true for success, false otherwise). /// \details This function is outlined to minimize host CPU code cache pressure. template -static NO_INLINE execute_status write_csr(STATE_ACCESS a, uint64_t mcycle, CSR_address csraddr, uint64_t val) { +static NO_INLINE execute_status write_csr(const STATE_ACCESS a, uint64_t mcycle, CSR_address csraddr, uint64_t val) { #if defined(DUMP_CSR) - fprintf(stderr, "csr_write: csr=0x%03x val=0x", static_cast(csraddr)); + d_printf("csr_write: csr=0x%03x val=0x", static_cast(csraddr)); print_uint64_t(val); - fprintf(stderr, "\n"); + d_printf("\n"); #endif if (unlikely(csr_is_read_only(csraddr))) { return execute_status::failure; @@ -2414,14 +2583,14 @@ static NO_INLINE execute_status write_csr(STATE_ACCESS a, uint64_t mcycle, CSR_a default: // Invalid CSRs #ifdef DUMP_INVALID_CSR - fprintf(stderr, "csr_write: invalid CSR=0x%x\n", static_cast(csraddr)); + d_printf("csr_write: invalid CSR=0x%x\n", static_cast(csraddr)); #endif return execute_status::failure; } } template -static FORCE_INLINE execute_status execute_csr_RW(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn, +static FORCE_INLINE execute_status execute_csr_RW(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn, const RS1VAL &rs1val) { auto csraddr = static_cast(insn_I_get_uimm(insn)); // Try to read old CSR value @@ -2454,22 +2623,22 @@ static FORCE_INLINE execute_status execute_csr_RW(STATE_ACCESS a, uint64_t &pc, /// \brief Implementation of the CSRRW instruction. template -static FORCE_INLINE execute_status execute_CSRRW(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { - dump_insn(a, pc, insn, "csrrw"); +static FORCE_INLINE execute_status execute_CSRRW(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "csrrw"); return execute_csr_RW(a, pc, mcycle, insn, - [](STATE_ACCESS a, uint32_t insn) -> uint64_t { return a.read_x(insn_get_rs1(insn)); }); + [](const STATE_ACCESS a, uint32_t insn) -> uint64_t { return a.read_x(insn_get_rs1(insn)); }); } /// \brief Implementation of the CSRRWI instruction. template -static FORCE_INLINE execute_status execute_CSRRWI(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { - dump_insn(a, pc, insn, "csrrwi"); +static FORCE_INLINE execute_status execute_CSRRWI(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "csrrwi"); return execute_csr_RW(a, pc, mcycle, insn, [](STATE_ACCESS, uint32_t insn) -> uint64_t { return static_cast(insn_get_rs1(insn)); }); } template -static FORCE_INLINE execute_status execute_csr_SC(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn, +static FORCE_INLINE execute_status execute_csr_SC(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn, const F &f) { auto csraddr = static_cast(insn_I_get_uimm(insn)); // Try to read old CSR value @@ -2504,20 +2673,20 @@ static FORCE_INLINE execute_status execute_csr_SC(STATE_ACCESS a, uint64_t &pc, /// \brief Implementation of the CSRRS instruction. template -static FORCE_INLINE execute_status execute_CSRRS(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { - dump_insn(a, pc, insn, "csrrs"); +static FORCE_INLINE execute_status execute_CSRRS(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "csrrs"); return execute_csr_SC(a, pc, mcycle, insn, [](uint64_t csr, uint64_t rs1) -> uint64_t { return csr | rs1; }); } /// \brief Implementation of the CSRRC instruction. template -static FORCE_INLINE execute_status execute_CSRRC(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { - dump_insn(a, pc, insn, "csrrc"); +static FORCE_INLINE execute_status execute_CSRRC(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "csrrc"); return execute_csr_SC(a, pc, mcycle, insn, [](uint64_t csr, uint64_t rs1) -> uint64_t { return csr & ~rs1; }); } template -static FORCE_INLINE execute_status execute_csr_SCI(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn, +static FORCE_INLINE execute_status execute_csr_SCI(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn, const F &f) { auto csraddr = static_cast(insn_I_get_uimm(insn)); // Try to read old CSR value @@ -2549,22 +2718,22 @@ static FORCE_INLINE execute_status execute_csr_SCI(STATE_ACCESS a, uint64_t &pc, /// \brief Implementation of the CSRRSI instruction. template -static FORCE_INLINE execute_status execute_CSRRSI(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { - dump_insn(a, pc, insn, "csrrsi"); +static FORCE_INLINE execute_status execute_CSRRSI(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "csrrsi"); return execute_csr_SCI(a, pc, mcycle, insn, [](uint64_t csr, uint32_t rs1) -> uint64_t { return csr | rs1; }); } /// \brief Implementation of the CSRRCI instruction. template -static FORCE_INLINE execute_status execute_CSRRCI(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { - dump_insn(a, pc, insn, "csrrci"); +static FORCE_INLINE execute_status execute_CSRRCI(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "csrrci"); return execute_csr_SCI(a, pc, mcycle, insn, [](uint64_t csr, uint32_t rs1) -> uint64_t { return csr & ~rs1; }); } /// \brief Implementation of the ECALL instruction. template -static FORCE_INLINE execute_status execute_ECALL(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "ecall"); +static FORCE_INLINE execute_status execute_ECALL(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "ecall"); auto prv = a.read_iprv(); pc = raise_exception(a, pc, MCAUSE_ECALL_BASE + prv, 0); return execute_status::failure; @@ -2572,16 +2741,16 @@ static FORCE_INLINE execute_status execute_ECALL(STATE_ACCESS a, uint64_t &pc, u /// \brief Implementation of the EBREAK instruction. template -static FORCE_INLINE execute_status execute_EBREAK(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "ebreak"); +static FORCE_INLINE execute_status execute_EBREAK(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "ebreak"); pc = raise_exception(a, pc, MCAUSE_BREAKPOINT, pc); return execute_status::failure; } /// \brief Implementation of the SRET instruction. template -static FORCE_INLINE execute_status execute_SRET(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "sret"); +static FORCE_INLINE execute_status execute_SRET(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "sret"); auto prv = a.read_iprv(); uint64_t mstatus = a.read_mstatus(); if (unlikely(prv < PRV_S || (prv == PRV_S && (mstatus & MSTATUS_TSR_MASK)))) { @@ -2610,8 +2779,8 @@ static FORCE_INLINE execute_status execute_SRET(STATE_ACCESS a, uint64_t &pc, ui /// \brief Implementation of the MRET instruction. template -static FORCE_INLINE execute_status execute_MRET(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "mret"); +static FORCE_INLINE execute_status execute_MRET(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "mret"); auto prv = a.read_iprv(); if (unlikely(prv < PRV_M)) { return raise_illegal_insn_exception(a, pc, insn); @@ -2642,8 +2811,9 @@ static FORCE_INLINE execute_status execute_MRET(STATE_ACCESS a, uint64_t &pc, ui /// \brief Implementation of the WFI instruction. /// \details This function is outlined to minimize host CPU code cache pressure. template -static FORCE_INLINE execute_status execute_WFI(STATE_ACCESS a, uint64_t &pc, uint64_t &mcycle, uint32_t insn) { - dump_insn(a, pc, insn, "wfi"); +static FORCE_INLINE execute_status execute_WFI(const STATE_ACCESS a, uint64_t &pc, uint64_t &mcycle, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "wfi"); + auto status = execute_status::success; // Check privileges and do nothing else auto prv = a.read_iprv(); const uint64_t mstatus = a.read_mstatus(); @@ -2653,14 +2823,11 @@ static FORCE_INLINE execute_status execute_WFI(STATE_ACCESS a, uint64_t &pc, uin } // We wait for interrupts until the next timer interrupt. const uint64_t mcycle_max = rtc_time_to_cycle(a.read_clint_mtimecmp()); - execute_status status = execute_status::success; - if (mcycle_max > mcycle) { - // Poll for external interrupts (e.g console or network), - // this may advance mcycle only when interactive mode is enabled - const auto [next_mcycle, interrupted] = a.poll_external_interrupts(mcycle, mcycle_max); - mcycle = next_mcycle; - if (interrupted) { - status = execute_status::success_and_serve_interrupts; + if constexpr (is_an_i_interactive_state_access_v) { + if (mcycle_max > mcycle) { + // Poll for external interrupts (e.g console or network), + // this may advance mcycle only when interactive mode is enabled + std::tie(mcycle, status) = a.poll_external_interrupts(mcycle, mcycle_max); } } return advance_to_next_insn(a, pc, status); @@ -2668,24 +2835,24 @@ static FORCE_INLINE execute_status execute_WFI(STATE_ACCESS a, uint64_t &pc, uin /// \brief Implementation of the FENCE instruction. template -static FORCE_INLINE execute_status execute_FENCE(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - INC_COUNTER(a.get_statistics(), fence); - dump_insn(a, pc, insn, "fence"); +static FORCE_INLINE execute_status execute_FENCE(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + DUMP_STATS_INCR(a, "fence"); + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fence"); // Really do nothing return advance_to_next_insn(a, pc); } /// \brief Implementation of the FENCE.I instruction. template -static FORCE_INLINE execute_status execute_FENCE_I(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - INC_COUNTER(a.get_statistics(), fence_i); - dump_insn(a, pc, insn, "fence.i"); +static FORCE_INLINE execute_status execute_FENCE_I(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + DUMP_STATS_INCR(a, "fence.i"); + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fence.i"); // Really do nothing return advance_to_next_insn(a, pc); } template -static FORCE_INLINE execute_status execute_arithmetic(STATE_ACCESS a, uint64_t &pc, uint32_t insn, const F &f) { +static FORCE_INLINE execute_status execute_arithmetic(const STATE_ACCESS a, uint64_t &pc, uint32_t insn, const F &f) { const uint32_t rd = insn_get_rd(insn); // Ensure rs1 and rs2 are loaded in order: do not nest with call to f() as // the order of evaluation of arguments in a function call is undefined. @@ -2698,8 +2865,8 @@ static FORCE_INLINE execute_status execute_arithmetic(STATE_ACCESS a, uint64_t & /// \brief Implementation of the ADD instruction. template -static FORCE_INLINE execute_status execute_ADD(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "add"); +static FORCE_INLINE execute_status execute_ADD(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "add"); if constexpr (rd_kind == rd_kind::x0) { return advance_to_next_insn(a, pc); } @@ -2712,8 +2879,8 @@ static FORCE_INLINE execute_status execute_ADD(STATE_ACCESS a, uint64_t &pc, uin /// \brief Implementation of the SUB instruction. template -static FORCE_INLINE execute_status execute_SUB(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "sub"); +static FORCE_INLINE execute_status execute_SUB(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "sub"); if constexpr (rd_kind == rd_kind::x0) { return advance_to_next_insn(a, pc); } @@ -2726,8 +2893,8 @@ static FORCE_INLINE execute_status execute_SUB(STATE_ACCESS a, uint64_t &pc, uin /// \brief Implementation of the SLL instruction. template -static FORCE_INLINE execute_status execute_SLL(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "sll"); +static FORCE_INLINE execute_status execute_SLL(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "sll"); if constexpr (rd_kind == rd_kind::x0) { return advance_to_next_insn(a, pc); } @@ -2737,8 +2904,8 @@ static FORCE_INLINE execute_status execute_SLL(STATE_ACCESS a, uint64_t &pc, uin /// \brief Implementation of the SLT instruction. template -static FORCE_INLINE execute_status execute_SLT(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "slt"); +static FORCE_INLINE execute_status execute_SLT(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "slt"); if constexpr (rd_kind == rd_kind::x0) { return advance_to_next_insn(a, pc); } @@ -2748,8 +2915,8 @@ static FORCE_INLINE execute_status execute_SLT(STATE_ACCESS a, uint64_t &pc, uin /// \brief Implementation of the SLTU instruction. template -static FORCE_INLINE execute_status execute_SLTU(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "sltu"); +static FORCE_INLINE execute_status execute_SLTU(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "sltu"); if constexpr (rd_kind == rd_kind::x0) { return advance_to_next_insn(a, pc); } @@ -2758,8 +2925,8 @@ static FORCE_INLINE execute_status execute_SLTU(STATE_ACCESS a, uint64_t &pc, ui /// \brief Implementation of the XOR instruction. template -static FORCE_INLINE execute_status execute_XOR(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "xor"); +static FORCE_INLINE execute_status execute_XOR(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "xor"); if constexpr (rd_kind == rd_kind::x0) { return advance_to_next_insn(a, pc); } @@ -2768,8 +2935,8 @@ static FORCE_INLINE execute_status execute_XOR(STATE_ACCESS a, uint64_t &pc, uin /// \brief Implementation of the SRL instruction. template -static FORCE_INLINE execute_status execute_SRL(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "srl"); +static FORCE_INLINE execute_status execute_SRL(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "srl"); if constexpr (rd_kind == rd_kind::x0) { return advance_to_next_insn(a, pc); } @@ -2779,8 +2946,8 @@ static FORCE_INLINE execute_status execute_SRL(STATE_ACCESS a, uint64_t &pc, uin /// \brief Implementation of the SRA instruction. template -static FORCE_INLINE execute_status execute_SRA(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "sra"); +static FORCE_INLINE execute_status execute_SRA(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "sra"); if constexpr (rd_kind == rd_kind::x0) { return advance_to_next_insn(a, pc); } @@ -2791,8 +2958,8 @@ static FORCE_INLINE execute_status execute_SRA(STATE_ACCESS a, uint64_t &pc, uin /// \brief Implementation of the OR instruction. template -static FORCE_INLINE execute_status execute_OR(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "or"); +static FORCE_INLINE execute_status execute_OR(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "or"); if constexpr (rd_kind == rd_kind::x0) { return advance_to_next_insn(a, pc); } @@ -2801,8 +2968,8 @@ static FORCE_INLINE execute_status execute_OR(STATE_ACCESS a, uint64_t &pc, uint /// \brief Implementation of the AND instruction. template -static FORCE_INLINE execute_status execute_AND(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "and"); +static FORCE_INLINE execute_status execute_AND(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "and"); if constexpr (rd_kind == rd_kind::x0) { return advance_to_next_insn(a, pc); } @@ -2811,8 +2978,8 @@ static FORCE_INLINE execute_status execute_AND(STATE_ACCESS a, uint64_t &pc, uin /// \brief Implementation of the MUL instruction. template -static FORCE_INLINE execute_status execute_MUL(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "mul"); +static FORCE_INLINE execute_status execute_MUL(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "mul"); if constexpr (rd_kind == rd_kind::x0) { return advance_to_next_insn(a, pc); } @@ -2827,8 +2994,8 @@ static FORCE_INLINE execute_status execute_MUL(STATE_ACCESS a, uint64_t &pc, uin /// \brief Implementation of the MULH instruction. template -static FORCE_INLINE execute_status execute_MULH(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "mulh"); +static FORCE_INLINE execute_status execute_MULH(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "mulh"); if constexpr (rd_kind == rd_kind::x0) { return advance_to_next_insn(a, pc); } @@ -2841,8 +3008,8 @@ static FORCE_INLINE execute_status execute_MULH(STATE_ACCESS a, uint64_t &pc, ui /// \brief Implementation of the MULHSU instruction. template -static FORCE_INLINE execute_status execute_MULHSU(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "mulhsu"); +static FORCE_INLINE execute_status execute_MULHSU(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "mulhsu"); if constexpr (rd_kind == rd_kind::x0) { return advance_to_next_insn(a, pc); } @@ -2855,8 +3022,8 @@ static FORCE_INLINE execute_status execute_MULHSU(STATE_ACCESS a, uint64_t &pc, /// \brief Implementation of the MULHU instruction. template -static FORCE_INLINE execute_status execute_MULHU(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "mulhu"); +static FORCE_INLINE execute_status execute_MULHU(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "mulhu"); if constexpr (rd_kind == rd_kind::x0) { return advance_to_next_insn(a, pc); } @@ -2867,8 +3034,8 @@ static FORCE_INLINE execute_status execute_MULHU(STATE_ACCESS a, uint64_t &pc, u /// \brief Implementation of the DIV instruction. template -static FORCE_INLINE execute_status execute_DIV(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "div"); +static FORCE_INLINE execute_status execute_DIV(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "div"); if constexpr (rd_kind == rd_kind::x0) { return advance_to_next_insn(a, pc); } @@ -2887,8 +3054,8 @@ static FORCE_INLINE execute_status execute_DIV(STATE_ACCESS a, uint64_t &pc, uin /// \brief Implementation of the DIVU instruction. template -static FORCE_INLINE execute_status execute_DIVU(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "divu"); +static FORCE_INLINE execute_status execute_DIVU(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "divu"); if constexpr (rd_kind == rd_kind::x0) { return advance_to_next_insn(a, pc); } @@ -2902,8 +3069,8 @@ static FORCE_INLINE execute_status execute_DIVU(STATE_ACCESS a, uint64_t &pc, ui /// \brief Implementation of the REM instruction. template -static FORCE_INLINE execute_status execute_REM(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "rem"); +static FORCE_INLINE execute_status execute_REM(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "rem"); if constexpr (rd_kind == rd_kind::x0) { return advance_to_next_insn(a, pc); } @@ -2922,8 +3089,8 @@ static FORCE_INLINE execute_status execute_REM(STATE_ACCESS a, uint64_t &pc, uin /// \brief Implementation of the REMU instruction. template -static FORCE_INLINE execute_status execute_REMU(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "remu"); +static FORCE_INLINE execute_status execute_REMU(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "remu"); if constexpr (rd_kind == rd_kind::x0) { return advance_to_next_insn(a, pc); } @@ -2936,7 +3103,7 @@ static FORCE_INLINE execute_status execute_REMU(STATE_ACCESS a, uint64_t &pc, ui } template -static FORCE_INLINE execute_status execute_arithmetic_immediate(STATE_ACCESS a, uint64_t &pc, uint32_t insn, +static FORCE_INLINE execute_status execute_arithmetic_immediate(const STATE_ACCESS a, uint64_t &pc, uint32_t insn, const F &f) { const uint32_t rd = insn_get_rd(insn); const uint64_t rs1 = a.read_x(insn_get_rs1(insn)); @@ -2947,8 +3114,8 @@ static FORCE_INLINE execute_status execute_arithmetic_immediate(STATE_ACCESS a, /// \brief Implementation of the SRLI instruction. template -static FORCE_INLINE execute_status execute_SRLI(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "srli"); +static FORCE_INLINE execute_status execute_SRLI(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "srli"); if constexpr (rd_kind == rd_kind::x0) { return advance_to_next_insn(a, pc); } @@ -2958,8 +3125,8 @@ static FORCE_INLINE execute_status execute_SRLI(STATE_ACCESS a, uint64_t &pc, ui /// \brief Implementation of the SRAI instruction. template -static FORCE_INLINE execute_status execute_SRAI(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "srai"); +static FORCE_INLINE execute_status execute_SRAI(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "srai"); if constexpr (rd_kind == rd_kind::x0) { return advance_to_next_insn(a, pc); } @@ -2970,8 +3137,8 @@ static FORCE_INLINE execute_status execute_SRAI(STATE_ACCESS a, uint64_t &pc, ui /// \brief Implementation of the ADDI instruction. template -static FORCE_INLINE execute_status execute_ADDI(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "addi"); +static FORCE_INLINE execute_status execute_ADDI(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "addi"); if constexpr (rd_kind == rd_kind::x0) { return advance_to_next_insn(a, pc); } @@ -2984,30 +3151,34 @@ static FORCE_INLINE execute_status execute_ADDI(STATE_ACCESS a, uint64_t &pc, ui /// \brief Implementation of the SLTI instruction. template -static FORCE_INLINE execute_status execute_SLTI(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "slti"); +static FORCE_INLINE execute_status execute_SLTI(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "slti"); if constexpr (rd_kind == rd_kind::x0) { return advance_to_next_insn(a, pc); } - return execute_arithmetic_immediate(a, pc, insn, - [](uint64_t rs1, int32_t imm) -> uint64_t { return static_cast(rs1) < static_cast(imm); }); + return execute_arithmetic_immediate(a, pc, insn, [](uint64_t rs1, int32_t imm) -> uint64_t { + // NOLINTNEXTLINE(modernize-use-integer-sign-comparison) + return static_cast(rs1) < static_cast(imm); + }); } /// \brief Implementation of the SLTIU instruction. template -static FORCE_INLINE execute_status execute_SLTIU(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "sltiu"); +static FORCE_INLINE execute_status execute_SLTIU(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "sltiu"); if constexpr (rd_kind == rd_kind::x0) { return advance_to_next_insn(a, pc); } - return execute_arithmetic_immediate(a, pc, insn, - [](uint64_t rs1, int32_t imm) -> uint64_t { return rs1 < static_cast(imm); }); + return execute_arithmetic_immediate(a, pc, insn, [](uint64_t rs1, int32_t imm) -> uint64_t { + // NOLINTNEXTLINE(modernize-use-integer-sign-comparison) + return rs1 < static_cast(imm); + }); } /// \brief Implementation of the XORI instruction. template -static FORCE_INLINE execute_status execute_XORI(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "xori"); +static FORCE_INLINE execute_status execute_XORI(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "xori"); if constexpr (rd_kind == rd_kind::x0) { return advance_to_next_insn(a, pc); } @@ -3016,8 +3187,8 @@ static FORCE_INLINE execute_status execute_XORI(STATE_ACCESS a, uint64_t &pc, ui /// \brief Implementation of the ORI instruction. template -static FORCE_INLINE execute_status execute_ORI(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "ori"); +static FORCE_INLINE execute_status execute_ORI(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "ori"); if constexpr (rd_kind == rd_kind::x0) { return advance_to_next_insn(a, pc); } @@ -3026,8 +3197,8 @@ static FORCE_INLINE execute_status execute_ORI(STATE_ACCESS a, uint64_t &pc, uin /// \brief Implementation of the ANDI instruction. template -static FORCE_INLINE execute_status execute_ANDI(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "andi"); +static FORCE_INLINE execute_status execute_ANDI(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "andi"); if constexpr (rd_kind == rd_kind::x0) { return advance_to_next_insn(a, pc); } @@ -3036,11 +3207,11 @@ static FORCE_INLINE execute_status execute_ANDI(STATE_ACCESS a, uint64_t &pc, ui /// \brief Implementation of the SLLI instruction. template -static FORCE_INLINE execute_status execute_SLLI(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { +static FORCE_INLINE execute_status execute_SLLI(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { if (unlikely((insn & (0b111111 << 26)) != 0)) { return raise_illegal_insn_exception(a, pc, insn); } - dump_insn(a, pc, insn, "slli"); + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "slli"); if constexpr (rd_kind == rd_kind::x0) { return advance_to_next_insn(a, pc); } @@ -3053,8 +3224,8 @@ static FORCE_INLINE execute_status execute_SLLI(STATE_ACCESS a, uint64_t &pc, ui /// \brief Implementation of the ADDIW instruction. template -static FORCE_INLINE execute_status execute_ADDIW(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "addiw"); +static FORCE_INLINE execute_status execute_ADDIW(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "addiw"); if constexpr (rd_kind == rd_kind::x0) { return advance_to_next_insn(a, pc); } @@ -3067,11 +3238,11 @@ static FORCE_INLINE execute_status execute_ADDIW(STATE_ACCESS a, uint64_t &pc, u /// \brief Implementation of the SLLIW instruction. template -static FORCE_INLINE execute_status execute_SLLIW(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { +static FORCE_INLINE execute_status execute_SLLIW(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { if (unlikely(insn_get_funct7(insn) != 0)) { return raise_illegal_insn_exception(a, pc, insn); } - dump_insn(a, pc, insn, "slliw"); + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "slliw"); if constexpr (rd_kind == rd_kind::x0) { return advance_to_next_insn(a, pc); } @@ -3085,8 +3256,8 @@ static FORCE_INLINE execute_status execute_SLLIW(STATE_ACCESS a, uint64_t &pc, u /// \brief Implementation of the SRLIW instruction. template -static FORCE_INLINE execute_status execute_SRLIW(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "srliw"); +static FORCE_INLINE execute_status execute_SRLIW(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "srliw"); if constexpr (rd_kind == rd_kind::x0) { return advance_to_next_insn(a, pc); } @@ -3100,13 +3271,15 @@ static FORCE_INLINE execute_status execute_SRLIW(STATE_ACCESS a, uint64_t &pc, u /// \brief Implementation of the SRAIW instruction. template -static FORCE_INLINE execute_status execute_SRAIW(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "sraiw"); +static FORCE_INLINE execute_status execute_SRAIW(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "sraiw"); + // When rd=0 the instruction is a HINT, and we consider it as a soft yield when rs1 == 31 if constexpr (rd_kind == rd_kind::x0) { - // When rd=0 the instruction is a HINT, and we consider it as a soft yield when rs1 == 31 - if (unlikely(insn_get_rs1(insn) == 31 && a.get_soft_yield())) { - // Force the main interpreter loop to break - return advance_to_next_insn(a, pc, execute_status::success_and_yield); + if constexpr (is_an_i_interactive_state_access_v) { + if ((insn_get_rs1(insn) == 31) && a.get_soft_yield()) { + // Force the main interpreter loop to break + return advance_to_next_insn(a, pc, execute_status::success_and_yield); + } } return advance_to_next_insn(a, pc); } @@ -3117,7 +3290,7 @@ static FORCE_INLINE execute_status execute_SRAIW(STATE_ACCESS a, uint64_t &pc, u } template -static FORCE_INLINE execute_status execute_S(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { +static FORCE_INLINE execute_status execute_S(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { const uint64_t vaddr = a.read_x(insn_get_rs1(insn)); const int32_t imm = insn_S_get_imm(insn); const uint64_t val = a.read_x(insn_get_rs2(insn)); @@ -3133,34 +3306,34 @@ static FORCE_INLINE execute_status execute_S(STATE_ACCESS a, uint64_t &pc, uint6 /// \brief Implementation of the SB instruction. template -static FORCE_INLINE execute_status execute_SB(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { - dump_insn(a, pc, insn, "sb"); +static FORCE_INLINE execute_status execute_SB(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "sb"); return execute_S(a, pc, mcycle, insn); } /// \brief Implementation of the SH instruction. template -static FORCE_INLINE execute_status execute_SH(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { - dump_insn(a, pc, insn, "sh"); +static FORCE_INLINE execute_status execute_SH(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "sh"); return execute_S(a, pc, mcycle, insn); } /// \brief Implementation of the SW instruction. template -static FORCE_INLINE execute_status execute_SW(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { - dump_insn(a, pc, insn, "sw"); +static FORCE_INLINE execute_status execute_SW(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "sw"); return execute_S(a, pc, mcycle, insn); } /// \brief Implementation of the SD instruction. template -static FORCE_INLINE execute_status execute_SD(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { - dump_insn(a, pc, insn, "sd"); +static FORCE_INLINE execute_status execute_SD(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "sd"); return execute_S(a, pc, mcycle, insn); } template -static FORCE_INLINE execute_status execute_L(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { +static FORCE_INLINE execute_status execute_L(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { const uint64_t vaddr = a.read_x(insn_get_rs1(insn)); const int32_t imm = insn_I_get_imm(insn); T val = 0; @@ -3183,55 +3356,55 @@ static FORCE_INLINE execute_status execute_L(STATE_ACCESS a, uint64_t &pc, uint6 /// \brief Implementation of the LB instruction. template -static FORCE_INLINE execute_status execute_LB(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { - dump_insn(a, pc, insn, "lb"); +static FORCE_INLINE execute_status execute_LB(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "lb"); return execute_L(a, pc, mcycle, insn); } /// \brief Implementation of the LH instruction. template -static FORCE_INLINE execute_status execute_LH(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { - dump_insn(a, pc, insn, "lh"); +static FORCE_INLINE execute_status execute_LH(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "lh"); return execute_L(a, pc, mcycle, insn); } /// \brief Implementation of the LW instruction. template -static FORCE_INLINE execute_status execute_LW(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { - dump_insn(a, pc, insn, "lw"); +static FORCE_INLINE execute_status execute_LW(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "lw"); return execute_L(a, pc, mcycle, insn); } /// \brief Implementation of the LD instruction. template -static FORCE_INLINE execute_status execute_LD(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { - dump_insn(a, pc, insn, "ld"); +static FORCE_INLINE execute_status execute_LD(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "ld"); return execute_L(a, pc, mcycle, insn); } /// \brief Implementation of the LBU instruction. template -static FORCE_INLINE execute_status execute_LBU(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { - dump_insn(a, pc, insn, "lbu"); +static FORCE_INLINE execute_status execute_LBU(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "lbu"); return execute_L(a, pc, mcycle, insn); } /// \brief Implementation of the LHU instruction. template -static FORCE_INLINE execute_status execute_LHU(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { - dump_insn(a, pc, insn, "lhu"); +static FORCE_INLINE execute_status execute_LHU(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "lhu"); return execute_L(a, pc, mcycle, insn); } /// \brief Implementation of the LWU instruction. template -static FORCE_INLINE execute_status execute_LWU(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { - dump_insn(a, pc, insn, "lwu"); +static FORCE_INLINE execute_status execute_LWU(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "lwu"); return execute_L(a, pc, mcycle, insn); } template -static FORCE_INLINE execute_status execute_branch(STATE_ACCESS a, uint64_t &pc, uint32_t insn, const F &f) { +static FORCE_INLINE execute_status execute_branch(const STATE_ACCESS a, uint64_t &pc, uint32_t insn, const F &f) { const uint64_t rs1 = a.read_x(insn_get_rs1(insn)); const uint64_t rs2 = a.read_x(insn_get_rs2(insn)); if (f(rs1, rs2)) { @@ -3243,52 +3416,52 @@ static FORCE_INLINE execute_status execute_branch(STATE_ACCESS a, uint64_t &pc, /// \brief Implementation of the BEQ instruction. template -static FORCE_INLINE execute_status execute_BEQ(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "beq"); +static FORCE_INLINE execute_status execute_BEQ(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "beq"); return execute_branch(a, pc, insn, [](uint64_t rs1, uint64_t rs2) -> bool { return rs1 == rs2; }); } /// \brief Implementation of the BNE instruction. template -static FORCE_INLINE execute_status execute_BNE(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "bne"); +static FORCE_INLINE execute_status execute_BNE(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "bne"); return execute_branch(a, pc, insn, [](uint64_t rs1, uint64_t rs2) -> bool { return rs1 != rs2; }); } /// \brief Implementation of the BLT instruction. template -static FORCE_INLINE execute_status execute_BLT(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "blt"); +static FORCE_INLINE execute_status execute_BLT(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "blt"); return execute_branch(a, pc, insn, [](uint64_t rs1, uint64_t rs2) -> bool { return static_cast(rs1) < static_cast(rs2); }); } /// \brief Implementation of the BGE instruction. template -static FORCE_INLINE execute_status execute_BGE(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "bge"); +static FORCE_INLINE execute_status execute_BGE(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "bge"); return execute_branch(a, pc, insn, [](uint64_t rs1, uint64_t rs2) -> bool { return static_cast(rs1) >= static_cast(rs2); }); } /// \brief Implementation of the BLTU instruction. template -static FORCE_INLINE execute_status execute_BLTU(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "bltu"); +static FORCE_INLINE execute_status execute_BLTU(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "bltu"); return execute_branch(a, pc, insn, [](uint64_t rs1, uint64_t rs2) -> bool { return rs1 < rs2; }); } /// \brief Implementation of the BGEU instruction. template -static FORCE_INLINE execute_status execute_BGEU(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "bgeu"); +static FORCE_INLINE execute_status execute_BGEU(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "bgeu"); return execute_branch(a, pc, insn, [](uint64_t rs1, uint64_t rs2) -> bool { return rs1 >= rs2; }); } /// \brief Implementation of the LUI instruction. template -static FORCE_INLINE execute_status execute_LUI(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "lui"); +static FORCE_INLINE execute_status execute_LUI(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "lui"); if constexpr (rd_kind == rd_kind::x0) { return advance_to_next_insn(a, pc); } @@ -3299,8 +3472,8 @@ static FORCE_INLINE execute_status execute_LUI(STATE_ACCESS a, uint64_t &pc, uin /// \brief Implementation of the AUIPC instruction. template -static FORCE_INLINE execute_status execute_AUIPC(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "auipc"); +static FORCE_INLINE execute_status execute_AUIPC(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "auipc"); if constexpr (rd_kind == rd_kind::x0) { return advance_to_next_insn(a, pc); } @@ -3311,8 +3484,8 @@ static FORCE_INLINE execute_status execute_AUIPC(STATE_ACCESS a, uint64_t &pc, u /// \brief Implementation of the JAL instruction. template -static FORCE_INLINE execute_status execute_JAL(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "jal"); +static FORCE_INLINE execute_status execute_JAL(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "jal"); const uint64_t new_pc = pc + insn_J_get_imm(insn); if constexpr (rd_kind == rd_kind::x0) { return execute_jump(a, pc, new_pc); @@ -3324,8 +3497,8 @@ static FORCE_INLINE execute_status execute_JAL(STATE_ACCESS a, uint64_t &pc, uin /// \brief Implementation of the JALR instruction. template -static FORCE_INLINE execute_status execute_JALR(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "jalr"); +static FORCE_INLINE execute_status execute_JALR(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "jalr"); const uint64_t val = pc + 4; const uint64_t new_pc = static_cast(a.read_x(insn_get_rs1(insn)) + insn_I_get_imm(insn)) & ~static_cast(1); @@ -3340,13 +3513,13 @@ static FORCE_INLINE execute_status execute_JALR(STATE_ACCESS a, uint64_t &pc, ui /// \brief Implementation of the SFENCE.VMA instruction. /// \details This function is outlined to minimize host CPU code cache pressure. template -static FORCE_INLINE execute_status execute_SFENCE_VMA(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { +static FORCE_INLINE execute_status execute_SFENCE_VMA(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { // rs1 and rs2 are arbitrary, rest is set if (unlikely((insn & 0b11111110000000000111111111111111) != 0b00010010000000000000000001110011)) { return raise_illegal_insn_exception(a, pc, insn); } - INC_COUNTER(a.get_statistics(), fence_vma); - dump_insn(a, pc, insn, "sfence.vma"); + DUMP_STATS_INCR(a, "fence_vma"); + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "sfence.vma"); auto prv = a.read_iprv(); const uint64_t mstatus = a.read_mstatus(); @@ -3358,34 +3531,34 @@ static FORCE_INLINE execute_status execute_SFENCE_VMA(STATE_ACCESS a, uint64_t & const uint32_t rs1 = insn_get_rs1(insn); [[maybe_unused]] const uint32_t rs2 = insn_get_rs2(insn); if (rs1 == 0) { - a.flush_all_tlb(); -#ifdef DUMP_COUNTERS - INC_COUNTER(a.get_statistics(), tlb_flush_all); + flush_all_tlb(a); +#ifdef DUMP_STATS + DUMP_STATS_INCR(a, "tlb.flush_all"); if (rs2 == 0) { // Invalidates all address-translation cache entries, for all address spaces - INC_COUNTER(a.get_statistics(), tlb_flush_fence_vma_all); + DUMP_STATS_INCR(a, "tlb.flush_fence_vma_all"); } else { // Invalidates all address-translation cache entries matching the // address space identified by integer register rs2, // except for entries containing global mappings. - INC_COUNTER(a.get_statistics(), tlb_flush_fence_vma_asid); + DUMP_STATS_INCR(a, "tlb.flush_fence_vma_asid"); } #endif } else { const uint64_t vaddr = a.read_x(rs1); - a.flush_tlb_vaddr(vaddr); -#ifdef DUMP_COUNTERS - INC_COUNTER(a.get_statistics(), tlb_flush_vaddr); + flush_tlb_vaddr(a, vaddr); +#ifdef DUMP_STATS + DUMP_STATS_INCR(a, "tlb.flush_vaddr"); if (rs2 == 0) { // Invalidates all address-translation cache entries that contain leaf page table entries // corresponding to the virtual address in rs1, for all address spaces. - INC_COUNTER(a.get_statistics(), tlb_flush_fence_vma_vaddr); + DUMP_STATS_INCR(a, "tlb.flush_fence_vma_vaddr"); } else { // Invalidates all address-translation cache entries that contain leaf page table entries // corresponding to the virtual address in rs1 // and that match the address space identified by integer register rs2, // except for entries containing global mappings. - INC_COUNTER(a.get_statistics(), tlb_flush_fence_vma_asid_vaddr); + DUMP_STATS_INCR(a, "tlb.flush_fence_vma_asid_vaddr"); } #endif } @@ -3393,7 +3566,7 @@ static FORCE_INLINE execute_status execute_SFENCE_VMA(STATE_ACCESS a, uint64_t & } template -static FORCE_INLINE execute_status execute_SRLI_SRAI(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { +static FORCE_INLINE execute_status execute_SRLI_SRAI(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { // Use ifs instead of a switch to produce fewer branches for the most frequent instructions const auto funct7_sr1 = static_cast(insn_get_funct7_sr1(insn)); if (funct7_sr1 == insn_SRLI_SRAI_funct7_sr1::SRLI) { @@ -3406,7 +3579,7 @@ static FORCE_INLINE execute_status execute_SRLI_SRAI(STATE_ACCESS a, uint64_t &p } template -static FORCE_INLINE execute_status execute_SRLIW_SRAIW(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { +static FORCE_INLINE execute_status execute_SRLIW_SRAIW(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { // Use ifs instead of a switch to produce fewer branches for the most frequent instructions const auto funct7 = static_cast(insn_get_funct7(insn)); if (funct7 == insn_SRLIW_SRAIW_funct7::SRLIW) { @@ -3419,7 +3592,7 @@ static FORCE_INLINE execute_status execute_SRLIW_SRAIW(STATE_ACCESS a, uint64_t } template -static FORCE_INLINE execute_status execute_AMO_W(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { +static FORCE_INLINE execute_status execute_AMO_W(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { switch (static_cast(insn_get_funct7_sr2(insn))) { case insn_AMO_funct7_sr2::AMOADD: return execute_AMOADD_W(a, pc, mcycle, insn); @@ -3449,7 +3622,7 @@ static FORCE_INLINE execute_status execute_AMO_W(STATE_ACCESS a, uint64_t &pc, u } template -static FORCE_INLINE execute_status execute_AMO_D(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { +static FORCE_INLINE execute_status execute_AMO_D(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { switch (static_cast(insn_get_funct7_sr2(insn))) { case insn_AMO_funct7_sr2::AMOADD: return execute_AMOADD_D(a, pc, mcycle, insn); @@ -3479,7 +3652,7 @@ static FORCE_INLINE execute_status execute_AMO_D(STATE_ACCESS a, uint64_t &pc, u } template -static FORCE_INLINE execute_status execute_ADD_MUL_SUB(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { +static FORCE_INLINE execute_status execute_ADD_MUL_SUB(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { // Use ifs instead of a switch to produce fewer branches for the most frequent instructions const auto funct7 = static_cast(insn_get_funct7(insn)); if (funct7 == insn_ADD_MUL_SUB_funct7::ADD) { @@ -3495,7 +3668,7 @@ static FORCE_INLINE execute_status execute_ADD_MUL_SUB(STATE_ACCESS a, uint64_t } template -static FORCE_INLINE execute_status execute_SLL_MULH(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { +static FORCE_INLINE execute_status execute_SLL_MULH(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { // Use ifs instead of a switch to produce fewer branches for the most frequent instructions const auto funct7 = static_cast(insn_get_funct7(insn)); if (funct7 == insn_SLL_MULH_funct7::SLL) { @@ -3508,7 +3681,7 @@ static FORCE_INLINE execute_status execute_SLL_MULH(STATE_ACCESS a, uint64_t &pc } template -static FORCE_INLINE execute_status execute_SLT_MULHSU(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { +static FORCE_INLINE execute_status execute_SLT_MULHSU(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { // Use ifs instead of a switch to produce fewer branches for the most frequent instructions const auto funct7 = static_cast(insn_get_funct7(insn)); if (funct7 == insn_SLT_MULHSU_funct7::SLT) { @@ -3521,7 +3694,7 @@ static FORCE_INLINE execute_status execute_SLT_MULHSU(STATE_ACCESS a, uint64_t & } template -static FORCE_INLINE execute_status execute_SLTU_MULHU(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { +static FORCE_INLINE execute_status execute_SLTU_MULHU(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { // Use ifs instead of a switch to produce fewer branches for the most frequent instructions const auto funct7 = static_cast(insn_get_funct7(insn)); if (funct7 == insn_SLTU_MULHU_funct7::SLTU) { @@ -3534,7 +3707,7 @@ static FORCE_INLINE execute_status execute_SLTU_MULHU(STATE_ACCESS a, uint64_t & } template -static FORCE_INLINE execute_status execute_XOR_DIV(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { +static FORCE_INLINE execute_status execute_XOR_DIV(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { // Use ifs instead of a switch to produce fewer branches for the most frequent instructions const auto funct7 = static_cast(insn_get_funct7(insn)); if (funct7 == insn_XOR_DIV_funct7::XOR) { @@ -3547,7 +3720,7 @@ static FORCE_INLINE execute_status execute_XOR_DIV(STATE_ACCESS a, uint64_t &pc, } template -static FORCE_INLINE execute_status execute_SRL_DIVU_SRA(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { +static FORCE_INLINE execute_status execute_SRL_DIVU_SRA(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { // Use ifs instead of a switch to produce fewer branches for the most frequent instructions const auto funct7 = static_cast(insn_get_funct7(insn)); if (funct7 == insn_SRL_DIVU_SRA_funct7::SRL) { @@ -3563,7 +3736,7 @@ static FORCE_INLINE execute_status execute_SRL_DIVU_SRA(STATE_ACCESS a, uint64_t } template -static FORCE_INLINE execute_status execute_OR_REM(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { +static FORCE_INLINE execute_status execute_OR_REM(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { // Use ifs instead of a switch to produce fewer branches for the most frequent instructions const auto funct7 = static_cast(insn_get_funct7(insn)); if (funct7 == insn_OR_REM_funct7::OR) { @@ -3576,7 +3749,7 @@ static FORCE_INLINE execute_status execute_OR_REM(STATE_ACCESS a, uint64_t &pc, } template -static FORCE_INLINE execute_status execute_AND_REMU(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { +static FORCE_INLINE execute_status execute_AND_REMU(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { // Use ifs instead of a switch to produce fewer branches for the most frequent instructions const auto funct7 = static_cast(insn_get_funct7(insn)); if (funct7 == insn_AND_REMU_funct7::AND) { @@ -3589,7 +3762,7 @@ static FORCE_INLINE execute_status execute_AND_REMU(STATE_ACCESS a, uint64_t &pc } template -static FORCE_INLINE execute_status execute_ADDW_MULW_SUBW(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { +static FORCE_INLINE execute_status execute_ADDW_MULW_SUBW(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { // Use ifs instead of a switch to produce fewer branches for the most frequent instructions const auto funct7 = static_cast(insn_get_funct7(insn)); if (funct7 == insn_ADDW_MULW_SUBW_funct7::ADDW) { @@ -3605,7 +3778,7 @@ static FORCE_INLINE execute_status execute_ADDW_MULW_SUBW(STATE_ACCESS a, uint64 } template -static FORCE_INLINE execute_status execute_SRLW_DIVUW_SRAW(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { +static FORCE_INLINE execute_status execute_SRLW_DIVUW_SRAW(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { // Use ifs instead of a switch to produce fewer branches for the most frequent instructions const auto funct7 = static_cast(insn_get_funct7(insn)); if (funct7 == insn_SRLW_DIVUW_SRAW_funct7::SRLW) { @@ -3621,7 +3794,8 @@ static FORCE_INLINE execute_status execute_SRLW_DIVUW_SRAW(STATE_ACCESS a, uint6 } template -static FORCE_INLINE execute_status execute_privileged(STATE_ACCESS a, uint64_t &pc, uint64_t &mcycle, uint32_t insn) { +static FORCE_INLINE execute_status execute_privileged(const STATE_ACCESS a, uint64_t &pc, uint64_t &mcycle, + uint32_t insn) { switch (static_cast(insn)) { case insn_privileged::ECALL: return execute_ECALL(a, pc, insn); @@ -3681,7 +3855,7 @@ static inline T float_unbox(uint64_t val) { } template -static FORCE_INLINE execute_status execute_float_ternary_op_rm(STATE_ACCESS a, uint64_t &pc, uint32_t insn, +static FORCE_INLINE execute_status execute_float_ternary_op_rm(const STATE_ACCESS a, uint64_t &pc, uint32_t insn, const F &f) { const uint64_t fcsr = a.read_fcsr(); // The rounding mode comes from the insn @@ -3703,7 +3877,8 @@ static FORCE_INLINE execute_status execute_float_ternary_op_rm(STATE_ACCESS a, u } template -static FORCE_INLINE execute_status execute_float_binary_op_rm(STATE_ACCESS a, uint64_t &pc, uint32_t insn, const F &f) { +static FORCE_INLINE execute_status execute_float_binary_op_rm(const STATE_ACCESS a, uint64_t &pc, uint32_t insn, + const F &f) { const uint64_t fcsr = a.read_fcsr(); // The rounding mode comes from the insn const uint32_t rm = insn_get_rm(insn, fcsr); @@ -3723,7 +3898,8 @@ static FORCE_INLINE execute_status execute_float_binary_op_rm(STATE_ACCESS a, ui } template -static FORCE_INLINE execute_status execute_float_unary_op_rm(STATE_ACCESS a, uint64_t &pc, uint32_t insn, const F &f) { +static FORCE_INLINE execute_status execute_float_unary_op_rm(const STATE_ACCESS a, uint64_t &pc, uint32_t insn, + const F &f) { const uint64_t fcsr = a.read_fcsr(); // Unary operation should have rs2 set to 0 if (unlikely(insn_get_rs2(insn) != 0)) { @@ -3746,7 +3922,7 @@ static FORCE_INLINE execute_status execute_float_unary_op_rm(STATE_ACCESS a, uin } template -static FORCE_INLINE execute_status execute_FS(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { +static FORCE_INLINE execute_status execute_FS(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { const uint64_t vaddr = a.read_x(insn_get_rs1(insn)); const int32_t imm = insn_S_get_imm(insn); // A narrower n-bit transfer out of the floating-point @@ -3763,8 +3939,8 @@ static FORCE_INLINE execute_status execute_FS(STATE_ACCESS a, uint64_t &pc, uint } template -static FORCE_INLINE execute_status execute_FSW(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { - dump_insn(a, pc, insn, "fsw"); +static FORCE_INLINE execute_status execute_FSW(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fsw"); // If FS is OFF, attempts to read or write the float state will cause an illegal instruction exception. if (unlikely((a.read_mstatus() & MSTATUS_FS_MASK) == MSTATUS_FS_OFF)) { return raise_illegal_insn_exception(a, pc, insn); @@ -3773,8 +3949,8 @@ static FORCE_INLINE execute_status execute_FSW(STATE_ACCESS a, uint64_t &pc, uin } template -static FORCE_INLINE execute_status execute_FSD(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { - dump_insn(a, pc, insn, "fsd"); +static FORCE_INLINE execute_status execute_FSD(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fsd"); // If FS is OFF, attempts to read or write the float state will cause an illegal instruction exception. if (unlikely((a.read_mstatus() & MSTATUS_FS_MASK) == MSTATUS_FS_OFF)) { return raise_illegal_insn_exception(a, pc, insn); @@ -3783,7 +3959,7 @@ static FORCE_INLINE execute_status execute_FSD(STATE_ACCESS a, uint64_t &pc, uin } template -static FORCE_INLINE execute_status execute_FL(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { +static FORCE_INLINE execute_status execute_FL(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { // Loads the float value from virtual memory const uint64_t vaddr = a.read_x(insn_get_rs1(insn)); const int32_t imm = insn_I_get_imm(insn); @@ -3799,8 +3975,8 @@ static FORCE_INLINE execute_status execute_FL(STATE_ACCESS a, uint64_t &pc, uint } template -static FORCE_INLINE execute_status execute_FLW(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { - dump_insn(a, pc, insn, "flw"); +static FORCE_INLINE execute_status execute_FLW(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "flw"); // If FS is OFF, attempts to read or write the float state will cause an illegal instruction exception. if (unlikely((a.read_mstatus() & MSTATUS_FS_MASK) == MSTATUS_FS_OFF)) { return raise_illegal_insn_exception(a, pc, insn); @@ -3809,8 +3985,8 @@ static FORCE_INLINE execute_status execute_FLW(STATE_ACCESS a, uint64_t &pc, uin } template -static FORCE_INLINE execute_status execute_FLD(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { - dump_insn(a, pc, insn, "fld"); +static FORCE_INLINE execute_status execute_FLD(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fld"); // If FS is OFF, attempts to read or write the float state will cause an illegal instruction exception. if (unlikely((a.read_mstatus() & MSTATUS_FS_MASK) == MSTATUS_FS_OFF)) { return raise_illegal_insn_exception(a, pc, insn); @@ -3819,8 +3995,8 @@ static FORCE_INLINE execute_status execute_FLD(STATE_ACCESS a, uint64_t &pc, uin } template -static FORCE_INLINE execute_status execute_FMADD_S(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "fmadd.s"); +static FORCE_INLINE execute_status execute_FMADD_S(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fmadd.s"); return execute_float_ternary_op_rm(a, pc, insn, [](uint32_t s1, uint32_t s2, uint32_t s3, uint32_t rm, uint32_t *fflags) -> uint32_t { return i_sfloat32::fma(s1, s2, s3, static_cast(rm), fflags); @@ -3828,8 +4004,8 @@ static FORCE_INLINE execute_status execute_FMADD_S(STATE_ACCESS a, uint64_t &pc, } template -static FORCE_INLINE execute_status execute_FMADD_D(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "fmadd.d"); +static FORCE_INLINE execute_status execute_FMADD_D(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fmadd.d"); return execute_float_ternary_op_rm(a, pc, insn, [](uint64_t s1, uint64_t s2, uint64_t s3, uint32_t rm, uint32_t *fflags) -> uint64_t { return i_sfloat64::fma(s1, s2, s3, static_cast(rm), fflags); @@ -3837,7 +4013,7 @@ static FORCE_INLINE execute_status execute_FMADD_D(STATE_ACCESS a, uint64_t &pc, } template -static FORCE_INLINE execute_status execute_FMADD(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { +static FORCE_INLINE execute_status execute_FMADD(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { // If FS is OFF, attempts to read or write the float state will cause an illegal instruction exception. if (unlikely((a.read_mstatus() & MSTATUS_FS_MASK) == MSTATUS_FS_OFF)) { return raise_illegal_insn_exception(a, pc, insn); @@ -3853,8 +4029,8 @@ static FORCE_INLINE execute_status execute_FMADD(STATE_ACCESS a, uint64_t &pc, u } template -static FORCE_INLINE execute_status execute_FMSUB_S(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "fmsub.s"); +static FORCE_INLINE execute_status execute_FMSUB_S(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fmsub.s"); return execute_float_ternary_op_rm(a, pc, insn, [](uint32_t s1, uint32_t s2, uint32_t s3, uint32_t rm, uint32_t *fflags) -> uint32_t { return i_sfloat32::fma(s1, s2, s3 ^ i_sfloat32::SIGN_MASK, static_cast(rm), fflags); @@ -3862,8 +4038,8 @@ static FORCE_INLINE execute_status execute_FMSUB_S(STATE_ACCESS a, uint64_t &pc, } template -static FORCE_INLINE execute_status execute_FMSUB_D(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "fmsub.d"); +static FORCE_INLINE execute_status execute_FMSUB_D(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fmsub.d"); return execute_float_ternary_op_rm(a, pc, insn, [](uint64_t s1, uint64_t s2, uint64_t s3, uint32_t rm, uint32_t *fflags) -> uint64_t { return i_sfloat64::fma(s1, s2, s3 ^ i_sfloat64::SIGN_MASK, static_cast(rm), fflags); @@ -3871,7 +4047,7 @@ static FORCE_INLINE execute_status execute_FMSUB_D(STATE_ACCESS a, uint64_t &pc, } template -static FORCE_INLINE execute_status execute_FMSUB(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { +static FORCE_INLINE execute_status execute_FMSUB(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { // If FS is OFF, attempts to read or write the float state will cause an illegal instruction exception. if (unlikely((a.read_mstatus() & MSTATUS_FS_MASK) == MSTATUS_FS_OFF)) { return raise_illegal_insn_exception(a, pc, insn); @@ -3887,8 +4063,8 @@ static FORCE_INLINE execute_status execute_FMSUB(STATE_ACCESS a, uint64_t &pc, u } template -static FORCE_INLINE execute_status execute_FNMADD_S(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "fnmadd.s"); +static FORCE_INLINE execute_status execute_FNMADD_S(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fnmadd.s"); return execute_float_ternary_op_rm(a, pc, insn, [](uint32_t s1, uint32_t s2, uint32_t s3, uint32_t rm, uint32_t *fflags) -> uint32_t { return i_sfloat32::fma(s1 ^ i_sfloat32::SIGN_MASK, s2, s3 ^ i_sfloat32::SIGN_MASK, @@ -3897,8 +4073,8 @@ static FORCE_INLINE execute_status execute_FNMADD_S(STATE_ACCESS a, uint64_t &pc } template -static FORCE_INLINE execute_status execute_FNMADD_D(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "fnmadd.d"); +static FORCE_INLINE execute_status execute_FNMADD_D(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fnmadd.d"); return execute_float_ternary_op_rm(a, pc, insn, [](uint64_t s1, uint64_t s2, uint64_t s3, uint32_t rm, uint32_t *fflags) -> uint64_t { return i_sfloat64::fma(s1 ^ i_sfloat64::SIGN_MASK, s2, s3 ^ i_sfloat64::SIGN_MASK, @@ -3907,7 +4083,7 @@ static FORCE_INLINE execute_status execute_FNMADD_D(STATE_ACCESS a, uint64_t &pc } template -static FORCE_INLINE execute_status execute_FNMADD(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { +static FORCE_INLINE execute_status execute_FNMADD(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { // If FS is OFF, attempts to read or write the float state will cause an illegal instruction exception. if (unlikely((a.read_mstatus() & MSTATUS_FS_MASK) == MSTATUS_FS_OFF)) { return raise_illegal_insn_exception(a, pc, insn); @@ -3923,8 +4099,8 @@ static FORCE_INLINE execute_status execute_FNMADD(STATE_ACCESS a, uint64_t &pc, } template -static FORCE_INLINE execute_status execute_FNMSUB_S(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "fnmsub.s"); +static FORCE_INLINE execute_status execute_FNMSUB_S(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fnmsub.s"); return execute_float_ternary_op_rm(a, pc, insn, [](uint32_t s1, uint32_t s2, uint32_t s3, uint32_t rm, uint32_t *fflags) -> uint32_t { return i_sfloat32::fma(s1 ^ i_sfloat32::SIGN_MASK, s2, s3, static_cast(rm), fflags); @@ -3932,8 +4108,8 @@ static FORCE_INLINE execute_status execute_FNMSUB_S(STATE_ACCESS a, uint64_t &pc } template -static FORCE_INLINE execute_status execute_FNMSUB_D(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "fnmsub.d"); +static FORCE_INLINE execute_status execute_FNMSUB_D(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fnmsub.d"); return execute_float_ternary_op_rm(a, pc, insn, [](uint64_t s1, uint64_t s2, uint64_t s3, uint32_t rm, uint32_t *fflags) -> uint64_t { return i_sfloat64::fma(s1 ^ i_sfloat64::SIGN_MASK, s2, s3, static_cast(rm), fflags); @@ -3941,7 +4117,7 @@ static FORCE_INLINE execute_status execute_FNMSUB_D(STATE_ACCESS a, uint64_t &pc } template -static FORCE_INLINE execute_status execute_FNMSUB(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { +static FORCE_INLINE execute_status execute_FNMSUB(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { // If FS is OFF, attempts to read or write the float state will cause an illegal instruction exception. if (unlikely((a.read_mstatus() & MSTATUS_FS_MASK) == MSTATUS_FS_OFF)) { return raise_illegal_insn_exception(a, pc, insn); @@ -3957,8 +4133,8 @@ static FORCE_INLINE execute_status execute_FNMSUB(STATE_ACCESS a, uint64_t &pc, } template -static FORCE_INLINE execute_status execute_FADD_S(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "fadd.s"); +static FORCE_INLINE execute_status execute_FADD_S(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fadd.s"); return execute_float_binary_op_rm(a, pc, insn, [](uint32_t s1, uint32_t s2, uint32_t rm, uint32_t *fflags) -> uint32_t { return i_sfloat32::add(s1, s2, static_cast(rm), fflags); @@ -3966,8 +4142,8 @@ static FORCE_INLINE execute_status execute_FADD_S(STATE_ACCESS a, uint64_t &pc, } template -static FORCE_INLINE execute_status execute_FADD_D(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "fadd.d"); +static FORCE_INLINE execute_status execute_FADD_D(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fadd.d"); return execute_float_binary_op_rm(a, pc, insn, [](uint64_t s1, uint64_t s2, uint32_t rm, uint32_t *fflags) -> uint64_t { return i_sfloat64::add(s1, s2, static_cast(rm), fflags); @@ -3975,8 +4151,8 @@ static FORCE_INLINE execute_status execute_FADD_D(STATE_ACCESS a, uint64_t &pc, } template -static FORCE_INLINE execute_status execute_FSUB_S(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "fsub.s"); +static FORCE_INLINE execute_status execute_FSUB_S(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fsub.s"); return execute_float_binary_op_rm(a, pc, insn, [](uint32_t s1, uint32_t s2, uint32_t rm, uint32_t *fflags) -> uint32_t { return i_sfloat32::add(s1, s2 ^ i_sfloat32::SIGN_MASK, static_cast(rm), fflags); @@ -3984,8 +4160,8 @@ static FORCE_INLINE execute_status execute_FSUB_S(STATE_ACCESS a, uint64_t &pc, } template -static FORCE_INLINE execute_status execute_FSUB_D(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "fsub.d"); +static FORCE_INLINE execute_status execute_FSUB_D(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fsub.d"); return execute_float_binary_op_rm(a, pc, insn, [](uint64_t s1, uint64_t s2, uint32_t rm, uint32_t *fflags) -> uint64_t { return i_sfloat64::add(s1, s2 ^ i_sfloat64::SIGN_MASK, static_cast(rm), fflags); @@ -3993,8 +4169,8 @@ static FORCE_INLINE execute_status execute_FSUB_D(STATE_ACCESS a, uint64_t &pc, } template -static FORCE_INLINE execute_status execute_FMUL_S(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "fmul.s"); +static FORCE_INLINE execute_status execute_FMUL_S(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fmul.s"); return execute_float_binary_op_rm(a, pc, insn, [](uint32_t s1, uint32_t s2, uint32_t rm, uint32_t *fflags) -> uint32_t { return i_sfloat32::mul(s1, s2, static_cast(rm), fflags); @@ -4002,8 +4178,8 @@ static FORCE_INLINE execute_status execute_FMUL_S(STATE_ACCESS a, uint64_t &pc, } template -static FORCE_INLINE execute_status execute_FMUL_D(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "fmul.d"); +static FORCE_INLINE execute_status execute_FMUL_D(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fmul.d"); return execute_float_binary_op_rm(a, pc, insn, [](uint64_t s1, uint64_t s2, uint32_t rm, uint32_t *fflags) -> uint64_t { return i_sfloat64::mul(s1, s2, static_cast(rm), fflags); @@ -4011,8 +4187,8 @@ static FORCE_INLINE execute_status execute_FMUL_D(STATE_ACCESS a, uint64_t &pc, } template -static FORCE_INLINE execute_status execute_FDIV_S(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "fdiv.s"); +static FORCE_INLINE execute_status execute_FDIV_S(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fdiv.s"); return execute_float_binary_op_rm(a, pc, insn, [](uint32_t s1, uint32_t s2, uint32_t rm, uint32_t *fflags) -> uint32_t { return i_sfloat32::div(s1, s2, static_cast(rm), fflags); @@ -4020,8 +4196,8 @@ static FORCE_INLINE execute_status execute_FDIV_S(STATE_ACCESS a, uint64_t &pc, } template -static FORCE_INLINE execute_status execute_FDIV_D(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "fdiv.d"); +static FORCE_INLINE execute_status execute_FDIV_D(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fdiv.d"); return execute_float_binary_op_rm(a, pc, insn, [](uint64_t s1, uint64_t s2, uint32_t rm, uint32_t *fflags) -> uint64_t { return i_sfloat64::div(s1, s2, static_cast(rm), fflags); @@ -4029,7 +4205,7 @@ static FORCE_INLINE execute_status execute_FDIV_D(STATE_ACCESS a, uint64_t &pc, } template -static FORCE_INLINE execute_status execute_FCLASS(STATE_ACCESS a, uint64_t &pc, uint32_t insn, const F &f) { +static FORCE_INLINE execute_status execute_FCLASS(const STATE_ACCESS a, uint64_t &pc, uint32_t insn, const F &f) { const uint32_t rd = insn_get_rd(insn); if (unlikely(rd == 0)) { return advance_to_next_insn(a, pc); @@ -4041,7 +4217,8 @@ static FORCE_INLINE execute_status execute_FCLASS(STATE_ACCESS a, uint64_t &pc, } template -static FORCE_INLINE execute_status execute_float_binary_op(STATE_ACCESS a, uint64_t &pc, uint32_t insn, const F &f) { +static FORCE_INLINE execute_status execute_float_binary_op(const STATE_ACCESS a, uint64_t &pc, uint32_t insn, + const F &f) { const uint64_t fcsr = a.read_fcsr(); // We must always check if input operands are properly NaN-boxed. T s1 = float_unbox(a.read_f(insn_get_rs1(insn))); @@ -4055,7 +4232,7 @@ static FORCE_INLINE execute_status execute_float_binary_op(STATE_ACCESS a, uint6 } template -static FORCE_INLINE execute_status execute_float_cmp_op(STATE_ACCESS a, uint64_t &pc, uint32_t insn, const F &f) { +static FORCE_INLINE execute_status execute_float_cmp_op(const STATE_ACCESS a, uint64_t &pc, uint32_t insn, const F &f) { const uint64_t fcsr = a.read_fcsr(); // We must always check if input operands are properly NaN-boxed. T s1 = float_unbox(a.read_f(insn_get_rs1(insn))); @@ -4073,8 +4250,8 @@ static FORCE_INLINE execute_status execute_float_cmp_op(STATE_ACCESS a, uint64_t } template -static FORCE_INLINE execute_status execute_FSGNJ_S(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "fsgnj.s"); +static FORCE_INLINE execute_status execute_FSGNJ_S(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fsgnj.s"); return execute_float_binary_op(a, pc, insn, [](uint32_t s1, uint32_t s2, const uint32_t * /*fflags*/) -> uint32_t { return (s1 & ~i_sfloat32::SIGN_MASK) | (s2 & i_sfloat32::SIGN_MASK); @@ -4082,8 +4259,8 @@ static FORCE_INLINE execute_status execute_FSGNJ_S(STATE_ACCESS a, uint64_t &pc, } template -static FORCE_INLINE execute_status execute_FSGNJN_S(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "fsgnjn.s"); +static FORCE_INLINE execute_status execute_FSGNJN_S(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fsgnjn.s"); return execute_float_binary_op(a, pc, insn, [](uint32_t s1, uint32_t s2, const uint32_t * /*fflags*/) -> uint32_t { return (s1 & ~i_sfloat32::SIGN_MASK) | ((s2 & i_sfloat32::SIGN_MASK) ^ i_sfloat32::SIGN_MASK); @@ -4091,8 +4268,8 @@ static FORCE_INLINE execute_status execute_FSGNJN_S(STATE_ACCESS a, uint64_t &pc } template -static FORCE_INLINE execute_status execute_FSGNJX_S(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "fsgnjx.s"); +static FORCE_INLINE execute_status execute_FSGNJX_S(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fsgnjx.s"); return execute_float_binary_op(a, pc, insn, [](uint32_t s1, uint32_t s2, const uint32_t * /*fflags*/) -> uint32_t { return s1 ^ (s2 & i_sfloat32::SIGN_MASK); @@ -4100,7 +4277,7 @@ static FORCE_INLINE execute_status execute_FSGNJX_S(STATE_ACCESS a, uint64_t &pc } template -static FORCE_INLINE execute_status execute_FSGN_S(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { +static FORCE_INLINE execute_status execute_FSGN_S(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { switch (static_cast(insn_get_funct3_000000000000(insn))) { case insn_FSGN_funct3_000000000000::J: return execute_FSGNJ_S(a, pc, insn); @@ -4114,8 +4291,8 @@ static FORCE_INLINE execute_status execute_FSGN_S(STATE_ACCESS a, uint64_t &pc, } template -static FORCE_INLINE execute_status execute_FSGNJ_D(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "fsgnj.d"); +static FORCE_INLINE execute_status execute_FSGNJ_D(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fsgnj.d"); return execute_float_binary_op(a, pc, insn, [](uint64_t s1, uint64_t s2, const uint32_t * /*fflags*/) -> uint64_t { return (s1 & ~i_sfloat64::SIGN_MASK) | (s2 & i_sfloat64::SIGN_MASK); @@ -4123,8 +4300,8 @@ static FORCE_INLINE execute_status execute_FSGNJ_D(STATE_ACCESS a, uint64_t &pc, } template -static FORCE_INLINE execute_status execute_FSGNJN_D(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "fsgnjn.d"); +static FORCE_INLINE execute_status execute_FSGNJN_D(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fsgnjn.d"); return execute_float_binary_op(a, pc, insn, [](uint64_t s1, uint64_t s2, const uint32_t * /*fflags*/) -> uint64_t { return (s1 & ~i_sfloat64::SIGN_MASK) | ((s2 & i_sfloat64::SIGN_MASK) ^ i_sfloat64::SIGN_MASK); @@ -4132,8 +4309,8 @@ static FORCE_INLINE execute_status execute_FSGNJN_D(STATE_ACCESS a, uint64_t &pc } template -static FORCE_INLINE execute_status execute_FSGNJX_D(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "fsgnjx.d"); +static FORCE_INLINE execute_status execute_FSGNJX_D(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fsgnjx.d"); return execute_float_binary_op(a, pc, insn, [](uint64_t s1, uint64_t s2, const uint32_t * /*fflags*/) -> uint64_t { return s1 ^ (s2 & i_sfloat64::SIGN_MASK); @@ -4141,7 +4318,7 @@ static FORCE_INLINE execute_status execute_FSGNJX_D(STATE_ACCESS a, uint64_t &pc } template -static FORCE_INLINE execute_status execute_FSGN_D(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { +static FORCE_INLINE execute_status execute_FSGN_D(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { switch (static_cast(insn_get_funct3_000000000000(insn))) { case insn_FSGN_funct3_000000000000::J: return execute_FSGNJ_D(a, pc, insn); @@ -4155,21 +4332,21 @@ static FORCE_INLINE execute_status execute_FSGN_D(STATE_ACCESS a, uint64_t &pc, } template -static FORCE_INLINE execute_status execute_FMIN_S(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "fmin.s"); +static FORCE_INLINE execute_status execute_FMIN_S(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fmin.s"); return execute_float_binary_op(a, pc, insn, [](uint32_t s1, uint32_t s2, uint32_t *fflags) -> uint32_t { return i_sfloat32::min(s1, s2, fflags); }); } template -static FORCE_INLINE execute_status execute_FMAX_S(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "fmax.s"); +static FORCE_INLINE execute_status execute_FMAX_S(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fmax.s"); return execute_float_binary_op(a, pc, insn, [](uint32_t s1, uint32_t s2, uint32_t *fflags) -> uint32_t { return i_sfloat32::max(s1, s2, fflags); }); } template -static FORCE_INLINE execute_status execute_FMINMAX_S(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { +static FORCE_INLINE execute_status execute_FMINMAX_S(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { switch (static_cast(insn_get_funct3_000000000000(insn))) { case insn_FMIN_FMAX_funct3_000000000000::MIN: return execute_FMIN_S(a, pc, insn); @@ -4181,21 +4358,21 @@ static FORCE_INLINE execute_status execute_FMINMAX_S(STATE_ACCESS a, uint64_t &p } template -static FORCE_INLINE execute_status execute_FMIN_D(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "fmin.d"); +static FORCE_INLINE execute_status execute_FMIN_D(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fmin.d"); return execute_float_binary_op(a, pc, insn, [](uint64_t s1, uint64_t s2, uint32_t *fflags) -> uint64_t { return i_sfloat64::min(s1, s2, fflags); }); } template -static FORCE_INLINE execute_status execute_FMAX_D(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "fmax.d"); +static FORCE_INLINE execute_status execute_FMAX_D(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fmax.d"); return execute_float_binary_op(a, pc, insn, [](uint64_t s1, uint64_t s2, uint32_t *fflags) -> uint64_t { return i_sfloat64::max(s1, s2, fflags); }); } template -static FORCE_INLINE execute_status execute_FMINMAX_D(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { +static FORCE_INLINE execute_status execute_FMINMAX_D(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { switch (static_cast(insn_get_funct3_000000000000(insn))) { case insn_FMIN_FMAX_funct3_000000000000::MIN: return execute_FMIN_D(a, pc, insn); @@ -4207,7 +4384,7 @@ static FORCE_INLINE execute_status execute_FMINMAX_D(STATE_ACCESS a, uint64_t &p } template -static FORCE_INLINE execute_status execute_FCVT_F_F(STATE_ACCESS a, uint64_t &pc, uint32_t insn, const F &f) { +static FORCE_INLINE execute_status execute_FCVT_F_F(const STATE_ACCESS a, uint64_t &pc, uint32_t insn, const F &f) { const uint64_t fcsr = a.read_fcsr(); // The rounding mode comes from the insn const uint32_t rm = insn_get_rm(insn, fcsr); @@ -4227,7 +4404,7 @@ static FORCE_INLINE execute_status execute_FCVT_F_F(STATE_ACCESS a, uint64_t &pc } template -static FORCE_INLINE execute_status execute_FCVT_X_F(STATE_ACCESS a, uint64_t &pc, uint32_t insn, const F &f) { +static FORCE_INLINE execute_status execute_FCVT_X_F(const STATE_ACCESS a, uint64_t &pc, uint32_t insn, const F &f) { const uint64_t fcsr = a.read_fcsr(); // The rounding mode comes from the insn const uint32_t rm = insn_get_rm(insn, fcsr); @@ -4249,7 +4426,7 @@ static FORCE_INLINE execute_status execute_FCVT_X_F(STATE_ACCESS a, uint64_t &pc } template -static FORCE_INLINE execute_status execute_FCVT_F_X(STATE_ACCESS a, uint64_t &pc, uint32_t insn, const F &f) { +static FORCE_INLINE execute_status execute_FCVT_F_X(const STATE_ACCESS a, uint64_t &pc, uint32_t insn, const F &f) { const uint64_t fcsr = a.read_fcsr(); // The rounding mode comes from the insn const uint32_t rm = insn_get_rm(insn, fcsr); @@ -4268,8 +4445,8 @@ static FORCE_INLINE execute_status execute_FCVT_F_X(STATE_ACCESS a, uint64_t &pc } template -static FORCE_INLINE execute_status execute_FCVT_S_D(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "fcvt.s.d"); +static FORCE_INLINE execute_status execute_FCVT_S_D(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fcvt.s.d"); return execute_FCVT_F_F(a, pc, insn, [](uint64_t s1, uint32_t rm, uint32_t *fflags) -> uint32_t { return sfloat_cvt_f64_f32(s1, static_cast(rm), fflags); @@ -4277,8 +4454,8 @@ static FORCE_INLINE execute_status execute_FCVT_S_D(STATE_ACCESS a, uint64_t &pc } template -static FORCE_INLINE execute_status execute_FCVT_D_S(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "fcvt.d.s"); +static FORCE_INLINE execute_status execute_FCVT_D_S(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fcvt.d.s"); return execute_FCVT_F_F(a, pc, insn, [](uint32_t s1, uint32_t /*rm*/, uint32_t *fflags) -> uint64_t { // FCVT.D.S will never round, since it's a widen operation. @@ -4287,47 +4464,47 @@ static FORCE_INLINE execute_status execute_FCVT_D_S(STATE_ACCESS a, uint64_t &pc } template -static FORCE_INLINE execute_status execute_FSQRT_S(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "fsqrt.s"); +static FORCE_INLINE execute_status execute_FSQRT_S(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fsqrt.s"); return execute_float_unary_op_rm(a, pc, insn, [](uint32_t s1, uint32_t rm, uint32_t *fflags) -> uint32_t { return i_sfloat32::sqrt(s1, static_cast(rm), fflags); }); } template -static FORCE_INLINE execute_status execute_FSQRT_D(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "fsqrt.d"); +static FORCE_INLINE execute_status execute_FSQRT_D(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fsqrt.d"); return execute_float_unary_op_rm(a, pc, insn, [](uint64_t s1, uint32_t rm, uint32_t *fflags) -> uint64_t { return i_sfloat64::sqrt(s1, static_cast(rm), fflags); }); } template -static FORCE_INLINE execute_status execute_FLE_S(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "fle.s"); +static FORCE_INLINE execute_status execute_FLE_S(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fle.s"); return execute_float_cmp_op(a, pc, insn, [](uint32_t s1, uint32_t s2, uint32_t *fflags) -> uint64_t { return static_cast(i_sfloat32::le(s1, s2, fflags)); }); } template -static FORCE_INLINE execute_status execute_FLT_S(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "flt.s"); +static FORCE_INLINE execute_status execute_FLT_S(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "flt.s"); return execute_float_cmp_op(a, pc, insn, [](uint32_t s1, uint32_t s2, uint32_t *fflags) -> uint64_t { return static_cast(i_sfloat32::lt(s1, s2, fflags)); }); } template -static FORCE_INLINE execute_status execute_FEQ_S(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "feq.s"); +static FORCE_INLINE execute_status execute_FEQ_S(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "feq.s"); return execute_float_cmp_op(a, pc, insn, [](uint32_t s1, uint32_t s2, uint32_t *fflags) -> uint64_t { return static_cast(i_sfloat32::eq(s1, s2, fflags)); }); } template -static FORCE_INLINE execute_status execute_FCMP_S(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { +static FORCE_INLINE execute_status execute_FCMP_S(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { switch (static_cast(insn_get_funct3_000000000000(insn))) { case insn_FCMP_funct3_000000000000::LT: return execute_FLT_S(a, pc, insn); @@ -4341,31 +4518,31 @@ static FORCE_INLINE execute_status execute_FCMP_S(STATE_ACCESS a, uint64_t &pc, } template -static FORCE_INLINE execute_status execute_FLE_D(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "fle.d"); +static FORCE_INLINE execute_status execute_FLE_D(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fle.d"); return execute_float_cmp_op(a, pc, insn, [](uint64_t s1, uint64_t s2, uint32_t *fflags) -> uint64_t { return static_cast(i_sfloat64::le(s1, s2, fflags)); }); } template -static FORCE_INLINE execute_status execute_FLT_D(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "flt.d"); +static FORCE_INLINE execute_status execute_FLT_D(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "flt.d"); return execute_float_cmp_op(a, pc, insn, [](uint64_t s1, uint64_t s2, uint32_t *fflags) -> uint64_t { return static_cast(i_sfloat64::lt(s1, s2, fflags)); }); } template -static FORCE_INLINE execute_status execute_FEQ_D(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "feq.d"); +static FORCE_INLINE execute_status execute_FEQ_D(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "feq.d"); return execute_float_cmp_op(a, pc, insn, [](uint64_t s1, uint64_t s2, uint32_t *fflags) -> uint64_t { return static_cast(i_sfloat64::eq(s1, s2, fflags)); }); } template -static FORCE_INLINE execute_status execute_FCMP_D(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { +static FORCE_INLINE execute_status execute_FCMP_D(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { switch (static_cast(insn_get_funct3_000000000000(insn))) { case insn_FCMP_funct3_000000000000::LT: return execute_FLT_D(a, pc, insn); @@ -4379,8 +4556,8 @@ static FORCE_INLINE execute_status execute_FCMP_D(STATE_ACCESS a, uint64_t &pc, } template -static FORCE_INLINE execute_status execute_FCVT_W_S(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "fcvt.w.s"); +static FORCE_INLINE execute_status execute_FCVT_W_S(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fcvt.w.s"); return execute_FCVT_X_F(a, pc, insn, [](uint32_t s1, uint32_t rm, uint32_t *fflags) -> uint64_t { const auto val = i_sfloat32::cvt_f_i(s1, static_cast(rm), fflags); // For XLEN > 32, FCVT.W.S sign-extends the 32-bit result. @@ -4389,8 +4566,8 @@ static FORCE_INLINE execute_status execute_FCVT_W_S(STATE_ACCESS a, uint64_t &pc } template -static FORCE_INLINE execute_status execute_FCVT_WU_S(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "fcvt.wu.s"); +static FORCE_INLINE execute_status execute_FCVT_WU_S(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fcvt.wu.s"); return execute_FCVT_X_F(a, pc, insn, [](uint32_t s1, uint32_t rm, uint32_t *fflags) -> uint64_t { const auto val = i_sfloat32::cvt_f_i(s1, static_cast(rm), fflags); // For XLEN > 32, FCVT.WU.S sign-extends the 32-bit result. @@ -4399,8 +4576,8 @@ static FORCE_INLINE execute_status execute_FCVT_WU_S(STATE_ACCESS a, uint64_t &p } template -static FORCE_INLINE execute_status execute_FCVT_L_S(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "fcvt.l.s"); +static FORCE_INLINE execute_status execute_FCVT_L_S(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fcvt.l.s"); return execute_FCVT_X_F(a, pc, insn, [](uint32_t s1, uint32_t rm, uint32_t *fflags) -> uint64_t { const auto val = i_sfloat32::cvt_f_i(s1, static_cast(rm), fflags); return static_cast(val); @@ -4408,16 +4585,16 @@ static FORCE_INLINE execute_status execute_FCVT_L_S(STATE_ACCESS a, uint64_t &pc } template -static FORCE_INLINE execute_status execute_FCVT_LU_S(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "fcvt.lu.s"); +static FORCE_INLINE execute_status execute_FCVT_LU_S(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fcvt.lu.s"); return execute_FCVT_X_F(a, pc, insn, [](uint32_t s1, uint32_t rm, uint32_t *fflags) -> uint64_t { return i_sfloat32::cvt_f_i(s1, static_cast(rm), fflags); }); } template -static FORCE_INLINE execute_status execute_FCVT_W_D(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "fcvt.w.d"); +static FORCE_INLINE execute_status execute_FCVT_W_D(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fcvt.w.d"); return execute_FCVT_X_F(a, pc, insn, [](uint64_t s1, uint32_t rm, uint32_t *fflags) -> uint64_t { const auto val = i_sfloat64::cvt_f_i(s1, static_cast(rm), fflags); // For RV64, FCVT.W.D sign-extends the 32-bit result. @@ -4426,8 +4603,8 @@ static FORCE_INLINE execute_status execute_FCVT_W_D(STATE_ACCESS a, uint64_t &pc } template -static FORCE_INLINE execute_status execute_FCVT_WU_D(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "fcvt.wu.d"); +static FORCE_INLINE execute_status execute_FCVT_WU_D(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fcvt.wu.d"); return execute_FCVT_X_F(a, pc, insn, [](uint64_t s1, uint32_t rm, uint32_t *fflags) -> uint64_t { const auto val = i_sfloat64::cvt_f_i(s1, static_cast(rm), fflags); // For RV64, FCVT.WU.D sign-extends the 32-bit result. @@ -4436,8 +4613,8 @@ static FORCE_INLINE execute_status execute_FCVT_WU_D(STATE_ACCESS a, uint64_t &p } template -static FORCE_INLINE execute_status execute_FCVT_L_D(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "fcvt.l.d"); +static FORCE_INLINE execute_status execute_FCVT_L_D(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fcvt.l.d"); return execute_FCVT_X_F(a, pc, insn, [](uint64_t s1, uint32_t rm, uint32_t *fflags) -> uint64_t { const auto val = i_sfloat64::cvt_f_i(s1, static_cast(rm), fflags); return static_cast(val); @@ -4445,79 +4622,79 @@ static FORCE_INLINE execute_status execute_FCVT_L_D(STATE_ACCESS a, uint64_t &pc } template -static FORCE_INLINE execute_status execute_FCVT_LU_D(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "fcvt.lu.d"); +static FORCE_INLINE execute_status execute_FCVT_LU_D(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fcvt.lu.d"); return execute_FCVT_X_F(a, pc, insn, [](uint64_t s1, uint32_t rm, uint32_t *fflags) -> uint64_t { return i_sfloat64::cvt_f_i(s1, static_cast(rm), fflags); }); } template -static FORCE_INLINE execute_status execute_FCVT_S_W(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "fcvt.s.w"); +static FORCE_INLINE execute_status execute_FCVT_S_W(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fcvt.s.w"); return execute_FCVT_F_X(a, pc, insn, [](uint64_t s1, uint32_t rm, uint32_t *fflags) -> uint32_t { return i_sfloat32::cvt_i_f(static_cast(s1), static_cast(rm), fflags); }); } template -static FORCE_INLINE execute_status execute_FCVT_S_WU(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "fcvt.s.wu"); +static FORCE_INLINE execute_status execute_FCVT_S_WU(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fcvt.s.wu"); return execute_FCVT_F_X(a, pc, insn, [](uint64_t s1, uint32_t rm, uint32_t *fflags) -> uint32_t { return i_sfloat32::cvt_i_f(static_cast(s1), static_cast(rm), fflags); }); } template -static FORCE_INLINE execute_status execute_FCVT_S_L(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "fcvt.s.l"); +static FORCE_INLINE execute_status execute_FCVT_S_L(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fcvt.s.l"); return execute_FCVT_F_X(a, pc, insn, [](uint64_t s1, uint32_t rm, uint32_t *fflags) -> uint32_t { return i_sfloat32::cvt_i_f(static_cast(s1), static_cast(rm), fflags); }); } template -static FORCE_INLINE execute_status execute_FCVT_S_LU(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "fcvt.s.lu"); +static FORCE_INLINE execute_status execute_FCVT_S_LU(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fcvt.s.lu"); return execute_FCVT_F_X(a, pc, insn, [](uint64_t s1, uint32_t rm, uint32_t *fflags) -> uint32_t { return i_sfloat32::cvt_i_f(s1, static_cast(rm), fflags); }); } template -static FORCE_INLINE execute_status execute_FCVT_D_W(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "fcvt.d.w"); +static FORCE_INLINE execute_status execute_FCVT_D_W(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fcvt.d.w"); return execute_FCVT_F_X(a, pc, insn, [](uint64_t s1, uint32_t rm, uint32_t *fflags) -> uint64_t { return i_sfloat64::cvt_i_f(static_cast(s1), static_cast(rm), fflags); }); } template -static FORCE_INLINE execute_status execute_FCVT_D_WU(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "fcvt.d.wu"); +static FORCE_INLINE execute_status execute_FCVT_D_WU(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fcvt.d.wu"); return execute_FCVT_F_X(a, pc, insn, [](uint64_t s1, uint32_t rm, uint32_t *fflags) -> uint64_t { return i_sfloat64::cvt_i_f(static_cast(s1), static_cast(rm), fflags); }); } template -static FORCE_INLINE execute_status execute_FCVT_D_L(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "fcvt.d.l"); +static FORCE_INLINE execute_status execute_FCVT_D_L(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fcvt.d.l"); return execute_FCVT_F_X(a, pc, insn, [](uint64_t s1, uint32_t rm, uint32_t *fflags) -> uint64_t { return i_sfloat64::cvt_i_f(static_cast(s1), static_cast(rm), fflags); }); } template -static FORCE_INLINE execute_status execute_FCVT_D_LU(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "fcvt.d.lu"); +static FORCE_INLINE execute_status execute_FCVT_D_LU(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fcvt.d.lu"); return execute_FCVT_F_X(a, pc, insn, [](uint64_t s1, uint32_t rm, uint32_t *fflags) -> uint64_t { return i_sfloat64::cvt_i_f(s1, static_cast(rm), fflags); }); } template -static FORCE_INLINE execute_status execute_FMV_F_X(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { +static FORCE_INLINE execute_status execute_FMV_F_X(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { // Should have funct3 set to 0 if (unlikely(insn_get_funct3(insn) != 0)) { return raise_illegal_insn_exception(a, pc, insn); @@ -4530,26 +4707,26 @@ static FORCE_INLINE execute_status execute_FMV_F_X(STATE_ACCESS a, uint64_t &pc, } template -static FORCE_INLINE execute_status execute_FMV_W_X(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "fmv.w.x"); +static FORCE_INLINE execute_status execute_FMV_W_X(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fmv.w.x"); return execute_FMV_F_X(a, pc, insn); } template -static FORCE_INLINE execute_status execute_FMV_D_X(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "fmv.d.x"); +static FORCE_INLINE execute_status execute_FMV_D_X(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fmv.d.x"); return execute_FMV_F_X(a, pc, insn); } template -static FORCE_INLINE execute_status execute_FCLASS_S(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "fclass.s"); +static FORCE_INLINE execute_status execute_FCLASS_S(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fclass.s"); return execute_FCLASS(a, pc, insn, [](uint32_t s1) -> uint64_t { return i_sfloat32::fclass(s1); }); } template -static FORCE_INLINE execute_status execute_FMV_X_W(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "fmv.x.w"); +static FORCE_INLINE execute_status execute_FMV_X_W(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fmv.x.w"); const uint32_t rd = insn_get_rd(insn); if (unlikely(rd == 0)) { return advance_to_next_insn(a, pc); @@ -4563,7 +4740,7 @@ static FORCE_INLINE execute_status execute_FMV_X_W(STATE_ACCESS a, uint64_t &pc, } template -static FORCE_INLINE execute_status execute_FMV_FCLASS_S(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { +static FORCE_INLINE execute_status execute_FMV_FCLASS_S(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { switch (static_cast(insn_get_funct3_000000000000(insn))) { case insn_FMV_FCLASS_funct3_000000000000::FMV: return execute_FMV_X_W(a, pc, insn); @@ -4575,14 +4752,14 @@ static FORCE_INLINE execute_status execute_FMV_FCLASS_S(STATE_ACCESS a, uint64_t } template -static FORCE_INLINE execute_status execute_FCLASS_D(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "fclass.d"); +static FORCE_INLINE execute_status execute_FCLASS_D(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fclass.d"); return execute_FCLASS(a, pc, insn, [](uint64_t s1) -> uint64_t { return i_sfloat64::fclass(s1); }); } template -static FORCE_INLINE execute_status execute_FMV_X_D(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, insn, "fmv.x.d"); +static FORCE_INLINE execute_status execute_FMV_X_D(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, insn, "fmv.x.d"); const uint32_t rd = insn_get_rd(insn); if (unlikely(rd == 0)) { return advance_to_next_insn(a, pc); @@ -4593,7 +4770,7 @@ static FORCE_INLINE execute_status execute_FMV_X_D(STATE_ACCESS a, uint64_t &pc, } template -static FORCE_INLINE execute_status execute_FMV_FCLASS_D(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { +static FORCE_INLINE execute_status execute_FMV_FCLASS_D(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { switch (static_cast(insn_get_funct3_000000000000(insn))) { case insn_FMV_FCLASS_funct3_000000000000::FMV: return execute_FMV_X_D(a, pc, insn); @@ -4605,7 +4782,7 @@ static FORCE_INLINE execute_status execute_FMV_FCLASS_D(STATE_ACCESS a, uint64_t } template -static FORCE_INLINE execute_status execute_FCVT_FMV_FCLASS(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { +static FORCE_INLINE execute_status execute_FCVT_FMV_FCLASS(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { switch (static_cast(insn_get_funct7_rs2(insn))) { case insn_FD_funct7_rs2::FCVT_W_S: return execute_FCVT_W_S(a, pc, insn); @@ -4657,7 +4834,7 @@ static FORCE_INLINE execute_status execute_FCVT_FMV_FCLASS(STATE_ACCESS a, uint6 } template -static FORCE_INLINE execute_status execute_FD(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { +static FORCE_INLINE execute_status execute_FD(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { // If FS is OFF, attempts to read or write the float state will cause an illegal instruction exception. if (unlikely((a.read_mstatus() & MSTATUS_FS_MASK) == MSTATUS_FS_OFF)) { return raise_illegal_insn_exception(a, pc, insn); @@ -4701,8 +4878,8 @@ static FORCE_INLINE execute_status execute_FD(STATE_ACCESS a, uint64_t &pc, uint } template -static FORCE_INLINE execute_status execute_C_L(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t rd, uint32_t rs1, - int32_t imm) { +static FORCE_INLINE execute_status execute_C_L(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t rd, + uint32_t rs1, int32_t imm) { const uint64_t vaddr = a.read_x(rs1); T val = 0; if (unlikely(!read_virtual_memory(a, pc, mcycle, vaddr + imm, &val))) { @@ -4718,7 +4895,7 @@ static FORCE_INLINE execute_status execute_C_L(STATE_ACCESS a, uint64_t &pc, uin } template -static FORCE_INLINE execute_status execute_C_S(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t rs2, +static FORCE_INLINE execute_status execute_C_S(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t rs2, uint32_t rs1, int32_t imm) { const uint64_t vaddr = a.read_x(rs1); const uint64_t val = a.read_x(rs2); @@ -4733,7 +4910,7 @@ static FORCE_INLINE execute_status execute_C_S(STATE_ACCESS a, uint64_t &pc, uin } template -static FORCE_INLINE execute_status execute_C_FL(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t rd, +static FORCE_INLINE execute_status execute_C_FL(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t rd, uint32_t rs1, int32_t imm) { // Loads the float value from virtual memory const uint64_t vaddr = a.read_x(rs1); @@ -4748,7 +4925,7 @@ static FORCE_INLINE execute_status execute_C_FL(STATE_ACCESS a, uint64_t &pc, ui } template -static FORCE_INLINE execute_status execute_C_FS(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t rs2, +static FORCE_INLINE execute_status execute_C_FS(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t rs2, uint32_t rs1, int32_t imm) { const uint64_t vaddr = a.read_x(rs1); // A narrower n-bit transfer out of the floating-point @@ -4766,8 +4943,8 @@ static FORCE_INLINE execute_status execute_C_FS(STATE_ACCESS a, uint64_t &pc, ui /// \brief Implementation of the C.ADDI4SPN instruction. template -static FORCE_INLINE execute_status execute_C_ADDI4SPN(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, static_cast(insn), "c.addi4spn"); +static FORCE_INLINE execute_status execute_C_ADDI4SPN(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, static_cast(insn), "c.addi4spn"); // rd cannot be zero (guaranteed by RISC-V spec design) const uint32_t rd = insn_get_CIW_CL_rd_CS_CA_rs2(insn); // imm cannot be zero (guaranteed by the jump table) @@ -4781,10 +4958,9 @@ static FORCE_INLINE execute_status execute_C_ADDI4SPN(STATE_ACCESS a, uint64_t & /// \brief Implementation of the C.FLD instruction. template -static FORCE_INLINE execute_status execute_C_FLD(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { - dump_insn(a, pc, static_cast(insn), "c.fld"); - // If FS is OFF, attempts to read or write the float state will cause an illegal instruction - // exception. +static FORCE_INLINE execute_status execute_C_FLD(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, static_cast(insn), "c.fld"); + // If FS is OFF, attempts to read or write the float state will cause an illegal instruction exception. if (unlikely((a.read_mstatus() & MSTATUS_FS_MASK) == MSTATUS_FS_OFF)) { return raise_illegal_insn_exception(a, pc, insn); } @@ -4796,8 +4972,8 @@ static FORCE_INLINE execute_status execute_C_FLD(STATE_ACCESS a, uint64_t &pc, u /// \brief Implementation of the C.LW instruction. template -static FORCE_INLINE execute_status execute_C_LW(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { - dump_insn(a, pc, static_cast(insn), "c.lw"); +static FORCE_INLINE execute_status execute_C_LW(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, static_cast(insn), "c.lw"); const uint32_t rd = insn_get_CIW_CL_rd_CS_CA_rs2(insn); const uint32_t rs1 = insn_get_CL_CS_CA_CB_rs1(insn); const int32_t imm = insn_get_C_LW_C_SW_imm(insn); @@ -4806,8 +4982,8 @@ static FORCE_INLINE execute_status execute_C_LW(STATE_ACCESS a, uint64_t &pc, ui /// \brief Implementation of the C.LD instruction. template -static FORCE_INLINE execute_status execute_C_LD(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { - dump_insn(a, pc, static_cast(insn), "c.ld"); +static FORCE_INLINE execute_status execute_C_LD(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, static_cast(insn), "c.ld"); const uint32_t rd = insn_get_CIW_CL_rd_CS_CA_rs2(insn); const uint32_t rs1 = insn_get_CL_CS_CA_CB_rs1(insn); const int32_t imm = insn_get_CL_CS_imm(insn); @@ -4816,10 +4992,9 @@ static FORCE_INLINE execute_status execute_C_LD(STATE_ACCESS a, uint64_t &pc, ui /// \brief Implementation of the C.FSD instruction. template -static FORCE_INLINE execute_status execute_C_FSD(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { - dump_insn(a, pc, static_cast(insn), "c.fsd"); - // If FS is OFF, attempts to read or write the float state will cause an illegal instruction - // exception. +static FORCE_INLINE execute_status execute_C_FSD(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, static_cast(insn), "c.fsd"); + // If FS is OFF, attempts to read or write the float state will cause an illegal instruction exception. if (unlikely((a.read_mstatus() & MSTATUS_FS_MASK) == MSTATUS_FS_OFF)) { return raise_illegal_insn_exception(a, pc, insn); } @@ -4831,8 +5006,8 @@ static FORCE_INLINE execute_status execute_C_FSD(STATE_ACCESS a, uint64_t &pc, u /// \brief Implementation of the C.SW instruction. template -static FORCE_INLINE execute_status execute_C_SW(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { - dump_insn(a, pc, static_cast(insn), "c.sw"); +static FORCE_INLINE execute_status execute_C_SW(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, static_cast(insn), "c.sw"); const uint32_t rs1 = insn_get_CL_CS_CA_CB_rs1(insn); const uint32_t rs2 = insn_get_CIW_CL_rd_CS_CA_rs2(insn); const int32_t imm = insn_get_C_LW_C_SW_imm(insn); @@ -4841,8 +5016,8 @@ static FORCE_INLINE execute_status execute_C_SW(STATE_ACCESS a, uint64_t &pc, ui /// \brief Implementation of the C.SD instruction. template -static FORCE_INLINE execute_status execute_C_SD(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { - dump_insn(a, pc, static_cast(insn), "c.sd"); +static FORCE_INLINE execute_status execute_C_SD(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, static_cast(insn), "c.sd"); const uint32_t rs1 = insn_get_CL_CS_CA_CB_rs1(insn); const uint32_t rs2 = insn_get_CIW_CL_rd_CS_CA_rs2(insn); const int32_t imm = insn_get_CL_CS_imm(insn); @@ -4851,16 +5026,16 @@ static FORCE_INLINE execute_status execute_C_SD(STATE_ACCESS a, uint64_t &pc, ui /// \brief Implementation of the C.NOP instruction. template -static FORCE_INLINE execute_status execute_C_NOP(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, static_cast(insn), "c.nop"); +static FORCE_INLINE execute_status execute_C_NOP(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, static_cast(insn), "c.nop"); // Really do nothing return advance_to_next_insn<2>(a, pc); } /// \brief Implementation of the C.ADDI instruction. template -static FORCE_INLINE execute_status execute_C_ADDI(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, static_cast(insn), "c.addi"); +static FORCE_INLINE execute_status execute_C_ADDI(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, static_cast(insn), "c.addi"); // rd cannot be zero (guaranteed by jump table) const uint32_t rd = insn_get_rd(insn); const int32_t imm = insn_get_CI_CB_imm_se(insn); @@ -4874,8 +5049,8 @@ static FORCE_INLINE execute_status execute_C_ADDI(STATE_ACCESS a, uint64_t &pc, /// \brief Implementation of the C.addiw instruction. template -static FORCE_INLINE execute_status execute_C_ADDIW(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, static_cast(insn), "c.addiw"); +static FORCE_INLINE execute_status execute_C_ADDIW(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, static_cast(insn), "c.addiw"); // rd cannot be zero (guaranteed by jump table) const uint32_t rd = insn_get_rd(insn); const uint64_t rd_value = a.read_x(rd); @@ -4888,8 +5063,8 @@ static FORCE_INLINE execute_status execute_C_ADDIW(STATE_ACCESS a, uint64_t &pc, /// \brief Implementation of the C.LI instruction. template -static FORCE_INLINE execute_status execute_C_LI(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, static_cast(insn), "c.li"); +static FORCE_INLINE execute_status execute_C_LI(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, static_cast(insn), "c.li"); // rd cannot be zero (guaranteed by jump table) const uint32_t rd = insn_get_rd(insn); const int32_t imm = insn_get_CI_CB_imm_se(insn); @@ -4899,8 +5074,8 @@ static FORCE_INLINE execute_status execute_C_LI(STATE_ACCESS a, uint64_t &pc, ui /// \brief Implementation of the C.ADDI16SP instruction. template -static FORCE_INLINE execute_status execute_C_ADDI16SP(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, static_cast(insn), "c.addi16sp"); +static FORCE_INLINE execute_status execute_C_ADDI16SP(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, static_cast(insn), "c.addi16sp"); // imm cannot be zero (guaranteed by the jump table) const int32_t imm = insn_get_C_ADDI16SP_imm(insn); const uint64_t rs1_value = a.read_x(2); @@ -4912,8 +5087,8 @@ static FORCE_INLINE execute_status execute_C_ADDI16SP(STATE_ACCESS a, uint64_t & /// \brief Implementation of the C.LUI instruction. template -static FORCE_INLINE execute_status execute_C_LUI(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, static_cast(insn), "c.lui"); +static FORCE_INLINE execute_status execute_C_LUI(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, static_cast(insn), "c.lui"); // imm cannot be zero (guaranteed by the jump table) const int32_t imm = insn_get_C_LUI_imm(insn); // rd cannot be zero (guaranteed by the jump table) @@ -4924,8 +5099,8 @@ static FORCE_INLINE execute_status execute_C_LUI(STATE_ACCESS a, uint64_t &pc, u /// \brief Implementation of the C.SRLI instruction. template -static FORCE_INLINE execute_status execute_C_SRLI(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, static_cast(insn), "c.srli"); +static FORCE_INLINE execute_status execute_C_SRLI(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, static_cast(insn), "c.srli"); const uint32_t rs1 = insn_get_CL_CS_CA_CB_rs1(insn); // imm cannot be zero (guaranteed by the jump table) const uint32_t imm = insn_get_CI_CB_imm(insn); @@ -4936,8 +5111,8 @@ static FORCE_INLINE execute_status execute_C_SRLI(STATE_ACCESS a, uint64_t &pc, /// \brief Implementation of the C.SRAI instruction. template -static FORCE_INLINE execute_status execute_C_SRAI(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, static_cast(insn), "c.srai"); +static FORCE_INLINE execute_status execute_C_SRAI(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, static_cast(insn), "c.srai"); const uint32_t rs1 = insn_get_CL_CS_CA_CB_rs1(insn); // imm cannot be zero (guaranteed by the jump table) const uint32_t imm = insn_get_CI_CB_imm(insn); @@ -4948,8 +5123,8 @@ static FORCE_INLINE execute_status execute_C_SRAI(STATE_ACCESS a, uint64_t &pc, /// \brief Implementation of the C.ANDI instruction. template -static FORCE_INLINE execute_status execute_C_ANDI(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, static_cast(insn), "c.andi"); +static FORCE_INLINE execute_status execute_C_ANDI(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, static_cast(insn), "c.andi"); const uint32_t rs1 = insn_get_CL_CS_CA_CB_rs1(insn); const int32_t imm = insn_get_CI_CB_imm_se(insn); const uint64_t rs1_value = a.read_x(rs1); @@ -4958,7 +5133,7 @@ static FORCE_INLINE execute_status execute_C_ANDI(STATE_ACCESS a, uint64_t &pc, } template -static FORCE_INLINE execute_status execute_C_arithmetic(STATE_ACCESS a, uint64_t &pc, uint32_t insn, const F &f) { +static FORCE_INLINE execute_status execute_C_arithmetic(const STATE_ACCESS a, uint64_t &pc, uint32_t insn, const F &f) { // Ensure rs1 and rs2 are loaded in order: do not nest with call to f() as // the order of evaluation of arguments in a function call is undefined. const uint32_t rs1 = insn_get_CL_CS_CA_CB_rs1(insn); @@ -4971,8 +5146,8 @@ static FORCE_INLINE execute_status execute_C_arithmetic(STATE_ACCESS a, uint64_t /// \brief Implementation of the C.SUB instruction. template -static FORCE_INLINE execute_status execute_C_SUB(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, static_cast(insn), "c.sub"); +static FORCE_INLINE execute_status execute_C_SUB(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, static_cast(insn), "c.sub"); return execute_C_arithmetic(a, pc, insn, [](uint64_t rs1_value, uint64_t rs2_value) -> uint64_t { uint64_t val = 0; __builtin_sub_overflow(rs1_value, rs2_value, &val); @@ -4982,32 +5157,32 @@ static FORCE_INLINE execute_status execute_C_SUB(STATE_ACCESS a, uint64_t &pc, u /// \brief Implementation of the C.XOR instruction. template -static FORCE_INLINE execute_status execute_C_XOR(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, static_cast(insn), "c.xor"); +static FORCE_INLINE execute_status execute_C_XOR(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, static_cast(insn), "c.xor"); return execute_C_arithmetic(a, pc, insn, [](uint64_t rs1_value, uint64_t rs2_value) -> uint64_t { return rs1_value ^ rs2_value; }); } /// \brief Implementation of the C.OR instruction. template -static FORCE_INLINE execute_status execute_C_OR(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, static_cast(insn), "c.or"); +static FORCE_INLINE execute_status execute_C_OR(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, static_cast(insn), "c.or"); return execute_C_arithmetic(a, pc, insn, [](uint64_t rs1_value, uint64_t rs2_value) -> uint64_t { return rs1_value | rs2_value; }); } /// \brief Implementation of the C.AND instruction. template -static FORCE_INLINE execute_status execute_C_AND(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, static_cast(insn), "c.and"); +static FORCE_INLINE execute_status execute_C_AND(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, static_cast(insn), "c.and"); return execute_C_arithmetic(a, pc, insn, [](uint64_t rs1_value, uint64_t rs2_value) -> uint64_t { return rs1_value & rs2_value; }); } /// \brief Implementation of the C.SUBW instruction. template -static FORCE_INLINE execute_status execute_C_SUBW(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, static_cast(insn), "c.subw"); +static FORCE_INLINE execute_status execute_C_SUBW(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, static_cast(insn), "c.subw"); return execute_C_arithmetic(a, pc, insn, [](uint64_t rs1_value, uint64_t rs2_value) -> uint64_t { // Convert 64-bit to 32-bit auto rs1w = static_cast(rs1_value); @@ -5020,8 +5195,8 @@ static FORCE_INLINE execute_status execute_C_SUBW(STATE_ACCESS a, uint64_t &pc, /// \brief Implementation of the C.ADDW instruction. template -static FORCE_INLINE execute_status execute_C_ADDW(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, static_cast(insn), "c.addw"); +static FORCE_INLINE execute_status execute_C_ADDW(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, static_cast(insn), "c.addw"); return execute_C_arithmetic(a, pc, insn, [](uint64_t rs1_value, uint64_t rs2_value) -> uint64_t { // Discard upper 32 bits auto rs1w = static_cast(rs1_value); @@ -5034,16 +5209,16 @@ static FORCE_INLINE execute_status execute_C_ADDW(STATE_ACCESS a, uint64_t &pc, /// \brief Implementation of the C_J instruction. template -static FORCE_INLINE execute_status execute_C_J(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, static_cast(insn), "c.j"); +static FORCE_INLINE execute_status execute_C_J(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, static_cast(insn), "c.j"); const uint64_t new_pc = pc + static_cast(insn_get_C_J_imm(insn)); return execute_jump(a, pc, new_pc); } /// \brief Implementation of the C.BEQZ instruction. template -static FORCE_INLINE execute_status execute_C_BEQZ(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, static_cast(insn), "c.beqz"); +static FORCE_INLINE execute_status execute_C_BEQZ(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, static_cast(insn), "c.beqz"); const uint32_t rs1 = insn_get_CL_CS_CA_CB_rs1(insn); if (a.read_x(rs1) == 0) { const int32_t imm = insn_get_C_BEQZ_BNEZ_imm(insn); @@ -5055,8 +5230,8 @@ static FORCE_INLINE execute_status execute_C_BEQZ(STATE_ACCESS a, uint64_t &pc, /// \brief Implementation of the C.BNEZ instruction. template -static FORCE_INLINE execute_status execute_C_BNEZ(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, static_cast(insn), "c.bnez"); +static FORCE_INLINE execute_status execute_C_BNEZ(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, static_cast(insn), "c.bnez"); const uint32_t rs1 = insn_get_CL_CS_CA_CB_rs1(insn); if (a.read_x(rs1) != 0) { const int32_t imm = insn_get_C_BEQZ_BNEZ_imm(insn); @@ -5068,8 +5243,8 @@ static FORCE_INLINE execute_status execute_C_BNEZ(STATE_ACCESS a, uint64_t &pc, /// \brief Implementation of the C.SLLI instruction. template -static FORCE_INLINE execute_status execute_C_SLLI(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, static_cast(insn), "c.slli"); +static FORCE_INLINE execute_status execute_C_SLLI(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, static_cast(insn), "c.slli"); // rd cannot be zero (guaranteed by jump table) const uint32_t rd = insn_get_rd(insn); // imm cannot be zero (guaranteed by jump table) @@ -5081,10 +5256,9 @@ static FORCE_INLINE execute_status execute_C_SLLI(STATE_ACCESS a, uint64_t &pc, /// \brief Implementation of the C.FLDSP instruction. template -static FORCE_INLINE execute_status execute_C_FLDSP(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { - dump_insn(a, pc, static_cast(insn), "c.fldsp"); - // If FS is OFF, attempts to read or write the float state will cause an illegal instruction - // exception. +static FORCE_INLINE execute_status execute_C_FLDSP(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, static_cast(insn), "c.fldsp"); + // If FS is OFF, attempts to read or write the float state will cause an illegal instruction exception. if (unlikely((a.read_mstatus() & MSTATUS_FS_MASK) == MSTATUS_FS_OFF)) { return raise_illegal_insn_exception(a, pc, insn); } @@ -5095,8 +5269,8 @@ static FORCE_INLINE execute_status execute_C_FLDSP(STATE_ACCESS a, uint64_t &pc, /// \brief Implementation of the C.LWSP instruction. template -static FORCE_INLINE execute_status execute_C_LWSP(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { - dump_insn(a, pc, static_cast(insn), "c.lwsp"); +static FORCE_INLINE execute_status execute_C_LWSP(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, static_cast(insn), "c.lwsp"); // rd cannot be zero (guaranteed by jump table) const uint32_t rd = insn_get_rd(insn); const int32_t imm = insn_get_C_LWSP_imm(insn); @@ -5105,8 +5279,8 @@ static FORCE_INLINE execute_status execute_C_LWSP(STATE_ACCESS a, uint64_t &pc, /// \brief Implementation of the C.LDSP instruction. template -static FORCE_INLINE execute_status execute_C_LDSP(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { - dump_insn(a, pc, static_cast(insn), "c.ldsp"); +static FORCE_INLINE execute_status execute_C_LDSP(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, static_cast(insn), "c.ldsp"); // rd cannot be zero (guaranteed by jump table) const uint32_t rd = insn_get_rd(insn); const int32_t imm = insn_get_C_FLDSP_LDSP_imm(insn); @@ -5115,8 +5289,8 @@ static FORCE_INLINE execute_status execute_C_LDSP(STATE_ACCESS a, uint64_t &pc, /// \brief Implementation of the C.JR instruction. template -static FORCE_INLINE execute_status execute_C_JR(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, static_cast(insn), "c.jr"); +static FORCE_INLINE execute_status execute_C_JR(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, static_cast(insn), "c.jr"); // rs1 cannot be zero (guaranteed by the jump table) const uint32_t rs1 = insn_get_rd(insn); const uint64_t new_pc = a.read_x(rs1) & ~static_cast(1); @@ -5125,8 +5299,8 @@ static FORCE_INLINE execute_status execute_C_JR(STATE_ACCESS a, uint64_t &pc, ui /// \brief Implementation of the C.MV instruction. template -static FORCE_INLINE execute_status execute_C_MV(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, static_cast(insn), "c.mv"); +static FORCE_INLINE execute_status execute_C_MV(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, static_cast(insn), "c.mv"); // rd cannot be zero (guaranteed by the jump table) const uint32_t rd = insn_get_rd(insn); const uint32_t rs2 = insn_get_CR_CSS_rs2(insn); @@ -5137,16 +5311,16 @@ static FORCE_INLINE execute_status execute_C_MV(STATE_ACCESS a, uint64_t &pc, ui /// \brief Implementation of the C.EBREAK instruction. template -static FORCE_INLINE execute_status execute_C_EBREAK(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, static_cast(insn), "c.ebreak"); +static FORCE_INLINE execute_status execute_C_EBREAK(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, static_cast(insn), "c.ebreak"); pc = raise_exception(a, pc, MCAUSE_BREAKPOINT, pc); return advance_to_raised_exception(a, pc); } /// \brief Implementation of the C.JALR instruction. template -static FORCE_INLINE execute_status execute_C_JALR(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, static_cast(insn), "c.jalr"); +static FORCE_INLINE execute_status execute_C_JALR(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, static_cast(insn), "c.jalr"); const uint32_t rs1 = insn_get_rd(insn); const uint64_t new_pc = a.read_x(rs1) & ~static_cast(1); const uint64_t val = pc + 2; @@ -5156,8 +5330,8 @@ static FORCE_INLINE execute_status execute_C_JALR(STATE_ACCESS a, uint64_t &pc, /// \brief Implementation of the C.ADD instruction. template -static FORCE_INLINE execute_status execute_C_ADD(STATE_ACCESS a, uint64_t &pc, uint32_t insn) { - dump_insn(a, pc, static_cast(insn), "c.add"); +static FORCE_INLINE execute_status execute_C_ADD(const STATE_ACCESS a, uint64_t &pc, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, static_cast(insn), "c.add"); // rd cannot be zero (guaranteed by the jump table) const uint32_t rd = insn_get_rd(insn); const uint32_t rs2 = insn_get_CR_CSS_rs2(insn); @@ -5171,10 +5345,9 @@ static FORCE_INLINE execute_status execute_C_ADD(STATE_ACCESS a, uint64_t &pc, u /// \brief Implementation of the C.FSDSP instruction. template -static FORCE_INLINE execute_status execute_C_FSDSP(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { - dump_insn(a, pc, static_cast(insn), "c.fsdsp"); - // If FS is OFF, attempts to read or write the float state will cause an illegal instruction - // exception. +static FORCE_INLINE execute_status execute_C_FSDSP(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, static_cast(insn), "c.fsdsp"); + // If FS is OFF, attempts to read or write the float state will cause an illegal instruction exception. if (unlikely((a.read_mstatus() & MSTATUS_FS_MASK) == MSTATUS_FS_OFF)) { return raise_illegal_insn_exception(a, pc, insn); } @@ -5185,8 +5358,8 @@ static FORCE_INLINE execute_status execute_C_FSDSP(STATE_ACCESS a, uint64_t &pc, /// \brief Implementation of the C.SWSP instruction. template -static FORCE_INLINE execute_status execute_C_SWSP(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { - dump_insn(a, pc, static_cast(insn), "c.swsp"); +static FORCE_INLINE execute_status execute_C_SWSP(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, static_cast(insn), "c.swsp"); const uint32_t rs2 = insn_get_CR_CSS_rs2(insn); const int32_t imm = insn_get_C_SWSP_imm(insn); return execute_C_S(a, pc, mcycle, rs2, 0x2, imm); @@ -5194,8 +5367,8 @@ static FORCE_INLINE execute_status execute_C_SWSP(STATE_ACCESS a, uint64_t &pc, /// \brief Implementation of the C.SDSP instruction. template -static FORCE_INLINE execute_status execute_C_SDSP(STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { - dump_insn(a, pc, static_cast(insn), "c.sdsp"); +static FORCE_INLINE execute_status execute_C_SDSP(const STATE_ACCESS a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { + [[maybe_unused]] auto note = dump_insn(a, pc, static_cast(insn), "c.sdsp"); const uint32_t rs2 = insn_get_CR_CSS_rs2(insn); const int32_t imm = insn_get_C_FSDSP_SDSP_imm(insn); return execute_C_S(a, pc, mcycle, rs2, 0x2, imm); @@ -5212,12 +5385,13 @@ enum class fetch_status : int { /// \param a Machine state accessor object. /// \param pc Virtual address for the current instruction being executed. /// \param vaddr Virtual address to be fetched. -/// \param phptr Receives host pointer. +/// \param vf_offset Receives vf_offset in the TLB slot +/// \param pma_index Receives the index of PMA where vaddr falls /// \return Returns fetch_status::success if load succeeded, fetch_status::exception if it caused an exception. // In that case, raise the exception. template -static FORCE_INLINE fetch_status fetch_translate_pc_slow(STATE_ACCESS a, uint64_t &pc, uint64_t vaddr, - unsigned char **phptr) { +static FORCE_INLINE fetch_status fetch_translate_pc_slow(const STATE_ACCESS a, uint64_t &pc, uint64_t vaddr, + i_state_access_fast_addr_t &vf_offset, uint64_t &pma_index) { uint64_t paddr{}; // Walk page table and obtain the physical address if (unlikely(!translate_virtual_address(a, &paddr, vaddr, PTE_XWR_X_SHIFT))) { @@ -5225,16 +5399,14 @@ static FORCE_INLINE fetch_status fetch_translate_pc_slow(STATE_ACCESS a, uint64_ return fetch_status::exception; } // Walk memory map to find the range that contains the physical address - auto &pma = find_pma_entry(a, paddr); + const auto &ar = find_pma(a, paddr, pma_index); // We only execute directly from RAM (as in "random access memory") // If the range is not memory or not executable, this as a PMA violation - if (unlikely(!pma.get_istart_M() || !pma.get_istart_X())) { + if (unlikely(!ar.is_memory() || !ar.is_executable())) { pc = raise_exception(a, pc, MCAUSE_INSN_ACCESS_FAULT, vaddr); return fetch_status::exception; } - unsigned char *hpage = a.template replace_tlb_entry(vaddr, paddr, pma); - const uint64_t hoffset = vaddr & PAGE_OFFSET_MASK; - *phptr = hpage + hoffset; + replace_tlb_entry(a, vaddr, paddr, pma_index, vf_offset); return fetch_status::success; } @@ -5243,19 +5415,24 @@ static FORCE_INLINE fetch_status fetch_translate_pc_slow(STATE_ACCESS a, uint64_ /// \param a Machine state accessor object. /// \param pc Virtual address for the current instruction being executed. /// \param vaddr Virtual address to be fetched. -/// \param phptr Receives the host pointer. +/// \param vf_offset Receives vf_offset in the TLB slot +/// \param pma_index Receives the index of PMA where vaddr falls /// \return Returns fetch_status::success if load succeeded, fetch_status::exception if it caused an exception. // In that case, raise the exception. template -static FORCE_INLINE fetch_status fetch_translate_pc(STATE_ACCESS a, uint64_t &pc, uint64_t vaddr, - unsigned char **phptr) { +static FORCE_INLINE fetch_status fetch_translate_pc(const STATE_ACCESS a, uint64_t &pc, uint64_t vaddr, + i_state_access_fast_addr_t &vf_offset, uint64_t &pma_index) { // Try to perform the address translation via TLB first - if (unlikely(!(a.template translate_vaddr_via_tlb(vaddr, phptr)))) { - INC_COUNTER(a.get_statistics(), tlb_cmiss); + const uint64_t slot_index = tlb_slot_index(vaddr); + const uint64_t slot_vaddr_page = a.template read_tlb_vaddr_page(slot_index); + if (unlikely(!tlb_is_hit(slot_vaddr_page, vaddr))) { + DUMP_STATS_INCR(a, "tlb.cmiss"); // Outline the slow path into a function call to minimize host CPU code cache pressure - return fetch_translate_pc_slow(a, pc, vaddr, phptr); + return fetch_translate_pc_slow(a, pc, vaddr, vf_offset, pma_index); } - INC_COUNTER(a.get_statistics(), tlb_chit); + vf_offset = a.template read_tlb_vf_offset(slot_index); + pma_index = a.template read_tlb_pma_index(slot_index); + DUMP_STATS_INCR(a, "tlb.chit"); return fetch_status::success; } @@ -5264,72 +5441,72 @@ static FORCE_INLINE fetch_status fetch_translate_pc(STATE_ACCESS a, uint64_t &pc /// \param a Machine state accessor object. /// \param pc Virtual address for the current instruction being executed. /// \param insn Receives the instruction. -/// \param fetch_vaddr_page Fetch virtual address translation page cache. -/// \param fetch_vh_offset Fetch virtual address host pointer offset cache. +/// \param last_vaddr_page Receives and updates vaddr_page for cache. +/// \param last_vf_offset Receives and updates vf_offset for cache. +/// \param last_pma_index Receives and updates pma_index for cache. /// \return Returns fetch_status::success if load succeeded, fetch_status::exception if it caused an exception. // In that case, raise the exception. template -static FORCE_INLINE fetch_status fetch_insn(STATE_ACCESS a, uint64_t &pc, uint32_t &insn, uint64_t &fetch_vaddr_page, - uint64_t &fetch_vh_offset) { - // Efficiently checks if current pc is in the same page as last pc fetch - // and it's not crossing a page boundary. - if (likely((pc ^ fetch_vaddr_page) < (PMA_PAGE_SIZE - 2))) { - // Fetch pc is in the same page as the last pc fetch and it's not crossing a page boundary, - // we can just reuse last fetch translation, skipping TLB or slow address translation altogether. - const unsigned char *hptr = cast_addr_to_ptr(pc + fetch_vh_offset); - - // Here we are sure that reading 4 bytes won't cross a page boundary. - // However pc may not be 4 byte aligned, at best it can only be 2-byte aligned, - // therefore we must perform a misaligned 4 byte read on a 2 byte aligned pointer. - // In case pc holds a compressed instruction, insn will store 2 additional bytes, - // but this is fine because later the instruction decoder will discard them. - insn = aliased_unaligned_read(hptr); - return fetch_status::success; - } - // Fetch pc is either not the same as last cache or crossing a page boundary. - - // Perform address translation - unsigned char *hptr = nullptr; - if (unlikely(fetch_translate_pc(a, pc, pc, &hptr) == fetch_status::exception)) { - return fetch_status::exception; +static FORCE_INLINE fetch_status fetch_insn(const STATE_ACCESS a, uint64_t &pc, uint32_t &insn, + uint64_t &last_vaddr_page, i_state_access_fast_addr_t &last_vf_offset, uint64_t &last_pma_index) { + [[maybe_unused]] auto note = a.make_scoped_note("fetch_insn"); + i_state_access_fast_addr_t faddr{0}; + const uint64_t pc_vaddr_page = tlb_addr_page(pc); + // If pc is in the same page as the last pc fetch, + // we can just reuse last fetch translation, skipping TLB or slow address translation altogether. + if (likely(pc_vaddr_page == last_vaddr_page)) { + faddr = pc + last_vf_offset; + } else { + // Not in the same page as last the fetch, we need to perform address translation + i_state_access_fast_addr_t pc_vf_offset{}; + uint64_t pc_pma_index{}; + if (unlikely(fetch_translate_pc(a, pc, pc, pc_vf_offset, pc_pma_index) == fetch_status::exception)) { + return fetch_status::exception; + } + // Update fetch address translation cache + last_vaddr_page = pc_vaddr_page; + last_vf_offset = pc_vf_offset; + last_pma_index = pc_pma_index; + faddr = pc + pc_vf_offset; } - // Update fetch address translation cache - fetch_vaddr_page = pc & ~PAGE_OFFSET_MASK; - fetch_vh_offset = cast_ptr_to_addr(hptr) - pc; - // The following code assumes pc is always 2-byte aligned, this is guaranteed by RISC-V spec. // If pc is pointing to the very last 2 bytes of a page, it's crossing a page boundary. if (unlikely(((~pc & PAGE_OFFSET_MASK) >> 1) == 0)) { // Here we are crossing page boundary, this is unlikely (1 in 2048 possible cases) - insn = aliased_aligned_read(hptr); + uint16_t insn16 = 0; + a.template read_memory_word(faddr, last_pma_index, &insn16); + insn = insn16; // If not a compressed instruction, we must read 2 additional bytes from the next page. if (unlikely(insn_is_uncompressed(insn))) { // We have to perform a new address translation to read the next 2 bytes since we changed pages. - const uint64_t vaddr = pc + 2; - if (unlikely(fetch_translate_pc(a, pc, vaddr, &hptr) == fetch_status::exception)) { + const uint64_t pc2 = pc + 2; + i_state_access_fast_addr_t pc2_vf_offset{}; + uint64_t pc2_pma_index{}; + if (unlikely(fetch_translate_pc(a, pc, pc2, pc2_vf_offset, pc2_pma_index) == fetch_status::exception)) { return fetch_status::exception; } - // Update fetch translation cache - fetch_vaddr_page = vaddr; - fetch_vh_offset = cast_ptr_to_addr(hptr) - vaddr; - // Produce the final 4-byte instruction - insn |= aliased_aligned_read(hptr) << 16; + last_vaddr_page = tlb_addr_page(pc2); + last_vf_offset = pc2_vf_offset; + last_pma_index = pc2_pma_index; + faddr = pc2 + last_vf_offset; + a.template read_memory_word(faddr, last_pma_index, &insn16); + insn |= insn16 << 16; } return fetch_status::success; } // Here we are sure that reading 4 bytes won't cross a page boundary. - // However pc may not be 4 byte aligned, at best it can only be 2-byte aligned, - // therefore we must perform a misaligned 4 byte read on a 2 byte aligned pointer. + // However pc may not be 4-byte aligned, at worst it could be only 2-byte aligned, + // therefore we must perform a misaligned 4-byte read on a 2-byte aligned pointer. // In case pc holds a compressed instruction, insn will store 2 additional bytes, // but this is fine because later the instruction decoder will discard them. - insn = aliased_unaligned_read(hptr); + a.template read_memory_word(faddr, last_pma_index, &insn); return fetch_status::success; } /// \brief Checks that false brk is consistent with rest of state template -static void assert_no_brk([[maybe_unused]] STATE_ACCESS a) { +static void assert_no_brk([[maybe_unused]] const STATE_ACCESS a) { assert(get_pending_irq_mask(a) == 0); // LCOV_EXCL_LINE assert(a.read_iflags_X() == 0); // LCOV_EXCL_LINE assert(a.read_iflags_Y() == 0); // LCOV_EXCL_LINE @@ -5338,7 +5515,7 @@ static void assert_no_brk([[maybe_unused]] STATE_ACCESS a) { /// \brief Interpreter hot loop template -static NO_INLINE execute_status interpret_loop(STATE_ACCESS a, uint64_t mcycle_end, uint64_t mcycle) { +static NO_INLINE execute_status interpret_loop(const STATE_ACCESS a, uint64_t mcycle_end, uint64_t mcycle) { // The interpret loop is constantly reading and modifying the pc and mcycle variables, // because of this care is taken to make them stack variables that are propagated across inline functions, // helping the C++ compiler optimize them into registers instead of stack variables when compiling, @@ -5352,13 +5529,14 @@ static NO_INLINE execute_status interpret_loop(STATE_ACCESS a, uint64_t mcycle_e uint64_t pc = a.read_pc(); // Initialize fetch address translation cache invalidated - uint64_t fetch_vaddr_page = ~pc; - uint64_t fetch_vh_offset = 0; + uint64_t fetch_vaddr_page = TLB_INVALID_PAGE; + uint64_t fetch_pma_index = TLB_INVALID_PMA_INDEX; + i_state_access_fast_addr_t fetch_vf_offset{}; // The outer loop continues until there is an interruption that should be handled // externally, or mcycle reaches mcycle_end while (mcycle < mcycle_end) { - INC_COUNTER(a.get_statistics(), outer_loop); + DUMP_STATS_INCR(a, "outer_loop"); if (rtc_is_tick(mcycle)) { // Set interrupt flag for RTC @@ -5367,8 +5545,10 @@ static NO_INLINE execute_status interpret_loop(STATE_ACCESS a, uint64_t mcycle_e // Polling external interrupts only in WFI instructions is not enough // because Linux won't execute WFI instructions while under heavy load, // yet external interrupts still need to be triggered. - // Therefore we poll for external interrupt once a while in the interpreter loop. - a.poll_external_interrupts(mcycle, mcycle); + // Therefore we poll for external interrupt once in a while in the interpreter loop. + if constexpr (is_an_i_interactive_state_access_v) { + a.poll_external_interrupts(mcycle, mcycle); + } } // Raise the highest priority pending interrupt, if any @@ -5385,12 +5565,13 @@ static NO_INLINE execute_status interpret_loop(STATE_ACCESS a, uint64_t mcycle_e // The inner loop continues until there is an interrupt condition // or mcycle reaches mcycle_tick_end while (mcycle < mcycle_tick_end) { - INC_COUNTER(a.get_statistics(), inner_loop); + DUMP_STATS_INCR(a, "inner_loop"); uint32_t insn = 0; // Try to fetch the next instruction - if (likely(fetch_insn(a, pc, insn, fetch_vaddr_page, fetch_vh_offset) == fetch_status::success)) { + if (likely(fetch_insn(a, pc, insn, fetch_vaddr_page, fetch_vf_offset, fetch_pma_index) == + fetch_status::success)) { // clang-format off // NOLINTBEGIN execute_status status; // explicit uninitialized as an optimization @@ -5848,6 +6029,12 @@ static NO_INLINE execute_status interpret_loop(STATE_ACCESS a, uint64_t mcycle_e // NOLINTEND // clang-format on +#ifdef DUMP_REGS + // Commit machine state + a.write_pc(pc); + a.write_mcycle(mcycle + 1); +#endif + // When execute status is above success, we have to deal with special loop conditions, // this is very unlikely to happen most of the time if (unlikely(status > execute_status::success)) { @@ -5856,7 +6043,7 @@ static NO_INLINE execute_status interpret_loop(STATE_ACCESS a, uint64_t mcycle_e // due to MRET/SRET instructions (execute_status::success_and_serve_interrupts) // As a simplification (and optimization), the next line will also invalidate in more cases, // but this it's fine. - fetch_vaddr_page = ~pc; + fetch_vaddr_page = TLB_INVALID_PAGE; // All status above execute_status::success_and_serve_interrupts will require breaking the loop if (unlikely(status >= execute_status::success_and_serve_interrupts)) { // Increment the cycle counter mcycle @@ -5892,19 +6079,20 @@ static NO_INLINE execute_status interpret_loop(STATE_ACCESS a, uint64_t mcycle_e } template -interpreter_break_reason interpret(STATE_ACCESS a, uint64_t mcycle_end) { +interpreter_break_reason interpret(const STATE_ACCESS a, uint64_t mcycle_end) { static_assert(__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__, "code assumes little-endian byte ordering"); - static_assert(is_an_i_state_access::value, "not an i_state_access"); + static_assert(is_an_i_state_access_v, "not an i_state_access"); + static_assert(is_an_i_accept_scoped_note_v, "not an i_accept_scoped_notes"); const uint64_t mcycle = a.read_mcycle(); // If the cpu is halted, we are done - if (a.read_iflags_H()) { + if (a.read_iflags_H() != 0) { return interpreter_break_reason::halted; } // If the cpu has yielded manually, we are done - if (a.read_iflags_Y()) { + if (a.read_iflags_Y() != 0) { return interpreter_break_reason::yielded_manually; } @@ -5938,8 +6126,8 @@ interpreter_break_reason interpret(STATE_ACCESS a, uint64_t mcycle_end) { } #ifdef MICROARCHITECTURE -// Explicit instantiation for uarch_machine_state_access -template interpreter_break_reason interpret(uarch_machine_state_access a, uint64_t mcycle_end); +// Explicit instantiation for machine_uarch_bridge_state_access +template interpreter_break_reason interpret(machine_uarch_bridge_state_access a, uint64_t mcycle_end); #else // Explicit instantiation for state_access template interpreter_break_reason interpret(state_access a, uint64_t mcycle_end); @@ -5947,6 +6135,8 @@ template interpreter_break_reason interpret(state_access a, uint64_t mcycle_end) template interpreter_break_reason interpret(record_step_state_access a, uint64_t mcycle_end); // Explicit instantiation for replay_step_state_access template interpreter_break_reason interpret(replay_step_state_access a, uint64_t mcycle_end); +// Explicit instantiation for collect_mcycle_hashes_state_access +template interpreter_break_reason interpret(collect_mcycle_hashes_state_access a, uint64_t mcycle_end); #endif // MICROARCHITECTURE } // namespace cartesi diff --git a/src/is-pristine.cpp b/src/is-pristine.cpp new file mode 100644 index 000000000..e41487615 --- /dev/null +++ b/src/is-pristine.cpp @@ -0,0 +1,66 @@ +// Copyright Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: LGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify it under +// the terms of the GNU Lesser General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) any +// later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +// PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License along +// with this program (see COPYING). If not, see . +// + +#include "is-pristine.h" + +#include +#include + +#include "address-range-constants.h" +#include "compiler-defines.h" + +namespace cartesi { + +template +FORCE_INLINE static bool is_pristine_impl(std::span data) noexcept { + unsigned char bits = 0; + // GCC and Clang are smart enough to use SIMD instructions with large words on this loop, + // however it may not unroll the loops, so we unroll it manually. + UNROLL_LOOP(64) + for (const unsigned char b : data) { + bits |= b; + } + return bits == 0; +} + +// Generic implementations + +MULTIVERSION_GENERIC bool is_pristine(std::span data) noexcept { + return is_pristine_impl(data); +} +MULTIVERSION_GENERIC bool is_pristine(std::span data) noexcept { + return is_pristine_impl(data); +} + +// x86_64 implementations + +#ifdef USE_MULTIVERSINING_AMD64 +MULTIVERSION_AMD64_AVX2 bool is_pristine(std::span data) noexcept { + return is_pristine_impl(data); +} +MULTIVERSION_AMD64_AVX2 bool is_pristine(std::span data) noexcept { + return is_pristine_impl(data); +} + +MULTIVERSION_AMD64_AVX512 bool is_pristine(std::span data) noexcept { + return is_pristine_impl(data); +} +MULTIVERSION_AMD64_AVX512 bool is_pristine(std::span data) noexcept { + return is_pristine_impl(data); +} +#endif + +} // namespace cartesi diff --git a/src/is-pristine.h b/src/is-pristine.h index 458d297fc..293c6d863 100644 --- a/src/is-pristine.h +++ b/src/is-pristine.h @@ -18,26 +18,37 @@ #define IS_PRISTINE_H #include "compiler-defines.h" -#include -#include + +#include +#include + +#include "address-range-constants.h" +#include "concepts.h" namespace cartesi { +/// \brief This is an optimized function for checking if data is pristine. +/// \param data Memory data. +/// \returns True if all values are 0, false otherwise. +/// \details It's to be used in situations where length is equal or less than a page size. +MULTIVERSION_GENERIC bool is_pristine(std::span data) noexcept; + /// \brief This is an optimized function for checking if memory page is pristine. -/// \param data Memory pointer -/// \param length Memory length -/// \details It's instead to be used in situations where length is equal or less than a page size. -// NOLINTNEXTLINE(clang-diagnostic-unknown-attributes) -static inline bool FORCE_OPTIMIZE_O3 is_pristine(const unsigned char *data, size_t length) { - // This tight for loop has no branches, and is optimized to SIMD instructions in x86_64, - // making it very fast to check if a given page is pristine. - unsigned char bits = 0; - for (size_t i = 0; i < length; ++i) { - bits |= data[i]; - } - return bits == 0; +MULTIVERSION_GENERIC bool is_pristine(std::span data) noexcept; + +template +bool is_pristine(R &&r) noexcept { // NOLINT(cppcoreguidelines-missing-std-forward) + return is_pristine(std::span{std::ranges::data(r), std::ranges::size(r)}); } +#ifdef USE_MULTIVERSINING_AMD64 +MULTIVERSION_AMD64_AVX2 bool is_pristine(std::span data) noexcept; +MULTIVERSION_AMD64_AVX2 bool is_pristine(std::span data) noexcept; + +MULTIVERSION_AMD64_AVX512 bool is_pristine(std::span data) noexcept; +MULTIVERSION_AMD64_AVX512 bool is_pristine(std::span data) noexcept; +#endif + } // namespace cartesi -#endif \ No newline at end of file +#endif diff --git a/src/json-util.cpp b/src/json-util.cpp index a80e85e07..8a4c212ea 100644 --- a/src/json-util.cpp +++ b/src/json-util.cpp @@ -14,12 +14,15 @@ // with this program (see COPYING). If not, see . // +#include "json-util.h" + #include #include #include #include #include #include +#include #include #include #include @@ -28,20 +31,31 @@ #include #include +#include + #include "access-log.h" +#include "address-range-description.h" +#include "back-merkle-tree.h" #include "base64.h" #include "bracket-note.h" +#include "hash-tree-constants.h" +#include "hash-tree-proof.h" +#include "hash-tree-stats.h" #include "interpret.h" -#include "json-util.h" -#include "json.hpp" +#include "jsonrpc-fork-result.h" #include "machine-config.h" -#include "machine-memory-range-descr.h" -#include "machine-merkle-tree.h" +#include "machine-hash.h" +#include "machine-reg.h" #include "machine-runtime-config.h" -#include "machine.h" +#include "mcycle-root-hashes.h" +#include "page-hash-tree-cache-stats.h" +#include "ranges.h" #include "semantic-version.h" -#include "uarch-config.h" +#include "shadow-registers.h" +#include "shadow-uarch-state.h" +#include "uarch-cycle-root-hashes.h" #include "uarch-interpret.h" +#include "variant-hasher.h" namespace cartesi { @@ -53,25 +67,11 @@ std::string to_string(const char *s) { return s; } -std::string encode_base64(const unsigned char *data, uint64_t length) { - // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) - const std::string input(reinterpret_cast(data), length); - return encode_base64(input); -} - -std::string encode_base64(const machine_merkle_tree::hash_type &hash) { - return encode_base64(hash.data(), hash.size()); -} - -std::string encode_base64(const access_data &data) { - return encode_base64(data.data(), data.size()); -} - /// \brief Converts between a register name and a register index /// \param name Register name /// \returns The register index static auto reg_from_name(const std::string &name) { - using reg = machine::reg; + using reg = machine_reg; const static std::unordered_map g_reg_name = { {"x0", reg::x0}, {"x1", reg::x1}, @@ -230,8 +230,33 @@ static auto reg_from_name(const std::string &name) { return got->second; } -static auto reg_to_name(machine::reg r) { - using reg = machine::reg; +static auto sharing_mode_to_name(sharing_mode sharing) { + switch (sharing) { + case sharing_mode::none: + return "none"; + case sharing_mode::config: + return "config"; + case sharing_mode::all: + return "all"; + default: + throw std::domain_error{"invalid sharing mode"}; + break; + } + return ""; +} + +static sharing_mode sharing_mode_from_name(const std::string &name) { + const static std::unordered_map g_sharing_name = {{"none", sharing_mode::none}, + {"config", sharing_mode::config}, {"all", sharing_mode::all}}; + auto got = g_sharing_name.find(name); + if (got == g_sharing_name.end()) { + throw std::domain_error{"invalid sharing mode"}; + } + return got->second; +} + +static auto reg_to_name(machine_reg r) { + using reg = machine_reg; switch (r) { case reg::x0: return "x0"; @@ -538,6 +563,38 @@ static auto reg_to_name(machine::reg r) { return ""; } +static std::string uarch_interpreter_break_reason_to_name(uarch_interpreter_break_reason reason) { + using R = uarch_interpreter_break_reason; + switch (reason) { + case R::uarch_halted: + return "uarch_halted"; + case R::reached_target_cycle: + return "reached_target_cycle"; + case R::cycle_overflow: + return "cycle_overflow"; + } + throw std::domain_error{"invalid uarch interpreter break reason"}; +} + +static std::string interpreter_break_reason_to_name(interpreter_break_reason reason) { + using R = interpreter_break_reason; + switch (reason) { + case R::failed: + return "failed"; + case R::halted: + return "halted"; + case R::yielded_manually: + return "yielded_manually"; + case R::yielded_automatically: + return "yielded_automatically"; + case R::yielded_softly: + return "yielded_softly"; + case R::reached_target_mcycle: + return "reached_target_mcycle"; + } + throw std::domain_error{"invalid interpreter break reason"}; +} + static interpreter_break_reason interpreter_break_reason_from_name(const std::string &name) { using ibr = interpreter_break_reason; const static std::unordered_map g_ibr_name = {{"failed", ibr::failed}, {"halted", ibr::halted}, @@ -552,16 +609,37 @@ static interpreter_break_reason interpreter_break_reason_from_name(const std::st static uarch_interpreter_break_reason uarch_interpreter_break_reason_from_name(const std::string &name) { using uibr = uarch_interpreter_break_reason; - if (name == "reached_target_cycle") { - return uibr::reached_target_cycle; + const static std::unordered_map g_uibr_name = { + {"reached_target_cycle", uibr::reached_target_cycle}, {"uarch_halted", uibr::uarch_halted}, + {"cycle_overflow", uibr::cycle_overflow}}; + auto got = g_uibr_name.find(name); + if (got == g_uibr_name.end()) { + throw std::domain_error{"invalid uarch interpreter break reason"}; } - if (name == "uarch_halted") { - return uibr::uarch_halted; + return got->second; +} + +static hash_function_type hash_function_from_name(const std::string &name) { + const static std::unordered_map g_hash_function_name = { + {"sha256", hash_function_type::sha256}, {"keccak256", hash_function_type::keccak256}}; + auto got = g_hash_function_name.find(name); + if (got == g_hash_function_name.end()) { + throw std::domain_error{"invalid hash function type"}; } - throw std::domain_error{"invalid uarch interpreter break reason"}; + return got->second; } -static std::string access_type_name(access_type at) { +static std::string hash_function_to_name(hash_function_type hf) { + switch (hf) { + case hash_function_type::keccak256: + return "keccak256"; + case hash_function_type::sha256: + return "sha256"; + } + throw std::domain_error{"invalid hash function type"}; +} + +static std::string access_type_to_name(access_type at) { switch (at) { case access_type::read: return "read"; @@ -571,7 +649,7 @@ static std::string access_type_name(access_type at) { throw std::domain_error{"invalid access type"}; } -static std::string bracket_type_name(bracket_type bt) { +static std::string bracket_type_to_name(bracket_type bt) { switch (bt) { case bracket_type::begin: return "begin"; @@ -581,14 +659,24 @@ static std::string bracket_type_name(bracket_type bt) { throw std::domain_error{"invalid bracket type"}; } +static bracket_type bracket_type_from_name(const std::string &name) { + if (name == "begin") { + return bracket_type::begin; + } + if (name == "end") { + return bracket_type::end; + } + throw std::domain_error{"invalid bracket type"}; +} + template void ju_get_opt_field(const nlohmann::json &j, const K &key, std::string &value, const std::string &path) { - if (!contains(j, key)) { + if (!contains(j, key, path)) { return; } const auto &jk = j[key]; if (!jk.is_string()) { - throw std::invalid_argument("field \""s + path + to_string(key) + "\" not a string"); + throw std::invalid_argument("\""s + path + to_string(key) + "\" not a string"); } value = jk.template get(); } @@ -600,12 +688,12 @@ template void ju_get_opt_field(const nlohmann::json &j, const std:: template void ju_get_opt_field(const nlohmann::json &j, const K &key, bool &value, const std::string &path) { - if (!contains(j, key)) { + if (!contains(j, key, path)) { return; } const auto &jk = j[key]; if (!jk.is_boolean()) { - throw std::invalid_argument("field \""s + path + to_string(key) + "\" not a Boolean"); + throw std::invalid_argument("\""s + path + to_string(key) + "\" not a Boolean"); } value = jk.template get(); } @@ -618,18 +706,18 @@ template void ju_get_opt_field(const nlohmann::json &j, const std:: template void ju_get_opt_field(const nlohmann::json &j, const K &key, uint64_t &value, const std::string &path) { - if (!contains(j, key)) { + if (!contains(j, key, path)) { return; } const auto &jk = j[key]; if (!jk.is_number_integer() && !jk.is_number_unsigned() && !jk.is_number_float()) { - throw std::invalid_argument("field \""s + path + to_string(key) + "\" not an unsigned integer"); + throw std::invalid_argument("\""s + path + to_string(key) + "\" not an unsigned integer"); } if (jk.is_number_float()) { auto f = jk.template get(); if (f < 0 || std::fmod(f, static_cast(1.0)) != 0 || f > static_cast(UINT64_MAX)) { - throw std::invalid_argument("field \""s + path + to_string(key) + "\" not an unsigned integer"); + throw std::invalid_argument("\""s + path + to_string(key) + "\" not an unsigned integer"); } value = static_cast(f); return; @@ -651,26 +739,26 @@ template void ju_get_opt_field(const nlohmann::json &j, const std:: template void ju_get_opt_field(const nlohmann::json &j, const K &key, uint32_t &value, const std::string &path) { - if (!contains(j, key)) { + if (!contains(j, key, path)) { return; } uint64_t value64 = 0; ju_get_field(j, key, value64, path); if (value64 > UINT32_MAX) { - throw std::invalid_argument("field \""s + path + to_string(key) + "\" out of range"); + throw std::invalid_argument("\""s + path + to_string(key) + "\" out of range"); } value = static_cast(value64); } template void ju_get_opt_field(const nlohmann::json &j, const K &key, uint16_t &value, const std::string &path) { - if (!contains(j, key)) { + if (!contains(j, key, path)) { return; } uint64_t value64 = 0; ju_get_field(j, key, value64, path); if (value64 > UINT16_MAX) { - throw std::invalid_argument("field \""s + path + to_string(key) + "\" out of range"); + throw std::invalid_argument("\""s + path + to_string(key) + "\" out of range"); } value = static_cast(value64); } @@ -689,12 +777,12 @@ template void ju_get_opt_field(const nlohmann::json &j, const std:: template void ju_get_opt_field(const nlohmann::json &j, const K &key, semantic_version &value, const std::string &path) { - if (!contains(j, key)) { + if (!contains(j, key, path)) { return; } const auto &jk = j[key]; if (!jk.is_object()) { - throw std::invalid_argument("field \""s + path + to_string(key) + "\" not a semantic version"); + throw std::invalid_argument("\""s + path + to_string(key) + "\" not a semantic version"); } const auto new_path = path + to_string(key) + "/"; ju_get_field(jk, "major"s, value.major, new_path); @@ -711,31 +799,49 @@ template void ju_get_opt_field(const nlohmann::json &j, const std:: const std::string &path); template -void ju_get_opt_field(const nlohmann::json &j, const K &key, machine::reg &value, const std::string &path) { - if (!contains(j, key)) { +void ju_get_opt_field(const nlohmann::json &j, const K &key, machine_reg &value, const std::string &path) { + if (!contains(j, key, path)) { return; } const auto &jk = j[key]; if (!jk.is_string()) { - throw std::invalid_argument("field \""s + path + to_string(key) + "\" not a string"); + throw std::invalid_argument("\""s + path + to_string(key) + "\" not a string"); } value = reg_from_name(jk.template get()); } -template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, machine::reg &value, +template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, machine_reg &value, + const std::string &path); + +template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, machine_reg &value, + const std::string &path); + +template +void ju_get_opt_field(const nlohmann::json &j, const K &key, sharing_mode &value, const std::string &path) { + if (!contains(j, key, path)) { + return; + } + const auto &jk = j[key]; + if (!jk.is_string()) { + throw std::invalid_argument("\""s + path + to_string(key) + "\" not a string"); + } + value = sharing_mode_from_name(jk.template get()); +} + +template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, sharing_mode &value, const std::string &path); -template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, machine::reg &value, +template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, sharing_mode &value, const std::string &path); template void ju_get_opt_field(const nlohmann::json &j, const K &key, interpreter_break_reason &value, const std::string &path) { - if (!contains(j, key)) { + if (!contains(j, key, path)) { return; } const auto &jk = j[key]; if (!jk.is_string()) { - throw std::invalid_argument("field \""s + path + to_string(key) + "\" not a string"); + throw std::invalid_argument("\""s + path + to_string(key) + "\" not a string"); } value = interpreter_break_reason_from_name(jk.template get()); } @@ -749,12 +855,12 @@ template void ju_get_opt_field(const nlohmann::json &j, const std:: template void ju_get_opt_field(const nlohmann::json &j, const K &key, uarch_interpreter_break_reason &value, const std::string &path) { - if (!contains(j, key)) { + if (!contains(j, key, path)) { return; } const auto &jk = j[key]; if (!jk.is_string()) { - throw std::invalid_argument("field \""s + path + to_string(key) + "\" not a string"); + throw std::invalid_argument("\""s + path + to_string(key) + "\" not a string"); } value = uarch_interpreter_break_reason_from_name(jk.template get()); } @@ -768,10 +874,10 @@ template void ju_get_opt_field(const nlohmann::json &j, const std:: template void ju_get_opt_field(const nlohmann::json &j, const K &key, concurrency_runtime_config &value, const std::string &path) { - if (!contains(j, key)) { + if (!contains(j, key, path)) { return; } - ju_get_opt_field(j[key], "update_merkle_tree"s, value.update_merkle_tree, path + to_string(key) + "/"); + ju_get_opt_field(j[key], "update_hash_tree"s, value.update_hash_tree, path + to_string(key) + "/"); } template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, @@ -782,7 +888,7 @@ template void ju_get_opt_field(const nlohmann::json &j, const std:: template void ju_get_opt_field(const nlohmann::json &j, const K &key, htif_runtime_config &value, const std::string &path) { - if (!contains(j, key)) { + if (!contains(j, key, path)) { return; } ju_get_opt_field(j[key], "no_console_putchar"s, value.no_console_putchar, path + to_string(key) + "/"); @@ -796,15 +902,14 @@ template void ju_get_opt_field(const nlohmann::json &j, const std:: template void ju_get_opt_field(const nlohmann::json &j, const K &key, machine_runtime_config &value, const std::string &path) { - if (!contains(j, key)) { + if (!contains(j, key, path)) { return; } ju_get_opt_field(j[key], "concurrency"s, value.concurrency, path + to_string(key) + "/"); ju_get_opt_field(j[key], "htif"s, value.htif, path + to_string(key) + "/"); - ju_get_opt_field(j[key], "skip_root_hash_check"s, value.skip_root_hash_check, path + to_string(key) + "/"); - ju_get_opt_field(j[key], "skip_root_hash_store"s, value.skip_root_hash_store, path + to_string(key) + "/"); ju_get_opt_field(j[key], "skip_version_check"s, value.skip_version_check, path + to_string(key) + "/"); ju_get_opt_field(j[key], "soft_yield"s, value.soft_yield, path + to_string(key) + "/"); + ju_get_opt_field(j[key], "no_reserve"s, value.no_reserve, path + to_string(key) + "/"); } template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, machine_runtime_config &value, @@ -814,52 +919,51 @@ template void ju_get_opt_field(const nlohmann::json &j, const std:: machine_runtime_config &value, const std::string &path); template -void ju_get_opt_field(const nlohmann::json &j, const K &key, machine_merkle_tree::proof_type::hash_type &value, - const std::string &path) { - if (!contains(j, key)) { +void ju_get_opt_field(const nlohmann::json &j, const K &key, machine_hash &value, const std::string &path) { + if (!contains(j, key, path)) { return; } const auto &jk = j[key]; if (!jk.is_string()) { - throw std::invalid_argument("field \""s + path + to_string(key) + "\" not a string"); + throw std::invalid_argument("\""s + path + to_string(key) + "\" not a string"); } std::string bin = decode_base64(jk.template get()); if (bin.size() != value.size()) { - throw std::invalid_argument("field \""s + path + to_string(key) + "\" not a base64-encoded 256-bit hash"); + throw std::invalid_argument("\""s + path + to_string(key) + "\" not a base64-encoded 256-bit hash"); } - std::copy(bin.begin(), bin.end(), value.data()); + std::ranges::copy(bin | views::cast_to, std::ranges::data(value)); } template -void ju_get_opt_field(const nlohmann::json &j, const K &key, - std::optional &optional, const std::string &path) { +void ju_get_opt_field(const nlohmann::json &j, const K &key, std::optional &optional, + const std::string &path) { optional = {}; - if (!contains(j, key)) { + if (!contains(j, key, path)) { return; } const auto &jk = j[key]; if (!jk.is_string()) { - throw std::invalid_argument("field \""s + path + to_string(key) + "\" not a string"); + throw std::invalid_argument("\""s + path + to_string(key) + "\" not a string"); } std::string bin = decode_base64(jk.template get()); optional.emplace(); if (bin.size() != optional.value().size()) { - throw std::invalid_argument("field \""s + path + to_string(key) + "\" not a base64-encoded 256-bit hash"); + throw std::invalid_argument("\""s + path + to_string(key) + "\" not a base64-encoded 256-bit hash"); } - std::copy(bin.begin(), bin.end(), optional.value().data()); + std::ranges::copy(bin | views::cast_to, std::ranges::data(optional.value())); } -template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, - machine_merkle_tree::proof_type::hash_type &value, const std::string &path); +template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, machine_hash &value, + const std::string &path); -template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, - machine_merkle_tree::proof_type::hash_type &value, const std::string &path); +template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, machine_hash &value, + const std::string &path); template -void ju_get_opt_field(const nlohmann::json &j, const K &key, - not_default_constructible &value, const std::string &path) { +void ju_get_opt_field(const nlohmann::json &j, const K &key, not_default_constructible &value, + const std::string &path) { value = {}; - if (!contains(j, key)) { + if (!contains(j, key, path)) { return; } const auto &jk = j[key]; @@ -867,54 +971,103 @@ void ju_get_opt_field(const nlohmann::json &j, const K &key, uint64_t log2_root_size = 0; ju_get_field(jk, "log2_root_size"s, log2_root_size, new_path); if (log2_root_size > INT_MAX) { - throw std::domain_error("field \""s + new_path + "log2_root_size\" is out of bounds"); + throw std::domain_error("\""s + new_path + "log2_root_size\" is out of bounds"); } uint64_t log2_target_size = 0; ju_get_field(jk, "log2_target_size"s, log2_target_size, new_path); if (log2_root_size > INT_MAX) { - throw std::domain_error("field \""s + new_path + "log2_target_size\" is out of bounds"); + throw std::domain_error("\""s + new_path + "log2_target_size\" is out of bounds"); } value.emplace(static_cast(log2_root_size), static_cast(log2_target_size)); auto &proof = value.value(); - machine_merkle_tree::proof_type::address_type target_address = 0; + uint64_t target_address = 0; ju_get_field(jk, "target_address"s, target_address, new_path); proof.set_target_address(target_address); - machine_merkle_tree::proof_type::hash_type target_hash; + machine_hash target_hash; ju_get_field(jk, "target_hash"s, target_hash, new_path); proof.set_target_hash(target_hash); - machine_merkle_tree::proof_type::hash_type root_hash; + machine_hash root_hash; ju_get_field(jk, "root_hash"s, root_hash, new_path); proof.set_root_hash(root_hash); - if (!contains(jk, "sibling_hashes")) { - throw std::invalid_argument("missing field \""s + new_path + "sibling_hashes\""s); + if (!contains(jk, "sibling_hashes", new_path)) { + throw std::invalid_argument("missing \""s + new_path + "sibling_hashes\""s); } const auto &sh = jk["sibling_hashes"]; if (!sh.is_array()) { - throw std::invalid_argument("field \""s + new_path + "sibling_hashes\" not an array"s); + throw std::invalid_argument("\""s + new_path + "sibling_hashes\" not an array"s); } const auto sibling_hashes_base = path + "sibling_hashes/"; for (int log2_size = proof.get_log2_target_size(), i = 0; log2_size < proof.get_log2_root_size(); ++log2_size, ++i) { - machine_merkle_tree::proof_type::hash_type sibling_hash; + machine_hash sibling_hash; ju_get_field(sh, i, sibling_hash, sibling_hashes_base); proof.set_sibling_hash(sibling_hash, log2_size); } } template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, - not_default_constructible &value, const std::string &path); + not_default_constructible &value, const std::string &path); + +template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, + not_default_constructible &value, const std::string &path); + +template +void ju_get_opt_field(const nlohmann::json &j, const K &key, page_hash_tree_cache_stats &value, + const std::string &path) { + if (!contains(j, key, path)) { + return; + } + const auto &jstats = j[key]; + const auto new_path = path + to_string(key) + "/"; + ju_get_opt_field(jstats, "page_hits"s, value.page_hits, new_path); + ju_get_opt_field(jstats, "page_misses"s, value.page_misses, new_path); + ju_get_opt_field(jstats, "word_hits"s, value.word_hits, new_path); + ju_get_opt_field(jstats, "word_misses"s, value.word_misses, new_path); + ju_get_opt_field(jstats, "page_changes"s, value.page_changes, new_path); + ju_get_opt_field(jstats, "inner_page_hashes"s, value.inner_page_hashes, new_path); + ju_get_opt_field(jstats, "pristine_pages"s, value.pristine_pages, new_path); + ju_get_opt_field(jstats, "non_pristine_pages"s, value.non_pristine_pages, new_path); +} + +template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, + page_hash_tree_cache_stats &value, const std::string &path); template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, - not_default_constructible &value, const std::string &path); + page_hash_tree_cache_stats &value, const std::string &path); + +template +void ju_get_opt_field(const nlohmann::json &j, const K &key, hash_tree_stats &value, const std::string &path) { + if (!contains(j, key, path)) { + return; + } + const auto &jstats = j[key]; + const auto new_path = path + to_string(key) + "/"; + ju_get_opt_field(jstats, "phtc"s, value.phtc, new_path); + ju_get_opt_field(jstats, "sparse_node_hashes"s, value.sparse_node_hashes, new_path); + const auto &jdnh = jstats["dense_node_hashes"]; + if (!jdnh.is_array()) { + throw std::invalid_argument("\""s + new_path + "dense_node_hashes\" not an array"s); + } + const auto dense_node_hashes_base = path + "dense_node_hashes/"; + for (uint64_t log2_size = 0; log2_size < value.dense_node_hashes.size(); ++log2_size) { + ju_get_field(jdnh, log2_size, value.dense_node_hashes[log2_size], dense_node_hashes_base); + } +} + +template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, hash_tree_stats &value, + const std::string &path); + +template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, hash_tree_stats &value, + const std::string &path); template void ju_get_opt_field(const nlohmann::json &j, const K &key, access_type &value, const std::string &path) { - if (!contains(j, key)) { + if (!contains(j, key, path)) { return; } const auto &jk = j[key]; if (!jk.is_string()) { - throw std::invalid_argument("field \""s + path + to_string(key) + "\" not a string"); + throw std::invalid_argument("\""s + path + to_string(key) + "\" not a string"); } const auto &v = jk.template get(); if (v == "read") { @@ -925,7 +1078,7 @@ void ju_get_opt_field(const nlohmann::json &j, const K &key, access_type &value, value = access_type::write; return; } - throw std::invalid_argument("field \""s + path + to_string(key) + "\" not an access type"); + throw std::invalid_argument("\""s + path + to_string(key) + "\" not an access type"); } template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, access_type &value, @@ -937,12 +1090,12 @@ template void ju_get_opt_field(const nlohmann::json &j, const std:: template void ju_get_opt_field(const nlohmann::json &j, const K &key, access_data &data, const std::string &path) { data.clear(); - if (!contains(j, key)) { + if (!contains(j, key, path)) { return; } const auto &jk = j[key]; if (!jk.is_string()) { - throw std::invalid_argument("field \""s + path + to_string(key) + "\" not a string"); + throw std::invalid_argument("\""s + path + to_string(key) + "\" not a string"); } const auto &bin = decode_base64(jk.template get()); std::copy(bin.begin(), bin.end(), std::back_inserter(data)); @@ -952,12 +1105,12 @@ template void ju_get_opt_field(const nlohmann::json &j, const K &key, std::optional &optional, const std::string &path) { optional = {}; - if (!contains(j, key)) { + if (!contains(j, key, path)) { return; } const auto &jk = j[key]; if (!jk.is_string()) { - throw std::invalid_argument("field \""s + path + to_string(key) + "\" not a string"); + throw std::invalid_argument("\""s + path + to_string(key) + "\" not a string"); } const auto &bin = decode_base64(jk.template get()); optional.emplace(); @@ -972,12 +1125,12 @@ template void ju_get_opt_field(const nlohmann::json &j, const std:: template void ju_get_opt_field(const nlohmann::json &j, const K &key, access &access, const std::string &path) { - if (!contains(j, key)) { + if (!contains(j, key, path)) { return; } const auto &jk = j[key]; if (!jk.is_object()) { - throw std::invalid_argument("field \""s + path + to_string(key) + "\" not an object"); + throw std::invalid_argument("\""s + path + to_string(key) + "\" not an object"); } const auto new_path = path + to_string(key) + "/"; access_type type = access_type::read; @@ -986,20 +1139,19 @@ void ju_get_opt_field(const nlohmann::json &j, const K &key, access &access, con uint64_t log2_size = 0; ju_get_field(jk, "log2_size"s, log2_size, new_path); if (log2_size >= 64) { - throw std::domain_error("field \""s + new_path + "log2_size\" is out of bounds"); + throw std::domain_error("\""s + new_path + "log2_size\" is out of bounds"); } access.set_log2_size(static_cast(log2_size)); - // Minimum logged data size is merkle tree word size - const uint64_t data_log2_size = - std::max(log2_size, static_cast(machine_merkle_tree::get_log2_word_size())); + // Minimum logged data size is hash-tree word size + const uint64_t data_log2_size = std::max(log2_size, static_cast(HASH_TREE_LOG2_WORD_SIZE)); uint64_t address = 0; ju_get_field(jk, "address"s, address, new_path); access.set_address(address); - machine_merkle_tree::proof_type::hash_type read_hash; + machine_hash read_hash; ju_get_field(jk, "read_hash", read_hash, new_path); access.set_read_hash(read_hash); - not_default_constructible written_hash; + not_default_constructible written_hash; ju_get_opt_field(jk, "written_hash", written_hash, new_path); if (written_hash.has_value()) { access.set_written_hash(written_hash.value()); @@ -1009,7 +1161,7 @@ void ju_get_opt_field(const nlohmann::json &j, const K &key, access &access, con ju_get_opt_field(jk, "read"s, read, new_path); if (read.has_value()) { if (read.value().size() != (UINT64_C(1) << data_log2_size)) { - throw std::invalid_argument("field \""s + new_path + "written\" has wrong length"); + throw std::invalid_argument("\""s + new_path + "written\" has wrong length"); } access.set_read(std::move(read.value())); } @@ -1018,19 +1170,19 @@ void ju_get_opt_field(const nlohmann::json &j, const K &key, access &access, con ju_get_opt_field(jk, "written"s, written, new_path); if (written.has_value()) { if (written.value().size() != (UINT64_C(1) << data_log2_size)) { - throw std::invalid_argument("field \""s + new_path + "written\" has wrong length"); + throw std::invalid_argument("\""s + new_path + "written\" has wrong length"); } access.set_written(std::move(written.value())); } } - if (contains(jk, "sibling_hashes")) { + if (contains(jk, "sibling_hashes", new_path)) { access.get_sibling_hashes().emplace(); // NOLINTNEXTLINE(bugprone-unchecked-optional-access) auto &sibling_hashes = access.get_sibling_hashes().value(); ju_get_vector_like_field(jk, "sibling_hashes"s, sibling_hashes, new_path); - auto expected_depth = static_cast(machine_merkle_tree::get_log2_root_size() - data_log2_size); + auto expected_depth = static_cast(HASH_TREE_LOG2_ROOT_SIZE - data_log2_size); if (sibling_hashes.size() != expected_depth) { - throw std::invalid_argument("field \""s + new_path + "sibling_hashes\" has wrong length"); + throw std::invalid_argument("\""s + new_path + "sibling_hashes\" has wrong length"); } } } @@ -1043,23 +1195,14 @@ template void ju_get_opt_field(const nlohmann::json &j, const std:: template void ju_get_opt_field(const nlohmann::json &j, const K &key, bracket_type &value, const std::string &path) { - if (!contains(j, key)) { + if (!contains(j, key, path)) { return; } const auto &jk = j[key]; if (!jk.is_string()) { - throw std::invalid_argument("field \""s + path + to_string(key) + "\" not a string"); - } - const auto &v = jk.template get(); - if (v == "begin") { - value = bracket_type::begin; - return; + throw std::invalid_argument("\""s + path + to_string(key) + "\" not a string"); } - if (v == "end") { - value = bracket_type::end; - return; - } - throw std::invalid_argument("field \""s + path + to_string(key) + "\" not a bracket type"); + value = bracket_type_from_name(jk.template get()); } template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, bracket_type &value, @@ -1070,12 +1213,12 @@ template void ju_get_opt_field(const nlohmann::json &j, const std:: template void ju_get_opt_field(const nlohmann::json &j, const K &key, bracket_note &value, const std::string &path) { - if (!contains(j, key)) { + if (!contains(j, key, path)) { return; } const auto &jk = j[key]; if (!jk.is_object()) { - throw std::invalid_argument("field \""s + path + to_string(key) + "\" not an object"); + throw std::invalid_argument("\""s + path + to_string(key) + "\" not an object"); } const auto new_path = path + to_string(key) + "/"; ju_get_field(jk, "type"s, value.type, new_path); @@ -1093,7 +1236,7 @@ template void ju_get_opt_field(const nlohmann::json &j, const K &key, not_default_constructible &optional, const std::string &path) { optional = {}; - if (!contains(j, key)) { + if (!contains(j, key, path)) { return; } const auto &jk = j[key]; @@ -1115,7 +1258,7 @@ template void ju_get_opt_field(const nlohmann::json &j, const K &key, not_default_constructible &optional, const std::string &path) { optional = {}; - if (!contains(j, key)) { + if (!contains(j, key, path)) { return; } const auto &jk = j[key]; @@ -1129,8 +1272,7 @@ void ju_get_opt_field(const nlohmann::json &j, const K &key, not_default_constru ju_get_vector_like_field(jk, "accesses"s, accesses, new_path); for (unsigned i = 0; i < accesses.size(); ++i) { if (!accesses[i].get_sibling_hashes().has_value()) { - throw std::invalid_argument( - "field \""s + new_path + "accesses/" + to_string(i) + "\" missing sibling hashes"); + throw std::invalid_argument("\""s + new_path + "accesses/" + to_string(i) + "\" missing sibling hashes"); } } std::vector brackets; @@ -1144,8 +1286,7 @@ void ju_get_opt_field(const nlohmann::json &j, const K &key, not_default_constru ju_get_vector_like_field(jk, "brackets"s, brackets, new_path); for (unsigned i = 0; i < brackets.size(); ++i) { if (brackets[i].where > accesses.size()) { - throw std::invalid_argument( - "field \""s + new_path + "brackets/" + to_string(i) + "/where\" is out of range"); + throw std::invalid_argument("\""s + new_path + "brackets/" + to_string(i) + "/where\" is out of range"); } } } @@ -1159,8 +1300,8 @@ template void ju_get_opt_field(const nlohmann::json &j, const std:: not_default_constructible &value, const std::string &path); template -void ju_get_opt_field(const nlohmann::json &j, const K &key, processor_config &value, const std::string &path) { - if (!contains(j, key)) { +void ju_get_opt_field(const nlohmann::json &j, const K &key, registers_state &value, const std::string &path) { + if (!contains(j, key, path)) { return; } const auto &jconfig = j[key]; @@ -1259,10 +1400,28 @@ void ju_get_opt_field(const nlohmann::json &j, const K &key, processor_config &v ju_get_opt_field(jconfig, "senvcfg"s, value.senvcfg, new_path); ju_get_opt_field(jconfig, "ilrsc"s, value.ilrsc, new_path); ju_get_opt_field(jconfig, "iprv"s, value.iprv, new_path); - ju_get_opt_field(jconfig, "iflags_X"s, value.iflags_X, new_path); - ju_get_opt_field(jconfig, "iflags_Y"s, value.iflags_Y, new_path); - ju_get_opt_field(jconfig, "iflags_H"s, value.iflags_H, new_path); + ju_get_opt_field(jconfig, "iflags"s, value.iflags, new_path); ju_get_opt_field(jconfig, "iunrep"s, value.iunrep, new_path); + ju_get_opt_field(jconfig, "clint"s, value.clint, new_path); + ju_get_opt_field(jconfig, "plic"s, value.plic, new_path); + ju_get_opt_field(jconfig, "htif"s, value.htif, new_path); +} + +template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, registers_state &value, + const std::string &path); + +template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, registers_state &value, + const std::string &path); + +template +void ju_get_opt_field(const nlohmann::json &j, const K &key, processor_config &value, const std::string &path) { + if (!contains(j, key, path)) { + return; + } + const auto &jconfig = j[key]; + const auto new_path = path + to_string(key) + "/"; + ju_get_opt_field(jconfig, "registers"s, value.registers, new_path); + ju_get_opt_field(jconfig, "backing_store"s, value.backing_store, new_path); } template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, processor_config &value, @@ -1273,7 +1432,7 @@ template void ju_get_opt_field(const nlohmann::json &j, const std:: template void ju_get_opt_field(const nlohmann::json &j, const K &key, dtb_config &value, const std::string &path) { - if (!contains(j, key)) { + if (!contains(j, key, path)) { return; } const auto &jconfig = j[key]; @@ -1281,7 +1440,7 @@ void ju_get_opt_field(const nlohmann::json &j, const K &key, dtb_config &value, ju_get_opt_field(jconfig, "bootargs"s, value.bootargs, new_path); ju_get_opt_field(jconfig, "init"s, value.init, new_path); ju_get_opt_field(jconfig, "entrypoint"s, value.entrypoint, new_path); - ju_get_opt_field(jconfig, "image_filename"s, value.image_filename, new_path); + ju_get_opt_field(jconfig, "backing_store"s, value.backing_store, new_path); } template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, dtb_config &value, @@ -1292,13 +1451,13 @@ template void ju_get_opt_field(const nlohmann::json &j, const std:: template void ju_get_opt_field(const nlohmann::json &j, const K &key, ram_config &value, const std::string &path) { - if (!contains(j, key)) { + if (!contains(j, key, path)) { return; } const auto &jconfig = j[key]; const auto new_path = path + to_string(key) + "/"; ju_get_field(jconfig, "length"s, value.length, new_path); - ju_get_opt_field(jconfig, "image_filename"s, value.image_filename, new_path); + ju_get_opt_field(jconfig, "backing_store"s, value.backing_store, new_path); } template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, ram_config &value, @@ -1309,15 +1468,15 @@ template void ju_get_opt_field(const nlohmann::json &j, const std:: template void ju_get_opt_field(const nlohmann::json &j, const K &key, memory_range_config &value, const std::string &path) { - if (!contains(j, key)) { + if (!contains(j, key, path)) { return; } const auto &jconfig = j[key]; const auto new_path = path + to_string(key) + "/"; ju_get_opt_field(jconfig, "start"s, value.start, new_path); ju_get_opt_field(jconfig, "length"s, value.length, new_path); - ju_get_opt_field(jconfig, "shared"s, value.shared, new_path); - ju_get_opt_field(jconfig, "image_filename"s, value.image_filename, new_path); + ju_get_opt_field(jconfig, "read_only"s, value.read_only, new_path); + ju_get_opt_field(jconfig, "backing_store"s, value.backing_store, new_path); } template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, memory_range_config &value, @@ -1327,30 +1486,90 @@ template void ju_get_opt_field(const nlohmann::json &j, const std:: const std::string &path); template -void ju_get_opt_field(const nlohmann::json &j, const K &key, cmio_buffer_config &value, const std::string &path) { - if (!contains(j, key)) { +void ju_get_opt_field(const nlohmann::json &j, const K &key, hash_function_type &value, const std::string &path) { + if (!contains(j, key, path)) { + return; + } + const auto &jk = j[key]; + if (!jk.is_string()) { + throw std::invalid_argument("\""s + path + to_string(key) + "\" not a string"); + } + value = hash_function_from_name(jk.template get()); +} + +template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, hash_function_type &value, + const std::string &path); + +template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, hash_function_type &value, + const std::string &path); + +template +void ju_get_opt_field(const nlohmann::json &j, const K &key, hash_tree_config &value, const std::string &path) { + if (!contains(j, key, path)) { return; } const auto &jconfig = j[key]; const auto new_path = path + to_string(key) + "/"; ju_get_opt_field(jconfig, "shared"s, value.shared, new_path); - ju_get_opt_field(jconfig, "image_filename"s, value.image_filename, new_path); + ju_get_opt_field(jconfig, "create"s, value.create, new_path); + ju_get_opt_field(jconfig, "sht_filename"s, value.sht_filename, new_path); + ju_get_opt_field(jconfig, "phtc_filename"s, value.phtc_filename, new_path); + ju_get_opt_field(jconfig, "phtc_size"s, value.phtc_size, new_path); + ju_get_opt_field(jconfig, "hash_function"s, value.hash_function, new_path); } -template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, cmio_buffer_config &value, +template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, hash_tree_config &value, const std::string &path); -template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, cmio_buffer_config &value, +template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, hash_tree_config &value, const std::string &path); +template +void ju_get_opt_field(const nlohmann::json &j, const K &key, backing_store_config &value, const std::string &path) { + if (!contains(j, key, path)) { + return; + } + const auto &jconfig = j[key]; + const auto new_path = path + to_string(key) + "/"; + ju_get_opt_field(jconfig, "shared"s, value.shared, new_path); + ju_get_opt_field(jconfig, "create"s, value.create, new_path); + ju_get_opt_field(jconfig, "truncate"s, value.truncate, new_path); + ju_get_opt_field(jconfig, "data_filename"s, value.data_filename, new_path); + ju_get_opt_field(jconfig, "dht_filename"s, value.dht_filename, new_path); + ju_get_opt_field(jconfig, "dpt_filename"s, value.dpt_filename, new_path); +} + +template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, backing_store_config &value, + const std::string &path); + +template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, + backing_store_config &value, const std::string &path); + +template +void ju_get_opt_field(const nlohmann::json &j, const K &key, backing_store_config_only &value, + const std::string &path) { + if (!contains(j, key, path)) { + return; + } + const auto &jconfig = j[key]; + const auto new_path = path + to_string(key) + "/"; + ju_get_opt_field(jconfig, "backing_store"s, value.backing_store, new_path); +} + +template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, backing_store_config_only &value, + const std::string &path); + +template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, + backing_store_config_only &value, const std::string &path); + template void ju_get_opt_field(const nlohmann::json &j, const K &key, flash_drive_configs &value, const std::string &path) { - if (!contains(j, key)) { + if (!contains(j, key, path)) { return; } const auto &jflash_drive = j[key]; if (!jflash_drive.is_array()) { - throw std::invalid_argument("field \""s + path + to_string(key) + "\" not an array"s); + throw std::invalid_argument("\""s + path + to_string(key) + "\" not an array"s); } const auto new_path = path + to_string(key) + "/"; value.resize(0); @@ -1368,7 +1587,7 @@ template void ju_get_opt_field(const nlohmann::json &j, const std:: template void ju_get_opt_field(const nlohmann::json &j, const K &key, virtio_device_config &value, const std::string &path) { - if (!contains(j, key)) { + if (!contains(j, key, path)) { return; } const auto &jconfig = j[key]; @@ -1387,7 +1606,7 @@ void ju_get_opt_field(const nlohmann::json &j, const K &key, virtio_device_confi if (jconfig.contains("hostfwd")) { const auto &jhostfwds = jconfig["hostfwd"]; if (!jhostfwds.is_array()) { - throw std::invalid_argument("field \""s + new_path + "hostfwd\" not an array"s); + throw std::invalid_argument("\""s + new_path + "hostfwd\" not an array"s); } for (const auto &el : jhostfwds.items()) { const auto &jhostfwd = el.value(); @@ -1419,17 +1638,17 @@ template void ju_get_opt_field(const nlohmann::json &j, const std:: template void ju_get_opt_field(const nlohmann::json &j, const K &key, virtio_configs &value, const std::string &path) { - if (!contains(j, key)) { + if (!contains(j, key, path)) { return; } const auto &jvirtio = j[key]; if (!jvirtio.is_array()) { - throw std::invalid_argument("field \""s + path + to_string(key) + "\" not an array"s); + throw std::invalid_argument("\""s + path + to_string(key) + "\" not an array"s); } const auto new_path = path + to_string(key) + "/"; value.resize(0); for (uint64_t i = 0; i < jvirtio.size(); ++i) { - value.push_back({}); + value.emplace_back(); ju_get_opt_field(jvirtio, i, value.back(), new_path); } } @@ -1441,24 +1660,26 @@ template void ju_get_opt_field(const nlohmann::json &j, const std:: const std::string &path); template -void ju_get_opt_field(const nlohmann::json &j, const K &key, tlb_config &value, const std::string &path) { - if (!contains(j, key)) { +void ju_get_opt_field(const nlohmann::json &j, const K &key, iflags_state &value, const std::string &path) { + if (!contains(j, key, path)) { return; } const auto &jconfig = j[key]; const auto new_path = path + to_string(key) + "/"; - ju_get_opt_field(jconfig, "image_filename"s, value.image_filename, new_path); + ju_get_opt_field(jconfig, "X"s, value.X, new_path); + ju_get_opt_field(jconfig, "Y"s, value.Y, new_path); + ju_get_opt_field(jconfig, "H"s, value.H, new_path); } -template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, tlb_config &value, +template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, iflags_state &value, const std::string &path); -template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, tlb_config &value, +template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, iflags_state &value, const std::string &path); template -void ju_get_opt_field(const nlohmann::json &j, const K &key, clint_config &value, const std::string &path) { - if (!contains(j, key)) { +void ju_get_opt_field(const nlohmann::json &j, const K &key, clint_state &value, const std::string &path) { + if (!contains(j, key, path)) { return; } const auto &jconfig = j[key]; @@ -1466,15 +1687,15 @@ void ju_get_opt_field(const nlohmann::json &j, const K &key, clint_config &value ju_get_opt_field(jconfig, "mtimecmp"s, value.mtimecmp, new_path); } -template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, clint_config &value, +template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, clint_state &value, const std::string &path); -template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, clint_config &value, +template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, clint_state &value, const std::string &path); template -void ju_get_opt_field(const nlohmann::json &j, const K &key, plic_config &value, const std::string &path) { - if (!contains(j, key)) { +void ju_get_opt_field(const nlohmann::json &j, const K &key, plic_state &value, const std::string &path) { + if (!contains(j, key, path)) { return; } const auto &jconfig = j[key]; @@ -1483,35 +1704,35 @@ void ju_get_opt_field(const nlohmann::json &j, const K &key, plic_config &value, ju_get_opt_field(jconfig, "girqsrvd"s, value.girqsrvd, new_path); } -template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, plic_config &value, +template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, plic_state &value, const std::string &path); -template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, plic_config &value, +template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, plic_state &value, const std::string &path); template -void ju_get_opt_field(const nlohmann::json &j, const K &key, htif_config &value, const std::string &path) { - if (!contains(j, key)) { +void ju_get_opt_field(const nlohmann::json &j, const K &key, htif_state &value, const std::string &path) { + if (!contains(j, key, path)) { return; } const auto &jconfig = j[key]; const auto new_path = path + to_string(key) + "/"; ju_get_opt_field(jconfig, "fromhost"s, value.fromhost, new_path); ju_get_opt_field(jconfig, "tohost"s, value.tohost, new_path); - ju_get_opt_field(jconfig, "console_getchar"s, value.console_getchar, new_path); - ju_get_opt_field(jconfig, "yield_manual"s, value.yield_manual, new_path); - ju_get_opt_field(jconfig, "yield_automatic"s, value.yield_automatic, new_path); + ju_get_opt_field(jconfig, "ihalt"s, value.ihalt, new_path); + ju_get_opt_field(jconfig, "iconsole"s, value.iconsole, new_path); + ju_get_opt_field(jconfig, "iyield"s, value.iyield, new_path); } -template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, htif_config &value, +template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, htif_state &value, const std::string &path); -template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, htif_config &value, +template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, htif_state &value, const std::string &path); template void ju_get_opt_field(const nlohmann::json &j, const K &key, cmio_config &value, const std::string &path) { - if (!contains(j, key)) { + if (!contains(j, key, path)) { return; } const auto &jconfig = j[key]; @@ -1527,8 +1748,8 @@ template void ju_get_opt_field(const nlohmann::json &j, const std:: const std::string &path); template -void ju_get_opt_field(const nlohmann::json &j, const K &key, uarch_processor_config &value, const std::string &path) { - if (!contains(j, key)) { +void ju_get_opt_field(const nlohmann::json &j, const K &key, uarch_registers_state &value, const std::string &path) { + if (!contains(j, key, path)) { return; } const auto &jconfig = j[key]; @@ -1570,31 +1791,32 @@ void ju_get_opt_field(const nlohmann::json &j, const K &key, uarch_processor_con ju_get_opt_field(jconfig, "halt_flag"s, value.halt_flag, new_path); } -template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, uarch_processor_config &value, +template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, uarch_registers_state &value, const std::string &path); template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, - uarch_processor_config &value, const std::string &path); + uarch_registers_state &value, const std::string &path); template -void ju_get_opt_field(const nlohmann::json &j, const K &key, uarch_ram_config &value, const std::string &path) { - if (!contains(j, key)) { +void ju_get_opt_field(const nlohmann::json &j, const K &key, uarch_processor_config &value, const std::string &path) { + if (!contains(j, key, path)) { return; } const auto &jconfig = j[key]; const auto new_path = path + to_string(key) + "/"; - ju_get_opt_field(jconfig, "image_filename"s, value.image_filename, new_path); + ju_get_opt_field(jconfig, "registers"s, value.registers, new_path); + ju_get_opt_field(jconfig, "backing_store"s, value.backing_store, new_path); } -template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, uarch_ram_config &value, +template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, uarch_processor_config &value, const std::string &path); -template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, uarch_ram_config &value, - const std::string &path); +template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, + uarch_processor_config &value, const std::string &path); template void ju_get_opt_field(const nlohmann::json &j, const K &key, uarch_config &value, const std::string &path) { - if (!contains(j, key)) { + if (!contains(j, key, path)) { return; } const auto &juarch = j[key]; @@ -1611,7 +1833,7 @@ template void ju_get_opt_field(const nlohmann::json &j, const std:: template void ju_get_opt_field(const nlohmann::json &j, const K &key, machine_config &value, const std::string &path) { - if (!contains(j, key)) { + if (!contains(j, key, path)) { return; } const auto &config = j[key]; @@ -1621,12 +1843,10 @@ void ju_get_opt_field(const nlohmann::json &j, const K &key, machine_config &val ju_get_opt_field(config, "dtb"s, value.dtb, new_path); ju_get_opt_field(config, "flash_drive"s, value.flash_drive, new_path); ju_get_opt_field(config, "virtio"s, value.virtio, new_path); - ju_get_opt_field(config, "tlb"s, value.tlb, new_path); - ju_get_opt_field(config, "clint"s, value.clint, new_path); - ju_get_opt_field(config, "plic"s, value.plic, new_path); - ju_get_opt_field(config, "htif"s, value.htif, new_path); - ju_get_opt_field(config, "uarch"s, value.uarch, new_path); ju_get_opt_field(config, "cmio"s, value.cmio, new_path); + ju_get_opt_field(config, "pmas"s, value.pmas, new_path); + ju_get_opt_field(config, "uarch"s, value.uarch, new_path); + ju_get_opt_field(config, "hash_tree"s, value.hash_tree, new_path); } template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, machine_config &value, @@ -1636,9 +1856,9 @@ template void ju_get_opt_field(const nlohmann::json &j, const std:: const std::string &path); template -void ju_get_opt_field(const nlohmann::json &j, const K &key, machine_memory_range_descr &value, +void ju_get_opt_field(const nlohmann::json &j, const K &key, address_range_description &value, const std::string &path) { - if (!contains(j, key)) { + if (!contains(j, key, path)) { return; } const auto &jconfig = j[key]; @@ -1648,27 +1868,27 @@ void ju_get_opt_field(const nlohmann::json &j, const K &key, machine_memory_rang ju_get_opt_field(jconfig, "description"s, value.description, new_path); } -template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, - machine_memory_range_descr &value, const std::string &path); +template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, address_range_description &value, + const std::string &path); template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, - machine_memory_range_descr &value, const std::string &path); + address_range_description &value, const std::string &path); template -void ju_get_opt_field(const nlohmann::json &j, const K &key, machine_memory_range_descrs &value, +void ju_get_opt_field(const nlohmann::json &j, const K &key, address_range_descriptions &value, const std::string &path) { ju_get_opt_vector_like_field(j, key, value, path); } template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, - machine_memory_range_descrs &value, const std::string &path); + address_range_descriptions &value, const std::string &path); template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, - machine_memory_range_descrs &value, const std::string &path); + address_range_descriptions &value, const std::string &path); template void ju_get_opt_field(const nlohmann::json &j, const K &key, fork_result &value, const std::string &path) { - if (!contains(j, key)) { + if (!contains(j, key, path)) { return; } const auto &jconfig = j[key]; @@ -1683,21 +1903,91 @@ template void ju_get_opt_field(const nlohmann::json &j, const uint64_t template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, fork_result &value, const std::string &path); -void to_json(nlohmann::json &j, const machine::reg ®) { +template +void ju_get_opt_field(const nlohmann::json &j, const K &key, mcycle_root_hashes &value, const std::string &path) { + if (!contains(j, key, path)) { + return; + } + const auto &jconfig = j[key]; + const auto new_path = path + to_string(key) + "/"; + ju_get_vector_like_field(jconfig, "hashes"s, value.hashes, new_path); + ju_get_field(jconfig, "mcycle_phase"s, value.mcycle_phase, new_path); + ju_get_field(jconfig, "break_reason"s, value.break_reason, new_path); + ju_get_opt_field(jconfig, "back_tree"s, value.back_tree, new_path); +} + +template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, mcycle_root_hashes &value, + const std::string &path); + +template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, mcycle_root_hashes &value, + const std::string &path); + +template +void ju_get_opt_field(const nlohmann::json &j, const K &key, uarch_cycle_root_hashes &value, const std::string &path) { + if (!contains(j, key, path)) { + return; + } + const auto &jconfig = j[key]; + const auto new_path = path + to_string(key) + "/"; + ju_get_vector_like_field(jconfig, "hashes"s, value.hashes, new_path); + ju_get_vector_like_field(jconfig, "reset_indices"s, value.reset_indices, new_path); + ju_get_field(jconfig, "break_reason"s, value.break_reason, new_path); +} + +template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, uarch_cycle_root_hashes &value, + const std::string &path); + +template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, + uarch_cycle_root_hashes &value, const std::string &path); + +template +void ju_get_opt_field(const nlohmann::json &j, const K &key, std::optional &value, + const std::string &path) { + value = {}; + if (!contains(j, key, path)) { + return; + } + const auto &jconfig = j[key]; + if (jconfig.is_null()) { + return; + } + const auto new_path = path + to_string(key) + "/"; + uint64_t log2_max_leaves{}; + hash_function_type hash_function{}; + uint64_t leaf_count{}; + machine_hashes context; + ju_get_field(jconfig, "log2_max_leaves"s, log2_max_leaves, new_path); + ju_get_field(jconfig, "hash_function"s, hash_function, new_path); + ju_get_field(jconfig, "leaf_count"s, leaf_count, new_path); + ju_get_vector_like_field(jconfig, "context"s, context, new_path); + value = back_merkle_tree{static_cast(log2_max_leaves), hash_function, leaf_count, context}; +} + +template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, + std::optional &value, const std::string &path); + +template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, + std::optional &value, const std::string &path); + +void to_json(nlohmann::json &j, const machine_reg ®) { j = reg_to_name(reg); } -void to_json(nlohmann::json &j, const machine_merkle_tree::hash_type &h) { - j = encode_base64(h); +void to_json(nlohmann::json &j, const sharing_mode &sharing) { + j = sharing_mode_to_name(sharing); } -void to_json(nlohmann::json &j, const std::vector &hs) { +void to_json(nlohmann::json &j, const base64_machine_hash &h) { + j = encode_base64(h.get()); +} + +void to_json(nlohmann::json &j, const base64_machine_hashes &hs) { j = nlohmann::json::array(); - std::transform(hs.cbegin(), hs.cend(), std::back_inserter(j), - [](const machine_merkle_tree::hash_type &h) -> nlohmann::json { return h; }); + std::ranges::transform(hs.get(), std::back_inserter(j), + [](const machine_hash &h) -> nlohmann::json { return base64_machine_hash(h); }); } -void to_json(nlohmann::json &j, const machine_merkle_tree::proof_type &p) { +void to_json(nlohmann::json &j, const hash_tree_proof &p) { nlohmann::json s = nlohmann::json::array(); for (int log2_size = p.get_log2_target_size(); log2_size < p.get_log2_root_size(); ++log2_size) { s.push_back(encode_base64(p.get_sibling_hash(log2_size))); @@ -1709,7 +1999,7 @@ void to_json(nlohmann::json &j, const machine_merkle_tree::proof_type &p) { void to_json(nlohmann::json &j, const access &a) { j = nlohmann::json{ - {"type", access_type_name(a.get_type())}, + {"type", access_type_to_name(a.get_type())}, {"address", a.get_address()}, {"log2_size", a.get_log2_size()}, }; @@ -1733,9 +2023,9 @@ void to_json(nlohmann::json &j, const access &a) { if (a.get_sibling_hashes().has_value()) { // NOLINTNEXTLINE(bugprone-unchecked-optional-access) const auto &sibling_hashes = a.get_sibling_hashes().value(); - // Minimum logged data size is merkle tree word size - auto data_log2_size = std::max(a.get_log2_size(), machine_merkle_tree::get_log2_word_size()); - auto depth = machine_merkle_tree::get_log2_root_size() - data_log2_size; + // Minimum logged data size is hash-tree word size + auto data_log2_size = std::max(a.get_log2_size(), HASH_TREE_LOG2_WORD_SIZE); + auto depth = HASH_TREE_LOG2_ROOT_SIZE - data_log2_size; nlohmann::json s = nlohmann::json::array(); for (int i = 0; i < depth; i++) { s.push_back(encode_base64(sibling_hashes[i])); @@ -1745,18 +2035,25 @@ void to_json(nlohmann::json &j, const access &a) { } void to_json(nlohmann::json &j, const bracket_note &b) { - j = nlohmann::json{{"type", bracket_type_name(b.type)}, {"where", b.where}, {"text", b.text}}; + j = nlohmann::json{{"type", bracket_type_to_name(b.type)}, {"where", b.where}, {"text", b.text}}; +} + +void to_json(nlohmann::json &j, const uarch_interpreter_break_reason &break_reason) { + j = uarch_interpreter_break_reason_to_name(break_reason); +} + +void to_json(nlohmann::json &j, const interpreter_break_reason &break_reason) { + j = interpreter_break_reason_to_name(break_reason); } void to_json(nlohmann::json &j, const std::vector &bs) { j = nlohmann::json::array(); - std::transform(bs.cbegin(), bs.cend(), std::back_inserter(j), - [](const bracket_note &b) -> nlohmann::json { return b; }); + std::ranges::transform(bs, std::back_inserter(j), [](const bracket_note &b) -> nlohmann::json { return b; }); } void to_json(nlohmann::json &j, const std::vector &as) { j = nlohmann::json::array(); - std::transform(as.cbegin(), as.cend(), std::back_inserter(j), [](const access &a) -> nlohmann::json { return a; }); + std::ranges::transform(as, std::back_inserter(j), [](const access &a) -> nlohmann::json { return a; }); } void to_json(nlohmann::json &j, const access_log::type &log_type) { @@ -1771,16 +2068,28 @@ void to_json(nlohmann::json &j, const access_log &log) { } } +void to_json(nlohmann::json &j, const backing_store_config &config) { + j = nlohmann::json{{"shared", config.shared}, {"create", config.create}, {"truncate", config.truncate}, + {"data_filename", config.data_filename}, {"dht_filename", config.dht_filename}, + {"dpt_filename", config.dpt_filename}}; +} + +void to_json(nlohmann::json &j, const backing_store_config_only &config) { + j = nlohmann::json{{"backing_store", config.backing_store}}; +} + void to_json(nlohmann::json &j, const memory_range_config &config) { - j = nlohmann::json{{"start", config.start}, {"length", config.length}, {"shared", config.shared}, - {"image_filename", config.image_filename}}; + j = nlohmann::json{{"start", config.start}, {"length", config.length}, {"read_only", config.read_only}, + {"backing_store", config.backing_store}}; } -void to_json(nlohmann::json &j, const cmio_buffer_config &config) { - j = nlohmann::json{{"shared", config.shared}, {"image_filename", config.image_filename}}; +void to_json(nlohmann::json &j, const hash_tree_config &config) { + j = nlohmann::json{{"shared", config.shared}, {"create", config.create}, {"sht_filename", config.sht_filename}, + {"phtc_filename", config.phtc_filename}, {"phtc_size", config.phtc_size}, + {"hash_function", hash_function_to_name(config.hash_function)}}; } -void to_json(nlohmann::json &j, const processor_config &config) { +void to_json(nlohmann::json &j, const registers_state &config) { j = nlohmann::json{{"x0", config.x[0]}, {"x1", config.x[1]}, {"x2", config.x[2]}, {"x3", config.x[3]}, {"x4", config.x[4]}, {"x5", config.x[5]}, {"x6", config.x[6]}, {"x7", config.x[7]}, {"x8", config.x[8]}, {"x9", config.x[9]}, {"x10", config.x[10]}, {"x11", config.x[11]}, {"x12", config.x[12]}, {"x13", config.x[13]}, @@ -1803,14 +2112,17 @@ void to_json(nlohmann::json &j, const processor_config &config) { {"medeleg", config.medeleg}, {"mideleg", config.mideleg}, {"mcounteren", config.mcounteren}, {"menvcfg", config.menvcfg}, {"stvec", config.stvec}, {"sscratch", config.sscratch}, {"sepc", config.sepc}, {"scause", config.scause}, {"stval", config.stval}, {"satp", config.satp}, {"scounteren", config.scounteren}, - {"senvcfg", config.senvcfg}, {"ilrsc", config.ilrsc}, {"iprv", config.iprv}, {"iflags_X", config.iflags_X}, - {"iflags_Y", config.iflags_Y}, {"iflags_H", config.iflags_H}, {"iunrep", config.iunrep}}; + {"senvcfg", config.senvcfg}, {"ilrsc", config.ilrsc}, {"iprv", config.iprv}, {"iflags", config.iflags}, + {"iunrep", config.iunrep}, {"clint", config.clint}, {"plic", config.plic}, {"htif", config.htif}}; +} + +void to_json(nlohmann::json &j, const processor_config &config) { + j = nlohmann::json{{"registers", config.registers}, {"backing_store", config.backing_store}}; } void to_json(nlohmann::json &j, const flash_drive_configs &fs) { j = nlohmann::json::array(); - std::transform(fs.cbegin(), fs.cend(), std::back_inserter(j), - [](const memory_range_config &m) -> nlohmann::json { return m; }); + std::ranges::transform(fs, std::back_inserter(j), [](const memory_range_config &m) -> nlohmann::json { return m; }); } void to_json(nlohmann::json &j, const virtio_device_config &config) { @@ -1824,7 +2136,7 @@ void to_json(nlohmann::json &j, const virtio_device_config &config) { {"host_directory", vdev_config.host_directory}}; } else if constexpr (std::is_same_v) { nlohmann::json jhostfwd = nlohmann::json::array(); - std::transform(vdev_config.hostfwd.cbegin(), vdev_config.hostfwd.cend(), std::back_inserter(jhostfwd), + std::ranges::transform(vdev_config.hostfwd, std::back_inserter(jhostfwd), [](const virtio_hostfwd_config &h) -> nlohmann::json { return nlohmann::json{ {"is_udp", h.is_udp}, @@ -1846,14 +2158,14 @@ void to_json(nlohmann::json &j, const virtio_device_config &config) { void to_json(nlohmann::json &j, const virtio_configs &vs) { j = nlohmann::json::array(); - std::transform(vs.cbegin(), vs.cend(), std::back_inserter(j), + std::ranges::transform(vs, std::back_inserter(j), [](const virtio_device_config &v) -> nlohmann::json { return v; }); } void to_json(nlohmann::json &j, const ram_config &config) { j = nlohmann::json{ {"length", config.length}, - {"image_filename", config.image_filename}, + {"backing_store", config.backing_store}, }; } @@ -1862,36 +2174,38 @@ void to_json(nlohmann::json &j, const dtb_config &config) { {"bootargs", config.bootargs}, {"init", config.init}, {"entrypoint", config.entrypoint}, - {"image_filename", config.image_filename}, + {"backing_store", config.backing_store}, }; } -void to_json(nlohmann::json &j, const tlb_config &config) { +void to_json(nlohmann::json &j, const iflags_state &config) { j = nlohmann::json{ - {"image_filename", config.image_filename}, + {"X", config.X}, + {"Y", config.Y}, + {"H", config.H}, }; } -void to_json(nlohmann::json &j, const clint_config &config) { +void to_json(nlohmann::json &j, const clint_state &config) { j = nlohmann::json{ {"mtimecmp", config.mtimecmp}, }; } -void to_json(nlohmann::json &j, const plic_config &config) { +void to_json(nlohmann::json &j, const plic_state &config) { j = nlohmann::json{ {"girqpend", config.girqpend}, {"girqsrvd", config.girqsrvd}, }; } -void to_json(nlohmann::json &j, const htif_config &config) { +void to_json(nlohmann::json &j, const htif_state &config) { j = nlohmann::json{ {"fromhost", config.fromhost}, {"tohost", config.tohost}, - {"console_getchar", config.console_getchar}, - {"yield_manual", config.yield_manual}, - {"yield_automatic", config.yield_automatic}, + {"ihalt", config.ihalt}, + {"iconsole", config.iconsole}, + {"iyield", config.iyield}, }; } @@ -1902,7 +2216,7 @@ void to_json(nlohmann::json &j, const cmio_config &config) { }; } -void to_json(nlohmann::json &j, const uarch_processor_config &config) { +void to_json(nlohmann::json &j, const uarch_registers_state &config) { j = nlohmann::json{ {"x0", config.x[0]}, {"x1", config.x[1]}, @@ -1942,10 +2256,8 @@ void to_json(nlohmann::json &j, const uarch_processor_config &config) { }; } -void to_json(nlohmann::json &j, const uarch_ram_config &config) { - j = nlohmann::json{ - {"image_filename", config.image_filename}, - }; +void to_json(nlohmann::json &j, const uarch_processor_config &config) { + j = nlohmann::json{{"registers", config.registers}, {"backing_store", config.backing_store}}; } void to_json(nlohmann::json &j, const uarch_config &config) { @@ -1962,18 +2274,16 @@ void to_json(nlohmann::json &j, const machine_config &config) { {"dtb", config.dtb}, {"flash_drive", config.flash_drive}, {"virtio", config.virtio}, - {"tlb", config.tlb}, - {"clint", config.clint}, - {"plic", config.plic}, - {"htif", config.htif}, - {"uarch", config.uarch}, {"cmio", config.cmio}, + {"pmas", config.pmas}, + {"uarch", config.uarch}, + {"hash_tree", config.hash_tree}, }; } void to_json(nlohmann::json &j, const concurrency_runtime_config &config) { j = nlohmann::json{ - {"update_merkle_tree", config.update_merkle_tree}, + {"update_hash_tree", config.update_hash_tree}, }; } @@ -1987,21 +2297,19 @@ void to_json(nlohmann::json &j, const machine_runtime_config &runtime) { j = nlohmann::json{ {"concurrency", runtime.concurrency}, {"htif", runtime.htif}, - {"skip_root_hash_check", runtime.skip_root_hash_check}, - {"skip_root_hash_store", runtime.skip_root_hash_store}, {"skip_version_check", runtime.skip_version_check}, {"soft_yield", runtime.soft_yield}, + {"no_reserve", runtime.no_reserve}, }; } -void to_json(nlohmann::json &j, const machine_memory_range_descr &mrd) { +void to_json(nlohmann::json &j, const address_range_description &mrd) { j = nlohmann::json{{"length", mrd.length}, {"start", mrd.start}, {"description", mrd.description}}; } -void to_json(nlohmann::json &j, const machine_memory_range_descrs &mrds) { +void to_json(nlohmann::json &j, const address_range_descriptions &mrds) { j = nlohmann::json::array(); - std::transform(mrds.cbegin(), mrds.cend(), std::back_inserter(j), - [](const auto &a) -> nlohmann::json { return a; }); + std::ranges::transform(mrds, std::back_inserter(j), [](const auto &a) -> nlohmann::json { return a; }); } void to_json(nlohmann::json &j, const fork_result &fork_result) { @@ -2018,4 +2326,51 @@ void to_json(nlohmann::json &j, const semantic_version &version) { }; } +void to_json(nlohmann::json &j, const hash_tree_stats &stats) { + auto jdnh = nlohmann::json::array(); + std::ranges::copy(stats.dense_node_hashes, std::back_inserter(jdnh)); + j = nlohmann::json{ + {"sparse_node_hashes", stats.sparse_node_hashes}, + {"dense_node_hashes", jdnh}, + {"phtc", stats.phtc}, + }; +} + +void to_json(nlohmann::json &j, const page_hash_tree_cache_stats &stats) { + j = nlohmann::json{ + {"page_hits", stats.page_hits}, + {"page_misses", stats.page_misses}, + {"word_hits", stats.word_hits}, + {"word_misses", stats.word_misses}, + {"page_changes", stats.page_changes}, + {"inner_page_hashes", stats.inner_page_hashes}, + {"pristine_pages", stats.pristine_pages}, + {"non_pristine_pages", stats.non_pristine_pages}, + }; +} + +void to_json(nlohmann::json &j, const std::vector &ints) { + j = nlohmann::json::array(); + std::ranges::copy(ints, std::back_inserter(j)); +} + +void to_json(nlohmann::json &j, const mcycle_root_hashes &result) { + j = nlohmann::json{{"hashes", base64_machine_hashes(result.hashes)}, {"mcycle_phase", result.mcycle_phase}, + {"break_reason", result.break_reason}}; + if (result.back_tree.has_value()) { + j["back_tree"] = result.back_tree; + } +} + +void to_json(nlohmann::json &j, const uarch_cycle_root_hashes &result) { + j = nlohmann::json{{"hashes", base64_machine_hashes(result.hashes)}, {"reset_indices", result.reset_indices}, + {"break_reason", result.break_reason}}; +} + +void to_json(nlohmann::json &j, const back_merkle_tree &back_tree) { + j = nlohmann::json{{"log2_max_leaves", back_tree.get_log2_max_leaves()}, + {"hash_function", hash_function_to_name(back_tree.get_hash_function())}, + {"leaf_count", back_tree.get_leaf_count()}, {"context", base64_machine_hashes(back_tree.get_context())}}; +} + } // namespace cartesi diff --git a/src/json-util.h b/src/json-util.h index 8a67c377e..f54185ef8 100644 --- a/src/json-util.h +++ b/src/json-util.h @@ -18,7 +18,6 @@ #define JSON_UTIL_H #include -#include #include #include #include @@ -28,17 +27,25 @@ #include #include "access-log.h" +#include "address-range-description.h" +#include "back-merkle-tree.h" #include "bracket-note.h" +#include "hash-tree-proof.h" +#include "hash-tree-stats.h" #include "interpret.h" #include "jsonrpc-fork-result.h" #include "machine-config.h" -#include "machine-memory-range-descr.h" -#include "machine-merkle-tree.h" +#include "machine-hash.h" +#include "machine-reg.h" #include "machine-runtime-config.h" -#include "machine.h" +#include "mcycle-root-hashes.h" +#include "page-hash-tree-cache-stats.h" #include "semantic-version.h" -#include "uarch-config.h" +#include "shadow-registers.h" +#include "shadow-uarch-state.h" +#include "uarch-cycle-root-hashes.h" #include "uarch-interpret.h" +#include "variant-hasher.h" namespace cartesi { @@ -68,18 +75,24 @@ void ju_get_field(const nlohmann::json &j, const K &key, T &value, const std::st // Allows use contains when the index is an integer and j contains an array template -inline bool contains(const nlohmann::json &j, T i) +inline bool contains(const nlohmann::json &j, T i, const std::string &path) requires(std::is_integral_v) { + if (!j.empty() && !j.is_array()) { + throw std::invalid_argument("\""s + path + "\" not an array"); + } if constexpr (std::is_signed_v) { - return j.is_array() && i >= 0 && i < static_cast(j.size()); + return i >= 0 && i < static_cast(j.size()); } else { - return j.is_array() && i < j.size(); + return i < j.size(); } } // Overload for case where index is a string and j contains an object -inline bool contains(const nlohmann::json &j, const std::string &s) { +inline bool contains(const nlohmann::json &j, const std::string &s, const std::string &path) { + if (!j.empty() && !j.is_object()) { + throw std::invalid_argument("\""s + path + "\" not an object"); + } return j.contains(s); } @@ -145,7 +158,16 @@ void ju_get_opt_field(const nlohmann::json &j, const K &key, semantic_version &v /// \param value Object to store value /// \param path Path to j template -void ju_get_opt_field(const nlohmann::json &j, const K &key, machine::reg &value, const std::string &path = "params/"); +void ju_get_opt_field(const nlohmann::json &j, const K &key, machine_reg &value, const std::string &path = "params/"); + +/// \brief Attempts to load an sharing_mode name from a field in a JSON object +/// \tparam K Key type (explicit extern declarations for uint64_t and std::string are provided) +/// \param j JSON object to load from +/// \param key Key to load value from +/// \param value Object to store value +/// \param path Path to j +template +void ju_get_opt_field(const nlohmann::json &j, const K &key, sharing_mode &value, const std::string &path = "params/"); /// \brief Attempts to load an interpreter_break_reason name from a field in a JSON object /// \tparam K Key type (explicit extern declarations for uint64_t and std::string are provided) @@ -204,8 +226,7 @@ void ju_get_opt_field(const nlohmann::json &j, const K &key, machine_runtime_con /// \param value Object to store value /// \param path Path to j template -void ju_get_opt_field(const nlohmann::json &j, const K &key, machine_merkle_tree::proof_type::hash_type &value, - const std::string &path = "params/"); +void ju_get_opt_field(const nlohmann::json &j, const K &key, machine_hash &value, const std::string &path = "params/"); /// \brief Attempts to load a hash from a field in a JSON object /// \tparam K Key type (explicit extern declarations for uint64_t and std::string are provided) @@ -214,18 +235,38 @@ void ju_get_opt_field(const nlohmann::json &j, const K &key, machine_merkle_tree /// \param value Object to store value /// \param path Path to j template -void ju_get_opt_field(const nlohmann::json &j, const K &key, - std::optional &optional, const std::string &path = "params/"); +void ju_get_opt_field(const nlohmann::json &j, const K &key, std::optional &optional, + const std::string &path = "params/"); -/// \brief Attempts to load an Merkle tree proof object from a field in a JSON object +/// \brief Attempts to load a hash-tree proof object from a field in a JSON object /// \tparam K Key type (explicit extern declarations for uint64_t and std::string are provided) /// \param j JSON object to load from /// \param key Key to load value from /// \param value Object to store value /// \param path Path to j template -void ju_get_opt_field(const nlohmann::json &j, const K &key, - not_default_constructible &value, const std::string &path = "params/"); +void ju_get_opt_field(const nlohmann::json &j, const K &key, not_default_constructible &value, + const std::string &path = "params/"); + +/// \brief Attempts to load a page_hash_tree_cache_stats object from a field in a JSON object +/// \tparam K Key type (explicit extern declarations for uint64_t and std::string are provided) +/// \param j JSON object to load from +/// \param key Key to load value from +/// \param value Object to store value +/// \param path Path to j +template +void ju_get_opt_field(const nlohmann::json &j, const K &key, page_hash_tree_cache_stats &value, + const std::string &path = "params/"); + +/// \brief Attempts to load a hash_tree_stats object from a field in a JSON object +/// \tparam K Key type (explicit extern declarations for uint64_t and std::string are provided) +/// \param j JSON object to load from +/// \param key Key to load value from +/// \param value Object to store value +/// \param path Path to j +template +void ju_get_opt_field(const nlohmann::json &j, const K &key, hash_tree_stats &value, + const std::string &path = "params/"); /// \brief Attempts to load an access_type name from a field in a JSON object /// \tparam K Key type (explicit extern declarations for uint64_t and std::string are provided) @@ -302,6 +343,16 @@ template void ju_get_opt_field(const nlohmann::json &j, const K &key, not_default_constructible &optional, const std::string &path = "params/"); +/// \brief Attempts to load a registers_state object from a field in a JSON object +/// \tparam K Key type (explicit extern declarations for uint64_t and std::string are provided) +/// \param j JSON object to load from +/// \param key Key to load value from +/// \param value Object to store value +/// \param path Path to j +template +void ju_get_opt_field(const nlohmann::json &j, const K &key, registers_state &value, + const std::string &path = "params/"); + /// \brief Attempts to load a processor_config object from a field in a JSON object /// \tparam K Key type (explicit extern declarations for uint64_t and std::string are provided) /// \param j JSON object to load from @@ -340,14 +391,24 @@ template void ju_get_opt_field(const nlohmann::json &j, const K &key, memory_range_config &value, const std::string &path = "params/"); -/// \brief Attempts to load a cmio_buffer_config object from a field in a JSON object +/// \brief Attempts to load a backing_store_config object from a field in a JSON object /// \tparam K Key type (explicit extern declarations for uint64_t and std::string are provided) /// \param j JSON object to load from /// \param key Key to load value from /// \param value Object to store value /// \param path Path to j template -void ju_get_opt_field(const nlohmann::json &j, const K &key, cmio_buffer_config &value, +void ju_get_opt_field(const nlohmann::json &j, const K &key, backing_store_config &value, + const std::string &path = "params/"); + +/// \brief Attempts to load a backing_store_config_only object from a field in a JSON object +/// \tparam K Key type (explicit extern declarations for uint64_t and std::string are provided) +/// \param j JSON object to load from +/// \param key Key to load value from +/// \param value Object to store value +/// \param path Path to j +template +void ju_get_opt_field(const nlohmann::json &j, const K &key, backing_store_config_only &value, const std::string &path = "params/"); /// \brief Attempts to load a flash_drive_configs object from a field in a JSON object @@ -380,41 +441,41 @@ template void ju_get_opt_field(const nlohmann::json &j, const K &key, virtio_configs &value, const std::string &path = "params/"); -/// \brief Attempts to load a tlb_config object from a field in a JSON object +/// \brief Attempts to load a iflags_state object from a field in a JSON object /// \tparam K Key type (explicit extern declarations for uint64_t and std::string are provided) /// \param j JSON object to load from /// \param key Key to load value from /// \param value Object to store value /// \param path Path to j template -void ju_get_opt_field(const nlohmann::json &j, const K &key, tlb_config &value, const std::string &path = "params/"); +void ju_get_opt_field(const nlohmann::json &j, const K &key, iflags_state &value, const std::string &path = "params/"); -/// \brief Attempts to load a clint_config object from a field in a JSON object +/// \brief Attempts to load a clint_state object from a field in a JSON object /// \tparam K Key type (explicit extern declarations for uint64_t and std::string are provided) /// \param j JSON object to load from /// \param key Key to load value from /// \param value Object to store value /// \param path Path to j template -void ju_get_opt_field(const nlohmann::json &j, const K &key, clint_config &value, const std::string &path = "params/"); +void ju_get_opt_field(const nlohmann::json &j, const K &key, clint_state &value, const std::string &path = "params/"); -/// \brief Attempts to load a plic_config object from a field in a JSON object +/// \brief Attempts to load a plic_state object from a field in a JSON object /// \tparam K Key type (explicit extern declarations for uint64_t and std::string are provided) /// \param j JSON object to load from /// \param key Key to load value from /// \param value Object to store value /// \param path Path to j template -void ju_get_opt_field(const nlohmann::json &j, const K &key, plic_config &value, const std::string &path = "params/"); +void ju_get_opt_field(const nlohmann::json &j, const K &key, plic_state &value, const std::string &path = "params/"); -/// \brief Attempts to load an htif_config object from a field in a JSON object +/// \brief Attempts to load an htif_state object from a field in a JSON object /// \tparam K Key type (explicit extern declarations for uint64_t and std::string are provided) /// \param j JSON object to load from /// \param key Key to load value from /// \param value Object to store value /// \param path Path to j template -void ju_get_opt_field(const nlohmann::json &j, const K &key, htif_config &value, const std::string &path = "params/"); +void ju_get_opt_field(const nlohmann::json &j, const K &key, htif_state &value, const std::string &path = "params/"); /// \brief Attempts to load a cmio_config object from a field in a JSON object /// \tparam K Key type (explicit extern declarations for uint64_t and std::string are provided) @@ -435,24 +496,24 @@ template void ju_get_opt_field(const nlohmann::json &j, const K &key, std::optional &optional, const std::string &path = "params/"); -/// \brief Attempts to load an uarch_processor_config object from a field in a JSON object +/// \brief Attempts to load an uarch_registers_state object from a field in a JSON object /// \tparam K Key type (explicit extern declarations for uint64_t and std::string are provided) /// \param j JSON object to load from /// \param key Key to load value from /// \param value Object to store value /// \param path Path to j template -void ju_get_opt_field(const nlohmann::json &j, const K &key, uarch_processor_config &value, +void ju_get_opt_field(const nlohmann::json &j, const K &key, uarch_registers_state &value, const std::string &path = "params/"); -/// \brief Attempts to load an uarch_ram_config object from a field in a JSON object +/// \brief Attempts to load an uarch_processor_config object from a field in a JSON object /// \tparam K Key type (explicit extern declarations for uint64_t and std::string are provided) /// \param j JSON object to load from /// \param key Key to load value from /// \param value Object to store value /// \param path Path to j template -void ju_get_opt_field(const nlohmann::json &j, const K &key, uarch_ram_config &value, +void ju_get_opt_field(const nlohmann::json &j, const K &key, uarch_processor_config &value, const std::string &path = "params/"); /// \brief Attempts to load an uarch_config object from a field in a JSON object @@ -464,6 +525,26 @@ void ju_get_opt_field(const nlohmann::json &j, const K &key, uarch_ram_config &v template void ju_get_opt_field(const nlohmann::json &j, const K &key, uarch_config &value, const std::string &path = "params/"); +/// \brief Attempts to load a hash_function_type object from a field in a JSON object +/// \tparam K Key type (explicit extern declarations for uint64_t and std::string are provided) +/// \param j JSON object to load from +/// \param key Key to load value from +/// \param value Object to store value +/// \param path Path to j +template +void ju_get_opt_field(const nlohmann::json &j, const K &key, hash_function_type &value, + const std::string &path = "params/"); + +/// \brief Attempts to load an hash_tree_config object from a field in a JSON object +/// \tparam K Key type (explicit extern declarations for uint64_t and std::string are provided) +/// \param j JSON object to load from +/// \param key Key to load value from +/// \param value Object to store value +/// \param path Path to j +template +void ju_get_opt_field(const nlohmann::json &j, const K &key, hash_tree_config &value, + const std::string &path = "params/"); + /// \brief Attempts to load a machine_config object from a field in a JSON object /// \tparam K Key type (explicit extern declarations for uint64_t and std::string are provided) /// \param j JSON object to load from @@ -474,24 +555,24 @@ template void ju_get_opt_field(const nlohmann::json &j, const K &key, machine_config &value, const std::string &path = "params/"); -/// \brief Attempts to load a machine_memory_range_descr object from a field in a JSON object +/// \brief Attempts to load a address_range_description object from a field in a JSON object /// \tparam K Key type (explicit extern declarations for uint64_t and std::string are provided) /// \param j JSON object to load from /// \param key Key to load value from /// \param value Object to store value /// \param path Path to j template -void ju_get_opt_field(const nlohmann::json &j, const K &key, machine_memory_range_descr &value, +void ju_get_opt_field(const nlohmann::json &j, const K &key, address_range_description &value, const std::string &path = "params/"); -/// \brief Attempts to load a machine_memory_range_descrs object from a field in a JSON object +/// \brief Attempts to load a address_range_descriptions object from a field in a JSON object /// \tparam K Key type (explicit extern declarations for uint64_t and std::string are provided) /// \param j JSON object to load from /// \param key Key to load value from /// \param value Object to store value /// \param path Path to j template -void ju_get_opt_field(const nlohmann::json &j, const K &key, machine_memory_range_descrs &value, +void ju_get_opt_field(const nlohmann::json &j, const K &key, address_range_descriptions &value, const std::string &path = "params/"); /// \brief Attempts to load a fork_result object from a field in a JSON object @@ -503,6 +584,36 @@ void ju_get_opt_field(const nlohmann::json &j, const K &key, machine_memory_rang template void ju_get_opt_field(const nlohmann::json &j, const K &key, fork_result &value, const std::string &path = "params/"); +/// \brief Attempts to load an mcycle_root_hashes object from a field in a JSON object +/// \tparam K Key type (explicit extern declarations for uint64_t and std::string are provided) +/// \param j JSON object to load from +/// \param key Key to load value from +/// \param value Object to store value +/// \param path Path to j +template +void ju_get_opt_field(const nlohmann::json &j, const K &key, mcycle_root_hashes &value, + const std::string &path = "params/"); + +/// \brief Attempts to load an uarch_cycle_root_hashes object from a field in a JSON object +/// \tparam K Key type (explicit extern declarations for uint64_t and std::string are provided) +/// \param j JSON object to load from +/// \param key Key to load value from +/// \param value Object to store value +/// \param path Path to j +template +void ju_get_opt_field(const nlohmann::json &j, const K &key, uarch_cycle_root_hashes &value, + const std::string &path = "params/"); + +/// \brief Attempts to load an std::optional object from a field in a JSON object +/// \tparam K Key type (explicit extern declarations for uint64_t and std::string are provided) +/// \param j JSON object to load from +/// \param key Key to load value from +/// \param value Object to store value +/// \param path Path to j +template +void ju_get_opt_field(const nlohmann::json &j, const K &key, std::optional &value, + const std::string &path = "params/"); + /// \brief Attempts to load a vector from a field in a JSON object /// \tparam K Key type (explicit extern declarations for uint64_t and std::string are provided) /// \param j JSON object to load from @@ -512,7 +623,7 @@ void ju_get_opt_field(const nlohmann::json &j, const K &key, fork_result &value, template void ju_get_opt_vector_like_field(const nlohmann::json &j, const K &key, A &value, const std::string &path) { value.clear(); - if (!contains(j, key)) { + if (!contains(j, key, path)) { return; } const auto &jk = j[key]; @@ -542,7 +653,7 @@ void ju_get_opt_vector_like_field(const nlohmann::json &j, const K &key, A &valu /// \detail Throws error if field is missing template void ju_get_vector_like_field(const nlohmann::json &j, const K &key, A &value, const std::string &path = "params/") { - if (!contains(j, key)) { + if (!contains(j, key, path)) { throw std::invalid_argument("missing field \""s + path + to_string(key) + "\""s); } return ju_get_opt_vector_like_field(j, key, value, path); @@ -557,7 +668,7 @@ void ju_get_vector_like_field(const nlohmann::json &j, const K &key, A &value, c template void ju_get_opt_field(const nlohmann::json &j, const K &key, optional_param &value, const std::string &path = "params/") { - if (contains(j, key)) { + if (contains(j, key, path)) { value.emplace(); ju_get_opt_field(j, key, value.value(), path); } @@ -572,63 +683,82 @@ void ju_get_opt_field(const nlohmann::json &j, const K &key, optional_param & /// \detail Throws error if field is missing template void ju_get_field(const nlohmann::json &j, const K &key, T &value, const std::string &path) { - if (!contains(j, key)) { + if (!contains(j, key, path)) { throw std::invalid_argument("missing field \""s + path + to_string(key) + "\""s); } ju_get_opt_field(j, key, value, path); } -/// \brief Encodes binary data into base64 -/// \param data Pointer to start of binary data -/// \param length Length of data -/// \returns Encoded data -std::string encode_base64(const unsigned char *data, uint64_t length); +template +class override_to_json { + const T &m_t; // NOLINT(cppcoreguidelines-avoid-const-or-ref-data-members) -/// \brief Encodes a hash as a base64 string -/// \param hash Hash to encode -/// \returns Encoded data -std::string encode_base64(const machine_merkle_tree::hash_type &hash); +public: + explicit override_to_json(const T &t) : m_t(t) {} + const T &get() const { + return m_t; + } +}; -/// \brief Encodes an access_data object as a base64 string -/// \param data Access data to encode -/// \returns Encoded data -std::string encode_base64(const access_data &data); +using base64_machine_hash = override_to_json; +using base64_machine_hashes = override_to_json; // Automatic conversion functions from Cartesi types to nlohmann::json void to_json(nlohmann::json &j, const access_log::type &log_type); -void to_json(nlohmann::json &j, const machine_merkle_tree::hash_type &h); -void to_json(nlohmann::json &j, const std::vector &hs); -void to_json(nlohmann::json &j, const machine_merkle_tree::proof_type &p); +void to_json(nlohmann::json &j, const base64_machine_hash &h); +void to_json(nlohmann::json &j, const base64_machine_hashes &hs); +void to_json(nlohmann::json &j, const hash_tree_proof &p); +void to_json(nlohmann::json &j, const page_hash_tree_cache_stats &s); +void to_json(nlohmann::json &j, const hash_tree_stats &s); void to_json(nlohmann::json &j, const access &a); void to_json(nlohmann::json &j, const bracket_note &b); void to_json(nlohmann::json &j, const std::vector &bs); void to_json(nlohmann::json &j, const std::vector &as); void to_json(nlohmann::json &j, const access_log &log); +void to_json(nlohmann::json &j, const interpreter_break_reason &break_reason); +void to_json(nlohmann::json &j, const uarch_interpreter_break_reason &break_reason); +void to_json(nlohmann::json &j, const backing_store_config &config); +void to_json(nlohmann::json &j, const backing_store_config_only &config); void to_json(nlohmann::json &j, const memory_range_config &config); -void to_json(nlohmann::json &j, const cmio_buffer_config &config); +void to_json(nlohmann::json &j, const registers_state &config); void to_json(nlohmann::json &j, const processor_config &config); void to_json(nlohmann::json &j, const flash_drive_configs &fs); void to_json(nlohmann::json &j, const virtio_device_config &config); void to_json(nlohmann::json &j, const virtio_configs &vs); void to_json(nlohmann::json &j, const ram_config &config); void to_json(nlohmann::json &j, const dtb_config &config); -void to_json(nlohmann::json &j, const tlb_config &config); -void to_json(nlohmann::json &j, const clint_config &config); -void to_json(nlohmann::json &j, const plic_config &config); -void to_json(nlohmann::json &j, const htif_config &config); +void to_json(nlohmann::json &j, const iflags_state &config); +void to_json(nlohmann::json &j, const clint_state &config); +void to_json(nlohmann::json &j, const plic_state &config); +void to_json(nlohmann::json &j, const htif_state &config); void to_json(nlohmann::json &j, const cmio_config &config); +void to_json(nlohmann::json &j, const uarch_registers_state &config); void to_json(nlohmann::json &j, const uarch_processor_config &config); -void to_json(nlohmann::json &j, const uarch_ram_config &config); void to_json(nlohmann::json &j, const uarch_config &config); +void to_json(nlohmann::json &j, const hash_tree_config &config); void to_json(nlohmann::json &j, const machine_config &config); void to_json(nlohmann::json &j, const concurrency_runtime_config &config); void to_json(nlohmann::json &j, const htif_runtime_config &config); void to_json(nlohmann::json &j, const machine_runtime_config &runtime); -void to_json(nlohmann::json &j, const machine::reg ®); -void to_json(nlohmann::json &j, const machine_memory_range_descr &mrd); -void to_json(nlohmann::json &j, const machine_memory_range_descrs &mrds); +void to_json(nlohmann::json &j, const machine_reg ®); +void to_json(nlohmann::json &j, const sharing_mode &sharing); +void to_json(nlohmann::json &j, const address_range_description &mrd); +void to_json(nlohmann::json &j, const address_range_descriptions &mrds); void to_json(nlohmann::json &j, const fork_result &fork_result); void to_json(nlohmann::json &j, const semantic_version &version); +void to_json(nlohmann::json &j, const std::vector &uints); +void to_json(nlohmann::json &j, const mcycle_root_hashes &result); +void to_json(nlohmann::json &j, const uarch_cycle_root_hashes &result); +void to_json(nlohmann::json &j, const back_merkle_tree &back_tree); + +template +void to_json(nlohmann::json &j, const std::optional &v) { + if (v.has_value()) { + to_json(j, *v); + } else { + j = nullptr; + } +} // Extern template declarations extern template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, std::string &value, @@ -651,9 +781,13 @@ extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &k const std::string &base = "params/"); extern template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, semantic_version &value, const std::string &base = "params/"); -extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, machine::reg &value, +extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, machine_reg &value, const std::string &base = "params/"); -extern template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, machine::reg &value, +extern template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, machine_reg &value, + const std::string &base = "params/"); +extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, sharing_mode &value, + const std::string &base = "params/"); +extern template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, sharing_mode &value, const std::string &base = "params/"); extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, interpreter_break_reason &value, const std::string &base = "params/"); @@ -675,14 +809,22 @@ extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &k const std::string &base = "params/"); extern template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, machine_runtime_config &value, const std::string &base = "params/"); +extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, machine_hash &value, + const std::string &base = "params/"); +extern template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, machine_hash &value, + const std::string &base = "params/"); extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, - machine_merkle_tree::proof_type::hash_type &value, const std::string &base = "params/"); + not_default_constructible &value, const std::string &base = "params/"); extern template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, - machine_merkle_tree::proof_type::hash_type &value, const std::string &base = "params/"); -extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, - not_default_constructible &value, const std::string &base = "params/"); + not_default_constructible &value, const std::string &base = "params/"); +extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, page_hash_tree_cache_stats &value, + const std::string &base = "params/"); extern template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, - not_default_constructible &value, const std::string &base = "params/"); + page_hash_tree_cache_stats &value, const std::string &base = "params/"); +extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, hash_tree_stats &value, + const std::string &base = "params/"); +extern template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, hash_tree_stats &value, + const std::string &base = "params/"); extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, access_type &value, const std::string &base = "params/"); extern template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, access_type &value, @@ -715,6 +857,10 @@ extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &k not_default_constructible &value, const std::string &base = "params/"); extern template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, not_default_constructible &value, const std::string &base = "params/"); +extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, registers_state &value, + const std::string &base = "params/"); +extern template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, registers_state &value, + const std::string &base = "params/"); extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, processor_config &value, const std::string &base = "params/"); extern template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, processor_config &value, @@ -731,9 +877,13 @@ extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &k const std::string &base = "params/"); extern template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, memory_range_config &value, const std::string &base = "params/"); -extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, cmio_buffer_config &value, +extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, backing_store_config &value, + const std::string &base = "params/"); +extern template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, backing_store_config &value, const std::string &base = "params/"); -extern template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, cmio_buffer_config &value, +extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, backing_store_config_only &value, + const std::string &base = "params/"); +extern template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, backing_store_config_only &value, const std::string &base = "params/"); extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, flash_drive_configs &value, const std::string &base = "params/"); @@ -747,21 +897,21 @@ extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &k const std::string &base = "params/"); extern template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, virtio_configs &value, const std::string &base = "params/"); -extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, tlb_config &value, +extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, iflags_state &value, const std::string &base = "params/"); -extern template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, tlb_config &value, +extern template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, iflags_state &value, const std::string &base = "params/"); -extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, clint_config &value, +extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, clint_state &value, const std::string &base = "params/"); -extern template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, clint_config &value, +extern template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, clint_state &value, const std::string &base = "params/"); -extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, plic_config &value, +extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, plic_state &value, const std::string &base = "params/"); -extern template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, plic_config &value, +extern template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, plic_state &value, const std::string &base = "params/"); -extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, htif_config &value, +extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, htif_state &value, const std::string &base = "params/"); -extern template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, htif_config &value, +extern template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, htif_state &value, const std::string &base = "params/"); extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, cmio_config &value, const std::string &base = "params/"); @@ -771,34 +921,50 @@ extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &k const std::string &base = "params/"); extern template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, std::optional &value, const std::string &base = "params/"); -extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, uarch_processor_config &value, +extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, uarch_registers_state &value, const std::string &base = "params/"); -extern template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, uarch_processor_config &value, +extern template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, uarch_registers_state &value, const std::string &base = "params/"); -extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, uarch_ram_config &value, +extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, uarch_processor_config &value, const std::string &base = "params/"); -extern template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, uarch_ram_config &value, +extern template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, uarch_processor_config &value, const std::string &base = "params/"); extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, uarch_config &value, const std::string &base = "params/"); extern template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, uarch_config &value, const std::string &base = "params/"); +extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, hash_tree_config &value, + const std::string &base = "params/"); +extern template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, hash_tree_config &value, + const std::string &base = "params/"); extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, machine_config &value, const std::string &base = "params/"); extern template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, machine_config &value, const std::string &base = "params/"); -extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, machine_memory_range_descr &value, +extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, address_range_description &value, const std::string &base = "params/"); -extern template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, - machine_memory_range_descr &value, const std::string &base = "params/"); -extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, machine_memory_range_descrs &value, +extern template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, address_range_description &value, + const std::string &base = "params/"); +extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, address_range_descriptions &value, const std::string &base = "params/"); extern template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, - machine_memory_range_descrs &value, const std::string &base = "params/"); + address_range_descriptions &value, const std::string &base = "params/"); extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, fork_result &value, const std::string &base = "params/"); extern template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, fork_result &value, const std::string &base = "params/"); +extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, mcycle_root_hashes &value, + const std::string &base = "params/"); +extern template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, mcycle_root_hashes &value, + const std::string &base = "params/"); +extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, uarch_cycle_root_hashes &value, + const std::string &base = "params/"); +extern template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, uarch_cycle_root_hashes &value, + const std::string &base = "params/"); +extern template void ju_get_opt_field(const nlohmann::json &j, const uint64_t &key, + std::optional &value, const std::string &base = "params/"); +extern template void ju_get_opt_field(const nlohmann::json &j, const std::string &key, + std::optional &value, const std::string &base = "params/"); template nlohmann::json to_json(const T &v) { @@ -808,11 +974,11 @@ nlohmann::json to_json(const T &v) { } template -T from_json(const char *s) { +T from_json(const char *s, const char *path) { T value{}; if (s) { - const nlohmann::json j = nlohmann::json{{"value", nlohmann::json::parse(s)}}; - ju_get_field(j, "value"s, value, ""s); + const nlohmann::json j = nlohmann::json{{path, nlohmann::json::parse(s)}}; + ju_get_field(j, std::string{path}, value, ""s); } return value; } diff --git a/src/jsonrpc-discover.json b/src/jsonrpc-discover.json index 24a90b15e..f09fe4ae8 100644 --- a/src/jsonrpc-discover.json +++ b/src/jsonrpc-discover.json @@ -2,7 +2,7 @@ "openrpc": "1.0.0-rc1", "info": { "title": "Remote Cartesi Machine", - "version": "0.5.0", + "version": "0.6.0", "description": "API for controlling a remote Cartesi Machine server", "license": { "name": "MIT" @@ -130,6 +130,13 @@ "schema": { "$ref": "#/components/schemas/MachineRuntimeConfig" } + }, + { + "name": "directory", + "description": "Directory to create an on-disk machine instance", + "schema": { + "type": "string" + } } ], "result": { @@ -159,6 +166,14 @@ "schema": { "$ref": "#/components/schemas/MachineRuntimeConfig" } + }, + { + "name": "sharing", + "description": "Backing stores sharing mode", + "required": false, + "schema": { + "$ref": "#/components/schemas/SharingMode" + } } ], "result": { @@ -192,6 +207,64 @@ "schema": { "type": "string" } + }, + { + "name": "sharing", + "description": "Backing stores sharing mode", + "required": true, + "schema": { + "$ref": "#/components/schemas/SharingMode" + } + } + ], + "result": { + "name": "status", + "description": "True when operation succeeded", + "schema": { + "type": "boolean" + } + } + }, + { + "name": "machine.clone_stored", + "summary": "Clones a machine stored from source directory to destination directory", + "params": [ + { + "name": "from_dir", + "description": "Source directory", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "to_dir", + "description": "Destination directory", + "required": true, + "schema": { + "type": "string" + } + } + ], + "result": { + "name": "status", + "description": "True when operation succeeded", + "schema": { + "type": "boolean" + } + } + }, + { + "name": "machine.remove_stored", + "summary": "Removes a stored machine instance from a directory", + "params": [ + { + "name": "dir", + "description": "Directory to stored machine instance", + "required": true, + "schema": { + "type": "string" + } } ], "result": { @@ -223,6 +296,51 @@ } } }, + { + "name": "machine.collect_mcycle_root_hashes", + "summary": "Collect root hashes periodically while running the emulator", + "params": [ + { + "name": "mcycle_end", + "description": "End machine cycle value", + "required": true, + "schema": { + "$ref": "#/components/schemas/UnsignedInteger" + } + }, + { + "name": "mcycle_period", + "description": "Number of machine cycles between root hashes to collect", + "required": true, + "schema": { + "$ref": "#/components/schemas/UnsignedInteger" + } + }, + { + "name": "mcycle_phase", + "description": "Number of machine cycles elapsed since last root hash collected", + "required": true, + "schema": { + "$ref": "#/components/schemas/UnsignedInteger" + } + }, + { + "name": "log2_bundle_mcycle_count", + "description": "Log base 2 of the amount of mcycle root hashes to bundle", + "required": true, + "schema": { + "$ref": "#/components/schemas/UnsignedInteger" + } + } + ], + "result": { + "name": "result", + "description": "Collected hashes", + "schema": { + "$ref": "#/components/schemas/McycleRootHashes" + } + } + }, { "name": "machine.run_uarch", "summary": "Runs the small emulator until a given cycle", @@ -244,6 +362,35 @@ } } }, + { + "name": "machine.collect_uarch_cycle_root_hashes", + "summary": "Collect root hashes while running the emulator's uarch", + "params": [ + { + "name": "mcycle_end", + "description": "End machine cycle value to execute, uarch cycle by uarch cycle", + "required": true, + "schema": { + "$ref": "#/components/schemas/UnsignedInteger" + } + }, + { + "name": "log2_bundle_uarch_cycle_count", + "description": "Log base 2 of the amount of uarch cycle root hashes to bundle", + "required": true, + "schema": { + "$ref": "#/components/schemas/UnsignedInteger" + } + } + ], + "result": { + "name": "result", + "description": "Collected hashes", + "schema": { + "$ref": "#/components/schemas/UarchCycleRootHashes" + } + } + }, { "name": "machine.log_step_uarch", "summary": "Runs the small emulator for one cycle and return a log of state accesses", @@ -341,11 +488,11 @@ }, { "name": "machine.get_root_hash", - "summary": "Obtains the Merkle hash of the current machine state", + "summary": "Obtains the hash-tree root hash of the current machine state", "params": [], "result": { "name": "hash", - "description": "Merkle hash", + "description": "Root hash", "schema": { "$ref": "#/components/schemas/Base64Hash" } @@ -353,7 +500,7 @@ }, { "name": "machine.get_proof", - "summary": "Obtains a Merkle proof for a range in the machine state", + "summary": "Obtains a hash-tree proof for a range in the machine state", "params": [ { "name": "address", @@ -380,6 +527,27 @@ } } }, + { + "name": "machine.get_hash_tree_stats", + "summary": "Returns statistics for the hash tree", + "params": [ + { + "name": "clear", + "description": "Whether to clear the stats after retrieving them", + "required": true, + "schema": { + "type": "boolean" + } + } + ], + "result": { + "name": "stats", + "description": "Statistics for the hash tree", + "schema": { + "$ref": "#/components/schemas/HashTreeStats" + } + } + }, { "name": "machine.read_word", "summary": "Reads a 64-bit word from memory (must be aligned)", @@ -401,6 +569,35 @@ } } }, + { + "name": "machine.write_word", + "summary": "Writes a 64-bit word from memory (must be aligned)", + "params": [ + { + "name": "address", + "description": "Starting physical address of word", + "required": true, + "schema": { + "$ref": "#/components/schemas/UnsignedInteger" + } + }, + { + "name": "value", + "description": "Value of word (little-endian)", + "required": true, + "schema": { + "$ref": "#/components/schemas/UnsignedInteger" + } + } + ], + "result": { + "name": "status", + "description": "True when operation succeeded", + "schema": { + "type": "boolean" + } + } + }, { "name": "machine.read_memory", "summary": "Reads a span of memory from the state (must be contained in the same memory range)", @@ -688,38 +885,26 @@ } }, { - "name": "machine.verify_merkle_tree", - "summary": "Verifies sanity of Merkle tree", + "name": "machine.verify_hash_tree", + "summary": "Verifies sanity of hash tree", "params": [], "result": { "name": "valid", - "description": "True if Merkle tree is sane", + "description": "True if hash tree is sane", "schema": { "type": "boolean" } } }, { - "name": "machine.verify_dirty_page_maps", - "summary": "Verifies sanity of dirty page maps", - "params": [], - "result": { - "name": "valid", - "description": "True if dirty page maps are sane", - "schema": { - "type": "boolean" - } - } - }, - { - "name": "machine.get_memory_ranges", - "summary": "Returns a list with descriptions for all of the machine's memory ranges", + "name": "machine.get_address_ranges", + "summary": "Returns a list with descriptions for all of the machine's address ranges", "params": [], "result": { "name": "ranges", - "description": "Array of memory range descriptions", + "description": "Array of address range descriptions", "schema": { - "$ref": "#/components/schemas/MemoryRangeDescriptionArray" + "$ref": "#/components/schemas/AddressRangeDescriptionArray" } } }, @@ -962,7 +1147,7 @@ "title": "ConcurrencyRuntimeConfig", "type": "object", "properties": { - "update_merkle_tree": { + "update_hash_tree": { "$ref": "#/components/schemas/UnsignedInteger" } } @@ -976,6 +1161,58 @@ } } }, + "HashTreeStats": { + "title": "HashTreeStats", + "type": "object", + "properties": { + "phtc": { + "$ref": "#/components/schemas/PageHashTreeCacheStats" + }, + "sparse_node_hashes": { + "$ref": "#/components/schemas/UnsignedInteger" + }, + "dense_node_hashes": { + "$ref": "#/components/schemas/UnsignedIntegerArray" + } + } + }, + "UnsignedIntegerArray": { + "title": "UnsignedIntegerArray", + "type": "array", + "items": { + "$ref": "#/components/schemas/UnsignedInteger" + } + }, + "PageHashTreeCacheStats": { + "title": "PageHashTreeCacheStats", + "type": "object", + "properties": { + "page_hits": { + "$ref": "#/components/schemas/UnsignedInteger" + }, + "page_misses": { + "$ref": "#/components/schemas/UnsignedInteger" + }, + "word_hits": { + "$ref": "#/components/schemas/UnsignedInteger" + }, + "word_misses": { + "$ref": "#/components/schemas/UnsignedInteger" + }, + "page_changes": { + "$ref": "#/components/schemas/UnsignedInteger" + }, + "inner_page_hashes": { + "$ref": "#/components/schemas/UnsignedInteger" + }, + "pristine_pages": { + "$ref": "#/components/schemas/UnsignedInteger" + }, + "non_pristine_pages": { + "$ref": "#/components/schemas/UnsignedInteger" + } + } + }, "MachineRuntimeConfig": { "title": "MachineRuntimeConfig", "type": "object", @@ -986,23 +1223,32 @@ "htif": { "$ref": "#/components/schemas/HTIFRuntimeConfig" }, - "skip_root_hash_check": { - "type": "boolean" - }, - "skip_root_hash_store": { - "type": "boolean" - }, "skip_version_check": { "type": "boolean" }, "soft_yield": { "type": "boolean" + }, + "no_reserve": { + "type": "boolean" } } }, "ProcessorConfig": { "title": "ProcessorConfig", "type": "object", + "properties": { + "registers": { + "$ref": "#/components/schemas/RegistersConfig" + }, + "backing_store": { + "$ref": "#/components/schemas/BackingStoreConfig" + } + } + }, + "RegistersConfig": { + "title": "RegistersConfig", + "type": "object", "properties": { "x0": { "$ref": "#/components/schemas/UnsignedInteger" @@ -1307,8 +1553,8 @@ "length": { "$ref": "#/components/schemas/UnsignedInteger" }, - "image_filename": { - "type": "string" + "backing_store": { + "$ref": "#/components/schemas/BackingStoreConfig" } }, "required": ["length"] @@ -1326,8 +1572,8 @@ "entrypoint": { "type": "string" }, - "image_filename": { - "type": "string" + "backing_store": { + "$ref": "#/components/schemas/BackingStoreConfig" } } }, @@ -1341,23 +1587,41 @@ "length": { "$ref": "#/components/schemas/UnsignedInteger" }, - "image_filename": { - "type": "string" - }, - "shared": { + "read_only": { "type": "boolean" + }, + "backing_store": { + "$ref": "#/components/schemas/BackingStoreConfig" } } }, - "CmioBufferConfig": { - "title": "CmioBufferConfig", + "BackingStoreConfig": { + "title": "BackingStoreConfig", "type": "object", "properties": { - "image_filename": { + "data_filename": { + "type": "string" + }, + "dht_filename": { "type": "string" }, "shared": { "type": "boolean" + }, + "create": { + "type": "boolean" + }, + "truncate": { + "type": "boolean" + } + } + }, + "CMIOBufferConfig": { + "title": "CMIOBufferConfig", + "type": "object", + "properties": { + "backing_store": { + "$ref": "#/components/schemas/BackingStoreConfig" } } }, @@ -1422,15 +1686,6 @@ "$ref": "#/components/schemas/MemoryRangeConfig" } }, - "TLBConfig": { - "title": "TLBConfig", - "type": "object", - "properties": { - "image_filename": { - "type": "string" - } - } - }, "CLINTConfig": { "title": "CLINTConfig", "type": "object", @@ -1476,6 +1731,18 @@ "UarchProcessorConfig": { "title": "UarchProcessorConfig", "type": "object", + "properties": { + "registers": { + "$ref": "#/components/schemas/UarchRegistersConfig" + }, + "backing_store": { + "$ref": "#/components/schemas/BackingStoreConfig" + } + } + }, + "UarchRegistersConfig": { + "title": "UarchRegistersConfig", + "type": "object", "properties": { "x0": { "$ref": "#/components/schemas/UnsignedInteger" @@ -1580,7 +1847,7 @@ "$ref": "#/components/schemas/UnsignedInteger" }, "halt_flag": { - "type": "boolean" + "$ref": "#/components/schemas/UnsignedInteger" } } }, @@ -1591,8 +1858,8 @@ "length": { "$ref": "#/components/schemas/UnsignedInteger" }, - "image_filename": { - "type": "string" + "backing_store": { + "$ref": "#/components/schemas/BackingStoreConfig" } } }, @@ -1608,15 +1875,48 @@ } } }, - "CmioConfig": { - "title": "CmioConfig", + "CMIOConfig": { + "title": "CMIOConfig", "type": "object", "properties": { "rx_buffer": { - "$ref": "#/components/schemas/CmioBufferConfig" + "$ref": "#/components/schemas/CMIOBufferConfig" }, "tx_buffer": { - "$ref": "#/components/schemas/CmioBufferConfig" + "$ref": "#/components/schemas/CMIOBufferConfig" + } + } + }, + "PMAsConfig": { + "title": "PMAsConfig", + "type": "object", + "properties": { + "backing_store": { + "$ref": "#/components/schemas/BackingStoreConfig" + } + } + }, + "HashTreeConfig": { + "title": "HashTreeConfig", + "type": "object", + "properties": { + "shared": { + "type": "boolean" + }, + "create": { + "type": "boolean" + }, + "sht_filename": { + "type": "string" + }, + "phtc_filename": { + "type": "string" + }, + "phtc_size": { + "$ref": "#/components/schemas/UnsignedInteger" + }, + "hash_function": { + "$ref": "#/components/schemas/HashFunctionType" } } }, @@ -1643,9 +1943,6 @@ "flash_drive": { "$ref": "#/components/schemas/FlashDriveConfigs" }, - "tlb": { - "$ref": "#/components/schemas/TLBConfig" - }, "clint": { "$ref": "#/components/schemas/CLINTConfig" }, @@ -1655,18 +1952,28 @@ "htif": { "$ref": "#/components/schemas/HTIFConfig" }, - "uarch": { - "$ref": "#/components/schemas/UarchConfig" + "virtio": { + "$ref": "#/components/schemas/VirtIOConfigs" }, "cmio": { - "$ref": "#/components/schemas/CmioConfig" + "$ref": "#/components/schemas/CMIOConfig" }, - "virtio": { - "$ref": "#/components/schemas/VirtIOConfigs" + "pmas": { + "$ref": "#/components/schemas/PMAsConfig" + }, + "uarch": { + "$ref": "#/components/schemas/UarchConfig" + }, + "hash_tree": { + "$ref": "#/components/schemas/HashTreeConfig" } }, "required": ["ram"] }, + "HashFunctionType": { + "title": "HashFunctionType", + "enum": ["keccak256", "sha256"] + }, "InterpreterBreakReason": { "title": "InterpreterBreakReason", "enum": [ @@ -1680,7 +1987,7 @@ }, "UarchInterpreterBreakReason": { "title": "UarchInterpreterBreakReason", - "enum": ["reached_target_cycle", "uarch_halted"] + "enum": ["reached_target_cycle", "uarch_halted", "cycle_overflow"] }, "Base64String": { "title": "Base64String", @@ -1842,6 +2149,10 @@ }, "required": ["log_type", "accesses"] }, + "SharingMode": { + "title": "SharingMode", + "enum": ["none", "config", "all"] + }, "REG": { "title": "REG", "enum": [ @@ -1996,8 +2307,8 @@ "uarch_halt_flag" ] }, - "MemoryRangeDescription": { - "title": "MemoryRangeDescription", + "AddressRangeDescription": { + "title": "AddressRangeDescription", "type": "object", "properties": { "start": { @@ -2011,12 +2322,71 @@ } } }, - "MemoryRangeDescriptionArray": { - "title": "MemoryRangeDescriptionArray", + "AddressRangeDescriptionArray": { + "title": "AddressRangeDescriptionArray", "type": "array", "items": { - "$ref": "#/components/schemas/MemoryRangeDescription" + "$ref": "#/components/schemas/AddressRangeDescription" } + }, + "McycleRootHashes": { + "title": "McycleRootHashes", + "type": "object", + "properties": { + "hashes": { + "$ref": "#/components/schemas/Base64HashArray" + }, + "mcycle_phase": { + "$ref": "#/components/schemas/UnsignedInteger" + }, + "break_reason": { + "$ref": "#/components/schemas/InterpreterBreakReason" + }, + "back_tree": { + "$ref": "#/components/schemas/BackMerkleTree" + } + }, + "required": ["hashes", "mcycle_phase", "break_reason"] + }, + "UarchCycleRootHashes": { + "title": "UarchCycleRootHashes", + "type": "object", + "properties": { + "hashes": { + "$ref": "#/components/schemas/Base64HashArray" + }, + "reset_indices": { + "$ref": "#/components/schemas/UnsignedIntegerArray" + }, + "break_reason": { + "$ref": "#/components/schemas/InterpreterBreakReason" + } + }, + "required": ["hashes", "reset_indices", "break_reason"] + }, + "BackMerkleTree": { + "title": "BackMerkleTree", + "type": "object", + "properties": { + "log2_max_leaves": { + "$ref": "#/components/schemas/UnsignedInteger" + }, + "hash_function": { + "$ref": "#/components/schemas/HashFunctionType" + }, + "leaf_count": { + "$ref": "#/components/schemas/UnsignedInteger" + }, + "context": { + "$ref": "#/components/schemas/Base64HashArray" + } + }, + "required": [ + "log2_max_leaves", + "hash_function", + "leaf_count", + "context" + ] } } } diff --git a/src/jsonrpc-fork-result.h b/src/jsonrpc-fork-result.h index 26a5ba73c..504ba6604 100644 --- a/src/jsonrpc-fork-result.h +++ b/src/jsonrpc-fork-result.h @@ -17,6 +17,9 @@ #ifndef JSONRPC_FORK_RESULT_H #define JSONRPC_FORK_RESULT_H +#include +#include + namespace cartesi { struct fork_result final { diff --git a/src/jsonrpc-machine-c-api.cpp b/src/jsonrpc-machine-c-api.cpp index a50429ba1..f1cf0c40e 100644 --- a/src/jsonrpc-machine-c-api.cpp +++ b/src/jsonrpc-machine-c-api.cpp @@ -14,78 +14,66 @@ // with this program (see COPYING). If not, see . // -#include -#include +#include "jsonrpc-machine-c-api.h" + #include -#include -#include -#include -#include -#include #include #include -#include "access-log.h" -#include "i-virtual-machine.h" +#include "i-machine.h" #include "json-util.h" -#include "jsonrpc-machine-c-api.h" -#include "jsonrpc-virtual-machine.h" +#include "jsonrpc-fork-result.h" +#include "jsonrpc-machine.h" #include "machine-c-api-internal.h" #include "machine-c-api.h" -#include "machine-config.h" -#include "machine-runtime-config.h" -#include "machine.h" -#include "os-features.h" -#include "os.h" -#include "semantic-version.h" using namespace std::string_literals; -static cartesi::jsonrpc_virtual_machine::cleanup_call convert_from_c(cm_jsonrpc_cleanup_call call) { +static cartesi::jsonrpc_machine::cleanup_call convert_from_c(cm_jsonrpc_cleanup_call call) { switch (call) { case CM_JSONRPC_DESTROY: - return cartesi::jsonrpc_virtual_machine::cleanup_call::destroy; + return cartesi::jsonrpc_machine::cleanup_call::destroy; case CM_JSONRPC_SHUTDOWN: - return cartesi::jsonrpc_virtual_machine::cleanup_call::shutdown; + return cartesi::jsonrpc_machine::cleanup_call::shutdown; case CM_JSONRPC_NOTHING: - return cartesi::jsonrpc_virtual_machine::cleanup_call::nothing; + return cartesi::jsonrpc_machine::cleanup_call::nothing; default: throw std::invalid_argument("invalid cleanup call"); } } -static cm_jsonrpc_cleanup_call convert_to_c(cartesi::jsonrpc_virtual_machine::cleanup_call call) { +static cm_jsonrpc_cleanup_call convert_to_c(cartesi::jsonrpc_machine::cleanup_call call) { switch (call) { - case cartesi::jsonrpc_virtual_machine::cleanup_call::destroy: + case cartesi::jsonrpc_machine::cleanup_call::destroy: return CM_JSONRPC_DESTROY; - case cartesi::jsonrpc_virtual_machine::cleanup_call::shutdown: + case cartesi::jsonrpc_machine::cleanup_call::shutdown: return CM_JSONRPC_SHUTDOWN; - case cartesi::jsonrpc_virtual_machine::cleanup_call::nothing: + case cartesi::jsonrpc_machine::cleanup_call::nothing: return CM_JSONRPC_NOTHING; default: throw std::invalid_argument("invalid cleanup call"); } } -static cartesi::jsonrpc_virtual_machine *convert_from_c(cm_machine *m) { +static cartesi::jsonrpc_machine *convert_from_c(cm_machine *m) { if (m == nullptr) { throw std::invalid_argument("invalid machine"); } // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) - auto *cpp_m = reinterpret_cast(m); - if (!cpp_m->is_jsonrpc_virtual_machine()) { + auto *cpp_m = reinterpret_cast(m); + if (!cpp_m->is_jsonrpc_machine()) { throw std::invalid_argument("not a JSONRPC remote machine"); } // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) - return reinterpret_cast(m); + return reinterpret_cast(m); } -static const cartesi::jsonrpc_virtual_machine *convert_from_c(const cm_machine *m) { +static const cartesi::jsonrpc_machine *convert_from_c(const cm_machine *m) { // NOLINTNEXTLINE(cppcoreguidelines-pro-type-const-cast) return convert_from_c(const_cast(m)); } -static cm_machine *convert_to_c(cartesi::i_virtual_machine *cpp_m) { +static cm_machine *convert_to_c(cartesi::i_machine *cpp_m) { // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) return reinterpret_cast(cpp_m); } @@ -98,7 +86,7 @@ cm_error cm_jsonrpc_connect_server(const char *address, int64_t connect_timeout_ if (new_m == nullptr) { throw std::invalid_argument("invalid new machine output"); } - *new_m = convert_to_c(new jsonrpc_virtual_machine(address, connect_timeout_ms)); + *new_m = convert_to_c(new jsonrpc_machine(address, connect_timeout_ms)); return cm_result_success(); } catch (...) { if (new_m != nullptr) { @@ -117,7 +105,7 @@ cm_error cm_jsonrpc_spawn_server(const char *address, int64_t spawn_timeout_ms, throw std::invalid_argument("invalid new machine output"); } fork_result spawned; - *new_m = convert_to_c(new jsonrpc_virtual_machine(address, spawn_timeout_ms, spawned)); + *new_m = convert_to_c(new jsonrpc_machine(address, spawn_timeout_ms, spawned)); if (bound_address != nullptr) { *bound_address = cm_set_temp_string(spawned.address); } @@ -152,7 +140,7 @@ cm_error cm_jsonrpc_fork_server(const cm_machine *m, cm_machine **forked_m, cons if (pid != nullptr) { *pid = forked.pid; } - auto *cpp_forked_m = new jsonrpc_virtual_machine(forked.address); + auto *cpp_forked_m = new jsonrpc_machine(forked.address); cpp_forked_m->set_cleanup_call(cpp_m->get_cleanup_call()); cpp_forked_m->set_timeout(cpp_m->get_timeout()); *forked_m = convert_to_c(cpp_forked_m); diff --git a/src/jsonrpc-virtual-machine.cpp b/src/jsonrpc-machine.cpp similarity index 77% rename from src/jsonrpc-virtual-machine.cpp rename to src/jsonrpc-machine.cpp index 1ba209e08..1f4e6eed1 100644 --- a/src/jsonrpc-virtual-machine.cpp +++ b/src/jsonrpc-machine.cpp @@ -14,16 +14,23 @@ // with this program (see COPYING). If not, see . // -#include "jsonrpc-virtual-machine.h" +#include "jsonrpc-machine.h" +#include "os-features.h" +#include #include #include #include #include #include +#include +#include #include #include -#include +#include +#include +#include +#include #include #include #include @@ -31,8 +38,6 @@ #include #include -#include "os-features.h" - #ifdef HAVE_FORK #include #include @@ -48,32 +53,40 @@ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdeprecated-declarations" -#include +#pragma GCC diagnostic ignored "-Wunused-but-set-variable" +#include // IWYU pragma: keep #include #include -#include -#include +#include // IWYU pragma: keep +#include // IWYU pragma: keep #include #pragma GCC diagnostic pop +#include + #include "access-log.h" +#include "address-range-description.h" +#include "back-merkle-tree.h" #include "base64.h" +#include "hash-tree-proof.h" +#include "hash-tree-stats.h" +#include "i-machine.h" #include "interpret.h" #include "json-util.h" -#include "json.hpp" +#include "jsonrpc-fork-result.h" #include "jsonrpc-version.h" #include "machine-config.h" -#include "machine-memory-range-descr.h" -#include "machine-merkle-tree.h" +#include "machine-hash.h" #include "machine-runtime-config.h" +#include "mcycle-root-hashes.h" #include "os.h" #include "scope-exit.h" #include "semantic-version.h" +#include "uarch-cycle-root-hashes.h" #include "uarch-interpret.h" using namespace std::string_literals; using json = nlohmann::json; -using hash_type = cartesi::machine_merkle_tree::hash_type; namespace beast = boost::beast; // from namespace http = beast::http; // from @@ -119,7 +132,7 @@ static asio::ip::tcp::endpoint parse_endpoint(const std::string &address) { throw std::runtime_error("invalid port number"s); } return {asio::ip::make_address(host), static_cast(port)}; - } catch (std::exception &e) { + } catch (const std::exception &e) { throw std::runtime_error("failed to parse endpoint from address \""s + address + "\": "s + e.what()); } } @@ -228,7 +241,7 @@ static std::string json_post(boost::asio::io_context &ioc, beast::tcp_stream &st beast::flat_buffer buffer; http::response_parser res_parser; res_parser.eager(true); - res_parser.body_limit(16777216U); // can receive up to 16MB + res_parser.body_limit(std::numeric_limits::max()); // can receive unlimited amount of data // Receive the HTTP response, we perform an asynchronous operation in order to support timeouts http::async_read(stream, buffer, res_parser, [&](beast::error_code ec, std::size_t /*bytes_transferred*/) { @@ -274,13 +287,13 @@ static void jsonrpc_request(std::unique_ptr &ioc, std:: std::string response_s; try { response_s = json_post(*ioc, *stream, remote_address, request, timeout_at, keep_alive); - } catch (std::exception &x) { + } catch (const std::exception &x) { throw std::runtime_error("jsonrpc error: post error contacting "s + remote_address + " ("s + x.what() + ")"s); } json response; try { response = json::parse(response_s); - } catch (std::exception &x) { + } catch (const std::exception &x) { throw std::runtime_error("jsonrpc server error: invalid response ("s + x.what() + ")"s); } if (!response.contains("jsonrpc")) { @@ -318,7 +331,7 @@ static void jsonrpc_request(std::unique_ptr &ioc, std:: } try { cartesi::ju_get_field(response, "result"s, result, ""s); - } catch (std::exception &x) { + } catch (const std::exception &x) { throw std::runtime_error("jsonrpc server error: "s + x.what()); } } @@ -326,7 +339,7 @@ static void jsonrpc_request(std::unique_ptr &ioc, std:: namespace cartesi { template -void jsonrpc_virtual_machine::request(const std::string &method, const std::tuple &tp, R &result, +void jsonrpc_machine::request(const std::string &method, const std::tuple &tp, R &result, bool keep_alive) const { // Determine request timeout time const auto timeout_at = m_timeout >= 0 ? (std::chrono::steady_clock::now() + std::chrono::milliseconds(m_timeout)) : @@ -336,12 +349,12 @@ void jsonrpc_virtual_machine::request(const std::string &method, const std::tupl } template -void jsonrpc_virtual_machine::request(const std::string &method, const std::tuple &tp, R &result, +void jsonrpc_machine::request(const std::string &method, const std::tuple &tp, R &result, std::chrono::time_point timeout_at, bool keep_alive) const { jsonrpc_request(m_ioc, m_stream, m_address, method, tp, result, timeout_at, keep_alive); } -void jsonrpc_virtual_machine::shutdown_server() { +void jsonrpc_machine::shutdown_server() { bool result = false; request("shutdown", std::tie(), result, false); // Destroy ASIO context early to release its socket before the destructor, @@ -351,28 +364,28 @@ void jsonrpc_virtual_machine::shutdown_server() { m_ioc.reset(); } -void jsonrpc_virtual_machine::delay_next_request(uint64_t ms) const { +void jsonrpc_machine::delay_next_request(uint64_t ms) const { bool result = false; request("delay_next_request", std::tie(ms), result); } -void jsonrpc_virtual_machine::set_timeout(int64_t ms) { +void jsonrpc_machine::set_timeout(int64_t ms) { m_timeout = ms; } -int64_t jsonrpc_virtual_machine::get_timeout() const { +int64_t jsonrpc_machine::get_timeout() const { return m_timeout; } -void jsonrpc_virtual_machine::set_cleanup_call(cleanup_call call) { +void jsonrpc_machine::set_cleanup_call(cleanup_call call) { m_call = call; } -auto jsonrpc_virtual_machine::get_cleanup_call() const -> cleanup_call { +auto jsonrpc_machine::get_cleanup_call() const -> cleanup_call { return m_call; } -const std::string &jsonrpc_virtual_machine::get_server_address() const { +const std::string &jsonrpc_machine::get_server_address() const { return m_address; } @@ -380,8 +393,7 @@ static inline std::string semver_to_string(uint32_t major, uint32_t minor) { return std::to_string(major) + "." + std::to_string(minor); } -void jsonrpc_virtual_machine::check_server_version( - std::chrono::time_point timeout_at) const { +void jsonrpc_machine::check_server_version(std::chrono::time_point timeout_at) const { semantic_version server_version; request("get_version", std::tie(), server_version, timeout_at, false); if (server_version.major != JSONRPC_VERSION_MAJOR || server_version.minor != JSONRPC_VERSION_MINOR) { @@ -391,7 +403,7 @@ void jsonrpc_virtual_machine::check_server_version( } } -jsonrpc_virtual_machine::jsonrpc_virtual_machine(std::string address, int64_t connect_timeout_ms) : +jsonrpc_machine::jsonrpc_machine(std::string address, int64_t connect_timeout_ms) : m_ioc(new boost::asio::io_context{1}), m_stream(new boost::beast::tcp_stream(*m_ioc)), m_address(std::move(address)) { @@ -407,7 +419,7 @@ jsonrpc_virtual_machine::jsonrpc_virtual_machine(std::string address, int64_t co check_server_version(timeout_at); } -jsonrpc_virtual_machine::jsonrpc_virtual_machine(std::string address) : +jsonrpc_machine::jsonrpc_machine(std::string address) : m_ioc(new boost::asio::io_context{1}), m_stream(new boost::beast::tcp_stream(*m_ioc)), m_address(std::move(address)) { @@ -426,7 +438,7 @@ static boost::asio::ip::tcp::endpoint address_to_endpoint(const std::string &add throw std::runtime_error{"invalid port"}; } return {boost::asio::ip::make_address(ip), static_cast(port)}; - } catch (std::exception &e) { + } catch (const std::exception &e) { throw std::runtime_error{"invalid endpoint address \"" + address + "\""}; } } @@ -437,8 +449,7 @@ static std::string endpoint_to_string(const boost::asio::ip::tcp::endpoint &endp return ss.str(); } -jsonrpc_virtual_machine::jsonrpc_virtual_machine(const std::string &address, int64_t spawn_timeout_ms, - fork_result &spawned) : +jsonrpc_machine::jsonrpc_machine(const std::string &address, int64_t spawn_timeout_ms, fork_result &spawned) : m_ioc(new boost::asio::io_context{1}), m_stream(new boost::beast::tcp_stream(*m_ioc)), m_call(cleanup_call::shutdown) { @@ -537,7 +548,7 @@ jsonrpc_virtual_machine::jsonrpc_virtual_machine(const std::string &address, int // Rebind the forked server to listen on the originally requested address std::string rebind_result; - request("rebind", std::tie(forked_grand_child.address), rebind_result, timeout_at, false); + request("rebind", std::tie(address), rebind_result, timeout_at, false); m_address = rebind_result; // At this point, we've confirmed the remote server is properly initialized and running @@ -547,26 +558,27 @@ jsonrpc_virtual_machine::jsonrpc_virtual_machine(const std::string &address, int #else -jsonrpc_virtual_machine::jsonrpc_virtual_machine(const std::string & /*address*/, fork_result & /*spawned*/) { +jsonrpc_machine::jsonrpc_machine(const std::string & /*address*/, fork_result & /*spawned*/) { throw std::runtime_error{"fork() is unsupported in this platform"s}; } #endif -void jsonrpc_virtual_machine::do_load(const std::string &directory, const machine_runtime_config &runtime) { +void jsonrpc_machine::do_load(const std::string &directory, const machine_runtime_config &runtime, + sharing_mode sharing) { bool result = false; - request("machine.load", std::tie(directory, runtime), result); + request("machine.load", std::tie(directory, runtime, sharing), result); } -bool jsonrpc_virtual_machine::do_is_empty() const { +bool jsonrpc_machine::do_is_empty() const { bool result = false; request("machine.is_empty", std::tie(), result); return result; } -i_virtual_machine *jsonrpc_virtual_machine::do_clone_empty() const { +i_machine *jsonrpc_machine::do_clone_empty() const { auto fork_result = fork_server(); - auto *clone = new jsonrpc_virtual_machine(fork_result.address); + auto *clone = new jsonrpc_machine(fork_result.address); try { if (!clone->is_empty()) { clone->destroy(); @@ -581,14 +593,15 @@ i_virtual_machine *jsonrpc_virtual_machine::do_clone_empty() const { throw; } return clone; -}; +} -void jsonrpc_virtual_machine::do_create(const machine_config &config, const machine_runtime_config &runtime) { +void jsonrpc_machine::do_create(const machine_config &config, const machine_runtime_config &runtime, + const std::string &dir) { bool result = false; - request("machine.create", std::tie(config, runtime), result); + request("machine.create", std::tie(config, runtime, dir), result); } -jsonrpc_virtual_machine::~jsonrpc_virtual_machine() { +jsonrpc_machine::~jsonrpc_machine() { // If configured to destroy machine, do it if (m_stream && m_call == cleanup_call::destroy) { try { @@ -613,76 +626,95 @@ jsonrpc_virtual_machine::~jsonrpc_virtual_machine() { } } -machine_config jsonrpc_virtual_machine::do_get_initial_config() const { +machine_config jsonrpc_machine::do_get_initial_config() const { machine_config result; request("machine.get_initial_config", std::tie(), result); return result; } -machine_runtime_config jsonrpc_virtual_machine::do_get_runtime_config() const { +machine_runtime_config jsonrpc_machine::do_get_runtime_config() const { machine_runtime_config result; request("machine.get_runtime_config", std::tie(), result); return result; } -void jsonrpc_virtual_machine::do_set_runtime_config(const machine_runtime_config &r) { +void jsonrpc_machine::do_set_runtime_config(const machine_runtime_config &r) { bool result = false; request("machine.set_runtime_config", std::tie(r), result); } -semantic_version jsonrpc_virtual_machine::get_server_version() const { +semantic_version jsonrpc_machine::get_server_version() const { semantic_version result; request("get_version", std::tie(), result); return result; } -interpreter_break_reason jsonrpc_virtual_machine::do_run(uint64_t mcycle_end) { +interpreter_break_reason jsonrpc_machine::do_run(uint64_t mcycle_end) { interpreter_break_reason result = interpreter_break_reason::failed; request("machine.run", std::tie(mcycle_end), result); return result; } -interpreter_break_reason jsonrpc_virtual_machine::do_log_step(uint64_t mcycle_count, const std::string &filename) { +mcycle_root_hashes jsonrpc_machine::do_collect_mcycle_root_hashes(uint64_t mcycle_end, uint64_t mcycle_period, + uint64_t mcycle_phase, int32_t log2_bundle_mcycle_count, + const std::optional &previous_back_tree) { + mcycle_root_hashes result; + request("machine.collect_mcycle_root_hashes", + std::tie(mcycle_end, mcycle_period, mcycle_phase, log2_bundle_mcycle_count, previous_back_tree), result); + return result; +} + +interpreter_break_reason jsonrpc_machine::do_log_step(uint64_t mcycle_count, const std::string &filename) { interpreter_break_reason result = interpreter_break_reason::failed; request("machine.log_step", std::tie(mcycle_count, filename), result); return result; } -void jsonrpc_virtual_machine::do_store(const std::string &directory) const { +void jsonrpc_machine::do_store(const std::string &directory, sharing_mode sharing) const { + bool result = false; + request("machine.store", std::tie(directory, sharing), result); +} + +void jsonrpc_machine::do_clone_stored(const std::string &from_dir, const std::string &to_dir) const { + bool result = false; + request("machine.clone_stored", std::tie(from_dir, to_dir), result); +} + +void jsonrpc_machine::do_remove_stored(const std::string &dir) const { bool result = false; - request("machine.store", std::tie(directory), result); + request("machine.remove_stored", std::tie(dir), result); } -uint64_t jsonrpc_virtual_machine::do_read_reg(reg r) const { +uint64_t jsonrpc_machine::do_read_reg(reg r) const { uint64_t result = 0; request("machine.read_reg", std::tie(r), result); return result; } -void jsonrpc_virtual_machine::do_write_reg(reg w, uint64_t val) { +void jsonrpc_machine::do_write_reg(reg w, uint64_t val) { bool result = false; request("machine.write_reg", std::tie(w, val), result); } -auto jsonrpc_virtual_machine::fork_server() const -> fork_result { +auto jsonrpc_machine::fork_server() const -> fork_result { fork_result result{}; request("fork", std::tie(), result, false); return result; } -std::string jsonrpc_virtual_machine::rebind_server(const std::string &address) { +std::string jsonrpc_machine::rebind_server(const std::string &address) { std::string result; request("rebind", std::tie(address), result, false); m_address = result; return result; } -void jsonrpc_virtual_machine::emancipate_server() const { +void jsonrpc_machine::emancipate_server() const { bool result = false; request("emancipate", std::tie(), result); } -void jsonrpc_virtual_machine::do_read_memory(uint64_t address, unsigned char *data, uint64_t length) const { +void jsonrpc_machine::do_read_memory(uint64_t address, unsigned char *data, uint64_t length) const { std::string result; request("machine.read_memory", std::tie(address, length), result); std::string bin = cartesi::decode_base64(result); @@ -692,13 +724,13 @@ void jsonrpc_virtual_machine::do_read_memory(uint64_t address, unsigned char *da std::memcpy(data, bin.data(), length); } -void jsonrpc_virtual_machine::do_write_memory(uint64_t address, const unsigned char *data, uint64_t length) { +void jsonrpc_machine::do_write_memory(uint64_t address, const unsigned char *data, uint64_t length) { bool result = false; - std::string b64 = cartesi::encode_base64(data, length); + std::string b64 = cartesi::encode_base64(std::span{data, length}); request("machine.write_memory", std::tie(address, b64), result); } -void jsonrpc_virtual_machine::do_read_virtual_memory(uint64_t address, unsigned char *data, uint64_t length) { +void jsonrpc_machine::do_read_virtual_memory(uint64_t address, unsigned char *data, uint64_t length) { std::string result; request("machine.read_virtual_memory", std::tie(address, length), result); std::string bin = cartesi::decode_base64(result); @@ -708,24 +740,24 @@ void jsonrpc_virtual_machine::do_read_virtual_memory(uint64_t address, unsigned std::memcpy(data, bin.data(), length); } -void jsonrpc_virtual_machine::do_write_virtual_memory(uint64_t address, const unsigned char *data, uint64_t length) { +void jsonrpc_machine::do_write_virtual_memory(uint64_t address, const unsigned char *data, uint64_t length) { bool result = false; - std::string b64 = cartesi::encode_base64(data, length); + std::string b64 = cartesi::encode_base64(std::span{data, length}); request("machine.write_virtual_memory", std::tie(address, b64), result); } -uint64_t jsonrpc_virtual_machine::do_translate_virtual_address(uint64_t vaddr) { +uint64_t jsonrpc_machine::do_translate_virtual_address(uint64_t vaddr) { uint64_t result = 0; request("machine.translate_virtual_address", std::tie(vaddr), result); return result; } -void jsonrpc_virtual_machine::do_reset_uarch() { +void jsonrpc_machine::do_reset_uarch() { bool result = false; request("machine.reset_uarch", std::tie(), result); } -access_log jsonrpc_virtual_machine::do_log_reset_uarch(const access_log::type &log_type) { +access_log jsonrpc_machine::do_log_reset_uarch(const access_log::type &log_type) { not_default_constructible result; request("machine.log_reset_uarch", std::tie(log_type), result); if (!result.has_value()) { @@ -734,12 +766,20 @@ access_log jsonrpc_virtual_machine::do_log_reset_uarch(const access_log::type &l return std::move(result).value(); } -void jsonrpc_virtual_machine::do_get_root_hash(hash_type &hash) const { +machine_hash jsonrpc_machine::do_get_root_hash() const { + machine_hash hash; request("machine.get_root_hash", std::tie(), hash); + return hash; +} + +machine_hash jsonrpc_machine::do_get_node_hash(uint64_t address, int log2_size) const { + machine_hash hash; + request("machine.get_node_hash", std::tie(address, log2_size), hash); + return hash; } -machine_merkle_tree::proof_type jsonrpc_virtual_machine::do_get_proof(uint64_t address, int log2_size) const { - not_default_constructible result; +hash_tree_proof jsonrpc_machine::do_get_proof(uint64_t address, int log2_size) const { + not_default_constructible result; request("machine.get_proof", std::tie(address, log2_size), result); if (!result.has_value()) { throw std::runtime_error("jsonrpc server error: missing result"); @@ -747,12 +787,12 @@ machine_merkle_tree::proof_type jsonrpc_virtual_machine::do_get_proof(uint64_t a return std::move(result).value(); } -void jsonrpc_virtual_machine::do_replace_memory_range(const memory_range_config &new_range) { +void jsonrpc_machine::do_replace_memory_range(const memory_range_config &new_range) { bool result = false; request("machine.replace_memory_range", std::tie(new_range), result); } -access_log jsonrpc_virtual_machine::do_log_step_uarch(const access_log::type &log_type) { +access_log jsonrpc_machine::do_log_step_uarch(const access_log::type &log_type) { not_default_constructible result; request("machine.log_step_uarch", std::tie(log_type), result); if (!result.has_value()) { @@ -761,51 +801,63 @@ access_log jsonrpc_virtual_machine::do_log_step_uarch(const access_log::type &lo return std::move(result).value(); } -void jsonrpc_virtual_machine::do_destroy() { +void jsonrpc_machine::do_destroy() { bool result = false; request("machine.destroy", std::tie(), result); } -bool jsonrpc_virtual_machine::do_verify_dirty_page_maps() const { - bool result = false; - request("machine.verify_dirty_page_maps", std::tie(), result); +uint64_t jsonrpc_machine::do_read_word(uint64_t address) const { + uint64_t result = 0; + request("machine.read_word", std::tie(address), result); return result; } -uint64_t jsonrpc_virtual_machine::do_read_word(uint64_t address) const { - uint64_t result = 0; - request("machine.read_word", std::tie(address), result); +void jsonrpc_machine::do_write_word(uint64_t address, uint64_t value) { + bool result = false; + request("machine.write_word", std::tie(address, value), result); +} + +hash_tree_stats jsonrpc_machine::do_get_hash_tree_stats(bool clear) { + hash_tree_stats result{}; + request("machine.get_hash_tree_stats", std::tie(clear), result); return result; } -bool jsonrpc_virtual_machine::do_verify_merkle_tree() const { +bool jsonrpc_machine::do_verify_hash_tree() const { bool result = false; - request("machine.verify_merkle_tree", std::tie(), result); + request("machine.verify_hash_tree", std::tie(), result); return result; } -uarch_interpreter_break_reason jsonrpc_virtual_machine::do_run_uarch(uint64_t uarch_cycle_end) { +uarch_interpreter_break_reason jsonrpc_machine::do_run_uarch(uint64_t uarch_cycle_end) { uarch_interpreter_break_reason result = uarch_interpreter_break_reason::reached_target_cycle; request("machine.run_uarch", std::tie(uarch_cycle_end), result); return result; } -machine_memory_range_descrs jsonrpc_virtual_machine::do_get_memory_ranges() const { - machine_memory_range_descrs result; - request("machine.get_memory_ranges", std::tie(), result); +uarch_cycle_root_hashes jsonrpc_machine::do_collect_uarch_cycle_root_hashes(uint64_t mcycle_end, + int32_t log2_bundle_uarch_cycle_count) { + uarch_cycle_root_hashes result; + request("machine.collect_uarch_cycle_root_hashes", std::tie(mcycle_end, log2_bundle_uarch_cycle_count), result); + return result; +} + +address_range_descriptions jsonrpc_machine::do_get_address_ranges() const { + address_range_descriptions result; + request("machine.get_address_ranges", std::tie(), result); return result; } -void jsonrpc_virtual_machine::do_send_cmio_response(uint16_t reason, const unsigned char *data, uint64_t length) { +void jsonrpc_machine::do_send_cmio_response(uint16_t reason, const unsigned char *data, uint64_t length) { bool result = false; - std::string b64 = cartesi::encode_base64(data, length); + std::string b64 = cartesi::encode_base64(std::span{data, length}); request("machine.send_cmio_response", std::tie(reason, b64), result); } -access_log jsonrpc_virtual_machine::do_log_send_cmio_response(uint16_t reason, const unsigned char *data, - uint64_t length, const access_log::type &log_type) { +access_log jsonrpc_machine::do_log_send_cmio_response(uint16_t reason, const unsigned char *data, uint64_t length, + const access_log::type &log_type) { not_default_constructible result; - std::string b64 = cartesi::encode_base64(data, length); + std::string b64 = cartesi::encode_base64(std::span{data, length}); request("machine.log_send_cmio_response", std::tie(reason, b64, log_type), result); if (!result.has_value()) { throw std::runtime_error("jsonrpc server error: missing result"); @@ -813,20 +865,20 @@ access_log jsonrpc_virtual_machine::do_log_send_cmio_response(uint16_t reason, c return std::move(result).value(); } -uint64_t jsonrpc_virtual_machine::do_get_reg_address(reg r) const { +uint64_t jsonrpc_machine::do_get_reg_address(reg r) const { uint64_t result = 0; request("machine.get_reg_address", std::tie(r), result); return result; } -machine_config jsonrpc_virtual_machine::do_get_default_config() const { +machine_config jsonrpc_machine::do_get_default_config() const { machine_config result; request("machine.get_default_config", std::tie(), result); return result; } -interpreter_break_reason jsonrpc_virtual_machine::do_verify_step(const hash_type &root_hash_before, - const std::string &log_filename, uint64_t mcycle_count, const hash_type &root_hash_after) const { +interpreter_break_reason jsonrpc_machine::do_verify_step(const machine_hash &root_hash_before, + const std::string &log_filename, uint64_t mcycle_count, const machine_hash &root_hash_after) const { interpreter_break_reason result = interpreter_break_reason::failed; auto b64_root_hash_before = encode_base64(root_hash_before); auto b64_root_hash_after = encode_base64(root_hash_after); @@ -835,33 +887,33 @@ interpreter_break_reason jsonrpc_virtual_machine::do_verify_step(const hash_type return result; } -void jsonrpc_virtual_machine::do_verify_step_uarch(const hash_type &root_hash_before, const access_log &log, - const hash_type &root_hash_after) const { +void jsonrpc_machine::do_verify_step_uarch(const machine_hash &root_hash_before, const access_log &log, + const machine_hash &root_hash_after) const { bool result = false; auto b64_root_hash_before = encode_base64(root_hash_before); auto b64_root_hash_after = encode_base64(root_hash_after); request("machine.verify_step_uarch", std::tie(b64_root_hash_before, log, b64_root_hash_after), result); } -void jsonrpc_virtual_machine::do_verify_reset_uarch(const hash_type &root_hash_before, const access_log &log, - const hash_type &root_hash_after) const { +void jsonrpc_machine::do_verify_reset_uarch(const machine_hash &root_hash_before, const access_log &log, + const machine_hash &root_hash_after) const { bool result = false; auto b64_root_hash_before = encode_base64(root_hash_before); auto b64_root_hash_after = encode_base64(root_hash_after); request("machine.verify_reset_uarch", std::tie(b64_root_hash_before, log, b64_root_hash_after), result); } -void jsonrpc_virtual_machine::do_verify_send_cmio_response(uint16_t reason, const unsigned char *data, uint64_t length, - const hash_type &root_hash_before, const access_log &log, const hash_type &root_hash_after) const { +void jsonrpc_machine::do_verify_send_cmio_response(uint16_t reason, const unsigned char *data, uint64_t length, + const machine_hash &root_hash_before, const access_log &log, const machine_hash &root_hash_after) const { bool result = false; - std::string b64_data = cartesi::encode_base64(data, length); + std::string b64_data = cartesi::encode_base64(std::span{data, length}); auto b64_root_hash_before = encode_base64(root_hash_before); auto b64_root_hash_after = encode_base64(root_hash_after); request("machine.verify_send_cmio_response", std::tie(reason, b64_data, b64_root_hash_before, log, b64_root_hash_after), result); } -bool jsonrpc_virtual_machine::do_is_jsonrpc_virtual_machine() const { +bool jsonrpc_machine::do_is_jsonrpc_machine() const { return true; } diff --git a/src/jsonrpc-virtual-machine.h b/src/jsonrpc-machine.h similarity index 66% rename from src/jsonrpc-virtual-machine.h rename to src/jsonrpc-machine.h index eae517539..d98dbb903 100644 --- a/src/jsonrpc-virtual-machine.h +++ b/src/jsonrpc-machine.h @@ -14,60 +14,67 @@ // with this program (see COPYING). If not, see . // -#ifndef JSONRPC_VIRTUAL_MACHINE_H -#define JSONRPC_VIRTUAL_MACHINE_H +#ifndef JSONRPC_MACHINE_H +#define JSONRPC_MACHINE_H +#include #include #include +#include #include +#include #include "access-log.h" -#include "i-virtual-machine.h" +#include "address-range-description.h" +#include "back-merkle-tree.h" +#include "hash-tree-proof.h" +#include "hash-tree-stats.h" +#include "i-machine.h" #include "interpret.h" #include "jsonrpc-fork-result.h" #include "machine-config.h" -#include "machine-memory-range-descr.h" -#include "machine-merkle-tree.h" +#include "machine-hash.h" #include "machine-runtime-config.h" +#include "mcycle-root-hashes.h" #include "semantic-version.h" +#include "uarch-cycle-root-hashes.h" #include "uarch-interpret.h" #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdeprecated-declarations" #include #include -#include #pragma GCC diagnostic pop namespace cartesi { -/// \class jsonrpc_virtual_machine -/// \brief JSONRPC implementation of the i_virtual_machine interface -class jsonrpc_virtual_machine final : public i_virtual_machine { +/// \class jsonrpc_machine +/// \brief JSONRPC implementation of the i_machine interface +class jsonrpc_machine final : public i_machine { public: enum class cleanup_call { nothing, destroy, shutdown }; /// \brief Constructor that connects to existing JSONRPC server and performs a sanity check on its version /// \param address Address of the server in "host:port" format to connect to /// \param connect_timeout_ms Timeout in milliseconds for the connection attempt - explicit jsonrpc_virtual_machine(std::string address, int64_t connect_timeout_ms); + explicit jsonrpc_machine(std::string address, int64_t connect_timeout_ms); /// \brief Constructor that connects to existing JSONRPC server without version verification /// \param address Address of the server in "host:port" format to connect to - explicit jsonrpc_virtual_machine(std::string address); + explicit jsonrpc_machine(std::string address); /// \brief Constructor that spawns a new server /// \param address The address to bind the server to /// \param spawned Output parameter that will contain information about the spawned server - jsonrpc_virtual_machine(const std::string &address, int64_t spawn_timeout_ms, fork_result &spawned); + jsonrpc_machine(const std::string &address, int64_t spawn_timeout_ms, fork_result &spawned); // no copies or assignments - jsonrpc_virtual_machine(const jsonrpc_virtual_machine &other) = delete; - jsonrpc_virtual_machine(jsonrpc_virtual_machine &&other) noexcept = delete; - jsonrpc_virtual_machine &operator=(const jsonrpc_virtual_machine &other) = delete; - jsonrpc_virtual_machine &operator=(jsonrpc_virtual_machine &&other) noexcept = delete; + jsonrpc_machine(const jsonrpc_machine &other) = delete; + jsonrpc_machine(jsonrpc_machine &&other) noexcept = delete; + jsonrpc_machine &operator=(const jsonrpc_machine &other) = delete; + jsonrpc_machine &operator=(jsonrpc_machine &&other) noexcept = delete; - ~jsonrpc_virtual_machine() override; + ~jsonrpc_machine() override; /// \brief Asks remote server to shutdown void shutdown_server(); @@ -103,14 +110,18 @@ class jsonrpc_virtual_machine final : public i_virtual_machine { const std::string &get_server_address() const; private: - machine_config do_get_initial_config() const override; - i_virtual_machine *do_clone_empty() const override; + i_machine *do_clone_empty() const override; bool do_is_empty() const override; - void do_create(const machine_config &config, const machine_runtime_config &runtime) override; - void do_load(const std::string &directory, const machine_runtime_config &runtime) override; + void do_create(const machine_config &config, const machine_runtime_config &runtime, + const std::string &dir) override; + void do_load(const std::string &directory, const machine_runtime_config &runtime, sharing_mode sharing) override; interpreter_break_reason do_run(uint64_t mcycle_end) override; + mcycle_root_hashes do_collect_mcycle_root_hashes(uint64_t mcycle_end, uint64_t mcycle_period, uint64_t mcycle_phase, + int32_t log2_bundle_mcycle_count, const std::optional &previous_back_tree) override; interpreter_break_reason do_log_step(uint64_t mcycle_count, const std::string &filename) override; - void do_store(const std::string &dir) const override; + void do_store(const std::string &dir, sharing_mode sharing) const override; + void do_clone_stored(const std::string &from_dir, const std::string &to_dir) const override; + void do_remove_stored(const std::string &dir) const override; uint64_t do_read_reg(reg r) const override; void do_write_reg(reg w, uint64_t val) override; void do_read_memory(uint64_t address, unsigned char *data, uint64_t length) const override; @@ -120,32 +131,38 @@ class jsonrpc_virtual_machine final : public i_virtual_machine { uint64_t do_translate_virtual_address(uint64_t vaddr) override; void do_reset_uarch() override; access_log do_log_reset_uarch(const access_log::type &log_type) override; - void do_get_root_hash(hash_type &hash) const override; - machine_merkle_tree::proof_type do_get_proof(uint64_t address, int log2_size) const override; + machine_hash do_get_root_hash() const override; + machine_hash do_get_node_hash(uint64_t address, int log2_size) const override; + hash_tree_proof do_get_proof(uint64_t address, int log2_size) const override; void do_replace_memory_range(const memory_range_config &new_range) override; access_log do_log_step_uarch(const access_log::type &log_type) override; machine_runtime_config do_get_runtime_config() const override; void do_set_runtime_config(const machine_runtime_config &r) override; void do_destroy() override; - bool do_verify_dirty_page_maps() const override; uint64_t do_read_word(uint64_t address) const override; - bool do_verify_merkle_tree() const override; + void do_write_word(uint64_t address, uint64_t value) override; + machine_config do_get_initial_config() const override; + hash_tree_stats do_get_hash_tree_stats(bool clear) override; + bool do_verify_hash_tree() const override; uarch_interpreter_break_reason do_run_uarch(uint64_t uarch_cycle_end) override; - machine_memory_range_descrs do_get_memory_ranges() const override; + uarch_cycle_root_hashes do_collect_uarch_cycle_root_hashes(uint64_t mcycle_end, + int32_t log2_bundle_uarch_cycle_count) override; + address_range_descriptions do_get_address_ranges() const override; void do_send_cmio_response(uint16_t reason, const unsigned char *data, uint64_t length) override; access_log do_log_send_cmio_response(uint16_t reason, const unsigned char *data, uint64_t length, const access_log::type &log_type) override; uint64_t do_get_reg_address(reg r) const override; machine_config do_get_default_config() const override; - interpreter_break_reason do_verify_step(const hash_type &root_hash_before, const std::string &log_filename, - uint64_t mcycle_count, const hash_type &root_hash_after) const override; - void do_verify_step_uarch(const hash_type &root_hash_before, const access_log &log, - const hash_type &root_hash_after) const override; - void do_verify_reset_uarch(const hash_type &root_hash_before, const access_log &log, - const hash_type &root_hash_after) const override; + interpreter_break_reason do_verify_step(const machine_hash &root_hash_before, const std::string &log_filename, + uint64_t mcycle_count, const machine_hash &root_hash_after) const override; + void do_verify_step_uarch(const machine_hash &root_hash_before, const access_log &log, + const machine_hash &root_hash_after) const override; + void do_verify_reset_uarch(const machine_hash &root_hash_before, const access_log &log, + const machine_hash &root_hash_after) const override; void do_verify_send_cmio_response(uint16_t reason, const unsigned char *data, uint64_t length, - const hash_type &root_hash_before, const access_log &log, const hash_type &root_hash_after) const override; - bool do_is_jsonrpc_virtual_machine() const override; + const machine_hash &root_hash_before, const access_log &log, + const machine_hash &root_hash_after) const override; + bool do_is_jsonrpc_machine() const override; void check_server_version(std::chrono::time_point timeout_at) const; template @@ -163,4 +180,4 @@ class jsonrpc_virtual_machine final : public i_virtual_machine { } // namespace cartesi -#endif // JSONRPC_VIRTUAL_MACHINE_H +#endif // JSONRPC_MACHINE_H diff --git a/src/jsonrpc-remote-machine.cpp b/src/jsonrpc-remote-machine.cpp index 99473ff60..7ded32e1e 100644 --- a/src/jsonrpc-remote-machine.cpp +++ b/src/jsonrpc-remote-machine.cpp @@ -32,7 +32,10 @@ #include #include #include +#include #include +#include +#include #include #include #include @@ -46,34 +49,33 @@ #include #include -#include +#include #include -#include -#include +#include // IWYU pragma: keep #include #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#pragma GCC diagnostic ignored "-Wunused-but-set-variable" #include -#include -#include +#include // IWYU pragma: keep +#include // IWYU pragma: keep #include #pragma GCC diagnostic pop #include #include "access-log.h" +#include "back-merkle-tree.h" #include "base64.h" -#include "interpret.h" #include "json-util.h" #include "jsonrpc-discover.h" +#include "jsonrpc-fork-result.h" #include "jsonrpc-version.h" #include "machine-config.h" -#include "machine-merkle-tree.h" +#include "machine-hash.h" #include "machine-runtime-config.h" #include "machine.h" -#include "os.h" -#include "uarch-interpret.h" #include "unique-c-ptr.h" #define SLOG_PREFIX log_prefix @@ -167,7 +169,7 @@ struct http_session : std::enable_shared_from_this { // Create a new request parser req_parser = std::make_unique>(); req_parser->eager(true); - req_parser->body_limit(16777216U); // can receive up to 16MB + req_parser->body_limit(std::numeric_limits::max()); // can receive unlimited amount of data // Read a request http::async_read(stream, buffer, *req_parser, @@ -362,9 +364,7 @@ struct http_handler : std::enable_shared_from_this { auto session = std::make_shared(std::move(socket), shared_from_this()); // Remove previous expired sessions - sessions.erase(std::remove_if(sessions.begin(), sessions.end(), - [](const std::weak_ptr &weak_session) { return weak_session.expired(); }), - sessions.end()); + std::erase_if(sessions, [](const std::weak_ptr &weak_session) { return weak_session.expired(); }); // Keep track of the new session sessions.push_back(session); @@ -380,7 +380,7 @@ struct http_handler : std::enable_shared_from_this { //------------------------------------------------------------------------------ /// \brief Names for JSONRPC error codes -enum jsonrpc_error_code : int { +enum class jsonrpc_error_code : int { parse_error = -32700, ///< When the request failed to parse invalid_request = -32600, ///< When the request was invalid (missing fields, wrong types etc) method_not_found = -32601, ///< When the method was not found @@ -460,7 +460,7 @@ static json jsonrpc_response_invalid_params(const json &j, const std::string &me static void jsonrpc_check_allowed_fields(const json &j, const std::unordered_set &keys, const std::string &base = "params/") { for (const auto &[key, val] : j.items()) { - if (keys.find(key) == keys.end()) { + if (!keys.contains(key)) { // NOLINTNEXTLINE(performance-inefficient-string-concatenation) throw std::invalid_argument("unexpected field \"/"s + base + key + "\""s); } @@ -647,7 +647,7 @@ static std::tuple parse_array_args(const json &j) { /// \param param_name Name of each parameter /// \returns tuple with arguments template -static std::tuple parse_object_args(const json &j, const char *(¶m_name)[sizeof...(ARGS)], +static std::tuple parse_object_args(const json &j, const char *const (¶m_name)[sizeof...(ARGS)], std::index_sequence /*unused*/) { std::tuple tp; (cartesi::ju_get_field(j, std::string(param_name[I]), std::get(tp)), ...); @@ -660,7 +660,7 @@ static std::tuple parse_object_args(const json &j, const char *(¶m_ /// \param param_name Name of each parameter /// \returns tuple with arguments template -static std::tuple parse_object_args(const json &j, const char *(¶m_name)[sizeof...(ARGS)]) { +static std::tuple parse_object_args(const json &j, const char *const (¶m_name)[sizeof...(ARGS)]) { return parse_object_args(j, param_name, std::make_index_sequence{}); } @@ -670,7 +670,7 @@ static std::tuple parse_object_args(const json &j, const char *(¶m_ /// \param param_name Name of each parameter /// \returns tuple with arguments template -static std::tuple parse_args(const json &j, const char *(¶m_name)[sizeof...(ARGS)]) { +static std::tuple parse_args(const json &j, const char *const (¶m_name)[sizeof...(ARGS)]) { constexpr auto mandatory_params = count_mandatory_params(); if (!j.contains("params")) { if constexpr (mandatory_params == 0) { @@ -757,7 +757,7 @@ static tcp::endpoint address_to_endpoint(const std::string &address) { throw std::runtime_error{"invalid port"}; } return {asio::ip::make_address(ip), static_cast(port)}; - } catch (std::exception &e) { + } catch (const std::exception &e) { throw std::runtime_error{"invalid endpoint address \"" + address + "\""}; } } @@ -779,6 +779,9 @@ static std::string endpoint_to_string(const tcp::endpoint &endpoint) { /// server. static json jsonrpc_fork_handler(const json &j, const std::shared_ptr &session) { jsonrpc_check_no_params(j); + if (session->handler->machine && session->handler->machine->has_shared_address_range()) { + throw std::domain_error("cannot fork machines with shared address ranges"); + } // Listen in desired port before fork so failures happen still in parent, // who can directly report them to client tcp::acceptor acceptor{session->handler->ioc, tcp::endpoint{session->handler->local_endpoint.address(), 0}}; @@ -822,7 +825,7 @@ static json jsonrpc_fork_handler(const json &j, const std::shared_ptr &session) { - static const char *param_name[] = {"address"}; + static const char *const param_name[] = {"address"}; auto args = parse_args(j, param_name); const std::string new_server_address = std::get<0>(args); const tcp::endpoint new_local_endpoint = address_to_endpoint(new_server_address); @@ -845,8 +848,9 @@ static json jsonrpc_machine_load_handler(const json &j, const std::shared_ptrhandler->machine) { return jsonrpc_response_invalid_request(j, "machine exists"); } - static const char *param_name[] = {"directory", "runtime_config"}; - auto args = parse_args>(j, param_name); + static const char *const param_name[] = {"directory", "runtime_config", "sharing"}; + auto args = parse_args, + cartesi::optional_param>(j, param_name); switch (count_args(args)) { case 1: session->handler->machine = std::make_unique(std::get<0>(args)); @@ -855,6 +859,11 @@ static json jsonrpc_machine_load_handler(const json &j, const std::shared_ptrhandler->machine = std::make_unique(std::get<0>(args), std::get<1>(args).value()); // NOLINT(bugprone-unchecked-optional-access) break; + case 3: + session->handler->machine = std::make_unique(std::get<0>(args), + std::get<1>(args).value(), // NOLINT(bugprone-unchecked-optional-access) + std::get<2>(args).value()); // NOLINT(bugprone-unchecked-optional-access) + break; default: throw std::runtime_error{"error detecting number of arguments"}; } @@ -869,9 +878,9 @@ static json jsonrpc_machine_create_handler(const json &j, const std::shared_ptr< if (session->handler->machine) { return jsonrpc_response_invalid_request(j, "machine exists"); } - static const char *param_name[] = {"config", "runtime_config"}; - auto args = - parse_args>(j, param_name); + static const char *const param_name[] = {"config", "runtime_config", "directory"}; + auto args = parse_args, + cartesi::optional_param>(j, param_name); switch (count_args(args)) { case 1: session->handler->machine = std::make_unique(std::get<0>(args)); @@ -880,12 +889,57 @@ static json jsonrpc_machine_create_handler(const json &j, const std::shared_ptr< session->handler->machine = std::make_unique(std::get<0>(args), std::get<1>(args).value()); // // NOLINT(bugprone-unchecked-optional-access) break; + case 3: + session->handler->machine = std::make_unique(std::get<0>(args), + std::get<1>(args).value(), // NOLINT(bugprone-unchecked-optional-access) + std::get<2>(args).value()); // NOLINT(bugprone-unchecked-optional-access) + break; default: throw std::runtime_error{"error detecting number of arguments"}; } return jsonrpc_response_ok(j); } +/// \brief JSONRPC handler for the machine.collect_mcycle_root_hashes method +/// \param j JSON request object +/// \param session HTTP session +/// \returns JSON response object +static json jsonrpc_machine_collect_mcycle_root_hashes(const json &j, const std::shared_ptr &session) { + if (!session->handler->machine) { + return jsonrpc_response_invalid_request(j, "no machine"); + } + static const char *const param_name[] = {"mcycle_end", "mcycle_period", "mcycle_phase", "log2_bundle_mcycle_count", + "previous_back_tree"}; + auto args = + parse_args>(j, param_name); + auto mcycle_end = std::get<0>(args); + auto mcycle_period = std::get<1>(args); + auto mcycle_phase = std::get<2>(args); + auto log2_bundle_mcycle_count = std::get<3>(args); + auto previous_back_tree = std::get<4>(args); + const auto result = session->handler->machine->collect_mcycle_root_hashes(mcycle_end, mcycle_period, mcycle_phase, + static_cast(log2_bundle_mcycle_count), previous_back_tree); + return jsonrpc_response_ok(j, result); +} + +/// \brief JSONRPC handler for the machine.collect_uarch_cycle_root_hashes method +/// \param j JSON request object +/// \param session HTTP session +/// \returns JSON response object +static json jsonrpc_machine_collect_uarch_cycle_root_hashes(const json &j, + const std::shared_ptr &session) { + if (!session->handler->machine) { + return jsonrpc_response_invalid_request(j, "no machine"); + } + static const char *const param_name[] = {"mcycle_end", "log2_bundle_uarch_cycle_count"}; + auto args = parse_args(j, param_name); + auto mcycle_end = std::get<0>(args); + auto log2_bundle_uarch_cycle_count = std::get<1>(args); + const auto result = session->handler->machine->collect_uarch_cycle_root_hashes(mcycle_end, + static_cast(log2_bundle_uarch_cycle_count)); + return jsonrpc_response_ok(j, result); +} + /// \brief JSONRPC handler for the machine.destroy method /// \param j JSON request object /// \param session HTTP session @@ -902,7 +956,7 @@ static json jsonrpc_machine_destroy_handler(const json &j, const std::shared_ptr /// \returns JSON response object /// \details This method causes the server to sleep for a number of milliseconds before the next call static json jsonrpc_delay_next_request_handler(const json &j, const std::shared_ptr &session) { - static const char *param_name[] = {"ms"}; + static const char *const param_name[] = {"ms"}; auto args = parse_args(j, param_name); session->handler->delay = std::get<0>(args); return jsonrpc_response_ok(j); @@ -935,32 +989,38 @@ static json jsonrpc_machine_store_handler(const json &j, const std::shared_ptrhandler->machine) { return jsonrpc_response_invalid_request(j, "no machine"); } - static const char *param_name[] = {"directory"}; - auto args = parse_args(j, param_name); - session->handler->machine->store(std::get<0>(args)); + static const char *const param_name[] = {"directory", "sharing"}; + auto args = parse_args(j, param_name); + session->handler->machine->store(std::get<0>(args), std::get<1>(args)); return jsonrpc_response_ok(j); } -/// \brief Translate an interpret_break_reason value to string -/// \param reason interpret_break_reason value to translate -/// \returns String representation of value -static std::string interpreter_break_reason_name(cartesi::interpreter_break_reason reason) { - using R = cartesi::interpreter_break_reason; - switch (reason) { - case R::failed: - return "failed"; - case R::halted: - return "halted"; - case R::yielded_manually: - return "yielded_manually"; - case R::yielded_automatically: - return "yielded_automatically"; - case R::yielded_softly: - return "yielded_softly"; - case R::reached_target_mcycle: - return "reached_target_mcycle"; - } - throw std::domain_error{"invalid interpreter break reason"}; +/// \brief JSONRPC handler for the machine.clone_stored method +/// \param j JSON request object +/// \param session HTTP session +/// \returns JSON response object +static json jsonrpc_machine_clone_stored_handler(const json &j, const std::shared_ptr &session) { + if (!session->handler->machine) { + return jsonrpc_response_invalid_request(j, "no machine"); + } + static const char *const param_name[] = {"from_dir", "to_dir"}; + auto args = parse_args(j, param_name); + cartesi::machine::clone_stored(std::get<0>(args), std::get<1>(args)); + return jsonrpc_response_ok(j); +} + +/// \brief JSONRPC handler for the machine.remove_stored method +/// \param j JSON request object +/// \param session HTTP session +/// \returns JSON response object +static json jsonrpc_machine_remove_stored_handler(const json &j, const std::shared_ptr &session) { + if (!session->handler->machine) { + return jsonrpc_response_invalid_request(j, "no machine"); + } + static const char *const param_name[] = {"dir"}; + auto args = parse_args(j, param_name); + cartesi::machine::remove_stored(std::get<0>(args)); + return jsonrpc_response_ok(j); } /// \brief JSONRPC handler for the machine.run method @@ -971,10 +1031,10 @@ static json jsonrpc_machine_run_handler(const json &j, const std::shared_ptrhandler->machine) { return jsonrpc_response_invalid_request(j, "no machine"); } - static const char *param_name[] = {"mcycle_end"}; + static const char *const param_name[] = {"mcycle_end"}; auto args = parse_args(j, param_name); auto reason = session->handler->machine->run(std::get<0>(args)); - return jsonrpc_response_ok(j, interpreter_break_reason_name(reason)); + return jsonrpc_response_ok(j, reason); } /// \brief JSONRPC handler for the machine.log_step method @@ -985,24 +1045,10 @@ static json jsonrpc_machine_log_step_handler(const json &j, const std::shared_pt if (!session->handler->machine) { return jsonrpc_response_invalid_request(j, "no machine"); } - static const char *param_name[] = {"mcycle_count", "filename"}; + static const char *const param_name[] = {"mcycle_count", "filename"}; auto args = parse_args(j, param_name); auto reason = session->handler->machine->log_step(std::get<0>(args), std::get<1>(args)); - return jsonrpc_response_ok(j, interpreter_break_reason_name(reason)); -} - -/// \brief Translate an uarch_interpret_break_reason value to string -/// \param reason uarch_interpret_break_reason value to translate -/// \returns String representation of value -static std::string uarch_interpreter_break_reason_name(cartesi::uarch_interpreter_break_reason reason) { - using R = cartesi::uarch_interpreter_break_reason; - switch (reason) { - case R::uarch_halted: - return "uarch_halted"; - case R::reached_target_cycle: - return "reached_target_cycle"; - } - throw std::domain_error{"invalid uarch interpreter break reason"}; + return jsonrpc_response_ok(j, reason); } /// \brief JSONRPC handler for the machine.run_uarch method @@ -1013,10 +1059,10 @@ static json jsonrpc_machine_run_uarch_handler(const json &j, const std::shared_p if (!session->handler->machine) { return jsonrpc_response_invalid_request(j, "no machine"); } - static const char *param_name[] = {"uarch_cycle_end"}; + static const char *const param_name[] = {"uarch_cycle_end"}; auto args = parse_args(j, param_name); auto reason = session->handler->machine->run_uarch(std::get<0>(args)); - return jsonrpc_response_ok(j, uarch_interpreter_break_reason_name(reason)); + return jsonrpc_response_ok(j, reason); } /// \brief JSONRPC handler for the machine.log_step_uarch method @@ -1027,7 +1073,7 @@ static json jsonrpc_machine_log_step_uarch_handler(const json &j, const std::sha if (!session->handler->machine) { return jsonrpc_response_invalid_request(j, "no machine"); } - static const char *param_name[] = {"log_type"}; + static const char *const param_name[] = {"log_type"}; auto args = parse_args>(j, param_name); // NOLINTNEXTLINE(bugprone-unchecked-optional-access) return jsonrpc_response_ok(j, session->handler->machine->log_step_uarch(std::get<0>(args).value())); @@ -1041,7 +1087,7 @@ static json jsonrpc_machine_log_reset_uarch_handler(const json &j, const std::sh if (!session->handler->machine) { return jsonrpc_response_invalid_request(j, "no machine"); } - static const char *param_name[] = {"log_type"}; + static const char *const param_name[] = {"log_type"}; auto args = parse_args>(j, param_name); // NOLINTNEXTLINE(bugprone-unchecked-optional-access) return jsonrpc_response_ok(j, session->handler->machine->log_reset_uarch(std::get<0>(args).value())); @@ -1053,13 +1099,12 @@ static json jsonrpc_machine_log_reset_uarch_handler(const json &j, const std::sh /// \returns JSON response object static json jsonrpc_machine_verify_step_handler(const json &j, const std::shared_ptr &session) { (void) session; - static const char *param_name[] = {"root_hash_before", "filename", "mcycle_count", "root_hash_after"}; - auto args = parse_args(j, param_name); + static const char *const param_name[] = {"root_hash_before", "filename", "mcycle_count", "root_hash_after"}; + auto args = parse_args(j, param_name); // NOLINTNEXTLINE(bugprone-unchecked-optional-access) auto reason = cartesi::machine::verify_step(std::get<0>(args), std::get<1>(args), std::get<2>(args), std::get<3>(args)); - return jsonrpc_response_ok(j, interpreter_break_reason_name(reason)); + return jsonrpc_response_ok(j, reason); } /// \brief JSONRPC handler for the machine.verify_step_uarch method @@ -1068,10 +1113,9 @@ static json jsonrpc_machine_verify_step_handler(const json &j, const std::shared /// \returns JSON response object static json jsonrpc_machine_verify_step_uarch_handler(const json &j, const std::shared_ptr & /*session*/) { - static const char *param_name[] = {"root_hash_before", "log", "root_hash_after"}; - auto args = - parse_args, - cartesi::machine_merkle_tree::hash_type>(j, param_name); + static const char *const param_name[] = {"root_hash_before", "log", "root_hash_after"}; + auto args = parse_args, + cartesi::machine_hash>(j, param_name); // NOLINTNEXTLINE(bugprone-unchecked-optional-access) cartesi::machine::verify_step_uarch(std::get<0>(args), std::get<1>(args).value(), std::get<2>(args)); return jsonrpc_response_ok(j); @@ -1083,10 +1127,9 @@ static json jsonrpc_machine_verify_step_uarch_handler(const json &j, /// \returns JSON response object static json jsonrpc_machine_verify_reset_uarch_handler(const json &j, const std::shared_ptr & /*session*/) { - static const char *param_name[] = {"root_hash_before", "log", "root_hash_after"}; - auto args = - parse_args, - cartesi::machine_merkle_tree::hash_type>(j, param_name); + static const char *const param_name[] = {"root_hash_before", "log", "root_hash_after"}; + auto args = parse_args, + cartesi::machine_hash>(j, param_name); // NOLINTNEXTLINE(bugprone-unchecked-optional-access) cartesi::machine::verify_reset_uarch(std::get<0>(args), std::get<1>(args).value(), std::get<2>(args)); return jsonrpc_response_ok(j); @@ -1100,7 +1143,7 @@ static json jsonrpc_machine_get_proof_handler(const json &j, const std::shared_p if (!session->handler->machine) { return jsonrpc_response_invalid_request(j, "no machine"); } - static const char *param_name[] = {"address", "log2_size"}; + static const char *const param_name[] = {"address", "log2_size"}; auto args = parse_args(j, param_name); if (std::get<1>(args) > INT_MAX) { throw std::domain_error("log2_size is out of range"); @@ -1109,16 +1152,29 @@ static json jsonrpc_machine_get_proof_handler(const json &j, const std::shared_p session->handler->machine->get_proof(std::get<0>(args), static_cast(std::get<1>(args)))); } -/// \brief JSONRPC handler for the machine.verify_merkle_tree method +/// \brief JSONRPC handler for the machine.get_hash_tree_stats method +/// \param j JSON request object +/// \param session HTTP session +/// \returns JSON response object +static json jsonrpc_machine_get_hash_tree_stats_handler(const json &j, const std::shared_ptr &session) { + if (!session->handler->machine) { + return jsonrpc_response_invalid_request(j, "no machine"); + } + static const char *const param_name[] = {"clear"}; + auto args = parse_args(j, param_name); + return jsonrpc_response_ok(j, session->handler->machine->get_hash_tree_stats(std::get<0>(args))); +} + +/// \brief JSONRPC handler for the machine.verify_hash_tree method /// \param j JSON request object /// \param session HTTP session /// \returns JSON response object -static json jsonrpc_machine_verify_merkle_tree_handler(const json &j, const std::shared_ptr &session) { +static json jsonrpc_machine_verify_hash_tree_handler(const json &j, const std::shared_ptr &session) { if (!session->handler->machine) { return jsonrpc_response_invalid_request(j, "no machine"); } jsonrpc_check_no_params(j); - return jsonrpc_response_ok(j, session->handler->machine->verify_merkle_tree()); + return jsonrpc_response_ok(j, session->handler->machine->verify_hash_tree()); } /// \brief JSONRPC handler for the machine.get_root_hash method @@ -1130,9 +1186,25 @@ static json jsonrpc_machine_get_root_hash_handler(const json &j, const std::shar return jsonrpc_response_invalid_request(j, "no machine"); } jsonrpc_check_no_params(j); - cartesi::machine_merkle_tree::hash_type hash; - session->handler->machine->get_root_hash(hash); - return jsonrpc_response_ok(j, cartesi::encode_base64(hash)); + auto hash = session->handler->machine->get_root_hash(); + return jsonrpc_response_ok(j, cartesi::base64_machine_hash(hash)); +} + +/// \brief JSONRPC handler for the machine.get_node_hash method +/// \param j JSON request object +/// \param session HTTP session +/// \returns JSON response object +static json jsonrpc_machine_get_node_hash_handler(const json &j, const std::shared_ptr &session) { + if (!session->handler->machine) { + return jsonrpc_response_invalid_request(j, "no machine"); + } + static const char *const param_name[] = {"address", "log2_size"}; + auto args = parse_args(j, param_name); + if (std::get<1>(args) > INT_MAX) { + throw std::domain_error("log2_size is out of range"); + } + auto hash = session->handler->machine->get_node_hash(std::get<0>(args), static_cast(std::get<1>(args))); + return jsonrpc_response_ok(j, cartesi::base64_machine_hash(hash)); } /// \brief JSONRPC handler for the machine.read_word method @@ -1143,12 +1215,28 @@ static json jsonrpc_machine_read_word_handler(const json &j, const std::shared_p if (!session->handler->machine) { return jsonrpc_response_invalid_request(j, "no machine"); } - static const char *param_name[] = {"address"}; + static const char *const param_name[] = {"address"}; auto args = parse_args(j, param_name); auto address = std::get<0>(args); return jsonrpc_response_ok(j, session->handler->machine->read_word(address)); } +/// \brief JSONRPC handler for the machine.write_word method +/// \param j JSON request object +/// \param session HTTP session +/// \returns JSON response object +static json jsonrpc_machine_write_word_handler(const json &j, const std::shared_ptr &session) { + if (!session->handler->machine) { + return jsonrpc_response_invalid_request(j, "no machine"); + } + static const char *const param_name[] = {"address", "value"}; + auto args = parse_args(j, param_name); + auto address = std::get<0>(args); + auto value = std::get<1>(args); + session->handler->machine->write_word(address, value); + return jsonrpc_response_ok(j); +} + /// \brief JSONRPC handler for the machine.read_memory method /// \param j JSON request object /// \param session HTTP session @@ -1157,13 +1245,13 @@ static json jsonrpc_machine_read_memory_handler(const json &j, const std::shared if (!session->handler->machine) { return jsonrpc_response_invalid_request(j, "no machine"); } - static const char *param_name[] = {"address", "length"}; + static const char *const param_name[] = {"address", "length"}; auto args = parse_args(j, param_name); auto address = std::get<0>(args); auto length = std::get<1>(args); - auto data = cartesi::unique_calloc(length); + auto data = cartesi::make_unique_calloc(length); session->handler->machine->read_memory(address, data.get(), length); - return jsonrpc_response_ok(j, cartesi::encode_base64(data.get(), length)); + return jsonrpc_response_ok(j, cartesi::encode_base64(std::span{data.get(), length})); } /// \brief JSONRPC handler for the machine.write_memory method @@ -1174,7 +1262,7 @@ static json jsonrpc_machine_write_memory_handler(const json &j, const std::share if (!session->handler->machine) { return jsonrpc_response_invalid_request(j, "no machine"); } - static const char *param_name[] = {"address", "data"}; + static const char *const param_name[] = {"address", "data"}; auto args = parse_args(j, param_name); auto address = std::get<0>(args); auto bin = cartesi::decode_base64(std::get<1>(args)); @@ -1191,13 +1279,13 @@ static json jsonrpc_machine_read_virtual_memory_handler(const json &j, const std if (!session->handler->machine) { return jsonrpc_response_invalid_request(j, "no machine"); } - static const char *param_name[] = {"address", "length"}; + static const char *const param_name[] = {"address", "length"}; auto args = parse_args(j, param_name); auto address = std::get<0>(args); auto length = std::get<1>(args); - auto data = cartesi::unique_calloc(length); + auto data = cartesi::make_unique_calloc(length); session->handler->machine->read_virtual_memory(address, data.get(), length); - return jsonrpc_response_ok(j, cartesi::encode_base64(data.get(), length)); + return jsonrpc_response_ok(j, cartesi::encode_base64(std::span{data.get(), length})); } /// \brief JSONRPC handler for the machine.write_virtual_memory method @@ -1208,7 +1296,7 @@ static json jsonrpc_machine_write_virtual_memory_handler(const json &j, const st if (!session->handler->machine) { return jsonrpc_response_invalid_request(j, "no machine"); } - static const char *param_name[] = {"address", "data"}; + static const char *const param_name[] = {"address", "data"}; auto args = parse_args(j, param_name); auto address = std::get<0>(args); auto bin = cartesi::decode_base64(std::get<1>(args)); @@ -1226,7 +1314,7 @@ static json jsonrpc_machine_translate_virtual_address_handler(const json &j, if (!session->handler->machine) { return jsonrpc_response_invalid_request(j, "no machine"); } - static const char *param_name[] = {"vaddr"}; + static const char *const param_name[] = {"vaddr"}; auto args = parse_args(j, param_name); auto vaddr = std::get<0>(args); return jsonrpc_response_ok(j, session->handler->machine->translate_virtual_address(vaddr)); @@ -1240,7 +1328,7 @@ static json jsonrpc_machine_replace_memory_range_handler(const json &j, const st if (!session->handler->machine) { return jsonrpc_response_invalid_request(j, "no machine"); } - static const char *param_name[] = {"range"}; + static const char *const param_name[] = {"range"}; auto args = parse_args(j, param_name); session->handler->machine->replace_memory_range(std::get<0>(args)); return jsonrpc_response_ok(j); @@ -1254,7 +1342,7 @@ static json jsonrpc_machine_read_reg_handler(const json &j, const std::shared_pt if (!session->handler->machine) { return jsonrpc_response_invalid_request(j, "no machine"); } - static const char *param_name[] = {"reg"}; + static const char *const param_name[] = {"reg"}; auto args = parse_args(j, param_name); return jsonrpc_response_ok(j, session->handler->machine->read_reg(std::get<0>(args))); } @@ -1267,7 +1355,7 @@ static json jsonrpc_machine_write_reg_handler(const json &j, const std::shared_p if (!session->handler->machine) { return jsonrpc_response_invalid_request(j, "no machine"); } - static const char *param_name[] = {"reg", "value"}; + static const char *const param_name[] = {"reg", "value"}; auto args = parse_args(j, param_name); session->handler->machine->write_reg(std::get<0>(args), std::get<1>(args)); return jsonrpc_response_ok(j); @@ -1278,7 +1366,7 @@ static json jsonrpc_machine_write_reg_handler(const json &j, const std::shared_p /// \param session HTTP session /// \returns JSON response object static json jsonrpc_machine_get_reg_address_handler(const json &j, const std::shared_ptr & /*session*/) { - static const char *param_name[] = {"reg"}; + static const char *const param_name[] = {"reg"}; auto args = parse_args(j, param_name); return jsonrpc_response_ok(j, cartesi::machine::get_reg_address(std::get<0>(args))); } @@ -1328,7 +1416,7 @@ static json jsonrpc_machine_set_runtime_config_handler(const json &j, const std: if (!session->handler->machine) { return jsonrpc_response_invalid_request(j, "no machine"); } - static const char *param_name[] = {"runtime_config"}; + static const char *const param_name[] = {"runtime_config"}; auto args = parse_args(j, param_name); session->handler->machine->set_runtime_config(std::get<0>(args)); return jsonrpc_response_ok(j); @@ -1344,29 +1432,16 @@ static json jsonrpc_machine_get_default_config_handler(const json &j, return jsonrpc_response_ok(j, cartesi::machine::get_default_config()); } -/// \brief JSONRPC handler for the machine.verify_dirty_page_maps method -/// \param j JSON request object -/// \param session HTTP session -/// \returns JSON response object -static json jsonrpc_machine_verify_dirty_page_maps_handler(const json &j, - const std::shared_ptr &session) { - if (!session->handler->machine) { - return jsonrpc_response_invalid_request(j, "no machine"); - } - jsonrpc_check_no_params(j); - return jsonrpc_response_ok(j, session->handler->machine->verify_dirty_page_maps()); -} - -/// \brief JSONRPC handler for the machine.get_memory_ranges method +/// \brief JSONRPC handler for the machine.get_address_ranges method /// \param j JSON request object /// \param session HTTP session /// \returns JSON response object -static json jsonrpc_machine_get_memory_ranges_handler(const json &j, const std::shared_ptr &session) { +static json jsonrpc_machine_get_address_ranges_handler(const json &j, const std::shared_ptr &session) { if (!session->handler->machine) { return jsonrpc_response_invalid_request(j, "no machine"); } jsonrpc_check_no_params(j); - return jsonrpc_response_ok(j, session->handler->machine->get_memory_ranges()); + return jsonrpc_response_ok(j, session->handler->machine->get_address_ranges()); } /// \brief JSONRPC handler for the machine.send_cmio_response method @@ -1377,7 +1452,7 @@ static json jsonrpc_machine_send_cmio_response_handler(const json &j, const std: if (!session->handler->machine) { return jsonrpc_response_invalid_request(j, "no machine"); } - static const char *param_name[] = {"reason", "data"}; + static const char *const param_name[] = {"reason", "data"}; auto args = parse_args(j, param_name); auto bin = cartesi::decode_base64(std::get<1>(args)); // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) @@ -1391,7 +1466,7 @@ static json jsonrpc_machine_log_send_cmio_response_handler(const json &j, if (!session->handler->machine) { return jsonrpc_response_invalid_request(j, "no machine"); } - static const char *param_name[] = {"reason", "data", "log_type"}; + static const char *const param_name[] = {"reason", "data", "log_type"}; auto args = parse_args>(j, param_name); auto bin = cartesi::decode_base64(std::get<1>(args)); @@ -1410,10 +1485,9 @@ static json jsonrpc_machine_log_send_cmio_response_handler(const json &j, /// \returns JSON response object static json jsonrpc_machine_verify_send_cmio_response_handler(const json &j, const std::shared_ptr & /*session*/) { - static const char *param_name[] = {"reason", "data", "root_hash_before", "log", "root_hash_after"}; - auto args = parse_args, cartesi::machine_merkle_tree::hash_type>(j, - param_name); + static const char *const param_name[] = {"reason", "data", "root_hash_before", "log", "root_hash_after"}; + auto args = parse_args, cartesi::machine_hash>(j, param_name); auto bin = cartesi::decode_base64(std::get<1>(args)); // NOLINTBEGIN(bugprone-unchecked-optional-access) @@ -1475,11 +1549,15 @@ static json jsonrpc_dispatch_method(const json &j, const std::shared_ptrsecond(j, session); } return jsonrpc_response_method_not_found(j, method); -} catch (std::invalid_argument &x) { +} catch (const std::invalid_argument &x) { return jsonrpc_response_invalid_params(j, x.what()); -} catch (std::exception &x) { +} catch (const std::exception &x) { return jsonrpc_response_internal_error(j, x.what()); } @@ -1573,7 +1653,7 @@ http::message_generator handle_request(HTTP_REQ &&rreq, const std::shared_ptr. +// + +#include "keccak-256-hasher.h" + +#include +#include +#include +#include +#include +#include + +#include "array2d.h" +#include "compiler-defines.h" +#include "i-hasher.h" +#include "keccakf.h" +#include "machine-hash.h" +#include "simd-vector-type.h" + +namespace cartesi { + +// This code is not portable to big-endian architectures. +// NOLINTNEXTLINE(misc-redundant-expression) +static_assert(std::endian::native == std::endian::little, "code assumes little-endian byte ordering"); + +constexpr size_t KECCAK_WORD_COUNT = 25; +constexpr size_t KECCAK_RSIZE = (KECCAK_WORD_COUNT * sizeof(uint64_t)) - (static_cast(2) * MACHINE_HASH_SIZE); + +template +struct alignas(uint64_vector_type::align) keccak_256_context final { + using word_vector_type = uint64_vector_type::type; + using word_bytes_array = uint8_t[KECCAK_WORD_COUNT][LaneCount][sizeof(uint64_t)]; + using data_span = std::span; + + static constexpr size_t word_vector_align = uint64_vector_type::align; // 64-bit m_words + + word_vector_type m_words[5][5]{}; ///< Buffer for Keccak-256 words, interleaved by lanes + + FORCE_INLINE void update(const std::array &data, size_t &pos) noexcept { + [[maybe_unused]] auto words_bytes = // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + reinterpret_cast(m_words); + // Assume all data spans have the same length + const size_t data_len = data[0].size(); + for (size_t i = 0; i < data_len;) { + // Interleave data while XORing + const size_t step = std::min(KECCAK_RSIZE - pos, data_len - i); + if constexpr (DataExtent != std::dynamic_extent && DataExtent % sizeof(uint64_t) == 0) { + // If data length is a multiple of word size, process a word at time + UNROLL_LOOP(128) + for (size_t j = 0; j < step; j += sizeof(uint64_t)) { + word_vector_type data_word; + UNROLL_LOOP_FULL() + for (size_t l = 0; l < LaneCount; ++l) { + uint64_t lane_word{}; + __builtin_memcpy(&lane_word, &data[l][i + j], sizeof(lane_word)); + data_word[l] = lane_word; + } + m_words[((pos + j) / sizeof(uint64_t)) / 5][((pos + j) / sizeof(uint64_t)) % 5] ^= data_word; + } + } else { // Otherwise, process a byte at time + UNROLL_LOOP(128) + for (size_t j = 0; j < step; j++) { + const size_t bi = (pos + j) / sizeof(uint64_t); + const size_t bj = (pos + j) % sizeof(uint64_t); + UNROLL_LOOP_FULL() + for (size_t l = 0; l < LaneCount; ++l) { + words_bytes[bi][l][bj] ^= data[l][i + j]; + } + } + } + i += step; + pos += step; + // Perform Keccak-256 permutation + if (pos >= KECCAK_RSIZE) [[unlikely]] { + keccakf_1600(m_words); + pos = 0; + } + } + } + + FORCE_INLINE void finish(const std::array &hashes, size_t pos) noexcept { + // Append delimiter suffix + constexpr uint64_t KECCAK_DSUFFIX = 0x01; + const size_t dsuffix_word_pos = pos / sizeof(uint64_t); + const size_t dsuffix_byte_pos = pos % sizeof(uint64_t); + m_words[dsuffix_word_pos / 5][dsuffix_word_pos % 5] ^= KECCAK_DSUFFIX << (dsuffix_byte_pos * 8); + // Append last bit + constexpr uint64_t KECCAK_LASTBIT = 0x80; + constexpr size_t lastbit_word_pos = (KECCAK_RSIZE - 1) / sizeof(uint64_t); + constexpr size_t lastbit_byte_pos = (KECCAK_RSIZE - 1) % sizeof(uint64_t); + m_words[lastbit_word_pos / 5][lastbit_word_pos % 5] ^= KECCAK_LASTBIT << (lastbit_byte_pos * 8); + // Perform last permutation + keccakf_1600(m_words); + // Deinterleave hash + UNROLL_LOOP_FULL() + for (size_t l = 0; l < LaneCount; ++l) { + UNROLL_LOOP_FULL() + for (size_t i = 0; i < MACHINE_HASH_SIZE; i += sizeof(uint64_t)) { + const uint64_t word = m_words[0][i / sizeof(uint64_t)][l]; + __builtin_memcpy(&hashes[l][i], &word, sizeof(uint64_t)); + } + } + } + template + FORCE_INLINE static void simd_concat_hash(array2d data, + std::array hashes) noexcept { + keccak_256_context ctx; + // Position is kept local to allow the compiler optimize it out when DataExtent is a compile time constant. + size_t pos = 0; + UNROLL_LOOP(4) + for (size_t i = 0; i < ConcatCount; ++i) { + ctx.update(data[i], pos); + } + ctx.finish(hashes, pos); + } +}; + +// Generic implementations + +MULTIVERSION_GENERIC size_t keccak_256_get_optimal_lane_count() noexcept { +#if defined(__x86_64__) + // On AMD64, SSE2 has 128-bit registers, supporting up to 2 lanes. + return 2; +#elif defined(__aarch64__) + // On ARM64, NEON has 128-bit registers, supporting up to 2 lanes. + return 2; +#elif defined(__riscv) && defined(__riscv_v) + // RISC-V with Vector extension, we assume 128-bit registers are available, supporting up to 2 lanes. + return 2; +#elif defined(__wasm_simd128__) + // WebAssembly with SIMD extension has 128-bit registers, supporting up to 2 lanes. + return 2; +#else + // For other architectures, we assume vector instructions are not available and use scalar implementation. + return 1; +#endif +} + +MULTIVERSION_GENERIC void keccak_256_data_1x1(const array2d, 1, 1> &data, + const std::array &hash) noexcept { + keccak_256_context<1>::simd_concat_hash<1>(data, hash); +} +MULTIVERSION_GENERIC void keccak_256_word_1x1(const array2d &data, + const std::array &hash) noexcept { + keccak_256_context<1, const_hash_tree_word_view::extent>::simd_concat_hash(data, hash); +} +MULTIVERSION_GENERIC void keccak_256_word_1x2(const array2d &data, + const std::array &hash) noexcept { + keccak_256_context<2, const_hash_tree_word_view::extent>::simd_concat_hash(data, hash); +} +MULTIVERSION_GENERIC void keccak_256_word_1x4(const array2d &data, + const std::array &hash) noexcept { + keccak_256_context<4, const_hash_tree_word_view::extent>::simd_concat_hash(data, hash); +} +MULTIVERSION_GENERIC void keccak_256_word_1x8(const array2d &data, + const std::array &hash) noexcept { + keccak_256_context<8, const_hash_tree_word_view::extent>::simd_concat_hash(data, hash); +} +MULTIVERSION_GENERIC void keccak_256_data_2x1(const array2d, 2, 1> &data, + const std::array &hash) noexcept { + keccak_256_context<1>::simd_concat_hash(data, hash); +} +MULTIVERSION_GENERIC void keccak_256_hash_2x1(const array2d &data, + const std::array &hash) noexcept { + keccak_256_context<1, const_machine_hash_view::extent>::simd_concat_hash(data, hash); +} +MULTIVERSION_GENERIC void keccak_256_hash_2x2(const array2d &data, + const std::array &hash) noexcept { + keccak_256_context<2, const_machine_hash_view::extent>::simd_concat_hash(data, hash); +} +MULTIVERSION_GENERIC void keccak_256_hash_2x4(const array2d &data, + const std::array &hash) noexcept { + keccak_256_context<4, const_machine_hash_view::extent>::simd_concat_hash(data, hash); +} +MULTIVERSION_GENERIC void keccak_256_hash_2x8(const array2d &data, + const std::array &hash) noexcept { + keccak_256_context<8, const_machine_hash_view::extent>::simd_concat_hash(data, hash); +} + +// x86_64 implementations + +#ifdef USE_MULTIVERSINING_AMD64 + +// AVX2 +MULTIVERSION_AMD64_AVX2_BMI_BMI2 size_t keccak_256_get_optimal_lane_count() noexcept { + // AVX2 has 256-bit registers, supporting up to 4 lanes. + return 4; +} +MULTIVERSION_AMD64_AVX2_BMI_BMI2 void keccak_256_data_1x1(const array2d, 1, 1> &data, + const std::array &hash) noexcept { + keccak_256_context<1>::simd_concat_hash<1>(data, hash); +} +MULTIVERSION_AMD64_AVX2_BMI_BMI2 void keccak_256_word_1x1(const array2d &data, + const std::array &hash) noexcept { + keccak_256_context<1, const_hash_tree_word_view::extent>::simd_concat_hash(data, hash); +} +MULTIVERSION_AMD64_AVX2_BMI_BMI2 void keccak_256_word_1x2(const array2d &data, + const std::array &hash) noexcept { + keccak_256_context<2, const_hash_tree_word_view::extent>::simd_concat_hash(data, hash); +} +MULTIVERSION_AMD64_AVX2_BMI_BMI2 void keccak_256_word_1x4(const array2d &data, + const std::array &hash) noexcept { + keccak_256_context<4, const_hash_tree_word_view::extent>::simd_concat_hash(data, hash); +} +MULTIVERSION_AMD64_AVX2_BMI_BMI2 void keccak_256_word_1x8(const array2d &data, + const std::array &hash) noexcept { + keccak_256_context<8, const_hash_tree_word_view::extent>::simd_concat_hash(data, hash); +} +MULTIVERSION_AMD64_AVX2_BMI_BMI2 void keccak_256_data_2x1(const array2d, 2, 1> &data, + const std::array &hash) noexcept { + keccak_256_context<1>::simd_concat_hash(data, hash); +} +MULTIVERSION_AMD64_AVX2_BMI_BMI2 void keccak_256_hash_2x1(const array2d &data, + const std::array &hash) noexcept { + keccak_256_context<1, const_machine_hash_view::extent>::simd_concat_hash(data, hash); +} +MULTIVERSION_AMD64_AVX2_BMI_BMI2 void keccak_256_hash_2x2(const array2d &data, + const std::array &hash) noexcept { + keccak_256_context<2, const_machine_hash_view::extent>::simd_concat_hash(data, hash); +} +MULTIVERSION_AMD64_AVX2_BMI_BMI2 void keccak_256_hash_2x4(const array2d &data, + const std::array &hash) noexcept { + keccak_256_context<4, const_machine_hash_view::extent>::simd_concat_hash(data, hash); +} +MULTIVERSION_AMD64_AVX2_BMI_BMI2 void keccak_256_hash_2x8(const array2d &data, + const std::array &hash) noexcept { + keccak_256_context<8, const_machine_hash_view::extent>::simd_concat_hash(data, hash); +} + +// AVX-512 + +MULTIVERSION_AMD64_AVX512_BMI_BMI2 size_t keccak_256_get_optimal_lane_count() noexcept { + // AVX-512 has 512-bit registers, supporting up to 8 lanes. + return 8; +} +MULTIVERSION_AMD64_AVX512_BMI_BMI2 void keccak_256_word_1x8(const array2d &data, + const std::array &hash) noexcept { + keccak_256_context<8, const_hash_tree_word_view::extent>::simd_concat_hash(data, hash); +} +MULTIVERSION_AMD64_AVX512_BMI_BMI2 void keccak_256_hash_2x8(const array2d &data, + const std::array &hash) noexcept { + keccak_256_context<8, const_machine_hash_view::extent>::simd_concat_hash(data, hash); +} + +#endif // USE_MULTIVERSINING_AMD64 + +} // namespace cartesi diff --git a/src/keccak-256-hasher.h b/src/keccak-256-hasher.h index a649d8d29..df96a0bde 100644 --- a/src/keccak-256-hasher.h +++ b/src/keccak-256-hasher.h @@ -17,60 +17,258 @@ #ifndef KECCAK_256_HASHER_H #define KECCAK_256_HASHER_H +#include #include -#include -#include +#include +#include "array2d.h" +#include "compiler-defines.h" +#include "hash-tree-constants.h" #include "i-hasher.h" - -extern "C" { -#include "sha3.h" -} +#include "machine-hash.h" namespace cartesi { -struct keccak_instance final { - union { - uint8_t b[200]; - uint64_t q[25]; - } st; - int pt; -}; - -class keccak_256_hasher final : public i_hasher> { - sha3_ctx_t m_ctx{}; - - friend i_hasher>; - - void do_begin() { - sha3_init(&m_ctx, 32, 0x01); - } +// Generic implementations +MULTIVERSION_GENERIC size_t keccak_256_get_optimal_lane_count() noexcept; +MULTIVERSION_GENERIC void keccak_256_data_1x1(const array2d, 1, 1> &data, + const std::array &hash) noexcept; +MULTIVERSION_GENERIC void keccak_256_word_1x1(const array2d &data, + const std::array &hash) noexcept; +MULTIVERSION_GENERIC void keccak_256_word_1x2(const array2d &data, + const std::array &hash) noexcept; +MULTIVERSION_GENERIC void keccak_256_word_1x4(const array2d &data, + const std::array &hash) noexcept; +MULTIVERSION_GENERIC void keccak_256_word_1x8(const array2d &data, + const std::array &hash) noexcept; +MULTIVERSION_GENERIC void keccak_256_data_2x1(const array2d, 2, 1> &data, + const std::array &hash) noexcept; +MULTIVERSION_GENERIC void keccak_256_hash_2x1(const array2d &data, + const std::array &hash) noexcept; +MULTIVERSION_GENERIC void keccak_256_hash_2x2(const array2d &data, + const std::array &hash) noexcept; +MULTIVERSION_GENERIC void keccak_256_hash_2x4(const array2d &data, + const std::array &hash) noexcept; +MULTIVERSION_GENERIC void keccak_256_hash_2x8(const array2d &data, + const std::array &hash) noexcept; - void do_add_data(const unsigned char *data, size_t length) { - sha3_update(&m_ctx, data, length); - } - - void do_end(hash_type &hash) { - sha3_final(hash.data(), &m_ctx); - } +// Optimized implementation for x86_64 architecture leveraging modern CPU instruction sets: +// - BMI1/BMI2 (Bit Manipulation Instructions) provide specialized bit operations: +// * RORX performs optimized bitwise rotation without requiring separate shift operations +// * ANDN efficiently computes (~x & y) in a single instruction +// - AVX2 for x4 SIMD hashing +// - AVX-512 for x8 SIMD hashing +#ifdef USE_MULTIVERSINING_AMD64 +// AVX2 implementation for x1, x2, x4, x8 SIMD hashing +MULTIVERSION_AMD64_AVX2_BMI_BMI2 size_t keccak_256_get_optimal_lane_count() noexcept; +MULTIVERSION_AMD64_AVX2_BMI_BMI2 void keccak_256_data_1x1(const array2d, 1, 1> &data, + const std::array &hash) noexcept; +MULTIVERSION_AMD64_AVX2_BMI_BMI2 void keccak_256_word_1x1(const array2d &data, + const std::array &hash) noexcept; +MULTIVERSION_AMD64_AVX2_BMI_BMI2 void keccak_256_word_1x2(const array2d &data, + const std::array &hash) noexcept; +MULTIVERSION_AMD64_AVX2_BMI_BMI2 void keccak_256_word_1x4(const array2d &data, + const std::array &hash) noexcept; +MULTIVERSION_AMD64_AVX2_BMI_BMI2 void keccak_256_word_1x8(const array2d &data, + const std::array &hash) noexcept; +MULTIVERSION_AMD64_AVX2_BMI_BMI2 void keccak_256_data_2x1(const array2d, 2, 1> &data, + const std::array &hash) noexcept; +MULTIVERSION_AMD64_AVX2_BMI_BMI2 void keccak_256_hash_2x1(const array2d &data, + const std::array &hash) noexcept; +MULTIVERSION_AMD64_AVX2_BMI_BMI2 void keccak_256_hash_2x2(const array2d &data, + const std::array &hash) noexcept; +MULTIVERSION_AMD64_AVX2_BMI_BMI2 void keccak_256_hash_2x4(const array2d &data, + const std::array &hash) noexcept; +MULTIVERSION_AMD64_AVX2_BMI_BMI2 void keccak_256_hash_2x8(const array2d &data, + const std::array &hash) noexcept; +// AVX-512 implementation for x8 SIMD hashing +MULTIVERSION_AMD64_AVX512_BMI_BMI2 size_t keccak_256_get_optimal_lane_count() noexcept; +MULTIVERSION_AMD64_AVX512_BMI_BMI2 void keccak_256_word_1x8(const array2d &data, + const std::array &hash) noexcept; +MULTIVERSION_AMD64_AVX512_BMI_BMI2 void keccak_256_hash_2x8(const array2d &data, + const std::array &hash) noexcept; +#endif +class keccak_256_hasher final : public i_hasher { public: - /// \brief Default constructor keccak_256_hasher() = default; - /// \brief Default destructor - ~keccak_256_hasher() = default; + static constexpr int MAX_LANE_COUNT = 8; + + template + static void do_simd_concat_hash(const array2d, ConcatCount, LaneCount> &data, + const std::array &hash) noexcept; - /// \brief No copy constructor - keccak_256_hasher(const keccak_256_hasher &) = delete; - /// \brief No move constructor - keccak_256_hasher(keccak_256_hasher &&) = delete; - /// \brief No copy assignment - keccak_256_hasher &operator=(const keccak_256_hasher &) = delete; - /// \brief No move assignment - keccak_256_hasher &operator=(keccak_256_hasher &&) = delete; + static size_t do_get_optimal_lane_count() noexcept { + return keccak_256_get_optimal_lane_count(); + } }; +template <> +inline void keccak_256_hasher::do_simd_concat_hash<1, 1, std::dynamic_extent>( + const array2d, 1, 1> &data, const std::array &hash) noexcept { + keccak_256_data_1x1(data, hash); +} +template <> +inline void keccak_256_hasher::do_simd_concat_hash<1, 1, HASH_TREE_WORD_SIZE>( + const array2d &data, const std::array &hash) noexcept { + keccak_256_word_1x1(data, hash); +} +template <> +inline void keccak_256_hasher::do_simd_concat_hash<1, 2, HASH_TREE_WORD_SIZE>( + const array2d &data, const std::array &hash) noexcept { + keccak_256_word_1x2(data, hash); +} +template <> +inline void keccak_256_hasher::do_simd_concat_hash<1, 4, HASH_TREE_WORD_SIZE>( + const array2d &data, const std::array &hash) noexcept { + keccak_256_word_1x4(data, hash); +} +template <> +inline void keccak_256_hasher::do_simd_concat_hash<1, 8, HASH_TREE_WORD_SIZE>( + const array2d &data, const std::array &hash) noexcept { + keccak_256_word_1x8(data, hash); +} +template <> +inline void keccak_256_hasher::do_simd_concat_hash<1, 16, HASH_TREE_WORD_SIZE>( + const array2d &data, const std::array &hash) noexcept { + // Keccak-256 does not support 16-way parallelism, we simulate it by splitting it into two 8-way hashes + keccak_256_word_1x8(array2d{{{ + data[0][0], + data[0][1], + data[0][2], + data[0][3], + data[0][4], + data[0][5], + data[0][6], + data[0][7], + }}}, + std::array{ + hash[0], + hash[1], + hash[2], + hash[3], + hash[4], + hash[5], + hash[6], + hash[7], + }); + keccak_256_word_1x8(array2d{{{ + data[0][8], + data[0][9], + data[0][10], + data[0][11], + data[0][12], + data[0][13], + data[0][14], + data[0][15], + }}}, + std::array{ + hash[8], + hash[9], + hash[10], + hash[11], + hash[12], + hash[13], + hash[14], + hash[15], + }); +} +template <> +inline void keccak_256_hasher::do_simd_concat_hash<2, 1, std::dynamic_extent>( + const array2d, 2, 1> &data, const std::array &hash) noexcept { + keccak_256_data_2x1(data, hash); +} +template <> +inline void keccak_256_hasher::do_simd_concat_hash<2, 1, MACHINE_HASH_SIZE>( + const array2d &data, const std::array &hash) noexcept { + keccak_256_hash_2x1(data, hash); +} +template <> +inline void keccak_256_hasher::do_simd_concat_hash<2, 2, MACHINE_HASH_SIZE>( + const array2d &data, const std::array &hash) noexcept { + keccak_256_hash_2x2(data, hash); +} +template <> +inline void keccak_256_hasher::do_simd_concat_hash<2, 4, MACHINE_HASH_SIZE>( + const array2d &data, const std::array &hash) noexcept { + keccak_256_hash_2x4(data, hash); +} +template <> +inline void keccak_256_hasher::do_simd_concat_hash<2, 8, MACHINE_HASH_SIZE>( + const array2d &data, const std::array &hash) noexcept { + keccak_256_hash_2x8(data, hash); +} +template <> +inline void keccak_256_hasher::do_simd_concat_hash<2, 16, MACHINE_HASH_SIZE>( + const array2d &data, const std::array &hash) noexcept { + // Keccak-256 does not support 16-way parallelism, we simulate it by splitting it into two 8-way hashes + keccak_256_hash_2x8(array2d{{ + { + data[0][0], + data[0][1], + data[0][2], + data[0][3], + data[0][4], + data[0][5], + data[0][6], + data[0][7], + }, + { + data[1][0], + data[1][1], + data[1][2], + data[1][3], + data[1][4], + data[1][5], + data[1][6], + data[1][7], + }, + }}, + std::array{ + hash[0], + hash[1], + hash[2], + hash[3], + hash[4], + hash[5], + hash[6], + hash[7], + }); + keccak_256_hash_2x8(array2d{{ + { + data[0][8], + data[0][9], + data[0][10], + data[0][11], + data[0][12], + data[0][13], + data[0][14], + data[0][15], + }, + { + data[1][8], + data[1][9], + data[1][10], + data[1][11], + data[1][12], + data[1][13], + data[1][14], + data[1][15], + }, + }}, + std::array{ + hash[8], + hash[9], + hash[10], + hash[11], + hash[12], + hash[13], + hash[14], + hash[15], + }); +} + } // namespace cartesi #endif diff --git a/src/keccakf.h b/src/keccakf.h new file mode 100644 index 000000000..9a4f41274 --- /dev/null +++ b/src/keccakf.h @@ -0,0 +1,2573 @@ +// Copyright Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: LGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify it under +// the terms of the GNU Lesser General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) any +// later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +// PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License along +// with this program (see COPYING). If not, see . +// + +#ifndef KECCAKF_H +#define KECCAKF_H + +#include +#include + +#include "compiler-defines.h" + +namespace cartesi { + +template +// NOLINTNEXTLINE(readability-function-size,hicpp-function-size,google-readability-function-size) +FORCE_INLINE static void keccakf_1600(word_vector_type A[5][5]) noexcept { + // This code is adapted from the XKCP generic64 Keccak implementation (public domain). + // Original source: + // https://github.com/XKCP/XKCP/blob/master/lib/low/KeccakP-1600/plain-64bits/KeccakP-1600-opt64.c + // Selected for its 64-bit optimization, vectorization-friendly structure, branchless design, and efficient + // memory bandwidth. + alignas(word_vector_align) static constexpr uint64_t KECCAKF1600_ROUND_CONSTANTS[24] = {0x0000000000000001, + 0x0000000000008082, 0x800000000000808a, 0x8000000080008000, 0x000000000000808b, 0x0000000080000001, + 0x8000000080008081, 0x8000000000008009, 0x000000000000008a, 0x0000000000000088, 0x0000000080008009, + 0x000000008000000a, 0x000000008000808b, 0x800000000000008b, 0x8000000000008089, 0x8000000000008003, + 0x8000000000008002, 0x8000000000000080, 0x000000000000800a, 0x800000008000000a, 0x8000000080008081, + 0x8000000000008080, 0x0000000080000001, 0x8000000080008008}; + word_vector_type B[5][5]; + word_vector_type C[5]; + word_vector_type D[5]; + word_vector_type E[5][5]; + C[0] = A[0][0] ^ A[1][0] ^ A[2][0] ^ A[3][0] ^ A[4][0]; + C[1] = A[0][1] ^ A[1][1] ^ A[2][1] ^ A[3][1] ^ A[4][1]; + C[2] = A[0][2] ^ A[1][2] ^ A[2][2] ^ A[3][2] ^ A[4][2]; + C[3] = A[0][3] ^ A[1][3] ^ A[2][3] ^ A[3][3] ^ A[4][3]; + C[4] = A[0][4] ^ A[1][4] ^ A[2][4] ^ A[3][4] ^ A[4][4]; + D[0] = C[4] ^ (((C[1]) << 1) ^ ((C[1]) >> (64 - 1))); + D[1] = C[0] ^ (((C[2]) << 1) ^ ((C[2]) >> (64 - 1))); + D[2] = C[1] ^ (((C[3]) << 1) ^ ((C[3]) >> (64 - 1))); + D[3] = C[2] ^ (((C[4]) << 1) ^ ((C[4]) >> (64 - 1))); + D[4] = C[3] ^ (((C[0]) << 1) ^ ((C[0]) >> (64 - 1))); + A[0][0] ^= D[0]; + B[0][0] = A[0][0]; + A[1][1] ^= D[1]; + B[0][1] = (((A[1][1]) << 44) ^ ((A[1][1]) >> (64 - 44))); + A[2][2] ^= D[2]; + B[0][2] = (((A[2][2]) << 43) ^ ((A[2][2]) >> (64 - 43))); + A[3][3] ^= D[3]; + B[0][3] = (((A[3][3]) << 21) ^ ((A[3][3]) >> (64 - 21))); + A[4][4] ^= D[4]; + B[0][4] = (((A[4][4]) << 14) ^ ((A[4][4]) >> (64 - 14))); + E[0][0] = B[0][0] ^ ((~B[0][1]) & B[0][2]); + E[0][0] ^= KECCAKF1600_ROUND_CONSTANTS[0]; + C[0] = E[0][0]; + E[0][1] = B[0][1] ^ ((~B[0][2]) & B[0][3]); + C[1] = E[0][1]; + E[0][2] = B[0][2] ^ ((~B[0][3]) & B[0][4]); + C[2] = E[0][2]; + E[0][3] = B[0][3] ^ ((~B[0][4]) & B[0][0]); + C[3] = E[0][3]; + E[0][4] = B[0][4] ^ ((~B[0][0]) & B[0][1]); + C[4] = E[0][4]; + A[0][3] ^= D[3]; + B[1][0] = (((A[0][3]) << 28) ^ ((A[0][3]) >> (64 - 28))); + A[1][4] ^= D[4]; + B[1][1] = (((A[1][4]) << 20) ^ ((A[1][4]) >> (64 - 20))); + A[2][0] ^= D[0]; + B[1][2] = (((A[2][0]) << 3) ^ ((A[2][0]) >> (64 - 3))); + A[3][1] ^= D[1]; + B[1][3] = (((A[3][1]) << 45) ^ ((A[3][1]) >> (64 - 45))); + A[4][2] ^= D[2]; + B[1][4] = (((A[4][2]) << 61) ^ ((A[4][2]) >> (64 - 61))); + E[1][0] = B[1][0] ^ ((~B[1][1]) & B[1][2]); + C[0] ^= E[1][0]; + E[1][1] = B[1][1] ^ ((~B[1][2]) & B[1][3]); + C[1] ^= E[1][1]; + E[1][2] = B[1][2] ^ ((~B[1][3]) & B[1][4]); + C[2] ^= E[1][2]; + E[1][3] = B[1][3] ^ ((~B[1][4]) & B[1][0]); + C[3] ^= E[1][3]; + E[1][4] = B[1][4] ^ ((~B[1][0]) & B[1][1]); + C[4] ^= E[1][4]; + A[0][1] ^= D[1]; + B[2][0] = (((A[0][1]) << 1) ^ ((A[0][1]) >> (64 - 1))); + A[1][2] ^= D[2]; + B[2][1] = (((A[1][2]) << 6) ^ ((A[1][2]) >> (64 - 6))); + A[2][3] ^= D[3]; + B[2][2] = (((A[2][3]) << 25) ^ ((A[2][3]) >> (64 - 25))); + A[3][4] ^= D[4]; + B[2][3] = (((A[3][4]) << 8) ^ ((A[3][4]) >> (64 - 8))); + A[4][0] ^= D[0]; + B[2][4] = (((A[4][0]) << 18) ^ ((A[4][0]) >> (64 - 18))); + E[2][0] = B[2][0] ^ ((~B[2][1]) & B[2][2]); + C[0] ^= E[2][0]; + E[2][1] = B[2][1] ^ ((~B[2][2]) & B[2][3]); + C[1] ^= E[2][1]; + E[2][2] = B[2][2] ^ ((~B[2][3]) & B[2][4]); + C[2] ^= E[2][2]; + E[2][3] = B[2][3] ^ ((~B[2][4]) & B[2][0]); + C[3] ^= E[2][3]; + E[2][4] = B[2][4] ^ ((~B[2][0]) & B[2][1]); + C[4] ^= E[2][4]; + A[0][4] ^= D[4]; + B[3][0] = (((A[0][4]) << 27) ^ ((A[0][4]) >> (64 - 27))); + A[1][0] ^= D[0]; + B[3][1] = (((A[1][0]) << 36) ^ ((A[1][0]) >> (64 - 36))); + A[2][1] ^= D[1]; + B[3][2] = (((A[2][1]) << 10) ^ ((A[2][1]) >> (64 - 10))); + A[3][2] ^= D[2]; + B[3][3] = (((A[3][2]) << 15) ^ ((A[3][2]) >> (64 - 15))); + A[4][3] ^= D[3]; + B[3][4] = (((A[4][3]) << 56) ^ ((A[4][3]) >> (64 - 56))); + E[3][0] = B[3][0] ^ ((~B[3][1]) & B[3][2]); + C[0] ^= E[3][0]; + E[3][1] = B[3][1] ^ ((~B[3][2]) & B[3][3]); + C[1] ^= E[3][1]; + E[3][2] = B[3][2] ^ ((~B[3][3]) & B[3][4]); + C[2] ^= E[3][2]; + E[3][3] = B[3][3] ^ ((~B[3][4]) & B[3][0]); + C[3] ^= E[3][3]; + E[3][4] = B[3][4] ^ ((~B[3][0]) & B[3][1]); + C[4] ^= E[3][4]; + A[0][2] ^= D[2]; + B[4][0] = (((A[0][2]) << 62) ^ ((A[0][2]) >> (64 - 62))); + A[1][3] ^= D[3]; + B[4][1] = (((A[1][3]) << 55) ^ ((A[1][3]) >> (64 - 55))); + A[2][4] ^= D[4]; + B[4][2] = (((A[2][4]) << 39) ^ ((A[2][4]) >> (64 - 39))); + A[3][0] ^= D[0]; + B[4][3] = (((A[3][0]) << 41) ^ ((A[3][0]) >> (64 - 41))); + A[4][1] ^= D[1]; + B[4][4] = (((A[4][1]) << 2) ^ ((A[4][1]) >> (64 - 2))); + E[4][0] = B[4][0] ^ ((~B[4][1]) & B[4][2]); + C[0] ^= E[4][0]; + E[4][1] = B[4][1] ^ ((~B[4][2]) & B[4][3]); + C[1] ^= E[4][1]; + E[4][2] = B[4][2] ^ ((~B[4][3]) & B[4][4]); + C[2] ^= E[4][2]; + E[4][3] = B[4][3] ^ ((~B[4][4]) & B[4][0]); + C[3] ^= E[4][3]; + E[4][4] = B[4][4] ^ ((~B[4][0]) & B[4][1]); + C[4] ^= E[4][4]; + D[0] = C[4] ^ (((C[1]) << 1) ^ ((C[1]) >> (64 - 1))); + D[1] = C[0] ^ (((C[2]) << 1) ^ ((C[2]) >> (64 - 1))); + D[2] = C[1] ^ (((C[3]) << 1) ^ ((C[3]) >> (64 - 1))); + D[3] = C[2] ^ (((C[4]) << 1) ^ ((C[4]) >> (64 - 1))); + D[4] = C[3] ^ (((C[0]) << 1) ^ ((C[0]) >> (64 - 1))); + E[0][0] ^= D[0]; + B[0][0] = E[0][0]; + E[1][1] ^= D[1]; + B[0][1] = (((E[1][1]) << 44) ^ ((E[1][1]) >> (64 - 44))); + E[2][2] ^= D[2]; + B[0][2] = (((E[2][2]) << 43) ^ ((E[2][2]) >> (64 - 43))); + E[3][3] ^= D[3]; + B[0][3] = (((E[3][3]) << 21) ^ ((E[3][3]) >> (64 - 21))); + E[4][4] ^= D[4]; + B[0][4] = (((E[4][4]) << 14) ^ ((E[4][4]) >> (64 - 14))); + A[0][0] = B[0][0] ^ ((~B[0][1]) & B[0][2]); + A[0][0] ^= KECCAKF1600_ROUND_CONSTANTS[1]; + C[0] = A[0][0]; + A[0][1] = B[0][1] ^ ((~B[0][2]) & B[0][3]); + C[1] = A[0][1]; + A[0][2] = B[0][2] ^ ((~B[0][3]) & B[0][4]); + C[2] = A[0][2]; + A[0][3] = B[0][3] ^ ((~B[0][4]) & B[0][0]); + C[3] = A[0][3]; + A[0][4] = B[0][4] ^ ((~B[0][0]) & B[0][1]); + C[4] = A[0][4]; + E[0][3] ^= D[3]; + B[1][0] = (((E[0][3]) << 28) ^ ((E[0][3]) >> (64 - 28))); + E[1][4] ^= D[4]; + B[1][1] = (((E[1][4]) << 20) ^ ((E[1][4]) >> (64 - 20))); + E[2][0] ^= D[0]; + B[1][2] = (((E[2][0]) << 3) ^ ((E[2][0]) >> (64 - 3))); + E[3][1] ^= D[1]; + B[1][3] = (((E[3][1]) << 45) ^ ((E[3][1]) >> (64 - 45))); + E[4][2] ^= D[2]; + B[1][4] = (((E[4][2]) << 61) ^ ((E[4][2]) >> (64 - 61))); + A[1][0] = B[1][0] ^ ((~B[1][1]) & B[1][2]); + C[0] ^= A[1][0]; + A[1][1] = B[1][1] ^ ((~B[1][2]) & B[1][3]); + C[1] ^= A[1][1]; + A[1][2] = B[1][2] ^ ((~B[1][3]) & B[1][4]); + C[2] ^= A[1][2]; + A[1][3] = B[1][3] ^ ((~B[1][4]) & B[1][0]); + C[3] ^= A[1][3]; + A[1][4] = B[1][4] ^ ((~B[1][0]) & B[1][1]); + C[4] ^= A[1][4]; + E[0][1] ^= D[1]; + B[2][0] = (((E[0][1]) << 1) ^ ((E[0][1]) >> (64 - 1))); + E[1][2] ^= D[2]; + B[2][1] = (((E[1][2]) << 6) ^ ((E[1][2]) >> (64 - 6))); + E[2][3] ^= D[3]; + B[2][2] = (((E[2][3]) << 25) ^ ((E[2][3]) >> (64 - 25))); + E[3][4] ^= D[4]; + B[2][3] = (((E[3][4]) << 8) ^ ((E[3][4]) >> (64 - 8))); + E[4][0] ^= D[0]; + B[2][4] = (((E[4][0]) << 18) ^ ((E[4][0]) >> (64 - 18))); + A[2][0] = B[2][0] ^ ((~B[2][1]) & B[2][2]); + C[0] ^= A[2][0]; + A[2][1] = B[2][1] ^ ((~B[2][2]) & B[2][3]); + C[1] ^= A[2][1]; + A[2][2] = B[2][2] ^ ((~B[2][3]) & B[2][4]); + C[2] ^= A[2][2]; + A[2][3] = B[2][3] ^ ((~B[2][4]) & B[2][0]); + C[3] ^= A[2][3]; + A[2][4] = B[2][4] ^ ((~B[2][0]) & B[2][1]); + C[4] ^= A[2][4]; + E[0][4] ^= D[4]; + B[3][0] = (((E[0][4]) << 27) ^ ((E[0][4]) >> (64 - 27))); + E[1][0] ^= D[0]; + B[3][1] = (((E[1][0]) << 36) ^ ((E[1][0]) >> (64 - 36))); + E[2][1] ^= D[1]; + B[3][2] = (((E[2][1]) << 10) ^ ((E[2][1]) >> (64 - 10))); + E[3][2] ^= D[2]; + B[3][3] = (((E[3][2]) << 15) ^ ((E[3][2]) >> (64 - 15))); + E[4][3] ^= D[3]; + B[3][4] = (((E[4][3]) << 56) ^ ((E[4][3]) >> (64 - 56))); + A[3][0] = B[3][0] ^ ((~B[3][1]) & B[3][2]); + C[0] ^= A[3][0]; + A[3][1] = B[3][1] ^ ((~B[3][2]) & B[3][3]); + C[1] ^= A[3][1]; + A[3][2] = B[3][2] ^ ((~B[3][3]) & B[3][4]); + C[2] ^= A[3][2]; + A[3][3] = B[3][3] ^ ((~B[3][4]) & B[3][0]); + C[3] ^= A[3][3]; + A[3][4] = B[3][4] ^ ((~B[3][0]) & B[3][1]); + C[4] ^= A[3][4]; + E[0][2] ^= D[2]; + B[4][0] = (((E[0][2]) << 62) ^ ((E[0][2]) >> (64 - 62))); + E[1][3] ^= D[3]; + B[4][1] = (((E[1][3]) << 55) ^ ((E[1][3]) >> (64 - 55))); + E[2][4] ^= D[4]; + B[4][2] = (((E[2][4]) << 39) ^ ((E[2][4]) >> (64 - 39))); + E[3][0] ^= D[0]; + B[4][3] = (((E[3][0]) << 41) ^ ((E[3][0]) >> (64 - 41))); + E[4][1] ^= D[1]; + B[4][4] = (((E[4][1]) << 2) ^ ((E[4][1]) >> (64 - 2))); + A[4][0] = B[4][0] ^ ((~B[4][1]) & B[4][2]); + C[0] ^= A[4][0]; + A[4][1] = B[4][1] ^ ((~B[4][2]) & B[4][3]); + C[1] ^= A[4][1]; + A[4][2] = B[4][2] ^ ((~B[4][3]) & B[4][4]); + C[2] ^= A[4][2]; + A[4][3] = B[4][3] ^ ((~B[4][4]) & B[4][0]); + C[3] ^= A[4][3]; + A[4][4] = B[4][4] ^ ((~B[4][0]) & B[4][1]); + C[4] ^= A[4][4]; + D[0] = C[4] ^ (((C[1]) << 1) ^ ((C[1]) >> (64 - 1))); + D[1] = C[0] ^ (((C[2]) << 1) ^ ((C[2]) >> (64 - 1))); + D[2] = C[1] ^ (((C[3]) << 1) ^ ((C[3]) >> (64 - 1))); + D[3] = C[2] ^ (((C[4]) << 1) ^ ((C[4]) >> (64 - 1))); + D[4] = C[3] ^ (((C[0]) << 1) ^ ((C[0]) >> (64 - 1))); + A[0][0] ^= D[0]; + B[0][0] = A[0][0]; + A[1][1] ^= D[1]; + B[0][1] = (((A[1][1]) << 44) ^ ((A[1][1]) >> (64 - 44))); + A[2][2] ^= D[2]; + B[0][2] = (((A[2][2]) << 43) ^ ((A[2][2]) >> (64 - 43))); + A[3][3] ^= D[3]; + B[0][3] = (((A[3][3]) << 21) ^ ((A[3][3]) >> (64 - 21))); + A[4][4] ^= D[4]; + B[0][4] = (((A[4][4]) << 14) ^ ((A[4][4]) >> (64 - 14))); + E[0][0] = B[0][0] ^ ((~B[0][1]) & B[0][2]); + E[0][0] ^= KECCAKF1600_ROUND_CONSTANTS[2]; + C[0] = E[0][0]; + E[0][1] = B[0][1] ^ ((~B[0][2]) & B[0][3]); + C[1] = E[0][1]; + E[0][2] = B[0][2] ^ ((~B[0][3]) & B[0][4]); + C[2] = E[0][2]; + E[0][3] = B[0][3] ^ ((~B[0][4]) & B[0][0]); + C[3] = E[0][3]; + E[0][4] = B[0][4] ^ ((~B[0][0]) & B[0][1]); + C[4] = E[0][4]; + A[0][3] ^= D[3]; + B[1][0] = (((A[0][3]) << 28) ^ ((A[0][3]) >> (64 - 28))); + A[1][4] ^= D[4]; + B[1][1] = (((A[1][4]) << 20) ^ ((A[1][4]) >> (64 - 20))); + A[2][0] ^= D[0]; + B[1][2] = (((A[2][0]) << 3) ^ ((A[2][0]) >> (64 - 3))); + A[3][1] ^= D[1]; + B[1][3] = (((A[3][1]) << 45) ^ ((A[3][1]) >> (64 - 45))); + A[4][2] ^= D[2]; + B[1][4] = (((A[4][2]) << 61) ^ ((A[4][2]) >> (64 - 61))); + E[1][0] = B[1][0] ^ ((~B[1][1]) & B[1][2]); + C[0] ^= E[1][0]; + E[1][1] = B[1][1] ^ ((~B[1][2]) & B[1][3]); + C[1] ^= E[1][1]; + E[1][2] = B[1][2] ^ ((~B[1][3]) & B[1][4]); + C[2] ^= E[1][2]; + E[1][3] = B[1][3] ^ ((~B[1][4]) & B[1][0]); + C[3] ^= E[1][3]; + E[1][4] = B[1][4] ^ ((~B[1][0]) & B[1][1]); + C[4] ^= E[1][4]; + A[0][1] ^= D[1]; + B[2][0] = (((A[0][1]) << 1) ^ ((A[0][1]) >> (64 - 1))); + A[1][2] ^= D[2]; + B[2][1] = (((A[1][2]) << 6) ^ ((A[1][2]) >> (64 - 6))); + A[2][3] ^= D[3]; + B[2][2] = (((A[2][3]) << 25) ^ ((A[2][3]) >> (64 - 25))); + A[3][4] ^= D[4]; + B[2][3] = (((A[3][4]) << 8) ^ ((A[3][4]) >> (64 - 8))); + A[4][0] ^= D[0]; + B[2][4] = (((A[4][0]) << 18) ^ ((A[4][0]) >> (64 - 18))); + E[2][0] = B[2][0] ^ ((~B[2][1]) & B[2][2]); + C[0] ^= E[2][0]; + E[2][1] = B[2][1] ^ ((~B[2][2]) & B[2][3]); + C[1] ^= E[2][1]; + E[2][2] = B[2][2] ^ ((~B[2][3]) & B[2][4]); + C[2] ^= E[2][2]; + E[2][3] = B[2][3] ^ ((~B[2][4]) & B[2][0]); + C[3] ^= E[2][3]; + E[2][4] = B[2][4] ^ ((~B[2][0]) & B[2][1]); + C[4] ^= E[2][4]; + A[0][4] ^= D[4]; + B[3][0] = (((A[0][4]) << 27) ^ ((A[0][4]) >> (64 - 27))); + A[1][0] ^= D[0]; + B[3][1] = (((A[1][0]) << 36) ^ ((A[1][0]) >> (64 - 36))); + A[2][1] ^= D[1]; + B[3][2] = (((A[2][1]) << 10) ^ ((A[2][1]) >> (64 - 10))); + A[3][2] ^= D[2]; + B[3][3] = (((A[3][2]) << 15) ^ ((A[3][2]) >> (64 - 15))); + A[4][3] ^= D[3]; + B[3][4] = (((A[4][3]) << 56) ^ ((A[4][3]) >> (64 - 56))); + E[3][0] = B[3][0] ^ ((~B[3][1]) & B[3][2]); + C[0] ^= E[3][0]; + E[3][1] = B[3][1] ^ ((~B[3][2]) & B[3][3]); + C[1] ^= E[3][1]; + E[3][2] = B[3][2] ^ ((~B[3][3]) & B[3][4]); + C[2] ^= E[3][2]; + E[3][3] = B[3][3] ^ ((~B[3][4]) & B[3][0]); + C[3] ^= E[3][3]; + E[3][4] = B[3][4] ^ ((~B[3][0]) & B[3][1]); + C[4] ^= E[3][4]; + A[0][2] ^= D[2]; + B[4][0] = (((A[0][2]) << 62) ^ ((A[0][2]) >> (64 - 62))); + A[1][3] ^= D[3]; + B[4][1] = (((A[1][3]) << 55) ^ ((A[1][3]) >> (64 - 55))); + A[2][4] ^= D[4]; + B[4][2] = (((A[2][4]) << 39) ^ ((A[2][4]) >> (64 - 39))); + A[3][0] ^= D[0]; + B[4][3] = (((A[3][0]) << 41) ^ ((A[3][0]) >> (64 - 41))); + A[4][1] ^= D[1]; + B[4][4] = (((A[4][1]) << 2) ^ ((A[4][1]) >> (64 - 2))); + E[4][0] = B[4][0] ^ ((~B[4][1]) & B[4][2]); + C[0] ^= E[4][0]; + E[4][1] = B[4][1] ^ ((~B[4][2]) & B[4][3]); + C[1] ^= E[4][1]; + E[4][2] = B[4][2] ^ ((~B[4][3]) & B[4][4]); + C[2] ^= E[4][2]; + E[4][3] = B[4][3] ^ ((~B[4][4]) & B[4][0]); + C[3] ^= E[4][3]; + E[4][4] = B[4][4] ^ ((~B[4][0]) & B[4][1]); + C[4] ^= E[4][4]; + D[0] = C[4] ^ (((C[1]) << 1) ^ ((C[1]) >> (64 - 1))); + D[1] = C[0] ^ (((C[2]) << 1) ^ ((C[2]) >> (64 - 1))); + D[2] = C[1] ^ (((C[3]) << 1) ^ ((C[3]) >> (64 - 1))); + D[3] = C[2] ^ (((C[4]) << 1) ^ ((C[4]) >> (64 - 1))); + D[4] = C[3] ^ (((C[0]) << 1) ^ ((C[0]) >> (64 - 1))); + E[0][0] ^= D[0]; + B[0][0] = E[0][0]; + E[1][1] ^= D[1]; + B[0][1] = (((E[1][1]) << 44) ^ ((E[1][1]) >> (64 - 44))); + E[2][2] ^= D[2]; + B[0][2] = (((E[2][2]) << 43) ^ ((E[2][2]) >> (64 - 43))); + E[3][3] ^= D[3]; + B[0][3] = (((E[3][3]) << 21) ^ ((E[3][3]) >> (64 - 21))); + E[4][4] ^= D[4]; + B[0][4] = (((E[4][4]) << 14) ^ ((E[4][4]) >> (64 - 14))); + A[0][0] = B[0][0] ^ ((~B[0][1]) & B[0][2]); + A[0][0] ^= KECCAKF1600_ROUND_CONSTANTS[3]; + C[0] = A[0][0]; + A[0][1] = B[0][1] ^ ((~B[0][2]) & B[0][3]); + C[1] = A[0][1]; + A[0][2] = B[0][2] ^ ((~B[0][3]) & B[0][4]); + C[2] = A[0][2]; + A[0][3] = B[0][3] ^ ((~B[0][4]) & B[0][0]); + C[3] = A[0][3]; + A[0][4] = B[0][4] ^ ((~B[0][0]) & B[0][1]); + C[4] = A[0][4]; + E[0][3] ^= D[3]; + B[1][0] = (((E[0][3]) << 28) ^ ((E[0][3]) >> (64 - 28))); + E[1][4] ^= D[4]; + B[1][1] = (((E[1][4]) << 20) ^ ((E[1][4]) >> (64 - 20))); + E[2][0] ^= D[0]; + B[1][2] = (((E[2][0]) << 3) ^ ((E[2][0]) >> (64 - 3))); + E[3][1] ^= D[1]; + B[1][3] = (((E[3][1]) << 45) ^ ((E[3][1]) >> (64 - 45))); + E[4][2] ^= D[2]; + B[1][4] = (((E[4][2]) << 61) ^ ((E[4][2]) >> (64 - 61))); + A[1][0] = B[1][0] ^ ((~B[1][1]) & B[1][2]); + C[0] ^= A[1][0]; + A[1][1] = B[1][1] ^ ((~B[1][2]) & B[1][3]); + C[1] ^= A[1][1]; + A[1][2] = B[1][2] ^ ((~B[1][3]) & B[1][4]); + C[2] ^= A[1][2]; + A[1][3] = B[1][3] ^ ((~B[1][4]) & B[1][0]); + C[3] ^= A[1][3]; + A[1][4] = B[1][4] ^ ((~B[1][0]) & B[1][1]); + C[4] ^= A[1][4]; + E[0][1] ^= D[1]; + B[2][0] = (((E[0][1]) << 1) ^ ((E[0][1]) >> (64 - 1))); + E[1][2] ^= D[2]; + B[2][1] = (((E[1][2]) << 6) ^ ((E[1][2]) >> (64 - 6))); + E[2][3] ^= D[3]; + B[2][2] = (((E[2][3]) << 25) ^ ((E[2][3]) >> (64 - 25))); + E[3][4] ^= D[4]; + B[2][3] = (((E[3][4]) << 8) ^ ((E[3][4]) >> (64 - 8))); + E[4][0] ^= D[0]; + B[2][4] = (((E[4][0]) << 18) ^ ((E[4][0]) >> (64 - 18))); + A[2][0] = B[2][0] ^ ((~B[2][1]) & B[2][2]); + C[0] ^= A[2][0]; + A[2][1] = B[2][1] ^ ((~B[2][2]) & B[2][3]); + C[1] ^= A[2][1]; + A[2][2] = B[2][2] ^ ((~B[2][3]) & B[2][4]); + C[2] ^= A[2][2]; + A[2][3] = B[2][3] ^ ((~B[2][4]) & B[2][0]); + C[3] ^= A[2][3]; + A[2][4] = B[2][4] ^ ((~B[2][0]) & B[2][1]); + C[4] ^= A[2][4]; + E[0][4] ^= D[4]; + B[3][0] = (((E[0][4]) << 27) ^ ((E[0][4]) >> (64 - 27))); + E[1][0] ^= D[0]; + B[3][1] = (((E[1][0]) << 36) ^ ((E[1][0]) >> (64 - 36))); + E[2][1] ^= D[1]; + B[3][2] = (((E[2][1]) << 10) ^ ((E[2][1]) >> (64 - 10))); + E[3][2] ^= D[2]; + B[3][3] = (((E[3][2]) << 15) ^ ((E[3][2]) >> (64 - 15))); + E[4][3] ^= D[3]; + B[3][4] = (((E[4][3]) << 56) ^ ((E[4][3]) >> (64 - 56))); + A[3][0] = B[3][0] ^ ((~B[3][1]) & B[3][2]); + C[0] ^= A[3][0]; + A[3][1] = B[3][1] ^ ((~B[3][2]) & B[3][3]); + C[1] ^= A[3][1]; + A[3][2] = B[3][2] ^ ((~B[3][3]) & B[3][4]); + C[2] ^= A[3][2]; + A[3][3] = B[3][3] ^ ((~B[3][4]) & B[3][0]); + C[3] ^= A[3][3]; + A[3][4] = B[3][4] ^ ((~B[3][0]) & B[3][1]); + C[4] ^= A[3][4]; + E[0][2] ^= D[2]; + B[4][0] = (((E[0][2]) << 62) ^ ((E[0][2]) >> (64 - 62))); + E[1][3] ^= D[3]; + B[4][1] = (((E[1][3]) << 55) ^ ((E[1][3]) >> (64 - 55))); + E[2][4] ^= D[4]; + B[4][2] = (((E[2][4]) << 39) ^ ((E[2][4]) >> (64 - 39))); + E[3][0] ^= D[0]; + B[4][3] = (((E[3][0]) << 41) ^ ((E[3][0]) >> (64 - 41))); + E[4][1] ^= D[1]; + B[4][4] = (((E[4][1]) << 2) ^ ((E[4][1]) >> (64 - 2))); + A[4][0] = B[4][0] ^ ((~B[4][1]) & B[4][2]); + C[0] ^= A[4][0]; + A[4][1] = B[4][1] ^ ((~B[4][2]) & B[4][3]); + C[1] ^= A[4][1]; + A[4][2] = B[4][2] ^ ((~B[4][3]) & B[4][4]); + C[2] ^= A[4][2]; + A[4][3] = B[4][3] ^ ((~B[4][4]) & B[4][0]); + C[3] ^= A[4][3]; + A[4][4] = B[4][4] ^ ((~B[4][0]) & B[4][1]); + C[4] ^= A[4][4]; + D[0] = C[4] ^ (((C[1]) << 1) ^ ((C[1]) >> (64 - 1))); + D[1] = C[0] ^ (((C[2]) << 1) ^ ((C[2]) >> (64 - 1))); + D[2] = C[1] ^ (((C[3]) << 1) ^ ((C[3]) >> (64 - 1))); + D[3] = C[2] ^ (((C[4]) << 1) ^ ((C[4]) >> (64 - 1))); + D[4] = C[3] ^ (((C[0]) << 1) ^ ((C[0]) >> (64 - 1))); + A[0][0] ^= D[0]; + B[0][0] = A[0][0]; + A[1][1] ^= D[1]; + B[0][1] = (((A[1][1]) << 44) ^ ((A[1][1]) >> (64 - 44))); + A[2][2] ^= D[2]; + B[0][2] = (((A[2][2]) << 43) ^ ((A[2][2]) >> (64 - 43))); + A[3][3] ^= D[3]; + B[0][3] = (((A[3][3]) << 21) ^ ((A[3][3]) >> (64 - 21))); + A[4][4] ^= D[4]; + B[0][4] = (((A[4][4]) << 14) ^ ((A[4][4]) >> (64 - 14))); + E[0][0] = B[0][0] ^ ((~B[0][1]) & B[0][2]); + E[0][0] ^= KECCAKF1600_ROUND_CONSTANTS[4]; + C[0] = E[0][0]; + E[0][1] = B[0][1] ^ ((~B[0][2]) & B[0][3]); + C[1] = E[0][1]; + E[0][2] = B[0][2] ^ ((~B[0][3]) & B[0][4]); + C[2] = E[0][2]; + E[0][3] = B[0][3] ^ ((~B[0][4]) & B[0][0]); + C[3] = E[0][3]; + E[0][4] = B[0][4] ^ ((~B[0][0]) & B[0][1]); + C[4] = E[0][4]; + A[0][3] ^= D[3]; + B[1][0] = (((A[0][3]) << 28) ^ ((A[0][3]) >> (64 - 28))); + A[1][4] ^= D[4]; + B[1][1] = (((A[1][4]) << 20) ^ ((A[1][4]) >> (64 - 20))); + A[2][0] ^= D[0]; + B[1][2] = (((A[2][0]) << 3) ^ ((A[2][0]) >> (64 - 3))); + A[3][1] ^= D[1]; + B[1][3] = (((A[3][1]) << 45) ^ ((A[3][1]) >> (64 - 45))); + A[4][2] ^= D[2]; + B[1][4] = (((A[4][2]) << 61) ^ ((A[4][2]) >> (64 - 61))); + E[1][0] = B[1][0] ^ ((~B[1][1]) & B[1][2]); + C[0] ^= E[1][0]; + E[1][1] = B[1][1] ^ ((~B[1][2]) & B[1][3]); + C[1] ^= E[1][1]; + E[1][2] = B[1][2] ^ ((~B[1][3]) & B[1][4]); + C[2] ^= E[1][2]; + E[1][3] = B[1][3] ^ ((~B[1][4]) & B[1][0]); + C[3] ^= E[1][3]; + E[1][4] = B[1][4] ^ ((~B[1][0]) & B[1][1]); + C[4] ^= E[1][4]; + A[0][1] ^= D[1]; + B[2][0] = (((A[0][1]) << 1) ^ ((A[0][1]) >> (64 - 1))); + A[1][2] ^= D[2]; + B[2][1] = (((A[1][2]) << 6) ^ ((A[1][2]) >> (64 - 6))); + A[2][3] ^= D[3]; + B[2][2] = (((A[2][3]) << 25) ^ ((A[2][3]) >> (64 - 25))); + A[3][4] ^= D[4]; + B[2][3] = (((A[3][4]) << 8) ^ ((A[3][4]) >> (64 - 8))); + A[4][0] ^= D[0]; + B[2][4] = (((A[4][0]) << 18) ^ ((A[4][0]) >> (64 - 18))); + E[2][0] = B[2][0] ^ ((~B[2][1]) & B[2][2]); + C[0] ^= E[2][0]; + E[2][1] = B[2][1] ^ ((~B[2][2]) & B[2][3]); + C[1] ^= E[2][1]; + E[2][2] = B[2][2] ^ ((~B[2][3]) & B[2][4]); + C[2] ^= E[2][2]; + E[2][3] = B[2][3] ^ ((~B[2][4]) & B[2][0]); + C[3] ^= E[2][3]; + E[2][4] = B[2][4] ^ ((~B[2][0]) & B[2][1]); + C[4] ^= E[2][4]; + A[0][4] ^= D[4]; + B[3][0] = (((A[0][4]) << 27) ^ ((A[0][4]) >> (64 - 27))); + A[1][0] ^= D[0]; + B[3][1] = (((A[1][0]) << 36) ^ ((A[1][0]) >> (64 - 36))); + A[2][1] ^= D[1]; + B[3][2] = (((A[2][1]) << 10) ^ ((A[2][1]) >> (64 - 10))); + A[3][2] ^= D[2]; + B[3][3] = (((A[3][2]) << 15) ^ ((A[3][2]) >> (64 - 15))); + A[4][3] ^= D[3]; + B[3][4] = (((A[4][3]) << 56) ^ ((A[4][3]) >> (64 - 56))); + E[3][0] = B[3][0] ^ ((~B[3][1]) & B[3][2]); + C[0] ^= E[3][0]; + E[3][1] = B[3][1] ^ ((~B[3][2]) & B[3][3]); + C[1] ^= E[3][1]; + E[3][2] = B[3][2] ^ ((~B[3][3]) & B[3][4]); + C[2] ^= E[3][2]; + E[3][3] = B[3][3] ^ ((~B[3][4]) & B[3][0]); + C[3] ^= E[3][3]; + E[3][4] = B[3][4] ^ ((~B[3][0]) & B[3][1]); + C[4] ^= E[3][4]; + A[0][2] ^= D[2]; + B[4][0] = (((A[0][2]) << 62) ^ ((A[0][2]) >> (64 - 62))); + A[1][3] ^= D[3]; + B[4][1] = (((A[1][3]) << 55) ^ ((A[1][3]) >> (64 - 55))); + A[2][4] ^= D[4]; + B[4][2] = (((A[2][4]) << 39) ^ ((A[2][4]) >> (64 - 39))); + A[3][0] ^= D[0]; + B[4][3] = (((A[3][0]) << 41) ^ ((A[3][0]) >> (64 - 41))); + A[4][1] ^= D[1]; + B[4][4] = (((A[4][1]) << 2) ^ ((A[4][1]) >> (64 - 2))); + E[4][0] = B[4][0] ^ ((~B[4][1]) & B[4][2]); + C[0] ^= E[4][0]; + E[4][1] = B[4][1] ^ ((~B[4][2]) & B[4][3]); + C[1] ^= E[4][1]; + E[4][2] = B[4][2] ^ ((~B[4][3]) & B[4][4]); + C[2] ^= E[4][2]; + E[4][3] = B[4][3] ^ ((~B[4][4]) & B[4][0]); + C[3] ^= E[4][3]; + E[4][4] = B[4][4] ^ ((~B[4][0]) & B[4][1]); + C[4] ^= E[4][4]; + D[0] = C[4] ^ (((C[1]) << 1) ^ ((C[1]) >> (64 - 1))); + D[1] = C[0] ^ (((C[2]) << 1) ^ ((C[2]) >> (64 - 1))); + D[2] = C[1] ^ (((C[3]) << 1) ^ ((C[3]) >> (64 - 1))); + D[3] = C[2] ^ (((C[4]) << 1) ^ ((C[4]) >> (64 - 1))); + D[4] = C[3] ^ (((C[0]) << 1) ^ ((C[0]) >> (64 - 1))); + E[0][0] ^= D[0]; + B[0][0] = E[0][0]; + E[1][1] ^= D[1]; + B[0][1] = (((E[1][1]) << 44) ^ ((E[1][1]) >> (64 - 44))); + E[2][2] ^= D[2]; + B[0][2] = (((E[2][2]) << 43) ^ ((E[2][2]) >> (64 - 43))); + E[3][3] ^= D[3]; + B[0][3] = (((E[3][3]) << 21) ^ ((E[3][3]) >> (64 - 21))); + E[4][4] ^= D[4]; + B[0][4] = (((E[4][4]) << 14) ^ ((E[4][4]) >> (64 - 14))); + A[0][0] = B[0][0] ^ ((~B[0][1]) & B[0][2]); + A[0][0] ^= KECCAKF1600_ROUND_CONSTANTS[5]; + C[0] = A[0][0]; + A[0][1] = B[0][1] ^ ((~B[0][2]) & B[0][3]); + C[1] = A[0][1]; + A[0][2] = B[0][2] ^ ((~B[0][3]) & B[0][4]); + C[2] = A[0][2]; + A[0][3] = B[0][3] ^ ((~B[0][4]) & B[0][0]); + C[3] = A[0][3]; + A[0][4] = B[0][4] ^ ((~B[0][0]) & B[0][1]); + C[4] = A[0][4]; + E[0][3] ^= D[3]; + B[1][0] = (((E[0][3]) << 28) ^ ((E[0][3]) >> (64 - 28))); + E[1][4] ^= D[4]; + B[1][1] = (((E[1][4]) << 20) ^ ((E[1][4]) >> (64 - 20))); + E[2][0] ^= D[0]; + B[1][2] = (((E[2][0]) << 3) ^ ((E[2][0]) >> (64 - 3))); + E[3][1] ^= D[1]; + B[1][3] = (((E[3][1]) << 45) ^ ((E[3][1]) >> (64 - 45))); + E[4][2] ^= D[2]; + B[1][4] = (((E[4][2]) << 61) ^ ((E[4][2]) >> (64 - 61))); + A[1][0] = B[1][0] ^ ((~B[1][1]) & B[1][2]); + C[0] ^= A[1][0]; + A[1][1] = B[1][1] ^ ((~B[1][2]) & B[1][3]); + C[1] ^= A[1][1]; + A[1][2] = B[1][2] ^ ((~B[1][3]) & B[1][4]); + C[2] ^= A[1][2]; + A[1][3] = B[1][3] ^ ((~B[1][4]) & B[1][0]); + C[3] ^= A[1][3]; + A[1][4] = B[1][4] ^ ((~B[1][0]) & B[1][1]); + C[4] ^= A[1][4]; + E[0][1] ^= D[1]; + B[2][0] = (((E[0][1]) << 1) ^ ((E[0][1]) >> (64 - 1))); + E[1][2] ^= D[2]; + B[2][1] = (((E[1][2]) << 6) ^ ((E[1][2]) >> (64 - 6))); + E[2][3] ^= D[3]; + B[2][2] = (((E[2][3]) << 25) ^ ((E[2][3]) >> (64 - 25))); + E[3][4] ^= D[4]; + B[2][3] = (((E[3][4]) << 8) ^ ((E[3][4]) >> (64 - 8))); + E[4][0] ^= D[0]; + B[2][4] = (((E[4][0]) << 18) ^ ((E[4][0]) >> (64 - 18))); + A[2][0] = B[2][0] ^ ((~B[2][1]) & B[2][2]); + C[0] ^= A[2][0]; + A[2][1] = B[2][1] ^ ((~B[2][2]) & B[2][3]); + C[1] ^= A[2][1]; + A[2][2] = B[2][2] ^ ((~B[2][3]) & B[2][4]); + C[2] ^= A[2][2]; + A[2][3] = B[2][3] ^ ((~B[2][4]) & B[2][0]); + C[3] ^= A[2][3]; + A[2][4] = B[2][4] ^ ((~B[2][0]) & B[2][1]); + C[4] ^= A[2][4]; + E[0][4] ^= D[4]; + B[3][0] = (((E[0][4]) << 27) ^ ((E[0][4]) >> (64 - 27))); + E[1][0] ^= D[0]; + B[3][1] = (((E[1][0]) << 36) ^ ((E[1][0]) >> (64 - 36))); + E[2][1] ^= D[1]; + B[3][2] = (((E[2][1]) << 10) ^ ((E[2][1]) >> (64 - 10))); + E[3][2] ^= D[2]; + B[3][3] = (((E[3][2]) << 15) ^ ((E[3][2]) >> (64 - 15))); + E[4][3] ^= D[3]; + B[3][4] = (((E[4][3]) << 56) ^ ((E[4][3]) >> (64 - 56))); + A[3][0] = B[3][0] ^ ((~B[3][1]) & B[3][2]); + C[0] ^= A[3][0]; + A[3][1] = B[3][1] ^ ((~B[3][2]) & B[3][3]); + C[1] ^= A[3][1]; + A[3][2] = B[3][2] ^ ((~B[3][3]) & B[3][4]); + C[2] ^= A[3][2]; + A[3][3] = B[3][3] ^ ((~B[3][4]) & B[3][0]); + C[3] ^= A[3][3]; + A[3][4] = B[3][4] ^ ((~B[3][0]) & B[3][1]); + C[4] ^= A[3][4]; + E[0][2] ^= D[2]; + B[4][0] = (((E[0][2]) << 62) ^ ((E[0][2]) >> (64 - 62))); + E[1][3] ^= D[3]; + B[4][1] = (((E[1][3]) << 55) ^ ((E[1][3]) >> (64 - 55))); + E[2][4] ^= D[4]; + B[4][2] = (((E[2][4]) << 39) ^ ((E[2][4]) >> (64 - 39))); + E[3][0] ^= D[0]; + B[4][3] = (((E[3][0]) << 41) ^ ((E[3][0]) >> (64 - 41))); + E[4][1] ^= D[1]; + B[4][4] = (((E[4][1]) << 2) ^ ((E[4][1]) >> (64 - 2))); + A[4][0] = B[4][0] ^ ((~B[4][1]) & B[4][2]); + C[0] ^= A[4][0]; + A[4][1] = B[4][1] ^ ((~B[4][2]) & B[4][3]); + C[1] ^= A[4][1]; + A[4][2] = B[4][2] ^ ((~B[4][3]) & B[4][4]); + C[2] ^= A[4][2]; + A[4][3] = B[4][3] ^ ((~B[4][4]) & B[4][0]); + C[3] ^= A[4][3]; + A[4][4] = B[4][4] ^ ((~B[4][0]) & B[4][1]); + C[4] ^= A[4][4]; + D[0] = C[4] ^ (((C[1]) << 1) ^ ((C[1]) >> (64 - 1))); + D[1] = C[0] ^ (((C[2]) << 1) ^ ((C[2]) >> (64 - 1))); + D[2] = C[1] ^ (((C[3]) << 1) ^ ((C[3]) >> (64 - 1))); + D[3] = C[2] ^ (((C[4]) << 1) ^ ((C[4]) >> (64 - 1))); + D[4] = C[3] ^ (((C[0]) << 1) ^ ((C[0]) >> (64 - 1))); + A[0][0] ^= D[0]; + B[0][0] = A[0][0]; + A[1][1] ^= D[1]; + B[0][1] = (((A[1][1]) << 44) ^ ((A[1][1]) >> (64 - 44))); + A[2][2] ^= D[2]; + B[0][2] = (((A[2][2]) << 43) ^ ((A[2][2]) >> (64 - 43))); + A[3][3] ^= D[3]; + B[0][3] = (((A[3][3]) << 21) ^ ((A[3][3]) >> (64 - 21))); + A[4][4] ^= D[4]; + B[0][4] = (((A[4][4]) << 14) ^ ((A[4][4]) >> (64 - 14))); + E[0][0] = B[0][0] ^ ((~B[0][1]) & B[0][2]); + E[0][0] ^= KECCAKF1600_ROUND_CONSTANTS[6]; + C[0] = E[0][0]; + E[0][1] = B[0][1] ^ ((~B[0][2]) & B[0][3]); + C[1] = E[0][1]; + E[0][2] = B[0][2] ^ ((~B[0][3]) & B[0][4]); + C[2] = E[0][2]; + E[0][3] = B[0][3] ^ ((~B[0][4]) & B[0][0]); + C[3] = E[0][3]; + E[0][4] = B[0][4] ^ ((~B[0][0]) & B[0][1]); + C[4] = E[0][4]; + A[0][3] ^= D[3]; + B[1][0] = (((A[0][3]) << 28) ^ ((A[0][3]) >> (64 - 28))); + A[1][4] ^= D[4]; + B[1][1] = (((A[1][4]) << 20) ^ ((A[1][4]) >> (64 - 20))); + A[2][0] ^= D[0]; + B[1][2] = (((A[2][0]) << 3) ^ ((A[2][0]) >> (64 - 3))); + A[3][1] ^= D[1]; + B[1][3] = (((A[3][1]) << 45) ^ ((A[3][1]) >> (64 - 45))); + A[4][2] ^= D[2]; + B[1][4] = (((A[4][2]) << 61) ^ ((A[4][2]) >> (64 - 61))); + E[1][0] = B[1][0] ^ ((~B[1][1]) & B[1][2]); + C[0] ^= E[1][0]; + E[1][1] = B[1][1] ^ ((~B[1][2]) & B[1][3]); + C[1] ^= E[1][1]; + E[1][2] = B[1][2] ^ ((~B[1][3]) & B[1][4]); + C[2] ^= E[1][2]; + E[1][3] = B[1][3] ^ ((~B[1][4]) & B[1][0]); + C[3] ^= E[1][3]; + E[1][4] = B[1][4] ^ ((~B[1][0]) & B[1][1]); + C[4] ^= E[1][4]; + A[0][1] ^= D[1]; + B[2][0] = (((A[0][1]) << 1) ^ ((A[0][1]) >> (64 - 1))); + A[1][2] ^= D[2]; + B[2][1] = (((A[1][2]) << 6) ^ ((A[1][2]) >> (64 - 6))); + A[2][3] ^= D[3]; + B[2][2] = (((A[2][3]) << 25) ^ ((A[2][3]) >> (64 - 25))); + A[3][4] ^= D[4]; + B[2][3] = (((A[3][4]) << 8) ^ ((A[3][4]) >> (64 - 8))); + A[4][0] ^= D[0]; + B[2][4] = (((A[4][0]) << 18) ^ ((A[4][0]) >> (64 - 18))); + E[2][0] = B[2][0] ^ ((~B[2][1]) & B[2][2]); + C[0] ^= E[2][0]; + E[2][1] = B[2][1] ^ ((~B[2][2]) & B[2][3]); + C[1] ^= E[2][1]; + E[2][2] = B[2][2] ^ ((~B[2][3]) & B[2][4]); + C[2] ^= E[2][2]; + E[2][3] = B[2][3] ^ ((~B[2][4]) & B[2][0]); + C[3] ^= E[2][3]; + E[2][4] = B[2][4] ^ ((~B[2][0]) & B[2][1]); + C[4] ^= E[2][4]; + A[0][4] ^= D[4]; + B[3][0] = (((A[0][4]) << 27) ^ ((A[0][4]) >> (64 - 27))); + A[1][0] ^= D[0]; + B[3][1] = (((A[1][0]) << 36) ^ ((A[1][0]) >> (64 - 36))); + A[2][1] ^= D[1]; + B[3][2] = (((A[2][1]) << 10) ^ ((A[2][1]) >> (64 - 10))); + A[3][2] ^= D[2]; + B[3][3] = (((A[3][2]) << 15) ^ ((A[3][2]) >> (64 - 15))); + A[4][3] ^= D[3]; + B[3][4] = (((A[4][3]) << 56) ^ ((A[4][3]) >> (64 - 56))); + E[3][0] = B[3][0] ^ ((~B[3][1]) & B[3][2]); + C[0] ^= E[3][0]; + E[3][1] = B[3][1] ^ ((~B[3][2]) & B[3][3]); + C[1] ^= E[3][1]; + E[3][2] = B[3][2] ^ ((~B[3][3]) & B[3][4]); + C[2] ^= E[3][2]; + E[3][3] = B[3][3] ^ ((~B[3][4]) & B[3][0]); + C[3] ^= E[3][3]; + E[3][4] = B[3][4] ^ ((~B[3][0]) & B[3][1]); + C[4] ^= E[3][4]; + A[0][2] ^= D[2]; + B[4][0] = (((A[0][2]) << 62) ^ ((A[0][2]) >> (64 - 62))); + A[1][3] ^= D[3]; + B[4][1] = (((A[1][3]) << 55) ^ ((A[1][3]) >> (64 - 55))); + A[2][4] ^= D[4]; + B[4][2] = (((A[2][4]) << 39) ^ ((A[2][4]) >> (64 - 39))); + A[3][0] ^= D[0]; + B[4][3] = (((A[3][0]) << 41) ^ ((A[3][0]) >> (64 - 41))); + A[4][1] ^= D[1]; + B[4][4] = (((A[4][1]) << 2) ^ ((A[4][1]) >> (64 - 2))); + E[4][0] = B[4][0] ^ ((~B[4][1]) & B[4][2]); + C[0] ^= E[4][0]; + E[4][1] = B[4][1] ^ ((~B[4][2]) & B[4][3]); + C[1] ^= E[4][1]; + E[4][2] = B[4][2] ^ ((~B[4][3]) & B[4][4]); + C[2] ^= E[4][2]; + E[4][3] = B[4][3] ^ ((~B[4][4]) & B[4][0]); + C[3] ^= E[4][3]; + E[4][4] = B[4][4] ^ ((~B[4][0]) & B[4][1]); + C[4] ^= E[4][4]; + D[0] = C[4] ^ (((C[1]) << 1) ^ ((C[1]) >> (64 - 1))); + D[1] = C[0] ^ (((C[2]) << 1) ^ ((C[2]) >> (64 - 1))); + D[2] = C[1] ^ (((C[3]) << 1) ^ ((C[3]) >> (64 - 1))); + D[3] = C[2] ^ (((C[4]) << 1) ^ ((C[4]) >> (64 - 1))); + D[4] = C[3] ^ (((C[0]) << 1) ^ ((C[0]) >> (64 - 1))); + E[0][0] ^= D[0]; + B[0][0] = E[0][0]; + E[1][1] ^= D[1]; + B[0][1] = (((E[1][1]) << 44) ^ ((E[1][1]) >> (64 - 44))); + E[2][2] ^= D[2]; + B[0][2] = (((E[2][2]) << 43) ^ ((E[2][2]) >> (64 - 43))); + E[3][3] ^= D[3]; + B[0][3] = (((E[3][3]) << 21) ^ ((E[3][3]) >> (64 - 21))); + E[4][4] ^= D[4]; + B[0][4] = (((E[4][4]) << 14) ^ ((E[4][4]) >> (64 - 14))); + A[0][0] = B[0][0] ^ ((~B[0][1]) & B[0][2]); + A[0][0] ^= KECCAKF1600_ROUND_CONSTANTS[7]; + C[0] = A[0][0]; + A[0][1] = B[0][1] ^ ((~B[0][2]) & B[0][3]); + C[1] = A[0][1]; + A[0][2] = B[0][2] ^ ((~B[0][3]) & B[0][4]); + C[2] = A[0][2]; + A[0][3] = B[0][3] ^ ((~B[0][4]) & B[0][0]); + C[3] = A[0][3]; + A[0][4] = B[0][4] ^ ((~B[0][0]) & B[0][1]); + C[4] = A[0][4]; + E[0][3] ^= D[3]; + B[1][0] = (((E[0][3]) << 28) ^ ((E[0][3]) >> (64 - 28))); + E[1][4] ^= D[4]; + B[1][1] = (((E[1][4]) << 20) ^ ((E[1][4]) >> (64 - 20))); + E[2][0] ^= D[0]; + B[1][2] = (((E[2][0]) << 3) ^ ((E[2][0]) >> (64 - 3))); + E[3][1] ^= D[1]; + B[1][3] = (((E[3][1]) << 45) ^ ((E[3][1]) >> (64 - 45))); + E[4][2] ^= D[2]; + B[1][4] = (((E[4][2]) << 61) ^ ((E[4][2]) >> (64 - 61))); + A[1][0] = B[1][0] ^ ((~B[1][1]) & B[1][2]); + C[0] ^= A[1][0]; + A[1][1] = B[1][1] ^ ((~B[1][2]) & B[1][3]); + C[1] ^= A[1][1]; + A[1][2] = B[1][2] ^ ((~B[1][3]) & B[1][4]); + C[2] ^= A[1][2]; + A[1][3] = B[1][3] ^ ((~B[1][4]) & B[1][0]); + C[3] ^= A[1][3]; + A[1][4] = B[1][4] ^ ((~B[1][0]) & B[1][1]); + C[4] ^= A[1][4]; + E[0][1] ^= D[1]; + B[2][0] = (((E[0][1]) << 1) ^ ((E[0][1]) >> (64 - 1))); + E[1][2] ^= D[2]; + B[2][1] = (((E[1][2]) << 6) ^ ((E[1][2]) >> (64 - 6))); + E[2][3] ^= D[3]; + B[2][2] = (((E[2][3]) << 25) ^ ((E[2][3]) >> (64 - 25))); + E[3][4] ^= D[4]; + B[2][3] = (((E[3][4]) << 8) ^ ((E[3][4]) >> (64 - 8))); + E[4][0] ^= D[0]; + B[2][4] = (((E[4][0]) << 18) ^ ((E[4][0]) >> (64 - 18))); + A[2][0] = B[2][0] ^ ((~B[2][1]) & B[2][2]); + C[0] ^= A[2][0]; + A[2][1] = B[2][1] ^ ((~B[2][2]) & B[2][3]); + C[1] ^= A[2][1]; + A[2][2] = B[2][2] ^ ((~B[2][3]) & B[2][4]); + C[2] ^= A[2][2]; + A[2][3] = B[2][3] ^ ((~B[2][4]) & B[2][0]); + C[3] ^= A[2][3]; + A[2][4] = B[2][4] ^ ((~B[2][0]) & B[2][1]); + C[4] ^= A[2][4]; + E[0][4] ^= D[4]; + B[3][0] = (((E[0][4]) << 27) ^ ((E[0][4]) >> (64 - 27))); + E[1][0] ^= D[0]; + B[3][1] = (((E[1][0]) << 36) ^ ((E[1][0]) >> (64 - 36))); + E[2][1] ^= D[1]; + B[3][2] = (((E[2][1]) << 10) ^ ((E[2][1]) >> (64 - 10))); + E[3][2] ^= D[2]; + B[3][3] = (((E[3][2]) << 15) ^ ((E[3][2]) >> (64 - 15))); + E[4][3] ^= D[3]; + B[3][4] = (((E[4][3]) << 56) ^ ((E[4][3]) >> (64 - 56))); + A[3][0] = B[3][0] ^ ((~B[3][1]) & B[3][2]); + C[0] ^= A[3][0]; + A[3][1] = B[3][1] ^ ((~B[3][2]) & B[3][3]); + C[1] ^= A[3][1]; + A[3][2] = B[3][2] ^ ((~B[3][3]) & B[3][4]); + C[2] ^= A[3][2]; + A[3][3] = B[3][3] ^ ((~B[3][4]) & B[3][0]); + C[3] ^= A[3][3]; + A[3][4] = B[3][4] ^ ((~B[3][0]) & B[3][1]); + C[4] ^= A[3][4]; + E[0][2] ^= D[2]; + B[4][0] = (((E[0][2]) << 62) ^ ((E[0][2]) >> (64 - 62))); + E[1][3] ^= D[3]; + B[4][1] = (((E[1][3]) << 55) ^ ((E[1][3]) >> (64 - 55))); + E[2][4] ^= D[4]; + B[4][2] = (((E[2][4]) << 39) ^ ((E[2][4]) >> (64 - 39))); + E[3][0] ^= D[0]; + B[4][3] = (((E[3][0]) << 41) ^ ((E[3][0]) >> (64 - 41))); + E[4][1] ^= D[1]; + B[4][4] = (((E[4][1]) << 2) ^ ((E[4][1]) >> (64 - 2))); + A[4][0] = B[4][0] ^ ((~B[4][1]) & B[4][2]); + C[0] ^= A[4][0]; + A[4][1] = B[4][1] ^ ((~B[4][2]) & B[4][3]); + C[1] ^= A[4][1]; + A[4][2] = B[4][2] ^ ((~B[4][3]) & B[4][4]); + C[2] ^= A[4][2]; + A[4][3] = B[4][3] ^ ((~B[4][4]) & B[4][0]); + C[3] ^= A[4][3]; + A[4][4] = B[4][4] ^ ((~B[4][0]) & B[4][1]); + C[4] ^= A[4][4]; + D[0] = C[4] ^ (((C[1]) << 1) ^ ((C[1]) >> (64 - 1))); + D[1] = C[0] ^ (((C[2]) << 1) ^ ((C[2]) >> (64 - 1))); + D[2] = C[1] ^ (((C[3]) << 1) ^ ((C[3]) >> (64 - 1))); + D[3] = C[2] ^ (((C[4]) << 1) ^ ((C[4]) >> (64 - 1))); + D[4] = C[3] ^ (((C[0]) << 1) ^ ((C[0]) >> (64 - 1))); + A[0][0] ^= D[0]; + B[0][0] = A[0][0]; + A[1][1] ^= D[1]; + B[0][1] = (((A[1][1]) << 44) ^ ((A[1][1]) >> (64 - 44))); + A[2][2] ^= D[2]; + B[0][2] = (((A[2][2]) << 43) ^ ((A[2][2]) >> (64 - 43))); + A[3][3] ^= D[3]; + B[0][3] = (((A[3][3]) << 21) ^ ((A[3][3]) >> (64 - 21))); + A[4][4] ^= D[4]; + B[0][4] = (((A[4][4]) << 14) ^ ((A[4][4]) >> (64 - 14))); + E[0][0] = B[0][0] ^ ((~B[0][1]) & B[0][2]); + E[0][0] ^= KECCAKF1600_ROUND_CONSTANTS[8]; + C[0] = E[0][0]; + E[0][1] = B[0][1] ^ ((~B[0][2]) & B[0][3]); + C[1] = E[0][1]; + E[0][2] = B[0][2] ^ ((~B[0][3]) & B[0][4]); + C[2] = E[0][2]; + E[0][3] = B[0][3] ^ ((~B[0][4]) & B[0][0]); + C[3] = E[0][3]; + E[0][4] = B[0][4] ^ ((~B[0][0]) & B[0][1]); + C[4] = E[0][4]; + A[0][3] ^= D[3]; + B[1][0] = (((A[0][3]) << 28) ^ ((A[0][3]) >> (64 - 28))); + A[1][4] ^= D[4]; + B[1][1] = (((A[1][4]) << 20) ^ ((A[1][4]) >> (64 - 20))); + A[2][0] ^= D[0]; + B[1][2] = (((A[2][0]) << 3) ^ ((A[2][0]) >> (64 - 3))); + A[3][1] ^= D[1]; + B[1][3] = (((A[3][1]) << 45) ^ ((A[3][1]) >> (64 - 45))); + A[4][2] ^= D[2]; + B[1][4] = (((A[4][2]) << 61) ^ ((A[4][2]) >> (64 - 61))); + E[1][0] = B[1][0] ^ ((~B[1][1]) & B[1][2]); + C[0] ^= E[1][0]; + E[1][1] = B[1][1] ^ ((~B[1][2]) & B[1][3]); + C[1] ^= E[1][1]; + E[1][2] = B[1][2] ^ ((~B[1][3]) & B[1][4]); + C[2] ^= E[1][2]; + E[1][3] = B[1][3] ^ ((~B[1][4]) & B[1][0]); + C[3] ^= E[1][3]; + E[1][4] = B[1][4] ^ ((~B[1][0]) & B[1][1]); + C[4] ^= E[1][4]; + A[0][1] ^= D[1]; + B[2][0] = (((A[0][1]) << 1) ^ ((A[0][1]) >> (64 - 1))); + A[1][2] ^= D[2]; + B[2][1] = (((A[1][2]) << 6) ^ ((A[1][2]) >> (64 - 6))); + A[2][3] ^= D[3]; + B[2][2] = (((A[2][3]) << 25) ^ ((A[2][3]) >> (64 - 25))); + A[3][4] ^= D[4]; + B[2][3] = (((A[3][4]) << 8) ^ ((A[3][4]) >> (64 - 8))); + A[4][0] ^= D[0]; + B[2][4] = (((A[4][0]) << 18) ^ ((A[4][0]) >> (64 - 18))); + E[2][0] = B[2][0] ^ ((~B[2][1]) & B[2][2]); + C[0] ^= E[2][0]; + E[2][1] = B[2][1] ^ ((~B[2][2]) & B[2][3]); + C[1] ^= E[2][1]; + E[2][2] = B[2][2] ^ ((~B[2][3]) & B[2][4]); + C[2] ^= E[2][2]; + E[2][3] = B[2][3] ^ ((~B[2][4]) & B[2][0]); + C[3] ^= E[2][3]; + E[2][4] = B[2][4] ^ ((~B[2][0]) & B[2][1]); + C[4] ^= E[2][4]; + A[0][4] ^= D[4]; + B[3][0] = (((A[0][4]) << 27) ^ ((A[0][4]) >> (64 - 27))); + A[1][0] ^= D[0]; + B[3][1] = (((A[1][0]) << 36) ^ ((A[1][0]) >> (64 - 36))); + A[2][1] ^= D[1]; + B[3][2] = (((A[2][1]) << 10) ^ ((A[2][1]) >> (64 - 10))); + A[3][2] ^= D[2]; + B[3][3] = (((A[3][2]) << 15) ^ ((A[3][2]) >> (64 - 15))); + A[4][3] ^= D[3]; + B[3][4] = (((A[4][3]) << 56) ^ ((A[4][3]) >> (64 - 56))); + E[3][0] = B[3][0] ^ ((~B[3][1]) & B[3][2]); + C[0] ^= E[3][0]; + E[3][1] = B[3][1] ^ ((~B[3][2]) & B[3][3]); + C[1] ^= E[3][1]; + E[3][2] = B[3][2] ^ ((~B[3][3]) & B[3][4]); + C[2] ^= E[3][2]; + E[3][3] = B[3][3] ^ ((~B[3][4]) & B[3][0]); + C[3] ^= E[3][3]; + E[3][4] = B[3][4] ^ ((~B[3][0]) & B[3][1]); + C[4] ^= E[3][4]; + A[0][2] ^= D[2]; + B[4][0] = (((A[0][2]) << 62) ^ ((A[0][2]) >> (64 - 62))); + A[1][3] ^= D[3]; + B[4][1] = (((A[1][3]) << 55) ^ ((A[1][3]) >> (64 - 55))); + A[2][4] ^= D[4]; + B[4][2] = (((A[2][4]) << 39) ^ ((A[2][4]) >> (64 - 39))); + A[3][0] ^= D[0]; + B[4][3] = (((A[3][0]) << 41) ^ ((A[3][0]) >> (64 - 41))); + A[4][1] ^= D[1]; + B[4][4] = (((A[4][1]) << 2) ^ ((A[4][1]) >> (64 - 2))); + E[4][0] = B[4][0] ^ ((~B[4][1]) & B[4][2]); + C[0] ^= E[4][0]; + E[4][1] = B[4][1] ^ ((~B[4][2]) & B[4][3]); + C[1] ^= E[4][1]; + E[4][2] = B[4][2] ^ ((~B[4][3]) & B[4][4]); + C[2] ^= E[4][2]; + E[4][3] = B[4][3] ^ ((~B[4][4]) & B[4][0]); + C[3] ^= E[4][3]; + E[4][4] = B[4][4] ^ ((~B[4][0]) & B[4][1]); + C[4] ^= E[4][4]; + D[0] = C[4] ^ (((C[1]) << 1) ^ ((C[1]) >> (64 - 1))); + D[1] = C[0] ^ (((C[2]) << 1) ^ ((C[2]) >> (64 - 1))); + D[2] = C[1] ^ (((C[3]) << 1) ^ ((C[3]) >> (64 - 1))); + D[3] = C[2] ^ (((C[4]) << 1) ^ ((C[4]) >> (64 - 1))); + D[4] = C[3] ^ (((C[0]) << 1) ^ ((C[0]) >> (64 - 1))); + E[0][0] ^= D[0]; + B[0][0] = E[0][0]; + E[1][1] ^= D[1]; + B[0][1] = (((E[1][1]) << 44) ^ ((E[1][1]) >> (64 - 44))); + E[2][2] ^= D[2]; + B[0][2] = (((E[2][2]) << 43) ^ ((E[2][2]) >> (64 - 43))); + E[3][3] ^= D[3]; + B[0][3] = (((E[3][3]) << 21) ^ ((E[3][3]) >> (64 - 21))); + E[4][4] ^= D[4]; + B[0][4] = (((E[4][4]) << 14) ^ ((E[4][4]) >> (64 - 14))); + A[0][0] = B[0][0] ^ ((~B[0][1]) & B[0][2]); + A[0][0] ^= KECCAKF1600_ROUND_CONSTANTS[9]; + C[0] = A[0][0]; + A[0][1] = B[0][1] ^ ((~B[0][2]) & B[0][3]); + C[1] = A[0][1]; + A[0][2] = B[0][2] ^ ((~B[0][3]) & B[0][4]); + C[2] = A[0][2]; + A[0][3] = B[0][3] ^ ((~B[0][4]) & B[0][0]); + C[3] = A[0][3]; + A[0][4] = B[0][4] ^ ((~B[0][0]) & B[0][1]); + C[4] = A[0][4]; + E[0][3] ^= D[3]; + B[1][0] = (((E[0][3]) << 28) ^ ((E[0][3]) >> (64 - 28))); + E[1][4] ^= D[4]; + B[1][1] = (((E[1][4]) << 20) ^ ((E[1][4]) >> (64 - 20))); + E[2][0] ^= D[0]; + B[1][2] = (((E[2][0]) << 3) ^ ((E[2][0]) >> (64 - 3))); + E[3][1] ^= D[1]; + B[1][3] = (((E[3][1]) << 45) ^ ((E[3][1]) >> (64 - 45))); + E[4][2] ^= D[2]; + B[1][4] = (((E[4][2]) << 61) ^ ((E[4][2]) >> (64 - 61))); + A[1][0] = B[1][0] ^ ((~B[1][1]) & B[1][2]); + C[0] ^= A[1][0]; + A[1][1] = B[1][1] ^ ((~B[1][2]) & B[1][3]); + C[1] ^= A[1][1]; + A[1][2] = B[1][2] ^ ((~B[1][3]) & B[1][4]); + C[2] ^= A[1][2]; + A[1][3] = B[1][3] ^ ((~B[1][4]) & B[1][0]); + C[3] ^= A[1][3]; + A[1][4] = B[1][4] ^ ((~B[1][0]) & B[1][1]); + C[4] ^= A[1][4]; + E[0][1] ^= D[1]; + B[2][0] = (((E[0][1]) << 1) ^ ((E[0][1]) >> (64 - 1))); + E[1][2] ^= D[2]; + B[2][1] = (((E[1][2]) << 6) ^ ((E[1][2]) >> (64 - 6))); + E[2][3] ^= D[3]; + B[2][2] = (((E[2][3]) << 25) ^ ((E[2][3]) >> (64 - 25))); + E[3][4] ^= D[4]; + B[2][3] = (((E[3][4]) << 8) ^ ((E[3][4]) >> (64 - 8))); + E[4][0] ^= D[0]; + B[2][4] = (((E[4][0]) << 18) ^ ((E[4][0]) >> (64 - 18))); + A[2][0] = B[2][0] ^ ((~B[2][1]) & B[2][2]); + C[0] ^= A[2][0]; + A[2][1] = B[2][1] ^ ((~B[2][2]) & B[2][3]); + C[1] ^= A[2][1]; + A[2][2] = B[2][2] ^ ((~B[2][3]) & B[2][4]); + C[2] ^= A[2][2]; + A[2][3] = B[2][3] ^ ((~B[2][4]) & B[2][0]); + C[3] ^= A[2][3]; + A[2][4] = B[2][4] ^ ((~B[2][0]) & B[2][1]); + C[4] ^= A[2][4]; + E[0][4] ^= D[4]; + B[3][0] = (((E[0][4]) << 27) ^ ((E[0][4]) >> (64 - 27))); + E[1][0] ^= D[0]; + B[3][1] = (((E[1][0]) << 36) ^ ((E[1][0]) >> (64 - 36))); + E[2][1] ^= D[1]; + B[3][2] = (((E[2][1]) << 10) ^ ((E[2][1]) >> (64 - 10))); + E[3][2] ^= D[2]; + B[3][3] = (((E[3][2]) << 15) ^ ((E[3][2]) >> (64 - 15))); + E[4][3] ^= D[3]; + B[3][4] = (((E[4][3]) << 56) ^ ((E[4][3]) >> (64 - 56))); + A[3][0] = B[3][0] ^ ((~B[3][1]) & B[3][2]); + C[0] ^= A[3][0]; + A[3][1] = B[3][1] ^ ((~B[3][2]) & B[3][3]); + C[1] ^= A[3][1]; + A[3][2] = B[3][2] ^ ((~B[3][3]) & B[3][4]); + C[2] ^= A[3][2]; + A[3][3] = B[3][3] ^ ((~B[3][4]) & B[3][0]); + C[3] ^= A[3][3]; + A[3][4] = B[3][4] ^ ((~B[3][0]) & B[3][1]); + C[4] ^= A[3][4]; + E[0][2] ^= D[2]; + B[4][0] = (((E[0][2]) << 62) ^ ((E[0][2]) >> (64 - 62))); + E[1][3] ^= D[3]; + B[4][1] = (((E[1][3]) << 55) ^ ((E[1][3]) >> (64 - 55))); + E[2][4] ^= D[4]; + B[4][2] = (((E[2][4]) << 39) ^ ((E[2][4]) >> (64 - 39))); + E[3][0] ^= D[0]; + B[4][3] = (((E[3][0]) << 41) ^ ((E[3][0]) >> (64 - 41))); + E[4][1] ^= D[1]; + B[4][4] = (((E[4][1]) << 2) ^ ((E[4][1]) >> (64 - 2))); + A[4][0] = B[4][0] ^ ((~B[4][1]) & B[4][2]); + C[0] ^= A[4][0]; + A[4][1] = B[4][1] ^ ((~B[4][2]) & B[4][3]); + C[1] ^= A[4][1]; + A[4][2] = B[4][2] ^ ((~B[4][3]) & B[4][4]); + C[2] ^= A[4][2]; + A[4][3] = B[4][3] ^ ((~B[4][4]) & B[4][0]); + C[3] ^= A[4][3]; + A[4][4] = B[4][4] ^ ((~B[4][0]) & B[4][1]); + C[4] ^= A[4][4]; + D[0] = C[4] ^ (((C[1]) << 1) ^ ((C[1]) >> (64 - 1))); + D[1] = C[0] ^ (((C[2]) << 1) ^ ((C[2]) >> (64 - 1))); + D[2] = C[1] ^ (((C[3]) << 1) ^ ((C[3]) >> (64 - 1))); + D[3] = C[2] ^ (((C[4]) << 1) ^ ((C[4]) >> (64 - 1))); + D[4] = C[3] ^ (((C[0]) << 1) ^ ((C[0]) >> (64 - 1))); + A[0][0] ^= D[0]; + B[0][0] = A[0][0]; + A[1][1] ^= D[1]; + B[0][1] = (((A[1][1]) << 44) ^ ((A[1][1]) >> (64 - 44))); + A[2][2] ^= D[2]; + B[0][2] = (((A[2][2]) << 43) ^ ((A[2][2]) >> (64 - 43))); + A[3][3] ^= D[3]; + B[0][3] = (((A[3][3]) << 21) ^ ((A[3][3]) >> (64 - 21))); + A[4][4] ^= D[4]; + B[0][4] = (((A[4][4]) << 14) ^ ((A[4][4]) >> (64 - 14))); + E[0][0] = B[0][0] ^ ((~B[0][1]) & B[0][2]); + E[0][0] ^= KECCAKF1600_ROUND_CONSTANTS[10]; + C[0] = E[0][0]; + E[0][1] = B[0][1] ^ ((~B[0][2]) & B[0][3]); + C[1] = E[0][1]; + E[0][2] = B[0][2] ^ ((~B[0][3]) & B[0][4]); + C[2] = E[0][2]; + E[0][3] = B[0][3] ^ ((~B[0][4]) & B[0][0]); + C[3] = E[0][3]; + E[0][4] = B[0][4] ^ ((~B[0][0]) & B[0][1]); + C[4] = E[0][4]; + A[0][3] ^= D[3]; + B[1][0] = (((A[0][3]) << 28) ^ ((A[0][3]) >> (64 - 28))); + A[1][4] ^= D[4]; + B[1][1] = (((A[1][4]) << 20) ^ ((A[1][4]) >> (64 - 20))); + A[2][0] ^= D[0]; + B[1][2] = (((A[2][0]) << 3) ^ ((A[2][0]) >> (64 - 3))); + A[3][1] ^= D[1]; + B[1][3] = (((A[3][1]) << 45) ^ ((A[3][1]) >> (64 - 45))); + A[4][2] ^= D[2]; + B[1][4] = (((A[4][2]) << 61) ^ ((A[4][2]) >> (64 - 61))); + E[1][0] = B[1][0] ^ ((~B[1][1]) & B[1][2]); + C[0] ^= E[1][0]; + E[1][1] = B[1][1] ^ ((~B[1][2]) & B[1][3]); + C[1] ^= E[1][1]; + E[1][2] = B[1][2] ^ ((~B[1][3]) & B[1][4]); + C[2] ^= E[1][2]; + E[1][3] = B[1][3] ^ ((~B[1][4]) & B[1][0]); + C[3] ^= E[1][3]; + E[1][4] = B[1][4] ^ ((~B[1][0]) & B[1][1]); + C[4] ^= E[1][4]; + A[0][1] ^= D[1]; + B[2][0] = (((A[0][1]) << 1) ^ ((A[0][1]) >> (64 - 1))); + A[1][2] ^= D[2]; + B[2][1] = (((A[1][2]) << 6) ^ ((A[1][2]) >> (64 - 6))); + A[2][3] ^= D[3]; + B[2][2] = (((A[2][3]) << 25) ^ ((A[2][3]) >> (64 - 25))); + A[3][4] ^= D[4]; + B[2][3] = (((A[3][4]) << 8) ^ ((A[3][4]) >> (64 - 8))); + A[4][0] ^= D[0]; + B[2][4] = (((A[4][0]) << 18) ^ ((A[4][0]) >> (64 - 18))); + E[2][0] = B[2][0] ^ ((~B[2][1]) & B[2][2]); + C[0] ^= E[2][0]; + E[2][1] = B[2][1] ^ ((~B[2][2]) & B[2][3]); + C[1] ^= E[2][1]; + E[2][2] = B[2][2] ^ ((~B[2][3]) & B[2][4]); + C[2] ^= E[2][2]; + E[2][3] = B[2][3] ^ ((~B[2][4]) & B[2][0]); + C[3] ^= E[2][3]; + E[2][4] = B[2][4] ^ ((~B[2][0]) & B[2][1]); + C[4] ^= E[2][4]; + A[0][4] ^= D[4]; + B[3][0] = (((A[0][4]) << 27) ^ ((A[0][4]) >> (64 - 27))); + A[1][0] ^= D[0]; + B[3][1] = (((A[1][0]) << 36) ^ ((A[1][0]) >> (64 - 36))); + A[2][1] ^= D[1]; + B[3][2] = (((A[2][1]) << 10) ^ ((A[2][1]) >> (64 - 10))); + A[3][2] ^= D[2]; + B[3][3] = (((A[3][2]) << 15) ^ ((A[3][2]) >> (64 - 15))); + A[4][3] ^= D[3]; + B[3][4] = (((A[4][3]) << 56) ^ ((A[4][3]) >> (64 - 56))); + E[3][0] = B[3][0] ^ ((~B[3][1]) & B[3][2]); + C[0] ^= E[3][0]; + E[3][1] = B[3][1] ^ ((~B[3][2]) & B[3][3]); + C[1] ^= E[3][1]; + E[3][2] = B[3][2] ^ ((~B[3][3]) & B[3][4]); + C[2] ^= E[3][2]; + E[3][3] = B[3][3] ^ ((~B[3][4]) & B[3][0]); + C[3] ^= E[3][3]; + E[3][4] = B[3][4] ^ ((~B[3][0]) & B[3][1]); + C[4] ^= E[3][4]; + A[0][2] ^= D[2]; + B[4][0] = (((A[0][2]) << 62) ^ ((A[0][2]) >> (64 - 62))); + A[1][3] ^= D[3]; + B[4][1] = (((A[1][3]) << 55) ^ ((A[1][3]) >> (64 - 55))); + A[2][4] ^= D[4]; + B[4][2] = (((A[2][4]) << 39) ^ ((A[2][4]) >> (64 - 39))); + A[3][0] ^= D[0]; + B[4][3] = (((A[3][0]) << 41) ^ ((A[3][0]) >> (64 - 41))); + A[4][1] ^= D[1]; + B[4][4] = (((A[4][1]) << 2) ^ ((A[4][1]) >> (64 - 2))); + E[4][0] = B[4][0] ^ ((~B[4][1]) & B[4][2]); + C[0] ^= E[4][0]; + E[4][1] = B[4][1] ^ ((~B[4][2]) & B[4][3]); + C[1] ^= E[4][1]; + E[4][2] = B[4][2] ^ ((~B[4][3]) & B[4][4]); + C[2] ^= E[4][2]; + E[4][3] = B[4][3] ^ ((~B[4][4]) & B[4][0]); + C[3] ^= E[4][3]; + E[4][4] = B[4][4] ^ ((~B[4][0]) & B[4][1]); + C[4] ^= E[4][4]; + D[0] = C[4] ^ (((C[1]) << 1) ^ ((C[1]) >> (64 - 1))); + D[1] = C[0] ^ (((C[2]) << 1) ^ ((C[2]) >> (64 - 1))); + D[2] = C[1] ^ (((C[3]) << 1) ^ ((C[3]) >> (64 - 1))); + D[3] = C[2] ^ (((C[4]) << 1) ^ ((C[4]) >> (64 - 1))); + D[4] = C[3] ^ (((C[0]) << 1) ^ ((C[0]) >> (64 - 1))); + E[0][0] ^= D[0]; + B[0][0] = E[0][0]; + E[1][1] ^= D[1]; + B[0][1] = (((E[1][1]) << 44) ^ ((E[1][1]) >> (64 - 44))); + E[2][2] ^= D[2]; + B[0][2] = (((E[2][2]) << 43) ^ ((E[2][2]) >> (64 - 43))); + E[3][3] ^= D[3]; + B[0][3] = (((E[3][3]) << 21) ^ ((E[3][3]) >> (64 - 21))); + E[4][4] ^= D[4]; + B[0][4] = (((E[4][4]) << 14) ^ ((E[4][4]) >> (64 - 14))); + A[0][0] = B[0][0] ^ ((~B[0][1]) & B[0][2]); + A[0][0] ^= KECCAKF1600_ROUND_CONSTANTS[11]; + C[0] = A[0][0]; + A[0][1] = B[0][1] ^ ((~B[0][2]) & B[0][3]); + C[1] = A[0][1]; + A[0][2] = B[0][2] ^ ((~B[0][3]) & B[0][4]); + C[2] = A[0][2]; + A[0][3] = B[0][3] ^ ((~B[0][4]) & B[0][0]); + C[3] = A[0][3]; + A[0][4] = B[0][4] ^ ((~B[0][0]) & B[0][1]); + C[4] = A[0][4]; + E[0][3] ^= D[3]; + B[1][0] = (((E[0][3]) << 28) ^ ((E[0][3]) >> (64 - 28))); + E[1][4] ^= D[4]; + B[1][1] = (((E[1][4]) << 20) ^ ((E[1][4]) >> (64 - 20))); + E[2][0] ^= D[0]; + B[1][2] = (((E[2][0]) << 3) ^ ((E[2][0]) >> (64 - 3))); + E[3][1] ^= D[1]; + B[1][3] = (((E[3][1]) << 45) ^ ((E[3][1]) >> (64 - 45))); + E[4][2] ^= D[2]; + B[1][4] = (((E[4][2]) << 61) ^ ((E[4][2]) >> (64 - 61))); + A[1][0] = B[1][0] ^ ((~B[1][1]) & B[1][2]); + C[0] ^= A[1][0]; + A[1][1] = B[1][1] ^ ((~B[1][2]) & B[1][3]); + C[1] ^= A[1][1]; + A[1][2] = B[1][2] ^ ((~B[1][3]) & B[1][4]); + C[2] ^= A[1][2]; + A[1][3] = B[1][3] ^ ((~B[1][4]) & B[1][0]); + C[3] ^= A[1][3]; + A[1][4] = B[1][4] ^ ((~B[1][0]) & B[1][1]); + C[4] ^= A[1][4]; + E[0][1] ^= D[1]; + B[2][0] = (((E[0][1]) << 1) ^ ((E[0][1]) >> (64 - 1))); + E[1][2] ^= D[2]; + B[2][1] = (((E[1][2]) << 6) ^ ((E[1][2]) >> (64 - 6))); + E[2][3] ^= D[3]; + B[2][2] = (((E[2][3]) << 25) ^ ((E[2][3]) >> (64 - 25))); + E[3][4] ^= D[4]; + B[2][3] = (((E[3][4]) << 8) ^ ((E[3][4]) >> (64 - 8))); + E[4][0] ^= D[0]; + B[2][4] = (((E[4][0]) << 18) ^ ((E[4][0]) >> (64 - 18))); + A[2][0] = B[2][0] ^ ((~B[2][1]) & B[2][2]); + C[0] ^= A[2][0]; + A[2][1] = B[2][1] ^ ((~B[2][2]) & B[2][3]); + C[1] ^= A[2][1]; + A[2][2] = B[2][2] ^ ((~B[2][3]) & B[2][4]); + C[2] ^= A[2][2]; + A[2][3] = B[2][3] ^ ((~B[2][4]) & B[2][0]); + C[3] ^= A[2][3]; + A[2][4] = B[2][4] ^ ((~B[2][0]) & B[2][1]); + C[4] ^= A[2][4]; + E[0][4] ^= D[4]; + B[3][0] = (((E[0][4]) << 27) ^ ((E[0][4]) >> (64 - 27))); + E[1][0] ^= D[0]; + B[3][1] = (((E[1][0]) << 36) ^ ((E[1][0]) >> (64 - 36))); + E[2][1] ^= D[1]; + B[3][2] = (((E[2][1]) << 10) ^ ((E[2][1]) >> (64 - 10))); + E[3][2] ^= D[2]; + B[3][3] = (((E[3][2]) << 15) ^ ((E[3][2]) >> (64 - 15))); + E[4][3] ^= D[3]; + B[3][4] = (((E[4][3]) << 56) ^ ((E[4][3]) >> (64 - 56))); + A[3][0] = B[3][0] ^ ((~B[3][1]) & B[3][2]); + C[0] ^= A[3][0]; + A[3][1] = B[3][1] ^ ((~B[3][2]) & B[3][3]); + C[1] ^= A[3][1]; + A[3][2] = B[3][2] ^ ((~B[3][3]) & B[3][4]); + C[2] ^= A[3][2]; + A[3][3] = B[3][3] ^ ((~B[3][4]) & B[3][0]); + C[3] ^= A[3][3]; + A[3][4] = B[3][4] ^ ((~B[3][0]) & B[3][1]); + C[4] ^= A[3][4]; + E[0][2] ^= D[2]; + B[4][0] = (((E[0][2]) << 62) ^ ((E[0][2]) >> (64 - 62))); + E[1][3] ^= D[3]; + B[4][1] = (((E[1][3]) << 55) ^ ((E[1][3]) >> (64 - 55))); + E[2][4] ^= D[4]; + B[4][2] = (((E[2][4]) << 39) ^ ((E[2][4]) >> (64 - 39))); + E[3][0] ^= D[0]; + B[4][3] = (((E[3][0]) << 41) ^ ((E[3][0]) >> (64 - 41))); + E[4][1] ^= D[1]; + B[4][4] = (((E[4][1]) << 2) ^ ((E[4][1]) >> (64 - 2))); + A[4][0] = B[4][0] ^ ((~B[4][1]) & B[4][2]); + C[0] ^= A[4][0]; + A[4][1] = B[4][1] ^ ((~B[4][2]) & B[4][3]); + C[1] ^= A[4][1]; + A[4][2] = B[4][2] ^ ((~B[4][3]) & B[4][4]); + C[2] ^= A[4][2]; + A[4][3] = B[4][3] ^ ((~B[4][4]) & B[4][0]); + C[3] ^= A[4][3]; + A[4][4] = B[4][4] ^ ((~B[4][0]) & B[4][1]); + C[4] ^= A[4][4]; + D[0] = C[4] ^ (((C[1]) << 1) ^ ((C[1]) >> (64 - 1))); + D[1] = C[0] ^ (((C[2]) << 1) ^ ((C[2]) >> (64 - 1))); + D[2] = C[1] ^ (((C[3]) << 1) ^ ((C[3]) >> (64 - 1))); + D[3] = C[2] ^ (((C[4]) << 1) ^ ((C[4]) >> (64 - 1))); + D[4] = C[3] ^ (((C[0]) << 1) ^ ((C[0]) >> (64 - 1))); + A[0][0] ^= D[0]; + B[0][0] = A[0][0]; + A[1][1] ^= D[1]; + B[0][1] = (((A[1][1]) << 44) ^ ((A[1][1]) >> (64 - 44))); + A[2][2] ^= D[2]; + B[0][2] = (((A[2][2]) << 43) ^ ((A[2][2]) >> (64 - 43))); + A[3][3] ^= D[3]; + B[0][3] = (((A[3][3]) << 21) ^ ((A[3][3]) >> (64 - 21))); + A[4][4] ^= D[4]; + B[0][4] = (((A[4][4]) << 14) ^ ((A[4][4]) >> (64 - 14))); + E[0][0] = B[0][0] ^ ((~B[0][1]) & B[0][2]); + E[0][0] ^= KECCAKF1600_ROUND_CONSTANTS[12]; + C[0] = E[0][0]; + E[0][1] = B[0][1] ^ ((~B[0][2]) & B[0][3]); + C[1] = E[0][1]; + E[0][2] = B[0][2] ^ ((~B[0][3]) & B[0][4]); + C[2] = E[0][2]; + E[0][3] = B[0][3] ^ ((~B[0][4]) & B[0][0]); + C[3] = E[0][3]; + E[0][4] = B[0][4] ^ ((~B[0][0]) & B[0][1]); + C[4] = E[0][4]; + A[0][3] ^= D[3]; + B[1][0] = (((A[0][3]) << 28) ^ ((A[0][3]) >> (64 - 28))); + A[1][4] ^= D[4]; + B[1][1] = (((A[1][4]) << 20) ^ ((A[1][4]) >> (64 - 20))); + A[2][0] ^= D[0]; + B[1][2] = (((A[2][0]) << 3) ^ ((A[2][0]) >> (64 - 3))); + A[3][1] ^= D[1]; + B[1][3] = (((A[3][1]) << 45) ^ ((A[3][1]) >> (64 - 45))); + A[4][2] ^= D[2]; + B[1][4] = (((A[4][2]) << 61) ^ ((A[4][2]) >> (64 - 61))); + E[1][0] = B[1][0] ^ ((~B[1][1]) & B[1][2]); + C[0] ^= E[1][0]; + E[1][1] = B[1][1] ^ ((~B[1][2]) & B[1][3]); + C[1] ^= E[1][1]; + E[1][2] = B[1][2] ^ ((~B[1][3]) & B[1][4]); + C[2] ^= E[1][2]; + E[1][3] = B[1][3] ^ ((~B[1][4]) & B[1][0]); + C[3] ^= E[1][3]; + E[1][4] = B[1][4] ^ ((~B[1][0]) & B[1][1]); + C[4] ^= E[1][4]; + A[0][1] ^= D[1]; + B[2][0] = (((A[0][1]) << 1) ^ ((A[0][1]) >> (64 - 1))); + A[1][2] ^= D[2]; + B[2][1] = (((A[1][2]) << 6) ^ ((A[1][2]) >> (64 - 6))); + A[2][3] ^= D[3]; + B[2][2] = (((A[2][3]) << 25) ^ ((A[2][3]) >> (64 - 25))); + A[3][4] ^= D[4]; + B[2][3] = (((A[3][4]) << 8) ^ ((A[3][4]) >> (64 - 8))); + A[4][0] ^= D[0]; + B[2][4] = (((A[4][0]) << 18) ^ ((A[4][0]) >> (64 - 18))); + E[2][0] = B[2][0] ^ ((~B[2][1]) & B[2][2]); + C[0] ^= E[2][0]; + E[2][1] = B[2][1] ^ ((~B[2][2]) & B[2][3]); + C[1] ^= E[2][1]; + E[2][2] = B[2][2] ^ ((~B[2][3]) & B[2][4]); + C[2] ^= E[2][2]; + E[2][3] = B[2][3] ^ ((~B[2][4]) & B[2][0]); + C[3] ^= E[2][3]; + E[2][4] = B[2][4] ^ ((~B[2][0]) & B[2][1]); + C[4] ^= E[2][4]; + A[0][4] ^= D[4]; + B[3][0] = (((A[0][4]) << 27) ^ ((A[0][4]) >> (64 - 27))); + A[1][0] ^= D[0]; + B[3][1] = (((A[1][0]) << 36) ^ ((A[1][0]) >> (64 - 36))); + A[2][1] ^= D[1]; + B[3][2] = (((A[2][1]) << 10) ^ ((A[2][1]) >> (64 - 10))); + A[3][2] ^= D[2]; + B[3][3] = (((A[3][2]) << 15) ^ ((A[3][2]) >> (64 - 15))); + A[4][3] ^= D[3]; + B[3][4] = (((A[4][3]) << 56) ^ ((A[4][3]) >> (64 - 56))); + E[3][0] = B[3][0] ^ ((~B[3][1]) & B[3][2]); + C[0] ^= E[3][0]; + E[3][1] = B[3][1] ^ ((~B[3][2]) & B[3][3]); + C[1] ^= E[3][1]; + E[3][2] = B[3][2] ^ ((~B[3][3]) & B[3][4]); + C[2] ^= E[3][2]; + E[3][3] = B[3][3] ^ ((~B[3][4]) & B[3][0]); + C[3] ^= E[3][3]; + E[3][4] = B[3][4] ^ ((~B[3][0]) & B[3][1]); + C[4] ^= E[3][4]; + A[0][2] ^= D[2]; + B[4][0] = (((A[0][2]) << 62) ^ ((A[0][2]) >> (64 - 62))); + A[1][3] ^= D[3]; + B[4][1] = (((A[1][3]) << 55) ^ ((A[1][3]) >> (64 - 55))); + A[2][4] ^= D[4]; + B[4][2] = (((A[2][4]) << 39) ^ ((A[2][4]) >> (64 - 39))); + A[3][0] ^= D[0]; + B[4][3] = (((A[3][0]) << 41) ^ ((A[3][0]) >> (64 - 41))); + A[4][1] ^= D[1]; + B[4][4] = (((A[4][1]) << 2) ^ ((A[4][1]) >> (64 - 2))); + E[4][0] = B[4][0] ^ ((~B[4][1]) & B[4][2]); + C[0] ^= E[4][0]; + E[4][1] = B[4][1] ^ ((~B[4][2]) & B[4][3]); + C[1] ^= E[4][1]; + E[4][2] = B[4][2] ^ ((~B[4][3]) & B[4][4]); + C[2] ^= E[4][2]; + E[4][3] = B[4][3] ^ ((~B[4][4]) & B[4][0]); + C[3] ^= E[4][3]; + E[4][4] = B[4][4] ^ ((~B[4][0]) & B[4][1]); + C[4] ^= E[4][4]; + D[0] = C[4] ^ (((C[1]) << 1) ^ ((C[1]) >> (64 - 1))); + D[1] = C[0] ^ (((C[2]) << 1) ^ ((C[2]) >> (64 - 1))); + D[2] = C[1] ^ (((C[3]) << 1) ^ ((C[3]) >> (64 - 1))); + D[3] = C[2] ^ (((C[4]) << 1) ^ ((C[4]) >> (64 - 1))); + D[4] = C[3] ^ (((C[0]) << 1) ^ ((C[0]) >> (64 - 1))); + E[0][0] ^= D[0]; + B[0][0] = E[0][0]; + E[1][1] ^= D[1]; + B[0][1] = (((E[1][1]) << 44) ^ ((E[1][1]) >> (64 - 44))); + E[2][2] ^= D[2]; + B[0][2] = (((E[2][2]) << 43) ^ ((E[2][2]) >> (64 - 43))); + E[3][3] ^= D[3]; + B[0][3] = (((E[3][3]) << 21) ^ ((E[3][3]) >> (64 - 21))); + E[4][4] ^= D[4]; + B[0][4] = (((E[4][4]) << 14) ^ ((E[4][4]) >> (64 - 14))); + A[0][0] = B[0][0] ^ ((~B[0][1]) & B[0][2]); + A[0][0] ^= KECCAKF1600_ROUND_CONSTANTS[13]; + C[0] = A[0][0]; + A[0][1] = B[0][1] ^ ((~B[0][2]) & B[0][3]); + C[1] = A[0][1]; + A[0][2] = B[0][2] ^ ((~B[0][3]) & B[0][4]); + C[2] = A[0][2]; + A[0][3] = B[0][3] ^ ((~B[0][4]) & B[0][0]); + C[3] = A[0][3]; + A[0][4] = B[0][4] ^ ((~B[0][0]) & B[0][1]); + C[4] = A[0][4]; + E[0][3] ^= D[3]; + B[1][0] = (((E[0][3]) << 28) ^ ((E[0][3]) >> (64 - 28))); + E[1][4] ^= D[4]; + B[1][1] = (((E[1][4]) << 20) ^ ((E[1][4]) >> (64 - 20))); + E[2][0] ^= D[0]; + B[1][2] = (((E[2][0]) << 3) ^ ((E[2][0]) >> (64 - 3))); + E[3][1] ^= D[1]; + B[1][3] = (((E[3][1]) << 45) ^ ((E[3][1]) >> (64 - 45))); + E[4][2] ^= D[2]; + B[1][4] = (((E[4][2]) << 61) ^ ((E[4][2]) >> (64 - 61))); + A[1][0] = B[1][0] ^ ((~B[1][1]) & B[1][2]); + C[0] ^= A[1][0]; + A[1][1] = B[1][1] ^ ((~B[1][2]) & B[1][3]); + C[1] ^= A[1][1]; + A[1][2] = B[1][2] ^ ((~B[1][3]) & B[1][4]); + C[2] ^= A[1][2]; + A[1][3] = B[1][3] ^ ((~B[1][4]) & B[1][0]); + C[3] ^= A[1][3]; + A[1][4] = B[1][4] ^ ((~B[1][0]) & B[1][1]); + C[4] ^= A[1][4]; + E[0][1] ^= D[1]; + B[2][0] = (((E[0][1]) << 1) ^ ((E[0][1]) >> (64 - 1))); + E[1][2] ^= D[2]; + B[2][1] = (((E[1][2]) << 6) ^ ((E[1][2]) >> (64 - 6))); + E[2][3] ^= D[3]; + B[2][2] = (((E[2][3]) << 25) ^ ((E[2][3]) >> (64 - 25))); + E[3][4] ^= D[4]; + B[2][3] = (((E[3][4]) << 8) ^ ((E[3][4]) >> (64 - 8))); + E[4][0] ^= D[0]; + B[2][4] = (((E[4][0]) << 18) ^ ((E[4][0]) >> (64 - 18))); + A[2][0] = B[2][0] ^ ((~B[2][1]) & B[2][2]); + C[0] ^= A[2][0]; + A[2][1] = B[2][1] ^ ((~B[2][2]) & B[2][3]); + C[1] ^= A[2][1]; + A[2][2] = B[2][2] ^ ((~B[2][3]) & B[2][4]); + C[2] ^= A[2][2]; + A[2][3] = B[2][3] ^ ((~B[2][4]) & B[2][0]); + C[3] ^= A[2][3]; + A[2][4] = B[2][4] ^ ((~B[2][0]) & B[2][1]); + C[4] ^= A[2][4]; + E[0][4] ^= D[4]; + B[3][0] = (((E[0][4]) << 27) ^ ((E[0][4]) >> (64 - 27))); + E[1][0] ^= D[0]; + B[3][1] = (((E[1][0]) << 36) ^ ((E[1][0]) >> (64 - 36))); + E[2][1] ^= D[1]; + B[3][2] = (((E[2][1]) << 10) ^ ((E[2][1]) >> (64 - 10))); + E[3][2] ^= D[2]; + B[3][3] = (((E[3][2]) << 15) ^ ((E[3][2]) >> (64 - 15))); + E[4][3] ^= D[3]; + B[3][4] = (((E[4][3]) << 56) ^ ((E[4][3]) >> (64 - 56))); + A[3][0] = B[3][0] ^ ((~B[3][1]) & B[3][2]); + C[0] ^= A[3][0]; + A[3][1] = B[3][1] ^ ((~B[3][2]) & B[3][3]); + C[1] ^= A[3][1]; + A[3][2] = B[3][2] ^ ((~B[3][3]) & B[3][4]); + C[2] ^= A[3][2]; + A[3][3] = B[3][3] ^ ((~B[3][4]) & B[3][0]); + C[3] ^= A[3][3]; + A[3][4] = B[3][4] ^ ((~B[3][0]) & B[3][1]); + C[4] ^= A[3][4]; + E[0][2] ^= D[2]; + B[4][0] = (((E[0][2]) << 62) ^ ((E[0][2]) >> (64 - 62))); + E[1][3] ^= D[3]; + B[4][1] = (((E[1][3]) << 55) ^ ((E[1][3]) >> (64 - 55))); + E[2][4] ^= D[4]; + B[4][2] = (((E[2][4]) << 39) ^ ((E[2][4]) >> (64 - 39))); + E[3][0] ^= D[0]; + B[4][3] = (((E[3][0]) << 41) ^ ((E[3][0]) >> (64 - 41))); + E[4][1] ^= D[1]; + B[4][4] = (((E[4][1]) << 2) ^ ((E[4][1]) >> (64 - 2))); + A[4][0] = B[4][0] ^ ((~B[4][1]) & B[4][2]); + C[0] ^= A[4][0]; + A[4][1] = B[4][1] ^ ((~B[4][2]) & B[4][3]); + C[1] ^= A[4][1]; + A[4][2] = B[4][2] ^ ((~B[4][3]) & B[4][4]); + C[2] ^= A[4][2]; + A[4][3] = B[4][3] ^ ((~B[4][4]) & B[4][0]); + C[3] ^= A[4][3]; + A[4][4] = B[4][4] ^ ((~B[4][0]) & B[4][1]); + C[4] ^= A[4][4]; + D[0] = C[4] ^ (((C[1]) << 1) ^ ((C[1]) >> (64 - 1))); + D[1] = C[0] ^ (((C[2]) << 1) ^ ((C[2]) >> (64 - 1))); + D[2] = C[1] ^ (((C[3]) << 1) ^ ((C[3]) >> (64 - 1))); + D[3] = C[2] ^ (((C[4]) << 1) ^ ((C[4]) >> (64 - 1))); + D[4] = C[3] ^ (((C[0]) << 1) ^ ((C[0]) >> (64 - 1))); + A[0][0] ^= D[0]; + B[0][0] = A[0][0]; + A[1][1] ^= D[1]; + B[0][1] = (((A[1][1]) << 44) ^ ((A[1][1]) >> (64 - 44))); + A[2][2] ^= D[2]; + B[0][2] = (((A[2][2]) << 43) ^ ((A[2][2]) >> (64 - 43))); + A[3][3] ^= D[3]; + B[0][3] = (((A[3][3]) << 21) ^ ((A[3][3]) >> (64 - 21))); + A[4][4] ^= D[4]; + B[0][4] = (((A[4][4]) << 14) ^ ((A[4][4]) >> (64 - 14))); + E[0][0] = B[0][0] ^ ((~B[0][1]) & B[0][2]); + E[0][0] ^= KECCAKF1600_ROUND_CONSTANTS[14]; + C[0] = E[0][0]; + E[0][1] = B[0][1] ^ ((~B[0][2]) & B[0][3]); + C[1] = E[0][1]; + E[0][2] = B[0][2] ^ ((~B[0][3]) & B[0][4]); + C[2] = E[0][2]; + E[0][3] = B[0][3] ^ ((~B[0][4]) & B[0][0]); + C[3] = E[0][3]; + E[0][4] = B[0][4] ^ ((~B[0][0]) & B[0][1]); + C[4] = E[0][4]; + A[0][3] ^= D[3]; + B[1][0] = (((A[0][3]) << 28) ^ ((A[0][3]) >> (64 - 28))); + A[1][4] ^= D[4]; + B[1][1] = (((A[1][4]) << 20) ^ ((A[1][4]) >> (64 - 20))); + A[2][0] ^= D[0]; + B[1][2] = (((A[2][0]) << 3) ^ ((A[2][0]) >> (64 - 3))); + A[3][1] ^= D[1]; + B[1][3] = (((A[3][1]) << 45) ^ ((A[3][1]) >> (64 - 45))); + A[4][2] ^= D[2]; + B[1][4] = (((A[4][2]) << 61) ^ ((A[4][2]) >> (64 - 61))); + E[1][0] = B[1][0] ^ ((~B[1][1]) & B[1][2]); + C[0] ^= E[1][0]; + E[1][1] = B[1][1] ^ ((~B[1][2]) & B[1][3]); + C[1] ^= E[1][1]; + E[1][2] = B[1][2] ^ ((~B[1][3]) & B[1][4]); + C[2] ^= E[1][2]; + E[1][3] = B[1][3] ^ ((~B[1][4]) & B[1][0]); + C[3] ^= E[1][3]; + E[1][4] = B[1][4] ^ ((~B[1][0]) & B[1][1]); + C[4] ^= E[1][4]; + A[0][1] ^= D[1]; + B[2][0] = (((A[0][1]) << 1) ^ ((A[0][1]) >> (64 - 1))); + A[1][2] ^= D[2]; + B[2][1] = (((A[1][2]) << 6) ^ ((A[1][2]) >> (64 - 6))); + A[2][3] ^= D[3]; + B[2][2] = (((A[2][3]) << 25) ^ ((A[2][3]) >> (64 - 25))); + A[3][4] ^= D[4]; + B[2][3] = (((A[3][4]) << 8) ^ ((A[3][4]) >> (64 - 8))); + A[4][0] ^= D[0]; + B[2][4] = (((A[4][0]) << 18) ^ ((A[4][0]) >> (64 - 18))); + E[2][0] = B[2][0] ^ ((~B[2][1]) & B[2][2]); + C[0] ^= E[2][0]; + E[2][1] = B[2][1] ^ ((~B[2][2]) & B[2][3]); + C[1] ^= E[2][1]; + E[2][2] = B[2][2] ^ ((~B[2][3]) & B[2][4]); + C[2] ^= E[2][2]; + E[2][3] = B[2][3] ^ ((~B[2][4]) & B[2][0]); + C[3] ^= E[2][3]; + E[2][4] = B[2][4] ^ ((~B[2][0]) & B[2][1]); + C[4] ^= E[2][4]; + A[0][4] ^= D[4]; + B[3][0] = (((A[0][4]) << 27) ^ ((A[0][4]) >> (64 - 27))); + A[1][0] ^= D[0]; + B[3][1] = (((A[1][0]) << 36) ^ ((A[1][0]) >> (64 - 36))); + A[2][1] ^= D[1]; + B[3][2] = (((A[2][1]) << 10) ^ ((A[2][1]) >> (64 - 10))); + A[3][2] ^= D[2]; + B[3][3] = (((A[3][2]) << 15) ^ ((A[3][2]) >> (64 - 15))); + A[4][3] ^= D[3]; + B[3][4] = (((A[4][3]) << 56) ^ ((A[4][3]) >> (64 - 56))); + E[3][0] = B[3][0] ^ ((~B[3][1]) & B[3][2]); + C[0] ^= E[3][0]; + E[3][1] = B[3][1] ^ ((~B[3][2]) & B[3][3]); + C[1] ^= E[3][1]; + E[3][2] = B[3][2] ^ ((~B[3][3]) & B[3][4]); + C[2] ^= E[3][2]; + E[3][3] = B[3][3] ^ ((~B[3][4]) & B[3][0]); + C[3] ^= E[3][3]; + E[3][4] = B[3][4] ^ ((~B[3][0]) & B[3][1]); + C[4] ^= E[3][4]; + A[0][2] ^= D[2]; + B[4][0] = (((A[0][2]) << 62) ^ ((A[0][2]) >> (64 - 62))); + A[1][3] ^= D[3]; + B[4][1] = (((A[1][3]) << 55) ^ ((A[1][3]) >> (64 - 55))); + A[2][4] ^= D[4]; + B[4][2] = (((A[2][4]) << 39) ^ ((A[2][4]) >> (64 - 39))); + A[3][0] ^= D[0]; + B[4][3] = (((A[3][0]) << 41) ^ ((A[3][0]) >> (64 - 41))); + A[4][1] ^= D[1]; + B[4][4] = (((A[4][1]) << 2) ^ ((A[4][1]) >> (64 - 2))); + E[4][0] = B[4][0] ^ ((~B[4][1]) & B[4][2]); + C[0] ^= E[4][0]; + E[4][1] = B[4][1] ^ ((~B[4][2]) & B[4][3]); + C[1] ^= E[4][1]; + E[4][2] = B[4][2] ^ ((~B[4][3]) & B[4][4]); + C[2] ^= E[4][2]; + E[4][3] = B[4][3] ^ ((~B[4][4]) & B[4][0]); + C[3] ^= E[4][3]; + E[4][4] = B[4][4] ^ ((~B[4][0]) & B[4][1]); + C[4] ^= E[4][4]; + D[0] = C[4] ^ (((C[1]) << 1) ^ ((C[1]) >> (64 - 1))); + D[1] = C[0] ^ (((C[2]) << 1) ^ ((C[2]) >> (64 - 1))); + D[2] = C[1] ^ (((C[3]) << 1) ^ ((C[3]) >> (64 - 1))); + D[3] = C[2] ^ (((C[4]) << 1) ^ ((C[4]) >> (64 - 1))); + D[4] = C[3] ^ (((C[0]) << 1) ^ ((C[0]) >> (64 - 1))); + E[0][0] ^= D[0]; + B[0][0] = E[0][0]; + E[1][1] ^= D[1]; + B[0][1] = (((E[1][1]) << 44) ^ ((E[1][1]) >> (64 - 44))); + E[2][2] ^= D[2]; + B[0][2] = (((E[2][2]) << 43) ^ ((E[2][2]) >> (64 - 43))); + E[3][3] ^= D[3]; + B[0][3] = (((E[3][3]) << 21) ^ ((E[3][3]) >> (64 - 21))); + E[4][4] ^= D[4]; + B[0][4] = (((E[4][4]) << 14) ^ ((E[4][4]) >> (64 - 14))); + A[0][0] = B[0][0] ^ ((~B[0][1]) & B[0][2]); + A[0][0] ^= KECCAKF1600_ROUND_CONSTANTS[15]; + C[0] = A[0][0]; + A[0][1] = B[0][1] ^ ((~B[0][2]) & B[0][3]); + C[1] = A[0][1]; + A[0][2] = B[0][2] ^ ((~B[0][3]) & B[0][4]); + C[2] = A[0][2]; + A[0][3] = B[0][3] ^ ((~B[0][4]) & B[0][0]); + C[3] = A[0][3]; + A[0][4] = B[0][4] ^ ((~B[0][0]) & B[0][1]); + C[4] = A[0][4]; + E[0][3] ^= D[3]; + B[1][0] = (((E[0][3]) << 28) ^ ((E[0][3]) >> (64 - 28))); + E[1][4] ^= D[4]; + B[1][1] = (((E[1][4]) << 20) ^ ((E[1][4]) >> (64 - 20))); + E[2][0] ^= D[0]; + B[1][2] = (((E[2][0]) << 3) ^ ((E[2][0]) >> (64 - 3))); + E[3][1] ^= D[1]; + B[1][3] = (((E[3][1]) << 45) ^ ((E[3][1]) >> (64 - 45))); + E[4][2] ^= D[2]; + B[1][4] = (((E[4][2]) << 61) ^ ((E[4][2]) >> (64 - 61))); + A[1][0] = B[1][0] ^ ((~B[1][1]) & B[1][2]); + C[0] ^= A[1][0]; + A[1][1] = B[1][1] ^ ((~B[1][2]) & B[1][3]); + C[1] ^= A[1][1]; + A[1][2] = B[1][2] ^ ((~B[1][3]) & B[1][4]); + C[2] ^= A[1][2]; + A[1][3] = B[1][3] ^ ((~B[1][4]) & B[1][0]); + C[3] ^= A[1][3]; + A[1][4] = B[1][4] ^ ((~B[1][0]) & B[1][1]); + C[4] ^= A[1][4]; + E[0][1] ^= D[1]; + B[2][0] = (((E[0][1]) << 1) ^ ((E[0][1]) >> (64 - 1))); + E[1][2] ^= D[2]; + B[2][1] = (((E[1][2]) << 6) ^ ((E[1][2]) >> (64 - 6))); + E[2][3] ^= D[3]; + B[2][2] = (((E[2][3]) << 25) ^ ((E[2][3]) >> (64 - 25))); + E[3][4] ^= D[4]; + B[2][3] = (((E[3][4]) << 8) ^ ((E[3][4]) >> (64 - 8))); + E[4][0] ^= D[0]; + B[2][4] = (((E[4][0]) << 18) ^ ((E[4][0]) >> (64 - 18))); + A[2][0] = B[2][0] ^ ((~B[2][1]) & B[2][2]); + C[0] ^= A[2][0]; + A[2][1] = B[2][1] ^ ((~B[2][2]) & B[2][3]); + C[1] ^= A[2][1]; + A[2][2] = B[2][2] ^ ((~B[2][3]) & B[2][4]); + C[2] ^= A[2][2]; + A[2][3] = B[2][3] ^ ((~B[2][4]) & B[2][0]); + C[3] ^= A[2][3]; + A[2][4] = B[2][4] ^ ((~B[2][0]) & B[2][1]); + C[4] ^= A[2][4]; + E[0][4] ^= D[4]; + B[3][0] = (((E[0][4]) << 27) ^ ((E[0][4]) >> (64 - 27))); + E[1][0] ^= D[0]; + B[3][1] = (((E[1][0]) << 36) ^ ((E[1][0]) >> (64 - 36))); + E[2][1] ^= D[1]; + B[3][2] = (((E[2][1]) << 10) ^ ((E[2][1]) >> (64 - 10))); + E[3][2] ^= D[2]; + B[3][3] = (((E[3][2]) << 15) ^ ((E[3][2]) >> (64 - 15))); + E[4][3] ^= D[3]; + B[3][4] = (((E[4][3]) << 56) ^ ((E[4][3]) >> (64 - 56))); + A[3][0] = B[3][0] ^ ((~B[3][1]) & B[3][2]); + C[0] ^= A[3][0]; + A[3][1] = B[3][1] ^ ((~B[3][2]) & B[3][3]); + C[1] ^= A[3][1]; + A[3][2] = B[3][2] ^ ((~B[3][3]) & B[3][4]); + C[2] ^= A[3][2]; + A[3][3] = B[3][3] ^ ((~B[3][4]) & B[3][0]); + C[3] ^= A[3][3]; + A[3][4] = B[3][4] ^ ((~B[3][0]) & B[3][1]); + C[4] ^= A[3][4]; + E[0][2] ^= D[2]; + B[4][0] = (((E[0][2]) << 62) ^ ((E[0][2]) >> (64 - 62))); + E[1][3] ^= D[3]; + B[4][1] = (((E[1][3]) << 55) ^ ((E[1][3]) >> (64 - 55))); + E[2][4] ^= D[4]; + B[4][2] = (((E[2][4]) << 39) ^ ((E[2][4]) >> (64 - 39))); + E[3][0] ^= D[0]; + B[4][3] = (((E[3][0]) << 41) ^ ((E[3][0]) >> (64 - 41))); + E[4][1] ^= D[1]; + B[4][4] = (((E[4][1]) << 2) ^ ((E[4][1]) >> (64 - 2))); + A[4][0] = B[4][0] ^ ((~B[4][1]) & B[4][2]); + C[0] ^= A[4][0]; + A[4][1] = B[4][1] ^ ((~B[4][2]) & B[4][3]); + C[1] ^= A[4][1]; + A[4][2] = B[4][2] ^ ((~B[4][3]) & B[4][4]); + C[2] ^= A[4][2]; + A[4][3] = B[4][3] ^ ((~B[4][4]) & B[4][0]); + C[3] ^= A[4][3]; + A[4][4] = B[4][4] ^ ((~B[4][0]) & B[4][1]); + C[4] ^= A[4][4]; + D[0] = C[4] ^ (((C[1]) << 1) ^ ((C[1]) >> (64 - 1))); + D[1] = C[0] ^ (((C[2]) << 1) ^ ((C[2]) >> (64 - 1))); + D[2] = C[1] ^ (((C[3]) << 1) ^ ((C[3]) >> (64 - 1))); + D[3] = C[2] ^ (((C[4]) << 1) ^ ((C[4]) >> (64 - 1))); + D[4] = C[3] ^ (((C[0]) << 1) ^ ((C[0]) >> (64 - 1))); + A[0][0] ^= D[0]; + B[0][0] = A[0][0]; + A[1][1] ^= D[1]; + B[0][1] = (((A[1][1]) << 44) ^ ((A[1][1]) >> (64 - 44))); + A[2][2] ^= D[2]; + B[0][2] = (((A[2][2]) << 43) ^ ((A[2][2]) >> (64 - 43))); + A[3][3] ^= D[3]; + B[0][3] = (((A[3][3]) << 21) ^ ((A[3][3]) >> (64 - 21))); + A[4][4] ^= D[4]; + B[0][4] = (((A[4][4]) << 14) ^ ((A[4][4]) >> (64 - 14))); + E[0][0] = B[0][0] ^ ((~B[0][1]) & B[0][2]); + E[0][0] ^= KECCAKF1600_ROUND_CONSTANTS[16]; + C[0] = E[0][0]; + E[0][1] = B[0][1] ^ ((~B[0][2]) & B[0][3]); + C[1] = E[0][1]; + E[0][2] = B[0][2] ^ ((~B[0][3]) & B[0][4]); + C[2] = E[0][2]; + E[0][3] = B[0][3] ^ ((~B[0][4]) & B[0][0]); + C[3] = E[0][3]; + E[0][4] = B[0][4] ^ ((~B[0][0]) & B[0][1]); + C[4] = E[0][4]; + A[0][3] ^= D[3]; + B[1][0] = (((A[0][3]) << 28) ^ ((A[0][3]) >> (64 - 28))); + A[1][4] ^= D[4]; + B[1][1] = (((A[1][4]) << 20) ^ ((A[1][4]) >> (64 - 20))); + A[2][0] ^= D[0]; + B[1][2] = (((A[2][0]) << 3) ^ ((A[2][0]) >> (64 - 3))); + A[3][1] ^= D[1]; + B[1][3] = (((A[3][1]) << 45) ^ ((A[3][1]) >> (64 - 45))); + A[4][2] ^= D[2]; + B[1][4] = (((A[4][2]) << 61) ^ ((A[4][2]) >> (64 - 61))); + E[1][0] = B[1][0] ^ ((~B[1][1]) & B[1][2]); + C[0] ^= E[1][0]; + E[1][1] = B[1][1] ^ ((~B[1][2]) & B[1][3]); + C[1] ^= E[1][1]; + E[1][2] = B[1][2] ^ ((~B[1][3]) & B[1][4]); + C[2] ^= E[1][2]; + E[1][3] = B[1][3] ^ ((~B[1][4]) & B[1][0]); + C[3] ^= E[1][3]; + E[1][4] = B[1][4] ^ ((~B[1][0]) & B[1][1]); + C[4] ^= E[1][4]; + A[0][1] ^= D[1]; + B[2][0] = (((A[0][1]) << 1) ^ ((A[0][1]) >> (64 - 1))); + A[1][2] ^= D[2]; + B[2][1] = (((A[1][2]) << 6) ^ ((A[1][2]) >> (64 - 6))); + A[2][3] ^= D[3]; + B[2][2] = (((A[2][3]) << 25) ^ ((A[2][3]) >> (64 - 25))); + A[3][4] ^= D[4]; + B[2][3] = (((A[3][4]) << 8) ^ ((A[3][4]) >> (64 - 8))); + A[4][0] ^= D[0]; + B[2][4] = (((A[4][0]) << 18) ^ ((A[4][0]) >> (64 - 18))); + E[2][0] = B[2][0] ^ ((~B[2][1]) & B[2][2]); + C[0] ^= E[2][0]; + E[2][1] = B[2][1] ^ ((~B[2][2]) & B[2][3]); + C[1] ^= E[2][1]; + E[2][2] = B[2][2] ^ ((~B[2][3]) & B[2][4]); + C[2] ^= E[2][2]; + E[2][3] = B[2][3] ^ ((~B[2][4]) & B[2][0]); + C[3] ^= E[2][3]; + E[2][4] = B[2][4] ^ ((~B[2][0]) & B[2][1]); + C[4] ^= E[2][4]; + A[0][4] ^= D[4]; + B[3][0] = (((A[0][4]) << 27) ^ ((A[0][4]) >> (64 - 27))); + A[1][0] ^= D[0]; + B[3][1] = (((A[1][0]) << 36) ^ ((A[1][0]) >> (64 - 36))); + A[2][1] ^= D[1]; + B[3][2] = (((A[2][1]) << 10) ^ ((A[2][1]) >> (64 - 10))); + A[3][2] ^= D[2]; + B[3][3] = (((A[3][2]) << 15) ^ ((A[3][2]) >> (64 - 15))); + A[4][3] ^= D[3]; + B[3][4] = (((A[4][3]) << 56) ^ ((A[4][3]) >> (64 - 56))); + E[3][0] = B[3][0] ^ ((~B[3][1]) & B[3][2]); + C[0] ^= E[3][0]; + E[3][1] = B[3][1] ^ ((~B[3][2]) & B[3][3]); + C[1] ^= E[3][1]; + E[3][2] = B[3][2] ^ ((~B[3][3]) & B[3][4]); + C[2] ^= E[3][2]; + E[3][3] = B[3][3] ^ ((~B[3][4]) & B[3][0]); + C[3] ^= E[3][3]; + E[3][4] = B[3][4] ^ ((~B[3][0]) & B[3][1]); + C[4] ^= E[3][4]; + A[0][2] ^= D[2]; + B[4][0] = (((A[0][2]) << 62) ^ ((A[0][2]) >> (64 - 62))); + A[1][3] ^= D[3]; + B[4][1] = (((A[1][3]) << 55) ^ ((A[1][3]) >> (64 - 55))); + A[2][4] ^= D[4]; + B[4][2] = (((A[2][4]) << 39) ^ ((A[2][4]) >> (64 - 39))); + A[3][0] ^= D[0]; + B[4][3] = (((A[3][0]) << 41) ^ ((A[3][0]) >> (64 - 41))); + A[4][1] ^= D[1]; + B[4][4] = (((A[4][1]) << 2) ^ ((A[4][1]) >> (64 - 2))); + E[4][0] = B[4][0] ^ ((~B[4][1]) & B[4][2]); + C[0] ^= E[4][0]; + E[4][1] = B[4][1] ^ ((~B[4][2]) & B[4][3]); + C[1] ^= E[4][1]; + E[4][2] = B[4][2] ^ ((~B[4][3]) & B[4][4]); + C[2] ^= E[4][2]; + E[4][3] = B[4][3] ^ ((~B[4][4]) & B[4][0]); + C[3] ^= E[4][3]; + E[4][4] = B[4][4] ^ ((~B[4][0]) & B[4][1]); + C[4] ^= E[4][4]; + D[0] = C[4] ^ (((C[1]) << 1) ^ ((C[1]) >> (64 - 1))); + D[1] = C[0] ^ (((C[2]) << 1) ^ ((C[2]) >> (64 - 1))); + D[2] = C[1] ^ (((C[3]) << 1) ^ ((C[3]) >> (64 - 1))); + D[3] = C[2] ^ (((C[4]) << 1) ^ ((C[4]) >> (64 - 1))); + D[4] = C[3] ^ (((C[0]) << 1) ^ ((C[0]) >> (64 - 1))); + E[0][0] ^= D[0]; + B[0][0] = E[0][0]; + E[1][1] ^= D[1]; + B[0][1] = (((E[1][1]) << 44) ^ ((E[1][1]) >> (64 - 44))); + E[2][2] ^= D[2]; + B[0][2] = (((E[2][2]) << 43) ^ ((E[2][2]) >> (64 - 43))); + E[3][3] ^= D[3]; + B[0][3] = (((E[3][3]) << 21) ^ ((E[3][3]) >> (64 - 21))); + E[4][4] ^= D[4]; + B[0][4] = (((E[4][4]) << 14) ^ ((E[4][4]) >> (64 - 14))); + A[0][0] = B[0][0] ^ ((~B[0][1]) & B[0][2]); + A[0][0] ^= KECCAKF1600_ROUND_CONSTANTS[17]; + C[0] = A[0][0]; + A[0][1] = B[0][1] ^ ((~B[0][2]) & B[0][3]); + C[1] = A[0][1]; + A[0][2] = B[0][2] ^ ((~B[0][3]) & B[0][4]); + C[2] = A[0][2]; + A[0][3] = B[0][3] ^ ((~B[0][4]) & B[0][0]); + C[3] = A[0][3]; + A[0][4] = B[0][4] ^ ((~B[0][0]) & B[0][1]); + C[4] = A[0][4]; + E[0][3] ^= D[3]; + B[1][0] = (((E[0][3]) << 28) ^ ((E[0][3]) >> (64 - 28))); + E[1][4] ^= D[4]; + B[1][1] = (((E[1][4]) << 20) ^ ((E[1][4]) >> (64 - 20))); + E[2][0] ^= D[0]; + B[1][2] = (((E[2][0]) << 3) ^ ((E[2][0]) >> (64 - 3))); + E[3][1] ^= D[1]; + B[1][3] = (((E[3][1]) << 45) ^ ((E[3][1]) >> (64 - 45))); + E[4][2] ^= D[2]; + B[1][4] = (((E[4][2]) << 61) ^ ((E[4][2]) >> (64 - 61))); + A[1][0] = B[1][0] ^ ((~B[1][1]) & B[1][2]); + C[0] ^= A[1][0]; + A[1][1] = B[1][1] ^ ((~B[1][2]) & B[1][3]); + C[1] ^= A[1][1]; + A[1][2] = B[1][2] ^ ((~B[1][3]) & B[1][4]); + C[2] ^= A[1][2]; + A[1][3] = B[1][3] ^ ((~B[1][4]) & B[1][0]); + C[3] ^= A[1][3]; + A[1][4] = B[1][4] ^ ((~B[1][0]) & B[1][1]); + C[4] ^= A[1][4]; + E[0][1] ^= D[1]; + B[2][0] = (((E[0][1]) << 1) ^ ((E[0][1]) >> (64 - 1))); + E[1][2] ^= D[2]; + B[2][1] = (((E[1][2]) << 6) ^ ((E[1][2]) >> (64 - 6))); + E[2][3] ^= D[3]; + B[2][2] = (((E[2][3]) << 25) ^ ((E[2][3]) >> (64 - 25))); + E[3][4] ^= D[4]; + B[2][3] = (((E[3][4]) << 8) ^ ((E[3][4]) >> (64 - 8))); + E[4][0] ^= D[0]; + B[2][4] = (((E[4][0]) << 18) ^ ((E[4][0]) >> (64 - 18))); + A[2][0] = B[2][0] ^ ((~B[2][1]) & B[2][2]); + C[0] ^= A[2][0]; + A[2][1] = B[2][1] ^ ((~B[2][2]) & B[2][3]); + C[1] ^= A[2][1]; + A[2][2] = B[2][2] ^ ((~B[2][3]) & B[2][4]); + C[2] ^= A[2][2]; + A[2][3] = B[2][3] ^ ((~B[2][4]) & B[2][0]); + C[3] ^= A[2][3]; + A[2][4] = B[2][4] ^ ((~B[2][0]) & B[2][1]); + C[4] ^= A[2][4]; + E[0][4] ^= D[4]; + B[3][0] = (((E[0][4]) << 27) ^ ((E[0][4]) >> (64 - 27))); + E[1][0] ^= D[0]; + B[3][1] = (((E[1][0]) << 36) ^ ((E[1][0]) >> (64 - 36))); + E[2][1] ^= D[1]; + B[3][2] = (((E[2][1]) << 10) ^ ((E[2][1]) >> (64 - 10))); + E[3][2] ^= D[2]; + B[3][3] = (((E[3][2]) << 15) ^ ((E[3][2]) >> (64 - 15))); + E[4][3] ^= D[3]; + B[3][4] = (((E[4][3]) << 56) ^ ((E[4][3]) >> (64 - 56))); + A[3][0] = B[3][0] ^ ((~B[3][1]) & B[3][2]); + C[0] ^= A[3][0]; + A[3][1] = B[3][1] ^ ((~B[3][2]) & B[3][3]); + C[1] ^= A[3][1]; + A[3][2] = B[3][2] ^ ((~B[3][3]) & B[3][4]); + C[2] ^= A[3][2]; + A[3][3] = B[3][3] ^ ((~B[3][4]) & B[3][0]); + C[3] ^= A[3][3]; + A[3][4] = B[3][4] ^ ((~B[3][0]) & B[3][1]); + C[4] ^= A[3][4]; + E[0][2] ^= D[2]; + B[4][0] = (((E[0][2]) << 62) ^ ((E[0][2]) >> (64 - 62))); + E[1][3] ^= D[3]; + B[4][1] = (((E[1][3]) << 55) ^ ((E[1][3]) >> (64 - 55))); + E[2][4] ^= D[4]; + B[4][2] = (((E[2][4]) << 39) ^ ((E[2][4]) >> (64 - 39))); + E[3][0] ^= D[0]; + B[4][3] = (((E[3][0]) << 41) ^ ((E[3][0]) >> (64 - 41))); + E[4][1] ^= D[1]; + B[4][4] = (((E[4][1]) << 2) ^ ((E[4][1]) >> (64 - 2))); + A[4][0] = B[4][0] ^ ((~B[4][1]) & B[4][2]); + C[0] ^= A[4][0]; + A[4][1] = B[4][1] ^ ((~B[4][2]) & B[4][3]); + C[1] ^= A[4][1]; + A[4][2] = B[4][2] ^ ((~B[4][3]) & B[4][4]); + C[2] ^= A[4][2]; + A[4][3] = B[4][3] ^ ((~B[4][4]) & B[4][0]); + C[3] ^= A[4][3]; + A[4][4] = B[4][4] ^ ((~B[4][0]) & B[4][1]); + C[4] ^= A[4][4]; + D[0] = C[4] ^ (((C[1]) << 1) ^ ((C[1]) >> (64 - 1))); + D[1] = C[0] ^ (((C[2]) << 1) ^ ((C[2]) >> (64 - 1))); + D[2] = C[1] ^ (((C[3]) << 1) ^ ((C[3]) >> (64 - 1))); + D[3] = C[2] ^ (((C[4]) << 1) ^ ((C[4]) >> (64 - 1))); + D[4] = C[3] ^ (((C[0]) << 1) ^ ((C[0]) >> (64 - 1))); + A[0][0] ^= D[0]; + B[0][0] = A[0][0]; + A[1][1] ^= D[1]; + B[0][1] = (((A[1][1]) << 44) ^ ((A[1][1]) >> (64 - 44))); + A[2][2] ^= D[2]; + B[0][2] = (((A[2][2]) << 43) ^ ((A[2][2]) >> (64 - 43))); + A[3][3] ^= D[3]; + B[0][3] = (((A[3][3]) << 21) ^ ((A[3][3]) >> (64 - 21))); + A[4][4] ^= D[4]; + B[0][4] = (((A[4][4]) << 14) ^ ((A[4][4]) >> (64 - 14))); + E[0][0] = B[0][0] ^ ((~B[0][1]) & B[0][2]); + E[0][0] ^= KECCAKF1600_ROUND_CONSTANTS[18]; + C[0] = E[0][0]; + E[0][1] = B[0][1] ^ ((~B[0][2]) & B[0][3]); + C[1] = E[0][1]; + E[0][2] = B[0][2] ^ ((~B[0][3]) & B[0][4]); + C[2] = E[0][2]; + E[0][3] = B[0][3] ^ ((~B[0][4]) & B[0][0]); + C[3] = E[0][3]; + E[0][4] = B[0][4] ^ ((~B[0][0]) & B[0][1]); + C[4] = E[0][4]; + A[0][3] ^= D[3]; + B[1][0] = (((A[0][3]) << 28) ^ ((A[0][3]) >> (64 - 28))); + A[1][4] ^= D[4]; + B[1][1] = (((A[1][4]) << 20) ^ ((A[1][4]) >> (64 - 20))); + A[2][0] ^= D[0]; + B[1][2] = (((A[2][0]) << 3) ^ ((A[2][0]) >> (64 - 3))); + A[3][1] ^= D[1]; + B[1][3] = (((A[3][1]) << 45) ^ ((A[3][1]) >> (64 - 45))); + A[4][2] ^= D[2]; + B[1][4] = (((A[4][2]) << 61) ^ ((A[4][2]) >> (64 - 61))); + E[1][0] = B[1][0] ^ ((~B[1][1]) & B[1][2]); + C[0] ^= E[1][0]; + E[1][1] = B[1][1] ^ ((~B[1][2]) & B[1][3]); + C[1] ^= E[1][1]; + E[1][2] = B[1][2] ^ ((~B[1][3]) & B[1][4]); + C[2] ^= E[1][2]; + E[1][3] = B[1][3] ^ ((~B[1][4]) & B[1][0]); + C[3] ^= E[1][3]; + E[1][4] = B[1][4] ^ ((~B[1][0]) & B[1][1]); + C[4] ^= E[1][4]; + A[0][1] ^= D[1]; + B[2][0] = (((A[0][1]) << 1) ^ ((A[0][1]) >> (64 - 1))); + A[1][2] ^= D[2]; + B[2][1] = (((A[1][2]) << 6) ^ ((A[1][2]) >> (64 - 6))); + A[2][3] ^= D[3]; + B[2][2] = (((A[2][3]) << 25) ^ ((A[2][3]) >> (64 - 25))); + A[3][4] ^= D[4]; + B[2][3] = (((A[3][4]) << 8) ^ ((A[3][4]) >> (64 - 8))); + A[4][0] ^= D[0]; + B[2][4] = (((A[4][0]) << 18) ^ ((A[4][0]) >> (64 - 18))); + E[2][0] = B[2][0] ^ ((~B[2][1]) & B[2][2]); + C[0] ^= E[2][0]; + E[2][1] = B[2][1] ^ ((~B[2][2]) & B[2][3]); + C[1] ^= E[2][1]; + E[2][2] = B[2][2] ^ ((~B[2][3]) & B[2][4]); + C[2] ^= E[2][2]; + E[2][3] = B[2][3] ^ ((~B[2][4]) & B[2][0]); + C[3] ^= E[2][3]; + E[2][4] = B[2][4] ^ ((~B[2][0]) & B[2][1]); + C[4] ^= E[2][4]; + A[0][4] ^= D[4]; + B[3][0] = (((A[0][4]) << 27) ^ ((A[0][4]) >> (64 - 27))); + A[1][0] ^= D[0]; + B[3][1] = (((A[1][0]) << 36) ^ ((A[1][0]) >> (64 - 36))); + A[2][1] ^= D[1]; + B[3][2] = (((A[2][1]) << 10) ^ ((A[2][1]) >> (64 - 10))); + A[3][2] ^= D[2]; + B[3][3] = (((A[3][2]) << 15) ^ ((A[3][2]) >> (64 - 15))); + A[4][3] ^= D[3]; + B[3][4] = (((A[4][3]) << 56) ^ ((A[4][3]) >> (64 - 56))); + E[3][0] = B[3][0] ^ ((~B[3][1]) & B[3][2]); + C[0] ^= E[3][0]; + E[3][1] = B[3][1] ^ ((~B[3][2]) & B[3][3]); + C[1] ^= E[3][1]; + E[3][2] = B[3][2] ^ ((~B[3][3]) & B[3][4]); + C[2] ^= E[3][2]; + E[3][3] = B[3][3] ^ ((~B[3][4]) & B[3][0]); + C[3] ^= E[3][3]; + E[3][4] = B[3][4] ^ ((~B[3][0]) & B[3][1]); + C[4] ^= E[3][4]; + A[0][2] ^= D[2]; + B[4][0] = (((A[0][2]) << 62) ^ ((A[0][2]) >> (64 - 62))); + A[1][3] ^= D[3]; + B[4][1] = (((A[1][3]) << 55) ^ ((A[1][3]) >> (64 - 55))); + A[2][4] ^= D[4]; + B[4][2] = (((A[2][4]) << 39) ^ ((A[2][4]) >> (64 - 39))); + A[3][0] ^= D[0]; + B[4][3] = (((A[3][0]) << 41) ^ ((A[3][0]) >> (64 - 41))); + A[4][1] ^= D[1]; + B[4][4] = (((A[4][1]) << 2) ^ ((A[4][1]) >> (64 - 2))); + E[4][0] = B[4][0] ^ ((~B[4][1]) & B[4][2]); + C[0] ^= E[4][0]; + E[4][1] = B[4][1] ^ ((~B[4][2]) & B[4][3]); + C[1] ^= E[4][1]; + E[4][2] = B[4][2] ^ ((~B[4][3]) & B[4][4]); + C[2] ^= E[4][2]; + E[4][3] = B[4][3] ^ ((~B[4][4]) & B[4][0]); + C[3] ^= E[4][3]; + E[4][4] = B[4][4] ^ ((~B[4][0]) & B[4][1]); + C[4] ^= E[4][4]; + D[0] = C[4] ^ (((C[1]) << 1) ^ ((C[1]) >> (64 - 1))); + D[1] = C[0] ^ (((C[2]) << 1) ^ ((C[2]) >> (64 - 1))); + D[2] = C[1] ^ (((C[3]) << 1) ^ ((C[3]) >> (64 - 1))); + D[3] = C[2] ^ (((C[4]) << 1) ^ ((C[4]) >> (64 - 1))); + D[4] = C[3] ^ (((C[0]) << 1) ^ ((C[0]) >> (64 - 1))); + E[0][0] ^= D[0]; + B[0][0] = E[0][0]; + E[1][1] ^= D[1]; + B[0][1] = (((E[1][1]) << 44) ^ ((E[1][1]) >> (64 - 44))); + E[2][2] ^= D[2]; + B[0][2] = (((E[2][2]) << 43) ^ ((E[2][2]) >> (64 - 43))); + E[3][3] ^= D[3]; + B[0][3] = (((E[3][3]) << 21) ^ ((E[3][3]) >> (64 - 21))); + E[4][4] ^= D[4]; + B[0][4] = (((E[4][4]) << 14) ^ ((E[4][4]) >> (64 - 14))); + A[0][0] = B[0][0] ^ ((~B[0][1]) & B[0][2]); + A[0][0] ^= KECCAKF1600_ROUND_CONSTANTS[19]; + C[0] = A[0][0]; + A[0][1] = B[0][1] ^ ((~B[0][2]) & B[0][3]); + C[1] = A[0][1]; + A[0][2] = B[0][2] ^ ((~B[0][3]) & B[0][4]); + C[2] = A[0][2]; + A[0][3] = B[0][3] ^ ((~B[0][4]) & B[0][0]); + C[3] = A[0][3]; + A[0][4] = B[0][4] ^ ((~B[0][0]) & B[0][1]); + C[4] = A[0][4]; + E[0][3] ^= D[3]; + B[1][0] = (((E[0][3]) << 28) ^ ((E[0][3]) >> (64 - 28))); + E[1][4] ^= D[4]; + B[1][1] = (((E[1][4]) << 20) ^ ((E[1][4]) >> (64 - 20))); + E[2][0] ^= D[0]; + B[1][2] = (((E[2][0]) << 3) ^ ((E[2][0]) >> (64 - 3))); + E[3][1] ^= D[1]; + B[1][3] = (((E[3][1]) << 45) ^ ((E[3][1]) >> (64 - 45))); + E[4][2] ^= D[2]; + B[1][4] = (((E[4][2]) << 61) ^ ((E[4][2]) >> (64 - 61))); + A[1][0] = B[1][0] ^ ((~B[1][1]) & B[1][2]); + C[0] ^= A[1][0]; + A[1][1] = B[1][1] ^ ((~B[1][2]) & B[1][3]); + C[1] ^= A[1][1]; + A[1][2] = B[1][2] ^ ((~B[1][3]) & B[1][4]); + C[2] ^= A[1][2]; + A[1][3] = B[1][3] ^ ((~B[1][4]) & B[1][0]); + C[3] ^= A[1][3]; + A[1][4] = B[1][4] ^ ((~B[1][0]) & B[1][1]); + C[4] ^= A[1][4]; + E[0][1] ^= D[1]; + B[2][0] = (((E[0][1]) << 1) ^ ((E[0][1]) >> (64 - 1))); + E[1][2] ^= D[2]; + B[2][1] = (((E[1][2]) << 6) ^ ((E[1][2]) >> (64 - 6))); + E[2][3] ^= D[3]; + B[2][2] = (((E[2][3]) << 25) ^ ((E[2][3]) >> (64 - 25))); + E[3][4] ^= D[4]; + B[2][3] = (((E[3][4]) << 8) ^ ((E[3][4]) >> (64 - 8))); + E[4][0] ^= D[0]; + B[2][4] = (((E[4][0]) << 18) ^ ((E[4][0]) >> (64 - 18))); + A[2][0] = B[2][0] ^ ((~B[2][1]) & B[2][2]); + C[0] ^= A[2][0]; + A[2][1] = B[2][1] ^ ((~B[2][2]) & B[2][3]); + C[1] ^= A[2][1]; + A[2][2] = B[2][2] ^ ((~B[2][3]) & B[2][4]); + C[2] ^= A[2][2]; + A[2][3] = B[2][3] ^ ((~B[2][4]) & B[2][0]); + C[3] ^= A[2][3]; + A[2][4] = B[2][4] ^ ((~B[2][0]) & B[2][1]); + C[4] ^= A[2][4]; + E[0][4] ^= D[4]; + B[3][0] = (((E[0][4]) << 27) ^ ((E[0][4]) >> (64 - 27))); + E[1][0] ^= D[0]; + B[3][1] = (((E[1][0]) << 36) ^ ((E[1][0]) >> (64 - 36))); + E[2][1] ^= D[1]; + B[3][2] = (((E[2][1]) << 10) ^ ((E[2][1]) >> (64 - 10))); + E[3][2] ^= D[2]; + B[3][3] = (((E[3][2]) << 15) ^ ((E[3][2]) >> (64 - 15))); + E[4][3] ^= D[3]; + B[3][4] = (((E[4][3]) << 56) ^ ((E[4][3]) >> (64 - 56))); + A[3][0] = B[3][0] ^ ((~B[3][1]) & B[3][2]); + C[0] ^= A[3][0]; + A[3][1] = B[3][1] ^ ((~B[3][2]) & B[3][3]); + C[1] ^= A[3][1]; + A[3][2] = B[3][2] ^ ((~B[3][3]) & B[3][4]); + C[2] ^= A[3][2]; + A[3][3] = B[3][3] ^ ((~B[3][4]) & B[3][0]); + C[3] ^= A[3][3]; + A[3][4] = B[3][4] ^ ((~B[3][0]) & B[3][1]); + C[4] ^= A[3][4]; + E[0][2] ^= D[2]; + B[4][0] = (((E[0][2]) << 62) ^ ((E[0][2]) >> (64 - 62))); + E[1][3] ^= D[3]; + B[4][1] = (((E[1][3]) << 55) ^ ((E[1][3]) >> (64 - 55))); + E[2][4] ^= D[4]; + B[4][2] = (((E[2][4]) << 39) ^ ((E[2][4]) >> (64 - 39))); + E[3][0] ^= D[0]; + B[4][3] = (((E[3][0]) << 41) ^ ((E[3][0]) >> (64 - 41))); + E[4][1] ^= D[1]; + B[4][4] = (((E[4][1]) << 2) ^ ((E[4][1]) >> (64 - 2))); + A[4][0] = B[4][0] ^ ((~B[4][1]) & B[4][2]); + C[0] ^= A[4][0]; + A[4][1] = B[4][1] ^ ((~B[4][2]) & B[4][3]); + C[1] ^= A[4][1]; + A[4][2] = B[4][2] ^ ((~B[4][3]) & B[4][4]); + C[2] ^= A[4][2]; + A[4][3] = B[4][3] ^ ((~B[4][4]) & B[4][0]); + C[3] ^= A[4][3]; + A[4][4] = B[4][4] ^ ((~B[4][0]) & B[4][1]); + C[4] ^= A[4][4]; + D[0] = C[4] ^ (((C[1]) << 1) ^ ((C[1]) >> (64 - 1))); + D[1] = C[0] ^ (((C[2]) << 1) ^ ((C[2]) >> (64 - 1))); + D[2] = C[1] ^ (((C[3]) << 1) ^ ((C[3]) >> (64 - 1))); + D[3] = C[2] ^ (((C[4]) << 1) ^ ((C[4]) >> (64 - 1))); + D[4] = C[3] ^ (((C[0]) << 1) ^ ((C[0]) >> (64 - 1))); + A[0][0] ^= D[0]; + B[0][0] = A[0][0]; + A[1][1] ^= D[1]; + B[0][1] = (((A[1][1]) << 44) ^ ((A[1][1]) >> (64 - 44))); + A[2][2] ^= D[2]; + B[0][2] = (((A[2][2]) << 43) ^ ((A[2][2]) >> (64 - 43))); + A[3][3] ^= D[3]; + B[0][3] = (((A[3][3]) << 21) ^ ((A[3][3]) >> (64 - 21))); + A[4][4] ^= D[4]; + B[0][4] = (((A[4][4]) << 14) ^ ((A[4][4]) >> (64 - 14))); + E[0][0] = B[0][0] ^ ((~B[0][1]) & B[0][2]); + E[0][0] ^= KECCAKF1600_ROUND_CONSTANTS[20]; + C[0] = E[0][0]; + E[0][1] = B[0][1] ^ ((~B[0][2]) & B[0][3]); + C[1] = E[0][1]; + E[0][2] = B[0][2] ^ ((~B[0][3]) & B[0][4]); + C[2] = E[0][2]; + E[0][3] = B[0][3] ^ ((~B[0][4]) & B[0][0]); + C[3] = E[0][3]; + E[0][4] = B[0][4] ^ ((~B[0][0]) & B[0][1]); + C[4] = E[0][4]; + A[0][3] ^= D[3]; + B[1][0] = (((A[0][3]) << 28) ^ ((A[0][3]) >> (64 - 28))); + A[1][4] ^= D[4]; + B[1][1] = (((A[1][4]) << 20) ^ ((A[1][4]) >> (64 - 20))); + A[2][0] ^= D[0]; + B[1][2] = (((A[2][0]) << 3) ^ ((A[2][0]) >> (64 - 3))); + A[3][1] ^= D[1]; + B[1][3] = (((A[3][1]) << 45) ^ ((A[3][1]) >> (64 - 45))); + A[4][2] ^= D[2]; + B[1][4] = (((A[4][2]) << 61) ^ ((A[4][2]) >> (64 - 61))); + E[1][0] = B[1][0] ^ ((~B[1][1]) & B[1][2]); + C[0] ^= E[1][0]; + E[1][1] = B[1][1] ^ ((~B[1][2]) & B[1][3]); + C[1] ^= E[1][1]; + E[1][2] = B[1][2] ^ ((~B[1][3]) & B[1][4]); + C[2] ^= E[1][2]; + E[1][3] = B[1][3] ^ ((~B[1][4]) & B[1][0]); + C[3] ^= E[1][3]; + E[1][4] = B[1][4] ^ ((~B[1][0]) & B[1][1]); + C[4] ^= E[1][4]; + A[0][1] ^= D[1]; + B[2][0] = (((A[0][1]) << 1) ^ ((A[0][1]) >> (64 - 1))); + A[1][2] ^= D[2]; + B[2][1] = (((A[1][2]) << 6) ^ ((A[1][2]) >> (64 - 6))); + A[2][3] ^= D[3]; + B[2][2] = (((A[2][3]) << 25) ^ ((A[2][3]) >> (64 - 25))); + A[3][4] ^= D[4]; + B[2][3] = (((A[3][4]) << 8) ^ ((A[3][4]) >> (64 - 8))); + A[4][0] ^= D[0]; + B[2][4] = (((A[4][0]) << 18) ^ ((A[4][0]) >> (64 - 18))); + E[2][0] = B[2][0] ^ ((~B[2][1]) & B[2][2]); + C[0] ^= E[2][0]; + E[2][1] = B[2][1] ^ ((~B[2][2]) & B[2][3]); + C[1] ^= E[2][1]; + E[2][2] = B[2][2] ^ ((~B[2][3]) & B[2][4]); + C[2] ^= E[2][2]; + E[2][3] = B[2][3] ^ ((~B[2][4]) & B[2][0]); + C[3] ^= E[2][3]; + E[2][4] = B[2][4] ^ ((~B[2][0]) & B[2][1]); + C[4] ^= E[2][4]; + A[0][4] ^= D[4]; + B[3][0] = (((A[0][4]) << 27) ^ ((A[0][4]) >> (64 - 27))); + A[1][0] ^= D[0]; + B[3][1] = (((A[1][0]) << 36) ^ ((A[1][0]) >> (64 - 36))); + A[2][1] ^= D[1]; + B[3][2] = (((A[2][1]) << 10) ^ ((A[2][1]) >> (64 - 10))); + A[3][2] ^= D[2]; + B[3][3] = (((A[3][2]) << 15) ^ ((A[3][2]) >> (64 - 15))); + A[4][3] ^= D[3]; + B[3][4] = (((A[4][3]) << 56) ^ ((A[4][3]) >> (64 - 56))); + E[3][0] = B[3][0] ^ ((~B[3][1]) & B[3][2]); + C[0] ^= E[3][0]; + E[3][1] = B[3][1] ^ ((~B[3][2]) & B[3][3]); + C[1] ^= E[3][1]; + E[3][2] = B[3][2] ^ ((~B[3][3]) & B[3][4]); + C[2] ^= E[3][2]; + E[3][3] = B[3][3] ^ ((~B[3][4]) & B[3][0]); + C[3] ^= E[3][3]; + E[3][4] = B[3][4] ^ ((~B[3][0]) & B[3][1]); + C[4] ^= E[3][4]; + A[0][2] ^= D[2]; + B[4][0] = (((A[0][2]) << 62) ^ ((A[0][2]) >> (64 - 62))); + A[1][3] ^= D[3]; + B[4][1] = (((A[1][3]) << 55) ^ ((A[1][3]) >> (64 - 55))); + A[2][4] ^= D[4]; + B[4][2] = (((A[2][4]) << 39) ^ ((A[2][4]) >> (64 - 39))); + A[3][0] ^= D[0]; + B[4][3] = (((A[3][0]) << 41) ^ ((A[3][0]) >> (64 - 41))); + A[4][1] ^= D[1]; + B[4][4] = (((A[4][1]) << 2) ^ ((A[4][1]) >> (64 - 2))); + E[4][0] = B[4][0] ^ ((~B[4][1]) & B[4][2]); + C[0] ^= E[4][0]; + E[4][1] = B[4][1] ^ ((~B[4][2]) & B[4][3]); + C[1] ^= E[4][1]; + E[4][2] = B[4][2] ^ ((~B[4][3]) & B[4][4]); + C[2] ^= E[4][2]; + E[4][3] = B[4][3] ^ ((~B[4][4]) & B[4][0]); + C[3] ^= E[4][3]; + E[4][4] = B[4][4] ^ ((~B[4][0]) & B[4][1]); + C[4] ^= E[4][4]; + D[0] = C[4] ^ (((C[1]) << 1) ^ ((C[1]) >> (64 - 1))); + D[1] = C[0] ^ (((C[2]) << 1) ^ ((C[2]) >> (64 - 1))); + D[2] = C[1] ^ (((C[3]) << 1) ^ ((C[3]) >> (64 - 1))); + D[3] = C[2] ^ (((C[4]) << 1) ^ ((C[4]) >> (64 - 1))); + D[4] = C[3] ^ (((C[0]) << 1) ^ ((C[0]) >> (64 - 1))); + E[0][0] ^= D[0]; + B[0][0] = E[0][0]; + E[1][1] ^= D[1]; + B[0][1] = (((E[1][1]) << 44) ^ ((E[1][1]) >> (64 - 44))); + E[2][2] ^= D[2]; + B[0][2] = (((E[2][2]) << 43) ^ ((E[2][2]) >> (64 - 43))); + E[3][3] ^= D[3]; + B[0][3] = (((E[3][3]) << 21) ^ ((E[3][3]) >> (64 - 21))); + E[4][4] ^= D[4]; + B[0][4] = (((E[4][4]) << 14) ^ ((E[4][4]) >> (64 - 14))); + A[0][0] = B[0][0] ^ ((~B[0][1]) & B[0][2]); + A[0][0] ^= KECCAKF1600_ROUND_CONSTANTS[21]; + C[0] = A[0][0]; + A[0][1] = B[0][1] ^ ((~B[0][2]) & B[0][3]); + C[1] = A[0][1]; + A[0][2] = B[0][2] ^ ((~B[0][3]) & B[0][4]); + C[2] = A[0][2]; + A[0][3] = B[0][3] ^ ((~B[0][4]) & B[0][0]); + C[3] = A[0][3]; + A[0][4] = B[0][4] ^ ((~B[0][0]) & B[0][1]); + C[4] = A[0][4]; + E[0][3] ^= D[3]; + B[1][0] = (((E[0][3]) << 28) ^ ((E[0][3]) >> (64 - 28))); + E[1][4] ^= D[4]; + B[1][1] = (((E[1][4]) << 20) ^ ((E[1][4]) >> (64 - 20))); + E[2][0] ^= D[0]; + B[1][2] = (((E[2][0]) << 3) ^ ((E[2][0]) >> (64 - 3))); + E[3][1] ^= D[1]; + B[1][3] = (((E[3][1]) << 45) ^ ((E[3][1]) >> (64 - 45))); + E[4][2] ^= D[2]; + B[1][4] = (((E[4][2]) << 61) ^ ((E[4][2]) >> (64 - 61))); + A[1][0] = B[1][0] ^ ((~B[1][1]) & B[1][2]); + C[0] ^= A[1][0]; + A[1][1] = B[1][1] ^ ((~B[1][2]) & B[1][3]); + C[1] ^= A[1][1]; + A[1][2] = B[1][2] ^ ((~B[1][3]) & B[1][4]); + C[2] ^= A[1][2]; + A[1][3] = B[1][3] ^ ((~B[1][4]) & B[1][0]); + C[3] ^= A[1][3]; + A[1][4] = B[1][4] ^ ((~B[1][0]) & B[1][1]); + C[4] ^= A[1][4]; + E[0][1] ^= D[1]; + B[2][0] = (((E[0][1]) << 1) ^ ((E[0][1]) >> (64 - 1))); + E[1][2] ^= D[2]; + B[2][1] = (((E[1][2]) << 6) ^ ((E[1][2]) >> (64 - 6))); + E[2][3] ^= D[3]; + B[2][2] = (((E[2][3]) << 25) ^ ((E[2][3]) >> (64 - 25))); + E[3][4] ^= D[4]; + B[2][3] = (((E[3][4]) << 8) ^ ((E[3][4]) >> (64 - 8))); + E[4][0] ^= D[0]; + B[2][4] = (((E[4][0]) << 18) ^ ((E[4][0]) >> (64 - 18))); + A[2][0] = B[2][0] ^ ((~B[2][1]) & B[2][2]); + C[0] ^= A[2][0]; + A[2][1] = B[2][1] ^ ((~B[2][2]) & B[2][3]); + C[1] ^= A[2][1]; + A[2][2] = B[2][2] ^ ((~B[2][3]) & B[2][4]); + C[2] ^= A[2][2]; + A[2][3] = B[2][3] ^ ((~B[2][4]) & B[2][0]); + C[3] ^= A[2][3]; + A[2][4] = B[2][4] ^ ((~B[2][0]) & B[2][1]); + C[4] ^= A[2][4]; + E[0][4] ^= D[4]; + B[3][0] = (((E[0][4]) << 27) ^ ((E[0][4]) >> (64 - 27))); + E[1][0] ^= D[0]; + B[3][1] = (((E[1][0]) << 36) ^ ((E[1][0]) >> (64 - 36))); + E[2][1] ^= D[1]; + B[3][2] = (((E[2][1]) << 10) ^ ((E[2][1]) >> (64 - 10))); + E[3][2] ^= D[2]; + B[3][3] = (((E[3][2]) << 15) ^ ((E[3][2]) >> (64 - 15))); + E[4][3] ^= D[3]; + B[3][4] = (((E[4][3]) << 56) ^ ((E[4][3]) >> (64 - 56))); + A[3][0] = B[3][0] ^ ((~B[3][1]) & B[3][2]); + C[0] ^= A[3][0]; + A[3][1] = B[3][1] ^ ((~B[3][2]) & B[3][3]); + C[1] ^= A[3][1]; + A[3][2] = B[3][2] ^ ((~B[3][3]) & B[3][4]); + C[2] ^= A[3][2]; + A[3][3] = B[3][3] ^ ((~B[3][4]) & B[3][0]); + C[3] ^= A[3][3]; + A[3][4] = B[3][4] ^ ((~B[3][0]) & B[3][1]); + C[4] ^= A[3][4]; + E[0][2] ^= D[2]; + B[4][0] = (((E[0][2]) << 62) ^ ((E[0][2]) >> (64 - 62))); + E[1][3] ^= D[3]; + B[4][1] = (((E[1][3]) << 55) ^ ((E[1][3]) >> (64 - 55))); + E[2][4] ^= D[4]; + B[4][2] = (((E[2][4]) << 39) ^ ((E[2][4]) >> (64 - 39))); + E[3][0] ^= D[0]; + B[4][3] = (((E[3][0]) << 41) ^ ((E[3][0]) >> (64 - 41))); + E[4][1] ^= D[1]; + B[4][4] = (((E[4][1]) << 2) ^ ((E[4][1]) >> (64 - 2))); + A[4][0] = B[4][0] ^ ((~B[4][1]) & B[4][2]); + C[0] ^= A[4][0]; + A[4][1] = B[4][1] ^ ((~B[4][2]) & B[4][3]); + C[1] ^= A[4][1]; + A[4][2] = B[4][2] ^ ((~B[4][3]) & B[4][4]); + C[2] ^= A[4][2]; + A[4][3] = B[4][3] ^ ((~B[4][4]) & B[4][0]); + C[3] ^= A[4][3]; + A[4][4] = B[4][4] ^ ((~B[4][0]) & B[4][1]); + C[4] ^= A[4][4]; + D[0] = C[4] ^ (((C[1]) << 1) ^ ((C[1]) >> (64 - 1))); + D[1] = C[0] ^ (((C[2]) << 1) ^ ((C[2]) >> (64 - 1))); + D[2] = C[1] ^ (((C[3]) << 1) ^ ((C[3]) >> (64 - 1))); + D[3] = C[2] ^ (((C[4]) << 1) ^ ((C[4]) >> (64 - 1))); + D[4] = C[3] ^ (((C[0]) << 1) ^ ((C[0]) >> (64 - 1))); + A[0][0] ^= D[0]; + B[0][0] = A[0][0]; + A[1][1] ^= D[1]; + B[0][1] = (((A[1][1]) << 44) ^ ((A[1][1]) >> (64 - 44))); + A[2][2] ^= D[2]; + B[0][2] = (((A[2][2]) << 43) ^ ((A[2][2]) >> (64 - 43))); + A[3][3] ^= D[3]; + B[0][3] = (((A[3][3]) << 21) ^ ((A[3][3]) >> (64 - 21))); + A[4][4] ^= D[4]; + B[0][4] = (((A[4][4]) << 14) ^ ((A[4][4]) >> (64 - 14))); + E[0][0] = B[0][0] ^ ((~B[0][1]) & B[0][2]); + E[0][0] ^= KECCAKF1600_ROUND_CONSTANTS[22]; + C[0] = E[0][0]; + E[0][1] = B[0][1] ^ ((~B[0][2]) & B[0][3]); + C[1] = E[0][1]; + E[0][2] = B[0][2] ^ ((~B[0][3]) & B[0][4]); + C[2] = E[0][2]; + E[0][3] = B[0][3] ^ ((~B[0][4]) & B[0][0]); + C[3] = E[0][3]; + E[0][4] = B[0][4] ^ ((~B[0][0]) & B[0][1]); + C[4] = E[0][4]; + A[0][3] ^= D[3]; + B[1][0] = (((A[0][3]) << 28) ^ ((A[0][3]) >> (64 - 28))); + A[1][4] ^= D[4]; + B[1][1] = (((A[1][4]) << 20) ^ ((A[1][4]) >> (64 - 20))); + A[2][0] ^= D[0]; + B[1][2] = (((A[2][0]) << 3) ^ ((A[2][0]) >> (64 - 3))); + A[3][1] ^= D[1]; + B[1][3] = (((A[3][1]) << 45) ^ ((A[3][1]) >> (64 - 45))); + A[4][2] ^= D[2]; + B[1][4] = (((A[4][2]) << 61) ^ ((A[4][2]) >> (64 - 61))); + E[1][0] = B[1][0] ^ ((~B[1][1]) & B[1][2]); + C[0] ^= E[1][0]; + E[1][1] = B[1][1] ^ ((~B[1][2]) & B[1][3]); + C[1] ^= E[1][1]; + E[1][2] = B[1][2] ^ ((~B[1][3]) & B[1][4]); + C[2] ^= E[1][2]; + E[1][3] = B[1][3] ^ ((~B[1][4]) & B[1][0]); + C[3] ^= E[1][3]; + E[1][4] = B[1][4] ^ ((~B[1][0]) & B[1][1]); + C[4] ^= E[1][4]; + A[0][1] ^= D[1]; + B[2][0] = (((A[0][1]) << 1) ^ ((A[0][1]) >> (64 - 1))); + A[1][2] ^= D[2]; + B[2][1] = (((A[1][2]) << 6) ^ ((A[1][2]) >> (64 - 6))); + A[2][3] ^= D[3]; + B[2][2] = (((A[2][3]) << 25) ^ ((A[2][3]) >> (64 - 25))); + A[3][4] ^= D[4]; + B[2][3] = (((A[3][4]) << 8) ^ ((A[3][4]) >> (64 - 8))); + A[4][0] ^= D[0]; + B[2][4] = (((A[4][0]) << 18) ^ ((A[4][0]) >> (64 - 18))); + E[2][0] = B[2][0] ^ ((~B[2][1]) & B[2][2]); + C[0] ^= E[2][0]; + E[2][1] = B[2][1] ^ ((~B[2][2]) & B[2][3]); + C[1] ^= E[2][1]; + E[2][2] = B[2][2] ^ ((~B[2][3]) & B[2][4]); + C[2] ^= E[2][2]; + E[2][3] = B[2][3] ^ ((~B[2][4]) & B[2][0]); + C[3] ^= E[2][3]; + E[2][4] = B[2][4] ^ ((~B[2][0]) & B[2][1]); + C[4] ^= E[2][4]; + A[0][4] ^= D[4]; + B[3][0] = (((A[0][4]) << 27) ^ ((A[0][4]) >> (64 - 27))); + A[1][0] ^= D[0]; + B[3][1] = (((A[1][0]) << 36) ^ ((A[1][0]) >> (64 - 36))); + A[2][1] ^= D[1]; + B[3][2] = (((A[2][1]) << 10) ^ ((A[2][1]) >> (64 - 10))); + A[3][2] ^= D[2]; + B[3][3] = (((A[3][2]) << 15) ^ ((A[3][2]) >> (64 - 15))); + A[4][3] ^= D[3]; + B[3][4] = (((A[4][3]) << 56) ^ ((A[4][3]) >> (64 - 56))); + E[3][0] = B[3][0] ^ ((~B[3][1]) & B[3][2]); + C[0] ^= E[3][0]; + E[3][1] = B[3][1] ^ ((~B[3][2]) & B[3][3]); + C[1] ^= E[3][1]; + E[3][2] = B[3][2] ^ ((~B[3][3]) & B[3][4]); + C[2] ^= E[3][2]; + E[3][3] = B[3][3] ^ ((~B[3][4]) & B[3][0]); + C[3] ^= E[3][3]; + E[3][4] = B[3][4] ^ ((~B[3][0]) & B[3][1]); + C[4] ^= E[3][4]; + A[0][2] ^= D[2]; + B[4][0] = (((A[0][2]) << 62) ^ ((A[0][2]) >> (64 - 62))); + A[1][3] ^= D[3]; + B[4][1] = (((A[1][3]) << 55) ^ ((A[1][3]) >> (64 - 55))); + A[2][4] ^= D[4]; + B[4][2] = (((A[2][4]) << 39) ^ ((A[2][4]) >> (64 - 39))); + A[3][0] ^= D[0]; + B[4][3] = (((A[3][0]) << 41) ^ ((A[3][0]) >> (64 - 41))); + A[4][1] ^= D[1]; + B[4][4] = (((A[4][1]) << 2) ^ ((A[4][1]) >> (64 - 2))); + E[4][0] = B[4][0] ^ ((~B[4][1]) & B[4][2]); + C[0] ^= E[4][0]; + E[4][1] = B[4][1] ^ ((~B[4][2]) & B[4][3]); + C[1] ^= E[4][1]; + E[4][2] = B[4][2] ^ ((~B[4][3]) & B[4][4]); + C[2] ^= E[4][2]; + E[4][3] = B[4][3] ^ ((~B[4][4]) & B[4][0]); + C[3] ^= E[4][3]; + E[4][4] = B[4][4] ^ ((~B[4][0]) & B[4][1]); + C[4] ^= E[4][4]; + D[0] = C[4] ^ (((C[1]) << 1) ^ ((C[1]) >> (64 - 1))); + D[1] = C[0] ^ (((C[2]) << 1) ^ ((C[2]) >> (64 - 1))); + D[2] = C[1] ^ (((C[3]) << 1) ^ ((C[3]) >> (64 - 1))); + D[3] = C[2] ^ (((C[4]) << 1) ^ ((C[4]) >> (64 - 1))); + D[4] = C[3] ^ (((C[0]) << 1) ^ ((C[0]) >> (64 - 1))); + E[0][0] ^= D[0]; + B[0][0] = E[0][0]; + E[1][1] ^= D[1]; + B[0][1] = (((E[1][1]) << 44) ^ ((E[1][1]) >> (64 - 44))); + E[2][2] ^= D[2]; + B[0][2] = (((E[2][2]) << 43) ^ ((E[2][2]) >> (64 - 43))); + E[3][3] ^= D[3]; + B[0][3] = (((E[3][3]) << 21) ^ ((E[3][3]) >> (64 - 21))); + E[4][4] ^= D[4]; + B[0][4] = (((E[4][4]) << 14) ^ ((E[4][4]) >> (64 - 14))); + A[0][0] = B[0][0] ^ ((~B[0][1]) & B[0][2]); + A[0][0] ^= KECCAKF1600_ROUND_CONSTANTS[23]; + A[0][1] = B[0][1] ^ ((~B[0][2]) & B[0][3]); + A[0][2] = B[0][2] ^ ((~B[0][3]) & B[0][4]); + A[0][3] = B[0][3] ^ ((~B[0][4]) & B[0][0]); + A[0][4] = B[0][4] ^ ((~B[0][0]) & B[0][1]); + E[0][3] ^= D[3]; + B[1][0] = (((E[0][3]) << 28) ^ ((E[0][3]) >> (64 - 28))); + E[1][4] ^= D[4]; + B[1][1] = (((E[1][4]) << 20) ^ ((E[1][4]) >> (64 - 20))); + E[2][0] ^= D[0]; + B[1][2] = (((E[2][0]) << 3) ^ ((E[2][0]) >> (64 - 3))); + E[3][1] ^= D[1]; + B[1][3] = (((E[3][1]) << 45) ^ ((E[3][1]) >> (64 - 45))); + E[4][2] ^= D[2]; + B[1][4] = (((E[4][2]) << 61) ^ ((E[4][2]) >> (64 - 61))); + A[1][0] = B[1][0] ^ ((~B[1][1]) & B[1][2]); + A[1][1] = B[1][1] ^ ((~B[1][2]) & B[1][3]); + A[1][2] = B[1][2] ^ ((~B[1][3]) & B[1][4]); + A[1][3] = B[1][3] ^ ((~B[1][4]) & B[1][0]); + A[1][4] = B[1][4] ^ ((~B[1][0]) & B[1][1]); + E[0][1] ^= D[1]; + B[2][0] = (((E[0][1]) << 1) ^ ((E[0][1]) >> (64 - 1))); + E[1][2] ^= D[2]; + B[2][1] = (((E[1][2]) << 6) ^ ((E[1][2]) >> (64 - 6))); + E[2][3] ^= D[3]; + B[2][2] = (((E[2][3]) << 25) ^ ((E[2][3]) >> (64 - 25))); + E[3][4] ^= D[4]; + B[2][3] = (((E[3][4]) << 8) ^ ((E[3][4]) >> (64 - 8))); + E[4][0] ^= D[0]; + B[2][4] = (((E[4][0]) << 18) ^ ((E[4][0]) >> (64 - 18))); + A[2][0] = B[2][0] ^ ((~B[2][1]) & B[2][2]); + A[2][1] = B[2][1] ^ ((~B[2][2]) & B[2][3]); + A[2][2] = B[2][2] ^ ((~B[2][3]) & B[2][4]); + A[2][3] = B[2][3] ^ ((~B[2][4]) & B[2][0]); + A[2][4] = B[2][4] ^ ((~B[2][0]) & B[2][1]); + E[0][4] ^= D[4]; + B[3][0] = (((E[0][4]) << 27) ^ ((E[0][4]) >> (64 - 27))); + E[1][0] ^= D[0]; + B[3][1] = (((E[1][0]) << 36) ^ ((E[1][0]) >> (64 - 36))); + E[2][1] ^= D[1]; + B[3][2] = (((E[2][1]) << 10) ^ ((E[2][1]) >> (64 - 10))); + E[3][2] ^= D[2]; + B[3][3] = (((E[3][2]) << 15) ^ ((E[3][2]) >> (64 - 15))); + E[4][3] ^= D[3]; + B[3][4] = (((E[4][3]) << 56) ^ ((E[4][3]) >> (64 - 56))); + A[3][0] = B[3][0] ^ ((~B[3][1]) & B[3][2]); + A[3][1] = B[3][1] ^ ((~B[3][2]) & B[3][3]); + A[3][2] = B[3][2] ^ ((~B[3][3]) & B[3][4]); + A[3][3] = B[3][3] ^ ((~B[3][4]) & B[3][0]); + A[3][4] = B[3][4] ^ ((~B[3][0]) & B[3][1]); + E[0][2] ^= D[2]; + B[4][0] = (((E[0][2]) << 62) ^ ((E[0][2]) >> (64 - 62))); + E[1][3] ^= D[3]; + B[4][1] = (((E[1][3]) << 55) ^ ((E[1][3]) >> (64 - 55))); + E[2][4] ^= D[4]; + B[4][2] = (((E[2][4]) << 39) ^ ((E[2][4]) >> (64 - 39))); + E[3][0] ^= D[0]; + B[4][3] = (((E[3][0]) << 41) ^ ((E[3][0]) >> (64 - 41))); + E[4][1] ^= D[1]; + B[4][4] = (((E[4][1]) << 2) ^ ((E[4][1]) >> (64 - 2))); + A[4][0] = B[4][0] ^ ((~B[4][1]) & B[4][2]); + A[4][1] = B[4][1] ^ ((~B[4][2]) & B[4][3]); + A[4][2] = B[4][2] ^ ((~B[4][3]) & B[4][4]); + A[4][3] = B[4][3] ^ ((~B[4][4]) & B[4][0]); + A[4][4] = B[4][4] ^ ((~B[4][0]) & B[4][1]); +} + +} // namespace cartesi + +#endif // KECCAKF_H diff --git a/src/local-machine.cpp b/src/local-machine.cpp new file mode 100644 index 000000000..77ccb7a41 --- /dev/null +++ b/src/local-machine.cpp @@ -0,0 +1,242 @@ +// Copyright Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: LGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify it under +// the terms of the GNU Lesser General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) any +// later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +// PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License along +// with this program (see COPYING). If not, see . +// + +#include "local-machine.h" + +#include +#include +#include + +#include "access-log.h" +#include "address-range-description.h" +#include "back-merkle-tree.h" +#include "hash-tree-proof.h" +#include "hash-tree-stats.h" +#include "i-machine.h" +#include "interpret.h" +#include "machine-config.h" +#include "machine-hash.h" +#include "machine-runtime-config.h" +#include "machine.h" +#include "mcycle-root-hashes.h" +#include "uarch-cycle-root-hashes.h" +#include "uarch-interpret.h" + +namespace cartesi { + +i_machine *local_machine::do_clone_empty() const { + return new local_machine(); +} + +bool local_machine::do_is_empty() const { + return m_machine == nullptr; +} + +void local_machine::do_create(const machine_config &config, const machine_runtime_config &runtime, + const std::string &dir) { + m_machine = new machine(config, runtime, dir); +} + +void local_machine::do_load(const std::string &directory, const machine_runtime_config &runtime, sharing_mode sharing) { + m_machine = new machine(directory, runtime, sharing); +} + +local_machine::~local_machine() { + delete m_machine; + m_machine = nullptr; +} + +machine *local_machine::get_machine() { + if (m_machine == nullptr) { + throw std::runtime_error{"no machine"}; + } + return m_machine; +} + +const machine *local_machine::get_machine() const { + if (m_machine == nullptr) { + throw std::runtime_error{"no machine"}; + } + return m_machine; +} + +void local_machine::do_store(const std::string &directory, sharing_mode sharing) const { + get_machine()->store(directory, sharing); +} + +void local_machine::do_clone_stored(const std::string &from_dir, const std::string &to_dir) const { + machine::clone_stored(from_dir, to_dir); +} + +void local_machine::do_remove_stored(const std::string &dir) const { + machine::remove_stored(dir); +} + +interpreter_break_reason local_machine::do_run(uint64_t mcycle_end) { + return get_machine()->run(mcycle_end); +} + +mcycle_root_hashes local_machine::do_collect_mcycle_root_hashes(uint64_t mcycle_end, uint64_t mcycle_period, + uint64_t mcycle_phase, int32_t log2_bundle_mcycle_count, + const std::optional &previous_back_tree) { + return get_machine()->collect_mcycle_root_hashes(mcycle_end, mcycle_period, mcycle_phase, log2_bundle_mcycle_count, + previous_back_tree); +} + +interpreter_break_reason local_machine::do_log_step(uint64_t mcycle_count, const std::string &filename) { + return m_machine->log_step(mcycle_count, filename); +} + +access_log local_machine::do_log_step_uarch(const access_log::type &log_type) { + return get_machine()->log_step_uarch(log_type); +} + +hash_tree_proof local_machine::do_get_proof(uint64_t address, int log2_size) const { + return get_machine()->get_proof(address, log2_size); +} + +machine_hash local_machine::do_get_root_hash() const { + return get_machine()->get_root_hash(); +} + +machine_hash local_machine::do_get_node_hash(uint64_t address, int log2_size) const { + return get_machine()->get_node_hash(address, log2_size); +} + +bool local_machine::do_verify_hash_tree() const { + return get_machine()->verify_hash_tree(); +} + +uint64_t local_machine::do_read_reg(reg r) const { + return get_machine()->read_reg(r); +} + +void local_machine::do_write_reg(reg w, uint64_t val) { + get_machine()->write_reg(w, val); +} + +void local_machine::do_read_memory(uint64_t address, unsigned char *data, uint64_t length) const { + get_machine()->read_memory(address, data, length); +} + +void local_machine::do_write_memory(uint64_t address, const unsigned char *data, uint64_t length) { + get_machine()->write_memory(address, data, length); +} + +void local_machine::do_read_virtual_memory(uint64_t address, unsigned char *data, uint64_t length) { + get_machine()->read_virtual_memory(address, data, length); +} + +void local_machine::do_write_virtual_memory(uint64_t address, const unsigned char *data, uint64_t length) { + get_machine()->write_virtual_memory(address, data, length); +} + +uint64_t local_machine::do_translate_virtual_address(uint64_t vaddr) { + return get_machine()->translate_virtual_address(vaddr); +} + +void local_machine::do_replace_memory_range(const memory_range_config &new_range) { + get_machine()->replace_memory_range(new_range); +} + +uint64_t local_machine::do_read_word(uint64_t address) const { + return get_machine()->read_word(address); +} + +void local_machine::do_write_word(uint64_t address, uint64_t value) { + get_machine()->write_word(address, value); +} + +hash_tree_stats local_machine::do_get_hash_tree_stats(bool clear) { + return get_machine()->get_hash_tree_stats(clear); +} + +machine_config local_machine::do_get_initial_config() const { + return get_machine()->get_initial_config(); +} + +machine_runtime_config local_machine::do_get_runtime_config() const { + return get_machine()->get_runtime_config(); +} + +void local_machine::do_set_runtime_config(const machine_runtime_config &r) { + get_machine()->set_runtime_config(r); +} + +void local_machine::do_destroy() { + delete get_machine(); + m_machine = nullptr; +} + +void local_machine::do_reset_uarch() { + get_machine()->reset_uarch(); +} + +access_log local_machine::do_log_reset_uarch(const access_log::type &log_type) { + return get_machine()->log_reset_uarch(log_type); +} + +uarch_interpreter_break_reason local_machine::do_run_uarch(uint64_t uarch_cycle_end) { + return get_machine()->run_uarch(uarch_cycle_end); +} + +uarch_cycle_root_hashes local_machine::do_collect_uarch_cycle_root_hashes(uint64_t mcycle_end, + int32_t log2_bundle_uarch_cycle_count) { + return get_machine()->collect_uarch_cycle_root_hashes(mcycle_end, log2_bundle_uarch_cycle_count); +} + +address_range_descriptions local_machine::do_get_address_ranges() const { + return get_machine()->get_address_ranges(); +} + +void local_machine::do_send_cmio_response(uint16_t reason, const unsigned char *data, uint64_t length) { + get_machine()->send_cmio_response(reason, data, length); +} + +access_log local_machine::do_log_send_cmio_response(uint16_t reason, const unsigned char *data, uint64_t length, + const access_log::type &log_type) { + return get_machine()->log_send_cmio_response(reason, data, length, log_type); +} + +uint64_t local_machine::do_get_reg_address(reg r) const { + return machine::get_reg_address(r); +} + +machine_config local_machine::do_get_default_config() const { + return machine::get_default_config(); +} + +interpreter_break_reason local_machine::do_verify_step(const machine_hash &root_hash_before, + const std::string &log_filename, uint64_t mcycle_count, const machine_hash &root_hash_after) const { + return machine::verify_step(root_hash_before, log_filename, mcycle_count, root_hash_after); +} + +void local_machine::do_verify_step_uarch(const machine_hash &root_hash_before, const access_log &log, + const machine_hash &root_hash_after) const { + machine::verify_step_uarch(root_hash_before, log, root_hash_after); +} + +void local_machine::do_verify_reset_uarch(const machine_hash &root_hash_before, const access_log &log, + const machine_hash &root_hash_after) const { + machine::verify_reset_uarch(root_hash_before, log, root_hash_after); +} + +void local_machine::do_verify_send_cmio_response(uint16_t reason, const unsigned char *data, uint64_t length, + const machine_hash &root_hash_before, const access_log &log, const machine_hash &root_hash_after) const { + machine::verify_send_cmio_response(reason, data, length, root_hash_before, log, root_hash_after); +} + +} // namespace cartesi diff --git a/src/virtual-machine.h b/src/local-machine.h similarity index 54% rename from src/virtual-machine.h rename to src/local-machine.h index 64e9340c7..de789d168 100644 --- a/src/virtual-machine.h +++ b/src/local-machine.h @@ -14,47 +14,59 @@ // with this program (see COPYING). If not, see . // -#ifndef VIRTUAL_MACHINE_H -#define VIRTUAL_MACHINE_H +#ifndef LOCAL_MACHINE_H +#define LOCAL_MACHINE_H #include +#include #include #include "access-log.h" -#include "i-virtual-machine.h" +#include "address-range-description.h" +#include "back-merkle-tree.h" +#include "hash-tree-proof.h" +#include "hash-tree-stats.h" +#include "i-machine.h" #include "interpret.h" #include "machine-config.h" -#include "machine-memory-range-descr.h" -#include "machine-merkle-tree.h" +#include "machine-hash.h" #include "machine-runtime-config.h" #include "machine.h" +#include "mcycle-root-hashes.h" +#include "uarch-cycle-root-hashes.h" #include "uarch-interpret.h" namespace cartesi { -/// \class virtual_machine -/// \brief i_virtual_machine implementation pointing to a local machine instance -class virtual_machine : public i_virtual_machine { +/// \class local_machine +/// \brief i_machine implementation pointing to a local machine instance +class local_machine final : public i_machine { public: - virtual_machine() = default; - virtual_machine(const virtual_machine &other) = delete; - virtual_machine(virtual_machine &&other) noexcept = delete; - virtual_machine &operator=(const virtual_machine &other) = delete; - virtual_machine &operator=(virtual_machine &&other) noexcept = delete; - ~virtual_machine() override; + local_machine() = default; + local_machine(const local_machine &other) = delete; + local_machine(local_machine &&other) noexcept = delete; + local_machine &operator=(const local_machine &other) = delete; + local_machine &operator=(local_machine &&other) noexcept = delete; + ~local_machine() override; private: - i_virtual_machine *do_clone_empty() const override; + i_machine *do_clone_empty() const override; bool do_is_empty() const override; - void do_create(const machine_config &config, const machine_runtime_config &runtime) override; - void do_load(const std::string &directory, const machine_runtime_config &runtime) override; + void do_create(const machine_config &config, const machine_runtime_config &runtime, + const std::string &dir) override; + void do_load(const std::string &directory, const machine_runtime_config &runtime, sharing_mode sharing) override; interpreter_break_reason do_run(uint64_t mcycle_end) override; + mcycle_root_hashes do_collect_mcycle_root_hashes(uint64_t mcycle_end, uint64_t mcycle_period, uint64_t mcycle_phase, + int32_t log2_bundle_mcycle_count, const std::optional &previous_back_tree) override; interpreter_break_reason do_log_step(uint64_t mcycle_count, const std::string &filename) override; - void do_store(const std::string &directory) const override; + void do_store(const std::string &directory, sharing_mode sharing) const override; + void do_clone_stored(const std::string &from_dir, const std::string &to_dir) const override; + void do_remove_stored(const std::string &dir) const override; access_log do_log_step_uarch(const access_log::type &log_type) override; - machine_merkle_tree::proof_type do_get_proof(uint64_t address, int log2_size) const override; - void do_get_root_hash(hash_type &hash) const override; - bool do_verify_merkle_tree() const override; + hash_tree_proof do_get_proof(uint64_t address, int log2_size) const override; + machine_hash do_get_root_hash() const override; + machine_hash do_get_node_hash(uint64_t address, int log2_size) const override; + bool do_verify_hash_tree() const override; uint64_t do_read_reg(reg r) const override; void do_write_reg(reg w, uint64_t val) override; void do_read_memory(uint64_t address, unsigned char *data, uint64_t length) const override; @@ -64,28 +76,32 @@ class virtual_machine : public i_virtual_machine { uint64_t do_translate_virtual_address(uint64_t vaddr) override; void do_replace_memory_range(const memory_range_config &new_range) override; uint64_t do_read_word(uint64_t address) const override; - bool do_verify_dirty_page_maps() const override; + void do_write_word(uint64_t address, uint64_t value) override; machine_config do_get_initial_config() const override; + hash_tree_stats do_get_hash_tree_stats(bool clear) override; machine_runtime_config do_get_runtime_config() const override; void do_set_runtime_config(const machine_runtime_config &r) override; void do_destroy() override; void do_reset_uarch() override; access_log do_log_reset_uarch(const access_log::type &log_type) override; uarch_interpreter_break_reason do_run_uarch(uint64_t uarch_cycle_end) override; - machine_memory_range_descrs do_get_memory_ranges() const override; + uarch_cycle_root_hashes do_collect_uarch_cycle_root_hashes(uint64_t mcycle_end, + int32_t log2_bundle_uarch_cycle_count) override; + address_range_descriptions do_get_address_ranges() const override; void do_send_cmio_response(uint16_t reason, const unsigned char *data, uint64_t length) override; access_log do_log_send_cmio_response(uint16_t reason, const unsigned char *data, uint64_t length, const access_log::type &log_type) override; uint64_t do_get_reg_address(reg r) const override; machine_config do_get_default_config() const override; - interpreter_break_reason do_verify_step(const hash_type &root_hash_before, const std::string &log_filename, - uint64_t mcycle_count, const hash_type &root_hash_after) const override; - void do_verify_step_uarch(const hash_type &root_hash_before, const access_log &log, - const hash_type &root_hash_after) const override; - void do_verify_reset_uarch(const hash_type &root_hash_before, const access_log &log, - const hash_type &root_hash_after) const override; + interpreter_break_reason do_verify_step(const machine_hash &root_hash_before, const std::string &log_filename, + uint64_t mcycle_count, const machine_hash &root_hash_after) const override; + void do_verify_step_uarch(const machine_hash &root_hash_before, const access_log &log, + const machine_hash &root_hash_after) const override; + void do_verify_reset_uarch(const machine_hash &root_hash_before, const access_log &log, + const machine_hash &root_hash_after) const override; void do_verify_send_cmio_response(uint16_t reason, const unsigned char *data, uint64_t length, - const hash_type &root_hash_before, const access_log &log, const hash_type &root_hash_after) const override; + const machine_hash &root_hash_before, const access_log &log, + const machine_hash &root_hash_after) const override; machine *get_machine(); const machine *get_machine() const; @@ -95,4 +111,4 @@ class virtual_machine : public i_virtual_machine { } // namespace cartesi -#endif // VIRTUAL_MACHINE_H +#endif // LOCAL_MACHINE_H diff --git a/src/machine-address-ranges.cpp b/src/machine-address-ranges.cpp new file mode 100644 index 000000000..c9cb4a7a9 --- /dev/null +++ b/src/machine-address-ranges.cpp @@ -0,0 +1,541 @@ +// Copyright Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: LGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify it under +// the terms of the GNU Lesser General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) any +// later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +// PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License along +// with this program (see COPYING). If not, see . +// + +/// \file +/// \brief Cartesi machine address ranges. + +#include "machine-address-ranges.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "address-range-constants.h" +#include "address-range-description.h" +#include "address-range.h" +#include "clint-address-range.h" +#include "htif-address-range.h" +#include "machine-config.h" +#include "machine-runtime-config.h" +#include "memory-address-range.h" +#include "meta.h" +#include "os-filesystem.h" +#include "plic-address-range.h" +#include "pmas-constants.h" +#include "pmas.h" +#include "processor-state.h" +#include "scope-remove.h" +#include "uarch-pristine.h" +#include "uarch-processor-state.h" +#include "virtio-address-range.h" +#include "virtio-console-address-range.h" +#include "virtio-net-tuntap-address-range.h" +#include "virtio-net-user-address-range.h" +#include "virtio-p9fs-address-range.h" + +namespace cartesi { + +using namespace std::string_literals; + +static const auto throw_invalid_argument = [](const char *err) { throw std::invalid_argument{err}; }; + +static inline auto make_pmas_address_range(const pmas_config &config) { + static constexpr pmas_flags flags{ + .M = true, + .IO = false, + .R = true, + .W = false, + .X = false, + .IR = true, + .IW = false, + .DID = PMA_ISTART_DID::memory, + }; + return std::make_unique("PMAs"s, AR_PMAS_START, AR_PMAS_LENGTH, flags, config.backing_store, + memory_address_range_config{.host_read_only = true}); +} + +static inline auto make_dtb_address_range(const dtb_config &config) { + // When we pass a RNG seed in a FDT stored in DTB, Linux will wipe out its contents as a security measure, + // therefore, we need to make DTB writable, otherwise boot will hang. + static constexpr pmas_flags dtb_flags{ + .M = true, + .IO = false, + .R = true, + .W = true, + .X = true, + .IR = true, + .IW = true, + .DID = PMA_ISTART_DID::memory, + }; + return std::make_unique("DTB"s, AR_DTB_START, AR_DTB_LENGTH, dtb_flags, config.backing_store); +} + +static inline auto make_shadow_state_address_range(const processor_config &config) { + static constexpr pmas_flags shadow_state_flags{ + .M = true, + .IO = false, + .R = false, + .W = false, + .X = false, + .IR = false, + .IW = false, + .DID = PMA_ISTART_DID::shadow_state, + }; + static constexpr memory_address_range_config shadow_state_config{.host_length = sizeof(processor_state)}; + return std::make_unique("shadow state", AR_SHADOW_STATE_START, AR_SHADOW_STATE_LENGTH, + shadow_state_flags, config.backing_store, shadow_state_config); +} + +static inline auto make_shadow_uarch_state_address_range(const uarch_processor_config &config) { + static constexpr pmas_flags shadow_uarch_state_flags{ + .M = true, + .IO = false, + .R = false, + .W = false, + .X = false, + .IR = false, + .IW = false, + .DID = PMA_ISTART_DID::shadow_uarch_state, + }; + static constexpr memory_address_range_config shadow_uarch_state_config{ + .host_length = sizeof(uarch_processor_state)}; + return std::make_unique("shadow uarch state", AR_SHADOW_UARCH_STATE_START, + AR_SHADOW_UARCH_STATE_LENGTH, shadow_uarch_state_flags, config.backing_store, shadow_uarch_state_config); +} + +void machine_address_ranges::mark_always_dirty_pages() { + m_all[m_shadow_uarch_state_index]->get_dirty_page_tree().mark_dirty_pages_and_up(0, AR_SHADOW_UARCH_STATE_LENGTH); + auto &shadow_dpt = m_all[m_shadow_state_index]->get_dirty_page_tree(); + shadow_dpt.mark_dirty_pages_and_up(AR_SHADOW_REGISTERS_START - AR_SHADOW_STATE_START, AR_SHADOW_REGISTERS_LENGTH); + shadow_dpt.mark_dirty_pages_and_up(AR_SHADOW_TLB_START - AR_SHADOW_STATE_START, AR_SHADOW_TLB_LENGTH); +} + +template + requires std::derived_from +AR &machine_address_ranges::push_back(std::unique_ptr &&ar_ptr, register_where where) { + AR &ar_ref = *ar_ptr; // Get reference to object, already in heap, to return later + check(ar_ref, where); // Check if we can register it + const auto index = m_all.size(); // Get index the new address range will occupy + m_all.push_back(std::move(ar_ptr)); // Move ptr to list of address ranges + if (where.pmas) { // Register as a PMA + m_pmas.push_back(index); + } + if (where.hash_tree) { // Register with the hash tree + m_hash_tree.push_back(index); + } + if constexpr (std::is_convertible_v) { // Register with VirtIO + m_virtio.push_back(&ar_ref); + } + return ar_ref; // Return reference to object in heap +} + +static void prepare_ar_backing_store(const backing_store_config &c, uint64_t length, scope_remove &remover) { + if (!c.shared) { + return; + } + + // Create data storage + if (!c.data_filename.empty()) { + if (c.create) { // Create new file + os::truncate_file(c.data_filename, length, true); + remover.add_file(c.data_filename); + } else if (c.truncate) { // Truncate existing file + os::truncate_file(c.data_filename, length, false); + } + } + + // Create dht storage only if needed + if (!c.dht_filename.empty() && (c.create || !os::exists(c.dht_filename))) { + os::truncate_file(c.dht_filename, memory_address_range::get_dht_storage_length(length), true); + remover.add_file(c.dht_filename); + } + + // Create dpt storage only if needed + if (!c.dpt_filename.empty() && (c.create || !os::exists(c.dpt_filename))) { + os::truncate_file(c.dpt_filename, memory_address_range::get_dpt_storage_length(length), true); + remover.add_file(c.dpt_filename); + } +} + +static void prepare_ar_backing_store_for_share(const backing_store_config &from_c, backing_store_config &to_c, + uint64_t length, bool read_only, scope_remove &remover) { + to_c.shared = true; + const uint64_t dht_length = memory_address_range::get_dht_storage_length(length); + const uint64_t dpt_length = memory_address_range::get_dpt_storage_length(length); + if (from_c.newly_created()) { // Nothing to copy + // Prepare data storage + os::truncate_file(to_c.data_filename, length, true); + remover.add_file(to_c.data_filename); + + // Prepare dht storage + os::truncate_file(to_c.dht_filename, dht_length, true); + remover.add_file(to_c.dht_filename); + + // Prepare dpt storage + os::truncate_file(to_c.dpt_filename, dpt_length, true); + remover.add_file(to_c.dpt_filename); + + // Necessary, so the memory is always mapped as writeable in memory address range constructor + to_c.create = true; + } else { + // Prepare data storage + os::copy_file(from_c.data_filename, to_c.data_filename, length); + remover.add_file(to_c.data_filename); + + // Prepare dht storage + if (os::exists(from_c.dht_filename) && os::file_size(from_c.dht_filename) == dht_length) { + os::copy_file(from_c.dht_filename, to_c.dht_filename, dht_length); + } else { + os::truncate_file(to_c.dht_filename, dht_length, true); + } + remover.add_file(to_c.dht_filename); + + // Prepare dpt storage + if (os::exists(from_c.dpt_filename) && os::file_size(from_c.dpt_filename) == dpt_length) { + os::copy_file(from_c.dpt_filename, to_c.dpt_filename, dpt_length); + } else { + os::truncate_file(to_c.dpt_filename, dpt_length, true); + } + remover.add_file(to_c.dpt_filename); + + os::change_writable(to_c.data_filename, !read_only); + } +} + +static void prepare_ar_backing_stores(const machine_config &c, scope_remove &remover) { + prepare_ar_backing_store(c.processor.backing_store, AR_SHADOW_STATE_LENGTH, remover); + prepare_ar_backing_store(c.pmas.backing_store, AR_PMAS_LENGTH, remover); + prepare_ar_backing_store(c.dtb.backing_store, AR_DTB_LENGTH, remover); + prepare_ar_backing_store(c.ram.backing_store, c.ram.length, remover); + prepare_ar_backing_store(c.cmio.rx_buffer.backing_store, AR_CMIO_RX_BUFFER_LENGTH, remover); + prepare_ar_backing_store(c.cmio.tx_buffer.backing_store, AR_CMIO_TX_BUFFER_LENGTH, remover); + prepare_ar_backing_store(c.uarch.processor.backing_store, AR_SHADOW_UARCH_STATE_LENGTH, remover); + prepare_ar_backing_store(c.uarch.ram.backing_store, AR_UARCH_RAM_LENGTH, remover); + for (const auto &f : c.flash_drive) { + prepare_ar_backing_store(f.backing_store, f.length, remover); + } +} + +static void prepare_ar_backing_stores_for_share(const machine_config &from_c, machine_config &to_c, + scope_remove &remover) { + prepare_ar_backing_store_for_share(from_c.processor.backing_store, to_c.processor.backing_store, + AR_SHADOW_STATE_LENGTH, false, remover); + prepare_ar_backing_store_for_share(from_c.pmas.backing_store, to_c.pmas.backing_store, AR_PMAS_LENGTH, false, + remover); + prepare_ar_backing_store_for_share(from_c.dtb.backing_store, to_c.dtb.backing_store, AR_DTB_LENGTH, false, remover); + prepare_ar_backing_store_for_share(from_c.ram.backing_store, to_c.ram.backing_store, to_c.ram.length, false, + remover); + prepare_ar_backing_store_for_share(from_c.cmio.rx_buffer.backing_store, to_c.cmio.rx_buffer.backing_store, + AR_CMIO_RX_BUFFER_LENGTH, false, remover); + prepare_ar_backing_store_for_share(from_c.cmio.tx_buffer.backing_store, to_c.cmio.tx_buffer.backing_store, + AR_CMIO_TX_BUFFER_LENGTH, false, remover); + prepare_ar_backing_store_for_share(from_c.uarch.processor.backing_store, to_c.uarch.processor.backing_store, + AR_SHADOW_UARCH_STATE_LENGTH, false, remover); + prepare_ar_backing_store_for_share(from_c.uarch.ram.backing_store, to_c.uarch.ram.backing_store, + AR_UARCH_RAM_LENGTH, false, remover); + for (size_t i = 0; i < to_c.flash_drive.size(); ++i) { + const auto &from_f = from_c.flash_drive[i]; + auto &to_f = to_c.flash_drive[i]; + prepare_ar_backing_store_for_share(from_f.backing_store, to_f.backing_store, from_f.length, from_f.read_only, + remover); + } +} + +machine_address_ranges::machine_address_ranges(const machine_config &config, + const machine_runtime_config &runtime_config, const std::string &dir, scope_remove &remover) { + // Copy config + machine_config c = config; + + // Prepare address ranges backing stores + if (!dir.empty()) { // Machine that it's fully on-disk + if (config.processor.registers.iunrep != 0) { + throw std::invalid_argument{"fully on-disk machines must not be unreproducible"s}; + } + // Adjust config + c.adjust_backing_stores(dir); + // Create backing store directory + os::create_directory(dir); + remover.add_directory(dir); + // Store config + remover.add_file(c.store(dir)); + // Copy backing stores and mark them as shared + prepare_ar_backing_stores_for_share(config, c, remover); + } else { // A machine that may be partially or fully in-memory + // Create and truncate backing stores as necessary + prepare_ar_backing_stores(c, remover); + } + + // Add all address ranges to m_all, and potentially to interpret and hash + m_shadow_state_index = static_cast(m_all.size()); // NOLINT(cppcoreguidelines-prefer-member-initializer) + push_back(make_shadow_state_address_range(c.processor), register_where{.hash_tree = true, .pmas = false}); + m_shadow_uarch_state_index = static_cast(m_all.size()); // NOLINT(cppcoreguidelines-prefer-member-initializer) + push_back(make_shadow_uarch_state_address_range(c.uarch.processor), + register_where{.hash_tree = true, .pmas = false}); + push_back_uarch_ram(c.uarch.ram); + push_back_ram(c.ram); + push_back(make_dtb_address_range(c.dtb), register_where{.hash_tree = true, .pmas = true}); + push_back_flash_drives(c.flash_drive, runtime_config); + push_back_cmio(c.cmio); + push_back(std::make_unique(throw_invalid_argument), + register_where{.hash_tree = false, .pmas = true}); + push_back(std::make_unique(throw_invalid_argument), + register_where{.hash_tree = false, .pmas = true}); + push_back(std::make_unique(throw_invalid_argument), + register_where{.hash_tree = false, .pmas = true}); + push_back(make_pmas_address_range(c.pmas), register_where{.hash_tree = true, .pmas = true}); + push_back_virtio(c.virtio, c.processor.registers.iunrep); + + // Sort indices visible to hash tree by the start of corresponding address range + std::ranges::sort( + m_hash_tree, [](const auto &a, const auto &b) { return a.get_start() < b.get_start(); }, + [this](const auto i) { return *m_all[i]; }); + + // Create descriptions and sort by start address + auto src = + m_all | std::views::filter([](auto &ar) { return !ar->is_empty(); }) | std::views::transform([](auto &ar) { + return address_range_description{.start = ar->get_start(), + .length = ar->get_length(), + .description = ar->get_description()}; + }); + std::ranges::copy(src, std::back_inserter(m_descrs)); + std::ranges::sort(m_descrs, [](auto &a, auto &b) { return a.start < b.start; }); +} + +void machine_address_ranges::push_back_uarch_ram(const uarch_ram_config &uram) { + // Register uarch RAM + static constexpr pmas_flags uram_flags{ + .M = true, + .IO = false, + .R = true, + .W = true, + .X = true, + .IR = true, + .IW = true, + .DID = PMA_ISTART_DID::memory, + }; + constexpr auto ram_description = "uarch RAM"; + auto &ar = push_back(std::make_unique(ram_description, AR_UARCH_RAM_START, + AR_UARCH_RAM_LENGTH, uram_flags, uram.backing_store), + register_where{.hash_tree = true, .pmas = false}); + // Initialize uarch RAM + if (uram.backing_store.newly_created()) { + if (uarch_pristine_ram_len > AR_UARCH_RAM_LENGTH) { + throw std::runtime_error("embedded uarch RAM image does not fit in uarch memory"); + } + memcpy(ar.get_host_memory(), uarch_pristine_ram, uarch_pristine_ram_len); + } +} + +void machine_address_ranges::check(const address_range &new_ar, register_where where) { + if (!where.pmas && !where.hash_tree) { + throw std::runtime_error{"address range "s + new_ar.get_description() + " must be registered somewhere"s}; + } + const auto start = new_ar.get_start(); + const auto end = new_ar.get_end(); + // Checks if new range is machine addressable space (safe unsigned overflows) + if (start > AR_ADDRESSABLE_MASK || end > AR_ADDRESSABLE_MASK) { + throw std::invalid_argument{ + "address range of "s + new_ar.get_description() + " must use at most 56 bits to be addressable"s}; + } + // Range A overlaps with B if A starts before B ends and A ends after B starts + for (const auto &ar : m_all) { + if (start < ar->get_end() && end > ar->get_start()) { + throw std::invalid_argument{"address range of "s + new_ar.get_description() + + " overlaps with address range of existing "s + ar->get_description()}; + } + } +} + +void machine_address_ranges::push_back_ram(const ram_config &ram) { + // Flags for RAM + static constexpr pmas_flags ram_flags{ + .M = true, + .IO = false, + .R = true, + .W = true, + .X = true, + .IR = true, + .IW = true, + .DID = PMA_ISTART_DID::memory, + }; + if (ram.length == 0) { + throw std::invalid_argument("RAM length cannot be zero"); + } + push_back(std::make_unique("RAM"s, AR_RAM_START, ram.length, ram_flags, ram.backing_store), + register_where{.hash_tree = true, .pmas = true}); +} + +void machine_address_ranges::push_back_flash_drives(const flash_drive_configs &flash_drive, + const machine_runtime_config &r) { + if (flash_drive.size() > FLASH_DRIVE_MAX) { + throw std::invalid_argument{"too many flash drives"}; + } + // Register all flash drives + int i = 0; // NOLINT(misc-const-correctness) + for (const auto &f : flash_drive) { + const std::string flash_description = "flash drive "s + std::to_string(i); + // Flags for the flash drive + const pmas_flags flash_flags{ + .M = true, + .IO = false, + .R = true, + .W = !f.read_only, + .X = false, + .IR = true, + .IW = !f.read_only, + .DID = PMA_ISTART_DID::flash_drive, + }; + push_back(std::make_unique(flash_description, f.start, f.length, flash_flags, + f.backing_store, + memory_address_range_config{.host_read_only = f.read_only, .host_no_reserve = r.no_reserve}), + register_where{.hash_tree = true, .pmas = true}); + i++; + } +} + +void machine_address_ranges::push_back_virtio(const virtio_configs &virtio, uint64_t iunrep) { + if (virtio.empty()) { + return; + } + if (virtio.size() > VIRTIO_DEVICE_MAX) { + throw std::invalid_argument{"too many VirtIO devices"}; + } + // VirtIO devices are disallowed in unreproducible mode + if (iunrep == 0) { + throw std::invalid_argument{"virtio devices are only supported in unreproducible machines"}; + } + uint32_t virtio_idx = 0; + for (const auto &c : virtio) { + const auto where = register_where{.hash_tree = false, .pmas = true}; + const auto visitor = overloads{ + [this, virtio_idx, where](const virtio_console_config &) { + const auto start = AR_FIRST_VIRTIO_START + (virtio_idx * AR_VIRTIO_LENGTH); + push_back(std::make_unique(start, AR_VIRTIO_LENGTH, virtio_idx), where); + }, + [this, virtio_idx, where](const virtio_p9fs_config &c) { +#ifdef HAVE_POSIX_FS + const auto start = AR_FIRST_VIRTIO_START + (virtio_idx * AR_VIRTIO_LENGTH); + push_back(std::make_unique(start, AR_VIRTIO_LENGTH, virtio_idx, c.tag, + c.host_directory), + where); +#else + (void) c; + (void) this; + (void) virtio_idx; + (void) where; + throw std::invalid_argument{"virtio 9p device is unsupported in this platform"}; +#endif + }, + [this, virtio_idx, where](const virtio_net_tuntap_config &c) { +#ifdef HAVE_TUNTAP + const auto start = AR_FIRST_VIRTIO_START + (virtio_idx * AR_VIRTIO_LENGTH); + push_back( + std::make_unique(start, AR_VIRTIO_LENGTH, virtio_idx, c.iface), + where); +#else + (void) c; + (void) this; + (void) virtio_idx; + (void) where; + throw std::invalid_argument("virtio network TUN/TAP device is unsupported in this platform"); +#endif + }, + [this, virtio_idx, where](const virtio_net_user_config &c) { +#ifdef HAVE_SLIRP + if (c.hostfwd.size() > VIRTIO_HOSTFWD_MAX) { + throw std::invalid_argument("too many virtio network user host-forwarding ports"); + } + const auto start = AR_FIRST_VIRTIO_START + (virtio_idx * AR_VIRTIO_LENGTH); + push_back(std::make_unique(start, AR_VIRTIO_LENGTH, virtio_idx, c), + where); +#else + (void) c; + (void) this; + (void) virtio_idx; + (void) where; + throw std::invalid_argument("virtio network user device is unsupported in this platform"); +#endif + }, + [](const auto &) { throw std::invalid_argument("invalid virtio device configuration"); }}; + std::visit(visitor, c); + ++virtio_idx; + } +} + +void machine_address_ranges::push_back_cmio(const cmio_config &c) { + const pmas_flags tx_flags{ + .M = true, + .IO = false, + .R = true, + .W = true, + .X = false, + .IR = true, + .IW = true, + .DID = PMA_ISTART_DID::cmio_tx_buffer, + }; + const pmas_flags rx_flags{ + .M = true, + .IO = false, + .R = true, + .W = false, + .X = false, + .IR = true, + .IW = true, + .DID = PMA_ISTART_DID::cmio_rx_buffer, + }; + push_back(std::make_unique("CMIO tx buffer"s, AR_CMIO_TX_BUFFER_START, + AR_CMIO_TX_BUFFER_LENGTH, tx_flags, c.tx_buffer.backing_store), + register_where{.hash_tree = true, .pmas = true}); + push_back(std::make_unique("CMIO rx buffer"s, AR_CMIO_RX_BUFFER_START, + AR_CMIO_RX_BUFFER_LENGTH, rx_flags, c.rx_buffer.backing_store), + register_where{.hash_tree = true, .pmas = true}); +} + +void machine_address_ranges::replace(const memory_range_config &config) { + for (auto &ar : m_all) { + if (ar->get_start() == config.start && ar->get_length() == config.length) { + if (!ar->is_memory() || pmas_is_protected(ar->get_driver_id())) { + throw std::invalid_argument{ + std::string{"attempted replace of protected memory range "}.append(ar->get_description())}; + } + if (ar->is_host_read_only() || !ar->is_writeable()) { + throw std::invalid_argument{ + std::string{"attempted replace of read-only memory range "}.append(ar->get_description())}; + } + if (config.read_only) { + throw std::invalid_argument{std::string{"attempted replace of read-write memory range "} + .append(ar->get_description()) + .append(" with read-only memory range")}; + } + // Replace range, preserving original flags. + // This will automatically start with all pages dirty. + ar = std::make_unique(ar->get_description(), ar->get_start(), ar->get_length(), + ar->get_flags(), config.backing_store); + return; + } + } + throw std::invalid_argument{"attempted replace of inexistent memory range"}; +} + +} // namespace cartesi diff --git a/src/machine-address-ranges.h b/src/machine-address-ranges.h new file mode 100644 index 000000000..8a7fa5bf9 --- /dev/null +++ b/src/machine-address-ranges.h @@ -0,0 +1,222 @@ +// Copyright Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: LGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify it under +// the terms of the GNU Lesser General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) any +// later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +// PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License along +// with this program (see COPYING). If not, see . +// + +#ifndef MACHINE_ADDRESS_RANGES_H +#define MACHINE_ADDRESS_RANGES_H + +#include +#include +#include +#include +#include +#include +#include +#include + +/// \file +/// \brief Cartesi machine address ranges. + +#include "address-range-description.h" +#include "address-range.h" +#include "machine-config.h" +#include "machine-runtime-config.h" +#include "scope-remove.h" +#include "virtio-address-range.h" + +namespace cartesi { + +/// \brief Machine address ranges. +/// \details Class holding all address ranges in the machine +class machine_address_ranges { + + std::vector> m_all; ///< Owner of all address ranges + std::vector m_hash_tree; ///< Indices of address ranges registered with hash tree + std::vector m_pmas; ///< Indices of address ranges registered as PMAs + std::vector m_virtio; ///< Pointers to all VirtIO address ranges + address_range_descriptions m_descrs; ///< Address range descriptions for users + int m_shadow_state_index; ///< Index of shadow state address range + int m_shadow_uarch_state_index; ///< Index of shadow uarch state address range + address_range m_sentinel{"sentinel"}; + +public: + /// \brief Constructor + /// \param config Machine configuration + /// \param runtime_config Runtime configuration + /// \param dir Directory for backing store files (can be empty) + /// \param remover Scope remove object to remove created files on failure + explicit machine_address_ranges(const machine_config &config, const machine_runtime_config &runtime_config, + const std::string &dir, scope_remove &remover); + + /// \brief Const view of all address ranges + auto all() const { + return std::views::transform(m_all, [](const auto &p) -> const address_range & { return *p; }); + } + + /// \brief Const view of address ranges registered with hash tree + auto hash_tree_view() const { + return std::views::transform(m_hash_tree, [this](auto i) -> const address_range & { return *m_all[i]; }); + } + + /// \brief View of address ranges registered with hash tree + auto hash_tree_view() { + return std::views::transform(m_hash_tree, [this](auto i) -> address_range & { return *m_all[i]; }); + } + + /// \brief Const view of address ranges registered as PMAs + auto pmas_view() const { + return std::views::transform(m_pmas, [this](auto i) -> const address_range & { return *m_all[i]; }); + } + + /// \brief Const view of address ranges registered as PMAs + auto pmas_view() { + return std::views::transform(m_pmas, [this](auto i) -> address_range & { return *m_all[i]; }); + } + + /// \brief View of all VirtIO address ranges + auto virtio_view() { + return std::views::transform(m_virtio, [](auto *p) -> virtio_address_range & { return *p; }); + } + + /// \brief View of all VirtIO address ranges + auto virtio_view() const { + return std::views::transform(m_virtio, [](auto *p) -> const virtio_address_range & { return *p; }); + } + + /// \brief Returns the address range corresponding to the ith PMA + const address_range &read_pma(uint64_t index) const noexcept { + if (index >= m_pmas.size()) [[unlikely]] { + return m_sentinel; + } + return *m_all[static_cast(m_pmas[static_cast(index)])]; + } + + /// \brief Returns const address range that covers a given physical memory region + /// \param paddr Target physical address of start of region. + /// \param length Length of region, in bytes. + /// \returns Corresponding address range if found, or an empty sentinel otherwise. + const address_range &find(uint64_t paddr, uint64_t length) const noexcept { + for (const auto &ar : m_all) { + if (ar->contains_absolute(paddr, length)) { + return *ar; + } + } + return m_sentinel; + } + + /// \brief Returns address range that covers a given physical memory region + /// \param paddr Target physical address of start of region. + /// \param length Length of region, in bytes. + /// \returns Corresponding address range if found, or an empty sentinel otherwise. + address_range &find(uint64_t paddr, uint64_t length) noexcept { + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-const-cast) + return const_cast(std::as_const(*this).find(paddr, length)); + } + + /// \brief Returns a const address range that covers a given word in physical memory + /// \tparam T Type of word. + /// \param paddr Target physical address of start of region. + /// \returns Corresponding address range if found, or an empty sentinel otherwise. + template + const address_range &find(uint64_t paddr) const { + return find(paddr, sizeof(T)); + } + + /// \brief Returns a const address range that covers a given word in physical memory + /// \tparam T Type of word. + /// \param paddr Target physical address of start of region. + /// \returns Corresponding address range if found, or an empty sentinel otherwise. + template + address_range &find(uint64_t paddr) { + return find(paddr, sizeof(T)); + } + + /// \brief Replaces a memory address range. + /// \param config Configuration of the new memory address range. + /// \details A memory address range matching the start and length specified in the config must exist. + void replace(const memory_range_config &config); + + /// \brief Returns descriptions of all address ranges. + const address_range_descriptions &descriptions_view() const { + return m_descrs; + } + + /// \brief Marks as dirty the pages of address ranges assumed dirty + void mark_always_dirty_pages(); + +private: + ///< Where to register an address range + struct register_where { + bool hash_tree; //< Register with hash tree, so it appears in the root hash + bool pmas; //< Register as a PMA, so the interpreter can see it + }; + + /// \brief Checks if address range can be registered. + /// \param ar Address range object to register. + /// \param where Where to register the address range. + void check(const address_range &ar, register_where where); + + /// \brief Registers a new address range. + /// \tparam AR An address range or derived type. + /// \param ar The address range object to register (as an r-value). + /// \param where Where to register the address range. + /// \returns Reference to address range object after it is moved inside the machine. + /// \details The r-value address range is moved to the heap, and a smart pointer holding it is added to a container. + /// Once the address range is moved to the heap, its address will remain valid until it is replaced by + /// a call to replace(), or until this object is destroyed. + /// This means pointers to address ranges remain valid even after subsequent calls to push_back(), + /// but may be invalidated by calls to replace(). + /// For a stronger guarantee, when an address range is replaced, the smart pointer holding the new address range + /// overwrites the smart pointer holding the old address range at the same index in the container. + /// This means the index into the container that owns all address ranges will always refer to same address range + /// after subsequent calls to register() and calls to replace() as well. + /// \details Besides the container that stores the address ranges, the machine maintains three subsets. + /// The "hash_tree" subset lists the indices of the address ranges that will be considered by the hash tree + /// during the computation of the state hash. + /// The "pmas" subset lists the indices of the address ranges registered as PMAs, and therefore visible + /// from within the interpreter. + /// The \p where parameter tells whether to register the address range with the hash tree and/or as a PMA. + /// Finally, the "virtio" subset lists every VirtIO address range that has been registered. + template + requires std::derived_from + AR &push_back(std::unique_ptr &&ar_ptr, register_where where); + + /// \brief Adds uarch RAM address range + /// \param c uarch RAM configuration + void push_back_uarch_ram(const uarch_ram_config &uram); + + /// \brief Adds RAM address range + /// \param ram RAM configuration + void push_back_ram(const ram_config &ram); + + /// \brief Adds flash drive address ranges + /// \param flash_drive Flash drive configurations + /// \param runtime_config Runtime configuration + /// \detail This modifies the flash drive configuration with automatic start/length from backing storage, if needed + void push_back_flash_drives(const flash_drive_configs &flash_drive, const machine_runtime_config &runtime_config); + + /// \brief Adds virtio address ranges + /// \param virtio VirtIO configurations + /// \param iunrep Initial value of iunrep CSR + void push_back_virtio(const virtio_configs &virtio, uint64_t iunrep); + + /// \brief Adds CMIO address ranges + /// \param c CMIO configuration + void push_back_cmio(const cmio_config &c); +}; + +} // namespace cartesi + +#endif diff --git a/src/machine-c-api.cpp b/src/machine-c-api.cpp index 920133f2a..f24ae0f37 100644 --- a/src/machine-c-api.cpp +++ b/src/machine-c-api.cpp @@ -16,6 +16,7 @@ #include "machine-c-api.h" +#include #include #include #include @@ -24,32 +25,59 @@ #include #include #include +#include #include +#include #include #include #include +#include #include #include #include "access-log.h" -#include "htif.h" -#include "i-virtual-machine.h" +#include "address-range-constants.h" +#include "address-range-defines.h" +#include "address-range-description.h" +#include "back-merkle-tree.h" +#include "htif-constants.h" +#include "htif-defines.h" +#include "i-machine.h" +#include "interpret.h" #include "json-util.h" +#include "keccak-256-hasher.h" +#include "local-machine.h" #include "machine-c-api-internal.h" #include "machine-config.h" -#include "machine-memory-range-descr.h" -#include "machine-merkle-tree.h" +#include "machine-hash.h" +#include "machine-reg.h" #include "machine-runtime-config.h" #include "machine.h" #include "os-features.h" -#include "pma-constants.h" -#include "virtual-machine.h" +#include "ranges.h" +#include "sha-256-hasher.h" static std::string &get_last_err_msg_storage() { static THREAD_LOCAL std::string last_err_msg; return last_err_msg; } +static_assert(AR_CMIO_RX_BUFFER_START_DEF == CM_AR_CMIO_RX_BUFFER_START); +static_assert(AR_CMIO_RX_BUFFER_LOG2_SIZE_DEF == CM_AR_CMIO_RX_BUFFER_LOG2_SIZE); +static_assert(AR_CMIO_TX_BUFFER_START_DEF == CM_AR_CMIO_TX_BUFFER_START); +static_assert(AR_CMIO_TX_BUFFER_LOG2_SIZE_DEF == CM_AR_CMIO_TX_BUFFER_LOG2_SIZE); +static_assert(AR_SHADOW_REVERT_ROOT_HASH_START_DEF == CM_AR_SHADOW_REVERT_ROOT_HASH_START); +static_assert(AR_RAM_START_DEF == CM_AR_RAM_START); + +static_assert(HTIF_YIELD_AUTOMATIC_REASON_PROGRESS_DEF == CM_CMIO_YIELD_AUTOMATIC_REASON_PROGRESS); +static_assert(HTIF_YIELD_AUTOMATIC_REASON_TX_OUTPUT_DEF == CM_CMIO_YIELD_AUTOMATIC_REASON_TX_OUTPUT); +static_assert(HTIF_YIELD_AUTOMATIC_REASON_TX_REPORT_DEF == CM_CMIO_YIELD_AUTOMATIC_REASON_TX_REPORT); +static_assert(HTIF_YIELD_MANUAL_REASON_RX_ACCEPTED_DEF == CM_CMIO_YIELD_MANUAL_REASON_RX_ACCEPTED); +static_assert(HTIF_YIELD_MANUAL_REASON_RX_REJECTED_DEF == CM_CMIO_YIELD_MANUAL_REASON_RX_REJECTED); +static_assert(HTIF_YIELD_MANUAL_REASON_TX_EXCEPTION_DEF == CM_CMIO_YIELD_MANUAL_REASON_TX_EXCEPTION); +static_assert(HTIF_YIELD_REASON_ADVANCE_STATE_DEF == CM_CMIO_YIELD_REASON_ADVANCE_STATE); +static_assert(HTIF_YIELD_REASON_INSPECT_STATE_DEF == CM_CMIO_YIELD_REASON_INSPECT_STATE); + const char *cm_get_last_error_message() { return get_last_err_msg_storage().c_str(); } @@ -60,53 +88,53 @@ const char *cm_set_temp_string(const std::string &s) { return temp_string.c_str(); } -cm_error cm_result_failure() try { throw; } catch (std::exception &e) { +cm_error cm_result_failure() try { throw; } catch (const std::exception &e) { try { get_last_err_msg_storage() = e.what(); throw; - } catch (std::invalid_argument &ex) { + } catch (const std::invalid_argument &ex) { return CM_ERROR_INVALID_ARGUMENT; - } catch (std::domain_error &ex) { + } catch (const std::domain_error &ex) { return CM_ERROR_DOMAIN_ERROR; - } catch (std::length_error &ex) { + } catch (const std::length_error &ex) { return CM_ERROR_LENGTH_ERROR; - } catch (std::out_of_range &ex) { + } catch (const std::out_of_range &ex) { return CM_ERROR_OUT_OF_RANGE; - } catch (std::logic_error &ex) { + } catch (const std::logic_error &ex) { return CM_ERROR_LOGIC_ERROR; - } catch (std::bad_optional_access &ex) { + } catch (const std::bad_optional_access &ex) { return CM_ERROR_BAD_OPTIONAL_ACCESS; - } catch (std::range_error &ex) { + } catch (const std::range_error &ex) { return CM_ERROR_RANGE_ERROR; - } catch (std::overflow_error &ex) { + } catch (const std::overflow_error &ex) { return CM_ERROR_OVERFLOW_ERROR; - } catch (std::underflow_error &ex) { + } catch (const std::underflow_error &ex) { return CM_ERROR_UNDERFLOW_ERROR; - } catch (std::regex_error &ex) { + } catch (const std::regex_error &ex) { return CM_ERROR_REGEX_ERROR; - } catch (std::system_error &ex) { + } catch (const std::system_error &ex) { return CM_ERROR_SYSTEM_ERROR; - } catch (std::runtime_error &ex) { + } catch (const std::runtime_error &ex) { return CM_ERROR_RUNTIME_ERROR; - } catch (std::bad_typeid &ex) { + } catch (const std::bad_typeid &ex) { return CM_ERROR_BAD_TYPEID; - } catch (std::bad_any_cast &ex) { + } catch (const std::bad_any_cast &ex) { return CM_ERROR_BAD_ANY_CAST; - } catch (std::bad_cast &ex) { + } catch (const std::bad_cast &ex) { return CM_ERROR_BAD_CAST; - } catch (std::bad_weak_ptr &ex) { + } catch (const std::bad_weak_ptr &ex) { return CM_ERROR_BAD_WEAK_PTR; - } catch (std::bad_function_call &ex) { + } catch (const std::bad_function_call &ex) { return CM_ERROR_BAD_FUNCTION_CALL; - } catch (std::bad_array_new_length &ex) { + } catch (const std::bad_array_new_length &ex) { return CM_ERROR_BAD_ARRAY_NEW_LENGTH; - } catch (std::bad_alloc &ex) { + } catch (const std::bad_alloc &ex) { return CM_ERROR_BAD_ALLOC; - } catch (std::bad_exception &ex) { + } catch (const std::bad_exception &ex) { return CM_ERROR_BAD_EXCEPTION; - } catch (std::bad_variant_access &ex) { + } catch (const std::bad_variant_access &ex) { return CM_ERROR_BAD_VARIANT_ACCESS; - } catch (std::exception &e) { + } catch (const std::exception &e) { return CM_ERROR_EXCEPTION; } } catch (...) { @@ -435,36 +463,58 @@ static cartesi::machine_reg convert_from_c(cm_reg r) { throw std::domain_error{"unknown register"}; } -static cartesi::i_virtual_machine *convert_from_c(cm_machine *m) { +static cartesi::sharing_mode convert_from_c(cm_sharing_mode sharing) { + switch (sharing) { + case CM_SHARING_NONE: + return cartesi::sharing_mode::none; + case CM_SHARING_CONFIG: + return cartesi::sharing_mode::config; + case CM_SHARING_ALL: + return cartesi::sharing_mode::all; + } + throw std::domain_error{"unknown sharing mode"}; +} + +static cartesi::i_machine *convert_from_c(cm_machine *m) { if (m == nullptr) { throw std::invalid_argument("invalid machine"); } // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) - return reinterpret_cast(m); + return reinterpret_cast(m); } -static const cartesi::i_virtual_machine *convert_from_c(const cm_machine *m) { +static const cartesi::i_machine *convert_from_c(const cm_machine *m) { if (m == nullptr) { throw std::invalid_argument("invalid machine"); } // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) - return reinterpret_cast(m); + return reinterpret_cast(m); } -static cm_machine *convert_to_c(cartesi::i_virtual_machine *cpp_m) { +static cm_machine *convert_to_c(cartesi::i_machine *cpp_m) { // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) return reinterpret_cast(cpp_m); } -static cartesi::machine_merkle_tree::hash_type convert_from_c(const cm_hash *c_hash) { +static cartesi::machine_hash convert_from_c(const cm_hash *c_hash) { if (c_hash == nullptr) { throw std::invalid_argument("invalid hash"); } - cartesi::machine_merkle_tree::hash_type cpp_hash; // In emulator this is std::array; - memcpy(cpp_hash.data(), c_hash, sizeof(cm_hash)); + cartesi::machine_hash cpp_hash; // In emulator this is std::array; + std::memcpy(cpp_hash.data(), c_hash, sizeof(cm_hash)); return cpp_hash; } +static void convert_to_c(cartesi::machine_hash cpp_hash, cm_hash *c_hash) { + if (c_hash == nullptr) { + throw std::invalid_argument("invalid hash output"); + } + using elem_t = std::ranges::range_value_t; + constexpr auto elem_n = std::extent_v; + static_assert(std::ranges::size(cpp_hash) == elem_n); + std::ranges::copy(cpp_hash | cartesi::views::cast_to, std::ranges::data(*c_hash)); +} + // ---------------------------------------------- // The C API implementation // ---------------------------------------------- @@ -473,7 +523,7 @@ cm_error cm_new(cm_machine **new_m) try { if (new_m == nullptr) { throw std::invalid_argument("invalid new machine output"); } - *new_m = convert_to_c(new cartesi::virtual_machine()); + *new_m = convert_to_c(new cartesi::local_machine()); return cm_result_success(); } catch (...) { if (new_m != nullptr) { @@ -507,43 +557,43 @@ cm_error cm_is_empty(const cm_machine *m, bool *yes) try { return cm_result_failure(); } -cm_error cm_create(cm_machine *m, const char *config, const char *runtime_config) try { +cm_error cm_create(cm_machine *m, const char *config, const char *runtime_config, const char *dir) try { auto *cpp_m = convert_from_c(m); if (config == nullptr) { throw std::invalid_argument("invalid machine configuration"); } - const auto c = cartesi::from_json(config); + const auto c = cartesi::from_json(config, "config"); cartesi::machine_runtime_config r; if (runtime_config != nullptr) { - r = cartesi::from_json(runtime_config); + r = cartesi::from_json(runtime_config, "runtime_config"); } - cpp_m->create(c, r); + cpp_m->create(c, r, dir != nullptr ? std::string(dir) : std::string()); return cm_result_success(); } catch (...) { return cm_result_failure(); } -cm_error cm_load(cm_machine *m, const char *dir, const char *runtime_config) try { +cm_error cm_load(cm_machine *m, const char *dir, const char *runtime_config, cm_sharing_mode sharing) try { auto *cpp_m = convert_from_c(m); if (dir == nullptr) { throw std::invalid_argument("invalid dir"); } cartesi::machine_runtime_config r; if (runtime_config != nullptr) { - r = cartesi::from_json(runtime_config); + r = cartesi::from_json(runtime_config, "runtime_config"); } - cpp_m->load(dir, r); + cpp_m->load(dir, r, convert_from_c(sharing)); return cm_result_success(); } catch (...) { return cm_result_failure(); } -cm_error cm_load_new(const char *dir, const char *runtime_config, cm_machine **new_m) { +cm_error cm_load_new(const char *dir, const char *runtime_config, cm_sharing_mode sharing, cm_machine **new_m) { auto err = cm_new(new_m); if (err != 0) { return err; } - err = cm_load(*new_m, dir, runtime_config); + err = cm_load(*new_m, dir, runtime_config, sharing); if (err != 0) { cm_delete(*new_m); *new_m = nullptr; @@ -551,12 +601,12 @@ cm_error cm_load_new(const char *dir, const char *runtime_config, cm_machine **n return err; } -cm_error cm_create_new(const char *config, const char *runtime_config, cm_machine **new_m) { +cm_error cm_create_new(const char *config, const char *runtime_config, const char *dir, cm_machine **new_m) { auto err = cm_new(new_m); if (err != 0) { return err; } - err = cm_create(*new_m, config, runtime_config); + err = cm_create(*new_m, config, runtime_config, dir); if (err != 0) { cm_delete(*new_m); *new_m = nullptr; @@ -564,12 +614,37 @@ cm_error cm_create_new(const char *config, const char *runtime_config, cm_machin return err; } -cm_error cm_store(const cm_machine *m, const char *dir) try { +cm_error cm_store(const cm_machine *m, const char *dir, cm_sharing_mode sharing) try { + if (dir == nullptr) { + throw std::invalid_argument("invalid dir"); + } + const auto *cpp_m = convert_from_c(m); + cpp_m->store(dir, convert_from_c(sharing)); + return cm_result_success(); +} catch (...) { + return cm_result_failure(); +} + +cm_error cm_clone_stored(const cm_machine *m, const char *from_dir, const char *to_dir) try { + if (from_dir == nullptr) { + throw std::invalid_argument("invalid from dir"); + } + if (to_dir == nullptr) { + throw std::invalid_argument("invalid to dir"); + } + const auto *cpp_m = convert_from_c(m); + cpp_m->clone_stored(from_dir, to_dir); + return cm_result_success(); +} catch (...) { + return cm_result_failure(); +} + +cm_error cm_remove_stored(const cm_machine *m, const char *dir) try { if (dir == nullptr) { throw std::invalid_argument("invalid dir"); } const auto *cpp_m = convert_from_c(m); - cpp_m->store(dir); + cpp_m->remove_stored(dir); return cm_result_success(); } catch (...) { return cm_result_failure(); @@ -589,6 +664,41 @@ cm_error cm_run(cm_machine *m, uint64_t mcycle_end, cm_break_reason *break_reaso return cm_result_failure(); } +cm_error cm_collect_mcycle_root_hashes(cm_machine *m, uint64_t mcycle_end, uint64_t mcycle_period, + uint64_t mcycle_phase, int32_t log2_bundle_mcycle_count, const char *previous_back_tree, const char **result) try { + if (result == nullptr) { + throw std::invalid_argument("invalid result output"); + } + auto *cpp_m = convert_from_c(m); + const auto cpp_previous_back_tree = + cartesi::from_json>(previous_back_tree, "previous_back_tree"); + const auto cpp_res = cpp_m->collect_mcycle_root_hashes(mcycle_end, mcycle_period, mcycle_phase, + log2_bundle_mcycle_count, cpp_previous_back_tree); + *result = cm_set_temp_string(cartesi::to_json(cpp_res).dump()); + return cm_result_success(); +} catch (...) { + if (result != nullptr) { + *result = nullptr; + } + return cm_result_failure(); +} + +cm_error cm_collect_uarch_cycle_root_hashes(cm_machine *m, uint64_t mcycle_end, int32_t log2_bundle_uarch_cycle_count, + const char **result) try { + if (result == nullptr) { + throw std::invalid_argument("invalid result output"); + } + auto *cpp_m = convert_from_c(m); + const auto cpp_res = cpp_m->collect_uarch_cycle_root_hashes(mcycle_end, log2_bundle_uarch_cycle_count); + *result = cm_set_temp_string(cartesi::to_json(cpp_res).dump()); + return cm_result_success(); +} catch (...) { + if (result != nullptr) { + *result = nullptr; + } + return cm_result_failure(); +} + cm_error cm_reset_uarch(cm_machine *m) try { auto *cpp_m = convert_from_c(m); cpp_m->reset_uarch(); @@ -627,7 +737,7 @@ cm_error cm_run_uarch(cm_machine *m, uint64_t uarch_cycle_end, cm_uarch_break_re return cm_result_failure(); } -CM_API cm_error cm_log_step(cm_machine *m, uint64_t mcycle_count, const char *log_filename, +cm_error cm_log_step(cm_machine *m, uint64_t mcycle_count, const char *log_filename, cm_break_reason *break_reason) try { if (log_filename == nullptr) { throw std::invalid_argument("invalid log_filename"); @@ -666,8 +776,8 @@ cm_error cm_verify_step(const cm_machine *m, const cm_hash *root_hash_before, co if (log_filename == nullptr) { throw std::invalid_argument("invalid log_filename"); } - const cartesi::machine::hash_type cpp_root_hash_before = convert_from_c(root_hash_before); - const cartesi::machine::hash_type cpp_root_hash_after = convert_from_c(root_hash_after); + const cartesi::machine_hash cpp_root_hash_before = convert_from_c(root_hash_before); + const cartesi::machine_hash cpp_root_hash_after = convert_from_c(root_hash_after); cartesi::interpreter_break_reason status{}; if (m != nullptr) { const auto *cpp_m = convert_from_c(m); @@ -692,9 +802,9 @@ cm_error cm_verify_step_uarch(const cm_machine *m, const cm_hash *root_hash_befo throw std::invalid_argument("invalid access log"); } const auto cpp_log = // NOLINTNEXTLINE(bugprone-unchecked-optional-access) - cartesi::from_json>(log).value(); - const cartesi::machine::hash_type cpp_root_hash_before = convert_from_c(root_hash_before); - const cartesi::machine::hash_type cpp_root_hash_after = convert_from_c(root_hash_after); + cartesi::from_json>(log, "log").value(); + const cartesi::machine_hash cpp_root_hash_before = convert_from_c(root_hash_before); + const cartesi::machine_hash cpp_root_hash_after = convert_from_c(root_hash_after); if (m != nullptr) { const auto *cpp_m = convert_from_c(m); cpp_m->verify_step_uarch(cpp_root_hash_before, cpp_log, cpp_root_hash_after); @@ -712,9 +822,9 @@ cm_error cm_verify_reset_uarch(const cm_machine *m, const cm_hash *root_hash_bef throw std::invalid_argument("invalid access log"); } const auto cpp_log = // NOLINTNEXTLINE(bugprone-unchecked-optional-access) - cartesi::from_json>(log).value(); - const cartesi::machine::hash_type cpp_root_hash_before = convert_from_c(root_hash_before); - const cartesi::machine::hash_type cpp_root_hash_after = convert_from_c(root_hash_after); + cartesi::from_json>(log, "log").value(); + const cartesi::machine_hash cpp_root_hash_before = convert_from_c(root_hash_before); + const cartesi::machine_hash cpp_root_hash_after = convert_from_c(root_hash_after); if (m != nullptr) { const auto *cpp_m = convert_from_c(m); cpp_m->verify_reset_uarch(cpp_root_hash_before, cpp_log, cpp_root_hash_after); @@ -726,12 +836,27 @@ cm_error cm_verify_reset_uarch(const cm_machine *m, const cm_hash *root_hash_bef return cm_result_failure(); } +cm_error cm_get_hash_tree_stats(cm_machine *m, bool clear, const char **stats) try { + if (stats == nullptr) { + throw std::invalid_argument("invalid stats output"); + } + auto *cpp_m = convert_from_c(m); + const auto cpp_stats = cpp_m->get_hash_tree_stats(clear); + *stats = cm_set_temp_string(cartesi::to_json(cpp_stats).dump()); + return cm_result_success(); +} catch (...) { + if (stats != nullptr) { + *stats = nullptr; + } + return cm_result_failure(); +} + cm_error cm_get_proof(const cm_machine *m, uint64_t address, int32_t log2_size, const char **proof) try { if (proof == nullptr) { throw std::invalid_argument("invalid proof output"); } const auto *cpp_m = convert_from_c(m); - const cartesi::machine_merkle_tree::proof_type cpp_proof = cpp_m->get_proof(address, log2_size); + const auto cpp_proof = cpp_m->get_proof(address, log2_size); *proof = cm_set_temp_string(cartesi::to_json(cpp_proof).dump()); return cm_result_success(); } catch (...) { @@ -742,24 +867,29 @@ cm_error cm_get_proof(const cm_machine *m, uint64_t address, int32_t log2_size, } cm_error cm_get_root_hash(const cm_machine *m, cm_hash *hash) try { - if (hash == nullptr) { - throw std::invalid_argument("invalid hash output"); - } const auto *cpp_m = convert_from_c(m); - cartesi::machine_merkle_tree::hash_type cpp_hash; - cpp_m->get_root_hash(cpp_hash); - memcpy(hash, static_cast(cpp_hash.data()), sizeof(cm_hash)); + const cartesi::machine_hash cpp_hash = cpp_m->get_root_hash(); + convert_to_c(cpp_hash, hash); return cm_result_success(); } catch (...) { return cm_result_failure(); } -cm_error cm_verify_merkle_tree(cm_machine *m, bool *result) try { +cm_error cm_get_node_hash(const cm_machine *m, uint64_t address, int log2_size, cm_hash *hash) try { + const auto *cpp_m = convert_from_c(m); + const cartesi::machine_hash cpp_hash = cpp_m->get_node_hash(address, log2_size); + convert_to_c(cpp_hash, hash); + return cm_result_success(); +} catch (...) { + return cm_result_failure(); +} + +cm_error cm_verify_hash_tree(cm_machine *m, bool *result) try { if (result == nullptr) { throw std::invalid_argument("invalid result output"); } auto *cpp_m = convert_from_c(m); - *result = cpp_m->verify_merkle_tree(); + *result = cpp_m->verify_hash_tree(); return cm_result_success(); } catch (...) { if (result != nullptr) { @@ -824,18 +954,25 @@ cm_error cm_read_word(const cm_machine *m, uint64_t address, uint64_t *val) try } return cm_result_failure(); } +cm_error cm_write_word(cm_machine *m, uint64_t address, uint64_t val) try { + auto *cpp_m = convert_from_c(m); + cpp_m->write_word(address, val); + return cm_result_success(); +} catch (...) { + return cm_result_failure(); +} -cm_error cm_read_memory(const cm_machine *m, uint64_t address, uint8_t *data, uint64_t length) try { +cm_error cm_read_memory(const cm_machine *m, uint64_t paddr, uint8_t *data, uint64_t length) try { const auto *cpp_m = convert_from_c(m); - cpp_m->read_memory(address, data, length); + cpp_m->read_memory(paddr, data, length); return cm_result_success(); } catch (...) { return cm_result_failure(); } -cm_error cm_write_memory(cm_machine *m, uint64_t address, const uint8_t *data, uint64_t length) try { +cm_error cm_write_memory(cm_machine *m, uint64_t paddr, const uint8_t *data, uint64_t length) try { auto *cpp_m = convert_from_c(m); - cpp_m->write_memory(address, data, length); + cpp_m->write_memory(paddr, data, length); return cm_result_success(); } catch (...) { return cm_result_failure(); @@ -868,20 +1005,6 @@ cm_error cm_translate_virtual_address(cm_machine *m, uint64_t vaddr, uint64_t *p return cm_result_failure(); } -cm_error cm_verify_dirty_page_maps(cm_machine *m, bool *result) try { - if (result == nullptr) { - throw std::invalid_argument("invalid result output"); - } - auto *cpp_m = convert_from_c(m); - *result = cpp_m->verify_dirty_page_maps(); - return cm_result_success(); -} catch (...) { - if (result != nullptr) { - *result = false; - } - return cm_result_failure(); -} - cm_error cm_get_initial_config(const cm_machine *m, const char **config) try { if (config == nullptr) { throw std::invalid_argument("invalid config output"); @@ -916,7 +1039,7 @@ cm_error cm_set_runtime_config(cm_machine *m, const char *runtime_config) try { if (runtime_config == nullptr) { throw std::invalid_argument("invalid machine runtime configuration"); } - auto r = cartesi::from_json(runtime_config); + auto r = cartesi::from_json(runtime_config, "runtime_config"); auto *cpp_m = convert_from_c(m); cpp_m->set_runtime_config(r); return cm_result_success(); @@ -944,14 +1067,12 @@ cm_error cm_get_default_config(const cm_machine *m, const char **config) try { return cm_result_failure(); } -cm_error cm_replace_memory_range(cm_machine *m, uint64_t start, uint64_t length, bool shared, - const char *image_filename) try { +cm_error cm_replace_memory_range(cm_machine *m, const char *range_config) try { auto *cpp_m = convert_from_c(m); - cartesi::memory_range_config cpp_range; - cpp_range.start = start; - cpp_range.length = length; - cpp_range.shared = shared; - cpp_range.image_filename = (image_filename != nullptr) ? image_filename : ""; + if (range_config == nullptr) { + throw std::invalid_argument("invalid memory range configuration"); + } + const auto cpp_range = cartesi::from_json(range_config, "range_config"); cpp_m->replace_memory_range(cpp_range); return cm_result_success(); } catch (...) { @@ -973,12 +1094,12 @@ cm_error cm_destroy(cm_machine *m) try { return cm_result_failure(); } -cm_error cm_get_memory_ranges(const cm_machine *m, const char **ranges) try { +cm_error cm_get_address_ranges(const cm_machine *m, const char **ranges) try { if (ranges == nullptr) { throw std::invalid_argument("invalid memory range output"); } const auto *cpp_m = convert_from_c(m); - const cartesi::machine_memory_range_descrs cpp_ranges = cpp_m->get_memory_ranges(); + const cartesi::address_range_descriptions cpp_ranges = cpp_m->get_address_ranges(); *ranges = cm_set_temp_string(cartesi::to_json(cpp_ranges).dump()); return cm_result_success(); } catch (...) { @@ -1021,7 +1142,7 @@ cm_error cm_receive_cmio_request(const cm_machine *m, uint8_t *cmd, uint16_t *re if (data_length > *length) { throw std::invalid_argument{"data buffer length is too small"}; } - cpp_m->read_memory(cartesi::PMA_CMIO_TX_BUFFER_START, data, data_length); + cpp_m->read_memory(cartesi::AR_CMIO_TX_BUFFER_START, data, data_length); } } if (cmd != nullptr) { @@ -1078,9 +1199,9 @@ cm_error cm_verify_send_cmio_response(const cm_machine *m, uint16_t reason, cons throw std::invalid_argument("invalid access log"); } const auto cpp_log = // NOLINTNEXTLINE(bugprone-unchecked-optional-access) - cartesi::from_json>(log).value(); - const cartesi::machine::hash_type cpp_root_hash_before = convert_from_c(root_hash_before); - const cartesi::machine::hash_type cpp_root_hash_after = convert_from_c(root_hash_after); + cartesi::from_json>(log, "log").value(); + const cartesi::machine_hash cpp_root_hash_before = convert_from_c(root_hash_before); + const cartesi::machine_hash cpp_root_hash_after = convert_from_c(root_hash_after); if (m != nullptr) { const auto *cpp_m = convert_from_c(m); cpp_m->verify_send_cmio_response(reason, data, length, cpp_root_hash_before, cpp_log, cpp_root_hash_after); @@ -1092,3 +1213,58 @@ cm_error cm_verify_send_cmio_response(const cm_machine *m, uint16_t reason, cons } catch (...) { return cm_result_failure(); } + +cm_error cm_get_hash(cm_hash_function hash_function, const uint8_t *data, uint64_t length, cm_hash *result) try { + if (data == nullptr && length > 0) { + throw std::invalid_argument("invalid data input"); + } + const std::span data_span{data, static_cast(length)}; + cartesi::machine_hash cpp_result; + + switch (hash_function) { + case CM_HASH_KECCAK256: { + cartesi::keccak_256_hasher hasher; + hasher.hash(data_span, cpp_result); + break; + } + case CM_HASH_SHA256: { + cartesi::sha_256_hasher hasher; + hasher.hash(data_span, cpp_result); + break; + } + default: + throw std::invalid_argument("invalid hash function"); + } + + convert_to_c(cpp_result, result); + return cm_result_success(); +} catch (...) { + return cm_result_failure(); +} + +cm_error cm_get_concat_hash(cm_hash_function hash_function, const cm_hash *left, const cm_hash *right, + cm_hash *result) try { + cartesi::machine_hash cpp_left = convert_from_c(left); + cartesi::machine_hash cpp_right = convert_from_c(right); + cartesi::machine_hash cpp_result; + + switch (hash_function) { + case CM_HASH_KECCAK256: { + cartesi::keccak_256_hasher h; + h.concat_hash(cpp_left, cpp_right, cpp_result); + break; + } + case CM_HASH_SHA256: { + cartesi::sha_256_hasher h; + h.concat_hash(cpp_left, cpp_right, cpp_result); + break; + } + default: + throw std::invalid_argument("invalid hash function"); + } + + convert_to_c(cpp_result, result); + return cm_result_success(); +} catch (...) { + return cm_result_failure(); +} diff --git a/src/machine-c-api.h b/src/machine-c-api.h index ae3e523ae..75eea760d 100644 --- a/src/machine-c-api.h +++ b/src/machine-c-api.h @@ -37,6 +37,7 @@ extern "C" { #endif #define CM_MCYCLE_MAX UINT64_MAX +#define CM_UARCH_CYCLE_MAX UINT64_C(1048576) // ----------------------------------------------------------------------------- // API enums and structures @@ -51,13 +52,14 @@ typedef enum cm_constant { } cm_constant; /// \brief Physical memory addresses (only the most useful are exposed in the API). -typedef enum cm_pma_constant { - CM_PMA_CMIO_RX_BUFFER_START = 0x60000000, - CM_PMA_CMIO_RX_BUFFER_LOG2_SIZE = 21, - CM_PMA_CMIO_TX_BUFFER_START = 0x60800000, - CM_PMA_CMIO_TX_BUFFER_LOG2_SIZE = 21, - CM_PMA_RAM_START = 0x80000000, -} cm_pma_constant; +typedef enum cm_pmas_constant { + CM_AR_CMIO_RX_BUFFER_START = 0x60000000, + CM_AR_CMIO_RX_BUFFER_LOG2_SIZE = 21, + CM_AR_CMIO_TX_BUFFER_START = 0x60800000, + CM_AR_CMIO_TX_BUFFER_LOG2_SIZE = 21, + CM_AR_SHADOW_REVERT_ROOT_HASH_START = 0xfe0, + CM_AR_RAM_START = 0x80000000, +} cm_pmas_constant; /// \brief Error codes returned from the C API. typedef enum cm_error { @@ -101,6 +103,7 @@ typedef enum cm_break_reason { typedef enum cm_uarch_break_reason { CM_UARCH_BREAK_REASON_REACHED_TARGET_CYCLE, CM_UARCH_BREAK_REASON_UARCH_HALTED, + CM_UARCH_BREAK_REASON_CYCLE_OVERFLOW, CM_UARCH_BREAK_REASON_FAILED, } cm_uarch_break_reason; @@ -128,9 +131,16 @@ typedef enum cm_cmio_yield_reason { CM_CMIO_YIELD_REASON_INSPECT_STATE = 1, ///< Input in rx buffer is an inspect state } cm_cmio_yield_reason; +/// \brief Sharing modes. +typedef enum cm_sharing_mode { + CM_SHARING_NONE = 0, ///< No sharing, all machine changes will be in-memory + CM_SHARING_CONFIG = 1, ///< Share backing stores marked as shared in the machine configuration + CM_SHARING_ALL = 2, ///< Share all backing stores, all machine changes will be on-disk +} cm_sharing_mode; + /// \brief Machine x, f, and control and status registers. typedef enum cm_reg { - // Processor x registers + // Machine x registers CM_REG_X0, CM_REG_X1, CM_REG_X2, @@ -163,7 +173,7 @@ typedef enum cm_reg { CM_REG_X29, CM_REG_X30, CM_REG_X31, - // Processor f registers + // Machine f registers CM_REG_F0, CM_REG_F1, CM_REG_F2, @@ -196,7 +206,7 @@ typedef enum cm_reg { CM_REG_F29, CM_REG_F30, CM_REG_F31, - // Processor CSRs + // Machine CSRs CM_REG_PC, CM_REG_FCSR, CM_REG_MVENDORID, @@ -291,6 +301,12 @@ typedef enum cm_reg { CM_REG_LAST_ = CM_REG_UARCH_HALT_FLAG, } cm_reg; +/// \brief Hash function types. +typedef enum cm_hash_function { + CM_HASH_KECCAK256 = 0, ///< Keccak-256 (recommended for fraud proofs using microarchitecture) + CM_HASH_SHA256 = 1, ///< SHA-256 (recommended for fraud proofs using zkVMs) +} cm_hash_function; + /// \brief Storage for machine hash. typedef uint8_t cm_hash[CM_HASH_SIZE]; @@ -377,10 +393,14 @@ CM_API void cm_delete(cm_machine *m); /// { /// "ram": { /// "length": 134217728, -/// "image_filename": "linux.bin" +/// "backing_store": { +/// "data_filename": "linux.bin" +/// } /// }, /// "flash_drive": [{ -/// "image_filename": "rootfs.ext2" +/// "backing_store": { +/// "data_filename": "rootfs.ext2" +/// } /// }], /// "dtb": { /// "entrypoint": "echo Hello world!" @@ -388,44 +408,75 @@ CM_API void cm_delete(cm_machine *m); /// } /// ``` /// \param runtime_config Machine runtime configuration as a JSON object in a string (can be NULL). +/// \param dir Directory to create an on-disk machine. If NULL or empty, the machine will be created in-memory. /// \returns 0 for success, non zero code for error. /// \details Use cm_destroy() to destroy the machine instance and remove it from the object. -CM_API cm_error cm_create(cm_machine *m, const char *config, const char *runtime_config); +CM_API cm_error cm_create(cm_machine *m, const char *config, const char *runtime_config, const char *dir); /// \brief Combines cm_new() and cm_create() for convenience. /// \param config Machine configuration as a JSON object in a string (at least RAM length must be set). /// \param runtime_config Machine runtime configuration as a JSON object in a string (can be NULL). /// \param new_m Receives the pointer to the new machine object with a machine instance. Set to NULL on failure. +/// \param dir Directory to create an on-disk machine. If NULL or empty, the machine will be created in-memory. /// \returns 0 for success, non zero code for error. /// \details Use cm_destroy() to destroy the machine instance and remove it from the object. /// \details Use cm_delete() to delete the object. /// \details See cm_new() and cm_create() for more details. -CM_API cm_error cm_create_new(const char *config, const char *runtime_config, cm_machine **new_m); +CM_API cm_error cm_create_new(const char *config, const char *runtime_config, const char *dir, cm_machine **new_m); /// \brief Loads a new machine instance from a previously stored directory. /// \param m Pointer to an empty machine object (does not hold a machine instance). /// \param dir Directory where previous machine is stored. +/// \param sharing Backing stores sharing mode. +/// CM_SHARING_NONE: Machine state will be fully in-memory. +/// CM_SHARING_CONFIG: Machine state will be on-disk for shared backing stores, but in-memory for everything else. +/// CM_SHARING_ALL: Machine state will be fully on-disk. +/// \param new_m Receives the pointer to the new machine object with a machine instance. Set to NULL on failure. /// \param runtime_config Machine runtime configuration as a JSON object in a string (can be NULL). /// \returns 0 for success, non zero code for error. /// \details Use cm_destroy() to destroy the machine instance and remove it from the object. -CM_API cm_error cm_load(cm_machine *m, const char *dir, const char *runtime_config); +CM_API cm_error cm_load(cm_machine *m, const char *dir, const char *runtime_config, cm_sharing_mode sharing); /// \brief Combines cm_new() and cm_load() for convenience. /// \param dir Directory where previous machine is stored. /// \param runtime_config Machine runtime configuration as a JSON object in a string (can be NULL). -/// \param new_m Receives the pointer to the new machine object with a machine instance. Set to NULL on failure. +/// \param sharing Backing stores sharing mode. /// \returns 0 for success, non zero code for error. /// \details Use cm_destroy() to destroy the machine instance and remove it from the object. /// \details Use cm_delete() to delete the object. -/// \details See cm_new() and cm_load() for more details. -CM_API cm_error cm_load_new(const char *dir, const char *runtime_config, cm_machine **new_m); +/// \details See cm_load() and cm_create() for more details. +CM_API cm_error cm_load_new(const char *dir, const char *runtime_config, cm_sharing_mode sharing, cm_machine **new_m); /// \brief Stores a machine instance to a directory, serializing its entire state. /// \param m Pointer to a non-empty machine object (holds a machine instance). /// \param dir Directory where the machine will be stored. +/// \param sharing Specifies how machine changes are reflected in the new store. +/// CM_SHARING_NONE: Copies backing stores from the initial machine state (rarely useful). +/// CM_SHARING_CONFIG: Store current machine state for shared backing stores; others are copied from the initial state. +/// CM_SHARING_ALL: Store current machine state for all backing stores. /// \returns 0 for success, non zero code for error. /// \details The function refuses to store into an existing directory (it will not overwrite an existing machine). -CM_API cm_error cm_store(const cm_machine *m, const char *dir); +CM_API cm_error cm_store(const cm_machine *m, const char *dir, cm_sharing_mode sharing); + +/// \brief Clones a machine stored from source directory to destination directory. +/// \param m Pointer to a machine object. Can be NULL (for local machines). +/// \param from_dir Path to the source directory containing a valid machine to be cloned. +/// \param to_dir Path to the destination directory where the cloned machine will be stored. +/// \returns 0 for success, non zero code for error. +/// \details Read-only files are copied using hard links on filesystems that support them. +/// Writable files are copied using reflinks on copy-on-write filesystems. +/// Sparsity of files is preserved. +/// The source directory machine must not have a running instance. +CM_API cm_error cm_clone_stored(const cm_machine *m, const char *from_dir, const char *to_dir); + +/// \brief Removes all files and the directory of a previously stored machine. +/// \param m Pointer to a machine object. Can be NULL (for local machines). +/// \param dir Path to the directory containing the stored machine to be removed. +/// \returns 0 for success, non zero code for error. +/// \details The stored machine must not be in use. +/// On success, the entire directory and all machine files are deleted. +/// On failure, some files or the directory may remain. +CM_API cm_error cm_remove_stored(const cm_machine *m, const char *dir); /// \brief Destroy a machine instance and remove it from the object. /// \param m Pointer to a non-empty machine object (holds a machine instance). @@ -452,17 +503,26 @@ CM_API cm_error cm_get_runtime_config(const cm_machine *m, const char **runtime_ /// \brief Replaces a memory range. /// \param m Pointer to a non-empty machine object (holds a machine instance). -/// \param start Range start physical address. -/// \param length Range length in bytes. -/// \param shared If true, changes to the range from inside the machine will be -/// written to the associated image file in the host. -/// \param image_filename Image file name to load into the range. If NULL, entire -/// range is cleared with zeros. +/// \param range_config Memory range configuration as a JSON object in a string. +/// Must have the same start, length, and read-only settings as an existing memory range. +/// For example: +/// ```json +/// { +/// "start": 0x80000000000000, +/// "length": 0x100000, +/// "read_only": false, +/// "backing_store": { +/// "data_filename": "linux.bin" +/// "dht_filename": "linux.dht" +/// "dpt_filename": "linux.dpt" +/// "shared": false +/// } +/// } +/// ``` /// \returns 0 for success, non zero code for error. /// \details The machine must have been initialized with an existing memory range that /// has the same start and length specified in the new range. -CM_API cm_error cm_replace_memory_range(cm_machine *m, uint64_t start, uint64_t length, bool shared, - const char *image_filename); +CM_API cm_error cm_replace_memory_range(cm_machine *m, const char *range_config); /// \brief Returns a JSON object with the machine config used to initialize the machine. /// \param m Pointer to a non-empty machine object (holds a machine instance). @@ -472,20 +532,28 @@ CM_API cm_error cm_replace_memory_range(cm_machine *m, uint64_t start, uint64_t /// \returns 0 for success, non zero code for error. CM_API cm_error cm_get_initial_config(const cm_machine *m, const char **config); -/// \brief Returns a list with all memory ranges in the machine. +/// \brief Returns a list with all address ranges in the machine. /// \param m Pointer to a non-empty machine object (holds a machine instance). -/// \param ranges Receives the memory ranges as a JSON object in a string, +/// \param ranges Receives the address ranges as a JSON object in a string, /// guaranteed to remain valid only until the next CM_API function is called from the same thread. /// \returns 0 for success, non zero code for error. -CM_API cm_error cm_get_memory_ranges(const cm_machine *m, const char **ranges); +CM_API cm_error cm_get_address_ranges(const cm_machine *m, const char **ranges); -/// \brief Obtains the root hash of the Merkle tree. +/// \brief Obtains the root hash of the hash tree. /// \param m Pointer to a non-empty machine object (holds a machine instance). /// \param hash Valid pointer to cm_hash structure that receives the hash. /// \returns 0 for success, non zero code for error. CM_API cm_error cm_get_root_hash(const cm_machine *m, cm_hash *hash); -/// \brief Obtains the proof for a node in the machine state Merkle tree. +/// \brief Obtains the hash of a node in the hash tree. +/// \param m Pointer to a non-empty machine object (holds a machine instance). +/// \param address Address of node. +/// \param log2_size Log2 of node size. +/// \param hash Valid pointer to cm_hash structure that receives the hash. +/// \returns 0 for success, non zero code for error. +CM_API cm_error cm_get_node_hash(const cm_machine *m, uint64_t address, int32_t log2_size, cm_hash *hash); + +/// \brief Obtains the proof for a node in the machine state hash tree. /// \param m Pointer to a non-empty machine object (holds a machine instance). /// \param address Address of target node. Must be aligned to a 2^log2_size boundary. /// \param log2_size The log base 2 of the size subtended by the target node. @@ -503,43 +571,51 @@ CM_API cm_error cm_get_proof(const cm_machine *m, uint64_t address, int32_t log2 /// \brief Reads the value of a word in the machine state, by its physical address. /// \param m Pointer to a non-empty machine object (holds a machine instance). /// \param address Word address (aligned to 64-bit boundary). -/// \param val Receives word value. +/// \param val Receives word value (little-endian). /// \returns 0 for success, non zero code for error. /// \warning The implementation is slow when the word falls in a memory range mapped to a device. CM_API cm_error cm_read_word(const cm_machine *m, uint64_t address, uint64_t *val); +/// \brief Writes the value of a word in the machine state, by its physical address. +/// \param m Pointer to a non-empty machine object (holds a machine instance). +/// \param address Word address (aligned to 64-bit boundary). +/// \param val Word value to write (little-endian). +/// \returns 0 for success, non zero code for error. +/// \warning The implementation is slow when the word falls in a memory range mapped to a device. +CM_API cm_error cm_write_word(cm_machine *m, uint64_t address, uint64_t val); + /// \brief Reads the value of a register. /// \param m Pointer to a non-empty machine object (holds a machine instance). /// \param reg Register to read. -/// \param val Receives the value. +/// \param val Receives the value (little-endian). /// \returns 0 for success, non zero code for error. CM_API cm_error cm_read_reg(const cm_machine *m, cm_reg reg, uint64_t *val); /// \brief Writes the value of a register. /// \param m Pointer to a non-empty machine object (holds a machine instance). /// \param reg Register to write. -/// \param val Value to write. +/// \param val Value to write (little-endian). /// \returns 0 for success, non zero code for error. CM_API cm_error cm_write_reg(cm_machine *m, cm_reg reg, uint64_t val); -/// \brief Reads a chunk of data from a machine memory range, by its physical address. +/// \brief Reads a chunk of data, by its target physical address and length. /// \param m Pointer to a non-empty machine object (holds a machine instance). -/// \param address Physical address to start reading. -/// \param data Receives chunk of memory. -/// \param length Size of chunk in bytes. +/// \param paddr Target physical address to start reading from. +/// \param data Buffer that receives data to read. Must be at least \p length bytes long. +/// \param length Number of bytes to read from \p paddr to \p data. /// \returns 0 for success, non zero code for error. -/// \details The entire chunk must be inside the same memory range. -CM_API cm_error cm_read_memory(const cm_machine *m, uint64_t address, uint8_t *data, uint64_t length); +/// \details The data can be anywhere in the entire address space. +CM_API cm_error cm_read_memory(const cm_machine *m, uint64_t paddr, uint8_t *data, uint64_t length); -/// \brief Writes a chunk of data to a machine memory range, by its physical address. +/// \brief Writes a chunk of data to machine memory, by its target physical address and length. /// \param m Pointer to a non-empty machine object (holds a machine instance). -/// \param address Physical address to start writing. -/// \param data Source for chunk of data. -/// \param length Size of chunk in bytes. +/// \param paddr Target physical address to start writing to. +/// \param data Buffer that contains data to write. Must be at least \p length bytes long. +/// \param length Number of bytes to write starting from \p data to \p paddr. /// \returns 0 for success, non zero code for error. -/// \details The entire chunk must be inside the same memory range. -/// Moreover, unlike cm_read_memory(), the memory range written to must not be mapped to a device. -CM_API cm_error cm_write_memory(cm_machine *m, uint64_t address, const uint8_t *data, uint64_t length); +/// \details Unlike read_memory(), the entire chunk of data, from \p paddr to \p paddr + \p length, +/// must reside entirely in the same memory range. Moreover, it cannot be mapped to a device. +CM_API cm_error cm_write_memory(cm_machine *m, uint64_t paddr, const uint8_t *data, uint64_t length); /// \brief Reads a chunk of data from a machine memory range, by its virtual address. /// \param m Pointer to a non-empty machine object (holds a machine instance). @@ -579,6 +655,50 @@ CM_API cm_error cm_translate_virtual_address(cm_machine *m, uint64_t vaddr, uint /// \details You may want to receive cmio requests depending on the run break reason. CM_API cm_error cm_run(cm_machine *m, uint64_t mcycle_end, cm_break_reason *break_reason); +/// \brief Collects the root hashes after every \p mcycle_period machine cycles +/// until mcycle reaches \p mcycle_end, the machine yields, or halts. +/// \param m Pointer to a non-empty machine object (holds a machine instance). +/// \param mcycle_end End machine cycle value. +/// \param mcycle_period Number of machine cycles between root hashes to collect. +/// \param mcycle_phase Number of machine cycles elapsed since last root hash collected. +/// \param log2_bundle_mcycle_count Log base 2 of the amount of mcycle root hashes to bundle. +/// If greater than 0, it collects subtree root hashes for 2^log2_bundle_mcycle_count root hashes. +/// \param previous_back_tree Optional context to continue collecting bundled root hashes. +/// If not NULL, it must be a JSON object as a string containing the back tree. +/// \param result Receives a JSON object as a string, guaranteed to remain valid only until +/// the next CM_API function is called from the same thread. +/// The field "hashes" is an array of root hashes (as base64-encoded strings) after each period. +/// The field "mcycle_phase" is the number of machine cycles elapsed since last root hash collected. +/// The field "break_reason" is a string with the reason why the function returned. +/// (Set to "failed" on failure.) +/// The optional field "back_tree" is the back tree context to continue collecting bundled root hashes. +/// For example: +/// ```json +/// { +/// "hashes": ["2av44Q3TaHKREgULA/iMoNe+btxCr+ja+hHGtXxaGjM=", ..., "FMAHn7mc2f+I30UjiywbAtOkw99iOa3NWS6fWql3UNU="], +/// "mcycle_phase": 1278, +/// "break_reason": "yielded_manually", +/// "back_tree": { ... }, +/// } +/// ``` +/// \returns 0 for success, non zero code for error. +/// \detail The first hash added to "hashes" is the root hash after (\p mcycle_period - \p mcycle_phase) +/// machine cycles (if the function managed to get that far before returning). +/// +/// If "break_reason" is "yielded_manually", "halted", or if mcycle reaches CM_MCYCLE_MAX (maximum mcycle value), +/// this means that execution stopped at a fixed point. +/// +/// When \p log2_bundle_mcycle_count is greater than 0 and execution stops at a fixed point, then +/// the next-to-last hash in "hashes" represents a bundle that is completed by padding with repetitions of the final +/// root hash, and the last hash in "hashes" represents a bundle consisting entirely of repetitions of that same final +/// root hash. +/// +/// When execution stops at a mcycle_period boundary or at a fixed point, the "back_tree" field is omitted. +/// Otherwise the "back_tree" field is included in the result and contains partial root hashes as its context, +/// and must be passed a subsequent function call to continue bundling root hashes properly. +CM_API cm_error cm_collect_mcycle_root_hashes(cm_machine *m, uint64_t mcycle_end, uint64_t mcycle_period, + uint64_t mcycle_phase, int32_t log2_bundle_mcycle_count, const char *previous_back_tree, const char **result); + /// \brief Runs the machine microarchitecture until CM_REG_UARCH_CYCLE reaches uarch_cycle_end or it halts. /// \param m Pointer to a non-empty machine object (holds a machine instance). /// \param uarch_cycle_end End micro cycle value. @@ -586,6 +706,46 @@ CM_API cm_error cm_run(cm_machine *m, uint64_t mcycle_end, cm_break_reason *brea /// \returns 0 for success, non zero code for error. CM_API cm_error cm_run_uarch(cm_machine *m, uint64_t uarch_cycle_end, cm_uarch_break_reason *uarch_break_reason); +/// \brief Collects the root hashes after every uarch cycle until mcycle reaches \p mcycle_end, +/// the machine yields, or halts. Implicitly resetting the uarch between mcycles. +/// \param m Pointer to a non-empty machine object (holds a machine instance). +/// \param mcycle_end End machine cycle value to execute, uarch cycle by uarch cycle. +/// \param log2_bundle_uarch_cycle_count Log base 2 of the amount of uarch cycle root hashes to bundle. +/// If greater than 0, it collects subtree root hashes for 2^log2_bundle_uarch_cycle_count root hashes. +/// \param result Receives an JSON object as a string, guaranteed to remain valid only until +/// the next CM_API function is called from the same thread. +/// The field "hashes" is an array (of hashes as base64-encoded strings) with the root hashes after each uarch cycle. +/// The field "reset_indices" is an array with indices of the root hashes after each implicit uarch reset +/// (i.e., after each machine cycle). +/// The field "break_reason" is a string with the reason why the function returned. +/// (Set to "failed" on failure.) +/// For example: +/// ```json +/// { +/// "hashes": ["2av44Q3TaHKREgULA/iMoNe+btxCr+ja+hHGtXxaGjM=", ..., "FMAHn7mc2f+I30UjiywbAtOkw99iOa3NWS6fWql3UNU="], +/// "reset_indices": [712, ..., 2293768], +/// "break_reason": "yielded_manually" +/// } +/// ``` +/// \returns 0 for success, non zero code for error. +/// \detail The first hash added to "hashes" is the root hash after the first uarch cycle, the last is the +/// root hash at the time function returns (for whatever reason), which always happens right after an uarch reset. +/// +/// If \p log2_bundle_uarch_cycle_count is greater than zero, the "hashes" array contains bundles of root hashes. +/// For each reset_index value in the "reset_indices" array: +/// - hashes[reset_index - 1] contains a bundle representing repeated root hashes at the point where the uarch halted +/// just before the uarch reset. +/// - hashes[reset_index] contains a bundle representing repeated root hashes at the point where the uarch halted, +/// followed by a single root hash immediately after the uarch reset. +/// +/// If "break_reason" is "yielded_manually", "halted", or if mcycle reaches CM_MCYCLE_MAX (maximum mcycle value), +/// this means that execution stopped at a fixed point. +/// In these cases, the function will attempt to execute one additional mcycle at this fixed point, +/// and collect the resulting root hashes as well. As a result, all root hashes collected after the next-to-last +/// reset index correspond to this fixed point. +CM_API cm_error cm_collect_uarch_cycle_root_hashes(cm_machine *m, uint64_t mcycle_end, + int32_t log2_bundle_uarch_cycle_count, const char **result); + /// \brief Resets the entire microarchitecture state to pristine values. /// \param m Pointer to a non-empty machine object (holds a machine instance). /// \returns 0 for success, non zero code for error. @@ -707,19 +867,43 @@ CM_API cm_error cm_verify_send_cmio_response(const cm_machine *m, uint16_t reaso // Integrity checking // ------------------------------------ -/// \brief Verifies integrity of Merkle tree against current machine state. +/// \brief Verifies integrity of hash tree against current machine state. /// \param m Pointer to a non-empty machine object (holds a machine instance). /// \param result True if tree is self-consistent, false otherwise. /// \returns 0 for success, non zero code for error. /// \details This method is used only for emulator internal tests. -CM_API cm_error cm_verify_merkle_tree(cm_machine *m, bool *result); +CM_API cm_error cm_verify_hash_tree(cm_machine *m, bool *result); -/// \brief Verifies integrity of dirty page maps. +/// \brief Obtains hash tree statistics /// \param m Pointer to a non-empty machine object (holds a machine instance). -/// \param result True if dirty page maps are consistent, false otherwise. +/// \param clear Whether to clear the statistics after retrieving them. +/// \param stats Receives the statistics as a JSON object in a string, +/// guaranteed to remain valid only until the next CM_API function is called from the same thread. /// \returns 0 for success, non zero code for error. -/// \details This method is used only for emulator internal tests. -CM_API cm_error cm_verify_dirty_page_maps(cm_machine *m, bool *result); +CM_API cm_error cm_get_hash_tree_stats(cm_machine *m, bool clear, const char **stats); + +// ------------------------------------ +// Hashing +// ------------------------------------ + +/// \brief Gets the hash of data. +/// \param hash_function Hash function to use. +/// \param data Data to hash. +/// \param length Length of data in bytes. +/// \param result Valid pointer to cm_hash (32-byte array) that receives the hash. +/// \returns 0 for success, non zero code for error. +/// \details This function is optimized to hash words of hash trees, but can be used for any data. +CM_API cm_error cm_get_hash(cm_hash_function hash_function, const uint8_t *data, uint64_t length, cm_hash *result); + +/// \brief Gets the hash of a concatenation of two hashes. +/// \param hash_function Hash function to use. +/// \param left Left hash to concatenate. +/// \param right Right hash to concatenate. +/// \param result Valid pointer to cm_hash (32-byte array) that receives the hash. +/// \returns 0 for success, non zero code for error. +/// \details This function is optimized and intended to be used for concatenating hashes of hash trees. +CM_API cm_error cm_get_concat_hash(cm_hash_function hash_function, const cm_hash *left, const cm_hash *right, + cm_hash *result); #ifdef __cplusplus } diff --git a/src/machine-config.cpp b/src/machine-config.cpp index 9f8961727..268da07df 100644 --- a/src/machine-config.cpp +++ b/src/machine-config.cpp @@ -16,62 +16,131 @@ #include "machine-config.h" -#include #include #include -#include #include -#include +#include +#include #include #include #include -#include +#include + +#include "address-range-constants.h" #include "json-util.h" -#include "json.hpp" -#include "pma-constants.h" +#include "os-filesystem.h" +#include "riscv-constants.h" -static constexpr uint32_t archive_version = 5; +static constexpr uint32_t archive_version = 6; namespace cartesi { -std::string machine_config::get_image_filename(const std::string &dir, uint64_t start, uint64_t length) { +std::string machine_config::get_data_filename(const std::string &dir, uint64_t start, uint64_t length) { std::ostringstream sout; sout << dir << "/" << std::hex << std::setw(16) << std::setfill('0') << start << "-" << length << ".bin"; return sout.str(); } -std::string machine_config::get_image_filename(const std::string &dir, const memory_range_config &c) { - return get_image_filename(dir, c.start, c.length); +std::string machine_config::get_dht_filename(const std::string &dir, uint64_t start, uint64_t length) { + std::ostringstream sout; + sout << dir << "/" << std::hex << std::setw(16) << std::setfill('0') << start << "-" << length << ".dht"; + return sout.str(); +} + +std::string machine_config::get_dpt_filename(const std::string &dir, uint64_t start, uint64_t length) { + std::ostringstream sout; + sout << dir << "/" << std::hex << std::setw(16) << std::setfill('0') << start << "-" << length << ".dpt"; + return sout.str(); +} + +std::string machine_config::get_sht_filename(const std::string &dir) { + return dir + "/hash_tree.sht"; +} + +std::string machine_config::get_phtc_filename(const std::string &dir) { + return dir + "/hash_tree.phtc"; } std::string machine_config::get_config_filename(const std::string &dir) { return dir + "/config.json"; } -static void adjust_image_filenames(machine_config &c, const std::string &dir) { - c.dtb.image_filename = machine_config::get_image_filename(dir, PMA_DTB_START, PMA_DTB_LENGTH); - c.ram.image_filename = machine_config::get_image_filename(dir, PMA_RAM_START, c.ram.length); - c.tlb.image_filename = machine_config::get_image_filename(dir, PMA_SHADOW_TLB_START, PMA_SHADOW_TLB_LENGTH); - for (auto &f : c.flash_drive) { - f.image_filename = machine_config::get_image_filename(dir, f); +void machine_config::adjust_ar_backing_store_config(uint64_t start, uint64_t length, const std::string &dir, + sharing_mode sharing, backing_store_config &c) { + // Convert sharing mode + switch (sharing) { + case sharing_mode::none: + c.shared = false; + break; + case sharing_mode::config: + // Preserve the shared setting as specified in the configuration + break; + case sharing_mode::all: + c.shared = true; + break; } - c.uarch.ram.image_filename = machine_config::get_image_filename(dir, PMA_UARCH_RAM_START, PMA_UARCH_RAM_LENGTH); - c.cmio.rx_buffer.image_filename = - machine_config::get_image_filename(dir, PMA_CMIO_RX_BUFFER_START, PMA_CMIO_RX_BUFFER_LENGTH); - c.cmio.tx_buffer.image_filename = - machine_config::get_image_filename(dir, PMA_CMIO_TX_BUFFER_START, PMA_CMIO_TX_BUFFER_LENGTH); + + // Strip create and truncate since backing store should be already created + c.create = false; + c.truncate = false; + + // Adjust filenames + c.data_filename = machine_config::get_data_filename(dir, start, length); + c.dht_filename = machine_config::get_dht_filename(dir, start, length); + c.dpt_filename = machine_config::get_dpt_filename(dir, start, length); } -machine_config machine_config::load(const std::string &dir) { +void machine_config::adjust_ht_backing_store_config(const std::string &dir, sharing_mode sharing, hash_tree_config &c) { + // Convert sharing mode + switch (sharing) { + case sharing_mode::none: + c.shared = false; + break; + case sharing_mode::config: + // Preserve the shared setting as specified in the configuration + break; + case sharing_mode::all: + c.shared = true; + break; + } + + // Strip create since backing store should be already created + c.create = false; + + c.sht_filename = machine_config::get_sht_filename(dir); + c.phtc_filename = machine_config::get_phtc_filename(dir); +} + +machine_config &machine_config::adjust_backing_stores(const std::string &dir, sharing_mode sharing) { + adjust_ar_backing_store_config(AR_RAM_START, ram.length, dir, sharing, ram.backing_store); + adjust_ar_backing_store_config(AR_DTB_START, AR_DTB_LENGTH, dir, sharing, dtb.backing_store); + for (auto &f : flash_drive) { + adjust_ar_backing_store_config(f.start, f.length, dir, sharing, f.backing_store); + } + adjust_ar_backing_store_config(AR_SHADOW_STATE_START, AR_SHADOW_STATE_LENGTH, dir, sharing, + processor.backing_store); + adjust_ar_backing_store_config(AR_CMIO_RX_BUFFER_START, AR_CMIO_RX_BUFFER_LENGTH, dir, sharing, + cmio.rx_buffer.backing_store); + adjust_ar_backing_store_config(AR_CMIO_TX_BUFFER_START, AR_CMIO_TX_BUFFER_LENGTH, dir, sharing, + cmio.tx_buffer.backing_store); + adjust_ar_backing_store_config(AR_PMAS_START, AR_PMAS_LENGTH, dir, sharing, pmas.backing_store); + adjust_ar_backing_store_config(AR_SHADOW_UARCH_STATE_START, AR_SHADOW_UARCH_STATE_LENGTH, dir, sharing, + uarch.processor.backing_store); + adjust_ar_backing_store_config(AR_UARCH_RAM_START, AR_UARCH_RAM_LENGTH, dir, sharing, uarch.ram.backing_store); + adjust_ht_backing_store_config(dir, sharing, hash_tree); + return *this; +} + +machine_config machine_config::load(const std::string &dir, sharing_mode sharing) { + if (dir.empty()) { + throw std::invalid_argument{"directory name cannot be empty"}; + } machine_config c; auto name = machine_config::get_config_filename(dir); - std::ifstream ifs(name, std::ios::binary); - if (!ifs) { - throw std::system_error{errno, std::generic_category(), "unable to open '" + name + "' for reading"}; - } + const auto [ptr, data] = os::read_file(name); try { - auto j = nlohmann::json::parse(ifs); + auto j = nlohmann::json::parse(data); if (!j.contains("archive_version")) { throw std::runtime_error("missing field \"archive_version\""); } @@ -79,28 +148,62 @@ machine_config machine_config::load(const std::string &dir) { if (!jv.is_number_integer()) { throw std::runtime_error("expected integer field \"archive_version\""); } - if (jv.get() != archive_version) { + if (jv.get() != archive_version) { throw std::runtime_error("expected \"archive_version\" " + std::to_string(archive_version) + " (got " + std::to_string(jv.get()) + ")"); } ju_get_field(j, std::string("config"), c, ""); - adjust_image_filenames(c, dir); - } catch (std::exception &e) { + c.adjust_backing_stores(dir, sharing); + } catch (const std::exception &e) { throw std::runtime_error{e.what()}; } return c; } -void machine_config::store(const std::string &dir) const { +std::string machine_config::store(const std::string &dir, sharing_mode sharing) const { + if (dir.empty()) { + throw std::invalid_argument{"directory name cannot be empty"}; + } auto name = get_config_filename(dir); nlohmann::json j; j["archive_version"] = archive_version; - j["config"] = *this; - std::ofstream ofs(name, std::ios::binary); - if (!ofs) { - throw std::system_error{errno, std::generic_category(), "unable to open '" + name + "' for writing"}; + j["config"] = machine_config(*this).adjust_backing_stores(".", sharing); // Strip relative directories + const std::string data = j.dump(); + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + os::create_file(name, std::span(reinterpret_cast(data.data()), data.size())); + return name; +} + +machine_config &machine_config::adjust_defaults() { + // Fill version registers + if (processor.registers.marchid == UINT64_C(-1)) { + processor.registers.marchid = MARCHID_INIT; + } + if (processor.registers.mvendorid == UINT64_C(-1)) { + processor.registers.mvendorid = MVENDORID_INIT; + } + if (processor.registers.mimpid == UINT64_C(-1)) { + processor.registers.mimpid = MIMPID_INIT; + } + // Auto detect flash drives start address and length + int i = 0; // NOLINT(misc-const-correctness) + for (auto &f : flash_drive) { + const std::string flash_description = "flash drive "s + std::to_string(i); + // Auto detect flash drive start address + if (f.start == UINT64_C(-1)) { + f.start = AR_DRIVE_START + (AR_DRIVE_OFFSET * i); + } + // Auto detect flash drive image length + if (f.length == UINT64_C(-1)) { + if (f.backing_store.data_filename.empty()) { + throw std::runtime_error{ + "unable to auto-detect length of "s.append(flash_description).append(" with empty image file")}; + } + f.length = os::file_size(f.backing_store.data_filename); + } + i += 1; } - ofs << j; + return *this; } } // namespace cartesi diff --git a/src/machine-config.h b/src/machine-config.h index 30beab497..72834b92f 100644 --- a/src/machine-config.h +++ b/src/machine-config.h @@ -17,18 +17,24 @@ #ifndef MACHINE_CONFIG_H #define MACHINE_CONFIG_H -#include #include #include #include +#include -#include - -#include "riscv-constants.h" -#include "uarch-config.h" +#include "shadow-registers.h" +#include "shadow-uarch-state.h" +#include "variant-hasher.h" namespace cartesi { +/// \brief Sharing modes +enum class sharing_mode { + none, ///< No sharing, all machine changes will be in-memory + config, ///< Share backing stores marked as shared in the machine configuration + all, ///< Share all backing stores, all machine changes will be on-disk +}; + /// \brief Machine config constants enum machine_config_constants { FLASH_DRIVE_MAX = 8, ///< Maximum number of flash drives @@ -36,100 +42,59 @@ enum machine_config_constants { VIRTIO_HOSTFWD_MAX = 16, ///< Maximum number of virtio net user host forward ports }; -/// \brief Processor state configuration +/// \brief Backing store config +struct backing_store_config final { + bool shared{false}; ///< Should changes be reflected in backing store? + bool create{false}; ///< Should backing store be created? + bool truncate{false}; ///< Should backing store be truncated to correct size? + std::string data_filename; ///< Backing store for associated memory address range + std::string dht_filename; ///< Backing store for corresponding dense hash-tree + std::string dpt_filename; ///< Backing store for corresponding dirty-page tree + + /// \brief Returns true if the backing store was just created. + /// Indicates the backing store is zero-initialized and may require further initialization. + bool newly_created() const { + return create || data_filename.empty(); + } +}; + +/// \brief Processor state config struct processor_config final { - std::array x{REG_X0, REG_X1, REG_X2, REG_X3, REG_X4, REG_X5, REG_X6, REG_X7, REG_X8, REG_X9, - REG_X10, REG_X11, REG_X12, REG_X13, REG_X14, REG_X15, REG_X16, REG_X17, REG_X18, REG_X19, REG_X20, REG_X21, - REG_X22, REG_X23, REG_X24, REG_X25, REG_X26, REG_X27, REG_X28, REG_X29, REG_X30, - REG_X31}; ///< Value of general-purpose registers - std::array f{}; ///< Value of floating-point registers - uint64_t pc{PC_INIT}; ///< Value of pc - uint64_t fcsr{FCSR_INIT}; ///< Value of fcsr CSR - uint64_t mvendorid{MVENDORID_INIT}; ///< Value of mvendorid CSR - uint64_t marchid{MARCHID_INIT}; ///< Value of marchid CSR - uint64_t mimpid{MIMPID_INIT}; ///< Value of mimpid CSR - uint64_t mcycle{MCYCLE_INIT}; ///< Value of mcycle CSR - uint64_t icycleinstret{ICYCLEINSTRET_INIT}; ///< Value of icycleinstret CSR - uint64_t mstatus{MSTATUS_INIT}; ///< Value of mstatus CSR - uint64_t mtvec{MTVEC_INIT}; ///< Value of mtvec CSR - uint64_t mscratch{MSCRATCH_INIT}; ///< Value of mscratch CSR - uint64_t mepc{MEPC_INIT}; ///< Value of mepc CSR - uint64_t mcause{MCAUSE_INIT}; ///< Value of mcause CSR - uint64_t mtval{MTVAL_INIT}; ///< Value of mtval CSR - uint64_t misa{MISA_INIT}; ///< Value of misa CSR - uint64_t mie{MIE_INIT}; ///< Value of mie CSR - uint64_t mip{MIP_INIT}; ///< Value of mip CSR - uint64_t medeleg{MEDELEG_INIT}; ///< Value of medeleg CSR - uint64_t mideleg{MIDELEG_INIT}; ///< Value of mideleg CSR - uint64_t mcounteren{MCOUNTEREN_INIT}; ///< Value of mcounteren CSR - uint64_t menvcfg{MENVCFG_INIT}; ///< Value of menvcfg CSR - uint64_t stvec{STVEC_INIT}; ///< Value of stvec CSR - uint64_t sscratch{SSCRATCH_INIT}; ///< Value of sscratch CSR - uint64_t sepc{SEPC_INIT}; ///< Value of sepc CSR - uint64_t scause{SCAUSE_INIT}; ///< Value of scause CSR - uint64_t stval{STVAL_INIT}; ///< Value of stval CSR - uint64_t satp{SATP_INIT}; ///< Value of satp CSR - uint64_t scounteren{SCOUNTEREN_INIT}; ///< Value of scounteren CSR - uint64_t senvcfg{SENVCFG_INIT}; ///< Value of senvcfg CSR - uint64_t ilrsc{ILRSC_INIT}; ///< Value of ilrsc CSR - uint64_t iprv{IPRV_INIT}; ///< Value of iprv CSR - uint64_t iflags_X{IFLAGS_X_INIT}; ///< Value of iflags_X CSR - uint64_t iflags_Y{IFLAGS_Y_INIT}; ///< Value of iflags_Y CSR - uint64_t iflags_H{IFLAGS_H_INIT}; ///< Value of iflags_H CSR - uint64_t iunrep{IUNREP_INIT}; ///< Value of iunrep CSR -}; - -/// \brief RAM state configuration + registers_state registers; + backing_store_config backing_store; +}; + +/// \brief Config with only backing store config field +struct backing_store_config_only final { + backing_store_config backing_store; +}; + +/// \brief RAM state config struct ram_config final { - uint64_t length{0}; ///< RAM length - std::string image_filename; ///< RAM image file name + uint64_t length{0}; ///< RAM length + backing_store_config backing_store; ///< Backing store }; -/// \brief DTB state configuration +/// \brief DTB state config struct dtb_config final { std::string bootargs{ "quiet earlycon=sbi console=hvc0 root=/dev/pmem0 rw init=/usr/sbin/cartesi-init"}; ///< Bootargs to pass ///< to kernel - std::string init; ///< Initialization commands to be executed as root on boot - std::string entrypoint; ///< Commands to execute the main application - std::string image_filename; ///< DTB image file + std::string init; ///< Initialization commands to be executed as root on boot + std::string entrypoint; ///< Commands to execute the main application + backing_store_config backing_store; ///< Backing store }; -/// \brief Memory range configuration +/// \brief Memory range config struct memory_range_config final { uint64_t start{0xffffffffffffffffUL}; ///< Memory range start position, default is to auto detect uint64_t length{0xffffffffffffffffUL}; ///< Memory range length, default is to auto detect - bool shared{false}; ///< Target changes to memory affect image file? - std::string image_filename; ///< Memory range image file name + bool read_only{false}; ///< Make memory range read-only to host + backing_store_config backing_store; ///< Backing store }; /// \brief List of flash drives -using flash_drive_configs = boost::container::static_vector; - -/// \brief TLB device state configuration -struct tlb_config final { - std::string image_filename; ///< TLB image file name -}; - -/// \brief CLINT device state configuration -struct clint_config final { - uint64_t mtimecmp{MTIMECMP_INIT}; ///< Value of mtimecmp CSR -}; - -/// \brief PLIC device state configuration -struct plic_config final { - uint64_t girqpend{GIRQPEND_INIT}; ///< Value of girqpend CSR - uint64_t girqsrvd{GIRQSRVD_INIT}; ///< Value of girqsrvd CSR -}; - -/// \brief HTIF device state configuration -struct htif_config final { - uint64_t fromhost{FROMHOST_INIT}; ///< Value of fromhost CSR - uint64_t tohost{TOHOST_INIT}; ///< Value of tohost CSR - bool console_getchar{false}; ///< Make console getchar available? - bool yield_manual{true}; ///< Make yield manual available? - bool yield_automatic{true}; ///< Make yield automatic available? -}; +using flash_drive_configs = std::vector; /// \brief VirtIO console device state config struct virtio_console_config final {}; @@ -150,7 +115,7 @@ struct virtio_hostfwd_config final { }; /// \brief List of VirtIO host forwards -using virtio_hostfwd_configs = boost::container::static_vector; +using virtio_hostfwd_configs = std::vector; /// \brief VirtIO user network device state config struct virtio_net_user_config final { @@ -170,49 +135,96 @@ using virtio_device_config = std::variant; /// \brief List of VirtIO devices -using virtio_configs = boost::container::static_vector; +using virtio_configs = std::vector; -/// \brief cmio buffer configuration -struct cmio_buffer_config final { - bool shared{false}; ///< Target changes to memory affect image file? - std::string image_filename; ///< Memory range image file name +/// \brief CMIO config +struct cmio_config final { + backing_store_config_only rx_buffer{}; ///< RX buffer config + backing_store_config_only tx_buffer{}; ///< TX buffer config }; -/// \brief Cmio configuration -struct cmio_config final { - cmio_buffer_config rx_buffer; ///< RX buffer configuration - cmio_buffer_config tx_buffer; ///< TX buffer configuration +/// \brief PMAS config +using pmas_config = backing_store_config_only; + +/// \brief Uarch RAM config +using uarch_ram_config = backing_store_config_only; + +/// \brief Uarch processor state config +struct uarch_processor_config final { + uarch_registers_state registers; ///< Uarch registers + backing_store_config backing_store; +}; + +/// \brief Uarch config +struct uarch_config final { + uarch_processor_config processor{}; ///< Uarch processor + uarch_ram_config ram{}; ///< Uarch RAM +}; + +/// \brief Hash tree config +struct hash_tree_config final { + bool shared{false}; ///< Should changes be reflected in backing store? + bool create{false}; ///< Should backing store be created? + std::string sht_filename; ///< Backing storage for sparse hash-tree + std::string phtc_filename; ///< Backing storage for page hash-tree cache + uint64_t phtc_size{4096}; ///< Max number of pages in page hash-tree cache + hash_function_type hash_function{hash_function_type::keccak256}; ///< Hash function type to use in the hash-tree }; -/// \brief Machine state configuration +/// \brief Machine state config struct machine_config final { - processor_config processor{}; ///< Processor state - ram_config ram{}; ///< RAM state - dtb_config dtb{}; ///< DTB state - flash_drive_configs flash_drive; ///< Flash drives state - tlb_config tlb{}; ///< TLB device state - clint_config clint{}; ///< CLINT device state - plic_config plic{}; ///< PLIC device state - htif_config htif{}; ///< HTIF device state - virtio_configs virtio; ///< VirtIO devices state - uarch_config uarch{}; ///< microarchitecture configuration - cmio_config cmio{}; ///< Cmio state + processor_config processor{}; ///< Processor config + ram_config ram{}; ///< RAM config + dtb_config dtb{}; ///< Device Tree config + flash_drive_configs flash_drive; ///< Flash drives config + virtio_configs virtio; ///< VirtIO devices config + cmio_config cmio{}; ///< Cartesi Machine IO config + pmas_config pmas{}; ///< Physical Memory Attributes config + uarch_config uarch{}; ///< Microarchitecture config + hash_tree_config hash_tree{}; ///< Hash-tree config /// \brief Get the name where config will be stored in a directory static std::string get_config_filename(const std::string &dir); - /// \brief Get the name where memory range will be stored in a directory - static std::string get_image_filename(const std::string &dir, uint64_t start, uint64_t length); - static std::string get_image_filename(const std::string &dir, const memory_range_config &c); + /// \brief Get the name where the data for an address range will be stored in a directory + static std::string get_data_filename(const std::string &dir, uint64_t start, uint64_t length); + + /// \brief Get the name where dense hash-tree for an address range will be stored in a directory + static std::string get_dht_filename(const std::string &dir, uint64_t start, uint64_t length); + + /// \brief Get the name where dirty page-tree for an address range will be stored in a directory + static std::string get_dpt_filename(const std::string &dir, uint64_t start, uint64_t length); + + /// \brief Get the name where global sparse hash-tree will be stored in a directory + static std::string get_sht_filename(const std::string &dir); + + /// \brief Get the name where global page hash-tree cache will be stored in a directory + static std::string get_phtc_filename(const std::string &dir); + + static void adjust_ar_backing_store_config(uint64_t start, uint64_t length, const std::string &dir, + sharing_mode sharing, backing_store_config &c); + + static void adjust_ht_backing_store_config(const std::string &dir, sharing_mode sharing, hash_tree_config &c); + + /// \brief Adjusts the machine config filling in default values + machine_config &adjust_defaults(); + + /// \brief Adjust backing stores to point to the directory + /// \param dir Directory where "config" is stored + /// \param sharing Sharing mode conversion policy to use for backing stores + machine_config &adjust_backing_stores(const std::string &dir, sharing_mode sharing = sharing_mode::config); /// \brief Loads a machine config from a directory /// \param dir Directory from whence "config" will be loaded + /// \param sharing Sharing mode conversion policy to use for backing stores /// \returns The config loaded - static machine_config load(const std::string &dir); + static machine_config load(const std::string &dir, sharing_mode sharing = sharing_mode::config); /// \brief Stores the machine config to a directory /// \param dir Directory where "config" will be stored - void store(const std::string &dir) const; + /// \param sharing Sharing mode conversion policy to use for backing stores + /// \returns The filename where the config was stored. + std::string store(const std::string &dir, sharing_mode sharing = sharing_mode::config) const; }; } // namespace cartesi diff --git a/src/shadow-tlb-factory.h b/src/machine-hash.h similarity index 64% rename from src/shadow-tlb-factory.h rename to src/machine-hash.h index 6c699a9e9..0aae53a80 100644 --- a/src/shadow-tlb-factory.h +++ b/src/machine-hash.h @@ -14,23 +14,24 @@ // with this program (see COPYING). If not, see . // -#ifndef SHADOW_TLB_FACTORY_H -#define SHADOW_TLB_FACTORY_H +#ifndef MACHINE_HASH_H +#define MACHINE_HASH_H /// \file -/// \brief TLB device factory. +/// \brief Storage for a hash -#include - -#include "pma.h" +#include +#include +#include +#include namespace cartesi { -/// \brief Creates a PMA entry for the TLB device -/// \param start Start address for memory range. -/// \param length Length of memory range. -/// \returns Corresponding PMA entry -pma_entry make_shadow_tlb_pma_entry(uint64_t start, uint64_t length); +static constexpr size_t MACHINE_HASH_SIZE = 32; +using machine_hash = std::array; +using machine_hash_view = std::span; +using const_machine_hash_view = std::span; +using machine_hashes = std::vector; } // namespace cartesi diff --git a/src/machine-merkle-tree.cpp b/src/machine-merkle-tree.cpp deleted file mode 100644 index a0c23bc57..000000000 --- a/src/machine-merkle-tree.cpp +++ /dev/null @@ -1,470 +0,0 @@ -// Copyright Cartesi and individual authors (see AUTHORS) -// SPDX-License-Identifier: LGPL-3.0-or-later -// -// This program is free software: you can redistribute it and/or modify it under -// the terms of the GNU Lesser General Public License as published by the Free -// Software Foundation, either version 3 of the License, or (at your option) any -// later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT ANY -// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A -// PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License along -// with this program (see COPYING). If not, see . -// - -#include "machine-merkle-tree.h" -#include "i-hasher.h" -#include "pristine-merkle-tree.h" - -#include -#include -#include -#include -#include -#include -#include - -/// \file -/// \brief Merkle tree implementation. - -namespace cartesi { - -// Initialize static pristine hashes -const cartesi::pristine_merkle_tree &machine_merkle_tree::pristine_hashes() { - static const cartesi::pristine_merkle_tree tree{machine_merkle_tree::get_log2_root_size(), - machine_merkle_tree::get_log2_word_size()}; - return tree; -} - -constexpr machine_merkle_tree::address_type machine_merkle_tree::get_page_index(address_type address) { - return address & m_page_index_mask; -} - -machine_merkle_tree::tree_node *machine_merkle_tree::get_page_node(address_type page_index) const { - // Look for entry in page map hash table - auto it = m_page_node_map.find(page_index); - if (it != m_page_node_map.end()) { - return it->second; - } - return nullptr; -} - -constexpr machine_merkle_tree::address_type machine_merkle_tree::get_offset_in_page(address_type address) { - return address & m_page_offset_mask; -} - -int machine_merkle_tree::set_page_node_map(address_type page_index, tree_node *node) { - m_page_node_map[page_index] = node; - return 1; -} - -// NOLINTNEXTLINE(readability-convert-member-functions-to-static) -machine_merkle_tree::tree_node *machine_merkle_tree::create_node() const { -#ifdef MERKLE_DUMP_STATS - m_num_nodes++; -#endif - return new tree_node{}; -} - -// NOLINTNEXTLINE(readability-convert-member-functions-to-static) -void machine_merkle_tree::destroy_node(tree_node *node) const { -#ifdef MERKLE_DUMP_STATS - --m_num_nodes; -#endif - delete node; -} - -machine_merkle_tree::tree_node *machine_merkle_tree::new_page_node(address_type page_index) { - // Start with the first bit in the address space - address_type bit_mask = UINT64_C(1) << (get_log2_root_size() - 1); - tree_node *node = m_root; - // Descend tree until we reach the node at the end of the - // path determined by the page index, - // creating the needed nodes along the way - while (true) { - const int bit = static_cast((page_index & bit_mask) != 0); - tree_node *child = node->child[bit]; - if (child == nullptr) { - child = create_node(); - if (child == nullptr) { - return nullptr; - } - child->parent = node; - node->child[bit] = child; - } - node = child; - bit_mask >>= 1; - if ((bit_mask & m_page_index_mask) == 0) { - break; - } - } - // Finally associate page node to page index - if (set_page_node_map(page_index, node) == 0) { - return nullptr; - } - // Only if all previous steps succeeded, do we return the node - return node; -} - -void machine_merkle_tree::get_page_node_hash(hasher_type &h, const unsigned char *start, int log2_size, - hash_type &hash) const { - if (log2_size > get_log2_word_size()) { - hash_type child0; - hash_type child1; - --log2_size; - const address_type size = UINT64_C(1) << log2_size; - get_page_node_hash(h, start, log2_size, child0); - get_page_node_hash(h, start + size, log2_size, child1); - get_concat_hash(h, child0, child1, hash); - } else { - h.begin(); - h.add_data(start, get_word_size()); - h.end(hash); - } -} - -void machine_merkle_tree::get_page_node_hash(hasher_type &h, const unsigned char *page_data, hash_type &hash) const { - if (page_data != nullptr) { - get_page_node_hash(h, page_data, get_log2_page_size(), hash); - } else { - hash = get_pristine_hash(get_log2_page_size()); - } -} - -void machine_merkle_tree::get_page_node_hash(address_type page_index, hash_type &hash) const { - assert(page_index == get_page_index(page_index)); - tree_node *node = get_page_node(page_index); - if (node == nullptr) { - hash = get_pristine_hash(get_log2_page_size()); - } else { - hash = node->hash; - } -} - -const machine_merkle_tree::hash_type &machine_merkle_tree::get_child_hash(int child_log2_size, const tree_node *node, - int bit) { - const tree_node *child = node->child[bit]; - return (child != nullptr) ? child->hash : get_pristine_hash(child_log2_size); -} - -void machine_merkle_tree::update_inner_node_hash(hasher_type &h, int log2_size, tree_node *node) { - get_concat_hash(h, get_child_hash(log2_size - 1, node, 0), get_child_hash(log2_size - 1, node, 1), node->hash); -} - -void machine_merkle_tree::dump_hash(const hash_type &hash) { - auto f = std::cerr.flags(); - for (const auto &b : hash) { - std::cerr << std::hex << std::setfill('0') << std::setw(2) << static_cast(b); - } - std::cerr << '\n'; - std::cerr.flags(f); -} - -const machine_merkle_tree::hash_type &machine_merkle_tree::get_pristine_hash(int log2_size) { - return pristine_hashes().get_hash(log2_size); -} - -void machine_merkle_tree::dump_merkle_tree(tree_node *node, uint64_t address, int log2_size) const { - for (int i = 0; i < get_log2_root_size() - log2_size; i++) { - std::cerr << ' '; - } - std::cerr << "0x" << std::setfill('0') << std::setw(16) << std::hex << address << ":" << std::setfill('0') - << std::setw(2) << std::dec << log2_size << ' '; - if (node != nullptr) { - dump_hash(node->hash); - if (log2_size > get_log2_page_size()) { - dump_merkle_tree(node->child[0], address, log2_size - 1); - dump_merkle_tree(node->child[1], address + (UINT64_C(1) << (log2_size - 1)), log2_size - 1); - } - } else { - std::cerr << "nullptr\n"; - } -} - -void machine_merkle_tree::destroy_merkle_tree(tree_node *node, int log2_size) { - if (node != nullptr) { - // If this is an inner node, invoke recursively - if (log2_size > get_log2_page_size()) { - destroy_merkle_tree(node->child[0], log2_size - 1); - destroy_merkle_tree(node->child[1], log2_size - 1); - } - destroy_node(node); - } -} - -void machine_merkle_tree::destroy_merkle_tree() { - destroy_merkle_tree(m_root_storage.child[0], get_log2_root_size() - 1); - destroy_merkle_tree(m_root_storage.child[1], get_log2_root_size() - 1); - memset(&m_root_storage, 0, sizeof(m_root_storage)); -} - -void machine_merkle_tree::get_inside_page_sibling_hashes(hasher_type &h, address_type address, int log2_size, - hash_type &hash, const unsigned char *curr_data, int log2_curr_size, hash_type &curr_hash, int parent_diverged, - int curr_diverged, proof_type &proof) const { - // If node currently being visited is larger than a - // word, invoke recursively - if (log2_curr_size > get_log2_word_size()) { - const int log2_child_size = log2_curr_size - 1; - const address_type child_size = UINT64_C(1) << log2_child_size; - hash_type first_hash; - hash_type second_hash; - const int child_bit = static_cast((address & child_size) != 0); - get_inside_page_sibling_hashes(h, address, log2_size, hash, curr_data, log2_child_size, first_hash, - static_cast((parent_diverged != 0) || (curr_diverged) != 0), static_cast(child_bit != 0), proof); - get_inside_page_sibling_hashes(h, address, log2_size, hash, curr_data + child_size, log2_child_size, - second_hash, static_cast((parent_diverged != 0) || (curr_diverged) != 0), - static_cast(child_bit != 1), proof); - // Compute curr_hash from hashes of its children - get_concat_hash(h, first_hash, second_hash, curr_hash); - // Otherwise directly compute hash of word - } else { - h.begin(); - h.add_data(curr_data, get_word_size()); - h.end(curr_hash); - } - if (parent_diverged == 0) { - // So if the parent belongs to the path, but the node currently being - // visited doesn't, it is a sibling and we store its hash. - if ((curr_diverged != 0) && log2_curr_size >= proof.get_log2_target_size()) { - proof.set_sibling_hash(curr_hash, log2_curr_size); - // Otherwise, if the node hasn't diverged either and - // it is has the same size as the target node, then it *is* - // the target node - } else if (log2_curr_size == log2_size) { - hash = curr_hash; - } - } -} - -void machine_merkle_tree::get_inside_page_sibling_hashes(address_type address, int log2_size, hash_type &hash, - const unsigned char *page_data, hash_type &page_hash, proof_type &proof) const { - hasher_type h; - get_inside_page_sibling_hashes(h, address, log2_size, hash, page_data, get_log2_page_size(), page_hash, - 0 /* parent hasn't diverted */, 0 /* curr node hasn't diverged */, proof); -} - -void machine_merkle_tree::dump_merkle_tree() const { - dump_merkle_tree(m_root, 0, get_log2_root_size()); -} - -bool machine_merkle_tree::begin_update() { - m_merkle_update_fifo.clear(); - return true; -} - -bool machine_merkle_tree::update_page_node_hash(address_type page_index, const hash_type &hash) { - assert(get_page_index(page_index) == page_index); - tree_node *node = get_page_node(page_index); - // If there is no page node for this page index, allocate a fresh one - if (node == nullptr) { - node = new_page_node(page_index); - } - // If allocation failed, we fail - if (node == nullptr) { - return false; - } - // Copy new hash value to node - node->hash = hash; - // Add parent to fifo so we propagate changes - if ((node->parent != nullptr) && node->parent->mark != m_merkle_update_nonce) { - m_merkle_update_fifo.emplace_back(get_log2_page_size() + 1, node->parent); - node->parent->mark = m_merkle_update_nonce; - } - return true; -} - -bool machine_merkle_tree::end_update(hasher_type &h) { - // Now go over the queue of inner nodes updating their hashes and - // enqueueing their parents until the queue is empty - while (!m_merkle_update_fifo.empty()) { - int log2_size{}; - tree_node *node{}; - std::tie(log2_size, node) = m_merkle_update_fifo.front(); - update_inner_node_hash(h, log2_size, node); - m_merkle_update_fifo.pop_front(); - if ((node->parent != nullptr) && node->parent->mark != m_merkle_update_nonce) { - m_merkle_update_fifo.emplace_back(log2_size + 1, node->parent); - node->parent->mark = m_merkle_update_nonce; - } - } - ++m_merkle_update_nonce; - return true; -} - -machine_merkle_tree::machine_merkle_tree() : m_root_storage{}, m_root{&m_root_storage} { - m_root->hash = get_pristine_hash(get_log2_root_size()); -#ifdef MERKLE_DUMP_STATS - m_num_nodes = 0; -#endif -} - -machine_merkle_tree::~machine_merkle_tree() { -#ifdef MERKLE_DUMP_STATS - std::cerr << "before destruction\n"; - std::cerr << " number of tree nodes: " << m_num_nodes << '\n'; -#endif - destroy_merkle_tree(); -#ifdef MERKLE_DUMP_STATS - std::cerr << "after destruction\n"; - std::cerr << " number of tree nodes: " << m_num_nodes << '\n'; -#endif -} - -void machine_merkle_tree::get_root_hash(hash_type &hash) const { - hash = m_root->hash; -} - -bool machine_merkle_tree::verify_tree() const { - hasher_type h; - return verify_tree(h, m_root, get_log2_root_size()); -} - -bool machine_merkle_tree::verify_tree(hasher_type &h, tree_node *node, int log2_size) const { - // pristine node is always correct - if (node == nullptr) { - return true; - } - // verify inner node - if (log2_size > get_log2_page_size()) { - const int child_log2_size = log2_size - 1; - auto first_ok = verify_tree(h, node->child[0], child_log2_size); - auto second_ok = verify_tree(h, node->child[1], child_log2_size); - if (!first_ok || !second_ok) { - return false; - } - hash_type hash; - get_concat_hash(h, get_child_hash(child_log2_size, node, 0), get_child_hash(child_log2_size, node, 1), hash); - return hash == node->hash; - // Assume page nodes are correct - } - return true; -} - -machine_merkle_tree::proof_type machine_merkle_tree::get_proof(address_type target_address, int log2_target_size, - const unsigned char *page_data) const { - // Check for valid target node size - if (log2_target_size > get_log2_root_size() || log2_target_size < get_log2_word_size()) { - throw std::runtime_error{"log2_target_size is out of bounds"}; - } - - // Check target address alignment - if ((target_address & ((~UINT64_C(0)) >> (get_log2_root_size() - log2_target_size))) != 0) { - throw std::runtime_error{"misaligned target address"}; - } - - proof_type proof{get_log2_root_size(), log2_target_size}; - - // Copy hashes for nodes larger than or equal to the page size - const int log2_stop_size = std::max(log2_target_size, get_log2_page_size()); - int log2_node_size = get_log2_root_size(); - const tree_node *node = m_root; - // Copy non-pristine siblings hashes directly from tree nodes - while ((node != nullptr) && log2_node_size > log2_stop_size) { - const int log2_child_size = log2_node_size - 1; - const int path_bit = static_cast((target_address & (UINT64_C(1) << (log2_child_size))) != 0); - proof.set_sibling_hash(get_child_hash(log2_child_size, node, static_cast(static_cast(path_bit) == 0)), - log2_child_size); - node = node->child[path_bit]; - log2_node_size = log2_child_size; - } - // At this point, there are three alternatives - // Case 1 - // We hit a pristine node along the path to the target node - if (node == nullptr) { - if (page_data != nullptr) { - throw std::runtime_error{"inconsistent merkle tree"}; - } - // All remaining siblings along the path are pristine - for (int i = log2_node_size - 1; i >= log2_target_size; --i) { - proof.set_sibling_hash(get_pristine_hash(i), i); - } - // Copy pristine hash into target - proof.set_target_hash(get_pristine_hash(log2_target_size)); - // Case 2 - // We hit a page node along the path to the target node - } else if (log2_node_size == get_log2_page_size()) { - assert(node); - hash_type page_hash; - // If target node is smaller than page size - if (log2_target_size < get_log2_page_size()) { - // If we were given the page data, compute from it - if (page_data != nullptr) { - get_inside_page_sibling_hashes(target_address, log2_target_size, proof.get_target_hash(), page_data, - page_hash, proof); - // Otherwise, if page is pristine - } else { - page_hash = get_pristine_hash(get_log2_page_size()); - for (int i = get_log2_page_size() - 1; i >= log2_target_size; --i) { - proof.set_sibling_hash(get_pristine_hash(i), i); - } - proof.set_target_hash(get_pristine_hash(log2_target_size)); - } - // Check if hash stored in node matches what we just computed - if (node->hash != page_hash) { - // Caller probably forgot to update the Merkle tree - throw std::runtime_error{"inconsistent merkle tree"}; - } - // If target node is the page itself - } else { - // Simply copy hash - proof.set_target_hash(node->hash); - } - // Case 3 - // We hit the target node itself - } else { - assert(node && (log2_node_size == log2_target_size)); - // Copy target node hash and nothing else to do - proof.set_target_hash(node->hash); - } - // Copy remaining proof values - proof.set_target_address(target_address); - proof.set_root_hash(m_root->hash); // NOLINT: m_root can't be nullptr -#ifndef NDEBUG - // Return proof only if it passes verification - hasher_type h; - if (!proof.verify(h)) { - throw std::runtime_error{"proof failed verification"}; - } -#endif - return proof; -} - -machine_merkle_tree::hash_type machine_merkle_tree::get_node_hash(address_type target_address, - int log2_target_size) const { - if (log2_target_size > get_log2_root_size() || log2_target_size < get_log2_word_size()) { - throw std::runtime_error{"log2_target_size is out of bounds"}; - } - if (log2_target_size < get_log2_page_size()) { - throw std::runtime_error{"log2_target_size is smaller than page size"}; - } - if ((target_address & ((static_cast(1) << log2_target_size) - 1)) != 0) { - throw std::invalid_argument{"address is not page-aligned"}; - } - int log2_node_size = get_log2_root_size(); - const tree_node *node = m_root; - // walk down the tree until we reach the target node - while (node != nullptr && log2_node_size > log2_target_size) { - const int log2_child_size = log2_node_size - 1; - const int path_bit = static_cast((target_address & (UINT64_C(1) << (log2_child_size))) != 0); - node = node->child[path_bit]; - log2_node_size = log2_child_size; - } - if (node == nullptr) { - // We hit a pristine node along the path to the target node - return get_pristine_hash(log2_target_size); - } - return node->hash; -} - -std::ostream &operator<<(std::ostream &out, const machine_merkle_tree::hash_type &hash) { - auto f = out.flags(); - for (const unsigned b : hash) { - out << std::hex << std::setfill('0') << std::setw(2) << b; - } - out.flags(f); - return out; -} - -} // namespace cartesi diff --git a/src/machine-merkle-tree.h b/src/machine-merkle-tree.h deleted file mode 100644 index 71293fed5..000000000 --- a/src/machine-merkle-tree.h +++ /dev/null @@ -1,368 +0,0 @@ -// Copyright Cartesi and individual authors (see AUTHORS) -// SPDX-License-Identifier: LGPL-3.0-or-later -// -// This program is free software: you can redistribute it and/or modify it under -// the terms of the GNU Lesser General Public License as published by the Free -// Software Foundation, either version 3 of the License, or (at your option) any -// later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT ANY -// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A -// PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License along -// with this program (see COPYING). If not, see . -// - -#ifndef MERKLE_TREE_H -#define MERKLE_TREE_H - -/// \file -/// \brief Merkle tree interface. - -#include -#include -#include -#include -#include -#include -#include - -#include "keccak-256-hasher.h" -#include "merkle-tree-proof.h" -#include "pristine-merkle-tree.h" - -namespace cartesi { - -/// \class machine_merkle_tree -/// \brief Merkle tree implementation. -/// -/// \details The machine_merkle_tree class implements a Merkle tree -/// covering LOG2_ROOT_SIZE bits of address space. -/// -/// Upon creation, the memory is *pristine*, i.e., completely -/// filled with zeros. -/// -/// To optimize for space, subtrees corresponding to pristine -/// memory are represented by nullptr nodes. -/// Additionally, the tree is truncated below *page* nodes -/// subintending LOG2_PAGE_SIZE bits of address space. -/// The trees corresponding to pages are rebuilt from the -/// original data whenever needed and never stored. -/// Pages are divided into *words* that cover LOG2_WORD_SIZE -/// bits of address space. -/// Tree leaves contain Keccak-256 hashes of individual words. -/// -/// Tree contents are updated page-by-page using calls to -/// machine_merkle_tree#begin_update, machine_merkle_tree#update_page, ..., -/// machine_merkle_tree#update_page, machine_merkle_tree#end_update. -class machine_merkle_tree final { -public: - using word_type = uint64_t; - - using address_type = uint64_t; - -private: - /// \brief LOG2_ROOT_SIZE Number of bits covered by the address space. - /// I.e., log2 of number of bytes subintended by the tree root. - static constexpr int LOG2_ROOT_SIZE = 64; - /// \brief LOG2_PAGE_SIZE Number of bits covered by a page. - /// I.e., log2 of number of bytes subintended by the - /// the deepest explicitly represented nodes. - static constexpr int LOG2_PAGE_SIZE = 12; - /// \brief LOG2_WORD_SIZE Number of bits covered by a word. - /// I.e., log2 of number of bytes subintended by the - /// the deepest tree nodes. - static constexpr int LOG2_WORD_SIZE = 5; - /// \brief DEPTH Depth of Merkle tree. - static constexpr int DEPTH = LOG2_ROOT_SIZE - LOG2_WORD_SIZE; - - static constexpr size_t m_word_size = static_cast(1) << LOG2_WORD_SIZE; - - static constexpr address_type m_page_index_mask = ((~UINT64_C(0)) >> (64 - LOG2_ROOT_SIZE)) << LOG2_PAGE_SIZE; - - static constexpr address_type m_page_offset_mask = ~m_page_index_mask; - - static constexpr size_t m_page_size = static_cast(1) << LOG2_PAGE_SIZE; - -public: - /// \brief Returns the LOG2_ROOT_SIZE parameter. - static constexpr int get_log2_root_size() { - return LOG2_ROOT_SIZE; - } - /// \brief Returns the LOG2_PAGE_SIZE parameter. - static constexpr int get_log2_page_size() { - return LOG2_PAGE_SIZE; - } - /// \brief Returns the LOG2_WORD_SIZE parameter. - static constexpr int get_log2_word_size() { - return LOG2_WORD_SIZE; - } - /// \brief Returns the tree DEPTH. - static constexpr int get_depth() { - return DEPTH; - } - /// \brief Returns the page size. - static constexpr size_t get_page_size() { - return m_page_size; - } - /// \brief Returns the word size. - static constexpr size_t get_word_size() { - return m_word_size; - } - - /// \brief Hasher class. - using hasher_type = keccak_256_hasher; - - /// \brief Storage for a hash. - using hash_type = hasher_type::hash_type; - - /// \brief Storage for the proof of a word value. - using proof_type = merkle_tree_proof; - - /// \brief Storage for the hashes of the siblings of all nodes along - /// the path from the root to target node. - using siblings_type = proof_type::sibling_hashes_type; - -private: - /// \brief Merkle tree node structure. - /// \details A node is known to be an inner-node or a page-node implicitly - /// based on its height in the tree. - //??D This is assumed to be a POD type in the implementation - struct tree_node { - hash_type hash; ///< Hash of subintended data. - tree_node *parent; ///< Pointer to parent node (nullptr for root). - std::array child; ///< Children nodes. - uint64_t mark; ///< Helper for traversal algorithms. - }; - - // Sparse map from virtual page index to the - // corresponding page node in the Merkle tree. - std::unordered_map m_page_node_map; - - // Root of the Merkle tree. - tree_node m_root_storage; - tree_node *m_root; - - // Used to mark visited nodes when traversing the tree - // bottom up in breadth to propagate changes from dirty - // pages all the way up to the tree root. - uint64_t m_merkle_update_nonce{1}; - // FIFO to process pages in bottom-up order. - std::deque> m_merkle_update_fifo; - - // For statistics. -#ifdef MERKLE_DUMP_STATS - mutable uint64_t m_num_nodes; -#endif - - /// \brief Maps a page_index to a node. - /// \param page_index Page index. - /// \param node Node subintending page. - /// \return 1 if succeeded, 0 otherwise. - int set_page_node_map(address_type page_index, tree_node *node); - - /// \brief Creates and returns a new tree node. - /// \return Newly created node or nullptr if out-of-memory. - tree_node *create_node() const; - - /// \brief Deallocates node. - /// \param node Node to be deallocated. - void destroy_node(tree_node *node) const; - - /// \brief Creates a new page node and insert it into the Merkle tree. - /// \param page_index Page index for node. - /// \return Newly created node. - /// \details Does *not* check if a node already exists for that page index. - /// Maps new node to the page index. - tree_node *new_page_node(address_type page_index); - - /// \brief Updates an inner node hash from its children. - /// \param h Hasher object. - /// \param log2_size log2 of size subintended by node. - /// \param node Node to be updated. - static void update_inner_node_hash(hasher_type &h, int log2_size, tree_node *node); - - /// \brief Dumps a hash to std::cerr. - /// \param hash Hash to be dumped. - static void dump_hash(const hash_type &hash); - - /// \brief Returns the hash for a child of a given node. - /// \param child_log2_size log2_size of child node. - /// \param node Node from which to obtain child. - /// \param bit Bit corresponding to child_log2_size in child node address. - /// \return Reference to child hash. If child pointer is null, - /// returns a pristine hash. - static const hash_type &get_child_hash(int child_log2_size, const tree_node *node, int bit); - - /// \brief Dumps tree rooted at node to std::cerr. - /// \param node Root of subtree. - /// \param address start of range subintended by \p node. - /// \param log2_size log2 of size of range subintended by \p node. - void dump_merkle_tree(tree_node *node, uint64_t address, int log2_size) const; - - /// \brief Dumps the entire tree rooted to std::cerr. - void dump_merkle_tree() const; - - /// \brief Destroys tree rooted at node. - /// \param node Root of subtree. - /// \param log2_size log2 of size subintended by \p node. - void destroy_merkle_tree(tree_node *node, int log2_size); - - /// \brief Destroys entire Merkle tree. - void destroy_merkle_tree(); - - /// \brief Verifies tree rooted at node. - /// \param h Hasher object. - /// \param node Root of subtree. - /// \param log2_size log2 of size subintended by \p node. - /// \returns True if tree is consistent, false otherwise. - bool verify_tree(hasher_type &h, tree_node *node, int log2_size) const; - - /// \brief Computes the page index for a memory address. - /// \param address Memory address. - /// \return The page index. - static constexpr address_type get_page_index(address_type address); - - /// \brief Computes the offset of a memory address within its page. - /// \param address Memory address. - /// \return The offset. - static constexpr address_type get_offset_in_page(address_type address); - - /// \brief Computes the offset of a memory address within its page. - /// \param page_index Page index associated to node. - /// \return The node, if found, or nullptr otherwise. - tree_node *get_page_node(address_type page_index) const; - - /// \brief Recursively builds hash for log2_size node - /// from contiguous memory. - /// \param h Hasher object. - /// \param start Start of contiguous memory subintended by node. - /// \param log2_size log2 of size subintended by node. - /// \param hash Receives the hash. - void get_page_node_hash(hasher_type &h, const unsigned char *start, int log2_size, hash_type &hash) const; - - /// \brief Gets the sibling hashes along the path from - /// the node currently being visited and a target node. - /// \param h Hasher object. - /// \param address Address of target node. - /// \param log2_size log2 of size subintended by target node. - /// \param hash Receives the target node hash if it is a - /// subnode of the node currently being visited. - /// \param curr_data Pointer to contiguous data for node currently - /// being visited. - /// \param log2_curr_size log2 of size subintended by node - /// currently being visited. - /// \param curr_hash Receives the hash for node currently being visited. - /// \param parent_diverged True if parent of node currently being visited - /// is not in path from root to target node. - /// \param curr_diverged True if node currently being visited is - /// itself not in path from root to target node. - /// \param proof Proof to receive sibling hashes. - void get_inside_page_sibling_hashes(hasher_type &h, address_type address, int log2_size, hash_type &hash, - const unsigned char *curr_data, int log2_curr_size, hash_type &curr_hash, int parent_diverged, - int curr_diverged, proof_type &proof) const; - - /// \brief Gets the sibling hashes along the path from a - /// page node towards a target node. - /// \param address Address of target node. - /// \param log2_size log2 of size subintended by target node. - /// \param hash Receives target node hash. - /// \param page_data Pointer to start of contiguous page data. - /// \param page_hash Receives the hash for the page. - /// \param proof Proof to receive sibling hashes. - void get_inside_page_sibling_hashes(address_type address, int log2_size, hash_type &hash, - const unsigned char *page_data, hash_type &page_hash, proof_type &proof) const; - - // Precomputed hashes of spans of zero bytes with - // increasing power-of-two sizes, from 2^LOG2_WORD_SIZE - // to 2^LOG2_ROOT_SIZE bytes. - static const pristine_merkle_tree &pristine_hashes(); - -public: - /// \brief Verifies the entire Merkle tree. - /// \return True if tree is consistent, false otherwise. - bool verify_tree() const; - - /// \brief Default constructor. - /// \details Initializes memory to zero. - machine_merkle_tree(); - - /// \brief No copy constructor - machine_merkle_tree(const machine_merkle_tree &) = delete; - /// \brief No copy assignment - machine_merkle_tree &operator=(const machine_merkle_tree &) = delete; - /// \brief No move constructor - machine_merkle_tree(machine_merkle_tree &&) = delete; - /// \brief No move assignment - machine_merkle_tree &operator=(machine_merkle_tree &&) = delete; - - /// \brief Destructor - /// \details Releases all used memory - ~machine_merkle_tree(); - - /// \brief Returns the root hash. - /// \param hash Receives the hash. - void get_root_hash(hash_type &hash) const; - - /// \brief Start tree update. - /// \returns True. - /// \details This method is not thread safe, so be careful when using - /// parallelization to compute Merkle trees - bool begin_update(); - - /// \brief Update tree with new hash for a page node. - /// \param page_index Page index for node. - /// \param hash New hash for node. - /// \returns True if succeeded, false otherwise. - /// \details This method is not thread safe, so be careful when using - /// parallelization to compute Merkle trees - bool update_page_node_hash(address_type page_index, const hash_type &hash); - - /// \brief End tree update. - /// \param h Hasher object. - /// \returns True if succeeded, false otherwise. - /// \details This method is not thread safe, so be careful when using - /// parallelization to compute Merkle trees - bool end_update(hasher_type &h); - - /// \brief Returns the proof for a node in the tree. - /// \param target_address Address of target node. Must be aligned - /// to a 2log2_target_size boundary. - /// \param log2_target_size log2 of size subintended by - /// target node. Must be between LOG2_WORD_SIZE and LOG2_ROOT_SIZE, - /// inclusive. - /// \param page_data When log2_target_size smaller than LOG2_PAGE_SIZE, - /// \p page_data must point to start of contiguous page containing - /// the node, or nullptr if the page is pristine (i.e., filled with zeros). - /// \returns Proof if successful, otherwise throws exception. - proof_type get_proof(address_type target_address, int log2_target_size, const unsigned char *page_data) const; - - /// \brief Recursively builds hash for page node from contiguous memory. - /// \param h Hasher object. - /// \param page_data Pointer to start of contiguous page data. - /// \param hash Receives the hash. - void get_page_node_hash(hasher_type &h, const unsigned char *page_data, hash_type &hash) const; - - /// \brief Gets currently stored hash for page node. - /// \param page_index Page index for node. - /// \param hash Receives the hash. - void get_page_node_hash(address_type page_index, hash_type &hash) const; - - /// \brief Get the hash of a node in the Merkle tree. - /// \param target_address Address of target node. - /// \param log2_target_size log2 of the node size. - /// \return Hash of the node. - hash_type get_node_hash(address_type target_address, int log2_target_size) const; - - /// \brief Returns the hash for a log2_size pristine node. - /// \param log2_size log2 of size subintended by node. - /// \return Reference to precomputed hash. - static const hash_type &get_pristine_hash(int log2_size); -}; - -std::ostream &operator<<(std::ostream &out, const machine_merkle_tree::hash_type &hash); - -} // namespace cartesi - -#endif diff --git a/src/machine-reg.h b/src/machine-reg.h index 0fc8e77bf..307a4e074 100644 --- a/src/machine-reg.h +++ b/src/machine-reg.h @@ -17,8 +17,9 @@ #ifndef MACHINE_REG_H #define MACHINE_REG_H -#include "pma-constants.h" -#include "shadow-state.h" +#include + +#include "shadow-registers.h" #include "shadow-uarch-state.h" /// \file @@ -28,154 +29,156 @@ namespace cartesi { /// \brief List of machine registers enum class machine_reg : uint64_t { - // Processor x registers - x0 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[0]), - x1 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[1]), - x2 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[2]), - x3 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[3]), - x4 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[4]), - x5 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[5]), - x6 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[6]), - x7 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[7]), - x8 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[8]), - x9 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[9]), - x10 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[10]), - x11 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[11]), - x12 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[12]), - x13 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[13]), - x14 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[14]), - x15 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[15]), - x16 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[16]), - x17 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[17]), - x18 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[18]), - x19 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[19]), - x20 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[20]), - x21 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[21]), - x22 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[22]), - x23 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[23]), - x24 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[24]), - x25 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[25]), - x26 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[26]), - x27 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[27]), - x28 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[28]), - x29 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[29]), - x30 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[30]), - x31 = PMA_SHADOW_STATE_START + offsetof(shadow_state, x[31]), - f0 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[0]), - f1 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[1]), - f2 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[2]), - f3 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[3]), - f4 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[4]), - f5 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[5]), - f6 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[6]), - f7 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[7]), - f8 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[8]), - f9 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[9]), - f10 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[10]), - f11 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[11]), - f12 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[12]), - f13 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[13]), - f14 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[14]), - f15 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[15]), - f16 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[16]), - f17 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[17]), - f18 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[18]), - f19 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[19]), - f20 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[20]), - f21 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[21]), - f22 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[22]), - f23 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[23]), - f24 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[24]), - f25 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[25]), - f26 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[26]), - f27 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[27]), - f28 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[28]), - f29 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[29]), - f30 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[30]), - f31 = PMA_SHADOW_STATE_START + offsetof(shadow_state, f[31]), - pc = PMA_SHADOW_STATE_START + offsetof(shadow_state, pc), - fcsr = PMA_SHADOW_STATE_START + offsetof(shadow_state, fcsr), - mvendorid = PMA_SHADOW_STATE_START + offsetof(shadow_state, mvendorid), - marchid = PMA_SHADOW_STATE_START + offsetof(shadow_state, marchid), - mimpid = PMA_SHADOW_STATE_START + offsetof(shadow_state, mimpid), - mcycle = PMA_SHADOW_STATE_START + offsetof(shadow_state, mcycle), - icycleinstret = PMA_SHADOW_STATE_START + offsetof(shadow_state, icycleinstret), - mstatus = PMA_SHADOW_STATE_START + offsetof(shadow_state, mstatus), - mtvec = PMA_SHADOW_STATE_START + offsetof(shadow_state, mtvec), - mscratch = PMA_SHADOW_STATE_START + offsetof(shadow_state, mscratch), - mepc = PMA_SHADOW_STATE_START + offsetof(shadow_state, mepc), - mcause = PMA_SHADOW_STATE_START + offsetof(shadow_state, mcause), - mtval = PMA_SHADOW_STATE_START + offsetof(shadow_state, mtval), - misa = PMA_SHADOW_STATE_START + offsetof(shadow_state, misa), - mie = PMA_SHADOW_STATE_START + offsetof(shadow_state, mie), - mip = PMA_SHADOW_STATE_START + offsetof(shadow_state, mip), - medeleg = PMA_SHADOW_STATE_START + offsetof(shadow_state, medeleg), - mideleg = PMA_SHADOW_STATE_START + offsetof(shadow_state, mideleg), - mcounteren = PMA_SHADOW_STATE_START + offsetof(shadow_state, mcounteren), - menvcfg = PMA_SHADOW_STATE_START + offsetof(shadow_state, menvcfg), - stvec = PMA_SHADOW_STATE_START + offsetof(shadow_state, stvec), - sscratch = PMA_SHADOW_STATE_START + offsetof(shadow_state, sscratch), - sepc = PMA_SHADOW_STATE_START + offsetof(shadow_state, sepc), - scause = PMA_SHADOW_STATE_START + offsetof(shadow_state, scause), - stval = PMA_SHADOW_STATE_START + offsetof(shadow_state, stval), - satp = PMA_SHADOW_STATE_START + offsetof(shadow_state, satp), - scounteren = PMA_SHADOW_STATE_START + offsetof(shadow_state, scounteren), - senvcfg = PMA_SHADOW_STATE_START + offsetof(shadow_state, senvcfg), - ilrsc = PMA_SHADOW_STATE_START + offsetof(shadow_state, ilrsc), - iprv = PMA_SHADOW_STATE_START + offsetof(shadow_state, iprv), - iflags_X = PMA_SHADOW_STATE_START + offsetof(shadow_state, iflags_X), - iflags_Y = PMA_SHADOW_STATE_START + offsetof(shadow_state, iflags_Y), - iflags_H = PMA_SHADOW_STATE_START + offsetof(shadow_state, iflags_H), - iunrep = PMA_SHADOW_STATE_START + offsetof(shadow_state, iunrep), - clint_mtimecmp = PMA_SHADOW_STATE_START + offsetof(shadow_state, clint_mtimecmp), - plic_girqpend = PMA_SHADOW_STATE_START + offsetof(shadow_state, plic_girqpend), - plic_girqsrvd = PMA_SHADOW_STATE_START + offsetof(shadow_state, plic_girqsrvd), - htif_tohost = PMA_SHADOW_STATE_START + offsetof(shadow_state, htif_tohost), - htif_fromhost = PMA_SHADOW_STATE_START + offsetof(shadow_state, htif_fromhost), - htif_ihalt = PMA_SHADOW_STATE_START + offsetof(shadow_state, htif_ihalt), - htif_iconsole = PMA_SHADOW_STATE_START + offsetof(shadow_state, htif_iconsole), - htif_iyield = PMA_SHADOW_STATE_START + offsetof(shadow_state, htif_iyield), + x0 = static_cast(shadow_registers_what::x0), + x1 = static_cast(shadow_registers_what::x1), + x2 = static_cast(shadow_registers_what::x2), + x3 = static_cast(shadow_registers_what::x3), + x4 = static_cast(shadow_registers_what::x4), + x5 = static_cast(shadow_registers_what::x5), + x6 = static_cast(shadow_registers_what::x6), + x7 = static_cast(shadow_registers_what::x7), + x8 = static_cast(shadow_registers_what::x8), + x9 = static_cast(shadow_registers_what::x9), + x10 = static_cast(shadow_registers_what::x10), + x11 = static_cast(shadow_registers_what::x11), + x12 = static_cast(shadow_registers_what::x12), + x13 = static_cast(shadow_registers_what::x13), + x14 = static_cast(shadow_registers_what::x14), + x15 = static_cast(shadow_registers_what::x15), + x16 = static_cast(shadow_registers_what::x16), + x17 = static_cast(shadow_registers_what::x17), + x18 = static_cast(shadow_registers_what::x18), + x19 = static_cast(shadow_registers_what::x19), + x20 = static_cast(shadow_registers_what::x20), + x21 = static_cast(shadow_registers_what::x21), + x22 = static_cast(shadow_registers_what::x22), + x23 = static_cast(shadow_registers_what::x23), + x24 = static_cast(shadow_registers_what::x24), + x25 = static_cast(shadow_registers_what::x25), + x26 = static_cast(shadow_registers_what::x26), + x27 = static_cast(shadow_registers_what::x27), + x28 = static_cast(shadow_registers_what::x28), + x29 = static_cast(shadow_registers_what::x29), + x30 = static_cast(shadow_registers_what::x30), + x31 = static_cast(shadow_registers_what::x31), + f0 = static_cast(shadow_registers_what::f0), + f1 = static_cast(shadow_registers_what::f1), + f2 = static_cast(shadow_registers_what::f2), + f3 = static_cast(shadow_registers_what::f3), + f4 = static_cast(shadow_registers_what::f4), + f5 = static_cast(shadow_registers_what::f5), + f6 = static_cast(shadow_registers_what::f6), + f7 = static_cast(shadow_registers_what::f7), + f8 = static_cast(shadow_registers_what::f8), + f9 = static_cast(shadow_registers_what::f9), + f10 = static_cast(shadow_registers_what::f10), + f11 = static_cast(shadow_registers_what::f11), + f12 = static_cast(shadow_registers_what::f12), + f13 = static_cast(shadow_registers_what::f13), + f14 = static_cast(shadow_registers_what::f14), + f15 = static_cast(shadow_registers_what::f15), + f16 = static_cast(shadow_registers_what::f16), + f17 = static_cast(shadow_registers_what::f17), + f18 = static_cast(shadow_registers_what::f18), + f19 = static_cast(shadow_registers_what::f19), + f20 = static_cast(shadow_registers_what::f20), + f21 = static_cast(shadow_registers_what::f21), + f22 = static_cast(shadow_registers_what::f22), + f23 = static_cast(shadow_registers_what::f23), + f24 = static_cast(shadow_registers_what::f24), + f25 = static_cast(shadow_registers_what::f25), + f26 = static_cast(shadow_registers_what::f26), + f27 = static_cast(shadow_registers_what::f27), + f28 = static_cast(shadow_registers_what::f28), + f29 = static_cast(shadow_registers_what::f29), + f30 = static_cast(shadow_registers_what::f30), + f31 = static_cast(shadow_registers_what::f31), + pc = static_cast(shadow_registers_what::pc), + fcsr = static_cast(shadow_registers_what::fcsr), + mvendorid = static_cast(shadow_registers_what::mvendorid), + marchid = static_cast(shadow_registers_what::marchid), + mimpid = static_cast(shadow_registers_what::mimpid), + mcycle = static_cast(shadow_registers_what::mcycle), + icycleinstret = static_cast(shadow_registers_what::icycleinstret), + mstatus = static_cast(shadow_registers_what::mstatus), + mtvec = static_cast(shadow_registers_what::mtvec), + mscratch = static_cast(shadow_registers_what::mscratch), + mepc = static_cast(shadow_registers_what::mepc), + mcause = static_cast(shadow_registers_what::mcause), + mtval = static_cast(shadow_registers_what::mtval), + misa = static_cast(shadow_registers_what::misa), + mie = static_cast(shadow_registers_what::mie), + mip = static_cast(shadow_registers_what::mip), + medeleg = static_cast(shadow_registers_what::medeleg), + mideleg = static_cast(shadow_registers_what::mideleg), + mcounteren = static_cast(shadow_registers_what::mcounteren), + menvcfg = static_cast(shadow_registers_what::menvcfg), + stvec = static_cast(shadow_registers_what::stvec), + sscratch = static_cast(shadow_registers_what::sscratch), + sepc = static_cast(shadow_registers_what::sepc), + scause = static_cast(shadow_registers_what::scause), + stval = static_cast(shadow_registers_what::stval), + satp = static_cast(shadow_registers_what::satp), + scounteren = static_cast(shadow_registers_what::scounteren), + senvcfg = static_cast(shadow_registers_what::senvcfg), + ilrsc = static_cast(shadow_registers_what::ilrsc), + iprv = static_cast(shadow_registers_what::iprv), + iflags_X = static_cast(shadow_registers_what::iflags_X), + iflags_Y = static_cast(shadow_registers_what::iflags_Y), + iflags_H = static_cast(shadow_registers_what::iflags_H), + iunrep = static_cast(shadow_registers_what::iunrep), + clint_mtimecmp = static_cast(shadow_registers_what::clint_mtimecmp), + plic_girqpend = static_cast(shadow_registers_what::plic_girqpend), + plic_girqsrvd = static_cast(shadow_registers_what::plic_girqsrvd), + htif_tohost = static_cast(shadow_registers_what::htif_tohost), + htif_fromhost = static_cast(shadow_registers_what::htif_fromhost), + htif_ihalt = static_cast(shadow_registers_what::htif_ihalt), + htif_iconsole = static_cast(shadow_registers_what::htif_iconsole), + htif_iyield = static_cast(shadow_registers_what::htif_iyield), first_ = x0, last_ = htif_iyield, - uarch_halt_flag = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, halt_flag), - uarch_cycle = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, cycle), - uarch_pc = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, pc), - uarch_x0 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[0]), - uarch_x1 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[1]), - uarch_x2 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[2]), - uarch_x3 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[3]), - uarch_x4 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[4]), - uarch_x5 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[5]), - uarch_x6 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[6]), - uarch_x7 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[7]), - uarch_x8 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[8]), - uarch_x9 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[9]), - uarch_x10 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[10]), - uarch_x11 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[11]), - uarch_x12 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[12]), - uarch_x13 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[13]), - uarch_x14 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[14]), - uarch_x15 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[15]), - uarch_x16 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[16]), - uarch_x17 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[17]), - uarch_x18 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[18]), - uarch_x19 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[19]), - uarch_x20 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[20]), - uarch_x21 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[21]), - uarch_x22 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[22]), - uarch_x23 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[23]), - uarch_x24 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[24]), - uarch_x25 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[25]), - uarch_x26 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[26]), - uarch_x27 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[27]), - uarch_x28 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[28]), - uarch_x29 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[29]), - uarch_x30 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[30]), - uarch_x31 = PMA_SHADOW_UARCH_STATE_START + offsetof(shadow_uarch_state, x[31]), + uarch_halt_flag = static_cast(shadow_uarch_state_what::uarch_halt_flag), + uarch_cycle = static_cast(shadow_uarch_state_what::uarch_cycle), + uarch_pc = static_cast(shadow_uarch_state_what::uarch_pc), + uarch_x0 = static_cast(shadow_uarch_state_what::uarch_x0), + uarch_x1 = static_cast(shadow_uarch_state_what::uarch_x1), + uarch_x2 = static_cast(shadow_uarch_state_what::uarch_x2), + uarch_x3 = static_cast(shadow_uarch_state_what::uarch_x3), + uarch_x4 = static_cast(shadow_uarch_state_what::uarch_x4), + uarch_x5 = static_cast(shadow_uarch_state_what::uarch_x5), + uarch_x6 = static_cast(shadow_uarch_state_what::uarch_x6), + uarch_x7 = static_cast(shadow_uarch_state_what::uarch_x7), + uarch_x8 = static_cast(shadow_uarch_state_what::uarch_x8), + uarch_x9 = static_cast(shadow_uarch_state_what::uarch_x9), + uarch_x10 = static_cast(shadow_uarch_state_what::uarch_x10), + uarch_x11 = static_cast(shadow_uarch_state_what::uarch_x11), + uarch_x12 = static_cast(shadow_uarch_state_what::uarch_x12), + uarch_x13 = static_cast(shadow_uarch_state_what::uarch_x13), + uarch_x14 = static_cast(shadow_uarch_state_what::uarch_x14), + uarch_x15 = static_cast(shadow_uarch_state_what::uarch_x15), + uarch_x16 = static_cast(shadow_uarch_state_what::uarch_x16), + uarch_x17 = static_cast(shadow_uarch_state_what::uarch_x17), + uarch_x18 = static_cast(shadow_uarch_state_what::uarch_x18), + uarch_x19 = static_cast(shadow_uarch_state_what::uarch_x19), + uarch_x20 = static_cast(shadow_uarch_state_what::uarch_x20), + uarch_x21 = static_cast(shadow_uarch_state_what::uarch_x21), + uarch_x22 = static_cast(shadow_uarch_state_what::uarch_x22), + uarch_x23 = static_cast(shadow_uarch_state_what::uarch_x23), + uarch_x24 = static_cast(shadow_uarch_state_what::uarch_x24), + uarch_x25 = static_cast(shadow_uarch_state_what::uarch_x25), + uarch_x26 = static_cast(shadow_uarch_state_what::uarch_x26), + uarch_x27 = static_cast(shadow_uarch_state_what::uarch_x27), + uarch_x28 = static_cast(shadow_uarch_state_what::uarch_x28), + uarch_x29 = static_cast(shadow_uarch_state_what::uarch_x29), + uarch_x30 = static_cast(shadow_uarch_state_what::uarch_x30), + uarch_x31 = static_cast(shadow_uarch_state_what::uarch_x31), uarch_first_ = uarch_halt_flag, uarch_last_ = uarch_x31, + // Something unknown + unknown_ = UINT64_C(1) << 63, // Outside of RISC-V address space + // Views of registers htif_tohost_dev, htif_tohost_cmd, @@ -185,17 +188,60 @@ enum class machine_reg : uint64_t { htif_fromhost_cmd, htif_fromhost_reason, htif_fromhost_data, - unknown_, + view_first_ = htif_tohost_dev, + view_last_ = htif_fromhost_data, + }; -constexpr uint64_t machine_reg_address(machine_reg reg, int i = 0) { +static constexpr uint64_t machine_reg_address(machine_reg reg, int i = 0) { return static_cast(reg) + (i * sizeof(uint64_t)); } -constexpr machine_reg machine_reg_enum(machine_reg reg, int i) { +static constexpr machine_reg machine_reg_enum(machine_reg reg, int i) { return static_cast(static_cast(reg) + (i * sizeof(uint64_t))); } +static constexpr machine_reg machine_reg_enum(shadow_registers_what reg) { + return static_cast(reg); +} + +static constexpr machine_reg machine_reg_enum(shadow_uarch_state_what reg) { + return static_cast(reg); +} + +static constexpr const char *machine_reg_get_name(machine_reg reg) { + const auto ureg = static_cast(reg); + if (ureg >= static_cast(machine_reg::first_) && ureg <= static_cast(machine_reg::last_)) { + return shadow_registers_get_what_name(static_cast(reg)); + } + if (ureg >= static_cast(machine_reg::uarch_first_) && + ureg <= static_cast(machine_reg::uarch_last_)) { + return shadow_uarch_state_get_what_name(static_cast(reg)); + } + switch (reg) { + case machine_reg::htif_tohost_dev: + return "htif.tohost_dev"; + case machine_reg::htif_tohost_cmd: + return "htif.tohost_cmd"; + case machine_reg::htif_tohost_reason: + return "htif.tohost_reason"; + case machine_reg::htif_tohost_data: + return "htif.tohost_data"; + case machine_reg::htif_fromhost_dev: + return "htif.fromhost_dev"; + case machine_reg::htif_fromhost_cmd: + return "htif.fromhost_cmd"; + case machine_reg::htif_fromhost_reason: + return "htif.fromhost_reason"; + case machine_reg::htif_fromhost_data: + return "htif.fromhost_data"; + case machine_reg::unknown_: + [[fallthrough]]; + default: + return "unknown"; + } +} + static_assert(machine_reg_address(machine_reg::uarch_first_) > machine_reg_address(machine_reg::last_)); static_assert(machine_reg_address(machine_reg::x0, 1) == machine_reg_address(machine_reg::x1)); diff --git a/src/machine-runtime-config.h b/src/machine-runtime-config.h index 649ea4aaf..a6c6c2ebd 100644 --- a/src/machine-runtime-config.h +++ b/src/machine-runtime-config.h @@ -26,7 +26,7 @@ namespace cartesi { /// \brief Concurrency runtime configuration struct concurrency_runtime_config { - uint64_t update_merkle_tree{}; + uint64_t update_hash_tree{}; }; /// \brief HTIF runtime configuration @@ -38,10 +38,9 @@ struct htif_runtime_config { struct machine_runtime_config { concurrency_runtime_config concurrency{}; htif_runtime_config htif{}; - bool skip_root_hash_check{}; - bool skip_root_hash_store{}; bool skip_version_check{}; bool soft_yield{}; + bool no_reserve{}; ///< Do not reserve swap memory when mapping flash drives }; /// \brief CONCURRENCY constants diff --git a/src/machine-state.h b/src/machine-state.h deleted file mode 100644 index e13edea6d..000000000 --- a/src/machine-state.h +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright Cartesi and individual authors (see AUTHORS) -// SPDX-License-Identifier: LGPL-3.0-or-later -// -// This program is free software: you can redistribute it and/or modify it under -// the terms of the GNU Lesser General Public License as published by the Free -// Software Foundation, either version 3 of the License, or (at your option) any -// later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT ANY -// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A -// PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License along -// with this program (see COPYING). If not, see . -// - -#ifndef STATE_H -#define STATE_H - -/// \file -/// \brief Cartesi machine state structure definition. - -#include -#include - -#ifdef DUMP_HIST -#include -#endif - -#include - -#include "pma-constants.h" -#include "pma.h" -#include "riscv-constants.h" -#include "shadow-tlb.h" - -namespace cartesi { - -/// \brief Machine state. -/// \details The machine_state structure contains the entire -/// state of a Cartesi machine. -struct machine_state { - machine_state() = default; - ~machine_state() = default; - - /// \brief No copy or move constructor or assignment - machine_state(const machine_state &other) = delete; - machine_state(machine_state &&other) = delete; - machine_state &operator=(const machine_state &other) = delete; - machine_state &operator=(machine_state &&other) = delete; - - // The following state fields are very hot, - // and are carefully ordered to have better data locality in the interpreter loop. - // The X registers are the very first to optimize access of registers in the interpreter. - std::array x{}; ///< Register file - uint64_t mcycle{}; ///< CSR mcycle. - uint64_t pc{}; ///< Program counter. - uint64_t fcsr{}; ///< CSR fcsr. - std::array f{}; ///< Floating-point register file. - - uint64_t iprv{}; ///< Privilege level (Cartesi-specific). - - uint64_t mstatus{}; ///< CSR mstatus. - uint64_t mtvec{}; ///< CSR mtvec. - uint64_t mscratch{}; ///< CSR mscratch. - uint64_t mepc{}; ///< CSR mepc. - uint64_t mcause{}; ///< CSR mcause. - uint64_t mtval{}; ///< CSR mtval. - uint64_t misa{}; ///< CSR misa. - - uint64_t mie{}; ///< CSR mie. - uint64_t mip{}; ///< CSR mip. - uint64_t medeleg{}; ///< CSR medeleg. - uint64_t mideleg{}; ///< CSR mideleg. - uint64_t mcounteren{}; ///< CSR mcounteren. - uint64_t menvcfg{}; ///< CSR menvcfg. - - uint64_t stvec{}; ///< CSR stvec. - uint64_t sscratch{}; ///< CSR sscratch. - uint64_t sepc{}; ///< CSR sepc. - uint64_t scause{}; ///< CSR scause. - uint64_t stval{}; ///< CSR stval. - uint64_t satp{}; ///< CSR satp. - uint64_t scounteren{}; ///< CSR scounteren. - uint64_t senvcfg{}; ///< CSR senvcfg. - - // Cartesi-specific state - uint64_t ilrsc{}; ///< For LR/SC instructions (Cartesi-specific). - uint64_t icycleinstret{}; ///< CSR icycleinstret (Cartesi-specific). - struct { - uint64_t X{}; ///< CPU has yielded with automatic reset (Cartesi-specific). - uint64_t Y{}; ///< CPU has yielded with manual reset (Cartesi-specific). - uint64_t H{}; ///< CPU has been permanently halted (Cartesi-specific). - } iflags; - uint64_t iunrep{}; ///< Unreproducible mode (Cartesi-specific). - - /// \brief CLINT state - struct { - uint64_t mtimecmp{}; ///< CSR mtimecmp. - } clint; - - /// \brief PLIC state - struct { - uint64_t girqpend{}; ///< CSR girqpend (global interrupts pending). - uint64_t girqsrvd{}; ///< CSR girqsrvd (global interrupts served). - } plic; - - /// \brief TLB state - shadow_tlb_state tlb{}; - - /// \brief HTIF state - struct { - uint64_t tohost{}; ///< CSR tohost. - uint64_t fromhost{}; ///< CSR fromhost. - uint64_t ihalt{}; ///< CSR ihalt. - uint64_t iconsole{}; ///< CSR iconsole. - uint64_t iyield{}; ///< CSR iyield. - } htif; - - /// Soft yield - bool soft_yield{}; - - /// Map of physical memory ranges - boost::container::static_vector pmas; - - pma_entry empty_pma; ///< fallback to PMA for empty range - - // Entries below this mark are not needed in the blockchain - -#ifdef DUMP_COUNTERS - machine_statistics stats; -#endif - -#ifdef DUMP_HIST - std::unordered_map insn_hist; -#endif -}; - -} // namespace cartesi - -#endif diff --git a/src/machine-statistics.h b/src/machine-statistics.h deleted file mode 100644 index e0221d68d..000000000 --- a/src/machine-statistics.h +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright Cartesi and individual authors (see AUTHORS) -// SPDX-License-Identifier: LGPL-3.0-or-later -// -// This program is free software: you can redistribute it and/or modify it under -// the terms of the GNU Lesser General Public License as published by the Free -// Software Foundation, either version 3 of the License, or (at your option) any -// later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT ANY -// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A -// PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License along -// with this program (see COPYING). If not, see . -// - -#ifndef MACHINE_STATISTICS_H -#define MACHINE_STATISTICS_H - -#include - -namespace cartesi { - -/// \brief Machine statistics -struct machine_statistics { - uint64_t inner_loop; ///< Counts executions of inner loop - uint64_t outer_loop; ///< Counts executions of outer loop - uint64_t sv_int; ///< Counts supervisor interrupts - uint64_t sv_ex; ///< Counts supervisor exceptions (except ECALL) - uint64_t m_int; ///< Counts machine interrupts - uint64_t m_ex; ///< Counts machine exceptions (except ECALL) - uint64_t atomic_mop; ///< Counts atomic memory operations - uint64_t flush_all; ///< Counts flush all calls - uint64_t flush_va; ///< Counts flush virtual address calls - uint64_t fence; ///< Counts fence calls - uint64_t fence_i; ///< Counts fence.i calls - uint64_t fence_vma; ///< Counts fence.vma calls - uint64_t max_asid; ///< Counts the maximum number of used ASIDs (only relevant when ASIDLEN > 0) - uint64_t prv_level[4]; ///< Counts changes to prvilege levels - - // TLB - uint64_t tlb_chit; ///< Counts TLB code access hits - uint64_t tlb_cmiss; ///< Counts TLB code access misses - uint64_t tlb_rhit; ///< Counts TLB read access hits - uint64_t tlb_rmiss; ///< Counts TLB read access misses - uint64_t tlb_whit; ///< Counts TLB write access hits - uint64_t tlb_wmiss; ///< Counts TLB write access misses - uint64_t tlb_flush_all; ///< Counts TLB flush all calls - uint64_t tlb_flush_vaddr; ///< Counts TLB flush virtual address calls - uint64_t tlb_flush_read; ///< Counts read TLB flush calls - uint64_t tlb_flush_write; ///< Counts write TLB flush calls - uint64_t tlb_flush_satp; ///< Counts TLB flush originated from satp changes - uint64_t tlb_flush_mstatus; ///< Counts TLB flush originated from mstatus changes - uint64_t tlb_flush_set_prv; ///< Counts TLB flush originated from set_prv changes - uint64_t tlb_flush_fence_vma_all; ///< Counts TLB flush originated from SFENCE.VMA (all) - uint64_t tlb_flush_fence_vma_asid; ///< Counts TLB flush originated from SFENCE.VMA (asid) - uint64_t tlb_flush_fence_vma_vaddr; ///< Counts TLB flush originated from SFENCE.VMA (vaddr) - uint64_t tlb_flush_fence_vma_asid_vaddr; ///< Counts TLB flush originated originated from SFENCE.VMA (vaddr,asid) -}; - -#ifdef DUMP_COUNTERS -// NOLINTBEGIN(cppcoreguidelines-macro-usage,cppcoreguidelines-avoid-do-while) -#define INC_COUNTER(stats, counter) \ - do { \ - stats.counter++; \ - } while (0) -// NOLINTEND(cppcoreguidelines-macro-usage,cppcoreguidelines-avoid-do-while) -#else -// NOLINTBEGIN(cppcoreguidelines-macro-usage,cppcoreguidelines-avoid-do-while) -#define INC_COUNTER(state, counter) \ - do { \ - } while (0) -// NOLINTEND(cppcoreguidelines-macro-usage,cppcoreguidelines-avoid-do-while) -#endif - -} // namespace cartesi - -#endif diff --git a/src/machine.cpp b/src/machine.cpp index 46ff2acd8..b5eadbfdb 100644 --- a/src/machine.cpp +++ b/src/machine.cpp @@ -17,582 +17,329 @@ #include "machine.h" #include -#include -#include +#include +#include #include #include #include -#include #include -#include +#include +#include +#include +#include #include #include -#include -#include #include -#include - -#include #include "access-log.h" -#include "bracket-note.h" -#include "clint-factory.h" +#include "address-range-constants.h" +#include "address-range.h" +#include "algorithm.h" +#include "assert-printf.h" +#include "back-merkle-tree.h" +#include "collect-mcycle-hashes-state-access.h" +#include "collect-uarch-cycle-hashes-state-access.h" +#include "compiler-defines.h" +#include "device-state-access.h" #include "dtb.h" -#include "htif-factory.h" -#include "htif.h" +#include "hash-tree-constants.h" +#include "hash-tree.h" +#include "host-addr.h" +#include "hot-tlb.h" +#include "htif-constants.h" #include "i-device-state-access.h" #include "interpret.h" #include "is-pristine.h" +#include "machine-address-ranges.h" #include "machine-config.h" -#include "machine-memory-range-descr.h" +#include "machine-hash.h" +#include "machine-reg.h" #include "machine-runtime-config.h" +#include "mcycle-root-hashes.h" +#include "memory-address-range.h" +#include "os-filesystem.h" +#include "os-mapped-memory.h" #include "os.h" -#include "plic-factory.h" -#include "pma-constants.h" -#include "pma-defines.h" -#include "pma.h" +#include "pmas-constants.h" +#include "pmas.h" +#include "processor-state.h" #include "record-send-cmio-state-access.h" #include "record-step-state-access.h" #include "replay-send-cmio-state-access.h" #include "replay-step-state-access.h" #include "riscv-constants.h" +#include "rtc.h" +#include "scope-exit.h" +#include "scope-remove.h" #include "send-cmio-response.h" -#include "shadow-pmas-factory.h" -#include "shadow-state-factory.h" -#include "shadow-state.h" -#include "shadow-tlb-factory.h" +#include "shadow-registers.h" #include "shadow-tlb.h" #include "shadow-uarch-state.h" #include "state-access.h" #include "strict-aliasing.h" #include "translate-virtual-address.h" +#include "uarch-constants.h" +#include "uarch-cycle-root-hashes.h" #include "uarch-interpret.h" -#include "uarch-pristine-state-hash.h" +#include "uarch-pristine.h" #include "uarch-record-state-access.h" #include "uarch-replay-state-access.h" #include "uarch-reset-state.h" #include "uarch-state-access.h" #include "uarch-step.h" -#include "unique-c-ptr.h" -#include "virtio-console.h" -#include "virtio-device.h" -#include "virtio-factory.h" -#include "virtio-net-carrier-slirp.h" -#include "virtio-net-carrier-tuntap.h" -#include "virtio-net.h" -#include "virtio-p9fs.h" +#include "variant-hasher.h" +#include "virtio-address-range.h" /// \file /// \brief Cartesi machine implementation namespace cartesi { -using namespace std::string_literals; - -const pma_entry::flags machine::m_ram_flags{.R = true, - .W = true, - .X = true, - .IR = true, - .IW = true, - .DID = PMA_ISTART_DID::memory}; - -// When we pass a RNG seed in a FDT stored in DTB, -// Linux will wipe out its contents as a security measure, -// therefore we need to make DTB writable, otherwise boot will hang. -const pma_entry::flags machine::m_dtb_flags{.R = true, - .W = true, - .X = true, - .IR = true, - .IW = true, - .DID = PMA_ISTART_DID::memory}; - -const pma_entry::flags machine::m_flash_drive_flags{.R = true, - .W = true, - .X = false, - .IR = true, - .IW = true, - .DID = PMA_ISTART_DID::flash_drive}; - -const pma_entry::flags machine::m_cmio_rx_buffer_flags{.R = true, - .W = false, - .X = false, - .IR = true, - .IW = true, - .DID = PMA_ISTART_DID::cmio_rx_buffer}; - -const pma_entry::flags machine::m_cmio_tx_buffer_flags{.R = true, - .W = true, - .X = false, - .IR = true, - .IW = true, - .DID = PMA_ISTART_DID::cmio_tx_buffer}; - -pma_entry machine::make_memory_range_pma_entry(const std::string &description, const memory_range_config &c) { - if (c.image_filename.empty()) { - return make_callocd_memory_pma_entry(description, c.start, c.length); - } - return make_mmapd_memory_pma_entry(description, c.start, c.length, c.image_filename, c.shared); -} - -pma_entry machine::make_flash_drive_pma_entry(const std::string &description, const memory_range_config &c) { - return make_memory_range_pma_entry(description, c).set_flags(m_flash_drive_flags); -} - -pma_entry machine::make_cmio_rx_buffer_pma_entry(const cmio_config &c) { - const auto description = "cmio rx buffer memory range"s; - if (!c.rx_buffer.image_filename.empty()) { - return make_mmapd_memory_pma_entry(description, PMA_CMIO_RX_BUFFER_START, PMA_CMIO_RX_BUFFER_LENGTH, - c.rx_buffer.image_filename, c.rx_buffer.shared) - .set_flags(m_cmio_rx_buffer_flags); - } - return make_callocd_memory_pma_entry(description, PMA_CMIO_RX_BUFFER_START, PMA_CMIO_RX_BUFFER_LENGTH) - .set_flags(m_cmio_rx_buffer_flags); -} +static_assert(static_cast(AR_LOG2_PAGE_SIZE) == static_cast(LOG2_PAGE_SIZE), + "address range page size must match RISC-V page size"); +static_assert(static_cast(AR_LOG2_PAGE_SIZE) == static_cast(HASH_TREE_LOG2_PAGE_SIZE), + "address range page size must match hash-tree page size"); -pma_entry machine::make_cmio_tx_buffer_pma_entry(const cmio_config &c) { - const auto description = "cmio tx buffer memory range"s; - if (!c.tx_buffer.image_filename.empty()) { - return make_mmapd_memory_pma_entry(description, PMA_CMIO_TX_BUFFER_START, PMA_CMIO_TX_BUFFER_LENGTH, - c.tx_buffer.image_filename, c.tx_buffer.shared) - .set_flags(m_cmio_tx_buffer_flags); - } - return make_callocd_memory_pma_entry(description, PMA_CMIO_TX_BUFFER_START, PMA_CMIO_TX_BUFFER_LENGTH) - .set_flags(m_cmio_tx_buffer_flags); -} +using namespace std::string_literals; -pma_entry &machine::register_pma_entry(pma_entry &&pma) { - if (decltype(m_s.pmas)::capacity() <= m_s.pmas.size()) { - throw std::runtime_error{"too many PMAs when adding "s + pma.get_description()}; - } - auto start = pma.get_start(); - if ((start & (PMA_PAGE_SIZE - 1)) != 0) { - throw std::invalid_argument{"start of "s + pma.get_description() + " ("s + std::to_string(start) + - ") must be aligned to page boundary of "s + std::to_string(PMA_PAGE_SIZE) + " bytes"s}; - } - auto length = pma.get_length(); - if ((length & (PMA_PAGE_SIZE - 1)) != 0) { - throw std::invalid_argument{"length of "s + pma.get_description() + " ("s + std::to_string(length) + - ") must be multiple of page size "s + std::to_string(PMA_PAGE_SIZE)}; - } - // Check PMA range, when not the sentinel PMA entry - if (length != 0 || start != 0) { - if (length == 0) { - throw std::invalid_argument{"length of "s + pma.get_description() + " cannot be zero"s}; - } - // Checks if PMA is in addressable range, safe unsigned overflows - if (start > PMA_ADDRESSABLE_MASK || (length - 1) > (PMA_ADDRESSABLE_MASK - start)) { - throw std::invalid_argument{ - "range of "s + pma.get_description() + " must use at most 56 bits to be addressable"s}; - } - } - // Range A overlaps with B if A starts before B ends and A ends after B starts - for (const auto &existing_pma : m_s.pmas) { - if (start < existing_pma.get_start() + existing_pma.get_length() && start + length > existing_pma.get_start()) { - throw std::invalid_argument{"range of "s + pma.get_description() + " overlaps with range of existing "s + - existing_pma.get_description()}; +void machine::init_uarch_processor(const uarch_processor_config &p) { + if (p.backing_store.newly_created()) { + // Initialize to default values first + *m_us = uarch_processor_state{}; + + // Initialize registers + write_reg(reg::uarch_pc, p.registers.pc); + write_reg(reg::uarch_cycle, p.registers.cycle); + write_reg(reg::uarch_halt_flag, p.registers.halt_flag); + // General purpose registers + for (int i = 1; i < UARCH_X_REG_COUNT; i++) { + write_reg(machine_reg_enum(reg::uarch_x0, i), p.registers.x[i]); } } - pma.set_index(static_cast(m_s.pmas.size())); - m_s.pmas.push_back(std::move(pma)); - return m_s.pmas.back(); -} -static bool DID_is_protected(PMA_ISTART_DID DID) { - switch (DID) { - case PMA_ISTART_DID::flash_drive: - case PMA_ISTART_DID::cmio_rx_buffer: - case PMA_ISTART_DID::cmio_tx_buffer: - return false; - default: - return true; + // Ensure uarch x0 is actually zero + if (m_us->registers.x[0] != 0) { + throw std::invalid_argument{"uarch x0 register is corrupt"}; } } -void machine::replace_memory_range(const memory_range_config &range) { - for (auto &pma : m_s.pmas) { - if (pma.get_start() == range.start && pma.get_length() == range.length) { - const auto curr = pma.get_istart_DID(); - if (DID_is_protected(curr)) { - throw std::invalid_argument{"attempt to replace a protected range "s + pma.get_description()}; - } - // replace range preserving original flags - pma = make_memory_range_pma_entry(pma.get_description(), range).set_flags(pma.get_flags()); - return; - } - } - throw std::invalid_argument{"attempt to replace inexistent memory range"}; -} +void machine::init_processor(const processor_config &p, const machine_runtime_config &r) { + if (p.backing_store.newly_created()) { + // Initialize to default values first + *m_s = processor_state{}; -template -static void load_tlb_entry(machine &m, uint64_t eidx, unsigned char *hmem) { - tlb_hot_entry &tlbhe = m.get_state().tlb.hot[ETYPE][eidx]; - tlb_cold_entry &tlbce = m.get_state().tlb.cold[ETYPE][eidx]; - auto vaddr_page = aliased_aligned_read(hmem + tlb_get_vaddr_page_rel_addr(eidx)); - auto paddr_page = aliased_aligned_read(hmem + tlb_get_paddr_page_rel_addr(eidx)); - auto pma_index = aliased_aligned_read(hmem + tlb_get_pma_index_rel_addr(eidx)); - if (vaddr_page != TLB_INVALID_PAGE) { - if ((vaddr_page & ~PAGE_OFFSET_MASK) != vaddr_page) { - throw std::invalid_argument{"misaligned virtual page address in TLB entry"}; - } - if ((paddr_page & ~PAGE_OFFSET_MASK) != paddr_page) { - throw std::invalid_argument{"misaligned physical page address in TLB entry"}; + // General purpose registers + for (int i = 1; i < X_REG_COUNT; i++) { + write_reg(machine_reg_enum(reg::x0, i), p.registers.x[i]); } - const pma_entry &pma = m.find_pma_entry(paddr_page); - // Checks if the PMA still valid - if (pma.get_length() == 0 || !pma.get_istart_M() || pma_index >= m.get_state().pmas.size() || - &pma != &m.get_state().pmas[pma_index]) { - throw std::invalid_argument{"invalid PMA for TLB entry"}; - } - const unsigned char *hpage = pma.get_memory().get_host_memory() + (paddr_page - pma.get_start()); - // Valid TLB entry - tlbhe.vaddr_page = vaddr_page; - tlbhe.vh_offset = cast_ptr_to_addr(hpage) - vaddr_page; - tlbce.paddr_page = paddr_page; - tlbce.pma_index = pma_index; - } else { // Empty or invalidated TLB entry - tlbhe.vaddr_page = vaddr_page; - tlbhe.vh_offset = 0; - tlbce.paddr_page = paddr_page; - tlbce.pma_index = pma_index; - } -} -template -static void init_tlb_entry(machine &m, uint64_t eidx) { - tlb_hot_entry &tlbhe = m.get_state().tlb.hot[ETYPE][eidx]; - tlb_cold_entry &tlbce = m.get_state().tlb.cold[ETYPE][eidx]; - tlbhe.vaddr_page = TLB_INVALID_PAGE; - tlbhe.vh_offset = 0; - tlbce.paddr_page = TLB_INVALID_PAGE; - tlbce.pma_index = TLB_INVALID_PMA; -} - -machine::machine(const machine_config &c, const machine_runtime_config &r) : m_c{c}, m_uarch{c.uarch}, m_r{r} { - - if (m_c.processor.marchid == UINT64_C(-1)) { - m_c.processor.marchid = MARCHID_INIT; - } - - if (m_c.processor.marchid != MARCHID_INIT && !r.skip_version_check) { - throw std::invalid_argument{"marchid mismatch, emulator version is incompatible"}; - } - - if (m_c.processor.mvendorid == UINT64_C(-1)) { - m_c.processor.mvendorid = MVENDORID_INIT; - } - - if (m_c.processor.mvendorid != MVENDORID_INIT && !r.skip_version_check) { - throw std::invalid_argument{"mvendorid mismatch, emulator version is incompatible"}; - } - - if (m_c.processor.mimpid == UINT64_C(-1)) { - m_c.processor.mimpid = MIMPID_INIT; - } - - if (m_c.processor.mimpid != MIMPID_INIT && !r.skip_version_check) { - throw std::invalid_argument{"mimpid mismatch, emulator version is incompatible"}; - } - - m_s.soft_yield = r.soft_yield; - - // General purpose registers - for (int i = 1; i < X_REG_COUNT; i++) { - write_reg(machine_reg_enum(reg::x0, i), m_c.processor.x[i]); - } - - // Floating-point registers - for (int i = 0; i < F_REG_COUNT; i++) { - write_reg(machine_reg_enum(reg::f0, i), m_c.processor.f[i]); - } - - // Named registers - write_reg(reg::pc, m_c.processor.pc); - write_reg(reg::fcsr, m_c.processor.fcsr); - write_reg(reg::mcycle, m_c.processor.mcycle); - write_reg(reg::icycleinstret, m_c.processor.icycleinstret); - write_reg(reg::mstatus, m_c.processor.mstatus); - write_reg(reg::mtvec, m_c.processor.mtvec); - write_reg(reg::mscratch, m_c.processor.mscratch); - write_reg(reg::mepc, m_c.processor.mepc); - write_reg(reg::mcause, m_c.processor.mcause); - write_reg(reg::mtval, m_c.processor.mtval); - write_reg(reg::misa, m_c.processor.misa); - write_reg(reg::mie, m_c.processor.mie); - write_reg(reg::mip, m_c.processor.mip); - write_reg(reg::medeleg, m_c.processor.medeleg); - write_reg(reg::mideleg, m_c.processor.mideleg); - write_reg(reg::mcounteren, m_c.processor.mcounteren); - write_reg(reg::menvcfg, m_c.processor.menvcfg); - write_reg(reg::stvec, m_c.processor.stvec); - write_reg(reg::sscratch, m_c.processor.sscratch); - write_reg(reg::sepc, m_c.processor.sepc); - write_reg(reg::scause, m_c.processor.scause); - write_reg(reg::stval, m_c.processor.stval); - write_reg(reg::satp, m_c.processor.satp); - write_reg(reg::scounteren, m_c.processor.scounteren); - write_reg(reg::senvcfg, m_c.processor.senvcfg); - write_reg(reg::ilrsc, m_c.processor.ilrsc); - write_reg(reg::iprv, m_c.processor.iprv); - write_reg(reg::iflags_X, m_c.processor.iflags_X); - write_reg(reg::iflags_Y, m_c.processor.iflags_Y); - write_reg(reg::iflags_H, m_c.processor.iflags_H); - write_reg(reg::iunrep, m_c.processor.iunrep); - - // Register RAM - if (m_c.ram.image_filename.empty()) { - register_pma_entry(make_callocd_memory_pma_entry("RAM"s, PMA_RAM_START, m_c.ram.length).set_flags(m_ram_flags)); - } else { - register_pma_entry(make_callocd_memory_pma_entry("RAM"s, PMA_RAM_START, m_c.ram.length, m_c.ram.image_filename) - .set_flags(m_ram_flags)); - } - - // Register DTB - pma_entry &dtb = register_pma_entry((m_c.dtb.image_filename.empty() ? - make_callocd_memory_pma_entry("DTB"s, PMA_DTB_START, PMA_DTB_LENGTH) : - make_callocd_memory_pma_entry("DTB"s, PMA_DTB_START, PMA_DTB_LENGTH, m_c.dtb.image_filename)) - .set_flags(m_dtb_flags)); - - // Register all flash drives - int i = 0; // NOLINT(misc-const-correctness) - for (auto &f : m_c.flash_drive) { - const std::string flash_description = "flash drive "s + std::to_string(i); - // Auto detect flash drive start address - if (f.start == UINT64_C(-1)) { - f.start = PMA_DRIVE_START + PMA_DRIVE_OFFSET_DEF * i; - } - // Auto detect flash drive image length - if (f.length == UINT64_C(-1)) { - auto fp = unique_fopen(f.image_filename.c_str(), "rb"); - if (fseek(fp.get(), 0, SEEK_END) != 0) { - throw std::system_error{errno, std::generic_category(), - "unable to obtain length of image file '"s + f.image_filename + "' when initializing "s + - flash_description}; - } - const auto length = ftell(fp.get()); - if (length < 0) { - throw std::system_error{errno, std::generic_category(), - "unable to obtain length of image file '"s + f.image_filename + "' when initializing "s + - flash_description}; - } - f.length = length; + // Floating-point registers + for (int i = 0; i < F_REG_COUNT; i++) { + write_reg(machine_reg_enum(reg::f0, i), p.registers.f[i]); } - register_pma_entry(make_flash_drive_pma_entry(flash_description, f)); - i++; - } - - // Register cmio memory ranges - register_pma_entry(make_cmio_tx_buffer_pma_entry(m_c.cmio)); - register_pma_entry(make_cmio_rx_buffer_pma_entry(m_c.cmio)); - - // Register HTIF device - register_pma_entry(make_htif_pma_entry(PMA_HTIF_START, PMA_HTIF_LENGTH)); - - // Copy HTIF state to from config to machine - write_reg(reg::htif_tohost, m_c.htif.tohost); - write_reg(reg::htif_fromhost, m_c.htif.fromhost); - // Only command in halt device is command 0 and it is always available - const uint64_t htif_ihalt = static_cast(true) << HTIF_HALT_CMD_HALT; - write_reg(reg::htif_ihalt, htif_ihalt); - const uint64_t htif_iconsole = static_cast(m_c.htif.console_getchar) << HTIF_CONSOLE_CMD_GETCHAR | - static_cast(true) << HTIF_CONSOLE_CMD_PUTCHAR; - write_reg(reg::htif_iconsole, htif_iconsole); - const uint64_t htif_iyield = static_cast(m_c.htif.yield_manual) << HTIF_YIELD_CMD_MANUAL | - static_cast(m_c.htif.yield_automatic) << HTIF_YIELD_CMD_AUTOMATIC; - write_reg(reg::htif_iyield, htif_iyield); - - // Register CLINT device - register_pma_entry(make_clint_pma_entry(PMA_CLINT_START, PMA_CLINT_LENGTH)); - // Copy CLINT state to from config to machine - write_reg(reg::clint_mtimecmp, m_c.clint.mtimecmp); - - // Register PLIC device - register_pma_entry(make_plic_pma_entry(PMA_PLIC_START, PMA_PLIC_LENGTH)); - // Copy PLIC state from config to machine - write_reg(reg::plic_girqpend, m_c.plic.girqpend); - write_reg(reg::plic_girqsrvd, m_c.plic.girqsrvd); - - // Register TLB device - register_pma_entry(make_shadow_tlb_pma_entry(PMA_SHADOW_TLB_START, PMA_SHADOW_TLB_LENGTH)); - - // Register state shadow device - register_pma_entry(make_shadow_state_pma_entry(PMA_SHADOW_STATE_START, PMA_SHADOW_STATE_LENGTH)); - - // Register memory range that holds shadow PMA state, keep pointer to populate later - shadow_pmas_state *shadow_pmas = nullptr; - { - auto shadow_pmas_pma_entry = make_shadow_pmas_pma_entry(PMA_SHADOW_PMAS_START, PMA_SHADOW_PMAS_LENGTH); - // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) - shadow_pmas = reinterpret_cast(shadow_pmas_pma_entry.get_memory().get_host_memory()); - register_pma_entry(std::move(shadow_pmas_pma_entry)); - } - // Initialize VirtIO devices - if (!m_c.virtio.empty()) { - // VirtIO devices are disallowed in unreproducible mode - if (m_c.processor.iunrep == 0) { - throw std::invalid_argument{"virtio devices are only supported in unreproducible machines"}; + // Named registers + write_reg(reg::pc, p.registers.pc); + write_reg(reg::fcsr, p.registers.fcsr); + write_reg(reg::mcycle, p.registers.mcycle); + write_reg(reg::icycleinstret, p.registers.icycleinstret); + write_reg(reg::mstatus, p.registers.mstatus); + write_reg(reg::mtvec, p.registers.mtvec); + write_reg(reg::mscratch, p.registers.mscratch); + write_reg(reg::mepc, p.registers.mepc); + write_reg(reg::mcause, p.registers.mcause); + write_reg(reg::mtval, p.registers.mtval); + write_reg(reg::misa, p.registers.misa); + write_reg(reg::mie, p.registers.mie); + write_reg(reg::mip, p.registers.mip); + write_reg(reg::medeleg, p.registers.medeleg); + write_reg(reg::mideleg, p.registers.mideleg); + write_reg(reg::mcounteren, p.registers.mcounteren); + write_reg(reg::menvcfg, p.registers.menvcfg); + write_reg(reg::stvec, p.registers.stvec); + write_reg(reg::sscratch, p.registers.sscratch); + write_reg(reg::sepc, p.registers.sepc); + write_reg(reg::scause, p.registers.scause); + write_reg(reg::stval, p.registers.stval); + write_reg(reg::satp, p.registers.satp); + write_reg(reg::scounteren, p.registers.scounteren); + write_reg(reg::senvcfg, p.registers.senvcfg); + write_reg(reg::ilrsc, p.registers.ilrsc); + write_reg(reg::iprv, p.registers.iprv); + write_reg(reg::iflags_X, p.registers.iflags.X); + write_reg(reg::iflags_Y, p.registers.iflags.Y); + write_reg(reg::iflags_H, p.registers.iflags.H); + write_reg(reg::iunrep, p.registers.iunrep); + + // HTIF registers + write_reg(reg::htif_tohost, p.registers.htif.tohost); + write_reg(reg::htif_fromhost, p.registers.htif.fromhost); + write_reg(reg::htif_ihalt, p.registers.htif.ihalt); + write_reg(reg::htif_iconsole, p.registers.htif.iconsole); + write_reg(reg::htif_iyield, p.registers.htif.iyield); + + // CLINT registers + write_reg(reg::clint_mtimecmp, p.registers.clint.mtimecmp); + + // PLIC registers + write_reg(reg::plic_girqpend, p.registers.plic.girqpend); + write_reg(reg::plic_girqsrvd, p.registers.plic.girqsrvd); + } + + // Check if registers are consistent + validate_processor_shadow(r.skip_version_check); +} + +void machine::validate_processor_shadow(bool skip_version_check) const { + // Ensure emulator version is compatible + if (!skip_version_check) { + if (m_s->shadow.registers.marchid != MARCHID_INIT) { + throw std::invalid_argument{"marchid mismatch, emulator version is incompatible"}; } - - for (const auto &vdev_config_entry : m_c.virtio) { - std::visit( - [&](const auto &vdev_config) { - using T = std::decay_t; - std::string pma_name = "VirtIO device"; // NOLINT(misc-const-correctness): // no, can't be const - std::unique_ptr vdev; - if constexpr (std::is_same_v) { - pma_name = "VirtIO Console"; - vdev = std::make_unique(m_vdevs.size()); - } else if constexpr (std::is_same_v) { -#ifdef HAVE_POSIX_FS - pma_name = "VirtIO 9P"; - vdev = std::make_unique(m_vdevs.size(), vdev_config.tag, - vdev_config.host_directory); -#else - throw std::invalid_argument("virtio 9p device is unsupported in this platform"); -#endif - } else if constexpr (std::is_same_v) { -#ifdef HAVE_SLIRP - pma_name = "VirtIO Net User"; - vdev = std::make_unique(m_vdevs.size(), - std::make_unique(vdev_config)); -#else - throw std::invalid_argument("virtio network user device is unsupported in this platform"); - -#endif - } else if constexpr (std::is_same_v) { -#ifdef HAVE_TUNTAP - pma_name = "VirtIO Net TUN/TAP"; - vdev = std::make_unique(m_vdevs.size(), - std::make_unique(vdev_config.iface)); -#else - - throw std::invalid_argument("virtio network TUN/TAP device is unsupported in this platform"); -#endif - } else { - throw std::invalid_argument("invalid virtio device configuration"); - } - register_pma_entry( - make_virtio_pma_entry(PMA_FIRST_VIRTIO_START + (vdev->get_virtio_index() * PMA_VIRTIO_LENGTH), - PMA_VIRTIO_LENGTH, pma_name, &virtio_driver, vdev.get())); - m_vdevs.push_back(std::move(vdev)); - }, - vdev_config_entry); + if (m_s->shadow.registers.mvendorid != MVENDORID_INIT) { + throw std::invalid_argument{"mvendorid mismatch, emulator version is incompatible"}; + } + if (m_s->shadow.registers.mimpid != MIMPID_INIT) { + throw std::invalid_argument{"mimpid mismatch, emulator version is incompatible"}; } } - - // Initialize DTB - if (m_c.dtb.image_filename.empty()) { - // Write the FDT (flattened device tree) into DTB - dtb_init(m_c, dtb.get_memory().get_host_memory(), PMA_DTB_LENGTH); + // Ensure x0 is actually zero + if (m_s->shadow.registers.x[0] != 0) { + throw std::invalid_argument{"x0 register is corrupt"}; } - - // Include machine PMAs in set considered by the Merkle tree. - for (auto &pma : m_s.pmas) { - m_merkle_pmas.push_back(&pma); + // Ensure padding bytes are consistent + if (!is_pristine(std::span{// NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + reinterpret_cast(&m_s->shadow.registers_padding_[0]), + sizeof(m_s->shadow.registers_padding_)}) || + !is_pristine(std::span{// NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + reinterpret_cast(&m_s->shadow.tlb_padding_[0]), sizeof(m_s->shadow.tlb_padding_)})) { + throw std::invalid_argument{"shadow padding bytes are corrupt"}; } +} - // Last, add empty sentinels until we reach capacity (need at least one sentinel) - register_pma_entry(make_empty_pma_entry("sentinel"s, 0, 0)); - - // NOLINTNEXTLINE(readability-static-accessed-through-instance) - if (m_s.pmas.capacity() != PMA_MAX) { - throw std::logic_error{"PMAs array must be able to hold PMA_MAX entries"}; - } - while (m_s.pmas.size() < PMA_MAX) { - register_pma_entry(make_empty_pma_entry("sentinel"s, 0, 0)); +void machine::init_pmas_contents(const pmas_config &config) { + static_assert(sizeof(pmas_state) == PMA_MAX * 2 * sizeof(uint64_t), "inconsistent PMAs state length"); + static_assert(AR_PMAS_LENGTH >= sizeof(pmas_state), "PMAs address range too short"); + auto &pmas = m_ars.find(AR_PMAS_START, AR_PMAS_LENGTH); + if (!pmas.is_memory()) { + throw std::runtime_error{"initialization error: PMAs memory address range not found"}; } + pmas_state pmas_state{}; + std::ranges::transform(m_ars.pmas_view(), pmas_state.begin(), + [](const auto &ar) { return pmas_entry{.istart = ar.get_istart(), .ilength = ar.get_ilength()}; }); - // Populate shadow PMAs - populate_shadow_pmas_state(m_s.pmas, shadow_pmas); - - // Include uarch PMAs in set considered by Merkle tree - m_merkle_pmas.push_back(&m_uarch.get_state().shadow_state); - m_merkle_pmas.push_back(&m_uarch.get_state().ram); - // Last, add sentinel PMA - m_merkle_pmas.push_back(&m_s.empty_pma); - - // Initialize TLB device - // this must be done after all PMA entries are already registered, so we can lookup page addresses - if (!m_c.tlb.image_filename.empty()) { - // Create a temporary PMA entry just to load TLB contents from an image file - pma_entry tlb_image_pma = make_mmapd_memory_pma_entry("shadow TLB device"s, PMA_SHADOW_TLB_START, - PMA_SHADOW_TLB_LENGTH, m_c.tlb.image_filename, false); - unsigned char *hmem = tlb_image_pma.get_memory().get_host_memory(); - for (uint64_t i = 0; i < PMA_TLB_SIZE; ++i) { - load_tlb_entry(*this, i, hmem); - load_tlb_entry(*this, i, hmem); - load_tlb_entry(*this, i, hmem); - } + if (config.backing_store.newly_created()) { + memcpy(pmas.get_host_memory(), pmas_state.data(), sizeof(pmas_state)); } else { - for (uint64_t i = 0; i < PMA_TLB_SIZE; ++i) { - init_tlb_entry(*this, i); - init_tlb_entry(*this, i); - init_tlb_entry(*this, i); + // Validate PMAs state is consistent + if (memcmp(pmas.get_host_memory(), pmas_state.data(), sizeof(pmas_state)) != 0) { + throw std::runtime_error{"PMA state is inconsistent machine address ranges"}; } } +} - // Initialize TTY if console input is enabled - if (m_c.htif.console_getchar || has_virtio_console()) { - if (m_c.processor.iunrep == 0) { - throw std::invalid_argument{"TTY stdin is only supported in unreproducible machines"}; +void machine::init_hot_tlb_contents() { + for (auto set_index : {TLB_CODE, TLB_READ, TLB_WRITE}) { + for (uint64_t slot_index = 0; slot_index < TLB_SET_SIZE; ++slot_index) { + const auto &shadow_slot = m_s->shadow.tlb[set_index][slot_index]; + const auto vaddr_page = shadow_slot.vaddr_page; + const auto vp_offset = shadow_slot.vp_offset; + const auto pma_index = shadow_slot.pma_index; + const auto zero_padding_ = shadow_slot.zero_padding_; + host_addr vh_offset{}; + if (zero_padding_ != 0) { + throw std::domain_error{"stored TLB is corrupt: inconsistent padding"}; + } + if (vaddr_page != TLB_INVALID_PAGE) { + const auto &ar = read_pma(pma_index); + if (!ar.is_memory()) { + throw std::invalid_argument{"stored TLB is corrupt: pma_index does not point to memory range"s}; + } + if ((vaddr_page & PAGE_OFFSET_MASK) != 0) { + throw std::invalid_argument{"stored TLB is corrupt: vaddr_page is not aligned"s}; + } + const auto paddr_page = vaddr_page + vp_offset; + if ((paddr_page & PAGE_OFFSET_MASK) != 0) { + throw std::invalid_argument{"stored TLB is corrupt: vp_offset is not aligned"s}; + } + const auto pmas_end = ar.get_start() + (ar.get_length() - AR_PAGE_SIZE); + if (paddr_page < ar.get_start() || paddr_page > pmas_end) { + throw std::invalid_argument{"stored TLB is corrupt: vp_offset is inconsistent with pma_index"s}; + } + vh_offset = get_host_addr(paddr_page, pma_index) - vaddr_page; + } else if (pma_index != TLB_INVALID_PMA_INDEX || vp_offset != 0) { + throw std::domain_error{"stored TLB is corrupt: inconsistent empty slot"}; + } + auto &hot_slot = m_s->penumbra.tlb[set_index][slot_index]; + hot_slot.vaddr_page = vaddr_page; + hot_slot.vh_offset = vh_offset; } - os_open_tty(); } - os_silence_putchar(m_r.htif.no_console_putchar); +} - // Initialize memory range descriptions returned by get_memory_ranges method - for (const auto *pma : m_merkle_pmas) { - if (pma->get_length() != 0) { - m_mrds.push_back(machine_memory_range_descr{.start = pma->get_start(), - .length = pma->get_length(), - .description = pma->get_description()}); +void machine::init_dtb_contents(const machine_config &config) { + if (config.dtb.backing_store.newly_created()) { + auto &dtb = m_ars.find(AR_DTB_START, AR_DTB_LENGTH); + if (!dtb.is_memory()) { + throw std::runtime_error{"initialization error: DTB memory address range not found"}; } - } - // Sort it by increasing start address - std::sort(m_mrds.begin(), m_mrds.end(), - [](const machine_memory_range_descr &a, const machine_memory_range_descr &b) { return a.start < b.start; }); - + dtb_init(config, dtb.get_host_memory(), dtb.get_length()); + } +} + +// ??D It is best to leave the std::move() on r because it may one day be necessary! +machine::machine(machine_config c, machine_runtime_config r, const std::string &dir, scope_remove remover) : + m_c{std::move(c).adjust_defaults()}, // NOLINT(hicpp-move-const-arg,performance-move-const-arg) + m_r{std::move(r)}, // NOLINT(hicpp-move-const-arg,performance-move-const-arg) + m_ars{m_c, m_r, dir, remover}, + m_ht{m_c.hash_tree, m_r.concurrency.update_hash_tree, m_ars, dir, remover}, + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + m_s{reinterpret_cast( + m_ars.find(AR_SHADOW_STATE_START, AR_SHADOW_STATE_LENGTH).get_host_memory())}, + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + m_us{reinterpret_cast( + m_ars.find(AR_SHADOW_UARCH_STATE_START, AR_SHADOW_UARCH_STATE_LENGTH).get_host_memory())} { + init_processor(m_c.processor, m_r); + init_uarch_processor(m_c.uarch.processor); + init_pmas_contents(m_c.pmas); + init_hot_tlb_contents(); + init_dtb_contents(m_c); + init_tty(); // Disable SIGPIPE handler, because this signal can be raised and terminate the emulator process // when calling write() on closed file descriptors. // This can happen with the stdout console file descriptors or network file descriptors. os_disable_sigpipe(); + // Construction succeeded, keep all created files and directories + remover.retain_all(); } -static void load_hash(const std::string &dir, machine::hash_type &h) { - auto name = dir + "/hash"; - auto fp = unique_fopen(name.c_str(), "rb"); - if (fread(h.data(), 1, h.size(), fp.get()) != h.size()) { - throw std::runtime_error{"error reading from '" + name + "'"}; +void machine::init_tty() { + // Initialize TTY if console input is enabled + if (has_htif_console() || has_virtio_console()) { + if (read_reg(reg::iunrep) == 0) { + throw std::invalid_argument{"TTY stdin is only supported in unreproducible machines"}; + } + os_open_tty(); + m_tty_opened = true; } } -machine::machine(const std::string &dir, const machine_runtime_config &r) : machine{machine_config::load(dir), r} { - if (r.skip_root_hash_check) { - return; - } - hash_type hstored; - hash_type hrestored; - load_hash(dir, hstored); - if (!update_merkle_tree()) { - throw std::runtime_error{"error updating Merkle tree"}; - } - m_t.get_root_hash(hrestored); - if (hstored != hrestored) { - throw std::runtime_error{"stored and restored hashes do not match"}; - } -} +// ??D It is best to leave the std::move() on r because it may one day be necessary! +machine::machine(const std::string &dir, machine_runtime_config r, sharing_mode sharing) : + // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) + machine{machine_config::load(dir, sharing), std::move(r)} {} void machine::prepare_virtio_devices_select(select_fd_sets *fds, uint64_t *timeout_us) { - for (auto &vdev : m_vdevs) { - vdev->prepare_select(fds, timeout_us); + for (auto &v : m_ars.virtio_view()) { + v.prepare_select(fds, timeout_us); } } // NOLINTNEXTLINE(readability-convert-member-functions-to-static) bool machine::poll_selected_virtio_devices(int select_ret, select_fd_sets *fds, i_device_state_access *da) { bool interrupt_requested = false; // NOLINT(misc-const-correctness) - for (auto &vdev : m_vdevs) { - interrupt_requested |= vdev->poll_selected(select_ret, fds, da); + for (auto &v : m_ars.virtio_view()) { + interrupt_requested |= v.poll_selected(select_ret, fds, da); } return interrupt_requested; } @@ -606,17 +353,17 @@ bool machine::poll_virtio_devices(uint64_t *timeout_us, i_device_state_access *d } bool machine::has_virtio_devices() const { - return !m_vdevs.empty(); + return !m_ars.virtio_view().empty(); } bool machine::has_virtio_console() const { // When present, the console device is guaranteed to be the first VirtIO device, // therefore we only need to check the first device. - return !m_vdevs.empty() && m_vdevs[0]->get_device_id() == VIRTIO_DEVICE_CONSOLE; + return has_virtio_devices() && m_ars.virtio_view().front().get_device_id() == VIRTIO_DEVICE_CONSOLE; } bool machine::has_htif_console() const { - return static_cast(read_reg(reg::htif_iconsole) & (1 << HTIF_CONSOLE_CMD_GETCHAR)); + return static_cast(read_reg(reg::htif_iconsole) & HTIF_CONSOLE_CMD_GETCHAR_MASK); } /// \brief Returns copy of initialization config. @@ -630,405 +377,393 @@ const machine_runtime_config &machine::get_runtime_config() const { } /// \brief Changes the machine runtime config. -void machine::set_runtime_config(const machine_runtime_config &r) { - m_r = r; - m_s.soft_yield = m_r.soft_yield; - os_silence_putchar(r.htif.no_console_putchar); +void machine::set_runtime_config(machine_runtime_config r) { + m_r = std::move(r); // NOLINT(hicpp-move-const-arg,performance-move-const-arg) } -machine_config machine::get_serialization_config() const { +void machine::store(const std::string &dir, sharing_mode sharing) const { + if (dir.empty()) { + throw std::invalid_argument{"directory name cannot be empty"}; + } if (read_reg(reg::iunrep) != 0) { - throw std::runtime_error{"cannot serialize configuration of unreproducible machines"}; - } - // Initialize with copy of original config - machine_config c = m_c; - // Copy current processor state to config - for (int i = 1; i < X_REG_COUNT; ++i) { - c.processor.x[i] = read_reg(machine_reg_enum(reg::x0, i)); - } - for (int i = 0; i < F_REG_COUNT; ++i) { - c.processor.f[i] = read_reg(machine_reg_enum(reg::f0, i)); - } - c.processor.pc = read_reg(reg::pc); - c.processor.fcsr = read_reg(reg::fcsr); - c.processor.mvendorid = read_reg(reg::mvendorid); - c.processor.marchid = read_reg(reg::marchid); - c.processor.mimpid = read_reg(reg::mimpid); - c.processor.mcycle = read_reg(reg::mcycle); - c.processor.icycleinstret = read_reg(reg::icycleinstret); - c.processor.mstatus = read_reg(reg::mstatus); - c.processor.mtvec = read_reg(reg::mtvec); - c.processor.mscratch = read_reg(reg::mscratch); - c.processor.mepc = read_reg(reg::mepc); - c.processor.mcause = read_reg(reg::mcause); - c.processor.mtval = read_reg(reg::mtval); - c.processor.misa = read_reg(reg::misa); - c.processor.mie = read_reg(reg::mie); - c.processor.mip = read_reg(reg::mip); - c.processor.medeleg = read_reg(reg::medeleg); - c.processor.mideleg = read_reg(reg::mideleg); - c.processor.mcounteren = read_reg(reg::mcounteren); - c.processor.menvcfg = read_reg(reg::menvcfg); - c.processor.stvec = read_reg(reg::stvec); - c.processor.sscratch = read_reg(reg::sscratch); - c.processor.sepc = read_reg(reg::sepc); - c.processor.scause = read_reg(reg::scause); - c.processor.stval = read_reg(reg::stval); - c.processor.satp = read_reg(reg::satp); - c.processor.scounteren = read_reg(reg::scounteren); - c.processor.senvcfg = read_reg(reg::senvcfg); - c.processor.ilrsc = read_reg(reg::ilrsc); - c.processor.iprv = read_reg(reg::iprv); - c.processor.iflags_X = read_reg(reg::iflags_X); - c.processor.iflags_Y = read_reg(reg::iflags_Y); - c.processor.iflags_H = read_reg(reg::iflags_H); - c.processor.iunrep = read_reg(reg::iunrep); - // Copy current CLINT state to config - c.clint.mtimecmp = read_reg(reg::clint_mtimecmp); - // Copy current PLIC state to config - c.plic.girqpend = read_reg(reg::plic_girqpend); - c.plic.girqsrvd = read_reg(reg::plic_girqsrvd); - // Copy current HTIF state to config - c.htif.tohost = read_reg(reg::htif_tohost); - c.htif.fromhost = read_reg(reg::htif_fromhost); - // c.htif.halt = read_reg(reg::htif_ihalt); // hard-coded to true - c.htif.console_getchar = static_cast(read_reg(reg::htif_iconsole) & (1 << HTIF_CONSOLE_CMD_GETCHAR)); - c.htif.yield_manual = static_cast(read_reg(reg::htif_iyield) & (1 << HTIF_YIELD_CMD_MANUAL)); - c.htif.yield_automatic = static_cast(read_reg(reg::htif_iyield) & (1 << HTIF_YIELD_CMD_AUTOMATIC)); - // Ensure we don't mess with DTB by writing the original bootargs - // over the potentially modified memory region we serialize - c.dtb.bootargs.clear(); - // Remove image filenames from serialization - // (they will be ignored by save and load for security reasons) - c.dtb.image_filename.clear(); - c.ram.image_filename.clear(); - c.uarch.ram.image_filename.clear(); - c.tlb.image_filename.clear(); - for (auto &f : c.flash_drive) { - f.image_filename.clear(); - } - c.cmio.rx_buffer.image_filename.clear(); - c.cmio.tx_buffer.image_filename.clear(); - c.uarch.processor.cycle = read_reg(reg::uarch_cycle); - c.uarch.processor.halt_flag = (read_reg(reg::uarch_halt_flag) != 0); - c.uarch.processor.pc = read_reg(reg::uarch_pc); - for (int i = 1; i < UARCH_X_REG_COUNT; i++) { - c.uarch.processor.x[i] = read_reg(machine_reg_enum(reg::uarch_x0, i)); + throw std::runtime_error{"cannot store unreproducible machines"}; } - return c; -} + scope_remove remover; + + // Create directory + os::create_directory(dir); + remover.add_directory(dir); -static void store_device_pma(const machine &m, const pma_entry &pma, const std::string &dir) { - if (!pma.get_istart_IO()) { - throw std::runtime_error{"attempt to save non-device PMA"}; - } - auto scratch = unique_calloc(PMA_PAGE_SIZE); // will throw if it fails - auto name = machine_config::get_image_filename(dir, pma.get_start(), pma.get_length()); - auto fp = unique_fopen(name.c_str(), "wb"); - for (uint64_t page_start_in_range = 0; page_start_in_range < pma.get_length(); - page_start_in_range += PMA_PAGE_SIZE) { - const unsigned char *page_data = nullptr; - auto peek = pma.get_peek(); - if (!peek(pma, m, page_start_in_range, &page_data, scratch.get())) { - throw std::runtime_error{"peek failed"}; + // Store config + remover.add_file(m_c.store(dir)); + + // Store all address ranges + auto store_address_range = [&](const backing_store_config &c, const address_range &ar, bool read_only) { + if (ar.is_empty()) { + throw std::runtime_error{"attempt to store empty address range "s.append(ar.get_description())}; } - if (page_data == nullptr) { - memset(scratch.get(), 0, PMA_PAGE_SIZE); - page_data = scratch.get(); + if (!ar.is_memory() || ar.get_host_memory() == nullptr) { + throw std::runtime_error{"attempt to store non-memory address range "s.append(ar.get_description())}; } - if (fwrite(page_data, 1, PMA_PAGE_SIZE, fp.get()) != PMA_PAGE_SIZE) { - throw std::system_error{errno, std::generic_category(), "error writing to '" + name + "'"}; + // Write the memory data to a file + const std::string data_filename = machine_config::get_data_filename(dir, ar.get_start(), ar.get_length()); + const std::string dht_filename = machine_config::get_dht_filename(dir, ar.get_start(), ar.get_length()); + const std::string dpt_filename = machine_config::get_dpt_filename(dir, ar.get_start(), ar.get_length()); + if (sharing == sharing_mode::all || (sharing == sharing_mode::config && c.shared)) { + os::create_file(data_filename, std::span{ar.get_host_memory(), static_cast(ar.get_length())}); + remover.add_file(data_filename); + os::create_file(dht_filename, ar.get_dense_hash_tree().get_storage_data()); + remover.add_file(dht_filename); + os::create_file(dpt_filename, ar.get_dirty_page_tree().get_storage_data()); + remover.add_file(dpt_filename); + } else { // Copy unshared backing store + if (c.data_filename.empty()) { + throw std::runtime_error{"attempt to rollback unbacked address range "s.append(ar.get_description())}; + } + if (c.shared) { + throw std::runtime_error{"attempt to rollback shared address range "s.append(ar.get_description())}; + } + os::copy_file(c.data_filename, data_filename, ar.get_length()); + remover.add_file(data_filename); + if (!c.dht_filename.empty()) { + os::copy_file(c.dht_filename, dht_filename); + remover.add_file(dht_filename); + } + if (!c.dpt_filename.empty()) { + os::copy_file(c.dpt_filename, dpt_filename); + remover.add_file(dpt_filename); + } } - } -} - -static void store_memory_pma(const pma_entry &pma, const std::string &dir) { - if (!pma.get_istart_M()) { - throw std::runtime_error{"attempt to save non-memory PMA"}; - } - auto name = machine_config::get_image_filename(dir, pma.get_start(), pma.get_length()); - auto fp = unique_fopen(name.c_str(), "wb"); - const pma_memory &mem = pma.get_memory(); - if (fwrite(mem.get_host_memory(), 1, pma.get_length(), fp.get()) != pma.get_length()) { - throw std::runtime_error{"error writing to '" + name + "'"}; - } -} - -pma_entry &machine::find_pma_entry(uint64_t paddr, uint64_t length) { - // NOLINTNEXTLINE(cppcoreguidelines-pro-type-const-cast): remove const to reuse code - return const_cast(std::as_const(*this).find_pma_entry(paddr, length)); -} - -const pma_entry &machine::find_pma_entry(uint64_t paddr, uint64_t length) const { - return find_pma_entry(m_s.pmas, paddr, length); -} - -template -pma_entry &machine::find_pma_entry(const CONTAINER &pmas, uint64_t paddr, uint64_t length) { - // NOLINTNEXTLINE(cppcoreguidelines-pro-type-const-cast): remove const to reuse code - return const_cast(std::as_const(*this).find_pma_entry(pmas, paddr, length)); -} - -template -const pma_entry &machine::find_pma_entry(const CONTAINER &pmas, uint64_t paddr, uint64_t length) const { - for (const auto &p : pmas) { - const auto &pma = deref(p); - // Stop at first empty PMA - if (pma.get_length() == 0) { - return pma; + if (read_only) { + // Mark host read-only ranges as read-only + os::change_writable(data_filename, false); } - // Check if data is in range - if (paddr >= pma.get_start() && pma.get_length() >= length && - paddr - pma.get_start() <= pma.get_length() - length) { - return pma; + }; + store_address_range(m_c.processor.backing_store, m_ars.find(AR_SHADOW_STATE_START), false); + store_address_range(m_c.pmas.backing_store, m_ars.find(AR_PMAS_START), true); + store_address_range(m_c.dtb.backing_store, m_ars.find(AR_DTB_START), false); + store_address_range(m_c.ram.backing_store, m_ars.find(AR_RAM_START), false); + store_address_range(m_c.cmio.rx_buffer.backing_store, m_ars.find(AR_CMIO_RX_BUFFER_START), false); + store_address_range(m_c.cmio.tx_buffer.backing_store, m_ars.find(AR_CMIO_TX_BUFFER_START), false); + store_address_range(m_c.uarch.processor.backing_store, m_ars.find(AR_SHADOW_UARCH_STATE_START), false); + store_address_range(m_c.uarch.ram.backing_store, m_ars.find(AR_UARCH_RAM_START), false); + for (const auto &f : m_c.flash_drive) { + store_address_range(f.backing_store, m_ars.find(f.start), f.read_only); + } + + // Store hash tree + const std::string sht_filename = machine_config::get_sht_filename(dir); + const std::string phtc_filename = machine_config::get_phtc_filename(dir); + if (sharing == sharing_mode::all || (sharing == sharing_mode::config && m_c.hash_tree.shared)) { + os::create_file(sht_filename, m_ht.get_sht_storage_data()); + remover.add_file(sht_filename); + os::create_file(phtc_filename, m_ht.get_phtc_storage_data()); + remover.add_file(phtc_filename); + } else { + os::truncate_file(sht_filename, m_ht.get_sht_storage_data().size(), true); + remover.add_file(sht_filename); + os::truncate_file(phtc_filename, m_ht.get_phtc_storage_data().size(), true); + remover.add_file(phtc_filename); + } + + // Retain all stored files + remover.retain_all(); +} + +void machine::clone_stored(const std::string &from_dir, const std::string &to_dir) { + if (from_dir.empty() || to_dir.empty()) { + throw std::invalid_argument{"directory name cannot be empty"}; + } + const auto from_c = machine_config::load(from_dir); + const auto to_c = machine_config(from_c).adjust_backing_stores(to_dir); + + scope_remove remover; + + // Create directory + os::create_directory(to_dir); + remover.add_directory(to_dir); + + // Clone config + const auto from_config_filename = machine_config::get_config_filename(from_dir); + const auto to_config_filename = machine_config::get_config_filename(to_dir); + os::clone_file(from_config_filename, to_config_filename); + remover.add_file(to_config_filename); + + // Clone all address ranges + auto clone_ar_backing_store = [&](const backing_store_config &from_bs, const backing_store_config &to_bs) { + os::clone_file(from_bs.data_filename, to_bs.data_filename); + remover.add_file(to_bs.data_filename); + os::clone_file(from_bs.dht_filename, to_bs.dht_filename); + remover.add_file(to_bs.dht_filename); + os::clone_file(from_bs.dpt_filename, to_bs.dpt_filename); + remover.add_file(to_bs.dpt_filename); + }; + clone_ar_backing_store(from_c.processor.backing_store, to_c.processor.backing_store); + clone_ar_backing_store(from_c.pmas.backing_store, to_c.pmas.backing_store); + clone_ar_backing_store(from_c.dtb.backing_store, to_c.dtb.backing_store); + clone_ar_backing_store(from_c.ram.backing_store, to_c.ram.backing_store); + clone_ar_backing_store(from_c.cmio.rx_buffer.backing_store, to_c.cmio.rx_buffer.backing_store); + clone_ar_backing_store(from_c.cmio.tx_buffer.backing_store, to_c.cmio.tx_buffer.backing_store); + clone_ar_backing_store(from_c.uarch.processor.backing_store, to_c.uarch.processor.backing_store); + clone_ar_backing_store(from_c.uarch.ram.backing_store, to_c.uarch.ram.backing_store); + for (size_t i = 0; i < to_c.flash_drive.size(); ++i) { + clone_ar_backing_store(from_c.flash_drive[i].backing_store, to_c.flash_drive[i].backing_store); + } + + // Clone hash tree + os::clone_file(from_c.hash_tree.sht_filename, to_c.hash_tree.sht_filename); + remover.add_file(to_c.hash_tree.sht_filename); + os::clone_file(from_c.hash_tree.phtc_filename, to_c.hash_tree.phtc_filename); + remover.add_file(to_c.hash_tree.phtc_filename); + + // Retain all stored files + remover.retain_all(); +} + +void machine::remove_stored(const std::string &dir) { + if (dir.empty()) { + throw std::invalid_argument{"directory name cannot be empty"}; + } + const auto config = machine_config::load(dir); + + // Remove all address ranges + os::remove_file(config.processor.backing_store.data_filename); + os::remove_file(config.processor.backing_store.dht_filename); + os::remove_file(config.processor.backing_store.dpt_filename); + os::remove_file(config.pmas.backing_store.data_filename); + os::remove_file(config.pmas.backing_store.dht_filename); + os::remove_file(config.pmas.backing_store.dpt_filename); + os::remove_file(config.dtb.backing_store.data_filename); + os::remove_file(config.dtb.backing_store.dht_filename); + os::remove_file(config.dtb.backing_store.dpt_filename); + os::remove_file(config.ram.backing_store.data_filename); + os::remove_file(config.ram.backing_store.dht_filename); + os::remove_file(config.ram.backing_store.dpt_filename); + os::remove_file(config.cmio.rx_buffer.backing_store.data_filename); + os::remove_file(config.cmio.rx_buffer.backing_store.dht_filename); + os::remove_file(config.cmio.rx_buffer.backing_store.dpt_filename); + os::remove_file(config.cmio.tx_buffer.backing_store.data_filename); + os::remove_file(config.cmio.tx_buffer.backing_store.dht_filename); + os::remove_file(config.cmio.tx_buffer.backing_store.dpt_filename); + os::remove_file(config.uarch.processor.backing_store.data_filename); + os::remove_file(config.uarch.processor.backing_store.dht_filename); + os::remove_file(config.uarch.processor.backing_store.dpt_filename); + os::remove_file(config.uarch.ram.backing_store.data_filename); + os::remove_file(config.uarch.ram.backing_store.dht_filename); + os::remove_file(config.uarch.ram.backing_store.dpt_filename); + for (const auto &f : config.flash_drive) { + os::remove_file(f.backing_store.data_filename); + os::remove_file(f.backing_store.dht_filename); + os::remove_file(f.backing_store.dpt_filename); + } + + // Remove hash tree files + os::remove_file(config.hash_tree.sht_filename); + os::remove_file(config.hash_tree.phtc_filename); + + // Remove config + os::remove_file(machine_config::get_config_filename(dir)); + + // Remove directory + os::remove_directory(dir); +} + +void machine::dump_insn_hist() { +#ifdef DUMP_INSN_HIST + d_printf("\nInstruction Histogram:\n"); + for (const auto &[key, val] : m_counters) { + if (key.starts_with("insn.")) { + d_printf("%s: %" PRIu64 "\n", key.c_str(), val); } } - - // Last PMA is always the empty range - return deref(pmas.back()); -} - -template -static inline T &deref(T &t) { - return t; // NOLINT(bugprone-return-const-ref-from-parameter) -} - -template -static inline T &deref(T *t) { - return *t; -} - -void machine::store_pmas(const machine_config &c, const std::string &dir) const { - if (read_reg(reg::iunrep) != 0) { - throw std::runtime_error{"cannot store PMAs of unreproducible machines"}; - } - store_memory_pma(find_pma_entry(PMA_DTB_START), dir); - store_memory_pma(find_pma_entry(PMA_RAM_START), dir); - store_device_pma(*this, find_pma_entry(PMA_SHADOW_TLB_START), dir); - // Could iterate over PMAs checking for those with a drive DID - // but this is easier - for (const auto &f : c.flash_drive) { - store_memory_pma(find_pma_entry(f.start), dir); - } - store_memory_pma(find_pma_entry(PMA_CMIO_RX_BUFFER_START), dir); - store_memory_pma(find_pma_entry(PMA_CMIO_TX_BUFFER_START), dir); - if (!m_uarch.get_state().ram.get_istart_E()) { - store_memory_pma(m_uarch.get_state().ram, dir); - } -} - -static void store_hash(const machine::hash_type &h, const std::string &dir) { - auto name = dir + "/hash"; - auto fp = unique_fopen(name.c_str(), "wb"); - if (fwrite(h.data(), 1, h.size(), fp.get()) != h.size()) { - throw std::runtime_error{"error writing to '" + name + "'"}; - } +#endif } -void machine::store(const std::string &dir) const { - if (os_mkdir(dir.c_str(), 0700) != 0) { - throw std::system_error{errno, std::generic_category(), "error creating directory '"s + dir + "'"s}; - } - if (!m_r.skip_root_hash_store) { - if (!update_merkle_tree()) { - throw std::runtime_error{"error updating Merkle tree"}; - } - hash_type h; - m_t.get_root_hash(h); - store_hash(h, dir); - } - auto c = get_serialization_config(); - c.store(dir); - store_pmas(c, dir); +void machine::dump_stats() { +#ifdef DUMP_STATS + const auto hr = [](uint64_t a, uint64_t b) { return static_cast(b) / static_cast(a + b); }; + + d_printf("\nMachine Counters:\n"); + d_printf("inner loops: %" PRIu64 "\n", m_counters["stats.inner_loop"]); + d_printf("outers loops: %" PRIu64 "\n", m_counters["stats.outer_loop"]); + d_printf("supervisor ints: %" PRIu64 "\n", m_counters["stats.sv_int"]); + d_printf("supervisor ex: %" PRIu64 "\n", m_counters["stats.sv_ex"]); + d_printf("machine ints: %" PRIu64 "\n", m_counters["stats.m_int"]); + d_printf("machine ex: %" PRIu64 "\n", m_counters["stats.m_ex"]); + d_printf("atomic mem ops: %" PRIu64 "\n", m_counters["stats.atomic_mop"]); + d_printf("fence: %" PRIu64 "\n", m_counters["stats.fence"]); + d_printf("fence.i: %" PRIu64 "\n", m_counters["stats.fence_i"]); + d_printf("fence.vma: %" PRIu64 "\n", m_counters["stats.fence_vma"]); + d_printf("max asid: %" PRIu64 "\n", m_counters["stats.max_asid"]); + d_printf("User mode: %" PRIu64 "\n", m_counters["stats.prv.U"]); + d_printf("Supervisor mode: %" PRIu64 "\n", m_counters["stats.prv.S"]); + d_printf("Machine mode: %" PRIu64 "\n", m_counters["stats.prv.M"]); + d_printf("tlb code hit ratio: %.4f\n", hr(m_counters["stats.tlb.cmiss"], m_counters["stats.tlb.chit"])); + d_printf("tlb read hit ratio: %.4f\n", hr(m_counters["stats.tlb.rmiss"], m_counters["stats.tlb.rhit"])); + d_printf("tlb write hit ratio: %.4f\n", hr(m_counters["stats.tlb.wmiss"], m_counters["stats.tlb.whit"])); + d_printf("tlb.chit: %" PRIu64 "\n", m_counters["stats.tlb.chit"]); + d_printf("tlb.cmiss: %" PRIu64 "\n", m_counters["stats.tlb.cmiss"]); + d_printf("tlb.rhit: %" PRIu64 "\n", m_counters["stats.tlb.rhit"]); + d_printf("tlb.rmiss: %" PRIu64 "\n", m_counters["stats.tlb.rmiss"]); + d_printf("tlb.whit: %" PRIu64 "\n", m_counters["stats.tlb.whit"]); + d_printf("tlb.wmiss: %" PRIu64 "\n", m_counters["stats.tlb.wmiss"]); + d_printf("tlb.flush_all: %" PRIu64 "\n", m_counters["stats.tlb.flush_all"]); + d_printf("tlb.flush_read: %" PRIu64 "\n", m_counters["stats.tlb.flush_read"]); + d_printf("tlb.flush_write: %" PRIu64 "\n", m_counters["stats.tlb.flush_write"]); + d_printf("tlb.flush_vaddr: %" PRIu64 "\n", m_counters["stats.tlb.flush_vaddr"]); + d_printf("tlb.flush_satp: %" PRIu64 "\n", m_counters["stats.tlb.flush_satp"]); + d_printf("tlb.flush_mstatus: %" PRIu64 "\n", m_counters["stats.tlb.flush_mstatus"]); + d_printf("tlb.flush_set_prv: %" PRIu64 "\n", m_counters["stats.tlb.flush_set_prv"]); + d_printf("tlb.flush_fence_vma_all: %" PRIu64 "\n", m_counters["stats.tlb.flush_fence_vma_all"]); + d_printf("tlb.flush_fence_vma_asid: %" PRIu64 "\n", m_counters["stats.tlb.flush_fence_vma_asid"]); + d_printf("tlb.flush_fence_vma_vaddr: %" PRIu64 "\n", m_counters["stats.tlb.flush_fence_vma_vaddr"]); + d_printf("tlb.flush_fence_vma_asid_vaddr: %" PRIu64 "\n", m_counters["stats.tlb.flush_fence_vma_asid_vaddr"]); +#undef TLB_HIT_RATIO +#endif } machine::~machine() { - // Cleanup TTY if console input was enabled - if (m_c.htif.console_getchar || has_virtio_console()) { + if (m_tty_opened) { os_close_tty(); } -#ifdef DUMP_HIST - std::ignore = fprintf(stderr, "\nInstruction Histogram:\n"); - for (auto v : m_s.insn_hist) { - std::ignore = fprintf(stderr, "%12" PRIu64 " %s\n", v.second, v.first.c_str()); - } -#endif -#if DUMP_COUNTERS -#define TLB_HIT_RATIO(s, a, b) (((double) (s).stats.b) / ((s).stats.a + (s).stats.b)) - std::ignore = fprintf(stderr, "\nMachine Counters:\n"); - std::ignore = fprintf(stderr, "inner loops: %" PRIu64 "\n", m_s.stats.inner_loop); - std::ignore = fprintf(stderr, "outers loops: %" PRIu64 "\n", m_s.stats.outer_loop); - std::ignore = fprintf(stderr, "supervisor ints: %" PRIu64 "\n", m_s.stats.sv_int); - std::ignore = fprintf(stderr, "supervisor ex: %" PRIu64 "\n", m_s.stats.sv_ex); - std::ignore = fprintf(stderr, "machine ints: %" PRIu64 "\n", m_s.stats.m_int); - std::ignore = fprintf(stderr, "machine ex: %" PRIu64 "\n", m_s.stats.m_ex); - std::ignore = fprintf(stderr, "atomic mem ops: %" PRIu64 "\n", m_s.stats.atomic_mop); - std::ignore = fprintf(stderr, "fence: %" PRIu64 "\n", m_s.stats.fence); - std::ignore = fprintf(stderr, "fence.i: %" PRIu64 "\n", m_s.stats.fence_i); - std::ignore = fprintf(stderr, "fence.vma: %" PRIu64 "\n", m_s.stats.fence_vma); - std::ignore = fprintf(stderr, "max asid: %" PRIu64 "\n", m_s.stats.max_asid); - std::ignore = fprintf(stderr, "User mode: %" PRIu64 "\n", m_s.stats.prv_level[PRV_U]); - std::ignore = fprintf(stderr, "Supervisor mode: %" PRIu64 "\n", m_s.stats.prv_level[PRV_S]); - std::ignore = fprintf(stderr, "Machine mode: %" PRIu64 "\n", m_s.stats.prv_level[PRV_M]); - - std::ignore = fprintf(stderr, "tlb code hit ratio: %.4f\n", TLB_HIT_RATIO(m_s, tlb_cmiss, tlb_chit)); - std::ignore = fprintf(stderr, "tlb read hit ratio: %.4f\n", TLB_HIT_RATIO(m_s, tlb_rmiss, tlb_rhit)); - std::ignore = fprintf(stderr, "tlb write hit ratio: %.4f\n", TLB_HIT_RATIO(m_s, tlb_wmiss, tlb_whit)); - std::ignore = fprintf(stderr, "tlb_chit: %" PRIu64 "\n", m_s.stats.tlb_chit); - std::ignore = fprintf(stderr, "tlb_cmiss: %" PRIu64 "\n", m_s.stats.tlb_cmiss); - std::ignore = fprintf(stderr, "tlb_rhit: %" PRIu64 "\n", m_s.stats.tlb_rhit); - std::ignore = fprintf(stderr, "tlb_rmiss: %" PRIu64 "\n", m_s.stats.tlb_rmiss); - std::ignore = fprintf(stderr, "tlb_whit: %" PRIu64 "\n", m_s.stats.tlb_whit); - std::ignore = fprintf(stderr, "tlb_wmiss: %" PRIu64 "\n", m_s.stats.tlb_wmiss); - std::ignore = fprintf(stderr, "tlb_flush_all: %" PRIu64 "\n", m_s.stats.tlb_flush_all); - std::ignore = fprintf(stderr, "tlb_flush_read: %" PRIu64 "\n", m_s.stats.tlb_flush_read); - std::ignore = fprintf(stderr, "tlb_flush_write: %" PRIu64 "\n", m_s.stats.tlb_flush_write); - std::ignore = fprintf(stderr, "tlb_flush_vaddr: %" PRIu64 "\n", m_s.stats.tlb_flush_vaddr); - std::ignore = fprintf(stderr, "tlb_flush_satp: %" PRIu64 "\n", m_s.stats.tlb_flush_satp); - std::ignore = fprintf(stderr, "tlb_flush_mstatus: %" PRIu64 "\n", m_s.stats.tlb_flush_mstatus); - std::ignore = fprintf(stderr, "tlb_flush_set_prv: %" PRIu64 "\n", m_s.stats.tlb_flush_set_prv); - std::ignore = fprintf(stderr, "tlb_flush_fence_vma_all: %" PRIu64 "\n", m_s.stats.tlb_flush_fence_vma_all); - std::ignore = fprintf(stderr, "tlb_flush_fence_vma_asid: %" PRIu64 "\n", m_s.stats.tlb_flush_fence_vma_asid); - std::ignore = fprintf(stderr, "tlb_flush_fence_vma_vaddr: %" PRIu64 "\n", m_s.stats.tlb_flush_fence_vma_vaddr); - std::ignore = - fprintf(stderr, "tlb_flush_fence_vma_asid_vaddr: %" PRIu64 "\n", m_s.stats.tlb_flush_fence_vma_asid_vaddr); -#endif + dump_insn_hist(); + dump_stats(); } uint64_t machine::read_reg(reg r) const { switch (r) { case reg::x0: - return m_s.x[0]; + return m_s->shadow.registers.x[0]; case reg::x1: - return m_s.x[1]; + return m_s->shadow.registers.x[1]; case reg::x2: - return m_s.x[2]; + return m_s->shadow.registers.x[2]; case reg::x3: - return m_s.x[3]; + return m_s->shadow.registers.x[3]; case reg::x4: - return m_s.x[4]; + return m_s->shadow.registers.x[4]; case reg::x5: - return m_s.x[5]; + return m_s->shadow.registers.x[5]; case reg::x6: - return m_s.x[6]; + return m_s->shadow.registers.x[6]; case reg::x7: - return m_s.x[7]; + return m_s->shadow.registers.x[7]; case reg::x8: - return m_s.x[8]; + return m_s->shadow.registers.x[8]; case reg::x9: - return m_s.x[9]; + return m_s->shadow.registers.x[9]; case reg::x10: - return m_s.x[10]; + return m_s->shadow.registers.x[10]; case reg::x11: - return m_s.x[11]; + return m_s->shadow.registers.x[11]; case reg::x12: - return m_s.x[12]; + return m_s->shadow.registers.x[12]; case reg::x13: - return m_s.x[13]; + return m_s->shadow.registers.x[13]; case reg::x14: - return m_s.x[14]; + return m_s->shadow.registers.x[14]; case reg::x15: - return m_s.x[15]; + return m_s->shadow.registers.x[15]; case reg::x16: - return m_s.x[16]; + return m_s->shadow.registers.x[16]; case reg::x17: - return m_s.x[17]; + return m_s->shadow.registers.x[17]; case reg::x18: - return m_s.x[18]; + return m_s->shadow.registers.x[18]; case reg::x19: - return m_s.x[19]; + return m_s->shadow.registers.x[19]; case reg::x20: - return m_s.x[20]; + return m_s->shadow.registers.x[20]; case reg::x21: - return m_s.x[21]; + return m_s->shadow.registers.x[21]; case reg::x22: - return m_s.x[22]; + return m_s->shadow.registers.x[22]; case reg::x23: - return m_s.x[23]; + return m_s->shadow.registers.x[23]; case reg::x24: - return m_s.x[24]; + return m_s->shadow.registers.x[24]; case reg::x25: - return m_s.x[25]; + return m_s->shadow.registers.x[25]; case reg::x26: - return m_s.x[26]; + return m_s->shadow.registers.x[26]; case reg::x27: - return m_s.x[27]; + return m_s->shadow.registers.x[27]; case reg::x28: - return m_s.x[28]; + return m_s->shadow.registers.x[28]; case reg::x29: - return m_s.x[29]; + return m_s->shadow.registers.x[29]; case reg::x30: - return m_s.x[30]; + return m_s->shadow.registers.x[30]; case reg::x31: - return m_s.x[31]; + return m_s->shadow.registers.x[31]; case reg::f0: - return m_s.f[0]; + return m_s->shadow.registers.f[0]; case reg::f1: - return m_s.f[1]; + return m_s->shadow.registers.f[1]; case reg::f2: - return m_s.f[2]; + return m_s->shadow.registers.f[2]; case reg::f3: - return m_s.f[3]; + return m_s->shadow.registers.f[3]; case reg::f4: - return m_s.f[4]; + return m_s->shadow.registers.f[4]; case reg::f5: - return m_s.f[5]; + return m_s->shadow.registers.f[5]; case reg::f6: - return m_s.f[6]; + return m_s->shadow.registers.f[6]; case reg::f7: - return m_s.f[7]; + return m_s->shadow.registers.f[7]; case reg::f8: - return m_s.f[8]; + return m_s->shadow.registers.f[8]; case reg::f9: - return m_s.f[9]; + return m_s->shadow.registers.f[9]; case reg::f10: - return m_s.f[10]; + return m_s->shadow.registers.f[10]; case reg::f11: - return m_s.f[11]; + return m_s->shadow.registers.f[11]; case reg::f12: - return m_s.f[12]; + return m_s->shadow.registers.f[12]; case reg::f13: - return m_s.f[13]; + return m_s->shadow.registers.f[13]; case reg::f14: - return m_s.f[14]; + return m_s->shadow.registers.f[14]; case reg::f15: - return m_s.f[15]; + return m_s->shadow.registers.f[15]; case reg::f16: - return m_s.f[16]; + return m_s->shadow.registers.f[16]; case reg::f17: - return m_s.f[17]; + return m_s->shadow.registers.f[17]; case reg::f18: - return m_s.f[18]; + return m_s->shadow.registers.f[18]; case reg::f19: - return m_s.f[19]; + return m_s->shadow.registers.f[19]; case reg::f20: - return m_s.f[20]; + return m_s->shadow.registers.f[20]; case reg::f21: - return m_s.f[21]; + return m_s->shadow.registers.f[21]; case reg::f22: - return m_s.f[22]; + return m_s->shadow.registers.f[22]; case reg::f23: - return m_s.f[23]; + return m_s->shadow.registers.f[23]; case reg::f24: - return m_s.f[24]; + return m_s->shadow.registers.f[24]; case reg::f25: - return m_s.f[25]; + return m_s->shadow.registers.f[25]; case reg::f26: - return m_s.f[26]; + return m_s->shadow.registers.f[26]; case reg::f27: - return m_s.f[27]; + return m_s->shadow.registers.f[27]; case reg::f28: - return m_s.f[28]; + return m_s->shadow.registers.f[28]; case reg::f29: - return m_s.f[29]; + return m_s->shadow.registers.f[29]; case reg::f30: - return m_s.f[30]; + return m_s->shadow.registers.f[30]; case reg::f31: - return m_s.f[31]; + return m_s->shadow.registers.f[31]; case reg::pc: - return m_s.pc; + return m_s->shadow.registers.pc; case reg::fcsr: - return m_s.fcsr; + return m_s->shadow.registers.fcsr; case reg::mvendorid: return MVENDORID_INIT; case reg::marchid: @@ -1036,165 +771,165 @@ uint64_t machine::read_reg(reg r) const { case reg::mimpid: return MIMPID_INIT; case reg::mcycle: - return m_s.mcycle; + return m_s->shadow.registers.mcycle; case reg::icycleinstret: - return m_s.icycleinstret; + return m_s->shadow.registers.icycleinstret; case reg::mstatus: - return m_s.mstatus; + return m_s->shadow.registers.mstatus; case reg::mtvec: - return m_s.mtvec; + return m_s->shadow.registers.mtvec; case reg::mscratch: - return m_s.mscratch; + return m_s->shadow.registers.mscratch; case reg::mepc: - return m_s.mepc; + return m_s->shadow.registers.mepc; case reg::mcause: - return m_s.mcause; + return m_s->shadow.registers.mcause; case reg::mtval: - return m_s.mtval; + return m_s->shadow.registers.mtval; case reg::misa: - return m_s.misa; + return m_s->shadow.registers.misa; case reg::mie: - return m_s.mie; + return m_s->shadow.registers.mie; case reg::mip: - return m_s.mip; + return m_s->shadow.registers.mip; case reg::medeleg: - return m_s.medeleg; + return m_s->shadow.registers.medeleg; case reg::mideleg: - return m_s.mideleg; + return m_s->shadow.registers.mideleg; case reg::mcounteren: - return m_s.mcounteren; + return m_s->shadow.registers.mcounteren; case reg::menvcfg: - return m_s.menvcfg; + return m_s->shadow.registers.menvcfg; case reg::stvec: - return m_s.stvec; + return m_s->shadow.registers.stvec; case reg::sscratch: - return m_s.sscratch; + return m_s->shadow.registers.sscratch; case reg::sepc: - return m_s.sepc; + return m_s->shadow.registers.sepc; case reg::scause: - return m_s.scause; + return m_s->shadow.registers.scause; case reg::stval: - return m_s.stval; + return m_s->shadow.registers.stval; case reg::satp: - return m_s.satp; + return m_s->shadow.registers.satp; case reg::scounteren: - return m_s.scounteren; + return m_s->shadow.registers.scounteren; case reg::senvcfg: - return m_s.senvcfg; + return m_s->shadow.registers.senvcfg; case reg::ilrsc: - return m_s.ilrsc; + return m_s->shadow.registers.ilrsc; case reg::iprv: - return m_s.iprv; + return m_s->shadow.registers.iprv; case reg::iflags_X: - return m_s.iflags.X; + return m_s->shadow.registers.iflags.X; case reg::iflags_Y: - return m_s.iflags.Y; + return m_s->shadow.registers.iflags.Y; case reg::iflags_H: - return m_s.iflags.H; + return m_s->shadow.registers.iflags.H; case reg::iunrep: - return m_s.iunrep; + return m_s->shadow.registers.iunrep; case reg::clint_mtimecmp: - return m_s.clint.mtimecmp; + return m_s->shadow.registers.clint.mtimecmp; case reg::plic_girqpend: - return m_s.plic.girqpend; + return m_s->shadow.registers.plic.girqpend; case reg::plic_girqsrvd: - return m_s.plic.girqsrvd; + return m_s->shadow.registers.plic.girqsrvd; case reg::htif_tohost: - return m_s.htif.tohost; + return m_s->shadow.registers.htif.tohost; case reg::htif_fromhost: - return m_s.htif.fromhost; + return m_s->shadow.registers.htif.fromhost; case reg::htif_ihalt: - return m_s.htif.ihalt; + return m_s->shadow.registers.htif.ihalt; case reg::htif_iconsole: - return m_s.htif.iconsole; + return m_s->shadow.registers.htif.iconsole; case reg::htif_iyield: - return m_s.htif.iyield; + return m_s->shadow.registers.htif.iyield; case reg::uarch_x0: - return m_uarch.get_state().x[0]; + return m_us->registers.x[0]; case reg::uarch_x1: - return m_uarch.get_state().x[1]; + return m_us->registers.x[1]; case reg::uarch_x2: - return m_uarch.get_state().x[2]; + return m_us->registers.x[2]; case reg::uarch_x3: - return m_uarch.get_state().x[3]; + return m_us->registers.x[3]; case reg::uarch_x4: - return m_uarch.get_state().x[4]; + return m_us->registers.x[4]; case reg::uarch_x5: - return m_uarch.get_state().x[5]; + return m_us->registers.x[5]; case reg::uarch_x6: - return m_uarch.get_state().x[6]; + return m_us->registers.x[6]; case reg::uarch_x7: - return m_uarch.get_state().x[7]; + return m_us->registers.x[7]; case reg::uarch_x8: - return m_uarch.get_state().x[8]; + return m_us->registers.x[8]; case reg::uarch_x9: - return m_uarch.get_state().x[9]; + return m_us->registers.x[9]; case reg::uarch_x10: - return m_uarch.get_state().x[10]; + return m_us->registers.x[10]; case reg::uarch_x11: - return m_uarch.get_state().x[11]; + return m_us->registers.x[11]; case reg::uarch_x12: - return m_uarch.get_state().x[12]; + return m_us->registers.x[12]; case reg::uarch_x13: - return m_uarch.get_state().x[13]; + return m_us->registers.x[13]; case reg::uarch_x14: - return m_uarch.get_state().x[14]; + return m_us->registers.x[14]; case reg::uarch_x15: - return m_uarch.get_state().x[15]; + return m_us->registers.x[15]; case reg::uarch_x16: - return m_uarch.get_state().x[16]; + return m_us->registers.x[16]; case reg::uarch_x17: - return m_uarch.get_state().x[17]; + return m_us->registers.x[17]; case reg::uarch_x18: - return m_uarch.get_state().x[18]; + return m_us->registers.x[18]; case reg::uarch_x19: - return m_uarch.get_state().x[19]; + return m_us->registers.x[19]; case reg::uarch_x20: - return m_uarch.get_state().x[20]; + return m_us->registers.x[20]; case reg::uarch_x21: - return m_uarch.get_state().x[21]; + return m_us->registers.x[21]; case reg::uarch_x22: - return m_uarch.get_state().x[22]; + return m_us->registers.x[22]; case reg::uarch_x23: - return m_uarch.get_state().x[23]; + return m_us->registers.x[23]; case reg::uarch_x24: - return m_uarch.get_state().x[24]; + return m_us->registers.x[24]; case reg::uarch_x25: - return m_uarch.get_state().x[25]; + return m_us->registers.x[25]; case reg::uarch_x26: - return m_uarch.get_state().x[26]; + return m_us->registers.x[26]; case reg::uarch_x27: - return m_uarch.get_state().x[27]; + return m_us->registers.x[27]; case reg::uarch_x28: - return m_uarch.get_state().x[28]; + return m_us->registers.x[28]; case reg::uarch_x29: - return m_uarch.get_state().x[29]; + return m_us->registers.x[29]; case reg::uarch_x30: - return m_uarch.get_state().x[30]; + return m_us->registers.x[30]; case reg::uarch_x31: - return m_uarch.get_state().x[31]; + return m_us->registers.x[31]; case reg::uarch_pc: - return m_uarch.get_state().pc; + return m_us->registers.pc; case reg::uarch_cycle: - return m_uarch.get_state().cycle; + return m_us->registers.cycle; case reg::uarch_halt_flag: - return static_cast(m_uarch.get_state().halt_flag); + return m_us->registers.halt_flag; case reg::htif_tohost_dev: - return HTIF_DEV_FIELD(m_s.htif.tohost); + return HTIF_DEV_FIELD(m_s->shadow.registers.htif.tohost); case reg::htif_tohost_cmd: - return HTIF_CMD_FIELD(m_s.htif.tohost); + return HTIF_CMD_FIELD(m_s->shadow.registers.htif.tohost); case reg::htif_tohost_reason: - return HTIF_REASON_FIELD(m_s.htif.tohost); + return HTIF_REASON_FIELD(m_s->shadow.registers.htif.tohost); case reg::htif_tohost_data: - return HTIF_DATA_FIELD(m_s.htif.tohost); + return HTIF_DATA_FIELD(m_s->shadow.registers.htif.tohost); case reg::htif_fromhost_dev: - return HTIF_DEV_FIELD(m_s.htif.fromhost); + return HTIF_DEV_FIELD(m_s->shadow.registers.htif.fromhost); case reg::htif_fromhost_cmd: - return HTIF_CMD_FIELD(m_s.htif.fromhost); + return HTIF_CMD_FIELD(m_s->shadow.registers.htif.fromhost); case reg::htif_fromhost_reason: - return HTIF_REASON_FIELD(m_s.htif.fromhost); + return HTIF_REASON_FIELD(m_s->shadow.registers.htif.fromhost); case reg::htif_fromhost_data: - return HTIF_DATA_FIELD(m_s.htif.fromhost); + return HTIF_DATA_FIELD(m_s->shadow.registers.htif.fromhost); default: throw std::invalid_argument{"unknown register"}; return 0; // never reached @@ -1206,199 +941,199 @@ void machine::write_reg(reg w, uint64_t value) { case reg::x0: throw std::invalid_argument{"register is read-only"}; case reg::x1: - m_s.x[1] = value; + m_s->shadow.registers.x[1] = value; break; case reg::x2: - m_s.x[2] = value; + m_s->shadow.registers.x[2] = value; break; case reg::x3: - m_s.x[3] = value; + m_s->shadow.registers.x[3] = value; break; case reg::x4: - m_s.x[4] = value; + m_s->shadow.registers.x[4] = value; break; case reg::x5: - m_s.x[5] = value; + m_s->shadow.registers.x[5] = value; break; case reg::x6: - m_s.x[6] = value; + m_s->shadow.registers.x[6] = value; break; case reg::x7: - m_s.x[7] = value; + m_s->shadow.registers.x[7] = value; break; case reg::x8: - m_s.x[8] = value; + m_s->shadow.registers.x[8] = value; break; case reg::x9: - m_s.x[9] = value; + m_s->shadow.registers.x[9] = value; break; case reg::x10: - m_s.x[10] = value; + m_s->shadow.registers.x[10] = value; break; case reg::x11: - m_s.x[11] = value; + m_s->shadow.registers.x[11] = value; break; case reg::x12: - m_s.x[12] = value; + m_s->shadow.registers.x[12] = value; break; case reg::x13: - m_s.x[13] = value; + m_s->shadow.registers.x[13] = value; break; case reg::x14: - m_s.x[14] = value; + m_s->shadow.registers.x[14] = value; break; case reg::x15: - m_s.x[15] = value; + m_s->shadow.registers.x[15] = value; break; case reg::x16: - m_s.x[16] = value; + m_s->shadow.registers.x[16] = value; break; case reg::x17: - m_s.x[17] = value; + m_s->shadow.registers.x[17] = value; break; case reg::x18: - m_s.x[18] = value; + m_s->shadow.registers.x[18] = value; break; case reg::x19: - m_s.x[19] = value; + m_s->shadow.registers.x[19] = value; break; case reg::x20: - m_s.x[20] = value; + m_s->shadow.registers.x[20] = value; break; case reg::x21: - m_s.x[21] = value; + m_s->shadow.registers.x[21] = value; break; case reg::x22: - m_s.x[22] = value; + m_s->shadow.registers.x[22] = value; break; case reg::x23: - m_s.x[23] = value; + m_s->shadow.registers.x[23] = value; break; case reg::x24: - m_s.x[24] = value; + m_s->shadow.registers.x[24] = value; break; case reg::x25: - m_s.x[25] = value; + m_s->shadow.registers.x[25] = value; break; case reg::x26: - m_s.x[26] = value; + m_s->shadow.registers.x[26] = value; break; case reg::x27: - m_s.x[27] = value; + m_s->shadow.registers.x[27] = value; break; case reg::x28: - m_s.x[28] = value; + m_s->shadow.registers.x[28] = value; break; case reg::x29: - m_s.x[29] = value; + m_s->shadow.registers.x[29] = value; break; case reg::x30: - m_s.x[30] = value; + m_s->shadow.registers.x[30] = value; break; case reg::x31: - m_s.x[31] = value; + m_s->shadow.registers.x[31] = value; break; case reg::f0: - m_s.f[0] = value; + m_s->shadow.registers.f[0] = value; break; case reg::f1: - m_s.f[1] = value; + m_s->shadow.registers.f[1] = value; break; case reg::f2: - m_s.f[2] = value; + m_s->shadow.registers.f[2] = value; break; case reg::f3: - m_s.f[3] = value; + m_s->shadow.registers.f[3] = value; break; case reg::f4: - m_s.f[4] = value; + m_s->shadow.registers.f[4] = value; break; case reg::f5: - m_s.f[5] = value; + m_s->shadow.registers.f[5] = value; break; case reg::f6: - m_s.f[6] = value; + m_s->shadow.registers.f[6] = value; break; case reg::f7: - m_s.f[7] = value; + m_s->shadow.registers.f[7] = value; break; case reg::f8: - m_s.f[8] = value; + m_s->shadow.registers.f[8] = value; break; case reg::f9: - m_s.f[9] = value; + m_s->shadow.registers.f[9] = value; break; case reg::f10: - m_s.f[10] = value; + m_s->shadow.registers.f[10] = value; break; case reg::f11: - m_s.f[11] = value; + m_s->shadow.registers.f[11] = value; break; case reg::f12: - m_s.f[12] = value; + m_s->shadow.registers.f[12] = value; break; case reg::f13: - m_s.f[13] = value; + m_s->shadow.registers.f[13] = value; break; case reg::f14: - m_s.f[14] = value; + m_s->shadow.registers.f[14] = value; break; case reg::f15: - m_s.f[15] = value; + m_s->shadow.registers.f[15] = value; break; case reg::f16: - m_s.f[16] = value; + m_s->shadow.registers.f[16] = value; break; case reg::f17: - m_s.f[17] = value; + m_s->shadow.registers.f[17] = value; break; case reg::f18: - m_s.f[18] = value; + m_s->shadow.registers.f[18] = value; break; case reg::f19: - m_s.f[19] = value; + m_s->shadow.registers.f[19] = value; break; case reg::f20: - m_s.f[20] = value; + m_s->shadow.registers.f[20] = value; break; case reg::f21: - m_s.f[21] = value; + m_s->shadow.registers.f[21] = value; break; case reg::f22: - m_s.f[22] = value; + m_s->shadow.registers.f[22] = value; break; case reg::f23: - m_s.f[23] = value; + m_s->shadow.registers.f[23] = value; break; case reg::f24: - m_s.f[24] = value; + m_s->shadow.registers.f[24] = value; break; case reg::f25: - m_s.f[25] = value; + m_s->shadow.registers.f[25] = value; break; case reg::f26: - m_s.f[26] = value; + m_s->shadow.registers.f[26] = value; break; case reg::f27: - m_s.f[27] = value; + m_s->shadow.registers.f[27] = value; break; case reg::f28: - m_s.f[28] = value; + m_s->shadow.registers.f[28] = value; break; case reg::f29: - m_s.f[29] = value; + m_s->shadow.registers.f[29] = value; break; case reg::f30: - m_s.f[30] = value; + m_s->shadow.registers.f[30] = value; break; case reg::f31: - m_s.f[31] = value; + m_s->shadow.registers.f[31] = value; break; case reg::pc: - m_s.pc = value; + m_s->shadow.registers.pc = value; break; case reg::fcsr: - m_s.fcsr = value; + m_s->shadow.registers.fcsr = value; break; case reg::mvendorid: throw std::invalid_argument{"register is read-only"}; @@ -1407,243 +1142,243 @@ void machine::write_reg(reg w, uint64_t value) { case reg::mimpid: throw std::invalid_argument{"register is read-only"}; case reg::mcycle: - m_s.mcycle = value; + m_s->shadow.registers.mcycle = value; break; case reg::icycleinstret: - m_s.icycleinstret = value; + m_s->shadow.registers.icycleinstret = value; break; case reg::mstatus: - m_s.mstatus = value; + m_s->shadow.registers.mstatus = value; break; case reg::mtvec: - m_s.mtvec = value; + m_s->shadow.registers.mtvec = value; break; case reg::mscratch: - m_s.mscratch = value; + m_s->shadow.registers.mscratch = value; break; case reg::mepc: - m_s.mepc = value; + m_s->shadow.registers.mepc = value; break; case reg::mcause: - m_s.mcause = value; + m_s->shadow.registers.mcause = value; break; case reg::mtval: - m_s.mtval = value; + m_s->shadow.registers.mtval = value; break; case reg::misa: - m_s.misa = value; + m_s->shadow.registers.misa = value; break; case reg::mie: - m_s.mie = value; + m_s->shadow.registers.mie = value; break; case reg::mip: - m_s.mip = value; + m_s->shadow.registers.mip = value; break; case reg::medeleg: - m_s.medeleg = value; + m_s->shadow.registers.medeleg = value; break; case reg::mideleg: - m_s.mideleg = value; + m_s->shadow.registers.mideleg = value; break; case reg::mcounteren: - m_s.mcounteren = value; + m_s->shadow.registers.mcounteren = value; break; case reg::menvcfg: - m_s.menvcfg = value; + m_s->shadow.registers.menvcfg = value; break; case reg::stvec: - m_s.stvec = value; + m_s->shadow.registers.stvec = value; break; case reg::sscratch: - m_s.sscratch = value; + m_s->shadow.registers.sscratch = value; break; case reg::sepc: - m_s.sepc = value; + m_s->shadow.registers.sepc = value; break; case reg::scause: - m_s.scause = value; + m_s->shadow.registers.scause = value; break; case reg::stval: - m_s.stval = value; + m_s->shadow.registers.stval = value; break; case reg::satp: - m_s.satp = value; + m_s->shadow.registers.satp = value; break; case reg::scounteren: - m_s.scounteren = value; + m_s->shadow.registers.scounteren = value; break; case reg::senvcfg: - m_s.senvcfg = value; + m_s->shadow.registers.senvcfg = value; break; case reg::ilrsc: - m_s.ilrsc = value; + m_s->shadow.registers.ilrsc = value; break; case reg::iprv: - m_s.iprv = value; + m_s->shadow.registers.iprv = value; break; case reg::iflags_X: - m_s.iflags.X = value; + m_s->shadow.registers.iflags.X = value; break; case reg::iflags_Y: - m_s.iflags.Y = value; + m_s->shadow.registers.iflags.Y = value; break; case reg::iflags_H: - m_s.iflags.H = value; + m_s->shadow.registers.iflags.H = value; break; case reg::iunrep: - m_s.iunrep = value; + m_s->shadow.registers.iunrep = value; break; case reg::clint_mtimecmp: - m_s.clint.mtimecmp = value; + m_s->shadow.registers.clint.mtimecmp = value; break; case reg::plic_girqpend: - m_s.plic.girqpend = value; + m_s->shadow.registers.plic.girqpend = value; break; case reg::plic_girqsrvd: - m_s.plic.girqsrvd = value; + m_s->shadow.registers.plic.girqsrvd = value; break; case reg::htif_tohost: - m_s.htif.tohost = value; + m_s->shadow.registers.htif.tohost = value; break; case reg::htif_fromhost: - m_s.htif.fromhost = value; + m_s->shadow.registers.htif.fromhost = value; break; case reg::htif_ihalt: - m_s.htif.ihalt = value; + m_s->shadow.registers.htif.ihalt = value; break; case reg::htif_iconsole: - m_s.htif.iconsole = value; + m_s->shadow.registers.htif.iconsole = value; break; case reg::htif_iyield: - m_s.htif.iyield = value; + m_s->shadow.registers.htif.iyield = value; break; case reg::uarch_x0: throw std::invalid_argument{"register is read-only"}; case reg::uarch_x1: - m_uarch.get_state().x[1] = value; + m_us->registers.x[1] = value; break; case reg::uarch_x2: - m_uarch.get_state().x[2] = value; + m_us->registers.x[2] = value; break; case reg::uarch_x3: - m_uarch.get_state().x[3] = value; + m_us->registers.x[3] = value; break; case reg::uarch_x4: - m_uarch.get_state().x[4] = value; + m_us->registers.x[4] = value; break; case reg::uarch_x5: - m_uarch.get_state().x[5] = value; + m_us->registers.x[5] = value; break; case reg::uarch_x6: - m_uarch.get_state().x[6] = value; + m_us->registers.x[6] = value; break; case reg::uarch_x7: - m_uarch.get_state().x[7] = value; + m_us->registers.x[7] = value; break; case reg::uarch_x8: - m_uarch.get_state().x[8] = value; + m_us->registers.x[8] = value; break; case reg::uarch_x9: - m_uarch.get_state().x[9] = value; + m_us->registers.x[9] = value; break; case reg::uarch_x10: - m_uarch.get_state().x[10] = value; + m_us->registers.x[10] = value; break; case reg::uarch_x11: - m_uarch.get_state().x[11] = value; + m_us->registers.x[11] = value; break; case reg::uarch_x12: - m_uarch.get_state().x[12] = value; + m_us->registers.x[12] = value; break; case reg::uarch_x13: - m_uarch.get_state().x[13] = value; + m_us->registers.x[13] = value; break; case reg::uarch_x14: - m_uarch.get_state().x[14] = value; + m_us->registers.x[14] = value; break; case reg::uarch_x15: - m_uarch.get_state().x[15] = value; + m_us->registers.x[15] = value; break; case reg::uarch_x16: - m_uarch.get_state().x[16] = value; + m_us->registers.x[16] = value; break; case reg::uarch_x17: - m_uarch.get_state().x[17] = value; + m_us->registers.x[17] = value; break; case reg::uarch_x18: - m_uarch.get_state().x[18] = value; + m_us->registers.x[18] = value; break; case reg::uarch_x19: - m_uarch.get_state().x[19] = value; + m_us->registers.x[19] = value; break; case reg::uarch_x20: - m_uarch.get_state().x[20] = value; + m_us->registers.x[20] = value; break; case reg::uarch_x21: - m_uarch.get_state().x[21] = value; + m_us->registers.x[21] = value; break; case reg::uarch_x22: - m_uarch.get_state().x[22] = value; + m_us->registers.x[22] = value; break; case reg::uarch_x23: - m_uarch.get_state().x[23] = value; + m_us->registers.x[23] = value; break; case reg::uarch_x24: - m_uarch.get_state().x[24] = value; + m_us->registers.x[24] = value; break; case reg::uarch_x25: - m_uarch.get_state().x[25] = value; + m_us->registers.x[25] = value; break; case reg::uarch_x26: - m_uarch.get_state().x[26] = value; + m_us->registers.x[26] = value; break; case reg::uarch_x27: - m_uarch.get_state().x[27] = value; + m_us->registers.x[27] = value; break; case reg::uarch_x28: - m_uarch.get_state().x[28] = value; + m_us->registers.x[28] = value; break; case reg::uarch_x29: - m_uarch.get_state().x[29] = value; + m_us->registers.x[29] = value; break; case reg::uarch_x30: - m_uarch.get_state().x[30] = value; + m_us->registers.x[30] = value; break; case reg::uarch_x31: - m_uarch.get_state().x[31] = value; + m_us->registers.x[31] = value; break; case reg::uarch_pc: - m_uarch.get_state().pc = value; + m_us->registers.pc = value; break; case reg::uarch_cycle: - m_uarch.get_state().cycle = value; + m_us->registers.cycle = value; break; case reg::uarch_halt_flag: - m_uarch.get_state().halt_flag = static_cast(value); + m_us->registers.halt_flag = value; break; case reg::htif_tohost_dev: - m_s.htif.tohost = HTIF_REPLACE_DEV(m_s.htif.tohost, value); + m_s->shadow.registers.htif.tohost = HTIF_REPLACE_DEV(m_s->shadow.registers.htif.tohost, value); break; case reg::htif_tohost_cmd: - m_s.htif.tohost = HTIF_REPLACE_CMD(m_s.htif.tohost, value); + m_s->shadow.registers.htif.tohost = HTIF_REPLACE_CMD(m_s->shadow.registers.htif.tohost, value); break; case reg::htif_tohost_reason: - m_s.htif.tohost = HTIF_REPLACE_REASON(m_s.htif.tohost, value); + m_s->shadow.registers.htif.tohost = HTIF_REPLACE_REASON(m_s->shadow.registers.htif.tohost, value); break; case reg::htif_tohost_data: - m_s.htif.tohost = HTIF_REPLACE_DATA(m_s.htif.tohost, value); + m_s->shadow.registers.htif.tohost = HTIF_REPLACE_DATA(m_s->shadow.registers.htif.tohost, value); break; case reg::htif_fromhost_dev: - m_s.htif.fromhost = HTIF_REPLACE_DEV(m_s.htif.fromhost, value); + m_s->shadow.registers.htif.fromhost = HTIF_REPLACE_DEV(m_s->shadow.registers.htif.fromhost, value); break; case reg::htif_fromhost_cmd: - m_s.htif.fromhost = HTIF_REPLACE_CMD(m_s.htif.fromhost, value); + m_s->shadow.registers.htif.fromhost = HTIF_REPLACE_CMD(m_s->shadow.registers.htif.fromhost, value); break; case reg::htif_fromhost_reason: - m_s.htif.fromhost = HTIF_REPLACE_REASON(m_s.htif.fromhost, value); + m_s->shadow.registers.htif.fromhost = HTIF_REPLACE_REASON(m_s->shadow.registers.htif.fromhost, value); break; case reg::htif_fromhost_data: - m_s.htif.fromhost = HTIF_REPLACE_DATA(m_s.htif.fromhost, value); + m_s->shadow.registers.htif.fromhost = HTIF_REPLACE_DATA(m_s->shadow.registers.htif.fromhost, value); break; default: throw std::invalid_argument{"unknown register"}; @@ -1663,335 +1398,255 @@ uint64_t machine::get_reg_address(reg r) { } void machine::mark_write_tlb_dirty_pages() const { - for (uint64_t i = 0; i < PMA_TLB_SIZE; ++i) { - const tlb_hot_entry &tlbhe = m_s.tlb.hot[TLB_WRITE][i]; - if (tlbhe.vaddr_page != TLB_INVALID_PAGE) { - const tlb_cold_entry &tlbce = m_s.tlb.cold[TLB_WRITE][i]; - if (tlbce.pma_index >= m_s.pmas.size()) { + auto &hot_set = m_s->penumbra.tlb[TLB_WRITE]; + auto &shadow_set = m_s->shadow.tlb[TLB_WRITE]; + for (uint64_t slot_index = 0; slot_index < TLB_SET_SIZE; ++slot_index) { + const auto &hot_slot = hot_set[slot_index]; + if (hot_slot.vaddr_page != TLB_INVALID_PAGE) { + const auto &shadow_slot = shadow_set[slot_index]; + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-const-cast) + auto &ar = const_cast(read_pma(shadow_slot.pma_index)); + if (!ar.is_memory()) { throw std::runtime_error{"could not mark dirty page for a TLB entry: TLB is corrupt"}; } - pma_entry &pma = m_s.pmas[tlbce.pma_index]; - if (!pma.contains(tlbce.paddr_page, PMA_PAGE_SIZE)) { + auto paddr_page = hot_slot.vaddr_page + shadow_slot.vp_offset; + if (!ar.contains_absolute(paddr_page, AR_PAGE_SIZE)) { throw std::runtime_error{"could not mark dirty page for a TLB entry: TLB is corrupt"}; } - pma.mark_dirty_page(tlbce.paddr_page - pma.get_start()); + ar.get_dirty_page_tree().mark_dirty_page_and_up(paddr_page - ar.get_start()); } } } -bool machine::verify_dirty_page_maps() const { - static_assert(PMA_PAGE_SIZE == machine_merkle_tree::get_page_size(), - "PMA and machine_merkle_tree page sizes must match"); - machine_merkle_tree::hasher_type h; - auto scratch = unique_calloc(PMA_PAGE_SIZE, std::nothrow_t{}); - if (!scratch) { - return false; - } - bool broken = false; - // Go over the write TLB and mark as dirty all pages currently there +bool machine::update_hash_tree() const { mark_write_tlb_dirty_pages(); - // Now go over all memory PMAs verifying that all dirty pages are marked - for (const auto &pma : m_s.pmas) { - auto peek = pma.get_peek(); - for (uint64_t page_start_in_range = 0; page_start_in_range < pma.get_length(); - page_start_in_range += PMA_PAGE_SIZE) { - const uint64_t page_address = pma.get_start() + page_start_in_range; - if (pma.get_istart_M()) { - const unsigned char *page_data = nullptr; - peek(pma, *this, page_start_in_range, &page_data, scratch.get()); - hash_type stored; - hash_type real; - m_t.get_page_node_hash(page_address, stored); - m_t.get_page_node_hash(h, page_data, real); - const bool marked_dirty = pma.is_page_marked_dirty(page_start_in_range); - const bool is_dirty = (real != stored); - if (is_dirty && !marked_dirty) { - broken = true; - std::cerr << std::setfill('0') << std::setw(8) << std::hex << page_address - << " should have been dirty\n"; - std::cerr << " expected " << stored << '\n'; - std::cerr << " got " << real << '\n'; - break; - } - } else if (pma.get_istart_IO()) { - if (!pma.is_page_marked_dirty(page_start_in_range)) { - broken = true; - std::cerr << std::setfill('0') << std::setw(8) << std::hex << page_address - << " should have been dirty\n"; - std::cerr << " all pages in IO PMAs must be set to dirty\n"; - break; - } - } - } - } - return !broken; -} - -static uint64_t get_task_concurrency(uint64_t value) { - const uint64_t concurrency = value > 0 ? value : std::max(os_get_concurrency(), UINT64_C(1)); - return std::min(concurrency, static_cast(THREADS_MAX)); + m_ars.mark_always_dirty_pages(); + return m_ht.update(m_ars); } -bool machine::update_merkle_tree() const { - machine_merkle_tree::hasher_type gh; - static_assert(PMA_PAGE_SIZE == machine_merkle_tree::get_page_size(), - "PMA and machine_merkle_tree page sizes must match"); - // Go over the write TLB and mark as dirty all pages currently there - mark_write_tlb_dirty_pages(); - // Now go over all PMAs and updating the Merkle tree - m_t.begin_update(); - for (const auto &pma : m_merkle_pmas) { - auto peek = pma->get_peek(); - // Each PMA has a number of pages - auto pages_in_range = (pma->get_length() + PMA_PAGE_SIZE - 1) / PMA_PAGE_SIZE; - // For each PMA, we launch as many threads (n) as defined on concurrency - // runtime config or as the hardware supports. - const uint64_t n = get_task_concurrency(m_r.concurrency.update_merkle_tree); - const bool succeeded = os_parallel_for(n, [&](int j, const parallel_for_mutex &mutex) -> bool { - auto scratch = unique_calloc(PMA_PAGE_SIZE, std::nothrow_t{}); - if (!scratch) { - return false; - } - machine_merkle_tree::hasher_type h; - // Thread j is responsible for page i if i % n == j. - for (uint64_t i = j; i < pages_in_range; i += n) { - const uint64_t page_start_in_range = i * PMA_PAGE_SIZE; - const uint64_t page_address = pma->get_start() + page_start_in_range; - const unsigned char *page_data = nullptr; - // Skip any clean pages - if (!pma->is_page_marked_dirty(page_start_in_range)) { - continue; - } - // If the peek failed, or if it returned a page for update but - // we failed updating it, the entire process failed - if (!peek(*pma, *this, page_start_in_range, &page_data, scratch.get())) { - return false; - } - if (page_data != nullptr) { - if (is_pristine(page_data, PMA_PAGE_SIZE)) { - // The update_page_node_hash function in the machine_merkle_tree is not thread - // safe, so we protect it with a mutex - const parallel_for_mutex_guard lock(mutex); - if (!m_t.update_page_node_hash(page_address, - machine_merkle_tree::get_pristine_hash(machine_merkle_tree::get_log2_page_size()))) { - return false; - } - } else { - hash_type hash; - m_t.get_page_node_hash(h, page_data, hash); - { - // The update_page_node_hash function in the machine_merkle_tree is not thread - // safe, so we protect it with a mutex - const parallel_for_mutex_guard lock(mutex); - if (!m_t.update_page_node_hash(page_address, hash)) { - return false; - } - } - } - } - } - return true; - }); - // If any thread failed, we also failed - if (!succeeded) { - m_t.end_update(gh); - return false; - } - // Otherwise, mark all pages in PMA as clean and move on to next - pma->mark_pages_clean(); +machine_hash machine::get_root_hash() const { + if (!update_hash_tree()) { + throw std::runtime_error{"update hash tree failed"}; } - const bool ret = m_t.end_update(gh); - return ret; + return m_ht.get_root_hash(); } -bool machine::update_merkle_tree_page(uint64_t address) { - static_assert(PMA_PAGE_SIZE == machine_merkle_tree::get_page_size(), - "PMA and machine_merkle_tree page sizes must match"); - // Align address to beginning of page - address &= ~(PMA_PAGE_SIZE - 1); - pma_entry &pma = find_pma_entry(m_merkle_pmas, address, sizeof(uint64_t)); - const uint64_t page_start_in_range = address - pma.get_start(); - machine_merkle_tree::hasher_type h; - auto scratch = unique_calloc(PMA_PAGE_SIZE, std::nothrow_t{}); - if (!scratch) { - return false; - } - m_t.begin_update(); - const unsigned char *page_data = nullptr; - auto peek = pma.get_peek(); - if (!peek(pma, *this, page_start_in_range, &page_data, scratch.get())) { - m_t.end_update(h); - return false; - } - if (page_data != nullptr) { - const uint64_t page_address = pma.get_start() + page_start_in_range; - hash_type hash; - m_t.get_page_node_hash(h, page_data, hash); - if (!m_t.update_page_node_hash(page_address, hash)) { - m_t.end_update(h); - return false; - } +const char *machine::get_what_name(uint64_t paddr) { + if (paddr >= AR_UARCH_RAM_START && paddr - AR_UARCH_RAM_START < AR_UARCH_RAM_LENGTH) { + return "uarch.ram"; } - pma.mark_clean_page(page_start_in_range); - return m_t.end_update(h); -} - -const boost::container::static_vector &machine::get_pmas() const { - return m_s.pmas; -} - -void machine::get_root_hash(hash_type &hash) const { - if (read_reg(reg::iunrep) != 0) { - throw std::runtime_error("cannot compute root hash of unreproducible machines"); + // If in shadow, return refined name + if (paddr >= AR_SHADOW_REGISTERS_START && paddr - AR_SHADOW_REGISTERS_START < AR_SHADOW_REGISTERS_LENGTH) { + return shadow_registers_get_what_name(shadow_registers_get_what(paddr)); } - if (!update_merkle_tree()) { - throw std::runtime_error{"error updating Merkle tree"}; + if (paddr >= AR_SHADOW_TLB_START && paddr - AR_SHADOW_TLB_START < AR_SHADOW_TLB_LENGTH) { + [[maybe_unused]] TLB_set_index set_index{}; + [[maybe_unused]] uint64_t slot_index{}; + return shadow_tlb_get_what_name(shadow_tlb_get_what(paddr, set_index, slot_index)); } - m_t.get_root_hash(hash); + if (paddr >= AR_PMAS_START && paddr - AR_PMAS_START < AR_PMAS_LENGTH) { + return pmas_get_what_name(pmas_get_what(paddr)); + } + if (paddr >= AR_SHADOW_UARCH_STATE_START && paddr - AR_SHADOW_UARCH_STATE_START < AR_SHADOW_UARCH_STATE_LENGTH) { + return shadow_uarch_state_get_what_name(shadow_uarch_state_get_what(paddr)); + } + return "memory"; } -machine::hash_type machine::get_merkle_tree_node_hash(uint64_t address, int log2_size, - skip_merkle_tree_update_t /* unused */) const { - return m_t.get_node_hash(address, log2_size); +machine_hash machine::get_node_hash(uint64_t address, int log2_size) const { + if (!update_hash_tree()) { + throw std::runtime_error{"update hash tree failed"}; + } + return get_node_hash(address, log2_size, skip_hash_tree_update); } -machine::hash_type machine::get_merkle_tree_node_hash(uint64_t address, int log2_size) const { - if (!update_merkle_tree()) { - throw std::runtime_error{"error updating Merkle tree"}; - } - return get_merkle_tree_node_hash(address, log2_size, skip_merkle_tree_update); +bool machine::verify_hash_tree() const { + mark_write_tlb_dirty_pages(); + m_ars.mark_always_dirty_pages(); + return m_ht.verify(m_ars); } -bool machine::verify_merkle_tree() const { - return m_t.verify_tree(); +machine::proof_type machine::get_proof(uint64_t address, int log2_size) const { + if (!update_hash_tree()) { + throw std::runtime_error{"update hash tree failed"}; + } + return get_proof(address, log2_size, skip_hash_tree_update); } -machine_merkle_tree::proof_type machine::get_proof(uint64_t address, int log2_size, - skip_merkle_tree_update_t /*unused*/) const { - static_assert(PMA_PAGE_SIZE == machine_merkle_tree::get_page_size(), - "PMA and machine_merkle_tree page sizes must match"); - // Check for valid target node size - if (log2_size > machine_merkle_tree::get_log2_root_size() || - log2_size < machine_merkle_tree::get_log2_word_size()) { - throw std::invalid_argument{"invalid log2_size"}; - } - // Check target address alignment - if ((address & ((~UINT64_C(0)) >> (64 - log2_size))) != 0) { - throw std::invalid_argument{"address not aligned to log2_size"}; - } - // If proof concerns range smaller than a page, we may need to rebuild part - // of the proof from the contents of a page inside some PMA range. - // PMA range starts and lengths are multiple of the page size, which is a - // power of 2. - // The size of the desired range is smaller than the page size, but its - // size is a power of 2, and it is aligned to its size. - // Therefore, it is is either entirely inside a PMA range, - // or entirely outside it. - if (log2_size < machine_merkle_tree::get_log2_page_size()) { - const uint64_t length = UINT64_C(1) << log2_size; - const pma_entry &pma = find_pma_entry(m_merkle_pmas, address, length); - auto scratch = unique_calloc(PMA_PAGE_SIZE); - const unsigned char *page_data = nullptr; - // If the PMA range is empty, we know the desired range is - // entirely outside of any non-pristine PMA. - // Therefore, the entire page where it lies is also pristine - // Otherwise, the entire desired range is inside it. - if (!pma.get_istart_E()) { - const uint64_t page_start_in_range = (address - pma.get_start()) & (~(PMA_PAGE_SIZE - 1)); - auto peek = pma.get_peek(); - if (!peek(pma, *this, page_start_in_range, &page_data, scratch.get())) { - throw std::runtime_error{"PMA peek failed"}; - } - } - return m_t.get_proof(address, log2_size, page_data); - // If proof concerns range bigger than a page, we already have its hash - // stored in the tree itself +template +static inline void foreach_aligned_chunk(uint64_t start, uint64_t length, uint64_t alignment, F f) { + // Optional first chunk brings start to alignment + if (const auto rem = start % alignment; rem != 0) { + const auto first_length = std::min(length, alignment - rem); + f(start, first_length); + start += first_length; + length -= first_length; + } + // Intermediate chunks start aligned and cover exactly alignment bytes + while (length >= alignment) { + f(start, alignment); + start += alignment; + length -= alignment; + } + // Last chunk completes the span + if (length != 0) { + f(start, length); } - return m_t.get_proof(address, log2_size, nullptr); } -machine_merkle_tree::proof_type machine::get_proof(uint64_t address, int log2_size) const { - if (!update_merkle_tree()) { - throw std::runtime_error{"error updating Merkle tree"}; +//??D I think we should have something like this in the address_range interface +//??D Something that combines a check with contains and returns an optional view +static std::optional> ar_host_memory_view(const address_range &ar, uint64_t offset, + uint64_t length) { + if (!ar.contains_relative(offset, length)) { + throw std::logic_error{"invalid access to address range "s.append(ar.get_description())}; + } + if (ar.is_memory()) { + if (ar.get_host_memory() == nullptr) { + throw std::runtime_error{ + "memory address range "s.append(ar.get_description()).append("has no host memory")}; + } + return std::span{ar.get_host_memory() + offset, static_cast(length)}; } - return get_proof(address, log2_size, skip_merkle_tree_update); + return {}; } -void machine::read_memory(uint64_t address, unsigned char *data, uint64_t length) const { +//??D change this to use a span, or perhaps a contiguous_range of byte-like elements +void machine::read_memory(uint64_t paddr, unsigned char *data, uint64_t length) const { if (length == 0) { return; } if (data == nullptr) { throw std::invalid_argument{"invalid data buffer"}; } - const pma_entry &pma = find_pma_entry(m_merkle_pmas, address, length); - if (pma.get_istart_M()) { - memcpy(data, pma.get_memory().get_host_memory() + (address - pma.get_start()), length); - return; - } - auto scratch = unique_calloc(PMA_PAGE_SIZE); - // relative request address inside pma - uint64_t shift = address - pma.get_start(); - // relative page address inside pma - constexpr const auto log2_page_size = PMA_constants::PMA_PAGE_SIZE_LOG2; - uint64_t page_address = (shift >> log2_page_size) << log2_page_size; - // relative request address inside page - shift -= page_address; - - const unsigned char *page_data = nullptr; - auto peek = pma.get_peek(); - - while (length != 0) { - const uint64_t bytes_to_write = std::min(length, PMA_PAGE_SIZE - shift); - // avoid copying to the intermediate buffer when getting the whole page - if (bytes_to_write == PMA_PAGE_SIZE) { - if (!peek(pma, *this, page_address, &page_data, data)) { - throw std::runtime_error{"peek failed"}; - } - if (page_data == nullptr) { - memset(data, 0, bytes_to_write); - } - } else { - if (!peek(pma, *this, page_address, &page_data, scratch.get())) { - throw std::runtime_error{"peek failed"}; + uint64_t gap_start = 0; + auto view = m_ars.hash_tree_view() | + std::views::drop_while([paddr, &gap_start](const address_range &ar) { // Only those that end after paddr + const auto ar_end = ar.get_end(); + if (paddr >= ar_end) { + gap_start = ar_end; + return true; } - if (page_data == nullptr) { - memset(data, 0, bytes_to_write); + return false; + }); + for (const auto &ar : view) { + const auto ar_start = ar.get_start(); + // Write as much as possible from pristine gap between last address range and current address range + if (paddr >= gap_start && paddr < ar_start) { + const auto from_gap = std::min(ar_start - paddr, length); + memset(data, 0, from_gap); + length -= from_gap; + paddr += from_gap; + data += from_gap; + } + gap_start = ar.get_end(); + // Write as much as possible from current address range + if (paddr >= ar_start && paddr < gap_start) { + const auto from_ar = std::min(gap_start - paddr, length); + auto data_view = std::span{data, static_cast(from_ar)}; + auto ar_view = ar_host_memory_view(ar, paddr - ar_start, from_ar); + if (ar_view) { + std::ranges::copy(*ar_view, data_view.begin()); } else { - memcpy(data, page_data + shift, bytes_to_write); + std::ranges::fill(data_view, 0); } + length -= from_ar; + if (length == 0) { + return; + } + paddr += from_ar; + data += from_ar; } - - page_address += PMA_PAGE_SIZE; - length -= bytes_to_write; - data += bytes_to_write; - shift = 0; + } + if (length != 0) { + // Finish up with pristine padding after last address range + memset(data, 0, length); } } -void machine::write_memory(uint64_t address, const unsigned char *data, uint64_t length) { +//??D change this to use a span, or perhaps a contiguous_range of byte-like elements +void machine::write_memory(uint64_t paddr, const unsigned char *data, uint64_t length) { if (length == 0) { return; } if (data == nullptr) { throw std::invalid_argument{"invalid data buffer"}; } - pma_entry &pma = find_pma_entry(m_merkle_pmas, address, length); - if (!pma.get_istart_M() || pma.get_istart_E()) { - throw std::invalid_argument{"address range not entirely in memory PMA"}; + auto &ar = m_ars.find(paddr, length); + const auto *ar_descr = ar.get_description(); + if (ar.is_device()) { + throw std::invalid_argument{"attempted write to device memory range "s.append(ar_descr)}; + } + if (!ar.is_memory()) { + throw std::invalid_argument{"attempted write is not entirely within single memory range"}; + } + if (ar.is_host_read_only()) { + throw std::invalid_argument{"attempted write to host-read-only memory range "s.append(ar_descr)}; + } + // Handle special case for writing to shadow memory, allowing manual snapshots + // for machines with shared layouts via read_memory()/write_memory() + if (paddr == AR_SHADOW_STATE_START && length == AR_SHADOW_STATE_LENGTH) { + // Save the current processor state for potential rollback + static const auto s = *m_s; + // Overwrite the processor shadow state with the provided data + static_assert(AR_SHADOW_STATE_LENGTH == sizeof(m_s->shadow)); + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + memcpy(reinterpret_cast(&m_s->shadow), data, sizeof(m_s->shadow)); + // Ensure processor state rollback in case of failure during subsequent operations + auto state_reverter = scope_fail([&] { *m_s = s; }); + // Ensure the new processor shadow state is consistent + validate_processor_shadow(m_r.skip_version_check); + // Reinitialize the hot TLB to reflect changes in the shadow TLB and verify consistency + init_hot_tlb_contents(); + return; } - pma.write_memory(address, data, length); + // Allow special case for writing the revert root hash + if (paddr == AR_SHADOW_REVERT_ROOT_HASH_START && length == sizeof(machine_hash)) { + ; + } else if (pmas_is_protected(ar.get_driver_id())) { + throw std::invalid_argument{"attempted write to protected memory range "s.append(ar_descr)}; + } + //??D In C++23, change this to use std::span and std::views::chunk std::views::drop/take and std::views::chain + foreach_aligned_chunk(paddr, length, AR_PAGE_SIZE, [&ar, paddr, data](auto chunk_start, auto chunk_length) { + const auto *src = data + (chunk_start - paddr); + const auto offset = chunk_start - ar.get_start(); + auto *dest = ar.get_host_memory() + offset; + if (memcmp(dest, src, chunk_length) != 0) { + // Page is different, we have to copy memory + memcpy(dest, src, chunk_length); + ar.get_dirty_page_tree().mark_dirty_page_and_up(offset); + } + }); } -void machine::fill_memory(uint64_t address, uint8_t data, uint64_t length) { +void machine::fill_memory(uint64_t paddr, uint8_t val, uint64_t length) { if (length == 0) { return; } - pma_entry &pma = find_pma_entry(m_merkle_pmas, address, length); - if (!pma.get_istart_M() || pma.get_istart_E()) { - throw std::invalid_argument{"address range not entirely in memory PMA"}; + auto &ar = m_ars.find(paddr, length); + const auto *ar_descr = ar.get_description(); + if (ar.is_device()) { + throw std::invalid_argument{"attempted fill to device memory range "s.append(ar_descr)}; + } + if (!ar.is_memory()) { + throw std::invalid_argument{"attempted fill is not entirely within single memory range"}; + } + if (ar.is_host_read_only()) { + throw std::invalid_argument{"attempted fill to host-read-only memory range "s.append(ar_descr)}; } - pma.fill_memory(address, data, length); + if (pmas_is_protected(ar.get_driver_id())) { + throw std::invalid_argument{"attempted fill to protected memory range "s.append(ar_descr)}; + } + // The case of filling a range with zeros is special and optimized for uarch reset + //??D In C++23, change this to use std::span and std::views::chunk std::views::drop/take and std::views::chain + foreach_aligned_chunk(paddr, length, AR_PAGE_SIZE, [&ar, val](auto chunk_start, auto chunk_length) { + const auto offset = chunk_start - ar.get_start(); + const auto dest = ar.get_host_memory() + offset; + if (val != 0 || !is_pristine(std::span{dest, static_cast(chunk_length)})) { + memset(dest, val, chunk_length); + ar.get_dirty_page_tree().mark_dirty_page_and_up(offset); + } + }); } void machine::read_virtual_memory(uint64_t vaddr_start, unsigned char *data, uint64_t length) { @@ -2003,17 +1658,17 @@ void machine::read_virtual_memory(uint64_t vaddr_start, unsigned char *data, uin throw std::invalid_argument{"invalid data buffer"}; } const uint64_t vaddr_limit = vaddr_start + length; - const uint64_t vaddr_page_start = vaddr_start & ~(PMA_PAGE_SIZE - 1); // align page backward - const uint64_t vaddr_page_limit = (vaddr_limit + PMA_PAGE_SIZE - 1) & ~(PMA_PAGE_SIZE - 1); // align page forward + const uint64_t vaddr_page_start = vaddr_start & ~(AR_PAGE_SIZE - 1); // align page backward + const uint64_t vaddr_page_limit = (vaddr_limit + AR_PAGE_SIZE - 1) & ~(AR_PAGE_SIZE - 1); // align page forward // copy page by page, because we need to perform address translation again for each page - for (uint64_t vaddr_page = vaddr_page_start; vaddr_page < vaddr_page_limit; vaddr_page += PMA_PAGE_SIZE) { + for (uint64_t vaddr_page = vaddr_page_start; vaddr_page < vaddr_page_limit; vaddr_page += AR_PAGE_SIZE) { uint64_t paddr_page = 0; if (!cartesi::translate_virtual_address(a, &paddr_page, vaddr_page, PTE_XWR_R_SHIFT)) { throw std::domain_error{"page fault"}; } uint64_t paddr = paddr_page; uint64_t vaddr = vaddr_page; - uint64_t chunklen = std::min(PMA_PAGE_SIZE, vaddr_limit - vaddr); + uint64_t chunklen = std::min(AR_PAGE_SIZE, vaddr_limit - vaddr); if (vaddr_page < vaddr_start) { const uint64_t off = vaddr_start - vaddr_page; paddr += off; @@ -2034,10 +1689,10 @@ void machine::write_virtual_memory(uint64_t vaddr_start, const unsigned char *da throw std::invalid_argument{"invalid data buffer"}; } const uint64_t vaddr_limit = vaddr_start + length; - const uint64_t vaddr_page_start = vaddr_start & ~(PMA_PAGE_SIZE - 1); // align page backward - const uint64_t vaddr_page_limit = (vaddr_limit + PMA_PAGE_SIZE - 1) & ~(PMA_PAGE_SIZE - 1); // align page forward + const uint64_t vaddr_page_start = vaddr_start & ~(AR_PAGE_SIZE - 1); // align page backward + const uint64_t vaddr_page_limit = (vaddr_limit + AR_PAGE_SIZE - 1) & ~(AR_PAGE_SIZE - 1); // align page forward // copy page by page, because we need to perform address translation again for each page - for (uint64_t vaddr_page = vaddr_page_start; vaddr_page < vaddr_page_limit; vaddr_page += PMA_PAGE_SIZE) { + for (uint64_t vaddr_page = vaddr_page_start; vaddr_page < vaddr_page_limit; vaddr_page += AR_PAGE_SIZE) { uint64_t paddr_page = 0; // perform address translation using read access mode, // so we can write any reachable virtual memory range @@ -2046,7 +1701,7 @@ void machine::write_virtual_memory(uint64_t vaddr_start, const unsigned char *da } uint64_t paddr = paddr_page; uint64_t vaddr = vaddr_page; - uint64_t chunklen = std::min(PMA_PAGE_SIZE, vaddr_limit - vaddr); + uint64_t chunklen = std::min(AR_PAGE_SIZE, vaddr_limit - vaddr); if (vaddr_page < vaddr_start) { const uint64_t off = vaddr_start - vaddr_page; paddr += off; @@ -2068,29 +1723,62 @@ uint64_t machine::translate_virtual_address(uint64_t vaddr) { return paddr; } -uint64_t machine::read_word(uint64_t word_address) const { +uint64_t machine::read_word(uint64_t paddr) const { // Make sure address is aligned - if ((word_address & (PMA_WORD_SIZE - 1)) != 0) { - throw std::invalid_argument{"address not aligned"}; - } - const pma_entry &pma = find_pma_entry(word_address); - // ??D We should split peek into peek_word and peek_page - // for performance. On the other hand, this function - // will almost never be used, so one wonders if it is worth it... - auto scratch = unique_calloc(PMA_PAGE_SIZE); - const unsigned char *page_data = nullptr; - const uint64_t page_start_in_range = (word_address - pma.get_start()) & (~(PMA_PAGE_SIZE - 1)); - auto peek = pma.get_peek(); - if (!peek(pma, *this, page_start_in_range, &page_data, scratch.get())) { - throw std::invalid_argument{"peek failed"}; - } - // If peek returns a page, read from it - if (page_data != nullptr) { - const uint64_t word_start_in_range = (word_address - pma.get_start()) & (PMA_PAGE_SIZE - 1); - return aliased_aligned_read(page_data + word_start_in_range); - // Otherwise, page is always pristine - } - return 0; + if ((paddr & (sizeof(uint64_t) - 1)) != 0) { + throw std::domain_error{"attempted misaligned read from word"}; + } + // Use read_memory + alignas(sizeof(uint64_t)) std::array scratch{}; + read_memory(paddr, scratch.data(), scratch.size()); + return aliased_aligned_read(scratch.data()); +} + +void machine::write_word(uint64_t paddr, uint64_t val) { + // Make sure address is aligned + if ((paddr & (sizeof(uint64_t) - 1)) != 0) { + throw std::domain_error{"attempted misaligned write to word"}; + } + // If in shadow, forward to write_reg + if (paddr >= AR_SHADOW_REGISTERS_START && paddr - AR_SHADOW_REGISTERS_START < AR_SHADOW_REGISTERS_LENGTH) { + auto reg = shadow_registers_get_what(paddr); + if (reg == shadow_registers_what::unknown_) { + throw std::runtime_error("unhandled write to shadow state"); + } + write_reg(machine_reg_enum(reg), val); + return; + } + // If in uarch shadow, forward to write_reg + if (paddr >= AR_SHADOW_UARCH_STATE_START && paddr - AR_SHADOW_UARCH_STATE_START < AR_SHADOW_UARCH_STATE_LENGTH) { + auto reg = shadow_uarch_state_get_what(paddr); + if (reg == shadow_uarch_state_what::unknown_) { + throw std::runtime_error("unhandled write to shadow uarch state"); + } + write_reg(machine_reg_enum(reg), val); + return; + } + auto &ar = m_ars.find(paddr, sizeof(uint64_t)); + if (!ar.is_memory() || ar.get_host_memory() == nullptr) { + std::ostringstream err; + err << "attempted memory write to " << ar.get_description() << " at address 0x" << std::hex << paddr << "(" + << std::dec << paddr << ")"; + throw std::runtime_error{err.str()}; + } + if (ar.is_host_read_only()) { + std::ostringstream err; + err << "attempted memory write to read-only " << ar.get_description() << " at address 0x" << std::hex << paddr + << "(" << std::dec << paddr << ")"; + throw std::runtime_error{err.str()}; + } + if (pmas_is_protected(ar.get_driver_id())) { + std::ostringstream err; + err << "attempted memory word write to protected memory range " << ar.get_description() << " at address 0x" + << std::hex << paddr << "(" << std::dec << paddr << ")"; + throw std::runtime_error{err.str()}; + } + const auto offset = paddr - ar.get_start(); + aliased_aligned_write(ar.get_host_memory() + offset, val); + ar.get_dirty_page_tree().mark_dirty_page_and_up(offset); } void machine::send_cmio_response(uint16_t reason, const unsigned char *data, uint64_t length) { @@ -2100,76 +1788,83 @@ void machine::send_cmio_response(uint16_t reason, const unsigned char *data, uin access_log machine::log_send_cmio_response(uint16_t reason, const unsigned char *data, uint64_t length, const access_log::type &log_type) { - hash_type root_hash_before; - get_root_hash(root_hash_before); + if (m_c.hash_tree.hash_function != hash_function_type::keccak256) { + throw std::runtime_error{ + "access logs can only be used with hash tree configured with Keccak-256 hash function"}; + } + auto root_hash_before = get_root_hash(); access_log log(log_type); // Call send_cmio_response with the recording state accessor - record_send_cmio_state_access a(*this, log); - a.push_bracket(bracket_type::begin, "send cmio response"); - cartesi::send_cmio_response(a, reason, data, length); - a.push_bracket(bracket_type::end, "send cmio response"); - // Verify access log before returning - hash_type root_hash_after; - update_merkle_tree(); - get_root_hash(root_hash_after); + const record_send_cmio_state_access a(*this, log); + { + [[maybe_unused]] auto note = a.make_scoped_note("send_cmio_response"); + cartesi::send_cmio_response(a, reason, data, length); + } + auto root_hash_after = get_root_hash(); verify_send_cmio_response(reason, data, length, root_hash_before, log, root_hash_after); return log; } void machine::verify_send_cmio_response(uint16_t reason, const unsigned char *data, uint64_t length, - const hash_type &root_hash_before, const access_log &log, const hash_type &root_hash_after) { - // There must be at least one access in log - if (log.get_accesses().empty()) { - throw std::invalid_argument{"too few accesses in log"}; - } - replay_send_cmio_state_access::context context(log, root_hash_before); + const machine_hash &root_hash_before, const access_log &log, const machine_hash &root_hash_after) { + replay_send_cmio_state_access::context context{log, root_hash_before, hash_function_type::keccak256}; // Verify all intermediate state transitions replay_send_cmio_state_access a(context); cartesi::send_cmio_response(a, reason, data, length); a.finish(); - // Make sure the access log ends at the same root hash as the state - hash_type obtained_root_hash; - a.get_root_hash(obtained_root_hash); + auto obtained_root_hash = a.get_root_hash(); if (obtained_root_hash != root_hash_after) { throw std::invalid_argument{"mismatch in root hash after replay"}; } } void machine::reset_uarch() { - uarch_state_access a(m_uarch.get_state(), get_state()); - uarch_reset_state(a); + if (m_c.hash_tree.hash_function != hash_function_type::keccak256) { + throw std::runtime_error{ + "microarchitecture can only be used with hash tree configured with Keccak-256 hash function"}; + } + write_reg(reg::uarch_halt_flag, UARCH_HALT_FLAG_INIT); + write_reg(reg::uarch_pc, UARCH_PC_INIT); + write_reg(reg::uarch_cycle, UARCH_CYCLE_INIT); + // General purpose registers + for (int i = 1; i < UARCH_X_REG_COUNT; i++) { + write_reg(machine_reg_enum(reg::uarch_x0, i), UARCH_X_INIT); + } + // Reset RAM to initial state + write_memory(AR_UARCH_RAM_START, uarch_pristine_ram, uarch_pristine_ram_len); + if (AR_UARCH_RAM_LENGTH > uarch_pristine_ram_len) { + fill_memory(AR_UARCH_RAM_START + uarch_pristine_ram_len, 0, AR_UARCH_RAM_LENGTH - uarch_pristine_ram_len); + } } access_log machine::log_reset_uarch(const access_log::type &log_type) { - hash_type root_hash_before; - get_root_hash(root_hash_before); + if (m_c.hash_tree.hash_function != hash_function_type::keccak256) { + throw std::runtime_error{ + "microarchitecture can only be used with hash tree configured with Keccak-256 hash function"}; + } + const machine_hash root_hash_before = get_root_hash(); // Call uarch_reset_state with a uarch_record_state_access object - uarch_record_state_access a(m_uarch.get_state(), *this, log_type); - a.push_bracket(bracket_type::begin, "reset uarch state"); - uarch_reset_state(a); - a.push_bracket(bracket_type::end, "reset uarch state"); - // Verify access log before returning - hash_type root_hash_after; - update_merkle_tree(); - get_root_hash(root_hash_after); - verify_reset_uarch(root_hash_before, *a.get_log(), root_hash_after); - return std::move(*a.get_log()); + access_log log(log_type); + uarch_record_state_access a(*this, log); + { + [[maybe_unused]] auto note = a.make_scoped_note("reset_uarch_state"); + uarch_reset_state(a); + } + const auto root_hash_after = get_root_hash(); + verify_reset_uarch(root_hash_before, log, root_hash_after); + return log; } -void machine::verify_reset_uarch(const hash_type &root_hash_before, const access_log &log, - const hash_type &root_hash_after) { - // There must be at least one access in log - if (log.get_accesses().empty()) { - throw std::invalid_argument{"too few accesses in log"}; - } +void machine::verify_reset_uarch(const machine_hash &root_hash_before, const access_log &log, + const machine_hash &root_hash_after) { // Verify all intermediate state transitions - uarch_replay_state_access a(log, root_hash_before); + uarch_replay_state_access::context context{log, root_hash_before}; + uarch_replay_state_access a(context); uarch_reset_state(a); a.finish(); // Make sure the access log ends at the same root hash as the state - hash_type obtained_root_hash; - a.get_root_hash(obtained_root_hash); + const machine_hash obtained_root_hash = a.get_root_hash(); if (obtained_root_hash != root_hash_after) { throw std::invalid_argument{"mismatch in root hash after replay"}; } @@ -2179,39 +1874,40 @@ void machine::verify_reset_uarch(const hash_type &root_hash_before, const access extern template UArchStepStatus uarch_step(uarch_record_state_access &a); access_log machine::log_step_uarch(const access_log::type &log_type) { - if (m_uarch.get_state().ram.get_istart_E()) { - throw std::runtime_error("microarchitecture RAM is not present"); + if (read_reg(reg::iunrep) != 0) { + throw std::runtime_error("microarchitecture cannot be used with unreproducible machines"); + } + if (m_c.hash_tree.hash_function != hash_function_type::keccak256) { + throw std::runtime_error{ + "microarchitecture can only be used with hash tree configured with Keccak-256 hash function"}; } - hash_type root_hash_before; - get_root_hash(root_hash_before); + auto root_hash_before = get_root_hash(); + access_log log(log_type); // Call interpret with a logged state access object - uarch_record_state_access a(m_uarch.get_state(), *this, log_type); - a.push_bracket(bracket_type::begin, "step"); - uarch_step(a); - a.push_bracket(bracket_type::end, "step"); + const uarch_record_state_access a(*this, log); + { + [[maybe_unused]] auto note = a.make_scoped_note("step"); + uarch_step(a); + } // Verify access log before returning - hash_type root_hash_after; - get_root_hash(root_hash_after); - verify_step_uarch(root_hash_before, *a.get_log(), root_hash_after); - return std::move(*a.get_log()); + auto root_hash_after = get_root_hash(); + os_silence_putchar(m_r.htif.no_console_putchar); + verify_step_uarch(root_hash_before, log, root_hash_after); + return log; } // Declaration of explicit instantiation in module uarch-step.cpp extern template UArchStepStatus uarch_step(uarch_replay_state_access &a); -void machine::verify_step_uarch(const hash_type &root_hash_before, const access_log &log, - const hash_type &root_hash_after) { - // There must be at least one access in log - if (log.get_accesses().empty()) { - throw std::invalid_argument{"too few accesses in log"}; - } +void machine::verify_step_uarch(const machine_hash &root_hash_before, const access_log &log, + const machine_hash &root_hash_after) { // Verify all intermediate state transitions - uarch_replay_state_access a(log, root_hash_before); + uarch_replay_state_access::context context{log, root_hash_before}; + uarch_replay_state_access a(context); uarch_step(a); a.finish(); // Make sure the access log ends at the same root hash as the state - hash_type obtained_root_hash; - a.get_root_hash(obtained_root_hash); + auto obtained_root_hash = a.get_root_hash(); if (obtained_root_hash != root_hash_after) { throw std::invalid_argument{"mismatch in root hash after replay"}; } @@ -2223,64 +1919,486 @@ machine_config machine::get_default_config() { // NOLINTNEXTLINE(readability-convert-member-functions-to-static) uarch_interpreter_break_reason machine::run_uarch(uint64_t uarch_cycle_end) { + if (m_c.hash_tree.hash_function != hash_function_type::keccak256) { + throw std::runtime_error{ + "microarchitecture can only be used with hash tree configured with Keccak-256 hash function"}; + } if (read_reg(reg::iunrep) != 0) { throw std::runtime_error("microarchitecture cannot be used with unreproducible machines"); } - if (m_uarch.get_state().ram.get_istart_E()) { - throw std::runtime_error("microarchitecture RAM is not present"); - } - uarch_state_access a(m_uarch.get_state(), get_state()); + const uarch_state_access a(*this); + os_silence_putchar(m_r.htif.no_console_putchar); return uarch_interpret(a, uarch_cycle_end); } interpreter_break_reason machine::log_step(uint64_t mcycle_count, const std::string &filename) { - if (!update_merkle_tree()) { - throw std::runtime_error{"error updating Merkle tree"}; - } - // Ensure that the microarchitecture is reset - auto current_uarch_state_hash = - get_merkle_tree_node_hash(PMA_SHADOW_UARCH_STATE_START, UARCH_STATE_LOG2_SIZE, skip_merkle_tree_update); - if (current_uarch_state_hash != get_uarch_pristine_state_hash()) { + if (read_reg(reg::uarch_cycle) != 0) { throw std::runtime_error{"microarchitecture is not reset"}; } - hash_type root_hash_before; - get_root_hash(root_hash_before); - record_step_state_access::context context(filename); + auto root_hash_before = get_root_hash(); + record_step_state_access::context context(filename, m_c.hash_tree.hash_function); record_step_state_access a(context, *this); - uint64_t mcycle_end{}; - if (__builtin_add_overflow(a.read_mcycle(), mcycle_count, &mcycle_end)) { - mcycle_end = UINT64_MAX; - } + const uint64_t mcycle_end = saturating_add(a.read_mcycle(), mcycle_count); + os_silence_putchar(m_r.htif.no_console_putchar); auto break_reason = interpret(a, mcycle_end); a.finish(); - hash_type root_hash_after; - get_root_hash(root_hash_after); + auto root_hash_after = get_root_hash(); verify_step(root_hash_before, filename, mcycle_count, root_hash_after); return break_reason; } -interpreter_break_reason machine::verify_step(const hash_type &root_hash_before, const std::string &filename, - uint64_t mcycle_count, const hash_type &root_hash_after) { - auto data_length = os_get_file_length(filename.c_str(), "step log file"); - auto *data = os_map_file(filename.c_str(), data_length, false /* not shared */); +interpreter_break_reason machine::verify_step(const machine_hash &root_hash_before, const std::string &filename, + uint64_t mcycle_count, const machine_hash &root_hash_after) { + auto data_length = os::file_size(filename); + auto mapped_data = os::mapped_memory(data_length, os::mapped_memory_flags{}, filename); replay_step_state_access::context context; - replay_step_state_access a(context, data, data_length, root_hash_before); - uint64_t mcycle_end{}; - if (__builtin_add_overflow(a.read_mcycle(), mcycle_count, &mcycle_end)) { - mcycle_end = UINT64_MAX; - } + replay_step_state_access a(context, mapped_data.get_ptr(), data_length, root_hash_before); + const uint64_t mcycle_end = saturating_add(a.read_mcycle(), mcycle_count); auto break_reason = interpret(a, mcycle_end); a.finish(root_hash_after); - os_unmap_file(data, data_length); return break_reason; } interpreter_break_reason machine::run(uint64_t mcycle_end) { - if (mcycle_end < read_reg(reg::mcycle)) { + const auto mcycle = read_reg(reg::mcycle); + if (mcycle_end < mcycle) { throw std::invalid_argument{"mcycle is past"}; } + const auto uarch_cycle = read_reg(reg::uarch_cycle); + if (uarch_cycle != 0) { + throw std::invalid_argument{"microarchitecture is not reset"}; + } const state_access a(*this); + os_silence_putchar(m_r.htif.no_console_putchar); return interpret(a, mcycle_end); } +//??D How come this function seems to never signal we have an inteerrupt??? +std::pair machine::poll_external_interrupts(uint64_t mcycle, uint64_t mcycle_max) { + const auto status = execute_status::success; + // Only poll external interrupts if we are in unreproducible mode + if (unlikely(m_s->shadow.registers.iunrep)) { + // Convert the relative interval of cycles we can wait to the interval of host time we can wait + uint64_t timeout_us = (mcycle_max - mcycle) / RTC_CYCLES_PER_US; + int64_t start_us = 0; + if (timeout_us > 0) { + start_us = os_now_us(); + } + const state_access a(*this); + device_state_access da(a, mcycle); + // Poll virtio for events (e.g console stdin, network sockets) + // Timeout may be decremented in case a device has deadline timers (e.g network device) + if (has_virtio_devices() && has_virtio_console()) { // VirtIO + VirtIO console + poll_virtio_devices(&timeout_us, &da); + // VirtIO console device will poll TTY + } else if (has_virtio_devices()) { // VirtIO without a console + poll_virtio_devices(&timeout_us, &da); + if (has_htif_console()) { // VirtIO + HTIF console + // Poll tty without waiting more time, because the pool above should have waited enough time + os_poll_tty(0); + } + } else if (has_htif_console()) { // Only HTIF console + os_poll_tty(timeout_us); + } else if (timeout_us > 0) { // No interrupts to check, just keep the CPU idle + os_sleep_us(timeout_us); + } + // If timeout is greater than zero, we should also increment mcycle relative to the elapsed time + if (timeout_us > 0) { + const int64_t end_us = os_now_us(); + const uint64_t elapsed_us = static_cast(std::max(end_us - start_us, 0)); + const uint64_t next_mcycle = mcycle + (elapsed_us * RTC_CYCLES_PER_US); + mcycle = std::min(std::max(next_mcycle, mcycle), mcycle_max); + } + } + return {mcycle, status}; +} + +std::string machine::get_counter_key(const char *name, const char *domain) { + if (name == nullptr) { + throw std::invalid_argument{"invalid name argument"}; + } + std::string key{(domain != nullptr) ? domain : name}; + if (domain != nullptr) { + key.append(name); + } + return key; +} + +void machine::increment_counter(const char *name, const char *domain) { + ++m_counters[get_counter_key(name, domain)]; +} + +uint64_t machine::read_counter(const char *name, const char *domain) { + return m_counters[get_counter_key(name, domain)]; +} + +void machine::write_counter(uint64_t val, const char *name, const char *domain) { + m_counters[get_counter_key(name, domain)] = val; +} + +mcycle_root_hashes machine::collect_mcycle_root_hashes(uint64_t mcycle_end, uint64_t mcycle_period, + uint64_t mcycle_phase, int32_t log2_bundle_mcycle_count, + const std::optional &previous_back_tree) { + const uint64_t mcycle_start = read_reg(reg::mcycle); + + // Check preconditions + if (mcycle_end < mcycle_start) { + throw std::runtime_error{"mcycle is past"}; + } + if (mcycle_period == 0) { + throw std::runtime_error{"mcycle_period cannot be 0"}; + } + if (mcycle_phase >= mcycle_period) { + throw std::runtime_error{"mcycle_phase must be in {0, ..., mcycle_period-1}"}; + } + if (previous_back_tree.has_value() && + (previous_back_tree->get_log2_max_leaves() != log2_bundle_mcycle_count || + previous_back_tree->get_hash_function() != m_c.hash_tree.hash_function)) { + throw std::runtime_error{"back tree context is incompatible"}; + } + if (read_reg(reg::iunrep) != 0) { + throw std::runtime_error{"cannot collect hashes from unreproducible machines"}; + } + if (m_r.soft_yield) { + throw std::runtime_error{"cannot collect hashes when soft yield is enabled"}; + } + if (!update_hash_tree()) { + throw std::runtime_error{"update hash tree failed"}; + } + if (read_reg(reg::uarch_cycle) != 0) { + throw std::runtime_error{"microarchitecture is not reset"}; + } + + // If the collection loop does not advance mcycle, keep the original mcycle_phase + // and set the break reason to indicate the target mcycle was reached + mcycle_root_hashes result; + result.mcycle_phase = mcycle_phase; + result.break_reason = interpreter_break_reason::reached_target_mcycle; + + // Check halted and yielded break reasons first to behave with same priority as the interpreter + if (read_reg(reg::iflags_H) != 0) { + result.break_reason = interpreter_break_reason::halted; + return result; + } + if (read_reg(reg::iflags_Y) != 0) { + result.break_reason = interpreter_break_reason::yielded_manually; + return result; + } + + // Initialize back tree + if (previous_back_tree.has_value()) { + result.back_tree = *previous_back_tree; + } else { + result.back_tree = back_merkle_tree{log2_bundle_mcycle_count, m_c.hash_tree.hash_function}; + } + auto &back_tree = result.back_tree.value(); + + collect_mcycle_hashes_state_access::context context{}; + const collect_mcycle_hashes_state_access a(context, *this); + os_silence_putchar(m_r.htif.no_console_putchar); + + // Reserve space before entering the loop to minimize dynamic memory allocations, + // the reserved sizes below are based on empirical benchmarks to balance performance and memory usage, + // and are clamped to avoid over-allocation + const uint64_t max_result_hashes = + ((mcycle_end - mcycle_start) / (mcycle_period * (UINT64_C(1) << log2_bundle_mcycle_count))) + 1; + result.hashes.reserve(std::clamp(max_result_hashes, 1, 16384)); + context.dirty_pages.reserve(std::clamp(mcycle_period * 4, 16, 4096)); + + // The first iteration will target the next mcycle_period boundary, + // but will not exceed mcycle_end if that limit is reached first + uint64_t mcycle_target = saturating_add(mcycle_start, mcycle_period - mcycle_phase, mcycle_end); + uint64_t mcycle_reached = read_reg(reg::mcycle); + bool at_fixed_point = false; + + // Run until reaching next mcycle target + while (mcycle_reached < mcycle_target) { + assert(!at_fixed_point); + + // Attempt to execute until finishing this period + result.break_reason = interpret(a, mcycle_target); + + // Mark dirty pages tracked by the context + for (const uint64_t paddr_page : context.dirty_pages) { + auto &ar = find_address_range(paddr_page, HASH_TREE_PAGE_SIZE); + ar.get_dirty_page_tree().mark_dirty_pages_and_up(paddr_page - ar.get_start(), HASH_TREE_PAGE_SIZE); + } + context.dirty_pages.clear(); + + // Update hash tree and retrieve root hash + if (!m_ht.update(m_ars)) { + throw std::runtime_error{"update hash tree failed"}; + } + + mcycle_reached = read_reg(reg::mcycle); + + // Check if we reached a fixed point + at_fixed_point = result.break_reason == interpreter_break_reason::halted || + result.break_reason == interpreter_break_reason::yielded_manually || mcycle_reached == UINT64_MAX; + + // Compute the new phase + result.mcycle_phase = (((mcycle_reached - mcycle_start) % mcycle_period) + mcycle_phase) % mcycle_period; + + // Add the current root hash to the back tree whenever we reach a period boundary or a fixed point + // This ensures we only append at the correct intervals, even when mcycle_end does not align with the period + if (result.mcycle_phase == 0 || at_fixed_point) { + // Append root hash relative to this period to the result + back_tree.push_back(m_ht.get_root_hash()); + + // When back tree is full, we can append the bundled root hash and reset it + if (back_tree.full()) { + result.hashes.emplace_back(back_tree.get_root_hash()); + back_tree.clear(); + } + } + + // If last iteration broke due to any other reason than reaching its target mcycle, we are done + if (at_fixed_point || result.break_reason != interpreter_break_reason::reached_target_mcycle) { + break; + } + + // The next iteration will target the next mcycle_period boundary, + // but will not exceed mcycle_end if that limit is reached first + mcycle_target = saturating_add(mcycle_target, mcycle_period, mcycle_end); + } + + // If the machine yielded manually or halted, then append bundled root hashes with padding + if (at_fixed_point && log2_bundle_mcycle_count > 0) { + // Construct pad tree containing repetitions of the current root hash + const auto pad_hashes = back_merkle_tree::make_pad_hashes(m_ht.get_root_hash(), log2_bundle_mcycle_count, + m_c.hash_tree.hash_function); + + // Pad back tree when partially filled and append its bundled root hash + if (!back_tree.empty()) { + assert(!back_tree.full()); + back_tree.pad_back(back_tree.get_remaining_leaf_count(), pad_hashes); + result.hashes.emplace_back(back_tree.get_root_hash()); + back_tree.clear(); + } + + // Append bundled root hash containing only repetitions of the last root hash + result.hashes.emplace_back(pad_hashes[log2_bundle_mcycle_count]); + } + + // There is no need to return the back tree when it's empty + if (back_tree.empty()) { + result.back_tree.reset(); + } + + return result; +} + +uarch_cycle_root_hashes machine::collect_uarch_cycle_root_hashes(uint64_t mcycle_end, + int32_t log2_bundle_uarch_cycle_count) { + const uint64_t mcycle_start = read_reg(reg::mcycle); + + // Check preconditions + if (mcycle_end < mcycle_start) { + throw std::runtime_error{"mcycle is past"}; + } + if (read_reg(reg::iunrep) != 0) { + throw std::runtime_error{"cannot collect hashes from unreproducible machines"}; + } + if (m_r.soft_yield) { + throw std::runtime_error{"cannot collect hashes when soft yield is enabled"}; + } + if (m_c.hash_tree.hash_function != hash_function_type::keccak256) { + throw std::runtime_error{ + "microarchitecture can only be used with hash tree configured with Keccak-256 hash function"}; + } + if (!update_hash_tree()) { + throw std::runtime_error{"update hash tree failed"}; + } + if (read_reg(reg::uarch_cycle) != 0) { + throw std::runtime_error{"microarchitecture is not reset"}; + } + + // If the collection loop does not advance mcycle, set the break reason to indicate the target mcycle was reached + uarch_cycle_root_hashes result; + result.break_reason = interpreter_break_reason::reached_target_mcycle; + + // Initialize back tree + back_merkle_tree back_tree(log2_bundle_uarch_cycle_count, m_c.hash_tree.hash_function); + + hash_tree::dirty_words_type reset_dirty_words; + collect_uarch_cycle_hashes_state_access::context context{}; + const collect_uarch_cycle_hashes_state_access a(context, *this); + os_silence_putchar(m_r.htif.no_console_putchar); + + // Reserve space before entering the loop to minimize dynamic memory allocations, + // the reserved sizes below are based on empirical benchmarks to balance performance and memory usage + const uint64_t mcycle_count = mcycle_end - mcycle_start; + result.hashes.reserve(mcycle_count * 512); + result.reset_indices.reserve(mcycle_count); + context.dirty_words.reserve(8); + reset_dirty_words.reserve(64); + + // The first iteration will target the next mcycle, + // but will not exceed mcycle_end if that limit is reached first + uint64_t mcycle_target = saturating_add(mcycle_start, UINT64_C(1), mcycle_end); + uint64_t mcycle_reached = read_reg(reg::mcycle); + + // In case we start at fixed point, we will attempt to execute one extra mcycle which + bool at_fixed_point = read_reg(reg::iflags_H) != 0 || read_reg(reg::iflags_Y) != 0 || mcycle_start == UINT64_MAX; + if (at_fixed_point) { + mcycle_target = mcycle_reached; + } + + // Run until reaching next mcycle target, or one more time when at a fixed point + while (mcycle_reached < mcycle_target || at_fixed_point) { + uint64_t uarch_cycle_reached = read_reg(reg::uarch_cycle); + assert(back_tree.empty()); + assert(uarch_cycle_reached == 0); + + // Add one hash after each uarch cycle + for (uint64_t uarch_cycle_target = 1; // First uarch cycle target should always be 1 + uarch_cycle_reached < uarch_cycle_target; // Run until reaching uarch cycle target + ++uarch_cycle_target) { + // Attempt to advance to the next uarch cycle + const auto uarch_break_reason = uarch_interpret(a, uarch_cycle_target); + uarch_cycle_reached = read_reg(reg::uarch_cycle); + + // Update hash tree and retrieve root hash + if (!m_ht.update_words(m_ars, context.dirty_words)) { + throw std::runtime_error{"update hash tree failed"}; + } + + // Track dirty uarch words to be reset later + for (auto word_paddr : context.dirty_words) { + if (word_paddr >= UARCH_STATE_START_ADDRESS && + word_paddr - UARCH_STATE_START_ADDRESS < UARCH_STATE_LENGTH) { + reset_dirty_words.insert(word_paddr); + } + } + context.dirty_words.clear(); + + // If the uarch cycle counter overflows, this is catastrophic and machine execution cannot continue + if (uarch_break_reason == uarch_interpreter_break_reason::cycle_overflow) { + throw std::runtime_error{"uarch reached its maximum cycle and cannot advance more uarch cycles"}; + } + // If the uarch is halted, we can break the loop + if (uarch_break_reason == uarch_interpreter_break_reason::uarch_halted) { + break; + } + + // Sanity check to ensure the loop is working correctly, this should always be true + assert(uarch_cycle_reached == uarch_cycle_target); + assert(uarch_break_reason == uarch_interpreter_break_reason::reached_target_cycle); + + // Append root hash to the result + back_tree.push_back(m_ht.get_root_hash()); + + // When back tree is full, we can append the bundled root hash and reset it + if (back_tree.full()) { + result.hashes.emplace_back(back_tree.get_root_hash()); + back_tree.clear(); + } + } + + // Sanity check to ensure the loop is working correctly, this should always be true + assert(read_reg(reg::uarch_halt_flag) != 0); + + const auto halt_root_hash = m_ht.get_root_hash(); + + // Perform reset uarch directly (as an optimization), we need to only revert dirty words in its state + // Reset uarch registers + m_us->registers = uarch_registers_state{}; + // Reset dirty words in uarch ram + auto &uram = find_address_range(AR_UARCH_RAM_START, AR_UARCH_RAM_LENGTH); + for (auto word_paddr : reset_dirty_words) { + if (word_paddr >= AR_UARCH_RAM_START && word_paddr - AR_UARCH_RAM_START < AR_UARCH_RAM_LENGTH) { + const uint64_t word_start_offset = word_paddr - AR_UARCH_RAM_START; + // Word is aligned and inside range, there can be no overflow + const uint64_t word_end_offset = word_start_offset + HASH_TREE_WORD_SIZE; + if (word_end_offset <= uarch_pristine_ram_len) { // Word is entirely inside pristine uarch ram image + std::memcpy(uram.get_host_memory() + word_start_offset, uarch_pristine_ram + word_start_offset, + HASH_TREE_WORD_SIZE); + } else if (word_start_offset >= uarch_pristine_ram_len) { // Word is entirely outside image + std::memset(uram.get_host_memory() + word_start_offset, 0, HASH_TREE_WORD_SIZE); + } else { // Word stradles image boundary + std::memcpy(uram.get_host_memory() + word_start_offset, uarch_pristine_ram + word_start_offset, + uarch_pristine_ram_len - word_start_offset); + std::memset(uram.get_host_memory() + uarch_pristine_ram_len, 0, + word_end_offset - uarch_pristine_ram_len); + } + } + } + // Update hash tree and retrieve root hash + if (!m_ht.update_words(m_ars, reset_dirty_words)) { + throw std::runtime_error{"update hash tree failed"}; + } + reset_dirty_words.clear(); + const auto reset_root_hash = m_ht.get_root_hash(); + + // Add one hash after the uarch reset + if (log2_bundle_uarch_cycle_count > 0) { + const auto halt_pad_hashes = back_merkle_tree::make_pad_hashes(halt_root_hash, + log2_bundle_uarch_cycle_count, m_c.hash_tree.hash_function); + + // Pad back tree when partially filled and then append its bundled root hash + if (!back_tree.empty()) { + assert(!back_tree.full()); + back_tree.pad_back(back_tree.get_remaining_leaf_count(), halt_pad_hashes); + result.hashes.emplace_back(back_tree.get_root_hash()); + back_tree.clear(); + } + + // Append bundled root hash containing only repetitions of the halt root hash + result.hashes.emplace_back(halt_pad_hashes[log2_bundle_uarch_cycle_count]); + + // Append bundled root hash containing repetitions of the halt root hash on the left + // and one reset root hash on the right + assert(back_tree.empty()); + back_tree.pad_back((1 << log2_bundle_uarch_cycle_count) - 1, halt_pad_hashes); + back_tree.push_back(reset_root_hash); + assert(back_tree.full()); + result.hashes.emplace_back(back_tree.get_root_hash()); + back_tree.clear(); + + // Add the index where reset happened + result.reset_indices.emplace_back(result.hashes.size() - 1); + } else { + result.hashes.push_back(reset_root_hash); + + // Add the index where reset happened + result.reset_indices.emplace_back(result.hashes.size() - 1); + } + + mcycle_reached = read_reg(reg::mcycle); + + // Sanity check to ensure the loop is working correctly, this should always be true + assert(mcycle_reached == mcycle_target); + + // Retrieve break reason + if (read_reg(reg::iflags_H) != 0) { + result.break_reason = interpreter_break_reason::halted; + } else if (read_reg(reg::iflags_Y) != 0) { + result.break_reason = interpreter_break_reason::yielded_manually; + } else if (read_reg(reg::iflags_X) != 0) { + result.break_reason = interpreter_break_reason::yielded_automatically; + break; + } + + // If we already attempted to advance one mcycle over a fixed point, we are done + if (at_fixed_point) { + break; + } + + // If the machine halted, yielded manually or reached mcycle UINT64_MAX, + // then we are at a fixed point and should always attempt to advance one more mcycle + at_fixed_point = result.break_reason == interpreter_break_reason::halted || + result.break_reason == interpreter_break_reason::yielded_manually || mcycle_reached == UINT64_MAX; + + if (!at_fixed_point) { + // The next iteration will target the next mcycle successor, + // but will not exceed mcycle_end if that limit is reached first + mcycle_target = saturating_add(mcycle_target, UINT64_C(1), mcycle_end); + } + } + + assert(back_tree.empty()); + + return result; +} + } // namespace cartesi diff --git a/src/machine.h b/src/machine.h index c7bb5f705..6510110c2 100644 --- a/src/machine.h +++ b/src/machine.h @@ -20,130 +20,170 @@ /// \file /// \brief Cartesi machine interface +#include #include -#include +#include +#include #include -#include - -#include +#include +#include #include "access-log.h" +#include "address-range.h" +#include "back-merkle-tree.h" +#include "hash-tree-stats.h" +#include "hash-tree.h" +#include "host-addr.h" #include "i-device-state-access.h" #include "interpret.h" +#include "machine-address-ranges.h" #include "machine-config.h" -#include "machine-memory-range-descr.h" -#include "machine-merkle-tree.h" +#include "machine-hash.h" #include "machine-reg.h" #include "machine-runtime-config.h" -#include "machine-state.h" +#include "mcycle-root-hashes.h" #include "os.h" -#include "pma-constants.h" -#include "pma.h" +#include "processor-state.h" +#include "scope-remove.h" +#include "shadow-tlb.h" +#include "uarch-cycle-root-hashes.h" #include "uarch-interpret.h" -#include "uarch-machine.h" -#include "virtio-device.h" +#include "uarch-processor-state.h" +#include "variant-hasher.h" namespace cartesi { -/// \brief Tag type used to indicate that merkle tree updates should be skipped. -struct skip_merkle_tree_update_t { - explicit skip_merkle_tree_update_t() = default; +/// \brief Tag type used to indicate that hash-tree updates should be skipped. +struct skip_hash_tree_update_t { + explicit skip_hash_tree_update_t() = default; }; -/// \brief Tag indicating that merkle tree updates should be skipped. -constexpr skip_merkle_tree_update_t skip_merkle_tree_update; +/// \brief Tag indicating that hash-tree updates should be skipped. +constexpr skip_hash_tree_update_t skip_hash_tree_update; /// \class machine /// \brief Cartesi Machine implementation class machine final { private: - //??D Ideally, we would hold a unique_ptr to the state. This - // would allow us to remove the machine-state.h include and - // therefore hide its contents from anyone who includes only - // machine.h. Maybe the compiler can do a good job we we are - // not constantly going through the extra indirection. We - // should test this. - - mutable machine_state m_s; ///< Opaque machine state - mutable machine_merkle_tree m_t; ///< Merkle tree of state - std::vector m_merkle_pmas; ///< PMAs considered by the Merkle tree: from big machine and uarch - machine_config m_c; ///< Copy of initialization config - uarch_machine m_uarch; ///< Microarchitecture machine - machine_runtime_config m_r; ///< Copy of initialization runtime config - machine_memory_range_descrs m_mrds; ///< List of memory ranges returned by get_memory_ranges(). - - boost::container::static_vector, VIRTIO_MAX> m_vdevs; ///< Array of VirtIO devices - - static const pma_entry::flags m_dtb_flags; ///< PMA flags used for DTB - static const pma_entry::flags m_ram_flags; ///< PMA flags used for RAM - static const pma_entry::flags m_flash_drive_flags; ///< PMA flags used for flash drives - static const pma_entry::flags m_cmio_rx_buffer_flags; ///< PMA flags used for cmio rx buffer - static const pma_entry::flags m_cmio_tx_buffer_flags; ///< PMA flags used for cmio tx buffer - - /// \brief Allocates a new PMA entry. - /// \param pma PMA entry to add to machine. - /// \returns Reference to corresponding entry in machine state. - pma_entry ®ister_pma_entry(pma_entry &&pma); - - /// \brief Creates a new PMA entry reflecting a memory range configuration. - /// \param description Informative description of PMA entry for use in error messages - /// \param c Memory range configuration. - /// \returns New PMA entry (with default flags). - static pma_entry make_memory_range_pma_entry(const std::string &description, const memory_range_config &c); - - /// \brief Creates a new flash drive PMA entry. - /// \param description Informative description of PMA entry for use in error messages - /// \param c Memory range configuration. - /// \returns New PMA entry with flash drive flags already set. - static pma_entry make_flash_drive_pma_entry(const std::string &description, const memory_range_config &c); - - /// \brief Creates a new cmio rx buffer PMA entry. - // \param c Optional cmio configuration - /// \returns New PMA entry with rx buffer flags already set. - static pma_entry make_cmio_rx_buffer_pma_entry(const cmio_config &cmio_config); - - /// \brief Creates a new cmio tx buffer PMA entry. - // \param c Optional cmio configuration - /// \returns New PMA entry with tx buffer flags already set. - static pma_entry make_cmio_tx_buffer_pma_entry(const cmio_config &cmio_config); - - /// \brief Saves PMAs into files for serialization - /// \param config Machine config to be stored - /// \param directory Directory where PMAs will be stored - void store_pmas(const machine_config &config, const std::string &directory) const; - - /// \brief Obtain PMA entry that covers a given physical memory region - /// \param pmas Container of pmas to be searched. - /// \param s Pointer to machine state. - /// \param paddr Start of physical memory region. - /// \param length Length of physical memory region. - /// \returns Corresponding entry if found, or a sentinel entry - /// for an empty range. - template - pma_entry &find_pma_entry(const CONTAINER &pmas, uint64_t paddr, uint64_t length); - - template - const pma_entry &find_pma_entry(const CONTAINER &pmas, uint64_t paddr, uint64_t length) const; + const machine_config m_c; ///< Copy of initialization config + machine_runtime_config m_r; ///< Copy of initialization runtime config + mutable machine_address_ranges m_ars; ///< Address ranges + mutable hash_tree m_ht; ///< Top level hash tree + processor_state *m_s; ///< Big machine processor state + uarch_processor_state *m_us; ///< Microarchitecture processor state + + std::unordered_map m_counters; ///< Counters used for statistics collection + bool m_tty_opened{false}; ///< Whether tty was opened + + /// \brief Returns offset that converts between machine host addresses and target physical addresses + /// \param pma_index Index of the memory PMA for the desired offset + host_addr get_hp_offset(uint64_t pma_index) const { + const auto &ar = m_ars.read_pma(pma_index); + if (!ar.is_memory()) [[unlikely]] { + throw std::domain_error{"attempt to get host to physical offset of PMA that is not a memory range"}; + } + auto haddr = cast_ptr_to_host_addr(ar.get_host_memory()); + auto paddr = ar.get_start(); + return paddr - haddr; + } + + /// \brief Initializes processor + /// \param p Processor configuration + /// \param r Machine runtime configuration + void init_processor(const processor_config &p, const machine_runtime_config &r); + + /// \brief Initializes microarchitecture processor + /// \param c Microarchitecture processor configuration + void init_uarch_processor(const uarch_processor_config &p); + + /// \brief Initializes TTY if needed + void init_tty(); + + /// \brief Initializes contents of the shadow PMAs memory + /// \param pmas_config PMAs configuration + /// \detail This can only be called after all PMAs have been added + void init_pmas_contents(const pmas_config &config); + + /// \brief Initialize hot TLB contents + /// \detail This can only be called after all PMAs have been added + void init_hot_tlb_contents(); + + /// \brief Initializes contents of machine DTB, if image was not available + /// \param config Machine configuration + void init_dtb_contents(const machine_config &config); + + /// \brief Dumps statistics + void dump_stats(); + + /// \brief Dumps instruction histogram + void dump_insn_hist(); + + /// \brief Returns key to counter + /// \param name Counter name. + /// \param domain Counter domain. Can be nullptr. Otherwise, should end with a dot '.' + /// \details The counter is key is the concatenation of \p domain with \p name. + static std::string get_counter_key(const char *name, const char *domain = nullptr); + + /// \brief Checks if the machine has VirtIO devices. + /// \returns True if at least one VirtIO device is present. + bool has_virtio_devices() const; + + /// \brief Checks if the machine has VirtIO console device. + /// \returns True if at least one VirtIO console is present. + bool has_virtio_console() const; + + /// \brief Checks if the machine has HTIF console device. + /// \returns True if HTIF console is present. + bool has_htif_console() const; public: - /// \brief Type of hash - using hash_type = machine_merkle_tree::hash_type; + /// \brief Shorthand for the proof type + using proof_type = hash_tree::proof_type; + /// \brief Shorthand for machine register type using reg = machine_reg; /// \brief Constructor from machine configuration /// \param config Machine config to use instantiating machine /// \param runtime Runtime config to use with machine - explicit machine(const machine_config &config, const machine_runtime_config &runtime = {}); + /// \param remover Object that may remove created files and directories if construction fails + explicit machine(machine_config config, machine_runtime_config runtime = {}, const std::string &dir = {}, + scope_remove remover = {}); /// \brief Constructor from previously serialized directory /// \param directory Directory to load stored machine from /// \param runtime Runtime config to use with machine - explicit machine(const std::string &directory, const machine_runtime_config &runtime = {}); + explicit machine(const std::string &directory, machine_runtime_config runtime = {}, + sharing_mode sharing = sharing_mode::none); /// \brief Serialize entire state to directory /// \param directory Directory to store machine into - void store(const std::string &directory) const; + void store(const std::string &directory, sharing_mode sharing = sharing_mode::all) const; + + /// \brief Clones a machine stored from source directory to destination directory. + /// \param from_dir Path to the source directory where the machine is stored. + /// \param to_dir Path to the destination directory where the cloned machine will be stored. + static void clone_stored(const std::string &from_dir, const std::string &to_dir); + + /// \brief Removes all files and the directory of a previously stored machine. + /// \param dir Path to the directory containing the stored machine to be removed. + static void remove_stored(const std::string &dir); + + /// \brief Returns address range that covers a given physical memory region + /// \param paddr Target physical address of start of region. + /// \param length Length of region, in bytes. + /// \returns Corresponding address range if found, or an empty sentinel otherwise. + const address_range &find_address_range(uint64_t paddr, uint64_t length) const noexcept { + return m_ars.find(paddr, length); + } + + /// \brief Returns address range that covers a given physical memory region + /// \param paddr Target physical address of start of region. + /// \param length Length of region, in bytes. + /// \returns Corresponding address range if found, or an empty sentinel otherwise. + address_range &find_address_range(uint64_t paddr, uint64_t length) noexcept { + return m_ars.find(paddr, length); + } /// \brief No default constructor machine() = delete; @@ -156,13 +196,42 @@ class machine final { /// \brief No move assignment machine &operator=(machine &&other) = delete; + /// \brief Destructor. + ~machine(); + + /// \brief Returns hash-tree statistics + /// \param clear Clear all statistics after collecting them + /// \returns Structure containing all statistics + hash_tree_stats get_hash_tree_stats(bool clear = false) noexcept { + return m_ht.get_stats(clear); + } + /// \brief Runs the machine until mcycle reaches mcycle_end, the machine halts or yields. /// \param mcycle_end Maximum value of mcycle before function returns. /// \returns The reason the machine was interrupted. /// \details Several conditions can cause the function to break before mcycle reaches mcycle_end. The most - /// frequent scenario is when the program executes a WFI instruction. Another example is when the machine halts. + /// frequent scenario is when the program executes a WFI instruction. Another example is when the machine + /// halts. interpreter_break_reason run(uint64_t mcycle_end); + /// \brief Collects the root hashes after every \p mcycle_period machine cycles + /// until mcycle reaches \p mcycle_end, the machine yields, or halts. + /// \param mcycle_end Maximum value of mcycle before function returns. + /// \param mcycle_period Number of machine cycles between root hashes to collect. + /// \param mcycle_phase Number of machine cycles elapsed since last root hash collected. + /// \param log2_bundle_mcycle_count Log base 2 of the amount of mcycle root hashes to bundle. + /// If greater than 0, it collects subtree root hashes for 2^log2_bundle_mcycle_count root hashes. + /// \param previous_back_tree Optional context to continue collecting bundled root hashes. + /// \returns The collected mcycle root hashes. + /// Stores into result.hashes the root hashes after each period. + /// Stores into result.mcycle_phase the number of machine cycles after last root hash collected. + /// Stores into result.break_reason the reason the function returned. + /// Stores into result.back_tree the back tree context to continue collecting bundled root hashes. + /// \detail The first hash added to \p result.hashes is the root hash after (\p mcycle_period - \p mcycle_phase) + /// machine cycles (if the function managed to get that far before returning). + mcycle_root_hashes collect_mcycle_root_hashes(uint64_t mcycle_end, uint64_t mcycle_period, uint64_t mcycle_phase, + int32_t log2_bundle_mcycle_count, const std::optional &previous_back_tree = {}); + /// \brief Runs the machine for the given mcycle count and generates a log of accessed pages and proof data. /// \param mcycle_count Number of mcycles to run the machine for. /// \param filename Name of the file to store the log. @@ -174,14 +243,27 @@ class machine final { /// \param log_filename Name of the file containing the log. /// \param mcycle_count Number of mcycles the machine was run for. /// \param root_hash_after Hash of the state after the step. - static interpreter_break_reason verify_step(const hash_type &root_hash_before, const std::string &log_filename, - uint64_t mcycle_count, const hash_type &root_hash_after); + static interpreter_break_reason verify_step(const machine_hash &root_hash_before, const std::string &log_filename, + uint64_t mcycle_count, const machine_hash &root_hash_after); /// \brief Runs the machine in the microarchitecture until the mcycles advances by one unit or the micro cycle /// counter (uarch_cycle) reaches uarch_cycle_end /// \param uarch_cycle_end uarch_cycle limit uarch_interpreter_break_reason run_uarch(uint64_t uarch_cycle_end); + /// \brief Collects the root hashes after every uarch cycle until mcycle reaches \p mcycle_end, + /// the machine yields, or halts. Implicitly resetting the uarch between mcycles. + /// \param mcycle_end End machine cycle value to execute, uarch cycle by uarch cycle. + /// \param log2_bundle_uarch_cycle_count Log base 2 of the amount of uarch cycle root hashes to bundle. + /// \returns The collected uarch cycle root hashes. + /// Stores into result.hashes the root hashes after each uarch cycle. + /// Stores into result.reset_indices the indices of the root hashes after each implicit uarch reset + /// (i.e., after each machine cycle). + /// Stores into result.break_reason the reason why the function returned. + /// \detail The first hash added to \p result.hashes is the root hash after the first uarch cycle, the last is the + /// root hash at the time function returns (for whatever reason), which always happens right after an uarch reset. + uarch_cycle_root_hashes collect_uarch_cycle_root_hashes(uint64_t mcycle_end, int32_t log2_bundle_uarch_cycle_count); + /// \brief Advances one micro step and returns a state access log. /// \param log_type Type of access log to generate. /// \returns The state access log. @@ -200,40 +282,58 @@ class machine final { /// \param root_hash_before State hash before step. /// \param log Step state access log. /// \param root_hash_after State hash after step. - static void verify_step_uarch(const hash_type &root_hash_before, const access_log &log, - const hash_type &root_hash_after); + static void verify_step_uarch(const machine_hash &root_hash_before, const access_log &log, + const machine_hash &root_hash_after); /// \brief Checks the validity of a state transition caused by log_reset_uarch. /// \param root_hash_before State hash before uarch reset /// \param log Step state access log. /// \param root_hash_after State hash after uarch reset. - static void verify_reset_uarch(const hash_type &root_hash_before, const access_log &log, - const hash_type &root_hash_after); + static void verify_reset_uarch(const machine_hash &root_hash_before, const access_log &log, + const machine_hash &root_hash_after); /// \brief Returns copy of default machine config static machine_config get_default_config(); + /// \brief Returns a list of descriptions for all PMA entries registered in the machine, sorted by start + const auto &get_address_ranges() const { + return m_ars.descriptions_view(); + } + /// \brief Returns machine state for direct access. - machine_state &get_state() { - return m_s; + processor_state &get_state() { + return *m_s; } /// \brief Returns machine state for direct read-only access. - const machine_state &get_state() const { - return m_s; + const processor_state &get_state() const { + return *m_s; } - /// \brief Returns a list of descriptions for all PMA entries registered in the machine, sorted by start - machine_memory_range_descrs get_memory_ranges() const { - return m_mrds; + /// \brief Returns uarch state for direct access. + uarch_processor_state &get_uarch_state() { + return *m_us; } - /// \brief Destructor. - ~machine(); + /// \brief Returns uarch state for direct read-only access. + const uarch_processor_state &get_uarch_state() const { + return *m_us; + } + + /// \brief Wait for external interrupts requests. + /// \param mcycle Current value of mcycle. + /// \param mcycle_max Maximum mcycle after wait. + /// \returns A pair {new_mcycle, status}, where new_mcycle gives new value for mcycle after wait, + /// and status will be execute_status::success_and_serve_interrupts if wait was stopped by an + /// external interrupt request. + /// \details When mcycle_max is greater than mcycle, this function will sleep until an external interrupt + /// is triggered or until the amount of time estimated for mcycle to reach mcycle_max has elapsed. + std::pair poll_external_interrupts(uint64_t mcycle, uint64_t mcycle_max); /// \brief Fill file descriptors to be polled by select() for all VirtIO devices. /// \param fds Pointer to sets of read, write and except file descriptors to be updated. - /// \param timeout_us Maximum amount of time to wait in microseconds, this may be updated (always to lower values). + /// \param timeout_us Maximum amount of time to wait in microseconds, this may be updated (always to lower + /// values). void prepare_virtio_devices_select(select_fd_sets *fds, uint64_t *timeout_us); /// \brief Poll file descriptors that were marked as ready by select() for all VirtIO devices. @@ -245,69 +345,67 @@ class machine final { /// \brief Poll file descriptors through select() for all VirtIO devices. /// \details Basically call prepare_virtio_devices_select(), select() and poll_selected_virtio_devices(). - /// \param timeout_us Maximum amount of time to wait in microseconds, this may be updated (always to lower values). + /// \param timeout_us Maximum amount of time to wait in microseconds, this may be updated (always to lower + /// values). /// \returns True if an interrupt was requested, false otherwise. bool poll_virtio_devices(uint64_t *timeout_us, i_device_state_access *da); - /// \brief Checks if the machine has VirtIO devices. - /// \returns True if at least one VirtIO device is present. - bool has_virtio_devices() const; - - /// \brief Checks if the machine has VirtIO console device. - /// \returns True if at least one VirtIO console is present. - bool has_virtio_console() const; - - /// \brief Checks if the machine has HTIF console device. - /// \returns True if HTIF console is present. - bool has_htif_console() const; - - /// \brief Update the Merkle tree so it matches the contents of the machine state. - /// \returns true if succeeded, false otherwise. - bool update_merkle_tree() const; + /// \brief Update the hash-tree so it matches the contents of the machine state. + /// \returns True if successful, false otherwise. + bool update_hash_tree() const; - /// \brief Update the Merkle tree after a page has been modified in the machine state. + /// \brief Update a single page in the hash-tree after it is modified in the machine state. /// \param address Any address inside modified page. /// \returns true if succeeded, false otherwise. - bool update_merkle_tree_page(uint64_t address); + bool update_hash_tree_page(uint64_t address) { + return m_ht.update_page(m_ars, address); + } - /// \brief Obtains the proof for a node in the Merkle tree. + /// \brief Obtains the proof for a node in the hash-tree. /// \param address Address of target node. Must be aligned to a 2log2_size boundary. /// \param log2_size log2 of size subintended by target node. /// Must be between 3 (for a word) and 64 (for the entire address space), inclusive. /// \param proof Receives the proof. /// \details If the node is /// smaller than a page size, then it must lie entirely inside the same PMA range. - machine_merkle_tree::proof_type get_proof(uint64_t address, int log2_size) const; + proof_type get_proof(uint64_t address, int log2_size) const; - /// \brief Obtains the proof for a node in the Merkle tree without making any modifications to the tree. + /// \brief Obtains the proof for a node in the hash-tree without making any modifications to the tree. /// \param address Address of target node. Must be aligned to a 2log2_size boundary. /// \param log2_size log2 of size subintended by target node. /// Must be between 3 (for a word) and 64 (for the entire address space), inclusive. /// \param proof Receives the proof. /// \details If the node is smaller than a page size, then it must lie entirely inside the same PMA range. - /// This overload is used to optimize proof generation when the caller knows that the tree is already up to date. - machine_merkle_tree::proof_type get_proof(uint64_t address, int log2_size, - skip_merkle_tree_update_t /*unused*/) const; + /// This overload is used to optimize proof generation when the caller knows that the tree is already up to + /// date. + proof_type get_proof(uint64_t address, int log2_size, skip_hash_tree_update_t /*unused*/) const { + return m_ht.get_proof(m_ars, address, log2_size); + } - /// \brief Obtains the root hash of the Merkle tree. - /// \param hash Receives the hash. - void get_root_hash(hash_type &hash) const; + /// \brief Obtains the root hash of the hash-tree. + /// \returns The hash. + machine_hash get_root_hash() const; - /// \brief Obtains the hash of a node in the Merkle tree. + /// \brief Obtains the hash of a node in the hash-tree. /// \param address Address of target node. Must be aligned to a 2log2_size boundary. /// \param log2_size log2 of size subintended by target node. - /// \returns The hash of the target node. - hash_type get_merkle_tree_node_hash(uint64_t address, int log2_size) const; + /// \returns The hash. + machine_hash get_node_hash(uint64_t address, int log2_size) const; - /// \brief Obtains the hash of a node in the Merkle tree without making any modifications to the tree. + /// \brief Obtains the hash of a node in the hash-tree without making any modifications to it. /// \param address Address of target node. Must be aligned to a 2log2_size boundary. /// \param log2_size log2 of size subintended by target node. - /// \returns The hash of the target node. - hash_type get_merkle_tree_node_hash(uint64_t address, int log2_size, skip_merkle_tree_update_t /*unused*/) const; + /// \returns The hash. + machine_hash get_node_hash(uint64_t address, int log2_size, skip_hash_tree_update_t /*unused*/) const { + return m_ht.get_node_hash(m_ars, address, log2_size); + } - /// \brief Verifies integrity of Merkle tree. + /// \brief Verifies integrity of hash tree without making any modifications to it tree. /// \returns True if tree is self-consistent, false otherwise. - bool verify_merkle_tree() const; + bool verify_hash_tree() const; + + /// \brief Validate integrity of processor shadow. + void validate_processor_shadow(bool skip_version_check) const; /// \brief Read the value of any register /// \param r Register to read @@ -324,34 +422,41 @@ class machine final { /// \returns The address of the register static uint64_t get_reg_address(reg r); - /// \brief Read the value of a word in the machine state. - /// \param address Word address (aligned to 64-bit boundary). + /// \brief Read the value of a word from the machine state. + /// \param paddr Word address (aligned to 64-bit boundary). /// \returns The value of word at address. - /// \warning The current implementation of this function is very slow! - uint64_t read_word(uint64_t address) const; - - /// \brief Reads a chunk of data from the machine memory. - /// \param address Physical address to start reading. - /// \param data Receives chunk of memory. - /// \param length Size of chunk. - /// \details The entire chunk, from \p address to \p address + \p length must - /// be inside the same PMA region. - void read_memory(uint64_t address, unsigned char *data, uint64_t length) const; - - /// \brief Writes a chunk of data to the machine memory. - /// \param address Physical address to start writing. - /// \param data Source for chunk of data. - /// \param length Size of chunk. - /// \details The entire chunk, from \p address to \p address + \p length must - /// be inside the same PMA region. Moreover, this PMA must be a memory PMA, - /// and not a device PMA. - void write_memory(uint64_t address, const unsigned char *data, uint64_t length); + /// \details The word can be anywhere in the entire address space. + uint64_t read_word(uint64_t paddr) const; + + /// \brief Writes the value of a word to the machine state. + /// \param paddr Word address (aligned to 64-bit boundary). + /// \details The word can be in a writeable area of the address space. + /// This includes the shadow state and the shadow uarch state. + /// (But does NOT include memory-mapped devices, shadow PMAs, or unnocupied memory regions.) + void write_word(uint64_t paddr, uint64_t val); + + /// \brief Reads a chunk of data, by its target physical address and length. + /// \param paddr Target physical address to start reading from. + /// \param data Buffer that receives data to read. Must be at least \p length bytes long. + /// \param length Number of bytes to read from \p paddr to \p data. + /// \details The data can be anywhere in the entire address space. + void read_memory(uint64_t paddr, unsigned char *data, uint64_t length) const; + + /// \brief Writes a chunk of data to machine memory, by its target physical address and length. + /// \param paddr Target physical address to start writing to. + /// \param data Buffer that contains data to write. Must be at least \p length bytes long. + /// \param length Number of bytes to write from \p data to \p paddr. + /// \details Unlike read_memory(), the entire chunk of data, from \p paddr to \p paddr + \p length, + /// must reside entirely in the same address range. Moreover, it cannot be mapped to a device. + void write_memory(uint64_t paddr, const unsigned char *data, uint64_t length); /// \brief Fills a memory range with a single byte. - /// \param address Physical address to start filling. - /// \param data Byte to fill memory with. - /// \param length Size of memory range to fill. - void fill_memory(uint64_t address, uint8_t data, uint64_t length); + /// \param paddr Target physical address to start filling. + /// \param val Byte to fill memory with. + /// \param length Number of bytes to write starting at \p paddr. + /// \details Unlike read_memory(), the entire chunk of data, from \p paddr to \p paddr + \p length, + /// must reside entirely in the same address range. Moreover, it cannot be mapped to a device. + void fill_memory(uint64_t paddr, uint8_t val, uint64_t length); /// \brief Reads a chunk of data from the machine virtual memory. /// \param vaddr_start Virtual address to start reading. @@ -370,43 +475,24 @@ class machine final { /// \returns The corresponding physical address. uint64_t translate_virtual_address(uint64_t vaddr); - /// \brief Get read-only access to container with all PMA entries. - /// \returns The container. - const boost::container::static_vector &get_pmas() const; - - /// \brief Obtain PMA entry from the machine state that covers a given physical memory region - /// \brief Microarchitecture PMAs are not considered. - /// \param s Pointer to machine state. - /// \param paddr Start of physical memory region. - /// \param length Length of physical memory region. - /// \returns Corresponding entry if found, or a sentinel entry - /// for an empty range. - pma_entry &find_pma_entry(uint64_t paddr, uint64_t length); - - const pma_entry &find_pma_entry(uint64_t paddr, uint64_t length) const; - - /// \brief Obtain PMA entry covering a physical memory word - /// \tparam T Type of word. - /// \param s Pointer to machine state. - /// \param paddr Target physical address. - /// \returns Corresponding entry if found, or a sentinel entry - /// for an empty range. - template - const pma_entry &find_pma_entry(uint64_t paddr) const { - return find_pma_entry(paddr, sizeof(T)); + /// \brief Returns the address range associated to the PMA at a given index + /// \param index Index of desired address range + /// \returns Desired address range, or an empty sentinel if index is out of bounds + const address_range &read_pma(uint64_t index) const noexcept { + return m_ars.read_pma(index); + } + + /// \brief Returns the address range associated to the PMA at a given index + /// \param index Index of desired address range + /// \returns Desired address range, or an empty sentinel if index is out of bounds + address_range &read_pma(uint64_t index) noexcept { + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-const-cast) + return const_cast(std::as_const(*this).read_pma(index)); } /// \brief Go over the write TLB and mark as dirty all pages currently there. void mark_write_tlb_dirty_pages() const; - /// \brief Verify if dirty page maps are consistent. - /// \returns true if they are, false if there is an error. - bool verify_dirty_page_maps() const; - - /// \brief Copies the current state into a configuration for serialization - /// \returns The configuration - machine_config get_serialization_config() const; - /// \brief Returns copy of initialization config. const machine_config &get_initial_config() const; @@ -416,13 +502,14 @@ class machine final { /// \brief Changes the machine runtime config. /// \param range Configuration of the new memory range. /// \details Some runtime options cannot be changed. - void set_runtime_config(const machine_runtime_config &r); + void set_runtime_config(machine_runtime_config r); - /// \brief Replaces a memory range. - /// \param range Configuration of the new memory range. - /// \details The machine must contain an existing memory range - /// matching the start and length specified in range. - void replace_memory_range(const memory_range_config &range); + /// \brief Replaces a memory address range. + /// \param config Configuration of the new memory address range. + /// \details A memory address range matching the start and length specified in the config must exist. + void replace_memory_range(const memory_range_config &config) { + m_ars.replace(config); + } /// \brief Sends cmio response /// \param reason Reason for sending response. @@ -430,6 +517,97 @@ class machine final { /// \param length Length of response data. void send_cmio_response(uint16_t reason, const unsigned char *data, uint64_t length); + /// \brief Converts from machine host address to target physical address + /// \param haddr Machine host address to convert + /// \param pma_index Index of PMA where address falls + /// \returns Corresponding target physical address + /// \details This method also converts from vh_offset to vp_offset + uint64_t get_paddr(host_addr haddr, uint64_t pma_index) const { + return static_cast(haddr + get_hp_offset(pma_index)); + } + + /// \brief Converts from target physical address to machine host address + /// \param paddr Target physical address to convert + /// \param pma_index Index of PMA where address falls + /// \returns Corresponding machine host address + /// \details This method also converts from vp_offset to vh_offset + host_addr get_host_addr(uint64_t paddr, uint64_t pma_index) const { + return host_addr{paddr} - get_hp_offset(pma_index); + } + + /// \brief Marks a page as dirty + /// \param haddr Machine host address within page + /// \param pma_index Index of PMA where address falls + void mark_dirty_page(host_addr haddr, uint64_t pma_index) { + auto paddr = get_paddr(haddr, pma_index); + mark_dirty_page(paddr, pma_index); + } + + /// \brief Marks a page as dirty + /// \param paddr Target phyislcal address within page + /// \param pma_index Index of PMA where address falls + void mark_dirty_page(uint64_t paddr, uint64_t pma_index) { + auto &ar = read_pma(pma_index); + ar.get_dirty_page_tree().mark_dirty_page_and_up(paddr - ar.get_start()); + } + + /// \brief Updates a TLB slot + /// \param set_index TLB_CODE, TLB_READ, or TLB_WRITE + /// \param slot_index Index of slot to update + /// \param vaddr_page Virtual address of page to map + /// \param vh_offset Offset from target virtual addresses to host addresses within page + /// \param pma_index Index of PMA where address falls + void write_tlb(TLB_set_index set_index, uint64_t slot_index, uint64_t vaddr_page, host_addr vh_offset, + uint64_t pma_index) { + uint64_t vp_offset = 0; + if (vaddr_page != TLB_INVALID_PAGE) [[likely]] { + vp_offset = get_paddr(vh_offset, pma_index); + } + m_s->penumbra.tlb[set_index][slot_index].vaddr_page = vaddr_page; + m_s->penumbra.tlb[set_index][slot_index].vh_offset = vh_offset; + m_s->shadow.tlb[set_index][slot_index].vaddr_page = vaddr_page; + m_s->shadow.tlb[set_index][slot_index].vp_offset = vp_offset; + m_s->shadow.tlb[set_index][slot_index].pma_index = pma_index; + m_s->shadow.tlb[set_index][slot_index].zero_padding_ = 0; + } + + /// \brief Updates a TLB slot + /// \param set_index TLB_CODE, TLB_READ, or TLB_WRITE + /// \param slot_index Index of slot to update + /// \param vaddr_page Virtual address of page to map + /// \param vp_offset Offset from target virtual addresses to target physical addresses within page + /// \param pma_index Index of PMA where address falls + void write_shadow_tlb(TLB_set_index set_index, uint64_t slot_index, uint64_t vaddr_page, uint64_t vp_offset, + uint64_t pma_index) { + if (vaddr_page != TLB_INVALID_PAGE) [[likely]] { + auto paddr_page = vaddr_page + vp_offset; + const auto vh_offset = get_host_addr(paddr_page, pma_index) - vaddr_page; + write_tlb(set_index, slot_index, vaddr_page, vh_offset, pma_index); + } else { + write_tlb(set_index, slot_index, TLB_INVALID_PAGE, host_addr{0}, TLB_INVALID_PMA_INDEX); + } + } + + /// \brief Reads a TLB register + /// \param set_index TLB_CODE, TLB_READ, or TLB_WRITE + /// \param slot_index Index of slot to read + /// \param reg Register to read from slot + /// \returns Value of register + uint64_t read_shadow_tlb(TLB_set_index set_index, uint64_t slot_index, shadow_tlb_what reg) const { + switch (reg) { + case shadow_tlb_what::vaddr_page: + return m_s->shadow.tlb[set_index][slot_index].vaddr_page; + case shadow_tlb_what::vp_offset: + return m_s->shadow.tlb[set_index][slot_index].vp_offset; + case shadow_tlb_what::pma_index: + return m_s->shadow.tlb[set_index][slot_index].pma_index; + case shadow_tlb_what::zero_padding_: + return m_s->shadow.tlb[set_index][slot_index].zero_padding_; + default: + throw std::domain_error{"unknown shadow TLB register"}; + } + } + /// \brief Sends cmio response and returns an access log /// \param reason Reason for sending response. /// \param data Response data. @@ -447,7 +625,51 @@ class machine final { /// \param log Log containing the state accesses performed by the load operation /// \param root_hash_after State hash after response was sent. static void verify_send_cmio_response(uint16_t reason, const unsigned char *data, uint64_t length, - const hash_type &root_hash_before, const access_log &log, const hash_type &root_hash_after); + const machine_hash &root_hash_before, const access_log &log, const machine_hash &root_hash_after); + + /// \brief Returns a description of what is at a given target physical address + /// \param paddr Target physical address of interest + /// \returns Description of what is at that address + static const char *get_what_name(uint64_t paddr); + + /// \brief Increments a counter + /// \param name Counter name. + /// \param domain Counter domain. Can be nullptr. Otherwise, should end with a dot '.' + /// \details The counter is identified by the concatenation of \p domain with \p name. + void increment_counter(const char *name, const char *domain = nullptr); + + /// \brief Writes value to counter + /// \param val Value to write. + /// \param name Counter name. + /// \param domain Counter domain. Can be nullptr. Otherwise, should end with a dot '.' + /// \details The counter is identified by the concatenation of \p domain with \p name. + void write_counter(uint64_t val, const char *name, const char *domain = nullptr); + + /// \brief Returns value in counter + /// \param name Counter name. + /// \param domain Counter domain. Can be nullptr. Otherwise, should end with a dot '.' + /// \details The counter is identified by the concatenation of \p domain with \p name. + uint64_t read_counter(const char *name, const char *domain = nullptr); + + /// \brief Returns all counters + const auto &get_counters() { + return m_counters; + } + + /// \brief Returns whether runtime soft yields are enabled + bool get_soft_yield() const { + return m_r.soft_yield; + } + + /// \brief Returns hash tree hash function + hash_function_type get_hash_function() const { + return m_c.hash_tree.hash_function; + } + + /// \brief Returns whether the machine contains a shared address range + bool has_shared_address_range() const { + return std::ranges::any_of(m_ars.all(), [](const auto &ar) { return ar.is_backing_store_shared(); }); + } }; } // namespace cartesi diff --git a/src/mcycle-root-hashes.h b/src/mcycle-root-hashes.h new file mode 100644 index 000000000..8152476c7 --- /dev/null +++ b/src/mcycle-root-hashes.h @@ -0,0 +1,39 @@ +// Copyright Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: LGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify it under +// the terms of the GNU Lesser General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) any +// later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +// PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License along +// with this program (see COPYING). If not, see . +// + +#ifndef MCYCLE_ROOT_HASHES_H +#define MCYCLE_ROOT_HASHES_H + +#include +#include + +#include "back-merkle-tree.h" +#include "interpret.h" +#include "machine-hash.h" + +namespace cartesi { + +/// \brief Collected mcycle root hashes +struct mcycle_root_hashes { + machine_hashes hashes; ///< Root hashes collected after each machine cycle period + uint64_t mcycle_phase{}; ///< Machine cycles elapsed since last collected root hash + interpreter_break_reason break_reason{}; ///< Reason why function returned + std::optional back_tree; ///< Root hashes context +}; + +} // namespace cartesi + +#endif diff --git a/src/memory-address-range.cpp b/src/memory-address-range.cpp new file mode 100644 index 000000000..22d03aa31 --- /dev/null +++ b/src/memory-address-range.cpp @@ -0,0 +1,82 @@ +// Copyright Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: LGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify it under +// the terms of the GNU Lesser General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) any +// later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +// PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License along +// with this program (see COPYING). If not, see . +// + +#include "memory-address-range.h" + +#include +#include +#include +#include +#include + +#include "address-range-constants.h" +#include "address-range.h" +#include "machine-config.h" +#include "os-mapped-memory.h" +#include "pmas.h" + +namespace cartesi { + +using namespace std::string_literals; + +static os::mapped_memory_flags check_mmap_flags(const pmas_flags &flags, const backing_store_config &backing_store, + const memory_address_range_config &memory_config) { + if (!flags.M) { + throw std::invalid_argument{"memory address range must be flagged memory"}; + } + if (backing_store.shared && backing_store.data_filename.empty()) { + throw std::invalid_argument{"shared backing store must have a filename"}; + } + if (backing_store.create && !backing_store.shared) { + throw std::invalid_argument{"created backing store must also be shared"}; + } + if (backing_store.truncate && !backing_store.shared) { + throw std::invalid_argument{"truncated backing store must also be shared"}; + } + if (memory_config.host_read_only && flags.W) { + throw std::invalid_argument{"backing store for writable memory address range cannot be read-only"}; + } + return os::mapped_memory_flags{ + .read_only = memory_config.host_read_only && !backing_store.newly_created(), + .shared = backing_store.shared, + .backing_gap = !backing_store.shared, + .no_reserve = memory_config.host_no_reserve, + }; +} + +static const auto throw_invalid_argument = [](const char *err) { throw std::invalid_argument{err}; }; + +memory_address_range::memory_address_range(const std::string &description, uint64_t start, uint64_t length, + const pmas_flags &flags, const backing_store_config &backing_store, + const memory_address_range_config &memory_config) try : + address_range(description.c_str(), start, length, flags, throw_invalid_argument), + m_mapped{std::max(memory_config.host_length, length), check_mmap_flags(flags, backing_store, memory_config), + backing_store.data_filename, length}, + m_config{memory_config}, + m_backing_store{backing_store}, + // TODO(edubart): as an optimization we could detect newly created backing stores and pre-initialize + // dirty hash tree as clean without even touching or causing page faults to the mapped memory + m_dpt{get_level_count(length), static_cast(length >> AR_LOG2_PAGE_SIZE), backing_store.dpt_filename, + !backing_store.dpt_filename.empty() && backing_store.shared}, + m_dht{get_level_count(length), static_cast(length >> AR_LOG2_PAGE_SIZE), backing_store.dht_filename, + !backing_store.dht_filename.empty() && backing_store.shared} { +} catch (const std::exception &e) { + throw std::invalid_argument{std::string{e.what()}.append(" when initializing ").append(description)}; +} catch (...) { + throw std::invalid_argument{"unknown exception when initializing "s.append(description)}; +} + +} // namespace cartesi diff --git a/src/memory-address-range.h b/src/memory-address-range.h new file mode 100644 index 000000000..74c44eac8 --- /dev/null +++ b/src/memory-address-range.h @@ -0,0 +1,128 @@ +// Copyright Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: LGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify it under +// the terms of the GNU Lesser General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) any +// later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +// PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License along +// with this program (see COPYING). If not, see . +// + +#ifndef MEMORY_ADDRESS_RANGE_H +#define MEMORY_ADDRESS_RANGE_H + +#include +#include + +#include "address-range.h" +#include "dense-hash-tree.h" +#include "dirty-page-tree.h" +#include "machine-config.h" +#include "os-mapped-memory.h" +#include "pmas.h" + +namespace cartesi { + +// Forward declarations +class machine; + +/// \brief Configuration for memory address range +struct memory_address_range_config { + ///< Whether memory is read-only on host. (If set, must be read-only on target as well) + bool host_read_only{false}; + ///< Whether host should not reserve memory when mapping the memory. + /// Only useful to create or load machines with large unused address space on host without enough memory or swap. + /// (Use with caution as the process may crash when running out of host memory)/ + bool host_no_reserve{false}; + ///< Total amount of memory allocated by host. (Must be larger than what is needed by target) + uint64_t host_length{0}; +}; + +/// \file +/// \brief An address range occupied by memory + +class memory_address_range final : public address_range { + os::mapped_memory m_mapped; ///< Pointer to mapped memory + memory_address_range_config m_config; ///< Memory configuration passed to constructor. + backing_store_config m_backing_store; ///< Backing store configuration passed to constructor. + dirty_page_tree m_dpt; ///< Tree of dirty pages. + dense_hash_tree m_dht; ///< Dense hash tree of pages. + +public: + /// \brief Constructor + /// \param description Description of address range for use in error messages + /// \param start Start of address range + /// \param length Length of address range + /// \param flags Address range flags + /// \param backing_store Configuration for underlying file backing. + /// \param memory_config Additional configuration for memory address range. + /// \p config.host_length Length of host memory to be mapped. + /// If set to 0, use \p length. + /// Otherwise, \p host_length cannot be smaller than \p length. + /// The effect is to allocate additional memory past \p length that is not visible by the address range, + /// and is not part of the backing storage, but can be used for other purposes. + /// \p config.host_read_only Marks memory as read-only on host itself. Requires \p flags.W to be cleared as well. + memory_address_range(const std::string &description, uint64_t start, uint64_t length, const pmas_flags &flags, + const backing_store_config &backing_store = {}, const memory_address_range_config &memory_config = {}); + + ~memory_address_range() override = default; + + // No copy or move constructors or assignments + memory_address_range(const memory_address_range &) = delete; + memory_address_range(memory_address_range &&) = delete; + memory_address_range &operator=(const memory_address_range &) = delete; + memory_address_range &operator=(memory_address_range &&) = delete; + + static uint64_t get_dht_storage_length(uint64_t length) { + return dense_hash_tree::get_storage_length(get_level_count(length), + static_cast(length >> AR_LOG2_PAGE_SIZE)); + } + + static uint64_t get_dpt_storage_length(uint64_t length) { + return dirty_page_tree::get_storage_length(get_level_count(length), + static_cast(length >> AR_LOG2_PAGE_SIZE)); + } + +private: + unsigned char *do_get_host_memory() noexcept override { + return m_mapped.get_ptr(); + } + + const unsigned char *do_get_host_memory() const noexcept override { + return m_mapped.get_ptr(); + } + + bool do_is_host_read_only() const noexcept override { + return m_config.host_read_only; + } + + bool do_is_backing_store_shared() const noexcept override { + return m_backing_store.shared; + } + + dirty_page_tree &do_get_dirty_page_tree() noexcept override { + return m_dpt; + } + + const dirty_page_tree &do_get_dirty_page_tree() const noexcept override { + return m_dpt; + } + + dense_hash_tree &do_get_dense_hash_tree() noexcept override { + return m_dht; + } + + const dense_hash_tree &do_get_dense_hash_tree() const noexcept override { + return m_dht; + } +}; + +} // namespace cartesi + +#endif diff --git a/src/meta.h b/src/meta.h index c0d5023d7..cea033337 100644 --- a/src/meta.h +++ b/src/meta.h @@ -42,14 +42,6 @@ struct is_template_base_of_helper { }; } // namespace detail -/// \class remove_cvref -/// \brief Provides a member typedef type with reference and topmost cv-qualifiers removed. -/// \note (This is directly available in C++20.) -template -struct remove_cvref { - using type = std::remove_reference_t>; -}; - /// \class is_template_base_of /// \brief SFINAE test if class is derived from from a base template class. /// \tparam BASE Base template. @@ -59,12 +51,18 @@ using is_template_base_of = std::integral_constant, const DERIVED &>, typename detail::is_template_base_of_helper::yes>>; +template