Major hash tree refactoring

.clang-format

@@ -16,4 +16,4 @@ ConstructorInitializerAllOnOneLineOrOnePerLine: true
 IndentCaseLabels: true
 IndentWidth: 4
 SpaceAfterCStyleCast: true
-Standard: c++20
+Standard: Latest

.clang-tidy

@@ -1,12 +1,10 @@
 Checks: >-
   boost*,
-  -boost-use-ranges,
   bugprone*,
   -bugprone-easily-swappable-parameters,
   cert*,
   -cert-int09-c,
   clang-analyzer*,
-  -clang-analyzer-optin.cplusplus.VirtualCall,
   cppcoreguidelines*,
   -cppcoreguidelines-avoid-magic-numbers,
   -cppcoreguidelines-owning-memory,
@@ -32,7 +30,6 @@ Checks: >-
   -modernize-use-nodiscard,
   -modernize-use-trailing-return-type,
   -modernize-avoid-c-arrays,
-  -modernize-use-ranges,
   performance*,
   -performance-enum-size,
   portability*,
@@ -41,16 +38,6 @@ Checks: >-
   -readability-identifier-length,
   -readability-magic-numbers,
   -readability-enum-initial-value,
+  -readability-use-concise-preprocessor-directives,
 WarningsAsErrors: >-
-  boost*,
-  bugprone*,
-  cert*,
-  clang-analyzer*,
-  cppcoreguidelines*,
-  google*,
-  hicpp*,
-  misc*,
-  modernize*,
-  performance*,
-  portability*,
-  readability*,
+  *

.dockerignore

@@ -2,8 +2,6 @@
 **/*.a
 **/*.swp
 **/*.d
-**/*.pb.cc
-**/*.pb.h
 **/*.gcno
 **/*.so
 **/*.dtb
@@ -15,7 +13,7 @@
 build
 third-party/downloads
 src/cartesi-jsonrpc-machine
-src/cartesi-merkle-tree-hash
+src/cartesi-hash-tree-hash
 
 doc/html
 doc/api.md

.github/workflows/build.yml

@@ -209,10 +209,6 @@ jobs:
         run: |
           docker run --rm -t -v cmio-templates:/tmp/cartesi-machine/tests/data cartesi/machine-emulator:tests /usr/share/cartesi-machine/tests/scripts/test-cmio.sh cartesi-jsonrpc-machine cartesi-machine lua
 
-      - name: Run Merkle tree tests
-        run: |
-          docker run --rm -t ${{ github.repository_owner }}/machine-emulator:tests test-merkle-tree-hash --log2-root-size=30 --log2-leaf-size=12 --input=/usr/bin/test-merkle-tree-hash
-
       - name: Run C API tests
         run: |
           docker run --rm -t ${{ github.repository_owner }}/machine-emulator:tests test-machine-c-api
@@ -227,7 +223,7 @@ jobs:
 
       - name: Run test suite with microarchitecture and host based interpreters comparing machine hashes at every step
         run: |
-          docker run --rm -t ${{ github.repository_owner }}/machine-emulator:tests cartesi-machine-tests --concurrency=update_merkle_tree:1 --test="^rv64ui.*$" --jobs=$(nproc) run_host_and_uarch
+          docker run --rm -t ${{ github.repository_owner }}/machine-emulator:tests cartesi-machine-tests --concurrency=update_hash_tree:1 --test="^rv64ui.*$" --jobs=$(nproc) run_host_and_uarch
 
       - name: Create uarch json logs to be used to test the Solidity based microarchitecture interpreter
         run: |
@@ -377,10 +373,6 @@ jobs:
         run: |
           docker run --platform linux/arm64 --rm -t -v cmio-templates:/tmp/cartesi-machine/tests/data cartesi/machine-emulator:tests /usr/share/cartesi-machine/tests/scripts/test-cmio.sh cartesi-jsonrpc-machine cartesi-machine lua
 
-      - name: Run Merkle tree tests
-        run: |
-          docker run --platform linux/arm64 --rm -t ${{ github.repository_owner }}/machine-emulator:tests test-merkle-tree-hash --log2-root-size=30 --log2-leaf-size=12 --input=/usr/bin/test-merkle-tree-hash
-
       - name: Run C API tests
         run: |
           docker run --platform linux/arm64 --rm -t ${{ github.repository_owner }}/machine-emulator:tests test-machine-c-api
@@ -395,7 +387,7 @@ jobs:
 
       - name: Run test suite with microarchitecture and host based interpreters comparing machine hashes at every step
         run: |
-          docker run --platform linux/arm64 --rm -t ${{ github.repository_owner }}/machine-emulator:tests cartesi-machine-tests  --test="^rv64ui%-v%-add.bin$" --concurrency=update_merkle_tree:1 --jobs=$(nproc) run_host_and_uarch
+          docker run --platform linux/arm64 --rm -t ${{ github.repository_owner }}/machine-emulator:tests cartesi-machine-tests  --test="^rv64ui%-v%-add.bin$" --concurrency=update_hash_tree:1 --jobs=$(nproc) run_host_and_uarch
 
       - name: Build machine-emulator "tests" docker image
         uses: docker/build-push-action@v5
@@ -529,6 +521,7 @@ jobs:
             GIT_COMMIT=${GITHUB_SHA}
             DEBUG=yes
             COVERAGE=yes
+            THREADS=no
             MACHINE_EMULATOR_VERSION=${{ env.MACHINE_EMULATOR_VERSION }}
           project: ${{ vars.DEPOT_PROJECT }}
           token: ${{ secrets.DEPOT_TOKEN }}
@@ -546,11 +539,12 @@ jobs:
           build-args: |
             DEBUG=yes
             COVERAGE=yes
+            THREADS=no
             MACHINE_EMULATOR_VERSION=${{ env.MACHINE_EMULATOR_VERSION }}
 
       - name: Run coverage
         run: |
-          docker run --name coverage-report -t ${{ github.repository_owner }}/machine-emulator:coverage make -j1 test-save-and-load test-machine test-hash test-lua test-jsonrpc test-c-api coverage-machine test-uarch-rv64ui test-uarch-interpreter coverage-uarch coverage-report coverage=yes
+          docker run --name coverage-report -t ${{ github.repository_owner }}/machine-emulator:coverage make -j1 test-save-and-load test-machine test-lua test-jsonrpc test-c-api coverage-machine test-uarch-rv64ui test-uarch-interpreter coverage-uarch coverage-report coverage=yes
           docker cp coverage-report:/usr/src/emulator/tests/build/coverage .
           docker rm coverage-report
 
@@ -569,7 +563,7 @@ jobs:
 
   sanitize:
     name: Sanitize
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
         with:
@@ -628,17 +622,17 @@ jobs:
             SANITIZE=yes
             MACHINE_EMULATOR_VERSION=${{ env.MACHINE_EMULATOR_VERSION }}
 
-      - name: MMAP rnd_bits workaround for the new GitHub ubuntu-22 runner
+      - name: MMAP rnd_bits workaround for the new GitHub Ubuntu runner
         run: sudo sysctl vm.mmap_rnd_bits=28
 
       - name: Run tests with sanitizer
         run: |
-          docker run --rm -t ${{ github.repository_owner }}/machine-emulator:sanitizer make sanitize=yes test-save-and-load test-machine test-hash test-lua test-jsonrpc test-c-api coverage-machine test-uarch-rv64ui test-uarch-interpreter coverage-uarch
+          docker run --rm -t ${{ github.repository_owner }}/machine-emulator:sanitizer make sanitize=yes test-save-and-load test-machine test-lua test-jsonrpc test-c-api coverage-machine test-uarch-rv64ui test-uarch-interpreter coverage-uarch
 
   publish_artifacts:
     name: Publish artifacts
     needs: [build, static-analysis, coverage, sanitize, test_amd64, test_arm64]
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout emulator source code
         uses: actions/checkout@v4

.gitignore

@@ -6,14 +6,15 @@
 *.a
 *.lib
 *.wasm
+*.TODO
+*.tmp
 
 build
 pkg
 third-party/downloads
 src/cartesi-jsonrpc-machine
-src/cartesi-merkle-tree-hash
+src/cartesi-hash-tree-hash
 src/tests/test-machine-c-api
-src/tests/test-merkle-tree-hash
 
 doc/html
 doc/api.md

.typos.toml

@@ -9,3 +9,5 @@ stap = "stap"
 wronly = "wronly"
 optin = "optin"
 sxl = "sxl"
+nd = "nd"
+mke = "mke"

Dockerfile

@@ -1,35 +1,17 @@
-FROM debian:bookworm-20250407 AS toolchain
+FROM debian:trixie-20250811 AS toolchain
 
 RUN apt-get update && \
     DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
-        build-essential vim wget git lcov \
-        libboost1.81-dev libssl-dev libslirp-dev \
+        build-essential vim wget git gcovr \
+        libomp-19-dev libboost1.83-dev libssl-dev libslirp-dev \
         ca-certificates pkg-config lua5.4 liblua5.4-dev \
-        luarocks xxd procps \
-        g++-12-riscv64-linux-gnu=12.2.0-13cross1 \
-        gcc-riscv64-unknown-elf=12.2.0-14+11+b1 && \
+        lua-check lua-socket lua-posix lua-lpeg \
+        xxd procps unzip gosu \
+        clang-tidy clang-format \
+        g++-14-riscv64-linux-gnu=14.2.0-19cross1 \
+        gcc-riscv64-unknown-elf=14.2.0+19 && \
     rm -rf /var/lib/apt/lists/*
 
-# Install clang 19
-RUN apt-get update && \
-    DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
-        wget software-properties-common gnupg && \
-    wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc && \
-    add-apt-repository -y 'deb http://apt.llvm.org/bookworm/  llvm-toolchain-bookworm-19 main' && \
-    add-apt-repository -y 'deb http://apt.llvm.org/bookworm/  llvm-toolchain-bookworm-19 main' && \
-    apt-get update && \
-    DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
-        clang-tidy-19 clang-format-19 && \
-    update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-19 120 && \
-    update-alternatives --install /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-19 120 && \
-    rm -rf /var/lib/apt/lists/*
-
-# Install lua packages
-RUN luarocks install --lua-version=5.4 luasocket && \
-    luarocks install --lua-version=5.4 luasec && \
-    luarocks install --lua-version=5.4 luaposix && \
-    luarocks install --lua-version=5.4 luacheck
-
 # Install stylua
 RUN cd /tmp && \
     wget https://github.com/JohnnyMorganz/StyLua/releases/download/v0.20.0/stylua-linux-`uname -m`.zip && \
@@ -44,15 +26,6 @@ RUN cd /tmp && \
 # Environment has the riscv64 toolchains
 ENV DEV_ENV_HAS_TOOLCHAIN=yes
 
-# Install su-exec
-RUN cd /tmp && \
-    git clone --branch v0.2 --depth 1 https://github.com/ncopa/su-exec.git && \
-    cd su-exec && \
-    if [ `git rev-parse --verify HEAD` != 'f85e5bde1afef399021fbc2a99c837cf851ceafa' ]; then exit 1; fi && \
-    make && \
-    cp su-exec /usr/local/bin/ && \
-    rm -rf /tmp/su-exec
-
 # Install workaround to run as current user
 COPY tools/docker-entrypoint.sh /usr/local/bin/entrypoint.sh
 RUN chmod +x /usr/local/bin/entrypoint.sh
@@ -69,32 +42,29 @@ FROM toolchain AS builder
 ARG GIT_COMMIT=""
 ARG DEBUG=no
 ARG COVERAGE=no
+ARG THREADS=yes
 ARG SANITIZE=no
 
 COPY . .
-RUN make -j$(nproc) git_commit=$GIT_COMMIT debug=$DEBUG coverage=$COVERAGE sanitize=$SANITIZE
+RUN make -j$(nproc) git_commit=$GIT_COMMIT debug=$DEBUG coverage=$COVERAGE threads=$THREADS sanitize=$SANITIZE
 
 ####################################################################################################
 FROM builder AS debian-packager
 
 RUN make install-uarch debian-package DESTDIR=$PWD/_install
 
 ####################################################################################################
-FROM debian:bookworm-20250407-slim
+FROM debian:trixie-20250811-slim
 ARG TARGETARCH
 
-COPY --from=debian-packager \
-    /usr/src/emulator/machine-emulator_${TARGETARCH}.deb \
-    machine-emulator.deb
-COPY --from=debian-packager /usr/local/lib/lua /usr/local/lib/lua
-COPY --from=debian-packager /usr/local/share/lua /usr/local/share/lua
+COPY --from=debian-packager /usr/src/emulator/machine-emulator_${TARGETARCH}.deb machine-emulator.deb
 
 RUN apt-get update && \
     apt-get install -y ./machine-emulator.deb && \
     rm -rf /var/lib/apt/lists/* /var/cache/apt/* machine-emulator.deb
 
-RUN addgroup --system --gid 102 cartesi && \
-    adduser --system --uid 102 --ingroup cartesi --disabled-login --no-create-home --home /nonexistent --gecos "cartesi user" --shell /bin/false cartesi
+RUN groupadd --system --gid 102 cartesi && \
+    useradd --system --uid 102 --gid 102 --no-create-home --home /nonexistent --comment "cartesi user" --shell /bin/false cartesi
 
 WORKDIR /opt/cartesi
 

LICENSES.md

@@ -8,11 +8,11 @@ This project includes several submodules and dependencies, each with its own lic
 
 - `tests/machine`: Licensed under the Apache License 2.0. See the license terms in [tests/machine/LICENSE](tests/machine/LICENSE).
 - `tests/uarch`: Licensed under the Apache License 2.0. Licensing details are available in [tests/uarch/LICENSE](tests/uarch/LICENSE).
+- `third-party/ankerl`: Licensed under the MIT License. The license can be found at [third-party/ankerl/LICENSE](third-party/ankerl/LICENSE).
 - `third-party/llvm-flang-uint128`: Licensed under the Apache License 2.0 with LLVM exceptions. The license can be found at [third-party/llvm-flang-uint128/LICENSE](third-party/llvm-flang-uint128/LICENSE).
 - `third-party/riscv-arch-test`: Source code licensed under the Apache 2.0 and BSD 3-Clause licenses. Documentation under `CC-BY-4.0`. License information is provided in README.md and other COPYING.* files like [third-party/riscv-arch-test/COPYING.APACHE](third-party/riscv-arch-test/COPYING.APACHE).
 - `third-party/riscv-tests`: Licensed under the BSD 3-Clause "New" or "Revised" License. See [third-party/riscv-tests/LICENSE](third-party/riscv-tests/LICENSE) for license details.
 - `third-party/riscv-tests/env`: Licensed under the BSD 3-Clause "New" or "Revised" License. License details are in [third-party/riscv-tests/env/LICENSE](third-party/riscv-tests/env/LICENSE).
-- `third-party/tiny_sha3`: Licensed under the MIT License. The license can be found at [third-party/tiny_sha3/LICENSE](third-party/tiny_sha3/LICENSE).
 - `third-party/nlohmann-json`: Licensed under the MIT License. The license can be found at [third-party/nlohmann-json/LICENSE.MIT](third-party/nlohmann-json/LICENSE.MIT).
 
 ## Debian Packages

Makefile

@@ -73,17 +73,17 @@ INSTALL_DIR= cp -RP
 SYMLINK= ln -sf
 CHMOD_EXEC= chmod 0755
 
-EMU_TO_BIN= src/cartesi-jsonrpc-machine src/cartesi-merkle-tree-hash
+EMU_TO_BIN= src/cartesi-jsonrpc-machine src/cartesi-hash-tree-hash
 EMU_TO_LIB= src/$(LIBCARTESI_SO) src/$(LIBCARTESI_SO_JSONRPC)
 EMU_TO_LIB_A= src/libcartesi.a src/libcartesi_jsonrpc.a src/libluacartesi.a src/libluacartesi_jsonrpc.a
 EMU_LUA_TO_BIN= src/cartesi-machine.lua src/cartesi-machine-stored-hash.lua
-EMU_TO_LUA_PATH= src/cartesi/util.lua src/cartesi/proof.lua src/cartesi/gdbstub.lua
+EMU_TO_LUA_PATH= src/cartesi/util.lua src/cartesi/gdbstub.lua
 EMU_TO_LUA_CPATH= src/cartesi.so
 EMU_TO_LUA_CARTESI_CPATH= src/cartesi/jsonrpc.so
 EMU_TO_INC= $(addprefix src/,jsonrpc-machine-c-api.h machine-c-api.h machine-c-version.h)
 UARCH_TO_SHARE= uarch-ram.bin
 
-TESTS_TO_BIN= tests/build/misc/test-merkle-tree-hash tests/build/misc/test-machine-c-api
+TESTS_TO_BIN= tests/build/misc/test-machine-c-api
 TESTS_LUA_TO_LUA_PATH=tests/lua/cartesi
 TESTS_LUA_TO_TEST_LUA_PATH=$(wildcard tests/lua/*.lua)
 TESTS_SCRIPTS_TO_TEST_SCRIPTS_PATH=$(wildcard tests/scripts/*.sh)
@@ -96,6 +96,9 @@ TESTSDIR = $(abspath tests)
 DOWNLOADDIR = $(DEPDIR)/downloads
 SUBCLEAN = $(addsuffix .clean,$(SRCDIR) uarch tests)
 
+# Pass down received UARCH_DEFS to sub-makefiles
+export UARCH_DEFS
+
 # Docker image tag
 TAG ?= devel
 DEBIAN_IMG ?= cartesi/machine-emulator:$(TAG).deb
@@ -206,10 +209,10 @@ doc:
 bundle-boost: $(DEPDIR)/downloads/boost
 $(DEPDIR)/downloads/boost:
 	mkdir -p $(DOWNLOADDIR)
-	wget -O $(DEPDIR)/downloads/boost_1_81_0.tar.gz https://archives.boost.io/release/1.81.0/source/boost_1_81_0.tar.gz
-	tar -C $(DEPDIR)/downloads -xzf $(DEPDIR)/downloads/boost_1_81_0.tar.gz boost_1_81_0/boost
-	mv $(DEPDIR)/downloads/boost_1_81_0/boost $(DEPDIR)/downloads/boost
-	rm -rf $(DEPDIR)/downloads/boost_1_81_0.tar.gz $(DEPDIR)/downloads/boost_1_81_0
+	wget -O $(DEPDIR)/downloads/boost_1_83_0.tar.gz https://archives.boost.io/release/1.83.0/source/boost_1_83_0.tar.gz
+	tar -C $(DEPDIR)/downloads -xzf $(DEPDIR)/downloads/boost_1_83_0.tar.gz boost_1_83_0/boost
+	mv $(DEPDIR)/downloads/boost_1_83_0/boost $(DEPDIR)/downloads/boost
+	rm -rf $(DEPDIR)/downloads/boost_1_83_0.tar.gz $(DEPDIR)/downloads/boost_1_83_0
 
 submodules:
 	git submodule update --init --recursive
@@ -250,22 +253,22 @@ $(SRCDIR)/interpret-jump-table.h:
 	@eval $$($(MAKE) -s --no-print-directory env); $(MAKE) -C $(SRCDIR) interpret-jump-table.h
 
 build-emulator-builder-image:
-	docker build $(DOCKER_PLATFORM) --build-arg DEBUG=$(debug) --build-arg COVERAGE=$(coverage) --build-arg SANITIZE=$(sanitize) --target builder -t cartesi/machine-emulator:builder -f Dockerfile .
+	docker build $(DOCKER_PLATFORM) --build-arg DEBUG=$(debug) --build-arg COVERAGE=$(coverage) --build-arg THREADS=$(threads) --build-arg SANITIZE=$(sanitize) --target builder -t cartesi/machine-emulator:builder -f Dockerfile .
 
 build-emulator-toolchain-image build-toolchain:
 	docker build $(DOCKER_PLATFORM) --target toolchain -t cartesi/machine-emulator:toolchain -f Dockerfile .
 
 build-emulator-image:
-	docker build $(DOCKER_PLATFORM) --build-arg DEBUG=$(debug) --build-arg COVERAGE=$(coverage) --build-arg SANITIZE=$(sanitize) -t cartesi/machine-emulator:$(TAG) -f Dockerfile .
+	docker build $(DOCKER_PLATFORM) --build-arg DEBUG=$(debug) --build-arg COVERAGE=$(coverage) --build-arg THREADS=$(threads) --build-arg SANITIZE=$(sanitize) -t cartesi/machine-emulator:$(TAG) -f Dockerfile .
 
 build-emulator-tests-image: build-emulator-builder-image build-emulator-image
-	docker build $(DOCKER_PLATFORM) --build-arg DEBUG=$(debug) --build-arg COVERAGE=$(coverage) --build-arg SANITIZE=$(sanitize) --build-arg TAG=$(TAG) -t cartesi/machine-emulator:tests -f tests/Dockerfile .
+	docker build $(DOCKER_PLATFORM) --build-arg DEBUG=$(debug) --build-arg COVERAGE=$(coverage) --build-arg THREADS=$(threads) --build-arg SANITIZE=$(sanitize) --build-arg TAG=$(TAG) -t cartesi/machine-emulator:tests -f tests/Dockerfile .
 
 build-emulator-tests-builder-image: build-emulator-builder-image
-	docker build $(DOCKER_PLATFORM) --target tests-builder --build-arg DEBUG=$(debug) --build-arg COVERAGE=$(coverage) --build-arg SANITIZE=$(sanitize) --build-arg TAG=$(TAG) -t cartesi/machine-emulator:tests-builder -f tests/Dockerfile .
+	docker build $(DOCKER_PLATFORM) --target tests-builder --build-arg DEBUG=$(debug) --build-arg COVERAGE=$(coverage) --build-arg THREADS=$(threads) --build-arg SANITIZE=$(sanitize) --build-arg TAG=$(TAG) -t cartesi/machine-emulator:tests-builder -f tests/Dockerfile .
 
 build-debian-package:
-	docker build $(DOCKER_PLATFORM) --target debian-packager --build-arg DEBUG=$(debug) --build-arg COVERAGE=$(coverage) --build-arg SANITIZE=$(sanitize) -t $(DEBIAN_IMG) -f Dockerfile .
+	docker build $(DOCKER_PLATFORM) --target debian-packager --build-arg DEBUG=$(debug) --build-arg COVERAGE=$(coverage) --build-arg THREADS=$(threads) --build-arg SANITIZE=$(sanitize) -t $(DEBIAN_IMG) -f Dockerfile .
 
 build-tests-debian-packages: build-emulator-builder-image
 	docker build $(DOCKER_PLATFORM) --target tests-debian-packager --build-arg TAG=$(TAG) -t cartesi/machine-emulator:tests-debian-packager -f tests/Dockerfile .
@@ -306,11 +309,12 @@ toolchain-env: check-toolchain
 		cartesi/machine-emulator:toolchain /bin/bash
 
 toolchain-exec: check-toolchain
-	@docker run --hostname toolchain --rm \
+	docker run --hostname toolchain --rm \
 		-e USER=$$(id -u -n) \
 		-e GROUP=$$(id -g -n) \
 		-e UID=$$(id -u) \
 		-e GID=$$(id -g) \
+		-e UARCH_DEFS="$(UARCH_DEFS)" \
 		-v `pwd`:/opt/cartesi/machine-emulator \
 		-w /opt/cartesi/machine-emulator \
 		cartesi/machine-emulator:toolchain /bin/bash -c "$(CONTAINER_COMMAND)"