Added microbenchmarks using google test

This suite measures performance of N-1 and N-2 transactions, low_level fundamental operations of the Tx processor, leveldb storage and deletion, and std::unordered_set storage and deletion. Signed-off-by: Michael Maurer <[email protected]>
mit-dci · Feb 6, 2023 · e9ff612 · e9ff612
1 parent 5f2781d
commit e9ff612
Show file tree

Hide file tree

Showing 9 changed files with 706 additions and 4 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -1,6 +1,5 @@
 build
 run
-benchmarks
 plots
 .vscode
 .deps

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -32,10 +32,11 @@ find_library(LEVELDB_LIBRARY leveldb REQUIRED)
 find_library(NURAFT_LIBRARY nuraft REQUIRED)
 find_library(GTEST_LIBRARY gtest REQUIRED)
 find_library(GTEST_MAIN_LIBRARY gtest_main REQUIRED)
+find_package(benchmark REQUIRED)
 
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-rtti")
 
-include_directories(3rdparty 3rdparty/secp256k1/include /usr/local/include /opt/homebrew/include)
+include_directories(3rdparty 3rdparty/secp256k1/include /usr/lib /usr/local/lib /usr/local/include /opt/homebrew/include)
 
 if(CMAKE_BUILD_TYPE STREQUAL "Debug")
     add_compile_options(-fprofile-arcs -ftest-coverage)
@@ -74,5 +75,6 @@ endif()
 
 add_subdirectory(src)
 add_subdirectory(tests)
+add_subdirectory(benchmarks)
 add_subdirectory(tools/bench)
 add_subdirectory(tools/shard-seeder)
diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt
@@ -0,0 +1,26 @@
+project(benchmarks)
+
+include_directories(. ../src ../tools/watchtower ../3rdparty ../3rdparty/secp256k1/include)
+set(SECP256K1_LIBRARY $<TARGET_FILE:secp256k1>)
+
+add_executable(run_benchmarks   low_level.cpp
+                                transactions.cpp
+                                uhs_leveldb.cpp
+                                uhs_set.cpp
+                                )
+
+target_compile_options(run_benchmarks PRIVATE -ftest-coverage -fprofile-arcs)
+target_link_options(run_benchmarks PRIVATE --coverage)
+target_link_libraries(run_benchmarks ${GTEST_LIBRARY}
+                                     ${GTEST_MAIN_LIBRARY}
+                                     benchmark::benchmark
+                                     util
+                                     shard
+                                     locking_shard
+                                     transaction
+                                     common
+                                     serialization
+                                     crypto
+                                     secp256k1
+                                     ${LEVELDB_LIBRARY}
+                                     ${CMAKE_THREAD_LIBS_INIT})
diff --git a/benchmarks/low_level.cpp b/benchmarks/low_level.cpp
@@ -0,0 +1,180 @@
+// Copyright (c) 2021 MIT Digital Currency Initiative,
+//                    Federal Reserve Bank of Boston
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+// Note: Contains call to BENCHMARK_MAIN
+
+#include "uhs/transaction/messages.hpp"
+#include "uhs/transaction/transaction.hpp"
+#include "uhs/transaction/validation.hpp"
+#include "uhs/transaction/wallet.hpp"
+#include "util/serialization/istream_serializer.hpp"
+#include "util/serialization/ostream_serializer.hpp"
+
+#include <benchmark/benchmark.h>
+#include <filesystem>
+#include <gtest/gtest.h>
+
+class low_level : public ::benchmark::Fixture {
+  protected:
+    void SetUp(const ::benchmark::State&) override {
+        auto mint_tx1 = wallet1.mint_new_coins(100, 2);
+        wallet1.confirm_transaction(mint_tx1);
+
+        m_of.open(m_BENCHMARK_File);
+
+        counter = 1;
+    }
+
+    void TearDown(const ::benchmark::State&) override {
+        m_if.close();
+        m_of.close();
+        std::filesystem::remove_all(m_BENCHMARK_File);
+    };
+
+    cbdc::transaction::wallet wallet1;
+    cbdc::transaction::wallet wallet2;
+
+    cbdc::transaction::full_tx m_valid_tx{};
+    cbdc::transaction::full_tx m_valid_tx_multi_inp{};
+
+    // for serialization
+    std::ifstream m_if;
+    std::ofstream m_of;
+
+    cbdc::istream_serializer m_is{m_if};
+    cbdc::ostream_serializer m_os{m_of};
+
+    static constexpr auto m_BENCHMARK_File = "serial_BENCHMARK_File.dat";
+
+    // used to ensure that transaction validation timing
+    // uses unique N values for N->1 transactions
+    int counter = 1;
+};
+
+// serialize full tx
+BENCHMARK_F(low_level, serialize_tx)(benchmark::State& state) {
+    m_valid_tx = wallet1.send_to(2, wallet2.generate_key(), true).value();
+    for(auto _ : state) {
+        m_os << m_valid_tx;
+    }
+    m_of.close();
+}
+
+// serialize compact tx
+BENCHMARK_F(low_level, serialize_compact_tx)(benchmark::State& state) {
+    // serialize 1-1 tx
+    m_valid_tx = wallet1.send_to(2, wallet2.generate_key(), true).value();
+    auto cp_tx = cbdc::transaction::compact_tx(m_valid_tx);
+    for(auto _ : state) {
+        m_os << cp_tx;
+    }
+    m_of.close();
+}
+
+// deserialize full tx
+BENCHMARK_F(low_level, deserialize_tx)(benchmark::State& state) {
+    // serialize 1-1 tx
+    auto read_tx = cbdc::transaction::full_tx();
+    m_valid_tx = wallet1.send_to(2, wallet2.generate_key(), true).value();
+    for(auto _ : state) {
+        state.PauseTiming();
+        m_os.reset();
+        m_os << m_valid_tx;
+        m_of.close();
+        m_if.open(m_BENCHMARK_File);
+
+        read_tx = cbdc::transaction::full_tx();
+        m_is.reset();
+        state.ResumeTiming();
+        m_is >> read_tx;
+        state.PauseTiming();
+        ASSERT_EQ(read_tx, m_valid_tx);
+    }
+
+    m_if.close();
+}
+
+// deserialize compact tx
+BENCHMARK_F(low_level, deserialize_compact_tx)(benchmark::State& state) {
+    // serialize 1-1 tx
+    auto read_tx = cbdc::transaction::full_tx();
+    auto read_cp = cbdc::transaction::compact_tx(read_tx);
+
+    m_valid_tx = wallet1.send_to(2, wallet2.generate_key(), true).value();
+    auto cp_tx = cbdc::transaction::compact_tx(m_valid_tx);
+    for(auto _ : state) {
+        state.PauseTiming();
+        m_os.reset();
+        m_os << cp_tx;
+        m_of.close();
+        m_if.open(m_BENCHMARK_File);
+
+        read_cp = cbdc::transaction::compact_tx(read_tx);
+        m_is.reset();
+        state.ResumeTiming();
+        m_is >> read_cp;
+        state.PauseTiming();
+        ASSERT_EQ(cp_tx, read_cp);
+    }
+
+    m_if.close();
+}
+
+// wallet sign tx
+BENCHMARK_F(low_level, sign_tx)(benchmark::State& state) {
+    // sign 1-1 tx
+    for(auto _ : state) {
+        state.PauseTiming();
+        m_valid_tx = wallet1.send_to(2, wallet1.generate_key(), true).value();
+
+        state.ResumeTiming();
+        wallet1.sign(m_valid_tx);
+        state.PauseTiming();
+
+        wallet1.confirm_transaction(m_valid_tx);
+    }
+}
+
+// tx validiation
+BENCHMARK_F(low_level, valid_tx)(benchmark::State& state) {
+    // validate n-1 tx
+    auto err = cbdc::transaction::validation::check_tx(m_valid_tx);
+    for(auto _ : state) {
+        m_valid_tx = wallet1.mint_new_coins(counter, 1);
+        wallet1.confirm_transaction(m_valid_tx);
+        m_valid_tx
+            = wallet1.send_to(counter, wallet2.generate_key(), true).value();
+        state.ResumeTiming();
+        err = cbdc::transaction::validation::check_tx(m_valid_tx);
+        state.PauseTiming();
+        counter++;
+    }
+}
+
+// test quick-failing validation
+BENCHMARK_F(low_level, no_inputs)(benchmark::State& state) {
+    m_valid_tx.m_inputs.clear();
+    auto err = cbdc::transaction::validation::check_tx(m_valid_tx);
+    for(auto _ : state) {
+        err = cbdc::transaction::validation::check_tx(m_valid_tx);
+    }
+}
+
+// calculate uhs id
+BENCHMARK_F(low_level, calculate_uhs_id)(benchmark::State& state) {
+    m_valid_tx = wallet1.send_to(2, wallet2.generate_key(), true).value();
+    auto cp_tx = cbdc::transaction::compact_tx(m_valid_tx);
+    auto engine = std::default_random_engine();
+    for(auto _ : state) {
+        state.PauseTiming();
+        uint64_t i = (uint64_t)engine();
+        state.ResumeTiming();
+        cbdc::transaction::uhs_id_from_output(cp_tx.m_id,
+                                              i,
+                                              m_valid_tx.m_outputs[0]);
+    }
+}
+
+BENCHMARK_MAIN();
diff --git a/benchmarks/transactions.cpp b/benchmarks/transactions.cpp
@@ -0,0 +1,138 @@
+// Copyright (c) 2021 MIT Digital Currency Initiative,
+//                    Federal Reserve Bank of Boston
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include "uhs/transaction/transaction.hpp"
+#include "uhs/transaction/validation.hpp"
+#include "uhs/transaction/wallet.hpp"
+
+#include <benchmark/benchmark.h>
+#include <gtest/gtest.h>
+
+#define SWEEP_MAX 32
+
+// reset wallets to default status
+void reset_wallets(cbdc::transaction::wallet& w1,
+                   cbdc::transaction::wallet& w2,
+                   uint32_t init_count) {
+    cbdc::transaction::full_tx m_valid_tx{};
+    if(w1.balance() + w2.balance() < init_count * 2) {
+        auto mint_tx = w1.mint_new_coins(1,
+                                         init_count * 2
+                                             - (w1.balance() + w2.balance()));
+        w1.confirm_transaction(mint_tx);
+    }
+    if(w2.balance() > 0) {
+        m_valid_tx = w2.send_to(w2.balance(), w1.generate_key(), true).value();
+        w1.confirm_transaction(m_valid_tx);
+        w2.confirm_transaction(m_valid_tx);
+    }
+    if(w1.count() != init_count) {
+        // generate an N-1 transaction
+        m_valid_tx = w1.send_to(w1.balance(), w1.generate_key(), true).value();
+        w1.confirm_transaction(m_valid_tx);
+        // fan to 10 (value-2 UTXOs)
+        m_valid_tx = w1.fan(init_count, 2, w1.generate_key(), true).value();
+        w1.confirm_transaction(m_valid_tx);
+    }
+}
+
+/// @brief Time an N-in, 1-out transaction.
+/// @brief Note: handles benchmark timing, do not time outside function.
+/// @param sender
+/// @param reciever
+/// @param n_in
+/// @param state
+/// @return
+inline bool generate_Nto1_tx(cbdc::transaction::wallet& sender,
+                             cbdc::transaction::wallet& reciever,
+                             uint32_t n_in,
+                             benchmark::State& state) {
+    std::optional<cbdc::transaction::full_tx> maybe_tx{};
+    state.ResumeTiming();
+    maybe_tx = sender.send_to(n_in * 2, reciever.generate_key(), true).value();
+    state.PauseTiming();
+    if(maybe_tx.has_value()) {
+        sender.confirm_transaction(*maybe_tx);
+        reciever.confirm_transaction(*maybe_tx);
+        return true;
+    }
+    return false;
+}
+
+/// @brief Time an N-in, 2-out transaction.
+/// @brief Note: handles benchmark timing, do not time outside function.
+/// @param sender
+/// @param reciever
+/// @param n_in
+/// @param state
+/// @return
+inline bool generate_Nto2_tx(cbdc::transaction::wallet& sender,
+                             cbdc::transaction::wallet& reciever,
+                             uint32_t n_in,
+                             benchmark::State& state) {
+    std::optional<cbdc::transaction::full_tx> maybe_tx{};
+    state.ResumeTiming();
+    maybe_tx
+        = sender.send_to(n_in * 2 - 1, reciever.generate_key(), true).value();
+    state.PauseTiming();
+    if(maybe_tx.has_value()) {
+        sender.confirm_transaction(*maybe_tx);
+        reciever.confirm_transaction(*maybe_tx);
+        return true;
+    }
+    return false;
+}
+
+// Benchmarkable N in 1 out transaction (for sweep)
+static void Nto1_tx(benchmark::State& state) {
+    cbdc::transaction::wallet wallet_a;
+    cbdc::transaction::wallet wallet_b;
+    bool valid;
+    for(auto _ : state) {
+        reset_wallets(wallet_a, wallet_b, SWEEP_MAX);
+        valid = generate_Nto1_tx(wallet_a, wallet_b, state.range(0), state);
+        if(!valid) {
+            GTEST_LOG_(ERROR) << state.range(0) << "-2 transaction invalid";
+            return;
+        }
+        ASSERT_EQ(wallet_a.balance(), SWEEP_MAX * 2 - 2 * state.range(0));
+        ASSERT_EQ(wallet_a.count(), SWEEP_MAX - state.range(0));
+        ASSERT_EQ(wallet_b.balance(), state.range(0) * 2);
+        ASSERT_EQ(wallet_b.count(), 1);
+    }
+    state.SetComplexityN(state.range(0));
+}
+
+// Benchmarkable N in 2 out transaction (for sweep)
+static void Nto2_tx(benchmark::State& state) {
+    cbdc::transaction::wallet wallet_a;
+    cbdc::transaction::wallet wallet_b;
+    bool valid;
+    for(auto _ : state) {
+        reset_wallets(wallet_a, wallet_b, SWEEP_MAX);
+        valid = generate_Nto2_tx(wallet_a, wallet_b, state.range(0), state);
+        if(!valid) {
+            GTEST_LOG_(ERROR) << state.range(0) << "-2 transaction invalid";
+            return;
+        }
+        ASSERT_EQ(wallet_a.balance(), SWEEP_MAX * 2 - 2 * state.range(0) + 1);
+        ASSERT_EQ(wallet_a.count(), SWEEP_MAX - state.range(0) + 1);
+        ASSERT_EQ(wallet_b.balance(), state.range(0) * 2 - 1);
+        ASSERT_EQ(wallet_b.count(), 1);
+    }
+    state.SetComplexityN(state.range(0));
+}
+
+// Benchmark declarations
+BENCHMARK(Nto1_tx)
+    ->RangeMultiplier(2)
+    ->Range(1, SWEEP_MAX)
+    ->Complexity(benchmark::oAuto);
+
+BENCHMARK(Nto2_tx)
+    ->RangeMultiplier(2)
+    ->Range(1, SWEEP_MAX)
+    ->Complexity(benchmark::oAuto);
+