Merged all to one patch also added scripts for running locally

.gitignore

@@ -35,6 +35,7 @@ CMakeCache.txt
 *.a
 *.log
 
+evpp/
 leveldb*/
 NuRaft*/
 benchmark-results/
@@ -63,3 +64,6 @@ cmake-build-*/
 build/gcov_data
 build/tests
 
+scripts/local_log/
+.cache/
+clang-*

2pc-local.cfg

@@ -0,0 +1,23 @@
+2pc=1
+sentinel_count=1
+sentinel0_endpoint="127.0.0.1:5858"
+sentinel0_loglevel="TRACE"
+sentinel0_private_key="0000000000000001000000000000000000000000000000000000000000000000"
+sentinel0_public_key="eaa649f21f51bdbae7be4ae34ce6e5217a58fdce7f47f9aa7f3b58fa2120e2b3"
+coordinator_count=1
+coordinator0_count=1
+coordinator0_loglevel="TRACE"
+coordinator0_0_endpoint="127.0.0.1:7777"
+coordinator0_0_raft_endpoint="127.0.0.1:7779"
+coordinator_max_threads=100
+shard_count=1
+shard0_start=0
+shard0_end=255
+shard0_count=1
+shard0_loglevel="TRACE"
+shard0_0_endpoint="127.0.0.1:6666"
+shard0_0_raft_endpoint="127.0.0.1:6667"
+shard0_0_readonly_endpoint="127.0.0.1:6767"
+rest_api_endpoint="127.0.0.1:8080"
+rest_api_enable="true"
+rest_api_loglevel="TRACE"

CMakeLists.txt

@@ -32,6 +32,7 @@ find_library(LEVELDB_LIBRARY leveldb REQUIRED)
 find_library(NURAFT_LIBRARY nuraft REQUIRED)
 find_library(GTEST_LIBRARY gtest REQUIRED)
 find_library(GTEST_MAIN_LIBRARY gtest_main REQUIRED)
+find_library(SNAPPY_LIBRARY snappy_compress REQUIRED)
 
 message("Disabling Run Time Type Information (RTTI) features.")
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-rtti")

atomizer-local.cfg

@@ -0,0 +1,24 @@
+atomizer_count=1
+atomizer0_endpoint="127.0.0.1:5555"
+atomizer0_raft_endpoint="127.0.0.1:6666"
+archiver_count=1
+archiver0_endpoint="127.0.0.1:4555"
+archiver0_db="archiver0_db"
+shard_count=1
+shard0_endpoint="127.0.0.1:6555"
+shard0_start=0
+shard0_end=255
+shard0_db="shard0"
+sentinel_count=1
+sentinel0_endpoint="127.0.0.1:7555"
+sentinel0_loglevel="INFO"
+sentinel0_private_key="0000000000000001000000000000000000000000000000000000000000000000"
+sentinel0_public_key="eaa649f21f51bdbae7be4ae34ce6e5217a58fdce7f47f9aa7f3b58fa2120e2b3"
+watchtower_count=1
+watchtower0_client_endpoint="127.0.0.1:8555"
+watchtower0_internal_endpoint="127.0.0.1:8556"
+watchtower_block_cache_size=50
+target_block_interval=3000
+rest_api_endpoint="127.0.0.1:8080";
+rest_api_enable="true"
+rest_api_loglevel="TRACE"

scripts/configure.sh

@@ -22,11 +22,25 @@ fi
 CPUS=1
 if [[ "$OSTYPE" == "linux-gnu"* ]]; then
   CPUS=$(grep -c ^processor /proc/cpuinfo)
+  if [ -f "/etc/arch-release" ]; then
+    pacman -Syu
+    pacman -S wget cmake gtest lcov git rsync gcc make snappy llvm
+    $SUDO ln -s -f $(which clang-format) /usr/local/bin/clang-format
+    $SUDO ln -s -f $(which clang-tidy) /usr/local/bin/clang-tidy
+  else
+    apt update
+    apt install -y build-essential wget cmake libgtest-dev lcov git software-properties-common rsync libsnappy-dev
+    wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | $SUDO apt-key add -
+    $SUDO add-apt-repository "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main"
+    $SUDO apt install -y clang-format-14 clang-tidy-14
+    $SUDO ln -s -f $(which clang-format-14) /usr/local/bin/clang-format
+    $SUDO ln -s -f $(which clang-tidy-14) /usr/local/bin/clang-tidy
+  fi
 elif [[ "$OSTYPE" == "darwin"* ]]; then
   CPUS=$(sysctl -n hw.ncpu)
   # ensure development environment is set correctly for clang
   $SUDO xcode-select -switch /Library/Developer/CommandLineTools
-  brew install llvm@14 googletest lcov make wget cmake
+  brew install llvm@14 googletest lcov make wget cmake snappy
   CLANG_TIDY=/usr/local/bin/clang-tidy
   if [ ! -L "$CLANG_TIDY" ]; then
     $SUDO ln -s $(brew --prefix)/opt/llvm@14/bin/clang-tidy /usr/local/bin/clang-tidy
@@ -37,17 +51,6 @@ elif [[ "$OSTYPE" == "darwin"* ]]; then
   fi
 fi
 
-if [[ "$OSTYPE" == "linux-gnu"* ]]; then
-  apt update
-  apt install -y build-essential wget cmake libgtest-dev lcov git software-properties-common rsync
-
-  wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | $SUDO apt-key add -
-  $SUDO add-apt-repository "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main"
-  $SUDO apt install -y clang-format-14 clang-tidy-14
-  $SUDO ln -s -f $(which clang-format-14) /usr/local/bin/clang-format
-  $SUDO ln -s -f $(which clang-tidy-14) /usr/local/bin/clang-tidy
-fi
-
 LEVELDB_VERSION="1.23"
 echo -e "${green}Building LevelDB from sources...${end}"
 wget https://github.com/google/leveldb/archive/${LEVELDB_VERSION}.tar.gz

scripts/local_run.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+function scriptDir
+{
+        SOURCE="${BASH_SOURCE[0]}"
+
+        while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
+                DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+                SOURCE="$(readlink "$SOURCE")"
+                [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+        done
+        DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+        echo $DIR
+}
+
+sdir="$( scriptDir )"
+
+mkdir -p $sdir/local_log || exit 1
+cd $sdir/local_log || exit 2
+
+#TODO:
+#sleep for fixed time isn't ideal but works for now
+#it looks like services needs some interval maybe some reconnect logic should
+#be added upon start to resolve this and let starting this daemons in parallel
+
+echo "Starting locking shard"
+$sdir/../build/src/uhs/twophase/locking_shard/locking-shardd $sdir/../2pc-local.cfg 0 0 &> locking-shardd.log&
+sleep 5
+echo "Starting coordinatord"
+$sdir/../build/src/uhs/twophase/coordinator/coordinatord $sdir/../2pc-local.cfg 0 0 &> coordinatord.log&
+sleep 5
+echo "Starting sentineld"
+$sdir/../build/src/uhs/twophase/sentinel_2pc/sentineld-2pc $sdir/../2pc-local.cfg 0 &> sentineld.log&
+
+trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT
+
+wait $(jobs -p)

src/uhs/atomizer/archiver/CMakeLists.txt

@@ -15,4 +15,5 @@ target_link_libraries(archiverd archiver
                                 ${LEVELDB_LIBRARY}
                                 ${NURAFT_LIBRARY}
                                 secp256k1
-                                ${CMAKE_THREAD_LIBS_INIT})
+                                ${CMAKE_THREAD_LIBS_INIT}
+				snappy)

src/uhs/atomizer/atomizer/CMakeLists.txt

@@ -23,4 +23,5 @@ target_link_libraries(atomizer-raftd atomizer_raft
                                      ${NURAFT_LIBRARY}
                                      ${LEVELDB_LIBRARY}
                                      secp256k1
-                                     ${CMAKE_THREAD_LIBS_INIT})
+                                     ${CMAKE_THREAD_LIBS_INIT}
+				     snappy)

src/uhs/atomizer/shard/CMakeLists.txt

@@ -16,4 +16,5 @@ target_link_libraries(shardd shard
                              ${LEVELDB_LIBRARY}
                              ${NURAFT_LIBRARY}
                              secp256k1
-                             ${CMAKE_THREAD_LIBS_INIT})
+                             ${CMAKE_THREAD_LIBS_INIT}
+			     snappy)

src/uhs/atomizer/watchtower/CMakeLists.txt

@@ -21,4 +21,5 @@ target_link_libraries(watchtowerd watchtower
                                   crypto
                                   ${NURAFT_LIBRARY}
                                   secp256k1
-                                  ${CMAKE_THREAD_LIBS_INIT})
+                                  ${CMAKE_THREAD_LIBS_INIT}
+				  snappy)

src/uhs/client/CMakeLists.txt

@@ -4,10 +4,10 @@ add_library(client atomizer_client.cpp
                    client.cpp
                    twophase_client.cpp)
 
+target_link_libraries(client bech32)
+
 add_executable(client-cli client-cli.cpp)
 target_link_libraries(client-cli client
-                                 atomizer
-                                 bech32
                                  atomizer
                                  watchtower
                                  sentinel

src/uhs/client/client-cli.cpp

@@ -4,21 +4,18 @@
 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 #include "atomizer_client.hpp"
-#include "bech32/bech32.h"
-#include "bech32/util/strencodings.h"
 #include "client.hpp"
 #include "crypto/sha256.h"
 #include "twophase_client.hpp"
-#include "uhs/transaction/messages.hpp"
 #include "util/common/config.hpp"
 #include "util/serialization/util.hpp"
+#include "uhs/transaction/messages.hpp"
+#include "bech32/bech32.h"
+#include "bech32/util/strencodings.h"
 
 #include <future>
 #include <iostream>
 
-static constexpr auto bits_per_byte = 8;
-static constexpr auto bech32_bits_per_symbol = 5;
-
 auto mint_command(cbdc::client& client, const std::vector<std::string>& args)
     -> bool {
     static constexpr auto min_mint_arg_count = 7;
@@ -38,36 +35,6 @@ auto mint_command(cbdc::client& client, const std::vector<std::string>& args)
     return true;
 }
 
-auto decode_address(const std::string& addr_str)
-    -> std::optional<cbdc::hash_t> {
-    // TODO: if/when bech32m is merged into Bitcoin Core, switch to that.
-    //       see: https://github.com/bitcoin/bitcoin/pull/20861
-    // TODO: move address encoding/decoding into cbdc::client.
-    const auto [hrp, enc_data] = bech32::Decode(addr_str);
-    if(hrp != cbdc::config::bech32_hrp) {
-        std::cout << "Invalid address encoding" << std::endl;
-        return std::nullopt;
-    }
-    auto data = std::vector<uint8_t>();
-    ConvertBits<bech32_bits_per_symbol, bits_per_byte, false>(
-        [&](uint8_t c) {
-            data.push_back(c);
-        },
-        enc_data.begin(),
-        enc_data.end());
-
-    auto pubkey = cbdc::hash_t();
-    if(data[0] != static_cast<uint8_t>(cbdc::client::address_type::public_key)
-       || data.size() != pubkey.size() + 1) {
-        std::cout << "Address is not a supported type" << std::endl;
-        return std::nullopt;
-    }
-    data.erase(data.begin());
-    std::copy_n(data.begin(), pubkey.size(), pubkey.begin());
-
-    return pubkey;
-}
-
 void print_tx_result(
     const std::optional<cbdc::transaction::full_tx>& tx,
     const std::optional<cbdc::sentinel::execute_response>& resp,
@@ -105,7 +72,7 @@ auto send_command(cbdc::client& client, const std::vector<std::string>& args)
 
     const auto value = std::stoul(args[5]);
     static constexpr auto address_arg_idx = 6;
-    auto pubkey = decode_address(args[address_arg_idx]);
+    auto pubkey = cbdc::address::decode(args[address_arg_idx]);
     if(!pubkey.has_value()) {
         std::cout << "Could not decode address" << std::endl;
         return false;
@@ -134,7 +101,7 @@ auto fan_command(cbdc::client& client, const std::vector<std::string>& args)
     const auto count = std::stoul(args[5]);
 
     static constexpr auto address_arg_idx = 7;
-    auto pubkey = decode_address(args[address_arg_idx]);
+    auto pubkey = cbdc::address::decode(args[address_arg_idx]);
     if(!pubkey.has_value()) {
         std::cout << "Could not decode address" << std::endl;
         return false;
@@ -163,7 +130,7 @@ void newaddress_command(cbdc::client& client) {
                 addr_vec.begin()
                     + sizeof(cbdc::client::address_type::public_key));
     auto data = std::vector<uint8_t>();
-    ConvertBits<bits_per_byte, bech32_bits_per_symbol, true>(
+    ConvertBits<cbdc::address::bits_per_byte, cbdc::address::bech32_bits_per_symbol, true>(
         [&](uint8_t c) {
             data.push_back(c);
         },

src/uhs/client/client.cpp

@@ -14,12 +14,45 @@
 #include "util/serialization/istream_serializer.hpp"
 #include "util/serialization/ostream_serializer.hpp"
 #include "util/serialization/util.hpp"
-
+#include "bech32/bech32.h"
+#include "bech32/util/strencodings.h"
 #include <filesystem>
 #include <iomanip>
 #include <utility>
 
 namespace cbdc {
+
+    namespace address{
+        auto decode(const std::string& addr_str)
+        -> std::optional<cbdc::hash_t> { (void) addr_str;
+            // TODO: if/when bech32m is merged into Bitcoin Core, switch to that.
+            //       see: https://github.com/bitcoin/bitcoin/pull/20861
+            const auto [hrp, enc_data] = bech32::Decode(addr_str);
+            if(hrp != cbdc::config::bech32_hrp) {
+                std::cout << "Invalid address encoding" << std::endl;
+                return std::nullopt;
+            }
+            auto data = std::vector<uint8_t>();
+            ConvertBits<bech32_bits_per_symbol, bits_per_byte, false>(
+                [&](uint8_t c) {
+                    data.push_back(c);
+                },
+                enc_data.begin(),enc_data.end());
+
+            auto pubkey = cbdc::hash_t();
+            if(data[0] != static_cast<uint8_t>(cbdc::client::address_type::public_key)
+                || data.size() != pubkey.size() + 1) {
+                std::cout << "Address is not a supported type" << std::endl;
+                return std::nullopt;
+            }
+
+            data.erase(data.begin());
+                std::copy_n(data.begin(), pubkey.size(), pubkey.begin());
+
+                return pubkey;
+        }
+    }
+
     client::client(cbdc::config::options opts,
                    std::shared_ptr<logging::log> logger,
                    std::string wallet_file,

src/uhs/client/client.hpp

@@ -11,6 +11,14 @@
 #include "uhs/transaction/wallet.hpp"
 
 namespace cbdc {
+    namespace address {
+        static constexpr auto bits_per_byte = 8;
+        static constexpr auto bech32_bits_per_symbol = 5; 
+
+        auto decode(const std::string& addr_str)
+        -> std::optional<cbdc::hash_t>;
+    }
+
     /// External client for sending new transactions to the system.
     class client {
       public:

src/uhs/twophase/coordinator/CMakeLists.txt

@@ -20,4 +20,5 @@ target_link_libraries(coordinatord coordinator
                                    ${NURAFT_LIBRARY}
                                    ${LEVELDB_LIBRARY}
                                    secp256k1
-                                   ${CMAKE_THREAD_LIBS_INIT})
+                                   ${CMAKE_THREAD_LIBS_INIT}
+				   snappy)

src/uhs/twophase/locking_shard/CMakeLists.txt

@@ -22,4 +22,5 @@ target_link_libraries(locking-shardd locking_shard
                                      secp256k1
                                      ${NURAFT_LIBRARY}
                                      ${LEVELDB_LIBRARY}
-                                     ${CMAKE_THREAD_LIBS_INIT})
+                                     ${CMAKE_THREAD_LIBS_INIT}
+				     snappy)

tests/integration/CMakeLists.txt

@@ -37,4 +37,5 @@ target_link_libraries(run_integration_tests ${GTEST_LIBRARY}
                                             secp256k1
                                             ${LEVELDB_LIBRARY}
                                             ${NURAFT_LIBRARY}
-                                            ${CMAKE_THREAD_LIBS_INIT})
+                                            ${CMAKE_THREAD_LIBS_INIT}
+					    snappy)

tests/unit/CMakeLists.txt

@@ -55,4 +55,5 @@ target_link_libraries(run_unit_tests ${GTEST_LIBRARY}
                                      secp256k1
                                      ${LEVELDB_LIBRARY}
                                      ${NURAFT_LIBRARY}
-                                     ${CMAKE_THREAD_LIBS_INIT})
+                                     ${CMAKE_THREAD_LIBS_INIT}
+				     snappy)

tools/shard-seeder/CMakeLists.txt

@@ -12,4 +12,5 @@ target_link_libraries(shard-seeder transaction
                                    crypto
                                    ${LEVELDB_LIBRARY}
                                    ${CMAKE_THREAD_LIBS_INIT}
-                                   secp256k1)
+                                   secp256k1
+				   snappy)