From ca23e90d0090d7408e4b5e33ded99db9c49afcfc Mon Sep 17 00:00:00 2001 From: paparuch Date: Wed, 7 Sep 2022 22:51:24 +0000 Subject: [PATCH 1/6] Added snappy library check and for multiple target linking Signed-off-by: paparuch --- CMakeLists.txt | 2 ++ src/uhs/atomizer/archiver/CMakeLists.txt | 3 ++- src/uhs/atomizer/atomizer/CMakeLists.txt | 3 ++- src/uhs/atomizer/shard/CMakeLists.txt | 3 ++- src/uhs/atomizer/watchtower/CMakeLists.txt | 3 ++- src/uhs/twophase/coordinator/CMakeLists.txt | 3 ++- src/uhs/twophase/locking_shard/CMakeLists.txt | 3 ++- tests/integration/CMakeLists.txt | 3 ++- tests/unit/CMakeLists.txt | 3 ++- tools/shard-seeder/CMakeLists.txt | 3 ++- 10 files changed, 20 insertions(+), 9 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 16ed4825f..ccc28af5b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -32,6 +32,8 @@ find_library(LEVELDB_LIBRARY leveldb REQUIRED) find_library(NURAFT_LIBRARY nuraft REQUIRED) find_library(GTEST_LIBRARY gtest REQUIRED) find_library(GTEST_MAIN_LIBRARY gtest_main REQUIRED) +include(CheckLibraryExists) +check_library_exists(snappy snappy_compress "" HAVE_SNAPPY) message("Disabling Run Time Type Information (RTTI) features.") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-rtti") diff --git a/src/uhs/atomizer/archiver/CMakeLists.txt b/src/uhs/atomizer/archiver/CMakeLists.txt index 375d4ef92..e1e417d11 100644 --- a/src/uhs/atomizer/archiver/CMakeLists.txt +++ b/src/uhs/atomizer/archiver/CMakeLists.txt @@ -15,4 +15,5 @@ target_link_libraries(archiverd archiver ${LEVELDB_LIBRARY} ${NURAFT_LIBRARY} secp256k1 - ${CMAKE_THREAD_LIBS_INIT}) + ${CMAKE_THREAD_LIBS_INIT} + snappy) diff --git a/src/uhs/atomizer/atomizer/CMakeLists.txt b/src/uhs/atomizer/atomizer/CMakeLists.txt index 5dabd0b8b..89e8e011d 100644 --- a/src/uhs/atomizer/atomizer/CMakeLists.txt +++ b/src/uhs/atomizer/atomizer/CMakeLists.txt @@ -23,4 +23,5 @@ target_link_libraries(atomizer-raftd atomizer_raft ${NURAFT_LIBRARY} ${LEVELDB_LIBRARY} secp256k1 - ${CMAKE_THREAD_LIBS_INIT}) + ${CMAKE_THREAD_LIBS_INIT} + snappy) diff --git a/src/uhs/atomizer/shard/CMakeLists.txt b/src/uhs/atomizer/shard/CMakeLists.txt index 844b314db..f380b724c 100644 --- a/src/uhs/atomizer/shard/CMakeLists.txt +++ b/src/uhs/atomizer/shard/CMakeLists.txt @@ -16,4 +16,5 @@ target_link_libraries(shardd shard ${LEVELDB_LIBRARY} ${NURAFT_LIBRARY} secp256k1 - ${CMAKE_THREAD_LIBS_INIT}) + ${CMAKE_THREAD_LIBS_INIT} + snappy) diff --git a/src/uhs/atomizer/watchtower/CMakeLists.txt b/src/uhs/atomizer/watchtower/CMakeLists.txt index 21612f7b7..24d6ed41f 100644 --- a/src/uhs/atomizer/watchtower/CMakeLists.txt +++ b/src/uhs/atomizer/watchtower/CMakeLists.txt @@ -21,4 +21,5 @@ target_link_libraries(watchtowerd watchtower crypto ${NURAFT_LIBRARY} secp256k1 - ${CMAKE_THREAD_LIBS_INIT}) + ${CMAKE_THREAD_LIBS_INIT} + snappy) diff --git a/src/uhs/twophase/coordinator/CMakeLists.txt b/src/uhs/twophase/coordinator/CMakeLists.txt index 9aee6bc37..2fec2caf7 100644 --- a/src/uhs/twophase/coordinator/CMakeLists.txt +++ b/src/uhs/twophase/coordinator/CMakeLists.txt @@ -20,4 +20,5 @@ target_link_libraries(coordinatord coordinator ${NURAFT_LIBRARY} ${LEVELDB_LIBRARY} secp256k1 - ${CMAKE_THREAD_LIBS_INIT}) + ${CMAKE_THREAD_LIBS_INIT} + snappy) diff --git a/src/uhs/twophase/locking_shard/CMakeLists.txt b/src/uhs/twophase/locking_shard/CMakeLists.txt index 42ad6b5e1..970b35088 100644 --- a/src/uhs/twophase/locking_shard/CMakeLists.txt +++ b/src/uhs/twophase/locking_shard/CMakeLists.txt @@ -22,4 +22,5 @@ target_link_libraries(locking-shardd locking_shard secp256k1 ${NURAFT_LIBRARY} ${LEVELDB_LIBRARY} - ${CMAKE_THREAD_LIBS_INIT}) + ${CMAKE_THREAD_LIBS_INIT} + snappy) diff --git a/tests/integration/CMakeLists.txt b/tests/integration/CMakeLists.txt index ef8968704..d0dde2990 100644 --- a/tests/integration/CMakeLists.txt +++ b/tests/integration/CMakeLists.txt @@ -37,4 +37,5 @@ target_link_libraries(run_integration_tests ${GTEST_LIBRARY} secp256k1 ${LEVELDB_LIBRARY} ${NURAFT_LIBRARY} - ${CMAKE_THREAD_LIBS_INIT}) + ${CMAKE_THREAD_LIBS_INIT} + snappy) diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index 7f9df8f6b..2feb04d73 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -55,4 +55,5 @@ target_link_libraries(run_unit_tests ${GTEST_LIBRARY} secp256k1 ${LEVELDB_LIBRARY} ${NURAFT_LIBRARY} - ${CMAKE_THREAD_LIBS_INIT}) + ${CMAKE_THREAD_LIBS_INIT} + snappy) diff --git a/tools/shard-seeder/CMakeLists.txt b/tools/shard-seeder/CMakeLists.txt index 3bac1adb3..5a82634c9 100644 --- a/tools/shard-seeder/CMakeLists.txt +++ b/tools/shard-seeder/CMakeLists.txt @@ -12,4 +12,5 @@ target_link_libraries(shard-seeder transaction crypto ${LEVELDB_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} - secp256k1) + secp256k1 + snappy) From 47ddade289d53adfe1e7138264c198d29d7eb764 Mon Sep 17 00:00:00 2001 From: paparuch Date: Sat, 10 Sep 2022 01:48:20 +0000 Subject: [PATCH 2/6] Moved decode_address from client-cli to client Signed-off-by: paparuch --- src/uhs/client/CMakeLists.txt | 4 ++-- src/uhs/client/client-cli.cpp | 45 +++++------------------------------ src/uhs/client/client.cpp | 35 ++++++++++++++++++++++++++- src/uhs/client/client.hpp | 8 +++++++ 4 files changed, 50 insertions(+), 42 deletions(-) diff --git a/src/uhs/client/CMakeLists.txt b/src/uhs/client/CMakeLists.txt index dcf328bd4..9dddf9df0 100644 --- a/src/uhs/client/CMakeLists.txt +++ b/src/uhs/client/CMakeLists.txt @@ -4,10 +4,10 @@ add_library(client atomizer_client.cpp client.cpp twophase_client.cpp) +target_link_libraries(client bech32) + add_executable(client-cli client-cli.cpp) target_link_libraries(client-cli client - atomizer - bech32 atomizer watchtower sentinel diff --git a/src/uhs/client/client-cli.cpp b/src/uhs/client/client-cli.cpp index 0801b79e0..a94eedb02 100644 --- a/src/uhs/client/client-cli.cpp +++ b/src/uhs/client/client-cli.cpp @@ -4,21 +4,18 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "atomizer_client.hpp" -#include "bech32/bech32.h" -#include "bech32/util/strencodings.h" #include "client.hpp" #include "crypto/sha256.h" #include "twophase_client.hpp" -#include "uhs/transaction/messages.hpp" #include "util/common/config.hpp" #include "util/serialization/util.hpp" +#include "uhs/transaction/messages.hpp" +#include "bech32/bech32.h" +#include "bech32/util/strencodings.h" #include #include -static constexpr auto bits_per_byte = 8; -static constexpr auto bech32_bits_per_symbol = 5; - auto mint_command(cbdc::client& client, const std::vector& args) -> bool { static constexpr auto min_mint_arg_count = 7; @@ -38,36 +35,6 @@ auto mint_command(cbdc::client& client, const std::vector& args) return true; } -auto decode_address(const std::string& addr_str) - -> std::optional { - // TODO: if/when bech32m is merged into Bitcoin Core, switch to that. - // see: https://github.com/bitcoin/bitcoin/pull/20861 - // TODO: move address encoding/decoding into cbdc::client. - const auto [hrp, enc_data] = bech32::Decode(addr_str); - if(hrp != cbdc::config::bech32_hrp) { - std::cout << "Invalid address encoding" << std::endl; - return std::nullopt; - } - auto data = std::vector(); - ConvertBits( - [&](uint8_t c) { - data.push_back(c); - }, - enc_data.begin(), - enc_data.end()); - - auto pubkey = cbdc::hash_t(); - if(data[0] != static_cast(cbdc::client::address_type::public_key) - || data.size() != pubkey.size() + 1) { - std::cout << "Address is not a supported type" << std::endl; - return std::nullopt; - } - data.erase(data.begin()); - std::copy_n(data.begin(), pubkey.size(), pubkey.begin()); - - return pubkey; -} - void print_tx_result( const std::optional& tx, const std::optional& resp, @@ -105,7 +72,7 @@ auto send_command(cbdc::client& client, const std::vector& args) const auto value = std::stoul(args[5]); static constexpr auto address_arg_idx = 6; - auto pubkey = decode_address(args[address_arg_idx]); + auto pubkey = cbdc::address::decode(args[address_arg_idx]); if(!pubkey.has_value()) { std::cout << "Could not decode address" << std::endl; return false; @@ -134,7 +101,7 @@ auto fan_command(cbdc::client& client, const std::vector& args) const auto count = std::stoul(args[5]); static constexpr auto address_arg_idx = 7; - auto pubkey = decode_address(args[address_arg_idx]); + auto pubkey = cbdc::address::decode(args[address_arg_idx]); if(!pubkey.has_value()) { std::cout << "Could not decode address" << std::endl; return false; @@ -163,7 +130,7 @@ void newaddress_command(cbdc::client& client) { addr_vec.begin() + sizeof(cbdc::client::address_type::public_key)); auto data = std::vector(); - ConvertBits( + ConvertBits( [&](uint8_t c) { data.push_back(c); }, diff --git a/src/uhs/client/client.cpp b/src/uhs/client/client.cpp index 7aaa16dc0..9eff7963d 100644 --- a/src/uhs/client/client.cpp +++ b/src/uhs/client/client.cpp @@ -14,12 +14,45 @@ #include "util/serialization/istream_serializer.hpp" #include "util/serialization/ostream_serializer.hpp" #include "util/serialization/util.hpp" - +#include "bech32/bech32.h" +#include "bech32/util/strencodings.h" #include #include #include namespace cbdc { + + namespace address{ + auto decode(const std::string& addr_str) + -> std::optional { (void) addr_str; + // TODO: if/when bech32m is merged into Bitcoin Core, switch to that. + // see: https://github.com/bitcoin/bitcoin/pull/20861 + const auto [hrp, enc_data] = bech32::Decode(addr_str); + if(hrp != cbdc::config::bech32_hrp) { + std::cout << "Invalid address encoding" << std::endl; + return std::nullopt; + } + auto data = std::vector(); + ConvertBits( + [&](uint8_t c) { + data.push_back(c); + }, + enc_data.begin(),enc_data.end()); + + auto pubkey = cbdc::hash_t(); + if(data[0] != static_cast(cbdc::client::address_type::public_key) + || data.size() != pubkey.size() + 1) { + std::cout << "Address is not a supported type" << std::endl; + return std::nullopt; + } + + data.erase(data.begin()); + std::copy_n(data.begin(), pubkey.size(), pubkey.begin()); + + return pubkey; + } + } + client::client(cbdc::config::options opts, std::shared_ptr logger, std::string wallet_file, diff --git a/src/uhs/client/client.hpp b/src/uhs/client/client.hpp index 2c2db7875..fa61d1b7a 100644 --- a/src/uhs/client/client.hpp +++ b/src/uhs/client/client.hpp @@ -11,6 +11,14 @@ #include "uhs/transaction/wallet.hpp" namespace cbdc { + namespace address { + static constexpr auto bits_per_byte = 8; + static constexpr auto bech32_bits_per_symbol = 5; + + auto decode(const std::string& addr_str) + -> std::optional; + } + /// External client for sending new transactions to the system. class client { public: From ceeee1971dac3a5d464fd5debef34b7d00476904 Mon Sep 17 00:00:00 2001 From: paparuch Date: Sat, 10 Sep 2022 14:57:24 +0000 Subject: [PATCH 3/6] Added scripts/local_run.sh for running 2pc locally for testing purposes Signed-off-by: paparuch --- .gitignore | 1 + 2pc-local.cfg | 23 +++++++++++++++++++++++ atomizer-local.cfg | 24 ++++++++++++++++++++++++ scripts/local_run.sh | 37 +++++++++++++++++++++++++++++++++++++ 4 files changed, 85 insertions(+) create mode 100644 2pc-local.cfg create mode 100644 atomizer-local.cfg create mode 100755 scripts/local_run.sh diff --git a/.gitignore b/.gitignore index ce54777ee..212bafa79 100644 --- a/.gitignore +++ b/.gitignore @@ -63,3 +63,4 @@ cmake-build-*/ build/gcov_data build/tests +scripts/local_log/ diff --git a/2pc-local.cfg b/2pc-local.cfg new file mode 100644 index 000000000..04935b13e --- /dev/null +++ b/2pc-local.cfg @@ -0,0 +1,23 @@ +2pc=1 +sentinel_count=1 +sentinel0_endpoint="127.0.0.1:5858" +sentinel0_loglevel="TRACE" +sentinel0_private_key="0000000000000001000000000000000000000000000000000000000000000000" +sentinel0_public_key="eaa649f21f51bdbae7be4ae34ce6e5217a58fdce7f47f9aa7f3b58fa2120e2b3" +coordinator_count=1 +coordinator0_count=1 +coordinator0_loglevel="TRACE" +coordinator0_0_endpoint="127.0.0.1:7777" +coordinator0_0_raft_endpoint="127.0.0.1:7779" +coordinator_max_threads=100 +shard_count=1 +shard0_start=0 +shard0_end=255 +shard0_count=1 +shard0_loglevel="TRACE" +shard0_0_endpoint="127.0.0.1:6666" +shard0_0_raft_endpoint="127.0.0.1:6667" +shard0_0_readonly_endpoint="127.0.0.1:6767" +rest_api_endpoint="127.0.0.1:8080" +rest_api_enable="true" +rest_api_loglevel="TRACE" diff --git a/atomizer-local.cfg b/atomizer-local.cfg new file mode 100644 index 000000000..ee70e8ff9 --- /dev/null +++ b/atomizer-local.cfg @@ -0,0 +1,24 @@ +atomizer_count=1 +atomizer0_endpoint="127.0.0.1:5555" +atomizer0_raft_endpoint="127.0.0.1:6666" +archiver_count=1 +archiver0_endpoint="127.0.0.1:4555" +archiver0_db="archiver0_db" +shard_count=1 +shard0_endpoint="127.0.0.1:6555" +shard0_start=0 +shard0_end=255 +shard0_db="shard0" +sentinel_count=1 +sentinel0_endpoint="127.0.0.1:7555" +sentinel0_loglevel="INFO" +sentinel0_private_key="0000000000000001000000000000000000000000000000000000000000000000" +sentinel0_public_key="eaa649f21f51bdbae7be4ae34ce6e5217a58fdce7f47f9aa7f3b58fa2120e2b3" +watchtower_count=1 +watchtower0_client_endpoint="127.0.0.1:8555" +watchtower0_internal_endpoint="127.0.0.1:8556" +watchtower_block_cache_size=50 +target_block_interval=3000 +rest_api_endpoint="127.0.0.1:8080"; +rest_api_enable="true" +rest_api_loglevel="TRACE" diff --git a/scripts/local_run.sh b/scripts/local_run.sh new file mode 100755 index 000000000..540d78883 --- /dev/null +++ b/scripts/local_run.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +function scriptDir +{ + SOURCE="${BASH_SOURCE[0]}" + + while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink + DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located + done + DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + echo $DIR +} + +sdir="$( scriptDir )" + +mkdir -p $sdir/local_log || exit 1 +cd $sdir/local_log || exit 2 + +#TODO: +#sleep for fixed time isn't ideal but works for now +#it looks like services needs some interval maybe some reconnect logic should +#be added upon start to resolve this and let starting this daemons in parallel + +echo "Starting locking shard" +$sdir/../build/src/uhs/twophase/locking_shard/locking-shardd $sdir/../2pc-local.cfg 0 0 &> locking-shardd.log& +sleep 5 +echo "Starting coordinatord" +$sdir/../build/src/uhs/twophase/coordinator/coordinatord $sdir/../2pc-local.cfg 0 0 &> coordinatord.log& +sleep 5 +echo "Starting sentineld" +$sdir/../build/src/uhs/twophase/sentinel_2pc/sentineld-2pc $sdir/../2pc-local.cfg 0 &> sentineld.log& + +trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT + +wait $(jobs -p) From 8915c7c549219d003f31e12bf391950d990d70b5 Mon Sep 17 00:00:00 2001 From: paparuch Date: Sat, 10 Sep 2022 19:58:59 +0000 Subject: [PATCH 4/6] Update .gitignore to ommit files downloaded by scripts/configure.sh Signed-off-by: paparuch --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 212bafa79..572c80fb5 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,7 @@ CMakeCache.txt *.a *.log +evpp/ leveldb*/ NuRaft*/ benchmark-results/ @@ -64,3 +65,5 @@ build/gcov_data build/tests scripts/local_log/ +.cache/ +clang-* From 483f72d11c72e88f9d715f5ca6eb0e8efe69faaf Mon Sep 17 00:00:00 2001 From: paparuch Date: Wed, 7 Sep 2022 21:39:38 +0000 Subject: [PATCH 5/6] Added Arch Linux support for scripts/configure.sh Signed-off-by: paparuch --- scripts/configure.sh | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/scripts/configure.sh b/scripts/configure.sh index adc87827f..4aecefa7c 100755 --- a/scripts/configure.sh +++ b/scripts/configure.sh @@ -22,11 +22,25 @@ fi CPUS=1 if [[ "$OSTYPE" == "linux-gnu"* ]]; then CPUS=$(grep -c ^processor /proc/cpuinfo) + if [ -f "/etc/arch-release" ]; then + pacman -Syu + pacman -S wget cmake gtest lcov git rsync gcc make snappy llvm + $SUDO ln -s -f $(which clang-format) /usr/local/bin/clang-format + $SUDO ln -s -f $(which clang-tidy) /usr/local/bin/clang-tidy + else + apt update + apt install -y build-essential wget cmake libgtest-dev lcov git software-properties-common rsync libsnappy-dev + wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | $SUDO apt-key add - + $SUDO add-apt-repository "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main" + $SUDO apt install -y clang-format-14 clang-tidy-14 + $SUDO ln -s -f $(which clang-format-14) /usr/local/bin/clang-format + $SUDO ln -s -f $(which clang-tidy-14) /usr/local/bin/clang-tidy + fi elif [[ "$OSTYPE" == "darwin"* ]]; then CPUS=$(sysctl -n hw.ncpu) # ensure development environment is set correctly for clang $SUDO xcode-select -switch /Library/Developer/CommandLineTools - brew install llvm@14 googletest lcov make wget cmake + brew install llvm@14 googletest lcov make wget cmake snappy CLANG_TIDY=/usr/local/bin/clang-tidy if [ ! -L "$CLANG_TIDY" ]; then $SUDO ln -s $(brew --prefix)/opt/llvm@14/bin/clang-tidy /usr/local/bin/clang-tidy @@ -37,17 +51,6 @@ elif [[ "$OSTYPE" == "darwin"* ]]; then fi fi -if [[ "$OSTYPE" == "linux-gnu"* ]]; then - apt update - apt install -y build-essential wget cmake libgtest-dev lcov git software-properties-common rsync - - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | $SUDO apt-key add - - $SUDO add-apt-repository "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main" - $SUDO apt install -y clang-format-14 clang-tidy-14 - $SUDO ln -s -f $(which clang-format-14) /usr/local/bin/clang-format - $SUDO ln -s -f $(which clang-tidy-14) /usr/local/bin/clang-tidy -fi - LEVELDB_VERSION="1.23" echo -e "${green}Building LevelDB from sources...${end}" wget https://github.com/google/leveldb/archive/${LEVELDB_VERSION}.tar.gz From 50172237352523752828526ae9c4d965a730fb69 Mon Sep 17 00:00:00 2001 From: paparuch Date: Tue, 13 Sep 2022 16:23:54 +0000 Subject: [PATCH 6/6] Merge branch 'arch-linux-snappy' into trunk Signed-off-by: paparuch --- CMakeLists.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ccc28af5b..03ab8e5ae 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -32,8 +32,7 @@ find_library(LEVELDB_LIBRARY leveldb REQUIRED) find_library(NURAFT_LIBRARY nuraft REQUIRED) find_library(GTEST_LIBRARY gtest REQUIRED) find_library(GTEST_MAIN_LIBRARY gtest_main REQUIRED) -include(CheckLibraryExists) -check_library_exists(snappy snappy_compress "" HAVE_SNAPPY) +find_library(SNAPPY_LIBRARY snappy_compress REQUIRED) message("Disabling Run Time Type Information (RTTI) features.") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-rtti")