Update to config generation branch based on merge comments

Signed-off-by: Yiannis Karavas <[email protected]>

README.md

@@ -121,38 +121,38 @@ Additionally, you can start the atomizer architecture by passing `--file docker-
 ## Setup test wallets and test them
 
 The following commands are all performed from within the second container we started in the previous step.
-In each of the below commands, you should pass `atomizer-compose.cfg` instead of `2pc-compose.cfg` if you started the atomizer architecture.
+In each of the below commands, you should pass `config/general/atomizer-compose.cfg` instead of `config/general/2pc-compose.cfg` if you started the atomizer architecture.
 
 * Mint new coins (e.g., 10 new UTXOs each with a value of 5 atomic units of currency)
   ```terminal
-  # ./build/src/uhs/client/client-cli 2pc-compose.cfg mempool0.dat wallet0.dat mint 10 5
+  # ./build/src/uhs/client/client-cli config/general/2pc-compose.cfg mempool0.dat wallet0.dat mint 10 5
   [2021-08-17 15:11:57.686] [WARN ] Existing wallet file not found
   [2021-08-17 15:11:57.686] [WARN ] Existing mempool not found
   4bc23da407c3a8110145c5b6c38199c8ec3b0e35ea66bbfd78f0ed65304ce6fa
   ```
 
   If using the atomizer architecture, you'll need to sync the wallet after:
   ```terminal
-  # ./build/src/uhs/client/client-cli atomizer-compose.cfg mempool0.dat wallet0.dat sync
+  # ./build/src/uhs/client/client-cli config/general/atomizer-compose.cfg mempool0.dat wallet0.dat sync
   ```
 
 * Inspect the balance of a wallet
   ```terminal
-  # ./build/src/uhs/client/client-cli 2pc-compose.cfg mempool0.dat wallet0.dat info
+  # ./build/src/uhs/client/client-cli config/general/2pc-compose.cfg mempool0.dat wallet0.dat info
   Balance: $0.50, UTXOs: 10, pending TXs: 0
   ```
 
 * Make a new wallet
   ```terminal
-  # ./build/src/uhs/client/client-cli 2pc-compose.cfg mempool1.dat wallet1.dat newaddress
+  # ./build/src/uhs/client/client-cli config/general/2pc-compose.cfg mempool1.dat wallet1.dat newaddress
   [2021-08-17 15:13:16.148] [WARN ] Existing wallet file not found
   [2021-08-17 15:13:16.148] [WARN ] Existing mempool not found
   usd1qrw038lx5n4wxx3yvuwdndpr7gnm347d6pn37uywgudzq90w7fsuk52kd5u
   ```
 
 * Send currency from one wallet to another (e.g., 30 atomic units of currency)
   ```terminal
-  # ./build/src/uhs/client/client-cli 2pc-compose.cfg mempool0.dat wallet0.dat send 30 usd1qrw038lx5n4wxx3yvuwdndpr7gnm347d6pn37uywgudzq90w7fsuk52kd5u
+  # ./build/src/uhs/client/client-cli config/general/2pc-compose.cfg mempool0.dat wallet0.dat send 30 usd1qrw038lx5n4wxx3yvuwdndpr7gnm347d6pn37uywgudzq90w7fsuk52kd5u
   tx_id:
   cc1f7dc708be5b07e23e125cf0674002ff8546a9342928114bc97031d8b96e75
   Data for recipient importinput:
@@ -162,20 +162,20 @@ In each of the below commands, you should pass `atomizer-compose.cfg` instead of
 
   If using the atomizer architecture, you'll need to sync the sending wallet after:
   ```terminal
-  # ./build/src/uhs/client/client-cli atomizer-compose.cfg mempool0.dat wallet0.dat sync
+  # ./build/src/uhs/client/client-cli config/general/atomizer-compose.cfg mempool0.dat wallet0.dat sync
   ```
 
 * Check that the currency is no longer available in the sending wallet
   ```terminal
-  # ./build/src/uhs/client/client-cli 2pc-compose.cfg mempool0.dat wallet0.dat info
+  # ./build/src/uhs/client/client-cli config/general/2pc-compose.cfg mempool0.dat wallet0.dat info
   Balance: $0.20, UTXOs: 4, pending TXs: 0
   ```
 
 * Import coins to the receiving wallet
   ```terminal
-  # ./build/src/uhs/client/client-cli 2pc-compose.cfg mempool1.dat wallet1.dat importinput cc1f7dc708be5b07e23e125cf0674002ff8546a9342928114bc97031d8b96e750000000000000000d0e4f689b550f623e9370edae235de50417860be0f2f8e924eca9f402fcefeaa1e00000000000000
-  # ./build/src/uhs/client/client-cli 2pc-compose.cfg mempool1.dat wallet1.dat sync
-  # ./build/src/uhs/client/client-cli 2pc-compose.cfg mempool1.dat wallet1.dat info
+  # ./build/src/uhs/client/client-cli config/general/2pc-compose.cfg mempool1.dat wallet1.dat importinput cc1f7dc708be5b07e23e125cf0674002ff8546a9342928114bc97031d8b96e750000000000000000d0e4f689b550f623e9370edae235de50417860be0f2f8e924eca9f402fcefeaa1e00000000000000
+  # ./build/src/uhs/client/client-cli config/general/2pc-compose.cfg mempool1.dat wallet1.dat sync
+  # ./build/src/uhs/client/client-cli config/general/2pc-compose.cfg mempool1.dat wallet1.dat info
   Balance: $0.30, UTXOs: 1, pending TXs: 0
   ```
 

config/tools/CMakeLists.txt

@@ -1,10 +1,10 @@
 project(config_gen)
 
-include_directories(../../src ../../config/tools ../../3rdparty/secp256k1/include)
+include_directories(../../src ../../3rdparty/secp256k1/include)
 
 add_library(config_generator config_generator.cpp)
 
-add_executable(generate_config generate_configd.cpp)
+add_executable(generate_config generate_config.cpp)
 
 target_link_libraries(generate_config config_generator
                                       util

config/tools/config_generator.cpp

@@ -6,32 +6,32 @@
 
 #include "config_generator.hpp"
 
+#include "util/common/config.hpp"
 #include "util/common/keys.hpp"
+#include "util/common/variant_overloaded.hpp"
 #include "util/network/tcp_listener.hpp"
 
 #include <algorithm>
 #include <cassert>
-#include <filesystem>
 #include <functional>
 #include <random>
 #include <set>
 
 // NORMAL CONFIGS
 
-// Integer that determines if this is 2PC or Atomizer. 1 means 2PC
-static constexpr auto two_phase_mode = "2pc";
-// Number of shards to be created
-static constexpr auto shard_count_key = "shard_count";
-// Number of sentinels to be created
-static constexpr auto sentinel_count_key = "sentinel_count";
-// Number of coordinators to be created
-static constexpr auto coordinator_count_key = "coordinator_count";
-// Number of archivers to be created
-static constexpr auto archiver_count_key = "archiver_count";
-// Number of atomizers to be created
-static constexpr auto atomizer_count_key = "atomizer_count";
-// Number of watchtowers to be created
-static constexpr auto watchtower_count_key = "watchtower_count";
+using namespace cbdc::config;
+
+// Using the following values from src/common/config.hpp:
+
+// "cbdc::config::two_phase_mode" : Name of value in config template file that
+// is an integer which determines if this is 2PC or Atomizer. 1 means 2PC
+// "cbdc::config::shard_count_key" : Number of shards to be created
+// "cbdc::config::sentinel_count_key" : Number of sentinels to be created
+// "cbdc::config::coordinator_count_key" : Number of coordinators to be created
+// "cbdc::config::archiver_count_key" : Number of archivers to be created
+// "cbdc::config::atomizer_count_key" : Number of atomizers to be created
+// "cbdc::config::watchtower_count_key" : Number of watchtowers to be created
+
 // Prefix of interest that denotes parameters in the file that are used
 // here to help generate the config file but will not be present in the
 // final product
@@ -69,18 +69,77 @@ std::set<std::string> log_levels
 namespace cbdc::generate_config {
 
     config_generator::config_generator(std::string& _template_config_file,
-                                       const size_t _start_port)
+                                       const size_t _start_port,
+                                       std::string _build_dir)
         : m_template_config_file(_template_config_file),
           m_current_port(_start_port) {
+        // Get Project root dir and build dir
+        std::filesystem::path build_dir = std::filesystem::current_path();
+        if(_build_dir.at(_build_dir.size() - 1) == '/') {
+            _build_dir.erase(_build_dir.end() - 1);
+        }
+        // This config generator class assumes project root is "opencbdc-tx"
+        while(build_dir.has_parent_path()) {
+            if(build_dir.filename() == "opencbdc-tx") {
+                m_project_root_dir = build_dir;
+                std::string delimiter = "/";
+                std::string tmp_str = _build_dir;
+                size_t pos = 0;
+                std::string token;
+                while((pos = tmp_str.find("/")) != std::string::npos) {
+                    token = tmp_str.substr(0, pos);
+                    build_dir = build_dir.append(token);
+                    tmp_str.erase(0, pos + delimiter.length());
+                }
+                token = tmp_str.substr(0, pos);
+                build_dir = build_dir.append(token);
+                tmp_str.erase(0, pos + delimiter.length());
+                m_build_dir = build_dir;
+                std::cout << "Build directory determined to be "
+                          << m_build_dir.string() << std::endl;
+                std::cout << "Project Root directory determined to be "
+                          << m_project_root_dir.string() << std::endl;
+                break;
+            } else {
+                build_dir = build_dir.parent_path();
+            }
+        }
         template_file_is_valid = true;
         if(!std::filesystem::exists(m_template_config_file)) {
             template_file_is_valid = false;
+            std::filesystem::path temp_config_tools_dir = m_project_root_dir;
+            temp_config_tools_dir.append("config").append("tools");
+            std::filesystem::path temp_build_config_tools_dir = m_build_dir;
+            temp_build_config_tools_dir.append("config").append("tools");
             std::cout << "Warning: File provided, " << m_template_config_file
                       << ", does not exist. Attempting to copy it from its "
-                         "original location to build/config/tools. Rerun with "
-                         "valid template file."
-                      << std::endl;
+                         "original location, "
+                      << temp_config_tools_dir.string() << " to "
+                      << temp_build_config_tools_dir.string() << std::endl;
             copy_templates_to_build_dir();
+            // Try to use newly copied tmplate files
+            std::string delimiter = "/";
+            std::string tmp_str = m_template_config_file;
+            size_t pos = 0;
+            std::string template_filename;
+            while((pos = tmp_str.find("/")) != std::string::npos) {
+                template_filename = tmp_str.substr(0, pos);
+                tmp_str.erase(0, pos + delimiter.length());
+            }
+            template_filename = tmp_str.substr(0, pos);
+            std::filesystem::path full_template_path_and_filename
+                = temp_build_config_tools_dir.append(template_filename);
+            if(std::filesystem::exists(full_template_path_and_filename)) {
+                m_template_config_file
+                    = full_template_path_and_filename.string();
+                template_file_is_valid = true;
+                std::cout << "Successfully copied " << template_filename
+                          << " from " << temp_config_tools_dir.string()
+                          << " to " << temp_build_config_tools_dir.string()
+                          << ". Using "
+                          << full_template_path_and_filename.string()
+                          << " as template file." << std::endl;
+            }
         }
         generator.seed(100);
     }
@@ -105,7 +164,7 @@ namespace cbdc::generate_config {
                 = std::make_pair(randNum, randNum);
             shard_info.at(i).numbers_covered = 1;
             shard_info.at(i).shard_id = i;
-            double overlap_percentage;
+            double overlap_percentage = 0;
             find_value(tmpl_avg_shard_start_end_overlap_percent,
                        overlap_percentage);
             shard_info.at(i).overlap_percentage_allowed = std::abs(
@@ -335,7 +394,7 @@ namespace cbdc::generate_config {
     [[nodiscard]] auto config_generator::get_param_from_template_file(
         const std::string option,
         std::map<std::string, std::string>& config_map)
-        -> std::variant<size_t, double, std::string> {
+        -> std::variant<std::string, size_t, double> {
         auto it = config_map.find(option);
         if(it != config_map.end()) {
             value_t parsed_val = parse_value(it->second, false);
@@ -346,10 +405,7 @@ namespace cbdc::generate_config {
             } else if(std::holds_alternative<std::string>(parsed_val)) {
                 return std::get<std::string>(parsed_val);
             } else {
-                std::string error_msg
-                    = "Warning: Unrecognized type for param, " + option + ".";
-                std::cout << error_msg << std::endl;
-                return error_msg;
+                __builtin_unreachable();
             }
         } else {
             std::string error_msg
@@ -376,13 +432,12 @@ namespace cbdc::generate_config {
 
     void config_generator::set_log_level(const std::string key,
                                          std::string& log_level) {
-        if(find_value(key, log_level) == false) {
-            find_value(tmpl_universal_override_log_level, log_level);
-        }
         if(log_levels.find(log_level) == log_levels.end()) {
             log_level = "DEBUG";
             std::cout << "Warning: Log level not recognized. Setting to DEBUG"
                       << std::endl;
+        } else if(find_value(key, log_level) == false) {
+            find_value(tmpl_universal_override_log_level, log_level);
         }
     }
 
@@ -396,8 +451,8 @@ namespace cbdc::generate_config {
         auto _default_debug_level = static_cast<std::string>("INFO");
         find_value(tmpl_default_log_level, _default_debug_level);
         if(type == shard_count_key) {
-            // Create Shards
-            size_t shard_size;
+            // Create Shards. Default value of 255
+            size_t shard_size = 255;
             find_value(tmpl_shard_size, shard_size);
             // Add one since this is 255 and we are 0 indexing but want 0-255
             // inclusive
@@ -642,27 +697,16 @@ namespace cbdc::generate_config {
         outFile.close();
     }
 
-    // This method assumes project root is "opencbdc-tx" and build dir is
-    // "build"
     [[nodiscard]] auto
     config_generator::copy_to_build_dir(const std::string filename) -> bool {
         std::filesystem::path cwd = std::filesystem::current_path();
         cwd.append(filename);
-        std::filesystem::path build_dir = std::filesystem::current_path();
-        while(build_dir.has_parent_path()) {
-            if(build_dir.filename() == "opencbdc-tx") {
-                build_dir = build_dir.append("build");
-                break;
-            } else {
-                build_dir = build_dir.parent_path();
-            }
-        }
         if(std::filesystem::exists(filename)) {
             const auto copyOptions
                 = std::filesystem::copy_options::overwrite_existing;
             // Copy and remove file if we are not in build currently
-            if(std::filesystem::current_path().filename() != "build") {
-                std::filesystem::copy(cwd, build_dir, copyOptions);
+            if(std::filesystem::current_path() != m_build_dir) {
+                std::filesystem::copy(cwd, m_build_dir, copyOptions);
                 std::filesystem::remove(cwd);
             }
             return true;
@@ -671,24 +715,11 @@ namespace cbdc::generate_config {
         }
     }
 
-    // This method assumes project root is "opencbdc-tx" and build dir is
-    // "build"
     void config_generator::copy_templates_to_build_dir() {
-        std::filesystem::path config_dir = std::filesystem::current_path();
-        std::filesystem::path build_dir = std::filesystem::current_path();
-        while(config_dir.has_parent_path()) {
-            if(config_dir.filename() == "opencbdc-tx") {
-                config_dir = config_dir.append("config");
-                config_dir = config_dir.append("tools");
-                build_dir = build_dir.append("build");
-                build_dir = build_dir.append("config");
-                build_dir = build_dir.append("tools");
-                break;
-            } else {
-                config_dir = config_dir.parent_path();
-                build_dir = build_dir.parent_path();
-            }
-        }
+        std::filesystem::path config_dir = m_project_root_dir;
+        config_dir.append("config").append("tools");
+        std::filesystem::path build_config_dir = m_build_dir;
+        build_config_dir.append("config").append("tools");
         for(auto const& dir_entry :
             std::filesystem::directory_iterator{config_dir}) {
             std::string filename = dir_entry.path().filename();
@@ -697,7 +728,11 @@ namespace cbdc::generate_config {
             if(tmp_str == match_str) {
                 const auto copyOptions
                     = std::filesystem::copy_options::overwrite_existing;
-                std::filesystem::copy(dir_entry, build_dir, copyOptions);
+                std::filesystem::copy(dir_entry,
+                                      build_config_dir,
+                                      copyOptions);
+                std::cout << "Copying " << dir_entry.path().string() << " to "
+                          << build_config_dir.string() << std::endl;
             }
         }
     }
@@ -728,10 +763,16 @@ namespace cbdc::generate_config {
         std::string output_filename = "tmp.cfg";
 
         if(!template_file_is_valid) {
+            std::filesystem::path temp_build_dir = m_build_dir;
+            temp_build_dir.append("config").append("tools");
             return_msg += "File provided, " + m_template_config_file
-                        + ", does not exist. Aborting operation. \n";
+                        + ", did not exist and could not be copied to "
+                        + temp_build_dir.string()
+                        + ". Aborting operation. Please rerun with proper "
+                          "template location \n";
             return return_msg;
         }
+
         std::map<std::string, std::string> config_map;
         load_template(m_template_config_file, config_map);
         // Deal with all config values in file with "tmpl_"
@@ -757,8 +798,7 @@ namespace cbdc::generate_config {
                         config_map_it->first,
                         std::get<std::string>(parsed_val));
                 } else {
-                    return_msg += "Warning: Unrecognized type for param, "
-                                + config_map_it->first + ". \n";
+                    __builtin_unreachable();
                 }
             }
         }
@@ -794,8 +834,7 @@ namespace cbdc::generate_config {
                     return_msg
                         += "Warning: Two-phase mode requires at least one "
                            "configured shard. Fix configuration template "
-                           "and "
-                           "rerun.\n";
+                           "and rerun.\n";
                     return return_msg;
                 } else {
                     create_2pc_component(shard_count_key,
@@ -853,8 +892,7 @@ namespace cbdc::generate_config {
                     return_msg
                         += "Warning: Sentinels require at least one "
                            "configured shard. Fix configuration template "
-                           "and "
-                           "rerun. \n";
+                           "and rerun. \n";
                     return return_msg;
                 } else {
                     create_atomizer_component(