Merge #1305: [Epic] Split Omni specific implementation from Bitcoin core

Pull request description: * Switch Omni protocol validation to event driven using validation interface * Switch Omni specific data storage out of bitcoin core databases * Move global coins cache to local one * Simple and efficient mempool and transaction storage * Remove txindex hard requirements * Rework all database records to use binary prefix instead of string data * Remove usage of core mutexes to allow validation to work in its own thread * Omnicore could be started as prune node, a lightweight client Todo's: * Remove Omni specific coin selection from Bitcoin core * Remove Omni specific functions from Bitcoin wallet interface Pending: * Full resync to measure time and storage Fixes: #1293 #1294
OmniLayer · Nov 10, 2023 · a2c114c · a2c114c
2 parents 5e8b398 + ea2f801
commit a2c114c
Show file tree

Hide file tree

Showing 102 changed files with 4,392 additions and 5,385 deletions.
diff --git a/.gitignore b/.gitignore
@@ -45,6 +45,7 @@ src/config/bitcoin-config.h
 src/config/bitcoin-config.h.in
 src/config/stamp-h1
 src/obj
+src/omnicore/res/omnicore-setup-win.nsi
 share/setup.nsi
 share/qt/Info.plist
 
@@ -171,3 +172,7 @@ dist/
 /guix-build-*
 
 /ci/scratch/
+
+# clangd language server
+compile_commands.json
+./cache/clangd/*
diff --git a/src/Makefile.omnicore.include b/src/Makefile.omnicore.include
@@ -1,9 +1,11 @@
 OMNICORE_H = \
   omnicore/activation.h \
+  omnicore/coinscache.h \
   omnicore/consensushash.h \
   omnicore/convert.h \
   omnicore/createpayload.h \
   omnicore/createtx.h \
+  omnicore/dbaddress.h \
   omnicore/dbbase.h \
   omnicore/dbfees.h \
   omnicore/dbspinfo.h \
@@ -16,6 +18,7 @@ OMNICORE_H = \
   omnicore/errors.h \
   omnicore/log.h \
   omnicore/mdex.h \
+  omnicore/mempool.h \
   omnicore/nftdb.h \
   omnicore/notifications.h \
   omnicore/omnicore.h \
@@ -38,17 +41,20 @@ OMNICORE_H = \
   omnicore/uint256_extensions.h \
   omnicore/utilsbitcoin.h \
   omnicore/utilsui.h \
+  omnicore/validationinterface.h \
   omnicore/version.h \
   omnicore/walletfetchtxs.h \
   omnicore/wallettxbuilder.h \
   omnicore/walletutils.h
 
 OMNICORE_CPP = \
   omnicore/activation.cpp \
+  omnicore/coinscache.cpp \
   omnicore/consensushash.cpp \
   omnicore/convert.cpp \
   omnicore/createpayload.cpp \
   omnicore/createtx.cpp \
+  omnicore/dbaddress.cpp \
   omnicore/dbbase.cpp \
   omnicore/dbfees.cpp \
   omnicore/dbspinfo.cpp \
@@ -60,6 +66,7 @@ OMNICORE_CPP = \
   omnicore/encoding.cpp \
   omnicore/log.cpp \
   omnicore/mdex.cpp \
+  omnicore/mempool.cpp \
   omnicore/nftdb.cpp \
   omnicore/notifications.cpp \
   omnicore/misc.cpp \
@@ -84,6 +91,7 @@ OMNICORE_CPP = \
   omnicore/tx.cpp \
   omnicore/utilsbitcoin.cpp \
   omnicore/utilsui.cpp \
+  omnicore/validationinterface.cpp \
   omnicore/version.cpp \
   omnicore/walletfetchtxs.cpp \
   omnicore/wallettxbuilder.cpp \

diff --git a/src/Makefile.omnitest.include b/src/Makefile.omnitest.include
@@ -12,6 +12,7 @@ OMNICORE_TEST_CPP = \
   omnicore/test/encoding_b_tests.cpp \
   omnicore/test/encoding_c_tests.cpp \
   omnicore/test/exodus_tests.cpp \
+  omnicore/test/iterator_tests.cpp \
   omnicore/test/lock_tests.cpp \
   omnicore/test/marker_tests.cpp \
   omnicore/test/mbstring_tests.cpp \

diff --git a/src/coins.h b/src/coins.h
@@ -20,69 +20,6 @@
 #include <functional>
 #include <unordered_map>
 
-struct CSpentIndexKey {
-    uint256 txid;
-    unsigned int outputIndex;
-
-    SERIALIZE_METHODS(CSpentIndexKey, obj) {
-        READWRITE(obj.txid, obj.outputIndex);
-    }
-
-    CSpentIndexKey(uint256 t, unsigned int i) {
-        txid = t;
-        outputIndex = i;
-    }
-
-    CSpentIndexKey() {
-        SetNull();
-    }
-
-    void SetNull() {
-        txid.SetNull();
-        outputIndex = 0;
-    }
-};
-
-struct CSpentIndexValue {
-    uint256 txid;
-    unsigned int inputIndex;
-    int blockHeight;
-    CAmount satoshis;
-    int addressType;
-    uint256 addressHash;
-
-    SERIALIZE_METHODS(CSpentIndexValue, obj) {
-        READWRITE(obj.txid, obj.inputIndex, obj.blockHeight,
-                  obj.satoshis, obj.addressType, obj.addressHash);
-    }
-
-    CSpentIndexValue(uint256 t, unsigned int i, int h, CAmount s, int type, uint256 a) {
-        txid = t;
-        inputIndex = i;
-        blockHeight = h;
-        satoshis = s;
-        addressType = type;
-        addressHash = a;
-    }
-
-    CSpentIndexValue() {
-        SetNull();
-    }
-
-    void SetNull() {
-        txid.SetNull();
-        inputIndex = 0;
-        blockHeight = 0;
-        satoshis = 0;
-        addressType = 0;
-        addressHash.SetNull();
-    }
-
-    bool IsNull() const {
-        return txid.IsNull();
-    }
-};
-
 /**
  * A UTXO entry.
  *

diff --git a/src/dbwrapper.h b/src/dbwrapper.h
@@ -182,6 +182,7 @@ class CDBIterator
 
 class CDBWrapper
 {
+    friend class COmniCoinsCache;
     friend const std::vector<unsigned char>& dbwrapper_private::GetObfuscateKey(const CDBWrapper &w);
 private:
     //! custom environment this database is using (may be nullptr in case of default environment)
@@ -235,15 +236,18 @@ class CDBWrapper
     CDBWrapper& operator=(const CDBWrapper&) = delete;
 
     template <typename K, typename V>
-    bool Read(const K& key, V& value) const
+    bool Read(const K& key, V& value, const leveldb::Snapshot* snapshot = nullptr) const
     {
         CDataStream ssKey(SER_DISK, CLIENT_VERSION);
         ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE);
         ssKey << key;
         leveldb::Slice slKey((const char*)ssKey.data(), ssKey.size());
 
+        auto options = readoptions;
+        options.snapshot = snapshot;
+
         std::string strValue;
-        leveldb::Status status = pdb->Get(readoptions, slKey, &strValue);
+        leveldb::Status status = pdb->Get(options, slKey, &strValue);
         if (!status.ok()) {
             if (status.IsNotFound())
                 return false;

diff --git a/src/httprpc.cpp b/src/httprpc.cpp
@@ -13,8 +13,6 @@
 #include <util/system.h>
 #include <walletinitinterface.h>
 
-#include <omnicore/rpcmbstring.h> // SanitizeInvalidUTF8
-
 #include <algorithm>
 #include <iterator>
 #include <map>
@@ -23,9 +21,6 @@
 #include <string>
 #include <vector>
 
-/** Sanitize UTF-8 encoded strings in RPC responses */
-static bool fSanitizeResponse = true;
-
 /** WWW-Authenticate to present with 401 Unauthorized response */
 static const char* WWW_AUTH_HEADER_DATA = "Basic realm=\"jsonrpc\"";
 
@@ -148,6 +143,10 @@ static bool RPCAuthorized(const std::string& strAuth, std::string& strAuthUserna
     return multiUserAuthorized(strUserPass);
 }
 
+namespace mastercore {
+    extern std::string SanitizeInvalidUTF8(const std::string&);
+}
+
 static bool HTTPReq_JSONRPC(const std::any& context, HTTPRequest* req)
 {
     // JSONRPC handles only POST
@@ -207,10 +206,6 @@ static bool HTTPReq_JSONRPC(const std::any& context, HTTPRequest* req)
 
             // Send reply
             strReply = JSONRPCReply(result, NullUniValue, jreq.id);
-            if (fSanitizeResponse) {
-                strReply = mastercore::SanitizeInvalidUTF8(strReply);
-            }
-
         // array of requests
         } else if (valRequest.isArray()) {
             if (user_has_whitelist) {
@@ -235,7 +230,7 @@ static bool HTTPReq_JSONRPC(const std::any& context, HTTPRequest* req)
             throw JSONRPCError(RPC_PARSE_ERROR, "Top-level object parse error");
 
         req->WriteHeader("Content-Type", "application/json");
-        req->WriteReply(HTTP_OK, strReply);
+        req->WriteReply(HTTP_OK, mastercore::SanitizeInvalidUTF8(strReply));
     } catch (const UniValue& objError) {
         JSONErrorReply(req, objError, jreq.id);
         return false;

diff --git a/src/init.cpp b/src/init.cpp
@@ -143,10 +143,8 @@ static const bool DEFAULT_REST_ENABLE = false;
 static const char* DEFAULT_ASMAP_FILENAME="ip_asn.map";
 
 // Omni Core initialization and shutdown handlers
-extern int mastercore_init();
+extern int mastercore_init(node::NodeContext&);
 extern int mastercore_shutdown();
-extern int CheckWalletUpdate();
-extern std::optional<std::reference_wrapper<node::NodeContext>> g_context;
 
 /**
  * The PID file facilities.
@@ -333,7 +331,6 @@ void Shutdown(NodeContext& node)
     node.fee_estimator.reset();
     node.chainman.reset();
     node.scheduler.reset();
-    g_context.reset();
 
     try {
         if (!fs::remove(GetPidFile(*node.args))) {
@@ -360,7 +357,6 @@ static void HandleSIGTERM(int)
 static void HandleSIGHUP(int)
 {
     LogInstance().m_reopen_file = true;
-    fReopenOmniCoreLog = true;
 }
 #else
 static BOOL WINAPI consoleCtrlHandler(DWORD dwCtrlType)
@@ -610,7 +606,6 @@ void SetupServerArgs(ArgsManager& argsman)
     argsman.AddArg("-server", "Accept command line and JSON-RPC commands", ArgsManager::ALLOW_ANY, OptionsCategory::RPC);
 
     argsman.AddArg("-startclean", "Clear all persistence files on startup; triggers reparsing of Omni transactions (default: 0)", false, OptionsCategory::OMNI);
-    argsman.AddArg("-omnitxcache", "The maximum number of transactions in the input transaction cache (default: 500000)", false, OptionsCategory::OMNI);
     argsman.AddArg("-omniprogressfrequency", "Time in seconds after which the initial scanning progress is reported (default: 30)", false, OptionsCategory::OMNI);
     argsman.AddArg("-omniseedblockfilter", "Set skipping of blocks without Omni transactions during initial scan (default: 1)", false, OptionsCategory::OMNI);
     argsman.AddArg("-omniskipstoringstate", "Don't store state during initial synchronization until block n (faster, but may have to restart syncing after a shutdown)(default: 770000)", false, OptionsCategory::OMNI);
@@ -912,44 +907,6 @@ bool AppInitParameterInteraction(const ArgsManager& args, bool use_syscall_sandb
         nLocalServices = ServiceFlags(nLocalServices | NODE_COMPACT_FILTERS);
     }
 
-    if (!gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX)) {
-        const fs::path configPathInfo = GetConfigFile(gArgs.GetArg("-conf", BITCOIN_CONF_FILENAME).c_str());
-        // ask the user if they would like us to modify their config file for them
-        auto msg = _("Disabled transaction index detected.\n\n"
-                    "Omni Core requires an enabled transaction index. To enable "
-                    "transaction indexing, please use the \"-txindex\" option as "
-                    "command line argument or add \"txindex=1\" to your client "
-                    "configuration file within your data directory.\n\n"
-                    "Configuration file: "); // allow translation of main text body while still allowing differing config file string
-        msg += _((configPathInfo.string() + "\n\n").c_str());
-        msg += _("Would you like Omni Core to attempt to update your configuration file accordingly?");
-        bool fRet = uiInterface.ThreadSafeMessageBox(msg, "", CClientUIInterface::MSG_INFORMATION | CClientUIInterface::BTN_OK | CClientUIInterface::MODAL | CClientUIInterface::BTN_ABORT);
-        if (fRet) {
-            // add txindex=1 to config file in GetConfigFile()
-            FILE *fp = fopen(configPathInfo.string().c_str(), "at");
-            if (!fp) {
-                auto failMsg = _("Unable to update configuration file at:\n");
-                failMsg += _((configPathInfo.string() + "\n\n").c_str());
-                failMsg += _("The file may be write protected or you may not have the required permissions to edit it.\n");
-                failMsg += _("Please add txindex=1 to your configuration file manually.\n\nOmni Core will now shutdown.");
-                return InitError(failMsg);
-            }
-            fprintf(fp, "\ntxindex=1\n");
-            fflush(fp);
-            fclose(fp);
-            auto strUpdated = _("Your configuration file has been updated.\n\n"
-                                "Omni Core will now shutdown - please restart the client for your new configuration to take effect.");
-            uiInterface.ThreadSafeMessageBox(strUpdated, "", CClientUIInterface::MSG_INFORMATION | CClientUIInterface::BTN_OK | CClientUIInterface::MODAL);
-            return false;
-        } else {
-            return InitError(_("Please add txindex=1 to your configuration file manually.\n\nOmni Core will now shutdown."));
-        }
-    }
-
-    if (args.GetIntArg("-prune", 0) || args.GetBoolArg("-reindex-chainstate", false)) {
-        gArgs.SoftSetBoolArg("-txindex", false);
-    }
-
     if (args.GetIntArg("-prune", 0)) {
         if (args.GetBoolArg("-txindex", DEFAULT_TXINDEX))
             return InitError(_("Prune mode is incompatible with -txindex."));
@@ -1469,9 +1426,6 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info)
     }
 #endif
 
-    // restore txindex
-    gArgs.SoftSetBoolArg("-txindex", true);
-
     // ********************************************************* Step 7: load block chain
 
     fReindex = args.GetBoolArg("-reindex", false);
@@ -1589,10 +1543,6 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info)
         return false;
     }
 
-    if (fAddressIndex != args.GetBoolArg("-experimental-btc-balances", DEFAULT_ADDRINDEX)) {
-        return InitError(_("You need to rebuild the database using -reindex-chainstate to change -experimental-btc-balances"));
-    }
-
     ChainstateManager& chainman = *Assert(node.chainman);
 
     assert(!node.peerman);
@@ -1626,21 +1576,14 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info)
         }
     }
 
-    g_context = std::ref(node);
-
-    uiInterface.InitMessage(_("Parsing Omni Layer transactions...").translated);
-
-    mastercore_init();
-
     // ********************************************************* Step 9: load wallet
     for (const auto& client : node.chain_clients) {
         if (!client->load()) {
             return false;
         }
     }
 
-    // Omni Core code should be initialized and wallet should now be loaded, perform an initial populat$
-    CheckWalletUpdate();
+    mastercore_init(node);
 
     // ********************************************************* Step 10: data directory maintenance
 

diff --git a/src/key_io.cpp b/src/key_io.cpp
@@ -306,42 +306,3 @@ bool IsValidDestinationString(const std::string& str)
 {
     return IsValidDestinationString(str, Params());
 }
-
-bool DecodeIndexKey(const std::string &str, uint256 &hashBytes, int &type)
-{
-    CTxDestination dest = DecodeDestination(str);
-    if (IsValidDestination(dest))
-    {
-        const PKHash *keyID = std::get_if<PKHash>(&dest);
-        if(keyID)
-        {
-            memcpy(&hashBytes, keyID, 20);
-            type = 1;
-            return true;
-        }
-
-        const ScriptHash *scriptID = std::get_if<ScriptHash>(&dest);
-        if(scriptID)
-        {
-            memcpy(&hashBytes, scriptID, 20);
-            type = 2;
-            return true;
-        }
-
-        const WitnessV0ScriptHash *witnessV0ScriptID = std::get_if<WitnessV0ScriptHash>(&dest);
-        if (witnessV0ScriptID) {
-            memcpy(&hashBytes, witnessV0ScriptID, 32);
-            type = 3;
-            return true;
-        }
-
-        const WitnessV0KeyHash *witnessV0KeyID = std::get_if<WitnessV0KeyHash>(&dest);
-        if (witnessV0KeyID) {
-            memcpy(&hashBytes, witnessV0KeyID, 20);
-            type = 4;
-            return true;
-        }
-    }
-
-    return false;
-}