Self healing open

CMakeLists.txt

@@ -380,6 +380,7 @@ endif()
 set(CCF_IMPL_SOURCE
     ${CCF_DIR}/src/enclave/main.cpp ${CCF_DIR}/src/enclave/thread_local.cpp
     ${CCF_DIR}/src/node/quote.cpp ${CCF_DIR}/src/node/uvm_endorsements.cpp
+    ${CCF_DIR}/src/node/self_healing_open_impl.cpp
 )
 
 add_ccf_static_library(
@@ -688,11 +689,20 @@ if(BUILD_TESTS)
     add_unit_test(
       frontend_test
       ${CMAKE_CURRENT_SOURCE_DIR}/src/node/rpc/test/frontend_test.cpp
-      ${CCF_DIR}/src/node/quote.cpp ${CCF_DIR}/src/node/uvm_endorsements.cpp
+      ${CCF_DIR}/src/node/quote.cpp
+      ${CCF_DIR}/src/node/uvm_endorsements.cpp
+      ${CMAKE_CURRENT_SOURCE_DIR}/src/node/self_healing_open_impl.cpp
     )
     target_link_libraries(
-      frontend_test PRIVATE ${CMAKE_THREAD_LIBS_INIT} http_parser ccf_js
-                            ccf_endpoints ccfcrypto ccf_kv
+      frontend_test
+      PRIVATE ${CMAKE_THREAD_LIBS_INIT}
+              http_parser
+              ccf_js
+              ccf_endpoints
+              ccfcrypto
+              ccf_kv
+              uv
+              curl
     )
 
     add_unit_test(
@@ -718,11 +728,20 @@ if(BUILD_TESTS)
     add_unit_test(
       node_frontend_test
       ${CMAKE_CURRENT_SOURCE_DIR}/src/node/rpc/test/node_frontend_test.cpp
-      ${CCF_DIR}/src/node/quote.cpp ${CCF_DIR}/src/node/uvm_endorsements.cpp
+      ${CCF_DIR}/src/node/quote.cpp
+      ${CCF_DIR}/src/node/uvm_endorsements.cpp
+      ${CMAKE_CURRENT_SOURCE_DIR}/src/node/self_healing_open_impl.cpp
     )
     target_link_libraries(
-      node_frontend_test PRIVATE ${CMAKE_THREAD_LIBS_INIT} http_parser ccf_js
-                                 ccf_endpoints ccfcrypto ccf_kv
+      node_frontend_test
+      PRIVATE ${CMAKE_THREAD_LIBS_INIT}
+              http_parser
+              ccf_js
+              ccf_endpoints
+              ccfcrypto
+              ccf_kv
+              uv
+              curl
     )
 
     add_unit_test(
@@ -1185,15 +1204,16 @@ if(BUILD_TESTS)
       10000
       --use-jwt
   )
-  add_test_bin(
-    curl_test ${CMAKE_CURRENT_SOURCE_DIR}/src/http/test/curl_test.cpp
-  )
-  target_link_libraries(curl_test PRIVATE curl uv http_parser)
 
   if(LONG_TESTS)
     add_e2e_test(
       NAME e2e_curl PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/e2e_curl.py
     )
+
+    add_test_bin(
+      curl_test ${CMAKE_CURRENT_SOURCE_DIR}/src/http/test/curl_test.cpp
+    )
+    target_link_libraries(curl_test PRIVATE curl uv http_parser)
   endif()
 endif()
 

doc/host_config_schema/cchost_config.json

@@ -417,6 +417,28 @@
                   "previous_sealed_ledger_secret_location": {
                     "type": ["string"],
                     "description": "Path to the sealed ledger secret folder, the ledger secrets for the recovered service will be unsealed from here instead of reconstructed from recovery shares."
+                  },
+                  "self_healing_open": {
+                    "type": "object",
+                    "properties": {
+                      "addresses": {
+                        "type": "array",
+                        "items": {
+                          "type": "string"
+                        },
+                        "description": "List of addresses (host:port) of the cluster that should open via self-healing-open"
+                      },
+                      "retry_timeout": {
+                        "type": "string",
+                        "default": "100ms",
+                        "description": "Interval (time string) at which the node re-sends self-healing-open messages. This should be leass than 'timeout'"
+                      },
+                      "timeout": {
+                        "type": "string",
+                        "default": "2000ms",
+                        "description": "Interval (time string) after which the node forcibly advances to the next phase of the self-healing-open protocol"
+                      }
+                    }
                   }
                 },
                 "required": ["previous_service_identity_file"],

doc/operations/recovery.rst

@@ -145,6 +145,94 @@ Which of these two paths is taken is noted in the `public:ccf.internal.last_reco
       ...
     $ /opt/ccf/bin/js_generic --config /path/to/config/file
 
+Self-Healing-Open recovery
+--------------------------
+
+In environments with limited orchestration or limited operator access, it is desirable to allow a limited disaster recovery without operator intervention.
+At a high level, Self-Healing-Open recovery allows recovering replicas to discover which replica has the most up-to-date ledger and automatically recover the network using that ledger.
+
+There are two paths, a election path, and a very-high-availablity failover path.
+The election path ensures that if: all nodes restart and have full network connectivity, a majority of nodes' on-disk ledger contains every committed transaction, and no timeouts trigger; then there will be only one recovered network, then all committed transaction will be persisted.
+However, the election path can become stuck, in which case the failover path is designed to ensure progress.
+
+In the election path, nodes first gossip with each other, learning of the ledgers of other nodes.
+Once they have heard from every node they vote for the node with the best ledger.
+If a node receives votes from a majority of nodes, it invokes `transition-to-open` and notifies the other nodes to restart and join it.
+This path is illustrated below, and is guaranteed to succeed if all nodes can communicate and no timeouts trigger.
+
+.. mermaid::
+
+    sequenceDiagram
+      participant N1
+      participant N2
+      participant N3
+
+      Note over N1, N3: Gossip
+
+      N1 ->> N2: Gossip(Tx=1)
+      N1 ->> N3: Gossip(Tx=1)
+      N2 ->> N3: Gossip(Tx=2)
+      N3 ->> N2: Gossip(Tx=3)
+
+      Note over N1, N3: Vote
+      N2 ->> N3: Vote
+      N3 ->> N3: Vote
+
+      Note over N1, N3: Open/Join
+      N3 ->> N1: IAmOpen
+      N3 ->> N2: IAmOpen
+
+      Note over N1, N2: Restart
+
+      Note over N3: Transition-to-open
+
+      Note over N3: Local unsealing
+
+      Note over N3: Open
+
+      N1 ->> N3: Join
+      N2 ->> N3: Join
+
+In the failover path, each phase has a timeout to skip to the next phase if a failure has occurred.
+For example, the election path requires all nodes to communicate to advance from the gossip phase to the vote phase.
+However, if any node fails to recover, the election path is stuck.
+In this case, after a timeout, nodes will advance to the vote phase regardless of whether they have heard from all nodes, and vote for the best ledger they have heard of at that point.
+
+Unfortunately, this can lead to multiple forks of the service if different nodes cannot communicate with each other and timeout.
+Hence, we recommend setting the timeout substantially higher than the highest expected recovery time, to minimise the chance of this happening.
+To audit if timeouts were used to open the service, the `public:ccf.gov.selfhealingopen.failover_open` table tracks this.
+
+This failover path is illustrated below.
+
+.. mermaid::
+
+    sequenceDiagram
+      participant N1
+      participant N2
+      participant N3
+
+      Note over N1, N3: Gossip
+
+      N2 ->> N3: Gossip(Tx=2)
+      N3 ->> N2: Gossip(Tx=3)
+
+      Note over N1: Timeout
+      Note over N3: Timeout
+
+      Note over N1, N3: Vote
+
+      N1 ->> N1: Vote
+      N3 ->> N3: Vote
+      N2 ->> N3: Vote
+
+      Note over N1, N3: Open/Join
+
+      Note over N1: Transition-to-open
+      Note over N3: Transition-to-open
+
+
+If the network fails during reconfiguration, each node will use its latest known configuration to recover. Since reconfiguration requires votes from a majority of nodes, the latest configuration should recover using the election path, however nodes in the previous configuration may recover using the election path.
+
 Notes
 -----
 

include/ccf/node/startup_config.h

@@ -102,6 +102,14 @@ namespace ccf
     Snapshots snapshots = {};
   };
 
+  struct SelfHealingOpenConfig
+  {
+    std::vector<std::string> addresses;
+    ccf::ds::TimeString retry_timeout = {"100ms"};
+    ccf::ds::TimeString timeout = {"2000ms"};
+    bool operator==(const SelfHealingOpenConfig&) const = default;
+  };
+
   struct StartupConfig : CCFConfig
   {
     StartupConfig() = default;
@@ -146,6 +154,7 @@ namespace ccf
         std::nullopt;
       std::optional<std::string> previous_sealed_ledger_secret_location =
         std::nullopt;
+      std::optional<SelfHealingOpenConfig> self_healing_open = std::nullopt;
     };
     Recover recover = {};
   };

include/ccf/service/tables/self_healing_open.h

@@ -0,0 +1,76 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the Apache 2.0 License.
+#pragma once
+
+#include "ccf/ds/enum_formatter.h"
+#include "ccf/ds/json.h"
+#include "ccf/ds/quote_info.h"
+#include "ccf/service/map.h"
+
+using IntrinsicIdentifier = std::string;
+
+struct SelfHealingOpenNodeInfo_t
+{
+  ccf::QuoteInfo quote_info;
+  std::string published_network_address;
+  std::vector<uint8_t> cert_der;
+  std::string service_identity;
+  IntrinsicIdentifier intrinsic_id;
+};
+
+DECLARE_JSON_TYPE(SelfHealingOpenNodeInfo_t);
+DECLARE_JSON_REQUIRED_FIELDS(
+  SelfHealingOpenNodeInfo_t,
+  quote_info,
+  published_network_address,
+  cert_der,
+  service_identity,
+  intrinsic_id);
+
+enum class SelfHealingOpenSM
+{
+  GOSSIPPING = 0,
+  VOTING,
+  OPENING, // by chosen replica
+  JOINING, // by all other replicas
+  OPEN,
+};
+
+DECLARE_JSON_ENUM(
+  SelfHealingOpenSM,
+  {{SelfHealingOpenSM::GOSSIPPING, "Gossipping"},
+   {SelfHealingOpenSM::VOTING, "Voting"},
+   {SelfHealingOpenSM::OPENING, "Opening"},
+   {SelfHealingOpenSM::JOINING, "Joining"},
+   {SelfHealingOpenSM::OPEN, "Open"}});
+
+namespace ccf
+{
+  using SelfHealingOpenNodeInfo =
+    ServiceMap<IntrinsicIdentifier, SelfHealingOpenNodeInfo_t>;
+  using SelfHealingOpenGossips =
+    ServiceMap<IntrinsicIdentifier, ccf::kv::Version>;
+  using SelfHealingOpenChosenReplica = ServiceValue<IntrinsicIdentifier>;
+  using SelfHealingOpenVotes = ServiceSet<IntrinsicIdentifier>;
+  using SelfHealingOpenSMState = ServiceValue<SelfHealingOpenSM>;
+  using SelfHealingOpenTimeoutSMState = ServiceValue<SelfHealingOpenSM>;
+  using SelfHealingOpenFailoverFlag = ServiceValue<bool>;
+
+  namespace Tables
+  {
+    static constexpr auto SELF_HEALING_OPEN_NODES =
+      "public:ccf.gov.selfhealingopen.nodes";
+    static constexpr auto SELF_HEALING_OPEN_GOSSIPS =
+      "public:ccf.gov.selfhealingopen.gossip";
+    static constexpr auto SELF_HEALING_OPEN_CHOSEN_REPLICA =
+      "public:ccf.gov.selfhealingopen.chosen_replica";
+    static constexpr auto SELF_HEALING_OPEN_VOTES =
+      "public:ccf.gov.selfhealingopen.votes";
+    static constexpr auto SELF_HEALING_OPEN_SM_STATE =
+      "public:ccf.gov.selfhealingopen.sm_state";
+    static constexpr auto SELF_HEALING_OPEN_TIMEOUT_SM_STATE =
+      "public:ccf.gov.selfhealingopen.timeout_sm_state";
+    static constexpr auto SELF_HEALING_OPEN_FAILOVER_FLAG =
+      "public:ccf.gov.selfhealingopen.failover_open";
+  }
+}

src/common/configuration.h

@@ -113,6 +113,10 @@ namespace ccf
     node_to_node_message_limit,
     historical_cache_soft_limit);
 
+  DECLARE_JSON_TYPE_WITH_OPTIONAL_FIELDS(SelfHealingOpenConfig);
+  DECLARE_JSON_REQUIRED_FIELDS(SelfHealingOpenConfig, addresses);
+  DECLARE_JSON_OPTIONAL_FIELDS(SelfHealingOpenConfig, retry_timeout, timeout);
+
   DECLARE_JSON_TYPE(StartupConfig::Start);
   DECLARE_JSON_REQUIRED_FIELDS(
     StartupConfig::Start, members, constitution, service_configuration);
@@ -127,9 +131,11 @@ namespace ccf
 
   DECLARE_JSON_TYPE(StartupConfig::Recover);
   DECLARE_JSON_REQUIRED_FIELDS(
+    StartupConfig::Recover, previous_service_identity);
+  DECLARE_JSON_OPTIONAL_FIELDS(
     StartupConfig::Recover,
-    previous_service_identity,
-    previous_sealed_ledger_secret_location);
+    previous_sealed_ledger_secret_location,
+    self_healing_open);
 
   DECLARE_JSON_TYPE_WITH_BASE(StartupConfig, CCFConfig);
   DECLARE_JSON_REQUIRED_FIELDS(

src/crypto/csr.h

@@ -13,7 +13,7 @@ namespace ccf::crypto
    * @param signing_request CSR to extract the public key from
    * @return extracted public key
    */
-  Pem public_key_pem_from_csr(const Pem& signing_request)
+  inline Pem public_key_pem_from_csr(const Pem& signing_request)
   {
     X509* icrt = NULL;
     OpenSSL::Unique_BIO mem(signing_request);

src/enclave/interface.h

@@ -27,7 +27,10 @@ enum AdminMessage : ringbuffer::Message
   DEFINE_RINGBUFFER_MSG_TYPE(tick),
 
   /// Notify the host of work done since last message. Enclave -> Host
-  DEFINE_RINGBUFFER_MSG_TYPE(work_stats)
+  DEFINE_RINGBUFFER_MSG_TYPE(work_stats),
+
+  /// Notify the host that it should restart
+  DEFINE_RINGBUFFER_MSG_TYPE(restart)
 };
 
 DECLARE_RINGBUFFER_MESSAGE_PAYLOAD(AdminMessage::fatal_error_msg, std::string);
@@ -36,6 +39,7 @@ DECLARE_RINGBUFFER_MESSAGE_NO_PAYLOAD(AdminMessage::stop_notice);
 DECLARE_RINGBUFFER_MESSAGE_NO_PAYLOAD(AdminMessage::stopped);
 DECLARE_RINGBUFFER_MESSAGE_NO_PAYLOAD(AdminMessage::tick);
 DECLARE_RINGBUFFER_MESSAGE_PAYLOAD(AdminMessage::work_stats, std::string);
+DECLARE_RINGBUFFER_MESSAGE_NO_PAYLOAD(AdminMessage::restart);
 
 /// Messages sent from app endpoints
 enum AppMessage : ringbuffer::Message

src/host/configuration.h

@@ -118,6 +118,8 @@ namespace host
         std::string previous_service_identity_file;
         std::optional<std::string> previous_sealed_ledger_secret_location =
           std::nullopt;
+        std::optional<ccf::SelfHealingOpenConfig> self_healing_open =
+          std::nullopt;
         bool operator==(const Recover&) const = default;
       };
       Recover recover = {};
@@ -168,7 +170,8 @@ namespace host
     CCHostConfig::Command::Recover,
     initial_service_certificate_validity_days,
     previous_service_identity_file,
-    previous_sealed_ledger_secret_location);
+    previous_sealed_ledger_secret_location,
+    self_healing_open);
 
   DECLARE_JSON_TYPE_WITH_OPTIONAL_FIELDS(CCHostConfig::Command);
   DECLARE_JSON_REQUIRED_FIELDS(CCHostConfig::Command, type);

src/host/handle_ring_buffer.h

@@ -5,6 +5,8 @@
 #include "../ds/files.h"
 #include "../enclave/interface.h"
 #include "ds/internal_logger.h"
+#include "ds/non_blocking.h"
+#include "self_healing_open.h"
 #include "timer.h"
 
 #include <chrono>
@@ -53,6 +55,11 @@ namespace asynchost
           uv_stop(uv_default_loop());
           LOG_INFO_FMT("Host stopped successfully");
         });
+
+      DISPATCHER_SET_MESSAGE_HANDLER(
+        bp, AdminMessage::restart, [&](const uint8_t*, size_t) {
+          ccf::SelfHealingOpenRBHandlerSingleton::instance()->trigger_restart();
+        });
     }
 
     void on_timer()