diff --git a/tests/apollo/CMakeLists.txt b/tests/apollo/CMakeLists.txt index dbcee91633..fe23df634b 100644 --- a/tests/apollo/CMakeLists.txt +++ b/tests/apollo/CMakeLists.txt @@ -149,3 +149,5 @@ apollo_test(NAME osexample_demo_tests SUITE test_osexample_demo) if(ENABLE_RESTART_RECOVERY_TESTS) apollo_test(NAME skvbc_restart_recovery_tests SUITE test_skvbc_restart_recovery NIGHTLY) endif() + +apollo_test(NAME skvbc_fullnode_tests SUITE test_skvbc_fullnode) diff --git a/tests/apollo/clientservice/request_pb2_grpc.py b/tests/apollo/clientservice/request_pb2_grpc.py index d1cee1b7df..b6d3314e76 100644 --- a/tests/apollo/clientservice/request_pb2_grpc.py +++ b/tests/apollo/clientservice/request_pb2_grpc.py @@ -2,8 +2,7 @@ """Client and server classes corresponding to protobuf-defined services.""" import grpc -import request_pb2 as request__pb2 - +import clientservice.request_pb2 as request__pb2 class RequestServiceStub(object): """Service error handling diff --git a/tests/apollo/test_skvbc_fullnode.py b/tests/apollo/test_skvbc_fullnode.py new file mode 100644 index 0000000000..5dd12c524b --- /dev/null +++ b/tests/apollo/test_skvbc_fullnode.py @@ -0,0 +1,146 @@ +# Concord +# +# Copyright (c) 2019 VMware, Inc. All Rights Reserved. +# +# This product is licensed to you under the Apache 2.0 license (the "License"). +# You may not use this product except in compliance with the Apache 2.0 License. +# +# This product may include a number of subcomponents with separate copyright +# notices and license terms. Your use of these subcomponents is subject to the +# terms and conditions of the subcomponent's license, as noted in the LICENSE +# file. +import os.path +import random +import unittest +from os import environ + +from google.protobuf import duration_pb2 as duration_proto + +import trio + +import os.path +import sys +sys.path.append(os.path.abspath("../../build/tests/apollo/util/")) +import skvbc_messages + +from util.test_base import ApolloTest +from util import skvbc as kvbc +from util.pyclient import bft_grpc_client as client +from util.bft import with_trio, with_bft_network, KEY_FILE_PREFIX + +def start_replica_cmd(builddir, replica_id): + """ + Return a command that starts an skvbc replica when passed to + subprocess.Popen. + + Note each arguments is an element in a list. + """ + statusTimerMilli = "500" + viewChangeTimeoutMilli = "10000" + path = os.path.join(builddir, "tests", "simpleKVBC", "TesterReplica", "skvbc_replica") + + if os.environ.get('BLOCKCHAIN_VERSION', default="1").lower() == "4" : + blockchain_version = "4" + else : + blockchain_version = "1" + + return [path, + "-k", KEY_FILE_PREFIX, + "-i", str(replica_id), + "-s", statusTimerMilli, + "-V",blockchain_version, + "-v", viewChangeTimeoutMilli, + "-e", str(True) + ] + +class SkvbcFullNodeTest(ApolloTest): + + @with_trio + @with_bft_network(start_replica_cmd=start_replica_cmd, num_fn=1, selected_configs=lambda n, f, c: n == 6) + async def test_read_write_request(self, bft_network): + + print(f"Running test_read_write_request") + bft_network.start_all_replicas() + bft_network.start_all_fns() + + await trio.sleep(5) + + bft_client = client.GrpcClient() + skvbc = kvbc.SimpleKVBCProtocol(bft_network) + + key = skvbc.random_key() + value = skvbc.random_value() + + print(f"Random Value {value}") + kv_pair = [(key, value)] + + msg = skvbc.write_req([], kv_pair, 0) + res = bft_client.sendRequest(msg) + #print(f"Write Response Raw {res} ") + + write_res = skvbc.parse_reply(res) + #print(f"Write Response Parsed {write_res.success} {write_res.last_block_id}") + + msg_read = skvbc.read_req([key]) + read_res = bft_client.sendRequest(msg_read, duration_proto.Duration(seconds =5), True) + #print(f"Read Response Raw {read_res} ") + + value_read_dict = skvbc.parse_reply(read_res) + value_read = value_read_dict[key] + + print(f"Random Value Read {value_read}") + + self.assertEqual(value, value_read, "A BFT Client failed to read a " \ + "key-value pair from a SimpleKVBC cluster matching the " \ + "key-value pair it wrote immediately prior to the read.") + + @with_trio + @with_bft_network(start_replica_cmd=start_replica_cmd, num_fn=1, selected_configs=lambda n, f, c: n == 6) + async def test_read_write_restart_fn(self, bft_network): + + print(f"Running test_read_write_restart_fn") + bft_network.start_all_replicas() + bft_network.start_all_fns() + + await trio.sleep(5) + + bft_client = client.GrpcClient() + skvbc = kvbc.SimpleKVBCProtocol(bft_network) + + key = skvbc.random_key() + value = skvbc.random_value() + + print(f"Random Value Before Restart {value}") + kv_pair = [(key, value)] + + msg = skvbc.write_req([], kv_pair, 0) + res = bft_client.sendRequest(msg) + #print(f"Write Response Raw {res} ") + + #write_res = skvbc.parse_reply(res) + #print(f"Write Response Parsed {write_res.success} {write_res.last_block_id}") + + bft_network.stop_fn() + + bft_network.start_all_fns() + await trio.sleep(5) + bft_client = client.GrpcClient() + + msg_read = skvbc.read_req([key]) + read_res = bft_client.sendRequest(msg_read, duration_proto.Duration(seconds =5), True) + #print(f"Read Response Raw {read_res} ") + + value_read_dict = skvbc.parse_reply(read_res) + value_read = value_read_dict[key] + + print(f"Random Value Read After Restart{value_read}") + + self.assertEqual(value, value_read, "A BFT Client failed to read a " \ + "key-value pair from a SimpleKVBC cluster matching the " \ + "key-value pair it wrote immediately prior to the read.") + + skvbc.parse_reply(res) + + print("Response received") + + diff --git a/tests/apollo/test_skvbc_reconfiguration.py b/tests/apollo/test_skvbc_reconfiguration.py index 815519d3c3..2eab94e25a 100644 --- a/tests/apollo/test_skvbc_reconfiguration.py +++ b/tests/apollo/test_skvbc_reconfiguration.py @@ -717,7 +717,7 @@ async def test_single_signature_scheme_to_no_single_signature_scheme(self, bft_n key_file_prefix=KEY_FILE_PREFIX, start_replica_cmd=lambda builddir, replica_id: start_replica_cmd(builddir, replica_id) + ["--key-exchange-on-start", "True"], - stop_replica_cmd=None, num_ro_replicas=0) + stop_replica_cmd=None, num_ro_replicas=0, num_fn=0) await bft_network.change_configuration(conf, generate_tls=False) bft_network.restart_clients(restart_replicas=True) @@ -1550,7 +1550,8 @@ async def test_remove_nodes(self, bft_network): key_file_prefix=KEY_FILE_PREFIX, start_replica_cmd=start_replica_cmd_with_key_exchange, stop_replica_cmd=None, - num_ro_replicas=0) + num_ro_replicas=0, + num_fn=0) await bft_network.change_configuration(conf) await op.add_remove_with_wedge(test_config, bft=False) await self.validate_epoch_number(bft_network, 1, bft_network.all_replicas()) @@ -1598,7 +1599,8 @@ async def test_remove_nodes_with_f_failures(self, bft_network): key_file_prefix=KEY_FILE_PREFIX, start_replica_cmd=start_replica_cmd_with_key_exchange, stop_replica_cmd=None, - num_ro_replicas=0) + num_ro_replicas=0, + num_fn=0) await bft_network.change_configuration(conf) await op.add_remove_with_wedge(test_config, bft=True, restart=True) await self.validate_epoch_number(bft_network, 1, bft_network.all_replicas()) @@ -1641,7 +1643,8 @@ async def test_remove_nodes_with_unwedge(self, bft_network): key_file_prefix=KEY_FILE_PREFIX, start_replica_cmd=start_replica_cmd_with_key_exchange, stop_replica_cmd=None, - num_ro_replicas=0) + num_ro_replicas=0, + num_fn=0) await bft_network.change_configuration(conf) await op.add_remove_with_wedge(test_config, bft=False, restart=False) await self.validate_stop_on_wedge_point(bft_network, skvbc, fullWedge=True) @@ -1691,7 +1694,8 @@ async def test_add_nodes(self, bft_network): key_file_prefix=KEY_FILE_PREFIX, start_replica_cmd=start_replica_cmd, stop_replica_cmd=None, - num_ro_replicas=0) + num_ro_replicas=0, + num_fn=0) await bft_network.change_configuration(conf, generate_tls=True) bft_network.restart_clients() await self.validate_epoch_number(bft_network, 1, bft_network.all_replicas()) @@ -1730,7 +1734,8 @@ async def test_key_exchange_after_add_nodes_with_failure(self, bft_network): key_file_prefix=KEY_FILE_PREFIX, start_replica_cmd=start_replica_cmd_with_key_exchange, stop_replica_cmd=None, - num_ro_replicas=0) + num_ro_replicas=0, + num_fn=0) await bft_network.change_configuration(conf, generate_tls=True) bft_network.restart_clients(restart_replicas=False) bft_network.start_replicas(replicas=bft_network.all_replicas(without=late_replica_set)) @@ -1816,7 +1821,8 @@ async def test_add_nodes_with_failures(self, bft_network): key_file_prefix=KEY_FILE_PREFIX, start_replica_cmd=start_replica_cmd_with_key_exchange, stop_replica_cmd=None, - num_ro_replicas=0) + num_ro_replicas=0, + num_fn=0) await bft_network.change_configuration(conf, generate_tls=True) bft_network.restart_clients(generate_tx_signing_keys=True, restart_replicas=False) @@ -1928,7 +1934,8 @@ async def test_reconfiguration_with_ror(self, bft_network): key_file_prefix=KEY_FILE_PREFIX, start_replica_cmd=start_replica_cmd_with_object_store_and_ke, stop_replica_cmd=None, - num_ro_replicas=1) + num_ro_replicas=1, + num_fn=0) await bft_network.change_configuration(conf) ro_replica_id = bft_network.config.n await bft_network.check_initial_key_exchange(stop_replicas=False) @@ -1965,7 +1972,8 @@ async def test_reconfiguration_with_ror(self, bft_network): key_file_prefix=KEY_FILE_PREFIX, start_replica_cmd=start_replica_cmd_with_object_store_and_ke, stop_replica_cmd=None, - num_ro_replicas=1) + num_ro_replicas=1, + num_fn=0) await bft_network.change_configuration(conf) ro_replica_id = bft_network.config.n await bft_network.check_initial_key_exchange(stop_replicas=False) diff --git a/tests/apollo/util/bft.py b/tests/apollo/util/bft.py index bea7ee5a11..d0ba42d97f 100644 --- a/tests/apollo/util/bft.py +++ b/tests/apollo/util/bft.py @@ -67,7 +67,8 @@ 'key_file_prefix', 'start_replica_cmd', 'stop_replica_cmd', - 'num_ro_replicas' + 'num_ro_replicas', + 'num_fn' ]) # NOTE: When the value is changed, then ensure to change in ReplicaConfig class' @@ -169,7 +170,7 @@ async def background_sender(arg1, arg2, *, task_status=trio.TASK_STATUS_IGNORED) nursery.cancel_scope.cancel() return wrapper -def with_bft_network(start_replica_cmd, selected_configs=None, num_clients=None, num_ro_replicas=0, +def with_bft_network(start_replica_cmd, selected_configs=None, num_clients=None, num_ro_replicas=0, num_fn=0, rotate_keys=False, bft_configs=None, with_cre=False, publish_master_keys=False, num_repeats=int(os.getenv("NUM_REPEATS", 1)), break_on_first_failure=bool(os.getenv('BREAK_ON_FAILURE') == 'TRUE'), @@ -211,7 +212,8 @@ async def wrapper(*args, **kwargs): key_file_prefix=KEY_FILE_PREFIX, start_replica_cmd=start_replica_cmd, stop_replica_cmd=None, - num_ro_replicas=num_ro_replicas) + num_ro_replicas=num_ro_replicas, + num_fn=num_fn) subtest_id = None with test_instance.subTest(config=f'{bft_config}', storage=f"v{os.environ.get('BLOCK_CHAIN_VERSION', default='1')}"): @@ -309,7 +311,7 @@ def __exit__(self, etype, value, tb): self._eliot_log_file = None def __init__(self, is_existing, origdir, config, testdir, certdir, builddir, toolsdir, - procs, replicas, clients, metrics, client_factory, background_nursery, ro_replicas=[], + procs, replicas, clients, metrics, client_factory, background_nursery, ro_replicas=[], fn_instances=[], case_name=""): self.is_existing = is_existing # An existing deployment might pass some of the folders paths as empty so we skip the next assertion. @@ -342,12 +344,13 @@ def __init__(self, is_existing, origdir, config, testdir, certdir, builddir, too self.test_start_time = None self.perf_proc = None self.ro_replicas = ro_replicas + self.fn_instances = fn_instances self.txn_signing_enabled = (os.environ.get('TXN_SIGNING_ENABLED', "").lower() in ["true", "on"]) self.with_cre = False self.use_unified_certs = False self.cre_proc = None self.cre_fds = None - self.cre_id = self.config.n + self.config.num_ro_replicas + self.config.num_clients + RESERVED_CLIENTS_QUOTA + self.cre_id = self.config.n + self.config.num_ro_replicas + self.config.num_fn + self.config.num_clients + RESERVED_CLIENTS_QUOTA self._logs_dir = Path(self.builddir) / "tests" / "apollo" / "logs" / logdir_timestamp() self._eliot_log_file = None self._suite_name = os.environ.get('TEST_NAME', None) @@ -391,6 +394,10 @@ def new(cls, config, background_nursery, client_factory=None, with_cre=False, us bft_config.bft_msg_port_from_node_id(i), bft_config.metrics_port_from_node_id(i)) for i in range(config.n, config.n + config.num_ro_replicas)], + fn_instances=[bft_config.Replica(i, "127.0.0.1", + bft_config.bft_msg_port_from_node_id(i), + bft_config.metrics_port_from_node_id(i)) + for i in range(config.n + config.num_ro_replicas, config.n + config.num_ro_replicas + config.num_fn)], case_name=case_name ) bft_network.with_cre = with_cre @@ -405,7 +412,7 @@ def new(cls, config, background_nursery, client_factory=None, with_cre=False, us bft_network._generate_crypto_keys() if bft_network.comm_type() == bft_config.COMM_TYPE_TCP_TLS: generate_cre = 0 if bft_network.with_cre is False else 1 - # Generate certificates for all replicas, clients, and reserved clients + # Generate certificates for all replicas, clients, and reserved clients ` bft_network.generate_tls_certs( bft_network.num_total_replicas() + config.num_clients + RESERVED_CLIENTS_QUOTA + generate_cre, use_unified_certs=use_unified_certs) @@ -417,6 +424,7 @@ def new(cls, config, background_nursery, client_factory=None, with_cre=False, us @classmethod def existing(cls, config, replicas, clients, client_factory=None, background_nursery=None): + log.log_message(message_type=f"Creating from existing ") certdir = None builddir = tempfile.mkdtemp(prefix='builddir') if not client_factory: @@ -496,6 +504,86 @@ def stop_cre(self): for fd in self.cre_fds: fd.close() p.wait() + + def start_fullnode(self, fullnode_id): + """ + Start the fullnode if it isn't already started. + Otherwise raise an AlreadyStoppedError. + """ + with log.start_action(action_type="start_fullnode"): + stdout_file = None + stderr_file = None + + keep_logs = os.environ.get('KEEP_APOLLO_LOGS', "").lower() in ["true", "on"] + print(f"start_fullnode {keep_logs}") + + if keep_logs: + test_name = os.environ.get('TEST_NAME') + if os.environ.get('BLOCKCHAIN_VERSION', default="1").lower() == "4" : + test_name = test_name + "_v4" + + fn_test_log_path = f"{self.current_test_case_path / 'stdout_fn.log' }" + + Path(self.current_test_case_path).mkdir(parents=True, exist_ok=True) + + stdout_file = open(fn_test_log_path, 'w+') + stderr_file = open(fn_test_log_path, 'w+') + + stdout_file.write("############################################\n") + stdout_file.flush() + stderr_file.write("############################################\n") + stderr_file.flush() + + statusTimerMilli = "500" + viewChangeTimeoutMilli = "10000" + path = os.path.join(self.builddir, "tests", "simpleKVBC", "TesterFullNode", "skvbc_fullnode") + + if os.environ.get('BLOCKCHAIN_VERSION', default="1").lower() == "4" : + blockchain_version = "4" + else : + blockchain_version = "1" + self.fn_fds = (stdout_file, stderr_file) + fn_cmd = [path, + "-k", KEY_FILE_PREFIX, + "-i", str(fullnode_id), + "-s", statusTimerMilli, + "-V", blockchain_version, + "-v", viewChangeTimeoutMilli, + "-c", self.certdir + "/" + str(fullnode_id), + "-e", str(True) + ] + digest = self.binary_digest(path) if Path(path).exists() else 'Unknown' + with log.start_action(action_type="start_fullnode_process", binary=path, + binary_digest=digest, cmd=' '.join(fn_cmd)): + my_env = os.environ.copy() + my_env["RID"] = str(fullnode_id) + self.fn_proc = subprocess.Popen( + fn_cmd, + stdout=stdout_file, + stderr=stderr_file, + close_fds=True, + env=my_env) + + def stop_fn(self): + with log.start_action(action_type="stop_fn"): + p = self.fn_proc + if os.environ.get('GRACEFUL_SHUTDOWN', "").lower() in set(["true", "on"]): + p.terminate() + else: + p.kill() + for fd in self.fn_fds: + fd.close() + p.wait() + + def start_all_fns(self): + with log.start_action(action_type="start_fullnodes"): + all_fullnodes = self.all_fullnodes() + for i in all_fullnodes: + try: + self.start_fullnode(i) + except AlreadyRunningError: + if not self.is_existing: + raise def transfer_db_files(self, source_id: int, dest_ids: Sequence[int]): with log.start_action(action_type="transfer db files", source_id=source_id, dests=dest_ids): @@ -549,6 +637,8 @@ def _generate_crypto_keys(self): args = [keygen, "-n", str(self.config.n), "-f", str(self.config.f)] if self.config.num_ro_replicas > 0: args.extend(["-r", str(self.config.num_ro_replicas)]) + if self.config.num_fn > 0: + args.extend(["-fn", str(self.config.num_fn)]) args.extend(["-o", self.config.key_file_prefix]) with log.start_action(action_type="Key Generation", cmdline=' '.join(args)): subprocess.run(args, check=True) @@ -614,7 +704,7 @@ def copy_certs_from_server_to_clients(self, src): shutil.copytree(src_cert, comp_cert_dir) def _create_clients(self): - start_id = self.config.n + self.config.num_ro_replicas + start_id = self.num_total_replicas() last_id = start_id + self.config.num_clients log_message(message_type=f"Creating clients", first_client_id=start_id, last_client_id=last_id, communication=str(BFT_CLIENT_TYPE), config_template=self._bft_config('client_id')) @@ -662,7 +752,7 @@ def _init_metrics(self): def random_client(self, without=None): if without == None: without = set() - + return random.choice(list(set(self.clients.values()) - without)) def random_clients(self, max_clients): @@ -705,7 +795,7 @@ def split(a, n): k, m = divmod(len(a), n) return [a[i * k + min(i, m):(i + 1) * k + min(i + 1, m)] for i in range(n)] - start_id = self.config.n + self.config.num_ro_replicas + start_id = self.num_total_replicas() client_ids = range(start_id, start_id + self.config.num_clients) start_id = self.num_total_replicas() + self.config.num_clients reserved_client_ids = range(start_id, start_id + RESERVED_CLIENTS_QUOTA + 1) @@ -1031,11 +1121,14 @@ def comm_type(self): raise NotImplementedError(f"{type(self.clients[self.config.n])} is not supported!") def num_total_replicas(self): - return self.config.n + self.config.num_ro_replicas + return self.config.n + self.config.num_ro_replicas + self.config.num_fn def num_total_clients(self): return self.config.num_clients + RESERVED_CLIENTS_QUOTA + def num_total_fns(self): + return self.config.num_fn + def node_id_from_bft_msg_port(self, bft_msg_port): assert ((bft_msg_port % 2 == 0) and (bft_msg_port < bft_config.START_METRICS_PORT) and @@ -1107,6 +1200,16 @@ def all_client_ids(self, without=None, with_reserved_clients=True): num_total_clients += RESERVED_CLIENTS_QUOTA return list(set(range(num_total_replicas, num_total_replicas + num_total_clients)) - without) + def all_fullnodes(self, without=None): + """ + Returns a list of all ACTIVE FN IDs excluding the "without" set + """ + if without == None: + without = set() + num_replicas = self.config.n + self.config.num_ro_replicas + num_fns = self.config.num_fn + return list(set(range(num_replicas, num_replicas + num_fns)) - without) + def random_set_of_replicas(self, size, without=None): """ Returns a random list of ACTIVE replica IDs excluding the "without" set, and without any RO replicas """ if without is None: diff --git a/tests/apollo/util/pyclient/bft_grpc_client.py b/tests/apollo/util/pyclient/bft_grpc_client.py new file mode 100644 index 0000000000..21e6fc59ca --- /dev/null +++ b/tests/apollo/util/pyclient/bft_grpc_client.py @@ -0,0 +1,44 @@ +from google.protobuf import duration_pb2 as duration_proto + +import grpc +import clientservice.request_pb2 as request_proto +import clientservice.request_pb2_grpc as request_grpc + +import os.path +import sys +sys.path.append(os.path.abspath("../../build/tests/apollo/util/")) +import skvbc_messages + +from util import skvbc as kvbc + +class GrpcClient(object): + """ + Client for gRPC functionality + """ + + def __init__(self): + self.host = '0.0.0.0' + self.server_port = 50051 + self.client_id = 100 + + # instantiate a channel + self.channel = grpc.insecure_channel( + 'localhost:50051', options=(('grpc.enable_http_proxy', 0),)) + + # bind the client and the server + self.request_stub = request_grpc.RequestServiceStub(self.channel) + + def sendRequest(self, request=b"csutil.py", timeout=duration_proto.Duration(seconds=5), + read_only=False, pre_execute=False, correlation_id="grpcclient-cid"): + + req = request_proto.Request( + raw_request=bytes(request), + timeout=timeout, + read_only=read_only, + pre_execute=pre_execute, + correlation_id=correlation_id + ) + + response = self.request_stub.Send(req) + + return response.raw_response \ No newline at end of file diff --git a/tests/simpleKVBC/CMakeLists.txt b/tests/simpleKVBC/CMakeLists.txt index 09c68299a3..ce160f34c2 100644 --- a/tests/simpleKVBC/CMakeLists.txt +++ b/tests/simpleKVBC/CMakeLists.txt @@ -2,6 +2,7 @@ add_subdirectory(cmf) add_subdirectory(TesterClient) add_subdirectory(TesterReplica) add_subdirectory(TesterCRE) +add_subdirectory(TesterFullNode) add_custom_target(copy_blockchain_scripts2 ALL COMMENT "Copying scripts abcd") add_custom_command(TARGET copy_blockchain_scripts2 COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR}/scripts ${CMAKE_CURRENT_BINARY_DIR}/scripts) diff --git a/tests/simpleKVBC/TesterFullNode/CMakeLists.txt b/tests/simpleKVBC/TesterFullNode/CMakeLists.txt new file mode 100644 index 0000000000..ca085e3c58 --- /dev/null +++ b/tests/simpleKVBC/TesterFullNode/CMakeLists.txt @@ -0,0 +1,19 @@ +cmake_minimum_required (VERSION 3.2) +project(skvbc_fullnode VERSION 0.1.0.0 LANGUAGES CXX) + +add_executable(skvbc_fullnode main.cpp + fnGrpc.cpp) + +if(${USE_COMM_PLAIN_TCP}) + target_compile_definitions(skvbc_fullnode PUBLIC USE_COMM_PLAIN_TCP) +endif() + +if(BUILD_ROCKSDB_STORAGE) + target_compile_definitions(skvbc_fullnode PUBLIC "USE_ROCKSDB=1") +endif() + +target_link_libraries(skvbc_fullnode PUBLIC skvbc_replica_lib bftclient_new clientservice-proto test_config_lib stdc++fs) + +target_include_directories(skvbc_fullnode PUBLIC ..) +target_include_directories(skvbc_fullnode PUBLIC ../../TesterReplica/) +target_include_directories(skvbc_fullnode PUBLIC ${libskvbc_replica_lib_SOURCE_DIR}) \ No newline at end of file diff --git a/tests/simpleKVBC/TesterFullNode/fnGrpc.cpp b/tests/simpleKVBC/TesterFullNode/fnGrpc.cpp new file mode 100644 index 0000000000..058e75c9d3 --- /dev/null +++ b/tests/simpleKVBC/TesterFullNode/fnGrpc.cpp @@ -0,0 +1,119 @@ +// Concord +// +// Copyright (c) 2019-2020 VMware, Inc. All Rights Reserved. +// +// This product is licensed to you under the Apache 2.0 license (the "License"). +// You may not use this product except in compliance with the Apache 2.0 +// License. +// +// This product may include a number of subcomponents with separate copyright +// notices and license terms. Your use of these subcomponents is subject to the +// terms and conditions of the subcomponent's license, as noted in the LICENSE +// file. + +#include "fnGrpc.hpp" + +constexpr size_t OUT_BUFFER_SIZE = 1024000; + +void RequestServiceImpl::sendToInternalHandler(const bft::client::RequestConfig req_config, + const bft::client::Msg&& msg, + const bftEngine::MsgFlag flags, + vmware::concord::client::request::v1::Response* response) { + // send to internal command handler + + concordUtils::SpanWrapper span_wrp_; + + bftEngine::IRequestsHandler::ExecutionRequestsQueue requests; + std::string req(msg.begin(), msg.end()); + + reply_buffer_ = std::string(OUT_BUFFER_SIZE, 0); + requests.push_back(bftEngine::IRequestsHandler::ExecutionRequest{6, + 1, + req_config.correlation_id, + flags, + static_cast(req.size()), + req.c_str(), + "", + static_cast(reply_buffer_.size()), + reply_buffer_.data()}); + + cmdHandler_->execute(requests, std::nullopt, "1", span_wrp_); + + bftEngine::IRequestsHandler::ExecutionRequest& single_request = requests.back(); + + LOG_INFO(GL, "Request Executed "); + LOG_INFO(GL, "Status " << single_request.outExecutionStatus); + LOG_INFO(GL, "Reply Size " << single_request.outActualReplySize); + LOG_INFO(GL, "Block ID " << single_request.blockId); + LOG_INFO(GL, "Reply String " << single_request.outReply); + + std::string ex_response(single_request.outReply, single_request.outActualReplySize); + response->set_raw_response(std::move(ex_response)); +} + +grpc::Status RequestServiceImpl::Send(grpc::ServerContext* context, + const vmware::concord::client::request::v1::Request* request, + vmware::concord::client::request::v1::Response* response) { + /*auto tracer = opentracing::MakeNoopTracer(); + auto raw_span = tracer->StartSpan("dummy"); + auto span_wrapper_ = concordUtils::startChildSpanFromContext(raw_span->context(), "dummy_1");*/ + + LOG_INFO(GL, "Received Request"); + + // send to bft client + bft::client::Msg msg; + msg = bft::client::Msg(request->raw_request().begin(), request->raw_request().end()); + + auto seconds = std::chrono::seconds{request->timeout().seconds()}; + auto nanos = std::chrono::nanoseconds{request->timeout().nanos()}; + auto timeout = std::chrono::duration_cast(seconds + nanos); + + bft::client::RequestConfig req_config; + req_config.pre_execute = request->pre_execute(); + req_config.timeout = timeout; + req_config.correlation_id = request->correlation_id(); + req_config.primary_only = request->primary_only(); + + if (request->read_only()) { + if (fn_execute_enable_) { + sendToInternalHandler(req_config, std::move(msg), bftEngine::MsgFlag::READ_ONLY_FLAG, response); + } else { + bft::client::ReadConfig config; + config.request = req_config; + + LOG_INFO(GL, "Sending Read Request to Validator Network"); + + bft::client::Reply reply; + reply = bft_client_->send(config, std::move(msg)); + + LOG_INFO(GL, "Received Read Reply" << reply.result); + + std::string data(reply.matched_data.begin(), reply.matched_data.end()); + + response->set_raw_response(std::move(data)); + } + + } else { + if (fn_execute_enable_) { + sendToInternalHandler(req_config, std::move(msg), bftEngine::MsgFlag::EMPTY_FLAGS, response); + } else { + bft::client::WriteConfig config; + config.request = req_config; + + LOG_INFO(GL, "Sending Write Request to Validator Network"); + + bft::client::Reply reply; + reply = bft_client_->send(config, std::move(msg)); + + LOG_INFO(GL, "Received Write Reply" << reply.result); + + std::string data(reply.matched_data.begin(), reply.matched_data.end()); + + response->set_raw_response(std::move(data)); + } + } + + auto status = grpc::Status(grpc::StatusCode::OK, "Received in Tester FullNode"); + + return status; +} \ No newline at end of file diff --git a/tests/simpleKVBC/TesterFullNode/fnGrpc.hpp b/tests/simpleKVBC/TesterFullNode/fnGrpc.hpp new file mode 100644 index 0000000000..ad4d67e094 --- /dev/null +++ b/tests/simpleKVBC/TesterFullNode/fnGrpc.hpp @@ -0,0 +1,45 @@ +// Concord +// +// Copyright (c) 2019-2023 VMware, Inc. All Rights Reserved. +// +// This product is licensed to you under the Apache 2.0 license (the "License"). +// You may not use this product except in compliance with the Apache 2.0 +// License. +// +// This product may include a number of subcomponents with separate copyright +// notices and license terms. Your use of these subcomponents is subject to the +// terms and conditions of the subcomponent's license, as noted in the LICENSE +// file. + +#pragma once + +#include +#include +#include "internalCommandsHandler.hpp" +#include "bftclient/bft_client.h" + +class RequestServiceImpl final : public vmware::concord::client::request::v1::RequestService::Service { + public: + RequestServiceImpl(const std::shared_ptr client, + const std::shared_ptr cmdHandler, + const bool fn_execute_enable) + : logger_(logging::getLogger("concord.client.clientservice.event")), + bft_client_(client), + cmdHandler_(cmdHandler), + fn_execute_enable_(fn_execute_enable){}; + grpc::Status Send(grpc::ServerContext* context, + const vmware::concord::client::request::v1::Request* request, + vmware::concord::client::request::v1::Response* response) override; + + private: + void sendToInternalHandler(const bft::client::RequestConfig req_config, + const bft::client::Msg&& msg, + const bftEngine::MsgFlag flags, + vmware::concord::client::request::v1::Response* response); + + logging::Logger logger_; + std::shared_ptr bft_client_; + std::shared_ptr cmdHandler_; + bool fn_execute_enable_; + std::string reply_buffer_; +}; \ No newline at end of file diff --git a/tests/simpleKVBC/TesterFullNode/main.cpp b/tests/simpleKVBC/TesterFullNode/main.cpp new file mode 100644 index 0000000000..9c99804a86 --- /dev/null +++ b/tests/simpleKVBC/TesterFullNode/main.cpp @@ -0,0 +1,207 @@ +// Concord +// +// Copyright (c) 2018-2019 VMware, Inc. All Rights Reserved. +// +// This product is licensed to you under the Apache 2.0 license (the "License"). +// You may not use this product except in compliance with the Apache 2.0 +// License. +// +// This product may include a number of subcomponents with separate copyright +// notices and license terms. Your use of these subcomponents is subject to the +// terms and conditions of the subcomponent's license, as noted in the LICENSE +// file. + +#include "fnGrpc.hpp" + +#include "setup.hpp" +#include "Replica.h" +#include "internalCommandsHandler.hpp" +#include "replica_state_sync_imp.hpp" +#include "block_metadata.hpp" +#include "SimpleBCStateTransfer.hpp" +#include "secrets/secrets_manager_plain.h" +#include "secrets/secrets_manager_enc.h" +#include "bftengine/ControlStateManager.hpp" +#include "messages/ReplicaRestartReadyMsg.hpp" +#include "bftengine/ReconfigurationCmd.hpp" +#include "client/reconfiguration/cre_interfaces.hpp" +#include "bftclient/bft_client.h" +#include "util/assertUtils.hpp" +#include "util/Metrics.hpp" +#include "diagnostics_server.hpp" +#include "communication/CommFactory.hpp" +#include "bftclient/config.h" +#include "bftclient/bft_client.h" +#include "config/test_comm_config.hpp" +#include +#include +#include + +#ifdef USE_ROCKSDB +#include "rocksdb/client.h" +#include "rocksdb/key_comparator.h" +#endif + +#include +#include + +using namespace std; +using namespace bftEngine; +using namespace bft::communication; +using namespace concord::client::reconfiguration; +using std::string; +using bft::client::ClientConfig; +using bft::client::ClientId; +using bft::client::Client; + +namespace concord::kvbc::test { + +std::shared_ptr replica; +std::shared_ptr bft_client; +std::shared_ptr fn_communication; +std::shared_ptr cmdHandler; +std::unique_ptr request_service; +std::unique_ptr fn_server; + +std::atomic_bool timeToExit = false; + +auto logger = logging::getLogger("skvbtest.fullnode"); + +ICommunication* createCommunication(const bft::client::ClientConfig& cc, + const std::string& commFileName, + const std::string& certFolder, + std::shared_ptr& sm, + bool& enc) { + TestCommConfig testCommConfig(logger); + uint16_t numOfReplicas = cc.all_replicas.size(); + uint16_t clients = cc.id.val; +#ifdef USE_COMM_PLAIN_TCP + PlainTcpConfig conf = testCommConfig.GetTCPConfig(false, cc.id.val, clients, numOfReplicas, commFileName); +#elif USE_COMM_TLS_TCP + TlsTcpConfig conf = testCommConfig.GetTlsTCPConfig( + false, cc.id.val, clients, numOfReplicas, commFileName, cc.use_unified_certs, certFolder); + if (conf.secretData_.has_value()) { + sm = std::make_shared(conf.secretData_.value()); + enc = true; + } else { + sm = std::make_shared(); + enc = false; + } +#else + PlainUdpConfig conf = testCommConfig.GetUDPConfig(false, cc.id.val, clients, numOfReplicas, commFileName); +#endif + + return CommFactory::create(conf); +} + +bft::client::ClientConfig setupClientConfig() { + bft::client::ClientConfig client_config; + + client_config.f_val = 0; + client_config.c_val = 0; + client_config.id = ClientId{6}; + + for (int i = 0; i < 4; i++) { + client_config.all_replicas.emplace(bft::client::ReplicaId{static_cast(i)}); + } + return client_config; +} + +void run_fn_grpc_server() { + while (true) { + grpc::ServerBuilder builder; + builder.AddListeningPort("localhost:50051", grpc::InsecureServerCredentials()); + builder.SetMaxReceiveMessageSize(52428800); + + builder.RegisterService(request_service.get()); + + fn_server = std::unique_ptr(builder.BuildAndStart()); + + LOG_INFO(GL, "skvbc_fullnode (fn grpc) running..."); + fn_server->Wait(); + LOG_INFO(GL, "skvbc_fullnode (fn grpc) Waiting End..."); + } +} + +void run_fullnode(int argc, char** argv) { + std::thread grpc_thread; + std::shared_ptr sm_; + bool enc; + + const auto setup = concord::kvbc::TestSetup::ParseArgs(argc, argv); + logging::initLogger(setup->getLogPropertiesFile()); + auto logger = setup->GetLogger(); + MDC_PUT(MDC_REPLICA_ID_KEY, std::to_string(setup->GetReplicaConfig().replicaId)); + MDC_PUT(MDC_THREAD_KEY, "main"); + + replica = std::make_shared( + setup->GetCommunication(), + setup->GetReplicaConfig(), + setup->GetStorageFactory(), + setup->GetMetricsServer().GetAggregator(), + setup->GetPerformanceManager(), + std::map{ + {VERSIONED_KV_CAT_ID, categorization::CATEGORY_TYPE::versioned_kv}, + {categorization::kExecutionEventGroupLatestCategory, categorization::CATEGORY_TYPE::versioned_kv}, + {BLOCK_MERKLE_CAT_ID, categorization::CATEGORY_TYPE::block_merkle}}, + setup->GetSecretManager()); + + std::unique_ptr comm_ptr{ + createCommunication(setupClientConfig(), "", setup->getCertsRootPath(), sm_, enc)}; + + bft_client = std::make_shared(std::move(comm_ptr), setupClientConfig()); + + auto* blockMetadata = new BlockMetadata(*replica); + cmdHandler = + std::make_shared(replica.get(), + replica.get(), + blockMetadata, + logger, + setup->AddAllKeysAsPublic(), + replica->kvBlockchain() ? &replica->kvBlockchain().value() : nullptr); + + request_service = std::make_unique(bft_client, cmdHandler, true); + + grpc_thread = std::thread(run_fn_grpc_server); + + while (true) { + if (timeToExit) { + LOG_INFO(GL, "skvbc_fullnode time to exit"); + fn_server->Shutdown(); + if (grpc_thread.joinable()) { + LOG_INFO(GL, "skvbc_fullnode join thread"); + grpc_thread.join(); + } + LOG_INFO(GL, "skvbc_fullnode exitting"); + break; + } else { + std::this_thread::sleep_for(std::chrono::seconds(1)); + } + } +} +} // namespace concord::kvbc::test + +namespace { +static void signal_handler(int signal_num) { + LOG_INFO(GL, "Program received signal " << signal_num); + concord::kvbc::test::timeToExit = true; +} +} // namespace + +int main(int argc, char** argv) { + struct sigaction sa; + LOG_INFO(GL, "skvbc_fullnode (concord-bft tester fullnode) starting..."); + memset(&sa, 0, sizeof(sa)); + sa.sa_handler = signal_handler; + sigfillset(&sa.sa_mask); + sigaction(SIGINT, &sa, NULL); + sigaction(SIGTERM, &sa, NULL); + + try { + concord::kvbc::test::run_fullnode(argc, argv); + } catch (const std::exception& e) { + LOG_FATAL(GL, "exception: " << e.what()); + } + LOG_INFO(GL, "skvbc_fullnode (concord-bft tester fullnode) shutting down..."); + return 0; +} \ No newline at end of file diff --git a/tests/simpleKVBC/TesterReplica/CMakeLists.txt b/tests/simpleKVBC/TesterReplica/CMakeLists.txt index a7431eb68d..c4dbe43599 100644 --- a/tests/simpleKVBC/TesterReplica/CMakeLists.txt +++ b/tests/simpleKVBC/TesterReplica/CMakeLists.txt @@ -1,14 +1,26 @@ cmake_minimum_required (VERSION 3.2) project(skvbc_replica VERSION 0.1.0.0 LANGUAGES CXX) -add_executable(skvbc_replica main.cpp - internalCommandsHandler.cpp - setup.cpp - WrapCommunication.cpp - strategy/ShufflePrePrepareMsgStrategy.cpp - strategy/StrategyUtils.cpp - strategy/MangledPreProcessResultMsgStrategy.cpp - strategy/CorruptCheckpointMsgStrategy.cpp) +add_library(skvbc_replica_lib + internalCommandsHandler.cpp + setup.cpp + WrapCommunication.cpp + strategy/ShufflePrePrepareMsgStrategy.cpp + strategy/StrategyUtils.cpp + strategy/MangledPreProcessResultMsgStrategy.cpp + strategy/CorruptCheckpointMsgStrategy.cpp) + +target_link_libraries(skvbc_replica_lib PUBLIC kvbc corebft threshsign util concord-crypto test_config_lib stdc++fs skvbc_messages_cmf) + +target_include_directories(skvbc_replica_lib PUBLIC ..) +target_include_directories(skvbc_replica_lib PUBLIC ../..) +target_include_directories(skvbc_replica_lib PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) +target_include_directories(skvbc_replica_lib PUBLIC ${libkvbc_SOURCE_DIR}/include) +target_include_directories(skvbc_replica_lib PRIVATE ${bftengine_SOURCE_DIR}/src/preprocessor) + +add_executable(skvbc_replica main.cpp) + +target_link_libraries(skvbc_replica PUBLIC skvbc_replica_lib) if(${USE_COMM_PLAIN_TCP}) target_compile_definitions(skvbc_replica PUBLIC USE_COMM_PLAIN_TCP) @@ -22,12 +34,7 @@ if(BUILD_ROCKSDB_STORAGE) target_compile_definitions(skvbc_replica PUBLIC "USE_ROCKSDB=1") endif() -target_link_libraries(skvbc_replica PUBLIC kvbc corebft threshsign util concord-crypto test_config_lib stdc++fs skvbc_messages_cmf) - target_include_directories(skvbc_replica PUBLIC ..) target_include_directories(skvbc_replica PUBLIC ../..) target_include_directories(skvbc_replica PUBLIC ${libkvbc_SOURCE_DIR}/include) -target_include_directories(skvbc_replica PRIVATE ${bftengine_SOURCE_DIR}/src/preprocessor) - - - +target_include_directories(skvbc_replica PRIVATE ${bftengine_SOURCE_DIR}/src/preprocessor) \ No newline at end of file diff --git a/tests/simpleKVBC/TesterReplica/setup.cpp b/tests/simpleKVBC/TesterReplica/setup.cpp index af8e1620df..7dddf5e30c 100644 --- a/tests/simpleKVBC/TesterReplica/setup.cpp +++ b/tests/simpleKVBC/TesterReplica/setup.cpp @@ -139,6 +139,7 @@ std::unique_ptr TestSetup::ParseArgs(int argc, char** argv) { {"blockchain-version", optional_argument, 0, 'V'}, {"enable-db-checkpoint", required_argument, 0, 'h'}, {"use-unified-certs", optional_argument, 0, 'U'}, + {"fn-cert-dir", optional_argument, 0, 'C'}, // direct options - assign directly ro a non-null flag {"publish-master-key-on-startup", no_argument, (int*)&replicaConfig.publishReplicasMasterKeyOnStartup, 1}, @@ -357,6 +358,8 @@ std::unique_ptr TestSetup::ParseArgs(int argc, char** argv) { } } + LOG_INFO(logger, "\nRead all the arguments \n"); + replicaConfig.singleSignatureScheme = singleSignatureScheme; if (keysFilePrefix.empty()) throw std::runtime_error("missing --key-file-prefix"); @@ -428,6 +431,7 @@ std::unique_ptr TestSetup::ParseArgs(int argc, char** argv) { persistMode == PersistencyMode::RocksDB, s3ConfigFile, logPropsFile, + certRootPath, cronEntryNumberOfExecutes, addAllKeysAsPublic != 0)); setup->sm_ = sm_; diff --git a/tests/simpleKVBC/TesterReplica/setup.hpp b/tests/simpleKVBC/TesterReplica/setup.hpp index ebe9d0f7f9..490a9bb281 100644 --- a/tests/simpleKVBC/TesterReplica/setup.hpp +++ b/tests/simpleKVBC/TesterReplica/setup.hpp @@ -45,9 +45,11 @@ class TestSetup { logging::Logger GetLogger() { return logger_; } const bool UsePersistentStorage() const { return usePersistentStorage_; } std::string getLogPropertiesFile() { return logPropsFile_; } + std::string getCertsRootPath() { return certRootPath_; } std::shared_ptr GetPerformanceManager() { return pm_; } std::optional GetCronEntryNumberOfExecutes() const { return cronEntryNumberOfExecutes_; } bool AddAllKeysAsPublic() const { return addAllKeysAsPublic_; } + bool getFNExecuteEnable() const { return fn_execute_enable_; } static inline constexpr auto kCronTableComponentId = 42; static inline constexpr auto kTickGeneratorPeriod = std::chrono::seconds{1}; @@ -60,6 +62,7 @@ class TestSetup { bool usePersistentStorage, const std::string& s3ConfigFile, const std::string& logPropsFile, + const std::string& certRootPath, const std::optional& cronEntryNumberOfExecutes, bool addAllKeysAsPublic) : replicaConfig_(config), @@ -69,6 +72,7 @@ class TestSetup { usePersistentStorage_(usePersistentStorage), s3ConfigFile_(s3ConfigFile), logPropsFile_(logPropsFile), + certRootPath_(certRootPath), pm_{std::make_shared()}, cronEntryNumberOfExecutes_{cronEntryNumberOfExecutes}, addAllKeysAsPublic_{addAllKeysAsPublic} {} @@ -88,10 +92,12 @@ class TestSetup { bool usePersistentStorage_; std::string s3ConfigFile_; std::string logPropsFile_; + std::string certRootPath_; std::shared_ptr pm_ = nullptr; std::optional cronEntryNumberOfExecutes_; std::shared_ptr sm_; bool addAllKeysAsPublic_{false}; + bool fn_execute_enable_{false}; }; } // namespace concord::kvbc diff --git a/tests/simpleTest/CMakeLists.txt b/tests/simpleTest/CMakeLists.txt index aa3d68abc9..e2badb628a 100644 --- a/tests/simpleTest/CMakeLists.txt +++ b/tests/simpleTest/CMakeLists.txt @@ -10,7 +10,6 @@ add_executable(simpleTest_client ${simpleTest_client_source_files} ) target_include_directories(simpleTest_client PRIVATE include - ${CMAKE_SOURCE_DIR}/util/include ../config ${concord_bft_tools_SOURCE_DIR} ${bftengine_SOURCE_DIR}/include diff --git a/tools/GenerateConcordKeys.cpp b/tools/GenerateConcordKeys.cpp index 5f0b46fecd..ec90e34bc1 100644 --- a/tools/GenerateConcordKeys.cpp +++ b/tools/GenerateConcordKeys.cpp @@ -97,6 +97,7 @@ int main(int argc, char** argv) { ReplicaConfig& config = ReplicaConfig::instance(); uint16_t n = 0; uint16_t ro = 0; + uint16_t fn = 0; std::string outputPrefix; std::string defaultSysType = "UninitializedCryptoSystem"; @@ -128,6 +129,9 @@ int main(int argc, char** argv) { } else if (option == "-r") { if (i >= argc - 1) throw std::runtime_error("Expected an argument to -r"); ro = parse(argv[i++ + 1], "-r"); + } else if (option == "-fn") { + if (i >= argc - 1) throw std::runtime_error("Expected an argument to -fn"); + fn = parse(argv[i++ + 1], "-fn"); } else if (option == "-o") { if (i >= argc - 1) throw std::runtime_error("Expected an argument to -o"); outputPrefix = argv[i++ + 1]; @@ -176,7 +180,7 @@ int main(int argc, char** argv) { std::vector> replicaKeyPairs; - for (uint16_t i = 0; i < n + ro; ++i) { + for (uint16_t i = 0; i < n + ro + fn; ++i) { if (ReplicaConfig::instance().replicaMsgSigningAlgo == SignatureAlgorithm::EdDSA) { replicaKeyPairs.push_back(generateEdDSAKeyPair()); } @@ -193,14 +197,20 @@ int main(int argc, char** argv) { for (uint16_t i = 0; i < n; ++i) { config.replicaId = i; config.replicaPrivateKey = replicaKeyPairs[i].first; - outputReplicaKeyfile(n, ro, config, outputPrefix + std::to_string(i), &cryptoSys); + outputReplicaKeyfile(n, ro, fn, config, outputPrefix + std::to_string(i), &cryptoSys); } for (uint16_t i = n; i < n + ro; ++i) { config.isReadOnly = true; config.replicaId = i; config.replicaPrivateKey = replicaKeyPairs[i].first; - outputReplicaKeyfile(n, ro, config, outputPrefix + std::to_string(i)); + outputReplicaKeyfile(n, ro, fn, config, outputPrefix + std::to_string(i)); + } + + for (uint16_t i = n + ro; i < n + ro + fn; ++i) { + config.replicaId = i; + config.replicaPrivateKey = replicaKeyPairs[i].first; + outputReplicaKeyfile(n, ro, fn, config, outputPrefix + std::to_string(i)); } } catch (std::exception& e) { std::cerr << "Exception: " << e.what() << std::endl; diff --git a/tools/KeyfileIOUtils.cpp b/tools/KeyfileIOUtils.cpp index 003803c6f8..ca5c17d3ae 100644 --- a/tools/KeyfileIOUtils.cpp +++ b/tools/KeyfileIOUtils.cpp @@ -30,6 +30,7 @@ using concord::crypto::Ed25519PublicKeyByteSize; void outputReplicaKeyfile(uint16_t numReplicas, uint16_t numRoReplicas, + uint16_t numFn, ReplicaConfig& config, const std::string& outputFilename, Cryptosystem* commonSys) { diff --git a/tools/KeyfileIOUtils.hpp b/tools/KeyfileIOUtils.hpp index 54ec9b2451..fcfc0bc70f 100644 --- a/tools/KeyfileIOUtils.hpp +++ b/tools/KeyfileIOUtils.hpp @@ -38,6 +38,7 @@ */ void outputReplicaKeyfile(uint16_t numReplicas, uint16_t numRoReplicas, + uint16_t numFn, bftEngine::ReplicaConfig& config, const std::string& outputFilename, Cryptosystem* commonSys = nullptr);