Merge master into fix-get-validators

Signed-off-by: cyc60 <[email protected]>

.env.example

@@ -44,6 +44,9 @@ CONSENSUS_ENDPOINTS=http://localhost:3500
 # The password file name should be the same as the keystore file name, but with .txt extension.
 # KEYSTORES_PASSWORD_DIR=/home/user/.stakewise/${VAULT_CONTRACT_ADDRESS}/keystores
 
+# URL to the remote signer. Default is None - using local keystores.
+# REMOTE_SIGNER_URL=http://remote-signer:9000
+
 # Path to the deposit_data.json file
 # Default is ${DATA_DIR}/${VAULT_CONTRACT_ADDRESS}/deposit_data.json
 # DEPOSIT_DATA_FILE=/home/user/.stakewise/${VAULT_CONTRACT_ADDRESS}/deposit_data.json

README.md

@@ -269,14 +269,88 @@ export $(grep -v '^#' .env | xargs)
 ./operator start
 ```
 
-## Misc commnads
+## Remote signer
+
+You may not want the operator service to have direct access to the validator
+keys. Validator keystores do not need to be present directly in the operator.
+The operator can query a remote signer to get signatures for validator
+exit messages. Because the validator exit signatures are split up and
+shared among oracles, the validator exit message needs to be signed by
+specific shares of the validator private key.
+
+These key shares therefore need to be present in your remote signer.
+
+### Remote signer setup
+
+This command will split up the private keys in the keystores directory
+into private key shares. The resulting private key shares are
+then imported to the remote signer. Local keystores are removed
+as a result of this command since they no longer need to be present.
+
+Notes:
+- You will need to run this command every time the oracle set
+  changes, or the threshold needed to recover exit signatures
+  (`exit_signature_recover_threshold`) changes.
+- In order to regenerate key shares, make sure to
+  adjust the `mnemonic_next_index` value in the vault config.json
+  to 0, then run the `create-keys` command, generating the full keystores
+  for all your validators. Next, run the `remote-signer-setup` command
+  to regenerate and import the new key shares for all your validators
+  into the remote signer.
+  You can remove the previously generated private key shares from the
+  remote signer, they will not be used anymore. This can optionally be
+  done by the setup command automatically by using the
+  `--remove-existing-keys` flag.
+
+```bash
+./operator remote-signer-setup \
+ --vault=0x3320ad928c20187602a2b2c04eeaa813fa899468 \
+ --remote-signer-url=http://signer:9000
+```
+
+```
+Successfully generated 11 key shares for 1 private key(s)!
+Successfully imported 11 key shares into remote signer.
+Removed keystores from local filesystem.
+Done. Successfully configured operator to use remote signer for 1 public key(s)!
+```
+
+#### `remote-signer-setup` options
+
+- `--vault` - The vault address.
+- `--remote-signer-url` - The base URL of the remote signer, e.g. http://signer:9000
+- `--remove-existing-keys` - Include this flag to remove any keys present in the signer that are not needed by the operator.
+  Can be used to remove outdated keyshares from the remote signer when the set of oracles changes,
+  see note above.
+- `--data-dir` - Path where the vault data is stored. Default is ~/.stakewise.
+- `--keystores-dir` - The directory with validator keys in the EIP-2335 standard.
+- `--execution-endpoints` - Comma separated list of API endpoints for execution nodes.
+- `--verbose` - Enable debug mode. Default is false.
+
+### Running the operator
+
+Provide the operator with the URL to your remote signer instance
+using the `--remote-signer-url` flag:
+
+```bash
+./operator start --remote-signer-url=http://remote-signer:9000 ...
+```
+
+You should see a message similar to this one after starting the operator:
+
+```
+Using remote signer at http://remote-signer:9000 for 10 public keys
+```
+
+
+## Misc commands
 
 ### Validators voluntary exit
 
 Performs a voluntary exit for active vault validators.
 
 ```bash
-./operator validator-exit
+./operator validators-exit
 ```
 
 ```sh
@@ -286,13 +360,14 @@ Are you sure you want to exit 3 validators with indexes: 513571, 513572, 513861?
 Validators 513571, 513572, 513861 exits successfully initiated
 ```
 
-#### `validator-exit` options
+#### `validators-exit` options
 
 - `--network` - The network of your vault.
 - `--vault` - The vault address.
 - `--consensus-endpoints` - Comma separated list of API endpoints for consensus nodes.
 - `--count` - The number of validators to exit. By default, command will force exit all active vault validators.
 - `--data-dir` - Path where the vault data is stored. Default is ~/.stakewise.
+- `--remote-signer-url` - URL to the remote signer instance.
 - `--verbose` - Enable debug mode. Default is false.
 
 ### Update vault deposit data

pyproject.toml

@@ -37,6 +37,7 @@ flake8-datetime-utcnow-plugin = "==0.1.2"
 flake8-print = "==5.0.0"
 types-pyyaml = "==6.0.12.11"
 coverage = "==7.3.1"
+aioresponses = "^0.7.4"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]
@@ -48,14 +49,16 @@ skips = ["B608"]
 
 [tool.pylint."pre-commit-hook"]
 disable = ["C0103", "C0114", "C0115", "C0116", "W0511", "W0703"]
-ignore-paths=["src/.*/tests/.*"]
+ignore-paths=["src/.*/tests/.*", "src/test_fixtures/.*"]
+ignore=["conftest.py"]
 
 [tool.pylint."BASIC"]
 good-names = ["db", "i", "w3"]
 ignored-modules=["milagro_bls_binding"]
 
 [tool.flake8]
 extend-ignore = ["E203", "E501"]  # line length will be checked by pylint
+exclude = ["conftest.py"]
 
 [tool.mypy]
 ignore_missing_imports = true

src/commands/create_keys.py

@@ -83,7 +83,7 @@ def create_keys(
 
     _export_keystores(
         credentials=credentials,
-        keystores_dir=str(keystores_dir),
+        keystores_dir=keystores_dir,
         password_file=str(password_file),
         per_keystore_password=per_keystore_password,
         pool_size=pool_size,
@@ -126,12 +126,13 @@ def _export_deposit_data_json(
 
 def _export_keystores(
     credentials: list[Credential],
-    keystores_dir: str,
+    keystores_dir: Path,
     password_file: str,
     per_keystore_password: bool,
     pool_size: int | None = None,
 ) -> None:
-    makedirs(path.abspath(keystores_dir), exist_ok=True)
+    keystores_dir.mkdir(exist_ok=True)
+
     if not per_keystore_password:
         password = get_or_create_password_file(password_file)
     with click.progressbar(
@@ -145,7 +146,7 @@ def _export_keystores(
                 cred.save_signing_keystore,
                 kwds={
                     'password': generate_password() if per_keystore_password else password,
-                    'folder': keystores_dir,
+                    'folder': str(keystores_dir),
                     'per_keystore_password': per_keystore_password,
                 },
                 callback=lambda x: progress_bar.update(1),
@@ -154,4 +155,5 @@ def _export_keystores(
         ]
 
         for result in results:
-            result.wait()
+            # Use result.get() to reraise exceptions
+            result.get()