Support for hashi vault structure with prefixed keys, 1 secret per key (

#439)

* Support for hashi vault structure with prefixed keys

* Code review feedback

.env.example

@@ -58,6 +58,12 @@ CONSENSUS_ENDPOINTS=http://localhost:3500
 # Internal structure of the secret must hold public validator keys in hex form without 0x as
 # secret keys, and signing keys in hex form without 0x prefix  as secret vault.
 # HASHI_VAULT_KEY_PATH=path/inside/hashi/vault/k/v/engine
+# A prefix in the K/V secret engine common for a group of signing keys.
+# Internal structure of keys under prefix must be as follows:
+# <prefix>/<public_key_hex_value_without_0x> -- {"<any_key>": "<secret_key_hex_value_without_0x>"}
+# "<any_key>" can be any key value like 'value' or 'key', the public key will be discovered
+# from the prefix anyway.
+# HASHI_VAULT_KEY_PREFIX=path/inside/hashi/vault/k/v/engine
 
 # Path to the deposit_data.json file
 # Default is ${DATA_DIR}/${VAULT_CONTRACT_ADDRESS}/deposit_data.json

src/commands/start.py

@@ -183,6 +183,12 @@
     multiple=True,
     help='Key path(s) in the K/V secret engine where validator signing keys are stored.',
 )
+@click.option(
+    '--hashi-vault-key-prefix',
+    envvar='HASHI_VAULT_KEY_PREFIX',
+    multiple=True,
+    help='Key prefix(es) in the K/V secret engine under which validator signing keys are stored.',
+)
 @click.option(
     '--hashi-vault-parallelism',
     envvar='HASHI_VAULT_PARALLELISM',
@@ -244,6 +250,7 @@ def start(
     keystores_password_file: str | None,
     remote_signer_url: str | None,
     hashi_vault_key_path: list[str] | None,
+    hashi_vault_key_prefix: list[str] | None,
     hashi_vault_token: str | None,
     hashi_vault_url: str | None,
     hashi_vault_parallelism: int,
@@ -278,6 +285,7 @@ def start(
         remote_signer_url=remote_signer_url,
         hashi_vault_token=hashi_vault_token,
         hashi_vault_key_paths=hashi_vault_key_path,
+        hashi_vault_key_prefixes=hashi_vault_key_prefix,
         hashi_vault_parallelism=hashi_vault_parallelism,
         hashi_vault_url=hashi_vault_url,
         hot_wallet_file=hot_wallet_file,

src/commands/validators_exit.py

@@ -98,6 +98,12 @@ class ValidatorExit:
     envvar='HASHI_VAULT_KEY_PATH',
     help='Key path in the K/V secret engine where validator signing keys are stored.',
 )
+@click.option(
+    '--hashi-vault-key-prefix',
+    envvar='HASHI_VAULT_KEY_PREFIX',
+    multiple=True,
+    help='Key prefix(es) in the K/V secret engine under which validator signing keys are stored.',
+)
 @click.option(
     '--hashi-vault-parallelism',
     envvar='HASHI_VAULT_PARALLELISM',
@@ -136,6 +142,7 @@ def validators_exit(
     consensus_endpoints: str,
     remote_signer_url: str,
     hashi_vault_key_path: list[str] | None,
+    hashi_vault_key_prefix: list[str] | None,
     hashi_vault_token: str | None,
     hashi_vault_url: str | None,
     hashi_vault_engine_name: str,
@@ -159,6 +166,7 @@ def validators_exit(
         remote_signer_url=remote_signer_url,
         hashi_vault_token=hashi_vault_token,
         hashi_vault_key_paths=hashi_vault_key_path,
+        hashi_vault_key_prefixes=hashi_vault_key_prefix,
         hashi_vault_url=hashi_vault_url,
         hashi_vault_engine_name=hashi_vault_engine_name,
         hashi_vault_parallelism=hashi_vault_parallelism,

src/config/settings.py

@@ -53,6 +53,7 @@ class Settings(metaclass=Singleton):
     remote_signer_use_deposit_data: bool
     dappnode: bool = False
     hashi_vault_key_paths: list[str] | None
+    hashi_vault_key_prefixes: list[str] | None
     hashi_vault_url: str | None
     hashi_vault_engine_name: str
     hashi_vault_token: str | None
@@ -116,6 +117,7 @@ def set(
         remote_signer_url: str | None = None,
         dappnode: bool = False,
         hashi_vault_key_paths: list[str] | None = None,
+        hashi_vault_key_prefixes: list[str] | None = None,
         hashi_vault_url: str | None = None,
         hashi_vault_engine_name: str = DEFAULT_HASHI_VAULT_ENGINE_NAME,
         hashi_vault_token: str | None = None,
@@ -183,6 +185,7 @@ def set(
         self.hashi_vault_url = hashi_vault_url
         self.hashi_vault_engine_name = hashi_vault_engine_name
         self.hashi_vault_key_paths = hashi_vault_key_paths
+        self.hashi_vault_key_prefixes = hashi_vault_key_prefixes
         self.hashi_vault_token = hashi_vault_token
         self.hashi_vault_parallelism = hashi_vault_parallelism
 

src/test_fixtures/hashi_vault.py

@@ -34,6 +34,26 @@ def mocked_hashi_vault(
         '8bc90a3110cf2b1ebaf8f5367bbfec1066797fca1f71ddbbf4f8f37ef74064404a78c31284c571656b7cb6efa29445ab': '56336628453e51cb9158da0651ea27dcb297eacdbd5cffdf0ea9d65fa154c327',
     }
 
+    # Generated via
+    # eth-staking-smith existing-mnemonic \
+    #   --chain holesky \
+    #   --num_validators 2 \
+    #   --mnemonic 'route flight verb churn work creek crane hole obscure young shaft area bird border refuse usage flash engage burden retreat drama bamboo profit sense'
+    _hashi_vault_prefixed_pk_sk_mapping1 = {
+        '8b09379ca969e8283a42a09285f430e8bd58c70bb33b44397ae81dac01b1403d0f631f156d211b6931a1c6284e2e469c': '5d88e114821bf871f321399d99fe58cb24d6434b416f112e8e46077e05399dc0',
+        '8979806d4e5d841758868b208df0dd961c12a0cf044e2de1d18e269ca0ad0308672be2f71d3d5606834764fe5b1d0bc4': '01352aec5cadb78eba6f716570d28b40f24b96c522dac535bc81375ceb54bf0b',
+    }
+
+    # Generated via
+    # eth-staking-smith existing-mnemonic \
+    #   --chain holesky \
+    #   --num_validators 2 \
+    #   --mnemonic 'lion toilet tooth guess excuse wise amateur evolve moment damage curtain image zebra dress drill circle luggage seminar similar symptom happy floor govern gravity'
+    _hashi_vault_prefixed_pk_sk_mapping2 = {
+        '859f3fc64e32a1e95aadc7a7ec35207f6305951e7dafacf9252aaa9edef3d1edf74d268041cb59ca64e703ba064890be': '17dd0ad25bd239092bfa47b53c94d7eec2f3621a99ffafc28cd3c6b25a72d7f9',
+        'a60dcf78a344afc297b4917f76b5b387924153182390361d5199c3455299d67fbb932b77943ffe5477150304f3cb600f': '4f768f0b9589fdff6e8371dd268d8d78b97bf968f6fc469657332cff48b1dea4',
+    }
+
     def _mocked_secret_path(data, url, **kwargs) -> CallbackResult:
         return CallbackResult(
             status=200,
@@ -46,13 +66,25 @@ def _mocked_secret_path(data, url, **kwargs) -> CallbackResult:
             ),  # type: ignore
         )
 
+    def _mocked_secrets_list(data, url, **kwargs) -> CallbackResult:
+        return CallbackResult(
+            status=200,
+            body=json.dumps(
+                dict(
+                    data=dict(
+                        keys=data,
+                    )
+                )
+            ),  # type: ignore
+        )
+
     def _mocked_error_path(url, **kwargs) -> CallbackResult:
         return CallbackResult(
             status=200, body=json.dumps(dict(errors=list('token not provided')))  # type: ignore
         )
 
     with aioresponses() as m:
-        # Mocked signing keys endpoints
+        # Mocked bundled signing keys endpoints
         m.get(
             f'{hashi_vault_url}/v1/secret/data/ethereum/signing/keystores',
             callback=partial(_mocked_secret_path, _hashi_vault_pk_sk_mapping_1),
@@ -68,7 +100,38 @@ def _mocked_error_path(url, **kwargs) -> CallbackResult:
             callback=partial(_mocked_secret_path, _hashi_vault_pk_sk_mapping_2),
             repeat=True,
         )
-        # Mocked signing keys endpoints with custom engine name
+        # Mocked prefixed signing keys endpoints
+        m.add(
+            f'{hashi_vault_url}/v1/secret/metadata/ethereum/signing/prefixed1',
+            callback=partial(
+                _mocked_secrets_list, list(_hashi_vault_prefixed_pk_sk_mapping1.keys())
+            ),
+            repeat=True,
+            method='LIST',
+        )
+        for _pk, _sk in _hashi_vault_prefixed_pk_sk_mapping1.items():
+            m.get(
+                f'{hashi_vault_url}/v1/secret/data/ethereum/signing/prefixed1/{_pk}',
+                callback=partial(_mocked_secret_path, {'value': _sk}),
+                repeat=True,
+            )
+
+        m.add(
+            f'{hashi_vault_url}/v1/secret/metadata/ethereum/signing/prefixed2',
+            callback=partial(
+                _mocked_secrets_list, list(_hashi_vault_prefixed_pk_sk_mapping2.keys())
+            ),
+            repeat=True,
+            method='LIST',
+        )
+        for _pk, _sk in _hashi_vault_prefixed_pk_sk_mapping2.items():
+            m.get(
+                f'{hashi_vault_url}/v1/secret/data/ethereum/signing/prefixed2/{_pk}',
+                callback=partial(_mocked_secret_path, {'value': _sk}),
+                repeat=True,
+            )
+
+        # Mocked bundled signing keys endpoints with custom engine name
         m.get(
             f'{hashi_vault_url}/v1/custom/data/ethereum/signing/keystores',
             callback=partial(_mocked_secret_path, _hashi_vault_pk_sk_mapping_1),