From d6234dae4c0b9aaf7cf094870e459af1917ca616 Mon Sep 17 00:00:00 2001 From: antares-sw <23400824+antares-sw@users.noreply.github.com> Date: Fri, 25 Aug 2023 11:28:22 +0300 Subject: [PATCH] Merge key-manager to operator --- .gitignore | 1 + poetry.lock | 113 +++++++++++++------- pyproject.toml | 3 + src/commands/sync_validator.py | 183 ++++++++++++++++++++++++++++++++ src/commands/sync_web3signer.py | 99 +++++++++++++++++ src/commands/update_db.py | 143 +++++++++++++++++++++++++ src/common/contrib.py | 15 +++ src/common/validators.py | 18 ++++ src/key_manager/__init__.py | 0 src/key_manager/database.py | 90 ++++++++++++++++ src/key_manager/encryptor.py | 41 +++++++ src/key_manager/typings.py | 10 ++ src/main.py | 6 ++ 13 files changed, 681 insertions(+), 41 deletions(-) create mode 100644 src/commands/sync_validator.py create mode 100644 src/commands/sync_web3signer.py create mode 100644 src/commands/update_db.py create mode 100644 src/key_manager/__init__.py create mode 100644 src/key_manager/database.py create mode 100644 src/key_manager/encryptor.py create mode 100644 src/key_manager/typings.py diff --git a/.gitignore b/.gitignore index 897dea07..774a2027 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ GIT_SHA build database *.db +.DS_Store diff --git a/poetry.lock b/poetry.lock index 56474023..4f02af06 100644 --- a/poetry.lock +++ b/poetry.lock @@ -162,13 +162,13 @@ wrapt = {version = ">=1.11,<2", markers = "python_version < \"3.11\""} [[package]] name = "async-timeout" -version = "4.0.2" +version = "4.0.3" description = "Timeout context manager for asyncio programs" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" files = [ - {file = "async-timeout-4.0.2.tar.gz", hash = "sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15"}, - {file = "async_timeout-4.0.2-py3-none-any.whl", hash = "sha256:8ca1e4fcf50d07413d66d1a5e416e42cfdf5851c981d679a09851a6853383b3c"}, + {file = "async-timeout-4.0.3.tar.gz", hash = "sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f"}, + {file = "async_timeout-4.0.3-py3-none-any.whl", hash = "sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028"}, ] [[package]] @@ -486,13 +486,13 @@ pycparser = "*" [[package]] name = "cfgv" -version = "3.3.1" +version = "3.4.0" description = "Validate configuration and produce human readable error messages." optional = false -python-versions = ">=3.6.1" +python-versions = ">=3.8" files = [ - {file = "cfgv-3.3.1-py2.py3-none-any.whl", hash = "sha256:c6a0883f3917a037485059700b9e75da2464e6c27051014ad85ba6aaa5884426"}, - {file = "cfgv-3.3.1.tar.gz", hash = "sha256:f5a830efb9ce7a445376bb66ec94c638a9787422f96264c98edc6bdeed8ab736"}, + {file = "cfgv-3.4.0-py2.py3-none-any.whl", hash = "sha256:b7265b1f29fd3316bfcd2b330d63d024f2bfd8bcb8b0272f8e19a504856c48f9"}, + {file = "cfgv-3.4.0.tar.gz", hash = "sha256:e52591d4c5f5dead8e0f673fb16db7949d2cfb3f7da4582893288f0ded8fe560"}, ] [[package]] @@ -980,13 +980,13 @@ test = ["hypothesis (>=4.43.0)", "mypy (==0.971)", "pytest (>=7.0.0)", "pytest-x [[package]] name = "exceptiongroup" -version = "1.1.2" +version = "1.1.3" description = "Backport of PEP 654 (exception groups)" optional = false python-versions = ">=3.7" files = [ - {file = "exceptiongroup-1.1.2-py3-none-any.whl", hash = "sha256:e346e69d186172ca7cf029c8c1d16235aa0e04035e5750b4b95039e65204328f"}, - {file = "exceptiongroup-1.1.2.tar.gz", hash = "sha256:12c3e887d6485d16943a309616de20ae5582633e0a2eda17f4e10fd61c1e8af5"}, + {file = "exceptiongroup-1.1.3-py3-none-any.whl", hash = "sha256:343280667a4585d195ca1cf9cef84a4e178c4b6cf2274caef9859782b567d5e3"}, + {file = "exceptiongroup-1.1.3.tar.gz", hash = "sha256:097acd85d473d75af5bb98e41b61ff7fe35efe6675e4f9370ec6ec5126d160e9"}, ] [package.extras] @@ -1200,13 +1200,13 @@ test = ["eth-utils (>=1.0.1,<3)", "hypothesis (>=3.44.24,<=6.31.6)", "pytest (>= [[package]] name = "identify" -version = "2.5.26" +version = "2.5.27" description = "File identification library for Python" optional = false python-versions = ">=3.8" files = [ - {file = "identify-2.5.26-py2.py3-none-any.whl", hash = "sha256:c22a8ead0d4ca11f1edd6c9418c3220669b3b7533ada0a0ffa6cc0ef85cf9b54"}, - {file = "identify-2.5.26.tar.gz", hash = "sha256:7243800bce2f58404ed41b7c002e53d4d22bcf3ae1b7900c2d7aefd95394bf7f"}, + {file = "identify-2.5.27-py2.py3-none-any.whl", hash = "sha256:fdb527b2dfe24602809b2201e033c2a113d7bdf716db3ca8e3243f735dcecaba"}, + {file = "identify-2.5.27.tar.gz", hash = "sha256:287b75b04a0e22d727bc9a41f0d4f3c1bcada97490fa6eabb5b28f0e9097e733"}, ] [package.extras] @@ -1807,24 +1807,44 @@ twisted = ["twisted"] [[package]] name = "protobuf" -version = "4.24.0" +version = "4.24.1" description = "" optional = false python-versions = ">=3.7" files = [ - {file = "protobuf-4.24.0-cp310-abi3-win32.whl", hash = "sha256:81cb9c4621d2abfe181154354f63af1c41b00a4882fb230b4425cbaed65e8f52"}, - {file = "protobuf-4.24.0-cp310-abi3-win_amd64.whl", hash = "sha256:6c817cf4a26334625a1904b38523d1b343ff8b637d75d2c8790189a4064e51c3"}, - {file = "protobuf-4.24.0-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:ae97b5de10f25b7a443b40427033e545a32b0e9dda17bcd8330d70033379b3e5"}, - {file = "protobuf-4.24.0-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:567fe6b0647494845d0849e3d5b260bfdd75692bf452cdc9cb660d12457c055d"}, - {file = "protobuf-4.24.0-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:a6b1ca92ccabfd9903c0c7dde8876221dc7d8d87ad5c42e095cc11b15d3569c7"}, - {file = "protobuf-4.24.0-cp37-cp37m-win32.whl", hash = "sha256:a38400a692fd0c6944c3c58837d112f135eb1ed6cdad5ca6c5763336e74f1a04"}, - {file = "protobuf-4.24.0-cp37-cp37m-win_amd64.whl", hash = "sha256:5ab19ee50037d4b663c02218a811a5e1e7bb30940c79aac385b96e7a4f9daa61"}, - {file = "protobuf-4.24.0-cp38-cp38-win32.whl", hash = "sha256:e8834ef0b4c88666ebb7c7ec18045aa0f4325481d724daa624a4cf9f28134653"}, - {file = "protobuf-4.24.0-cp38-cp38-win_amd64.whl", hash = "sha256:8bb52a2be32db82ddc623aefcedfe1e0eb51da60e18fcc908fb8885c81d72109"}, - {file = "protobuf-4.24.0-cp39-cp39-win32.whl", hash = "sha256:ae7a1835721086013de193311df858bc12cd247abe4ef9710b715d930b95b33e"}, - {file = "protobuf-4.24.0-cp39-cp39-win_amd64.whl", hash = "sha256:44825e963008f8ea0d26c51911c30d3e82e122997c3c4568fd0385dd7bacaedf"}, - {file = "protobuf-4.24.0-py3-none-any.whl", hash = "sha256:82e6e9ebdd15b8200e8423676eab38b774624d6a1ad696a60d86a2ac93f18201"}, - {file = "protobuf-4.24.0.tar.gz", hash = "sha256:5d0ceb9de6e08311832169e601d1fc71bd8e8c779f3ee38a97a78554945ecb85"}, + {file = "protobuf-4.24.1-cp310-abi3-win32.whl", hash = "sha256:d414199ca605eeb498adc4d2ba82aedc0379dca4a7c364ff9bc9a179aa28e71b"}, + {file = "protobuf-4.24.1-cp310-abi3-win_amd64.whl", hash = "sha256:5906c5e79ff50fe38b2d49d37db5874e3c8010826f2362f79996d83128a8ed9b"}, + {file = "protobuf-4.24.1-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:970c701ee16788d74f3de20938520d7a0aebc7e4fff37096a48804c80d2908cf"}, + {file = "protobuf-4.24.1-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:fc361148e902949dcb953bbcb148c99fe8f8854291ad01107e4120361849fd0e"}, + {file = "protobuf-4.24.1-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:5d32363d14aca6e5c9e9d5918ad8fb65b091b6df66740ae9de50ac3916055e43"}, + {file = "protobuf-4.24.1-cp37-cp37m-win32.whl", hash = "sha256:df015c47d6855b8efa0b9be706c70bf7f050a4d5ac6d37fb043fbd95157a0e25"}, + {file = "protobuf-4.24.1-cp37-cp37m-win_amd64.whl", hash = "sha256:d4af4fd9e9418e819be30f8df2a16e72fbad546a7576ac7f3653be92a6966d30"}, + {file = "protobuf-4.24.1-cp38-cp38-win32.whl", hash = "sha256:302e8752c760549ed4c7a508abc86b25d46553c81989343782809e1a062a2ef9"}, + {file = "protobuf-4.24.1-cp38-cp38-win_amd64.whl", hash = "sha256:06437f0d4bb0d5f29e3d392aba69600188d4be5ad1e0a3370e581a9bf75a3081"}, + {file = "protobuf-4.24.1-cp39-cp39-win32.whl", hash = "sha256:0b2b224e9541fe9f046dd7317d05f08769c332b7e4c54d93c7f0f372dedb0b1a"}, + {file = "protobuf-4.24.1-cp39-cp39-win_amd64.whl", hash = "sha256:bd39b9094a4cc003a1f911b847ab379f89059f478c0b611ba1215053e295132e"}, + {file = "protobuf-4.24.1-py3-none-any.whl", hash = "sha256:55dd644adc27d2a624339332755fe077c7f26971045b469ebb9732a69ce1f2ca"}, + {file = "protobuf-4.24.1.tar.gz", hash = "sha256:44837a5ed9c9418ad5d502f89f28ba102e9cd172b6668bc813f21716f9273348"}, +] + +[[package]] +name = "psycopg2" +version = "2.9.7" +description = "psycopg2 - Python-PostgreSQL Database Adapter" +optional = false +python-versions = ">=3.6" +files = [ + {file = "psycopg2-2.9.7-cp310-cp310-win32.whl", hash = "sha256:1a6a2d609bce44f78af4556bea0c62a5e7f05c23e5ea9c599e07678995609084"}, + {file = "psycopg2-2.9.7-cp310-cp310-win_amd64.whl", hash = "sha256:b22ed9c66da2589a664e0f1ca2465c29b75aaab36fa209d4fb916025fb9119e5"}, + {file = "psycopg2-2.9.7-cp311-cp311-win32.whl", hash = "sha256:44d93a0109dfdf22fe399b419bcd7fa589d86895d3931b01fb321d74dadc68f1"}, + {file = "psycopg2-2.9.7-cp311-cp311-win_amd64.whl", hash = "sha256:91e81a8333a0037babfc9fe6d11e997a9d4dac0f38c43074886b0d9dead94fe9"}, + {file = "psycopg2-2.9.7-cp37-cp37m-win32.whl", hash = "sha256:d1210fcf99aae6f728812d1d2240afc1dc44b9e6cba526a06fb8134f969957c2"}, + {file = "psycopg2-2.9.7-cp37-cp37m-win_amd64.whl", hash = "sha256:e9b04cbef584310a1ac0f0d55bb623ca3244c87c51187645432e342de9ae81a8"}, + {file = "psycopg2-2.9.7-cp38-cp38-win32.whl", hash = "sha256:d5c5297e2fbc8068d4255f1e606bfc9291f06f91ec31b2a0d4c536210ac5c0a2"}, + {file = "psycopg2-2.9.7-cp38-cp38-win_amd64.whl", hash = "sha256:8275abf628c6dc7ec834ea63f6f3846bf33518907a2b9b693d41fd063767a866"}, + {file = "psycopg2-2.9.7-cp39-cp39-win32.whl", hash = "sha256:c7949770cafbd2f12cecc97dea410c514368908a103acf519f2a346134caa4d5"}, + {file = "psycopg2-2.9.7-cp39-cp39-win_amd64.whl", hash = "sha256:b6bd7d9d3a7a63faae6edf365f0ed0e9b0a1aaf1da3ca146e6b043fb3eb5d723"}, + {file = "psycopg2-2.9.7.tar.gz", hash = "sha256:f00cc35bd7119f1fed17b85bd1007855194dde2cbd8de01ab8ebb17487440ad8"}, ] [[package]] @@ -2000,13 +2020,13 @@ hook-testing = ["execnet (>=1.5.0)", "psutil", "pytest (>=2.7.3)"] [[package]] name = "pyinstaller-hooks-contrib" -version = "2023.6" +version = "2023.7" description = "Community maintained hooks for PyInstaller" optional = false python-versions = ">=3.7" files = [ - {file = "pyinstaller-hooks-contrib-2023.6.tar.gz", hash = "sha256:596a72009d8692b043e0acbf5e1b476d93149900142ba01845dded91a0770cb5"}, - {file = "pyinstaller_hooks_contrib-2023.6-py2.py3-none-any.whl", hash = "sha256:aa6d7d038814df6aa7bec7bdbebc7cb4c693d3398df858f6062957f0797d397b"}, + {file = "pyinstaller-hooks-contrib-2023.7.tar.gz", hash = "sha256:0c436a4c3506020e34116a8a7ddfd854c1ad6ddca9a8cd84500bd6e69c9e68f9"}, + {file = "pyinstaller_hooks_contrib-2023.7-py2.py3-none-any.whl", hash = "sha256:3c10df14c0f71ab388dfbf1625375b087e7330d9444cbfd2b310ba027fa0cff0"}, ] [[package]] @@ -2520,18 +2540,18 @@ tornado = ["tornado (>=5)"] [[package]] name = "setuptools" -version = "68.0.0" +version = "68.1.2" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "setuptools-68.0.0-py3-none-any.whl", hash = "sha256:11e52c67415a381d10d6b462ced9cfb97066179f0e871399e006c4ab101fc85f"}, - {file = "setuptools-68.0.0.tar.gz", hash = "sha256:baf1fdb41c6da4cd2eae722e135500da913332ab3f2f5c7d33af9b492acb5235"}, + {file = "setuptools-68.1.2-py3-none-any.whl", hash = "sha256:3d8083eed2d13afc9426f227b24fd1659489ec107c0e86cec2ffdde5c92e790b"}, + {file = "setuptools-68.1.2.tar.gz", hash = "sha256:3d4dfa6d95f1b101d695a6160a7626e15583af71a5f52176efa5d39a054d475d"}, ] [package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5,<=7.1.2)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] [[package]] @@ -2667,6 +2687,17 @@ files = [ {file = "toolz-0.12.0.tar.gz", hash = "sha256:88c570861c440ee3f2f6037c4654613228ff40c93a6c25e0eba70d17282c6194"}, ] +[[package]] +name = "types-pyyaml" +version = "6.0.12.11" +description = "Typing stubs for PyYAML" +optional = false +python-versions = "*" +files = [ + {file = "types-PyYAML-6.0.12.11.tar.gz", hash = "sha256:7d340b19ca28cddfdba438ee638cd4084bde213e501a3978738543e27094775b"}, + {file = "types_PyYAML-6.0.12.11-py3-none-any.whl", hash = "sha256:a461508f3096d1d5810ec5ab95d7eeecb651f3a15b71959999988942063bf01d"}, +] + [[package]] name = "types-requests" version = "2.31.0.2" @@ -2732,13 +2763,13 @@ files = [ [[package]] name = "virtualenv" -version = "20.24.2" +version = "20.24.3" description = "Virtual Python Environment builder" optional = false python-versions = ">=3.7" files = [ - {file = "virtualenv-20.24.2-py3-none-any.whl", hash = "sha256:43a3052be36080548bdee0b42919c88072037d50d56c28bd3f853cbe92b953ff"}, - {file = "virtualenv-20.24.2.tar.gz", hash = "sha256:fd8a78f46f6b99a67b7ec5cf73f92357891a7b3a40fd97637c27f854aae3b9e0"}, + {file = "virtualenv-20.24.3-py3-none-any.whl", hash = "sha256:95a6e9398b4967fbcb5fef2acec5efaf9aa4972049d9ae41f95e0972a683fd02"}, + {file = "virtualenv-20.24.3.tar.gz", hash = "sha256:e5c3b4ce817b0b328af041506a2a299418c98747c4b1e68cb7527e74ced23efc"}, ] [package.dependencies] @@ -3036,4 +3067,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.11" -content-hash = "5aa890447b8c88b93b7d0a4e13338662d28f99615a2685bcbb72a75847717753" +content-hash = "12ebffcf308cc8a753d80f1d12b991830e22993c8392f0f85d8d17489abb3917" diff --git a/pyproject.toml b/pyproject.toml index d6b3b10d..ea7b00c2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,8 @@ click = "==8.1.3" tomli = "~2" eciespy = "==0.3.13" prometheus-client = "==0.17.0" +psycopg2 = "==2.9.7" +pyyaml = "==6.0.1" [tool.poetry.group.dev.dependencies] pylint = "==2.16.2" @@ -34,6 +36,7 @@ pyinstaller = "==5.7.0" faker = "==18.10.1" flake8-datetime-utcnow-plugin = "==0.1.2" flake8-print = "==5.0.0" +types-pyyaml = "==6.0.12.11" [build-system] requires = ["poetry-core>=1.0.0"] diff --git a/src/commands/sync_validator.py b/src/commands/sync_validator.py new file mode 100644 index 00000000..2639bd60 --- /dev/null +++ b/src/commands/sync_validator.py @@ -0,0 +1,183 @@ +import json +from pathlib import Path + +import click +import yaml +from eth_typing import HexStr + +from src.common.validators import validate_db_uri, validate_eth_address +from src.key_manager.database import Database, check_db_connection + +VALIDATOR_DEFINITIONS_FILENAME = 'validator_definitions.yml' +SIGNER_KEYS_FILENAME = 'signer_keys.yml' +PROPOSER_CONFIG_FILENAME = 'proposer_config.json' + + +@click.option( + '--validator-index', + help='The validator index to generate the configuration files.', + prompt='Enter the validator index to generate the configuration files', + type=int, +) +@click.option( + '--total-validators', + help='The total number of validators connected to the web3signer.', + prompt='Enter the total number of validators connected to the web3signer', + type=click.IntRange(min=1), +) +@click.option( + '--db-url', + help='The database connection address.', + prompt="Enter the database connection string, ex. 'postgresql://username:pass@hostname/dbname'", + callback=validate_db_uri, +) +@click.option( + '--web3signer-endpoint', + help='The endpoint of the web3signer service.', + prompt='Enter the endpoint of the web3signer service', + type=str, +) +@click.option( + '--fee-recipient', + help='The recipient address for MEV & priority fees.', + prompt='Enter the recipient address for MEV & priority fees', + type=str, + callback=validate_eth_address, +) +@click.option( + '--disable-proposal-builder', + is_flag=True, + default=False, + help='Disable proposal builder for Teku and Prysm clients.', +) +@click.option( + '--output-dir', + required=False, + help='The directory to save configuration files. Defaults to ./data/configs.', + default='./data/configs', + type=click.Path(exists=False, file_okay=False, dir_okay=True), +) +@click.command( + help='Creates validator configuration files for Lighthouse, ' + 'Prysm, and Teku clients to sign data using keys from database.' +) +# pylint: disable-next=too-many-arguments,too-many-locals +def sync_validator( + validator_index: int, + total_validators: int, + db_url: str, + web3signer_endpoint: str, + fee_recipient: str, + disable_proposal_builder: bool, + output_dir: str, +) -> None: + check_db_connection(db_url) + check_validator_index(validator_index, total_validators) + + database = Database(db_url=db_url) + public_keys_count = database.fetch_public_keys_count() + + keys_per_validator = public_keys_count // total_validators + + start_index = keys_per_validator * validator_index + if validator_index == total_validators - 1: + end_index = public_keys_count + else: + end_index = start_index + keys_per_validator + + public_keys = database.fetch_public_keys_by_range(start_index=start_index, end_index=end_index) + + if not public_keys: + raise click.ClickException('Database does not contain in range') + + Path.mkdir(Path(output_dir), exist_ok=True, parents=True) + + # lighthouse + validator_definitions_filepath = str(Path(output_dir, VALIDATOR_DEFINITIONS_FILENAME)) + _generate_lighthouse_config( + public_keys=public_keys, + web3signer_url=web3signer_endpoint, + fee_recipient=fee_recipient, + filepath=validator_definitions_filepath, + ) + + # teku/prysm + signer_keys_filepath = str(Path(output_dir, SIGNER_KEYS_FILENAME)) + _generate_signer_keys_config(public_keys=public_keys, filepath=signer_keys_filepath) + + proposer_config_filepath = str(Path(output_dir, PROPOSER_CONFIG_FILENAME)) + _generate_proposer_config( + fee_recipient=fee_recipient, + proposal_builder_enabled=not disable_proposal_builder, + filepath=proposer_config_filepath, + ) + + click.clear() + click.secho( + f'Done. ' + f'Generated configs with {len(public_keys)} keys for validator #{validator_index}.\n' + f'Validator definitions for Lighthouse saved to {validator_definitions_filepath} file.\n' + f'Signer keys for Teku\\Prysm saved to {signer_keys_filepath} file.\n' + f'Proposer config for Teku\\Prysm saved to {proposer_config_filepath} file.\n', + bold=True, + fg='green', + ) + + +def _generate_lighthouse_config( + public_keys: list[HexStr], + web3signer_url: str, + fee_recipient: str, + filepath: str, +) -> None: + """ + Generate config for Lighthouse clients + """ + items = [ + { + 'enabled': True, + 'voting_public_key': public_key, + 'type': 'web3signer', + 'url': web3signer_url, + 'suggested_fee_recipient': fee_recipient, + } + for public_key in public_keys + ] + + with open(filepath, 'w', encoding='utf-8') as f: + yaml.dump(items, f, explicit_start=True) + + +def _generate_signer_keys_config(public_keys: list[HexStr], filepath: str) -> None: + """ + Generate config for Teku and Prysm clients + """ + keys = ','.join([f'"{public_key}"' for public_key in public_keys]) + config = f"""validators-external-signer-public-keys: [{keys}]""" + with open(filepath, 'w', encoding='utf-8') as f: + f.write(config) + + +def _generate_proposer_config( + fee_recipient: str, + proposal_builder_enabled: bool, + filepath: str, +) -> None: + """ + Generate proposal config for Teku and Prysm clients + """ + config = { + 'default_config': { + 'fee_recipient': fee_recipient, + 'builder': { + 'enabled': proposal_builder_enabled, + }, + }, + } + with open(filepath, 'w', encoding='utf-8') as f: + json.dump(config, f, ensure_ascii=False, indent=4) + + +def check_validator_index(validator_index, total_validators): + if not total_validators or total_validators <= validator_index: + raise click.BadParameter('validator index must be less than total validators') diff --git a/src/commands/sync_web3signer.py b/src/commands/sync_web3signer.py new file mode 100644 index 00000000..8c83160d --- /dev/null +++ b/src/commands/sync_web3signer.py @@ -0,0 +1,99 @@ +import glob +import os +from os import mkdir +from os.path import exists +from typing import List + +import click +import yaml +from eth_utils import add_0x_prefix +from web3 import Web3 +from web3.types import HexStr + +from src.common.contrib import is_lists_equal +from src.common.validators import validate_db_uri, validate_env_name +from src.key_manager.database import Database, check_db_connection +from src.key_manager.encryptor import Encryptor + +DECRYPTION_KEY_ENV = 'DECRYPTION_KEY' + + +@click.option( + '--db-url', + help='The database connection address.', + prompt="Enter the database connection string, ex. 'postgresql://username:pass@hostname/dbname'", + callback=validate_db_uri, +) +@click.option( + '--output-dir', + help='The folder where web3signer keystores will be saved.', + prompt='Enter the folder where web3signer keystores will be saved', + type=click.Path(exists=False, file_okay=False, dir_okay=True), +) +@click.option( + '--decryption-key-env', + help='The environment variable with the decryption key for private keys in the database.', + default=DECRYPTION_KEY_ENV, + callback=validate_env_name, +) +@click.command(help='Synchronizes web3signer private keys from the database') +# pylint: disable-next=too-many-locals +def sync_web3signer(db_url: str, output_dir: str, decryption_key_env: str) -> None: + """ + The command is running by the init container in web3signer pods. + Fetch and decrypt keys for web3signer and store them as keypairs in the output_dir. + """ + check_db_connection(db_url) + + database = Database(db_url=db_url) + keys_records = database.fetch_keys() + + # decrypt private keys + decryption_key = os.environ[decryption_key_env] + decryptor = Encryptor(decryption_key) + private_keys: List[str] = [] + for key_record in keys_records: + key = decryptor.decrypt(data=key_record.private_key, nonce=key_record.nonce) + key_hex = Web3.to_hex(int(key)) + # pylint: disable-next=unsubscriptable-object + key_hex = HexStr(key_hex[2:].zfill(64)) # pad missing leading zeros + private_keys.append(add_0x_prefix(key_hex)) + + if not exists(output_dir): + mkdir(output_dir) + + # check current keys + current_keys = [] + for filename in glob.glob(os.path.join(output_dir, '*.yaml')): + with open(filename, 'r', encoding='utf-8') as f: + content = yaml.safe_load(f.read()) + current_keys.append(content.get('privateKey')) + + if is_lists_equal(current_keys, private_keys): + click.secho( + 'Keys already synced to the last version.\n', + bold=True, + fg='green', + ) + return + + # save key files + for index, private_key in enumerate(private_keys): + filename = f'key_{index}.yaml' + with open(os.path.join(output_dir, filename), 'w', encoding='utf-8') as f: + f.write(_generate_key_file(private_key)) + + click.secho( + f'Web3Signer now uses {len(private_keys)} private keys.\n', + bold=True, + fg='green', + ) + + +def _generate_key_file(private_key: str) -> str: + item = { + 'type': 'file-raw', + 'keyType': 'BLS', + 'privateKey': private_key, + } + return yaml.dump(item) diff --git a/src/commands/update_db.py b/src/commands/update_db.py new file mode 100644 index 00000000..7ab5531b --- /dev/null +++ b/src/commands/update_db.py @@ -0,0 +1,143 @@ +import glob +import json +import os +from pathlib import Path + +import click +from py_ecc.bls import G2ProofOfPossession +from staking_deposit.key_handling.keystore import ScryptKeystore +from web3 import Web3 + +from src.common.contrib import bytes_to_str +from src.common.validators import validate_db_uri +from src.config.settings import DATA_DIR +from src.key_manager.database import Database, check_db_connection +from src.key_manager.encryptor import Encryptor +from src.key_manager.typings import DatabaseKeyRecord + +w3 = Web3() + + +@click.option( + '--vault', + prompt='Enter your vault address', + help='Vault address', + type=str, +) +@click.option( + '--keystores-dir', + required=False, + help='The directory with validator keys in the EIP-2335 standard.', + type=click.Path(exists=True, file_okay=False, dir_okay=True), +) +@click.option( + '--keystores-password-file', + required=False, + help='The path to file with password for encrypting the keystores.', + type=click.Path(exists=True, file_okay=True, dir_okay=False), +) +@click.option( + '--db-url', + help='The database connection address.', + prompt="Enter the database connection string, ex. 'postgresql://username:pass@hostname/dbname'", + callback=validate_db_uri, +) +@click.option( + '--encryption-key', + help='The key for encrypting database record. ' + 'If you are upload new keystores use the same encryption key.', + required=False, + prompt=False, +) +@click.option( + '--no-confirm', + is_flag=True, + default=False, + help='Skips confirmation messages when provided.', +) +@click.command(help='Encrypt and load validator keys from the keystores into the database.') +# pylint: disable-next=too-many-arguments,too-many-locals +def update_db( + vault: str, + keystores_dir: str | Path | None, + keystores_password_file: str | Path | None, + db_url: str, + encryption_key: str | None, + no_confirm: bool, +) -> None: + check_db_connection(db_url) + + vault_dir = DATA_DIR / vault + + keystores_dir = keystores_dir or vault_dir / 'keystores' + keystores_password_file = keystores_password_file or vault_dir / 'keystores' / 'password.txt' + + with open(str(keystores_password_file), 'r', encoding='utf-8') as f: + keystores_password = f.read().strip() + + private_keys = [] + + with click.progressbar( + glob.glob(os.path.join(str(keystores_dir), 'keystore-*.json')), + label='Loading keystores...\t\t', + show_percent=False, + show_pos=True, + ) as _keystore_files: + for filename in _keystore_files: + try: + keystore = ScryptKeystore.from_file(filename).decrypt(keystores_password) + private_keys.append(int.from_bytes(keystore, 'big')) + except (json.JSONDecodeError, KeyError) as e: + click.secho( + f'Failed to load keystore {filename}. Error: {str(e)}.', + fg='red', + ) + + database = Database( + db_url=db_url, + ) + encryptor = Encryptor(encryption_key) + + database_records = _encrypt_private_keys( + private_keys=private_keys, + encryptor=encryptor, + ) + if not no_confirm: + click.confirm( + f'Fetched {len(private_keys)} validator keys, upload them to the database?', + default=True, + abort=True, + ) + database.upload_keys(keys=database_records) + total_keys_count = database.fetch_public_keys_count() + + click.clear() + + click.secho( + f'The database contains {total_keys_count} validator keys.\n' + f"The decryption key: '{encryptor.str_key}'", + bold=True, + fg='green', + ) + + +def _encrypt_private_keys(private_keys: list[int], encryptor: Encryptor) -> list[DatabaseKeyRecord]: + """ + Returns prepared database key records from the private keys. + """ + + click.secho('Encrypting database keys...', bold=True) + key_records: list[DatabaseKeyRecord] = [] + for private_key in private_keys: + encrypted_private_key, nonce = encryptor.encrypt(str(private_key)) + + key_record = DatabaseKeyRecord( + public_key=w3.to_hex(G2ProofOfPossession.SkToPk(private_key)), + private_key=bytes_to_str(encrypted_private_key), + nonce=bytes_to_str(nonce), + ) + + if key_record not in key_records: + key_records.append(key_record) + + return key_records diff --git a/src/common/contrib.py b/src/common/contrib.py index 071b0fd2..2f567e5d 100644 --- a/src/common/contrib.py +++ b/src/common/contrib.py @@ -1,3 +1,6 @@ +import collections +from base64 import b64decode, b64encode + import click @@ -8,3 +11,15 @@ def chunkify(items, size): def greenify(value): return click.style(value, bold=True, fg='green') + + +def bytes_to_str(value: bytes) -> str: + return b64encode(value).decode('ascii') + + +def str_to_bytes(value: str) -> bytes: + return b64decode(value) + + +def is_lists_equal(x: list, y: list) -> bool: + return collections.Counter(x) == collections.Counter(y) diff --git a/src/common/validators.py b/src/common/validators.py index ff383b4e..23c9768f 100644 --- a/src/common/validators.py +++ b/src/common/validators.py @@ -1,3 +1,6 @@ +import os +import re + import click from eth_utils import is_address, to_checksum_address @@ -21,3 +24,18 @@ def validate_eth_address(ctx, param, value): pass raise click.BadParameter('Invalid Ethereum address') + + +# pylint: disable-next=unused-argument +def validate_db_uri(ctx, param, value): + pattern = re.compile(r'.+:\/\/.+:.*@.+\/.+') + if not pattern.match(value): + raise click.BadParameter('Invalid database connection string') + return value + + +# pylint: disable-next=unused-argument +def validate_env_name(ctx, param, value): + if not os.getenv(value): + raise click.BadParameter(f'Empty environment variable {value}') + return value diff --git a/src/key_manager/__init__.py b/src/key_manager/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/key_manager/database.py b/src/key_manager/database.py new file mode 100644 index 00000000..a1f439e1 --- /dev/null +++ b/src/key_manager/database.py @@ -0,0 +1,90 @@ +import click +import psycopg2 +from eth_typing import HexStr +from psycopg2.extras import execute_values + +from src.key_manager.typings import DatabaseKeyRecord + + +class Database: + def __init__(self, db_url: str): + self.db_url = db_url + + def upload_keys(self, keys: list[DatabaseKeyRecord]) -> None: + """Updates database records to new state.""" + with _get_db_connection(self.db_url) as conn: + with conn.cursor() as cur: + # recreate table + cur.execute( + """ + CREATE TABLE IF NOT EXISTS keys ( + public_key TEXT UNIQUE NOT NULL, + private_key TEXT UNIQUE NOT NULL, + nonce TEXT NOT NULL + ); + """ + ) + + # insert keys + execute_values( + cur, + 'INSERT INTO keys (public_key, private_key, nonce) ' + 'VALUES %s ON CONFLICT DO NOTHING', + [ + ( + x.public_key, + x.private_key, + x.nonce, + ) + for x in keys + ], + ) + + def fetch_public_keys_by_range(self, start_index: int, end_index: int) -> list[HexStr]: + with _get_db_connection(self.db_url) as conn: + with conn.cursor() as cur: + cur.execute( + 'SELECT public_key FROM keys ORDER BY public_key LIMIT %s OFFSET %s;', + (end_index - start_index, start_index), + ) + rows = cur.fetchall() + return [row[0] for row in rows] + + def fetch_public_keys_count(self) -> int: + with _get_db_connection(self.db_url) as conn: + with conn.cursor() as cur: + cur.execute( + 'SELECT COUNT(public_key) FROM keys', + ) + row = cur.fetchone() + return row[0] + + def fetch_keys(self) -> list[DatabaseKeyRecord]: + with _get_db_connection(self.db_url) as conn: + with conn.cursor() as cur: + cur.execute('SELECT * FROM keys ORDER BY public_key') + rows = cur.fetchall() + return [ + DatabaseKeyRecord( + public_key=row[0], + private_key=row[1], + nonce=row[2], + ) + for row in rows + ] + + +def check_db_connection(db_url): + connection = _get_db_connection(db_url=db_url) + try: + cur = connection.cursor() + cur.execute('SELECT 1') + except psycopg2.OperationalError as e: + raise click.ClickException( + f'Error: failed to connect to the database server with provided URL. ' + f'Error details: {e}', + ) + + +def _get_db_connection(db_url): + return psycopg2.connect(dsn=db_url) diff --git a/src/key_manager/encryptor.py b/src/key_manager/encryptor.py new file mode 100644 index 00000000..298ef9d6 --- /dev/null +++ b/src/key_manager/encryptor.py @@ -0,0 +1,41 @@ +from typing import cast + +# pycryptodome lib used +from Crypto.Cipher import AES # nosec +from Crypto.Cipher._mode_eax import EaxMode # nosec +from Crypto.Random import get_random_bytes # nosec + +from src.common.contrib import bytes_to_str, str_to_bytes + +CIPHER_KEY_LENGTH = 32 + + +class Encryptor: + def __init__(self, key: str | None = None): + if key: + self.str_key = key + self.bytes_key = str_to_bytes(key) + else: + self.bytes_key = self._generate_cipher_key() + self.str_key = bytes_to_str(self.bytes_key) + + def encrypt(self, data: str): + cipher = self._get_cipher() + encrypted_data = cipher.encrypt(bytes(data, 'ascii')) + return encrypted_data, cipher.nonce + + def decrypt(self, data: str, nonce: str) -> str: + cipher = self._restore_cipher(nonce=nonce) + private_key = cipher.decrypt(str_to_bytes(data)) + return private_key.decode('ascii') + + def _restore_cipher(self, nonce: str) -> EaxMode: + cipher = AES.new(self.bytes_key, AES.MODE_EAX, nonce=str_to_bytes(nonce)) + return cast(EaxMode, cipher) + + def _generate_cipher_key(self) -> bytes: + return get_random_bytes(CIPHER_KEY_LENGTH) + + def _get_cipher(self) -> EaxMode: + cipher = AES.new(self.bytes_key, AES.MODE_EAX) + return cast(EaxMode, cipher) diff --git a/src/key_manager/typings.py b/src/key_manager/typings.py new file mode 100644 index 00000000..ee52d299 --- /dev/null +++ b/src/key_manager/typings.py @@ -0,0 +1,10 @@ +from dataclasses import dataclass + +from eth_typing import HexStr + + +@dataclass +class DatabaseKeyRecord: + public_key: HexStr + private_key: str + nonce: str diff --git a/src/main.py b/src/main.py index b6491cf8..49cbb1e9 100644 --- a/src/main.py +++ b/src/main.py @@ -12,6 +12,9 @@ from src.commands.merge_deposit_data import merge_deposit_data from src.commands.recover import recover from src.commands.start import start +from src.commands.sync_validator import sync_validator +from src.commands.sync_web3signer import sync_web3signer +from src.commands.update_db import update_db from src.commands.validators_exit import validators_exit @@ -28,6 +31,9 @@ def cli() -> None: cli.add_command(start) cli.add_command(recover) cli.add_command(get_validators_root) +cli.add_command(sync_validator) +cli.add_command(sync_web3signer) +cli.add_command(update_db) if __name__ == '__main__':